yellowdog-python-examples 7.15.2__tar.gz → 7.15.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {yellowdog_python_examples-7.15.2/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.15.4}/PKG-INFO +1 -1
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/README.md +14 -9
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/pyproject.toml +2 -2
- yellowdog_python_examples-7.15.4/yellowdog_cli/__init__.py +1 -0
- yellowdog_python_examples-7.15.2/yellowdog_cli/analyse.py → yellowdog_python_examples-7.15.4/yellowdog_cli/compare.py +10 -9
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/submit.py +42 -19
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/args.py +7 -7
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/config_types.py +1 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/follow_utils.py +6 -2
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/load_config.py +1 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/property_names.py +2 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/settings.py +2 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4/yellowdog_python_examples.egg-info}/PKG-INFO +1 -1
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/SOURCES.txt +1 -1
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/entry_points.txt +1 -1
- yellowdog_python_examples-7.15.2/yellowdog_cli/__init__.py +0 -1
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/LICENSE +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/PYPI_README.md +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/requirements.txt +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/setup.cfg +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_create_remove.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_demos.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_dryruns.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_entrypoints.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_gui.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_list.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_objects.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_variable_processing.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/abort.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/admin.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/boost.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/cancel.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/cloudwizard.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/create.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/delete.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/download.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/follow.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/format_json.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/hold.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/instantiate.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/jsonnet2json.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/list.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/provision.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/remove.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/resize.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/show.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/shutdown.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/start.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/terminate.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/upload.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/__init__.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/check_imports.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_aws.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/compact_json.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/csv_data.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/entity_utils.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/interactive.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/items.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/load_resources.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/misc_utils.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/printing.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/provision_utils.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/start_hold_common.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/submit_utils.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/type_check.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/upload_utils.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/validate_properties.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/variables.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/wrapper.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/ydid_utils.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/version.py +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/requires.txt +0 -0
- {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.15.
|
|
3
|
+
Version: 7.15.4
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
6
|
Project-URL: Homepage, https://github.com/yellowdog/python-examples
|
|
@@ -123,10 +123,10 @@
|
|
|
123
123
|
* [yd-hold](#yd-hold)
|
|
124
124
|
* [yd-boost](#yd-boost)
|
|
125
125
|
* [yd-show](#yd-show)
|
|
126
|
-
* [yd-
|
|
126
|
+
* [yd-compare](#yd-compare)
|
|
127
127
|
|
|
128
128
|
<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
|
|
129
|
-
<!-- Added by: pwt, at:
|
|
129
|
+
<!-- Added by: pwt, at: Wed Feb 26 15:47:37 GMT 2025 -->
|
|
130
130
|
|
|
131
131
|
<!--te-->
|
|
132
132
|
|
|
@@ -160,7 +160,7 @@ The commands provide the following capabilities:
|
|
|
160
160
|
- **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, and Namespace Policies with the **`yd-create`** and **`yd-remove`** commands
|
|
161
161
|
- **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
|
|
162
162
|
- **Showing** the details of any YellowDog entity using its YellowDog ID with the **`yd-show`** command
|
|
163
|
-
- **
|
|
163
|
+
- **Comparing** whether worker pools are a match for task groups with the **`yd-compare`** command
|
|
164
164
|
|
|
165
165
|
The operation of the commands is controlled using TOML configuration files and/or environment variables and command line arguments. In addition, Work Requirements and Worker Pools can be defined using JSON files providing extensive configurability.
|
|
166
166
|
|
|
@@ -614,6 +614,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
|
|
|
614
614
|
| `providers` | Constrains the YellowDog Scheduler only to execute tasks from the associated Task Group on the specified providers. E.g., `["AWS", "GOOGLE"]`. | Yes | Yes | Yes | |
|
|
615
615
|
| `ram` | Range constraint on GB of RAM that are required to execute Tasks. E.g., `[2.5, 4.0]`. | Yes | Yes | Yes | |
|
|
616
616
|
| `regions` | Constrains the YellowDog Scheduler only to execute Tasks from the associated Task Group in the specified regions. E.g., `["eu-west-2]`. | Yes | Yes | Yes | |
|
|
617
|
+
| `setTaskNames` | Set this to `false` to suppress automatic generation of task names. Defaults to `true`. Task names that are set by the user will still be observed. Note that Task names must be set if any outputs are specified. | Yes | Yes | Yes | Yes |
|
|
617
618
|
| `tag` | A tag that can be associated with a Work Requirement, Task Group or Task. Note there is **no property inheritance** for these tags. | Yes | Yes | Yes | Yes |
|
|
618
619
|
| `taskBatchSize` | Determines the batch size used to add Tasks to Task Groups. Default is 2,000. | Yes | | | |
|
|
619
620
|
| `taskCount` | The number of times to execute the Task. | Yes | Yes | Yes | |
|
|
@@ -643,7 +644,7 @@ In addition to the property inheritance mechanism, some properties are set autom
|
|
|
643
644
|
|
|
644
645
|
- The **Work Requirement** name is automatically set using a concatenation of the `tag` property, and a UTC timestamp: e.g.: `mytag_221024-15552480`.
|
|
645
646
|
- **Task Group** names are automatically created for any Task Group that is not explicitly named, using names of the form `task_group_1` (or `task_group_01`, etc., for larger numbers of Task Groups). Task Group numbers can also be included in user-defined Task Group names using the `{{task_group_number}}` variable substitution discussed below.
|
|
646
|
-
- **Task** names are automatically created for any Task that is not explicitly named, using names of the form `task_1` (or `task_01`, etc., for larger numbers of Tasks). The Task counter resets for each different Task Group. Task numbers can also be included in user-defined Task names using the `{{task_number}}` variable substitution discussed below.
|
|
647
|
+
- **Task** names are automatically created for any Task that is not explicitly named, using names of the form `task_1` (or `task_01`, etc., for larger numbers of Tasks). The Task counter resets for each different Task Group. Task numbers can also be included in user-defined Task names using the `{{task_number}}` variable substitution discussed below. Automatic Task name generation can be suppressed by setting the `setTaskNames` property to `false`, in which case the `task_name` variable will be set to `none`. Note that Task names must be set for any tasks that specify outputs.
|
|
647
648
|
|
|
648
649
|
#### Obtaining Names/Context from Environment Variables at Task Run Time
|
|
649
650
|
|
|
@@ -814,6 +815,7 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
|
|
|
814
815
|
providers = ["AWS"]
|
|
815
816
|
ram = [0.5, 2.0]
|
|
816
817
|
regions = ["eu-west-2"]
|
|
818
|
+
setTaskNames = false
|
|
817
819
|
tag = "my_tag"
|
|
818
820
|
taskBatchSize = 1000
|
|
819
821
|
taskCount = 100
|
|
@@ -870,6 +872,7 @@ Showing all possible properties at the Work Requirement level:
|
|
|
870
872
|
"providers": ["AWS"],
|
|
871
873
|
"ram": [0.5, 2],
|
|
872
874
|
"regions": ["eu-west-2"],
|
|
875
|
+
"setTaskNames": false,
|
|
873
876
|
"tag": "my_tag"
|
|
874
877
|
"taskCount": 100,
|
|
875
878
|
"taskData": "my_task_data_string",
|
|
@@ -932,6 +935,7 @@ Showing all possible properties at the Task Group level:
|
|
|
932
935
|
"providers": ["AWS"],
|
|
933
936
|
"ram": [0.5, 2],
|
|
934
937
|
"regions": ["eu-west-2"],
|
|
938
|
+
"setTaskNames": false,
|
|
935
939
|
"tag": "my_tag",
|
|
936
940
|
"taskCount": 5,
|
|
937
941
|
"taskData": "my_task_data_string",
|
|
@@ -986,6 +990,7 @@ Showing all possible properties at the Task level:
|
|
|
986
990
|
"outputs": ["results.txt"],
|
|
987
991
|
"outputsOther": [{"directoryName": "my_output_dir", "filePattern": "out.txt", "required": true}],
|
|
988
992
|
"outputsRequired": ["results_required.txt"],
|
|
993
|
+
"setTaskNames": false,
|
|
989
994
|
"tag": "my_tag",
|
|
990
995
|
"taskData": "my_task_data_string",
|
|
991
996
|
"taskDataFile": "my_data_file.txt",
|
|
@@ -1060,7 +1065,7 @@ To examine the JSON that will actually be sent to the YellowDog API after all pr
|
|
|
1060
1065
|
|
|
1061
1066
|
A dry-run is useful for inspecting the results of all the processing that's been performed. To suppress all output except for the JSON itself, add the `--quiet` (`-q`) command line option.
|
|
1062
1067
|
|
|
1063
|
-
Note that the generated JSON is a consolidated form of what would be submitted to the YellowDog API, and Tasks are presented directly within their Task Groups for ease of comprehension. In actual API submissions, the Work Requirement with
|
|
1068
|
+
Note that the generated JSON is a consolidated form of what would be submitted to the YellowDog API, and Tasks are presented directly within their Task Groups for ease of comprehension. In actual API submissions, the Work Requirement with zero or more Task Groups is submitted first, and Tasks are then added to their Task Groups separately, in subsequent API calls. Task Groups and Tasks can also later be added to the Work Requirement.
|
|
1064
1069
|
|
|
1065
1070
|
A simple example of the JSON output is shown below, showing a Work Requirement with a single Task Group, containing a single Task.
|
|
1066
1071
|
|
|
@@ -2771,17 +2776,17 @@ The `yd-show` command will show the details (in JSON) of any YellowDog entity th
|
|
|
2771
2776
|
- Keyrings
|
|
2772
2777
|
- Allowances
|
|
2773
2778
|
|
|
2774
|
-
## yd-
|
|
2779
|
+
## yd-compare
|
|
2775
2780
|
|
|
2776
|
-
The `yd-
|
|
2781
|
+
The `yd-compare` command takes a Work Requirement or Task Group ID and one or more Worker Pool IDs, and compares the selected Task Group(s) against the available Nodes/Workers in the Worker Pool(s). If a Work Requirement ID is supplied, all Task Groups in the Work Requirement will be compared.
|
|
2777
2782
|
|
|
2778
2783
|
```commandline
|
|
2779
|
-
yd-
|
|
2784
|
+
yd-compare ydid:taskgrp:000000:83587010-5e26-4174-92a7-c7cc2612638d:1 ydid:wrkrpool:000000:3666e4c5-382e-4512-a2c7-33dbb839f75
|
|
2780
2785
|
```
|
|
2781
2786
|
|
|
2782
2787
|
The command checks if the **Run Specification** of a Task Group matches the properties of the Worker Pools and their registered Nodes and Workers, meaning there are Workers in the Worker Pool that could be claimed by the Task Group and that the Worker Pool would be a candidate for scaling up to meet the demands of the Task Group.
|
|
2783
2788
|
|
|
2784
|
-
A detailed matching report showing the
|
|
2789
|
+
A detailed matching report showing the comparison against each specific property is created, which can be used to determine which properties are preventing a Worker Pool match.
|
|
2785
2790
|
|
|
2786
2791
|
The match status of a Worker Pool falls into one of four categories:
|
|
2787
2792
|
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
|
|
43
43
|
[project.scripts]
|
|
44
44
|
yd-abort = "yellowdog_cli.abort:main"
|
|
45
|
-
yd-analyse = "yellowdog_cli.analyse:main"
|
|
46
45
|
yd-boost = "yellowdog_cli.boost:main"
|
|
47
46
|
yd-cancel = "yellowdog_cli.cancel:main"
|
|
48
47
|
yd-cloudwizard = "yellowdog_cli.cloudwizard:main"
|
|
48
|
+
yd-compare = "yellowdog_cli.compare:main"
|
|
49
49
|
yd-create = "yellowdog_cli.create:main"
|
|
50
50
|
yd-delete = "yellowdog_cli.delete:main"
|
|
51
51
|
yd-download = "yellowdog_cli.download:main"
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
yd-resize = "yellowdog_cli.resize:main"
|
|
61
61
|
yd-show = "yellowdog_cli.show:main"
|
|
62
62
|
yd-shutdown = "yellowdog_cli.shutdown:main"
|
|
63
|
-
yd-submit = "yellowdog_cli.submit:main"
|
|
64
63
|
yd-start = "yellowdog_cli.start:main"
|
|
64
|
+
yd-submit = "yellowdog_cli.submit:main"
|
|
65
65
|
yd-terminate = "yellowdog_cli.terminate:main"
|
|
66
66
|
yd-upload = "yellowdog_cli.upload:main"
|
|
67
67
|
yd-version = "yellowdog_cli.version:main"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "7.15.4"
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
|
|
3
3
|
"""
|
|
4
|
-
A script to
|
|
4
|
+
A script to compare work requirements and task groups with worker pools,
|
|
5
|
+
and check for matches.
|
|
5
6
|
"""
|
|
6
7
|
|
|
7
8
|
from dataclasses import dataclass
|
|
@@ -130,7 +131,7 @@ class MatchReport:
|
|
|
130
131
|
else:
|
|
131
132
|
match_str = "NON-MATCHING"
|
|
132
133
|
print_log(
|
|
133
|
-
f"Detailed report for {match_str} ({self.worker_pool_status}) worker pool "
|
|
134
|
+
f"Detailed comparison report for {match_str} ({self.worker_pool_status}) worker pool "
|
|
134
135
|
f"'{self.worker_pool_name}' ({self.worker_pool_id})",
|
|
135
136
|
override_quiet=True,
|
|
136
137
|
)
|
|
@@ -607,12 +608,12 @@ def _get_worker_pool_by_id(worker_pool_id) -> WorkerPool:
|
|
|
607
608
|
)
|
|
608
609
|
|
|
609
610
|
|
|
610
|
-
def
|
|
611
|
+
def _compare_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
|
|
611
612
|
"""
|
|
612
|
-
|
|
613
|
+
Compare a Task Group.
|
|
613
614
|
"""
|
|
614
615
|
print_log(
|
|
615
|
-
f"
|
|
616
|
+
f"Comparing Task Group '{task_group.name}' ({task_group.id})",
|
|
616
617
|
override_quiet=True,
|
|
617
618
|
)
|
|
618
619
|
|
|
@@ -652,7 +653,7 @@ def _analyse_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
|
|
|
652
653
|
for match_report in match_reports:
|
|
653
654
|
match_report.print_detailed_report()
|
|
654
655
|
|
|
655
|
-
print_log("Task Group
|
|
656
|
+
print_log("Task Group comparison complete")
|
|
656
657
|
|
|
657
658
|
|
|
658
659
|
@main_wrapper
|
|
@@ -670,7 +671,7 @@ def main():
|
|
|
670
671
|
|
|
671
672
|
# Task group
|
|
672
673
|
if get_ydid_type(ARGS_PARSER.wr_or_tg_id) == YDIDType.TASK_GROUP:
|
|
673
|
-
|
|
674
|
+
_compare_task_group(
|
|
674
675
|
_get_task_group_by_id(ARGS_PARSER.wr_or_tg_id), worker_pools
|
|
675
676
|
)
|
|
676
677
|
|
|
@@ -678,12 +679,12 @@ def main():
|
|
|
678
679
|
elif get_ydid_type(ARGS_PARSER.wr_or_tg_id) == YDIDType.WORK_REQUIREMENT:
|
|
679
680
|
work_requirement = _get_work_requirement_by_id(ARGS_PARSER.wr_or_tg_id)
|
|
680
681
|
print_log(
|
|
681
|
-
f"
|
|
682
|
+
f"Comparing all Task Groups in Work Requirement '{work_requirement.name}' "
|
|
682
683
|
f"({work_requirement.id})",
|
|
683
684
|
override_quiet=True,
|
|
684
685
|
)
|
|
685
686
|
for task_group in work_requirement.taskGroups:
|
|
686
|
-
|
|
687
|
+
_compare_task_group(task_group, worker_pools)
|
|
687
688
|
|
|
688
689
|
else:
|
|
689
690
|
raise Exception(
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/submit.py
RENAMED
|
@@ -59,6 +59,7 @@ from yellowdog_cli.utils.settings import (
|
|
|
59
59
|
MAX_BATCH_SUBMIT_ATTEMPTS,
|
|
60
60
|
NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR,
|
|
61
61
|
VAR_CLOSING_DELIMITER,
|
|
62
|
+
VAR_NAME_OF_UNNAMED_TASK,
|
|
62
63
|
VAR_OPENING_DELIMITER,
|
|
63
64
|
)
|
|
64
65
|
from yellowdog_cli.utils.submit_utils import (
|
|
@@ -736,18 +737,33 @@ def generate_batch_of_tasks_for_task_group(
|
|
|
736
737
|
for task_number in range(start_task_number, end_task_number):
|
|
737
738
|
task_group_data = wr_data[TASK_GROUPS][tg_number]
|
|
738
739
|
task = tasks[task_number] if task_count is None else tasks[0]
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
740
|
+
|
|
741
|
+
set_task_names = check_bool(
|
|
742
|
+
task.get(
|
|
743
|
+
SET_TASK_NAMES,
|
|
744
|
+
task_group_data.get(
|
|
745
|
+
SET_TASK_NAMES,
|
|
746
|
+
wr_data.get(SET_TASK_NAMES, CONFIG_WR.set_task_names),
|
|
747
|
+
),
|
|
747
748
|
)
|
|
748
749
|
)
|
|
749
750
|
|
|
750
|
-
|
|
751
|
+
task_name = get_task_name(
|
|
752
|
+
task.get(NAME, task.get(TASK_NAME, CONFIG_WR.task_name)),
|
|
753
|
+
set_task_names,
|
|
754
|
+
task_number,
|
|
755
|
+
num_tasks,
|
|
756
|
+
tg_number,
|
|
757
|
+
num_task_groups,
|
|
758
|
+
task_group.name,
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
task_name = None if task_name is None else format_yd_name(task_name)
|
|
762
|
+
|
|
763
|
+
add_or_update_substitution(
|
|
764
|
+
L_TASK_NAME,
|
|
765
|
+
VAR_NAME_OF_UNNAMED_TASK if task_name is None else task_name,
|
|
766
|
+
)
|
|
751
767
|
add_or_update_substitution(
|
|
752
768
|
L_TASK_NUMBER, formatted_number_str(task_number, num_tasks)
|
|
753
769
|
)
|
|
@@ -1180,13 +1196,13 @@ def on_update(work_req: WorkRequirement):
|
|
|
1180
1196
|
|
|
1181
1197
|
def cleanup_on_failure(work_requirement: WorkRequirement) -> None:
|
|
1182
1198
|
"""
|
|
1183
|
-
Clean up the Work Requirement and any uploaded Objects on failure
|
|
1199
|
+
Clean up the Work Requirement and any uploaded Objects on failure.
|
|
1184
1200
|
"""
|
|
1185
1201
|
if ARGS_PARSER.dry_run:
|
|
1186
1202
|
return
|
|
1187
1203
|
|
|
1188
1204
|
CLIENT.work_client.cancel_work_requirement(work_requirement)
|
|
1189
|
-
|
|
1205
|
+
print_warning(f"Cancelled Work Requirement '{work_requirement.name}'")
|
|
1190
1206
|
|
|
1191
1207
|
# Delete uploaded objects
|
|
1192
1208
|
UPLOADED_FILES.delete()
|
|
@@ -1195,7 +1211,7 @@ def cleanup_on_failure(work_requirement: WorkRequirement) -> None:
|
|
|
1195
1211
|
def deduplicate_inputs(task_inputs: List[TaskInput]) -> List[TaskInput]:
|
|
1196
1212
|
"""
|
|
1197
1213
|
Deduplicate a list of TaskInputs. This is useful when wildcards
|
|
1198
|
-
are used. Note that TaskInputs
|
|
1214
|
+
are used. Note that TaskInputs which differ only in their verification
|
|
1199
1215
|
type will be caught by 'check_for_duplicates_in_file_lists()'.
|
|
1200
1216
|
"""
|
|
1201
1217
|
deduplicated_task_inputs: List[TaskInput] = []
|
|
@@ -1245,12 +1261,13 @@ def formatted_number_str(
|
|
|
1245
1261
|
|
|
1246
1262
|
def get_task_name(
|
|
1247
1263
|
name: Optional[str],
|
|
1264
|
+
set_task_names: bool,
|
|
1248
1265
|
task_number: int,
|
|
1249
1266
|
num_tasks: int,
|
|
1250
1267
|
task_group_number: int,
|
|
1251
1268
|
num_task_groups: int,
|
|
1252
1269
|
task_group_name: str,
|
|
1253
|
-
) -> str:
|
|
1270
|
+
) -> Optional[str]:
|
|
1254
1271
|
"""
|
|
1255
1272
|
Create the name of a Task.
|
|
1256
1273
|
Supports lazy substitution.
|
|
@@ -1278,9 +1295,12 @@ def get_task_name(
|
|
|
1278
1295
|
task_group_name,
|
|
1279
1296
|
)
|
|
1280
1297
|
|
|
1281
|
-
|
|
1298
|
+
elif set_task_names:
|
|
1282
1299
|
name = "task_" + formatted_number_str(task_number, num_tasks)
|
|
1283
1300
|
|
|
1301
|
+
else:
|
|
1302
|
+
name = None
|
|
1303
|
+
|
|
1284
1304
|
return name
|
|
1285
1305
|
|
|
1286
1306
|
|
|
@@ -1320,7 +1340,7 @@ def create_task(
|
|
|
1320
1340
|
wr_data: Dict,
|
|
1321
1341
|
task_group_data: Dict,
|
|
1322
1342
|
task_data: Dict,
|
|
1323
|
-
task_name: str,
|
|
1343
|
+
task_name: Optional[str],
|
|
1324
1344
|
task_number: int,
|
|
1325
1345
|
tg_name: str,
|
|
1326
1346
|
tg_number: int,
|
|
@@ -1350,13 +1370,16 @@ def create_task(
|
|
|
1350
1370
|
if inputs is None or len(inputs) == 0:
|
|
1351
1371
|
flatten_input_paths = None
|
|
1352
1372
|
|
|
1373
|
+
if task_name is None and len(outputs) > 0:
|
|
1374
|
+
raise Exception(f"Tasks must be named if outputs are specified")
|
|
1375
|
+
|
|
1353
1376
|
return Task(
|
|
1354
1377
|
name=task_name,
|
|
1355
1378
|
taskType=task_type,
|
|
1356
|
-
arguments=args,
|
|
1357
|
-
inputs=inputs,
|
|
1358
|
-
environment=env_copy,
|
|
1359
|
-
outputs=outputs,
|
|
1379
|
+
arguments=None if len(args) == 0 else args,
|
|
1380
|
+
inputs=None if len(inputs) == 0 else inputs,
|
|
1381
|
+
environment=None if len(env_copy) == 0 else env_copy,
|
|
1382
|
+
outputs=None if len(outputs) == 0 else outputs,
|
|
1360
1383
|
flattenInputPaths=flatten_input_paths,
|
|
1361
1384
|
taskData=task_data_property,
|
|
1362
1385
|
timeout=task_timeout,
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/args.py
RENAMED
|
@@ -115,7 +115,7 @@ class CLIParser:
|
|
|
115
115
|
)
|
|
116
116
|
|
|
117
117
|
# Module-specific argument sets
|
|
118
|
-
if not any(module in sys.argv[0] for module in ["
|
|
118
|
+
if not any(module in sys.argv[0] for module in ["compare"]):
|
|
119
119
|
parser.add_argument(
|
|
120
120
|
"--variable",
|
|
121
121
|
"-v",
|
|
@@ -137,7 +137,7 @@ class CLIParser:
|
|
|
137
137
|
"cloudwizard",
|
|
138
138
|
"follow",
|
|
139
139
|
"list",
|
|
140
|
-
"
|
|
140
|
+
"compare",
|
|
141
141
|
]
|
|
142
142
|
):
|
|
143
143
|
self.namespace_required = True
|
|
@@ -1121,13 +1121,13 @@ class CLIParser:
|
|
|
1121
1121
|
),
|
|
1122
1122
|
)
|
|
1123
1123
|
|
|
1124
|
-
if "
|
|
1124
|
+
if "compare" in sys.argv[0]:
|
|
1125
1125
|
parser.add_argument(
|
|
1126
1126
|
"wr_or_tg_id",
|
|
1127
1127
|
metavar="<work-requirement-or-task-group-ID>",
|
|
1128
1128
|
type=str,
|
|
1129
1129
|
help=(
|
|
1130
|
-
"the YellowDog ID of the work requirement or task group to be
|
|
1130
|
+
"the YellowDog ID of the work requirement or task group to be compared"
|
|
1131
1131
|
),
|
|
1132
1132
|
)
|
|
1133
1133
|
parser.add_argument(
|
|
@@ -1135,7 +1135,7 @@ class CLIParser:
|
|
|
1135
1135
|
metavar="<worker-pool-ID>",
|
|
1136
1136
|
type=str,
|
|
1137
1137
|
nargs="+",
|
|
1138
|
-
help="the YellowDog ID(s) of the worker pool(s) to
|
|
1138
|
+
help="the YellowDog ID(s) of the worker pool(s) to compare",
|
|
1139
1139
|
)
|
|
1140
1140
|
|
|
1141
1141
|
self.args = parser.parse_args()
|
|
@@ -1681,9 +1681,9 @@ def lookup_module_description(module_name: str) -> Optional[str]:
|
|
|
1681
1681
|
suffix = "boosting Allowances"
|
|
1682
1682
|
elif "show" in module_name:
|
|
1683
1683
|
suffix = "showing the JSON details of entities referenced by their YDIDs"
|
|
1684
|
-
elif "
|
|
1684
|
+
elif "compare" in module_name:
|
|
1685
1685
|
suffix = (
|
|
1686
|
-
"
|
|
1686
|
+
"comparing whether a work requirement or task group is matched by "
|
|
1687
1687
|
"workers in the specified worker pools"
|
|
1688
1688
|
)
|
|
1689
1689
|
|
|
@@ -51,6 +51,7 @@ class ConfigWorkRequirement:
|
|
|
51
51
|
providers: Optional[List[str]] = None
|
|
52
52
|
ram: Optional[List[float]] = None
|
|
53
53
|
regions: Optional[List[str]] = None
|
|
54
|
+
set_task_names: bool = True
|
|
54
55
|
task_batch_size: int = TASK_BATCH_SIZE_DEFAULT
|
|
55
56
|
task_count: int = 1
|
|
56
57
|
task_data: Optional[str] = None
|
|
@@ -95,8 +95,12 @@ def follow_events(ydid: str, ydid_type: YDIDType):
|
|
|
95
95
|
)
|
|
96
96
|
|
|
97
97
|
if response.status_code != 200:
|
|
98
|
-
|
|
99
|
-
|
|
98
|
+
try:
|
|
99
|
+
error_text = response.json()["message"]
|
|
100
|
+
except:
|
|
101
|
+
error_text = "(JSON error cannot be decoded)"
|
|
102
|
+
print_error(f"'{ydid}': {error_text}")
|
|
103
|
+
break
|
|
100
104
|
|
|
101
105
|
if response.encoding is None:
|
|
102
106
|
response.encoding = "utf-8"
|
|
@@ -307,6 +307,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
|
|
|
307
307
|
providers=wr_section.get(PROVIDERS, None),
|
|
308
308
|
ram=wr_section.get(RAM, None),
|
|
309
309
|
regions=wr_section.get(REGIONS, None),
|
|
310
|
+
set_task_names=wr_section.get(SET_TASK_NAMES, True),
|
|
310
311
|
task_batch_size=task_batch_size,
|
|
311
312
|
task_count=task_count,
|
|
312
313
|
task_data=wr_section.get(TASK_DATA, None),
|
|
@@ -60,6 +60,7 @@ RAM = "ram" # List of two Floats
|
|
|
60
60
|
REGIONS = "regions" # List of Strings
|
|
61
61
|
REQUIRED = "required" # Boolean
|
|
62
62
|
SECRET = "secret" # String
|
|
63
|
+
SET_TASK_NAMES = "setTaskNames" # Set to False to suppress task naming
|
|
63
64
|
TARGET_INSTANCE_COUNT = "targetInstanceCount" # Integer
|
|
64
65
|
TASKS = "tasks" # List of Tasks
|
|
65
66
|
TASKS_PER_WORKER = "tasksPerWorker" # Integer
|
|
@@ -160,6 +161,7 @@ ALL_KEYS = [
|
|
|
160
161
|
REGIONS,
|
|
161
162
|
REQUIRED,
|
|
162
163
|
SECRET,
|
|
164
|
+
SET_TASK_NAMES,
|
|
163
165
|
TARGET_INSTANCE_COUNT,
|
|
164
166
|
TASKS,
|
|
165
167
|
TASKS_PER_WORKER,
|
|
@@ -40,6 +40,8 @@ TABLE_TYPE_TAG = "table" + TYPE_TAG_TERMINATOR
|
|
|
40
40
|
FORMAT_NAME_TYPE_TAG = "format_name" + TYPE_TAG_TERMINATOR
|
|
41
41
|
TOML_VAR_NESTED_DEPTH = 3
|
|
42
42
|
|
|
43
|
+
VAR_NAME_OF_UNNAMED_TASK = "none"
|
|
44
|
+
|
|
43
45
|
DEFAULT_LOG_WIDTH = 120
|
|
44
46
|
MAX_LINES_COLOURED_FORMATTING = 1024
|
|
45
47
|
ERROR_STYLE = "bold red3"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.15.
|
|
3
|
+
Version: 7.15.4
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
6
|
Project-URL: Homepage, https://github.com/yellowdog/python-examples
|
|
@@ -14,10 +14,10 @@ tests/test_variable_processing.py
|
|
|
14
14
|
yellowdog_cli/__init__.py
|
|
15
15
|
yellowdog_cli/abort.py
|
|
16
16
|
yellowdog_cli/admin.py
|
|
17
|
-
yellowdog_cli/analyse.py
|
|
18
17
|
yellowdog_cli/boost.py
|
|
19
18
|
yellowdog_cli/cancel.py
|
|
20
19
|
yellowdog_cli/cloudwizard.py
|
|
20
|
+
yellowdog_cli/compare.py
|
|
21
21
|
yellowdog_cli/create.py
|
|
22
22
|
yellowdog_cli/delete.py
|
|
23
23
|
yellowdog_cli/download.py
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
[console_scripts]
|
|
2
2
|
yd-abort = yellowdog_cli.abort:main
|
|
3
|
-
yd-analyse = yellowdog_cli.analyse:main
|
|
4
3
|
yd-boost = yellowdog_cli.boost:main
|
|
5
4
|
yd-cancel = yellowdog_cli.cancel:main
|
|
6
5
|
yd-cloudwizard = yellowdog_cli.cloudwizard:main
|
|
6
|
+
yd-compare = yellowdog_cli.compare:main
|
|
7
7
|
yd-create = yellowdog_cli.create:main
|
|
8
8
|
yd-delete = yellowdog_cli.delete:main
|
|
9
9
|
yd-download = yellowdog_cli.download:main
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "7.15.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_create_remove.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_entrypoints.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/abort.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/admin.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/boost.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/cancel.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/cloudwizard.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/create.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/delete.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/download.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/follow.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/format_json.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/instantiate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/jsonnet2json.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/provision.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/remove.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/resize.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/shutdown.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/start.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/terminate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/upload.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/items.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/wrapper.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/version.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|