yellowdog-python-examples 7.15.2__tar.gz → 7.15.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {yellowdog_python_examples-7.15.2/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.15.4}/PKG-INFO +1 -1
  2. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/README.md +14 -9
  3. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/pyproject.toml +2 -2
  4. yellowdog_python_examples-7.15.4/yellowdog_cli/__init__.py +1 -0
  5. yellowdog_python_examples-7.15.2/yellowdog_cli/analyse.py → yellowdog_python_examples-7.15.4/yellowdog_cli/compare.py +10 -9
  6. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/submit.py +42 -19
  7. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/args.py +7 -7
  8. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/config_types.py +1 -0
  9. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/follow_utils.py +6 -2
  10. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/load_config.py +1 -0
  11. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/property_names.py +2 -0
  12. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/settings.py +2 -0
  13. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4/yellowdog_python_examples.egg-info}/PKG-INFO +1 -1
  14. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/SOURCES.txt +1 -1
  15. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/entry_points.txt +1 -1
  16. yellowdog_python_examples-7.15.2/yellowdog_cli/__init__.py +0 -1
  17. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/LICENSE +0 -0
  18. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/PYPI_README.md +0 -0
  19. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/requirements.txt +0 -0
  20. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/setup.cfg +0 -0
  21. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_create_remove.py +0 -0
  22. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_demos.py +0 -0
  23. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_dryruns.py +0 -0
  24. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_entrypoints.py +0 -0
  25. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_gui.py +0 -0
  26. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_list.py +0 -0
  27. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_objects.py +0 -0
  28. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/tests/test_variable_processing.py +0 -0
  29. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/abort.py +0 -0
  30. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/admin.py +0 -0
  31. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/boost.py +0 -0
  32. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/cancel.py +0 -0
  33. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/cloudwizard.py +0 -0
  34. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/create.py +0 -0
  35. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/delete.py +0 -0
  36. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/download.py +0 -0
  37. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/follow.py +0 -0
  38. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/format_json.py +0 -0
  39. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/hold.py +0 -0
  40. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/instantiate.py +0 -0
  41. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/jsonnet2json.py +0 -0
  42. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/list.py +0 -0
  43. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/provision.py +0 -0
  44. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/remove.py +0 -0
  45. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/resize.py +0 -0
  46. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/show.py +0 -0
  47. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/shutdown.py +0 -0
  48. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/start.py +0 -0
  49. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/terminate.py +0 -0
  50. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/upload.py +0 -0
  51. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/__init__.py +0 -0
  52. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/check_imports.py +0 -0
  53. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_aws.py +0 -0
  54. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
  55. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
  56. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
  57. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
  58. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/compact_json.py +0 -0
  59. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/csv_data.py +0 -0
  60. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/entity_utils.py +0 -0
  61. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/interactive.py +0 -0
  62. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/items.py +0 -0
  63. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/load_resources.py +0 -0
  64. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/misc_utils.py +0 -0
  65. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/printing.py +0 -0
  66. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/provision_utils.py +0 -0
  67. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
  68. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/start_hold_common.py +0 -0
  69. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/submit_utils.py +0 -0
  70. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/type_check.py +0 -0
  71. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/upload_utils.py +0 -0
  72. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/validate_properties.py +0 -0
  73. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/variables.py +0 -0
  74. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/wrapper.py +0 -0
  75. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/utils/ydid_utils.py +0 -0
  76. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_cli/version.py +0 -0
  77. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
  78. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/requires.txt +0 -0
  79. {yellowdog_python_examples-7.15.2 → yellowdog_python_examples-7.15.4}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: yellowdog-python-examples
3
- Version: 7.15.2
3
+ Version: 7.15.4
4
4
  Summary: Python CLI commands using the YellowDog Python SDK
5
5
  Author-email: YellowDog Limited <support@yellowdog.co>
6
6
  Project-URL: Homepage, https://github.com/yellowdog/python-examples
@@ -123,10 +123,10 @@
123
123
  * [yd-hold](#yd-hold)
124
124
  * [yd-boost](#yd-boost)
125
125
  * [yd-show](#yd-show)
126
- * [yd-analyse](#yd-analyse)
126
+ * [yd-compare](#yd-compare)
127
127
 
128
128
  <!-- Created by https://github.com/ekalinin/github-markdown-toc -->
129
- <!-- Added by: pwt, at: Sun Feb 23 10:18:32 GMT 2025 -->
129
+ <!-- Added by: pwt, at: Wed Feb 26 15:47:37 GMT 2025 -->
130
130
 
131
131
  <!--te-->
132
132
 
@@ -160,7 +160,7 @@ The commands provide the following capabilities:
160
160
  - **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, and Namespace Policies with the **`yd-create`** and **`yd-remove`** commands
161
161
  - **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
162
162
  - **Showing** the details of any YellowDog entity using its YellowDog ID with the **`yd-show`** command
163
- - **Analysing** whether worker pools are match for task groups
163
+ - **Comparing** whether worker pools are a match for task groups with the **`yd-compare`** command
164
164
 
165
165
  The operation of the commands is controlled using TOML configuration files and/or environment variables and command line arguments. In addition, Work Requirements and Worker Pools can be defined using JSON files providing extensive configurability.
166
166
 
@@ -614,6 +614,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
614
614
  | `providers` | Constrains the YellowDog Scheduler only to execute tasks from the associated Task Group on the specified providers. E.g., `["AWS", "GOOGLE"]`. | Yes | Yes | Yes | |
615
615
  | `ram` | Range constraint on GB of RAM that are required to execute Tasks. E.g., `[2.5, 4.0]`. | Yes | Yes | Yes | |
616
616
  | `regions` | Constrains the YellowDog Scheduler only to execute Tasks from the associated Task Group in the specified regions. E.g., `["eu-west-2]`. | Yes | Yes | Yes | |
617
+ | `setTaskNames` | Set this to `false` to suppress automatic generation of task names. Defaults to `true`. Task names that are set by the user will still be observed. Note that Task names must be set if any outputs are specified. | Yes | Yes | Yes | Yes |
617
618
  | `tag` | A tag that can be associated with a Work Requirement, Task Group or Task. Note there is **no property inheritance** for these tags. | Yes | Yes | Yes | Yes |
618
619
  | `taskBatchSize` | Determines the batch size used to add Tasks to Task Groups. Default is 2,000. | Yes | | | |
619
620
  | `taskCount` | The number of times to execute the Task. | Yes | Yes | Yes | |
@@ -643,7 +644,7 @@ In addition to the property inheritance mechanism, some properties are set autom
643
644
 
644
645
  - The **Work Requirement** name is automatically set using a concatenation of the `tag` property, and a UTC timestamp: e.g.: `mytag_221024-15552480`.
645
646
  - **Task Group** names are automatically created for any Task Group that is not explicitly named, using names of the form `task_group_1` (or `task_group_01`, etc., for larger numbers of Task Groups). Task Group numbers can also be included in user-defined Task Group names using the `{{task_group_number}}` variable substitution discussed below.
646
- - **Task** names are automatically created for any Task that is not explicitly named, using names of the form `task_1` (or `task_01`, etc., for larger numbers of Tasks). The Task counter resets for each different Task Group. Task numbers can also be included in user-defined Task names using the `{{task_number}}` variable substitution discussed below.
647
+ - **Task** names are automatically created for any Task that is not explicitly named, using names of the form `task_1` (or `task_01`, etc., for larger numbers of Tasks). The Task counter resets for each different Task Group. Task numbers can also be included in user-defined Task names using the `{{task_number}}` variable substitution discussed below. Automatic Task name generation can be suppressed by setting the `setTaskNames` property to `false`, in which case the `task_name` variable will be set to `none`. Note that Task names must be set for any tasks that specify outputs.
647
648
 
648
649
  #### Obtaining Names/Context from Environment Variables at Task Run Time
649
650
 
@@ -814,6 +815,7 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
814
815
  providers = ["AWS"]
815
816
  ram = [0.5, 2.0]
816
817
  regions = ["eu-west-2"]
818
+ setTaskNames = false
817
819
  tag = "my_tag"
818
820
  taskBatchSize = 1000
819
821
  taskCount = 100
@@ -870,6 +872,7 @@ Showing all possible properties at the Work Requirement level:
870
872
  "providers": ["AWS"],
871
873
  "ram": [0.5, 2],
872
874
  "regions": ["eu-west-2"],
875
+ "setTaskNames": false,
873
876
  "tag": "my_tag"
874
877
  "taskCount": 100,
875
878
  "taskData": "my_task_data_string",
@@ -932,6 +935,7 @@ Showing all possible properties at the Task Group level:
932
935
  "providers": ["AWS"],
933
936
  "ram": [0.5, 2],
934
937
  "regions": ["eu-west-2"],
938
+ "setTaskNames": false,
935
939
  "tag": "my_tag",
936
940
  "taskCount": 5,
937
941
  "taskData": "my_task_data_string",
@@ -986,6 +990,7 @@ Showing all possible properties at the Task level:
986
990
  "outputs": ["results.txt"],
987
991
  "outputsOther": [{"directoryName": "my_output_dir", "filePattern": "out.txt", "required": true}],
988
992
  "outputsRequired": ["results_required.txt"],
993
+ "setTaskNames": false,
989
994
  "tag": "my_tag",
990
995
  "taskData": "my_task_data_string",
991
996
  "taskDataFile": "my_data_file.txt",
@@ -1060,7 +1065,7 @@ To examine the JSON that will actually be sent to the YellowDog API after all pr
1060
1065
 
1061
1066
  A dry-run is useful for inspecting the results of all the processing that's been performed. To suppress all output except for the JSON itself, add the `--quiet` (`-q`) command line option.
1062
1067
 
1063
- Note that the generated JSON is a consolidated form of what would be submitted to the YellowDog API, and Tasks are presented directly within their Task Groups for ease of comprehension. In actual API submissions, the Work Requirement with its Task Groups is submitted first, and Tasks are then added to their Task Groups separately, in subsequent API calls.
1068
+ Note that the generated JSON is a consolidated form of what would be submitted to the YellowDog API, and Tasks are presented directly within their Task Groups for ease of comprehension. In actual API submissions, the Work Requirement with zero or more Task Groups is submitted first, and Tasks are then added to their Task Groups separately, in subsequent API calls. Task Groups and Tasks can also later be added to the Work Requirement.
1064
1069
 
1065
1070
  A simple example of the JSON output is shown below, showing a Work Requirement with a single Task Group, containing a single Task.
1066
1071
 
@@ -2771,17 +2776,17 @@ The `yd-show` command will show the details (in JSON) of any YellowDog entity th
2771
2776
  - Keyrings
2772
2777
  - Allowances
2773
2778
 
2774
- ## yd-analyse
2779
+ ## yd-compare
2775
2780
 
2776
- The `yd-analyse` command takes a Work Requirement or Task Group ID and one or more Worker Pool IDs, and matches the selected Task Group(s) against the available Nodes/Workers in the Worker Pool(s). If a Work Requirement ID is supplied, all Task Groups in the Work Requirement will be analysed.
2781
+ The `yd-compare` command takes a Work Requirement or Task Group ID and one or more Worker Pool IDs, and compares the selected Task Group(s) against the available Nodes/Workers in the Worker Pool(s). If a Work Requirement ID is supplied, all Task Groups in the Work Requirement will be compared.
2777
2782
 
2778
2783
  ```commandline
2779
- yd-analyse ydid:taskgrp:000000:83587010-5e26-4174-92a7-c7cc2612638d:1 ydid:wrkrpool:000000:3666e4c5-382e-4512-a2c7-33dbb839f75
2784
+ yd-compare ydid:taskgrp:000000:83587010-5e26-4174-92a7-c7cc2612638d:1 ydid:wrkrpool:000000:3666e4c5-382e-4512-a2c7-33dbb839f75
2780
2785
  ```
2781
2786
 
2782
2787
  The command checks if the **Run Specification** of a Task Group matches the properties of the Worker Pools and their registered Nodes and Workers, meaning there are Workers in the Worker Pool that could be claimed by the Task Group and that the Worker Pool would be a candidate for scaling up to meet the demands of the Task Group.
2783
2788
 
2784
- A detailed matching report showing the match against each specific property is created, which can be used to determine which properties are preventing a Worker Pool match.
2789
+ A detailed matching report showing the comparison against each specific property is created, which can be used to determine which properties are preventing a Worker Pool match.
2785
2790
 
2786
2791
  The match status of a Worker Pool falls into one of four categories:
2787
2792
 
@@ -42,10 +42,10 @@
42
42
 
43
43
  [project.scripts]
44
44
  yd-abort = "yellowdog_cli.abort:main"
45
- yd-analyse = "yellowdog_cli.analyse:main"
46
45
  yd-boost = "yellowdog_cli.boost:main"
47
46
  yd-cancel = "yellowdog_cli.cancel:main"
48
47
  yd-cloudwizard = "yellowdog_cli.cloudwizard:main"
48
+ yd-compare = "yellowdog_cli.compare:main"
49
49
  yd-create = "yellowdog_cli.create:main"
50
50
  yd-delete = "yellowdog_cli.delete:main"
51
51
  yd-download = "yellowdog_cli.download:main"
@@ -60,8 +60,8 @@
60
60
  yd-resize = "yellowdog_cli.resize:main"
61
61
  yd-show = "yellowdog_cli.show:main"
62
62
  yd-shutdown = "yellowdog_cli.shutdown:main"
63
- yd-submit = "yellowdog_cli.submit:main"
64
63
  yd-start = "yellowdog_cli.start:main"
64
+ yd-submit = "yellowdog_cli.submit:main"
65
65
  yd-terminate = "yellowdog_cli.terminate:main"
66
66
  yd-upload = "yellowdog_cli.upload:main"
67
67
  yd-version = "yellowdog_cli.version:main"
@@ -0,0 +1 @@
1
+ __version__ = "7.15.4"
@@ -1,7 +1,8 @@
1
1
  #!/usr/bin/env python3
2
2
 
3
3
  """
4
- A script to analyse task groups and check for matching worker pools.
4
+ A script to compare work requirements and task groups with worker pools,
5
+ and check for matches.
5
6
  """
6
7
 
7
8
  from dataclasses import dataclass
@@ -130,7 +131,7 @@ class MatchReport:
130
131
  else:
131
132
  match_str = "NON-MATCHING"
132
133
  print_log(
133
- f"Detailed report for {match_str} ({self.worker_pool_status}) worker pool "
134
+ f"Detailed comparison report for {match_str} ({self.worker_pool_status}) worker pool "
134
135
  f"'{self.worker_pool_name}' ({self.worker_pool_id})",
135
136
  override_quiet=True,
136
137
  )
@@ -607,12 +608,12 @@ def _get_worker_pool_by_id(worker_pool_id) -> WorkerPool:
607
608
  )
608
609
 
609
610
 
610
- def _analyse_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
611
+ def _compare_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
611
612
  """
612
- Analyse a Task Group.
613
+ Compare a Task Group.
613
614
  """
614
615
  print_log(
615
- f"Analysing Task Group '{task_group.name}' ({task_group.id})",
616
+ f"Comparing Task Group '{task_group.name}' ({task_group.id})",
616
617
  override_quiet=True,
617
618
  )
618
619
 
@@ -652,7 +653,7 @@ def _analyse_task_group(task_group: TaskGroup, worker_pools: WorkerPools):
652
653
  for match_report in match_reports:
653
654
  match_report.print_detailed_report()
654
655
 
655
- print_log("Task Group analysis complete")
656
+ print_log("Task Group comparison complete")
656
657
 
657
658
 
658
659
  @main_wrapper
@@ -670,7 +671,7 @@ def main():
670
671
 
671
672
  # Task group
672
673
  if get_ydid_type(ARGS_PARSER.wr_or_tg_id) == YDIDType.TASK_GROUP:
673
- _analyse_task_group(
674
+ _compare_task_group(
674
675
  _get_task_group_by_id(ARGS_PARSER.wr_or_tg_id), worker_pools
675
676
  )
676
677
 
@@ -678,12 +679,12 @@ def main():
678
679
  elif get_ydid_type(ARGS_PARSER.wr_or_tg_id) == YDIDType.WORK_REQUIREMENT:
679
680
  work_requirement = _get_work_requirement_by_id(ARGS_PARSER.wr_or_tg_id)
680
681
  print_log(
681
- f"Analysing all Task Groups in Work Requirement '{work_requirement.name}' "
682
+ f"Comparing all Task Groups in Work Requirement '{work_requirement.name}' "
682
683
  f"({work_requirement.id})",
683
684
  override_quiet=True,
684
685
  )
685
686
  for task_group in work_requirement.taskGroups:
686
- _analyse_task_group(task_group, worker_pools)
687
+ _compare_task_group(task_group, worker_pools)
687
688
 
688
689
  else:
689
690
  raise Exception(
@@ -59,6 +59,7 @@ from yellowdog_cli.utils.settings import (
59
59
  MAX_BATCH_SUBMIT_ATTEMPTS,
60
60
  NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR,
61
61
  VAR_CLOSING_DELIMITER,
62
+ VAR_NAME_OF_UNNAMED_TASK,
62
63
  VAR_OPENING_DELIMITER,
63
64
  )
64
65
  from yellowdog_cli.utils.submit_utils import (
@@ -736,18 +737,33 @@ def generate_batch_of_tasks_for_task_group(
736
737
  for task_number in range(start_task_number, end_task_number):
737
738
  task_group_data = wr_data[TASK_GROUPS][tg_number]
738
739
  task = tasks[task_number] if task_count is None else tasks[0]
739
- task_name = format_yd_name(
740
- get_task_name(
741
- task.get(NAME, task.get(TASK_NAME, CONFIG_WR.task_name)),
742
- task_number,
743
- num_tasks,
744
- tg_number,
745
- num_task_groups,
746
- task_group.name,
740
+
741
+ set_task_names = check_bool(
742
+ task.get(
743
+ SET_TASK_NAMES,
744
+ task_group_data.get(
745
+ SET_TASK_NAMES,
746
+ wr_data.get(SET_TASK_NAMES, CONFIG_WR.set_task_names),
747
+ ),
747
748
  )
748
749
  )
749
750
 
750
- add_or_update_substitution(L_TASK_NAME, str(task_name))
751
+ task_name = get_task_name(
752
+ task.get(NAME, task.get(TASK_NAME, CONFIG_WR.task_name)),
753
+ set_task_names,
754
+ task_number,
755
+ num_tasks,
756
+ tg_number,
757
+ num_task_groups,
758
+ task_group.name,
759
+ )
760
+
761
+ task_name = None if task_name is None else format_yd_name(task_name)
762
+
763
+ add_or_update_substitution(
764
+ L_TASK_NAME,
765
+ VAR_NAME_OF_UNNAMED_TASK if task_name is None else task_name,
766
+ )
751
767
  add_or_update_substitution(
752
768
  L_TASK_NUMBER, formatted_number_str(task_number, num_tasks)
753
769
  )
@@ -1180,13 +1196,13 @@ def on_update(work_req: WorkRequirement):
1180
1196
 
1181
1197
  def cleanup_on_failure(work_requirement: WorkRequirement) -> None:
1182
1198
  """
1183
- Clean up the Work Requirement and any uploaded Objects on failure
1199
+ Clean up the Work Requirement and any uploaded Objects on failure.
1184
1200
  """
1185
1201
  if ARGS_PARSER.dry_run:
1186
1202
  return
1187
1203
 
1188
1204
  CLIENT.work_client.cancel_work_requirement(work_requirement)
1189
- print_log(f"Cancelled Work Requirement '{work_requirement.name}'")
1205
+ print_warning(f"Cancelled Work Requirement '{work_requirement.name}'")
1190
1206
 
1191
1207
  # Delete uploaded objects
1192
1208
  UPLOADED_FILES.delete()
@@ -1195,7 +1211,7 @@ def cleanup_on_failure(work_requirement: WorkRequirement) -> None:
1195
1211
  def deduplicate_inputs(task_inputs: List[TaskInput]) -> List[TaskInput]:
1196
1212
  """
1197
1213
  Deduplicate a list of TaskInputs. This is useful when wildcards
1198
- are used. Note that TaskInputs that differ only in their verification
1214
+ are used. Note that TaskInputs which differ only in their verification
1199
1215
  type will be caught by 'check_for_duplicates_in_file_lists()'.
1200
1216
  """
1201
1217
  deduplicated_task_inputs: List[TaskInput] = []
@@ -1245,12 +1261,13 @@ def formatted_number_str(
1245
1261
 
1246
1262
  def get_task_name(
1247
1263
  name: Optional[str],
1264
+ set_task_names: bool,
1248
1265
  task_number: int,
1249
1266
  num_tasks: int,
1250
1267
  task_group_number: int,
1251
1268
  num_task_groups: int,
1252
1269
  task_group_name: str,
1253
- ) -> str:
1270
+ ) -> Optional[str]:
1254
1271
  """
1255
1272
  Create the name of a Task.
1256
1273
  Supports lazy substitution.
@@ -1278,9 +1295,12 @@ def get_task_name(
1278
1295
  task_group_name,
1279
1296
  )
1280
1297
 
1281
- else:
1298
+ elif set_task_names:
1282
1299
  name = "task_" + formatted_number_str(task_number, num_tasks)
1283
1300
 
1301
+ else:
1302
+ name = None
1303
+
1284
1304
  return name
1285
1305
 
1286
1306
 
@@ -1320,7 +1340,7 @@ def create_task(
1320
1340
  wr_data: Dict,
1321
1341
  task_group_data: Dict,
1322
1342
  task_data: Dict,
1323
- task_name: str,
1343
+ task_name: Optional[str],
1324
1344
  task_number: int,
1325
1345
  tg_name: str,
1326
1346
  tg_number: int,
@@ -1350,13 +1370,16 @@ def create_task(
1350
1370
  if inputs is None or len(inputs) == 0:
1351
1371
  flatten_input_paths = None
1352
1372
 
1373
+ if task_name is None and len(outputs) > 0:
1374
+ raise Exception(f"Tasks must be named if outputs are specified")
1375
+
1353
1376
  return Task(
1354
1377
  name=task_name,
1355
1378
  taskType=task_type,
1356
- arguments=args,
1357
- inputs=inputs,
1358
- environment=env_copy,
1359
- outputs=outputs,
1379
+ arguments=None if len(args) == 0 else args,
1380
+ inputs=None if len(inputs) == 0 else inputs,
1381
+ environment=None if len(env_copy) == 0 else env_copy,
1382
+ outputs=None if len(outputs) == 0 else outputs,
1360
1383
  flattenInputPaths=flatten_input_paths,
1361
1384
  taskData=task_data_property,
1362
1385
  timeout=task_timeout,
@@ -115,7 +115,7 @@ class CLIParser:
115
115
  )
116
116
 
117
117
  # Module-specific argument sets
118
- if not any(module in sys.argv[0] for module in ["analyse"]):
118
+ if not any(module in sys.argv[0] for module in ["compare"]):
119
119
  parser.add_argument(
120
120
  "--variable",
121
121
  "-v",
@@ -137,7 +137,7 @@ class CLIParser:
137
137
  "cloudwizard",
138
138
  "follow",
139
139
  "list",
140
- "analyse",
140
+ "compare",
141
141
  ]
142
142
  ):
143
143
  self.namespace_required = True
@@ -1121,13 +1121,13 @@ class CLIParser:
1121
1121
  ),
1122
1122
  )
1123
1123
 
1124
- if "analyse" in sys.argv[0]:
1124
+ if "compare" in sys.argv[0]:
1125
1125
  parser.add_argument(
1126
1126
  "wr_or_tg_id",
1127
1127
  metavar="<work-requirement-or-task-group-ID>",
1128
1128
  type=str,
1129
1129
  help=(
1130
- "the YellowDog ID of the work requirement or task group to be analysed"
1130
+ "the YellowDog ID of the work requirement or task group to be compared"
1131
1131
  ),
1132
1132
  )
1133
1133
  parser.add_argument(
@@ -1135,7 +1135,7 @@ class CLIParser:
1135
1135
  metavar="<worker-pool-ID>",
1136
1136
  type=str,
1137
1137
  nargs="+",
1138
- help="the YellowDog ID(s) of the worker pool(s) to analyse",
1138
+ help="the YellowDog ID(s) of the worker pool(s) to compare",
1139
1139
  )
1140
1140
 
1141
1141
  self.args = parser.parse_args()
@@ -1681,9 +1681,9 @@ def lookup_module_description(module_name: str) -> Optional[str]:
1681
1681
  suffix = "boosting Allowances"
1682
1682
  elif "show" in module_name:
1683
1683
  suffix = "showing the JSON details of entities referenced by their YDIDs"
1684
- elif "analyse" in module_name:
1684
+ elif "compare" in module_name:
1685
1685
  suffix = (
1686
- "analysing if a work requirement or single task group is matched by "
1686
+ "comparing whether a work requirement or task group is matched by "
1687
1687
  "workers in the specified worker pools"
1688
1688
  )
1689
1689
 
@@ -51,6 +51,7 @@ class ConfigWorkRequirement:
51
51
  providers: Optional[List[str]] = None
52
52
  ram: Optional[List[float]] = None
53
53
  regions: Optional[List[str]] = None
54
+ set_task_names: bool = True
54
55
  task_batch_size: int = TASK_BATCH_SIZE_DEFAULT
55
56
  task_count: int = 1
56
57
  task_data: Optional[str] = None
@@ -95,8 +95,12 @@ def follow_events(ydid: str, ydid_type: YDIDType):
95
95
  )
96
96
 
97
97
  if response.status_code != 200:
98
- print_error(f"'{ydid}': {response.json()['message']}")
99
- return
98
+ try:
99
+ error_text = response.json()["message"]
100
+ except:
101
+ error_text = "(JSON error cannot be decoded)"
102
+ print_error(f"'{ydid}': {error_text}")
103
+ break
100
104
 
101
105
  if response.encoding is None:
102
106
  response.encoding = "utf-8"
@@ -307,6 +307,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
307
307
  providers=wr_section.get(PROVIDERS, None),
308
308
  ram=wr_section.get(RAM, None),
309
309
  regions=wr_section.get(REGIONS, None),
310
+ set_task_names=wr_section.get(SET_TASK_NAMES, True),
310
311
  task_batch_size=task_batch_size,
311
312
  task_count=task_count,
312
313
  task_data=wr_section.get(TASK_DATA, None),
@@ -60,6 +60,7 @@ RAM = "ram" # List of two Floats
60
60
  REGIONS = "regions" # List of Strings
61
61
  REQUIRED = "required" # Boolean
62
62
  SECRET = "secret" # String
63
+ SET_TASK_NAMES = "setTaskNames" # Set to False to suppress task naming
63
64
  TARGET_INSTANCE_COUNT = "targetInstanceCount" # Integer
64
65
  TASKS = "tasks" # List of Tasks
65
66
  TASKS_PER_WORKER = "tasksPerWorker" # Integer
@@ -160,6 +161,7 @@ ALL_KEYS = [
160
161
  REGIONS,
161
162
  REQUIRED,
162
163
  SECRET,
164
+ SET_TASK_NAMES,
163
165
  TARGET_INSTANCE_COUNT,
164
166
  TASKS,
165
167
  TASKS_PER_WORKER,
@@ -40,6 +40,8 @@ TABLE_TYPE_TAG = "table" + TYPE_TAG_TERMINATOR
40
40
  FORMAT_NAME_TYPE_TAG = "format_name" + TYPE_TAG_TERMINATOR
41
41
  TOML_VAR_NESTED_DEPTH = 3
42
42
 
43
+ VAR_NAME_OF_UNNAMED_TASK = "none"
44
+
43
45
  DEFAULT_LOG_WIDTH = 120
44
46
  MAX_LINES_COLOURED_FORMATTING = 1024
45
47
  ERROR_STYLE = "bold red3"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: yellowdog-python-examples
3
- Version: 7.15.2
3
+ Version: 7.15.4
4
4
  Summary: Python CLI commands using the YellowDog Python SDK
5
5
  Author-email: YellowDog Limited <support@yellowdog.co>
6
6
  Project-URL: Homepage, https://github.com/yellowdog/python-examples
@@ -14,10 +14,10 @@ tests/test_variable_processing.py
14
14
  yellowdog_cli/__init__.py
15
15
  yellowdog_cli/abort.py
16
16
  yellowdog_cli/admin.py
17
- yellowdog_cli/analyse.py
18
17
  yellowdog_cli/boost.py
19
18
  yellowdog_cli/cancel.py
20
19
  yellowdog_cli/cloudwizard.py
20
+ yellowdog_cli/compare.py
21
21
  yellowdog_cli/create.py
22
22
  yellowdog_cli/delete.py
23
23
  yellowdog_cli/download.py
@@ -1,9 +1,9 @@
1
1
  [console_scripts]
2
2
  yd-abort = yellowdog_cli.abort:main
3
- yd-analyse = yellowdog_cli.analyse:main
4
3
  yd-boost = yellowdog_cli.boost:main
5
4
  yd-cancel = yellowdog_cli.cancel:main
6
5
  yd-cloudwizard = yellowdog_cli.cloudwizard:main
6
+ yd-compare = yellowdog_cli.compare:main
7
7
  yd-create = yellowdog_cli.create:main
8
8
  yd-delete = yellowdog_cli.delete:main
9
9
  yd-download = yellowdog_cli.download:main
@@ -1 +0,0 @@
1
- __version__ = "7.15.2"