yellowdog-python-examples 7.18.0__tar.gz → 7.18.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {yellowdog_python_examples-7.18.0/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.18.2}/PKG-INFO +3 -2
  2. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/README.md +28 -21
  3. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/requirements.txt +2 -1
  4. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_demos.py +1 -1
  5. yellowdog_python_examples-7.18.2/yellowdog_cli/__init__.py +1 -0
  6. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/list.py +3 -0
  7. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/show.py +3 -0
  8. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/args.py +13 -0
  9. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/config_types.py +1 -1
  10. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/load_config.py +32 -1
  11. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/printing.py +7 -0
  12. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/property_names.py +8 -8
  13. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/submit_utils.py +9 -9
  14. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/variables.py +2 -0
  15. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2/yellowdog_python_examples.egg-info}/PKG-INFO +3 -2
  16. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/requires.txt +2 -1
  17. yellowdog_python_examples-7.18.0/yellowdog_cli/__init__.py +0 -1
  18. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/LICENSE +0 -0
  19. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/PYPI_README.md +0 -0
  20. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/pyproject.toml +0 -0
  21. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/setup.cfg +0 -0
  22. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_create_remove.py +0 -0
  23. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_dryruns.py +0 -0
  24. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_entrypoints.py +0 -0
  25. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_gui.py +0 -0
  26. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_list.py +0 -0
  27. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_objects.py +0 -0
  28. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_variable_processing.py +0 -0
  29. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/abort.py +0 -0
  30. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/admin.py +0 -0
  31. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/boost.py +0 -0
  32. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/cancel.py +0 -0
  33. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/cloudwizard.py +0 -0
  34. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/compare.py +0 -0
  35. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/create.py +0 -0
  36. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/delete.py +0 -0
  37. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/download.py +0 -0
  38. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/follow.py +0 -0
  39. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/format_json.py +0 -0
  40. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/hold.py +0 -0
  41. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/instantiate.py +0 -0
  42. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/jsonnet2json.py +0 -0
  43. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/provision.py +0 -0
  44. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/remove.py +0 -0
  45. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/resize.py +0 -0
  46. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/shutdown.py +0 -0
  47. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/start.py +0 -0
  48. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/submit.py +0 -0
  49. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/terminate.py +0 -0
  50. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/upload.py +0 -0
  51. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/__init__.py +0 -0
  52. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/check_imports.py +0 -0
  53. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_aws.py +0 -0
  54. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
  55. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
  56. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
  57. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
  58. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/compact_json.py +0 -0
  59. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/csv_data.py +0 -0
  60. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/entity_utils.py +0 -0
  61. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/follow_utils.py +0 -0
  62. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/interactive.py +0 -0
  63. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/items.py +0 -0
  64. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/load_resources.py +0 -0
  65. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/misc_utils.py +0 -0
  66. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/provision_utils.py +0 -0
  67. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
  68. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/settings.py +0 -0
  69. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/start_hold_common.py +0 -0
  70. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/type_check.py +0 -0
  71. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/upload_utils.py +0 -0
  72. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/validate_properties.py +0 -0
  73. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/wrapper.py +0 -0
  74. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/ydid_utils.py +0 -0
  75. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/version.py +0 -0
  76. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/SOURCES.txt +0 -0
  77. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
  78. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/entry_points.txt +0 -0
  79. {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: yellowdog-python-examples
3
- Version: 7.18.0
3
+ Version: 7.18.2
4
4
  Summary: Python CLI commands using the YellowDog Python SDK
5
5
  Author-email: YellowDog Limited <support@yellowdog.co>
6
6
  Project-URL: Homepage, https://github.com/yellowdog/python-examples
@@ -15,11 +15,12 @@ Description-Content-Type: text/markdown
15
15
  License-File: LICENSE
16
16
  Requires-Dist: PyPAC>=0.16.4
17
17
  Requires-Dist: dateparser
18
+ Requires-Dist: dotenv
18
19
  Requires-Dist: requests
19
20
  Requires-Dist: rich==13.9.4
20
21
  Requires-Dist: tabulate>=0.9.0
21
22
  Requires-Dist: toml
22
- Requires-Dist: yellowdog-sdk>=9.2.3
23
+ Requires-Dist: yellowdog-sdk>=9.3.3
23
24
  Provides-Extra: jsonnet
24
25
  Requires-Dist: jsonnet; extra == "jsonnet"
25
26
  Provides-Extra: cloudwizard
@@ -161,7 +161,7 @@ The commands provide the following capabilities:
161
161
  - **Listing** YellowDog items using the **`yd-list`** command
162
162
  - **Resizing** Worker Pools and Compute Requirements with the **`yd-resize`** command
163
163
  - **Boosting** Allowances with the **`yd-boost`** command
164
- - **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, and Namespace Policies with the **`yd-create`** and **`yd-remove`** commands
164
+ - **Creating, Updating and Removing** Compute Source Templates, Compute Requirement Templates, Keyrings, Credentials, Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, Namespace Policies, Groups, and Applications with the **`yd-create`** and **`yd-remove`** commands
165
165
  - **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
166
166
  - **Showing** the details of any YellowDog entity using its YellowDog ID with the **`yd-show`** command
167
167
  - **Comparing** whether worker pools are a match for task groups with the **`yd-compare`** command
@@ -328,7 +328,7 @@ An example `common` section is shown below:
328
328
 
329
329
  Indentation is optional in TOML files and is for readability only.
330
330
 
331
- ## Importing `common` properties
331
+ ## Importing `common` Properties
332
332
 
333
333
  The `common` section can import properties from a separate TOML file, using the `importCommon` property. For example, the `key` and `secret` might be in a shared TOML file called `app_credentials.toml`, with the following contents:
334
334
 
@@ -383,6 +383,12 @@ When setting the value of the above properties, a property set on the command li
383
383
 
384
384
  If all the required common properties are set using the command line or environment variables, then the entire `common` section of the TOML file can be omitted.
385
385
 
386
+ ## Support for `.env` Files
387
+
388
+ Environment variables can also be set in a `.env` file, typically in the user's home directory. Entries in the `.env` file will not overwrite existing environment variables -- i.e., environment variables take precedence over entries in the `.env` file.
389
+
390
+ Environment variables sourced from a `.env` file whose names start with `YD` will be reported on the command line. Variables that do not start with `YD` will not be reported, but they will still be applied.
391
+
386
392
  ## Variable Substitutions in Common Properties
387
393
 
388
394
  Note the use of `{{username}}` in the value of the `tag` property example above: this is a **variable substitution** that can optionally be used to insert the login username of the user running the commands. So, for username `abc`, the `tag` would be set to `testing-abc`. This can be helpful to disambiguate multiple users running with the same configuration data.
@@ -464,7 +470,8 @@ The precedence order for setting variables is:
464
470
  1. Command line
465
471
  2. `YD_VAR` environment variables
466
472
  3. General environment variables
467
- 4. TOML file
473
+ 4. Variables in a `.env` file
474
+ 5. TOML file
468
475
 
469
476
  This method can also be used to override the default variables, e.g., setting `-v username="other-user"` will override the default `{{username}}` variable.
470
477
 
@@ -619,6 +626,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
619
626
  | `providers` | Constrains the YellowDog Scheduler only to execute tasks from the associated Task Group on the specified providers. E.g., `["AWS", "GOOGLE"]`. | Yes | Yes | Yes | |
620
627
  | `ram` | Range constraint on GB of RAM that are required to execute Tasks. E.g., `[2.5, 4.0]`. | Yes | Yes | Yes | |
621
628
  | `regions` | Constrains the YellowDog Scheduler only to execute Tasks from the associated Task Group in the specified regions. E.g., `["eu-west-2]`. | Yes | Yes | Yes | |
629
+ | `retryableErrors` | A list of error condition combinations under which Tasks will be retried (up to `maximumTaskRetries`). Retries will always be attempted if the list is empty (the default). See the TOML/JSON section for examples. | Yes | Yes | Yes | |
622
630
  | `setTaskNames` | Set this to `false` to suppress automatic generation of Task names. Defaults to `true`. Task names that are set by the user will still be observed. Note that Task names must be set if any outputs are specified. | Yes | Yes | Yes | Yes |
623
631
  | `tag` | A tag that can be associated with a Work Requirement, Task Group or Task. Note there is **no property inheritance** for these tags. | Yes | Yes | Yes | Yes |
624
632
  | `taskBatchSize` | Determines the batch size used to add Tasks to Task Groups. Default is 2,000. | Yes | | | |
@@ -630,7 +638,6 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
630
638
  | `taskName` | The name to use for the Task. Only usable in the TOML file. Mostly useful in conjunction with CSV Task data. E.g., `"my_task_number_{{task_number}}"`. | Yes | | | |
631
639
  | `taskGroupCount` | Create `taskGroupCount` duplicates of a single Task Group. | Yes | Yes | | |
632
640
  | `taskGroupName` | The name to use for the Task Group. Only usable in the TOML file. E.g., `"my_tg_number_{{task_group_number}}"`. | Yes | | | |
633
- | `taskRetryErrorMatchers` | A list of error condition combinationss under which Tasks will be retried (up to `maximumTaskRetries`). Retries will always be attempted if the list is empty (the default). See the TOML/JSON section for examples. | Yes | Yes | Yes | |
634
641
  | `taskTimeout` | The timeout in minutes after which an executing Task will be terminated and reported as `FAILED`. E.g. `120.0`. The default is no timeout. | Yes | Yes | Yes | |
635
642
  | `timeout` | As above, but set at the individual Task level, which overrides the group level `taskTimeout` property (if present). | Yes | | | Yes |
636
643
  | `taskType` | The Task Type of a Task. E.g., `"docker"`. | Yes | | | Yes |
@@ -824,6 +831,9 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
824
831
  providers = ["AWS"]
825
832
  ram = [0.5, 2.0]
826
833
  regions = ["eu-west-2"]
834
+ retryableErrors = [
835
+ {processExitCodes = [143], statusesAtFailure = ["FAILED"], errorTypes = ["ALLOCATION_LOST"]},
836
+ ]
827
837
  setTaskNames = false
828
838
  tag = "my_tag"
829
839
  taskBatchSize = 1000
@@ -844,9 +854,6 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
844
854
  taskTimeout = 120.0
845
855
  taskType = "docker"
846
856
  tasksPerWorker = 1
847
- taskRetryErrorMatchers = [
848
- {taskRetryErrorExitCodes = [143], taskRetryErrorStatuses = ["FAILED"], taskRetryErrorTypes = ["ALLOCATION_LOST"]},
849
- ]
850
857
  uploadFiles = [{localPath = "file_1.txt", uploadPath = "file_1.txt"}]
851
858
  uploadTaskProcessOutput = true
852
859
  vcpus = [1, 4]
@@ -892,6 +899,13 @@ Showing all possible properties at the Work Requirement level:
892
899
  "providers": ["AWS"],
893
900
  "ram": [0.5, 2],
894
901
  "regions": ["eu-west-2"],
902
+ "retryableErrors": [
903
+ {
904
+ "processExitCodes": [143],
905
+ "statusesAtFailure" : ["FAILED"],
906
+ "errorTypes": ["ALLOCATION_LOST"]
907
+ }
908
+ ]
895
909
  "setTaskNames": false,
896
910
  "tag": "my_tag"
897
911
  "taskCount": 100,
@@ -906,13 +920,6 @@ Showing all possible properties at the Work Requirement level:
906
920
  {"alwaysUpload": false, "destination": "dest_path_2", "source": "out_src_path_2"}
907
921
  ],
908
922
  "taskGroupCount": 5,
909
- "taskRetryErrorMatchers": [
910
- {
911
- "taskRetryErrorExitCodes": [143],
912
- "taskRetryErrorStatuses" : ["FAILED"],
913
- "taskRetryErrorTypes": ["ALLOCATION_LOST"]
914
- }
915
- ]
916
923
  "taskTimeout": 120.0,
917
924
  "taskTypes": ["docker"],
918
925
  "tasksPerWorker": 1,
@@ -970,6 +977,13 @@ Showing all possible properties at the Task Group level:
970
977
  "providers": ["AWS"],
971
978
  "ram": [0.5, 2],
972
979
  "regions": ["eu-west-2"],
980
+ "retryableErrors": [
981
+ {
982
+ "processExitCodes": [143],
983
+ "statusesAtFailure" : ["FAILED"],
984
+ "errorTypes": ["ALLOCATION_LOST"]
985
+ }
986
+ ]
973
987
  "setTaskNames": false,
974
988
  "tag": "my_tag",
975
989
  "taskCount": 5,
@@ -983,13 +997,6 @@ Showing all possible properties at the Task Group level:
983
997
  {"alwaysUpload": true, "destination": "dest_path_1", "source": "out_src_path_1"},
984
998
  {"alwaysUpload": false, "destination": "dest_path_2", "source": "out_src_path_2"}
985
999
  ],
986
- "taskRetryErrorMatchers": [
987
- {
988
- "taskRetryErrorExitCodes": [143],
989
- "taskRetryErrorStatuses" : ["FAILED"],
990
- "taskRetryErrorTypes": ["ALLOCATION_LOST"]
991
- }
992
- ]
993
1000
  "taskTimeout": 120.0,
994
1001
  "taskTypes": ["docker"],
995
1002
  "tasksPerWorker": 1,
@@ -1,7 +1,8 @@
1
1
  PyPAC >= 0.16.4
2
2
  dateparser
3
+ dotenv
3
4
  requests
4
5
  rich == 13.9.4
5
6
  tabulate >= 0.9.0
6
7
  toml
7
- yellowdog-sdk >= 9.2.3
8
+ yellowdog-sdk >= 9.3.3
@@ -63,7 +63,7 @@ class TestDemos:
63
63
 
64
64
  def test_montecarlo_graviton(self):
65
65
  result = shell(
66
- f"cd {DEMO_DIR}/montecarlo && yd-provision -c config_graviton.toml && yd-submit -f && yd-terminate -y && yd-delete -y"
66
+ f"cd {DEMO_DIR}/montecarlo && yd-provision -c config-graviton.toml && yd-submit -f && yd-terminate -y && yd-delete -y"
67
67
  )
68
68
  assert result.exit_code == 0
69
69
 
@@ -0,0 +1 @@
1
+ __version__ = "7.18.2"
@@ -93,6 +93,9 @@ def main():
93
93
  # Always use interactive mode for selections
94
94
  ARGS_PARSER.interactive = True
95
95
 
96
+ if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
97
+ print_log("Omitting YellowDog IDs from detailed JSON objects")
98
+
96
99
  if ARGS_PARSER.output_file and ARGS_PARSER.details:
97
100
  if exists(ARGS_PARSER.output_file):
98
101
  if not confirmed(
@@ -33,6 +33,9 @@ def main():
33
33
  # and the 'quiet' option is enabled
34
34
  generate_json_list = len(ARGS_PARSER.yellowdog_ids) > 1 and ARGS_PARSER.quiet
35
35
 
36
+ if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
37
+ print_log("Omitting YellowDog IDs from detailed JSON objects")
38
+
36
39
  if generate_json_list:
37
40
  print("[")
38
41
 
@@ -1169,6 +1169,14 @@ class CLIParser:
1169
1169
  "names in detailed compute requirement templates"
1170
1170
  ),
1171
1171
  )
1172
+ parser.add_argument(
1173
+ "--strip-ids",
1174
+ action="store_true",
1175
+ required=False,
1176
+ help=(
1177
+ "don't include objects' YellowDog IDs in their JSON representation"
1178
+ ),
1179
+ )
1172
1180
 
1173
1181
  if "compare" in sys.argv[0]:
1174
1182
  parser.add_argument(
@@ -1734,6 +1742,11 @@ class CLIParser:
1734
1742
  def running_nodes_only(self) -> Optional[bool]:
1735
1743
  return self.args.running_nodes_only
1736
1744
 
1745
+ @property
1746
+ @allow_missing_attribute
1747
+ def strip_ids(self) -> Optional[bool]:
1748
+ return self.args.strip_ids
1749
+
1737
1750
 
1738
1751
  def lookup_module_description(module_name: str) -> Optional[str]:
1739
1752
  """
@@ -52,6 +52,7 @@ class ConfigWorkRequirement:
52
52
  providers: Optional[List[str]] = None
53
53
  ram: Optional[List[float]] = None
54
54
  regions: Optional[List[str]] = None
55
+ retryable_errors: Optional[List[Dict]] = None
55
56
  set_task_names: bool = True
56
57
  task_batch_size: int = TASK_BATCH_SIZE_DEFAULT
57
58
  task_count: int = 1
@@ -63,7 +64,6 @@ class ConfigWorkRequirement:
63
64
  task_group_name: Optional[str] = None
64
65
  task_level_timeout: Optional[float] = None
65
66
  task_name: Optional[str] = None
66
- task_retry_error_matchers: Optional[List[Dict]] = None
67
67
  task_timeout: Optional[float] = None
68
68
  task_type: Optional[str] = None
69
69
  tasks_per_worker: Optional[int] = None
@@ -9,6 +9,7 @@ from pathlib import Path
9
9
  from sys import exit
10
10
  from typing import Dict
11
11
 
12
+ from dotenv import dotenv_values, find_dotenv, load_dotenv
12
13
  from toml import TomlDecodeError
13
14
 
14
15
  from yellowdog_cli.utils.args import ARGS_PARSER
@@ -101,6 +102,10 @@ def load_config_common() -> ConfigCommon:
101
102
  common_section_imported.update(common_section)
102
103
  common_section = common_section_imported
103
104
 
105
+ # Load extra environment variables from a .env file if it exists;
106
+ # do not override existing variables (environment takes precedence)
107
+ _load_dotenv()
108
+
104
109
  # Replace common section properties with command line or
105
110
  # environment variable overrides. Precedence is:
106
111
  # command line > environment variable > config file
@@ -308,6 +313,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
308
313
  providers=wr_section.get(PROVIDERS, None),
309
314
  ram=wr_section.get(RAM, None),
310
315
  regions=wr_section.get(REGIONS, None),
316
+ retryable_errors=wr_section.get(RETRYABLE_ERRORS, None),
311
317
  set_task_names=wr_section.get(SET_TASK_NAMES, True),
312
318
  task_batch_size=task_batch_size,
313
319
  task_count=task_count,
@@ -318,7 +324,6 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
318
324
  task_group_count=task_group_count,
319
325
  task_group_name=wr_section.get(TASK_GROUP_NAME, None),
320
326
  task_name=wr_section.get(TASK_NAME, None),
321
- task_retry_error_matchers=wr_section.get(TASK_RETRY_ERROR_MATCHERS, None),
322
327
  task_timeout=wr_section.get(TASK_TIMEOUT, None),
323
328
  task_type=task_type,
324
329
  tasks_per_worker=wr_section.get(TASKS_PER_WORKER, None),
@@ -445,3 +450,29 @@ def load_config_worker_pool() -> ConfigWorkerPool:
445
450
  except ValueError as e:
446
451
  print_error(f"Invalid type for configuration: {e}")
447
452
  exit(1)
453
+
454
+
455
+ def _load_dotenv():
456
+ """
457
+ Load extra environment variables from a .env file if it exists.
458
+ Do not override existing variables (environment takes precedence).
459
+ Report on YD vars that are taken from .env.
460
+ """
461
+ dotenv_file = find_dotenv()
462
+ if dotenv_file == "":
463
+ return
464
+
465
+ dotenv_yd_substitutions = [ # Find 'YD' variables
466
+ f"'{key}'"
467
+ for key in dotenv_values(dotenv_file).keys()
468
+ if key.startswith("YD") and os.environ.get(key) is None
469
+ ]
470
+
471
+ if len(dotenv_yd_substitutions) > 0:
472
+ print_log(
473
+ f"Adding 'YD' environment variables from '.env' file '{dotenv_file}': "
474
+ f"{', '.join(dotenv_yd_substitutions)}"
475
+ )
476
+
477
+ # Actually load the variables (including non-'YD' variables)
478
+ load_dotenv(dotenv_file, override=False)
@@ -71,6 +71,7 @@ from yellowdog_cli.utils.settings import (
71
71
  MAX_LINES_COLOURED_FORMATTING,
72
72
  MAX_TABLE_DESCRIPTION,
73
73
  NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR,
74
+ PROP_ID,
74
75
  PROP_RESOURCE,
75
76
  WARNING_STYLE,
76
77
  )
@@ -1062,6 +1063,11 @@ def print_yd_object(
1062
1063
  using the compact JSON encoder.
1063
1064
  """
1064
1065
  object_data: object = Json.dump(yd_object)
1066
+
1067
+ # Remove the 'id' property?
1068
+ if ARGS_PARSER.strip_ids:
1069
+ object_data.pop(PROP_ID, None)
1070
+
1065
1071
  if add_fields is not None:
1066
1072
  # Requires a copy of the 'object' datatype to be made,
1067
1073
  # in order to insert additional fields
@@ -1071,6 +1077,7 @@ def print_yd_object(
1071
1077
  for key, value in add_fields.items():
1072
1078
  object_data_new[key] = value
1073
1079
  object_data = object_data_new
1080
+
1074
1081
  print_json(object_data, initial_indent, drop_first_line, with_final_comma)
1075
1082
 
1076
1083
 
@@ -22,6 +22,7 @@ DOCKER_PASSWORD = "dockerPassword" # String
22
22
  DOCKER_REGISTRY = "dockerRegistry" # String
23
23
  DOCKER_USERNAME = "dockerUsername" # String
24
24
  ENV = "environment" # Dictionary
25
+ ERROR_TYPES = "errorTypes" # List of Strings
25
26
  EXCLUSIVE_WORKERS = "exclusiveWorkers" # Boolean
26
27
  EXECUTABLE = "executable" # String
27
28
  FILE_PATTERN = "filePattern" # String
@@ -56,12 +57,15 @@ OUTPUTS_OTHER = "outputsOther" # List of Dicts
56
57
  OUTPUTS_REQUIRED = "outputsRequired" # List of Strings
57
58
  PARALLEL_BATCHES = "parallelBatches" # Integer
58
59
  PRIORITY = "priority" # Float
60
+ PROCESS_EXIT_CODES = "processExitCodes" # List of Ints
59
61
  PROVIDERS = "providers" # List of Strings
60
62
  RAM = "ram" # List of two Floats
61
63
  REGIONS = "regions" # List of Strings
62
64
  REQUIRED = "required" # Boolean
65
+ RETRYABLE_ERRORS = "retryableErrors" # List of Dicts
63
66
  SECRET = "secret" # String
64
67
  SET_TASK_NAMES = "setTaskNames" # Set to False to suppress task naming
68
+ STATUSES_AT_FAILURE = "statusesAtFailure" # List of Strings
65
69
  TARGET_INSTANCE_COUNT = "targetInstanceCount" # Integer
66
70
  TASKS = "tasks" # List of Tasks
67
71
  TASKS_PER_WORKER = "tasksPerWorker" # Integer
@@ -79,10 +83,6 @@ TASK_GROUP_NAME = "taskGroupName" # String
79
83
  TASK_GROUP_TAG = "tag" # String
80
84
  TASK_LEVEL_TIMEOUT = "timeout" # Float
81
85
  TASK_NAME = "taskName" # String
82
- TASK_RETRY_ERROR_EXIT_CODES = "taskRetryErrorExitCodes" # List of Ints
83
- TASK_RETRY_ERROR_MATCHERS = "taskRetryErrorMatchers" # List of Dicts
84
- TASK_RETRY_ERROR_STATUSES = "taskRetryErrorStatuses" # List of Strings
85
- TASK_RETRY_ERROR_TYPES = "taskRetryErrorTypes" # List of Strings
86
86
  TASK_TAG = "tag" # String
87
87
  TASK_TIMEOUT = "taskTimeout" # Float
88
88
  TASK_TYPE = "taskType" # String
@@ -135,6 +135,7 @@ ALL_KEYS = [
135
135
  DOCKER_REGISTRY,
136
136
  DOCKER_USERNAME,
137
137
  ENV,
138
+ ERROR_TYPES,
138
139
  EXCLUSIVE_WORKERS,
139
140
  EXECUTABLE,
140
141
  FILE_PATTERN,
@@ -168,12 +169,15 @@ ALL_KEYS = [
168
169
  OUTPUTS_REQUIRED,
169
170
  PARALLEL_BATCHES,
170
171
  PRIORITY,
172
+ PROCESS_EXIT_CODES,
171
173
  PROVIDERS,
172
174
  RAM,
173
175
  REGIONS,
174
176
  REQUIRED,
177
+ RETRYABLE_ERRORS,
175
178
  SECRET,
176
179
  SET_TASK_NAMES,
180
+ STATUSES_AT_FAILURE,
177
181
  TARGET_INSTANCE_COUNT,
178
182
  TASKS,
179
183
  TASKS_PER_WORKER,
@@ -191,10 +195,6 @@ ALL_KEYS = [
191
195
  TASK_GROUP_TAG,
192
196
  TASK_LEVEL_TIMEOUT,
193
197
  TASK_NAME,
194
- TASK_RETRY_ERROR_EXIT_CODES,
195
- TASK_RETRY_ERROR_MATCHERS,
196
- TASK_RETRY_ERROR_STATUSES,
197
- TASK_RETRY_ERROR_TYPES,
198
198
  TASK_TAG,
199
199
  TASK_TIMEOUT,
200
200
  TASK_TYPE,
@@ -24,10 +24,10 @@ from yellowdog_client.model import (
24
24
  from yellowdog_cli.utils.config_types import ConfigCommon, ConfigWorkRequirement
25
25
  from yellowdog_cli.utils.printing import print_error, print_log
26
26
  from yellowdog_cli.utils.property_names import (
27
- TASK_RETRY_ERROR_EXIT_CODES,
28
- TASK_RETRY_ERROR_MATCHERS,
29
- TASK_RETRY_ERROR_STATUSES,
30
- TASK_RETRY_ERROR_TYPES,
27
+ ERROR_TYPES,
28
+ PROCESS_EXIT_CODES,
29
+ RETRYABLE_ERRORS,
30
+ STATUSES_AT_FAILURE,
31
31
  )
32
32
  from yellowdog_cli.utils.settings import NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR
33
33
  from yellowdog_cli.utils.type_check import check_list
@@ -350,8 +350,8 @@ def generate_task_error_matchers_list(
350
350
  """
351
351
  error_matchers: Optional[List[Dict]] = check_list(
352
352
  tg_data.get(
353
- TASK_RETRY_ERROR_MATCHERS,
354
- wr_data.get(TASK_RETRY_ERROR_MATCHERS, config_wr.task_retry_error_matchers),
353
+ RETRYABLE_ERRORS,
354
+ wr_data.get(RETRYABLE_ERRORS, config_wr.retryable_errors),
355
355
  )
356
356
  )
357
357
 
@@ -372,7 +372,7 @@ def _generate_task_error_matcher(task_error_matcher_data: Dict) -> TaskErrorMatc
372
372
  try:
373
373
 
374
374
  exit_codes_str: Optional[List[int]] = check_list(
375
- task_error_matcher_data.get(TASK_RETRY_ERROR_EXIT_CODES, None)
375
+ task_error_matcher_data.get(PROCESS_EXIT_CODES, None)
376
376
  )
377
377
  try:
378
378
  # Ensure ints
@@ -385,7 +385,7 @@ def _generate_task_error_matcher(task_error_matcher_data: Dict) -> TaskErrorMatc
385
385
  raise Exception(f"Unable to process error exit codes: {e}")
386
386
 
387
387
  statuses_str: Optional[List[str]] = check_list(
388
- task_error_matcher_data.get(TASK_RETRY_ERROR_STATUSES, None)
388
+ task_error_matcher_data.get(STATUSES_AT_FAILURE, None)
389
389
  )
390
390
  try:
391
391
  statuses = (
@@ -397,7 +397,7 @@ def _generate_task_error_matcher(task_error_matcher_data: Dict) -> TaskErrorMatc
397
397
  raise Exception(f"Unable to process error status: {e}")
398
398
 
399
399
  error_types: Optional[List[str]] = check_list(
400
- task_error_matcher_data.get(TASK_RETRY_ERROR_TYPES, None)
400
+ task_error_matcher_data.get(ERROR_TYPES, None)
401
401
  )
402
402
 
403
403
  return TaskErrorMatcher(
@@ -397,6 +397,8 @@ def process_typed_variable_substitution(
397
397
  f"(Use JSON syntax?) ({e})"
398
398
  )
399
399
 
400
+ return None
401
+
400
402
 
401
403
  def resolve_filename(files_directory: str, filename: str) -> str:
402
404
  """
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: yellowdog-python-examples
3
- Version: 7.18.0
3
+ Version: 7.18.2
4
4
  Summary: Python CLI commands using the YellowDog Python SDK
5
5
  Author-email: YellowDog Limited <support@yellowdog.co>
6
6
  Project-URL: Homepage, https://github.com/yellowdog/python-examples
@@ -15,11 +15,12 @@ Description-Content-Type: text/markdown
15
15
  License-File: LICENSE
16
16
  Requires-Dist: PyPAC>=0.16.4
17
17
  Requires-Dist: dateparser
18
+ Requires-Dist: dotenv
18
19
  Requires-Dist: requests
19
20
  Requires-Dist: rich==13.9.4
20
21
  Requires-Dist: tabulate>=0.9.0
21
22
  Requires-Dist: toml
22
- Requires-Dist: yellowdog-sdk>=9.2.3
23
+ Requires-Dist: yellowdog-sdk>=9.3.3
23
24
  Provides-Extra: jsonnet
24
25
  Requires-Dist: jsonnet; extra == "jsonnet"
25
26
  Provides-Extra: cloudwizard
@@ -1,10 +1,11 @@
1
1
  PyPAC>=0.16.4
2
2
  dateparser
3
+ dotenv
3
4
  requests
4
5
  rich==13.9.4
5
6
  tabulate>=0.9.0
6
7
  toml
7
- yellowdog-sdk>=9.2.3
8
+ yellowdog-sdk>=9.3.3
8
9
 
9
10
  [cloudwizard]
10
11
  boto3
@@ -1 +0,0 @@
1
- __version__ = "7.18.0"