yellowdog-python-examples 7.18.0__tar.gz → 7.18.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {yellowdog_python_examples-7.18.0/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.18.2}/PKG-INFO +3 -2
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/README.md +28 -21
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/requirements.txt +2 -1
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_demos.py +1 -1
- yellowdog_python_examples-7.18.2/yellowdog_cli/__init__.py +1 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/list.py +3 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/show.py +3 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/args.py +13 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/config_types.py +1 -1
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/load_config.py +32 -1
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/printing.py +7 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/property_names.py +8 -8
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/submit_utils.py +9 -9
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/variables.py +2 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2/yellowdog_python_examples.egg-info}/PKG-INFO +3 -2
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/requires.txt +2 -1
- yellowdog_python_examples-7.18.0/yellowdog_cli/__init__.py +0 -1
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/LICENSE +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/PYPI_README.md +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/pyproject.toml +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/setup.cfg +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_create_remove.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_dryruns.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_entrypoints.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_gui.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_list.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_objects.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_variable_processing.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/abort.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/admin.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/boost.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/cancel.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/cloudwizard.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/compare.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/create.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/delete.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/download.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/follow.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/format_json.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/hold.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/instantiate.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/jsonnet2json.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/provision.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/remove.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/resize.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/shutdown.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/start.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/submit.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/terminate.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/upload.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/__init__.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/check_imports.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_aws.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_aws_types.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_azure.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_common.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/cloudwizard_gcp.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/compact_json.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/csv_data.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/entity_utils.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/follow_utils.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/interactive.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/items.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/load_resources.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/misc_utils.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/provision_utils.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/rich_console_input_fixed.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/settings.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/start_hold_common.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/type_check.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/upload_utils.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/validate_properties.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/wrapper.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/ydid_utils.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/version.py +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/SOURCES.txt +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/dependency_links.txt +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/entry_points.txt +0 -0
- {yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.18.
|
|
3
|
+
Version: 7.18.2
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
6
|
Project-URL: Homepage, https://github.com/yellowdog/python-examples
|
|
@@ -15,11 +15,12 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Requires-Dist: PyPAC>=0.16.4
|
|
17
17
|
Requires-Dist: dateparser
|
|
18
|
+
Requires-Dist: dotenv
|
|
18
19
|
Requires-Dist: requests
|
|
19
20
|
Requires-Dist: rich==13.9.4
|
|
20
21
|
Requires-Dist: tabulate>=0.9.0
|
|
21
22
|
Requires-Dist: toml
|
|
22
|
-
Requires-Dist: yellowdog-sdk>=9.
|
|
23
|
+
Requires-Dist: yellowdog-sdk>=9.3.3
|
|
23
24
|
Provides-Extra: jsonnet
|
|
24
25
|
Requires-Dist: jsonnet; extra == "jsonnet"
|
|
25
26
|
Provides-Extra: cloudwizard
|
|
@@ -161,7 +161,7 @@ The commands provide the following capabilities:
|
|
|
161
161
|
- **Listing** YellowDog items using the **`yd-list`** command
|
|
162
162
|
- **Resizing** Worker Pools and Compute Requirements with the **`yd-resize`** command
|
|
163
163
|
- **Boosting** Allowances with the **`yd-boost`** command
|
|
164
|
-
- **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials,
|
|
164
|
+
- **Creating, Updating and Removing** Compute Source Templates, Compute Requirement Templates, Keyrings, Credentials, Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, Namespace Policies, Groups, and Applications with the **`yd-create`** and **`yd-remove`** commands
|
|
165
165
|
- **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
|
|
166
166
|
- **Showing** the details of any YellowDog entity using its YellowDog ID with the **`yd-show`** command
|
|
167
167
|
- **Comparing** whether worker pools are a match for task groups with the **`yd-compare`** command
|
|
@@ -328,7 +328,7 @@ An example `common` section is shown below:
|
|
|
328
328
|
|
|
329
329
|
Indentation is optional in TOML files and is for readability only.
|
|
330
330
|
|
|
331
|
-
## Importing `common`
|
|
331
|
+
## Importing `common` Properties
|
|
332
332
|
|
|
333
333
|
The `common` section can import properties from a separate TOML file, using the `importCommon` property. For example, the `key` and `secret` might be in a shared TOML file called `app_credentials.toml`, with the following contents:
|
|
334
334
|
|
|
@@ -383,6 +383,12 @@ When setting the value of the above properties, a property set on the command li
|
|
|
383
383
|
|
|
384
384
|
If all the required common properties are set using the command line or environment variables, then the entire `common` section of the TOML file can be omitted.
|
|
385
385
|
|
|
386
|
+
## Support for `.env` Files
|
|
387
|
+
|
|
388
|
+
Environment variables can also be set in a `.env` file, typically in the user's home directory. Entries in the `.env` file will not overwrite existing environment variables -- i.e., environment variables take precedence over entries in the `.env` file.
|
|
389
|
+
|
|
390
|
+
Environment variables sourced from a `.env` file whose names start with `YD` will be reported on the command line. Variables that do not start with `YD` will not be reported, but they will still be applied.
|
|
391
|
+
|
|
386
392
|
## Variable Substitutions in Common Properties
|
|
387
393
|
|
|
388
394
|
Note the use of `{{username}}` in the value of the `tag` property example above: this is a **variable substitution** that can optionally be used to insert the login username of the user running the commands. So, for username `abc`, the `tag` would be set to `testing-abc`. This can be helpful to disambiguate multiple users running with the same configuration data.
|
|
@@ -464,7 +470,8 @@ The precedence order for setting variables is:
|
|
|
464
470
|
1. Command line
|
|
465
471
|
2. `YD_VAR` environment variables
|
|
466
472
|
3. General environment variables
|
|
467
|
-
4.
|
|
473
|
+
4. Variables in a `.env` file
|
|
474
|
+
5. TOML file
|
|
468
475
|
|
|
469
476
|
This method can also be used to override the default variables, e.g., setting `-v username="other-user"` will override the default `{{username}}` variable.
|
|
470
477
|
|
|
@@ -619,6 +626,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
|
|
|
619
626
|
| `providers` | Constrains the YellowDog Scheduler only to execute tasks from the associated Task Group on the specified providers. E.g., `["AWS", "GOOGLE"]`. | Yes | Yes | Yes | |
|
|
620
627
|
| `ram` | Range constraint on GB of RAM that are required to execute Tasks. E.g., `[2.5, 4.0]`. | Yes | Yes | Yes | |
|
|
621
628
|
| `regions` | Constrains the YellowDog Scheduler only to execute Tasks from the associated Task Group in the specified regions. E.g., `["eu-west-2]`. | Yes | Yes | Yes | |
|
|
629
|
+
| `retryableErrors` | A list of error condition combinations under which Tasks will be retried (up to `maximumTaskRetries`). Retries will always be attempted if the list is empty (the default). See the TOML/JSON section for examples. | Yes | Yes | Yes | |
|
|
622
630
|
| `setTaskNames` | Set this to `false` to suppress automatic generation of Task names. Defaults to `true`. Task names that are set by the user will still be observed. Note that Task names must be set if any outputs are specified. | Yes | Yes | Yes | Yes |
|
|
623
631
|
| `tag` | A tag that can be associated with a Work Requirement, Task Group or Task. Note there is **no property inheritance** for these tags. | Yes | Yes | Yes | Yes |
|
|
624
632
|
| `taskBatchSize` | Determines the batch size used to add Tasks to Task Groups. Default is 2,000. | Yes | | | |
|
|
@@ -630,7 +638,6 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
|
|
|
630
638
|
| `taskName` | The name to use for the Task. Only usable in the TOML file. Mostly useful in conjunction with CSV Task data. E.g., `"my_task_number_{{task_number}}"`. | Yes | | | |
|
|
631
639
|
| `taskGroupCount` | Create `taskGroupCount` duplicates of a single Task Group. | Yes | Yes | | |
|
|
632
640
|
| `taskGroupName` | The name to use for the Task Group. Only usable in the TOML file. E.g., `"my_tg_number_{{task_group_number}}"`. | Yes | | | |
|
|
633
|
-
| `taskRetryErrorMatchers` | A list of error condition combinationss under which Tasks will be retried (up to `maximumTaskRetries`). Retries will always be attempted if the list is empty (the default). See the TOML/JSON section for examples. | Yes | Yes | Yes | |
|
|
634
641
|
| `taskTimeout` | The timeout in minutes after which an executing Task will be terminated and reported as `FAILED`. E.g. `120.0`. The default is no timeout. | Yes | Yes | Yes | |
|
|
635
642
|
| `timeout` | As above, but set at the individual Task level, which overrides the group level `taskTimeout` property (if present). | Yes | | | Yes |
|
|
636
643
|
| `taskType` | The Task Type of a Task. E.g., `"docker"`. | Yes | | | Yes |
|
|
@@ -824,6 +831,9 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
|
|
|
824
831
|
providers = ["AWS"]
|
|
825
832
|
ram = [0.5, 2.0]
|
|
826
833
|
regions = ["eu-west-2"]
|
|
834
|
+
retryableErrors = [
|
|
835
|
+
{processExitCodes = [143], statusesAtFailure = ["FAILED"], errorTypes = ["ALLOCATION_LOST"]},
|
|
836
|
+
]
|
|
827
837
|
setTaskNames = false
|
|
828
838
|
tag = "my_tag"
|
|
829
839
|
taskBatchSize = 1000
|
|
@@ -844,9 +854,6 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
|
|
|
844
854
|
taskTimeout = 120.0
|
|
845
855
|
taskType = "docker"
|
|
846
856
|
tasksPerWorker = 1
|
|
847
|
-
taskRetryErrorMatchers = [
|
|
848
|
-
{taskRetryErrorExitCodes = [143], taskRetryErrorStatuses = ["FAILED"], taskRetryErrorTypes = ["ALLOCATION_LOST"]},
|
|
849
|
-
]
|
|
850
857
|
uploadFiles = [{localPath = "file_1.txt", uploadPath = "file_1.txt"}]
|
|
851
858
|
uploadTaskProcessOutput = true
|
|
852
859
|
vcpus = [1, 4]
|
|
@@ -892,6 +899,13 @@ Showing all possible properties at the Work Requirement level:
|
|
|
892
899
|
"providers": ["AWS"],
|
|
893
900
|
"ram": [0.5, 2],
|
|
894
901
|
"regions": ["eu-west-2"],
|
|
902
|
+
"retryableErrors": [
|
|
903
|
+
{
|
|
904
|
+
"processExitCodes": [143],
|
|
905
|
+
"statusesAtFailure" : ["FAILED"],
|
|
906
|
+
"errorTypes": ["ALLOCATION_LOST"]
|
|
907
|
+
}
|
|
908
|
+
]
|
|
895
909
|
"setTaskNames": false,
|
|
896
910
|
"tag": "my_tag"
|
|
897
911
|
"taskCount": 100,
|
|
@@ -906,13 +920,6 @@ Showing all possible properties at the Work Requirement level:
|
|
|
906
920
|
{"alwaysUpload": false, "destination": "dest_path_2", "source": "out_src_path_2"}
|
|
907
921
|
],
|
|
908
922
|
"taskGroupCount": 5,
|
|
909
|
-
"taskRetryErrorMatchers": [
|
|
910
|
-
{
|
|
911
|
-
"taskRetryErrorExitCodes": [143],
|
|
912
|
-
"taskRetryErrorStatuses" : ["FAILED"],
|
|
913
|
-
"taskRetryErrorTypes": ["ALLOCATION_LOST"]
|
|
914
|
-
}
|
|
915
|
-
]
|
|
916
923
|
"taskTimeout": 120.0,
|
|
917
924
|
"taskTypes": ["docker"],
|
|
918
925
|
"tasksPerWorker": 1,
|
|
@@ -970,6 +977,13 @@ Showing all possible properties at the Task Group level:
|
|
|
970
977
|
"providers": ["AWS"],
|
|
971
978
|
"ram": [0.5, 2],
|
|
972
979
|
"regions": ["eu-west-2"],
|
|
980
|
+
"retryableErrors": [
|
|
981
|
+
{
|
|
982
|
+
"processExitCodes": [143],
|
|
983
|
+
"statusesAtFailure" : ["FAILED"],
|
|
984
|
+
"errorTypes": ["ALLOCATION_LOST"]
|
|
985
|
+
}
|
|
986
|
+
]
|
|
973
987
|
"setTaskNames": false,
|
|
974
988
|
"tag": "my_tag",
|
|
975
989
|
"taskCount": 5,
|
|
@@ -983,13 +997,6 @@ Showing all possible properties at the Task Group level:
|
|
|
983
997
|
{"alwaysUpload": true, "destination": "dest_path_1", "source": "out_src_path_1"},
|
|
984
998
|
{"alwaysUpload": false, "destination": "dest_path_2", "source": "out_src_path_2"}
|
|
985
999
|
],
|
|
986
|
-
"taskRetryErrorMatchers": [
|
|
987
|
-
{
|
|
988
|
-
"taskRetryErrorExitCodes": [143],
|
|
989
|
-
"taskRetryErrorStatuses" : ["FAILED"],
|
|
990
|
-
"taskRetryErrorTypes": ["ALLOCATION_LOST"]
|
|
991
|
-
}
|
|
992
|
-
]
|
|
993
1000
|
"taskTimeout": 120.0,
|
|
994
1001
|
"taskTypes": ["docker"],
|
|
995
1002
|
"tasksPerWorker": 1,
|
|
@@ -63,7 +63,7 @@ class TestDemos:
|
|
|
63
63
|
|
|
64
64
|
def test_montecarlo_graviton(self):
|
|
65
65
|
result = shell(
|
|
66
|
-
f"cd {DEMO_DIR}/montecarlo && yd-provision -c
|
|
66
|
+
f"cd {DEMO_DIR}/montecarlo && yd-provision -c config-graviton.toml && yd-submit -f && yd-terminate -y && yd-delete -y"
|
|
67
67
|
)
|
|
68
68
|
assert result.exit_code == 0
|
|
69
69
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "7.18.2"
|
|
@@ -93,6 +93,9 @@ def main():
|
|
|
93
93
|
# Always use interactive mode for selections
|
|
94
94
|
ARGS_PARSER.interactive = True
|
|
95
95
|
|
|
96
|
+
if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
|
|
97
|
+
print_log("Omitting YellowDog IDs from detailed JSON objects")
|
|
98
|
+
|
|
96
99
|
if ARGS_PARSER.output_file and ARGS_PARSER.details:
|
|
97
100
|
if exists(ARGS_PARSER.output_file):
|
|
98
101
|
if not confirmed(
|
|
@@ -33,6 +33,9 @@ def main():
|
|
|
33
33
|
# and the 'quiet' option is enabled
|
|
34
34
|
generate_json_list = len(ARGS_PARSER.yellowdog_ids) > 1 and ARGS_PARSER.quiet
|
|
35
35
|
|
|
36
|
+
if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
|
|
37
|
+
print_log("Omitting YellowDog IDs from detailed JSON objects")
|
|
38
|
+
|
|
36
39
|
if generate_json_list:
|
|
37
40
|
print("[")
|
|
38
41
|
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/args.py
RENAMED
|
@@ -1169,6 +1169,14 @@ class CLIParser:
|
|
|
1169
1169
|
"names in detailed compute requirement templates"
|
|
1170
1170
|
),
|
|
1171
1171
|
)
|
|
1172
|
+
parser.add_argument(
|
|
1173
|
+
"--strip-ids",
|
|
1174
|
+
action="store_true",
|
|
1175
|
+
required=False,
|
|
1176
|
+
help=(
|
|
1177
|
+
"don't include objects' YellowDog IDs in their JSON representation"
|
|
1178
|
+
),
|
|
1179
|
+
)
|
|
1172
1180
|
|
|
1173
1181
|
if "compare" in sys.argv[0]:
|
|
1174
1182
|
parser.add_argument(
|
|
@@ -1734,6 +1742,11 @@ class CLIParser:
|
|
|
1734
1742
|
def running_nodes_only(self) -> Optional[bool]:
|
|
1735
1743
|
return self.args.running_nodes_only
|
|
1736
1744
|
|
|
1745
|
+
@property
|
|
1746
|
+
@allow_missing_attribute
|
|
1747
|
+
def strip_ids(self) -> Optional[bool]:
|
|
1748
|
+
return self.args.strip_ids
|
|
1749
|
+
|
|
1737
1750
|
|
|
1738
1751
|
def lookup_module_description(module_name: str) -> Optional[str]:
|
|
1739
1752
|
"""
|
|
@@ -52,6 +52,7 @@ class ConfigWorkRequirement:
|
|
|
52
52
|
providers: Optional[List[str]] = None
|
|
53
53
|
ram: Optional[List[float]] = None
|
|
54
54
|
regions: Optional[List[str]] = None
|
|
55
|
+
retryable_errors: Optional[List[Dict]] = None
|
|
55
56
|
set_task_names: bool = True
|
|
56
57
|
task_batch_size: int = TASK_BATCH_SIZE_DEFAULT
|
|
57
58
|
task_count: int = 1
|
|
@@ -63,7 +64,6 @@ class ConfigWorkRequirement:
|
|
|
63
64
|
task_group_name: Optional[str] = None
|
|
64
65
|
task_level_timeout: Optional[float] = None
|
|
65
66
|
task_name: Optional[str] = None
|
|
66
|
-
task_retry_error_matchers: Optional[List[Dict]] = None
|
|
67
67
|
task_timeout: Optional[float] = None
|
|
68
68
|
task_type: Optional[str] = None
|
|
69
69
|
tasks_per_worker: Optional[int] = None
|
|
@@ -9,6 +9,7 @@ from pathlib import Path
|
|
|
9
9
|
from sys import exit
|
|
10
10
|
from typing import Dict
|
|
11
11
|
|
|
12
|
+
from dotenv import dotenv_values, find_dotenv, load_dotenv
|
|
12
13
|
from toml import TomlDecodeError
|
|
13
14
|
|
|
14
15
|
from yellowdog_cli.utils.args import ARGS_PARSER
|
|
@@ -101,6 +102,10 @@ def load_config_common() -> ConfigCommon:
|
|
|
101
102
|
common_section_imported.update(common_section)
|
|
102
103
|
common_section = common_section_imported
|
|
103
104
|
|
|
105
|
+
# Load extra environment variables from a .env file if it exists;
|
|
106
|
+
# do not override existing variables (environment takes precedence)
|
|
107
|
+
_load_dotenv()
|
|
108
|
+
|
|
104
109
|
# Replace common section properties with command line or
|
|
105
110
|
# environment variable overrides. Precedence is:
|
|
106
111
|
# command line > environment variable > config file
|
|
@@ -308,6 +313,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
|
|
|
308
313
|
providers=wr_section.get(PROVIDERS, None),
|
|
309
314
|
ram=wr_section.get(RAM, None),
|
|
310
315
|
regions=wr_section.get(REGIONS, None),
|
|
316
|
+
retryable_errors=wr_section.get(RETRYABLE_ERRORS, None),
|
|
311
317
|
set_task_names=wr_section.get(SET_TASK_NAMES, True),
|
|
312
318
|
task_batch_size=task_batch_size,
|
|
313
319
|
task_count=task_count,
|
|
@@ -318,7 +324,6 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
|
|
|
318
324
|
task_group_count=task_group_count,
|
|
319
325
|
task_group_name=wr_section.get(TASK_GROUP_NAME, None),
|
|
320
326
|
task_name=wr_section.get(TASK_NAME, None),
|
|
321
|
-
task_retry_error_matchers=wr_section.get(TASK_RETRY_ERROR_MATCHERS, None),
|
|
322
327
|
task_timeout=wr_section.get(TASK_TIMEOUT, None),
|
|
323
328
|
task_type=task_type,
|
|
324
329
|
tasks_per_worker=wr_section.get(TASKS_PER_WORKER, None),
|
|
@@ -445,3 +450,29 @@ def load_config_worker_pool() -> ConfigWorkerPool:
|
|
|
445
450
|
except ValueError as e:
|
|
446
451
|
print_error(f"Invalid type for configuration: {e}")
|
|
447
452
|
exit(1)
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _load_dotenv():
|
|
456
|
+
"""
|
|
457
|
+
Load extra environment variables from a .env file if it exists.
|
|
458
|
+
Do not override existing variables (environment takes precedence).
|
|
459
|
+
Report on YD vars that are taken from .env.
|
|
460
|
+
"""
|
|
461
|
+
dotenv_file = find_dotenv()
|
|
462
|
+
if dotenv_file == "":
|
|
463
|
+
return
|
|
464
|
+
|
|
465
|
+
dotenv_yd_substitutions = [ # Find 'YD' variables
|
|
466
|
+
f"'{key}'"
|
|
467
|
+
for key in dotenv_values(dotenv_file).keys()
|
|
468
|
+
if key.startswith("YD") and os.environ.get(key) is None
|
|
469
|
+
]
|
|
470
|
+
|
|
471
|
+
if len(dotenv_yd_substitutions) > 0:
|
|
472
|
+
print_log(
|
|
473
|
+
f"Adding 'YD' environment variables from '.env' file '{dotenv_file}': "
|
|
474
|
+
f"{', '.join(dotenv_yd_substitutions)}"
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
# Actually load the variables (including non-'YD' variables)
|
|
478
|
+
load_dotenv(dotenv_file, override=False)
|
|
@@ -71,6 +71,7 @@ from yellowdog_cli.utils.settings import (
|
|
|
71
71
|
MAX_LINES_COLOURED_FORMATTING,
|
|
72
72
|
MAX_TABLE_DESCRIPTION,
|
|
73
73
|
NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR,
|
|
74
|
+
PROP_ID,
|
|
74
75
|
PROP_RESOURCE,
|
|
75
76
|
WARNING_STYLE,
|
|
76
77
|
)
|
|
@@ -1062,6 +1063,11 @@ def print_yd_object(
|
|
|
1062
1063
|
using the compact JSON encoder.
|
|
1063
1064
|
"""
|
|
1064
1065
|
object_data: object = Json.dump(yd_object)
|
|
1066
|
+
|
|
1067
|
+
# Remove the 'id' property?
|
|
1068
|
+
if ARGS_PARSER.strip_ids:
|
|
1069
|
+
object_data.pop(PROP_ID, None)
|
|
1070
|
+
|
|
1065
1071
|
if add_fields is not None:
|
|
1066
1072
|
# Requires a copy of the 'object' datatype to be made,
|
|
1067
1073
|
# in order to insert additional fields
|
|
@@ -1071,6 +1077,7 @@ def print_yd_object(
|
|
|
1071
1077
|
for key, value in add_fields.items():
|
|
1072
1078
|
object_data_new[key] = value
|
|
1073
1079
|
object_data = object_data_new
|
|
1080
|
+
|
|
1074
1081
|
print_json(object_data, initial_indent, drop_first_line, with_final_comma)
|
|
1075
1082
|
|
|
1076
1083
|
|
|
@@ -22,6 +22,7 @@ DOCKER_PASSWORD = "dockerPassword" # String
|
|
|
22
22
|
DOCKER_REGISTRY = "dockerRegistry" # String
|
|
23
23
|
DOCKER_USERNAME = "dockerUsername" # String
|
|
24
24
|
ENV = "environment" # Dictionary
|
|
25
|
+
ERROR_TYPES = "errorTypes" # List of Strings
|
|
25
26
|
EXCLUSIVE_WORKERS = "exclusiveWorkers" # Boolean
|
|
26
27
|
EXECUTABLE = "executable" # String
|
|
27
28
|
FILE_PATTERN = "filePattern" # String
|
|
@@ -56,12 +57,15 @@ OUTPUTS_OTHER = "outputsOther" # List of Dicts
|
|
|
56
57
|
OUTPUTS_REQUIRED = "outputsRequired" # List of Strings
|
|
57
58
|
PARALLEL_BATCHES = "parallelBatches" # Integer
|
|
58
59
|
PRIORITY = "priority" # Float
|
|
60
|
+
PROCESS_EXIT_CODES = "processExitCodes" # List of Ints
|
|
59
61
|
PROVIDERS = "providers" # List of Strings
|
|
60
62
|
RAM = "ram" # List of two Floats
|
|
61
63
|
REGIONS = "regions" # List of Strings
|
|
62
64
|
REQUIRED = "required" # Boolean
|
|
65
|
+
RETRYABLE_ERRORS = "retryableErrors" # List of Dicts
|
|
63
66
|
SECRET = "secret" # String
|
|
64
67
|
SET_TASK_NAMES = "setTaskNames" # Set to False to suppress task naming
|
|
68
|
+
STATUSES_AT_FAILURE = "statusesAtFailure" # List of Strings
|
|
65
69
|
TARGET_INSTANCE_COUNT = "targetInstanceCount" # Integer
|
|
66
70
|
TASKS = "tasks" # List of Tasks
|
|
67
71
|
TASKS_PER_WORKER = "tasksPerWorker" # Integer
|
|
@@ -79,10 +83,6 @@ TASK_GROUP_NAME = "taskGroupName" # String
|
|
|
79
83
|
TASK_GROUP_TAG = "tag" # String
|
|
80
84
|
TASK_LEVEL_TIMEOUT = "timeout" # Float
|
|
81
85
|
TASK_NAME = "taskName" # String
|
|
82
|
-
TASK_RETRY_ERROR_EXIT_CODES = "taskRetryErrorExitCodes" # List of Ints
|
|
83
|
-
TASK_RETRY_ERROR_MATCHERS = "taskRetryErrorMatchers" # List of Dicts
|
|
84
|
-
TASK_RETRY_ERROR_STATUSES = "taskRetryErrorStatuses" # List of Strings
|
|
85
|
-
TASK_RETRY_ERROR_TYPES = "taskRetryErrorTypes" # List of Strings
|
|
86
86
|
TASK_TAG = "tag" # String
|
|
87
87
|
TASK_TIMEOUT = "taskTimeout" # Float
|
|
88
88
|
TASK_TYPE = "taskType" # String
|
|
@@ -135,6 +135,7 @@ ALL_KEYS = [
|
|
|
135
135
|
DOCKER_REGISTRY,
|
|
136
136
|
DOCKER_USERNAME,
|
|
137
137
|
ENV,
|
|
138
|
+
ERROR_TYPES,
|
|
138
139
|
EXCLUSIVE_WORKERS,
|
|
139
140
|
EXECUTABLE,
|
|
140
141
|
FILE_PATTERN,
|
|
@@ -168,12 +169,15 @@ ALL_KEYS = [
|
|
|
168
169
|
OUTPUTS_REQUIRED,
|
|
169
170
|
PARALLEL_BATCHES,
|
|
170
171
|
PRIORITY,
|
|
172
|
+
PROCESS_EXIT_CODES,
|
|
171
173
|
PROVIDERS,
|
|
172
174
|
RAM,
|
|
173
175
|
REGIONS,
|
|
174
176
|
REQUIRED,
|
|
177
|
+
RETRYABLE_ERRORS,
|
|
175
178
|
SECRET,
|
|
176
179
|
SET_TASK_NAMES,
|
|
180
|
+
STATUSES_AT_FAILURE,
|
|
177
181
|
TARGET_INSTANCE_COUNT,
|
|
178
182
|
TASKS,
|
|
179
183
|
TASKS_PER_WORKER,
|
|
@@ -191,10 +195,6 @@ ALL_KEYS = [
|
|
|
191
195
|
TASK_GROUP_TAG,
|
|
192
196
|
TASK_LEVEL_TIMEOUT,
|
|
193
197
|
TASK_NAME,
|
|
194
|
-
TASK_RETRY_ERROR_EXIT_CODES,
|
|
195
|
-
TASK_RETRY_ERROR_MATCHERS,
|
|
196
|
-
TASK_RETRY_ERROR_STATUSES,
|
|
197
|
-
TASK_RETRY_ERROR_TYPES,
|
|
198
198
|
TASK_TAG,
|
|
199
199
|
TASK_TIMEOUT,
|
|
200
200
|
TASK_TYPE,
|
|
@@ -24,10 +24,10 @@ from yellowdog_client.model import (
|
|
|
24
24
|
from yellowdog_cli.utils.config_types import ConfigCommon, ConfigWorkRequirement
|
|
25
25
|
from yellowdog_cli.utils.printing import print_error, print_log
|
|
26
26
|
from yellowdog_cli.utils.property_names import (
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
27
|
+
ERROR_TYPES,
|
|
28
|
+
PROCESS_EXIT_CODES,
|
|
29
|
+
RETRYABLE_ERRORS,
|
|
30
|
+
STATUSES_AT_FAILURE,
|
|
31
31
|
)
|
|
32
32
|
from yellowdog_cli.utils.settings import NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR
|
|
33
33
|
from yellowdog_cli.utils.type_check import check_list
|
|
@@ -350,8 +350,8 @@ def generate_task_error_matchers_list(
|
|
|
350
350
|
"""
|
|
351
351
|
error_matchers: Optional[List[Dict]] = check_list(
|
|
352
352
|
tg_data.get(
|
|
353
|
-
|
|
354
|
-
wr_data.get(
|
|
353
|
+
RETRYABLE_ERRORS,
|
|
354
|
+
wr_data.get(RETRYABLE_ERRORS, config_wr.retryable_errors),
|
|
355
355
|
)
|
|
356
356
|
)
|
|
357
357
|
|
|
@@ -372,7 +372,7 @@ def _generate_task_error_matcher(task_error_matcher_data: Dict) -> TaskErrorMatc
|
|
|
372
372
|
try:
|
|
373
373
|
|
|
374
374
|
exit_codes_str: Optional[List[int]] = check_list(
|
|
375
|
-
task_error_matcher_data.get(
|
|
375
|
+
task_error_matcher_data.get(PROCESS_EXIT_CODES, None)
|
|
376
376
|
)
|
|
377
377
|
try:
|
|
378
378
|
# Ensure ints
|
|
@@ -385,7 +385,7 @@ def _generate_task_error_matcher(task_error_matcher_data: Dict) -> TaskErrorMatc
|
|
|
385
385
|
raise Exception(f"Unable to process error exit codes: {e}")
|
|
386
386
|
|
|
387
387
|
statuses_str: Optional[List[str]] = check_list(
|
|
388
|
-
task_error_matcher_data.get(
|
|
388
|
+
task_error_matcher_data.get(STATUSES_AT_FAILURE, None)
|
|
389
389
|
)
|
|
390
390
|
try:
|
|
391
391
|
statuses = (
|
|
@@ -397,7 +397,7 @@ def _generate_task_error_matcher(task_error_matcher_data: Dict) -> TaskErrorMatc
|
|
|
397
397
|
raise Exception(f"Unable to process error status: {e}")
|
|
398
398
|
|
|
399
399
|
error_types: Optional[List[str]] = check_list(
|
|
400
|
-
task_error_matcher_data.get(
|
|
400
|
+
task_error_matcher_data.get(ERROR_TYPES, None)
|
|
401
401
|
)
|
|
402
402
|
|
|
403
403
|
return TaskErrorMatcher(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yellowdog-python-examples
|
|
3
|
-
Version: 7.18.
|
|
3
|
+
Version: 7.18.2
|
|
4
4
|
Summary: Python CLI commands using the YellowDog Python SDK
|
|
5
5
|
Author-email: YellowDog Limited <support@yellowdog.co>
|
|
6
6
|
Project-URL: Homepage, https://github.com/yellowdog/python-examples
|
|
@@ -15,11 +15,12 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
License-File: LICENSE
|
|
16
16
|
Requires-Dist: PyPAC>=0.16.4
|
|
17
17
|
Requires-Dist: dateparser
|
|
18
|
+
Requires-Dist: dotenv
|
|
18
19
|
Requires-Dist: requests
|
|
19
20
|
Requires-Dist: rich==13.9.4
|
|
20
21
|
Requires-Dist: tabulate>=0.9.0
|
|
21
22
|
Requires-Dist: toml
|
|
22
|
-
Requires-Dist: yellowdog-sdk>=9.
|
|
23
|
+
Requires-Dist: yellowdog-sdk>=9.3.3
|
|
23
24
|
Provides-Extra: jsonnet
|
|
24
25
|
Requires-Dist: jsonnet; extra == "jsonnet"
|
|
25
26
|
Provides-Extra: cloudwizard
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "7.18.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_create_remove.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/tests/test_entrypoints.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/abort.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/admin.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/boost.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/cancel.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/cloudwizard.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/compare.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/create.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/delete.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/download.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/follow.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/format_json.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/instantiate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/jsonnet2json.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/provision.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/remove.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/resize.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/shutdown.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/start.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/submit.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/terminate.py
RENAMED
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/upload.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/items.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/wrapper.py
RENAMED
|
File without changes
|
|
File without changes
|
{yellowdog_python_examples-7.18.0 → yellowdog_python_examples-7.18.2}/yellowdog_cli/version.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|