yellowdog-python-examples 7.18.1__tar.gz → 7.18.2__tar.gz

{yellowdog_python_examples-7.18.1/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.18.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yellowdog-python-examples
-Version: 7.18.1
+Version: 7.18.2
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 Project-URL: Homepage, https://github.com/yellowdog/python-examples
@@ -15,11 +15,12 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: PyPAC>=0.16.4
 Requires-Dist: dateparser
+Requires-Dist: dotenv
 Requires-Dist: requests
 Requires-Dist: rich==13.9.4
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: toml
-Requires-Dist: yellowdog-sdk>=9.2.3
+Requires-Dist: yellowdog-sdk>=9.3.3
 Provides-Extra: jsonnet
 Requires-Dist: jsonnet; extra == "jsonnet"
 Provides-Extra: cloudwizard

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/README.md

@@ -161,7 +161,7 @@ The commands provide the following capabilities:
 - **Listing** YellowDog items using the **`yd-list`** command
 - **Resizing** Worker Pools and Compute Requirements with the **`yd-resize`** command
 - **Boosting** Allowances with the **`yd-boost`** command
-- **Creating, Updating and Removing** Source Templates, Compute Templates, Keyrings, Credentials, Namespace Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, and Namespace Policies with the **`yd-create`** and **`yd-remove`** commands
+- **Creating, Updating and Removing** Compute Source Templates, Compute Requirement Templates, Keyrings, Credentials, Storage Configurations, Image Families, Allowances, Configured Worker Pools, User Attributes, Namespace Policies, Groups, and Applications with the **`yd-create`** and **`yd-remove`** commands
 - **Following Event Streams** for Work Requirements, Worker Pools and Compute Requirements with the **`yd-follow`** command
 - **Showing** the details of any YellowDog entity using its YellowDog ID with the **`yd-show`** command
 - **Comparing** whether worker pools are a match for task groups with the **`yd-compare`** command
@@ -328,7 +328,7 @@ An example `common` section is shown below:
 
 Indentation is optional in TOML files and is for readability only.
 
-## Importing `common` properties
+## Importing `common` Properties
 
 The `common` section can import properties from a separate TOML file, using the `importCommon` property. For example, the `key` and `secret` might be in a shared TOML file called `app_credentials.toml`, with the following contents:
 
@@ -383,6 +383,12 @@ When setting the value of the above properties, a property set on the command li
 
 If all the required common properties are set using the command line or environment variables, then the entire `common` section of the TOML file can be omitted.
 
+## Support for `.env` Files
+
+Environment variables can also be set in a `.env` file, typically in the user's home directory. Entries in the `.env` file will not overwrite existing environment variables -- i.e., environment variables take precedence over entries in the `.env` file.
+
+Environment variables sourced from a `.env` file whose names start with `YD` will be reported on the command line. Variables that do not start with `YD` will not be reported, but they will still be applied.
+
 ## Variable Substitutions in Common Properties
 
 Note the use of `{{username}}` in the value of the `tag` property example above: this is a **variable substitution** that can optionally be used to insert the login username of the user running the commands. So, for username `abc`, the `tag` would be set to `testing-abc`. This can be helpful to disambiguate multiple users running with the same configuration data.
@@ -464,7 +470,8 @@ The precedence order for setting variables is:
 1. Command line
 2. `YD_VAR` environment variables
 3. General environment variables
-4. TOML file
+4. Variables in a `.env` file
+5. TOML file
 
 This method can also be used to override the default variables, e.g., setting `-v username="other-user"` will override the default `{{username}}` variable.
 
@@ -824,6 +831,9 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
     providers = ["AWS"]
     ram = [0.5, 2.0]
     regions = ["eu-west-2"]
+    retryableErrors = [
+      {processExitCodes = [143], statusesAtFailure = ["FAILED"], errorTypes = ["ALLOCATION_LOST"]},
+    ]
     setTaskNames = false
     tag = "my_tag"
     taskBatchSize = 1000
@@ -844,9 +854,6 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
     taskTimeout = 120.0
     taskType = "docker"
     tasksPerWorker = 1
-    retryableErrors = [
-        {processExitCodes = [143], statusesAtFailure = ["FAILED"], errorTypes = ["ALLOCATION_LOST"]},
-    ]
     uploadFiles = [{localPath = "file_1.txt", uploadPath = "file_1.txt"}]
     uploadTaskProcessOutput = true
     vcpus = [1, 4]
@@ -892,6 +899,13 @@ Showing all possible properties at the Work Requirement level:
   "providers": ["AWS"],
   "ram": [0.5, 2],
   "regions": ["eu-west-2"],
+  "retryableErrors": [
+    {
+      "processExitCodes": [143],
+      "statusesAtFailure" : ["FAILED"],
+      "errorTypes": ["ALLOCATION_LOST"]
+    }
+  ]
   "setTaskNames": false,
   "tag": "my_tag"
   "taskCount": 100,
@@ -906,13 +920,6 @@ Showing all possible properties at the Work Requirement level:
     {"alwaysUpload": false, "destination": "dest_path_2", "source": "out_src_path_2"}
   ],
   "taskGroupCount": 5,
-  "retryableErrors": [
-    {
-      "processExitCodes": [143],
-      "statusesAtFailure" : ["FAILED"],
-      "errorTypes": ["ALLOCATION_LOST"]
-    }
-  ]
   "taskTimeout": 120.0,
   "taskTypes": ["docker"],
   "tasksPerWorker": 1,
@@ -970,6 +977,13 @@ Showing all possible properties at the Task Group level:
       "providers": ["AWS"],
       "ram": [0.5, 2],
       "regions": ["eu-west-2"],
+      "retryableErrors": [
+        {
+          "processExitCodes": [143],
+          "statusesAtFailure" : ["FAILED"],
+          "errorTypes": ["ALLOCATION_LOST"]
+        }
+      ]
       "setTaskNames": false,
       "tag": "my_tag",
       "taskCount": 5,
@@ -983,13 +997,6 @@ Showing all possible properties at the Task Group level:
         {"alwaysUpload": true, "destination": "dest_path_1", "source": "out_src_path_1"},
         {"alwaysUpload": false, "destination": "dest_path_2", "source": "out_src_path_2"}
       ],
-      "retryableErrors": [
-        {
-          "processExitCodes": [143],
-          "statusesAtFailure" : ["FAILED"],
-          "errorTypes": ["ALLOCATION_LOST"]
-        }
-      ]
       "taskTimeout": 120.0,
       "taskTypes": ["docker"],
       "tasksPerWorker": 1,

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/requirements.txt

@@ -1,7 +1,8 @@
 PyPAC >= 0.16.4
 dateparser
+dotenv
 requests
 rich == 13.9.4
 tabulate >= 0.9.0
 toml
-yellowdog-sdk >= 9.2.3
+yellowdog-sdk >= 9.3.3

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/tests/test_demos.py

@@ -63,7 +63,7 @@ class TestDemos:
 
     def test_montecarlo_graviton(self):
         result = shell(
-            f"cd {DEMO_DIR}/montecarlo && yd-provision -c config_graviton.toml && yd-submit -f && yd-terminate -y && yd-delete -y"
+            f"cd {DEMO_DIR}/montecarlo && yd-provision -c config-graviton.toml && yd-submit -f && yd-terminate -y && yd-delete -y"
         )
         assert result.exit_code == 0
 

yellowdog_python_examples-7.18.2/yellowdog_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "7.18.2"

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_cli/list.py

@@ -93,6 +93,9 @@ def main():
     # Always use interactive mode for selections
     ARGS_PARSER.interactive = True
 
+    if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
+        print_log("Omitting YellowDog IDs from detailed JSON objects")
+
     if ARGS_PARSER.output_file and ARGS_PARSER.details:
         if exists(ARGS_PARSER.output_file):
             if not confirmed(

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_cli/show.py

@@ -33,6 +33,9 @@ def main():
     # and the 'quiet' option is enabled
     generate_json_list = len(ARGS_PARSER.yellowdog_ids) > 1 and ARGS_PARSER.quiet
 
+    if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
+        print_log("Omitting YellowDog IDs from detailed JSON objects")
+
     if generate_json_list:
         print("[")
 

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/args.py

@@ -1169,6 +1169,14 @@ class CLIParser:
                     "names in detailed compute requirement templates"
                 ),
             )
+            parser.add_argument(
+                "--strip-ids",
+                action="store_true",
+                required=False,
+                help=(
+                    "don't include objects' YellowDog IDs in their JSON representation"
+                ),
+            )
 
         if "compare" in sys.argv[0]:
             parser.add_argument(
@@ -1734,6 +1742,11 @@ class CLIParser:
     def running_nodes_only(self) -> Optional[bool]:
         return self.args.running_nodes_only
 
+    @property
+    @allow_missing_attribute
+    def strip_ids(self) -> Optional[bool]:
+        return self.args.strip_ids
+
 
 def lookup_module_description(module_name: str) -> Optional[str]:
     """

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/load_config.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from sys import exit
 from typing import Dict
 
+from dotenv import dotenv_values, find_dotenv, load_dotenv
 from toml import TomlDecodeError
 
 from yellowdog_cli.utils.args import ARGS_PARSER
@@ -101,6 +102,10 @@ def load_config_common() -> ConfigCommon:
             common_section_imported.update(common_section)
             common_section = common_section_imported
 
+        # Load extra environment variables from a .env file if it exists;
+        # do not override existing variables (environment takes precedence)
+        _load_dotenv()
+
         # Replace common section properties with command line or
         # environment variable overrides. Precedence is:
         # command line > environment variable > config file
@@ -445,3 +450,29 @@ def load_config_worker_pool() -> ConfigWorkerPool:
     except ValueError as e:
         print_error(f"Invalid type for configuration: {e}")
         exit(1)
+
+
+def _load_dotenv():
+    """
+    Load extra environment variables from a .env file if it exists.
+    Do not override existing variables (environment takes precedence).
+    Report on YD vars that are taken from .env.
+    """
+    dotenv_file = find_dotenv()
+    if dotenv_file == "":
+        return
+
+    dotenv_yd_substitutions = [  # Find 'YD' variables
+        f"'{key}'"
+        for key in dotenv_values(dotenv_file).keys()
+        if key.startswith("YD") and os.environ.get(key) is None
+    ]
+
+    if len(dotenv_yd_substitutions) > 0:
+        print_log(
+            f"Adding 'YD' environment variables from '.env' file '{dotenv_file}': "
+            f"{', '.join(dotenv_yd_substitutions)}"
+        )
+
+    # Actually load the variables (including non-'YD' variables)
+    load_dotenv(dotenv_file, override=False)

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/printing.py

@@ -71,6 +71,7 @@ from yellowdog_cli.utils.settings import (
     MAX_LINES_COLOURED_FORMATTING,
     MAX_TABLE_DESCRIPTION,
     NAMESPACE_OBJECT_STORE_PREFIX_SEPARATOR,
+    PROP_ID,
     PROP_RESOURCE,
     WARNING_STYLE,
 )
@@ -1062,6 +1063,11 @@ def print_yd_object(
     using the compact JSON encoder.
     """
     object_data: object = Json.dump(yd_object)
+
+    # Remove the 'id' property?
+    if ARGS_PARSER.strip_ids:
+        object_data.pop(PROP_ID, None)
+
     if add_fields is not None:
         # Requires a copy of the 'object' datatype to be made,
         # in order to insert additional fields
@@ -1071,6 +1077,7 @@ def print_yd_object(
         for key, value in add_fields.items():
             object_data_new[key] = value
         object_data = object_data_new
+
     print_json(object_data, initial_indent, drop_first_line, with_final_comma)
 
 

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_cli/utils/variables.py

@@ -397,6 +397,8 @@ def process_typed_variable_substitution(
                 f"(Use JSON syntax?) ({e})"
             )
 
+    return None
+
 
 def resolve_filename(files_directory: str, filename: str) -> str:
     """

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2/yellowdog_python_examples.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yellowdog-python-examples
-Version: 7.18.1
+Version: 7.18.2
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 Project-URL: Homepage, https://github.com/yellowdog/python-examples
@@ -15,11 +15,12 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: PyPAC>=0.16.4
 Requires-Dist: dateparser
+Requires-Dist: dotenv
 Requires-Dist: requests
 Requires-Dist: rich==13.9.4
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: toml
-Requires-Dist: yellowdog-sdk>=9.2.3
+Requires-Dist: yellowdog-sdk>=9.3.3
 Provides-Extra: jsonnet
 Requires-Dist: jsonnet; extra == "jsonnet"
 Provides-Extra: cloudwizard

{yellowdog_python_examples-7.18.1 → yellowdog_python_examples-7.18.2}/yellowdog_python_examples.egg-info/requires.txt

@@ -1,10 +1,11 @@
 PyPAC>=0.16.4
 dateparser
+dotenv
 requests
 rich==13.9.4
 tabulate>=0.9.0
 toml
-yellowdog-sdk>=9.2.3
+yellowdog-sdk>=9.3.3
 
 [cloudwizard]
 boto3

yellowdog_python_examples-7.18.1/yellowdog_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "7.18.1"