yellowdog-python-examples 8.3.1__tar.gz → 8.4.1__tar.gz

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yellowdog-python-examples
-Version: 8.3.1
+Version: 8.4.1
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 License-Expression: Apache-2.0
@@ -25,8 +25,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: requests
 Requires-Dist: rich==13.9.4
 Requires-Dist: tabulate>=0.9.0
-Requires-Dist: toml
-Requires-Dist: yellowdog-sdk>=12.0.0
+Requires-Dist: tomli>=2.4.0
+Requires-Dist: yellowdog-sdk>=13.1.0
 Provides-Extra: jsonnet
 Requires-Dist: jsonnet; extra == "jsonnet"
 Provides-Extra: cloudwizard

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/README.md

@@ -622,6 +622,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
 | `csvFiles`                  | A list of CSV files used to derive Task data. E.g. `["file.csv", "file_2.csv:2]`.                                                                                                                                                          | Yes  |     |      |      |
 | `dependencies`              | The names of other Task Groups within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `["task_group_1", "task_group_2"]`.                                                             |      |     | Yes  |      |
 | `dependentOn`               | [Deprecated in favour of `dependencies`] The name of another Task Group within the same Work Requirement that must be successfully completed before the Task Group is started. E.g. `"task_group_1"`.                                      |      |     | Yes  |      |
+| `disablePreallocation`      | If `true`, tasks are only allocated to nodes as workers become idle and are not queued on the node. Default: `false`.                                                                                                                      | Yes  | Yes | Yes  |      |
 | `dockerEnvironment`         | The environment to be passed to a Docker container. Only used by the `docker` Task Type. E.g., JSON: `{"VAR_1": "abc"}`, TOML: `{VAR_1 = "abc", VAR_2 = "def"}`.                                                                           | Yes  | Yes | Yes  | Yes  |
 | `dockerOptions`             | Additional options to be passed to the docker run command. Only used by the `docker` Task Type. E.g.,`["--runtime nvidia, "--gpus all"]`.                                                                                                  | Yes  | Yes | Yes  | Yes  |
 | `dockerPassword`            | The password for the Docker container registry; only used by the `docker` Task Type. E,g., `"my_password"`.                                                                                                                                | Yes  | Yes | Yes  | Yes  |

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/requirements.txt

@@ -4,5 +4,5 @@ python-dotenv
 requests
 rich == 13.9.4
 tabulate >= 0.9.0
-toml
-yellowdog-sdk >= 12.0.0
+tomli >= 2.4.0
+yellowdog-sdk >= 13.1.0

yellowdog_python_examples-8.4.1/yellowdog_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "8.4.1"

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/follow.py

@@ -5,11 +5,16 @@ A script to follow event streams.
 """
 
 from yellowdog_cli.utils.follow_utils import follow_ids
+from yellowdog_cli.utils.printing import print_info
 from yellowdog_cli.utils.wrapper import ARGS_PARSER, main_wrapper
 
 
 @main_wrapper
 def main():
+    if len(ARGS_PARSER.yellowdog_ids) == 0:
+        print_info("No YellowDog IDs to follow")
+        return
+
     follow_ids(ARGS_PARSER.yellowdog_ids, ARGS_PARSER.auto_cr)
 
 

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/show.py

@@ -16,6 +16,7 @@ from yellowdog_cli.utils.entity_utils import (
 from yellowdog_cli.utils.printing import (
     print_error,
     print_info,
+    print_json,
     print_to_file,
     print_yd_object,
 )
@@ -32,13 +33,37 @@ from yellowdog_cli.utils.settings import (
     RN_ROLE,
     RN_SOURCE_TEMPLATE,
 )
-from yellowdog_cli.utils.wrapper import ARGS_PARSER, CLIENT, main_wrapper
+from yellowdog_cli.utils.variables import get_user_variable
+from yellowdog_cli.utils.wrapper import ARGS_PARSER, CLIENT, CONFIG_COMMON, main_wrapper
 from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
 
 
 @main_wrapper
 def main():
 
+    if ARGS_PARSER.parse_config:
+        username = get_user_variable("username")
+        if ARGS_PARSER.quiet:
+            # Output properties in JSON format
+            print_json(
+                {
+                    "namespace": CONFIG_COMMON.namespace,
+                    "tag": CONFIG_COMMON.name_tag,
+                    "username": username,
+                }
+            )
+        else:
+            # Print properties inline
+            print_info("Showing configuration values:")
+            print(f"                      namespace = {CONFIG_COMMON.namespace}")
+            print(f"                      tag       = {CONFIG_COMMON.name_tag}")
+            print(f"                      username  = {username}")
+        return
+
+    if len(ARGS_PARSER.yellowdog_ids) == 0:
+        print_info("No YellowDog IDs to show")
+        return
+
     # Generate a JSON list of resources if there are multiple YDIDs
     # and the 'quiet' option is enabled
     generate_json_list = len(ARGS_PARSER.yellowdog_ids) > 1 and ARGS_PARSER.quiet

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/submit.py

@@ -544,6 +544,12 @@ def create_task_group(
         retryableErrors=generate_task_error_matchers_list(
             config_wr, wr_data, task_group_data
         ),
+        disablePreallocation=check_bool(
+            task_group_data.get(
+                DISABLE_PREALLOCATION,
+                wr_data.get(DISABLE_PREALLOCATION, config_wr.disable_prealloaction),
+            )
+        ),
     )
     ctttl_data = check_float_or_int(
         task_group_data.get(

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/args.py

@@ -994,13 +994,21 @@ class CLIParser:
             verb = "follow" if "follow" in sys.argv[0] else "show"
             parser.add_argument(
                 "yellowdog_ids",
-                nargs="+",
+                nargs="*",
                 default=[],
                 metavar="<yellowdog-id>",
                 type=str,
                 help=f"the YellowDog ID(s) of the item(s) to {verb}",
             )
 
+        if any(module in sys.argv[0] for module in ["show"]):
+            parser.add_argument(
+                "--parse-config",
+                action="store_true",
+                required=False,
+                help="parse the config and return namespace, tag, username, then exit",
+            )
+
         if any(
             module in sys.argv[0]
             for module in ["upload", "submit", "provision", "instantiate"]
@@ -1311,6 +1319,11 @@ class CLIParser:
     def print_pid(self) -> Optional[bool]:
         return self.args.print_pid
 
+    @property
+    @allow_missing_attribute
+    def parse_config(self) -> Optional[bool]:
+        return self.args.parse_config
+
     @property
     @allow_missing_attribute
     def no_config(self) -> Optional[bool]:

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/config_types.py

@@ -26,6 +26,7 @@ class ConfigWorkRequirement:
     batch_allocation: Optional[bool] = None
     completed_task_ttl: Optional[float] = None  # In minutes
     csv_files: Optional[List[str]] = None
+    disable_prealloaction: Optional[bool] = None
     docker_env: Optional[Dict] = None
     docker_options: Optional[List] = None
     docker_password: Optional[str] = None

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/csv_data.py

@@ -10,7 +10,7 @@ from json import load as json_load
 from os.path import relpath
 from typing import Dict, List, Optional, Tuple
 
-from toml import load as toml_load
+from tomli import load as toml_load
 
 from yellowdog_cli.utils.args import ARGS_PARSER
 from yellowdog_cli.utils.config_types import ConfigWorkRequirement
@@ -165,7 +165,7 @@ def load_toml_file_with_csv_task_expansion(
     data.
     """
 
-    with open(resolve_filename(files_directory, toml_file), "r") as f:
+    with open(resolve_filename(files_directory, toml_file), "rb") as f:
         wr_data = toml_load(f)
 
     return perform_csv_task_expansion(wr_data, csv_files, files_directory)

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/entity_utils.py

@@ -921,7 +921,7 @@ def get_application_id_by_name(client: PlatformClient, app_name: str) -> Optiona
 @lru_cache
 def get_application_details(client: PlatformClient) -> ApplicationDetails:
     """
-    Load and cache the Application's details
+    Load and cache the Application's details.
     """
     return client.application_client.get_application_details()
 
@@ -952,23 +952,28 @@ def get_all_roles_and_namespaces_for_application(
     client: PlatformClient, application_id: str
 ) -> Dict:
     """
-    Get a list of roles and the namespaces to which they apply, for a given application.
+    Get a list of roles and the namespaces to which they apply, for a given
+    application.
+
     Returns {role_name: [namespace, ...]}, sorted by role name.
     """
-    # Iterate through groups, roles, accumulate namespaces
+    # Iterate through groups, roles, accumulate unique namespaces
     roles = dict()
     for group in get_application_groups(client, application_id):
         for role in group.roles:
             if roles.get(role.role.name) is None:
-                roles[role.role.name] = []
+                # Set of namespaces to suppress duplicates
+                roles[role.role.name] = set()
             if role.scope.global_:
-                roles[role.role.name] += ["GLOBAL"]
+                roles[role.role.name].update(["GLOBAL"])
             else:
-                roles[role.role.name] += [
-                    namespace.namespace for namespace in role.scope.namespaces
-                ]
+                roles[role.role.name].update(
+                    [namespace.namespace for namespace in role.scope.namespaces]
+                )
 
-    return {role: namespaces for role, namespaces in sorted(roles.items())}
+    return {
+        role: sorted(list(namespaces)) for role, namespaces in sorted(roles.items())
+    }
 
 
 def clear_application_caches():

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/load_config.py

@@ -9,7 +9,7 @@ from pathlib import Path
 from sys import exit
 from typing import Dict
 
-from toml import TomlDecodeError
+from tomli import TOMLDecodeError
 
 from yellowdog_cli.utils.args import ARGS_PARSER
 from yellowdog_cli.utils.config_types import (
@@ -102,7 +102,7 @@ else:
         CONFIG_TOML = {COMMON_SECTION: {}}
         CONFIG_FILE_DIR = os.getcwd()
 
-    except (PermissionError, TomlDecodeError) as e:
+    except (PermissionError, TOMLDecodeError) as e:
         print_error(
             f"Unable to load configuration data from '{CONFIG_FILE}': {e}",
         )
@@ -196,7 +196,7 @@ def load_config_common() -> ConfigCommon:
             print_info(
                 f"Setting environment variable '{requests_ca_bundle}' to '{certificates}'"
             )
-            os.environ["REQUESTS_CA_BUNDLE"] = certificates
+            os.environ[requests_ca_bundle] = certificates
 
         return ConfigCommon(
             # Required
@@ -222,7 +222,7 @@ def import_toml(filename: str) -> Dict:
     try:
         common_config: Dict = load_toml_file_with_variable_substitutions(filename)
         return common_config[COMMON_SECTION]
-    except (FileNotFoundError, PermissionError, TomlDecodeError) as e:
+    except (FileNotFoundError, PermissionError, TOMLDecodeError) as e:
         print_error(f"Unable to load imported common configuration data: {e}")
         exit(1)
 
@@ -313,6 +313,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
             upload_taskoutput=wr_section.get(UPLOAD_TASKOUTPUT, False),
             completed_task_ttl=wr_section.get(COMPLETED_TASK_TTL, None),
             csv_files=csv_files,
+            disable_prealloaction=wr_section.get(DISABLE_PREALLOCATION, None),
             docker_env=wr_section.get(DOCKER_ENV, None),
             docker_options=wr_section.get(DOCKER_OPTIONS, None),
             docker_password=wr_section.get(DOCKER_PASSWORD, None),

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/property_names.py

@@ -15,6 +15,7 @@ COMPUTE_REQUIREMENT_SECTION = "computeRequirement"  # No value
 CR_TAG = "requirementTag"  # String
 CSV_FILE = "csvFile"  # String
 CSV_FILES = "csvFiles"  # List of Strings
+DISABLE_PREALLOCATION = "disablePreallocation"
 DEPENDENCIES = "dependencies"  # List of Strings
 DEPENDENT_ON = "dependentOn"  # String (Deprecated)
 DIRECTORY_NAME = "directoryName"  # String
@@ -130,6 +131,7 @@ ALL_KEYS = [
     CR_TAG,
     CSV_FILE,
     CSV_FILES,
+    DISABLE_PREALLOCATION,
     DEPENDENCIES,
     DEPENDENT_ON,
     DIRECTORY_NAME,

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/settings.py

@@ -66,6 +66,7 @@ HIGHLIGHTED_STATES = [
     r"(?P<active>ALIVE)",
     r"(?P<active>MATCHING)",
     r"(?P<active>MAYBE MATCHING)",
+    r"(?P<active>FINISHING)",
     r"(?P<cancelled>ABORTED)",
     r"(?P<cancelled>CANCELLED)",
     r"(?P<cancelled>CANCELLING)",

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_cli/utils/variables.py

@@ -12,7 +12,7 @@ from json import loads as json_loads
 from random import randint
 from typing import Dict, List, Optional, Union
 
-from toml import load as toml_load
+from tomli import load as toml_load
 
 from yellowdog_cli.utils.args import ARGS_PARSER
 from yellowdog_cli.utils.check_imports import check_jsonnet_import
@@ -130,6 +130,13 @@ def add_or_update_substitution(key: str, value: str):
     VARIABLE_SUBSTITUTIONS[key] = str(value)
 
 
+def get_user_variable(variable_name: str) -> Optional[str]:
+    """
+    Get the value of a variable.
+    """
+    return VARIABLE_SUBSTITUTIONS.get(variable_name)
+
+
 def process_variable_substitutions_insitu(
     data: Union[Dict, List], prefix: str = "", postfix: str = ""
 ) -> Union[Dict, List]:
@@ -493,7 +500,7 @@ def load_toml_file_with_variable_substitutions(
     Takes a TOML filename and returns a dictionary with its variable
     substitutions processed.
     """
-    with open(resolve_filename(files_directory, filename), "r") as f:
+    with open(resolve_filename(files_directory, filename), "rb") as f:
         config = toml_load(f)
 
     # Add any variable substitutions in the TOML file before processing the

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_python_examples.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yellowdog-python-examples
-Version: 8.3.1
+Version: 8.4.1
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 License-Expression: Apache-2.0
@@ -25,8 +25,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: requests
 Requires-Dist: rich==13.9.4
 Requires-Dist: tabulate>=0.9.0
-Requires-Dist: toml
-Requires-Dist: yellowdog-sdk>=12.0.0
+Requires-Dist: tomli>=2.4.0
+Requires-Dist: yellowdog-sdk>=13.1.0
 Provides-Extra: jsonnet
 Requires-Dist: jsonnet; extra == "jsonnet"
 Provides-Extra: cloudwizard

{yellowdog_python_examples-8.3.1 → yellowdog_python_examples-8.4.1}/yellowdog_python_examples.egg-info/requires.txt

@@ -4,8 +4,8 @@ python-dotenv
 requests
 rich==13.9.4
 tabulate>=0.9.0
-toml
-yellowdog-sdk>=12.0.0
+tomli>=2.4.0
+yellowdog-sdk>=13.1.0
 
 [cloudwizard]
 boto3

yellowdog_python_examples-8.3.1/yellowdog_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "8.3.1"