yellowdog-python-examples 8.1.6__tar.gz → 8.2.0__tar.gz

{yellowdog_python_examples-8.1.6/yellowdog_python_examples.egg-info → yellowdog_python_examples-8.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yellowdog-python-examples
-Version: 8.1.6
+Version: 8.2.0
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 License-Expression: Apache-2.0
@@ -26,7 +26,7 @@ Requires-Dist: requests
 Requires-Dist: rich==13.9.4
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: toml
-Requires-Dist: yellowdog-sdk>=11.6.0
+Requires-Dist: yellowdog-sdk>=11.7.0
 Provides-Extra: jsonnet
 Requires-Dist: jsonnet; extra == "jsonnet"
 Provides-Extra: cloudwizard

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/README.md

@@ -614,6 +614,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
 | `addYDEnvironmentVariables` | Automatically add YellowDog environment variables to each Task's environment.                                                                                                                                                              | Yes  | Yes | Yes  | Yes  |
 | `alwaysUpload`              | Whether to attempt to upload task outputs on failure. Default: `true`.                                                                                                                                                                     | Yes  | Yes | Yes  | Yes  |
 | `arguments`                 | The list of arguments to be passed to the Task when it is executed. E.g.: `[1, "Two"]`.                                                                                                                                                    | Yes  | Yes | Yes  | Yes  |
+| `batchAllocation`           | Enables the batch allocation of tasks to nodes. Boolean, Default: `false`. Requires `exclusiveWorkers = true`.                                                                                                                             | Yes  | Yes | Yes  |      |
 | `completedTaskTtl`          | The time (in minutes) to live for completed Tasks. If set, Tasks that have been completed for longer than this period will be deleted. E.g.: `10.0`.                                                                                       | Yes  | Yes | Yes  |      |
 | `csvFile`                   | The name of the CSV file used to derive Task data. An alternative to `csvFiles` that can be used when there's only a single CSV file. E.g. `"file.csv"`.                                                                                   | Yes  |     |      |      |
 | `csvFiles`                  | A list of CSV files used to derive Task data. E.g. `["file.csv", "file_2.csv:2]`.                                                                                                                                                          | Yes  |     |      |      |
@@ -819,6 +820,7 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
     addYDEnvironmentVariables = true
     alwaysUpload = true
     arguments = ["1", "TWO"]
+    batchAllocation = false
     completedTaskTtl = 10
     csvFile = "file1.csv"
     csvFiles = ["file1.csv", "file3.csv:3"]
@@ -894,6 +896,7 @@ Showing all possible properties at the Work Requirement level:
   "addYDEnvironmentVariables": true,
   "alwaysUpload": true,
   "arguments": [1, "TWO"],
+  "batchAllocation": false,
   "completedTaskTtl": 10,
   "dockerEnvironment": {"MY_DOCKER_VAR": 100},
   "dockerPassword": "myPassword",
@@ -973,6 +976,7 @@ Showing all possible properties at the Task Group level:
       "addYDEnvironmentVariables": true,
       "alwaysUpload": true,
       "arguments": [1, "TWO"],
+      "batchAllocation": false,
       "completedTaskTtl": 10,
       "dockerEnvironment": {"MY_DOCKER_VAR": 100},
       "dockerPassword": "myPassword",
@@ -1992,7 +1996,7 @@ yd-remove --ids ydid:crt:D9C548:2a09093d-c74c-4bde-95d1-c576c6f03b13 ydid:imgfam
 
 Resources match on **resource names** and (where applicable) **resource namespaces** rather than on YellowDog IDs. This is done for flexibility and to allow the `yd-create` and `yd-remove` commands to be stateless (i.e., we don't need to keep a local record of the YellowDog IDs of the resources created).
 
-However, this means that **caution is required** when updating or removing resources, since resource matching is done using **only** the **namespace/name** of the resource -- i.e., the system-generated `ydid` IDs are not used. This means that a resource with a given name could have been removed and replaced in Platform by some other means, and the resource specification(s) would still match it.
+However, this means that **caution is required** when updating or removing resources, since resource matching is done using **only** the **namespace/name** of the resource -- i.e., the system-generated `ydid` IDs are not used. This means that a resource with a given name could have been removed and replaced in the platform by some other means, and the resource specification(s) would still match it.
 
 ## Resource Specification Definitions
 

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/requirements.txt

@@ -5,4 +5,4 @@ requests
 rich == 13.9.4
 tabulate >= 0.9.0
 toml
-yellowdog-sdk >= 11.6.0
+yellowdog-sdk >= 11.7.0

yellowdog_python_examples-8.2.0/yellowdog_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "8.2.0"

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/abort.py

@@ -37,9 +37,9 @@ def main():
         return
 
     print_log(
-        "Finding active Work Requirements in"
-        f"namespace '{CONFIG_COMMON.namespace}' with "
-        f"'{CONFIG_COMMON.name_tag}' in tag"
+        "Finding active Work Requirements in "
+        f"namespace '{CONFIG_COMMON.namespace}' with tags "
+        f"including '{CONFIG_COMMON.name_tag}'"
     )
 
     # Abort Tasks is always interactive

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/cancel.py

@@ -32,8 +32,8 @@ def main():
 
     print_log(
         "Cancelling Work Requirements in namespace "
-        f"'{CONFIG_COMMON.namespace}' with "
-        f"'{CONFIG_COMMON.name_tag}' in tag"
+        f"'{CONFIG_COMMON.namespace}' with tags "
+        f"including '{CONFIG_COMMON.name_tag}'"
     )
 
     selected_work_requirement_summaries: List[WorkRequirementSummary] = (

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/create.py

@@ -7,7 +7,7 @@ A script to create or update YellowDog resources.
 from copy import deepcopy
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Optional, Set, Tuple
 
 import yellowdog_client.model as model
 from dateparser import parse as date_parse
@@ -278,7 +278,7 @@ def create_compute_source_template(resource: Dict):
         compute_source = CLIENT.compute_client.add_compute_source_template(
             compute_source_template
         )
-        print_log(f"Created Compute Source Template '{name}'" f" ({compute_source.id})")
+        print_log(f"Created Compute Source Template '{name}' ({compute_source.id})")
     else:
         if not confirmed(f"Update existing Compute Source Template '{name}'?"):
             return
@@ -405,9 +405,7 @@ def create_compute_requirement_template(resource: Dict):
     template = CLIENT.compute_client.update_compute_requirement_template(
         compute_template
     )
-    print_log(
-        f"Updated existing Compute Requirement Template '{name}'" f" ({template.id})"
-    )
+    print_log(f"Updated existing Compute Requirement Template '{name}' ({template.id})")
     if ARGS_PARSER.quiet:
         print(template.id)
 
@@ -446,7 +444,7 @@ def create_keyring(resource: Dict, show_secrets: bool = False):
         print_error(f"Failed to create Keyring '{name}': {e}")
 
 
-def create_keyring_via_api(name: str, description: str) -> (Keyring, str):
+def create_keyring_via_api(name: str, description: str) -> Tuple[Keyring, str]:
     """
     Temporary direct API call to create a Keyring and return the shown-once
     password. The password is not available via the SDK call.

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/finish.py

@@ -31,8 +31,8 @@ def main():
 
     print_log(
         "Finishing Work Requirements in namespace "
-        f"'{CONFIG_COMMON.namespace}' with "
-        f"'{CONFIG_COMMON.name_tag}' in tag"
+        f"'{CONFIG_COMMON.namespace}' with tags "
+        f"including '{CONFIG_COMMON.name_tag}'"
     )
 
     selected_work_requirement_summaries: List[WorkRequirementSummary] = (
@@ -59,7 +59,7 @@ def main():
         )
 
     if len(selected_work_requirement_summaries) > 0 and confirmed(
-        f"Finish {len(selected_work_requirement_summaries)} " f"Work Requirement(s)?"
+        f"Finish {len(selected_work_requirement_summaries)} Work Requirement(s)?"
     ):
         for work_summary in selected_work_requirement_summaries:
             if work_summary.status != WorkRequirementStatus.FINISHING:

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/list.py

@@ -115,7 +115,7 @@ def main():
         ARGS_PARSER.details = True
 
     if ARGS_PARSER.details and ARGS_PARSER.strip_ids:
-        print_log("Omitting YellowDog IDs (etc.) from detailed JSON objects")
+        print_log("Stripping YellowDog IDs (etc.) from detailed JSON objects")
 
     if ARGS_PARSER.output_file and ARGS_PARSER.details:
         if exists(ARGS_PARSER.output_file):

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/show.py

@@ -44,7 +44,7 @@ def main():
     generate_json_list = len(ARGS_PARSER.yellowdog_ids) > 1 and ARGS_PARSER.quiet
 
     if ARGS_PARSER.strip_ids:
-        print_log("Omitting YellowDog IDs (etc.) from detailed JSON objects")
+        print_log("Stripping YellowDog IDs (etc.) from detailed JSON objects")
 
     if generate_json_list:
         print("[")

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/shutdown.py

@@ -16,11 +16,12 @@ from yellowdog_client.model import (
 from yellowdog_cli.utils.entity_utils import (
     get_worker_pool_by_id,
     get_worker_pool_id_by_name,
+    get_worker_pools,
 )
 from yellowdog_cli.utils.follow_utils import follow_ids
 from yellowdog_cli.utils.interactive import confirmed, select
 from yellowdog_cli.utils.misc_utils import link_entity
-from yellowdog_cli.utils.printing import print_error, print_log
+from yellowdog_cli.utils.printing import print_error, print_log, print_warning
 from yellowdog_cli.utils.wrapper import ARGS_PARSER, CLIENT, CONFIG_COMMON, main_wrapper
 from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
 
@@ -33,13 +34,14 @@ def main():
 
     print_log(
         "Shutting down Worker Pools in "
-        f"namespace '{CONFIG_COMMON.namespace}' and "
+        f"namespace '{CONFIG_COMMON.namespace}' with "
         f"names including '{CONFIG_COMMON.name_tag}'"
     )
 
-    worker_pool_summaries: List[WorkerPoolSummary] = (
-        CLIENT.worker_pool_client.find_all_worker_pools()
+    worker_pool_summaries: List[WorkerPoolSummary] = get_worker_pools(
+        CLIENT, CONFIG_COMMON.namespace, CONFIG_COMMON.name_tag
     )
+
     shutdown_count = 0
 
     selected_worker_pool_summaries: List[WorkerPoolSummary] = []
@@ -73,7 +75,7 @@ def main():
                 print_log(f"Shut down {link_entity(CONFIG_COMMON.url, worker_pool)}")
                 optionally_terminate_compute_requirement(worker_pool_summary.id)
             except Exception as e:
-                print_error(f"Unable to shut down '{worker_pool_summary.name}': {e}")
+                print_error(f"Failed to shut down '{worker_pool_summary.name}': {e}")
 
     if shutdown_count > 0:
         print_log(f"Shut down {shutdown_count} Worker Pool(s)")
@@ -101,7 +103,7 @@ def shutdown_by_names_or_ids(names_or_ids: List[str]):
         else:
             worker_pool_id = get_worker_pool_id_by_name(CLIENT, name_or_id)
             if worker_pool_id is None:
-                print_error(f"Worker Pool '{name_or_id}' not found")
+                print_warning(f"Worker Pool '{name_or_id}' not found")
                 continue
         worker_pool_ids.append(worker_pool_id)
 
@@ -120,14 +122,14 @@ def shutdown_by_names_or_ids(names_or_ids: List[str]):
             print_log(f"Shut down Worker Pool '{worker_pool_id}'")
             optionally_terminate_compute_requirement(worker_pool_id)
         except Exception as e:
-            print_error(f"Unable to shut down Worker Pool '{worker_pool_id}': ({e})")
+            print_error(f"Failed to shut down Worker Pool '{worker_pool_id}': ({e})")
 
     for node_id in node_ids:
         try:
             CLIENT.worker_pool_client.shutdown_node_by_id(node_id)
             print_log(f"Shut down Node '{node_id}'")
         except Exception as e:
-            print_error(f"Unable to shut down Node '{node_id}': ({e})")
+            print_error(f"Failed to to shut down Node '{node_id}': ({e})")
 
     if ARGS_PARSER.follow:
         follow_ids(worker_pool_ids, auto_cr=ARGS_PARSER.auto_cr)
@@ -151,7 +153,7 @@ def optionally_terminate_compute_requirement(worker_pool_id: str):
             f"Terminated associated Compute Requirement '{worker_pool.computeRequirementId}'"
         )
     except Exception as e:
-        print_error(f"Error terminating Compute Requirement: ({e})")
+        print_error(f"Failed to terminate Compute Requirement: ({e})")
 
 
 # Entry point

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/submit.py

@@ -477,6 +477,7 @@ def create_task_group(
         if providers_data is None
         else [CloudProvider(provider) for provider in providers_data]
     )
+
     task_timeout_minutes: Optional[float] = check_float_or_int(
         task_group_data.get(TASK_TIMEOUT, config_wr.task_timeout)
     )
@@ -486,6 +487,24 @@ def create_task_group(
         else timedelta(minutes=task_timeout_minutes)
     )
 
+    exclusive_workers = check_bool(
+        task_group_data.get(
+            EXCLUSIVE_WORKERS,
+            wr_data.get(EXCLUSIVE_WORKERS, config_wr.exclusive_workers),
+        )
+    )
+    batch_allocation = check_bool(
+        task_group_data.get(
+            BATCH_ALLOCATION,
+            wr_data.get(BATCH_ALLOCATION, config_wr.batch_allocation),
+        )
+    )
+    if batch_allocation and not exclusive_workers:
+        raise Exception(
+            f"Property '{EXCLUSIVE_WORKERS}' must be set when "
+            f"'{BATCH_ALLOCATION}' is specified"
+        )
+
     run_specification = RunSpecification(
         taskTypes=task_types,
         maximumTaskRetries=check_int(
@@ -498,12 +517,8 @@ def create_task_group(
                 WORKER_TAGS, wr_data.get(WORKER_TAGS, config_wr.worker_tags)
             )
         ),
-        exclusiveWorkers=check_bool(
-            task_group_data.get(
-                EXCLUSIVE_WORKERS,
-                wr_data.get(EXCLUSIVE_WORKERS, config_wr.exclusive_workers),
-            )
-        ),
+        exclusiveWorkers=exclusive_workers,
+        batchAllocation=batch_allocation,
         instanceTypes=check_list(
             task_group_data.get(
                 INSTANCE_TYPES, wr_data.get(INSTANCE_TYPES, config_wr.instance_types)

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/terminate.py

@@ -40,7 +40,7 @@ def main():
 
     print_log(
         "Terminating Compute Requirements in "
-        f"namespace '{CONFIG_COMMON.namespace}' and tag "
+        f"namespace '{CONFIG_COMMON.namespace}' with tags "
         f"including '{CONFIG_COMMON.name_tag}'"
     )
 
@@ -77,7 +77,7 @@ def main():
                 )
             except Exception as e:
                 print_error(
-                    f"Unable to terminate '{compute_requirement_summary.name}': {e}"
+                    f"Failed to terminate '{compute_requirement_summary.name}': {e}"
                 )
 
     if terminated_count > 0:
@@ -125,7 +125,7 @@ def terminate_cr_by_name_or_id(names_or_ids: List[str]):
             )
             print_log(f"Terminated '{compute_requirement_id}'")
         except Exception as e:
-            print_error(f"Unable to terminate '{compute_requirement_id}': ({e})")
+            print_error(f"Failed to terminate '{compute_requirement_id}': ({e})")
 
     if ARGS_PARSER.follow:
         follow_ids(compute_requirement_ids)

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/utils/config_types.py

@@ -23,6 +23,7 @@ class ConfigWorkRequirement:
     add_yd_env_vars: bool = False
     always_upload: bool = True
     args: List[str] = field(default_factory=list)
+    batch_allocation: Optional[bool] = None
     completed_task_ttl: Optional[float] = None  # In minutes
     csv_files: Optional[List[str]] = None
     docker_env: Optional[Dict] = None

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/utils/entity_utils.py

@@ -54,7 +54,7 @@ from yellowdog_client.model import (
 
 from yellowdog_cli.utils.args import ARGS_PARSER
 from yellowdog_cli.utils.interactive import confirmed, select
-from yellowdog_cli.utils.printing import print_log, print_warning
+from yellowdog_cli.utils.printing import print_log
 from yellowdog_cli.utils.settings import NAMESPACE_PREFIX_SEPARATOR
 from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
 

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/utils/load_config.py

@@ -309,6 +309,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
             add_yd_env_vars=wr_section.get(ADD_YD_ENV_VARS, False),
             always_upload=wr_section.get(ALWAYS_UPLOAD, True),
             args=wr_section.get(ARGS, []),
+            batch_allocation=wr_section.get(BATCH_ALLOCATION, None),
             upload_taskoutput=wr_section.get(UPLOAD_TASKOUTPUT, False),
             completed_task_ttl=wr_section.get(COMPLETED_TASK_TTL, None),
             csv_files=csv_files,

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/utils/property_names.py

@@ -5,6 +5,7 @@ Property names.
 ADD_YD_ENV_VARS = "addYDEnvironmentVariables"
 ALWAYS_UPLOAD = "alwaysUpload"  # Bool
 ARGS = "arguments"  # List
+BATCH_ALLOCATION = "batchAllocation"  # Bool
 CERTIFICATES = "certificates"
 COMMON_SECTION = "common"  # No value
 COMPLETED_TASK_TTL = "completedTaskTtl"  # Float
@@ -37,7 +38,7 @@ IMAGES_ID = "imagesId"  # String
 IMPORT_COMMON = "importCommon"  # String
 INPUTS_OPTIONAL = "inputsOptional"  # List of Strings
 INPUTS_REQUIRED = "inputs"  # List of Strings
-INSTANCE_TAGS = "instanceTags"  # List of Strings
+INSTANCE_TAGS = "instanceTags"  # Dictionary
 INSTANCE_TYPES = "instanceTypes"  # List of Strings
 KEY = "key"  # String
 LOCAL_PATH = "localPath"  # String
@@ -119,6 +120,7 @@ ALL_KEYS = [
     ADD_YD_ENV_VARS,
     ALWAYS_UPLOAD,
     ARGS,
+    BATCH_ALLOCATION,
     CERTIFICATES,
     COMMON_SECTION,
     COMPLETED_TASK_TTL,

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_cli/utils/provision_utils.py

@@ -11,7 +11,6 @@ from yellowdog_cli.utils.config_types import ConfigWorkerPool
 from yellowdog_cli.utils.entity_utils import (
     find_compute_requirement_template_id_by_name,
     find_image_name_or_id,
-    split_namespace_and_name,
 )
 from yellowdog_cli.utils.load_config import CONFIG_FILE_DIR
 from yellowdog_cli.utils.printing import print_log

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0/yellowdog_python_examples.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yellowdog-python-examples
-Version: 8.1.6
+Version: 8.2.0
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 License-Expression: Apache-2.0
@@ -26,7 +26,7 @@ Requires-Dist: requests
 Requires-Dist: rich==13.9.4
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: toml
-Requires-Dist: yellowdog-sdk>=11.6.0
+Requires-Dist: yellowdog-sdk>=11.7.0
 Provides-Extra: jsonnet
 Requires-Dist: jsonnet; extra == "jsonnet"
 Provides-Extra: cloudwizard

{yellowdog_python_examples-8.1.6 → yellowdog_python_examples-8.2.0}/yellowdog_python_examples.egg-info/requires.txt

@@ -5,7 +5,7 @@ requests
 rich==13.9.4
 tabulate>=0.9.0
 toml
-yellowdog-sdk>=11.6.0
+yellowdog-sdk>=11.7.0
 
 [cloudwizard]
 boto3

yellowdog_python_examples-8.1.6/yellowdog_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "8.1.6"