yellowdog-python-examples 7.15.0__tar.gz → 7.15.2__tar.gz

{yellowdog_python_examples-7.15.0/yellowdog_python_examples.egg-info → yellowdog_python_examples-7.15.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: yellowdog-python-examples
-Version: 7.15.0
+Version: 7.15.2
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 Project-URL: Homepage, https://github.com/yellowdog/python-examples

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/README.md

@@ -34,7 +34,7 @@
          * [Bash, Python, PowerShell and cmd/bat Tasks](#bash-python-powershell-and-cmdbat-tasks)
          * [Docker Tasks](#docker-tasks)
          * [Bash, Python, PowerShell, cmd.exe/batch, and Docker without Automatic Processing](#bash-python-powershell-cmdexebatch-and-docker-without-automatic-processing)
-      * [Task Counts](#task-counts)
+      * [Task and Task Group Counts](#task-and-task-group-counts)
    * [Examples](#examples)
       * [TOML Properties in the workRequirement Section](#toml-properties-in-the-workrequirement-section)
       * [JSON Properties at the Work Requirement Level](#json-properties-at-the-work-requirement-level)
@@ -126,7 +126,7 @@
    * [yd-analyse](#yd-analyse)
 
 <!-- Created by https://github.com/ekalinin/github-markdown-toc -->
-<!-- Added by: pwt, at: Wed Feb 12 12:13:59 GMT 2025 -->
+<!-- Added by: pwt, at: Sun Feb 23 10:18:32 GMT 2025 -->
 
 <!--te-->
 
@@ -620,6 +620,7 @@ All properties are optional except for **`taskType`** (or **`taskTypes`**).
 | `taskData`                  | The data to be passed to the Worker when the Task is started. E.g., `"mydata"`. Becomes file `taskdata.txt` in the Task's working directory when The Task executes.                                                                        | Yes  | Yes | Yes  | Yes  |
 | `taskDataFile`              | Populate the `taskData` property above with the contents of the specified file. E.g., `"my_task_data_file.txt"`.                                                                                                                           | Yes  | Yes | Yes  | Yes  |
 | `taskName`                  | The name to use for the Task. Only usable in the TOML file. Mostly useful in conjunction with CSV Task data. E.g., `"my_task_number_{{task_number}}"`.                                                                                     | Yes  |     |      |      |
+| `taskGroupCount`            | Create `taskGroupCount` duplicates of a single Task Group.                                                                                                                                                                                 | Yes  | Yes |      |      |
 | `taskGroupName`             | The name to use for the Task Group. Only usable in the TOML file. E.g., `"my_tg_number_{{task_group_number}}"`.                                                                                                                            | Yes  |     |      |      |
 | `taskTimeout`               | The timeout in minutes after which an executing Task will be terminated and reported as `FAILED`. E.g. `120.0`. The default is no timeout.                                                                                                 | Yes  | Yes | Yes  |      |
 | `timeout`                   | As above, but set at the individual Task level, which overrides the group level `taskTimeout` property (if present).                                                                                                                       | Yes  |     |      | Yes  |
@@ -763,11 +764,11 @@ arguments = ["--runtime=nvidia", "--gpus=all", "my_dockerhubrepo/my_container_im
 
 If the `executable` property is not supplied, none of the automatic processing described above for `bash`, `python`, `powershell`, `cmd` (or `bat`) and `docker` task types is applied.
 
-### Task Counts
+### Task and Task Group Counts
 
-The `taskCount` property can be used to expand the number of Tasks within a Task Group, by creating duplicates of a single Task; this can be handy for testing and demos. In JSON specifications, there must be zero or one Task(s) listed within each Task Group or `taskCount` is ignored.
+The `taskCount` property can be used to expand the number of Tasks within a Task Group, by creating duplicates of a single Task; this can be handy for testing and demos. In JSON specifications, there must be zero or one Task(s) listed within each Task Group or `taskCount` is ignored. This property can also be set on the command line using the `--task-count`/`-C` option of `yd-submit` followed by the required number of Tasks.
 
-This property can also be set on the command line using the `--task-count`/`-C` option of `yd-submit` followed by the required number of Tasks.
+Also useful for testing, the `taskGroupCount` property or the command line option `--task-group-count`/`-G` can be set to expand the number of Task Groups in the Work Requirement, by creating duplicates of a single Task Group. If used, the `taskCount` property will apply to every Task Group, i.e., the total number of tasks is the multiple of `taskGroupCount` and `taskGroup`.
 
 ## Examples
 
@@ -819,6 +820,7 @@ Here's an example of the `workRequirement` section of a TOML configuration file,
     taskData = "my_data_string"
     taskDataFile = "my_data_file.txt"
     taskName = "my_task_number_{{task_number}}"
+    taskGroupCount = 5
     taskGroupName = "my_task_group_number_{{task_group_number}}"
     taskTimeout = 120.0
     taskType = "docker"
@@ -872,6 +874,7 @@ Showing all possible properties at the Work Requirement level:
   "taskCount": 100,
   "taskData": "my_task_data_string",
   "taskDataFile": "my_data_file.txt",
+  "taskGroupCount": 5,
   "taskTimeout": 120.0,
   "taskTypes": ["docker"],
   "tasksPerWorker": 1,
@@ -1247,7 +1250,7 @@ It's possible to make Tasks dependent on the presence of files in the Object Sto
 
 Note that a given file can only appear in *one* of the `inputs`, `verifyAtStart` or `verifyWait` lists.
 
-Tasks with `verifyAtStart` dependencies will fail immediately if the required files are not present when the Task is submitted. Tasks with `verifyWait` dependencies will not become `READY` to be scheduled to Workers until their dependencies are satisfied.
+Tasks with `verifyAtStart` dependencies will fail immediately if the required files are not present when the Task is submitted. Tasks with `verifyWait` dependencies will not become `READY` to be allocated to Workers until their dependencies are satisfied.
 
 When specifying files in the `verifyAtStart` and `verifyWait` lists, as with the `uploadPath` property discussed above, the file locations can be (1) relative to the Work Requirement name in the current namespace (the default), (2) relative to the root of the current namespace, or (3) relative to the root of a different namespace in the user's Account.
 
@@ -1258,12 +1261,12 @@ When specifying files in the `verifyAtStart` and `verifyWait` lists, as with the
 
 2. For files relative to the root of the current namespace, prefix the file path with `::`, e.g.
 ```shell
-"verifyWait": ["::file_1.txt"] -> development:file_1.txt
+"verifyWait": ["::file_1.txt"] -> development::file_1.txt
 ```
 
 3. For files relative to the root of a different namespace, prefix the file path with the namespace name and `::`, e.g.
 ```shell
-"verifyWait": ["other_namespace::file_1.txt"] -> other_namespace:file_1.txt
+"verifyWait": ["other_namespace::file_1.txt"] -> other_namespace::file_1.txt
 ```
 
 The use of the three different forms can be mixed within a single list, e.g.:

yellowdog_python_examples-7.15.2/yellowdog_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "7.15.2"

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/analyse.py

@@ -82,11 +82,11 @@ class MatchReport:
         self._property_match_list = [
             self._worker_tags,
             self._namespaces,
-            self._task_types,
             self._instance_types,
             self._providers,
-            self._regions,
             self._ram,
+            self._regions,
+            self._task_types,
             self._vcpus,
         ]
 
@@ -124,7 +124,7 @@ class MatchReport:
         if self.summary() == MatchType.YES:
             match_str = "MATCHING"
         elif self.summary() == MatchType.MAYBE:
-            match_str = "POSSIBLY MATCHING"
+            match_str = "MAYBE MATCHING"
         elif self.summary() == MatchType.PARTIAL:
             match_str = "PARTIALLY MATCHING"
         else:

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/list.py

@@ -36,7 +36,6 @@ from yellowdog_client.model import (
     Task,
     TaskGroup,
     Worker,
-    WorkerPool,
     WorkerPoolStatus,
     WorkerPoolSummary,
     WorkerStatus,

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/show.py

@@ -4,15 +4,10 @@
 Command to show the JSON details of YellowDog entities via their IDs.
 """
 
-from typing import Optional
-
 from yellowdog_client.model import ConfiguredWorkerPool, Task
 
 from yellowdog_cli.list import get_keyring
-from yellowdog_cli.utils.entity_utils import (
-    get_task_by_id,
-    substitute_ids_for_names_in_crt,
-)
+from yellowdog_cli.utils.entity_utils import substitute_ids_for_names_in_crt
 from yellowdog_cli.utils.printing import print_error, print_log, print_yd_object
 from yellowdog_cli.utils.settings import (
     RESOURCE_PROPERTY_NAME,
@@ -151,22 +146,11 @@ def show_details(ydid: str, initial_indent: int = 0, with_final_comma: bool = Fa
 
         elif ydid_type == YDIDType.TASK:
             print_log(f"Showing details of Task ID '{ydid}'")
-            work_requirement = CLIENT.work_client.get_work_requirement_by_id(
-                ydid.rsplit(":", 2)[0].replace("task", "workreq")
-            )
-            for task_group in work_requirement.taskGroups:
-                if task_group.id == ydid.rsplit(":", 1)[0].replace("task", "taskgrp"):
-                    break
-            else:
-                print_error(f"Task Group ID '{ydid}' not found")
-                return
-            task: Optional[Task] = get_task_by_id(
-                CLIENT, work_requirement.id, task_group.id, ydid
-            )
-            if task is not None:
-                print_yd_object(task)
-            else:
-                print_error(f"Task ID '{ydid}' not found")
+            try:
+                print_yd_object(CLIENT.work_client.get_task_by_id(ydid))
+            except Exception as e:
+                if "404" in str(e):
+                    print_error(f"Task ID '{ydid}' not found")
 
         elif ydid_type == YDIDType.IMAGE_FAMILY:
             print_log(f"Showing details of Image Family ID '{ydid}'")

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/submit.py

@@ -290,7 +290,28 @@ def submit_work_requirement(
         wr_data.get(FLATTEN_UPLOAD_PATHS, CONFIG_WR.flatten_upload_paths)
     )
 
-    # Create the list of Task Groups
+    # Expand number of task groups if there's a single task group
+    # and taskGroupCount is set
+    task_group_count = check_float_or_int(
+        wr_data.get(TASK_GROUP_COUNT, CONFIG_WR.task_group_count)
+    )
+    if task_group_count > 1:
+        if len(wr_data[TASK_GROUPS]) == 1:
+            print_log(
+                f"Expanding number of Task Groups to '{TASK_GROUP_COUNT}="
+                f"{task_group_count}'"
+            )
+            wr_data[TASK_GROUPS] = [
+                deepcopy(wr_data[TASK_GROUPS][0]) for _ in range(task_group_count)
+            ]
+        elif len(wr_data[TASK_GROUPS]) > 1:
+            print_warning(
+                f"Note: Work Requirement already contains"
+                f" {len(wr_data[TASK_GROUPS])} Task Groups: ignoring expansion "
+                f"using '{TASK_GROUP_COUNT} = {int(task_group_count)}'"
+            )
+
+    # Create the list of TaskGroup objects
     task_groups: List[TaskGroup] = []
     for tg_number, task_group_data in enumerate(wr_data[TASK_GROUPS]):
         task_groups.append(create_task_group(tg_number, wr_data, task_group_data))
@@ -571,16 +592,16 @@ def add_tasks_to_task_group(
             # Expand the number of Tasks to match the specified Task count
             print_log(
                 f"Expanding number of Tasks in Task Group '{task_group.name}' to"
-                f" 'taskCount={task_group_task_count}' Tasks"
+                f" '{TASK_COUNT}={task_group_task_count}' Tasks"
             )
             for _ in range(1, task_group_task_count):
                 wr_data[TASK_GROUPS][tg_number][TASKS].append(
                     deepcopy(wr_data[TASK_GROUPS][tg_number][TASKS][0])
                 )
         elif task_group_task_count > 1:
-            print_log(
+            print_warning(
                 f"Note: Task Group '{task_group.name}' already contains"
-                f" {num_tasks} Tasks: ignoring expansion using 'taskCount ="
+                f" {num_tasks} Tasks: ignoring expansion using '{TASK_COUNT} ="
                 f" {int(task_group_task_count)}'"
             )
 

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/args.py

@@ -245,9 +245,17 @@ class CLIParser:
                 "-C",
                 type=int,
                 required=False,
-                help="the number of times to submit the Task",
+                help="the number of tasks to submit (copies of a single task)",
                 metavar="<task_count>",
             )
+            parser.add_argument(
+                "--task-group-count",
+                "-G",
+                type=int,
+                required=False,
+                help="the number of task groups to submit (copies of a single task group)",
+                metavar="<task_group_count>",
+            )
             parser.add_argument(
                 "--task-batch-size",
                 "-b",
@@ -1330,6 +1338,11 @@ class CLIParser:
     def task_count(self) -> Optional[int]:
         return self.args.task_count
 
+    @property
+    @allow_missing_attribute
+    def task_group_count(self) -> Optional[int]:
+        return self.args.task_group_count
+
     @property
     @allow_missing_attribute
     def csv_files(self) -> Optional[List[str]]:
@@ -1670,7 +1683,8 @@ def lookup_module_description(module_name: str) -> Optional[str]:
         suffix = "showing the JSON details of entities referenced by their YDIDs"
     elif "analyse" in module_name:
         suffix = (
-            "analysing if a task group is matched by workers in selected worker pools"
+            "analysing if a work requirement or single task group is matched by "
+            "workers in the specified worker pools"
         )
 
     return None if suffix is None else prefix + suffix

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/config_types.py

@@ -55,6 +55,7 @@ class ConfigWorkRequirement:
     task_count: int = 1
     task_data: Optional[str] = None
     task_data_file: Optional[str] = None
+    task_group_count: int = 1
     task_group_name: Optional[str] = None
     task_level_timeout: Optional[float] = None
     task_name: Optional[str] = None

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/entity_utils.py

@@ -440,23 +440,10 @@ def list_matching_object_paths(
     ]
 
 
-def get_task_by_id(
-    client: PlatformClient, wr_id: str, task_group_id: str, task_id: str
-) -> Optional[Task]:
-    """
-    Find a task by its ID.
-    """
-    tasks: List[Task] = get_tasks(client, wr_id, task_group_id)
-    for task in tasks:
-        if task.id == task_id:
-            return task
-
-
 @lru_cache
 def get_tasks(client: PlatformClient, wr_id: str, task_group_id: str) -> List[Task]:
     """
     Return all the tasks in a task group, with caching.
-    There is no native way to search for a task by its id.
     """
     task_search = TaskSearch(
         workRequirementId=wr_id,

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/follow_utils.py

@@ -3,6 +3,7 @@ Utility function to follow event streams.
 """
 
 from threading import Thread
+from time import sleep
 from typing import List, Optional
 
 import requests
@@ -19,6 +20,8 @@ from yellowdog_cli.utils.printing import (
 from yellowdog_cli.utils.wrapper import CLIENT, CONFIG_COMMON
 from yellowdog_cli.utils.ydid_utils import YDIDType, get_ydid_type
 
+EVENT_STREAM_RETRY_INTERVAL = 5.0  # Seconds
+
 
 def follow_ids(ydids: List[str], auto_cr: bool = False):
     """
@@ -82,25 +85,39 @@ def follow_events(ydid: str, ydid_type: YDIDType):
     """
     Follow events.
     """
-    response = requests.get(
-        headers={"Authorization": f"yd-key {CONFIG_COMMON.key}:{CONFIG_COMMON.secret}"},
-        url=get_event_url(ydid, ydid_type),
-        stream=True,
-    )
-
-    if response.status_code != 200:
-        print_error(f"'{ydid}': {response.json()['message']}")
-        return
+    while True:
+        response = requests.get(
+            headers={
+                "Authorization": f"yd-key {CONFIG_COMMON.key}:{CONFIG_COMMON.secret}"
+            },
+            url=get_event_url(ydid, ydid_type),
+            stream=True,
+        )
+
+        if response.status_code != 200:
+            print_error(f"'{ydid}': {response.json()['message']}")
+            return
+
+        if response.encoding is None:
+            response.encoding = "utf-8"
 
-    if response.encoding is None:
-        response.encoding = "utf-8"
-
-    try:
-        for event in response.iter_lines(decode_unicode=True):
-            if event:
-                print_event(event, ydid_type)
-    except Exception as e:
-        print_warning(f"Event stream error: {e}")
+        try:
+            for event in response.iter_lines(decode_unicode=True):
+                if event:
+                    print_event(event, ydid_type)
+            break
+
+        except Exception as e:
+            if "Connection broken" in str(e):
+                print_warning(
+                    f"Event stream interruption for '{ydid}'"
+                    f"(retrying in {EVENT_STREAM_RETRY_INTERVAL}s): {e}"
+                )
+                sleep(EVENT_STREAM_RETRY_INTERVAL)
+                continue
+            else:
+                print_error(f"Event stream error: {e}")
+                break
 
     print_log(f"Event stream concluded for '{ydid}'")
 

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/load_config.py

@@ -266,6 +266,12 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
             else wr_section.get(TASK_COUNT, 1)
         )
 
+        task_group_count = (
+            ARGS_PARSER.task_group_count
+            if ARGS_PARSER.task_group_count is not None
+            else wr_section.get(TASK_GROUP_COUNT, 1)
+        )
+
         return ConfigWorkRequirement(
             add_yd_env_vars=wr_section.get(ADD_YD_ENV_VARS, False),
             always_upload=wr_section.get(ALWAYS_UPLOAD, True),
@@ -305,6 +311,7 @@ def load_config_work_requirement() -> ConfigWorkRequirement:
             task_count=task_count,
             task_data=wr_section.get(TASK_DATA, None),
             task_data_file=wr_section.get(TASK_DATA_FILE, None),
+            task_group_count=task_group_count,
             task_group_name=wr_section.get(TASK_GROUP_NAME, None),
             task_name=wr_section.get(TASK_NAME, None),
             task_timeout=wr_section.get(TASK_TIMEOUT, None),

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/property_names.py

@@ -68,6 +68,7 @@ TASK_COUNT = "taskCount"  # Integer
 TASK_DATA = "taskData"  # String
 TASK_DATA_FILE = "taskDataFile"  # String
 TASK_GROUPS = "taskGroups"  # List of Task Groups
+TASK_GROUP_COUNT = "taskGroupCount"  # Integer
 TASK_GROUP_NAME = "taskGroupName"  # String
 TASK_GROUP_TAG = "tag"  # String
 TASK_LEVEL_TIMEOUT = "timeout"  # Float
@@ -167,6 +168,7 @@ ALL_KEYS = [
     TASK_DATA,
     TASK_DATA_FILE,
     TASK_GROUPS,
+    TASK_GROUP_COUNT,
     TASK_GROUP_NAME,
     TASK_GROUP_TAG,
     TASK_LEVEL_TIMEOUT,

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2}/yellowdog_cli/utils/settings.py

@@ -55,6 +55,7 @@ HIGHLIGHTED_STATES = [
     r"(?P<active>RUNNING)",
     r"(?P<active>TARGET)",
     r"(?P<active>ALIVE)",
+    r"(?P<active>MATCHING)",
     r"(?P<cancelled>ABORTED)",
     r"(?P<cancelled>CANCELLED)",
     r"(?P<cancelled>CANCELLING)",
@@ -66,6 +67,7 @@ HIGHLIGHTED_STATES = [
     r"(?P<failed>FAILED)",
     r"(?P<failed>FAILING)",
     r"(?P<failed>LOST)",
+    r"(?P<failed>NON-MATCHING)",
     r"(?P<idle>EMPTY)",
     r"(?P<idle>FOUND)",
     r"(?P<idle>IDLE)",

{yellowdog_python_examples-7.15.0 → yellowdog_python_examples-7.15.2/yellowdog_python_examples.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: yellowdog-python-examples
-Version: 7.15.0
+Version: 7.15.2
 Summary: Python CLI commands using the YellowDog Python SDK
 Author-email: YellowDog Limited <support@yellowdog.co>
 Project-URL: Homepage, https://github.com/yellowdog/python-examples

yellowdog_python_examples-7.15.0/yellowdog_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "7.15.0"