yellowdog-python-examples 5.0.0__tar.gz → 5.0.2__tar.gz

{yellowdog-python-examples-5.0.0/yellowdog_python_examples.egg-info → yellowdog-python-examples-5.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: yellowdog-python-examples
-Version: 5.0.0
+Version: 5.0.2
 Summary: Example Python commands using the YellowDog Python SDK
 Home-page: https://github.com/yellowdog/python-examples
 Author: YellowDog Limited

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2}/README.md

@@ -15,6 +15,7 @@
 * [Variable Substitutions](#variable-substitutions)
    * [Default Variables](#default-variables)
    * [User-Defined Variables](#user-defined-variables)
+   * [Nested Variables](#nested-variables)
 * [Work Requirement Properties](#work-requirement-properties)
    * [Work Requirement JSON File Structure](#work-requirement-json-file-structure)
    * [Property Inheritance](#property-inheritance)
@@ -45,6 +46,11 @@
       * [Files Downloaded to a Node for use in Task Execution](#files-downloaded-to-a-node-for-use-in-task-execution)
       * [Files Uploaded from a Node to the Object Store after Task Execution](#files-uploaded-from-a-node-to-the-object-store-after-task-execution)
       * [Files Downloaded from the Object Store to Local Storage](#files-downloaded-from-the-object-store-to-local-storage)
+   * [Task Execution Context](#task-execution-context)
+      * [Task Execution Steps](#task-execution-steps)
+      * [The User and Group used for Tasks](#the-user-and-group-used-for-tasks)
+      * [Home Directory for yd-agent](#home-directory-for-yd-agent)
+      * [Task Execution Directory](#task-execution-directory)
    * [Specifying Work Requirements using CSV Data](#specifying-work-requirements-using-csv-data)
       * [Work Requirement CSV Data Example](#work-requirement-csv-data-example)
       * [CSV Variable Substitutions](#csv-variable-substitutions)
@@ -78,7 +84,7 @@
       * [Test-Running a Dynamic Template](#test-running-a-dynamic-template)
    * [yd-terminate](#yd-terminate)
 
-<!-- Added by: pwt, at: Mon Mar 13 14:01:55 GMT 2023 -->
+<!-- Added by: pwt, at: Fri Apr 14 15:14:49 BST 2023 -->
 
 <!--te-->
 
@@ -335,6 +341,26 @@ Directives set on the command line take precedence over directives set in enviro
 
 This method can be used to override the default directives, e.g., setting `-v username="other-user"` will override the default `{{username}}` directive.
 
+## Nested Variables
+
+In the case of **TOML properties only**, variable substitutions can be nested.
+
+For example, if one wanted to select a different `templateId` for a Worker Pool depending on the value of a `region` variable, one could use the following:
+
+```toml
+[common.variables]
+    template_london = "ydid:crt:65EF4F:a4d757cf-b67a-4eb6-bd39-8a6ffd46c8f4"
+    template_phoenix = "ydid:crt:65EF4F:e4239dec-78c2-421c-a7f3-71e61b72946f"
+    template_frankfurt = "ydid:crt:65EF4F:329602cf-5945-4aad-a288-ea424d64d55e"
+
+[common.workerPool]
+    templateId = "{{template_{{region}}}}"
+```
+
+Then, if one used `yd-provision -v region=phoenix`, the `templateId` property would first resolve to `"{{template_pheonix}}"`, and then to `"ydid:crt:65EF4F:e4239dec-78c2-421c-a7f3-71e61b72946f"`.
+
+Nesting can be up to three levels deep including the top level.
+
 # Work Requirement Properties
 
 The `workRequirement` section of the configuration file is optional. It's used only by the `yd-submit` command, and controls the Work Requirement that is submitted to the Platform.
@@ -969,7 +995,7 @@ When a Task is started by the Agent, its working directory has a pattern somethi
 
 Files that are downloaded by the Agent prior to Task execution are located as follows:
 
-1. If the `flattenInputPaths` property is set to `false` for the Task (this is the default), the downloaded objects are placed in subdirectories that mirror those in the Object Store, including the Work Requirement name, situated beneath the working directory.
+1. If the `flattenInputPaths` property is set to the default of `false` for the Task, the downloaded objects are placed in subdirectories that mirror those in the Object Store, including the Work Requirement name, situated beneath the working directory.
 
 
 2. If the `flattenInputPaths` property is set to `true` for the Task, the downloaded objects are all placed directly in root of the Task's working directory.
@@ -977,7 +1003,7 @@ Files that are downloaded by the Agent prior to Task execution are located as fo
 For example:
 
 ```shell
-If the required object is: development:testrun_221108-120404-7d2/dev/file_1.txt
+If the required object is: development::testrun_221108-120404-7d2/dev/file_1.txt
 
 then, if flattenInputPaths is false, the file will be found at:
  -> <working_directory>/testrun_221108-120404-7d2/dev/file_1.txt
@@ -989,9 +1015,17 @@ where <working_directory> is:
   /var/opt/yellowdog/yd-agent-4/data/workers/1/ydid_task_D0D0D0_68f5e5be-dc93-49eb-a824-1fcdb52f9195_1_1/
 ```
 
+Note that Work Requirement name (e.g., `testrun_221108-120404-7d2`) is available via the variable substitution `wr_name`, so this could be supplied to the Task to help it locate its downloaded files. For example, in the `workRequirement` section of the `config.toml` file, I could specify:
+
+```toml
+environment = {WR_DIRECTORY = "{{wr_name}}"}
+```
+
+The Work Requirement name would then be available to the Task in the environment variable `$WR_DIRECTORY`.
+
 ### Files Uploaded from a Node to the Object Store after Task Execution
 
-After Task completion, the Agent will upload specified output files to the Object Store. The files to be uploaded are those listed in the `outputs` property for the Task.
+After Task completion, the Agent will upload specified output files to the Object Store. The files to be uploaded are those listed in the `outputs` and `outputsRequired` properties for the Task.
 
 In addition, the console output of the Task is captured in a file called `taskoutput.txt` in the root of the Task's working directory. Whether the `taskoutput.txt` file is uploaded to the Object Store is determined by the `captureTaskOutput` property for the Task, and this is set to 'true' by default.
 
@@ -1036,6 +1070,69 @@ Note that everything within the `namespace::work-requirement` directory in the O
 
 If the `development` directory already exists, `yd-download` will try `development.01`, etc., to avoid overwriting previous downloads.
 
+## Task Execution Context
+
+This section discusses the context within which a Task operates when it's executed by a Worker on a node. It applies specifically to the YellowDog Agent running on a Linux node, and configured using the default username, directories, etc. Configurations can vary.
+
+### Task Execution Steps
+
+When a Task is allocated to a Worker on a node by the YellowDog Scheduler, the following steps are followed:
+
+1. The Agent running on the node downloads the Task's properties: its `taskType`,  `arguments`, `environment`, `taskdata`, and (from the Object Store) any files in the `inputs` list and any available files in the `inputsOptional` list.
+2. These files are placed in an ephemeral directory created for this Task's execution, and into which any output files are also placed by default.
+2. The Agent runs the command specified for the `taskType` in the Agent's `application.yaml` configuration file. This done as a simple `exec` of a subprocess.
+3. When the Task concludes, the Agent uses the exit code of the subprocess to report success (zero) or failure (non-zero).
+4. The Agent then gathers any files in the `outputs` and `outputsRequired` lists and uploads them to the Object Store. If a file in the `outputsRequired` list is not found, the Task will be reported as failed. The Agent will also optionally upload the console output (including both `stdout` and `stderr`) of the Task, contained in the `taskoutput.txt` file.
+5. The ephemeral Task directory is then deleted.
+
+Note that if a Task is aborted during execution, the Task's subprocess is sent a `SIGINT`, allowing the Task an opportunity to terminate any child processes or other resources (e.g., containers) that may have been started as part of Task execution.
+
+Once the steps above have been completed, the Worker is ready to accept its next Task from the YellowDog scheduler.
+
+Note that if the Agent on a node has multiple Workers, then Tasks are executed in parallel on the node and can start and stop independently.
+
+### The User and Group used for Tasks
+
+By default, the Agent runs as user and group `yd-agent`, and hence Tasks also execute under this user.
+
+`yd-agent` does not have `sudo` privileges as standard, but this can be added if required at instance boot time via the `userData` property of a provisioning request. E.g. (for Ubuntu):
+
+```shell
+usermod -aG wheel yd-agent
+echo -e "yd-agent\tALL=(ALL)\tNOPASSWD: ALL" > /etc/sudoers.d/020-yd-agent
+```
+
+### Home Directory for `yd-agent`
+
+By default, the home directory of the `yd-agent` user is `/opt/yellowdog/agent`. This directory typically contains the `application.yaml` file used to configure the Agent, as well as any scripts that are used to execute the Task Types that the node supports.
+
+If one wants to SSH to an instance as user `yd-agent`, perhaps for debugging purposes, SSH keys can be inserted via instance `userData`, e.g.:
+
+```shell
+YDA_HOME=/opt/yellowdog/agent
+mkdir -p $YDA_HOME/.ssh
+chmod og-rwx $YDA_HOME/.ssh
+cat >> $YDA_HOME/.ssh/authorized_keys << EOF
+<<Insert_Public_key_Here>>
+EOF
+chmod og-rw $YDA_HOME/.ssh/authorized_keys
+chown -R yd-agent:yd-agent $YDA_HOME/.ssh
+```
+
+### Task Execution Directory
+
+Ephemeral Task directories are created under `/var/opt/yellowdog/agent/data/workers`. This directory contains one or more numbered subdirectories where each numbered directory corresponds to a Worker on the node.
+
+When a Task is allocated to a node, an ephemeral directory is created under the applicable Worker directory, with a name corresponding the YellowDog ID for the Task. For example, this is an ephemeral directory being used by Worker number `1`:
+
+`/var/opt/yellowdog/agent/data/workers/1/ydid_task_559EBE_74949336-ac2b-4811-a7d5-f3ecd9739908_1_1`
+
+This is the directory into which downloaded objects are placed, and in which output files are created by default. The console output `taskoutput.txt` file will also be created in this directory.
+
+See the [Files Downloaded to a Node](#files-downloaded-to-a-node-for-use-in-task-execution) section above for more details on how files in this directory are handled.
+
+At the conclusion of the Task, after any files requested for upload have been uploaded to the Object Store (see the [Files Uploaded from a Node](#files-uploaded-from-a-node-to-the-object-store-after-task-execution) section for more information), the `ydid_task_559EBE_74949336-ac2b-4811-a7d5-f3ecd9739908_1_1` will be removed.
+
 ## Specifying Work Requirements using CSV Data
 
 CSV data files can be used to drive the generation of lists of Tasks, as follows:
@@ -1477,15 +1574,19 @@ Variable substitution is performed before Jsonnet expansion into JSON, and again
 
 ## Checking Jsonnet Processing
 
-To inspect the basic conversion of Jsonnet into JSON, without any additional processing by the Python Examples scripts, the `yd-jsonnet2json` command can be used. This takes a single command line argument which is the name of the jsonnet file to be processed:
+There are three possibilities for verifying that a Jsonnet specification is doing what is intended:
+
+1. To inspect the basic conversion of Jsonnet into JSON, without any additional processing by the Python Examples scripts, the `yd-jsonnet2json` command can be used. This takes a single command line argument which is the name of the jsonnet file to be processed:
 
 ```shell
 yd-jsonnet2json my_file.jsonnet
 ```
 
-The `jsonnet-dry-run` (`-J`) option of the `yd-submit`, `yd-provision` and `yd-instantiate` commands will generate JSON output representing the Jsonnet to JSON processing only, including applicable variable substitutions, but before full property expansion into the JSON that will be submitted to the Platform.
 
-The `dry-run` (`-D`) option will generate JSON output representing the full processing of the Jsonnet file into what will be submitted to the API. This allows inspection to check that the output matches expectations, prior to submitting to the Platform.
+2. The `jsonnet-dry-run` (`-J`) option of the `yd-submit`, `yd-provision` and `yd-instantiate` commands will generate JSON output representing the Jsonnet to JSON processing only, including applicable variable substitutions, but before full property expansion into the JSON that will be submitted to the Platform.
+
+
+3. The `dry-run` (`-D`) option will generate JSON output representing the full processing of the Jsonnet file into what will be submitted to the API. This allows inspection to check that the output matches expectations, prior to submitting to the Platform.
 
 ## Jsonnet Example
 

yellowdog-python-examples-5.0.2/yd_commands/__init__.py

@@ -0,0 +1 @@
+__version__ = "5.0.2"

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2}/yd_commands/csv_data.py

@@ -350,6 +350,7 @@ def csv_expand_toml_tasks(config_wr: ConfigWorkRequirement, csv_file: str) -> Di
         (config_wr.output_files_required, OUTPUT_FILES_REQUIRED),
         (config_wr.task_data, TASK_DATA),
         (config_wr.task_data_file, TASK_DATA_FILE),
+        (config_wr.task_group_name, TASK_GROUP_NAME),  # Note: oddity
         (config_wr.task_name, TASK_NAME),
         (config_wr.task_type, TASK_TYPE),
         (config_wr.upload_files, UPLOAD_FILES),

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2}/yd_commands/instantiate.py

@@ -114,12 +114,16 @@ def main():
 
             if ARGS_PARSER.report:
                 print_log("Generating provisioning report only")
-                test_result: ComputeRequirementTemplateTestResult = (
-                    CLIENT.compute_client.test_compute_requirement_template(
-                        compute_requirement_template_usage
+                try:
+                    test_result: ComputeRequirementTemplateTestResult = (
+                        CLIENT.compute_client.test_compute_requirement_template(
+                            compute_requirement_template_usage
+                        )
                     )
-                )
-                print_compute_template_test_result(test_result)
+                    print_compute_template_test_result(test_result)
+                except requests.HTTPError as http_error:
+                    if "No sources" in http_error.response.text:
+                        print_log("No Compute Sources match the Template's constraints")
                 return
 
             if not ARGS_PARSER.dry_run:

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2}/yd_commands/submit.py

@@ -209,7 +209,7 @@ def submit_work_requirement(
     # Overwrite the WR name?
     global ID, CONFIG_WR
     ID = format_yd_name(
-        wr_data.get(L_WR_NAME, ID if CONFIG_WR.wr_name is None else CONFIG_WR.wr_name)
+        wr_data.get(NAME, ID if CONFIG_WR.wr_name is None else CONFIG_WR.wr_name)
     )
     # Lazy substitution of the Work Requirement name, now it's defined
     add_substitutions(subs={L_WR_NAME: ID})
@@ -304,6 +304,12 @@ def create_task_group(
     # Name the Task Group
     num_task_groups = len(wr_data[TASK_GROUPS])
     num_tasks = len(task_group_data[TASKS])
+    # The following handles possible CSV substitution at the config.toml level
+    try:
+        if task_group_data.get(NAME, None) is None:
+            task_group_data[NAME] = task_group_data[TASKS][0][TASK_GROUP_NAME]
+    except KeyError:
+        pass
     task_group_name = format_yd_name(
         get_task_group_name(
             task_group_data.get(NAME, CONFIG_WR.task_group_name),
@@ -482,7 +488,7 @@ def add_tasks_to_task_group(
             task = tasks[task_number] if task_count is None else tasks[0]
             task_name = format_yd_name(
                 get_task_name(
-                    task.get(NAME, CONFIG_WR.task_name),
+                    task.get(NAME, task.get(TASK_NAME, CONFIG_WR.task_name)),
                     task_number,
                     num_tasks,
                     tg_number,

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2}/yd_commands/submit_utils.py

@@ -181,4 +181,7 @@ def format_yd_name(yd_name: str) -> str:
     # Make obvious substitutions
     yd_name = yd_name.replace("/", "-").replace(" ", "_").lower()
     # Enforce acceptable regex and name length
-    return re.sub("[^a-z0-9_-]", "", yd_name)[:60]
+    yd_name = re.sub("[^a-z0-9_-]", "", yd_name)
+    if not yd_name[0].isalpha():
+        yd_name = f"z_{yd_name}"
+    return yd_name[:60]

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2}/yd_commands/variables.py

@@ -41,6 +41,9 @@ if "submit" in sys.argv[0]:
 NUMBER_SUB = "num:"
 BOOL_SUB = "bool:"
 
+# Nested variables depth supported in TOML files
+NESTED_DEPTH = 3
+
 # Add user-defined variable substitutions
 # Can supersede the existing substitutions above
 ENV_VAR_PREFIX = "YD_VAR_"
@@ -251,7 +254,9 @@ def load_toml_file_with_variable_substitutions(filename: str, prefix: str = "")
     except KeyError:
         pass
 
-    process_variable_substitutions(config, prefix=prefix)
+    for _ in range(NESTED_DEPTH):
+        process_variable_substitutions(config, prefix=prefix)
+
     return config
 
 

{yellowdog-python-examples-5.0.0 → yellowdog-python-examples-5.0.2/yellowdog_python_examples.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: yellowdog-python-examples
-Version: 5.0.0
+Version: 5.0.2
 Summary: Example Python commands using the YellowDog Python SDK
 Home-page: https://github.com/yellowdog/python-examples
 Author: YellowDog Limited

yellowdog-python-examples-5.0.0/yd_commands/__init__.py

@@ -1 +0,0 @@
-__version__ = "5.0.0"