yellowdog-python-examples 7.8.2__tar.gz → 7.8.3__tar.gz

{yellowdog-python-examples-7.8.2/yellowdog_python_examples.egg-info → yellowdog-python-examples-7.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: yellowdog-python-examples
-Version: 7.8.2
+Version: 7.8.3
 Summary: Example Python commands using the YellowDog Python SDK
 Home-page: https://github.com/yellowdog/python-examples
 Author: YellowDog Limited
@@ -19,7 +19,7 @@ Requires-Dist: yellowdog-sdk>=8.0.0
 Requires-Dist: toml
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: PyPAC>=0.16.4
-Requires-Dist: rich>=13.5.2
+Requires-Dist: rich>=13.7.1
 Requires-Dist: requests
 Requires-Dist: boto3
 Requires-Dist: google-cloud-compute

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/requirements.txt

@@ -2,7 +2,7 @@ yellowdog-sdk >= 8.0.0
 toml
 tabulate >= 0.9.0
 PyPAC >= 0.16.4
-rich >= 13.5.2
+rich >= 13.7.1
 requests
 boto3
 google-cloud-compute

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/tests/test_demos.py

@@ -8,6 +8,7 @@ from cli_test_helpers import shell
 
 DEMO_DIR = "../python-examples-demos"
 CMD_SEQ = "yd-provision && yd-submit -f && yd-terminate -y && yd-delete -y"
+NEXTFLOW = "/Users/pwt/nextflow/nextflow"
 
 
 @pytest.mark.demos
@@ -51,3 +52,24 @@ class TestDemos:
     def test_cmd_exe(self):
         result = shell(f"cd {DEMO_DIR}/cmd.exe && {CMD_SEQ}")
         assert result.exit_code == 0
+
+    def test_nextflow_image_montage(self):
+        result = shell(
+            f"cd {DEMO_DIR}/nextflow/image-montage && {NEXTFLOW} main.nf "
+            "&& cd .. && ./cleanup.sh"
+        )
+        assert result.exit_code == 0
+
+    def test_nextflow_salmon_rna(self):
+        result = shell(
+            f"cd {DEMO_DIR}/nextflow/salmon-rna && {NEXTFLOW} main.nf "
+            "&& cd .. && ./cleanup.sh"
+        )
+        assert result.exit_code == 0
+
+    def test_cmd_modelled_on_premise(self):
+        result = shell(
+            f"cd {DEMO_DIR}/modelled-on-premise && yd-instantiate "
+            "&& sleep 120 && yd-terminate -y"
+        )
+        assert result.exit_code == 0

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/tests/test_entrypoints.py

@@ -42,3 +42,9 @@ def test_entrypoints():
     assert result.exit_code == 0
     result = shell("yd-cloudwizard --help")
     assert result.exit_code == 0
+    result = shell("yd-start --help")
+    assert result.exit_code == 0
+    result = shell("yd-hold --help")
+    assert result.exit_code == 0
+    result = shell("yd-boost --help")
+    assert result.exit_code == 0

yellowdog-python-examples-7.8.3/yd_commands/__init__.py

@@ -0,0 +1 @@
+__version__ = "7.8.3"

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/args.py

@@ -131,8 +131,6 @@ class CLIParser:
                 "remove",
                 "cloudwizard",
                 "follow",
-                "hold",
-                "start",
             ]
         ):
             parser.add_argument(
@@ -338,6 +336,8 @@ class CLIParser:
                 "resize",
                 "cloudwizard",
                 "boost",
+                "hold",
+                "start",
             ]
         ):
             parser.add_argument(
@@ -345,7 +345,7 @@ class CLIParser:
                 "-y",
                 action="store_true",
                 required=False,
-                help="perform destructive actions without requiring user confirmation",
+                help="perform modifying actions without requiring user confirmation",
             )
 
         if any(module in sys.argv[0] for module in ["delete", "download"]):
@@ -639,18 +639,19 @@ class CLIParser:
             )
 
         if "boost" in sys.argv[0]:
-            parser.add_argument(
-                "allowance",
-                metavar="<allowance-ID>",
-                type=str,
-                help=("the YellowDog ID of the allowance to boost"),
-            )
             parser.add_argument(
                 "boost_hours",
                 metavar="<boost hours>",
                 type=int,
                 help="the number of hours to boost the allowance by",
             )
+            parser.add_argument(
+                "allowances",
+                metavar="<allowance-ID> [<allowance-ID>]",
+                nargs="+",
+                type=str,
+                help="the YellowDog ID(s) of the allowance(s) to boost",
+            )
 
         if "shutdown" in sys.argv[0]:
             parser.add_argument(
@@ -1401,8 +1402,8 @@ class CLIParser:
 
     @property
     @allow_missing_attribute
-    def allowance(self) -> str:
-        return self.args.allowance
+    def allowance_list(self) -> List[str]:
+        return self.args.allowances
 
     @property
     @allow_missing_attribute

yellowdog-python-examples-7.8.3/yd_commands/boost.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+"""
+A script to boost allowances.
+"""
+
+from yd_commands.interactive import confirmed
+from yd_commands.printing import print_error, print_log
+from yd_commands.wrapper import ARGS_PARSER, CLIENT, main_wrapper
+
+
+@main_wrapper
+def main():
+
+    count = 0
+    for allowance in ARGS_PARSER.allowance_list:
+        if not confirmed(
+            f"Boost Allowance {allowance} by {ARGS_PARSER.boost_hours} hours?"
+        ):
+            continue
+        try:
+            CLIENT.allowances_client.boost_allowance_by_id(
+                allowance, ARGS_PARSER.boost_hours
+            )
+            print_log(
+                f"Boosted Allowance {allowance} by {ARGS_PARSER.boost_hours} hours"
+            )
+            count += 1
+        except Exception as e:
+            print_error(f"Unable to boost Allowance {allowance}: {e}")
+
+    if count > 1:
+        print_log(f"Boosted {count} allowances by {ARGS_PARSER.boost_hours} hours")
+
+
+# Standalone entry point
+if __name__ == "__main__":
+    main()

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/cloudwizard_aws.py

@@ -79,42 +79,44 @@ AWS_YD_IMAGE_REGIONS = [
 
 YELLOWDOG_POLICY = {
     "Version": "2012-10-17",
-    "Statement": [{
-        "Sid": "VisualEditor0",
-        "Effect": "Allow",
-        "Action": [
-            "EC2:CancelSpotInstanceRequests",
-            "EC2:CreateFleet",
-            "EC2:CreateLaunchTemplate",
-            "EC2:CreatePlacementGroup",
-            "EC2:CreateTags",
-            "EC2:DeleteFleets",
-            "EC2:DeleteLaunchTemplate",
-            "EC2:DeletePlacementGroup",
-            "EC2:DescribeFleets",
-            "EC2:DescribeInstanceTypes",
-            "EC2:DescribeInstances",
-            "EC2:DescribeLaunchTemplates",
-            "EC2:DescribePlacementGroups",
-            "EC2:DescribeSpotInstanceRequests",
-            "EC2:ModifyFleet",
-            "EC2:RebootInstances",
-            "EC2:RequestSpotInstances",
-            "EC2:RunInstances",
-            "EC2:StartInstances",
-            "EC2:StopInstances",
-            "EC2:TerminateInstances",
-            "S3:AbortMultipartUpload",
-            "S3:CreateBucket",
-            "S3:DeleteBucket",
-            "S3:DeleteObject",
-            "S3:GetObject",
-            "S3:ListBucketMultipartUploads",
-            "S3:ListMultipartUploadParts",
-            "S3:PutObject",
-        ],
-        "Resource": "*",
-    }],
+    "Statement": [
+        {
+            "Sid": "VisualEditor0",
+            "Effect": "Allow",
+            "Action": [
+                "EC2:CancelSpotInstanceRequests",
+                "EC2:CreateFleet",
+                "EC2:CreateLaunchTemplate",
+                "EC2:CreatePlacementGroup",
+                "EC2:CreateTags",
+                "EC2:DeleteFleets",
+                "EC2:DeleteLaunchTemplate",
+                "EC2:DeletePlacementGroup",
+                "EC2:DescribeFleets",
+                "EC2:DescribeInstanceTypes",
+                "EC2:DescribeInstances",
+                "EC2:DescribeLaunchTemplates",
+                "EC2:DescribePlacementGroups",
+                "EC2:DescribeSpotInstanceRequests",
+                "EC2:ModifyFleet",
+                "EC2:RebootInstances",
+                "EC2:RequestSpotInstances",
+                "EC2:RunInstances",
+                "EC2:StartInstances",
+                "EC2:StopInstances",
+                "EC2:TerminateInstances",
+                "S3:AbortMultipartUpload",
+                "S3:CreateBucket",
+                "S3:DeleteBucket",
+                "S3:DeleteObject",
+                "S3:GetObject",
+                "S3:ListBucketMultipartUploads",
+                "S3:ListMultipartUploadParts",
+                "S3:PutObject",
+            ],
+            "Resource": "*",
+        }
+    ],
 }
 
 
@@ -182,12 +184,14 @@ class AWSConfig(CommonCloudConfig):
         A list of regions can be supplied as an argument.
         The 'operation' argument must be 'add-ssh' or 'remove-ssh'.
         """
-        ssh_ipv4_ingress_rule = [{
-            "IpProtocol": "tcp",
-            "FromPort": 22,
-            "ToPort": 22,
-            "IpRanges": [{"CidrIp": f"0.0.0.0/0"}],
-        }]
+        ssh_ipv4_ingress_rule = [
+            {
+                "IpProtocol": "tcp",
+                "FromPort": 22,
+                "ToPort": 22,
+                "IpRanges": [{"CidrIp": f"0.0.0.0/0"}],
+            }
+        ]
         for region in (
             AWS_YD_IMAGE_REGIONS if selected_region is None else [selected_region]
         ):
@@ -359,11 +363,13 @@ class AWSConfig(CommonCloudConfig):
             "Creating YellowDog Namespace Configuration"
             f" 'S3:{self._get_s3_bucket_name()}' -> '{YD_NAMESPACE}'"
         )
-        create_resources([
-            self._generate_yd_namespace_configuration(
-                namespace=YD_NAMESPACE, s3_bucket_name=self._get_s3_bucket_name()
-            )
-        ])
+        create_resources(
+            [
+                self._generate_yd_namespace_configuration(
+                    namespace=YD_NAMESPACE, s3_bucket_name=self._get_s3_bucket_name()
+                )
+            ]
+        )
 
         # Sequence the Compute Requirement Templates before the Compute Source
         # Templates for subsequent removals.
@@ -390,11 +396,13 @@ class AWSConfig(CommonCloudConfig):
         self._remove_keyring(keyring_name=YD_KEYRING_NAME)
 
         # Remove the Namespace Configuration
-        remove_resources([
-            self._generate_yd_namespace_configuration(
-                YD_NAMESPACE, self._get_s3_bucket_name()
-            )
-        ])
+        remove_resources(
+            [
+                self._generate_yd_namespace_configuration(
+                    YD_NAMESPACE, self._get_s3_bucket_name()
+                )
+            ]
+        )
 
     def _gather_aws_network_information(self):
         """
@@ -961,23 +969,25 @@ class AWSConfig(CommonCloudConfig):
         """
         assert self._aws_user is not None
         s3_bucket_name = self._get_s3_bucket_name()
-        return json.dumps({
-            "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Effect": "Allow",
-                    "Principal": {"AWS": self._aws_user.arn},
-                    "Action": "s3:*",
-                    "Resource": f"arn:aws:s3:::{s3_bucket_name}/*",
-                },
-                {
-                    "Effect": "Allow",
-                    "Principal": {"AWS": self._aws_user.arn},
-                    "Action": "s3:ListBucket",
-                    "Resource": f"arn:aws:s3:::{s3_bucket_name}",
-                },
-            ],
-        })
+        return json.dumps(
+            {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Principal": {"AWS": self._aws_user.arn},
+                        "Action": "s3:*",
+                        "Resource": f"arn:aws:s3:::{s3_bucket_name}/*",
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Principal": {"AWS": self._aws_user.arn},
+                        "Action": "s3:ListBucket",
+                        "Resource": f"arn:aws:s3:::{s3_bucket_name}",
+                    },
+                ],
+            }
+        )
 
     def _get_s3_bucket_name(self) -> str:
         """

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/cloudwizard_azure.py

@@ -491,11 +491,13 @@ class AzureConfig(CommonCloudConfig):
 
         # Create namespace configuration (Keyring/Credential creation must come first)
         print_log(f"Creating YellowDog Namespace Configuration '{YD_NAMESPACE}'")
-        create_resources([
-            self._generate_yd_namespace_configuration(
-                namespace=YD_NAMESPACE, storage_blob_name=STORAGE_BLOB_NAME
-            )
-        ])
+        create_resources(
+            [
+                self._generate_yd_namespace_configuration(
+                    namespace=YD_NAMESPACE, storage_blob_name=STORAGE_BLOB_NAME
+                )
+            ]
+        )
 
         # Save the list of resources
         # Sequence the Compute Requirement Templates before the Compute Source

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/cloudwizard_common.py

@@ -186,12 +186,14 @@ class CommonCloudConfig(ABC):
                     "type": "co.yellowdog.platform.model.StringAttributeConstraint",
                 },
             ],
-            "preferences": [{
-                "attribute": "yd.cost",
-                "rankOrder": "PREFER_LOWER",
-                "type": "co.yellowdog.platform.model.NumericAttributePreference",
-                "weight": 1,
-            }],
+            "preferences": [
+                {
+                    "attribute": "yd.cost",
+                    "rankOrder": "PREFER_LOWER",
+                    "type": "co.yellowdog.platform.model.NumericAttributePreference",
+                    "weight": 1,
+                }
+            ],
             "maximumSourceCount": 5,
             "minimumSourceCount": 1,
             "strategyType": f"co.yellowdog.platform.model.{strategy}ProvisionStrategy",

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/cloudwizard_gcp.py

@@ -158,13 +158,15 @@ class GCPConfig(CommonCloudConfig):
             client=self._client, name_prefix=YD_RESOURCE_PREFIX
         )
         self._remove_keyring(YD_KEYRING_NAME)
-        remove_resources([
-            self._generate_namespace_configuration(
-                YD_NAMESPACE,
-                self._generate_bucket_name(),
-                credential_name=f"{YD_KEYRING_NAME}/{YD_CREDENTIAL_NAME}",
-            )
-        ])
+        remove_resources(
+            [
+                self._generate_namespace_configuration(
+                    YD_NAMESPACE,
+                    self._generate_bucket_name(),
+                    credential_name=f"{YD_KEYRING_NAME}/{YD_CREDENTIAL_NAME}",
+                )
+            ]
+        )
 
     def _gather_regions(self):
         """

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/interactive.py

@@ -1,5 +1,5 @@
 """
-User interaction processing
+User interaction processing utilities.
 """
 
 from os import getenv
@@ -76,8 +76,6 @@ def select(
             else (f"Please select items (e.g.: 1,2,4-7 / *){cancel_string}:")
         )
         selector_string = CONSOLE.input(print_string(input_string) + " ")
-        if selector_string == "":  # A quirk of Rich?
-            print()
         if selector_string.strip() == "*":
             selector_string = f"1-{len(objects)}"
         selector_list = selector_string.split(",")
@@ -156,8 +154,6 @@ def confirmed(msg: str) -> bool:
     # Seek user confirmation
     while True:
         response = CONSOLE.input(print_string(f"{msg} (y/N):") + " ")
-        if response == "":  # Seems to be a quirk of Rich
-            print()
         if response.lower() in ["y", "yes"]:
             print_log("Action confirmed by user")
             return True

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/list.py

@@ -519,10 +519,12 @@ def list_namespaces():
     # Assemble and print the table
     headings = [field.capitalize() for field in all_fields]
     headings.insert(0, "#")
-    rows = sorted([
-        [index + 1] + [namespace.get(field, "") for field in all_fields]
-        for index, namespace in enumerate(namespace_list)
-    ])
+    rows = sorted(
+        [
+            [index + 1] + [namespace.get(field, "") for field in all_fields]
+            for index, namespace in enumerate(namespace_list)
+        ]
+    )
     print()
     CONSOLE_TABLE.print(
         indent(tabulate(rows, headings, tablefmt="simple_outline")),
@@ -544,7 +546,7 @@ def list_allowances():
     )
     allowances: List[Allowance] = search_client.list_all()
     if len(allowances) == 0:
-        print_log("No allowances to display")
+        print_log("No Allowances to display")
         return
 
     if not ARGS_PARSER.details:

{yellowdog-python-examples-7.8.2 → yellowdog-python-examples-7.8.3}/yd_commands/load_config.py

@@ -4,7 +4,7 @@ Common utility functions, mostly related to loading configuration data.
 
 import os
 from os import getenv
-from os.path import dirname, relpath
+from os.path import dirname, join, relpath
 from sys import exit
 from typing import Dict, Optional
 
@@ -41,18 +41,18 @@ from yd_commands.variables import (
 )
 
 # CLI > YD_CONF > 'config.toml'
-config_file = relpath(
+CONFIG_FILE = relpath(
     getenv("YD_CONF", "config.toml")
     if ARGS_PARSER.config_file is None
     else ARGS_PARSER.config_file
 )
 
 try:
-    CONFIG_FILE_DIR = dirname(config_file)
-    print_log(f"Loading configuration data from: '{config_file}'")
-    CONFIG_TOML: Dict = load_toml_file_with_variable_substitutions(config_file)
+    CONFIG_FILE_DIR = dirname(CONFIG_FILE)
+    print_log(f"Loading configuration data from: '{CONFIG_FILE}'")
+    CONFIG_TOML: Dict = load_toml_file_with_variable_substitutions(CONFIG_FILE)
     try:
-        validate_properties(CONFIG_TOML, f"'{config_file}'")
+        validate_properties(CONFIG_TOML, f"'{CONFIG_FILE}'")
     except Exception as e:
         print_error(e)
         exit(1)
@@ -71,7 +71,7 @@ except FileNotFoundError as e:
 
 except (PermissionError, TomlDecodeError) as e:
     print_error(
-        f"Unable to load configuration data from '{config_file}': {e}",
+        f"Unable to load configuration data from '{CONFIG_FILE}': {e}",
     )
     exit(1)
 
@@ -87,7 +87,7 @@ def load_config_common() -> ConfigCommon:
     try:
         common_section = CONFIG_TOML.get(COMMON_SECTION, {})
 
-        # Check for IMPORT directive (common section in a separate file)
+        # Check for IMPORT directive ('common' section in a separate file)
         common_section_import_file = common_section.get(IMPORT_COMMON, None)
         if common_section_import_file is not None:
             common_section = import_toml(common_section_import_file)
@@ -160,6 +160,7 @@ def load_config_common() -> ConfigCommon:
 
 
 def import_toml(filename: str) -> Dict:
+    filename = relpath(join(CONFIG_FILE_DIR, process_variable_substitutions(filename)))
     print_log(f"Loading imported common configuration data from: '{filename}'")
     try:
         common_config: Dict = load_toml_file_with_variable_substitutions(filename)