aws-parallelcluster-node 3.16.1__tar.gz → 3.16.2__tar.gz

{aws_parallelcluster_node-3.16.1/src/aws_parallelcluster_node.egg-info → aws_parallelcluster_node-3.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-parallelcluster-node
-Version: 3.16.1
+Version: 3.16.2
 Summary: aws-parallelcluster-node provides the scripts for an AWS ParallelCluster node.
 Home-page: https://github.com/aws/aws-parallelcluster-node
 Author: Amazon Web Services
@@ -10,7 +10,7 @@ Classifier: Environment :: Console
 Classifier: Programming Language :: Python
 Classifier: Topic :: Scientific/Engineering
 Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.9
+Requires-Python: >=3.14
 License-File: LICENSE.txt
 License-File: NOTICE.txt
 Requires-Dist: boto3>=1.7.55

{aws_parallelcluster_node-3.16.1 → aws_parallelcluster_node-3.16.2}/setup.py

@@ -32,7 +32,7 @@ console_scripts = [
     "clustermgtd = slurm_plugin.clustermgtd:main",
     "computemgtd = slurm_plugin.computemgtd:main",
 ]
-version = "3.16.1"
+version = "3.16.2"
 requires = ["boto3>=1.7.55", "retrying>=1.3.3", "setuptools"]
 
 setup(
@@ -44,7 +44,7 @@ setup(
     license="Apache License 2.0",
     packages=find_packages("src", exclude=["tests"]),
     package_dir={"": "src"},
-    python_requires=">=3.9",
+    python_requires=">=3.14",
     install_requires=requires,
     entry_points=dict(console_scripts=console_scripts),
     zip_safe=False,

{aws_parallelcluster_node-3.16.1 → aws_parallelcluster_node-3.16.2/src/aws_parallelcluster_node.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-parallelcluster-node
-Version: 3.16.1
+Version: 3.16.2
 Summary: aws-parallelcluster-node provides the scripts for an AWS ParallelCluster node.
 Home-page: https://github.com/aws/aws-parallelcluster-node
 Author: Amazon Web Services
@@ -10,7 +10,7 @@ Classifier: Environment :: Console
 Classifier: Programming Language :: Python
 Classifier: Topic :: Scientific/Engineering
 Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.9
+Requires-Python: >=3.14
 License-File: LICENSE.txt
 License-File: NOTICE.txt
 Requires-Dist: boto3>=1.7.55

{aws_parallelcluster_node-3.16.1 → aws_parallelcluster_node-3.16.2}/src/common/schedulers/slurm_commands.py

@@ -75,10 +75,15 @@ SCONTROL_NODE_INFO_FIELD_REGEX = re.compile(
     r"|(LastBusyTime=\S+)"
     r"|(ReservationName=\S+)"
     r"|(InstanceId=\S+)"
-    r"|(Reason=.*)",
+    r"|(Reason=.*)"
+    r"|(Comment=.*)"
+    r"|(Extra=.*)",
     re.MULTILINE,
 )
 
+# Fields matched up to the end of the line and discarded, to prevent their free-text content from being parsed as fields
+SCONTROL_DISCARDED_FIELDS = ("Comment", "Extra")
+
 # Fields extracted from `scontrol show partitions` output. `(?<!Next)` ensures `State` is matched but
 # `NextState` is not.
 SCONTROL_PARTITION_INFO_FIELD_REGEX = re.compile(
@@ -426,6 +431,8 @@ def _extract_scontrol_records(raw_output: str, field_regex) -> List[Dict[str, st
         fields = {}
         for match in field_regex.finditer(record):
             key, _, value = match.group(0).partition("=")
+            if key in SCONTROL_DISCARDED_FIELDS:
+                continue
             fields[key] = value
         if fields:
             records.append(fields)

{aws_parallelcluster_node-3.16.1 → aws_parallelcluster_node-3.16.2}/src/common/schedulers/slurm_reservation_commands.py

@@ -39,7 +39,7 @@ logger = logging.getLogger(__name__)
 # Fields extracted from raw `scontrol show reservations` output. Only `ReservationName` is anchored at the
 # start of a line; `(?<!Next)` ensures `State` is matched but `NextState` is not.
 SCONTROL_RESERVATION_INFO_FIELD_REGEX = re.compile(
-    r"^(ReservationName=\S+)" r"|(?<!Next)(State=\S+)" r"|(Users=\S+)" r"|(Nodes=\S+)",
+    r"^(ReservationName=\S+)" r"|(?<!Next)(State=\S+)" r"|(Users=\S+)" r"|(Nodes=\S+)" r"|(Comment=.*)",
     re.MULTILINE,
 )
 

{aws_parallelcluster_node-3.16.1 → aws_parallelcluster_node-3.16.2}/src/slurm_plugin/fleet_manager.py

@@ -21,6 +21,7 @@ from common.ec2_utils import get_private_ip_address_and_dns_name
 from common.utils import setup_logging_filter
 from retrying import retry
 from slurm_plugin.common import print_with_count
+from slurm_plugin.slurm_resources import SlurmNode
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +31,32 @@ logger = logging.getLogger(__name__)
 INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT = 90
 INSTANCE_INFO_RETRIEVAL_MAX_BACKOFF = 30
 
+# The only error code EC2 returns when a launch request is throttled, see
+# https://docs.aws.amazon.com/ec2/latest/devguide/ec2-api-throttling.html
+LAUNCH_THROTTLING_ERROR_CODE = "RequestLimitExceeded"
+
+# An override that CreateFleet did not fulfill because another override failed is reported with this exact
+# code and message pair, which points at the real error rather than being one.
+UNFULFILLED_OVERRIDE_ERROR = (
+    "UnfulfillableCapacity",
+    "Failed to fulfill capacity. Please review errors in the response.",
+)
+
+# Errors confined to one instance type and subnet pool. Reported alongside throttling, they leave the throttled
+# pools able to serve the batch once the rate limit refills, so the throttling is still retried. Any other error
+# alongside throttling would fail the retry on every pool alike and is reported instead.
+POOL_LEVEL_ERROR_CODES = frozenset(
+    SlurmNode.EC2_ICE_ERROR_CODES
+    | {
+        "InsufficientFreeAddressesInSubnet",
+        "InvalidSubnetID.NotFound",
+        "InvalidSubnet",
+        "InsufficientVolumeCapacity",
+        "VolumeTypeNotAvailableInZone",
+        "ServiceUnavailable",
+    }
+)
+
 
 class EC2Instance:
     def __init__(self, id, private_ip, hostname, all_private_ips, launch_time):
@@ -430,7 +457,40 @@ class Ec2CreateFleetManager(FleetManager):
             if partial_instance_ids:
                 logger.error("Unable to retrieve instance info for instances: %s", partial_instance_ids)
 
-            if not instances and len(err_list) == 1:
+            if not instances:
+                # Drop the entries that only point at the real error, unless the response carries nothing else.
+                real_errors = [
+                    err
+                    for err in err_list
+                    if (err.get("ErrorCode"), err.get("ErrorMessage")) != UNFULFILLED_OVERRIDE_ERROR
+                ]
+                if real_errors:
+                    err_list = real_errors
+                # A single cause is normally left. Should there be several, throttling is retried only when every
+                # other cause is confined to a pool, since the throttled pools can then still serve the batch once the
+                # rate limit refills. Any other cause would fail the retry on every pool alike and is reported instead.
+                # For example:
+                # - [RequestLimitExceeded, InsufficientInstanceCapacity] -> retry, other pools can still serve the batch
+                # - [RequestLimitExceeded, VcpuLimitExceeded] -> report VcpuLimitExceeded, a retry hits the same limit
+                throttling = next(
+                    (err for err in err_list if err.get("ErrorCode") == LAUNCH_THROTTLING_ERROR_CODE), None
+                )
+                if throttling:
+                    blocking = next(
+                        (
+                            err
+                            for err in err_list
+                            if err.get("ErrorCode") != LAUNCH_THROTTLING_ERROR_CODE
+                            and err.get("ErrorCode") not in POOL_LEVEL_ERROR_CODES
+                        ),
+                        None,
+                    )
+                    chosen = blocking or throttling
+                    raise LaunchInstancesError(chosen.get("ErrorCode"), chosen.get("ErrorMessage"))
+            # Normally a single cause is left. Reporting the first one of several is a second safety net: the
+            # caller otherwise records a hardcoded InsufficientInstanceCapacity, and any code EC2 actually
+            # returned is more useful than an invented one, whichever of them the response happens to list first.
+            if not instances and err_list:
                 raise LaunchInstancesError(err_list[0].get("ErrorCode"), err_list[0].get("ErrorMessage"))
             return {"Instances": instances}
         except ClientError as e:

{aws_parallelcluster_node-3.16.1 → aws_parallelcluster_node-3.16.2}/src/slurm_plugin/slurm_resources.py

@@ -126,8 +126,16 @@ class SlurmResumeJob(SlurmJob):
         reservation="",
         features="",
         extra="",
+        exclusive=None,
+        priority=None,
+        **unknown_fields,
     ):
-        """Initialize Slurm job with attributes."""
+        """
+        Initialize Slurm job with attributes.
+
+        Slurm adds fields to the jobs of the Resume File over time (exclusive and priority in 26.05), so fields this
+        class does not know are ignored rather than rejected: rejecting them fails the resume of every node.
+        """
         super().__init__(
             job_id,
         )
@@ -139,9 +147,10 @@ class SlurmResumeJob(SlurmJob):
         self.reservation = reservation
         self.features = features
         self.extra = extra
-
-    def is_exclusive(self):
-        return self.oversubscribe == JobOversubscribe.NO
+        self.exclusive = exclusive
+        self.priority = priority
+        if unknown_fields:
+            logger.debug("Ignoring unknown fields of job %s in Slurm Resume File: %s", job_id, unknown_fields)
 
     def __eq__(self, other):
         """Compare 2 SlurmJob objects."""