aws-parallelcluster-node 3.15.1__tar.gz → 3.16.1__tar.gz

{aws_parallelcluster_node-3.15.1/src/aws_parallelcluster_node.egg-info → aws_parallelcluster_node-3.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-parallelcluster-node
-Version: 3.15.1
+Version: 3.16.1
 Summary: aws-parallelcluster-node provides the scripts for an AWS ParallelCluster node.
 Home-page: https://github.com/aws/aws-parallelcluster-node
 Author: Amazon Web Services

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/setup.py

@@ -32,7 +32,7 @@ console_scripts = [
     "clustermgtd = slurm_plugin.clustermgtd:main",
     "computemgtd = slurm_plugin.computemgtd:main",
 ]
-version = "3.15.1"
+version = "3.16.1"
 requires = ["boto3>=1.7.55", "retrying>=1.3.3", "setuptools"]
 
 setup(

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1/src/aws_parallelcluster_node.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-parallelcluster-node
-Version: 3.15.1
+Version: 3.16.1
 Summary: aws-parallelcluster-node provides the scripts for an AWS ParallelCluster node.
 Home-page: https://github.com/aws/aws-parallelcluster-node
 Author: Amazon Web Services

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/common/schedulers/slurm_commands.py

@@ -13,6 +13,11 @@ import json
 import logging
 import os
 import re
+import shlex
+
+# A nosec comment is appended to the following line in order to disable the B404 check.
+# In this file the input of the module subprocess is trusted.
+import subprocess  # nosec B404
 from datetime import datetime, timezone
 from typing import Dict, List
 
@@ -60,10 +65,25 @@ SLURM_CONF_DIR = os.path.join(os.path.split(SLURM_BINARIES_DIR)[0], "etc")
 SCONTROL = f"sudo {SLURM_BINARIES_DIR}/scontrol"
 SINFO = f"{SLURM_BINARIES_DIR}/sinfo"
 
-SCONTROL_OUTPUT_AWK_PARSER = (
-    'awk \'BEGIN{{RS="\\n\\n" ; ORS="######\\n";}} {{print}}\' | '
-    + "grep -oP '^(NodeName=\\S+)|(NodeAddr=\\S+)|(NodeHostName=\\S+)|(?<!Next)(State=\\S+)|"
-    + "(Partitions=\\S+)|(SlurmdStartTime=\\S+)|(LastBusyTime=\\S+)|(ReservationName=\\S+)|(Reason=.*)|(######)'"
+SCONTROL_NODE_INFO_FIELD_REGEX = re.compile(
+    r"^(NodeName=\S+)"
+    r"|(NodeAddr=\S+)"
+    r"|(NodeHostName=\S+)"
+    r"|(?<!Next)(State=\S+)"
+    r"|(Partitions=\S+)"
+    r"|(SlurmdStartTime=\S+)"
+    r"|(LastBusyTime=\S+)"
+    r"|(ReservationName=\S+)"
+    r"|(InstanceId=\S+)"
+    r"|(Reason=.*)",
+    re.MULTILINE,
+)
+
+# Fields extracted from `scontrol show partitions` output. `(?<!Next)` ensures `State` is matched but
+# `NextState` is not.
+SCONTROL_PARTITION_INFO_FIELD_REGEX = re.compile(
+    r"^(PartitionName=\S+)" r"|(?<!Next)(State=\S+)",
+    re.MULTILINE,
 )
 
 # Set default timeouts for running different slurm commands.
@@ -129,6 +149,7 @@ def update_nodes(
     nodes,
     nodeaddrs=None,
     nodehostnames=None,
+    instance_ids=None,
     state=None,
     reason=None,
     raise_on_error=True,
@@ -150,8 +171,11 @@ def update_nodes(
     For example, if updating a state cause failure, but updating nodeaddr cause no failure.
     if we run scontrol update state=fail_state nodeaddr=good_addr nodename=name,
     the scontrol command will fail but nodeaddr will be updated to good_addr.
+
+    InstanceId is set in the same batched command as NodeAddr so that the node and its backing
+    instance are associated atomically.
     """
-    batched_node_info = _batch_node_info(nodes, nodeaddrs, nodehostnames, batch_size=100)
+    batched_node_info = _batch_node_info(nodes, nodeaddrs, nodehostnames, instance_ids, batch_size=100)
 
     update_cmd = f"{SCONTROL} update"
     if state:
@@ -160,7 +184,7 @@ def update_nodes(
     if reason:
         validate_subprocess_argument(reason)
         update_cmd += f' reason="{reason}"'
-    for nodenames, addrs, hostnames in batched_node_info:
+    for nodenames, addrs, hostnames, ids in batched_node_info:
         validate_subprocess_argument(nodenames)
         node_info = f"nodename={nodenames}"
         if addrs:
@@ -169,6 +193,9 @@ def update_nodes(
         if hostnames:
             validate_subprocess_argument(hostnames)
             node_info += f" nodehostname={hostnames}"
+        if ids:
+            validate_subprocess_argument(ids)
+            node_info += f" instanceid={ids}"
         # It's safe to use the function affected by B604 since the command is fully built in this code
         run_command(  # nosec B604
             f"{update_cmd} {node_info}", raise_on_error=raise_on_error, timeout=command_timeout, shell=True
@@ -213,6 +240,44 @@ def update_all_partitions(state, reset_node_addrs_hostname):
         return False
 
 
+def reset_nodes_in_inactive_partitions():
+    """
+    Reset nodeaddr/nodehostname and set to down the nodes belonging to INACTIVE partitions.
+
+    This is meant to be called when starting the compute fleet, before bringing partitions back UP, to clean up nodes
+    that an INACTIVE partition may have left behind (e.g. dynamic nodes that were powering up without a backing
+    instance when protected mode disabled the partition). If not reset, those nodes would be detected as bootstrap
+    failures right after start and send the cluster back into protected mode. Only INACTIVE partitions are considered,
+    so nodes of active partitions (which may be running jobs) are not affected.
+    """
+    try:
+        # Collect the node names of INACTIVE partitions, skipping partitions with no nodes to avoid building a
+        # malformed (e.g. empty or with consecutive commas) node list.
+        inactive_partition_nodes = [
+            part.nodenames
+            for part in get_partitions_info()
+            if PartitionStatus(part.state) == PartitionStatus.INACTIVE and part.nodenames
+        ]
+        if not inactive_partition_nodes:
+            log.info("No nodes in INACTIVE partitions, nothing to reset.")
+            return
+        # Reset only the nodes that actually need it (e.g. nodeaddr still set), mirroring clustermgtd's inactive
+        # cleanup.
+        nodes_to_reset = {
+            node.name for node in get_nodes_info(",".join(inactive_partition_nodes)) if node.needs_reset_when_inactive()
+        }
+        if nodes_to_reset:
+            log.info(
+                "Resetting nodeaddr/nodehostname and setting to down the following nodes of INACTIVE partitions: %s",
+                sorted(nodes_to_reset),
+            )
+            reset_nodes(nodes_to_reset, state="down", reason="inactive partition", raise_on_error=False)
+        else:
+            log.info("No nodes of INACTIVE partitions need to be reset.")
+    except Exception as e:
+        log.error("Failed when resetting nodes of INACTIVE partitions with error %s", e)
+
+
 def _batch_attribute(attribute, batch_size, expected_length=None):
     """Parse an attribute into batches."""
     if type(attribute) is str:
@@ -223,29 +288,30 @@ def _batch_attribute(attribute, batch_size, expected_length=None):
     return [",".join(batch) for batch in grouper(attribute, batch_size)]
 
 
-def _batch_node_info(nodenames, nodeaddrs, nodehostnames, batch_size):
-    """Group nodename, nodeaddrs, nodehostnames into batches."""
+def _batch_optional_attribute(attribute, attribute_label, nodenames, default_batch, batch_size):
+    """Batch an optional per-node attribute, raising if its entry count does not match the nodes."""
+    if not attribute:
+        return default_batch
+    try:
+        return _batch_attribute(attribute, batch_size, expected_length=len(nodenames))
+    except ValueError:
+        log.error("Nodename %s and %s %s contain different number of entries", nodenames, attribute_label, attribute)
+        raise
+
+
+def _batch_node_info(nodenames, nodeaddrs, nodehostnames, instance_ids, batch_size):
+    """Group nodename, nodeaddrs, nodehostnames, instance_ids into batches."""
     if type(nodenames) is str:
         # Only split on , if there is ] before
         # For ex. "node-[1,3,4-5],node-[20,30]" should split into ["node-[1,3,4-5]","node-[20,30]"]
         nodenames = re.split("(?<=]),", nodenames)
     nodename_batch = _batch_attribute(nodenames, batch_size)
-    nodeaddrs_batch = [None] * len(nodename_batch)
-    nodehostnames_batch = [None] * len(nodename_batch)
-    if nodeaddrs:
-        try:
-            nodeaddrs_batch = _batch_attribute(nodeaddrs, batch_size, expected_length=len(nodenames))
-        except ValueError:
-            log.error("Nodename %s and NodeAddr %s contain different number of entries", nodenames, nodeaddrs)
-            raise
-    if nodehostnames:
-        try:
-            nodehostnames_batch = _batch_attribute(nodehostnames, batch_size, expected_length=len(nodenames))
-        except ValueError:
-            log.error("Nodename %s and NodeHostname %s contain different number of entries", nodenames, nodehostnames)
-            raise
+    default_batch = [None] * len(nodename_batch)
+    nodeaddrs_batch = _batch_optional_attribute(nodeaddrs, "NodeAddr", nodenames, default_batch, batch_size)
+    nodehostnames_batch = _batch_optional_attribute(nodehostnames, "NodeHostName", nodenames, default_batch, batch_size)
+    instance_ids_batch = _batch_optional_attribute(instance_ids, "InstanceId", nodenames, default_batch, batch_size)
 
-    return zip(nodename_batch, nodeaddrs_batch, nodehostnames_batch)
+    return zip(nodename_batch, nodeaddrs_batch, nodehostnames_batch, instance_ids_batch)
 
 
 def set_nodes_down(nodes, reason):
@@ -289,6 +355,83 @@ def set_nodes_idle(nodes, reason=None, reset_node_addrs_hostname=False):
         update_nodes(nodes=nodes, state="resume", reason=reason, raise_on_error=False)
 
 
+def _run_scontrol_command(
+    scontrol_args: str,
+    command_timeout: int = DEFAULT_GET_INFO_COMMAND_TIMEOUT,
+    raise_on_error: bool = True,
+) -> str:
+    """
+    Run a `scontrol <args>` command as a standalone subprocess and return its stdout.
+
+    scontrol's exit code, stderr and raw stdout are logged. When scontrol exits non-zero but still returns
+    output (e.g. one of the requested nodes does not exist: Slurm prints "Node <x> not found" and exits 1
+    while still returning the other nodes), the output is returned and a warning is logged. When it exits
+    non-zero with no output, an error is logged and, when raise_on_error, a CalledProcessError is raised.
+    """
+    command = f"{SCONTROL} {scontrol_args}"
+    log.debug("Executing scontrol command: %s", command)
+    try:
+        # nosec B603: command is built from trusted/validated input and run without a shell.
+        result = subprocess.run(  # nosec B603
+            shlex.split(command),
+            timeout=command_timeout,
+            check=False,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            encoding="utf-8",
+        )
+    except subprocess.TimeoutExpired:
+        log.error("scontrol command timed out after %s seconds: '%s'", command_timeout, command)
+        raise
+    except OSError as e:
+        log.error("Unable to execute scontrol command '%s'. Failed with exception: %s", command, e)
+        raise
+
+    stdout = result.stdout or ""
+    stderr = (result.stderr or "").strip()
+    # Raw scontrol output is logged at DEBUG only, so it is not emitted at the default INFO level.
+    log.debug("scontrol command '%s' exited with code %s. Raw output:\n%s", command, result.returncode, stdout)
+
+    if result.returncode != 0:
+        if stdout.strip():
+            log.warning(
+                "scontrol command '%s' returned non-zero exit code %s but produced output; proceeding with "
+                "the output. stderr: '%s'",
+                command,
+                result.returncode,
+                stderr,
+            )
+        else:
+            log.error(
+                "scontrol command '%s' failed with exit code %s and produced no output. stderr: '%s'",
+                command,
+                result.returncode,
+                stderr,
+            )
+            if raise_on_error:
+                raise subprocess.CalledProcessError(result.returncode, command, output=stdout, stderr=result.stderr)
+
+    return stdout
+
+
+def _extract_scontrol_records(raw_output: str, field_regex) -> List[Dict[str, str]]:
+    """
+    Split raw scontrol output into records and extract the relevant `key=value` fields of each record.
+
+    Records are separated by blank lines. For each record, only the fields matched by field_regex are kept,
+    returned as a dict mapping the Slurm field name to its value. Records with no matching field are skipped.
+    """
+    records = []
+    for record in re.split(r"\n\n+", raw_output.strip()):
+        fields = {}
+        for match in field_regex.finditer(record):
+            key, _, value = match.group(0).partition("=")
+            fields[key] = value
+        if fields:
+            records.append(fields)
+    return records
+
+
 def get_nodes_info(nodes: str = "", command_timeout=DEFAULT_GET_INFO_COMMAND_TIMEOUT) -> List[SlurmNode]:
     """
     Retrieve SlurmNode list from slurm nodelist notation.
@@ -304,15 +447,11 @@ def get_nodes_info(nodes: str = "", command_timeout=DEFAULT_GET_INFO_COMMAND_TIM
     if nodes == "":
         nodes = _get_all_partition_nodes(",".join(PartitionNodelistMapping.instance().get_partitions()))
 
-    # Validation to sanitize the input argument and make it safe to use the function affected by B604
     validate_subprocess_argument(nodes)
 
-    # awk is used to replace the \n\n record separator with '######\n'
     # Note: In case the node does not belong to any partition the Partitions field is missing from Slurm output
-    show_node_info_command = f"{SCONTROL} show nodes {nodes} | {SCONTROL_OUTPUT_AWK_PARSER}"
-    nodeinfo_str = check_command_output(show_node_info_command, timeout=command_timeout, shell=True)  # nosec B604
-
-    return _parse_nodes_info(nodeinfo_str)
+    raw_node_info = _run_scontrol_command(f"show nodes {nodes}", command_timeout=command_timeout)
+    return _parse_nodes_info(_extract_scontrol_records(raw_node_info, SCONTROL_NODE_INFO_FIELD_REGEX))
 
 
 def get_partitions_info(command_timeout=DEFAULT_GET_INFO_COMMAND_TIMEOUT) -> List[SlurmPartition]:
@@ -321,16 +460,10 @@ def get_partitions_info(command_timeout=DEFAULT_GET_INFO_COMMAND_TIMEOUT) -> Lis
 
     This function considers only partitions managed by ParallelCluster.
     """
-    partitions = list(PartitionNodelistMapping.instance().get_partitions())
-    grep_filter = _get_partition_grep_filter(partitions)
-    show_partition_info_command = (
-        f"{SCONTROL} show partitions -o {grep_filter} " + '| grep -oP "^PartitionName=\\K(\\S+)| State=\\K(\\S+)"'
-    )
-    # It's safe to use the function affected by B604 since the command is fully built in this code
-    partition_info_str = check_command_output(
-        show_partition_info_command, timeout=command_timeout, shell=True  # nosec B604
-    )
-    partitions_info = _parse_partition_name_and_state(partition_info_str)
+    partitions = set(PartitionNodelistMapping.instance().get_partitions())
+    raw_partition_info = _run_scontrol_command("show partitions", command_timeout=command_timeout)
+    partition_records = _extract_scontrol_records(raw_partition_info, SCONTROL_PARTITION_INFO_FIELD_REGEX)
+    partitions_info = _parse_partitions_info(partition_records, managed_partitions=partitions or None)
     return [
         SlurmPartition(
             partition_name,
@@ -341,15 +474,6 @@ def get_partitions_info(command_timeout=DEFAULT_GET_INFO_COMMAND_TIMEOUT) -> Lis
     ]
 
 
-def _get_partition_grep_filter(partitions: List[str]) -> str:
-    grep_filter = ""
-    if partitions:
-        grep_filter += " | grep"
-        for partition in partitions:
-            grep_filter += f' -e "PartitionName={partition}"'
-    return grep_filter
-
-
 def resume_powering_down_nodes():
     """Resume nodes that are powering_down so that are set in power state right away."""
     # TODO: This function was added due to Slurm ticket 12915. The bug is not reproducible and the ticket was then
@@ -359,9 +483,23 @@ def resume_powering_down_nodes():
     update_nodes(nodes=powering_down_nodes, state="resume", raise_on_error=False)
 
 
-def _parse_partition_name_and_state(partition_info):
-    """Parse partition name and state from scontrol output."""
-    return grouper(partition_info.splitlines(), 2)
+def _parse_partitions_info(partition_records, managed_partitions=None):
+    """
+    Extract (partition_name, partition_state) tuples from scontrol partition records.
+
+    Only partitions in managed_partitions are returned; if managed_partitions is None, all partitions are
+    returned. Records missing the name or state are skipped.
+    """
+    partitions_info = []
+    for fields in partition_records:
+        partition_name = fields.get("PartitionName")
+        partition_state = fields.get("State")
+        if not partition_name or not partition_state:
+            continue
+        if managed_partitions is not None and partition_name not in managed_partitions:
+            continue
+        partitions_info.append((partition_name, partition_state))
+    return partitions_info
 
 
 def _get_all_partition_nodes(partition_name, command_timeout=DEFAULT_GET_INFO_COMMAND_TIMEOUT):
@@ -386,43 +524,14 @@ def _get_slurm_nodes(states=None, partition_name=None, command_timeout=DEFAULT_G
     return check_command_output(sinfo_command, timeout=command_timeout, shell=True).splitlines()  # nosec B604
 
 
-def _parse_nodes_info(slurm_node_info: str) -> List[SlurmNode]:
-    """Parse slurm node info into SlurmNode objects."""
-    # [ec2-user@ip-10-0-0-58 ~]$ /opt/slurm/bin/scontrol show nodes compute-dy-c5xlarge-[1-3],compute-dy-c5xlarge-50001\
-    # | awk 'BEGIN{{RS="\n\n" ; ORS="######\n";}} {{print}}' | grep -oP "^(NodeName=\S+)|(NodeAddr=\S+)
-    # |(NodeHostName=\S+)|(?<!Next)(State=\S+)|(Partitions=\S+)|(SlurmdStartTime=\S+)|(LastBusyTime=\\S+)
-    # |(ReservationName=\S+)|(Reason=.*)|(######)"
-    # NodeName=compute-dy-c5xlarge-1
-    # NodeAddr=1.2.3.4
-    # NodeHostName=compute-dy-c5xlarge-1
-    # State=IDLE+CLOUD+POWER
-    # Partitions=compute,compute2
-    # SlurmdStartTime=2023-01-26T09:57:15
-    # Reason=some reason
-    # ReservationName=root_1
-    # ######
-    # NodeName=compute-dy-c5xlarge-2
-    # NodeAddr=1.2.3.4
-    # NodeHostName=compute-dy-c5xlarge-2
-    # State=IDLE+CLOUD+POWER
-    # Partitions=compute,compute2
-    # SlurmdStartTime=2023-01-26T09:57:15
-    # Reason=(Code:InsufficientInstanceCapacity)Failure when resuming nodes
-    # ######
-    # NodeName=compute-dy-c5xlarge-3
-    # NodeAddr=1.2.3.4
-    # NodeHostName=compute-dy-c5xlarge-3
-    # State=IDLE+CLOUD+POWER
-    # Partitions=compute,compute2
-    # SlurmdStartTime=2023-01-26T09:57:15
-    # ######
-    # NodeName=compute-dy-c5xlarge-50001
-    # NodeAddr=1.2.3.4
-    # NodeHostName=compute-dy-c5xlarge-50001
-    # State=IDLE+CLOUD+POWER
-    # SlurmdStartTime=None
-    # ######
+def _parse_nodes_info(node_records: List[Dict[str, str]]) -> List[SlurmNode]:
+    """
+    Build SlurmNode objects from scontrol node records extracted by _extract_scontrol_records.
 
+    Each record is a dict mapping Slurm field names to their values, e.g.:
+    {"NodeName": "compute-dy-c5xlarge-1", "NodeAddr": "1.2.3.4", "NodeHostName": "compute-dy-c5xlarge-1",
+     "State": "IDLE+CLOUD+POWER", "Partitions": "compute,compute2", "SlurmdStartTime": "2023-01-26T09:57:15"}
+    """
     map_slurm_key_to_arg = {
         "NodeName": "name",
         "NodeAddr": "nodeaddr",
@@ -433,32 +542,32 @@ def _parse_nodes_info(slurm_node_info: str) -> List[SlurmNode]:
         "SlurmdStartTime": "slurmdstarttime",
         "LastBusyTime": "lastbusytime",
         "ReservationName": "reservation_name",
+        "InstanceId": "instance_id",
     }
 
     date_fields = ["SlurmdStartTime", "LastBusyTime"]
 
-    node_info = slurm_node_info.split("######\n")
     slurm_nodes = []
-    for node in node_info:
-        lines = node.splitlines()
+    for fields in node_records:
+        if "NodeName" not in fields:
+            continue
         kwargs = {}
-        for line in lines:
-            key, value = line.split("=", 1)
+        for key, value in fields.items():
             if key in date_fields:
                 if value not in ["None", "Unknown"]:
                     value = datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").astimezone(tz=timezone.utc)
                 else:
                     value = None
+            elif key == "InstanceId" and value == "(null)":
+                # Slurm reports an unset InstanceId as "(null)"
+                value = None
             kwargs[map_slurm_key_to_arg[key]] = value
-        if lines:
-            try:
-                if is_static_node(kwargs["name"]):
-                    node = StaticNode(**kwargs)
-                    slurm_nodes.append(node)
-                else:
-                    node = DynamicNode(**kwargs)
-                    slurm_nodes.append(node)
-            except InvalidNodenameError:
-                log.warning("Ignoring node %s because it has an invalid name", kwargs["name"])
+        try:
+            if is_static_node(kwargs["name"]):
+                slurm_nodes.append(StaticNode(**kwargs))
+            else:
+                slurm_nodes.append(DynamicNode(**kwargs))
+        except InvalidNodenameError:
+            log.warning("Ignoring node %s because it has an invalid name", kwargs["name"])
 
     return slurm_nodes

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/common/schedulers/slurm_reservation_commands.py

@@ -9,6 +9,7 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import re
 
 # A nosec comment is appended to the following line in order to disable the B404 check.
 # In this file the input of the module subprocess is trusted.
@@ -16,7 +17,12 @@ import subprocess  # nosec B404
 from datetime import datetime
 from typing import List, Union
 
-from common.schedulers.slurm_commands import DEFAULT_SCONTROL_COMMAND_TIMEOUT, SCONTROL
+from common.schedulers.slurm_commands import (
+    DEFAULT_SCONTROL_COMMAND_TIMEOUT,
+    SCONTROL,
+    _extract_scontrol_records,
+    _run_scontrol_command,
+)
 from common.utils import (
     SlurmCommandError,
     SlurmCommandErrorHandler,
@@ -30,9 +36,11 @@ from slurm_plugin.slurm_resources import SlurmReservation
 logger = logging.getLogger(__name__)
 
 
-SCONTROL_SHOW_RESERVATION_OUTPUT_AWK_PARSER = (
-    'awk \'BEGIN{{RS="\\n\\n" ; ORS="######\\n";}} {{print}}\' | '
-    + "grep -oP '^(ReservationName=\\S+)|(?<!Next)(State=\\S+)|(Users=\\S+)|(Nodes=\\S+)|(######)'"
+# Fields extracted from raw `scontrol show reservations` output. Only `ReservationName` is anchored at the
+# start of a line; `(?<!Next)` ensures `State` is matched but `NextState` is not.
+SCONTROL_RESERVATION_INFO_FIELD_REGEX = re.compile(
+    r"^(ReservationName=\S+)" r"|(?<!Next)(State=\S+)" r"|(Users=\S+)" r"|(Nodes=\S+)",
+    re.MULTILINE,
 )
 
 
@@ -256,41 +264,26 @@ def get_slurm_reservations_info(
 
     Official documentation is https://slurm.schedmd.com/reservations.html
     """
-    # awk is used to replace the \n\n record separator with '######\n'
-    show_reservations_command = f"{SCONTROL} show reservations | {SCONTROL_SHOW_RESERVATION_OUTPUT_AWK_PARSER}"
-    slurm_reservations_info = check_command_output(
-        show_reservations_command, raise_on_error=raise_on_error, timeout=command_timeout, shell=True
-    )  # nosec B604
-
-    return _parse_reservations_info(slurm_reservations_info)
-
-
-def _parse_reservations_info(slurm_reservations_info: str) -> List[SlurmReservation]:
-    """Parse slurm reservations info into SlurmReservation objects."""
-    # $ /opt/slurm/bin/scontrol show reservations awk 'BEGIN{{RS="\n\n" ; ORS="######\n";}} {{print}}' |
-    # grep -oP '^(ReservationName=\S+)|(?<!Next)(State=\S+)|(Users=\S+)|(Nodes=\S+)|(######)'
-    # ReservationName=root_8
-    # Nodes=queuep4d-dy-crp4d-[1-5]
-    # Users=root
-    # State=ACTIVE
-    # ######
-    # ReservationName=root_9
-    # Nodes=queue1-st-crt2micro-1
-    # Users=root
-    # State=ACTIVE
-    # ######
+    raw_reservations_info = _run_scontrol_command(
+        "show reservations", command_timeout=command_timeout, raise_on_error=raise_on_error
+    )
+    reservation_records = _extract_scontrol_records(raw_reservations_info, SCONTROL_RESERVATION_INFO_FIELD_REGEX)
+
+    return _parse_reservations_info(reservation_records)
+
+
+def _parse_reservations_info(reservation_records: List[dict]) -> List[SlurmReservation]:
+    """
+    Build SlurmReservation objects from scontrol reservation records extracted by _extract_scontrol_records.
+
+    Each record is a dict mapping Slurm field names to their values, e.g.:
+    {"ReservationName": "root_8", "Nodes": "queuep4d-dy-crp4d-[1-5]", "Users": "root", "State": "ACTIVE"}
+    """
     map_slurm_key_to_arg = {"ReservationName": "name", "Nodes": "nodes", "Users": "users", "State": "state"}
 
-    reservation_info = slurm_reservations_info.split("######\n")
     slurm_reservations = []
-    for reservation in reservation_info:
-        lines = reservation.splitlines()
-        kwargs = {}
-        for line in lines:
-            key, value = line.split("=")
-            kwargs[map_slurm_key_to_arg[key]] = value
-        if lines:
-            reservation = SlurmReservation(**kwargs)
-            slurm_reservations.append(reservation)
+    for fields in reservation_records:
+        kwargs = {map_slurm_key_to_arg[key]: value for key, value in fields.items()}
+        slurm_reservations.append(SlurmReservation(**kwargs))
 
     return slurm_reservations

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/slurm_plugin/clustermgtd.py

@@ -44,6 +44,7 @@ from slurm_plugin.capacity_block_manager import CapacityBlockManager
 from slurm_plugin.cluster_event_publisher import ClusterEventPublisher
 from slurm_plugin.common import TIMESTAMP_FORMAT, ScalingStrategy, log_exception, print_with_count
 from slurm_plugin.console_logger import ConsoleLogger
+from slurm_plugin.fleet_manager import INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT
 from slurm_plugin.instance_manager import InstanceManager
 from slurm_plugin.slurm_resources import (
     CONFIG_FILE_DIR,
@@ -97,14 +98,30 @@ class ComputeFleetStatusManager:
     COMPUTE_FLEET_STATUS_ATTRIBUTE = "status"
     COMPUTE_FLEET_LAST_UPDATED_TIME_ATTRIBUTE = "lastStatusUpdatedTime"
 
+    # Retry is sized to outlast the transient IMDS unavailability windows observed in the fleet, which are in the
+    # order of a few seconds: waits are 1s, 2s and 4s, plus a jitter of up to 1s each to avoid retrying in lockstep
+    # with the other daemons polling IMDS. The overall wait is kept well below the clustermgtd loop time, because
+    # the fleet status is retrieved synchronously at the beginning of every loop.
+    @staticmethod
+    @retry(
+        wait_exponential_multiplier=500,
+        wait_exponential_max=seconds(4),
+        wait_jitter_max=seconds(1),
+        stop_max_attempt_number=4,
+    )
+    def _get_fleet_status():
+        # Failures are logged by get_status only once the retries are exhausted, so that the ones recovered by the
+        # retries do not pollute the log with errors that have no impact on the cluster.
+        compute_fleet_raw_data = check_command_output("get-compute-fleet-status.sh", log_error=False)
+        log.debug("Retrieved compute fleet data: %s", compute_fleet_raw_data)
+        return ComputeFleetStatus(
+            json.loads(compute_fleet_raw_data).get(ComputeFleetStatusManager.COMPUTE_FLEET_STATUS_ATTRIBUTE)
+        )
+
     @staticmethod
     def get_status(fallback=None):
         try:
-            compute_fleet_raw_data = check_command_output("get-compute-fleet-status.sh")
-            log.debug("Retrieved compute fleet data: %s", compute_fleet_raw_data)
-            return ComputeFleetStatus(
-                json.loads(compute_fleet_raw_data).get(ComputeFleetStatusManager.COMPUTE_FLEET_STATUS_ATTRIBUTE)
-            )
+            return ComputeFleetStatusManager._get_fleet_status()
         except Exception as e:
             if isinstance(e, CalledProcessError):
                 error = e.stdout.rstrip()
@@ -115,7 +132,6 @@ class ComputeFleetStatusManager:
                 error,
                 fallback,
             )
-
             return fallback
 
     @staticmethod
@@ -143,6 +159,7 @@ class ClustermgtdConfig:
         "run_instances_overrides": "/opt/slurm/etc/pcluster/run_instances_overrides.json",
         "create_fleet_overrides": "/opt/slurm/etc/pcluster/create_fleet_overrides.json",
         "fleet_config_file": "/etc/parallelcluster/slurm_plugin/fleet-config.json",
+        "instance_info_retrieval_timeout": INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT,
         # Terminate configs
         "terminate_max_batch_size": 1000,
         # Timeout to wait for node initialization, should be the same as ResumeTimeout
@@ -253,6 +270,11 @@ class ClustermgtdConfig:
             "clustermgtd", "create_fleet_overrides", fallback=self.DEFAULTS.get("create_fleet_overrides")
         )
         self.create_fleet_overrides = read_json(create_fleet_overrides_file, default={})
+        self.instance_info_retrieval_timeout = config.getint(
+            "clustermgtd",
+            "instance_info_retrieval_timeout",
+            fallback=self.DEFAULTS.get("instance_info_retrieval_timeout"),
+        )
 
     def _get_health_check_config(self, config):
         self.disable_ec2_health_check = config.getboolean(
@@ -448,6 +470,7 @@ class ClusterManager:
             run_instances_overrides=config.run_instances_overrides,
             create_fleet_overrides=config.create_fleet_overrides,
             fleet_config=config.fleet_config,
+            instance_info_retrieval_timeout=config.instance_info_retrieval_timeout,
         )
 
     def _initialize_executor(self, config):
@@ -729,13 +752,13 @@ class ClusterManager:
         return is_instance_status_unhealthy
 
     def _update_static_nodes_in_replacement(self, slurm_nodes: List[SlurmNode]):
-        """Remove from self.static_nodes_in_replacement nodes that are up or that are in maintenance."""
+        """Remove from self.static_nodes_in_replacement nodes that are up."""
         nodename_to_slurm_nodes_map = {node.name: node for node in slurm_nodes}
         nodes_still_in_replacement = set()
         for nodename in self._static_nodes_in_replacement:
             node = nodename_to_slurm_nodes_map.get(nodename)
-            # Consider nodename still in replacement if node is not up and not in maintenance
-            if node and not node.is_up() and not node.is_in_maintenance():
+            # Consider nodename still in replacement if node is not up
+            if node and not node.is_up():
                 nodes_still_in_replacement.add(nodename)
 
         # override self._static_nodes_in_replacement with updated list
@@ -949,7 +972,11 @@ class ClusterManager:
             log.info("Found the following unhealthy dynamic nodes: %s", print_with_count(unhealthy_dynamic_nodes))
             self._handle_unhealthy_dynamic_nodes(unhealthy_dynamic_nodes)
         if unhealthy_static_nodes:
-            log.info("Found the following unhealthy static nodes: %s", print_with_count(unhealthy_static_nodes))
+            # Include the node states, which carry the reason the nodes are considered unhealthy.
+            log.info(
+                "Found the following unhealthy static nodes: %s",
+                print_with_count([f"{node} in state {node.state_string}" for node in unhealthy_static_nodes]),
+            )
             self._handle_unhealthy_static_nodes(unhealthy_static_nodes)
 
         # evaluate partitions to put in protected mode and ICEs nodes to terminate
@@ -1145,15 +1172,14 @@ class ClusterManager:
 
     @staticmethod
     def _update_slurm_nodes_with_ec2_info(nodes, cluster_instances):
+        """Associate EC2 instances with Slurm nodes by matching on instance ID."""
         if cluster_instances:
-            ip_to_slurm_node_map = {node.nodeaddr: node for node in nodes}
+            instance_id_to_slurm_node_map = {node.instance_id: node for node in nodes if node.instance_id}
             for instance in cluster_instances:
-                for private_ip in instance.all_private_ips:
-                    if private_ip in ip_to_slurm_node_map:
-                        slurm_node = ip_to_slurm_node_map.get(private_ip)
-                        slurm_node.instance = instance
-                        instance.slurm_node = slurm_node
-                        break
+                if instance.id in instance_id_to_slurm_node_map:
+                    slurm_node = instance_id_to_slurm_node_map[instance.id]
+                    slurm_node.instance = instance
+                    instance.slurm_node = slurm_node
 
     @staticmethod
     def get_instance_id_to_active_node_map(partitions: List[SlurmPartition]) -> Dict:

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/slurm_plugin/fleet_manager.py

@@ -24,6 +24,12 @@ from slurm_plugin.common import print_with_count
 
 logger = logging.getLogger(__name__)
 
+# Total time budget (seconds) and per-attempt backoff cap for retrying DescribeInstances after a CreateFleet
+# launch, to tolerate EC2 API eventual consistency.
+# See https://docs.aws.amazon.com/ec2/latest/devguide/eventual-consistency.html
+INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT = 90
+INSTANCE_INFO_RETRIEVAL_MAX_BACKOFF = 30
+
 
 class EC2Instance:
     def __init__(self, id, private_ip, hostname, all_private_ips, launch_time):
@@ -94,6 +100,7 @@ class FleetManagerFactory:
         all_or_nothing,
         run_instances_overrides,
         create_fleet_overrides,
+        instance_info_retrieval_timeout=INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT,
     ):
         try:
             queue_config = fleet_config[queue]
@@ -120,6 +127,7 @@ class FleetManagerFactory:
                 compute_resource_config,
                 all_or_nothing,
                 create_fleet_overrides.get(queue, {}).get(compute_resource, {}),
+                instance_info_retrieval_timeout=instance_info_retrieval_timeout,
             )
         elif api == "run-instances":
             return Ec2RunInstancesManager(
@@ -272,6 +280,7 @@ class Ec2CreateFleetManager(FleetManager):
         compute_resource_config,
         all_or_nothing,
         launch_overrides,
+        instance_info_retrieval_timeout=INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT,
     ):
         super().__init__(
             cluster_name,
@@ -283,6 +292,7 @@ class Ec2CreateFleetManager(FleetManager):
             all_or_nothing,
             launch_overrides,
         )
+        self._instance_info_retrieval_timeout = instance_info_retrieval_timeout
 
     def _evaluate_template_overrides(self) -> list:
         """Build and return the list of Launch Template Overrides to be applied in the CreateFleet request.
@@ -436,22 +446,39 @@ class Ec2CreateFleetManager(FleetManager):
         """
         Describe instances to retrieve info not available from create-fleet response.
 
+        Right after a CreateFleet launch, DescribeInstances may return instances with missing info
+        (e.g. PrivateIpAddress) or even InvalidInstanceID.NotFound due to EC2 API eventual consistency.
+        Retry with exponential backoff (capped per attempt) until the configured total timeout is reached,
+        as recommended at https://docs.aws.amazon.com/ec2/latest/devguide/eventual-consistency.html
+
         :raises ClientError in case of boto3 failure
         :return list of instances with complete information and list of IDs for instances with incomplete information
         """
         instances = []
         partial_instance_ids = instance_ids
 
-        retries = 5
         attempt_count = 0
+        # Budget is tracked against the un-jittered backoff; jitter is added only to the actual sleep.
+        elapsed_backoff = 0
         # Wait for instances to be available in EC2
         time.sleep(0.1)
-        while attempt_count < retries and partial_instance_ids:
+        while partial_instance_ids:
             complete_instances, partial_instance_ids = self._retrieve_instances_info_from_ec2(partial_instance_ids)
             instances.extend(complete_instances)
+            if not partial_instance_ids:
+                break
+            base_backoff = min(0.3 * 2 ** (attempt_count + 1), INSTANCE_INFO_RETRIEVAL_MAX_BACKOFF)
+            if elapsed_backoff + base_backoff > self._instance_info_retrieval_timeout:
+                logger.warning(
+                    "Unable to retrieve complete info for instances %s within %s seconds, giving up after %s attempts.",
+                    print_with_count(partial_instance_ids),
+                    self._instance_info_retrieval_timeout,
+                    attempt_count + 1,
+                )
+                break
+            elapsed_backoff += base_backoff
             attempt_count += 1
-            if attempt_count < retries:
-                time.sleep(0.3 * 2**attempt_count + (secrets.randbelow(500) / 1000))
+            time.sleep(base_backoff + (secrets.randbelow(500) / 1000))
 
         return instances, partial_instance_ids
 

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/slurm_plugin/fleet_status_manager.py

@@ -19,7 +19,11 @@ from configparser import ConfigParser
 from logging.config import fileConfig
 
 from botocore.config import Config
-from common.schedulers.slurm_commands import resume_powering_down_nodes, update_all_partitions
+from common.schedulers.slurm_commands import (
+    reset_nodes_in_inactive_partitions,
+    resume_powering_down_nodes,
+    update_all_partitions,
+)
 from slurm_plugin.clustermgtd import ComputeFleetStatus, ComputeFleetStatusManager
 from slurm_plugin.common import log_exception
 from slurm_plugin.instance_manager import InstanceManager
@@ -91,6 +95,7 @@ def _manage_fleet_status_transition(config, computefleet_status_data_path):
 
 def _start_partitions():
     log.info("Setting slurm partitions to UP and resuming nodes...")
+    reset_nodes_in_inactive_partitions()
     update_all_partitions(PartitionStatus.UP, reset_node_addrs_hostname=False)
     # TODO: This function was added due to Slurm ticket 12915. The bug is not reproducible and the ticket was then
     #  closed. This operation may now be useless: we need to check this.

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/slurm_plugin/instance_manager.py

@@ -25,7 +25,11 @@ from common.ec2_utils import get_private_ip_address_and_dns_name
 from common.schedulers.slurm_commands import get_nodes_info, update_nodes
 from common.utils import grouper, setup_logging_filter
 from slurm_plugin.common import ComputeInstanceDescriptor, ScalingStrategy, log_exception, print_with_count
-from slurm_plugin.fleet_manager import EC2Instance, FleetManagerFactory
+from slurm_plugin.fleet_manager import (
+    INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT,
+    EC2Instance,
+    FleetManagerFactory,
+)
 from slurm_plugin.slurm_resources import (
     EC2_HEALTH_STATUS_UNHEALTHY_STATES,
     EC2_INSTANCE_ALIVE_STATES,
@@ -85,6 +89,7 @@ class InstanceManager:
         run_instances_overrides: dict = None,
         create_fleet_overrides: dict = None,
         job_level_scaling: bool = False,
+        instance_info_retrieval_timeout: int = INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT,
     ):
         """Initialize InstanceLauncher with required attributes."""
         self._region = region
@@ -101,6 +106,7 @@ class InstanceManager:
         self._fleet_config = fleet_config
         self._run_instances_overrides = run_instances_overrides or {}
         self._create_fleet_overrides = create_fleet_overrides or {}
+        self._instance_info_retrieval_timeout = instance_info_retrieval_timeout
         self._boto3_resource_factory = lambda resource_name: boto3.session.Session().resource(
             resource_name, region_name=region, config=boto3_config
         )
@@ -262,7 +268,8 @@ class InstanceManager:
         """
         Get instances that are associated with the cluster.
 
-        Instances without all the info set are ignored and not returned
+        Instances with missing info (e.g. PrivateIpAddress due to EC2 eventual consistency) are kept with
+        empty IP fields so that clustermgtd can still match them to Slurm nodes by instance ID.
         """
         ec2_client = boto3.client("ec2", region_name=self._region, config=self._boto3_config)
         paginator = ec2_client.get_paginator("describe_instances")
@@ -290,12 +297,17 @@ class InstanceManager:
                     )
                 )
             except Exception as e:
+                # Keep the instance with empty IP info so it can still be matched by instance ID in clustermgtd.
                 logger.warning(
-                    "Ignoring instance %s because not all EC2 info are available, exception: %s, message: %s",
+                    "Incomplete EC2 info for instance %s, keeping it for instance-ID matching, "
+                    "exception: %s, message: %s",
                     instance_info["InstanceId"],
                     type(e).__name__,
                     e,
                 )
+                instances.append(
+                    EC2Instance(instance_info["InstanceId"], "", "", set(), instance_info.get("LaunchTime"))
+                )
 
         return instances
 
@@ -1008,6 +1020,7 @@ class InstanceManager:
             all_or_nothing=all_or_nothing_batch,
             run_instances_overrides=self._run_instances_overrides,
             create_fleet_overrides=self._create_fleet_overrides,
+            instance_info_retrieval_timeout=self._instance_info_retrieval_timeout,
         )
         return fleet_manager
 
@@ -1077,6 +1090,7 @@ class InstanceManager:
                 slurm_nodes,
                 nodeaddrs=[instance.private_ip for instance in launched_instances],
                 nodehostnames=node_hostnames,
+                instance_ids=[instance.id for instance in launched_instances],
             )
             logger.info(
                 "Nodes are now configured with instances %s",

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/slurm_plugin/resume.py

@@ -22,6 +22,7 @@ from common.schedulers.slurm_commands import get_nodes_info, set_nodes_down
 from common.utils import read_json
 from slurm_plugin.cluster_event_publisher import ClusterEventPublisher
 from slurm_plugin.common import ScalingStrategy, is_clustermgtd_heartbeat_valid, print_with_count
+from slurm_plugin.fleet_manager import INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT
 from slurm_plugin.instance_manager import InstanceManager
 from slurm_plugin.slurm_resources import CONFIG_FILE_DIR
 
@@ -47,6 +48,7 @@ class SlurmResumeConfig:
         "fleet_config_file": "/etc/parallelcluster/slurm_plugin/fleet-config.json",
         "job_level_scaling": True,
         "scaling_strategy": "all-or-nothing",
+        "instance_info_retrieval_timeout": INSTANCE_INFO_RETRIEVAL_TIMEOUT_DEFAULT,
     }
 
     def __init__(self, config_file_path):
@@ -96,6 +98,11 @@ class SlurmResumeConfig:
         self.job_level_scaling = config.getboolean(
             "slurm_resume", "job_level_scaling", fallback=self.DEFAULTS.get("job_level_scaling")
         )
+        self.instance_info_retrieval_timeout = config.getint(
+            "slurm_resume",
+            "instance_info_retrieval_timeout",
+            fallback=self.DEFAULTS.get("instance_info_retrieval_timeout"),
+        )
         fleet_config_file = config.get(
             "slurm_resume", "fleet_config_file", fallback=self.DEFAULTS.get("fleet_config_file")
         )
@@ -206,6 +213,7 @@ def _resume(arg_nodes, resume_config, slurm_resume):
         run_instances_overrides=resume_config.run_instances_overrides,
         create_fleet_overrides=resume_config.create_fleet_overrides,
         job_level_scaling=resume_config.job_level_scaling,
+        instance_info_retrieval_timeout=resume_config.instance_info_retrieval_timeout,
     )
     instance_manager.add_instances(
         slurm_resume=slurm_resume,

{aws_parallelcluster_node-3.15.1 → aws_parallelcluster_node-3.16.1}/src/slurm_plugin/slurm_resources.py

@@ -203,7 +203,11 @@ class SlurmNode(metaclass=ABCMeta):
     SLURM_SCONTROL_POWER_UP_STATE = "POWERING_UP"
     SLURM_SCONTROL_ONLINE_STATES = {"IDLE+CLOUD", "MIXED+CLOUD", "ALLOCATED+CLOUD", "COMPLETING+CLOUD"}
     SLURM_SCONTROL_POWER_WITH_JOB_STATE = {"MIXED", "CLOUD", "POWERED_DOWN"}
-    SLURM_SCONTROL_RESUME_FAILED_STATE = {"DOWN", "CLOUD", "POWERED_DOWN", "NOT_RESPONDING"}
+    # In Slurm 25.11:
+    # When ResumeTimeout fires on a cloud node, Slurm explicitly clears NODE_STATE_NO_RESPOND (along with DRAIN,
+    # POWER_DOWN, POWERING_UP), adds POWERED_DOWN, and then set node to DOWN with reason "ResumeTimeout reached".
+    # The resulting state is therefore DOWN+CLOUD+POWERED_DOWN and does NOT include NOT_RESPONDING.
+    SLURM_SCONTROL_RESUME_FAILED_STATE = {"DOWN", "CLOUD", "POWERED_DOWN"}
     SLURM_SCONTROL_NODE_DOWN_NOT_RESPONDING_STATE = {"DOWN", "CLOUD", "NOT_RESPONDING"}
     # Due to a bug in Slurm a powered down node can enter IDLE+CLOUD+POWER_DOWN+POWERED_DOWN state
     SLURM_SCONTROL_POWER_STATES = [{"IDLE", "CLOUD", "POWERED_DOWN"}, {"IDLE", "CLOUD", "POWERED_DOWN", "POWER_DOWN"}]
@@ -239,6 +243,7 @@ class SlurmNode(metaclass=ABCMeta):
         slurmdstarttime: datetime = None,
         lastbusytime: datetime = None,
         reservation_name: str = None,
+        instance_id: str = None,
     ):
         """Initialize slurm node with attributes."""
         self.name = name
@@ -249,6 +254,7 @@ class SlurmNode(metaclass=ABCMeta):
         self.partitions = partitions.strip().split(",") if partitions else None
         self.reason = reason
         self.instance = instance
+        self.instance_id = instance_id
         self.slurmdstarttime = slurmdstarttime
         self.lastbusytime = lastbusytime
         self.reservation_name = reservation_name
@@ -529,6 +535,7 @@ class StaticNode(SlurmNode):
         slurmdstarttime=None,
         lastbusytime=None,
         reservation_name=None,
+        instance_id=None,
     ):
         """Initialize slurm node with attributes."""
         super().__init__(
@@ -542,6 +549,7 @@ class StaticNode(SlurmNode):
             slurmdstarttime,
             lastbusytime=lastbusytime,
             reservation_name=reservation_name,
+            instance_id=instance_id,
         )
 
     def is_healthy(
@@ -581,20 +589,30 @@ class StaticNode(SlurmNode):
                 return True
             else:
                 if log_warn_if_unhealthy:
-                    logger.warning("Node state check: node %s in DRAINED, node state: %s", self, self.state_string)
+                    logger.warning(
+                        "Node state check: node %s in DRAINED and not within the replacement protection window, "
+                        "it is considered unhealthy and will be replaced, node state: %s",
+                        self,
+                        self.state_string,
+                    )
                 return False
         # Check to see if node is in DOWN, ignoring any node currently being replaced
         elif self.is_down() and consider_down_as_unhealthy:
             if self.is_being_replaced:
                 logger.debug(
-                    "Node state check: node %s in DOWN but is currently being replaced, ignoring. Node state: ",
+                    "Node state check: node %s in DOWN but is currently being replaced, ignoring, node state: %s",
                     self,
                     self.state_string,
                 )
                 return True
             else:
                 if log_warn_if_unhealthy:
-                    logger.warning("Node state check: node %s in DOWN, node state: %s", self, self.state_string)
+                    logger.warning(
+                        "Node state check: node %s in DOWN and not within the replacement protection window, "
+                        "it is considered unhealthy and will be replaced, node state: %s",
+                        self,
+                        self.state_string,
+                    )
                 return False
         return True
 
@@ -663,6 +681,7 @@ class DynamicNode(SlurmNode):
         slurmdstarttime=None,
         lastbusytime=None,
         reservation_name=None,
+        instance_id=None,
     ):
         """Initialize slurm node with attributes."""
         super().__init__(
@@ -676,6 +695,7 @@ class DynamicNode(SlurmNode):
             slurmdstarttime,
             lastbusytime=lastbusytime,
             reservation_name=reservation_name,
+            instance_id=instance_id,
         )
 
     def is_state_healthy(self, consider_drain_as_unhealthy, consider_down_as_unhealthy, log_warn_if_unhealthy=True):
@@ -730,7 +750,7 @@ class DynamicNode(SlurmNode):
                 self.state_string,
             )
             return True
-        # Dynamic node in DOWN+CLOUD+POWERED_DOWN+NOT_RESPONDING state
+        # Dynamic node in SLURM_SCONTROL_RESUME_FAILED_STATE.
         elif self.is_bootstrap_timeout():
             # We need to check if nodeaddr is set to avoid counting powering up nodes as bootstrap failure nodes during
             # cluster start/stop.