aws-parallelcluster-node 3.14.2__tar.gz → 3.15.0__tar.gz

{aws_parallelcluster_node-3.14.2/src/aws_parallelcluster_node.egg-info → aws_parallelcluster_node-3.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-parallelcluster-node
-Version: 3.14.2
+Version: 3.15.0
 Summary: aws-parallelcluster-node provides the scripts for an AWS ParallelCluster node.
 Home-page: https://github.com/aws/aws-parallelcluster-node
 Author: Amazon Web Services
@@ -15,6 +15,7 @@ License-File: LICENSE.txt
 License-File: NOTICE.txt
 Requires-Dist: boto3>=1.7.55
 Requires-Dist: retrying>=1.3.3
+Requires-Dist: setuptools
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{aws_parallelcluster_node-3.14.2 → aws_parallelcluster_node-3.15.0}/setup.py

@@ -32,8 +32,8 @@ console_scripts = [
     "clustermgtd = slurm_plugin.clustermgtd:main",
     "computemgtd = slurm_plugin.computemgtd:main",
 ]
-version = "3.14.2"
-requires = ["boto3>=1.7.55", "retrying>=1.3.3"]
+version = "3.15.0"
+requires = ["boto3>=1.7.55", "retrying>=1.3.3", "setuptools"]
 
 setup(
     name="aws-parallelcluster-node",

{aws_parallelcluster_node-3.14.2 → aws_parallelcluster_node-3.15.0/src/aws_parallelcluster_node.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-parallelcluster-node
-Version: 3.14.2
+Version: 3.15.0
 Summary: aws-parallelcluster-node provides the scripts for an AWS ParallelCluster node.
 Home-page: https://github.com/aws/aws-parallelcluster-node
 Author: Amazon Web Services
@@ -15,6 +15,7 @@ License-File: LICENSE.txt
 License-File: NOTICE.txt
 Requires-Dist: boto3>=1.7.55
 Requires-Dist: retrying>=1.3.3
+Requires-Dist: setuptools
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{aws_parallelcluster_node-3.14.2 → aws_parallelcluster_node-3.15.0}/src/aws_parallelcluster_node.egg-info/requires.txt

@@ -1,2 +1,3 @@
 boto3>=1.7.55
 retrying>=1.3.3
+setuptools

{aws_parallelcluster_node-3.14.2 → aws_parallelcluster_node-3.15.0}/src/slurm_plugin/cluster_event_publisher.py

@@ -104,6 +104,10 @@ CLUSTER_INSTANCE_COUNT = {
     "message": "Count of EC2 instances backing the cluster compute fleet",
     "event_type": "cluster-instance-count",
 }
+CLUSTERMGTD_HEARTBEAT = {
+    "message": "Clustermgtd heartbeat",
+    "event_type": "clustermgtd-heartbeat",
+}
 
 
 class ClusterEventPublisher:
@@ -474,6 +478,37 @@ class ClusterEventPublisher:
             event_supplier=({"detail": self._describe_node(node)} for node in compute_nodes),
         )
 
+    # Example event generated by this function:
+    # {
+    #     "datetime": "2023-04-03T18:10:13.089+00:00",
+    #     "version": 0,
+    #     "scheduler": "slurm",
+    #     "cluster-name": "integ-tests-9hbu2op3liukbqqz-develop",
+    #     "node-role": "HeadNode",
+    #     "component": "clustermgtd",
+    #     "level": "INFO",
+    #     "instance-id": "i-04886f8b56e5967ee",
+    #     "event-type": "clustermgtd-heartbeat",
+    #     "message": "Clustermgtd heartbeat",
+    #     "detail": {
+    #         "heartbeat-timestamp": "2023-04-03T18:10:13.089+00:00"
+    #     }
+    # }
+    @log_exception(logger, "publish_heartbeat_event", catch_exception=Exception, raise_on_error=False)
+    def publish_heartbeat_event(self, heartbeat_timestamp: datetime):
+        """Publish heartbeat event."""
+        timestamp = ClusterEventPublisher.timestamp()
+        self.publish_event(
+            logging.INFO,
+            **CLUSTERMGTD_HEARTBEAT,
+            timestamp=timestamp,
+            detail={
+                "heartbeat-timestamp": (
+                    heartbeat_timestamp.isoformat(timespec="milliseconds") if heartbeat_timestamp else None
+                ),
+            },
+        )
+
     # Slurm Resume Events
     # Note: This uses the same schema as `publish_unhealthy_static_node_events` from clustermgtd
     # Example event generated by this function:

{aws_parallelcluster_node-3.14.2 → aws_parallelcluster_node-3.15.0}/src/slurm_plugin/clustermgtd.py

@@ -577,10 +577,13 @@ class ClusterManager:
     def _write_timestamp_to_file(self):
         """Write timestamp into shared file so compute nodes can determine if head node is online."""
         # Make clustermgtd heartbeat readable to all users
+        heartbeat_time = datetime.now(tz=timezone.utc)
         with open(os.open(self._config.heartbeat_file_path, os.O_WRONLY | os.O_CREAT, 0o644), "w") as timestamp_file:
             # Note: heartbeat must be written with datetime.strftime to convert localized datetime into str
             # datetime.strptime will not work with str(datetime)
-            timestamp_file.write(datetime.now(tz=timezone.utc).strftime(TIMESTAMP_FORMAT))
+            timestamp_file.write(heartbeat_time.strftime(TIMESTAMP_FORMAT))
+        # Publish heartbeat event to events log
+        self._event_publisher.publish_heartbeat_event(heartbeat_time)
 
     @staticmethod
     @retry(stop_max_attempt_number=2, wait_fixed=1000)
@@ -1262,7 +1265,8 @@ class ClusterManager:
             return
         log.info(
             "The following compute resources are in down state due to insufficient capacity: %s, "
-            "compute resources will be reset after insufficient capacity timeout (%s seconds) expired",
+            "compute resources will be reset after insufficient capacity timeout (%s seconds) expired. "
+            "Check the slurm_resume log for EC2 error codes.",
             self._insufficient_capacity_compute_resources,
             self._config.insufficient_capacity_timeout,
         )

{aws_parallelcluster_node-3.14.2 → aws_parallelcluster_node-3.15.0}/src/slurm_plugin/resume.py

@@ -227,7 +227,11 @@ def _resume(arg_nodes, resume_config, slurm_resume):
             print_with_count(failed_nodes),
         )
         for error_code, node_list in instance_manager.failed_nodes.items():
-            _handle_failed_nodes(node_list, reason=f"(Code:{error_code})Failure when resuming nodes")
+            _handle_failed_nodes(
+                node_list,
+                reason=f"(Code:{error_code})Failure when resuming nodes - "
+                f"Check the slurm_resume log for EC2 error codes",
+            )
 
         event_publisher = ClusterEventPublisher.create_with_default_publisher(
             event_logger,