squidient 4.0.2__tar.gz → 4.0.4__tar.gz

{squidient-4.0.2 → squidient-4.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 4.0.2
+Version: 4.0.4
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-4.0.2 → squidient-4.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "4.0.2"
+version = "4.0.4"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-4.0.2 → squidient-4.0.4}/src/squidient/definitions.py

@@ -20,6 +20,8 @@
 
 test_default_timeout = "00:01:00"
 test_max_timeout = "00:05:00"
+test_timeout_grace_seconds = 15
+test_timeout_grace_max_seconds = 3600
 merged_max_timeout = "02:00:00"
 queue_max_timeout = "02:00:00"
 queue_test_sleep_before_get_launch_time = 30  # seconds

{squidient-4.0.2 → squidient-4.0.4}/src/squidient/testing/cases/instance.py

@@ -30,6 +30,8 @@ from ...definitions import (
     test_default_timeout,
     test_max_retry,
     test_max_timeout,
+    test_timeout_grace_seconds,
+    test_timeout_grace_max_seconds,
 )
 from ...systems.queue_tester import QueueTester
 from ...utils.log import logger
@@ -535,10 +537,14 @@ class Instance:
             elif file_contains(self._report_path + "/post.err", "Fatal error in PMPI_Init: Unknown error class, error stack"):
                 self._report["error"] = "job"
                 retry_required = True
+            timeout_seconds = timetosecond(self._timeout)
+            grace_seconds = test_timeout_grace_seconds if timeout_seconds <= test_timeout_grace_max_seconds else 0
             if not self.compute_times():
                 self._report["error"] = "timeout"
                 retry_required = True
-            elif self._merged and timetosecond(self._report["jobTime"]) > timetosecond(self._timeout):
+            elif timetosecond(self._report["jobTime"]) > timeout_seconds + grace_seconds:
+                logger.warning("Instance %s exceeded its total runtime limit: measured=%s, allowed=%s, grace=%ss, merged=%s, attempt=%s",
+                               self._id, self._report["jobTime"], self._timeout, grace_seconds, self._merged, self._retry + 1)
                 self._report["error"] = "timeout"
                 retry_required = True
         if self._report["error"] in ("report", "slurm", "timeout", "overload", "binding", "job"):