squidient 3.4.5__tar.gz → 3.4.7__tar.gz

{squidient-3.4.5 → squidient-3.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 3.4.5
+Version: 3.4.7
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-3.4.5 → squidient-3.4.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "3.4.5"
+version = "3.4.7"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-3.4.5 → squidient-3.4.7}/src/squidient/builds/build_orchestrator.py

@@ -152,6 +152,8 @@ class BuildOrchestrator:
         return len(self._launched_builds)
 
     def next_valid_build(self, show_progress=True, single_snapshot=None, stop_on_error=False, output_level=1):
+        if single_snapshot is None:
+            single_snapshot = not console.interactive()
         display_status = show_progress
         while len(self._pending_builds) == 0:
             if self.check_builds(

{squidient-3.4.5 → squidient-3.4.7}/src/squidient/data/builds/builds.json

@@ -12,6 +12,7 @@
         ],
         "platform": "gpp",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited",
             "export SLURM_CPU_BIND=none"
         ],
@@ -45,6 +46,7 @@
         ],
         "platform": "gpp",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited",
             "export SLURM_CPU_BIND=none"
         ],
@@ -156,6 +158,7 @@
             "export DLB_ARGS=--talp"
         ],
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited",
             "export SLURM_CPU_BIND=none"
         ],
@@ -233,6 +236,7 @@
         ],
         "platform": "gpp",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited"
         ],
         "system": "bsc",
@@ -316,6 +320,7 @@
         ],
         "platform": "gpp",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited"
         ],
         "system": "bsc",
@@ -342,6 +347,7 @@
         ],
         "platform": "acc",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited",
             "export SLURM_CPU_BIND=none"
         ],
@@ -515,6 +521,7 @@
         ],
         "platform": "acc",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited",
             "export SLURM_CPU_BIND=none"
         ],
@@ -573,6 +580,7 @@
         ],
         "platform": "grace",
         "postModules": [
+            "unset MAKEFLAGS",
             "ulimit -s unlimited",
             "export SLURM_CPU_BIND=none"
         ],

{squidient-3.4.5 → squidient-3.4.7}/src/squidient/staging/test_relauncher.py

@@ -40,7 +40,7 @@ class StagingTestRelaunchManager:
             stop_on_exceeded=True,
             retry_message="Maximum number of retries exceeded"):
         retry_count = 0
-        while self._workflow.should_relaunch_tests() and (status or stop_on_exceeded):
+        while self._workflow.should_relaunch_tests():
             retry_count += 1
             logger.debug(
                 "Waiting for relaunched tests: retry=%s, status=%s, stop_on_exceeded=%s",
@@ -53,6 +53,7 @@ class StagingTestRelaunchManager:
                 console.error(retry_message, level=1)
                 if stop_on_exceeded:
                     raise WorkflowFailure(retry_message)
+                status = False
                 break
             console.section("Some tests have been relaunched")
             self._workflow.wait_current_tests(single_snapshot=single_snapshot)

{squidient-3.4.5 → squidient-3.4.7}/src/squidient/testing/cases/instance.py

@@ -461,14 +461,22 @@ class Instance:
             except RuntimeError:
                 console.group("Instance " + self._id, level=2)
                 console.error("Cannot download the report for instance " + self._id, level=3)
-        error = "unknown"
+        error = "report"
         if not self._job_valid and self._job_terminated:
             error = "slurm"
         try:
             self._report = open_critical_json(self._report_path + "/report.json")
-        except (OSError, ValueError, RuntimeError):
+            if (not isinstance(self._report, dict)
+                    or not isinstance(self._report.get("status"), bool)
+                    or not isinstance(self._report.get("error"), str)
+                    or not self._report["error"]
+                    or self._report["error"] in ("skipped", "unknown")):
+                raise ValueError("Report must contain a boolean status and a non-placeholder error")
+        except (OSError, ValueError, RuntimeError) as exception:
+            logger.warning("Missing or invalid report for instance %s, attempt %s: %s",
+                           self._id, self._retry + 1, exception)
             console.group("Instance " + self._id, level=2)
-            console.warning("Report not found; creating a dummy report for instance " + self._id, level=3)
+            console.warning("Report missing or invalid; creating a dummy report for instance " + self._id, level=3)
             self.build_failed_report(error)
         shutil.copy2(self._json_path, self._report_path)
         self._report["path"] = self._test_path
@@ -483,18 +491,17 @@ class Instance:
 
         self._relaunch = False
 
-        if self._report["error"] in ["slurm"]:
-            self.retry()
-        else:
+        retry_required = self._report["error"] in ("slurm", "report")
+        if self._report["error"] != "slurm":
             if file_contains(self._report_path + "/test.err", "DUE TO TIME LIMIT"):
                 self._report["error"] = "timeout"
-                self.retry()
+                retry_required = True
             elif file_contains(self._report_path + "/post.err", "already completing or completed"):
                 self._report["error"] = "timeout"
-                self.retry()
+                retry_required = True
             elif file_contains(self._report_path + "/post.err", "Expired or invalid job"):
                 self._report["error"] = "timeout"
-                self.retry()
+                retry_required = True
             elif file_contains(self._report_path + "/alya.err", "All nodes which are allocated for this job are already filled"):
                 self._report["error"] = "overload"
             elif (
@@ -503,31 +510,45 @@ class Instance:
             ):
                 self._report["error"] = "binding"
             elif file_contains(self._report_path + "/alya.err", "Integer overflow in xmallocarray"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             elif not file_contains(self._report_path + "/alya.out", "START ALYA"):
                 self._report["error"] = "job"
-                self.retry()
+                retry_required = True
             elif file_contains(self._report_path + "/alya.err", "Program received signal SIGSEGV"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             elif file_contains(self._report_path + "/alya.err", "Bus error"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             elif file_contains(self._report_path + "/alya.err", "Transport retry count exceeded on"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             elif file_contains(self._report_path + "/post.err", "Transport retry count exceeded on"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             elif file_contains(self._report_path + "/post.err", "srun: error: Unable to create step for job"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             elif file_contains(self._report_path + "/post.err", "Fatal error in PMPI_Init: Unknown error class, error stack"):
-                self.retry()
+                self._report["error"] = "job"
+                retry_required = True
             if not self.compute_times():
                 self._report["error"] = "timeout"
-                self.retry()
+                retry_required = True
             elif self._merged and timetosecond(self._report["jobTime"]) > timetosecond(self._timeout):
                 self._report["error"] = "timeout"
+                retry_required = True
+        if self._report["error"] in ("report", "slurm", "timeout", "overload", "binding", "job"):
+            self._report["status"] = False
+        if retry_required:
+            # Several diagnostics can fail in one execution; count that attempt only once.
+            self.retry()
+            logger.debug("Instance %s: error=%s, attempt=%s, max_attempts=%s, relaunch=%s",
+                         self._id, self._report["error"], self._report["retries"] + 1,
+                         self._max_retry, self._relaunch)
         self.build_performance()
         self.build_counters()
-        if self._relaunch:
-            self._report["error"] = "skipped"
         save_json(self._report, self._report_path + "/report.json")
         if self._relaunch:
             self.backup_outputs()