squidient 2.5.2__tar.gz → 2.6.0__tar.gz

{squidient-2.5.2 → squidient-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 2.5.2
+Version: 2.6.0
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-2.5.2 → squidient-2.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "2.5.2"
+version = "2.6.0"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/configure.py

@@ -27,6 +27,7 @@ from typing import Optional, Sequence, Dict, Any
 
 from .builds.predefinedbuild import *
 from .builds.build import *
+from .definitions import benchmark_failure_copy, benchmark_failure_sender
 from .utils.message import console
 
 from pathlib import Path
@@ -364,6 +365,8 @@ def generate_json():
     config["benchmarks"]["builds"] = predefined_benchmarks["all"]
     config["benchmarks"]["tests"] = []
     config["benchmarks"]["lowDiskUsage"] = True
+    config["benchmarks"]["failureSender"] = benchmark_failure_sender
+    config["benchmarks"]["failureCopy"] = benchmark_failure_copy
     # staging
     config["staging"] = {}
     config["staging"]["builds"] = predefined_builds["fast"]["gpp"]
@@ -391,7 +394,7 @@ def generate_json():
 
 def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system="bsc", platform="gpp", hosts=None, install=False,
                   type=None, api=None, reservation=False, project=None, alamak=False, alamak_branch=None,
-                  alamak_revision=None):
+                  alamak_revision=None, notify_failures=False):
 
     predefined_benchmarks = PredefinedBuild("benchmarks").get_pb()
     predefined_builds = PredefinedBuild("builds").get_pb()
@@ -443,6 +446,9 @@ def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system
     # various
     console.section("Configuring flex")
     config["flex"] = flex
+    config["benchmarks"]["notifyFailures"] = notify_failures
+    config["benchmarks"]["failureSender"] = benchmark_failure_sender
+    config["benchmarks"]["failureCopy"] = benchmark_failure_copy
     console.section("Configuring clean log")
     config["cleanLog"] = False
     console.section("Configuring system")
@@ -590,6 +596,7 @@ def configure_help():
     console.write("\t--api:[api]                           Define GitLab API")
     console.write("\t--alamak-branch:[branch]              Define which alamak branch to use")
     console.write("\t--alamak-revision:[revision]          Define which alamak revision to use")
+    console.write("\t--notify-failures                     Notify benchmark authors when a benchmark fails")
     console.write("\t--reservation                         Enable reservation")
     line_jump()
 
@@ -643,6 +650,7 @@ def _run_cli(argv: Sequence[str]) -> int:
         alamak = False
         alamak_branch = None
         alamak_revision = None
+        notify_failures = False
 
         for arg in argv[0:]:
             if "--pipeline:" in arg:
@@ -686,8 +694,13 @@ def _run_cli(argv: Sequence[str]) -> int:
                 console.step("Project " + project)
             if "--alamak-branch:" in arg:
                 alamak_branch = arg.replace("--alamak-branch:", "")
+                console.step("Alamak branch " + alamak_branch)
             if "--alamak-revision:" in arg:
                 alamak_revision = arg.replace("--alamak-revision:", "")
+                console.step("Alamak revision " + alamak_revision)
+            if "--notify-failures" in arg:
+                notify_failures = True
+                console.step("Benchmark failure notifications enabled")
 
         gitlab_config(
             pipeline=pipeline,
@@ -706,6 +719,7 @@ def _run_cli(argv: Sequence[str]) -> int:
             alamak=alamak,
             alamak_branch=alamak_branch,
             alamak_revision=alamak_revision,
+            notify_failures=notify_failures,
         )
         return 0
 

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/connection/job.py

@@ -87,12 +87,16 @@ class JobThread (threading.Thread):
                 self._current_jobs = self.get_job_numbers()
                 if self._current_jobs > self._max_jobs:
                     time.sleep(20)
-            output = self._ssh.ssh(
+            result = self._ssh.ssh(
                 cmd=job["command"],
                 system=self._system,
                 platform=self._platform,
                 server_path=job["server_path"],
-            ).stdout
+            )
+            output = result.stdout
+            job["stdout"] = output
+            job["stderr"] = getattr(result, "stderr", "")
+            job["returncode"] = getattr(result, "returncode", 0)
             try:
                 m = re.search(r"Submitted batch job (\d+)", output)
                 if not m:
@@ -128,6 +132,7 @@ class Job:
         self._jobs_not_actualized = {}
         self._jobs_launched = {}
         self._jobs_not_launched = {}
+        self._job_submissions = {}
         self._job_list = {}
         self._input_queues = {}
         self._output_queues = {}
@@ -138,6 +143,7 @@ class Job:
             self._jobs_not_actualized[system] = {}
             self._jobs_launched[system] = {}
             self._jobs_not_launched[system] = {}
+            self._job_submissions[system] = {}
             self._job_list[system] = {}
             self._input_queues[system] = {}
             self._output_queues[system] = {}
@@ -174,6 +180,7 @@ class Job:
             self._jobs_not_launched[system][platform] = {}
             self._jobs_launched[system][platform] = {}
             self._jobs_not_actualized[system][platform] = {}
+            self._job_submissions[system][platform] = {}
             self._jobs[system][platform] = {}
         job = {}
         job["id"] = id
@@ -281,11 +288,18 @@ class Job:
         job_id = -1
         if id in self._jobs_launched[system][platform]:
             job_id = self._jobs_launched[system][platform][id]["job_id"]
+            self._job_submissions[system][platform][job_id] = self._jobs_launched[system][platform][id]
             self._jobs[system][platform][job_id] = job_id
             self._jobs_not_actualized[system][platform][job_id] = job_id
             self._jobs_launched[system][platform].pop(id)
         return job_id
 
+    def get_job_submission(self, job_id, system, platform):
+        try:
+            return self._job_submissions[system][platform].get(job_id, {})
+        except:
+            return {}
+
     def kill_all(self):
         for system in self._job_threads:
             for platform in self._job_threads[system]:

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/data/web/btest.html

@@ -36,6 +36,7 @@
         <th>Openmp</th>
         <th>Json</th>
         <th>Job</th>
+        <th>Sbatch</th>
         <th>Output</th>
         <th>Alya</th>
         <th>Postprocess</th>
@@ -77,6 +78,9 @@
           dir=testsuiteJSON.benchmark_tests + name + "/" + t[x].build + "/" + t[x].mpi + "_" + t[x].openmp + "/"
           t[x].job=testsuiteJSON.setUrl(testsuiteJSON.FILE_CLOUD, dir + "/JOB.SB", null)
           t[x].json=testsuiteJSON.setUrl(testsuiteJSON.FILE_CONFIG, dir + "/" + name + ".json", null)
+          t[x].sbatch_out=testsuiteJSON.setUrl(testsuiteJSON.FILE_OK, dir + "/sbatch.out", null)
+          t[x].sbatch_err=testsuiteJSON.setUrl(testsuiteJSON.FILE_KO, dir + "/sbatch.err", null)
+          t[x].sbatch=t[x].sbatch_out + " " + t[x].sbatch_err
           t[x].output_out=testsuiteJSON.setUrl(testsuiteJSON.FILE_OK, dir + "/test.out", null)
           t[x].output_err=testsuiteJSON.setUrl(testsuiteJSON.FILE_KO, dir + "/test.err", null)
           t[x].output=t[x].output_out + " " + t[x].output_err

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/definitions.py

@@ -67,4 +67,16 @@ squidient_local = "squidient/"
 squidient_remote = "squidient"
 cmake_link_dir = "ALYA_CMAKE_INSTALL_DIR"
 max_advised_builds = 2
-database_name = {"test": "rooster_test", "production": "rooster"}
+database_name = {"test": "rooster_test", "production": "rooster"}
+benchmark_failure_sender = "damien.dosimont@bsc.es"
+benchmark_failure_copy = ["damien.dosimont@bsc.es"]
+benchmark_failure_log_files = [
+    "sbatch.out",
+    "sbatch.err",
+    "test.out",
+    "test.err",
+    "alya.out",
+    "alya.err",
+    "post.out",
+    "post.err",
+]

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/monitor/monitor.py

@@ -44,6 +44,7 @@ class Monitor(Staging):
             self.run_all_monitor_tests(terraform)
             status = self.finish_all_monitor_tests()
             self._report.write_tests(status, self._tester.get_total(), self._tester.get_failed())
+            self.write_notification_failures()
             jump()
             self._db.end_session()
             self._report.valid(True)
@@ -105,6 +106,23 @@ class Monitor(Staging):
         self._tester.wait_tests()
         return self._tester.build_report()
 
+    def write_notification_failures(self):
+        if not hasattr(self._tester, "get_notification_failures"):
+            return
+        failures = self._tester.get_notification_failures()
+        if len(failures) > 0:
+            self._report.write_notification_failures(failures)
+
+    def validate(self):
+        failures = []
+        if hasattr(self._report, "get_notification_failures"):
+            failures = self._report.get_notification_failures()
+        for failure in failures:
+            console.error(failure.get("message", str(failure)))
+        if len(failures) > 0:
+            exit(1)
+        super().validate()
+
     def build(self, terraform):
         with self.timer("monitor.build"):
             self._db = MonitorDB(self._config, enable=False)
@@ -208,4 +226,10 @@ class Monitor(Staging):
             revision=self._report.get_alya_revision(),
             db=self._db,
             directories=self._test_directories,
-            test_list=self._test_list)
+            test_list=self._test_list,
+            notify_failures=(
+                getattr(self, "_gitlab", False)
+                and self._config.get("benchmarks", {}).get("notifyFailures", False)
+            ),
+            notification_sender=self._config.get("benchmarks", {}).get("failureSender"),
+            notification_copy=self._config.get("benchmarks", {}).get("failureCopy"))

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/reports/report.py

@@ -61,6 +61,8 @@ class Report:
             self._valid = self._valid and self._data["builds"]["status"]
         if "tests" in self._data:
             self._valid = self._valid and self._data["tests"]["status"]
+        if len(self._data.get("notificationFailures", [])) > 0:
+            self._valid = False
         self._data["valid"] = self._valid
         self.save()
 
@@ -107,3 +109,10 @@ class Report:
 
     def get_valid(self):
         return self._valid
+
+    def get_notification_failures(self):
+        return self._data.get("notificationFailures", [])
+
+    def write_notification_failures(self, failures):
+        self._data["notificationFailures"] = failures
+        self.valid(len(failures) == 0)

squidient-2.6.0/src/squidient/testing/benchmarkfailurenotifier.py

@@ -0,0 +1,146 @@
+#########################################################################
+#                                                                       #
+#  This file is part of squidient.                                      #
+#                                                                       #
+#  squidient is free software: you can redistribute it and/or modify    #
+#  it under the terms of the GNU General Public License as published by #
+#  the Free Software Foundation, either version 3 of the License, or    #
+#  (at your option) any later version.                                  #
+#                                                                       #
+#  squidient is distributed in the hope that it will be useful,         #
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of       #
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the        #
+#  GNU General Public License for more details.                         #
+#                                                                       #
+#  You should have received a copy of the GNU General Public License    #
+#  along with squidient. If not, see <https://www.gnu.org/licenses/>.   #
+#                                                                       #
+#########################################################################
+
+from email.message import EmailMessage
+from ..definitions import benchmark_failure_log_files
+import os
+import shutil
+import smtplib
+import subprocess
+
+
+class BenchmarkFailureNotificationError(Exception):
+
+    def __init__(self, recipients, subject, body, error):
+        self.recipients = recipients
+        self.subject = subject
+        self.body = body
+        self.error = str(error)
+        super().__init__(self.message())
+
+    def message(self):
+        return (
+            "Benchmark failure email could not be sent.\n"
+            "Recipients: " + ", ".join(self.recipients) + "\n"
+            "Subject: " + self.subject + "\n"
+            "Transport error: " + self.error + "\n"
+            "\n"
+            "Mail body:\n"
+            + self.body
+        )
+
+    def as_dict(self):
+        return {
+            "recipients": self.recipients,
+            "subject": self.subject,
+            "body": self.body,
+            "error": self.error,
+            "message": self.message(),
+        }
+
+
+class BenchmarkFailureNotifier:
+
+    def __init__(
+            self,
+            report_dir,
+            from_address=None,
+            copy_recipients=None,
+            mail_sender=None):
+        self._report_dir = report_dir
+        self._from_address = from_address
+        if copy_recipients is None:
+            copy_recipients = []
+        self._copy_recipients = copy_recipients
+        self._mail_sender = mail_sender or self.send_mail
+
+    def notify(self, report):
+        recipients = self.recipients(report.get("authors", []))
+        if len(recipients) == 0:
+            return
+        subject = "[squidient] Benchmark failed: " + report.get("name", "unknown")
+        body = self.mail_body(report)
+        try:
+            self._mail_sender(recipients, subject, body)
+        except Exception as error:
+            raise BenchmarkFailureNotificationError(recipients, subject, body, error)
+
+    def recipients(self, authors):
+        recipients = []
+        for author in authors:
+            if isinstance(author, str):
+                candidate = author
+            elif isinstance(author, dict):
+                candidate = author.get("email", "")
+            else:
+                candidate = ""
+            if "@" in candidate and candidate not in recipients:
+                recipients.append(candidate)
+        for copy in self._copy_recipients:
+            if "@" in copy and copy not in recipients:
+                recipients.append(copy)
+        return recipients
+
+    def mail_body(self, report):
+        lines = [
+            "Benchmark: " + report.get("name", "unknown"),
+            "Error: " + report.get("error", "unknown"),
+            "",
+        ]
+        for instance in report.get("instances", []):
+            if instance.get("status", True):
+                continue
+            instance_dir = os.path.join(
+                self._report_dir,
+                report.get("name", ""),
+                str(instance.get("build", "")),
+                str(instance.get("mpi", "")) + "_" + str(instance.get("openmp", "")),
+            )
+            lines.extend([
+                "Instance: " + instance.get("name", "unknown"),
+                "Build: " + str(instance.get("build", "")),
+                "MPI/OpenMP: " + str(instance.get("mpi", "")) + "/" + str(instance.get("openmp", "")),
+                "Job ID: " + str(instance.get("jobId", "")),
+                "Error: " + str(instance.get("error", "")),
+                "",
+            ])
+            for filename in benchmark_failure_log_files:
+                lines.append("----- " + filename + " -----")
+                try:
+                    with open(os.path.join(instance_dir, filename), "r", errors="replace") as log:
+                        content = log.read()
+                except Exception:
+                    content = "<missing>"
+                lines.append(content)
+                lines.append("")
+        return "\n".join(lines)
+
+    def send_mail(self, recipients, subject, body):
+        message = EmailMessage()
+        if self._from_address is not None:
+            message["From"] = self._from_address
+        message["To"] = ", ".join(recipients)
+        message["Subject"] = subject
+        message.set_content(body)
+        sendmail = shutil.which("sendmail")
+        if sendmail is not None:
+            subprocess.run([sendmail, "-t"], input=message.as_string(), text=True, check=True)
+            return
+        with smtplib.SMTP("localhost") as smtp:
+            smtp.send_message(message)

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/testing/benchmarktester.py

@@ -23,6 +23,7 @@ from ..definitions import *
 from ..git import *
 from .tester import Tester
 from .benchmarktest import BenchmarkTest
+from .benchmarkfailurenotifier import BenchmarkFailureNotificationError, BenchmarkFailureNotifier
 import os
 import shutil
 
@@ -33,7 +34,7 @@ logger = logging.getLogger(logging_context)
 class BenchmarkTester(Tester):
 
     def __init__(self, systemManager, hash, job, ssh, config, benchmarks, revision, db, directories=["."],
-                 test_list=[]):
+                 test_list=[], notify_failures=False, notification_sender=None, notification_copy=None, mail_sender=None):
 
         super().__init__(systemManager, hash, job, ssh, config, revision, directories, test_list)
         self._merge = False
@@ -45,6 +46,14 @@ class BenchmarkTester(Tester):
         self._report_dir = benchmark_test_report_dir
         self._source_dir = self._remote_dir + "/.repositories"
         self._db = db
+        self._notify_failures = notify_failures
+        self._notification_failures = []
+        self._failure_notifier = BenchmarkFailureNotifier(
+            self._report_dir,
+            from_address=notification_sender,
+            copy_recipients=notification_copy,
+            mail_sender=mail_sender,
+        )
 
     def get_benchmark(self, benchmark, fake=False):
         b = self._benchmarks[benchmark]
@@ -155,6 +164,7 @@ class BenchmarkTester(Tester):
             report[t.get_name()] = t.get_report()
             if not report[t.get_name()]["status"]:
                 self._failed += 1
+                self.notify_failure(report[t.get_name()])
             self._status = report[t.get_name()]["status"] and self._status
             if t.get_relaunch():
                 self._relaunch = True
@@ -163,3 +173,17 @@ class BenchmarkTester(Tester):
         json2js("benchmark_tests", f)
         print_line()
         return self._status
+
+    def notify_failure(self, report):
+        if not getattr(self, "_notify_failures", False):
+            return
+        try:
+            self._failure_notifier.notify(report)
+        except BenchmarkFailureNotificationError as error:
+            self._notification_failures.append(error.as_dict())
+            console.warning(error.message())
+        except Exception as error:
+            console.warning("Cannot send benchmark failure email: " + str(error))
+
+    def get_notification_failures(self):
+        return self._notification_failures

{squidient-2.5.2 → squidient-2.6.0}/src/squidient/testing/instance.py

@@ -29,6 +29,9 @@ from ..systems.queuetester import *
 
 logger = logging.getLogger(logging_context)
 
+SBATCH_OUTPUT_FILE = "sbatch.out"
+SBATCH_ERROR_FILE = "sbatch.err"
+
 
 def file_contains(path, text):
     try:
@@ -99,6 +102,7 @@ class Instance:
         self._job_terminated = False
         self._job_launched = False
         self._job_id = 0
+        self._job_submission = {}
         self._relaunch = False
         self._retry = 0
         self._build_report = True
@@ -395,6 +399,13 @@ class Instance:
                 self._job_id = self._job.get_job_id(system=self._system,
                                                     platform=self._platform,
                                                     id=self._id + "_" + str(self._retry))
+                if hasattr(self._job, "get_job_submission"):
+                    self._job_submission = self._job.get_job_submission(job_id=self._job_id,
+                                                                        system=self._system,
+                                                                        platform=self._platform)
+                else:
+                    self._job_submission = {}
+                self.write_sbatch_outputs()
             else:
                 return "Scheduled"
         if self._job_id == -1:
@@ -442,6 +453,8 @@ class Instance:
         self._report["relaunched"] = self._retry > 0
         self._report["retries"] = self._retry
         self._report["jobId"] = self._job_id
+        self._report["sbatchOutput"] = SBATCH_OUTPUT_FILE
+        self._report["sbatchError"] = SBATCH_ERROR_FILE
 
         if self._remove_env:
             self._report["env"] = {}
@@ -495,6 +508,15 @@ class Instance:
             self.backup_outputs()
         self._build_report = False
 
+    def write_sbatch_outputs(self):
+        os.makedirs(self._report_path, exist_ok=True)
+        stdout = self._job_submission.get("stdout", "")
+        stderr = self._job_submission.get("stderr", "")
+        with open(self._report_path + "/" + SBATCH_OUTPUT_FILE, "w") as output:
+            output.write(stdout)
+        with open(self._report_path + "/" + SBATCH_ERROR_FILE, "w") as error:
+            error.write(stderr)
+
     def build_performance(self):
 
         self._report["performance"] = {}
@@ -610,6 +632,8 @@ class Instance:
         open(self._report_path + "/alya.err", 'a+').close()
         open(self._report_path + "/post.out", 'a+').close()
         open(self._report_path + "/post.err", 'a+').close()
+        open(self._report_path + "/" + SBATCH_OUTPUT_FILE, 'a+').close()
+        open(self._report_path + "/" + SBATCH_ERROR_FILE, 'a+').close()
         open(self._report_path + "/time.out", 'a+').close()
 
     def backup_outputs(self):
@@ -621,6 +645,8 @@ class Instance:
             "alya.err",
             "post.out",
             "post.err",
+            SBATCH_OUTPUT_FILE,
+            SBATCH_ERROR_FILE,
             "time.out",
         ]:
             source = os.path.join(self._report_path, filename)