squidient 2.5.0__tar.gz → 2.5.2__tar.gz

{squidient-2.5.0 → squidient-2.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 2.5.0
+Version: 2.5.2
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-2.5.0 → squidient-2.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "2.5.0"
+version = "2.5.2"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/builds/builder.py

@@ -18,6 +18,8 @@
 #########################################################################
 import sys
 import shutil
+from shlex import quote as shell_quote
+from urllib.parse import quote as url_quote
 
 from ..utils.utils import *
 from ..terraform.terraform import *
@@ -286,12 +288,19 @@ class Builder:
                 self.cd_push(self._config["alamak"]["path"])
                 critical_command("./alamak.sh --clean")
                 if self._config["alamak"]["revision"] is not None:
-                    ref = "--revision " + self._config["alamak"]["revision"]
+                    ref = "--revision " + shell_quote(self._config["alamak"]["revision"])
                 else:
-                    ref = "--branch " + self._config["alamak"]["branch"]
+                    ref = "--branch " + shell_quote(self._config["alamak"]["branch"])
                 alamak_command = ""
                 if user != "" and token != "":
-                    alamak_command = "./alamak.sh --https " + ref + " --user " + user + " --token " + token
+                    alamak_command = (
+                        "./alamak.sh --https "
+                        + ref
+                        + " --user "
+                        + shell_quote(user)
+                        + " --token "
+                        + shell_quote(url_quote(token, safe=""))
+                    )
                 else:
                     alamak_command = "./alamak.sh --ssh " + ref
                 try:

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/builds/executablebuild.py

@@ -170,19 +170,57 @@ class ExecutableBuild(Build):
         options.options("$" + cmake_link_dir, spath + remote_cc_dir + "/" + self._build_id, (spath + "/" + alya_dir).replace("//", "/"), mpirun_options)
         self._options = options
 
+    def _resolve_slurm_resources(self):
+        cpus_per_task = self._sbatch.get("--cpus-per-task", self._sys.get_cpus_per_task())
+        if "--ntasks-per-node" in self._sbatch:
+            return {
+                "--cpus-per-task": cpus_per_task,
+                "--ntasks-per-node": self._sbatch["--ntasks-per-node"],
+            }
+
+        tasks_per_node = self._sys.get_tasks_per_node()
+        try:
+            node_cpus = int(self._sys.get_cpus_per_task()) * int(self._sys.get_tasks_per_node())
+            requested_cpus_per_task = int(cpus_per_task)
+            requested_tasks = max(1, int(self._threads))
+        except (TypeError, ValueError):
+            return {
+                "--cpus-per-task": cpus_per_task,
+                "--ntasks-per-node": tasks_per_node,
+            }
+        if requested_cpus_per_task <= 0:
+            return {
+                "--cpus-per-task": cpus_per_task,
+                "--ntasks-per-node": tasks_per_node,
+            }
+
+        tasks_per_node = min(
+            int(tasks_per_node),
+            requested_tasks,
+            max(1, node_cpus // requested_cpus_per_task),
+        )
+        return {
+            "--cpus-per-task": cpus_per_task,
+            "--ntasks-per-node": tasks_per_node,
+        }
+
     def generate_job(self):
         self.set_queue()
         spath = self._server_path + "/"
+        slurm_resources = self._resolve_slurm_resources()
         script = open(self._report_dir + "/JOB.SB", 'w')
         script.write("#!/bin/bash\n")
         script.write("#SBATCH -D " + spath + self._build_path + "\n")
         script.write("#SBATCH --output=" + build_out_file + "\n")
         script.write("#SBATCH --error=" + build_err_file + "\n")
         script.write("#SBATCH --job-name=" + jtitle + self._hash + "\n")
-        script.write("#SBATCH --ntasks-per-node=" + str(self._sys.get_tasks_per_node()) + "\n")
+        script.write("#SBATCH --cpus-per-task=" + str(slurm_resources["--cpus-per-task"]) + "\n")
+        script.write("#SBATCH --ntasks-per-node=" + str(slurm_resources["--ntasks-per-node"]) + "\n")
         script.write("#SBATCH --ntasks=" + str(self._threads)+"\n")
         script.write("#SBATCH --time=" + self._timeout + "\n")
         for parameter in self._sbatch:
+            if parameter in slurm_resources:
+                continue
             if str(self._sbatch[parameter]) == "":
                 line = "#SBATCH " + parameter + "\n"
             else:

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/compare/methods/diff.py

@@ -33,6 +33,7 @@ class Diff(ScalarTolerance):
 
     def read(self):
         self._total = 0
+        return self._status
 
     def compare_pair(self, vref : str, vtest : str) -> bool:
         pass

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/configure.py

@@ -204,6 +204,7 @@ def set_cc_ref(com=""):
                                                 "\t0a604c815ead8bb5ebc5fd944c39ea9c31dbf694\n" +
                                                 "\torigin/master\n", [r"\w{8}", r"\w{40}", "origin/master"],
                                                 "This does not look like a git commit", "origin/master", com)
+    save_json(config, config_file)
 
 
 def set_builds(com=""):
@@ -490,7 +491,7 @@ def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system
         config_project = open_json(project_file)
         if project not in config_project:
             console.error("Project " + project + " not found")
-            exit(1)
+            raise RuntimeError("Project " + project + " not found")
         config_project = config_project[project]
         for platform in config_project["systems"][system]["platforms"]:
             if platform in config["systems"][system]["platforms"]:
@@ -518,7 +519,7 @@ def set_mr(type):
             config = open_json(config_file)
             config["db"]["user"] = "alya_user"
             config["db"]["database"] = database_name["test"]
-            config["benchmarks"]["tests"] = ["elbow", "sphere-16M"]
+            config["benchmarks"]["tests"] = ["elbow", "sphere-16M", "bolund-32M"]
             save_json(config, config_file)
 
 

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/data/builds/predefined.json

@@ -60,7 +60,8 @@
             "cmake-acc-nvhpc-24.11-i4-gpu-ndimepar-vecsize2048k"
         ],
         "mr": [
-            "cmake-gpp-gnu-13.2.0-i4-ndimepar"
+            "cmake-gpp-gnu-13.2.0-i4-ndimepar",
+            "cmake-acc-nvhpc-24.11-i4-gpu-ndimepar-vecsize2048k"
         ]
     }
 }

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/git.py

@@ -22,6 +22,7 @@
 # -*- coding: utf-8 -*-
 import os.path
 import shutil
+from urllib.parse import quote
 
 from .utils.utils import *
 from .utils.message import console
@@ -58,7 +59,7 @@ class Git:
                 if self._password == "":
                     self._password = getpass.getpass(prompt="Enter your password for the HTTPS repository \n" + self._git_https + ":")
                 self._git_https_user_password = self._git_https.replace("https://", "https://" + self._user + ":" +
-                                                                        percent_encoding(self._password) + "@")
+                                                                        quote(self._password, safe="") + "@")
         else:
             self._git_https_user_password = self._git_https
 
@@ -128,7 +129,7 @@ class Git:
             else:
                 c = command('git remote set-url origin ' + self._git_https).success
             if password != "":
-                self.replace_git_config_text(percent_encoding(password), "PASSWORD")
+                self.replace_git_config_text(quote(password, safe=""), "PASSWORD")
         self.cd_pop()
         return c
 
@@ -138,7 +139,7 @@ class Git:
             self._original_remote = command('git remote get-url origin').stdout.strip()
             c = command('git remote set-url origin ' + remote, silent=True).success
             if password != "":
-                self.replace_git_config_text("PASSWORD", percent_encoding(password))
+                self.replace_git_config_text("PASSWORD", quote(password, safe=""))
         self.cd_pop()
         return c
 
@@ -213,6 +214,7 @@ class Git:
             command("git clean -x -f")
             if not command("git checkout " + self._branch, log_output=True).success:
                 logger.error("Cannot checkout branch " + self._branch)
+                self.cd_pop()
                 return False
             command("git reset --hard HEAD")
             command("git clean -x -f")

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/reports/report.py

@@ -48,6 +48,7 @@ class Report:
     def open(self):
         try:
             self._data = open_critical_json(self._report)
+            self._valid = self._data.get("valid", True)
         except:
             self.init()
 

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/squidient.py

@@ -183,7 +183,7 @@ def _run_cli(argv: Sequence[str]) -> int:
 
             else:
                 help()
-                exit(1)
+                return 1
 
         return 0
 

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/squidientmonitor.py

@@ -107,7 +107,7 @@ def _run_cli(argv: Sequence[str]) -> int:
 
         else:
             help()
-            exit(1)
+            return 1
 
         return 0
 

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/testing/instance.py

@@ -140,13 +140,47 @@ class Instance:
         return int(self._sys.get_test_max_tasks())<int(self._mpi)*int(max(1,int(self._openmp)))
 
     def set_sbatch(self):
-
-        self._sbatch["--cpus-per-task"] = self._sys.get_cpus_per_task()
-        self._sbatch["--ntasks-per-node"] = self._sys.get_tasks_per_node()
-        if "sbatch" in self._execution:
-            for parameter in self._execution["sbatch"]:
-                self._sbatch[parameter] = self._execution["sbatch"][parameter]
-                self._mergeable = False
+        execution_sbatch = self._execution.get("sbatch", {})
+        self._sbatch.update(self._resolve_slurm_resources(execution_sbatch))
+
+        for parameter in execution_sbatch:
+            if parameter not in self._sbatch:
+                self._sbatch[parameter] = execution_sbatch[parameter]
+            self._mergeable = False
+
+    def _resolve_slurm_resources(self, execution_sbatch):
+        cpus_per_task = execution_sbatch.get("--cpus-per-task", self._sys.get_cpus_per_task())
+        if "--ntasks-per-node" in execution_sbatch:
+            return {
+                "--cpus-per-task": cpus_per_task,
+                "--ntasks-per-node": execution_sbatch["--ntasks-per-node"],
+            }
+
+        tasks_per_node = self._sys.get_tasks_per_node()
+        try:
+            node_cpus = int(self._sys.get_cpus_per_task()) * int(self._sys.get_tasks_per_node())
+            requested_cpus_per_task = int(cpus_per_task)
+            requested_tasks = max(1, int(self.get_tasks()))
+        except (TypeError, ValueError):
+            return {
+                "--cpus-per-task": cpus_per_task,
+                "--ntasks-per-node": tasks_per_node,
+            }
+        if requested_cpus_per_task <= 0:
+            return {
+                "--cpus-per-task": cpus_per_task,
+                "--ntasks-per-node": tasks_per_node,
+            }
+
+        tasks_per_node = min(
+            int(tasks_per_node),
+            requested_tasks,
+            max(1, node_cpus // requested_cpus_per_task),
+        )
+        return {
+            "--cpus-per-task": cpus_per_task,
+            "--ntasks-per-node": tasks_per_node,
+        }
 
     def set_bind_and_map(self):
         if "commands" in self._execution:

{squidient-2.5.0 → squidient-2.5.2}/src/squidient/utils/utils.py

@@ -23,9 +23,6 @@
 from .log import *
 from datetime import datetime
 
-import smtplib
-from email.mime.multipart import MIMEMultipart
-from email.mime.text import MIMEText
 from shutil import copyfile
 
 logger = logging.getLogger(logging_context)
@@ -36,6 +33,7 @@ def exist_json(name):
     """
        Test if Json file exist
     """
+    name = os.fspath(name)
     return os.path.isfile(name)
 
 
@@ -43,6 +41,7 @@ def open_critical_json(name):
     """
         Open a Json file.
     """
+    name = os.fspath(name)
     try:
         with open(name) as json_data:
             d = json.load(json_data)
@@ -64,6 +63,7 @@ def open_json(name):
     """
         Open a Json file.
     """
+    name = os.fspath(name)
     try:
         with open(name) as json_data:
             d = json.load(json_data)
@@ -85,10 +85,11 @@ def save_json(d, name):
     """
         Save a json file
     """
+    name = os.fspath(name)
     if(type(d) != type({})):
         logger.error("Date type is not a dict")
         raise TypeError("First parameter data type expected: dict " +
-                        "but find: "+type(d))
+                        "but find: " + str(type(d)))
     try:
         with open(name, 'w') as fp:
             json.dump(d, fp, indent=4, sort_keys=True)
@@ -292,25 +293,6 @@ def write_text_file(file, data):
     f.close()
 
 
-def send_email(html, fr, to, tag, copy=None):
-    # Create message container - the correct MIME type is multipart/alternative
-    msg = MIMEMultipart('alternative')
-    if tag == "":
-        msg['Subject'] = "[AlyaTestSuite] Report Test Email"
-    else:
-        msg['Subject'] = "[AlyaTestSuite][" + tag + "] Report Test Email"
-    msg['From'] = fr
-    to_list = [to, 'damien.dosimont@bsc.es']
-    msg['To'] = ", ".join(to_list)
-    if copy is not None:
-        msg['CC'] = copy
-    part2 = MIMEText(html, 'html', "utf-8")
-    msg.attach(part2)
-    s = smtplib.SMTP_SSL('mail.bsc.es', port=465)
-    s.sendmail(fr, to_list, msg.as_string())
-    s.quit()
-
-
 def copy_json(source, destination):
     """
        Copy a Json file
@@ -334,28 +316,6 @@ def is_option(value, short, long=""):
     return value == short or (long != "" and value == long)
 
 
-def percent_encoding(str):
-    str2 = str.replace('!', '%21')
-    str2 = str2.replace('#', '%23')
-    str2 = str2.replace('$', '%24')
-    str2 = str2.replace('&', '%26')
-    str2 = str2.replace("'", '%27')
-    str2 = str2.replace('(', '%28')
-    str2 = str2.replace(')', '%29')
-    str2 = str2.replace('*', '%2A')
-    str2 = str2.replace('+', '%2B')
-    str2 = str2.replace(',', '%2C')
-    str2 = str2.replace('/', '%2F')
-    str2 = str2.replace(':', '%3A')
-    str2 = str2.replace(';', '%3B')
-    str2 = str2.replace('=', '%3D')
-    str2 = str2.replace('?', '%3F')
-    str2 = str2.replace('@', '%40')
-    str2 = str2.replace('[', '%5B')
-    str2 = str2.replace(']', '%5D')
-    return str2
-
-
 def duration2str(start, end):
     duration = int(end - start)
     return secondstotime(duration)