squidient 2.7.0__tar.gz → 3.0.0__tar.gz

{squidient-2.7.0 → squidient-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 2.7.0
+Version: 3.0.0
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-2.7.0 → squidient-3.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "2.7.0"
+version = "3.0.0"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]
@@ -29,8 +29,8 @@ Homepage = "https://alya.gitlab.bsc.es/alya/ci-cd/squidient"
 
 [project.scripts]
 squidient = "squidient.cli:main"
-sqreport = "squidient.sqreport.squidientcasereport:main"
-sqdetails = "squidient.sqreport.squidientdetailedreport:main"
+sqcase = "squidient.reports.cases.case:main"
+sqdetails = "squidient.reports.cases.detailed:main"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/squidient"]

squidient-2.7.0/src/squidient/builds/benchmarkbuild.py → squidient-3.0.0/src/squidient/builds/benchmark_build_execution.py

@@ -21,19 +21,17 @@
 
 # -*- coding: utf-8 -*-
 
-from .executablebuild import *
-from ..utils.log import *
+from .build_execution import BuildExecution
+from ..definitions import max_jobs_monitoring
 
-logger = logging.getLogger(logging_context)
 
+class BenchmarkBuildExecution(BuildExecution):
 
-class BenchmarkBuild(ExecutableBuild):
-
-    def __init__(self, systemManager, hash, build_id, job, ssh, config, report_directory, db):
-        super().__init__(systemManager, hash, build_id, job, ssh, config, report_directory)
+    def __init__(self, system_manager, run_id, build_id, job, ssh, config, report_directory, db):
+        super().__init__(system_manager, run_id, build_id, job, ssh, config, report_directory)
         self._queue_type = "benchmarks"
         self._db = db
-        self._max_jobs = max_jobs_monitor
+        self._max_jobs = max_jobs_monitoring
 
     def add_to_db(self):
         db_build = {}
@@ -46,7 +44,7 @@ class BenchmarkBuild(ExecutableBuild):
         db_build["codeCoverage"] = self._code_coverage
         db_build["system"] = self._system
         db_build["platform"] = self._platform
-        db_build["type"] = self._type
+        db_build["type"] = self._build_type
         db_build["ctest"] = self._ctest
         db_configuration = self._options.get_configuration()
         self._db.add_build(name=self._build_id, compiler=self._compiler, version=self._version,
@@ -55,4 +53,3 @@ class BenchmarkBuild(ExecutableBuild):
     def launch_job(self):
         self.add_to_db()
         return super().launch_job()
-

squidient-2.7.0/src/squidient/builds/benchmarkbuilder.py → squidient-3.0.0/src/squidient/builds/benchmark_build_orchestrator.py

@@ -17,21 +17,23 @@
 #                                                                       #
 #########################################################################
 
-
-
-from .builder import *
-from .benchmarkbuild import BenchmarkBuild
+from .build_orchestrator import BuildOrchestrator
+from .benchmark_build_execution import BenchmarkBuildExecution
 import shutil
 
+from ..definitions import benchmark_build_report_dir
+from ..utils.message import console
+from ..utils.utils import json2js, save_json
+
 
-class BenchmarkBuilder(Builder):
+class BenchmarkBuildOrchestrator(BuildOrchestrator):
 
-    def __init__(self, systemManager, hash, job, ssh, config, db, password=""):
-        super().__init__(systemManager, hash, job, ssh, config, password)
+    def __init__(self, system_manager, run_id, job, ssh, config, db, password=""):
+        super().__init__(system_manager, run_id, job, ssh, config, password)
         self._db = db
 
-    def set_build_list(self):
-        self._build_list = self._config["benchmarks"]["builds"]
+    def set_requested_build_ids(self):
+        self._requested_build_ids = self._config["benchmarks"]["builds"]
 
     def set_report_dir(self):
         self._report_dir = benchmark_build_report_dir
@@ -40,10 +42,10 @@ class BenchmarkBuilder(Builder):
         shutil.rmtree(benchmark_build_report_dir, ignore_errors=True)
 
     def create_build(self, build):
-        return BenchmarkBuild(systemManager=self._systemManager,
-                              hash=self._hash,
+        return BenchmarkBuildExecution(system_manager=self._system_manager,
+                              run_id=self._run_id,
                               build_id=build,
-                              job=self._job,
+                              job=self._job_manager,
                               ssh=self._ssh,
                               config=self._config,
                               report_directory=self._report_dir,
@@ -53,10 +55,10 @@ class BenchmarkBuilder(Builder):
         console.section("Building benchmark build report")
         report = {}
         console.step("Building per-build reports")
-        for b in self._builds:
-            console.group("Build " + b, level=2)
-            self._builds[b].build_report()
-            report[b] = self._builds[b].get_report()
-        f = self._report_dir+"/benchmark_builds.json"
-        save_json(report, f)
-        json2js("benchmark_builds", f)
+        for build_id in self._launched_builds:
+            console.group("Build " + build_id, level=2)
+            self._launched_builds[build_id].build_report()
+            report[build_id] = self._launched_builds[build_id].get_report()
+        report_file = self._report_dir+"/benchmark_builds.json"
+        save_json(report, report_file)
+        json2js("benchmark_builds", report_file)

squidient-2.7.0/src/squidient/builds/build.py → squidient-3.0.0/src/squidient/builds/build_definition.py

@@ -23,17 +23,14 @@
 
 from ..utils.utils import open_critical_json
 from ..definitions import build_file, default_file
-from ..utils.log import *
 
-logger = logging.getLogger(logging_context)
-
-class Build:
+class BuildDefinition:
 
     def __init__(self, build_id):
 
         self._build_id = build_id
 
-        #These variables propagates
+        #These variables propagate
 
         self._pre_modules = []
         self._modules = []
@@ -47,13 +44,13 @@ class Build:
         self._exclusive_tests = ""
         self._flex = ""
         self._file = ""
-        self._job = ""
+        self._job_config = ""
         self._platform = ""
         self._lock = ""
         self._runnable = ""
         self._timeout = ""
         self._timeoffset = ""
-        self._type = ""
+        self._build_type = ""
         self._ctest = ""
         self._threads = ""
         self._version = ""
@@ -67,12 +64,12 @@ class Build:
 
         #Configure build
         self._builds = open_critical_json(build_file)
-        self.fill(build_id)
+        self.load_build_definition(build_id)
 
         # configure default
         try:
             self._alias_default = open_critical_json(default_file)[self._system][self._platform]
-        except:
+        except KeyError:
             self._alias_default = None
 
         if self._timeout == "":
@@ -86,7 +83,7 @@ class Build:
         if type(self._exclusive_tests) != bool:
             self._exclusive_tests = False
 
-    def fill(self, build_id):
+    def load_build_definition(self, build_id):
         build = self._builds[build_id]
         if len(self._modules) == 0 and "modules" in build:
             self._modules = build["modules"]
@@ -116,8 +113,8 @@ class Build:
             self._file = build["file"]
         if self._flex == "" and "flex" in build:
             self._flex = build["flex"]
-        if self._job == "" and "job" in build:
-            self._job = build["job"]
+        if self._job_config == "" and "job" in build:
+            self._job_config = build["job"]
         if self._system == "" and "system" in build:
             self._system = build["system"]
         if self._platform == "" and "platform" in build:
@@ -128,8 +125,8 @@ class Build:
             self._runnable = build["runnable"]
         if self._threads == "" and "threads" in build:
             self._threads = build["threads"]
-        if self._type == "" and "type" in build:
-            self._type = build["type"]
+        if self._build_type == "" and "type" in build:
+            self._build_type = build["type"]
         if self._ctest == "" and "ctest" in build:
             self._ctest = build["ctest"]
         if self._timeout == "" and "timeout" in build:
@@ -149,7 +146,7 @@ class Build:
             else:
                 self._installable = False
         if "from" in build:
-            self.fill(build["from"])
+            self.load_build_definition(build["from"])
 
 
     def get_tags(self):

squidient-2.7.0/src/squidient/builds/executablebuild.py → squidient-3.0.0/src/squidient/builds/build_execution.py

@@ -18,15 +18,22 @@
 #########################################################################
 
 
-from .build import Build
+from .build_definition import BuildDefinition
 # -*- coding: utf-8 -*-
 import os
 
-from ..compilation.cmakeoptions import *
-from ..deployment.modulegenerator import *
-from ..systems.queuetester import *
-
-logger = logging.getLogger(logging_context)
+from ..compilation.cmake_options import CMakeOptions
+from ..definitions import (
+    alya_dir,
+    cmake_link_dir,
+    max_jobs_staging,
+    remote_bin_dir,
+    remote_build_dir,
+    remote_cc_dir,
+)
+from ..deployment.module_generator import ModuleGenerator
+from ..systems.queue_tester import QueueTester
+from ..utils.utils import duration2str, jtitle, read_text_file, save_json
 
 time_file = "time.out"
 env_file = "env.out"
@@ -41,9 +48,9 @@ clean_string = "find . -name 'CMakeCache.txt' -print0 -o -name '*.cmake' -print0
                "-o -name '*.build' -print0 | xargs -0 rm -fr && "
 
 
-class ExecutableBuild(Build):
+class BuildExecution(BuildDefinition):
 
-    def __init__(self, systemManager, hash, build_id, job, ssh, config, report_directory):
+    def __init__(self, system_manager, run_id, build_id, job, ssh, config, report_directory):
         super().__init__(build_id)
         self._queue_type = "builds"
 
@@ -52,36 +59,36 @@ class ExecutableBuild(Build):
         os.makedirs(self._report_dir, exist_ok=True)
 
         #class configuration
-        self._hash = hash
-        self._systemManager = systemManager
-        self._jobm = job
+        self._run_id = run_id
+        self._system_manager = system_manager
+        self._job_manager = job
         self._ssh = ssh
         self._config = config
         self._build_path = remote_build_dir + "/" + build_id
 
         #System/platform
-        self._sys = systemManager.get_platform(self._system, self._platform)
+        self._system_platform = system_manager.get_platform(self._system, self._platform)
 
         #Merge
-        self._merge = self._sys.get_merge()
+        self._merge = self._system_platform.get_merge()
 
         #Server, installation and module directories
-        install_dir = self._sys.get_application_path()
-        module_dir = self._sys.get_module_path()
+        install_dir = self._system_platform.get_application_path()
+        module_dir = self._system_platform.get_module_path()
         self._install_path = install_dir + "/" + self._alias
         self._install = self._config["installation"] and self._installable
         if self._install:
             self._queue_type = "install"
         self._module_path = module_dir + "/" + self._alias
         self._module_dir = module_dir
-        self._server_path = self._sys.get_path()
+        self._server_path = self._system_platform.get_path()
 
-        self._sjobexitmod = self._sys.get_sjobexitmod()
+        self._sjobexitmod = self._system_platform.get_sjobexitmod()
 
         self._queue = ""
         self._slurm_flag = ""
         self._exclusive = False
-        self._account = self._sys.get_account()
+        self._account = self._system_platform.get_account()
 
         self._options = None
         self._job_id = ""
@@ -91,96 +98,33 @@ class ExecutableBuild(Build):
         self._max_jobs = max_jobs_staging
 
     def set_queue(self):
-        queueTester = QueueTester(self._ssh, self._sys, self._hash, self._jobm)
-        self._queue, self._slurm_flag, self._exclusive = queueTester.which_qos(
+        queue_tester = QueueTester(self._ssh, self._system_platform, self._run_id, self._job_manager)
+        self._queue, self._slurm_flag, self._exclusive = queue_tester.which_qos(
             queue_type=self._queue_type,
             cpus=self._threads,
             timeout=self._timeout,
             flex=self._flex)
 
-    def fill(self, build_id):
-        build = self._builds[build_id]
-        if len(self._modules) == 0 and "modules" in build:
-            self._modules = build["modules"]
-        if len(self._env) == 0 and "environment" in build:
-            self._env = build["environment"]
-        if len(self._pre_exec) == 0 and "preExec" in build:
-            self._pre_exec = build["preExec"]
-        if len(self._pre_modules) == 0 and "preModules" in build:
-            self._pre_modules = build["preModules"]
-        if len(self._post_modules) == 0 and "postModules" in build:
-            self._post_modules = build["postModules"]
-        if len(self._tags) == 0 and "tags" in build:
-            self._tags = build["tags"]
-        if len(self._sbatch) == 0 and "sbatch" in build:
-            self._sbatch = build["sbatch"]
-        if self._cc_tool == "" and "cc_tool" in build:
-            self._cc_tool = build["cc_tool"]
-        if self._compiler == "" and "compiler" in build:
-            self._compiler = build["compiler"]
-        if self._code_coverage == "" and "codeCoverage" in build:
-            self._code_coverage = build["codeCoverage"]
-        if self._exclusive_tests == "" and "exclusive" in build:
-            self._exclusive_tests = build["exclusive"]
-        if self._configuration == "" and "configuration" in build:
-            self._configuration = build["configuration"]
-        if self._file == "" and "file" in build:
-            self._file = build["file"]
-        if self._flex == "" and "flex" in build:
-            self._flex = build["flex"]
-        if self._job == "" and "job" in build:
-            self._job = build["job"]
-        if self._system == "" and "system" in build:
-            self._system = build["system"]
-        if self._platform == "" and "platform" in build:
-            self._platform = build["platform"]
-        if self._lock == "" and "lock" in build:
-            self._lock = build["lock"]
-        if self._runnable == "" and "runnable" in build:
-            self._runnable = build["runnable"]
-        if self._threads == "" and "threads" in build:
-            self._threads = build["threads"]
-        if self._ctest == "" and "ctest" in build:
-            self._ctest = build["ctest"]
-        if self._timeout == "" and "timeout" in build:
-            self._timeout = build["timeout"]
-        if self._timeoffset == "" and "timeOffset" in build:
-            self._timeoffset = build["timeOffset"]
-        if self._version == "" and "version" in build:
-            self._version = build["version"]
-        if self._alias == "":
-            if "alias" in build:
-                self._alias = build["alias"]
-            else:
-                self._alias = self._build_id
-        if self._installable == "":
-            if "installable" in build:
-                self._installable = build["installable"]
-            else:
-                self._installable = False
-        if "from" in build:
-            self.fill(build["from"])
-
     def generate_configuration(self):
         spath = self._server_path + "/"
         options = CMakeOptions(self._file, self._configuration)
         if self._link_dir == "":
             self._link_dir = spath + remote_bin_dir + "/" + self._build_id
-        mpirun_options = self._sys.get_mpirun_options(self._modules)
-        options.options("$" + cmake_link_dir, spath + remote_cc_dir + "/" + self._build_id, (spath + "/" + alya_dir).replace("//", "/"), mpirun_options)
+        mpirun_options = self._system_platform.get_mpirun_options(self._modules)
+        options.render_options("$" + cmake_link_dir, spath + remote_cc_dir + "/" + self._build_id, (spath + "/" + alya_dir).replace("//", "/"), mpirun_options)
         self._options = options
 
     def _resolve_slurm_resources(self):
-        cpus_per_task = self._sbatch.get("--cpus-per-task", self._sys.get_cpus_per_task())
+        cpus_per_task = self._sbatch.get("--cpus-per-task", self._system_platform.get_cpus_per_task())
         if "--ntasks-per-node" in self._sbatch:
             return {
                 "--cpus-per-task": cpus_per_task,
                 "--ntasks-per-node": self._sbatch["--ntasks-per-node"],
             }
 
-        tasks_per_node = self._sys.get_tasks_per_node()
+        tasks_per_node = self._system_platform.get_tasks_per_node()
         try:
-            node_cpus = int(self._sys.get_cpus_per_task()) * int(self._sys.get_tasks_per_node())
+            node_cpus = int(self._system_platform.get_cpus_per_task()) * int(self._system_platform.get_tasks_per_node())
             requested_cpus_per_task = int(cpus_per_task)
             requested_tasks = max(1, int(self._threads))
         except (TypeError, ValueError):
@@ -213,7 +157,7 @@ class ExecutableBuild(Build):
         script.write("#SBATCH -D " + spath + self._build_path + "\n")
         script.write("#SBATCH --output=" + build_out_file + "\n")
         script.write("#SBATCH --error=" + build_err_file + "\n")
-        script.write("#SBATCH --job-name=" + jtitle + self._hash + "\n")
+        script.write("#SBATCH --job-name=" + jtitle + self._run_id + "\n")
         script.write("#SBATCH --cpus-per-task=" + str(slurm_resources["--cpus-per-task"]) + "\n")
         script.write("#SBATCH --ntasks-per-node=" + str(slurm_resources["--ntasks-per-node"]) + "\n")
         script.write("#SBATCH --ntasks=" + str(self._threads)+"\n")
@@ -243,8 +187,8 @@ class ExecutableBuild(Build):
         for post_module in self._post_modules:
             script.write(post_module + "\n")
         script.write("\n")
-        for env in self._env:
-            script.write(env + "\n")
+        for environment_command in self._env:
+            script.write(environment_command + "\n")
         script.write("\n")
         rm_string = "rm " + spath + remote_bin_dir + "/alya_" + self._build_id + "\n"
         rm_string += "rm " + spath + remote_bin_dir + "/alya2pos_" + self._build_id + "\n"
@@ -304,8 +248,8 @@ class ExecutableBuild(Build):
             install_string += "rm -fr " + self._install_path + "\n"
             install_string += "make install -j" + str(self._threads) + "\n"
             script.write(install_string)
-            m = ModuleGenerator(self._alias, self._modules, self._install_path, self._module_path)
-            script.write(m.getmodule())
+            module_generator = ModuleGenerator(self._alias, self._modules, self._install_path, self._module_path)
+            script.write(module_generator.generate_module_script())
             if self._alias == self._alias_default:
                 script.write("cp " + self._module_path + " " + self._module_dir + "/default\n")
         configuration = open(self._report_dir + "/config.in", 'w')
@@ -342,7 +286,7 @@ class ExecutableBuild(Build):
         self._ssh.scp_send(system=self._system, local=self._report_dir + "/*", remote=self._build_path + "/", server_path=True, critical=True)
 
     def launch_job(self):
-        self._job_id = self._jobm.launch_job(id=self._build_id, path=self._build_path + "/JOB.SB",
+        self._job_id = self._job_manager.launch_job(internal_job_id=self._build_id, path=self._build_path + "/JOB.SB",
                                              system=self._system, platform=self._platform,
                                              server_path=True, key_lock=self._lock, max_jobs=self._max_jobs)
         return self._job_id
@@ -351,7 +295,7 @@ class ExecutableBuild(Build):
         return self._job_id
 
     def check_job_status(self):
-        return self._jobm.check_job(job_id=self._job_id, system=self._system, platform=self._platform)
+        return self._job_manager.check_job(job_id=self._job_id, system=self._system, platform=self._platform)
 
     def check_build_status(self):
         status = True
@@ -385,17 +329,17 @@ class ExecutableBuild(Build):
                                           server_path=True, system=self._system, platform=self._platform).stdout.strip():
                 return "timeout"
         try:
-            t = read_text_file(self._report_dir + "/" + time_file).split("\n")
-            start = int(t[0])
-            end = int(t[1])
+            time_lines = read_text_file(self._report_dir + "/" + time_file).split("\n")
+            start = int(time_lines[0])
+            end = int(time_lines[1])
             return duration2str(start, end)
-        except:
+        except (OSError, ValueError, IndexError):
             return "timeout"
 
     def find_hostname(self):
         try:
             return read_text_file(self._report_dir + "/" + env_file).strip().replace("HOSTNAME=", "")
-        except:
+        except OSError:
             return "unknown"
 
     def get_report(self):