squidient 1.0.0__py3-none-any.whl

squidient/__init__.py

@@ -0,0 +1 @@
+__all__ = []

squidient/builds/benchmarkbuild.py

@@ -0,0 +1,58 @@
+#########################################################################
+#                                                                       #
+#  This file is part of squidient.                                      #
+#                                                                       #
+#  squidient is free software: you can redistribute it and/or modify    #
+#  it under the terms of the GNU General Public License as published by #
+#  the Free Software Foundation, either version 3 of the License, or    #
+#  (at your option) any later version.                                  #
+#                                                                       #
+#  squidient is distributed in the hope that it will be useful,         #
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of       #
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the        # 
+#  GNU General Public License for more details.                         #
+#                                                                       #
+#  You should have received a copy of the GNU General Public License    #
+#  along with squidient. If not, see <https://www.gnu.org/licenses/>.   #
+#                                                                       #
+#########################################################################
+
+
+
+# -*- coding: utf-8 -*-
+
+from .executablebuild import *
+from ..utils.log import *
+
+logger = logging.getLogger(logging_context)
+
+
+class BenchmarkBuild(ExecutableBuild):
+
+    def __init__(self, systemManager, hash, build_id, job, ssh, config, report_directory, db):
+        super().__init__(systemManager, hash, build_id, job, ssh, config, report_directory)
+        self._queue_type = "benchmarks"
+        self._db = db
+        self._max_jobs = max_jobs_monitor
+
+    def add_to_db(self):
+        db_build = {}
+        db_build["modules"] = self._modules
+        db_build["environment"] = self._env
+        db_build["preExec"] = self._pre_exec
+        db_build["preModules"] = self._pre_modules
+        db_build["postModules"] = self._post_modules
+        db_build["tags"] = self._tags
+        db_build["codeCoverage"] = self._code_coverage
+        db_build["system"] = self._system
+        db_build["platform"] = self._platform
+        db_build["type"] = self._type
+        db_build["ctest"] = self._ctest
+        db_configuration = self._options.get_configuration()
+        self._db.add_build(name=self._build_id, compiler=self._compiler, version=self._version,
+                           system=self._system, platform=self._platform, build=db_build, configuration=db_configuration)
+
+    def launch_job(self):
+        self.add_to_db()
+        return super().launch_job()
+

squidient/builds/benchmarkbuilder.py

@@ -0,0 +1,61 @@
+#########################################################################
+#                                                                       #
+#  This file is part of squidient.                                      #
+#                                                                       #
+#  squidient is free software: you can redistribute it and/or modify    #
+#  it under the terms of the GNU General Public License as published by #
+#  the Free Software Foundation, either version 3 of the License, or    #
+#  (at your option) any later version.                                  #
+#                                                                       #
+#  squidient is distributed in the hope that it will be useful,         #
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of       #
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the        # 
+#  GNU General Public License for more details.                         #
+#                                                                       #
+#  You should have received a copy of the GNU General Public License    #
+#  along with squidient. If not, see <https://www.gnu.org/licenses/>.   #
+#                                                                       #
+#########################################################################
+
+
+
+from .builder import *
+from .benchmarkbuild import BenchmarkBuild
+
+
+class BenchmarkBuilder(Builder):
+
+    def __init__(self, systemManager, hash, job, ssh, config, db, password=""):
+        super().__init__(systemManager, hash, job, ssh, config, password)
+        self._db = db
+
+    def set_build_list(self):
+        self._build_list = self._config["benchmarks"]["builds"]
+
+    def set_report_dir(self):
+        self._report_dir = benchmark_build_report_dir
+
+    def clean_builds(self):
+        command("rm -fr " + benchmark_build_report_dir)
+
+    def create_build(self, build):
+        return BenchmarkBuild(systemManager=self._systemManager,
+                              hash=self._hash,
+                              build_id=build,
+                              job=self._job,
+                              ssh=self._ssh,
+                              config=self._config,
+                              report_directory=self._report_dir,
+                              db=self._db)
+
+    def build_report(self):
+        jump()
+        print("Building monitor build report")
+        print_line()
+        report = {}
+        for b in self._builds:
+            self._builds[b].build_report()
+            report[b] = self._builds[b].get_report()
+        f = self._report_dir+"/benchmark_builds.json"
+        save_json(report, f)
+        json2js("benchmark_builds", f)

squidient/builds/build.py

@@ -0,0 +1,195 @@
+#########################################################################
+#                                                                       #
+#  This file is part of squidient.                                      #
+#                                                                       #
+#  squidient is free software: you can redistribute it and/or modify    #
+#  it under the terms of the GNU General Public License as published by #
+#  the Free Software Foundation, either version 3 of the License, or    #
+#  (at your option) any later version.                                  #
+#                                                                       #
+#  squidient is distributed in the hope that it will be useful,         #
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of       #
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the        #
+#  GNU General Public License for more details.                         #
+#                                                                       #
+#  You should have received a copy of the GNU General Public License    #
+#  along with squidient. If not, see <https://www.gnu.org/licenses/>.   #
+#                                                                       #
+#########################################################################
+
+
+
+# -*- coding: utf-8 -*-
+
+from ..utils.utils import open_critical_json
+from ..definitions import build_file
+from ..utils.log import *
+
+logger = logging.getLogger(logging_context)
+
+class Build:
+
+    def __init__(self, build_id):
+
+        self._build_id = build_id
+
+        #These variables propagates
+
+        self._pre_modules = []
+        self._modules = []
+        self._post_modules = []
+        self._pre_exec = []
+        self._env = []
+        self._tags = []
+        self._compiler = ""
+        self._code_coverage = ""
+        self._configuration = ""
+        self._exclusive_tests = ""
+        self._flex = ""
+        self._file = ""
+        self._job = ""
+        self._platform = ""
+        self._lock = ""
+        self._runnable = ""
+        self._timeout = ""
+        self._timeoffset = ""
+        self._type = ""
+        self._ctest = ""
+        self._threads = ""
+        self._version = ""
+        self._system = ""
+        self._cc_tool = ""
+        self._sbatch = {}
+
+        #these variables don't propagate
+        self._installable = ""
+        self._alias = ""
+
+        #Configure build
+        self._builds = open_critical_json(build_file)
+        self.fill(build_id)
+
+        if self._timeout == "":
+            self._timeout = "02:00:00"
+
+        if self._timeoffset == "":
+            self._timeoffset = "00:00:00"
+
+        if type(self._ctest) != bool:
+            self._ctest = True
+        if type(self._exclusive_tests) != bool:
+            self._exclusive_tests = False
+
+    def fill(self, build_id):
+        build = self._builds[build_id]
+        if len(self._modules) == 0 and "modules" in build:
+            self._modules = build["modules"]
+        if len(self._env) == 0 and "environment" in build:
+            self._env = build["environment"]
+        if len(self._pre_exec) == 0 and "preExec" in build:
+            self._pre_exec = build["preExec"]
+        if len(self._pre_modules) == 0 and "preModules" in build:
+            self._pre_modules = build["preModules"]
+        if len(self._post_modules) == 0 and "postModules" in build:
+            self._post_modules = build["postModules"]
+        if len(self._tags) == 0 and "tags" in build:
+            self._tags = build["tags"]
+        if len(self._sbatch) == 0 and "sbatch" in build:
+            self._sbatch = build["sbatch"]
+        if self._cc_tool == "" and "cc_tool" in build:
+            self._cc_tool = build["cc_tool"]
+        if self._compiler == "" and "compiler" in build:
+            self._compiler = build["compiler"]
+        if self._code_coverage == "" and "codeCoverage" in build:
+            self._code_coverage = build["codeCoverage"]
+        if self._exclusive_tests == "" and "exclusive" in build:
+            self._exclusive_tests = build["exclusive"]
+        if self._configuration == "" and "configuration" in build:
+            self._configuration = build["configuration"]
+        if self._file == "" and "file" in build:
+            self._file = build["file"]
+        if self._flex == "" and "flex" in build:
+            self._flex = build["flex"]
+        if self._job == "" and "job" in build:
+            self._job = build["job"]
+        if self._system == "" and "system" in build:
+            self._system = build["system"]
+        if self._platform == "" and "platform" in build:
+            self._platform = build["platform"]
+        if self._lock == "" and "lock" in build:
+            self._lock = build["lock"]
+        if self._runnable == "" and "runnable" in build:
+            self._runnable = build["runnable"]
+        if self._threads == "" and "threads" in build:
+            self._threads = build["threads"]
+        if self._type == "" and "type" in build:
+            self._type = build["type"]
+        if self._ctest == "" and "ctest" in build:
+            self._ctest = build["ctest"]
+        if self._timeout == "" and "timeout" in build:
+            self._timeout = build["timeout"]
+        if self._timeoffset == "" and "timeOffset" in build:
+            self._timeoffset = build["timeOffset"]
+        if self._version == "" and "version" in build:
+            self._version = build["version"]
+        if self._alias == "":
+            if "alias" in build:
+                self._alias = build["alias"]
+            else:
+                self._alias = self._build_id
+        if self._installable == "":
+            if "installable" in build:
+                self._installable = build["installable"]
+            else:
+                self._installable = False
+        if "from" in build:
+            self.fill(build["from"])
+
+
+    def get_tags(self):
+        return self._tags
+
+    def get_modules(self):
+        return self._modules
+
+    def get_pre_modules(self):
+        return self._pre_modules
+
+    def get_post_modules(self):
+        return self._post_modules
+
+    def get_pre_exec(self):
+        return self._pre_exec
+
+    def get_env(self):
+        return self._env
+
+    def get_compiler(self):
+        return self._compiler
+
+    def get_runnable(self):
+        return self._runnable
+
+    def get_version(self):
+        return self._version
+
+    def get_system(self):
+        return self._system
+
+    def get_platform(self):
+        return self._platform
+
+    def get_build_id(self):
+        return self._build_id
+
+    def get_code_coverage(self):
+        return self._code_coverage
+
+    def get_exclusive_tests(self):
+        return self._exclusive_tests
+
+    def get_cc_tool(self):
+        return self._cc_tool
+
+    def get_timeoffset(self):
+        return self._timeoffset

squidient/builds/builder.py

@@ -0,0 +1,339 @@
+#########################################################################
+#                                                                       #
+#  This file is part of squidient.                                      #
+#                                                                       #
+#  squidient is free software: you can redistribute it and/or modify    #
+#  it under the terms of the GNU General Public License as published by #
+#  the Free Software Foundation, either version 3 of the License, or    #
+#  (at your option) any later version.                                  #
+#                                                                       #
+#  squidient is distributed in the hope that it will be useful,         #
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of       #
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the        # 
+#  GNU General Public License for more details.                         #
+#                                                                       #
+#  You should have received a copy of the GNU General Public License    #
+#  along with squidient. If not, see <https://www.gnu.org/licenses/>.   #
+#                                                                       #
+#########################################################################
+import sys
+
+from ..utils.utils import *
+from ..terraform.terraform import *
+from ..git import Git
+from .executablebuild import ExecutableBuild
+import time
+
+from ..definitions import *
+
+logger = logging.getLogger(logging_context)
+
+
+class Builder:
+
+    def __init__(self, systemManager, hash, job, ssh, config, password=""):
+        self._systemManager = systemManager
+        self._hash = hash
+        self._alya_repository = None
+        self._config = config
+        self._builds = {}
+        self._start_time = time.time()
+        self._finished_builds = {}
+        self._pending_builds = {}
+        self._gotten_builds = {}
+        self._unlaunched_builds = {}
+        self._platform_ready = {}
+        self._ssh = ssh
+        self._job = job
+        self._password = password
+        self._build_list = {}
+        self.set_build_list()
+        self._report_dir = ""
+        self.set_report_dir()
+        self._p = ""
+
+    def cd_push(self, path):
+        """
+        Switch to git repository
+        """
+        self._p = os.getcwd()
+
+        if not os.path.isdir(path):
+            logger.error(path + " does not exist")
+            return False
+        try:
+            os.chdir(path)
+        except:
+            logger.error("Changing directory to " + path + " failed")
+            return False
+        return True
+
+    def cd_pop(self):
+        try:
+            os.chdir(self._p)
+        except:
+            logger.error("Changing directory back failed")
+            raise RuntimeError("An internal error happened...")
+
+    def set_build_list(self):
+        self._build_list = self._config["staging"]["builds"]
+
+    def set_report_dir(self):
+        self._report_dir = build_report_dir
+
+    def clean_builds(self):
+        command("rm -fr " + build_report_dir)
+
+    def run_builds(self, terraform):
+        self.send_alya()
+        self.clean_builds()
+        sorted_builds = {}
+        for system in self._systemManager.get_systems():
+            sorted_builds[system] = {}
+            for platform in self._systemManager.get_platforms(system):
+                sorted_builds[system][platform] = []
+        for b in self._build_list:
+            bui = self.init_build(b)
+            self._unlaunched_builds[b] = bui
+            sorted_builds[bui.get_system()][bui.get_platform()].append(b)
+        while len(self._unlaunched_builds) > 0:
+            for system in self._systemManager.get_systems():
+                for platform in self._systemManager.get_platforms(system):
+                    if terraform.get_platform_state(self._ssh, system, platform):
+                        for b in sorted_builds[system][platform]:
+                            self.run_build(self._unlaunched_builds[b])
+                            if self._unlaunched_builds[b].get_job_id() != -1:
+                                self._builds[b] = self._unlaunched_builds[b]
+                                del self._unlaunched_builds[b]
+                            else:
+                                raise Exception
+                        sorted_builds[system][platform] = []
+            time.sleep(60)
+        print_line()
+
+    def terminate_builds(self):
+        finished = False
+        jump()
+        print("Waiting for builds to finish:")
+        print_line()
+        while not finished:
+            finished = self.check_builds()
+        return self.get_status()
+
+    def get_status(self):
+        for b in self._finished_builds:
+            if not self._finished_builds[b]:
+                return False
+        return True
+
+    def get_failed(self):
+        failed = 0
+        for b in self._finished_builds:
+            if not self._finished_builds[b]:
+                failed += 1
+        return failed
+
+    def get_total(self):
+        return len(self._builds)
+
+    def next_valid_build(self, _print=True, oneprint=False, stop_on_error=False):
+        __print = _print
+        while len(self._pending_builds) == 0:
+            if self.check_builds(__print, __print, False):
+                if self.get_failed() > 1 and stop_on_error:
+                    return None
+                break
+            __print = __print and not oneprint
+        build = None
+        for b in self._pending_builds:
+            build = b
+            break
+        if build is not None:
+            self._pending_builds.pop(build)
+            return self._builds[build]
+        else:
+            return None
+
+    def get_build(self):
+        self.check_builds(False, False, False)
+        if len(self._pending_builds) > 0:
+            for build in self._pending_builds:
+                if build not in self._gotten_builds:
+                    self._gotten_builds[build] = self._pending_builds[build]
+                    return self._builds[build]
+        for build in self._builds:
+            if build not in self._gotten_builds:
+                self._gotten_builds[build] = True
+                return self._builds[build]
+        return None
+
+    def ready(self, build):
+        self.check_builds(False, False, False)
+        if build.get_build_id() in self._pending_builds:
+            self._pending_builds.pop(build.get_build_id())
+            return True
+        return False
+
+    def build_report(self):
+        jump()
+        print("Building build report")
+        print_line()
+        report = {}
+        for b in self._builds:
+            if b in self._finished_builds:
+                self._builds[b].build_report()
+                report[b] = self._builds[b].get_report()
+        f = self._report_dir+"/builds.json"
+        save_json(report, f)
+        json2js("builds", f)
+
+    def check_build_job_status(self, build):
+        status = self._builds[build].check_job_status()
+        if "Finished" in status:
+            finished = True
+        else:
+            finished = False
+        return {"finished": finished, "status": status}
+
+    def check_build_status(self, build):
+        status = self._builds[build].check_build_status()
+        return status
+
+    def check_builds(self, _print=True, minutes=True, back=True):
+        count = len(self._builds) + 1
+        if not minutes:
+            count -= 1
+        count = str(count)
+        check = True
+        time.sleep(1)
+        status = {}
+        one_success = False
+        for b in self._builds:
+            if b in self._finished_builds:
+                status["finished"] = True
+                test = True
+                if self._finished_builds[b]:
+                    status["status"] = "Success"
+                else:
+                    status["status"] = "Failed"
+            else:
+                status = self.check_build_job_status(b)
+                if status["finished"]:
+                    test = True
+                    if self.check_build_status(b):
+                        status["status"] = "Success"
+                        self._finished_builds[b] = True
+                        self._pending_builds[b] = True
+                        one_success = not back
+                    else:
+                        status["status"] = "Failed"
+                        self._finished_builds[b] = False
+                    self._builds[b].build_report()
+                else:
+                    test = False
+            check = check and test
+            if _print:
+                print('{:<35}{:>16}'.format("- " + b[:31] + ": ", status["status"]))
+        if _print and minutes:
+            print("Elapsed time: ", int((time.time() - self._start_time) / 60), "minutes")
+        if (not check) and _print and not one_success:
+            print("\033[" + count + "A", end='')
+        time.sleep(1)
+        return check
+
+    def get_alya(self, fetch=True, force_https=False):
+        jump()
+        self._alya_repository = Git(git_ssh=self._config["alyaGitRepository"]["ssh"],
+                                    git_https=self._config["alyaGitRepository"]["https"],
+                                    user=self._config["alyaGitRepository"]["user"],
+                                    branch=self._config["alyaGitRepository"]["branch"],
+                                    dir="alya",
+                                    password=self._password,
+                                    force_https=force_https)
+        if fetch:
+            print("Cloning/Updating the alya git repository...")
+            self._alya_repository.clone_or_update_tests()
+            print_line()
+        self._revision = self._alya_repository.revision_short()
+        self._revision_long = self._alya_repository.revision_long()
+        return self._alya_repository
+
+    def download_alamak(self, user="", token=""):
+        jump()
+        if self._config["alamak"]["enable"]:
+            print("Cloning alamak")
+            self.cd_push(self._config["alamak"]["path"])
+            critical_command("./alamak.sh --clean")
+            if self._config["alamak"]["revision"] is not None:
+                ref = "--revision " + self._config["alamak"]["revision"]
+            else:
+                ref = "--branch " + self._config["alamak"]["branch"]
+            alamak_command = ""
+            if user != "" and token != "":
+                alamak_command = "./alamak.sh --https " + ref + " --user " + user + " --token " + token
+            else:
+                alamak_command = "./alamak.sh --ssh " + ref
+            try:
+                critical_command(alamak_command, silent=True)
+            except:
+                print("The cloning of alamak has failed. Showing the error and terminating...")
+                command("./alamak.sh --clean")
+                print(command(alamak_command, output=True, silent=True))
+                raise Exception
+            self.cd_pop()
+            print_line()
+
+    def get_alya_repository(self):
+        return self._alya_repository
+
+    def send_alya(self):
+        jump()
+        for system in self._systemManager.get_systems():
+            print("System: " + system)
+            print("Cleaning the remote build directory...")
+            self._ssh.rmdir(system=system, path=remote_bin_dir, server_path=True)
+            if self._config["clean"]:
+                self._ssh.rmdir(system=system, path=remote_build_dir, server_path=True)
+            self._ssh.rmdir(system=system, path=remote_cc_dir, server_path=True)
+            #self._ssh.ssh(system=system, cmd="rm " + remote_cc_dir + "/*/*.dyn", server_path=True)
+            print("Creating the remote build directory infrastructure...")
+            self._ssh.mkdir(system=system, path=remote_bin_dir, server_path=True)
+            self._ssh.mkdir(system=system, path=remote_build_dir, server_path=True)
+            self._ssh.mkdir(system=system, path=remote_cc_dir, server_path=True)
+            print("Uploading alya...")
+            self._ssh.rsync_send(system=system, local=alya_dir, options="--exclude .git/", delete=True, server_path=True, critical=True)
+        print_line()
+
+    def create_build(self, build):
+        return ExecutableBuild(systemManager=self._systemManager, hash=self._hash, build_id=build, job=self._job, ssh=self._ssh,
+                               config=self._config, report_directory=self._report_dir)
+
+    def init_build(self, build):
+        jump()
+        print("Build: " + build)
+        b = self.create_build(build)
+        print("\tGenerating alya configuration file...")
+        b.generate_configuration()
+        print("\tGenerating job file...")
+        b.generate_job()
+        print("\tUploading files...")
+        b.send_files()
+        return b
+
+    def run_build(self, b):
+        time.sleep(5)
+        print("\tSubmitting job...")
+        job_id = b.launch_job()
+        if job_id == -1:
+            print("\tJob submission has failed!")
+        else:
+            print("\tJob id: " + str(job_id))
+
+    def get_builds(self):
+        return self._builds
+
+    def get_alya_revision(self):
+        return self._revision
+
+    def get_alya_revision_long(self):
+        return self._revision_long