squidient 2.6.1__tar.gz → 2.7.0__tar.gz

{squidient-2.6.1 → squidient-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 2.6.1
+Version: 2.7.0
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-2.6.1 → squidient-2.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "2.6.1"
+version = "2.7.0"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/builds/benchmarkbuilder.py

@@ -50,9 +50,11 @@ class BenchmarkBuilder(Builder):
                               db=self._db)
 
     def build_report(self):
-        console.section("Building monitor build report")
+        console.section("Building benchmark build report")
         report = {}
+        console.step("Building per-build reports")
         for b in self._builds:
+            console.group("Build " + b, level=2)
             self._builds[b].build_report()
             report[b] = self._builds[b].get_report()
         f = self._report_dir+"/benchmark_builds.json"

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/builds/builder.py

@@ -82,7 +82,7 @@ class Builder:
             os.chdir(self._p)
         except:
             logger.error("Changing directory back failed")
-            raise RuntimeError("An internal error happened...")
+            raise RuntimeError("An internal error happened")
 
     def set_build_list(self):
         self._build_list = self._config["staging"]["builds"]
@@ -102,10 +102,12 @@ class Builder:
                 sorted_builds[system] = {}
                 for platform in self._systemManager.get_platforms(system):
                     sorted_builds[system][platform] = []
+            console.section("Preparing builds")
             for b in self._build_list:
                 bui = self.init_build(b)
                 self._unlaunched_builds[b] = bui
                 sorted_builds[bui.get_system()][bui.get_platform()].append(b)
+            console.section("Submitting build jobs")
             while len(self._unlaunched_builds) > 0:
                 for system in self._systemManager.get_systems():
                     for platform in self._systemManager.get_platforms(system):
@@ -121,7 +123,6 @@ class Builder:
                                     raise Exception
                             sorted_builds[system][platform] = []
                 time.sleep(60)
-            print_line()
 
     def terminate_builds(self):
         with self.timer("builds.wait", local=False):
@@ -147,14 +148,15 @@ class Builder:
     def get_total(self):
         return len(self._builds)
 
-    def next_valid_build(self, show_progress=True, single_snapshot=None, stop_on_error=False):
+    def next_valid_build(self, show_progress=True, single_snapshot=None, stop_on_error=False, output_level=1):
         display_status = show_progress
         while len(self._pending_builds) == 0:
             if self.check_builds(
                     show_progress=display_status,
                     show_elapsed=display_status,
                     rewind=False,
-                    single_snapshot=single_snapshot):
+                    single_snapshot=single_snapshot,
+                    output_level=output_level):
                 if self.get_failed() > 1 and stop_on_error:
                     return None
                 break
@@ -194,8 +196,10 @@ class Builder:
         with self.timer("builds.report"):
             console.section("Building build report")
             report = {}
+            console.step("Building per-build reports")
             for b in self._builds:
                 if b in self._finished_builds:
+                    console.group("Build " + b, level=2)
                     self._builds[b].build_report()
                     report[b] = self._builds[b].get_report()
             f = self._report_dir+"/builds.json"
@@ -214,7 +218,7 @@ class Builder:
         status = self._builds[build].check_build_status()
         return status
 
-    def check_builds(self, show_progress=True, show_elapsed=True, rewind=True, single_snapshot=None):
+    def check_builds(self, show_progress=True, show_elapsed=True, rewind=True, single_snapshot=None, output_level=1):
         line_count = len(self._builds) + (1 if show_elapsed else 0)
         live = LiveOutput(
             line_count,
@@ -250,9 +254,9 @@ class Builder:
                     test = False
             check = check and test
             if live.should_print():
-                console.status(b, status["status"])
+                console.status(b, status["status"], level=output_level)
         if live.should_print() and show_elapsed:
-            console.write("Elapsed time: {} minutes".format(int((time.time() - self._start_time) / 60)))
+            console.elapsed_time(int((time.time() - self._start_time) / 60), level=output_level)
             live.printed()
             self._build_status_printed = True
         elif live.should_print():
@@ -276,9 +280,12 @@ class Builder:
                                         force_https=force_https)
             if fetch:
                 console.section("Cloning or updating the alya git repository")
+                console.step("Fetching Alya source tree")
                 self._alya_repository.clone_or_update_tests()
             self._revision = self._alya_repository.revision_short()
             self._revision_long = self._alya_repository.revision_long()
+            if fetch:
+                console.step("Fetched revision " + self._revision, level=2)
             return self._alya_repository
 
     def download_alamak(self, user="", token=""):
@@ -286,6 +293,7 @@ class Builder:
             if self._config["alamak"]["enable"]:
                 console.section("Cloning alamak")
                 self.cd_push(self._config["alamak"]["path"])
+                console.step("Cleaning alamak workspace")
                 critical_command("./alamak.sh --clean")
                 if self._config["alamak"]["revision"] is not None:
                     ref = "--revision " + shell_quote(self._config["alamak"]["revision"])
@@ -301,14 +309,16 @@ class Builder:
                         + " --token "
                         + shell_quote(url_quote(token, safe=""))
                     )
+                    console.step("Downloading alamak through HTTPS")
                 else:
                     alamak_command = "./alamak.sh --ssh " + ref
+                    console.step("Downloading alamak through SSH")
                 try:
                     critical_command(alamak_command, silent=True)
                 except:
-                    console.error("The cloning of alamak has failed")
+                    console.error("The cloning of alamak has failed", level=1)
                     command("./alamak.sh --clean")
-                    console.write(command(alamak_command, silent=True).stdout)
+                    console.detail(command(alamak_command, silent=True).stdout, level=2)
                     raise Exception
                 self.cd_pop()
 
@@ -320,17 +330,17 @@ class Builder:
             console.section("Uploading alya")
             for system in self._systemManager.get_systems():
                 with self.timer("builds.send_alya.system", local=False, system=system):
-                    console.step("System " + system)
-                    console.step("Cleaning the remote build directory")
+                    console.group("System " + system)
+                    console.step("Cleaning the remote build directory", level=2)
                     self._ssh.rmdir(system=system, path=remote_bin_dir, server_path=True)
                     if self._config["clean"]:
                         self._ssh.rmdir(system=system, path=remote_build_dir, server_path=True)
                     self._ssh.rmdir(system=system, path=remote_cc_dir, server_path=True)
-                    console.step("Creating the remote build directory infrastructure")
+                    console.step("Creating the remote build directory infrastructure", level=2)
                     self._ssh.mkdir(system=system, path=remote_bin_dir, server_path=True)
                     self._ssh.mkdir(system=system, path=remote_build_dir, server_path=True)
                     self._ssh.mkdir(system=system, path=remote_cc_dir, server_path=True)
-                    console.step("Uploading alya source tree")
+                    console.step("Uploading alya source tree", level=2)
                     self._ssh.rsync_send(system=system, local=alya_dir, options="--exclude .git/", delete=True, server_path=True, critical=True)
 
     def create_build(self, build):
@@ -339,13 +349,13 @@ class Builder:
 
     def init_build(self, build):
         with self.timer("builds.prepare_build", comment=build):
-            console.section("Preparing build " + build)
+            console.group("Build " + build)
             b = self.create_build(build)
-            console.step("Generating alya configuration file")
+            console.step("Generating alya configuration file", level=2)
             b.generate_configuration()
-            console.step("Generating job file")
+            console.step("Generating job file", level=2)
             b.generate_job()
-            console.step("Uploading files")
+            console.step("Uploading files", level=2)
             with self.timer("builds.prepare_build.upload", local=False, system=b.get_system(), platform=b.get_platform(), comment=build):
                 b.send_files()
             return b
@@ -353,12 +363,13 @@ class Builder:
     def run_build(self, b):
         with self.timer("builds.submit_job", local=False, system=b.get_system(), platform=b.get_platform(), comment=b.get_build_id()):
             time.sleep(5)
-            console.step("Submitting job")
+            console.group("Build " + b.get_build_id())
+            console.step("Submitting build job", level=2)
             job_id = b.launch_job()
             if job_id == -1:
-                console.error("Job submission has failed")
+                console.error("Job submission has failed", level=3)
             else:
-                console.success("Job id: " + str(job_id))
+                console.success("Job id: " + str(job_id), level=3)
 
     def get_builds(self):
         return self._builds

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/cli.py

@@ -30,22 +30,17 @@ from .squidientmonitor import main as mmain
 from .utils.package import check_latest_package_any_index
 
 def help():
-    console.write("squidient [option]")
-    console.blank()
-    console.write("\tsquidient help                   Print this help")
-    console.blank()
-    console.write("\tInitialization options:")
-    console.write("\tsquidient init [directory]       Initialize the directory")
-    console.blank()
-    console.write("\tConfiguration options:")
-    console.write("\tsquidient configure [options]    Configure squidient")
+    console.section("squidient")
+    console.option("squidient help", "Print this help")
+    console.section("Initialization options")
+    console.option("squidient init [directory]", "Initialize the directory")
+    console.section("Configuration options")
+    console.option("squidient configure [options]", "Configure squidient")
     chelp()
-    console.blank()
-    console.write("\tBuild and test options:")
-    console.write("\tsquidient run [options]")
+    console.section("Build and test options")
+    console.option("squidient run [options]", "Run the staging workflow")
     shelp()
-    console.blank()
-    console.write("\tsquidient monitor [options]")
+    console.option("squidient monitor [options]", "Run the monitor workflow")
     mhelp()
 
 def main():
@@ -90,8 +85,9 @@ def main():
         return 1
 
     except Exception as e:
-        console.error(str(e))
-        console.error("Exception")
+        console.section("Command error")
+        console.error(str(e), level=1)
+        console.error("Exception", level=1)
         return 1
 
 

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/codecoverage/codecoverage.py

@@ -98,6 +98,7 @@ class CodeCoverage(ABC):
     def build_report(self):
         with self.timer("code_coverage.report", comment=self._tool):
             console.section("Building code coverage report")
+            console.step("Writing code coverage report files")
             f = cc_report_dir + "/cc.json"
             save_json(self._report, f)
             json2js("cc", f)

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/codecoverage/intelcodecoverage.py

@@ -107,11 +107,13 @@ class IntelCodeCoverage(CodeCoverage):
         with self.platform_timer("code_coverage.process", comment=self._tool):
             console.step("Merging code coverage files")
             if not self.merge():
-                console.error("Merging has failed")
+                console.section("Code coverage result")
+                console.error("Merging has failed", level=1)
                 return False
             console.step("Generating code coverage html files")
             if not self.codecov():
-                console.error("Code coverage has failed")
+                console.section("Code coverage result")
+                console.error("Code coverage has failed", level=1)
                 return False
             return True
 
@@ -210,7 +212,7 @@ class IntelCodeCoverage(CodeCoverage):
                     continue
                 f["name"] = rows[i]["file"]
                 if show_details:
-                    console.step("File " + f["name"])
+                    console.group("File " + f["name"], level=2)
                 path = rows[i]["path"].split("alya/", 1)[1]
                 f["path"] = path
                 realPath = self.alya_realpath(path)

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/compare/methods/tolerance.py

@@ -107,7 +107,7 @@ class Tolerance(ABC):
             rows = range(len(self._refContent))
             logger.debug("There are " + str(len(self._refContent)) + " rows")
             if len(self._refContent) == 0:
-                logger.warning("Base does not have rows!")
+                logger.warning("Base does not have rows")
                 self._status = False
                 self._error = "empty base"
 

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/configure.py

@@ -223,15 +223,15 @@ def set_builds(com=""):
     if com == "":
         console.section("Available builds")
         if is_bsc_system():
-            console.write("Presets:")
-            console.write("\t- all        \n\t\tall the builds required to validate alya")
-            console.write("\t- gpp        \n\t\tall the marenostrum 5 GPP builds required to validate alya")
-            console.write("\t- gnu        \n\t\tall the gnu builds (gpp only)")
-            console.write("\t- oneapi     \n\t\tall the oneapi builds (gpp only)")
-            console.write("\t- benchmarks \n\t\tall the benchmarks builds")
-        console.write("Unitary:")
+            console.detail("Presets", level=1)
+            console.option("all", "all the builds required to validate alya", level=2)
+            console.option("gpp", "all the marenostrum 5 GPP builds required to validate alya", level=2)
+            console.option("gnu", "all the gnu builds (gpp only)", level=2)
+            console.option("oneapi", "all the oneapi builds (gpp only)", level=2)
+            console.option("benchmarks", "all the benchmarks builds", level=2)
+        console.detail("Unitary", level=1)
         for build in selectedBuilds:
-            console.write("\t- " + build)
+            console.option(build, "", level=2)
         line_jump()
         b = input("Choose one or more builds:\n")
     else:
@@ -246,13 +246,14 @@ def set_builds(com=""):
         if a in b:
             b.extend(predefined_builds[a])
     b = set(b)
+    console.section("Build selection result")
     for build in b:
         if build in predefined_builds:
             continue
         if build not in builds:
-            console.error(build + " is not a valid option")
+            console.error(build + " is not a valid option", level=1)
         else:
-            console.success("Build " + build + " added correctly")
+            console.success("Build " + build + " added correctly", level=1)
             config["staging"]["builds"].append(build)
     line_jump()
     save_json(config, config_file)
@@ -268,7 +269,7 @@ def set_cc(com=""):
     if com == "":
         console.section("Available code coverage configurations")
         for cc in ccs:
-            console.write("\t- " + cc)
+            console.option(cc, "", level=2)
         line_jump()
         c = input("Choose a code coverage configuration:\n")
     else:
@@ -292,7 +293,7 @@ def set_systems(com=""):
     if com == "":
         console.section("Available systems")
         for system in config["systems"]:
-            console.write("\t- " + system)
+            console.option(system, "", level=2)
         line_jump()
         s = input("Choose one or more systems:\n")
     else:
@@ -301,11 +302,12 @@ def set_systems(com=""):
     if s[0] == '':
         s[0] = "bsc"
 
+    console.section("System selection result")
     for system in s:
         if system not in config["systems"]:
-            console.error(system + " is not a valid option")
+            console.error(system + " is not a valid option", level=1)
         else:
-            console.success("System " + system + " added correctly")
+            console.success("System " + system + " added correctly", level=1)
             systems[system] = config["systems"][system]
     line_jump()
     config["systems"] = systems
@@ -496,7 +498,7 @@ def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system
     if project is not None:
         config_project = open_json(project_file)
         if project not in config_project:
-            console.error("Project " + project + " not found")
+            console.error("Project " + project + " not found", level=1)
             raise RuntimeError("Project " + project + " not found")
         config_project = config_project[project]
         for platform in config_project["systems"][system]["platforms"]:
@@ -562,42 +564,41 @@ def ensure_initialized(root: Path = Path.cwd()) -> None:
         )
 
 def configure_help():
-    console.write("squidient configuration options:")
-    console.write("[argument] between brackets are optional")
-    line_jump()
-    console.write("\t-h  | --help                          Show this information")
-    console.write("Global Configuration")
-    console.write("\t-a  | --all                           Configure the main options")
-    console.write("\t-r  | --re                            Reconfigure the main options")
-    console.write("\t-A  | --ALL                           Configure all the options")
-    console.write("\t-s  | --systems          [systems]    Change the set of systems")
-    console.write("\t-b  | --builds           [builds]     Change the set of builds")
-    console.write("\t-p  | --path             [path]       Change the server path")
-    console.write("\t-k  | --ssh-key          [ssh-key]    Define the SSH key path for the configured systems")
-    console.write("Alamak")
-    console.write("\t--alamak                              Enable alamak")
-    console.write("Git configuration")
-    console.write("\t-u  | --git-user         [git-user]   Set up the git user (gitlab/https user)")
-    console.write("\t-br | --git-branch       [git-branch] Select alya git branch/commit/tag")
-    console.write("Coverage configuration")
-    console.write("\t-c  | -c                 [tool]       Define code coverage tool")
-    console.write("\t-cc-ref  | --cc-ref      [git-commit] Define code coverage reference commit")
-    console.write("GitLab configuration")
-    console.write("\t--gitlab                              Enable GitLab features")
-    console.write("\t--pipeline:[pipeline]                 Pipeline number")
-    console.write("\t--directory:[directory]               Change directory name")
-    console.write("\t--branch:[reference]                  alya reference")
-    console.write("\t--alya:[reference]                    alya reference (priority inferior to branch)")
-    console.write("\t--flex                                Enable flex queue")
-    console.write("\t--install                             Enable install mode")
-    console.write("\t--system:[system]                     Choose system")
-    console.write("\t--platform:[platform]                 Choose platform for fast build")
-    console.write("\t--type:[type]                         Choose execution type")
-    console.write("\t--api:[api]                           Define GitLab API")
-    console.write("\t--alamak-branch:[branch]              Define which alamak branch to use")
-    console.write("\t--alamak-revision:[revision]          Define which alamak revision to use")
-    console.write("\t--notify-failures                     Notify benchmark authors when a benchmark fails")
-    console.write("\t--reservation                         Enable reservation")
+    console.section("squidient configuration options")
+    console.detail("[argument] between brackets are optional")
+    console.option("-h | --help", "Show this information")
+    console.section("Global configuration")
+    console.option("-a | --all", "Configure the main options")
+    console.option("-r | --re", "Reconfigure the main options")
+    console.option("-A | --ALL", "Configure all the options")
+    console.option("-s | --systems [systems]", "Change the set of systems")
+    console.option("-b | --builds [builds]", "Change the set of builds")
+    console.option("-p | --path [path]", "Change the server path")
+    console.option("-k | --ssh-key [ssh-key]", "Define the SSH key path for the configured systems")
+    console.section("Alamak")
+    console.option("--alamak", "Enable alamak")
+    console.section("Git configuration")
+    console.option("-u | --git-user [git-user]", "Set up the git user (gitlab/https user)")
+    console.option("-br | --git-branch [git-branch]", "Select alya git branch/commit/tag")
+    console.section("Coverage configuration")
+    console.option("-c | --cc [tool]", "Define code coverage tool")
+    console.option("-cc-ref | --cc-ref [git-commit]", "Define code coverage reference commit")
+    console.section("GitLab configuration")
+    console.option("--gitlab", "Enable GitLab features")
+    console.option("--pipeline:[pipeline]", "Pipeline number")
+    console.option("--directory:[directory]", "Change directory name")
+    console.option("--branch:[reference]", "alya reference")
+    console.option("--alya:[reference]", "alya reference (priority inferior to branch)")
+    console.option("--flex", "Enable flex queue")
+    console.option("--install", "Enable install mode")
+    console.option("--system:[system]", "Choose system")
+    console.option("--platform:[platform]", "Choose platform for fast build")
+    console.option("--type:[type]", "Choose execution type")
+    console.option("--api:[api]", "Define GitLab API")
+    console.option("--alamak-branch:[branch]", "Define which alamak branch to use")
+    console.option("--alamak-revision:[revision]", "Define which alamak revision to use")
+    console.option("--notify-failures", "Notify benchmark authors when a benchmark fails")
+    console.option("--reservation", "Enable reservation")
     line_jump()
 
 
@@ -613,11 +614,12 @@ def _run_cli(argv: Sequence[str]) -> int:
     argument = ""
 
     if base_config["fileFormat"] == -1:
-        console.warning("This is your first use of squidient; it requires a full configuration")
+        console.section("First use configuration")
+        console.warning("This is your first use of squidient; it requires a full configuration", level=1)
         if len(argv) < 1:
             option = "--all"
         elif not is_option(argv[0], "-A", "--ALL") and not is_option(argv[0], "--gitlab", "--gitlab"):
-            console.warning("Ignoring arguments")
+            console.warning("Ignoring arguments", level=1)
             option = "--all"
         else:
             option = argv[0]
@@ -797,7 +799,8 @@ def _run_cli(argv: Sequence[str]) -> int:
         configure_help()
 
     else:
-        console.error(option + " is not a valid argument")
+        console.section("Configuration error")
+        console.error(option + " is not a valid argument", level=1)
         configure_help()
 
     return 0

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/connection/job.py

@@ -111,10 +111,10 @@ class JobThread (threading.Thread):
     def get_job_numbers(self):
         job_number = 0
         try:
-            #job_number = int(self._ssh.ssh("squeue | wc -l", system=self._system, platform=self._platform).stdout.strip()) - 1
             job_number = max(len(self._ssh.ssh("squeue", system=self._system, platform=self._platform).stdout.split("\n")) - 2, 0)
         except:
-            console.warning("Cannot determine running jobs")
+            console.group("System " + self._system + ", platform " + self._platform)
+            console.warning("Cannot determine running jobs", level=2)
             logger.warning("Cannot determine job number")
         logger.debug("Job number of " + self._platform + " retrieved with squeue: " + str(job_number))
         return job_number
@@ -158,7 +158,8 @@ class Job:
         dep = ""
         if key_lock != "":
             if asynchronous:
-                console.error("You cannot launch an asynchronous job with a dependency")
+                console.section("Job submission")
+                console.error("You cannot launch an asynchronous job with a dependency", level=1)
                 raise Exception
             lock_id = self._build_lock.get_lock(key_lock)
             if lock_id != -1:
@@ -168,7 +169,8 @@ class Job:
             self._output_queues[system][platform] = queue.Queue()
             self._job_threads[system][platform] = []
             if self._thread_number[system][platform] < 1:
-                console.error("Thread number must be greater than 0")
+                console.section("Job submission")
+                console.error("Thread number must be greater than 0", level=1)
                 raise Exception
             for i in range(self._thread_number[system][platform]):
                 self._job_threads[system][platform].append(JobThread(self._ssh, system, platform,

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/git.py

@@ -82,7 +82,7 @@ class Git:
             os.chdir(self._path)
         except:
             logger.error("Changing directory back failed")
-            raise RuntimeError("An internal error happened...")
+            raise RuntimeError("An internal error happened")
 
     def clone_ssh(self):
         self.cd_pop()
@@ -242,25 +242,22 @@ class Git:
         """
            Clone or update the repository
         """
-        #print("Updating the repository")
         self.test_git()
         if not self._force_https:
             if not self.update_ssh():
                 logger.info("SSH update failed")
-                #print("Update failed, cloning the tests repository")
                 if not self.clone_ssh():
                     logger.info("SSH clone failed")
-                    raise RuntimeError("Git repository cannot be cloned using the ssh protocol!\nUse the option --https during the execution of squidient.")
+                    raise RuntimeError("Git repository cannot be cloned using the ssh protocol\nUse the option --https during the execution of squidient")
         else:
             if not self.update_https():
                 logger.info("HTTPS update failed")
-                # print("Update failed, cloning the tests repository")
                 if not self.clone_https():
                     logger.info("HTTPS clone failed")
                     raise RuntimeError("Git repository cannot be cloned")
         console.step("Switching git directory to branch " + self._branch)
         if not self.branch():
-            raise RuntimeError("Cannot switch to " + self._branch + " and rebase it!")
+            raise RuntimeError("Cannot switch to " + self._branch + " and rebase it")
 
     def revision_short(self):
         revision = ""

{squidient-2.6.1 → squidient-2.7.0}/src/squidient/init.py

@@ -84,4 +84,4 @@ def init_squidient(dist_dir: str=".") -> None:
 
 if __name__ == "__main__":
     init_squidient()
-    console.success("Copied squidient/data/* into current directory")
+    console.success("Copied squidient/data/* into current directory", level=1)