squidient 2.2.0__tar.gz → 2.4.0__tar.gz

{squidient-2.2.0 → squidient-2.4.0}/.gitignore

@@ -51,6 +51,7 @@ dist
 /email/
 /icons/
 /terraform/
+/timers/
 /version/
 /web/
 /sqreport/

{squidient-2.2.0 → squidient-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: squidient
-Version: 2.2.0
+Version: 2.4.0
 Summary: CI/CD framework for alya
 Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
 Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues

{squidient-2.2.0 → squidient-2.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "squidient"
-version = "2.2.0"
+version = "2.4.0"
 authors = [
   { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
 ]

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/builds/benchmarkbuilder.py

@@ -50,9 +50,7 @@ class BenchmarkBuilder(Builder):
                               db=self._db)
 
     def build_report(self):
-        jump()
-        print("Building monitor build report")
-        print_line()
+        console.section("Building monitor build report")
         report = {}
         for b in self._builds:
             self._builds[b].build_report()

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/builds/builder.py

@@ -27,6 +27,7 @@ import time
 
 from ..definitions import *
 from ..timers import Timer
+from ..utils.message import LiveOutput, console
 
 logger = logging.getLogger(logging_context)
 
@@ -53,6 +54,7 @@ class Builder:
         self._report_dir = ""
         self.set_report_dir()
         self._p = ""
+        self._build_status_printed = False
 
     def timer(self, name, local=True, system="", platform="", comment=""):
         return Timer.scope(name, local=local, system=system, platform=platform, comment=comment)
@@ -122,9 +124,7 @@ class Builder:
     def terminate_builds(self):
         with self.timer("builds.wait", local=False):
             finished = False
-            jump()
-            print("Waiting for builds to finish:")
-            print_line()
+            console.section("Waiting for builds to finish")
             while not finished:
                 finished = self.check_builds()
             return self.get_status()
@@ -145,14 +145,19 @@ class Builder:
     def get_total(self):
         return len(self._builds)
 
-    def next_valid_build(self, _print=True, oneprint=False, stop_on_error=False):
-        __print = _print
+    def next_valid_build(self, show_progress=True, single_snapshot=None, stop_on_error=False):
+        display_status = show_progress
         while len(self._pending_builds) == 0:
-            if self.check_builds(__print, __print, False):
+            if self.check_builds(
+                    show_progress=display_status,
+                    show_elapsed=display_status,
+                    rewind=False,
+                    single_snapshot=single_snapshot):
                 if self.get_failed() > 1 and stop_on_error:
                     return None
                 break
-            __print = __print and not oneprint
+            if single_snapshot is True:
+                display_status = False
         build = None
         for b in self._pending_builds:
             build = b
@@ -185,9 +190,7 @@ class Builder:
 
     def build_report(self):
         with self.timer("builds.report"):
-            jump()
-            print("Building build report")
-            print_line()
+            console.section("Building build report")
             report = {}
             for b in self._builds:
                 if b in self._finished_builds:
@@ -209,11 +212,13 @@ class Builder:
         status = self._builds[build].check_build_status()
         return status
 
-    def check_builds(self, _print=True, minutes=True, back=True):
-        count = len(self._builds) + 1
-        if not minutes:
-            count -= 1
-        count = str(count)
+    def check_builds(self, show_progress=True, show_elapsed=True, rewind=True, single_snapshot=None):
+        line_count = len(self._builds) + (1 if show_elapsed else 0)
+        live = LiveOutput(
+            line_count,
+            enabled=show_progress,
+            single_snapshot=single_snapshot if single_snapshot is not None else not rewind,
+            printed=self._build_status_printed)
         check = True
         time.sleep(1)
         status = {}
@@ -234,7 +239,7 @@ class Builder:
                         status["status"] = "Success"
                         self._finished_builds[b] = True
                         self._pending_builds[b] = True
-                        one_success = not back
+                        one_success = not rewind
                     else:
                         status["status"] = "Failed"
                         self._finished_builds[b] = False
@@ -242,18 +247,24 @@ class Builder:
                 else:
                     test = False
             check = check and test
-            if _print:
-                print('{:<35}{:>16}'.format("- " + b[:31] + ": ", status["status"]))
-        if _print and minutes:
-            print("Elapsed time: ", int((time.time() - self._start_time) / 60), "minutes")
-        if (not check) and _print and not one_success:
-            print("\033[" + count + "A", end='')
+            if live.should_print():
+                console.status(b, status["status"])
+        if live.should_print() and show_elapsed:
+            console.write("Elapsed time: {} minutes".format(int((time.time() - self._start_time) / 60)))
+            live.printed()
+            self._build_status_printed = True
+        elif live.should_print():
+            live.printed()
+            self._build_status_printed = True
+        if check:
+            self._build_status_printed = False
+        if (not check) and not one_success:
+            live.rewind()
         time.sleep(1)
         return check
 
     def get_alya(self, fetch=True, force_https=False):
         with self.timer("builds.get_alya", comment="fetch={}".format(fetch)):
-            jump()
             self._alya_repository = Git(git_ssh=self._config["alyaGitRepository"]["ssh"],
                                         git_https=self._config["alyaGitRepository"]["https"],
                                         user=self._config["alyaGitRepository"]["user"],
@@ -262,18 +273,16 @@ class Builder:
                                         password=self._password,
                                         force_https=force_https)
             if fetch:
-                print("Cloning/Updating the alya git repository...")
+                console.section("Cloning or updating the alya git repository")
                 self._alya_repository.clone_or_update_tests()
-                print_line()
             self._revision = self._alya_repository.revision_short()
             self._revision_long = self._alya_repository.revision_long()
             return self._alya_repository
 
     def download_alamak(self, user="", token=""):
         with self.timer("builds.download_alamak"):
-            jump()
             if self._config["alamak"]["enable"]:
-                print("Cloning alamak")
+                console.section("Cloning alamak")
                 self.cd_push(self._config["alamak"]["path"])
                 critical_command("./alamak.sh --clean")
                 if self._config["alamak"]["revision"] is not None:
@@ -288,34 +297,32 @@ class Builder:
                 try:
                     critical_command(alamak_command, silent=True)
                 except:
-                    print("The cloning of alamak has failed. Showing the error and terminating...")
+                    console.error("The cloning of alamak has failed")
                     command("./alamak.sh --clean")
-                    print(command(alamak_command, silent=True).stdout)
+                    console.write(command(alamak_command, silent=True).stdout)
                     raise Exception
                 self.cd_pop()
-                print_line()
 
     def get_alya_repository(self):
         return self._alya_repository
 
     def send_alya(self):
         with self.timer("builds.send_alya", local=False):
-            jump()
+            console.section("Uploading alya")
             for system in self._systemManager.get_systems():
                 with self.timer("builds.send_alya.system", local=False, system=system):
-                    print("System: " + system)
-                    print("Cleaning the remote build directory...")
+                    console.step("System " + system)
+                    console.step("Cleaning the remote build directory")
                     self._ssh.rmdir(system=system, path=remote_bin_dir, server_path=True)
                     if self._config["clean"]:
                         self._ssh.rmdir(system=system, path=remote_build_dir, server_path=True)
                     self._ssh.rmdir(system=system, path=remote_cc_dir, server_path=True)
-                    print("Creating the remote build directory infrastructure...")
+                    console.step("Creating the remote build directory infrastructure")
                     self._ssh.mkdir(system=system, path=remote_bin_dir, server_path=True)
                     self._ssh.mkdir(system=system, path=remote_build_dir, server_path=True)
                     self._ssh.mkdir(system=system, path=remote_cc_dir, server_path=True)
-                    print("Uploading alya...")
+                    console.step("Uploading alya source tree")
                     self._ssh.rsync_send(system=system, local=alya_dir, options="--exclude .git/", delete=True, server_path=True, critical=True)
-            print_line()
 
     def create_build(self, build):
         return ExecutableBuild(systemManager=self._systemManager, hash=self._hash, build_id=build, job=self._job, ssh=self._ssh,
@@ -323,14 +330,13 @@ class Builder:
 
     def init_build(self, build):
         with self.timer("builds.prepare_build", comment=build):
-            jump()
-            print("Build: " + build)
+            console.section("Preparing build " + build)
             b = self.create_build(build)
-            print("\tGenerating alya configuration file...")
+            console.step("Generating alya configuration file")
             b.generate_configuration()
-            print("\tGenerating job file...")
+            console.step("Generating job file")
             b.generate_job()
-            print("\tUploading files...")
+            console.step("Uploading files")
             with self.timer("builds.prepare_build.upload", local=False, system=b.get_system(), platform=b.get_platform(), comment=build):
                 b.send_files()
             return b
@@ -338,12 +344,12 @@ class Builder:
     def run_build(self, b):
         with self.timer("builds.submit_job", local=False, system=b.get_system(), platform=b.get_platform(), comment=b.get_build_id()):
             time.sleep(5)
-            print("\tSubmitting job...")
+            console.step("Submitting job")
             job_id = b.launch_job()
             if job_id == -1:
-                print("\tJob submission has failed!")
+                console.error("Job submission has failed")
             else:
-                print("\tJob id: " + str(job_id))
+                console.success("Job id: " + str(job_id))
 
     def get_builds(self):
         return self._builds

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/cli.py

@@ -21,7 +21,7 @@ import sys
 
 from .configure import configure_help as chelp
 from .init import copy_packaged_tree, init_squidient
-from .utils.message import jump
+from .utils.message import console
 from .squidient import help as shelp
 from .squidientmonitor import help as mhelp
 from .configure import main as cmain
@@ -30,35 +30,34 @@ from .squidientmonitor import main as mmain
 from .utils.package import check_latest_package_any_index
 
 def help():
-    print("squidient [option]")
-    jump()
-    print("\tsquidient help                   Print this help")
-    jump()
-    print("\tIntialization options:")
-    print("\tsquidient init [directory]       Initialize the directory")
-    jump()
-    print("\tConfiguration options:")
-    print("\tsquidient configure [options]    Configure squidient")
+    console.write("squidient [option]")
+    console.blank()
+    console.write("\tsquidient help                   Print this help")
+    console.blank()
+    console.write("\tInitialization options:")
+    console.write("\tsquidient init [directory]       Initialize the directory")
+    console.blank()
+    console.write("\tConfiguration options:")
+    console.write("\tsquidient configure [options]    Configure squidient")
     chelp()
-    jump()
-    print("\tBuild and test options:")
-    print("\tsquidient run [options]")
+    console.blank()
+    console.write("\tBuild and test options:")
+    console.write("\tsquidient run [options]")
     shelp()
-    jump()
-    print("\tsquidient monitor [options]")
+    console.blank()
+    console.write("\tsquidient monitor [options]")
     mhelp()
 
 def main():
     info = check_latest_package_any_index("squidient")
 
     if not info["up_to_date"]:
-        print(
-            f"? squidient {info['installed']} is outdated. "
-            f"Latest is {info['latest']} on {info['latest_index']}."
+        console.warning(
+            f"squidient {info['installed']} is older than "
+            f"{info['latest']} on {info['latest_index']}."
         )
-        print("Please update to the last version running python3 -m pip install --user -U squidient")
     else:
-        print(
+        console.success(
             f"squidient {info['installed']} is up to date "
             f"(checked: {info['checked']})"
         )
@@ -91,10 +90,10 @@ def main():
         return 1
 
     except Exception as e:
-        print(e)
-        print("Exception")
+        console.error(str(e))
+        console.error("Exception")
         return 1
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main())

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/codecoverage/codecoverage.py

@@ -92,13 +92,12 @@ class CodeCoverage(ABC):
         pass
 
     @abstractmethod
-    def postprocess(self, _print):
+    def postprocess(self, show_details):
         pass
 
     def build_report(self):
         with self.timer("code_coverage.report", comment=self._tool):
-            print("Building code coverage report")
-            print_line()
+            console.section("Building code coverage report")
             f = cc_report_dir + "/cc.json"
             save_json(self._report, f)
             json2js("cc", f)

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/codecoverage/gcovrcodecoverage.py

@@ -35,7 +35,7 @@ def read_base_xml(filename):
 
 def create_package_file(cobertura, package, destination_path):
     filename = "cobertura-{}.xml".format(package.attrib['name'])
-    print("\t\tCreating package file {}".format(filename))
+    console.step("Creating package file {}".format(filename))
     xml_to_write = copy.deepcopy(cobertura)
     packages_node = xml_to_write.find('packages')
 
@@ -64,17 +64,17 @@ class GcovrCodeCoverage(CodeCoverage):
 
     def process(self):
         with self.platform_timer("code_coverage.process", comment=self._tool):
-            print("\tGenerating code coverage file...")
+            console.step("Generating code coverage file")
             if not self._ssh.ssh(system=self._system, platform=self._platform,
                           cmd=self.gcovr(),
                           server_path=True, path=remote_cc_dir):
                 return False
-            print("\tGenerating cobertura report...")
+            console.step("Generating cobertura report")
             if not self._ssh.ssh(system=self._system, platform=self._platform,
                           cmd=self.cobertura(),
                           server_path=True, path=remote_cc_dir):
                 return False
-            print("\tGenerating detailed html report...")
+            console.step("Generating detailed html report")
             if not self._ssh.ssh(system=self._system, platform=self._platform,
                           cmd=self.html(),
                           server_path=True, path=remote_cc_dir):
@@ -107,11 +107,11 @@ class GcovrCodeCoverage(CodeCoverage):
                                        server_path=True, critical=True):
                 raise Exception
 
-    def postprocess(self, _print):
+    def postprocess(self, show_details):
         with self.timer("code_coverage.postprocess", comment=self._tool):
-            print("\tComputing code coverage")
+            console.step("Computing code coverage")
             self.cc_finding()
-            print("\tSplitting cobertura files...")
+            console.step("Splitting cobertura files")
             self.cobertura_split()
 
     def cc_finding(self):

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/codecoverage/intelcodecoverage.py

@@ -105,17 +105,13 @@ class IntelCodeCoverage(CodeCoverage):
 
     def process(self):
         with self.platform_timer("code_coverage.process", comment=self._tool):
-            print("\tMerging code coverage files...")
+            console.step("Merging code coverage files")
             if not self.merge():
-                jump()
-                print("\tMerging has failed!")
-                print_line()
+                console.error("Merging has failed")
                 return False
-            print("\tGenerating code coverage html files...")
+            console.step("Generating code coverage html files")
             if not self.codecov():
-                jump()
-                print("\tCode coverage has failed!")
-                print_line()
+                console.error("Code coverage has failed")
                 return False
             return True
 
@@ -160,14 +156,14 @@ class IntelCodeCoverage(CodeCoverage):
                                        local=cc_report_dir, delete=True, server_path=True, critical=True):
                 raise Exception
 
-    def postprocess(self, _print):
+    def postprocess(self, show_details):
         with self.timer("code_coverage.postprocess", comment=self._tool):
-            print("\tSetting up input file list...")
+            console.step("Setting up input file list")
             self.set_html_files()
-            print("\tBuilding code coverage summary...")
+            console.step("Building code coverage summary")
             self.summary()
-            self.details(True, _print=_print)
-            self.details(False, _print=_print)
+            self.details(True, show_details=show_details)
+            self.details(False, show_details=show_details)
 
     def set_html_files(self):
         files = []
@@ -196,13 +192,13 @@ class IntelCodeCoverage(CodeCoverage):
                 d.append(c)
             self._report["summary"] = d
 
-    def details(self, covered, _print=True):
+    def details(self, covered, show_details=True):
         with self.timer("code_coverage.intel.details", comment="covered={}".format(covered)):
             if covered:
-                print("Building code coverage details (covered files)...")
+                console.step("Building code coverage details for covered files")
                 input = self._covered
             else:
-                print("Building code coverage details (uncovered files)...")
+                console.step("Building code coverage details for uncovered files")
                 input = self._uncovered
 
             rows = _source_rows(input)
@@ -213,8 +209,8 @@ class IntelCodeCoverage(CodeCoverage):
                 if rows[i]["file"] == "":
                     continue
                 f["name"] = rows[i]["file"]
-                if _print:
-                    print("File: " + f["name"])
+                if show_details:
+                    console.step("File " + f["name"])
                 path = rows[i]["path"].split("alya/", 1)[1]
                 f["path"] = path
                 realPath = self.alya_realpath(path)

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/connection/job.py

@@ -110,7 +110,7 @@ class JobThread (threading.Thread):
             #job_number = int(self._ssh.ssh("squeue | wc -l", system=self._system, platform=self._platform).stdout.strip()) - 1
             job_number = max(len(self._ssh.ssh("squeue", system=self._system, platform=self._platform).stdout.split("\n")) - 2, 0)
         except:
-            print("Cannot determine running jobs")
+            console.warning("Cannot determine running jobs")
             logger.warning("Cannot determine job number")
         logger.debug("Job number of " + self._platform + " retrieved with squeue: " + str(job_number))
         return job_number
@@ -152,7 +152,7 @@ class Job:
         dep = ""
         if key_lock != "":
             if asynchronous:
-                print("You cannot launch an asynchronous job with a dependency!")
+                console.error("You cannot launch an asynchronous job with a dependency")
                 raise Exception
             lock_id = self._build_lock.get_lock(key_lock)
             if lock_id != -1:
@@ -162,7 +162,7 @@ class Job:
             self._output_queues[system][platform] = queue.Queue()
             self._job_threads[system][platform] = []
             if self._thread_number[system][platform] < 1:
-                print("Thread number must be greater than 0!")
+                console.error("Thread number must be greater than 0")
                 raise Exception
             for i in range(self._thread_number[system][platform]):
                 self._job_threads[system][platform].append(JobThread(self._ssh, system, platform,
@@ -287,12 +287,11 @@ class Job:
         return job_id
 
     def kill_all(self):
-        jump()
         for system in self._job_threads:
             for platform in self._job_threads[system]:
                 self._ssh.ssh(cmd="scancel --name=" + jtitle + self._hash, system=system, platform=platform)
         self._jobs.clear()
-        print()
+        console.blank()
 
     def job_list(self, force=False):
         timet = time.time()

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/git.py

@@ -24,6 +24,7 @@ import os.path
 import shutil
 
 from .utils.utils import *
+from .utils.message import console
 import getpass
 
 logger = logging.getLogger(logging_context)
@@ -255,7 +256,7 @@ class Git:
                 if not self.clone_https():
                     logger.info("HTTPS clone failed")
                     raise RuntimeError("Git repository cannot be cloned")
-        print("Switching git directory to branch " + self._branch + "...")
+        console.step("Switching git directory to branch " + self._branch)
         if not self.branch():
             raise RuntimeError("Cannot switch to " + self._branch + " and rebase it!")
 

{squidient-2.2.0 → squidient-2.4.0}/src/squidient/init.py

@@ -5,6 +5,7 @@ import shutil
 from pathlib import Path
 from typing import Union
 import squidient
+from .utils.message import console
 
 try:
     from importlib.resources import files as ir_files
@@ -83,4 +84,4 @@ def init_squidient(dist_dir: str=".") -> None:
 
 if __name__ == "__main__":
     init_squidient()
-    print("OK: copied squidient/data/* into current directory")
+    console.success("Copied squidient/data/* into current directory")