gitlab-runner-tart-driver 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

gitlab_runner_tart_driver/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.3"
+__version__ = "0.3.5"

gitlab_runner_tart_driver/cli.py

@@ -9,8 +9,7 @@ from gitlab_runner_tart_driver.commands.run import run
 
 @click.group()
 @click.version_option(__version__, "--version", "-v", message="%(version)s")
-def cli():
-    ...
+def cli(): ...
 
 
 cli.add_command(config)

gitlab_runner_tart_driver/commands/prepare.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import time
 
 import click
 
@@ -115,6 +116,12 @@ def prepare(
     if not p.tart_executor_timeout:
         p.tart_executor_timeout = timeout
 
+    # retrieve the SYSTEM_FAILURE_EXIT_CODE from the environment
+    system_failure_exit_code = os.getenv("SYSTEM_FAILURE_EXIT_CODE", None)
+    if not system_failure_exit_code:
+        click.secho("[WARNING] SYSTEM_FAILURE_EXIT_CODE not set, defaulting to '1'", fg="red")
+        system_failure_exit_code = 1
+
     tart = Tart(exec_path=tart_executable)
     tart_images = tart.list()
     tart_vm_map = {}
@@ -157,7 +164,7 @@ def prepare(
             tart.pull(p.ci_job_image)
         except:
             click.secho(f"[ERROR] Failed to pull image '{p.ci_job_image}'", fg="red")
-            sys.exit(1)
+            sys.exit(system_failure_exit_code)
     else:
         click.echo(f"[INFO] Skipping '{p.ci_job_image}' [pull_policy={p.pull_policy}]")
 
@@ -182,14 +189,14 @@ def prepare(
         click.secho(
             f"[ERROR] The limit of running VMs [{p.tart_max_vm_count}] is exceeded [{len(list_running_vms)}].", fg="red"
         )
-        sys.exit(1)
+        sys.exit(system_failure_exit_code)
 
     click.echo(f"[INFO] Cloning VM instance '{tart_vm_name}' from '{p.ci_job_image}'")
     try:
         tart.clone(p.ci_job_image, tart_vm_name)
     except:
         click.secho(f"[ERROR] failed to clone image f'{p.ci_job_image}'", fg="red")
-        sys.exit(1)
+        sys.exit(system_failure_exit_code)
 
     if cpu or memory or p.display:
         click.echo(f"[INFO] Configuring instance '{tart_vm_name}' from '{p.ci_job_image}'")
@@ -228,24 +235,19 @@ def prepare(
     for v in volumes:
         volume_mounts.append(TartVolume.from_string(v))
 
-    try:
-        tart.run(tart_vm_name, volume_mounts, no_graphics=p.headless, softnet=p.softnet_enabled)
-    except:
-        click.secho(f"[ERROR] Failed to start VM '{tart_vm_name}'", fg="red")
-        sys.exit(1)
-
-    try:
-        ip = tart.ip(tart_vm_name, timeout=p.timeout)
-    except:
-        click.secho(f"[ERROR] Failed to get IP of VM '{tart_vm_name}'", fg="red")
-        sys.exit(1)
-
-    if not ip:
-        click.echo(f"[ERROR] Error, VM was not reacheable after '{p.timeout}' seconds")
-        sys.exit(1)
+    _create_vm(
+        tart_client=tart,
+        vm_name=tart_vm_name,
+        volume_mounts=volume_mounts,
+        params=p,
+        system_failure_exit_code=system_failure_exit_code,
+    )
+    _get_vm_ip(tart_client=tart, vm_name=tart_vm_name, params=p, system_failure_exit_code=system_failure_exit_code)
+    ssh_session = _create_ssh_session(
+        tart_client=tart, vm_name=tart_vm_name, params=p, system_failure_exit_code=system_failure_exit_code
+    )
 
     try:
-        ssh_session = tart.ssh_session(name=p.vm_name(), username=p.ssh_username, password=p.ssh_password)
         ssh_session.exec_ssh_command(
             f"sudo mkdir -p {remote_script_dir} && sudo chown {p.ssh_username}:{p.ssh_username} {remote_script_dir}",
         )
@@ -268,7 +270,7 @@ def prepare(
             )
     except:
         click.secho(f"[ERROR] Failed so prepare VM '{tart_vm_name}'", fg="red")
-        sys.exit(1)
+        sys.exit(system_failure_exit_code)
 
     if p.install_gitlab_runner:
         click.echo(
@@ -278,8 +280,90 @@ def prepare(
             tart.install_gitlab_runner(name=tart_vm_name, username=p.ssh_username, password=p.ssh_password)
         except:
             click.secho(f"[ERROR] Failed to install GitLab Runner '{gitlab_runner_version}'", fg="red")
-            sys.exit(1)
+            sys.exit(system_failure_exit_code)
 
     tart.print_spec(tart_vm_name)
 
     sys.exit(0)
+
+
+def _create_vm(
+    tart_client, vm_name, volume_mounts, params, system_failure_exit_code, max_retries=3, retry_timeout_in_sec=5
+):
+    try:
+        tart_client.run(vm_name, volume_mounts, no_graphics=params.headless, softnet=params.softnet_enabled)
+        # check if vm is listed
+        retry_count = 0
+        while True:
+            try:
+                vm = tart_client.get(vm_name)
+                if vm:
+                    click.echo(f"[INFO] VM '{vm_name}' is running")
+                    break
+                else:
+                    raise Exception("VM not found")
+            except Exception:
+                if retry_count < max_retries:
+                    retry_count += 1
+                    click.secho(
+                        f"[WARNING] Failed to get IP of VM '{vm_name}'. [{retry_count+1}/{max_retries}] Retrying in '{retry_timeout_in_sec}' seconds...",
+                        fg="yellow",
+                    )
+                    time.sleep(retry_timeout_in_sec)
+                else:
+                    click.secho(f"[ERROR] VM '{vm_name}' could not be started.")
+                    sys.exit(system_failure_exit_code)
+    except Exception:
+        click.secho(f"[ERROR] Failed to start VM '{vm_name}'", fg="red")
+        sys.exit(system_failure_exit_code)
+
+
+def _get_vm_ip(tart_client, vm_name, params, system_failure_exit_code, max_retries=3, retry_timeout_in_sec=5):
+    retry_count = 0
+    while True:
+        try:
+            vm_ip_address = tart_client.ip(vm_name, timeout=params.timeout)
+            if vm_ip_address:
+                break
+            else:
+                raise Exception("VM IP not found")
+        except Exception:
+            if retry_count < max_retries:
+                retry_count += 1
+                click.secho(
+                    f"[WARNING] VM with name '{vm_name}' was not found. Retrying in '{retry_timeout_in_sec}' seconds...",
+                    fg="yellow",
+                )
+                time.sleep(retry_timeout_in_sec)
+            else:
+                click.secho(f"[ERROR] Failed to get IP of VM '{vm_name}'.")
+                sys.exit(system_failure_exit_code)
+
+    return vm_ip_address
+
+
+def _create_ssh_session(tart_client, vm_name, params, system_failure_exit_code, max_retries=3, retry_timeout_in_sec=5):
+    ssh_session = None
+    retry_count = 0
+    while True:
+        try:
+            ssh_session = tart_client.ssh_session(
+                name=vm_name, username=params.ssh_username, password=params.ssh_password
+            )
+            if ssh_session:
+                break
+            else:
+                raise Exception("SSH Session could not be established.")
+        except Exception:
+            if retry_count < max_retries:
+                retry_count += 1
+                click.secho(
+                    f"[WARNING] Failed to setup ssh connection with '{vm_name}'. Retrying in '{retry_timeout_in_sec}' seconds...",
+                    fg="yellow",
+                )
+                time.sleep(retry_timeout_in_sec)
+            else:
+                click.secho(f"[ERROR] Failed to setup ssh connection with '{vm_name}'.")
+                sys.exit(system_failure_exit_code)
+
+    return ssh_session

gitlab_runner_tart_driver/commands/run.py

@@ -32,6 +32,18 @@ def run(ssh_timeout, tart_executable, shell, script, stage):
     """Run commands."""
     p = GitLabCustomCommandConfig()
 
+    # retrieve the BUILD_FAILURE_EXIT_CODE from the environment
+    build_failure_exit_code = os.getenv("BUILD_FAILURE_EXIT_CODE", None)
+    if not build_failure_exit_code:
+        click.secho("[WARNING] BUILD_FAILURE_EXIT_CODE not set, defaulting to '1'", fg="red")
+        build_failure_exit_code = 1
+
+    # retrieve the SYSTEM_FAILURE_EXIT_CODE from the environment
+    system_failure_exit_code = os.getenv("SYSTEM_FAILURE_EXIT_CODE", None)
+    if not system_failure_exit_code:
+        click.secho("[WARNING] SYSTEM_FAILURE_EXIT_CODE not set, defaulting to '1'", fg="red")
+        system_failure_exit_code = 1
+
     if not p.tart_executor_shell:
         p.tart_executor_shell = shell
     ######################################################################
@@ -48,13 +60,13 @@ def run(ssh_timeout, tart_executable, shell, script, stage):
             f"[{stage}][ERROR] Could not establish SSH conntection to '{tart_vm_name}' after '{ssh_timeout}' seconds.",
             fg="red",
         )
-        sys.exit(1)
+        sys.exit(system_failure_exit_code)
 
     try:
         ssh_session = tart.ssh_session(name=p.vm_name(), username=p.ssh_username, password=p.ssh_password)
     except:
         click.secho(f"[{stage}][ERROR] Could not establish SSH session with '{p.ssh_username}@{tart_ip}'", fg="red")
-        sys.exit(1)
+        sys.exit(system_failure_exit_code)
 
     remote_temp_dir = "/opt/temp"
     script_name = os.path.basename(script)
@@ -67,4 +79,8 @@ def run(ssh_timeout, tart_executable, shell, script, stage):
     # ssh_session.exec_ssh_command(f"cd {remote_build_dir}")
     script_exit_code = ssh_session.exec_ssh_command(f"{p.shell} -l {remote_script_path}", get_pty=True)
 
-    sys.exit(script_exit_code)
+    if script_exit_code != 0:
+        click.secho(f"[{stage}][ERROR] Script '{script}' failed with exit code '{script_exit_code}'", fg="red")
+        sys.exit(build_failure_exit_code)
+
+    sys.exit(0)

{gitlab_runner_tart_driver-0.3.3.dist-info → gitlab_runner_tart_driver-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlab-runner-tart-driver
-Version: 0.3.3
+Version: 0.3.5
 Home-page: https://gitlab.com/schmieder.matthias/gitlab-runner-tart-driver
 Author: Matthias Schmieder
 Author-email: schmieder.matthias@gmail.com
@@ -13,7 +13,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: click
-Requires-Dist: pydantic >=2.0.0
+Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pydantic-settings
 Requires-Dist: paramiko
 Requires-Dist: psutil

{gitlab_runner_tart_driver-0.3.3.dist-info → gitlab_runner_tart_driver-0.3.5.dist-info}/RECORD

@@ -1,20 +1,20 @@
-gitlab_runner_tart_driver/__init__.py,sha256=8KcCYTXH99C2-gCLuPILJvtT9YftRWJsartIx6TQ2ZY,22
+gitlab_runner_tart_driver/__init__.py,sha256=ThnCuF3X7rsQSd5PAea_jfYA70ZmhLvkFcLBxBPwZnY,22
 gitlab_runner_tart_driver/__main__.py,sha256=FiyMv64vDC-R8i3CGEP9QP48djZFHm7utyChwZJWCeY,107
-gitlab_runner_tart_driver/cli.py,sha256=1eFi9m8L4zcBO5eE4g4EP8phtE1znF54qt-xAigPsOE,596
+gitlab_runner_tart_driver/cli.py,sha256=rCtFzi7i4JUX7VXteBHyII4sEbMzpn8SIpyjyKrQCBI,592
 gitlab_runner_tart_driver/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gitlab_runner_tart_driver/commands/cleanup.py,sha256=nFe7TGdxsVSXNR9onm-3iCjmUo6SzujV5LrA_y439fs,1422
 gitlab_runner_tart_driver/commands/config.py,sha256=cacJ9ms5r3nZGZ_sS2d21Uoz7S-bMjB__lORUmeXZ-4,760
-gitlab_runner_tart_driver/commands/prepare.py,sha256=FCINFtBakV8hiKbHGba3ZWwyNwICp6UV5mKZVCyP8h4,10487
-gitlab_runner_tart_driver/commands/run.py,sha256=2grP79TARRL0xzmppRITbSpIH0WL4YPxlxq1fYQZxlM,2235
+gitlab_runner_tart_driver/commands/prepare.py,sha256=JTxTlPnPELC9C2TMOLqvcu23jOL0sofMxdp9OwpfddU,13964
+gitlab_runner_tart_driver/commands/run.py,sha256=kXQxi8nxv6QQRFzGrlqifLtFtbjXAYgjCiMVqXkb2_A,3065
 gitlab_runner_tart_driver/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gitlab_runner_tart_driver/modules/gitlab_custom_command_config.py,sha256=XvE83QmQUcqtVMYZY3okGOgO86DBX6Y7xhXNenmKxFY,3086
 gitlab_runner_tart_driver/modules/gitlab_custom_driver_config.py,sha256=ujxlzP1ZbfsZyLIsghID5c_jLPSt-1pDazE2B5CPORY,447
 gitlab_runner_tart_driver/modules/tart.py,sha256=rD6IPtpz57dwsPGVBWaeKeVoqI1MoBvdtkqozXgmAxA,9914
 gitlab_runner_tart_driver/modules/utils.py,sha256=7ipKjy_5tC5iW67Na_A9XhF4o2lKcAqO8LRTTSJKNd0,923
 gitlab_runner_tart_driver/scripts/install-gitlab-runner.sh.j2,sha256=-rBzxZ92w7lMrTCVcjMHhlZgqPIK1RJKVoOc2wjZvck,2533
-gitlab_runner_tart_driver-0.3.3.dist-info/LICENSE.txt,sha256=TiYXQpEfbzcPBNGb7k1NslUngq_un-5cxGyT4W1S_f4,3303
-gitlab_runner_tart_driver-0.3.3.dist-info/METADATA,sha256=0rUA-uUqMDTOx657qWQbVNiMhQzuroumn8aIENgVZzs,20706
-gitlab_runner_tart_driver-0.3.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-gitlab_runner_tart_driver-0.3.3.dist-info/entry_points.txt,sha256=xmvpGQf1wvFPy5wqDWXu8k5FD_k9KkCNCkpuworTsr0,82
-gitlab_runner_tart_driver-0.3.3.dist-info/top_level.txt,sha256=JjRzCs2sr24xG4SV_5tIBPpNC1Tec7I4AbK4IKMDqOc,26
-gitlab_runner_tart_driver-0.3.3.dist-info/RECORD,,
+gitlab_runner_tart_driver-0.3.5.dist-info/LICENSE.txt,sha256=TiYXQpEfbzcPBNGb7k1NslUngq_un-5cxGyT4W1S_f4,3303
+gitlab_runner_tart_driver-0.3.5.dist-info/METADATA,sha256=dI44fmUwC6k_iCygHAOKtp31CoAKM7uBme8-OPdYVo0,20705
+gitlab_runner_tart_driver-0.3.5.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+gitlab_runner_tart_driver-0.3.5.dist-info/entry_points.txt,sha256=xmvpGQf1wvFPy5wqDWXu8k5FD_k9KkCNCkpuworTsr0,82
+gitlab_runner_tart_driver-0.3.5.dist-info/top_level.txt,sha256=JjRzCs2sr24xG4SV_5tIBPpNC1Tec7I4AbK4IKMDqOc,26
+gitlab_runner_tart_driver-0.3.5.dist-info/RECORD,,

{gitlab_runner_tart_driver-0.3.3.dist-info → gitlab_runner_tart_driver-0.3.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any