gitlab-runner-tart-driver 0.1.1__tar.gz → 0.1.3__tar.gz

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlab-runner-tart-driver
-Version: 0.1.1
+Version: 0.1.3
 Home-page: https://gitlab.com/schmieder.matthias/gitlab-runner-tart-driver
 Author: Matthias Schmieder
 Author-email: schmieder.matthias@gmail.com
@@ -114,15 +114,18 @@ The `config.toml` should be located at `~/.gitlab-runner/config.toml`. Open it a
 [runners.custom]
     config_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     config_args = [ "config", '-x', '/opt/homebrew/bin/tart' ]
+    config_exec_timeout = 200
 
     prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     prepare_args = [ "prepare", '-x', '/opt/homebrew/bin/tart' ]
+    prepare_exec_timeout = 200
 
     run_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     run_args = [ "run", '-x', '/opt/homebrew/bin/tart' ]
 
     cleanup_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     cleanup_args = [ "cleanup", '-x', '/opt/homebrew/bin/tart' ]
+    cleanup_exec_timeout = 200
 ```
 
 Now restart the gitlab-runner with `gitlab-runner restart`
@@ -266,6 +269,32 @@ shutdown_timeout = 0
     cleanup_args = [ "cleanup", '-x', '/opt/homebrew/bin/tart' ]
 ```
 
+### Host Cache and Builds directories
+
+With `tart` it is possible to mount local directories into the VM. Mounting local directories brings a number of benefits like up tp 30% higher IO performance as well as a correct caching mechanism.
+To enable Host-local caching and builds, simply pass `--builds-dir` and `--cache-dir` to the `prepare` command
+
+```ini
+prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+prepare_args = [ "prepare", '-x', '/opt/homebrew/bin/tart', '--concurrency', '2', '--auto-resources', '--cache-dir', '/Users/gitlab/gitlab-runner/cache', '--builds-dir', '/Users/gitlab/gitlab-runner/builds']
+```
+
+### Volume Mounts
+
+Just like with `docker` the `gitlab-runner-tart-driver` allows you to mount any arbitrary host path into your VM. This can be especially useful if you have to mount runner-local resources like shared test-data or additional caches into the executor. You can follow the standard docker syntax for *volume mounts* `--volume /my/local/dir:/opt/remote/dir`. Additionally you can also pass the `ro` flag to make the mount *read-only* i.e.  `--volume /my/local/dir:/opt/remote/dir:ro`.
+
+```ini
+[runners.custom]
+  config_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+  config_args = [ "config" ]
+
+  prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+  prepare_args = [ "prepare",
+    '-x', '/opt/homebrew/bin/tart',
+    '--concurrency', '2',
+    '--auto-resources',
+    '--volume /data/models/:/opt/data/models:ro']
+```
 ### Auto Host Resource Distribution
 
 `tart` images come with a pre-defined number of CPUs and Memory allocation. Typical numbers for default images are `cpu_count=4` and `memory=8192`. With the concurrency limitation of two VMs/images running at the same time this might not utilize your host system completely.
@@ -324,11 +353,13 @@ Usage: gitlab-runner-tart-driver prepare [OPTIONS]
   Prepare the environment and start the tart VM.
 
 Options:
+  --default-ssh-username TEXT     username to login to a tart vm
+  --default-ssh-password TEXT     password to login to a tart vm
   --pull-policy [always|if-not-present|never]
                                   define how runners pull tart images from
                                   registries
   --registry-username TEXT        username to login to a oci registry
-  --registry-password TEXT        passowrd to login to a oci registry
+  --registry-password TEXT        password to login to a oci registry
   --registry TEXT                 username to login to a oci registry
   --cpu INTEGER                   Number of CPUs associated to VM
   --memory INTEGER                VM memory size in megabytes associated to VM
@@ -340,8 +371,14 @@ Options:
   --concurrency INTEGER           Number of concurrent processes that are
                                   supported. ATTENTION tart currently only
                                   support two concurrent VMs
+  --cache-dir TEXT                Caching dir to be used.
+  --builds-dir TEXT               Path to the builds directory.
+  --timeout INTEGER               Timeout in seconds for the VM to be
+                                  reachable via SSH.
+  --volume TEXT                   Volume mount definition with docker syntax.
+                                  <host_dir>:<vm_dir>[:ro]
   -x, --tart-executable TEXT      Path to the tart executable.
-  --help                          Show this message and exit.
+  --help                          Show this message and exit.                      Show this message and exit.
 ```
 
 ### Command `run`
@@ -353,7 +390,7 @@ Usage: gitlab-runner-tart-driver run [OPTIONS] SCRIPT STAGE
 
 Options:
   --default-ssh-username TEXT  username to login to a tart vm
-  --default-ssh-password TEXT  passowrd to login to a tart vm
+  --default-ssh-password TEXT  password to login to a tart vm
   -x, --tart-executable TEXT   Path to the tart executable.
   --shell TEXT                 Path to the shell to be used for commands over
                                ssh.

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/README.md

@@ -99,15 +99,18 @@ The `config.toml` should be located at `~/.gitlab-runner/config.toml`. Open it a
 [runners.custom]
     config_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     config_args = [ "config", '-x', '/opt/homebrew/bin/tart' ]
+    config_exec_timeout = 200
 
     prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     prepare_args = [ "prepare", '-x', '/opt/homebrew/bin/tart' ]
+    prepare_exec_timeout = 200
 
     run_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     run_args = [ "run", '-x', '/opt/homebrew/bin/tart' ]
 
     cleanup_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     cleanup_args = [ "cleanup", '-x', '/opt/homebrew/bin/tart' ]
+    cleanup_exec_timeout = 200
 ```
 
 Now restart the gitlab-runner with `gitlab-runner restart`
@@ -251,6 +254,32 @@ shutdown_timeout = 0
     cleanup_args = [ "cleanup", '-x', '/opt/homebrew/bin/tart' ]
 ```
 
+### Host Cache and Builds directories
+
+With `tart` it is possible to mount local directories into the VM. Mounting local directories brings a number of benefits like up tp 30% higher IO performance as well as a correct caching mechanism.
+To enable Host-local caching and builds, simply pass `--builds-dir` and `--cache-dir` to the `prepare` command
+
+```ini
+prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+prepare_args = [ "prepare", '-x', '/opt/homebrew/bin/tart', '--concurrency', '2', '--auto-resources', '--cache-dir', '/Users/gitlab/gitlab-runner/cache', '--builds-dir', '/Users/gitlab/gitlab-runner/builds']
+```
+
+### Volume Mounts
+
+Just like with `docker` the `gitlab-runner-tart-driver` allows you to mount any arbitrary host path into your VM. This can be especially useful if you have to mount runner-local resources like shared test-data or additional caches into the executor. You can follow the standard docker syntax for *volume mounts* `--volume /my/local/dir:/opt/remote/dir`. Additionally you can also pass the `ro` flag to make the mount *read-only* i.e.  `--volume /my/local/dir:/opt/remote/dir:ro`.
+
+```ini
+[runners.custom]
+  config_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+  config_args = [ "config" ]
+
+  prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+  prepare_args = [ "prepare",
+    '-x', '/opt/homebrew/bin/tart',
+    '--concurrency', '2',
+    '--auto-resources',
+    '--volume /data/models/:/opt/data/models:ro']
+```
 ### Auto Host Resource Distribution
 
 `tart` images come with a pre-defined number of CPUs and Memory allocation. Typical numbers for default images are `cpu_count=4` and `memory=8192`. With the concurrency limitation of two VMs/images running at the same time this might not utilize your host system completely.
@@ -309,11 +338,13 @@ Usage: gitlab-runner-tart-driver prepare [OPTIONS]
   Prepare the environment and start the tart VM.
 
 Options:
+  --default-ssh-username TEXT     username to login to a tart vm
+  --default-ssh-password TEXT     password to login to a tart vm
   --pull-policy [always|if-not-present|never]
                                   define how runners pull tart images from
                                   registries
   --registry-username TEXT        username to login to a oci registry
-  --registry-password TEXT        passowrd to login to a oci registry
+  --registry-password TEXT        password to login to a oci registry
   --registry TEXT                 username to login to a oci registry
   --cpu INTEGER                   Number of CPUs associated to VM
   --memory INTEGER                VM memory size in megabytes associated to VM
@@ -325,8 +356,14 @@ Options:
   --concurrency INTEGER           Number of concurrent processes that are
                                   supported. ATTENTION tart currently only
                                   support two concurrent VMs
+  --cache-dir TEXT                Caching dir to be used.
+  --builds-dir TEXT               Path to the builds directory.
+  --timeout INTEGER               Timeout in seconds for the VM to be
+                                  reachable via SSH.
+  --volume TEXT                   Volume mount definition with docker syntax.
+                                  <host_dir>:<vm_dir>[:ro]
   -x, --tart-executable TEXT      Path to the tart executable.
-  --help                          Show this message and exit.
+  --help                          Show this message and exit.                      Show this message and exit.
 ```
 
 ### Command `run`
@@ -338,7 +375,7 @@ Usage: gitlab-runner-tart-driver run [OPTIONS] SCRIPT STAGE
 
 Options:
   --default-ssh-username TEXT  username to login to a tart vm
-  --default-ssh-password TEXT  passowrd to login to a tart vm
+  --default-ssh-password TEXT  password to login to a tart vm
   -x, --tart-executable TEXT   Path to the tart executable.
   --shell TEXT                 Path to the shell to be used for commands over
                                ssh.

gitlab-runner-tart-driver-0.1.3/gitlab_runner_tart_driver/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.3"

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/gitlab_runner_tart_driver/commands/config.py

@@ -14,8 +14,8 @@ def config():
     https://docs.gitlab.com/runner/executors/custom.html#config."""
 
     c = GitLabCustomDriverConfig(
-        builds_dir="/Users/admin/build",
-        cache_dir="/Users/admin/cache",
+        builds_dir="/opt/builds",
+        cache_dir="/opt/cache",
         hostname=socket.gethostname(),
         driver=GitLabCustomDriver(name="tart", version=__version__),
     )

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/gitlab_runner_tart_driver/commands/prepare.py

@@ -1,14 +1,22 @@
+import os
 import sys
 
 import click
 
 from gitlab_runner_tart_driver.modules.gitlab_custom_command_config import GitLabCustomCommandConfig
 from gitlab_runner_tart_driver.modules.tart import Tart
+from gitlab_runner_tart_driver.modules.tart import TartVolume
 from gitlab_runner_tart_driver.modules.utils import get_host_spec
 from gitlab_runner_tart_driver.modules.utils import print_host_spec
 
 
 @click.command()
+@click.option(
+    "--default-ssh-username", default="admin", required=False, type=str, help="username to login to a tart vm"
+)
+@click.option(
+    "--default-ssh-password", default="admin", required=False, type=str, help="password to login to a tart vm"
+)
 @click.option(
     "--pull-policy",
     default="if-not-present",
@@ -16,7 +24,7 @@ from gitlab_runner_tart_driver.modules.utils import print_host_spec
     help="define how runners pull tart images from registries",
 )
 @click.option("--registry-username", required=False, default=None, type=str, help="username to login to a oci registry")
-@click.option("--registry-password", required=False, default=None, type=str, help="passowrd to login to a oci registry")
+@click.option("--registry-password", required=False, default=None, type=str, help="password to login to a oci registry")
 @click.option("--registry", required=False, default=None, type=str, help="username to login to a oci registry")
 @click.option("--cpu", required=False, default=None, type=int, help="Number of CPUs associated to VM")
 @click.option("--memory", required=False, default=None, type=int, help="VM memory size in megabytes associated to VM")
@@ -42,8 +50,40 @@ from gitlab_runner_tart_driver.modules.utils import print_host_spec
     type=int,
     help="Number of concurrent processes that are supported. ATTENTION tart currently only support two concurrent VMs",
 )
+@click.option(
+    "--cache-dir",
+    required=False,
+    default=None,
+    type=str,
+    help="Caching dir to be used.",
+)
+@click.option(
+    "--builds-dir",
+    required=False,
+    default=None,
+    type=str,
+    help="Path to the builds directory.",
+)
+@click.option(
+    "--timeout",
+    required=False,
+    default=60,
+    type=int,
+    help="Timeout in seconds for the VM to be reachable via SSH.",
+)
+@click.option(
+    "--volume",
+    "volumes",
+    required=False,
+    default=[],
+    type=str,
+    multiple=True,
+    help="Volume mount definition with docker syntax. <host_dir>:<vm_dir>[:ro]",
+)
 @click.option("-x", "--tart-executable", required=False, default="tart", type=str, help="Path to the tart executable.")
 def prepare(
+    default_ssh_username,
+    default_ssh_password,
     registry_username,
     registry_password,
     registry,
@@ -53,6 +93,10 @@ def prepare(
     pull_policy,
     auto_resources,
     concurrency,
+    cache_dir,
+    builds_dir,
+    timeout,
+    volumes,
     tart_executable,
 ):
     """Prepare the environment and start the tart VM."""
@@ -61,6 +105,11 @@ def prepare(
 
     p = GitLabCustomCommandConfig()
 
+    if not p.tart_ssh_username:
+        p.tart_ssh_username = default_ssh_username
+    if not p.tart_ssh_password:
+        p.tart_ssh_password = default_ssh_password
+
     tart = Tart(exec_path=tart_executable)
     tart_images = tart.list()
     tart_vm_map = {}
@@ -78,7 +127,6 @@ def prepare(
         tart.login(username=p.ci_registry_user, password=p.ci_registry_password, host=p.ci_registry)
     except:
         click.secho(f"[ERROR] Failed to login to '{p.ci_registry}'", fg="red")
-        sys.exit(1)
 
     if registry_username and registry_password and registry:
         click.echo(f"[INFO] Logging into OCI Registry '{registry}'")
@@ -86,7 +134,6 @@ def prepare(
             tart.login(username=registry_username, password=registry_password, host=registry)
         except:
             click.secho(f"[ERROR] Failed to login to '{registry}'", fg="red")
-            sys.exit(1)
 
     if p.tart_registry_username and p.tart_registry_password and p.tart_registry:
         click.echo(f"[INFO] Logging into custom OCI Registry '{p.tart_registry}'")
@@ -94,7 +141,6 @@ def prepare(
             tart.login(username=p.tart_registry_username, password=p.tart_registry_password, host=p.tart_registry)
         except:
             click.secho(f"[ERROR] Failed to login to '{p.tart_registry}'", fg="red")
-            sys.exit(1)
 
     ######################################################################
     # PULL
@@ -140,8 +186,58 @@ def prepare(
 
     click.echo(f"[INFO] Starting VM instance '{tart_vm_name}'")
 
-    tart.run(tart_vm_name)
-    tart.ip(tart_vm_name, timeout=30)
+    remote_build_dir = "/opt/builds"
+    remote_script_dir = "/opt/temp"
+    remote_cache_dir = "/opt/cache"
+
+    volume_mounts = []
+    if cache_dir:
+        cache_dir = os.path.abspath(os.path.expanduser(cache_dir))
+        os.makedirs(cache_dir, exist_ok=True)
+        click.echo(f"[INFO] Cache directory set to '{cache_dir}'")
+        volume_mounts.append(TartVolume(source=cache_dir, dest=remote_cache_dir, name="cache", ro=False))
+    if builds_dir:
+        # Concurrency compatible builds directory
+        # see https://docs.gitlab.com/runner/executors/shell.html#run-scripts-as-a-privileged-user
+        # <builds_dir>/<short-token>/<concurrent-id>/<namespace>/<project-name>.
+        builds_dir = os.path.join(
+            os.path.abspath(os.path.expanduser(builds_dir)), p.ci_runner_short_token, p.ci_concurrent_project_id
+        )
+        os.makedirs(builds_dir, exist_ok=True)
+        click.echo(f"[INFO] Builds directory set to '{builds_dir}'")
+        volume_mounts.append(TartVolume(source=builds_dir, dest=remote_build_dir, name="builds", ro=False))
+
+    for v in volumes:
+        volume_mounts.append(TartVolume.from_string(v))
+
+    tart.run(tart_vm_name, volume_mounts)
+    ip = tart.ip(tart_vm_name, timeout=timeout)
+    if not ip:
+        click.echo(f"[ERROR] Error, VM was not reacheable after '{timeout}' seconds")
+        sys.exit(1)
+
+    ssh_session = tart.ssh_session(name=p.vm_name(), username=p.tart_ssh_username, password=p.tart_ssh_password)
+    ssh_session.exec_ssh_command(
+        f"sudo mkdir -p {remote_script_dir} && sudo chown {p.tart_ssh_username}:{p.tart_ssh_username} {remote_script_dir}",
+    )
+
+    for volume in volume_mounts:
+        click.echo("[INFO] Setting up volume mount '{volume.name}'")
+        ssh_session.exec_ssh_command(
+            f"sudo mkdir -p $(dirname {volume.dest}); sudo ln -sf '/Volumes/My Shared Files/{volume.name}' {volume.dest}",
+        )
+
+    # if cache and builds volumes are not mounted, make sure to create them locally inside the VM
+    if not cache_dir:
+        ssh_session.exec_ssh_command(
+            f"sudo mkdir -p {remote_cache_dir} && sudo chown {p.tart_ssh_username}:{p.tart_ssh_username} {remote_cache_dir}",
+        )
+
+    if not builds_dir:
+        ssh_session.exec_ssh_command(
+            f"sudo mkdir -p {remote_build_dir} && sudo chown {p.tart_ssh_username}:{p.tart_ssh_username} {remote_build_dir}",
+        )
+
     tart.print_spec(tart_vm_name)
 
     sys.exit(0)

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/gitlab_runner_tart_driver/commands/run.py

@@ -2,8 +2,6 @@ import os
 import sys
 
 import click
-from paramiko import MissingHostKeyPolicy
-from paramiko import SSHClient
 
 from gitlab_runner_tart_driver.modules.gitlab_custom_command_config import GitLabCustomCommandConfig
 from gitlab_runner_tart_driver.modules.tart import Tart
@@ -14,7 +12,7 @@ from gitlab_runner_tart_driver.modules.tart import Tart
     "--default-ssh-username", default="admin", required=False, type=str, help="username to login to a tart vm"
 )
 @click.option(
-    "--default-ssh-password", default="admin", required=False, type=str, help="passowrd to login to a tart vm"
+    "--default-ssh-password", default="admin", required=False, type=str, help="password to login to a tart vm"
 )
 @click.option(
     "-x",
@@ -50,46 +48,19 @@ def run(default_ssh_username, default_ssh_password, tart_executable, shell, scri
     tart_ip = tart.ip(tart_vm_name, timeout=30)
     click.echo(f"[INFO] Establishing SSH conntection to '{p.tart_ssh_username}@{tart_ip}'")
 
-    ssh_client = SSHClient()
-    ssh_client.set_missing_host_key_policy(MissingHostKeyPolicy())
-    ssh_client.connect(tart_ip, username=p.tart_ssh_username, password=p.tart_ssh_password)
+    ssh_session = tart.ssh_session(name=p.vm_name(), username=p.tart_ssh_username, password=p.tart_ssh_password)
 
-    click.echo("[INFO] Preparing workspace'")
-    remote_build_dir = "/opt/build/"
-    remote_script_dir = "/opt/temp"
-    remote_cache_dir = "/opt/cache"
-    exec_ssh_command(
-        ssh_client,
-        f"sudo mkdir -p {remote_build_dir} && sudo chown {p.tart_ssh_username}:{p.tart_ssh_username} {remote_build_dir}",
-    )
-    exec_ssh_command(
-        ssh_client,
-        f"sudo mkdir -p {remote_script_dir} && sudo chown {p.tart_ssh_username}:{p.tart_ssh_username} {remote_script_dir}",
-    )
-    exec_ssh_command(
-        ssh_client,
-        f"sudo mkdir -p {remote_cache_dir} && sudo chown {p.tart_ssh_username}:{p.tart_ssh_username} {remote_cache_dir}",
-    )
+    click.echo("[INFO] Preparing workspace")
+    remote_temp_dir = "/opt/temp"
 
     script_name = os.path.basename(script)
-    remote_script_path = os.path.join(remote_script_dir, stage + "-" + script_name)
+    remote_script_path = os.path.join(remote_temp_dir, stage + "-" + script_name)
 
-    sftp = ssh_client.open_sftp()
+    sftp = ssh_session.ssh_client.open_sftp()
     sftp.put(script, remote_script_path)
     sftp.close()
 
-    exec_ssh_command(ssh_client, f"cd {remote_build_dir}")
-    script_exit_code = exec_ssh_command(ssh_client, f"{shell} -l {remote_script_path}", get_pty=True)
+    # ssh_session.exec_ssh_command(f"cd {remote_build_dir}")
+    script_exit_code = ssh_session.exec_ssh_command(f"{shell} -l {remote_script_path}", get_pty=True)
 
     sys.exit(script_exit_code)
-
-
-def exec_ssh_command(ssh_client, command, get_pty=True):
-    """Executes an ssh command and prints it's output continously to stdout/stderr"""
-    _, stdout, stderr = ssh_client.exec_command(command, get_pty=get_pty)
-    for line in iter(stdout.readline, ""):
-        click.echo(line, nl=False)
-    for line in iter(stderr.readline, ""):
-        click.echo(line, nl=False, err=True)
-
-    return stdout.channel.recv_exit_status()

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/gitlab_runner_tart_driver/modules/gitlab_custom_command_config.py

@@ -10,6 +10,8 @@ class GitLabCustomCommandConfig(BaseSettings):
     ci_pipeline_id: str
     ci_job_id: str
     ci_concurrent_id: str
+    ci_concurrent_project_id: str
+    ci_runner_short_token: str
     ci_project_name: str
     ci_registry: str
     ci_registry_user: str

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/gitlab_runner_tart_driver/modules/tart.py

@@ -1,6 +1,10 @@
+import os
 import re
 import subprocess
 
+import click
+from paramiko import MissingHostKeyPolicy
+from paramiko import SSHClient
 from pydantic import BaseModel
 from pydantic import Field
 from tabulate import tabulate
@@ -22,6 +26,50 @@ class TartVmSpec(BaseModel):
     ip_address: str
 
 
+class TartVolume(BaseModel):
+    name: str
+    source: str
+    dest: str
+    ro: bool = Field(default=False)
+
+    @classmethod
+    def from_string(cls, value):
+        components = value.split(":")
+        if len(components) < 2:
+            raise ValueError(f"'{value}' is not a valid volume mount definition")
+
+        source = os.path.abspath(os.path.expanduser(components[0])).rstrip("/")
+        dest = components[1].rstrip("/")
+        name = dest.strip("/").replace("/", "__")
+        ro = False
+        if len(components) > 2:
+            if "ro" == components[2]:
+                ro = True
+            else:
+                raise ValueError(f"'{components[2]}' flag unknown")
+
+        return cls(name=name, source=source, dest=dest, ro=ro)
+
+
+class TartSshSession:
+    def __init__(self, username, password, ip):
+        self.user = username
+        self.ip = ip
+        self.password = password
+        self.ssh_client = SSHClient()
+        self.ssh_client.set_missing_host_key_policy(MissingHostKeyPolicy())
+        self.ssh_client.connect(ip, username=username, password=password)
+
+    def exec_ssh_command(self, command, get_pty=True):
+        """Executes an ssh command and prints it's output continously to stdout/stderr"""
+        _, stdout, stderr = self.ssh_client.exec_command(command, get_pty=get_pty)
+        for line in iter(stdout.readline, ""):
+            click.echo(line, nl=False)
+        for line in iter(stderr.readline, ""):
+            click.echo(line, nl=False, err=True)
+        return stdout.channel.recv_exit_status()
+
+
 class Tart(object):
     def __init__(self, exec_path="tart"):
         self.tart_executable = exec_path
@@ -102,12 +150,18 @@ class Tart(object):
         """stops a given tart VM"""
         self.exec(["stop", name])
 
-    def run(self, name, no_graphics=True) -> None:
+    def run(self, name: str, volumes: list[TartVolume] = [], no_graphics=True) -> None:
         """starts a given tart VM"""
         args = ["run", name]
         if no_graphics:
             args.append("--no-graphics")
 
+        if volumes:
+            for d in volumes:
+                source_path = os.path.abspath(os.path.expanduser(d.source))
+                if d.ro:
+                    source_path = f"{source_path}:ro"
+                args.extend(["--dir", f"{d.name}:{source_path}"])
         try:
             self.spawn_exec(args)
         except Exception as e:
@@ -157,3 +211,10 @@ class Tart(object):
         if cmd:
             exec_cmd.extend(cmd)
         subprocess.Popen(exec_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+    def ssh_session(self, name, username, password) -> TartSshSession:
+        ip = self.ip(name=name)
+        if not ip:
+            raise ValueError("Could not retrievew IP")
+
+        return TartSshSession(ip=ip, username=username, password=password)

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/gitlab_runner_tart_driver.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlab-runner-tart-driver
-Version: 0.1.1
+Version: 0.1.3
 Home-page: https://gitlab.com/schmieder.matthias/gitlab-runner-tart-driver
 Author: Matthias Schmieder
 Author-email: schmieder.matthias@gmail.com
@@ -114,15 +114,18 @@ The `config.toml` should be located at `~/.gitlab-runner/config.toml`. Open it a
 [runners.custom]
     config_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     config_args = [ "config", '-x', '/opt/homebrew/bin/tart' ]
+    config_exec_timeout = 200
 
     prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     prepare_args = [ "prepare", '-x', '/opt/homebrew/bin/tart' ]
+    prepare_exec_timeout = 200
 
     run_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     run_args = [ "run", '-x', '/opt/homebrew/bin/tart' ]
 
     cleanup_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
     cleanup_args = [ "cleanup", '-x', '/opt/homebrew/bin/tart' ]
+    cleanup_exec_timeout = 200
 ```
 
 Now restart the gitlab-runner with `gitlab-runner restart`
@@ -266,6 +269,32 @@ shutdown_timeout = 0
     cleanup_args = [ "cleanup", '-x', '/opt/homebrew/bin/tart' ]
 ```
 
+### Host Cache and Builds directories
+
+With `tart` it is possible to mount local directories into the VM. Mounting local directories brings a number of benefits like up tp 30% higher IO performance as well as a correct caching mechanism.
+To enable Host-local caching and builds, simply pass `--builds-dir` and `--cache-dir` to the `prepare` command
+
+```ini
+prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+prepare_args = [ "prepare", '-x', '/opt/homebrew/bin/tart', '--concurrency', '2', '--auto-resources', '--cache-dir', '/Users/gitlab/gitlab-runner/cache', '--builds-dir', '/Users/gitlab/gitlab-runner/builds']
+```
+
+### Volume Mounts
+
+Just like with `docker` the `gitlab-runner-tart-driver` allows you to mount any arbitrary host path into your VM. This can be especially useful if you have to mount runner-local resources like shared test-data or additional caches into the executor. You can follow the standard docker syntax for *volume mounts* `--volume /my/local/dir:/opt/remote/dir`. Additionally you can also pass the `ro` flag to make the mount *read-only* i.e.  `--volume /my/local/dir:/opt/remote/dir:ro`.
+
+```ini
+[runners.custom]
+  config_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+  config_args = [ "config" ]
+
+  prepare_exec = "/opt/homebrew/bin/gitlab-runner-tart-driver"
+  prepare_args = [ "prepare",
+    '-x', '/opt/homebrew/bin/tart',
+    '--concurrency', '2',
+    '--auto-resources',
+    '--volume /data/models/:/opt/data/models:ro']
+```
 ### Auto Host Resource Distribution
 
 `tart` images come with a pre-defined number of CPUs and Memory allocation. Typical numbers for default images are `cpu_count=4` and `memory=8192`. With the concurrency limitation of two VMs/images running at the same time this might not utilize your host system completely.
@@ -324,11 +353,13 @@ Usage: gitlab-runner-tart-driver prepare [OPTIONS]
   Prepare the environment and start the tart VM.
 
 Options:
+  --default-ssh-username TEXT     username to login to a tart vm
+  --default-ssh-password TEXT     password to login to a tart vm
   --pull-policy [always|if-not-present|never]
                                   define how runners pull tart images from
                                   registries
   --registry-username TEXT        username to login to a oci registry
-  --registry-password TEXT        passowrd to login to a oci registry
+  --registry-password TEXT        password to login to a oci registry
   --registry TEXT                 username to login to a oci registry
   --cpu INTEGER                   Number of CPUs associated to VM
   --memory INTEGER                VM memory size in megabytes associated to VM
@@ -340,8 +371,14 @@ Options:
   --concurrency INTEGER           Number of concurrent processes that are
                                   supported. ATTENTION tart currently only
                                   support two concurrent VMs
+  --cache-dir TEXT                Caching dir to be used.
+  --builds-dir TEXT               Path to the builds directory.
+  --timeout INTEGER               Timeout in seconds for the VM to be
+                                  reachable via SSH.
+  --volume TEXT                   Volume mount definition with docker syntax.
+                                  <host_dir>:<vm_dir>[:ro]
   -x, --tart-executable TEXT      Path to the tart executable.
-  --help                          Show this message and exit.
+  --help                          Show this message and exit.                      Show this message and exit.
 ```
 
 ### Command `run`
@@ -353,7 +390,7 @@ Usage: gitlab-runner-tart-driver run [OPTIONS] SCRIPT STAGE
 
 Options:
   --default-ssh-username TEXT  username to login to a tart vm
-  --default-ssh-password TEXT  passowrd to login to a tart vm
+  --default-ssh-password TEXT  password to login to a tart vm
   -x, --tart-executable TEXT   Path to the tart executable.
   --shell TEXT                 Path to the shell to be used for commands over
                                ssh.

{gitlab-runner-tart-driver-0.1.1 → gitlab-runner-tart-driver-0.1.3}/setup.py

@@ -10,7 +10,7 @@ this_directory = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f:
     long_description = f.read()
 
-version = "0.1.1"
+version = "0.1.3"
 
 version = f"{version}{os.environ.get('PIP_VERSION_POSTFIX','')}"
 

gitlab-runner-tart-driver-0.1.1/gitlab_runner_tart_driver/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.1"