bitbucket-pipeline-runner 0.0.1__tar.gz

bitbucket_pipeline_runner-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Mathieu Lemay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bitbucket_pipeline_runner-0.0.1/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.1
+Name: bitbucket-pipeline-runner
+Version: 0.0.1
+Summary: Run a bitbucket pipeline locally
+Home-page: https://github.com/mathieu-lemay/pipeline-runner
+License: MIT
+Author: Mathieu Lemay
+Author-email: acidrain1@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: GitPython (>=3.1.12,<4.0.0)
+Requires-Dist: PyYAML (>=6.0,<7.0)
+Requires-Dist: appdirs (>=1.4.4,<2.0.0)
+Requires-Dist: boto3 (>=1.28.65,<2.0.0)
+Requires-Dist: click (>=8.0.1,<9.0.0)
+Requires-Dist: coloredlogs (>=15.0,<16.0)
+Requires-Dist: cryptography (>=42.0.2,<43.0.0)
+Requires-Dist: docker (>=7.0.0,<8.0.0)
+Requires-Dist: pydantic (>=2.0,<3.0)
+Requires-Dist: pydantic-settings (>=2.0.3,<3.0.0)
+Requires-Dist: pyfzf (>=0.3.0,<0.4.0)
+Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
+Requires-Dist: python-slugify (>=8.0.0,<9.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: tenacity (>=8.0.1,<9.0.0)
+Project-URL: Repository, https://github.com/mathieu-lemay/pipeline-runner
+Description-Content-Type: text/markdown
+
+# Bitbucket Pipeline Runner
+
+Tool to run Bitbucket Pipelines locally.
+
+## Installation
+```shell
+pip install bitbucket-pipeline-runner
+```
+
+## Basic usage
+To run a pipeline
+```shell
+cd <project-directory>
+pipeline-runner run <pipeline-name>
+```
+
+To list available pipelines
+```shell
+cd <project-directory>
+pipeline-runner list
+```
+
+## Environment variables
+bitbucket pipeline runner already sets all `BITBUCKET_*` enviromnent variables in the step's run enviromnent.
+It will also source any `.env` file in the current directory, for all project specific enviromnent variables.
+
+## Artifacts and logs
+Persistent data like artifacts generated from your pipelines and execution logs can be found in your user's data directory.
+
+On Linux:
+
+    ${XDG_DATA_HOME:-~/.local/share}/pipeline-runner
+
+On macOS:
+
+    ~/Library/Application Support/pipeline-runner
+
+## Caches
+Caches defined in your pipelines are stored in your user's cache directory.
+
+On Linux:
+
+    ${XDG_CACHE_HOME:-~/.cache}/pipeline-runner
+
+On macOS:
+
+    ~/Library/Caches/pipeline-runner
+

bitbucket_pipeline_runner-0.0.1/README.md

@@ -0,0 +1,47 @@
+# Bitbucket Pipeline Runner
+
+Tool to run Bitbucket Pipelines locally.
+
+## Installation
+```shell
+pip install bitbucket-pipeline-runner
+```
+
+## Basic usage
+To run a pipeline
+```shell
+cd <project-directory>
+pipeline-runner run <pipeline-name>
+```
+
+To list available pipelines
+```shell
+cd <project-directory>
+pipeline-runner list
+```
+
+## Environment variables
+bitbucket pipeline runner already sets all `BITBUCKET_*` enviromnent variables in the step's run enviromnent.
+It will also source any `.env` file in the current directory, for all project specific enviromnent variables.
+
+## Artifacts and logs
+Persistent data like artifacts generated from your pipelines and execution logs can be found in your user's data directory.
+
+On Linux:
+
+    ${XDG_DATA_HOME:-~/.local/share}/pipeline-runner
+
+On macOS:
+
+    ~/Library/Application Support/pipeline-runner
+
+## Caches
+Caches defined in your pipelines are stored in your user's cache directory.
+
+On Linux:
+
+    ${XDG_CACHE_HOME:-~/.cache}/pipeline-runner
+
+On macOS:
+
+    ~/Library/Caches/pipeline-runner

bitbucket_pipeline_runner-0.0.1/pipeline_runner/__init__.py

@@ -0,0 +1 @@
+APP_NAME = "pipeline-runner"

bitbucket_pipeline_runner-0.0.1/pipeline_runner/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()

bitbucket_pipeline_runner-0.0.1/pipeline_runner/artifacts.py

@@ -0,0 +1,89 @@
+import io
+import logging
+import os.path
+import tarfile
+from tarfile import TarInfo
+from time import time as ts
+from uuid import UUID
+
+from .config import config
+from .container import ContainerRunner
+from .utils import FileStreamer, get_human_readable_size, safe_extract_tar
+
+logger = logging.getLogger(__name__)
+
+
+class ArtifactManager:
+    def __init__(self, container: ContainerRunner, artifact_directory: str, step_uuid: UUID) -> None:
+        self._container = container
+        self._artifact_directory = artifact_directory
+        self._step_uuid = step_uuid
+
+    def upload(self) -> None:
+        logger.info("Loading artifacts")
+
+        t = ts()
+
+        tar_data = io.BytesIO()
+
+        with tarfile.open(fileobj=tar_data, mode="w|") as tar:
+            for root, _, files in os.walk(self._artifact_directory):
+                for af in files:
+                    full_path = os.path.join(root, af)
+
+                    relpath = os.path.relpath(full_path, self._artifact_directory)
+                    ti = TarInfo(relpath)
+
+                    stat = os.stat(full_path)
+                    ti.size = stat.st_size
+                    ti.mode = stat.st_mode
+
+                    with open(full_path, "rb") as f:
+                        tar.addfile(ti, f)
+
+        res = self._container.put_archive(config.build_dir, tar_data.getvalue())
+        if not res:
+            raise Exception(f"Error loading artifact: {af}")
+
+        t = ts() - t
+
+        logger.info("Artifacts loaded in %.3fs", t)
+
+    def download(self, artifacts: list[str]) -> None:
+        if not artifacts:
+            return
+
+        artifact_file = f"artifacts-{self._step_uuid}.tar"
+        artifact_remote_path = os.path.join(config.build_dir, artifact_file)
+        artifact_local_directory = self._artifact_directory
+
+        logger.info("Collecting artifacts")
+
+        t = ts()
+
+        path_filters = " -o ".join(f"-path './{a}'" for a in artifacts)
+        prepare_artifacts_cmd = ["find", "-type", "f", r"\(", path_filters, r"\)"]
+        prepare_artifacts_cmd += ["|", "tar", "cf", artifact_file, "-C", config.build_dir, "-T", "-"]
+
+        self._container.run_command(prepare_artifacts_cmd)
+        if not self._container.path_exists(artifact_remote_path):
+            logger.info("No artifacts found. Skipping")
+            return
+
+        data, stats = self._container.get_archive(artifact_remote_path, encode_stream=True)
+        logger.debug("artifacts stats: %s", stats)
+
+        # FileStreamer only implements `read` which is all that is needed.
+        with tarfile.open(fileobj=FileStreamer(data), mode="r|") as wrapper_tar:  # type: ignore[abstract]
+            for entry in wrapper_tar:
+                with tarfile.open(fileobj=wrapper_tar.extractfile(entry), mode="r|") as tar:
+                    safe_extract_tar(tar, artifact_local_directory)
+
+        t = ts() - t
+
+        logger.info(
+            "Artifacts saved %s to %s in %.3fs",
+            get_human_readable_size(stats["size"]),
+            artifact_local_directory,
+            t,
+        )

bitbucket_pipeline_runner-0.0.1/pipeline_runner/cache.py

@@ -0,0 +1,224 @@
+import logging
+import os.path
+from tempfile import NamedTemporaryFile
+from time import time as ts
+
+from . import utils
+from .config import config
+from .container import ContainerRunner
+
+logger = logging.getLogger(__name__)
+
+DOCKER_IMAGES_ARCHIVE_FILE_NAME = "images.tar"
+
+
+class CacheManager:
+    def __init__(self, container: ContainerRunner, cache_directory: str, cache_definitions: dict[str, str]) -> None:
+        self._container = container
+        self._cache_directory = cache_directory
+        self._cache_definitions = cache_definitions
+
+        self._ignored_caches = {"docker"}
+
+    def upload(self, cache_names: list[str]) -> None:
+        for name in cache_names:
+            cu = CacheRestoreFactory.get(self._container, self._cache_directory, self._cache_definitions, name)
+            cu.restore()
+
+    def download(self, cache_names: list[str]) -> None:
+        for name in cache_names:
+            cd = CacheSaveFactory.get(self._container, self._cache_directory, self._cache_definitions, name)
+            cd.save()
+
+
+class CacheRestore:
+    def __init__(
+        self, container: ContainerRunner, cache_directory: str, cache_definitions: dict[str, str], cache_name: str
+    ) -> None:
+        self._container = container
+        self._cache_directory = cache_directory
+        self._cache_definitions = cache_definitions
+        self._cache_name = cache_name
+
+    def restore(self) -> None:
+        cache_file = self._get_local_cache_file()
+
+        if not cache_file:
+            logger.info("Cache '%s': Not found: Skipping", self._cache_name)
+            return
+
+        self._upload_cache(cache_file)
+        self._restore_cache()
+
+    def _get_local_cache_file(self) -> str | None:
+        local_cache_archive_path = get_local_cache_archive_path(self._cache_directory, self._cache_name)
+        if not os.path.exists(local_cache_archive_path):
+            return None
+
+        return local_cache_archive_path
+
+    def _upload_cache(self, cache_file: str) -> None:
+        remote_cache_directory = get_remote_temp_directory(self._cache_name)
+        remote_cache_parent_directory = os.path.dirname(remote_cache_directory)
+
+        cache_archive_size = os.path.getsize(cache_file)
+
+        logger.info("Cache '%s': Uploading", self._cache_name)
+
+        t = ts()
+
+        prepare_cache_dir_cmd = (
+            f'[ -d "{remote_cache_directory}" ] && rm -rf "{remote_cache_directory}"; '
+            f'mkdir -p "{remote_cache_parent_directory}"'
+        )
+        res, output = self._container.run_command(prepare_cache_dir_cmd)
+        if res != 0:
+            logger.error("Remote command failed: %s", output.decode())
+            raise Exception(f"Error uploading cache: {self._cache_name}")
+
+        with open(cache_file, "rb") as f:
+            success = self._container.put_archive(remote_cache_parent_directory, f)
+            if not success:
+                raise Exception(f"Error uploading cache: {self._cache_name}")
+
+        t = ts() - t
+
+        logger.info(
+            "Cache '%s': Uploaded %s in %.3fs", self._cache_name, utils.get_human_readable_size(cache_archive_size), t
+        )
+
+    def _restore_cache(self) -> None:
+        temp_dir = get_remote_temp_directory(self._cache_name)
+        target_dir = sanitize_remote_path(self._cache_definitions[self._cache_name])
+
+        logger.info("Cache '%s': Restoring", self._cache_name)
+
+        t = ts()
+
+        restore_cache_script = [
+            f'if [ -e "{target_dir}" ]; then rm -rf "{target_dir}"; fi',
+            f'mkdir -p "$(dirname "{target_dir}")"',
+            f'mv "{temp_dir}" "{target_dir}"',
+        ]
+
+        exit_code, output = self._container.run_command("\n".join(restore_cache_script))
+        if exit_code != 0:
+            raise Exception(f"Error restoring cache: {self._cache_name}: {output.decode()}")
+
+        t = ts() - t
+
+        logger.info("Cache '%s': Restored in %.3fs", self._cache_name, t)
+
+
+class NullCacheRestore(CacheRestore):
+    def restore(self) -> None:
+        logger.info("Cache '%s': Ignoring", self._cache_name)
+
+
+class CacheRestoreFactory:
+    @staticmethod
+    def get(
+        container: ContainerRunner, cache_directory: str, cache_definitions: dict[str, str], cache_name: str
+    ) -> CacheRestore:
+        cls: type[CacheRestore | NullCacheRestore]
+
+        cls = NullCacheRestore if cache_name == "docker" else CacheRestore
+
+        return cls(container, cache_directory, cache_definitions, cache_name)
+
+
+class CacheSave:
+    def __init__(
+        self, container: ContainerRunner, cache_directory: str, cache_definitions: dict[str, str], cache_name: str
+    ) -> None:
+        self._container = container
+        self._cache_directory = cache_directory
+        self._cache_definitions = cache_definitions
+        self._cache_name = cache_name
+
+    def save(self) -> None:
+        remote_cache_directory = self._prepare()
+
+        local_cache_archive_path = get_local_cache_archive_path(self._cache_directory, self._cache_name)
+        self._download(remote_cache_directory, local_cache_archive_path)
+
+    def _prepare(self) -> str:
+        remote_dir = sanitize_remote_path(self._cache_definitions[self._cache_name])
+        target_dir = get_remote_temp_directory(self._cache_name)
+
+        logger.info("Cache '%s': Preparing", self._cache_name)
+
+        t = ts()
+
+        prepare_cache_cmd = f'if [ -e "{remote_dir}" ]; then mv "{remote_dir}" "{target_dir}"; fi'
+
+        exit_code, output = self._container.run_command(prepare_cache_cmd)
+        if exit_code != 0:
+            raise Exception(f"Error preparing cache: {self._cache_name}: {output.decode()}")
+
+        t = ts() - t
+
+        logger.info("Cache '%s': Prepared in %.3fs", self._cache_name, t)
+
+        return target_dir
+
+    def _download(self, src: str, dst: str) -> None:
+        if not self._container.path_exists(src):
+            logger.info("Cache '%s': Not found", self._cache_name)
+            return
+
+        logger.info("Cache '%s': Downloading", self._cache_name)
+
+        t = ts()
+
+        with NamedTemporaryFile(dir=self._cache_directory, delete=False) as f:
+            try:
+                logger.debug("Downloading cache folder '%s' to '%s'", src, f.name)
+                data, _ = self._container.get_archive(src)
+                size = 0
+                for chunk in data:
+                    size += len(chunk)
+                    f.write(chunk)
+            except Exception:
+                logger.exception("Error getting cache from container: %s", self._cache_name)
+                os.unlink(f.name)
+                return
+            else:
+                logger.debug("Moving temp cache archive %s to %s", f.name, dst)
+                os.rename(f.name, dst)
+
+        t = ts() - t
+
+        logger.info("Cache '%s': Downloaded %s in %.3fs", self._cache_name, utils.get_human_readable_size(size), t)
+
+
+class NullCacheSave(CacheSave):
+    def save(self) -> None:
+        logger.info("Cache '%s': Ignoring", self._cache_name)
+
+
+class CacheSaveFactory:
+    @staticmethod
+    def get(
+        container: ContainerRunner, cache_directory: str, cache_definitions: dict[str, str], cache_name: str
+    ) -> CacheSave:
+        cls: type[CacheSave | NullCacheSave]
+
+        cls = NullCacheSave if cache_name == "docker" else CacheSave
+
+        return cls(container, cache_directory, cache_definitions, cache_name)
+
+
+def get_local_cache_archive_path(cache_directory: str, cache_name: str) -> str:
+    return os.path.join(cache_directory, f"{cache_name}.tar")
+
+
+def get_remote_temp_directory(cache_name: str) -> str:
+    return os.path.join(config.caches_dir, cache_name)
+
+
+def sanitize_remote_path(path: str) -> str:
+    if path.startswith("~"):
+        path = path.replace("~", "$HOME", 1)
+
+    return path

bitbucket_pipeline_runner-0.0.1/pipeline_runner/cli.py

@@ -0,0 +1,207 @@
+import logging
+import logging.config
+import os
+import shutil
+import sys
+from importlib.metadata import version
+
+import click
+from pyfzf import FzfPrompt  # type: ignore[import-untyped]
+
+from pipeline_runner.errors import InvalidPipelineError
+
+from . import utils
+from .config import config
+from .parse import parse_pipeline_file
+from .runner import PipelineRunner, PipelineRunRequest
+
+logger = logging.getLogger(__name__)
+
+
+def _init_logger() -> None:
+    logging.config.dictConfig(config.log_config)
+
+
+def _get_pipelines_list(pipeline_file: str) -> list[str]:
+    pipelines_definition = parse_pipeline_file(pipeline_file)
+
+    return pipelines_definition.get_available_pipelines()
+
+
+def _prompt_for_pipeline(pipeline_file: str) -> str | None:
+    pipeline = None
+    pipelines = _get_pipelines_list(pipeline_file)
+
+    try:
+        fzf = FzfPrompt()
+        pipeline = next(iter(fzf.prompt(pipelines)), None)
+        if not pipeline:
+            logger.warning("No pipeline selected")
+    except SystemError:
+        logger.warning("fzf executable not found, disabling interactive pipeline selection.")
+
+    return pipeline
+
+
+@click.group("Pipeline Runner", invoke_without_command=True)
+@click.option(
+    "-V",
+    "--version",
+    "show_version",
+    is_flag=True,
+    help="Print project version and exit.",
+)
+@click.pass_context
+def main(ctx: click.Context, *, show_version: bool) -> None:
+    if show_version:
+        click.echo(f"Pipeline Runner {version('bitbucket_pipeline_runner')}")
+        ctx.exit()
+
+    if not ctx.invoked_subcommand:
+        click.echo(ctx.get_help())
+        ctx.exit(1)
+
+
+@main.command()
+@click.argument("pipeline", required=False)
+@click.option(
+    "-r",
+    "--repository-path",
+    help="Path to the git repository. Defaults to current directory.",
+)
+@click.option(
+    "-s",
+    "--step",
+    "steps",
+    multiple=True,
+    help="Steps to run. If none are specified, they will all be run. Can be specified multiple times.",
+)
+@click.option(
+    "-e",
+    "--env-file",
+    "env_files",
+    multiple=True,
+    help="Read in a file of environment variables. Can be specified multiple times.",
+)
+@click.option(
+    "-c",
+    "--color/--no-color",
+    default=True,
+    help="Enable colored output. Default: True",
+)
+@click.option(
+    "--cpu-limits/--no-cpu-limits",
+    default=False,
+    help="Enable to enforce cpu limits for the runner. Default: False",
+)
+@click.option(
+    "--ssh/--no-ssh",
+    "expose_ssh_agent",
+    default=False,
+    help="Expose the local ssh agent to the container. Default: False",
+)
+def run(
+    pipeline: str | None,
+    repository_path: str,
+    steps: list[str],
+    env_files: list[str],
+    *,
+    color: bool,
+    cpu_limits: bool,
+    expose_ssh_agent: bool,
+) -> None:
+    """
+    Runs the pipeline PIPELINE.
+
+    PIPELINE is the full path to the pipeline to run. Ex: branches.master
+    """
+    config.color = color
+    config.cpu_limits = cpu_limits
+    config.expose_ssh_agent = expose_ssh_agent
+
+    _init_logger()
+
+    if not pipeline:
+        pipeline = _prompt_for_pipeline(os.path.join(repository_path or ".", "bitbucket-pipelines.yml"))
+
+    if not pipeline:
+        logger.error("pipeline not specified")
+        sys.exit(2)
+
+    req = PipelineRunRequest(pipeline, repository_path, steps, env_files)
+
+    runner = PipelineRunner(req)
+    try:
+        runner.run()
+    except Exception:
+        logger.exception("Error running pipeline")
+        sys.exit(1)
+
+
+@main.command("list")
+@click.option(
+    "-r",
+    "--repository-path",
+    help="Path to the git repository. Defaults to current directory.",
+)
+@click.option(
+    "-c",
+    "--color/--no-color",
+    default=True,
+    help="Enable colored output",
+)
+def list_(repository_path: str, *, color: bool) -> None:
+    """
+    List the available pipelines.
+    """
+    config.color = color
+
+    _init_logger()
+
+    pipelines = _get_pipelines_list(os.path.join(repository_path or ".", "bitbucket-pipelines.yml"))
+
+    logger.info("Available pipelines:\n\t%s", "\n\t".join(sorted(pipelines)))
+
+
+@main.command()
+@click.argument("pipeline", required=False)
+@click.option(
+    "-r",
+    "--repository-path",
+    help="Path to the git repository. Defaults to current directory.",
+)
+def parse(pipeline: str | None, repository_path: str) -> None:
+    """
+    Parse the pipeline file.
+    """
+    pipeline_file = os.path.join(repository_path or ".", "bitbucket-pipelines.yml")
+
+    pipelines_definition = parse_pipeline_file(pipeline_file)
+
+    if pipeline:
+        parsed = pipelines_definition.get_pipeline(pipeline)
+        if not parsed:
+            raise InvalidPipelineError(pipeline)
+        click.echo(parsed.model_dump_json(indent=2))
+    else:
+        click.echo(pipelines_definition.model_dump_json(indent=2))
+
+
+@main.command()
+@click.argument("action", type=click.Choice(["clear", "list"]))
+def cache(action: str) -> None:
+    cache_dir = utils.get_cache_directory()
+    if not os.path.isdir(cache_dir):
+        return
+
+    projects = sorted(os.listdir(cache_dir))
+    if action == "list":
+        click.echo("Caches:")
+        click.echo("\n".join(f"\t{p}" for p in projects))
+    elif action == "clear":
+        for p in projects:
+            shutil.rmtree(os.path.join(cache_dir, p))
+
+
+if __name__ == "__main__":
+    main()