pypeline-runner 1.7.0__py3-none-any.whl → 1.8.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "1.7.0"
+__version__ = "1.8.0"

pypeline/kickstart/create.py

@@ -1,55 +1,55 @@
-import shutil
-from pathlib import Path
-from typing import List, Optional, Union
-
-from py_app_dev.core.exceptions import UserNotificationException
-from py_app_dev.core.logging import logger
-
-
-class ProjectBuilder:
-    def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
-        self.project_dir = project_dir
-        self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
-
-        self.dirs: List[Path] = []
-        self.check_target_directory_flag = True
-
-    def with_disable_target_directory_check(self) -> "ProjectBuilder":
-        self.check_target_directory_flag = False
-        return self
-
-    def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
-        self.dirs.append(self.resolve_file_path(dir))
-        return self
-
-    def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
-        return [self.resolve_file_path(file) for file in files]
-
-    def resolve_file_path(self, file: Union[Path, str]) -> Path:
-        return self.input_dir.joinpath(file) if isinstance(file, str) else file
-
-    @staticmethod
-    def _check_target_directory(project_dir: Path) -> None:
-        if project_dir.is_dir() and any(project_dir.iterdir()):
-            raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
-
-    def build(self) -> None:
-        if self.check_target_directory_flag:
-            self._check_target_directory(self.project_dir)
-        for dir in self.dirs:
-            shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
-
-
-class KickstartProject:
-    def __init__(self, project_dir: Path, force: bool = False) -> None:
-        self.logger = logger.bind()
-        self.project_dir = project_dir
-        self.force = force
-
-    def run(self) -> None:
-        self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
-        project_builder = ProjectBuilder(self.project_dir)
-        if self.force:
-            project_builder.with_disable_target_directory_check()
-        project_builder.with_dir("project")
-        project_builder.build()
+import shutil
+from pathlib import Path
+from typing import List, Optional, Union
+
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.logging import logger
+
+
+class ProjectBuilder:
+    def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
+        self.project_dir = project_dir
+        self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
+
+        self.dirs: List[Path] = []
+        self.check_target_directory_flag = True
+
+    def with_disable_target_directory_check(self) -> "ProjectBuilder":
+        self.check_target_directory_flag = False
+        return self
+
+    def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
+        self.dirs.append(self.resolve_file_path(dir))
+        return self
+
+    def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
+        return [self.resolve_file_path(file) for file in files]
+
+    def resolve_file_path(self, file: Union[Path, str]) -> Path:
+        return self.input_dir.joinpath(file) if isinstance(file, str) else file
+
+    @staticmethod
+    def _check_target_directory(project_dir: Path) -> None:
+        if project_dir.is_dir() and any(project_dir.iterdir()):
+            raise UserNotificationException(f"Project directory '{project_dir}' is not empty. Use --force to override.")
+
+    def build(self) -> None:
+        if self.check_target_directory_flag:
+            self._check_target_directory(self.project_dir)
+        for dir in self.dirs:
+            shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
+
+
+class KickstartProject:
+    def __init__(self, project_dir: Path, force: bool = False) -> None:
+        self.logger = logger.bind()
+        self.project_dir = project_dir
+        self.force = force
+
+    def run(self) -> None:
+        self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
+        project_builder = ProjectBuilder(self.project_dir)
+        if self.force:
+            project_builder.with_disable_target_directory_check()
+        project_builder.with_dir("project")
+        project_builder.build()

pypeline/kickstart/templates/project/bootstrap.py

@@ -0,0 +1,461 @@
+#!/usr/bin/python3
+# Source: https://github.com/avengineers/bootstrap
+# Tag: v1.15.1
+import configparser
+import ensurepip
+import hashlib
+import json
+import logging
+import os
+import re
+import subprocess  # nosec
+import sys
+import tempfile
+import venv
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from functools import total_ordering
+from pathlib import Path
+from typing import List, Optional, Tuple
+from urllib.parse import urlparse
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("bootstrap")
+
+
+bootstrap_json_path = Path.cwd() / "bootstrap.json"
+if bootstrap_json_path.exists():
+    with bootstrap_json_path.open("r") as f:
+        config = json.load(f)
+    package_manager = config.get("python_package_manager", "poetry>=2.0")
+    package_manager_args = config.get("python_package_manager_args", [])
+else:
+    package_manager = "poetry>=2.0"
+    package_manager_args = []
+
+
+@total_ordering
+class Version:
+    def __init__(self, version_str: str) -> None:
+        self.version = self.parse_version(version_str)
+
+    @staticmethod
+    def parse_version(version_str: str) -> Tuple[int, ...]:
+        """Convert a version string into a tuple of integers for comparison."""
+        return tuple(map(int, re.split(r"\D+", version_str)))
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, Version) and self.version == other.version
+
+    def __lt__(self, other: object) -> bool:
+        return isinstance(other, Version) and self.version < other.version
+
+    def __repr__(self) -> str:
+        return f"Version({'.'.join(map(str, self.version))})"
+
+
+@dataclass
+class PyPiSource:
+    name: str
+    url: str
+
+
+@dataclass
+class TomlSection:
+    name: str
+    content: str
+
+    def __str__(self) -> str:
+        return f"[{self.name}]\n{self.content}"
+
+
+class PyPiSourceParser:
+    @staticmethod
+    def from_pyproject(project_dir: Path) -> Optional[PyPiSource]:
+        pyproject_toml = project_dir / "pyproject.toml"
+        pipfile = project_dir / "Pipfile"
+        if pyproject_toml.exists():
+            return PyPiSourceParser.from_toml_content(pyproject_toml.read_text(), "tool.poetry.source")
+        elif pipfile.exists():
+            return PyPiSourceParser.from_toml_content(pipfile.read_text(), "source")
+        else:
+            return None
+
+    @staticmethod
+    def from_toml_content(content: str, source_section_name: str) -> Optional[PyPiSource]:
+        sections = PyPiSourceParser.get_toml_sections(content)
+        for section in sections:
+            if section.name == source_section_name:
+                try:
+                    parser = configparser.ConfigParser()
+                    parser.read_string(str(section))
+                    name = parser[section.name]["name"].strip('"')
+                    url = parser[section.name]["url"].strip('"')
+                    return PyPiSource(name, url)
+                except KeyError:
+                    raise UserNotificationException(
+                        f"Could not parse PyPi source from section {section.name}. "
+                        f"Please make sure the section has the following format:\n"
+                        f"[{source_section_name}]\n"
+                        f'name = "name"\n'
+                        f'url = "https://url"\n'
+                        f"verify_ssl = true"
+                    ) from None
+        return None
+
+    @staticmethod
+    def get_toml_sections(toml_content: str) -> List[TomlSection]:
+        # Use a regular expression to find all sections with [ or [[ at the beginning of the line
+        raw_sections = re.findall(r"^\[+.*\]+\n(?:[^[]*\n)*", toml_content, re.MULTILINE)
+
+        # Process each section
+        sections = []
+        for section in raw_sections:
+            # Split the lines, from the first line extract the section name
+            # and merge all the other lines into the content
+            lines = section.splitlines()
+            name_match = re.match(r"^\[+([^]]*)\]+", lines[0])
+            if name_match:
+                name = name_match.group(1).strip()
+                content = "\n".join(lines[1:]).strip()
+                sections.append(TomlSection(name, content))
+
+        return sections
+
+
+class Runnable(ABC):
+    @abstractmethod
+    def run(self) -> int:
+        """Run stage"""
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """Get stage name"""
+
+    @abstractmethod
+    def get_inputs(self) -> List[Path]:
+        """Get stage dependencies"""
+
+    @abstractmethod
+    def get_outputs(self) -> List[Path]:
+        """Get stage outputs"""
+
+
+class RunInfoStatus(Enum):
+    MATCH = (False, "Nothing has changed, previous execution information matches.")
+    NO_INFO = (True, "No previous execution information found.")
+    FILE_CHANGED = (True, "Dependencies have been changed.")
+
+    def __init__(self, should_run: bool, message: str) -> None:
+        self.should_run = should_run
+        self.message = message
+
+
+class Executor:
+    """Accepts Runnable objects and executes them.
+    It create a file with the same name as the runnable's name
+    and stores the inputs and outputs with their hashes.
+    If the file exists, it checks the hashes of the inputs and outputs
+    and if they match, it skips the execution."""
+
+    RUN_INFO_FILE_EXTENSION = ".deps.json"
+
+    def __init__(self, cache_dir: Path) -> None:
+        self.cache_dir = cache_dir
+
+    @staticmethod
+    def get_file_hash(path: Path) -> str:
+        """Get the hash of a file.
+        Returns an empty string if the file does not exist."""
+        if path.is_file():
+            with open(path, "rb") as file:
+                bytes = file.read()
+                readable_hash = hashlib.sha256(bytes).hexdigest()
+                return readable_hash
+        else:
+            return ""
+
+    def store_run_info(self, runnable: Runnable) -> None:
+        file_info = {
+            "inputs": {str(path): self.get_file_hash(path) for path in runnable.get_inputs()},
+            "outputs": {str(path): self.get_file_hash(path) for path in runnable.get_outputs()},
+        }
+
+        run_info_path = self.get_runnable_run_info_file(runnable)
+        run_info_path.parent.mkdir(parents=True, exist_ok=True)
+        with run_info_path.open("w") as f:
+            # pretty print the json file
+            json.dump(file_info, f, indent=4)
+
+    def get_runnable_run_info_file(self, runnable: Runnable) -> Path:
+        return self.cache_dir / f"{runnable.get_name()}{self.RUN_INFO_FILE_EXTENSION}"
+
+    def previous_run_info_matches(self, runnable: Runnable) -> RunInfoStatus:
+        run_info_path = self.get_runnable_run_info_file(runnable)
+        if not run_info_path.exists():
+            return RunInfoStatus.NO_INFO
+
+        with run_info_path.open() as f:
+            previous_info = json.load(f)
+
+        for file_type in ["inputs", "outputs"]:
+            for path_str, previous_hash in previous_info[file_type].items():
+                path = Path(path_str)
+                if self.get_file_hash(path) != previous_hash:
+                    return RunInfoStatus.FILE_CHANGED
+        return RunInfoStatus.MATCH
+
+    def execute(self, runnable: Runnable) -> int:
+        run_info_status = self.previous_run_info_matches(runnable)
+        if run_info_status.should_run:
+            logger.info(f"Runnable '{runnable.get_name()}' must run. {run_info_status.message}")
+            exit_code = runnable.run()
+            self.store_run_info(runnable)
+            return exit_code
+        logger.info(f"Runnable '{runnable.get_name()}' execution skipped. {run_info_status.message}")
+
+        return 0
+
+
+class UserNotificationException(Exception):
+    pass
+
+
+class SubprocessExecutor:
+    def __init__(
+        self,
+        command: List[str | Path],
+        cwd: Optional[Path] = None,
+        capture_output: bool = True,
+    ):
+        self.command = " ".join([str(cmd) for cmd in command])
+        self.current_working_directory = cwd
+        self.capture_output = capture_output
+
+    def execute(self) -> None:
+        result = None
+        try:
+            current_dir = (self.current_working_directory or Path.cwd()).as_posix()
+            logger.info(f"Running command: {self.command} in {current_dir}")
+            # print all virtual environment variables
+            logger.debug(json.dumps(dict(os.environ), indent=4))
+            result = subprocess.run(
+                self.command.split(),
+                cwd=current_dir,
+                capture_output=self.capture_output,
+                text=True,  # to get stdout and stderr as strings instead of bytes
+            )  # nosec
+            result.check_returncode()
+        except subprocess.CalledProcessError as e:
+            raise UserNotificationException(f"Command '{self.command}' failed with:\n" f"{result.stdout if result else ''}\n" f"{result.stderr if result else e}") from e
+
+
+class VirtualEnvironment(ABC):
+    def __init__(self, venv_dir: Path) -> None:
+        self.venv_dir = venv_dir
+
+    def create(self) -> None:
+        """
+        Create a new virtual environment. This should configure the virtual environment such that
+        subsequent calls to `pip` and `run` operate within this environment.
+        """
+        try:
+            venv.create(env_dir=self.venv_dir, with_pip=True)
+        except PermissionError as e:
+            if "python.exe" in str(e):
+                raise UserNotificationException(
+                    f"Failed to create virtual environment in {self.venv_dir}.\n" f"Virtual environment python.exe is still running. Please kill all instances and run again.\n" f"Error: {e}"
+                ) from e
+            raise UserNotificationException(f"Failed to create virtual environment in {self.venv_dir}.\n" f"Please make sure you have the necessary permissions.\n" f"Error: {e}") from e
+
+    def pip_configure(self, index_url: str, verify_ssl: bool = True) -> None:
+        """
+        Configure pip to use the given index URL and SSL verification setting. This method should
+        behave as if the user had activated the virtual environment and run `pip config set
+        global.index-url <index_url>` and `pip config set global.cert <verify_ssl>` from the
+        command line.
+
+        Args:
+        ----
+            index_url: The index URL to use for pip.
+            verify_ssl: Whether to verify SSL certificates when using pip.
+
+        """
+        # The pip configuration file should be in the virtual environment directory %VIRTUAL_ENV%
+        pip_ini_path = self.pip_config_path()
+        with open(pip_ini_path, "w") as pip_ini_file:
+            pip_ini_file.write(f"[global]\nindex-url = {index_url}\n")
+            if not verify_ssl:
+                pip_ini_file.write("cert = false\n")
+
+    def pip(self, args: List[str]) -> None:
+        SubprocessExecutor([self.pip_path().as_posix(), *args]).execute()
+
+    @abstractmethod
+    def pip_path(self) -> Path:
+        """
+        Get the path to the pip executable within the virtual environment.
+        """
+
+    @abstractmethod
+    def pip_config_path(self) -> Path:
+        """
+        Get the path to the pip configuration file within the virtual environment.
+        """
+
+    @abstractmethod
+    def run(self, args: List[str], capture_output: bool = True) -> None:
+        """
+        Run an arbitrary command within the virtual environment. This method should behave as if the
+        user had activated the virtual environment and run the given command from the command line.
+
+        Args:
+        ----
+            *args: Command-line arguments. For example, `run('python', 'setup.py', 'install')`
+                   should behave similarly to `python setup.py install` at the command line.
+
+        """
+
+
+class WindowsVirtualEnvironment(VirtualEnvironment):
+    def __init__(self, venv_dir: Path) -> None:
+        super().__init__(venv_dir)
+        self.activate_script = self.venv_dir.joinpath("Scripts/activate")
+
+    def pip_path(self) -> Path:
+        return self.venv_dir.joinpath("Scripts/pip.exe")
+
+    def pip_config_path(self) -> Path:
+        return self.venv_dir.joinpath("pip.ini")
+
+    def run(self, args: List[str], capture_output: bool = True) -> None:
+        SubprocessExecutor(
+            command=[f"cmd /c {self.activate_script.as_posix()} && ", *args],
+            capture_output=capture_output,
+        ).execute()
+
+
+class UnixVirtualEnvironment(VirtualEnvironment):
+    def __init__(self, venv_dir: Path) -> None:
+        super().__init__(venv_dir)
+        self.activate_script = self.venv_dir.joinpath("bin/activate")
+
+    def pip_path(self) -> Path:
+        return self.venv_dir.joinpath("bin/pip")
+
+    def pip_config_path(self) -> Path:
+        return self.venv_dir.joinpath("pip.conf")
+
+    def run(self, args: List[str], capture_output: bool = True) -> None:
+        # Create a temporary shell script
+        with tempfile.NamedTemporaryFile("w", delete=False, suffix=".sh") as f:
+            f.write("#!/bin/bash\n")  # Add a shebang line
+            f.write(f"source {self.activate_script.as_posix()}\n")  # Write the activate command
+            f.write(" ".join(args))  # Write the provided command
+            temp_script_path = f.name  # Get the path of the temporary script
+
+        # Make the temporary script executable
+        SubprocessExecutor(["chmod", "+x", temp_script_path]).execute()
+        # Run the temporary script
+        SubprocessExecutor(
+            command=[f"{Path(temp_script_path).as_posix()}"],
+            capture_output=capture_output,
+        ).execute()
+        # Delete the temporary script
+        os.remove(temp_script_path)
+
+
+class CreateVirtualEnvironment(Runnable):
+    def __init__(self, root_dir: Path) -> None:
+        self.root_dir = root_dir
+        self.venv_dir = self.root_dir / ".venv"
+        self.bootstrap_dir = self.root_dir / ".bootstrap"
+        self.virtual_env = self.instantiate_os_specific_venv(self.venv_dir)
+
+    @property
+    def package_manager_name(self) -> str:
+        match = re.match(r"^([a-zA-Z0-9_-]+)", package_manager)
+
+        if match:
+            return match.group(1)
+        else:
+            raise UserNotificationException(f"Could not extract the package manager name from {package_manager}")
+
+    def run(self) -> int:
+        # Create the virtual environment if pip executable does not exist
+        if not self.virtual_env.pip_path().exists():
+            self.virtual_env.create()
+
+        # Get the PyPi source from pyproject.toml or Pipfile if it is defined
+        pypi_source = PyPiSourceParser.from_pyproject(self.root_dir)
+        if pypi_source:
+            self.virtual_env.pip_configure(index_url=pypi_source.url, verify_ssl=True)
+        # We need pip-system-certs in venv to use certificates, that are stored in the system's trust store,
+        pip_args = ["install", package_manager, "pip-system-certs"]
+        # but to install it, we need either a pip version with the trust store feature or to trust the host
+        # (trust store feature enabled by default since 24.2)
+        if Version(ensurepip.version()) < Version("24.2"):
+            # Add trusted host of configured source for older Python versions
+            if pypi_source:
+                pip_args.extend(["--trusted-host", urlparse(pypi_source.url).hostname])
+            else:
+                pip_args.extend(["--trusted-host", "pypi.org", "--trusted-host", "pypi.python.org", "--trusted-host", "files.pythonhosted.org"])
+        self.virtual_env.pip(pip_args)
+        self.virtual_env.run(["python", "-m", self.package_manager_name, "install", *package_manager_args])
+        return 0
+
+    @staticmethod
+    def instantiate_os_specific_venv(venv_dir: Path) -> VirtualEnvironment:
+        if sys.platform.startswith("win32"):
+            return WindowsVirtualEnvironment(venv_dir)
+        elif sys.platform.startswith("linux") or sys.platform.startswith("darwin"):
+            return UnixVirtualEnvironment(venv_dir)
+        else:
+            raise UserNotificationException(f"Unsupported operating system: {sys.platform}")
+
+    def get_name(self) -> str:
+        return "create-virtual-environment"
+
+    def get_inputs(self) -> List[Path]:
+        venv_relevant_files = [
+            "poetry.lock",
+            "poetry.toml",
+            "pyproject.toml",
+            ".env",
+            "Pipfile",
+            "Pipfile.lock",
+            "bootstrap.json",
+            ".bootstrap/bootstrap.ps1",
+            ".bootstrap/bootstrap.py",
+            "bootstrap.ps1",
+            "bootstrap.py",
+        ]
+        return [self.root_dir / file for file in venv_relevant_files]
+
+    def get_outputs(self) -> List[Path]:
+        return []
+
+
+def print_environment_info() -> None:
+    str_bar = "".join(["-" for _ in range(80)])
+    logger.debug(str_bar)
+    logger.debug("Environment: \n" + json.dumps(dict(os.environ), indent=4))
+    logger.info(str_bar)
+    logger.info(f"Arguments: {sys.argv[1:]}")
+    logger.info(str_bar)
+
+
+def main() -> int:
+    try:
+        # print_environment_info()
+        creator = CreateVirtualEnvironment(Path.cwd())
+        Executor(creator.venv_dir).execute(creator)
+    except UserNotificationException as e:
+        logger.error(e)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pypeline/kickstart/templates/project/poetry.toml

@@ -1,2 +1,2 @@
-[virtualenvs]
-in-project = true
+[virtualenvs]
+in-project = true

pypeline/kickstart/templates/project/pypeline.ps1

@@ -1,7 +1,7 @@
-Push-Location $PSScriptRoot
-try {
-    .\.venv\Scripts\pypeline.exe $args
-}
-finally {
-    Pop-Location
-}
+Push-Location $PSScriptRoot
+try {
+    .\.venv\Scripts\pypeline.exe $args
+}
+finally {
+    Pop-Location
+}

pypeline/kickstart/templates/project/pypeline.yaml

@@ -3,7 +3,8 @@ pipeline:
     module: pypeline.steps.create_venv
     config:
       bootstrap_script: .bootstrap/bootstrap.py
-  - step: ScoopInstall
-    module: pypeline.steps.scoop_install
+  - step: WestInstall
+    module: pypeline.steps.west_install
+    description: Download external modules
   - step: MyStep
     file: steps/my_step.py

pypeline/kickstart/templates/project/pyproject.toml

@@ -1,9 +1,11 @@
-[tool.poetry]
-name = "Hello Pypeline"
-version = "0.0.1"
-description = "A simple generated project to get you started"
-authors = ["Your Name"]
-
-[tool.poetry.dependencies]
-python = ">=3.10,<3.13"
-pypeline-runner = "*"
+[tool.poetry]
+name = "Hello Pypeline"
+version = "0.0.1"
+description = "A simple generated project to get you started"
+authors = ["Your Name"]
+package-mode = false
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+pypeline-runner = "*"
+west = "*"

pypeline/kickstart/templates/project/steps/my_step.py

@@ -1,26 +1,26 @@
-from pathlib import Path
-from typing import List
-
-from py_app_dev.core.logging import logger
-
-from pypeline.domain.execution_context import ExecutionContext
-from pypeline.domain.pipeline import PipelineStep
-
-
-class MyStep(PipelineStep[ExecutionContext]):
-    def run(self) -> None:
-        logger.info(f"Run {self.get_name()} found install dirs:")
-        for install_dir in self.execution_context.install_dirs:
-            logger.info(f" {install_dir}")
-
-    def get_inputs(self) -> List[Path]:
-        return []
-
-    def get_outputs(self) -> List[Path]:
-        return []
-
-    def get_name(self) -> str:
-        return self.__class__.__name__
-
-    def update_execution_context(self) -> None:
-        pass
+from pathlib import Path
+from typing import List
+
+from py_app_dev.core.logging import logger
+
+from pypeline.domain.execution_context import ExecutionContext
+from pypeline.domain.pipeline import PipelineStep
+
+
+class MyStep(PipelineStep[ExecutionContext]):
+    def run(self) -> None:
+        logger.info(f"Run {self.get_name()} found install dirs:")
+        for install_dir in self.execution_context.install_dirs:
+            logger.info(f" {install_dir}")
+
+    def get_inputs(self) -> List[Path]:
+        return []
+
+    def get_outputs(self) -> List[Path]:
+        return []
+
+    def get_name(self) -> str:
+        return self.__class__.__name__
+
+    def update_execution_context(self) -> None:
+        pass

pypeline/kickstart/templates/project/west.yaml

@@ -0,0 +1,10 @@
+manifest:
+  remotes:
+    - name: gtest
+      url-base: https://github.com/google
+
+  projects:
+    - name: googletest
+      remote: gtest
+      revision: v1.14.0
+      path: external/gtest

pypeline/steps/create_venv.py

@@ -9,10 +9,12 @@ from py_app_dev.core.logging import logger
 from ..domain.execution_context import ExecutionContext
 from ..domain.pipeline import PipelineStep
 
+DEFAULT_BOOTSTRAP_SCRIPT = "bootstrap.py"
+
 
 @dataclass
 class CreateVEnvConfig(DataClassDictMixin):
-    bootstrap_script: str = "bootstrap.py"
+    bootstrap_script: str = DEFAULT_BOOTSTRAP_SCRIPT
 
 
 class CreateVEnv(PipelineStep[ExecutionContext]):
@@ -32,9 +34,14 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
         config = CreateVEnvConfig.from_dict(self.config) if self.config else CreateVEnvConfig()
         bootstrap_script = self.project_root_dir / config.bootstrap_script
         if not bootstrap_script.exists():
-            raise UserNotificationException("Failed to find bootstrap script. Make sure that the project is initialized correctly.")
+            if config.bootstrap_script == DEFAULT_BOOTSTRAP_SCRIPT:
+                raise UserNotificationException(f"Failed to find bootstrap script '{config.bootstrap_script}'. Make sure that the project is initialized correctly.")
+            else:  # Fallback to default bootstrap script
+                bootstrap_script = self.project_root_dir / DEFAULT_BOOTSTRAP_SCRIPT
+                if not bootstrap_script.exists():
+                    raise UserNotificationException("Failed to find bootstrap script. Make sure that the project is initialized correctly.")
         self.execution_context.create_process_executor(
-            ["python", bootstrap_script.as_posix()],
+            ["python3", bootstrap_script.as_posix()],
             cwd=self.project_root_dir,
         ).execute()
         self.execution_context.add_install_dirs(self.install_dirs)

{pypeline_runner-1.7.0.dist-info → pypeline_runner-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pypeline-runner
-Version: 1.7.0
+Version: 1.8.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 Author: cuinixam
@@ -26,7 +26,13 @@ Project-URL: Documentation, https://pypeline-runner.readthedocs.io
 Project-URL: Repository, https://github.com/cuinixam/pypeline
 Description-Content-Type: text/markdown
 
-# Pypeline
+<p align="center">
+<a href="https://pypeline-runner.readthedocs.io">
+<img align="center" src="https://github.com/cuinixam/pypeline/raw/main/logo.png" width="400"/>
+</a>
+</p>
+
+# pypeline - Define Your CI/CD Pipeline Once, Run It Anywhere
 
 <p align="center">
   <a href="https://github.com/cuinixam/pypeline/actions/workflows/ci.yml?query=branch%3Amain">
@@ -58,28 +64,71 @@ Description-Content-Type: text/markdown
   <img src="https://img.shields.io/pypi/l/pypeline-runner.svg?style=flat-square" alt="License">
 </p>
 
-Pypeline is a Python application designed to streamline and automate the software development lifecycle, particularly the pipeline execution processes across various environments such as GitHub and Jenkins.
-The primary motivation for developing Pypeline stemmed from the need to unify and simplify the creation of build, test, and deployment pipelines that are traditionally defined separately across these platforms using GitHub workflows (YAML) and Jenkins pipelines (Jenkinsfile).
+Pypeline lets you define your build, test, and deployment pipeline in a single YAML file and run it _consistently_ across your local development environment and _any_ CI/CD platform (GitHub Actions, Jenkins, etc.). No more platform-specific configurations – write once, run anywhere.
 
 **Key Features**
 
 - **Unified Pipeline Definition**: Users can define their entire pipeline in a single YAML file, eliminating the need to switch between different syntaxes and configurations for different CI/CD tools.
 
-- **Extensibility**: Pypeline supports execution steps defined not only through local scripts but also from installed Python packages.
-
-- **Execution Context**: Each step in the pipeline receives an execution context that can be updated during step execution. This allows for the sharing of information and state between steps.
+- **Extensibility**: Pypeline supports execution steps defined not only through installed Python packages but also from local scripts.
 
-- **Dependency Handling**: Dependency management ensures that only the necessary steps are executed, reducing runtime and resource usage by avoiding unnecessary operations.
+- **Execution Context**: Allow sharing information and state between steps. Each step in the pipeline receives an execution context that can be updated during step execution.
 
-- **Ease of Use**: With Pypeline, setting up and running pipelines becomes more straightforward, enabling developers to focus more on coding and less on configuring pipeline specifics.
+- **Dependency Handling**: Every step can register its dependencies and will only be scheduled if anything has changed.
 
 ## Installation
 
-Install this via pip (or your favourite package manager):
+Use pipx (or your favorite package manager) to install and run it in an isolated environment:
+
+```shell
+pipx install pypeline-runner
+```
+
+This will install the `pypeline` command globally, which you can use to run your pipelines.
 
-`pip install pypeline-runner`
+> [!NOTE]
+> The Python package is called `pypeline-runner` because the name `pypeline` was already taken on PyPI.
+> The command-line interface is `pypeline`.
 
-## Start developing
+Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
+
+## Walkthrough: Getting Started with Pypeline
+
+To get started run the `init` command to create a sample project:
+
+```shell
+pypeline init --project-dir my-pipeline
+```
+
+The example project pipeline is defined in the `pipeline.yaml` file.
+
+```yaml
+pipeline:
+  - step: CreateVEnv
+    module: pypeline.steps.create_venv
+    config:
+      bootstrap_script: .bootstrap/bootstrap.py
+  - step: WestInstall
+    module: pypeline.steps.west_install
+    description: Download external modules
+  - step: MyStep
+    file: steps/my_step.py
+    description: Run a custom script
+```
+
+This pipeline consists of three steps:
+
+- `CreateVEnv`: This is a built-in step that creates a Python virtual environment.
+- `WestInstall`: This is a built-in step that downloads external modules using the `west` tool.
+- `MyStep`: This is a custom step that runs a script defined in the `steps/my_step.py` file.
+
+You can run the pipeline using the `run` command:
+
+```shell
+pypeline run --project-dir my-pipeline
+```
+
+## Contributing
 
 The project uses Poetry for dependencies management and packaging.
 Run the `bootstrap.ps1` script to install Python and create the virtual environment.
@@ -88,43 +137,22 @@ Run the `bootstrap.ps1` script to install Python and create the virtual environm
 .\bootstrap.ps1
 ```
 
-This will also generate a `poetry.lock` file, you should track this file in version control.
-
 To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
 
 ```shell
 .venv/Scripts/poetry run pytest
 ```
 
-Check out the Poetry documentation for more information on the available commands.
-
 For those using [VS Code](https://code.visualstudio.com/) there are tasks defined for the most common commands:
 
-- bootstrap
-- install dependencies
 - run tests
 - run all checks configured for pre-commit
 - generate documentation
 
 See the `.vscode/tasks.json` for more details.
 
-## Committing changes
-
 This repository uses [commitlint](https://github.com/conventional-changelog/commitlint) for checking if the commit message meets the [conventional commit format](https://www.conventionalcommits.org/en).
 
-## Contributors ✨
-
-Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
-
-<!-- prettier-ignore-start -->
-<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
-<!-- markdownlint-disable -->
-<!-- markdownlint-enable -->
-<!-- ALL-CONTRIBUTORS-LIST:END -->
-<!-- prettier-ignore-end -->
-
-This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
-
 ## Credits
 
 This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.

{pypeline_runner-1.7.0.dist-info → pypeline_runner-1.8.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-pypeline/__init__.py,sha256=lEEx5lwinUby-YNl7w-VFGUcjcibsJ9EuzgJ2W7ocrU,22
+pypeline/__init__.py,sha256=Oc_xF94AMAHKZkZlB5rBt1iO0TXWFalg65MP4T2qt-A,22
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/domain/artifacts.py,sha256=5k7cVfHhLmvWXNuHKxXb9ca4Lxu0JytGQqazENCeKEU,1404
@@ -7,24 +7,26 @@ pypeline/domain/execution_context.py,sha256=ho-WvCVRMUfYo1532eQYabXCHtXDgvSNUkX8
 pypeline/domain/pipeline.py,sha256=2BsN2lw2znUxLH--Novyqe6SubVKs6XeHQSQf9yxirw,7788
 pypeline/domain/project_slurper.py,sha256=e3BLV88GvfW3efh0agUWKqMk3oWnL602P5u9jER_o9U,971
 pypeline/kickstart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/kickstart/create.py,sha256=dWqSdDsqNh7_AnNEsJzsmheR2-xyO6rpUwF-7AwzCYY,2188
+pypeline/kickstart/create.py,sha256=iaB8MMC7PinpPBwRmz3rWZuE-DRbsLh2NtvczYaVgi0,2133
 pypeline/kickstart/templates/project/.gitignore,sha256=y8GJoVvRPez1LBokf1NaDOt2X1XtGwKFMF5yjA8AVS0,24
 pypeline/kickstart/templates/project/bootstrap.ps1,sha256=eR8cyIJwDVt-bA2H3GWmxUew3igJaKYKv4rtg7MqhsY,766
-pypeline/kickstart/templates/project/poetry.toml,sha256=q5gF2MWexTFIahJ6StUa3y62HDUrRW7t8kGFszZhgp4,34
-pypeline/kickstart/templates/project/pypeline.ps1,sha256=s7CDfnagg8BIO42fpCfLF1l8uK7PNzui-7t9_suokzc,111
-pypeline/kickstart/templates/project/pypeline.yaml,sha256=EV5Tnu3H33gMT3Ov0t14-jKwnv9naSMb0wEDzaG0H2Q,238
-pypeline/kickstart/templates/project/pyproject.toml,sha256=yc6RCo-bUo1PXF91XfM-dButgfxU16Uud34NidgJ0zQ,225
+pypeline/kickstart/templates/project/bootstrap.py,sha256=9cJp_sbU0SKvDjJluvyQfh0_xIsf6E1ct7sa7rRecNU,17244
+pypeline/kickstart/templates/project/poetry.toml,sha256=qgVxBdPcJZOHdHCTOBoZYna3cke4VGgRkNZ0bKgN6rs,32
+pypeline/kickstart/templates/project/pypeline.ps1,sha256=PjCJULG8XA3AHKbNt3oHrIgD04huvvpIue_gjSo3PMA,104
+pypeline/kickstart/templates/project/pypeline.yaml,sha256=7FeIu7OOqq7iToPVdP_a4MehIi9lUkBPbXnl8mYpxSo,279
+pypeline/kickstart/templates/project/pyproject.toml,sha256=frr2i_bresciD1LsW5_y65Mwh_QTHia5MVipwsrc1-Q,248
 pypeline/kickstart/templates/project/scoopfile.json,sha256=DcfZ8jYf9hmPHM-AWwnPKQJCzRG3fCuYtMeoY01nkag,219
-pypeline/kickstart/templates/project/steps/my_step.py,sha256=_zx01qAVuwn6IMPBUBwKY-IBjS9Gs2m-d51L9sayGug,733
+pypeline/kickstart/templates/project/steps/my_step.py,sha256=iZYTzWtL-qxEW_t7q079d-xpnRST_tumSzxqiQDW7sM,707
+pypeline/kickstart/templates/project/west.yaml,sha256=ZfVym7M4yzzC-Nm0vESdhqNYs6EaJuMQWGJBht_i0b4,188
 pypeline/main.py,sha256=54_-aINmJY6IILSp6swL2kYqNoBQqJw4W2duxY7gcQ4,3607
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pypeline.py,sha256=FHGS2iNtiuiM4dZDHCbeL-UW1apdCtZeFzl1xZW9qBw,6482
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/steps/create_venv.py,sha256=vTPSA0gGGzq_QhI1jAgsGchS2s8_ZpEtnWcb0uL8BHE,1934
+pypeline/steps/create_venv.py,sha256=xCc73Hk62bbuJIM3btvapuoWeQy_Se5MsgffokYc-r0,2429
 pypeline/steps/scoop_install.py,sha256=_YdoCMXLON0eIwck8PJOcNhayx_ka1krBAidw_oRuFE,3373
 pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
-pypeline_runner-1.7.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-1.7.0.dist-info/METADATA,sha256=O-Y7JgczzWmfE0m2W7CRH23JJmNDNTrCdxc0P0ZLpkI,7154
-pypeline_runner-1.7.0.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-pypeline_runner-1.7.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-1.7.0.dist-info/RECORD,,
+pypeline_runner-1.8.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-1.8.0.dist-info/METADATA,sha256=PNVemc7HHUUK7pjR6pzw9EWlTfn-FFm506Ca798Y5Gs,7557
+pypeline_runner-1.8.0.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+pypeline_runner-1.8.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-1.8.0.dist-info/RECORD,,