pypeline-runner 1.24.0__tar.gz → 1.25.0__tar.gz

{pypeline_runner-1.24.0 → pypeline_runner-1.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pypeline-runner
-Version: 1.24.0
+Version: 1.25.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 License-File: LICENSE
@@ -19,10 +19,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: py-app-dev (>=2.10,<3.0)
-Requires-Dist: pyyaml (>=6.0,<7.0)
-Requires-Dist: typer (>=0,<1)
-Requires-Dist: west (>=1.0,<2.0)
+Requires-Dist: py-app-dev (>=2.18.0,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
+Requires-Dist: typer (>=0.21.1,<0.22.0)
+Requires-Dist: west (>=1.5.0,<2.0.0)
 Project-URL: Bug Tracker, https://github.com/cuinixam/pypeline/issues
 Project-URL: Changelog, https://github.com/cuinixam/pypeline/blob/main/CHANGELOG.md
 Project-URL: Documentation, https://pypeline-runner.readthedocs.io
@@ -95,6 +95,15 @@ This will install the `pypeline` command globally, which you can use to run your
 
 Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
 
+## Quick Start
+
+```shell
+pipx install pypeline-runner
+pypeline init --project-dir my-pipeline
+cd my-pipeline
+pypeline run
+```
+
 ## Walkthrough: Getting Started with Pypeline
 
 To get started run the `init` command to create a sample project:
@@ -134,10 +143,19 @@ pypeline run --project-dir my-pipeline
 ## Contributing
 
 The project uses Poetry for dependencies management and packaging.
-Run the `bootstrap.ps1` script to install Python and create the virtual environment.
+You can set up the development environment using one of the following methods:
+
+**Option 1: Using Poetry directly** (minimal setup)
 
-```powershell
-.\bootstrap.ps1
+```shell
+poetry install
+```
+
+**Option 2: Using pypeline-runner** (runs the full pipeline including tests and checks)
+
+```shell
+pipx install pypeline-runner
+pypeline run
 ```
 
 To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
@@ -158,5 +176,5 @@ This repository uses [commitlint](https://github.com/conventional-changelog/comm
 
 ## Credits
 
-This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [browniebroke/pypackage-template](https://github.com/browniebroke/pypackage-template) project template.
 

{pypeline_runner-1.24.0 → pypeline_runner-1.25.0}/README.md

@@ -64,6 +64,15 @@ This will install the `pypeline` command globally, which you can use to run your
 
 Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
 
+## Quick Start
+
+```shell
+pipx install pypeline-runner
+pypeline init --project-dir my-pipeline
+cd my-pipeline
+pypeline run
+```
+
 ## Walkthrough: Getting Started with Pypeline
 
 To get started run the `init` command to create a sample project:
@@ -103,10 +112,19 @@ pypeline run --project-dir my-pipeline
 ## Contributing
 
 The project uses Poetry for dependencies management and packaging.
-Run the `bootstrap.ps1` script to install Python and create the virtual environment.
+You can set up the development environment using one of the following methods:
+
+**Option 1: Using Poetry directly** (minimal setup)
 
-```powershell
-.\bootstrap.ps1
+```shell
+poetry install
+```
+
+**Option 2: Using pypeline-runner** (runs the full pipeline including tests and checks)
+
+```shell
+pipx install pypeline-runner
+pypeline run
 ```
 
 To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
@@ -127,4 +145,4 @@ This repository uses [commitlint](https://github.com/conventional-changelog/comm
 
 ## Credits
 
-This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [browniebroke/pypackage-template](https://github.com/browniebroke/pypackage-template) project template.

{pypeline_runner-1.24.0 → pypeline_runner-1.25.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "1.24.0"
+version = "1.25.0"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"
@@ -27,26 +27,25 @@ pypeline = "pypeline.main:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-py-app-dev = "^2.10"
-typer = "^0"
-pyyaml = "^6.0"
-west = "^1.0"
+py-app-dev = "^2.18.0"
+typer = "^0.21.1"
+pyyaml = "^6.0.3"
+west = "^1.5.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^8.3"
-pytest-cov = "^6.0"
-pre-commit = "^4.0"
-ruff = "^0.9"
+pytest = "^9.0.2"
+pytest-cov = "^7.0.0"
+pre-commit = "^4.5.1"
+ruff = "^0.14.14"
 
 [tool.poetry.group.docs.dependencies]
-myst-parser = ">=0.16"
-sphinx = ">=4.0"
-sphinxcontrib-mermaid = "^0.8"
-mlx-traceability = "^10.0"
-sphinx-copybutton = "^0.5"
-sphinx-new-tab-link = "^0.2"
-sphinx-book-theme = "^1.1"
-sphinx-design = "^0.5"
+myst-parser = ">=4.0.0"
+sphinx = ">=8.0.0"
+sphinxcontrib-mermaid = "^2.0.0"
+sphinx-copybutton = "^0.5.2"
+sphinx-new-tab-link = "^0.8.1"
+sphinx-book-theme = "^1.1.4"
+sphinx-design = "^0.6.0"
 
 [tool.semantic_release]
 version_toml = ["pyproject.toml:tool.poetry.version"]

pypeline_runner-1.25.0/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.25.0"

{pypeline_runner-1.24.0 → pypeline_runner-1.25.0}/src/pypeline/steps/create_venv.py

@@ -316,7 +316,7 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
 
     def run(self) -> int:
         self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        bootstrap_config: Dict[str, Any] = {}
+        bootstrap_config = CreateVEnvConfig()
         is_managed = False
         # Determine target script and mode
         if self.user_config.bootstrap_script:
@@ -356,17 +356,17 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
             for field_name in self.user_config.get_all_properties_names(["bootstrap_script", "python_executable"]):
                 val = getattr(self.user_config, field_name)
                 if val is not None:
-                    bootstrap_config[field_name] = val
+                    setattr(bootstrap_config, field_name, val)
 
             # Priority: input python_version takes precedence over config python_version
             input_python_version = self.execution_context.get_input("python_version")
             if input_python_version:
-                bootstrap_config["python_version"] = input_python_version
+                bootstrap_config.python_version = input_python_version
 
             # Write bootstrap.json if any configuration is provided
-            if bootstrap_config:
+            if bootstrap_config.is_any_property_set():
                 self.bootstrap_config_file.parent.mkdir(exist_ok=True)
-                CreateVEnvConfig.from_dict(bootstrap_config).to_json_file(self.bootstrap_config_file)
+                bootstrap_config.to_json_file(self.bootstrap_config_file)
                 self.logger.info(f"Created bootstrap configuration at {self.bootstrap_config_file}")
 
             # Build bootstrap script arguments

pypeline_runner-1.25.0/src/pypeline/steps/west_install.py

@@ -0,0 +1,310 @@
+import io
+import json
+import traceback
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional
+
+import yaml
+from mashumaro.config import BaseConfig
+from mashumaro.mixins.json import DataClassJSONMixin
+from py_app_dev.core.config import BaseConfigDictMixin
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.logging import logger
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+from pypeline.domain.execution_context import ExecutionContext
+from pypeline.domain.pipeline import PipelineStep
+
+
+class BaseConfigJSONMixin(DataClassJSONMixin):
+    class Config(BaseConfig):
+        """Mashumaro configuration for JSON serialization."""
+
+        omit_none = True
+        serialize_by_alias = True
+
+    def to_json_string(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+    def to_json_file(self, file_path: Path) -> None:
+        file_path.write_text(self.to_json_string())
+
+
+@dataclass
+class WestDependency(BaseConfigDictMixin):
+    #: Project name
+    name: str
+    #: Remote name
+    remote: str
+    #: Revision (tag, branch, or commit)
+    revision: str
+    #: Path where the dependency will be installed
+    path: str
+
+
+@dataclass
+class WestRemote(BaseConfigDictMixin):
+    #: Remote name
+    name: str
+    #: URL base
+    url_base: str = field(metadata={"alias": "url-base"})
+
+
+@dataclass
+class WestManifest(BaseConfigDictMixin):
+    #: Remote configurations
+    remotes: list[WestRemote] = field(default_factory=list)
+    #: Project dependencies
+    projects: list[WestDependency] = field(default_factory=list)
+
+
+@dataclass
+class WestManifestFile(BaseConfigDictMixin):
+    manifest: WestManifest
+    # This field is intended to keep track of where configuration was loaded from and
+    # it is automatically added when configuration is loaded from file
+    file: Optional[Path] = None
+
+    @classmethod
+    def from_file(cls, config_file: Path) -> "WestManifestFile":
+        config_dict = cls.parse_to_dict(config_file)
+        return cls.from_dict(config_dict)
+
+    @staticmethod
+    def parse_to_dict(config_file: Path) -> dict[str, Any]:
+        try:
+            with open(config_file) as fs:
+                config_dict = yaml.safe_load(fs)
+                # Add file name to config to keep track of where configuration was loaded from
+                config_dict["file"] = config_file
+            return config_dict
+        except ScannerError as e:
+            raise UserNotificationException(f"Failed scanning west manifest file '{config_file}'. \nError: {e}") from e
+        except ParserError as e:
+            raise UserNotificationException(f"Failed parsing west manifest file '{config_file}'. \nError: {e}") from e
+
+
+@dataclass
+class WestInstallResult(DataClassJSONMixin):
+    """Tracks paths of installed west dependencies."""
+
+    installed_dirs: list[Path] = field(default_factory=list)
+
+    class Config(BaseConfig):
+        """Mashumaro configuration for JSON serialization."""
+
+        omit_none = True
+
+    @classmethod
+    def from_json_file(cls, file_path: Path) -> "WestInstallResult":
+        try:
+            result = cls.from_dict(json.loads(file_path.read_text()))
+        except Exception as e:
+            output = io.StringIO()
+            traceback.print_exc(file=output)
+            raise UserNotificationException(output.getvalue()) from e
+        return result
+
+    def to_json_string(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+    def to_json_file(self, file_path: Path) -> None:
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(self.to_json_string())
+
+
+@dataclass
+class WestWorkspaceDir:
+    """West workspace directory path for data registry sharing."""
+
+    path: Path
+
+
+@dataclass
+class WestInstallConfig(DataClassJSONMixin):
+    """Configuration for WestInstall step."""
+
+    #: Relative path from project root for west workspace directory
+    workspace_dir: Optional[str] = None
+
+
+class WestInstall(PipelineStep[ExecutionContext]):
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[dict[str, Any]] = None) -> None:
+        super().__init__(execution_context, group_name, config)
+        self.logger = logger.bind()
+        self.install_result = WestInstallResult()
+        self.user_config = WestInstallConfig.from_dict(config) if config else WestInstallConfig()
+
+        self._west_workspace_dir = self._resolve_workspace_dir()
+        self._manifests = self._collect_manifests()
+
+    def _resolve_workspace_dir(self) -> Path:
+        """Resolve workspace directory from data registry (priority) or config."""
+        # Check data registry first (highest priority)
+        registry_entries = self.execution_context.data_registry.find_data(WestWorkspaceDir)
+        if registry_entries:
+            return registry_entries[0].path
+
+        # Check config
+        if self.user_config.workspace_dir:
+            return self.project_root_dir / self.user_config.workspace_dir
+
+        # Fallback to build dir
+        return self.execution_context.create_artifacts_locator().build_dir
+
+    def _collect_manifests(self) -> list[WestManifestFile]:
+        manifests: list[WestManifestFile] = []
+
+        if self._source_manifest_file.exists():
+            try:
+                manifests.append(WestManifestFile.from_file(self._source_manifest_file))
+            except Exception as e:
+                self.logger.warning(f"Failed to parse source west.yaml: {e}")
+
+        # Check if there are registered manifests in the execution context data registry
+        manifests.extend(self.execution_context.data_registry.find_data(WestManifestFile))
+        return manifests
+
+    @property
+    def _source_manifest_file(self) -> Path:
+        """Optional west.yaml in project root (input)."""
+        return self.project_root_dir / "west.yaml"
+
+    @property
+    def _output_manifest_file(self) -> Path:
+        """Generated west.yaml (output)."""
+        return self.output_dir / "west.yaml"
+
+    @property
+    def _install_result_file(self) -> Path:
+        """Tracks installed dependency directories."""
+        return self.output_dir / "west_install_result.json"
+
+    @property
+    def installed_dirs(self) -> list[Path]:
+        return self.install_result.installed_dirs
+
+    def get_name(self) -> str:
+        return self.__class__.__name__
+
+    def get_config(self) -> dict[str, str] | None:
+        if self.user_config.workspace_dir:
+            return {"workspace_dir": self.user_config.workspace_dir}
+        return None
+
+    def _merge_manifests(self, manifests: list[WestManifest]) -> WestManifest:
+        """Merge multiple manifests, preserving order. First occurrence wins."""
+        merged = WestManifest()
+        for manifest in manifests:
+            for remote in manifest.remotes:
+                if remote not in merged.remotes:
+                    merged.remotes.append(remote)
+            for project in manifest.projects:
+                if project not in merged.projects:
+                    merged.projects.append(project)
+        return merged
+
+    def _write_west_manifest_file(self, manifest: WestManifest) -> None:
+        """Write merged manifest to west.yaml file."""
+        if not manifest.remotes and not manifest.projects:
+            self.logger.info("No West dependencies found. Skipping west.yaml generation.")
+            return
+
+        west_config = {
+            "manifest": {
+                "remotes": [remote.to_dict() for remote in manifest.remotes],
+                "projects": [project.to_dict() for project in manifest.projects],
+            }
+        }
+
+        # Convert url_base back to url-base for west compatibility
+        for remote in west_config["manifest"]["remotes"]:
+            if "url_base" in remote:
+                remote["url-base"] = remote.pop("url_base")
+
+        self._output_manifest_file.parent.mkdir(parents=True, exist_ok=True)
+        with open(self._output_manifest_file, "w") as f:
+            yaml.dump(west_config, f, default_flow_style=False)
+
+        self.logger.info(f"Generated west.yaml with {len(manifest.projects)} dependencies")
+
+    def _run_west_init(self) -> None:
+        """Initialize west workspace."""
+        self.execution_context.create_process_executor(
+            [
+                "west",
+                "init",
+                "-l",
+                "--mf",
+                self._output_manifest_file.as_posix(),
+                self._west_workspace_dir.joinpath("do_not_care").as_posix(),
+            ],
+            cwd=self.project_root_dir,
+        ).execute()
+
+    def _run_west_update(self) -> None:
+        """Update/download dependencies."""
+        self.execution_context.create_process_executor(
+            ["west", "update"],
+            cwd=self._west_workspace_dir,
+        ).execute()
+
+    def run(self) -> int:
+        self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
+
+        try:
+            merged_manifest = self._merge_manifests([mf.manifest for mf in self._manifests])
+            self._write_west_manifest_file(merged_manifest)
+
+            if not merged_manifest.projects:
+                self.logger.info("No West dependencies to install.")
+                return 0
+
+            self._run_west_init()
+            self._run_west_update()
+            self._record_installed_directories(merged_manifest)
+            self.install_result.to_json_file(self._install_result_file)
+
+        except Exception as e:
+            raise UserNotificationException(f"Failed to initialize and update with west: {e}") from e
+
+        return 0
+
+    def _record_installed_directories(self, manifest: WestManifest) -> None:
+        """Record directories created by west."""
+        dirs: list[Path] = []
+
+        if self._west_workspace_dir.exists():
+            dirs.append(self._west_workspace_dir)
+
+        for project in manifest.projects:
+            dep_dir = self._west_workspace_dir / project.path
+            if dep_dir.exists():
+                dirs.append(dep_dir)
+                self.logger.debug(f"Tracked dependency directory: {dep_dir}")
+
+        self.install_result.installed_dirs = list(dict.fromkeys(dirs))
+
+    def get_inputs(self) -> list[Path]:
+        inputs: list[Path] = []
+        for manifest_file in self._manifests:
+            if manifest_file.file and manifest_file.file.exists():
+                inputs.append(manifest_file.file)
+        return inputs
+
+    def get_outputs(self) -> list[Path]:
+        outputs: list[Path] = [self._output_manifest_file, self._install_result_file]
+        if self.install_result.installed_dirs:
+            outputs.extend(self.install_result.installed_dirs)
+        elif self._manifests:
+            outputs.append(self._west_workspace_dir)
+        return outputs
+
+    def update_execution_context(self) -> None:
+        if self._install_result_file.exists():
+            result = WestInstallResult.from_json_file(self._install_result_file)
+            if result.installed_dirs:
+                unique_paths = list(dict.fromkeys(result.installed_dirs))
+                self.execution_context.add_install_dirs(unique_paths)

pypeline_runner-1.24.0/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.24.0"

pypeline_runner-1.24.0/src/pypeline/steps/west_install.py

@@ -1,54 +0,0 @@
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from py_app_dev.core.exceptions import UserNotificationException
-from py_app_dev.core.logging import logger
-
-from ..domain.execution_context import ExecutionContext
-from ..domain.pipeline import PipelineStep
-
-
-class WestInstall(PipelineStep[ExecutionContext]):
-    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, group_name, config)
-        self.logger = logger.bind()
-        self.artifacts_locator = execution_context.create_artifacts_locator()
-
-    def get_name(self) -> str:
-        return self.__class__.__name__
-
-    @property
-    def west_manifest_file(self) -> Path:
-        return self.project_root_dir.joinpath("west.yaml")
-
-    def run(self) -> int:
-        self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        try:
-            self.execution_context.create_process_executor(
-                [
-                    "west",
-                    "init",
-                    "-l",
-                    "--mf",
-                    self.west_manifest_file.as_posix(),
-                    self.artifacts_locator.build_dir.joinpath("west").as_posix(),
-                ],
-                cwd=self.project_root_dir,
-            ).execute()
-            self.execution_context.create_process_executor(
-                ["west", "update"],
-                cwd=self.artifacts_locator.build_dir,
-            ).execute()
-        except Exception as e:
-            raise UserNotificationException(f"Failed to initialize and update with west: {e}") from e
-
-        return 0
-
-    def get_inputs(self) -> List[Path]:
-        return [self.west_manifest_file]
-
-    def get_outputs(self) -> List[Path]:
-        return []
-
-    def update_execution_context(self) -> None:
-        pass