pypeline-runner 1.24.0__py3-none-any.whl → 1.25.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "1.24.0"
+__version__ = "1.25.0"

pypeline/steps/create_venv.py

@@ -316,7 +316,7 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
 
     def run(self) -> int:
         self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        bootstrap_config: Dict[str, Any] = {}
+        bootstrap_config = CreateVEnvConfig()
         is_managed = False
         # Determine target script and mode
         if self.user_config.bootstrap_script:
@@ -356,17 +356,17 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
             for field_name in self.user_config.get_all_properties_names(["bootstrap_script", "python_executable"]):
                 val = getattr(self.user_config, field_name)
                 if val is not None:
-                    bootstrap_config[field_name] = val
+                    setattr(bootstrap_config, field_name, val)
 
             # Priority: input python_version takes precedence over config python_version
             input_python_version = self.execution_context.get_input("python_version")
             if input_python_version:
-                bootstrap_config["python_version"] = input_python_version
+                bootstrap_config.python_version = input_python_version
 
             # Write bootstrap.json if any configuration is provided
-            if bootstrap_config:
+            if bootstrap_config.is_any_property_set():
                 self.bootstrap_config_file.parent.mkdir(exist_ok=True)
-                CreateVEnvConfig.from_dict(bootstrap_config).to_json_file(self.bootstrap_config_file)
+                bootstrap_config.to_json_file(self.bootstrap_config_file)
                 self.logger.info(f"Created bootstrap configuration at {self.bootstrap_config_file}")
 
             # Build bootstrap script arguments

pypeline/steps/west_install.py

@@ -1,54 +1,310 @@
+import io
+import json
+import traceback
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
+import yaml
+from mashumaro.config import BaseConfig
+from mashumaro.mixins.json import DataClassJSONMixin
+from py_app_dev.core.config import BaseConfigDictMixin
 from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
 
-from ..domain.execution_context import ExecutionContext
-from ..domain.pipeline import PipelineStep
+from pypeline.domain.execution_context import ExecutionContext
+from pypeline.domain.pipeline import PipelineStep
+
+
+class BaseConfigJSONMixin(DataClassJSONMixin):
+    class Config(BaseConfig):
+        """Mashumaro configuration for JSON serialization."""
+
+        omit_none = True
+        serialize_by_alias = True
+
+    def to_json_string(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+    def to_json_file(self, file_path: Path) -> None:
+        file_path.write_text(self.to_json_string())
+
+
+@dataclass
+class WestDependency(BaseConfigDictMixin):
+    #: Project name
+    name: str
+    #: Remote name
+    remote: str
+    #: Revision (tag, branch, or commit)
+    revision: str
+    #: Path where the dependency will be installed
+    path: str
+
+
+@dataclass
+class WestRemote(BaseConfigDictMixin):
+    #: Remote name
+    name: str
+    #: URL base
+    url_base: str = field(metadata={"alias": "url-base"})
+
+
+@dataclass
+class WestManifest(BaseConfigDictMixin):
+    #: Remote configurations
+    remotes: list[WestRemote] = field(default_factory=list)
+    #: Project dependencies
+    projects: list[WestDependency] = field(default_factory=list)
+
+
+@dataclass
+class WestManifestFile(BaseConfigDictMixin):
+    manifest: WestManifest
+    # This field is intended to keep track of where configuration was loaded from and
+    # it is automatically added when configuration is loaded from file
+    file: Optional[Path] = None
+
+    @classmethod
+    def from_file(cls, config_file: Path) -> "WestManifestFile":
+        config_dict = cls.parse_to_dict(config_file)
+        return cls.from_dict(config_dict)
+
+    @staticmethod
+    def parse_to_dict(config_file: Path) -> dict[str, Any]:
+        try:
+            with open(config_file) as fs:
+                config_dict = yaml.safe_load(fs)
+                # Add file name to config to keep track of where configuration was loaded from
+                config_dict["file"] = config_file
+            return config_dict
+        except ScannerError as e:
+            raise UserNotificationException(f"Failed scanning west manifest file '{config_file}'. \nError: {e}") from e
+        except ParserError as e:
+            raise UserNotificationException(f"Failed parsing west manifest file '{config_file}'. \nError: {e}") from e
+
+
+@dataclass
+class WestInstallResult(DataClassJSONMixin):
+    """Tracks paths of installed west dependencies."""
+
+    installed_dirs: list[Path] = field(default_factory=list)
+
+    class Config(BaseConfig):
+        """Mashumaro configuration for JSON serialization."""
+
+        omit_none = True
+
+    @classmethod
+    def from_json_file(cls, file_path: Path) -> "WestInstallResult":
+        try:
+            result = cls.from_dict(json.loads(file_path.read_text()))
+        except Exception as e:
+            output = io.StringIO()
+            traceback.print_exc(file=output)
+            raise UserNotificationException(output.getvalue()) from e
+        return result
+
+    def to_json_string(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+
+    def to_json_file(self, file_path: Path) -> None:
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(self.to_json_string())
+
+
+@dataclass
+class WestWorkspaceDir:
+    """West workspace directory path for data registry sharing."""
+
+    path: Path
+
+
+@dataclass
+class WestInstallConfig(DataClassJSONMixin):
+    """Configuration for WestInstall step."""
+
+    #: Relative path from project root for west workspace directory
+    workspace_dir: Optional[str] = None
 
 
 class WestInstall(PipelineStep[ExecutionContext]):
-    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[dict[str, Any]] = None) -> None:
         super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
-        self.artifacts_locator = execution_context.create_artifacts_locator()
+        self.install_result = WestInstallResult()
+        self.user_config = WestInstallConfig.from_dict(config) if config else WestInstallConfig()
+
+        self._west_workspace_dir = self._resolve_workspace_dir()
+        self._manifests = self._collect_manifests()
+
+    def _resolve_workspace_dir(self) -> Path:
+        """Resolve workspace directory from data registry (priority) or config."""
+        # Check data registry first (highest priority)
+        registry_entries = self.execution_context.data_registry.find_data(WestWorkspaceDir)
+        if registry_entries:
+            return registry_entries[0].path
+
+        # Check config
+        if self.user_config.workspace_dir:
+            return self.project_root_dir / self.user_config.workspace_dir
+
+        # Fallback to build dir
+        return self.execution_context.create_artifacts_locator().build_dir
+
+    def _collect_manifests(self) -> list[WestManifestFile]:
+        manifests: list[WestManifestFile] = []
+
+        if self._source_manifest_file.exists():
+            try:
+                manifests.append(WestManifestFile.from_file(self._source_manifest_file))
+            except Exception as e:
+                self.logger.warning(f"Failed to parse source west.yaml: {e}")
+
+        # Check if there are registered manifests in the execution context data registry
+        manifests.extend(self.execution_context.data_registry.find_data(WestManifestFile))
+        return manifests
+
+    @property
+    def _source_manifest_file(self) -> Path:
+        """Optional west.yaml in project root (input)."""
+        return self.project_root_dir / "west.yaml"
+
+    @property
+    def _output_manifest_file(self) -> Path:
+        """Generated west.yaml (output)."""
+        return self.output_dir / "west.yaml"
+
+    @property
+    def _install_result_file(self) -> Path:
+        """Tracks installed dependency directories."""
+        return self.output_dir / "west_install_result.json"
+
+    @property
+    def installed_dirs(self) -> list[Path]:
+        return self.install_result.installed_dirs
 
     def get_name(self) -> str:
         return self.__class__.__name__
 
-    @property
-    def west_manifest_file(self) -> Path:
-        return self.project_root_dir.joinpath("west.yaml")
+    def get_config(self) -> dict[str, str] | None:
+        if self.user_config.workspace_dir:
+            return {"workspace_dir": self.user_config.workspace_dir}
+        return None
+
+    def _merge_manifests(self, manifests: list[WestManifest]) -> WestManifest:
+        """Merge multiple manifests, preserving order. First occurrence wins."""
+        merged = WestManifest()
+        for manifest in manifests:
+            for remote in manifest.remotes:
+                if remote not in merged.remotes:
+                    merged.remotes.append(remote)
+            for project in manifest.projects:
+                if project not in merged.projects:
+                    merged.projects.append(project)
+        return merged
+
+    def _write_west_manifest_file(self, manifest: WestManifest) -> None:
+        """Write merged manifest to west.yaml file."""
+        if not manifest.remotes and not manifest.projects:
+            self.logger.info("No West dependencies found. Skipping west.yaml generation.")
+            return
+
+        west_config = {
+            "manifest": {
+                "remotes": [remote.to_dict() for remote in manifest.remotes],
+                "projects": [project.to_dict() for project in manifest.projects],
+            }
+        }
+
+        # Convert url_base back to url-base for west compatibility
+        for remote in west_config["manifest"]["remotes"]:
+            if "url_base" in remote:
+                remote["url-base"] = remote.pop("url_base")
+
+        self._output_manifest_file.parent.mkdir(parents=True, exist_ok=True)
+        with open(self._output_manifest_file, "w") as f:
+            yaml.dump(west_config, f, default_flow_style=False)
+
+        self.logger.info(f"Generated west.yaml with {len(manifest.projects)} dependencies")
+
+    def _run_west_init(self) -> None:
+        """Initialize west workspace."""
+        self.execution_context.create_process_executor(
+            [
+                "west",
+                "init",
+                "-l",
+                "--mf",
+                self._output_manifest_file.as_posix(),
+                self._west_workspace_dir.joinpath("do_not_care").as_posix(),
+            ],
+            cwd=self.project_root_dir,
+        ).execute()
+
+    def _run_west_update(self) -> None:
+        """Update/download dependencies."""
+        self.execution_context.create_process_executor(
+            ["west", "update"],
+            cwd=self._west_workspace_dir,
+        ).execute()
 
     def run(self) -> int:
         self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
+
         try:
-            self.execution_context.create_process_executor(
-                [
-                    "west",
-                    "init",
-                    "-l",
-                    "--mf",
-                    self.west_manifest_file.as_posix(),
-                    self.artifacts_locator.build_dir.joinpath("west").as_posix(),
-                ],
-                cwd=self.project_root_dir,
-            ).execute()
-            self.execution_context.create_process_executor(
-                ["west", "update"],
-                cwd=self.artifacts_locator.build_dir,
-            ).execute()
+            merged_manifest = self._merge_manifests([mf.manifest for mf in self._manifests])
+            self._write_west_manifest_file(merged_manifest)
+
+            if not merged_manifest.projects:
+                self.logger.info("No West dependencies to install.")
+                return 0
+
+            self._run_west_init()
+            self._run_west_update()
+            self._record_installed_directories(merged_manifest)
+            self.install_result.to_json_file(self._install_result_file)
+
         except Exception as e:
             raise UserNotificationException(f"Failed to initialize and update with west: {e}") from e
 
         return 0
 
-    def get_inputs(self) -> List[Path]:
-        return [self.west_manifest_file]
+    def _record_installed_directories(self, manifest: WestManifest) -> None:
+        """Record directories created by west."""
+        dirs: list[Path] = []
+
+        if self._west_workspace_dir.exists():
+            dirs.append(self._west_workspace_dir)
+
+        for project in manifest.projects:
+            dep_dir = self._west_workspace_dir / project.path
+            if dep_dir.exists():
+                dirs.append(dep_dir)
+                self.logger.debug(f"Tracked dependency directory: {dep_dir}")
+
+        self.install_result.installed_dirs = list(dict.fromkeys(dirs))
+
+    def get_inputs(self) -> list[Path]:
+        inputs: list[Path] = []
+        for manifest_file in self._manifests:
+            if manifest_file.file and manifest_file.file.exists():
+                inputs.append(manifest_file.file)
+        return inputs
 
-    def get_outputs(self) -> List[Path]:
-        return []
+    def get_outputs(self) -> list[Path]:
+        outputs: list[Path] = [self._output_manifest_file, self._install_result_file]
+        if self.install_result.installed_dirs:
+            outputs.extend(self.install_result.installed_dirs)
+        elif self._manifests:
+            outputs.append(self._west_workspace_dir)
+        return outputs
 
     def update_execution_context(self) -> None:
-        pass
+        if self._install_result_file.exists():
+            result = WestInstallResult.from_json_file(self._install_result_file)
+            if result.installed_dirs:
+                unique_paths = list(dict.fromkeys(result.installed_dirs))
+                self.execution_context.add_install_dirs(unique_paths)

{pypeline_runner-1.24.0.dist-info → pypeline_runner-1.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pypeline-runner
-Version: 1.24.0
+Version: 1.25.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 License-File: LICENSE
@@ -19,10 +19,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: py-app-dev (>=2.10,<3.0)
-Requires-Dist: pyyaml (>=6.0,<7.0)
-Requires-Dist: typer (>=0,<1)
-Requires-Dist: west (>=1.0,<2.0)
+Requires-Dist: py-app-dev (>=2.18.0,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
+Requires-Dist: typer (>=0.21.1,<0.22.0)
+Requires-Dist: west (>=1.5.0,<2.0.0)
 Project-URL: Bug Tracker, https://github.com/cuinixam/pypeline/issues
 Project-URL: Changelog, https://github.com/cuinixam/pypeline/blob/main/CHANGELOG.md
 Project-URL: Documentation, https://pypeline-runner.readthedocs.io
@@ -95,6 +95,15 @@ This will install the `pypeline` command globally, which you can use to run your
 
 Documentation: [pypeline-runner.readthedocs.io](https://pypeline-runner.readthedocs.io)
 
+## Quick Start
+
+```shell
+pipx install pypeline-runner
+pypeline init --project-dir my-pipeline
+cd my-pipeline
+pypeline run
+```
+
 ## Walkthrough: Getting Started with Pypeline
 
 To get started run the `init` command to create a sample project:
@@ -134,10 +143,19 @@ pypeline run --project-dir my-pipeline
 ## Contributing
 
 The project uses Poetry for dependencies management and packaging.
-Run the `bootstrap.ps1` script to install Python and create the virtual environment.
+You can set up the development environment using one of the following methods:
+
+**Option 1: Using Poetry directly** (minimal setup)
 
-```powershell
-.\bootstrap.ps1
+```shell
+poetry install
+```
+
+**Option 2: Using pypeline-runner** (runs the full pipeline including tests and checks)
+
+```shell
+pipx install pypeline-runner
+pypeline run
 ```
 
 To execute the test suite, call pytest inside Poetry's virtual environment via `poetry run`:
@@ -158,5 +176,5 @@ This repository uses [commitlint](https://github.com/conventional-changelog/comm
 
 ## Credits
 
-This package was created with [Copier](https://copier.readthedocs.io/) and the [cuinixam/pypackage-template](https://github.com/cuinixam/pypackage-template) project template.
+This package was created with [Copier](https://copier.readthedocs.io/) and the [browniebroke/pypackage-template](https://github.com/browniebroke/pypackage-template) project template.
 

{pypeline_runner-1.24.0.dist-info → pypeline_runner-1.25.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-pypeline/__init__.py,sha256=Iat5kowFGlxZk2HgCHj6JjzGlHfXn2mvuhQ5AbHTC94,23
+pypeline/__init__.py,sha256=nnWtTD0Cs5IqX2RSzxsnBvgnHIH1dYlciKuzG2ViNCk,23
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/bootstrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/bootstrap/run.py,sha256=H5rxSa_owbAUpMA2lw-UhgBFB0eDgnQC0B4jYO8j3sE,33463
@@ -21,12 +21,12 @@ pypeline/main.py,sha256=k1CkeFGRvQ-zLv6C-AMLC2ed1iyFzDUdvEam3HLHy2E,4210
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pypeline.py,sha256=mDKUnTuMDw8l-kSDJCHRNbn6zrxAfXhAIAqc5HyHd5M,8758
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/steps/create_venv.py,sha256=8zZRFyya8DgNcYu4x2R_HLr5RdVCOddU2Ln96MgdegI,19051
+pypeline/steps/create_venv.py,sha256=s8uL0cLfAF-AFvrU77KtdUMTE0v5ZDY2Ie_fO4ZqKjg,19050
 pypeline/steps/env_setup_script.py,sha256=L8TwGo_Ugo2r4Z10MxtE0P8w0ApAxMKCHMnW-NkyG3w,4968
 pypeline/steps/scoop_install.py,sha256=2MhsJ0iPmL8ueQhI52sKjVY9fqzj5xOQweQ65C0onfE,4117
-pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
-pypeline_runner-1.24.0.dist-info/METADATA,sha256=bKMBdaJ56y2FiOqv4Xd9fkl2iTUixhVSnv7loKCqBS0,7659
-pypeline_runner-1.24.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pypeline_runner-1.24.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-1.24.0.dist-info/licenses/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-1.24.0.dist-info/RECORD,,
+pypeline/steps/west_install.py,sha256=J-eSD0tBuJ7LS5KuGpPCuu7mcwJ68g8tMBxJ2bvEsUw,11499
+pypeline_runner-1.25.0.dist-info/METADATA,sha256=ITheZZgSUmCIZVZfTaO6LyYhVjhPkHrz5UMWiWHCH5s,8002
+pypeline_runner-1.25.0.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
+pypeline_runner-1.25.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-1.25.0.dist-info/licenses/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-1.25.0.dist-info/RECORD,,

{pypeline_runner-1.24.0.dist-info → pypeline_runner-1.25.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.2.1
+Generator: poetry-core 2.3.0
 Root-Is-Purelib: true
 Tag: py3-none-any