pypeline-runner 1.11.0__py3-none-any.whl → 1.13.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "1.11.0"
+__version__ = "1.13.0"

pypeline/domain/config.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Literal, Optional
 
 import yaml
 from mashumaro import DataClassDictMixin
@@ -10,10 +10,23 @@ from yaml.scanner import ScannerError
 
 from .pipeline import PipelineConfig
 
+InputType = Literal["string", "integer", "boolean"]
+
+
+@dataclass
+class ProjectInput(DataClassDictMixin):
+    """Represents a single input parameter for a pipeline step similar to GitHub workflows inputs."""
+
+    type: InputType
+    description: Optional[str] = None
+    default: Optional[Any] = None
+    required: bool = False
+
 
 @dataclass
 class ProjectConfig(DataClassDictMixin):
     pipeline: PipelineConfig
+    inputs: Optional[Dict[str, ProjectInput]] = None
     # This field is intended to keep track of where configuration was loaded from and
     # it is automatically added when configuration is loaded from file
     file: Optional[Path] = None

pypeline/domain/execution_context.py

@@ -1,7 +1,7 @@
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from py_app_dev.core.data_registry import DataRegistry
 from py_app_dev.core.subprocess import SubprocessExecutor
@@ -16,13 +16,23 @@ class ExecutionContext:
     install_dirs: List[Path] = field(default_factory=list)
     # Data registry to exchange data of any type
     data_registry: DataRegistry = field(default_factory=DataRegistry)
+    # Inputs provided for the pipeline run
+    inputs: Dict[str, Any] = field(default_factory=dict)
+    # Environment variables to be passed to the subprocesses
+    env_vars: Dict[str, Any] = field(default_factory=dict)
+
+    def get_input(self, name: str) -> Optional[Any]:
+        return self.inputs.get(name, None)
 
     def add_install_dirs(self, install_dirs: List[Path]) -> None:
         self.install_dirs.extend(install_dirs)
 
+    def add_env_vars(self, env_vars: Dict[str, Any]) -> None:
+        self.env_vars.update(env_vars)
+
     def create_process_executor(self, command: List[str | Path], cwd: Optional[Path] = None) -> SubprocessExecutor:
-        # Add the install directories to the PATH
         env = os.environ.copy()
+        env.update(self.env_vars)
         env["PATH"] = os.pathsep.join([path.absolute().as_posix() for path in self.install_dirs] + [env["PATH"]])
         # When started from a windows shell (e.g. cmd on Jenkins) the shell parameter must be set to True
         shell = True if os.name == "nt" else False

pypeline/domain/project_slurper.py

@@ -21,3 +21,7 @@ class ProjectSlurper:
     @property
     def project_dir(self) -> Path:
         return self.artifacts_locator.project_root_dir
+
+    @property
+    def project_config(self) -> ProjectConfig:
+        return self.user_config

pypeline/inputs_parser.py

@@ -0,0 +1,88 @@
+import argparse
+import distutils.util
+from typing import Any, Dict, List
+
+from py_app_dev.core.exceptions import UserNotificationException
+
+from .domain.config import InputType, ProjectInput
+
+
+def _map_type_for_argparse(input_type: InputType) -> Any:
+    if input_type == "string":
+        return str
+    elif input_type == "integer":
+        return int
+    elif input_type == "boolean":
+        return lambda x: bool(distutils.util.strtobool(x))
+    else:
+        raise ValueError(f"Unsupported input type specified: {input_type}")
+
+
+def create_argument_parser_from_definitions(
+    input_definitions: Dict[str, ProjectInput],
+    description: str = "Pypeline inputs parser.",
+) -> argparse.ArgumentParser:
+    """Creates and configures an ArgumentParser based on input definitions."""
+    parser = argparse.ArgumentParser(
+        description=description,
+        exit_on_error=False,
+    )
+
+    for name, definition in input_definitions.items():
+        arg_type = _map_type_for_argparse(definition.type)
+
+        help_text = definition.description or f"Input parameter '{name}'."
+        help_text += f" (Type: {definition.type}"
+        if definition.default is not None:
+            help_text += f", Default: {definition.default}"
+        help_text += ")"
+
+        parser.add_argument(
+            f"--{name}",
+            dest=name,  # Attribute name in the parsed namespace
+            help=help_text,
+            type=arg_type,  # Type conversion function
+            required=definition.required,
+            default=definition.default,
+        )
+
+    return parser
+
+
+class InputsParser:
+    """Parses input arguments based on definitions using argparse."""
+
+    def __init__(self, parser: argparse.ArgumentParser) -> None:
+        self.parser = parser
+
+    @classmethod
+    def from_inputs_definitions(
+        cls,
+        input_definitions: Dict[str, ProjectInput],
+        description: str = "Pypeline inputs parser.",
+    ) -> "InputsParser":
+        """Factory method to create an InputsParser instance from input definitions."""
+        return cls(create_argument_parser_from_definitions(input_definitions, description))
+
+    def parse_inputs(self, inputs: List[str]) -> Dict[str, Any]:
+        """Parses and validates the provided input strings against the configured parser. Inputs are expected as a list of 'name=value' elements."""
+        try:
+            args = []
+            for item in inputs:
+                if "=" not in item:
+                    raise UserNotificationException(f"Invalid input format: '{item}', expected 'name=value'")
+                name, value = item.split("=", 1)
+                args.append(f"--{name}")
+                args.append(value)
+
+            parsed_namespace = self.parser.parse_args(args)
+            return vars(parsed_namespace)
+        except argparse.ArgumentError as e:
+            error_message = f"Input validation error: {e}"
+            raise UserNotificationException(error_message) from e
+        except SystemExit as e:
+            error_message = f"Input parsing failed: {e}"
+            raise UserNotificationException(error_message) from e
+        except Exception as e:
+            error_message = f"An unexpected error occurred during input parsing: {e}"
+            raise UserNotificationException(error_message) from e

pypeline/kickstart/templates/project/pypeline.yaml

@@ -1,3 +1,9 @@
+inputs:
+  my_input:
+    type: string
+    description: My input description
+    default: my_input_default
+
 pipeline:
   - step: CreateVEnv
     module: pypeline.steps.create_venv

pypeline/main.py

@@ -10,6 +10,7 @@ from pypeline import __version__
 from pypeline.domain.execution_context import ExecutionContext
 from pypeline.domain.pipeline import PipelineConfigIterator
 from pypeline.domain.project_slurper import ProjectSlurper
+from pypeline.inputs_parser import InputsParser
 from pypeline.kickstart.create import KickstartProject
 from pypeline.pypeline import PipelineScheduler, PipelineStepsExecutor
 
@@ -48,6 +49,12 @@ def run(
     print: bool = typer.Option(False, help="Print the pipeline steps."),
     force_run: bool = typer.Option(False, help="Force the execution of a step even if it is not dirty."),
     dry_run: bool = typer.Option(False, help="Do not run any step, just print the steps that would be executed."),
+    inputs: Optional[List[str]] = typer.Option(  # noqa: B008
+        None,
+        "--input",
+        "-i",
+        help="Provide input parameters as key=value pairs (e.g., -i name=value -i flag=true).",
+    ),
 ) -> None:
     project_slurper = ProjectSlurper(project_dir, config_file)
     if print:
@@ -65,8 +72,15 @@ def run(
     if not steps_references:
         logger.info("No steps to run.")
         return
-
-    PipelineStepsExecutor[ExecutionContext](ExecutionContext(project_dir), steps_references, force_run, dry_run).run()
+    # Parse the inputs
+    input_definitions = project_slurper.project_config.inputs
+    if input_definitions is None and inputs:
+        raise UserNotificationException(f"Inputs are not accepted because there are no inputs defined in the '{project_slurper.project_config.file}' configuration.")
+    if input_definitions and inputs:
+        inputs_dict = InputsParser.from_inputs_definitions(input_definitions).parse_inputs(inputs)
+    else:
+        inputs_dict = {}
+    PipelineStepsExecutor[ExecutionContext](ExecutionContext(project_dir, inputs=inputs_dict), steps_references, force_run, dry_run).run()
 
 
 def main() -> None:

pypeline/steps/env_setup_script.py

@@ -31,6 +31,10 @@ class GenerateEnvSetupScript(PipelineStep[ExecutionContext]):
         else:
             logger.warning(f".env file not found: {dot_env_file}")
             env_vars = {}
+
+        # Merge execution context environment variables
+        env_vars.update(self.execution_context.env_vars)
+
         # Generate the environment setup scripts
         BatEnvSetupScriptGenerator(
             install_dirs=self.execution_context.install_dirs,

pypeline/steps/scoop_install.py

@@ -1,7 +1,7 @@
 import io
 import json
 import traceback
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, ClassVar, Dict, List, Optional
 
@@ -18,6 +18,7 @@ from ..domain.pipeline import PipelineStep
 @dataclass
 class ScoopInstallExecutionInfo(DataClassJSONMixin):
     install_dirs: List[Path]
+    env_vars: Dict[str, Any] = field(default_factory=dict)
 
     class Config(BaseConfig):
         """Base configuration for JSON serialization with omitted None values."""
@@ -72,6 +73,8 @@ class ScoopInstall(PipelineStep[ExecutionContext]):
         for app in installed_apps:
             self.logger.debug(f" - {app.name} ({app.version})")
             self.install_dirs.extend(app.get_all_required_paths())
+            # Collect environment variables from each app
+            self.execution_info.env_vars.update(app.env_vars)
         self.execution_info.to_json_file(self.execution_info_file)
         return 0
 
@@ -82,8 +85,10 @@ class ScoopInstall(PipelineStep[ExecutionContext]):
         return [self.execution_info_file, *self.install_dirs]
 
     def update_execution_context(self) -> None:
-        install_dirs = ScoopInstallExecutionInfo.from_json_file(self.execution_info_file).install_dirs
+        execution_info = ScoopInstallExecutionInfo.from_json_file(self.execution_info_file)
         # Make the list unique and keep the order
-        unique_paths = list(dict.fromkeys(install_dirs))
+        unique_paths = list(dict.fromkeys(execution_info.install_dirs))
         # Update the install directories for the subsequent steps
         self.execution_context.add_install_dirs(unique_paths)
+        if execution_info.env_vars:
+            self.execution_context.add_env_vars(execution_info.env_vars)

{pypeline_runner-1.11.0.dist-info → pypeline_runner-1.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pypeline-runner
-Version: 1.11.0
+Version: 1.13.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 Author: cuinixam

{pypeline_runner-1.11.0.dist-info → pypeline_runner-1.13.0.dist-info}/RECORD

@@ -1,11 +1,12 @@
-pypeline/__init__.py,sha256=uTVPjS5n_41_n-e8UW1W9n8Wt-3YLQI1WS0r4xGphOk,23
+pypeline/__init__.py,sha256=qyRV1Az4m5yhA_Nx1r_oZmIwZAZJpLTJdDw9oDJaqr8,23
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/domain/artifacts.py,sha256=5k7cVfHhLmvWXNuHKxXb9ca4Lxu0JytGQqazENCeKEU,1404
-pypeline/domain/config.py,sha256=AlavAaz5hSxa6yaKYnj-x71ClhOtA41yv5Qf2JIE47k,1650
-pypeline/domain/execution_context.py,sha256=BwjKxkv5nBIc2htCzMgXwnBFCORHpWZYWSKQD5wpcMA,1424
+pypeline/domain/config.py,sha256=6vWdHi7B6MA7NGi9wWXQE-YhSg1COSRmc3b1ji6AdAk,2053
+pypeline/domain/execution_context.py,sha256=cDdUOU32hKg2Mv6FNXCG1co8GUrBf5QOVKhaKO-xVxI,1848
 pypeline/domain/pipeline.py,sha256=2BsN2lw2znUxLH--Novyqe6SubVKs6XeHQSQf9yxirw,7788
-pypeline/domain/project_slurper.py,sha256=e3BLV88GvfW3efh0agUWKqMk3oWnL602P5u9jER_o9U,971
+pypeline/domain/project_slurper.py,sha256=64aqgVsrLgAK-c5QOM2N0wGCkOM1uNMio8yKjO2zDLU,1069
+pypeline/inputs_parser.py,sha256=8WJnWbACNmw_iYLhbAwcgGohCZWult8HtiJC9KiG3R0,3389
 pypeline/kickstart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/kickstart/create.py,sha256=iaB8MMC7PinpPBwRmz3rWZuE-DRbsLh2NtvczYaVgi0,2133
 pypeline/kickstart/templates/project/.gitignore,sha256=y8GJoVvRPez1LBokf1NaDOt2X1XtGwKFMF5yjA8AVS0,24
@@ -13,21 +14,21 @@ pypeline/kickstart/templates/project/bootstrap.ps1,sha256=eR8cyIJwDVt-bA2H3GWmxU
 pypeline/kickstart/templates/project/bootstrap.py,sha256=9cJp_sbU0SKvDjJluvyQfh0_xIsf6E1ct7sa7rRecNU,17244
 pypeline/kickstart/templates/project/poetry.toml,sha256=qgVxBdPcJZOHdHCTOBoZYna3cke4VGgRkNZ0bKgN6rs,32
 pypeline/kickstart/templates/project/pypeline.ps1,sha256=PjCJULG8XA3AHKbNt3oHrIgD04huvvpIue_gjSo3PMA,104
-pypeline/kickstart/templates/project/pypeline.yaml,sha256=7FeIu7OOqq7iToPVdP_a4MehIi9lUkBPbXnl8mYpxSo,279
+pypeline/kickstart/templates/project/pypeline.yaml,sha256=dyzn8vmHugpINVyB9l0QujK0J-oi--JQXunjUj9n3_k,385
 pypeline/kickstart/templates/project/pyproject.toml,sha256=A60HZ6Aqf8KTFLoC35SexuJ2Ze1I-khuYmYUKhphfNY,247
 pypeline/kickstart/templates/project/scoopfile.json,sha256=DcfZ8jYf9hmPHM-AWwnPKQJCzRG3fCuYtMeoY01nkag,219
 pypeline/kickstart/templates/project/steps/my_step.py,sha256=iZYTzWtL-qxEW_t7q079d-xpnRST_tumSzxqiQDW7sM,707
 pypeline/kickstart/templates/project/west.yaml,sha256=ZfVym7M4yzzC-Nm0vESdhqNYs6EaJuMQWGJBht_i0b4,188
-pypeline/main.py,sha256=Xk92H0z7z-WnrC6qdSrD5xHlTN65WDc64lSs6M4Jn_g,3324
+pypeline/main.py,sha256=zkDqmYuTjMuYLsKdAHO-8RiEXUAgNLWoS3pbFbIp0Eg,4070
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pypeline.py,sha256=-mquLfFlEvESk-HORhvjRMESIzdlVAgBLPjwUDOPLqg,7452
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/steps/create_venv.py,sha256=sajEl43Yqcd1f0BN9lZOhm93UIRXj4Qx4TdAOSxLkTU,2509
-pypeline/steps/env_setup_script.py,sha256=j6wrrnZSXXu1fq01zUdgiFBTTlpR4XXl_uMWwVCcQSI,2221
-pypeline/steps/scoop_install.py,sha256=TIrU2hPCS3SzHc2_L2A2EgGFkTQIGdrconDUeNkMx64,3484
+pypeline/steps/env_setup_script.py,sha256=u08A6pvMccFQbcnU0xruFvpU30PbDrttnbOjl1gDqog,2340
+pypeline/steps/scoop_install.py,sha256=E9wd5ygxRCuF2LD4X3wwTTnKa4XDmZ5fqeiH4dO0vjE,3788
 pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
-pypeline_runner-1.11.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-1.11.0.dist-info/METADATA,sha256=ggajX1hHht_OzArfOo-AskPyZN2aum8z6rExafdgYuw,7553
-pypeline_runner-1.11.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-pypeline_runner-1.11.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-1.11.0.dist-info/RECORD,,
+pypeline_runner-1.13.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-1.13.0.dist-info/METADATA,sha256=mXar7ok3VkWZqyqsiDqIOCyv-sw_t-LenSKDphG8Q8M,7553
+pypeline_runner-1.13.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+pypeline_runner-1.13.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-1.13.0.dist-info/RECORD,,