pypeline-runner 1.15.1__tar.gz → 1.16.0__tar.gz

{pypeline_runner-1.15.1 → pypeline_runner-1.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pypeline-runner
-Version: 1.15.1
+Version: 1.16.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 Author: cuinixam

{pypeline_runner-1.15.1 → pypeline_runner-1.16.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pypeline-runner"
-version = "1.15.1"
+version = "1.16.0"
 description = "Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines)."
 authors = ["cuinixam <me@cuinixam.com>"]
 license = "MIT"

pypeline_runner-1.16.0/src/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.16.0"

{pypeline_runner-1.15.1 → pypeline_runner-1.16.0}/src/pypeline/bootstrap/run.py

@@ -1,7 +1,6 @@
 import argparse
 import configparser
 import ensurepip
-import hashlib
 import json
 import logging
 import os
@@ -12,7 +11,6 @@ import tempfile
 import venv
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from enum import Enum
 from functools import total_ordering
 from pathlib import Path
 from typing import List, Optional, Tuple
@@ -153,106 +151,6 @@ class PyPiSourceParser:
         return sections
 
 
-class Runnable(ABC):
-    @abstractmethod
-    def run(self) -> int:
-        """Run stage."""
-
-    @abstractmethod
-    def get_name(self) -> str:
-        """Get stage name."""
-
-    @abstractmethod
-    def get_inputs(self) -> List[Path]:
-        """Get stage dependencies."""
-
-    @abstractmethod
-    def get_outputs(self) -> List[Path]:
-        """Get stage outputs."""
-
-
-class RunInfoStatus(Enum):
-    MATCH = (False, "Nothing has changed, previous execution information matches.")
-    NO_INFO = (True, "No previous execution information found.")
-    FILE_CHANGED = (True, "Dependencies have been changed.")
-
-    def __init__(self, should_run: bool, message: str) -> None:
-        self.should_run = should_run
-        self.message = message
-
-
-class Executor:
-    """
-    Accepts Runnable objects and executes them.
-
-    It create a file with the same name as the runnable's name
-    and stores the inputs and outputs with their hashes.
-    If the file exists, it checks the hashes of the inputs and outputs
-    and if they match, it skips the execution.
-    """
-
-    RUN_INFO_FILE_EXTENSION = ".deps.json"
-
-    def __init__(self, cache_dir: Path) -> None:
-        self.cache_dir = cache_dir
-
-    @staticmethod
-    def get_file_hash(path: Path) -> str:
-        """
-        Get the hash of a file.
-
-        Returns an empty string if the file does not exist.
-        """
-        if path.is_file():
-            with open(path, "rb") as file:
-                bytes = file.read()
-                readable_hash = hashlib.sha256(bytes).hexdigest()
-                return readable_hash
-        else:
-            return ""
-
-    def store_run_info(self, runnable: Runnable) -> None:
-        file_info = {
-            "inputs": {str(path): self.get_file_hash(path) for path in runnable.get_inputs()},
-            "outputs": {str(path): self.get_file_hash(path) for path in runnable.get_outputs()},
-        }
-
-        run_info_path = self.get_runnable_run_info_file(runnable)
-        run_info_path.parent.mkdir(parents=True, exist_ok=True)
-        with run_info_path.open("w") as f:
-            # pretty print the json file
-            json.dump(file_info, f, indent=4)
-
-    def get_runnable_run_info_file(self, runnable: Runnable) -> Path:
-        return self.cache_dir / f"{runnable.get_name()}{self.RUN_INFO_FILE_EXTENSION}"
-
-    def previous_run_info_matches(self, runnable: Runnable) -> RunInfoStatus:
-        run_info_path = self.get_runnable_run_info_file(runnable)
-        if not run_info_path.exists():
-            return RunInfoStatus.NO_INFO
-
-        with run_info_path.open() as f:
-            previous_info = json.load(f)
-
-        for file_type in ["inputs", "outputs"]:
-            for path_str, previous_hash in previous_info[file_type].items():
-                path = Path(path_str)
-                if self.get_file_hash(path) != previous_hash:
-                    return RunInfoStatus.FILE_CHANGED
-        return RunInfoStatus.MATCH
-
-    def execute(self, runnable: Runnable) -> int:
-        run_info_status = self.previous_run_info_matches(runnable)
-        if run_info_status.should_run:
-            logger.info(f"Runnable '{runnable.get_name()}' must run. {run_info_status.message}")
-            exit_code = runnable.run()
-            self.store_run_info(runnable)
-            return exit_code
-        logger.info(f"Runnable '{runnable.get_name()}' execution skipped. {run_info_status.message}")
-
-        return 0
-
-
 class UserNotificationException(Exception):
     pass
 
@@ -399,7 +297,7 @@ class UnixVirtualEnvironment(VirtualEnvironment):
         os.remove(temp_script_path)
 
 
-class CreateVirtualEnvironment(Runnable):
+class CreateVirtualEnvironment:
     def __init__(self, root_dir: Path, package_manager: str) -> None:
         self.root_dir = root_dir
         self.venv_dir = self.root_dir / ".venv"
@@ -422,9 +320,7 @@ class CreateVirtualEnvironment(Runnable):
         return "install"
 
     def run(self) -> int:
-        # Create the virtual environment if pip executable does not exist
-        if not self.virtual_env.pip_path().exists():
-            self.virtual_env.create()
+        self.virtual_env.create()
 
         # Get the PyPi source from pyproject.toml or Pipfile if it is defined
         pypi_source = PyPiSourceParser.from_pyproject(self.root_dir)
@@ -454,24 +350,6 @@ class CreateVirtualEnvironment(Runnable):
         else:
             raise UserNotificationException(f"Unsupported operating system: {sys.platform}")
 
-    def get_name(self) -> str:
-        return "create-virtual-environment"
-
-    def get_inputs(self) -> List[Path]:
-        venv_relevant_files = [
-            "uv.lock",
-            "poetry.lock",
-            "poetry.toml",
-            "pyproject.toml",
-            ".env",
-            "Pipfile",
-            "Pipfile.lock",
-        ]
-        return [self.root_dir / file for file in venv_relevant_files] + [get_bootstrap_script()]
-
-    def get_outputs(self) -> List[Path]:
-        return []
-
 
 def print_environment_info() -> None:
     str_bar = "".join(["-" for _ in range(80)])
@@ -501,8 +379,7 @@ def main() -> int:
         )
         args = parser.parse_args()
 
-        creator = CreateVirtualEnvironment(args.project_dir, package_manager=args.package_manager)
-        Executor(creator.venv_dir).execute(creator)
+        CreateVirtualEnvironment(args.project_dir, package_manager=args.package_manager).run()
     except UserNotificationException as e:
         logger.error(e)
         return 1

pypeline_runner-1.16.0/src/pypeline/steps/create_venv.py

@@ -0,0 +1,110 @@
+import re
+from dataclasses import dataclass
+from enum import Enum, auto
+from pathlib import Path
+from typing import Any, ClassVar, Dict, List, Optional
+
+from mashumaro import DataClassDictMixin
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.logging import logger
+
+from pypeline.bootstrap.run import get_bootstrap_script
+
+from ..domain.execution_context import ExecutionContext
+from ..domain.pipeline import PipelineStep
+
+
+@dataclass
+class CreateVEnvConfig(DataClassDictMixin):
+    bootstrap_script: Optional[str] = None
+    python_executable: Optional[str] = None
+    package_manager: Optional[str] = None
+
+
+class BootstrapScriptType(Enum):
+    CUSTOM = auto()
+    INTERNAL = auto()
+
+
+class CreateVEnv(PipelineStep[ExecutionContext]):
+    DEFAULT_PACKAGE_MANAGER = "uv>=0.6"
+    DEFAULT_PYTHON_EXECUTABLE = "python311"
+    SUPPORTED_PACKAGE_MANAGERS: ClassVar[Dict[str, List[str]]] = {
+        "uv": ["uv.lock", "pyproject.toml"],
+        "pipenv": ["Pipfile", "Pipfile.lock"],
+        "poetry": ["pyproject.toml", "poetry.lock"],
+    }
+
+    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        self.user_config = CreateVEnvConfig.from_dict(config) if config else CreateVEnvConfig()
+        self.bootstrap_script_type = BootstrapScriptType.CUSTOM if self.user_config.bootstrap_script else BootstrapScriptType.INTERNAL
+        super().__init__(execution_context, group_name, config)
+        self.logger = logger.bind()
+        self.bootstrap_script = get_bootstrap_script()
+        self.package_manager = self.user_config.package_manager if self.user_config.package_manager else self.DEFAULT_PACKAGE_MANAGER
+        self.python_executable = self.user_config.python_executable if self.user_config.python_executable else self.DEFAULT_PYTHON_EXECUTABLE
+        self.venv_dir = self.project_root_dir / ".venv"
+
+    @property
+    def install_dirs(self) -> List[Path]:
+        return [self.project_root_dir / dir for dir in [".venv/Scripts", ".venv/bin"] if (self.project_root_dir / dir).exists()]
+
+    @property
+    def package_manager_name(self) -> str:
+        match = re.match(r"^([a-zA-Z0-9_-]+)", self.package_manager)
+        if match:
+            result = match.group(1)
+            if result in self.SUPPORTED_PACKAGE_MANAGERS:
+                return result
+            else:
+                raise UserNotificationException(f"Package manager {result} is not supported. Supported package managers are: {', '.join(self.SUPPORTED_PACKAGE_MANAGERS)}")
+        else:
+            raise UserNotificationException(f"Could not extract the package manager name from {self.package_manager}")
+
+    def get_name(self) -> str:
+        return self.__class__.__name__
+
+    def run(self) -> int:
+        self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
+
+        if self.user_config.bootstrap_script:
+            bootstrap_script = self.project_root_dir / self.user_config.bootstrap_script
+            if not bootstrap_script.exists():
+                raise UserNotificationException(f"Bootstrap script {bootstrap_script} does not exist.")
+            self.execution_context.create_process_executor(
+                [self.python_executable, bootstrap_script.as_posix()],
+                cwd=self.project_root_dir,
+            ).execute()
+        else:
+            # The internal bootstrap script supports arguments.
+            bootstrap_args = [
+                "--project-dir",
+                self.project_root_dir.as_posix(),
+                "--package-manager",
+                f'"{self.package_manager}"',
+            ]
+            self.execution_context.create_process_executor(
+                [self.python_executable, self.bootstrap_script.as_posix(), *bootstrap_args],
+                cwd=self.project_root_dir,
+            ).execute()
+        return 0
+
+    def get_inputs(self) -> List[Path]:
+        package_manager_relevant_file = self.SUPPORTED_PACKAGE_MANAGERS.get(self.package_manager_name, [])
+        return [self.project_root_dir / file for file in package_manager_relevant_file] + [self.bootstrap_script]
+
+    def get_outputs(self) -> List[Path]:
+        return [self.venv_dir]
+
+    def get_config(self) -> Optional[dict[str, str]]:
+        return {
+            "bootstrap_script": self.bootstrap_script.as_posix(),
+            "python_executable": self.python_executable,
+            "package_manager": self.package_manager,
+        }
+
+    def update_execution_context(self) -> None:
+        self.execution_context.add_install_dirs(self.install_dirs)
+
+    def get_needs_dependency_management(self) -> bool:
+        return False if self.bootstrap_script_type == BootstrapScriptType.CUSTOM else True

pypeline_runner-1.15.1/src/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.15.1"

pypeline_runner-1.15.1/src/pypeline/steps/create_venv.py

@@ -1,62 +0,0 @@
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from mashumaro import DataClassDictMixin
-from py_app_dev.core.logging import logger
-
-from pypeline.bootstrap.run import get_bootstrap_script
-
-from ..domain.execution_context import ExecutionContext
-from ..domain.pipeline import PipelineStep
-
-
-@dataclass
-class CreateVEnvConfig(DataClassDictMixin):
-    bootstrap_script: str = "bootstrap.py"
-    python_executable: str = "python3"
-    package_manager: Optional[str] = None
-
-
-class CreateVEnv(PipelineStep[ExecutionContext]):
-    def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
-        super().__init__(execution_context, group_name, config)
-        self.logger = logger.bind()
-
-    @property
-    def install_dirs(self) -> List[Path]:
-        return [self.project_root_dir / dir for dir in [".venv/Scripts", ".venv/bin"] if (self.project_root_dir / dir).exists()]
-
-    def get_name(self) -> str:
-        return self.__class__.__name__
-
-    def run(self) -> int:
-        self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        config = CreateVEnvConfig.from_dict(self.config) if self.config else CreateVEnvConfig()
-        bootstrap_script = self.project_root_dir / config.bootstrap_script
-        bootstrap_args = []
-        if not bootstrap_script.exists():
-            self.logger.warning(f"Bootstrap script {bootstrap_script} does not exist. Use pypeline internal `bootstrap.py`.")
-            bootstrap_script = get_bootstrap_script()
-            # Only the internal bootstrap.py script supports arguments.
-            bootstrap_args = ["--project-dir", self.project_root_dir.as_posix()]
-            if config.package_manager:
-                bootstrap_args.extend(["--package-manager", f'"{config.package_manager}"'])
-        self.execution_context.create_process_executor(
-            [config.python_executable, bootstrap_script.as_posix(), *bootstrap_args],
-            cwd=self.project_root_dir,
-        ).execute()
-        self.execution_context.add_install_dirs(self.install_dirs)
-        return 0
-
-    def get_inputs(self) -> List[Path]:
-        return []
-
-    def get_outputs(self) -> List[Path]:
-        return []
-
-    def update_execution_context(self) -> None:
-        pass
-
-    def get_needs_dependency_management(self) -> bool:
-        return False