pypeline-runner 1.15.1__py3-none-any.whl → 1.16.0__py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "1.15.1"
+__version__ = "1.16.0"

pypeline/bootstrap/run.py

@@ -1,7 +1,6 @@
 import argparse
 import configparser
 import ensurepip
-import hashlib
 import json
 import logging
 import os
@@ -12,7 +11,6 @@ import tempfile
 import venv
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from enum import Enum
 from functools import total_ordering
 from pathlib import Path
 from typing import List, Optional, Tuple
@@ -153,106 +151,6 @@ class PyPiSourceParser:
         return sections
 
 
-class Runnable(ABC):
-    @abstractmethod
-    def run(self) -> int:
-        """Run stage."""
-
-    @abstractmethod
-    def get_name(self) -> str:
-        """Get stage name."""
-
-    @abstractmethod
-    def get_inputs(self) -> List[Path]:
-        """Get stage dependencies."""
-
-    @abstractmethod
-    def get_outputs(self) -> List[Path]:
-        """Get stage outputs."""
-
-
-class RunInfoStatus(Enum):
-    MATCH = (False, "Nothing has changed, previous execution information matches.")
-    NO_INFO = (True, "No previous execution information found.")
-    FILE_CHANGED = (True, "Dependencies have been changed.")
-
-    def __init__(self, should_run: bool, message: str) -> None:
-        self.should_run = should_run
-        self.message = message
-
-
-class Executor:
-    """
-    Accepts Runnable objects and executes them.
-
-    It create a file with the same name as the runnable's name
-    and stores the inputs and outputs with their hashes.
-    If the file exists, it checks the hashes of the inputs and outputs
-    and if they match, it skips the execution.
-    """
-
-    RUN_INFO_FILE_EXTENSION = ".deps.json"
-
-    def __init__(self, cache_dir: Path) -> None:
-        self.cache_dir = cache_dir
-
-    @staticmethod
-    def get_file_hash(path: Path) -> str:
-        """
-        Get the hash of a file.
-
-        Returns an empty string if the file does not exist.
-        """
-        if path.is_file():
-            with open(path, "rb") as file:
-                bytes = file.read()
-                readable_hash = hashlib.sha256(bytes).hexdigest()
-                return readable_hash
-        else:
-            return ""
-
-    def store_run_info(self, runnable: Runnable) -> None:
-        file_info = {
-            "inputs": {str(path): self.get_file_hash(path) for path in runnable.get_inputs()},
-            "outputs": {str(path): self.get_file_hash(path) for path in runnable.get_outputs()},
-        }
-
-        run_info_path = self.get_runnable_run_info_file(runnable)
-        run_info_path.parent.mkdir(parents=True, exist_ok=True)
-        with run_info_path.open("w") as f:
-            # pretty print the json file
-            json.dump(file_info, f, indent=4)
-
-    def get_runnable_run_info_file(self, runnable: Runnable) -> Path:
-        return self.cache_dir / f"{runnable.get_name()}{self.RUN_INFO_FILE_EXTENSION}"
-
-    def previous_run_info_matches(self, runnable: Runnable) -> RunInfoStatus:
-        run_info_path = self.get_runnable_run_info_file(runnable)
-        if not run_info_path.exists():
-            return RunInfoStatus.NO_INFO
-
-        with run_info_path.open() as f:
-            previous_info = json.load(f)
-
-        for file_type in ["inputs", "outputs"]:
-            for path_str, previous_hash in previous_info[file_type].items():
-                path = Path(path_str)
-                if self.get_file_hash(path) != previous_hash:
-                    return RunInfoStatus.FILE_CHANGED
-        return RunInfoStatus.MATCH
-
-    def execute(self, runnable: Runnable) -> int:
-        run_info_status = self.previous_run_info_matches(runnable)
-        if run_info_status.should_run:
-            logger.info(f"Runnable '{runnable.get_name()}' must run. {run_info_status.message}")
-            exit_code = runnable.run()
-            self.store_run_info(runnable)
-            return exit_code
-        logger.info(f"Runnable '{runnable.get_name()}' execution skipped. {run_info_status.message}")
-
-        return 0
-
-
 class UserNotificationException(Exception):
     pass
 
@@ -399,7 +297,7 @@ class UnixVirtualEnvironment(VirtualEnvironment):
         os.remove(temp_script_path)
 
 
-class CreateVirtualEnvironment(Runnable):
+class CreateVirtualEnvironment:
     def __init__(self, root_dir: Path, package_manager: str) -> None:
         self.root_dir = root_dir
         self.venv_dir = self.root_dir / ".venv"
@@ -422,9 +320,7 @@ class CreateVirtualEnvironment(Runnable):
         return "install"
 
     def run(self) -> int:
-        # Create the virtual environment if pip executable does not exist
-        if not self.virtual_env.pip_path().exists():
-            self.virtual_env.create()
+        self.virtual_env.create()
 
         # Get the PyPi source from pyproject.toml or Pipfile if it is defined
         pypi_source = PyPiSourceParser.from_pyproject(self.root_dir)
@@ -454,24 +350,6 @@ class CreateVirtualEnvironment(Runnable):
         else:
             raise UserNotificationException(f"Unsupported operating system: {sys.platform}")
 
-    def get_name(self) -> str:
-        return "create-virtual-environment"
-
-    def get_inputs(self) -> List[Path]:
-        venv_relevant_files = [
-            "uv.lock",
-            "poetry.lock",
-            "poetry.toml",
-            "pyproject.toml",
-            ".env",
-            "Pipfile",
-            "Pipfile.lock",
-        ]
-        return [self.root_dir / file for file in venv_relevant_files] + [get_bootstrap_script()]
-
-    def get_outputs(self) -> List[Path]:
-        return []
-
 
 def print_environment_info() -> None:
     str_bar = "".join(["-" for _ in range(80)])
@@ -501,8 +379,7 @@ def main() -> int:
         )
         args = parser.parse_args()
 
-        creator = CreateVirtualEnvironment(args.project_dir, package_manager=args.package_manager)
-        Executor(creator.venv_dir).execute(creator)
+        CreateVirtualEnvironment(args.project_dir, package_manager=args.package_manager).run()
     except UserNotificationException as e:
         logger.error(e)
         return 1

pypeline/steps/create_venv.py

@@ -1,8 +1,11 @@
+import re
 from dataclasses import dataclass
+from enum import Enum, auto
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional
 
 from mashumaro import DataClassDictMixin
+from py_app_dev.core.exceptions import UserNotificationException
 from py_app_dev.core.logging import logger
 
 from pypeline.bootstrap.run import get_bootstrap_script
@@ -13,50 +16,95 @@ from ..domain.pipeline import PipelineStep
 
 @dataclass
 class CreateVEnvConfig(DataClassDictMixin):
-    bootstrap_script: str = "bootstrap.py"
-    python_executable: str = "python3"
+    bootstrap_script: Optional[str] = None
+    python_executable: Optional[str] = None
     package_manager: Optional[str] = None
 
 
+class BootstrapScriptType(Enum):
+    CUSTOM = auto()
+    INTERNAL = auto()
+
+
 class CreateVEnv(PipelineStep[ExecutionContext]):
+    DEFAULT_PACKAGE_MANAGER = "uv>=0.6"
+    DEFAULT_PYTHON_EXECUTABLE = "python311"
+    SUPPORTED_PACKAGE_MANAGERS: ClassVar[Dict[str, List[str]]] = {
+        "uv": ["uv.lock", "pyproject.toml"],
+        "pipenv": ["Pipfile", "Pipfile.lock"],
+        "poetry": ["pyproject.toml", "poetry.lock"],
+    }
+
     def __init__(self, execution_context: ExecutionContext, group_name: str, config: Optional[Dict[str, Any]] = None) -> None:
+        self.user_config = CreateVEnvConfig.from_dict(config) if config else CreateVEnvConfig()
+        self.bootstrap_script_type = BootstrapScriptType.CUSTOM if self.user_config.bootstrap_script else BootstrapScriptType.INTERNAL
         super().__init__(execution_context, group_name, config)
         self.logger = logger.bind()
+        self.bootstrap_script = get_bootstrap_script()
+        self.package_manager = self.user_config.package_manager if self.user_config.package_manager else self.DEFAULT_PACKAGE_MANAGER
+        self.python_executable = self.user_config.python_executable if self.user_config.python_executable else self.DEFAULT_PYTHON_EXECUTABLE
+        self.venv_dir = self.project_root_dir / ".venv"
 
     @property
     def install_dirs(self) -> List[Path]:
         return [self.project_root_dir / dir for dir in [".venv/Scripts", ".venv/bin"] if (self.project_root_dir / dir).exists()]
 
+    @property
+    def package_manager_name(self) -> str:
+        match = re.match(r"^([a-zA-Z0-9_-]+)", self.package_manager)
+        if match:
+            result = match.group(1)
+            if result in self.SUPPORTED_PACKAGE_MANAGERS:
+                return result
+            else:
+                raise UserNotificationException(f"Package manager {result} is not supported. Supported package managers are: {', '.join(self.SUPPORTED_PACKAGE_MANAGERS)}")
+        else:
+            raise UserNotificationException(f"Could not extract the package manager name from {self.package_manager}")
+
     def get_name(self) -> str:
         return self.__class__.__name__
 
     def run(self) -> int:
         self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
-        config = CreateVEnvConfig.from_dict(self.config) if self.config else CreateVEnvConfig()
-        bootstrap_script = self.project_root_dir / config.bootstrap_script
-        bootstrap_args = []
-        if not bootstrap_script.exists():
-            self.logger.warning(f"Bootstrap script {bootstrap_script} does not exist. Use pypeline internal `bootstrap.py`.")
-            bootstrap_script = get_bootstrap_script()
-            # Only the internal bootstrap.py script supports arguments.
-            bootstrap_args = ["--project-dir", self.project_root_dir.as_posix()]
-            if config.package_manager:
-                bootstrap_args.extend(["--package-manager", f'"{config.package_manager}"'])
-        self.execution_context.create_process_executor(
-            [config.python_executable, bootstrap_script.as_posix(), *bootstrap_args],
-            cwd=self.project_root_dir,
-        ).execute()
-        self.execution_context.add_install_dirs(self.install_dirs)
+
+        if self.user_config.bootstrap_script:
+            bootstrap_script = self.project_root_dir / self.user_config.bootstrap_script
+            if not bootstrap_script.exists():
+                raise UserNotificationException(f"Bootstrap script {bootstrap_script} does not exist.")
+            self.execution_context.create_process_executor(
+                [self.python_executable, bootstrap_script.as_posix()],
+                cwd=self.project_root_dir,
+            ).execute()
+        else:
+            # The internal bootstrap script supports arguments.
+            bootstrap_args = [
+                "--project-dir",
+                self.project_root_dir.as_posix(),
+                "--package-manager",
+                f'"{self.package_manager}"',
+            ]
+            self.execution_context.create_process_executor(
+                [self.python_executable, self.bootstrap_script.as_posix(), *bootstrap_args],
+                cwd=self.project_root_dir,
+            ).execute()
         return 0
 
     def get_inputs(self) -> List[Path]:
-        return []
+        package_manager_relevant_file = self.SUPPORTED_PACKAGE_MANAGERS.get(self.package_manager_name, [])
+        return [self.project_root_dir / file for file in package_manager_relevant_file] + [self.bootstrap_script]
 
     def get_outputs(self) -> List[Path]:
-        return []
+        return [self.venv_dir]
+
+    def get_config(self) -> Optional[dict[str, str]]:
+        return {
+            "bootstrap_script": self.bootstrap_script.as_posix(),
+            "python_executable": self.python_executable,
+            "package_manager": self.package_manager,
+        }
 
     def update_execution_context(self) -> None:
-        pass
+        self.execution_context.add_install_dirs(self.install_dirs)
 
     def get_needs_dependency_management(self) -> bool:
-        return False
+        return False if self.bootstrap_script_type == BootstrapScriptType.CUSTOM else True

{pypeline_runner-1.15.1.dist-info → pypeline_runner-1.16.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pypeline-runner
-Version: 1.15.1
+Version: 1.16.0
 Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
 License: MIT
 Author: cuinixam

{pypeline_runner-1.15.1.dist-info → pypeline_runner-1.16.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-pypeline/__init__.py,sha256=lJTHPTq_lku8cPevlR7mFSxJNTqHgZJVZuDdqSiDNeM,23
+pypeline/__init__.py,sha256=zXjo5icgcGx5X40awUvUOk5_ohh0Pz1N3VFKYCfYEcM,23
 pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
 pypeline/bootstrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/bootstrap/run.py,sha256=VqBZI9eioGXpWI87GrGrQZaA6Zd-qtf1pi3efyGZezk,20201
+pypeline/bootstrap/run.py,sha256=iZS-VZGk2KnMRVOb3QuYPrTeINQqVdWkvgjEDpl8oQA,15927
 pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/domain/artifacts.py,sha256=5k7cVfHhLmvWXNuHKxXb9ca4Lxu0JytGQqazENCeKEU,1404
 pypeline/domain/config.py,sha256=6vWdHi7B6MA7NGi9wWXQE-YhSg1COSRmc3b1ji6AdAk,2053
@@ -21,12 +21,12 @@ pypeline/main.py,sha256=2mC2BDB1OWIXhaijBXG6Y1vfT8_yMZ4Dj55w5u7g7-w,4158
 pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/pypeline.py,sha256=-mquLfFlEvESk-HORhvjRMESIzdlVAgBLPjwUDOPLqg,7452
 pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/steps/create_venv.py,sha256=oNdgBvfYDI0obqgI-I9-vfJYjS6OsDmWOuYq1py9NGk,2488
+pypeline/steps/create_venv.py,sha256=NeS2Q0a0USwEc-PpcqDxOOCGEDRENlnzAxI0OjTL3N4,4861
 pypeline/steps/env_setup_script.py,sha256=u08A6pvMccFQbcnU0xruFvpU30PbDrttnbOjl1gDqog,2340
 pypeline/steps/scoop_install.py,sha256=DDXBD-5TVaT-u6Yf7A85uWoCgBVmLvj9nPGrZ8OQCz0,3853
 pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
-pypeline_runner-1.15.1.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
-pypeline_runner-1.15.1.dist-info/METADATA,sha256=7EXTte2u8T8ZQDDOho7VbKAqqvOz7d_zEWsjKOw_b34,7553
-pypeline_runner-1.15.1.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-pypeline_runner-1.15.1.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
-pypeline_runner-1.15.1.dist-info/RECORD,,
+pypeline_runner-1.16.0.dist-info/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
+pypeline_runner-1.16.0.dist-info/METADATA,sha256=LdGV3SefgqFcV2tki0BBK2pr4wNzIBCOZ7Ws7Dx-Jew,7553
+pypeline_runner-1.16.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+pypeline_runner-1.16.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
+pypeline_runner-1.16.0.dist-info/RECORD,,