pypeline-runner 0.1.0__py3-none-any.whl

pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

pypeline/__run.py

@@ -0,0 +1,12 @@
+"""
+Used to run pypeline from the command line when run from this repository.
+
+This is required because pypeline module is not visible when running from the repository.
+"""
+
+import runpy
+import sys
+from pathlib import Path
+
+sys.path.insert(0, Path(__file__).parent.parent.absolute().as_posix())
+runpy.run_module("pypeline.main", run_name="__main__")

pypeline/domain/artifacts.py

@@ -0,0 +1,36 @@
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from py_app_dev.core.exceptions import UserNotificationException
+
+CONFIG_FILENAME = "pypeline.yaml"
+
+
+class ProjectArtifactsLocator:
+    """Provides paths to project artifacts."""
+
+    def __init__(
+        self,
+        project_root_dir: Path,
+    ) -> None:
+        self.project_root_dir = project_root_dir
+        self.build_dir = project_root_dir / "build"
+        self.config_file = project_root_dir / CONFIG_FILENAME
+        self.external_dependencies_dir = self.build_dir / "external"
+        scripts_dir = "Scripts" if sys.platform.startswith("win32") else "bin"
+        self.venv_scripts_dir = self.project_root_dir.joinpath(".venv").joinpath(scripts_dir)
+
+    def locate_artifact(self, artifact: str, first_search_paths: List[Optional[Path]]) -> Path:
+        search_paths = []
+        for path in first_search_paths:
+            if path:
+                search_paths.append(path.parent if path.is_file() else path)
+        for dir in [
+            *search_paths,
+            self.project_root_dir,
+        ]:
+            if dir and (artifact_path := Path(dir).joinpath(artifact)).exists():
+                return artifact_path
+        else:
+            raise UserNotificationException(f"Artifact '{artifact}' not found in the project.")

pypeline/domain/config.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+from mashumaro import DataClassDictMixin
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.pipeline import PipelineConfig
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+
+@dataclass
+class ProjectConfig(DataClassDictMixin):
+    pipeline: PipelineConfig
+    # This field is intended to keep track of where configuration was loaded from and
+    # it is automatically added when configuration is loaded from file
+    file: Optional[Path] = None
+
+    @classmethod
+    def from_file(cls, config_file: Path) -> "ProjectConfig":
+        config_dict = cls.parse_to_dict(config_file)
+        return cls.from_dict(config_dict)
+
+    @staticmethod
+    def parse_to_dict(config_file: Path) -> Dict[str, Any]:
+        try:
+            with open(config_file) as fs:
+                config_dict = yaml.safe_load(fs)
+                # Add file name to config to keep track of where configuration was loaded from
+                config_dict["file"] = config_file
+            return config_dict
+        except ScannerError as e:
+            raise UserNotificationException(f"Failed scanning configuration file '{config_file}'. \nError: {e}") from e
+        except ParserError as e:
+            raise UserNotificationException(f"Failed parsing configuration file '{config_file}'. \nError: {e}") from e

pypeline/domain/execution_context.py

@@ -0,0 +1,27 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+from py_app_dev.core.subprocess import SubprocessExecutor
+
+from .artifacts import ProjectArtifactsLocator
+
+
+@dataclass
+class ExecutionContext:
+    project_root_dir: Path
+    # Keep track of all install directories, updated by any step for the subsequent steps
+    install_dirs: List[Path] = field(default_factory=list)
+
+    def add_install_dirs(self, install_dirs: List[Path]) -> None:
+        self.install_dirs.extend(install_dirs)
+
+    def create_process_executor(self, command: List[str | Path], cwd: Optional[Path] = None) -> SubprocessExecutor:
+        # Add the install directories to the PATH
+        env = os.environ.copy()
+        env["PATH"] = os.pathsep.join([path.absolute().as_posix() for path in self.install_dirs] + [env["PATH"]])
+        return SubprocessExecutor(command, cwd=cwd, env=env, shell=True)  # noqa: S604
+
+    def create_artifacts_locator(self) -> ProjectArtifactsLocator:
+        return ProjectArtifactsLocator(self.project_root_dir)

pypeline/domain/pipeline.py

@@ -0,0 +1,28 @@
+from abc import abstractmethod
+from pathlib import Path
+from typing import Type
+
+from py_app_dev.core.runnable import Runnable
+
+from .execution_context import ExecutionContext
+
+
+class PipelineStep(Runnable):
+    def __init__(self, execution_context: ExecutionContext, output_dir: Path) -> None:
+        self.execution_context = execution_context
+        self.output_dir = output_dir
+        self.project_root_dir = self.execution_context.project_root_dir
+
+    @abstractmethod
+    def update_execution_context(self) -> None:
+        pass
+
+
+class PipelineStepReference:
+    def __init__(self, group_name: str, _class: Type[PipelineStep]) -> None:
+        self.group_name = group_name
+        self._class = _class
+
+    @property
+    def name(self) -> str:
+        return self._class.__name__

pypeline/domain/project_slurper.py

@@ -0,0 +1,22 @@
+from pathlib import Path
+
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.logging import logger
+
+from .artifacts import ProjectArtifactsLocator
+from .config import PipelineConfig, ProjectConfig
+
+
+class ProjectSlurper:
+    def __init__(self, project_dir: Path) -> None:
+        self.logger = logger.bind()
+        self.artifacts_locator = ProjectArtifactsLocator(project_dir)
+        try:
+            self.user_config: ProjectConfig = ProjectConfig.from_file(self.artifacts_locator.config_file)
+        except FileNotFoundError:
+            raise UserNotificationException(f"Project configuration file '{self.artifacts_locator.config_file}' not found.") from None
+        self.pipeline: PipelineConfig = self.user_config.pipeline
+
+    @property
+    def project_dir(self) -> Path:
+        return self.artifacts_locator.project_root_dir

pypeline/kickstart/create.py

@@ -0,0 +1,58 @@
+import shutil
+from pathlib import Path
+from typing import List, Optional, Union
+
+from py_app_dev.core.exceptions import UserNotificationException
+from py_app_dev.core.logging import logger
+
+
+class ProjectBuilder:
+    def __init__(self, project_dir: Path, input_dir: Optional[Path] = None) -> None:
+        self.project_dir = project_dir
+        self.input_dir = input_dir if input_dir else Path(__file__).parent.joinpath("templates")
+
+        self.dirs: List[Path] = []
+        self.check_target_directory_flag = True
+
+    def with_disable_target_directory_check(self) -> "ProjectBuilder":
+        self.check_target_directory_flag = False
+        return self
+
+    def with_dir(self, dir: Union[Path, str]) -> "ProjectBuilder":
+        self.dirs.append(self.resolve_file_path(dir))
+        return self
+
+    def resolve_file_paths(self, files: List[Path | str]) -> List[Path]:
+        return [self.resolve_file_path(file) for file in files]
+
+    def resolve_file_path(self, file: Union[Path, str]) -> Path:
+        return self.input_dir.joinpath(file) if isinstance(file, str) else file
+
+    @staticmethod
+    def _check_target_directory(project_dir: Path) -> None:
+        if project_dir.is_dir() and any(project_dir.iterdir()):
+            raise UserNotificationException(f"Project directory '{project_dir}' is not empty." " The target directory shall either be empty or not exist.")
+
+    def build(self) -> None:
+        if self.check_target_directory_flag:
+            self._check_target_directory(self.project_dir)
+        for dir in self.dirs:
+            shutil.copytree(dir, self.project_dir, dirs_exist_ok=True)
+
+
+class KickstartProject:
+    def __init__(self, project_dir: Path, bootstrap_only: bool = False, force: bool = False) -> None:
+        self.logger = logger.bind()
+        self.project_dir = project_dir
+        self.bootstrap_only = bootstrap_only
+        self.force = force
+
+    def run(self) -> None:
+        self.logger.info(f"Kickstart new project in '{self.project_dir.absolute().as_posix()}'")
+        project_builder = ProjectBuilder(self.project_dir)
+        project_builder.with_dir("bootstrap")
+        if self.bootstrap_only or self.force:
+            project_builder.with_disable_target_directory_check()
+        if not self.bootstrap_only:
+            project_builder.with_dir("project")
+        project_builder.build()

pypeline/kickstart/templates/bootstrap/bootstrap.ps1

@@ -0,0 +1,137 @@
+<#
+.DESCRIPTION
+    Script to install python and start the bootstrap.py.
+    This file was generated by yanga.
+    VERSION: 0.6.1
+#>
+
+$ErrorActionPreference = "Stop"
+
+###################################################################################################
+# Configuration
+###################################################################################################
+$bootstrapJsonPath = Join-Path $PSScriptRoot "bootstrap.json"
+if (Test-Path $bootstrapJsonPath) {
+    $json = Get-Content $bootstrapJsonPath | ConvertFrom-Json
+    $config = @{
+        pythonVersion = $json.python_version
+        scoopInstaller = $json.scoop_installer
+        scoopMainBucketBaseUrl = $json.scoop_main_bucket_base_url
+        scoopPythonBucketBaseUrl = $json.scoop_python_bucket_base_url
+    }
+} else {
+    $config = @{
+        pythonVersion = "3.10"
+        scoopInstaller = "https://raw.githubusercontent.com/ScoopInstaller/Install/master/install.ps1"
+        scoopMainBucketBaseUrl = "https://raw.githubusercontent.com/ScoopInstaller/Main/master/bucket"
+        scoopPythonBucketBaseUrl = "https://raw.githubusercontent.com/ScoopInstaller/Versions/master/bucket"
+    }
+}
+
+###################################################################################################
+# Utility functions
+###################################################################################################
+
+# Update/Reload current environment variable PATH with settings from registry
+Function Initialize-EnvPath {
+    # workaround for system-wide installations
+    if ($Env:USER_PATH_FIRST) {
+        $Env:Path = [System.Environment]::GetEnvironmentVariable("Path", "User") + ";" + [System.Environment]::GetEnvironmentVariable("Path", "Machine")
+    }
+    else {
+        $Env:Path = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path", "User")
+    }
+}
+
+Function Invoke-CommandLine {
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingInvokeExpression', '', Justification = 'Usually this statement must be avoided (https://learn.microsoft.com/en-us/powershell/scripting/learn/deep-dives/avoid-using-invoke-expression?view=powershell-7.3), here it is OK as it does not execute unknown code.')]
+    param (
+        [Parameter(Mandatory = $true, Position = 0)]
+        [string]$CommandLine,
+        [Parameter(Mandatory = $false, Position = 1)]
+        [bool]$StopAtError = $true,
+        [Parameter(Mandatory = $false, Position = 2)]
+        [bool]$Silent = $false
+    )
+    if (-Not $Silent) {
+        Write-Output "Executing: $CommandLine"
+    }
+    $global:LASTEXITCODE = 0
+    Invoke-Expression $CommandLine
+    if ($global:LASTEXITCODE -ne 0) {
+        if ($StopAtError) {
+            Write-Error "Command line call `"$CommandLine`" failed with exit code $global:LASTEXITCODE"
+        }
+        else {
+            if (-Not $Silent) {
+                Write-Output "Command line call `"$CommandLine`" failed with exit code $global:LASTEXITCODE, continuing ..."
+            }
+        }
+    }
+}
+
+Function Install-Scoop {
+    # Initial Scoop installation
+    if (-Not (Get-Command 'scoop' -ErrorAction SilentlyContinue)) {
+        $tempDir = [System.IO.Path]::GetTempPath()
+        $tempFile = "$tempDir\install.ps1"
+        Invoke-RestMethod $config.scoopInstaller -OutFile $tempFile
+        if ((New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
+            & $tempFile -RunAsAdmin
+        }
+        else {
+            & $tempFile
+        }
+        Initialize-EnvPath
+        Remove-Item $tempFile
+    }
+
+    # Install needed tools
+    Invoke-CommandLine "scoop update"
+    # Avoid deadlocks while updating scoop buckets
+    Invoke-CommandLine "scoop config autostash_on_conflict $true" -Silent $true
+
+    Initialize-EnvPath
+}
+
+###################################################################################################
+# Main
+###################################################################################################
+
+# python executable name
+$python = "python" + $config.pythonVersion.Replace(".", "")
+
+# Check if scoop is installed
+$scoopPath = (Get-Command scoop -ErrorAction SilentlyContinue).Source
+if ($scoopPath -eq $null) {
+    Write-Output "Scoop not found. Trying to install scoop ..."
+    Install-Scoop
+}
+else {
+    Write-Output "Found scoop under $scoopPath."
+}
+# Check if python is installed
+$pythonPath = (Get-Command $python -ErrorAction SilentlyContinue).Source
+if ($pythonPath -eq $null) {
+    Write-Output "$python not found. Try to install $python via scoop ..."
+    # Install Python installer dependencies
+    Invoke-CommandLine "scoop install $($config.scoopMainBucketBaseUrl)/dark.json"
+    Invoke-CommandLine "scoop install $($config.scoopMainBucketBaseUrl)/lessmsi.json"
+    # Install python
+    Invoke-CommandLine "scoop install $($config.scoopPythonBucketBaseUrl)/$python.json"
+}
+else {
+    Write-Output "$python found in $pythonPath"
+    # Extract the directory of python exe file and add it to PATH. It needs to be the first entry in PATH
+    # such that this version is used when the user calls python and not python311
+    $pythonDir = [System.IO.Path]::GetDirectoryName($pythonPath)
+    Write-Output "Adding $pythonDir to PATH"
+    $Env:Path += ";$pythonDir"
+}
+
+# Call the bootstrap.py if it exists with all provided arguments
+$buildScript = Join-Path $PSScriptRoot "bootstrap.py"
+if (Test-Path $buildScript) {
+    Write-Output "Calling $buildScript ..."
+    & $python $buildScript $args
+}