pypeline-runner 1.0.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/PKG-INFO +2 -1
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/pyproject.toml +1 -1
- pypeline_runner-1.1.0/src/pypeline/__init__.py +1 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/create.py +0 -1
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/.gitignore +1 -0
- pypeline_runner-1.1.0/src/pypeline/kickstart/templates/project/bootstrap.ps1 +31 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/pypeline.yaml +2 -0
- pypeline_runner-1.0.0/src/pypeline/__init__.py +0 -1
- pypeline_runner-1.0.0/src/pypeline/kickstart/templates/bootstrap/bootstrap.ps1 +0 -137
- pypeline_runner-1.0.0/src/pypeline/kickstart/templates/bootstrap/bootstrap.py +0 -428
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/LICENSE +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/README.md +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/__run.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/domain/__init__.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/domain/artifacts.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/domain/config.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/domain/execution_context.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/domain/pipeline.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/domain/project_slurper.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/__init__.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/poetry.toml +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/pypeline.ps1 +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/pyproject.toml +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/scoopfile.json +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/steps/my_step.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/main.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/py.typed +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/pypeline.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/steps/__init__.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/steps/create_venv.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/steps/scoop_install.py +0 -0
- {pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/steps/west_install.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pypeline-runner
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
|
|
5
5
|
Home-page: https://github.com/cuinixam/pypeline
|
|
6
6
|
License: MIT
|
|
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.10
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
20
|
Classifier: Topic :: Software Development :: Libraries
|
|
20
21
|
Requires-Dist: py-app-dev (>=2.2,<3.0)
|
|
21
22
|
Requires-Dist: pyyaml (>=6.0,<7.0)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.1.0"
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
<#
|
|
2
|
+
.DESCRIPTION
|
|
3
|
+
Wrapper for installing dependencies
|
|
4
|
+
#>
|
|
5
|
+
|
|
6
|
+
function Invoke-Bootstrap {
|
|
7
|
+
# Download bootstrap scripts from external repository
|
|
8
|
+
Invoke-RestMethod https://raw.githubusercontent.com/avengineers/bootstrap-installer/v1.15.0/install.ps1 | Invoke-Expression
|
|
9
|
+
# Execute bootstrap script
|
|
10
|
+
. .\.bootstrap\bootstrap.ps1
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
## start of script
|
|
14
|
+
# Always set the $InformationPreference variable to "Continue" globally,
|
|
15
|
+
# this way it gets printed on execution and continues execution afterwards.
|
|
16
|
+
$InformationPreference = "Continue"
|
|
17
|
+
|
|
18
|
+
# Stop on first error
|
|
19
|
+
$ErrorActionPreference = "Stop"
|
|
20
|
+
|
|
21
|
+
Push-Location $PSScriptRoot
|
|
22
|
+
Write-Output "Running in ${pwd}"
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
# bootstrap environment
|
|
26
|
+
Invoke-Bootstrap
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
Pop-Location
|
|
30
|
+
}
|
|
31
|
+
## end of script
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "1.0.0"
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
<#
|
|
2
|
-
.DESCRIPTION
|
|
3
|
-
Script to install python and start the bootstrap.py.
|
|
4
|
-
This file was generated by yanga.
|
|
5
|
-
VERSION: 0.6.1
|
|
6
|
-
#>
|
|
7
|
-
|
|
8
|
-
$ErrorActionPreference = "Stop"
|
|
9
|
-
|
|
10
|
-
###################################################################################################
|
|
11
|
-
# Configuration
|
|
12
|
-
###################################################################################################
|
|
13
|
-
$bootstrapJsonPath = Join-Path $PSScriptRoot "bootstrap.json"
|
|
14
|
-
if (Test-Path $bootstrapJsonPath) {
|
|
15
|
-
$json = Get-Content $bootstrapJsonPath | ConvertFrom-Json
|
|
16
|
-
$config = @{
|
|
17
|
-
pythonVersion = $json.python_version
|
|
18
|
-
scoopInstaller = $json.scoop_installer
|
|
19
|
-
scoopMainBucketBaseUrl = $json.scoop_main_bucket_base_url
|
|
20
|
-
scoopPythonBucketBaseUrl = $json.scoop_python_bucket_base_url
|
|
21
|
-
}
|
|
22
|
-
} else {
|
|
23
|
-
$config = @{
|
|
24
|
-
pythonVersion = "3.10"
|
|
25
|
-
scoopInstaller = "https://raw.githubusercontent.com/ScoopInstaller/Install/master/install.ps1"
|
|
26
|
-
scoopMainBucketBaseUrl = "https://raw.githubusercontent.com/ScoopInstaller/Main/master/bucket"
|
|
27
|
-
scoopPythonBucketBaseUrl = "https://raw.githubusercontent.com/ScoopInstaller/Versions/master/bucket"
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
###################################################################################################
|
|
32
|
-
# Utility functions
|
|
33
|
-
###################################################################################################
|
|
34
|
-
|
|
35
|
-
# Update/Reload current environment variable PATH with settings from registry
|
|
36
|
-
Function Initialize-EnvPath {
|
|
37
|
-
# workaround for system-wide installations
|
|
38
|
-
if ($Env:USER_PATH_FIRST) {
|
|
39
|
-
$Env:Path = [System.Environment]::GetEnvironmentVariable("Path", "User") + ";" + [System.Environment]::GetEnvironmentVariable("Path", "Machine")
|
|
40
|
-
}
|
|
41
|
-
else {
|
|
42
|
-
$Env:Path = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path", "User")
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
Function Invoke-CommandLine {
|
|
47
|
-
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingInvokeExpression', '', Justification = 'Usually this statement must be avoided (https://learn.microsoft.com/en-us/powershell/scripting/learn/deep-dives/avoid-using-invoke-expression?view=powershell-7.3), here it is OK as it does not execute unknown code.')]
|
|
48
|
-
param (
|
|
49
|
-
[Parameter(Mandatory = $true, Position = 0)]
|
|
50
|
-
[string]$CommandLine,
|
|
51
|
-
[Parameter(Mandatory = $false, Position = 1)]
|
|
52
|
-
[bool]$StopAtError = $true,
|
|
53
|
-
[Parameter(Mandatory = $false, Position = 2)]
|
|
54
|
-
[bool]$Silent = $false
|
|
55
|
-
)
|
|
56
|
-
if (-Not $Silent) {
|
|
57
|
-
Write-Output "Executing: $CommandLine"
|
|
58
|
-
}
|
|
59
|
-
$global:LASTEXITCODE = 0
|
|
60
|
-
Invoke-Expression $CommandLine
|
|
61
|
-
if ($global:LASTEXITCODE -ne 0) {
|
|
62
|
-
if ($StopAtError) {
|
|
63
|
-
Write-Error "Command line call `"$CommandLine`" failed with exit code $global:LASTEXITCODE"
|
|
64
|
-
}
|
|
65
|
-
else {
|
|
66
|
-
if (-Not $Silent) {
|
|
67
|
-
Write-Output "Command line call `"$CommandLine`" failed with exit code $global:LASTEXITCODE, continuing ..."
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
Function Install-Scoop {
|
|
74
|
-
# Initial Scoop installation
|
|
75
|
-
if (-Not (Get-Command 'scoop' -ErrorAction SilentlyContinue)) {
|
|
76
|
-
$tempDir = [System.IO.Path]::GetTempPath()
|
|
77
|
-
$tempFile = "$tempDir\install.ps1"
|
|
78
|
-
Invoke-RestMethod $config.scoopInstaller -OutFile $tempFile
|
|
79
|
-
if ((New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
|
|
80
|
-
& $tempFile -RunAsAdmin
|
|
81
|
-
}
|
|
82
|
-
else {
|
|
83
|
-
& $tempFile
|
|
84
|
-
}
|
|
85
|
-
Initialize-EnvPath
|
|
86
|
-
Remove-Item $tempFile
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
# Install needed tools
|
|
90
|
-
Invoke-CommandLine "scoop update"
|
|
91
|
-
# Avoid deadlocks while updating scoop buckets
|
|
92
|
-
Invoke-CommandLine "scoop config autostash_on_conflict $true" -Silent $true
|
|
93
|
-
|
|
94
|
-
Initialize-EnvPath
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
###################################################################################################
|
|
98
|
-
# Main
|
|
99
|
-
###################################################################################################
|
|
100
|
-
|
|
101
|
-
# python executable name
|
|
102
|
-
$python = "python" + $config.pythonVersion.Replace(".", "")
|
|
103
|
-
|
|
104
|
-
# Check if scoop is installed
|
|
105
|
-
$scoopPath = (Get-Command scoop -ErrorAction SilentlyContinue).Source
|
|
106
|
-
if ($scoopPath -eq $null) {
|
|
107
|
-
Write-Output "Scoop not found. Trying to install scoop ..."
|
|
108
|
-
Install-Scoop
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
Write-Output "Found scoop under $scoopPath."
|
|
112
|
-
}
|
|
113
|
-
# Check if python is installed
|
|
114
|
-
$pythonPath = (Get-Command $python -ErrorAction SilentlyContinue).Source
|
|
115
|
-
if ($pythonPath -eq $null) {
|
|
116
|
-
Write-Output "$python not found. Try to install $python via scoop ..."
|
|
117
|
-
# Install Python installer dependencies
|
|
118
|
-
Invoke-CommandLine "scoop install $($config.scoopMainBucketBaseUrl)/dark.json"
|
|
119
|
-
Invoke-CommandLine "scoop install $($config.scoopMainBucketBaseUrl)/lessmsi.json"
|
|
120
|
-
# Install python
|
|
121
|
-
Invoke-CommandLine "scoop install $($config.scoopPythonBucketBaseUrl)/$python.json"
|
|
122
|
-
}
|
|
123
|
-
else {
|
|
124
|
-
Write-Output "$python found in $pythonPath"
|
|
125
|
-
# Extract the directory of python exe file and add it to PATH. It needs to be the first entry in PATH
|
|
126
|
-
# such that this version is used when the user calls python and not python311
|
|
127
|
-
$pythonDir = [System.IO.Path]::GetDirectoryName($pythonPath)
|
|
128
|
-
Write-Output "Adding $pythonDir to PATH"
|
|
129
|
-
$Env:Path += ";$pythonDir"
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
# Call the bootstrap.py if it exists with all provided arguments
|
|
133
|
-
$buildScript = Join-Path $PSScriptRoot "bootstrap.py"
|
|
134
|
-
if (Test-Path $buildScript) {
|
|
135
|
-
Write-Output "Calling $buildScript ..."
|
|
136
|
-
& $python $buildScript $args
|
|
137
|
-
}
|
|
@@ -1,428 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# This file was generated by yanga.
|
|
3
|
-
# VERSION: 0.6.1
|
|
4
|
-
import configparser
|
|
5
|
-
import hashlib
|
|
6
|
-
import json
|
|
7
|
-
import logging
|
|
8
|
-
import os
|
|
9
|
-
import re
|
|
10
|
-
import subprocess # nosec
|
|
11
|
-
import sys
|
|
12
|
-
import tempfile
|
|
13
|
-
import venv
|
|
14
|
-
from abc import ABC, abstractmethod
|
|
15
|
-
from dataclasses import dataclass
|
|
16
|
-
from enum import Enum
|
|
17
|
-
from pathlib import Path
|
|
18
|
-
from typing import List, Optional
|
|
19
|
-
|
|
20
|
-
logging.basicConfig(level=logging.INFO)
|
|
21
|
-
logger = logging.getLogger("bootstrap")
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
this_dir = Path(__file__).parent
|
|
25
|
-
this_file = Path(__file__).name
|
|
26
|
-
bootstrap_json_path = Path(__file__).parent / "bootstrap.json"
|
|
27
|
-
if bootstrap_json_path.exists():
|
|
28
|
-
with bootstrap_json_path.open("r") as f:
|
|
29
|
-
config = json.load(f)
|
|
30
|
-
package_manager = config.get("python_package_manager", "poetry>=1.7.1")
|
|
31
|
-
else:
|
|
32
|
-
package_manager = "poetry>=1.7.1"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
@dataclass
|
|
36
|
-
class PyPiSource:
|
|
37
|
-
name: str
|
|
38
|
-
url: str
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
@dataclass
|
|
42
|
-
class TomlSection:
|
|
43
|
-
name: str
|
|
44
|
-
content: str
|
|
45
|
-
|
|
46
|
-
def __str__(self) -> str:
|
|
47
|
-
return f"[{self.name}]\n{self.content}"
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
class PyPiSourceParser:
|
|
51
|
-
tool_poetry_source_section = "tool.poetry.source"
|
|
52
|
-
|
|
53
|
-
@staticmethod
|
|
54
|
-
def from_pyproject_toml(pyproject_toml: Path) -> Optional[PyPiSource]:
|
|
55
|
-
if not pyproject_toml.exists():
|
|
56
|
-
return None
|
|
57
|
-
return PyPiSourceParser.from_pyproject_toml_content(pyproject_toml.read_text())
|
|
58
|
-
|
|
59
|
-
@staticmethod
|
|
60
|
-
def from_pyproject_toml_content(content: str) -> Optional[PyPiSource]:
|
|
61
|
-
sections = PyPiSourceParser.get_toml_sections(content)
|
|
62
|
-
for section in sections:
|
|
63
|
-
if section.name == PyPiSourceParser.tool_poetry_source_section:
|
|
64
|
-
try:
|
|
65
|
-
parser = configparser.ConfigParser()
|
|
66
|
-
parser.read_string(str(section))
|
|
67
|
-
name = parser[section.name]["name"].strip('"')
|
|
68
|
-
url = parser[section.name]["url"].strip('"')
|
|
69
|
-
return PyPiSource(name, url)
|
|
70
|
-
except KeyError as e:
|
|
71
|
-
raise UserNotificationException(
|
|
72
|
-
f"Could not parse PyPi source from pyproject.toml section {section.name}. "
|
|
73
|
-
f"Please make sure the section has the following format:\n"
|
|
74
|
-
f"[{PyPiSourceParser.tool_poetry_source_section}]\n"
|
|
75
|
-
f'name = "name"\n'
|
|
76
|
-
f'url = "https://url"\n'
|
|
77
|
-
f"verify_ssl = true"
|
|
78
|
-
) from e
|
|
79
|
-
return None
|
|
80
|
-
|
|
81
|
-
@staticmethod
|
|
82
|
-
def get_toml_sections(toml_content: str) -> List[TomlSection]:
|
|
83
|
-
# Use a regular expression to find all sections with [ or [[ at the beginning of the line
|
|
84
|
-
raw_sections = re.findall(r"^\[+.*\]+\n(?:[^[]*\n)*", toml_content, re.MULTILINE)
|
|
85
|
-
|
|
86
|
-
# Process each section
|
|
87
|
-
sections = []
|
|
88
|
-
for section in raw_sections:
|
|
89
|
-
# Split the lines, from the first line extract the section name
|
|
90
|
-
# and merge all the other lines into the content
|
|
91
|
-
lines = section.splitlines()
|
|
92
|
-
name_match = re.match(r"^\[+([^]]*)\]+", lines[0])
|
|
93
|
-
if name_match:
|
|
94
|
-
name = name_match.group(1).strip()
|
|
95
|
-
content = "\n".join(lines[1:]).strip()
|
|
96
|
-
sections.append(TomlSection(name, content))
|
|
97
|
-
|
|
98
|
-
return sections
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
class Runnable(ABC):
|
|
102
|
-
@abstractmethod
|
|
103
|
-
def run(self) -> int:
|
|
104
|
-
"""Run stage"""
|
|
105
|
-
|
|
106
|
-
@abstractmethod
|
|
107
|
-
def get_name(self) -> str:
|
|
108
|
-
"""Get stage name"""
|
|
109
|
-
|
|
110
|
-
@abstractmethod
|
|
111
|
-
def get_inputs(self) -> List[Path]:
|
|
112
|
-
"""Get stage dependencies"""
|
|
113
|
-
|
|
114
|
-
@abstractmethod
|
|
115
|
-
def get_outputs(self) -> List[Path]:
|
|
116
|
-
"""Get stage outputs"""
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
class RunInfoStatus(Enum):
|
|
120
|
-
MATCH = (False, "Nothing changed. Previous execution info matches.")
|
|
121
|
-
NO_INFO = (True, "No previous execution info found.")
|
|
122
|
-
FILE_NOT_FOUND = (True, "File not found.")
|
|
123
|
-
FILE_CHANGED = (True, "File has changed.")
|
|
124
|
-
|
|
125
|
-
def __init__(self, should_run: bool, message: str) -> None:
|
|
126
|
-
self.should_run = should_run
|
|
127
|
-
self.message = message
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
class Executor:
|
|
131
|
-
"""
|
|
132
|
-
Accepts Runnable objects and executes them.
|
|
133
|
-
It create a file with the same name as the runnable's name
|
|
134
|
-
and stores the inputs and outputs with their hashes.
|
|
135
|
-
If the file exists, it checks the hashes of the inputs and outputs
|
|
136
|
-
and if they match, it skips the execution.
|
|
137
|
-
"""
|
|
138
|
-
|
|
139
|
-
RUN_INFO_FILE_EXTENSION = ".deps.json"
|
|
140
|
-
|
|
141
|
-
def __init__(self, cache_dir: Path) -> None:
|
|
142
|
-
self.cache_dir = cache_dir
|
|
143
|
-
|
|
144
|
-
@staticmethod
|
|
145
|
-
def get_file_hash(path: Path) -> str:
|
|
146
|
-
with open(path, "rb") as file:
|
|
147
|
-
bytes = file.read()
|
|
148
|
-
readable_hash = hashlib.sha256(bytes).hexdigest()
|
|
149
|
-
return readable_hash
|
|
150
|
-
|
|
151
|
-
def store_run_info(self, runnable: Runnable) -> None:
|
|
152
|
-
file_info = {
|
|
153
|
-
"inputs": {str(path): self.get_file_hash(path) for path in runnable.get_inputs()},
|
|
154
|
-
"outputs": {str(path): self.get_file_hash(path) for path in runnable.get_outputs()},
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
run_info_path = self.get_runnable_run_info_file(runnable)
|
|
158
|
-
run_info_path.parent.mkdir(parents=True, exist_ok=True)
|
|
159
|
-
with run_info_path.open("w") as f:
|
|
160
|
-
# pretty print the json file
|
|
161
|
-
json.dump(file_info, f, indent=4)
|
|
162
|
-
|
|
163
|
-
def get_runnable_run_info_file(self, runnable: Runnable) -> Path:
|
|
164
|
-
return self.cache_dir / f"{runnable.get_name()}{self.RUN_INFO_FILE_EXTENSION}"
|
|
165
|
-
|
|
166
|
-
def previous_run_info_matches(self, runnable: Runnable) -> RunInfoStatus:
|
|
167
|
-
run_info_path = self.get_runnable_run_info_file(runnable)
|
|
168
|
-
if not run_info_path.exists():
|
|
169
|
-
return RunInfoStatus.NO_INFO
|
|
170
|
-
|
|
171
|
-
with run_info_path.open() as f:
|
|
172
|
-
previous_info = json.load(f)
|
|
173
|
-
|
|
174
|
-
for file_type in ["inputs", "outputs"]:
|
|
175
|
-
for path_str, previous_hash in previous_info[file_type].items():
|
|
176
|
-
path = Path(path_str)
|
|
177
|
-
if not path.exists():
|
|
178
|
-
return RunInfoStatus.FILE_NOT_FOUND
|
|
179
|
-
elif self.get_file_hash(path) != previous_hash:
|
|
180
|
-
return RunInfoStatus.FILE_CHANGED
|
|
181
|
-
return RunInfoStatus.MATCH
|
|
182
|
-
|
|
183
|
-
def execute(self, runnable: Runnable) -> int:
|
|
184
|
-
run_info_status = self.previous_run_info_matches(runnable)
|
|
185
|
-
if run_info_status.should_run:
|
|
186
|
-
logger.info(f"Runnable '{runnable.get_name()}' must run. {run_info_status.message}")
|
|
187
|
-
exit_code = runnable.run()
|
|
188
|
-
self.store_run_info(runnable)
|
|
189
|
-
return exit_code
|
|
190
|
-
logger.info(f"Runnable '{runnable.get_name()}' execution skipped. {run_info_status.message}")
|
|
191
|
-
|
|
192
|
-
return 0
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
class UserNotificationException(Exception):
|
|
196
|
-
pass
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
class SubprocessExecutor:
|
|
200
|
-
def __init__(
|
|
201
|
-
self,
|
|
202
|
-
command: List[str | Path],
|
|
203
|
-
cwd: Optional[Path] = None,
|
|
204
|
-
capture_output: bool = True,
|
|
205
|
-
):
|
|
206
|
-
self.command = " ".join([str(cmd) for cmd in command])
|
|
207
|
-
self.current_working_directory = cwd
|
|
208
|
-
self.capture_output = capture_output
|
|
209
|
-
|
|
210
|
-
def execute(self) -> None:
|
|
211
|
-
result = None
|
|
212
|
-
try:
|
|
213
|
-
current_dir = (self.current_working_directory or Path.cwd()).as_posix()
|
|
214
|
-
logger.info(f"Running command: {self.command} in {current_dir}")
|
|
215
|
-
# print all virtual environment variables
|
|
216
|
-
logger.debug(json.dumps(dict(os.environ), indent=4))
|
|
217
|
-
result = subprocess.run(
|
|
218
|
-
self.command.split(), # noqa: S603
|
|
219
|
-
cwd=current_dir,
|
|
220
|
-
capture_output=self.capture_output,
|
|
221
|
-
text=True, # to get stdout and stderr as strings instead of bytes
|
|
222
|
-
) # nosec
|
|
223
|
-
result.check_returncode()
|
|
224
|
-
except subprocess.CalledProcessError as e:
|
|
225
|
-
raise UserNotificationException(
|
|
226
|
-
f"Command '{self.command}' failed with:\n"
|
|
227
|
-
f"{result.stdout if result else ''}\n"
|
|
228
|
-
f"{result.stderr if result else e}"
|
|
229
|
-
) from e
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
class VirtualEnvironment(ABC):
|
|
233
|
-
def __init__(self, venv_dir: Path) -> None:
|
|
234
|
-
self.venv_dir = venv_dir
|
|
235
|
-
|
|
236
|
-
def create(self, clear: bool = False) -> None:
|
|
237
|
-
"""
|
|
238
|
-
Create a new virtual environment. This should configure the virtual environment such that
|
|
239
|
-
subsequent calls to `pip` and `run` operate within this environment.
|
|
240
|
-
"""
|
|
241
|
-
try:
|
|
242
|
-
venv.create(self.venv_dir, with_pip=True, clear=clear)
|
|
243
|
-
except PermissionError as e:
|
|
244
|
-
if "python.exe" in str(e):
|
|
245
|
-
raise UserNotificationException(
|
|
246
|
-
f"Failed to create virtual environment in {self.venv_dir}.\n"
|
|
247
|
-
f"Virtual environment python.exe is still running. Please kill all instances and run again.\n"
|
|
248
|
-
f"Error: {e}"
|
|
249
|
-
) from e
|
|
250
|
-
raise UserNotificationException(
|
|
251
|
-
f"Failed to create virtual environment in {self.venv_dir}.\n"
|
|
252
|
-
f"Please make sure you have the necessary permissions.\n"
|
|
253
|
-
f"Error: {e}"
|
|
254
|
-
) from e
|
|
255
|
-
|
|
256
|
-
def pip_configure(self, index_url: str, verify_ssl: bool) -> None:
|
|
257
|
-
"""
|
|
258
|
-
Configure pip to use the given index URL and SSL verification setting. This method should
|
|
259
|
-
behave as if the user had activated the virtual environment and run `pip config set
|
|
260
|
-
global.index-url <index_url>` and `pip config set global.cert <verify_ssl>` from the
|
|
261
|
-
command line.
|
|
262
|
-
|
|
263
|
-
Args:
|
|
264
|
-
----
|
|
265
|
-
index_url: The index URL to use for pip.
|
|
266
|
-
verify_ssl: Whether to verify SSL certificates when using pip.
|
|
267
|
-
|
|
268
|
-
"""
|
|
269
|
-
# The pip configuration file should be in the virtual environment directory %VIRTUAL_ENV%
|
|
270
|
-
pip_ini_path = self.pip_config_path()
|
|
271
|
-
with open(pip_ini_path, "w") as pip_ini_file:
|
|
272
|
-
match_host = re.match(r"https?://([^/]+)", index_url)
|
|
273
|
-
pip_ini_file.write(f"[global]\nindex-url = {index_url}\n")
|
|
274
|
-
if match_host:
|
|
275
|
-
pip_ini_file.write(f"trusted-host = {match_host.group(1)}\n")
|
|
276
|
-
if not verify_ssl:
|
|
277
|
-
pip_ini_file.write("cert = false\n")
|
|
278
|
-
|
|
279
|
-
def pip(self, args: List[str]) -> None:
|
|
280
|
-
SubprocessExecutor([self.pip_path().as_posix(), *args], this_dir).execute()
|
|
281
|
-
|
|
282
|
-
@abstractmethod
|
|
283
|
-
def pip_path(self) -> Path:
|
|
284
|
-
"""
|
|
285
|
-
Get the path to the pip executable within the virtual environment.
|
|
286
|
-
"""
|
|
287
|
-
|
|
288
|
-
@abstractmethod
|
|
289
|
-
def pip_config_path(self) -> Path:
|
|
290
|
-
"""
|
|
291
|
-
Get the path to the pip configuration file within the virtual environment.
|
|
292
|
-
"""
|
|
293
|
-
|
|
294
|
-
@abstractmethod
|
|
295
|
-
def run(self, args: List[str], capture_output: bool = True) -> None:
|
|
296
|
-
"""
|
|
297
|
-
Run an arbitrary command within the virtual environment. This method should behave as if the
|
|
298
|
-
user had activated the virtual environment and run the given command from the command line.
|
|
299
|
-
|
|
300
|
-
Args:
|
|
301
|
-
----
|
|
302
|
-
*args: Command-line arguments. For example, `run('python', 'setup.py', 'install')`
|
|
303
|
-
should behave similarly to `python setup.py install` at the command line.
|
|
304
|
-
|
|
305
|
-
"""
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
class WindowsVirtualEnvironment(VirtualEnvironment):
|
|
309
|
-
def __init__(self, venv_dir: Path) -> None:
|
|
310
|
-
super().__init__(venv_dir)
|
|
311
|
-
self.activate_script = self.venv_dir.joinpath("Scripts/activate")
|
|
312
|
-
|
|
313
|
-
def pip_path(self) -> Path:
|
|
314
|
-
return self.venv_dir.joinpath("Scripts/pip")
|
|
315
|
-
|
|
316
|
-
def pip_config_path(self) -> Path:
|
|
317
|
-
return self.venv_dir.joinpath("pip.ini")
|
|
318
|
-
|
|
319
|
-
def run(self, args: List[str], capture_output: bool = True) -> None:
|
|
320
|
-
SubprocessExecutor(
|
|
321
|
-
[f"cmd /c {self.activate_script.as_posix()} && ", *args],
|
|
322
|
-
this_dir,
|
|
323
|
-
capture_output,
|
|
324
|
-
).execute()
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
class UnixVirtualEnvironment(VirtualEnvironment):
|
|
328
|
-
def __init__(self, venv_dir: Path) -> None:
|
|
329
|
-
super().__init__(venv_dir)
|
|
330
|
-
self.activate_script = self.venv_dir.joinpath("bin/activate")
|
|
331
|
-
|
|
332
|
-
def pip_path(self) -> Path:
|
|
333
|
-
return self.venv_dir.joinpath("bin/pip")
|
|
334
|
-
|
|
335
|
-
def pip_config_path(self) -> Path:
|
|
336
|
-
return self.venv_dir.joinpath("pip.conf")
|
|
337
|
-
|
|
338
|
-
def run(self, args: List[str], capture_output: bool = True) -> None:
|
|
339
|
-
# Create a temporary shell script
|
|
340
|
-
with tempfile.NamedTemporaryFile("w", delete=False, suffix=".sh") as f:
|
|
341
|
-
f.write("#!/bin/bash\n") # Add a shebang line
|
|
342
|
-
f.write(f"source {self.activate_script.as_posix()}\n") # Write the activate command
|
|
343
|
-
f.write(" ".join(args)) # Write the provided command
|
|
344
|
-
temp_script_path = f.name # Get the path of the temporary script
|
|
345
|
-
|
|
346
|
-
# Make the temporary script executable
|
|
347
|
-
SubprocessExecutor(["chmod", "+x", temp_script_path]).execute()
|
|
348
|
-
# Run the temporary script
|
|
349
|
-
SubprocessExecutor([f"{Path(temp_script_path).as_posix()}"], this_dir, capture_output).execute()
|
|
350
|
-
# Delete the temporary script
|
|
351
|
-
os.remove(temp_script_path)
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
class CreateVirtualEnvironment(Runnable):
|
|
355
|
-
def __init__(
|
|
356
|
-
self,
|
|
357
|
-
) -> None:
|
|
358
|
-
self.root_dir = this_dir
|
|
359
|
-
self.venv_dir = self.root_dir / ".venv"
|
|
360
|
-
self.virtual_env = self.instantiate_os_specific_venv(self.venv_dir)
|
|
361
|
-
|
|
362
|
-
@property
|
|
363
|
-
def package_manager_name(self) -> str:
|
|
364
|
-
match = re.match(r"^([a-zA-Z0-9_-]+)", package_manager)
|
|
365
|
-
|
|
366
|
-
if match:
|
|
367
|
-
return match.group(1)
|
|
368
|
-
else:
|
|
369
|
-
raise UserNotificationException(f"Could not extract the package manager name from {package_manager}")
|
|
370
|
-
|
|
371
|
-
def run(self) -> int:
|
|
372
|
-
logger.info("Running project build script")
|
|
373
|
-
self.virtual_env.create(clear=self.venv_dir.exists())
|
|
374
|
-
pypi_source = PyPiSourceParser.from_pyproject_toml(self.root_dir / "pyproject.toml")
|
|
375
|
-
if pypi_source:
|
|
376
|
-
self.virtual_env.pip_configure(index_url=pypi_source.url, verify_ssl=True)
|
|
377
|
-
self.virtual_env.pip(["install", package_manager])
|
|
378
|
-
self.virtual_env.run([self.package_manager_name, "install"])
|
|
379
|
-
return 0
|
|
380
|
-
|
|
381
|
-
@staticmethod
|
|
382
|
-
def instantiate_os_specific_venv(venv_dir: Path) -> VirtualEnvironment:
|
|
383
|
-
if sys.platform.startswith("win32"):
|
|
384
|
-
return WindowsVirtualEnvironment(venv_dir)
|
|
385
|
-
elif sys.platform.startswith("linux") or sys.platform.startswith("darwin"):
|
|
386
|
-
return UnixVirtualEnvironment(venv_dir)
|
|
387
|
-
else:
|
|
388
|
-
raise UserNotificationException(f"Unsupported operating system: {sys.platform}")
|
|
389
|
-
|
|
390
|
-
def get_name(self) -> str:
|
|
391
|
-
return "create-virtual-environment"
|
|
392
|
-
|
|
393
|
-
def get_inputs(self) -> List[Path]:
|
|
394
|
-
bootstrap_files = list(self.root_dir.glob("bootstrap.*"))
|
|
395
|
-
venv_relevant_files = ["poetry.lock", "poetry.toml", "pyproject.toml"]
|
|
396
|
-
return [self.root_dir / file for file in venv_relevant_files] + bootstrap_files
|
|
397
|
-
|
|
398
|
-
def get_outputs(self) -> List[Path]:
|
|
399
|
-
return []
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
def print_environment_info() -> None:
|
|
403
|
-
str_bar = "".join(["-" for _ in range(80)])
|
|
404
|
-
logger.debug(str_bar)
|
|
405
|
-
logger.debug("Environment: \n" + json.dumps(dict(os.environ), indent=4))
|
|
406
|
-
logger.info(str_bar)
|
|
407
|
-
logger.info(f"Arguments: {sys.argv[1:]}")
|
|
408
|
-
logger.info(str_bar)
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
def main() -> int:
|
|
412
|
-
try:
|
|
413
|
-
# print_environment_info()
|
|
414
|
-
build = CreateVirtualEnvironment()
|
|
415
|
-
Executor(build.venv_dir).execute(build)
|
|
416
|
-
except UserNotificationException as e:
|
|
417
|
-
logger.error(e)
|
|
418
|
-
return 1
|
|
419
|
-
return 0
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
if __name__ == "__main__":
|
|
423
|
-
sys.exit(main())
|
|
424
|
-
|
|
425
|
-
if __name__ == "__test_main__":
|
|
426
|
-
"""This is used to execute the build script from a test and
|
|
427
|
-
it shall not call sys.exit()"""
|
|
428
|
-
main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pypeline_runner-1.0.0 → pypeline_runner-1.1.0}/src/pypeline/kickstart/templates/project/poetry.toml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|