pypeline-runner 1.19.1__py3-none-any.whl → 1.21.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pypeline/__init__.py +1 -1
- pypeline/bootstrap/run.py +543 -145
- pypeline/kickstart/templates/project/pypeline.yaml +1 -1
- pypeline/kickstart/templates/project/pyproject.toml +1 -1
- pypeline/main.py +2 -1
- pypeline/steps/create_venv.py +166 -6
- pypeline/steps/scoop_install.py +7 -0
- {pypeline_runner-1.19.1.dist-info → pypeline_runner-1.21.0.dist-info}/METADATA +4 -2
- {pypeline_runner-1.19.1.dist-info → pypeline_runner-1.21.0.dist-info}/RECORD +12 -12
- {pypeline_runner-1.19.1.dist-info → pypeline_runner-1.21.0.dist-info}/WHEEL +1 -1
- {pypeline_runner-1.19.1.dist-info → pypeline_runner-1.21.0.dist-info}/entry_points.txt +0 -0
- {pypeline_runner-1.19.1.dist-info → pypeline_runner-1.21.0.dist-info/licenses}/LICENSE +0 -0
pypeline/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "1.
|
|
1
|
+
__version__ = "1.21.0"
|
pypeline/bootstrap/run.py
CHANGED
|
@@ -1,30 +1,93 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import configparser
|
|
3
3
|
import ensurepip
|
|
4
|
+
import hashlib
|
|
4
5
|
import json
|
|
5
6
|
import logging
|
|
6
7
|
import os
|
|
7
8
|
import re
|
|
8
|
-
import
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess # nosec
|
|
9
11
|
import sys
|
|
10
|
-
import tempfile
|
|
11
12
|
import venv
|
|
12
13
|
from abc import ABC, abstractmethod
|
|
13
|
-
from dataclasses import dataclass
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from enum import Enum
|
|
14
16
|
from functools import total_ordering
|
|
15
17
|
from pathlib import Path
|
|
16
|
-
from typing import List, Optional, Tuple
|
|
18
|
+
from typing import Any, List, Optional, Sequence, Tuple
|
|
17
19
|
from urllib.parse import urlparse
|
|
18
20
|
|
|
19
21
|
logging.basicConfig(level=logging.INFO)
|
|
20
22
|
logger = logging.getLogger("bootstrap")
|
|
21
23
|
|
|
22
24
|
|
|
25
|
+
DEFAULT_PACKAGE_MANAGER = "poetry>=2.1.0"
|
|
26
|
+
DEFAULT_BOOTSTRAP_PACKAGES = ["pip-system-certs>=4.0,<5.0"]
|
|
27
|
+
BOOTSTRAP_COMPLETE_MARKER = ".bootstrap-complete"
|
|
28
|
+
VENV_PYTHON_VERSION_MARKER = ".python_version"
|
|
29
|
+
|
|
30
|
+
|
|
23
31
|
def get_bootstrap_script() -> Path:
|
|
24
32
|
"""Get the path to the internal bootstrap script."""
|
|
25
33
|
return Path(__file__)
|
|
26
34
|
|
|
27
35
|
|
|
36
|
+
@dataclass
|
|
37
|
+
class BootstrapConfig:
|
|
38
|
+
"""Configuration for the bootstrap process loaded from bootstrap.json."""
|
|
39
|
+
|
|
40
|
+
python_version: str = ""
|
|
41
|
+
package_manager: str = DEFAULT_PACKAGE_MANAGER
|
|
42
|
+
package_manager_args: List[str] = field(default_factory=list)
|
|
43
|
+
bootstrap_packages: List[str] = field(default_factory=lambda: list(DEFAULT_BOOTSTRAP_PACKAGES))
|
|
44
|
+
bootstrap_cache_dir: Optional[Path] = None
|
|
45
|
+
venv_install_command: Optional[str] = None
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def from_json_file(cls, json_path: Path) -> "BootstrapConfig":
|
|
49
|
+
"""Load configuration from a JSON file."""
|
|
50
|
+
if not json_path.exists():
|
|
51
|
+
return cls()
|
|
52
|
+
|
|
53
|
+
with json_path.open("r") as file_handle:
|
|
54
|
+
data = json.load(file_handle)
|
|
55
|
+
|
|
56
|
+
bootstrap_packages = data.get("bootstrap_packages", list(DEFAULT_BOOTSTRAP_PACKAGES))
|
|
57
|
+
|
|
58
|
+
cache_dir_str = data.get("bootstrap_cache_dir")
|
|
59
|
+
cache_dir = Path(cache_dir_str).expanduser() if cache_dir_str else None
|
|
60
|
+
|
|
61
|
+
return cls(
|
|
62
|
+
python_version=data.get("python_version", ""),
|
|
63
|
+
package_manager=data.get("python_package_manager", DEFAULT_PACKAGE_MANAGER),
|
|
64
|
+
package_manager_args=data.get("python_package_manager_args", []),
|
|
65
|
+
bootstrap_packages=bootstrap_packages,
|
|
66
|
+
bootstrap_cache_dir=cache_dir,
|
|
67
|
+
venv_install_command=data.get("venv_install_command"),
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def get_bootstrap_cache_dir(self) -> Path:
|
|
71
|
+
"""Return the bootstrap cache directory, defaulting to ~/.bootstrap."""
|
|
72
|
+
if self.bootstrap_cache_dir:
|
|
73
|
+
return self.bootstrap_cache_dir
|
|
74
|
+
return Path.home() / ".bootstrap"
|
|
75
|
+
|
|
76
|
+
def compute_bootstrap_env_hash(self) -> str:
|
|
77
|
+
"""Compute a hash for the bootstrap environment based on configuration."""
|
|
78
|
+
if self.python_version:
|
|
79
|
+
python_major_minor = ".".join(self.python_version.split(".")[:2])
|
|
80
|
+
else:
|
|
81
|
+
python_major_minor = f"{sys.version_info[0]}.{sys.version_info[1]}"
|
|
82
|
+
components = [
|
|
83
|
+
f"python={python_major_minor}",
|
|
84
|
+
f"manager={self.package_manager}",
|
|
85
|
+
f"packages={sorted(self.bootstrap_packages)}",
|
|
86
|
+
]
|
|
87
|
+
content = "|".join(str(component) for component in components)
|
|
88
|
+
return hashlib.sha256(content.encode()).hexdigest()[:12]
|
|
89
|
+
|
|
90
|
+
|
|
28
91
|
@total_ordering
|
|
29
92
|
class Version:
|
|
30
93
|
def __init__(self, version_str: str) -> None:
|
|
@@ -32,7 +95,6 @@ class Version:
|
|
|
32
95
|
|
|
33
96
|
@staticmethod
|
|
34
97
|
def parse_version(version_str: str) -> Tuple[int, ...]:
|
|
35
|
-
"""Convert a version string into a tuple of integers for comparison."""
|
|
36
98
|
return tuple(map(int, re.split(r"\D+", version_str)))
|
|
37
99
|
|
|
38
100
|
def __eq__(self, other: object) -> bool:
|
|
@@ -40,9 +102,7 @@ class Version:
|
|
|
40
102
|
return NotImplemented
|
|
41
103
|
return self.version == other.version
|
|
42
104
|
|
|
43
|
-
def __lt__(self, other:
|
|
44
|
-
if not isinstance(other, Version):
|
|
45
|
-
return NotImplemented
|
|
105
|
+
def __lt__(self, other: "Version") -> bool:
|
|
46
106
|
return self.version < other.version
|
|
47
107
|
|
|
48
108
|
def __repr__(self) -> str:
|
|
@@ -65,72 +125,36 @@ class TomlSection:
|
|
|
65
125
|
|
|
66
126
|
|
|
67
127
|
class PyPiSourceParser:
|
|
68
|
-
@staticmethod
|
|
69
|
-
def find_pypi_source_in_content(content: str) -> Optional[PyPiSource]:
|
|
70
|
-
"""Parses TOML content, finds the first section containing 'name' and 'url' keys, and returns it as a PyPiSource."""
|
|
71
|
-
sections = PyPiSourceParser.get_toml_sections(content)
|
|
72
|
-
logger.debug(f"Found {len(sections)} potential sections in TOML content.")
|
|
73
|
-
|
|
74
|
-
for section in sections:
|
|
75
|
-
logger.debug(f"Checking section: [{section.name}]")
|
|
76
|
-
try:
|
|
77
|
-
parser = configparser.ConfigParser(interpolation=None) # Disable interpolation
|
|
78
|
-
# Provide the section string directly to read_string
|
|
79
|
-
# The TomlSection.__str__ method formats it correctly
|
|
80
|
-
parser.read_string(str(section))
|
|
81
|
-
|
|
82
|
-
# Check if the section was parsed and contains the required keys
|
|
83
|
-
if section.name in parser and "name" in parser[section.name] and "url" in parser[section.name]:
|
|
84
|
-
name = parser[section.name]["name"].strip("\"' ") # Strip quotes and whitespace
|
|
85
|
-
url = parser[section.name]["url"].strip("\"' ") # Strip quotes and whitespace
|
|
86
|
-
|
|
87
|
-
# Ensure values are not empty after stripping
|
|
88
|
-
if name and url:
|
|
89
|
-
logger.info(f"Found valid PyPI source in section '[{section.name}]': name='{name}', url='{url}'")
|
|
90
|
-
return PyPiSource(name=name, url=url)
|
|
91
|
-
else:
|
|
92
|
-
logger.debug(f"Section '[{section.name}]' contains 'name' and 'url' keys, but one or both values are empty.")
|
|
93
|
-
else:
|
|
94
|
-
logger.debug(f"Section '[{section.name}]' does not contain both 'name' and 'url' keys.")
|
|
95
|
-
|
|
96
|
-
except configparser.Error as e:
|
|
97
|
-
# This might happen if the section content is not valid INI/config format
|
|
98
|
-
# or if the section name itself causes issues (though get_toml_sections should handle it)
|
|
99
|
-
logger.debug(f"Could not parse section '[{section.name}]' with configparser: {e}")
|
|
100
|
-
# Continue to the next section
|
|
101
|
-
continue
|
|
102
|
-
|
|
103
|
-
logger.info("No suitable PyPI source section found in the provided TOML content.")
|
|
104
|
-
return None
|
|
105
|
-
|
|
106
128
|
@staticmethod
|
|
107
129
|
def from_pyproject(project_dir: Path) -> Optional[PyPiSource]:
|
|
108
|
-
"""Reads pyproject.toml or Pipfile and finds the PyPI source configuration without relying on a specific section name."""
|
|
109
130
|
pyproject_toml = project_dir / "pyproject.toml"
|
|
110
131
|
pipfile = project_dir / "Pipfile"
|
|
111
|
-
content = None
|
|
112
|
-
file_checked = None
|
|
113
|
-
|
|
114
132
|
if pyproject_toml.exists():
|
|
115
|
-
|
|
116
|
-
content = pyproject_toml.read_text()
|
|
117
|
-
file_checked = pyproject_toml
|
|
133
|
+
return PyPiSourceParser.from_toml_content(pyproject_toml.read_text(), "tool.poetry.source")
|
|
118
134
|
elif pipfile.exists():
|
|
119
|
-
|
|
120
|
-
content = pipfile.read_text()
|
|
121
|
-
file_checked = pipfile
|
|
122
|
-
|
|
123
|
-
if content:
|
|
124
|
-
source = PyPiSourceParser.find_pypi_source_in_content(content)
|
|
125
|
-
if source:
|
|
126
|
-
return source
|
|
127
|
-
else:
|
|
128
|
-
logger.debug(f"No PyPI source definition found in {file_checked}")
|
|
129
|
-
return None
|
|
135
|
+
return PyPiSourceParser.from_toml_content(pipfile.read_text(), "source")
|
|
130
136
|
else:
|
|
131
|
-
logger.debug("Neither pyproject.toml nor Pipfile found in the project directory.")
|
|
132
137
|
return None
|
|
133
138
|
|
|
139
|
+
@staticmethod
|
|
140
|
+
def from_toml_content(content: str, source_section_name: str) -> Optional[PyPiSource]:
|
|
141
|
+
sections = PyPiSourceParser.get_toml_sections(content)
|
|
142
|
+
for section in sections:
|
|
143
|
+
if section.name == source_section_name:
|
|
144
|
+
try:
|
|
145
|
+
parser = configparser.ConfigParser()
|
|
146
|
+
parser.read_string(str(section))
|
|
147
|
+
name = parser[section.name]["name"].strip('"')
|
|
148
|
+
url = parser[section.name]["url"].strip('"')
|
|
149
|
+
return PyPiSource(name, url)
|
|
150
|
+
except KeyError:
|
|
151
|
+
raise UserNotificationException(
|
|
152
|
+
f"Could not parse PyPi source from section {section.name}. "
|
|
153
|
+
f"Please make sure the section has the following format:\n"
|
|
154
|
+
f"[{source_section_name}]\nname = 'name'\nurl = 'https://url'\nverify_ssl = true"
|
|
155
|
+
) from None
|
|
156
|
+
return None
|
|
157
|
+
|
|
134
158
|
@staticmethod
|
|
135
159
|
def get_toml_sections(toml_content: str) -> List[TomlSection]:
|
|
136
160
|
# Use a regular expression to find all sections with [ or [[ at the beginning of the line
|
|
@@ -151,6 +175,118 @@ class PyPiSourceParser:
|
|
|
151
175
|
return sections
|
|
152
176
|
|
|
153
177
|
|
|
178
|
+
class Runnable(ABC):
|
|
179
|
+
@abstractmethod
|
|
180
|
+
def run(self) -> int:
|
|
181
|
+
"""Run stage."""
|
|
182
|
+
|
|
183
|
+
@abstractmethod
|
|
184
|
+
def get_name(self) -> str:
|
|
185
|
+
"""Get stage name."""
|
|
186
|
+
|
|
187
|
+
@abstractmethod
|
|
188
|
+
def get_inputs(self) -> List[Path]:
|
|
189
|
+
"""Get stage dependencies."""
|
|
190
|
+
|
|
191
|
+
@abstractmethod
|
|
192
|
+
def get_outputs(self) -> List[Path]:
|
|
193
|
+
"""Get stage outputs."""
|
|
194
|
+
|
|
195
|
+
def get_config(self) -> Optional[dict[str, Any]]:
|
|
196
|
+
"""Get stage configuration for change detection."""
|
|
197
|
+
return None
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class RunInfoStatus(Enum):
|
|
201
|
+
MATCH = (False, "Nothing has changed, previous execution information matches.")
|
|
202
|
+
NO_INFO = (True, "No previous execution information found.")
|
|
203
|
+
FILE_CHANGED = (True, "Dependencies have been changed.")
|
|
204
|
+
CONFIG_CHANGED = (True, "Configuration has been changed.")
|
|
205
|
+
|
|
206
|
+
def __init__(self, should_run: bool, message: str) -> None:
|
|
207
|
+
self.should_run = should_run
|
|
208
|
+
self.message = message
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class Executor:
|
|
212
|
+
"""
|
|
213
|
+
Accepts Runnable objects and executes them.
|
|
214
|
+
|
|
215
|
+
It create a file with the same name as the runnable's name
|
|
216
|
+
and stores the inputs and outputs with their hashes.
|
|
217
|
+
If the file exists, it checks the hashes of the inputs and outputs
|
|
218
|
+
and if they match, it skips the execution.
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
RUN_INFO_FILE_EXTENSION = ".deps.json"
|
|
222
|
+
|
|
223
|
+
def __init__(self, cache_dir: Path) -> None:
|
|
224
|
+
self.cache_dir = cache_dir
|
|
225
|
+
|
|
226
|
+
@staticmethod
|
|
227
|
+
def get_file_hash(path: Path) -> str:
|
|
228
|
+
"""
|
|
229
|
+
Get the hash of a file.
|
|
230
|
+
|
|
231
|
+
Returns an empty string if the file does not exist.
|
|
232
|
+
"""
|
|
233
|
+
if path.is_file():
|
|
234
|
+
with open(path, "rb") as file:
|
|
235
|
+
bytes = file.read()
|
|
236
|
+
readable_hash = hashlib.sha256(bytes).hexdigest()
|
|
237
|
+
return readable_hash
|
|
238
|
+
else:
|
|
239
|
+
return ""
|
|
240
|
+
|
|
241
|
+
def store_run_info(self, runnable: Runnable) -> None:
|
|
242
|
+
file_info = {
|
|
243
|
+
"inputs": {str(path): self.get_file_hash(path) for path in runnable.get_inputs()},
|
|
244
|
+
"outputs": {str(path): self.get_file_hash(path) for path in runnable.get_outputs()},
|
|
245
|
+
"config": runnable.get_config() or {},
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
run_info_path = self.get_runnable_run_info_file(runnable)
|
|
249
|
+
run_info_path.parent.mkdir(parents=True, exist_ok=True)
|
|
250
|
+
with run_info_path.open("w") as f:
|
|
251
|
+
# pretty print the json file
|
|
252
|
+
json.dump(file_info, f, indent=4)
|
|
253
|
+
|
|
254
|
+
def get_runnable_run_info_file(self, runnable: Runnable) -> Path:
|
|
255
|
+
return self.cache_dir / f"{runnable.get_name()}{self.RUN_INFO_FILE_EXTENSION}"
|
|
256
|
+
|
|
257
|
+
def previous_run_info_matches(self, runnable: Runnable) -> RunInfoStatus:
|
|
258
|
+
run_info_path = self.get_runnable_run_info_file(runnable)
|
|
259
|
+
if not run_info_path.exists():
|
|
260
|
+
return RunInfoStatus.NO_INFO
|
|
261
|
+
|
|
262
|
+
with run_info_path.open() as f:
|
|
263
|
+
previous_info = json.load(f)
|
|
264
|
+
|
|
265
|
+
# Check if config has changed
|
|
266
|
+
current_config = runnable.get_config() or {}
|
|
267
|
+
previous_config = previous_info.get("config", {})
|
|
268
|
+
if current_config != previous_config:
|
|
269
|
+
return RunInfoStatus.CONFIG_CHANGED
|
|
270
|
+
|
|
271
|
+
for file_type in ["inputs", "outputs"]:
|
|
272
|
+
for path_str, previous_hash in previous_info[file_type].items():
|
|
273
|
+
path = Path(path_str)
|
|
274
|
+
if self.get_file_hash(path) != previous_hash:
|
|
275
|
+
return RunInfoStatus.FILE_CHANGED
|
|
276
|
+
return RunInfoStatus.MATCH
|
|
277
|
+
|
|
278
|
+
def execute(self, runnable: Runnable) -> int:
|
|
279
|
+
run_info_status = self.previous_run_info_matches(runnable)
|
|
280
|
+
if run_info_status.should_run:
|
|
281
|
+
logger.info(f"Executing '{runnable.get_name()}': {run_info_status.message}")
|
|
282
|
+
exit_code = runnable.run()
|
|
283
|
+
self.store_run_info(runnable)
|
|
284
|
+
return exit_code
|
|
285
|
+
logger.info(f"Skipping '{runnable.get_name()}': {run_info_status.message}")
|
|
286
|
+
|
|
287
|
+
return 0
|
|
288
|
+
|
|
289
|
+
|
|
154
290
|
class UserNotificationException(Exception):
|
|
155
291
|
pass
|
|
156
292
|
|
|
@@ -158,7 +294,7 @@ class UserNotificationException(Exception):
|
|
|
158
294
|
class SubprocessExecutor:
|
|
159
295
|
def __init__(
|
|
160
296
|
self,
|
|
161
|
-
command:
|
|
297
|
+
command: Sequence[str | Path],
|
|
162
298
|
cwd: Optional[Path] = None,
|
|
163
299
|
capture_output: bool = True,
|
|
164
300
|
):
|
|
@@ -173,7 +309,12 @@ class SubprocessExecutor:
|
|
|
173
309
|
logger.info(f"Running command: {self.command} in {current_dir}")
|
|
174
310
|
# print all virtual environment variables
|
|
175
311
|
logger.debug(json.dumps(dict(os.environ), indent=4))
|
|
176
|
-
result = subprocess.run(
|
|
312
|
+
result = subprocess.run(
|
|
313
|
+
self.command.split(), # noqa: S603
|
|
314
|
+
cwd=current_dir,
|
|
315
|
+
capture_output=self.capture_output,
|
|
316
|
+
text=True, # to get stdout and stderr as strings instead of bytes
|
|
317
|
+
)
|
|
177
318
|
result.check_returncode()
|
|
178
319
|
except subprocess.CalledProcessError as e:
|
|
179
320
|
raise UserNotificationException(f"Command '{self.command}' failed with:\n{result.stdout if result else ''}\n{result.stderr if result else e}") from e
|
|
@@ -196,8 +337,9 @@ class VirtualEnvironment(ABC):
|
|
|
196
337
|
except PermissionError as e:
|
|
197
338
|
if "python.exe" in str(e):
|
|
198
339
|
raise UserNotificationException(
|
|
199
|
-
f"Failed to create virtual environment in {self.venv_dir}.\
|
|
200
|
-
f"
|
|
340
|
+
f"Failed to create virtual environment in {self.venv_dir}.\n"
|
|
341
|
+
f"Virtual environment python.exe is still running. "
|
|
342
|
+
f"Please kill all instances and run again.\nError: {e}"
|
|
201
343
|
) from e
|
|
202
344
|
raise UserNotificationException(f"Failed to create virtual environment in {self.venv_dir}.\nPlease make sure you have the necessary permissions.\nError: {e}") from e
|
|
203
345
|
|
|
@@ -211,8 +353,8 @@ class VirtualEnvironment(ABC):
|
|
|
211
353
|
"""
|
|
212
354
|
Configure pip to use the given index URL and SSL verification setting.
|
|
213
355
|
|
|
214
|
-
This method should behave as if the user had activated the virtual environment
|
|
215
|
-
|
|
356
|
+
This method should behave as if the user had activated the virtual environment and run
|
|
357
|
+
`pip config set global.index-url <index_url>` and
|
|
216
358
|
`pip config set global.cert <verify_ssl>` from the command line.
|
|
217
359
|
|
|
218
360
|
Args:
|
|
@@ -230,6 +372,10 @@ class VirtualEnvironment(ABC):
|
|
|
230
372
|
def pip(self, args: List[str]) -> None:
|
|
231
373
|
SubprocessExecutor([self.pip_path().as_posix(), *args]).execute()
|
|
232
374
|
|
|
375
|
+
@abstractmethod
|
|
376
|
+
def python_path(self) -> Path:
|
|
377
|
+
"""Get the path to the Python executable within the virtual environment."""
|
|
378
|
+
|
|
233
379
|
@abstractmethod
|
|
234
380
|
def pip_path(self) -> Path:
|
|
235
381
|
"""Get the path to the pip executable within the virtual environment."""
|
|
@@ -239,120 +385,341 @@ class VirtualEnvironment(ABC):
|
|
|
239
385
|
"""Get the path to the pip configuration file within the virtual environment."""
|
|
240
386
|
|
|
241
387
|
@abstractmethod
|
|
242
|
-
def
|
|
388
|
+
def scripts_path(self) -> Path:
|
|
389
|
+
"""Get the path to the Scripts (Windows) or bin (Unix) directory within the virtual environment."""
|
|
390
|
+
|
|
391
|
+
def run(self, args: List[str], capture_output: bool = True, cwd: Optional[Path] = None) -> None:
|
|
243
392
|
"""
|
|
244
|
-
Run an arbitrary command within the virtual environment.
|
|
393
|
+
Run an arbitrary command within the virtual environment using the venv's Python.
|
|
245
394
|
|
|
246
|
-
|
|
247
|
-
|
|
395
|
+
If the first argument is 'python', it will be replaced with the full path
|
|
396
|
+
to the virtual environment's Python executable.
|
|
397
|
+
|
|
398
|
+
Args:
|
|
399
|
+
----
|
|
400
|
+
args: Command-line arguments. For example, `run(['python', '-m', 'poetry', 'install'])`
|
|
401
|
+
capture_output: Whether to capture stdout/stderr.
|
|
402
|
+
cwd: Working directory for the command.
|
|
248
403
|
|
|
249
404
|
"""
|
|
405
|
+
command = list(args)
|
|
406
|
+
if command and command[0] == "python":
|
|
407
|
+
command[0] = self.python_path().as_posix()
|
|
408
|
+
SubprocessExecutor(command, cwd=cwd, capture_output=capture_output).execute()
|
|
250
409
|
|
|
251
410
|
|
|
252
411
|
class WindowsVirtualEnvironment(VirtualEnvironment):
|
|
253
412
|
def __init__(self, venv_dir: Path) -> None:
|
|
254
413
|
super().__init__(venv_dir)
|
|
255
|
-
|
|
414
|
+
|
|
415
|
+
def python_path(self) -> Path:
|
|
416
|
+
return self.scripts_path().joinpath("python.exe")
|
|
256
417
|
|
|
257
418
|
def pip_path(self) -> Path:
|
|
258
|
-
return self.
|
|
419
|
+
return self.scripts_path().joinpath("pip.exe")
|
|
259
420
|
|
|
260
421
|
def pip_config_path(self) -> Path:
|
|
261
422
|
return self.venv_dir.joinpath("pip.ini")
|
|
262
423
|
|
|
263
|
-
def
|
|
264
|
-
|
|
265
|
-
command=[f"cmd /c {self.activate_script.as_posix()} && ", *args],
|
|
266
|
-
capture_output=capture_output,
|
|
267
|
-
).execute()
|
|
424
|
+
def scripts_path(self) -> Path:
|
|
425
|
+
return self.venv_dir.joinpath("Scripts")
|
|
268
426
|
|
|
269
427
|
|
|
270
428
|
class UnixVirtualEnvironment(VirtualEnvironment):
|
|
271
429
|
def __init__(self, venv_dir: Path) -> None:
|
|
272
430
|
super().__init__(venv_dir)
|
|
273
|
-
|
|
431
|
+
|
|
432
|
+
def python_path(self) -> Path:
|
|
433
|
+
return self.scripts_path().joinpath("python")
|
|
274
434
|
|
|
275
435
|
def pip_path(self) -> Path:
|
|
276
|
-
return self.
|
|
436
|
+
return self.scripts_path().joinpath("pip")
|
|
277
437
|
|
|
278
438
|
def pip_config_path(self) -> Path:
|
|
279
439
|
return self.venv_dir.joinpath("pip.conf")
|
|
280
440
|
|
|
281
|
-
def
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
441
|
+
def scripts_path(self) -> Path:
|
|
442
|
+
return self.venv_dir.joinpath("bin")
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def instantiate_os_specific_venv(venv_dir: Path) -> VirtualEnvironment:
|
|
446
|
+
"""Create an OS-specific VirtualEnvironment instance."""
|
|
447
|
+
if sys.platform.startswith("win32"):
|
|
448
|
+
return WindowsVirtualEnvironment(venv_dir)
|
|
449
|
+
elif sys.platform.startswith("linux") or sys.platform.startswith("darwin"):
|
|
450
|
+
return UnixVirtualEnvironment(venv_dir)
|
|
451
|
+
else:
|
|
452
|
+
raise UserNotificationException(f"Unsupported operating system: {sys.platform}")
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def extract_package_manager_name(package_manager_spec: str) -> str:
|
|
456
|
+
"""Extract the package manager name from a specification like 'poetry>=1.7.1'."""
|
|
457
|
+
match = re.match(r"^([a-zA-Z0-9_-]+)", package_manager_spec)
|
|
458
|
+
if match:
|
|
459
|
+
return match.group(1)
|
|
460
|
+
raise UserNotificationException(f"Could not extract the package manager name from {package_manager_spec}")
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
class CreateBootstrapEnvironment(Runnable):
|
|
464
|
+
"""
|
|
465
|
+
Creates a shared bootstrap environment with the package manager installed.
|
|
466
|
+
|
|
467
|
+
The bootstrap environment is stored in a user-level cache directory
|
|
468
|
+
(default: ~/.bootstrap/<hash>/) and is shared across projects with
|
|
469
|
+
the same configuration.
|
|
470
|
+
"""
|
|
471
|
+
|
|
472
|
+
def __init__(self, config: BootstrapConfig, project_dir: Path) -> None:
|
|
473
|
+
self.config = config
|
|
474
|
+
self.project_dir = project_dir
|
|
475
|
+
self.env_hash = config.compute_bootstrap_env_hash()
|
|
476
|
+
self.bootstrap_env_dir = config.get_bootstrap_cache_dir() / self.env_hash
|
|
477
|
+
self.venv_dir = self.bootstrap_env_dir / ".venv"
|
|
478
|
+
self.virtual_env = instantiate_os_specific_venv(self.venv_dir)
|
|
479
|
+
self.marker_file = self.bootstrap_env_dir / BOOTSTRAP_COMPLETE_MARKER
|
|
480
|
+
|
|
481
|
+
def run(self) -> int:
|
|
482
|
+
self._create_environment_atomic()
|
|
483
|
+
return 0
|
|
484
|
+
|
|
485
|
+
def _is_valid_environment(self) -> bool:
|
|
486
|
+
"""Check if the bootstrap environment exists and is valid."""
|
|
487
|
+
if not self.marker_file.exists():
|
|
488
|
+
return False
|
|
489
|
+
|
|
490
|
+
try:
|
|
491
|
+
stored_hash = self.marker_file.read_text().strip()
|
|
492
|
+
if stored_hash != self.env_hash:
|
|
493
|
+
logger.info(f"Bootstrap environment hash mismatch: {stored_hash} != {self.env_hash}")
|
|
494
|
+
return False
|
|
495
|
+
except OSError:
|
|
496
|
+
return False
|
|
497
|
+
|
|
498
|
+
if not self.virtual_env.pip_path().exists():
|
|
499
|
+
logger.info("Bootstrap environment pip not found, will recreate.")
|
|
500
|
+
return False
|
|
501
|
+
|
|
502
|
+
return True
|
|
503
|
+
|
|
504
|
+
def _create_environment_atomic(self) -> None:
|
|
505
|
+
"""Create the bootstrap environment, replacing any existing invalid environment."""
|
|
506
|
+
try:
|
|
507
|
+
# Remove existing directory if present (invalid or leftover from failed attempt)
|
|
508
|
+
if self.bootstrap_env_dir.exists():
|
|
509
|
+
logger.info(f"Removing existing bootstrap environment at {self.bootstrap_env_dir}")
|
|
510
|
+
shutil.rmtree(self.bootstrap_env_dir)
|
|
511
|
+
|
|
512
|
+
# Create bootstrap environment directory
|
|
513
|
+
self.bootstrap_env_dir.mkdir(parents=True, exist_ok=True)
|
|
514
|
+
bootstrap_venv = instantiate_os_specific_venv(self.venv_dir)
|
|
515
|
+
|
|
516
|
+
logger.info(f"Creating bootstrap environment in {self.bootstrap_env_dir}")
|
|
517
|
+
venv.create(env_dir=self.venv_dir, with_pip=True)
|
|
518
|
+
|
|
519
|
+
# Configure pip with PyPI source if available
|
|
520
|
+
pypi_source = PyPiSourceParser.from_pyproject(self.project_dir)
|
|
521
|
+
if pypi_source:
|
|
522
|
+
bootstrap_venv.pip_configure(index_url=pypi_source.url, verify_ssl=True)
|
|
523
|
+
|
|
524
|
+
# Build pip install arguments
|
|
525
|
+
packages_to_install = [self.config.package_manager, *self.config.bootstrap_packages]
|
|
526
|
+
pip_args = ["install", *packages_to_install]
|
|
527
|
+
|
|
528
|
+
# Handle SSL certificates for older pip versions
|
|
529
|
+
if Version(ensurepip.version()) < Version("24.2"):
|
|
530
|
+
if pypi_source and (hostname := urlparse(pypi_source.url).hostname):
|
|
531
|
+
pip_args.extend(["--trusted-host", hostname])
|
|
532
|
+
else:
|
|
533
|
+
pip_args.extend(
|
|
534
|
+
[
|
|
535
|
+
"--trusted-host",
|
|
536
|
+
"pypi.org",
|
|
537
|
+
"--trusted-host",
|
|
538
|
+
"pypi.python.org",
|
|
539
|
+
"--trusted-host",
|
|
540
|
+
"files.pythonhosted.org",
|
|
541
|
+
]
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
logger.info(f"Installing bootstrap packages: {packages_to_install}")
|
|
545
|
+
bootstrap_venv.pip(pip_args)
|
|
546
|
+
|
|
547
|
+
# Write the completion marker
|
|
548
|
+
marker_path = self.bootstrap_env_dir / BOOTSTRAP_COMPLETE_MARKER
|
|
549
|
+
marker_path.write_text(self.env_hash)
|
|
550
|
+
|
|
551
|
+
# Update the virtual_env reference
|
|
552
|
+
self.virtual_env = instantiate_os_specific_venv(self.venv_dir)
|
|
553
|
+
|
|
554
|
+
logger.info(f"Bootstrap environment created successfully at {self.bootstrap_env_dir}")
|
|
555
|
+
|
|
556
|
+
except Exception as exc:
|
|
557
|
+
logger.error(f"Bootstrap environment creation failed at {self.bootstrap_env_dir}")
|
|
558
|
+
raise UserNotificationException(f"Failed to create bootstrap environment: {exc}") from exc
|
|
559
|
+
|
|
560
|
+
def get_name(self) -> str:
|
|
561
|
+
return "create-bootstrap-environment"
|
|
562
|
+
|
|
563
|
+
def get_inputs(self) -> List[Path]:
|
|
564
|
+
# No file-based inputs for shared bootstrap environment
|
|
565
|
+
return []
|
|
566
|
+
|
|
567
|
+
def get_outputs(self) -> List[Path]:
|
|
568
|
+
return [self.marker_file]
|
|
569
|
+
|
|
570
|
+
def get_config(self) -> Optional[dict[str, Any]]:
|
|
571
|
+
"""Return configuration that affects the bootstrap environment."""
|
|
572
|
+
return {
|
|
573
|
+
"package_manager": self.config.package_manager,
|
|
574
|
+
"bootstrap_packages": sorted(self.config.bootstrap_packages),
|
|
575
|
+
"python_version": self.config.python_version,
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
class CreateVirtualEnvironment(Runnable):
|
|
580
|
+
"""Creates the project virtual environment using the bootstrap environment's package manager."""
|
|
581
|
+
|
|
582
|
+
def __init__(
|
|
583
|
+
self,
|
|
584
|
+
root_dir: Path,
|
|
585
|
+
bootstrap_env: CreateBootstrapEnvironment,
|
|
586
|
+
) -> None:
|
|
302
587
|
self.root_dir = root_dir
|
|
303
588
|
self.venv_dir = self.root_dir / ".venv"
|
|
304
|
-
self.
|
|
305
|
-
self.
|
|
306
|
-
self.
|
|
307
|
-
self.
|
|
589
|
+
self.bootstrap_dir = self.root_dir / ".bootstrap"
|
|
590
|
+
self.virtual_env = instantiate_os_specific_venv(self.venv_dir)
|
|
591
|
+
self.bootstrap_env = bootstrap_env
|
|
592
|
+
self.config = bootstrap_env.config
|
|
593
|
+
self.python_version_marker = self.venv_dir / VENV_PYTHON_VERSION_MARKER
|
|
308
594
|
|
|
309
595
|
@property
|
|
310
596
|
def package_manager_name(self) -> str:
|
|
311
|
-
|
|
312
|
-
if match:
|
|
313
|
-
return match.group(1)
|
|
314
|
-
else:
|
|
315
|
-
raise UserNotificationException(f"Could not extract the package manager name from {self.package_manager}")
|
|
597
|
+
return extract_package_manager_name(self.config.package_manager)
|
|
316
598
|
|
|
317
|
-
def
|
|
318
|
-
"""
|
|
599
|
+
def _check_python_version_compatibility(self) -> None:
|
|
600
|
+
"""
|
|
601
|
+
Check if the existing venv was created with the same Python version.
|
|
602
|
+
|
|
603
|
+
If the Python version has changed (e.g., switching branches), delete the
|
|
604
|
+
existing venv so it can be recreated by the package manager.
|
|
605
|
+
"""
|
|
606
|
+
if not self.venv_dir.exists():
|
|
607
|
+
return
|
|
608
|
+
|
|
609
|
+
current_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
|
|
610
|
+
|
|
611
|
+
if not self.python_version_marker.exists():
|
|
612
|
+
logger.info(
|
|
613
|
+
f"No Python version marker found in {self.venv_dir}. "
|
|
614
|
+
f"This venv may have been created before version tracking was added. "
|
|
615
|
+
f"Deleting {self.venv_dir} to ensure clean state."
|
|
616
|
+
)
|
|
617
|
+
shutil.rmtree(self.venv_dir)
|
|
618
|
+
return
|
|
619
|
+
|
|
620
|
+
try:
|
|
621
|
+
stored_version = self.python_version_marker.read_text().strip()
|
|
622
|
+
if stored_version != current_version:
|
|
623
|
+
logger.info(f"Python version changed from {stored_version} to {current_version}. Deleting {self.venv_dir} for recreation.")
|
|
624
|
+
shutil.rmtree(self.venv_dir)
|
|
625
|
+
except OSError as exc:
|
|
626
|
+
logger.warning(f"Could not read Python version marker: {exc}")
|
|
627
|
+
|
|
628
|
+
def _write_python_version_marker(self, version: str) -> None:
|
|
629
|
+
"""Write the Python version marker to track the venv's Python version."""
|
|
630
|
+
try:
|
|
631
|
+
self.python_version_marker.write_text(version)
|
|
632
|
+
except OSError as exc:
|
|
633
|
+
logger.warning(f"Could not write Python version marker: {exc}")
|
|
634
|
+
|
|
635
|
+
def _ensure_in_project_venv(self) -> None:
|
|
636
|
+
"""Configure package managers to create venv in-project (.venv in repository)."""
|
|
637
|
+
if self.package_manager_name == "poetry":
|
|
638
|
+
# Set environment variable for poetry to create venv in-project
|
|
639
|
+
os.environ["POETRY_VIRTUALENVS_IN_PROJECT"] = "true"
|
|
640
|
+
elif self.package_manager_name == "pipenv":
|
|
641
|
+
# Set environment variable for pipenv
|
|
642
|
+
os.environ["PIPENV_VENV_IN_PROJECT"] = "1"
|
|
643
|
+
# UV creates .venv in-project by default, no configuration needed
|
|
644
|
+
|
|
645
|
+
def _ensure_correct_python_version(self) -> None:
|
|
646
|
+
"""Ensure the correct Python version is used in the virtual environment."""
|
|
647
|
+
if self.package_manager_name == "poetry":
|
|
648
|
+
# Make Poetry use the Python interpreter it's being run with
|
|
649
|
+
os.environ["POETRY_VIRTUALENVS_PREFER_ACTIVE_PYTHON"] = "false"
|
|
650
|
+
os.environ["POETRY_VIRTUALENVS_USE_POETRY_PYTHON"] = "true"
|
|
651
|
+
|
|
652
|
+
def _get_install_argument(self) -> str:
|
|
319
653
|
if self.package_manager_name == "uv":
|
|
320
654
|
return "sync"
|
|
321
655
|
return "install"
|
|
322
656
|
|
|
657
|
+
def _get_install_command(self) -> List[str]:
|
|
658
|
+
if self.config.venv_install_command:
|
|
659
|
+
return self.config.venv_install_command.split()
|
|
660
|
+
|
|
661
|
+
return [
|
|
662
|
+
str(self.bootstrap_env.virtual_env.scripts_path() / self.package_manager_name),
|
|
663
|
+
self._get_install_argument(),
|
|
664
|
+
*self.config.package_manager_args,
|
|
665
|
+
]
|
|
666
|
+
|
|
323
667
|
def run(self) -> int:
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
logger.info("Skipping virtual environment creation as requested.")
|
|
668
|
+
self._check_python_version_compatibility()
|
|
669
|
+
self._ensure_in_project_venv()
|
|
670
|
+
self._ensure_correct_python_version()
|
|
328
671
|
|
|
329
672
|
# Get the PyPi source from pyproject.toml or Pipfile if it is defined
|
|
330
673
|
pypi_source = PyPiSourceParser.from_pyproject(self.root_dir)
|
|
331
|
-
|
|
674
|
+
|
|
675
|
+
# Use the bootstrap environment's package manager to install dependencies
|
|
676
|
+
# The package manager will create the .venv if it doesn't exist
|
|
677
|
+
logger.info(f"Using bootstrap environment at {self.bootstrap_env.venv_dir}")
|
|
678
|
+
self.bootstrap_env.virtual_env.run(self._get_install_command(), capture_output=True, cwd=self.root_dir)
|
|
679
|
+
|
|
680
|
+
# Write Python version marker after package manager creates/updates venv
|
|
681
|
+
if self.venv_dir.exists():
|
|
682
|
+
current_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
|
|
683
|
+
self._write_python_version_marker(current_version)
|
|
684
|
+
|
|
685
|
+
# Configure pip if needed (after venv is created by package manager)
|
|
686
|
+
if pypi_source and self.venv_dir.exists():
|
|
332
687
|
self.virtual_env.pip_configure(index_url=pypi_source.url, verify_ssl=True)
|
|
333
|
-
|
|
334
|
-
pip_args = ["install", self.package_manager, "pip-system-certs>=4.0,<5.0"]
|
|
335
|
-
# but to install it, we need either a pip version with the trust store feature or to trust the host
|
|
336
|
-
# (trust store feature enabled by default since 24.2)
|
|
337
|
-
if Version(ensurepip.version()) < Version("24.2"):
|
|
338
|
-
# Add trusted host of configured source for older Python versions
|
|
339
|
-
if pypi_source and pypi_source.url:
|
|
340
|
-
if hostname := urlparse(pypi_source.url).hostname:
|
|
341
|
-
pip_args.extend(["--trusted-host", hostname])
|
|
342
|
-
else:
|
|
343
|
-
pip_args.extend(["--trusted-host", "pypi.org", "--trusted-host", "pypi.python.org", "--trusted-host", "files.pythonhosted.org"])
|
|
344
|
-
self.virtual_env.pip(pip_args)
|
|
345
|
-
self.virtual_env.run(["python", "-m", self.package_manager_name, self.get_install_argument()])
|
|
688
|
+
|
|
346
689
|
return 0
|
|
347
690
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
691
|
+
def get_name(self) -> str:
|
|
692
|
+
return "create-virtual-environment"
|
|
693
|
+
|
|
694
|
+
def get_inputs(self) -> List[Path]:
|
|
695
|
+
venv_relevant_files = [
|
|
696
|
+
"uv.lock",
|
|
697
|
+
"poetry.lock",
|
|
698
|
+
"poetry.toml",
|
|
699
|
+
"pyproject.toml",
|
|
700
|
+
".env",
|
|
701
|
+
"Pipfile",
|
|
702
|
+
"Pipfile.lock",
|
|
703
|
+
"bootstrap.json",
|
|
704
|
+
".bootstrap/bootstrap.ps1",
|
|
705
|
+
".bootstrap/bootstrap.py",
|
|
706
|
+
"bootstrap.ps1",
|
|
707
|
+
"bootstrap.py",
|
|
708
|
+
str(self.bootstrap_env.marker_file),
|
|
709
|
+
]
|
|
710
|
+
return [self.root_dir / file for file in venv_relevant_files]
|
|
711
|
+
|
|
712
|
+
def get_outputs(self) -> List[Path]:
|
|
713
|
+
"""
|
|
714
|
+
Return the Scripts/bin directories for both bootstrap and project environments.
|
|
715
|
+
|
|
716
|
+
These paths are recorded in the .deps.json file, allowing other tools to discover
|
|
717
|
+
the package manager location (bootstrap env) and project tools (project env).
|
|
718
|
+
"""
|
|
719
|
+
return [
|
|
720
|
+
self.virtual_env.scripts_path(),
|
|
721
|
+
self.bootstrap_env.virtual_env.scripts_path(),
|
|
722
|
+
]
|
|
356
723
|
|
|
357
724
|
|
|
358
725
|
def print_environment_info() -> None:
|
|
@@ -388,11 +755,42 @@ def main() -> int:
|
|
|
388
755
|
default=False,
|
|
389
756
|
help="Skip the virtual environment creation process.",
|
|
390
757
|
)
|
|
758
|
+
parser.add_argument(
|
|
759
|
+
"--config",
|
|
760
|
+
type=Path,
|
|
761
|
+
required=False,
|
|
762
|
+
help="Path to bootstrap.json configuration file.",
|
|
763
|
+
)
|
|
391
764
|
args = parser.parse_args()
|
|
392
765
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
766
|
+
project_dir = args.project_dir
|
|
767
|
+
|
|
768
|
+
# Load configuration from bootstrap.json if provided, otherwise use CLI args
|
|
769
|
+
if args.config:
|
|
770
|
+
config_path = args.config if args.config.is_absolute() else project_dir / args.config
|
|
771
|
+
config = BootstrapConfig.from_json_file(config_path)
|
|
772
|
+
else:
|
|
773
|
+
# Use CLI arguments for backward compatibility
|
|
774
|
+
config = BootstrapConfig(
|
|
775
|
+
package_manager=args.package_manager,
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
# Step 1: Create the bootstrap environment (shared cache)
|
|
779
|
+
bootstrap_env = CreateBootstrapEnvironment(config, project_dir)
|
|
780
|
+
bootstrap_executor = Executor(bootstrap_env.bootstrap_env_dir)
|
|
781
|
+
bootstrap_executor.execute(bootstrap_env)
|
|
782
|
+
|
|
783
|
+
# Step 2: Create the project virtual environment using the bootstrap env
|
|
784
|
+
# Skip if requested (e.g., when running from within the venv)
|
|
785
|
+
if not args.skip_venv_creation:
|
|
786
|
+
project_venv = CreateVirtualEnvironment(project_dir, bootstrap_env)
|
|
787
|
+
project_executor = Executor(project_venv.venv_dir)
|
|
788
|
+
project_executor.execute(project_venv)
|
|
789
|
+
else:
|
|
790
|
+
logger.info("Skipping virtual environment creation as requested.")
|
|
791
|
+
|
|
792
|
+
except UserNotificationException as exc:
|
|
793
|
+
logger.error(exc)
|
|
396
794
|
return 1
|
|
397
795
|
return 0
|
|
398
796
|
|
pypeline/main.py
CHANGED
|
@@ -41,7 +41,7 @@ def init(
|
|
|
41
41
|
project_dir: Path = typer.Option(Path.cwd().absolute(), help="The project directory"), # noqa: B008
|
|
42
42
|
force: bool = typer.Option(False, help="Force the initialization of the project even if the directory is not empty."),
|
|
43
43
|
) -> None:
|
|
44
|
-
KickstartProject(project_dir, force).run()
|
|
44
|
+
KickstartProject(project_dir.absolute(), force).run()
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
@app.command()
|
|
@@ -61,6 +61,7 @@ def run(
|
|
|
61
61
|
help="Provide input parameters as key=value pairs (e.g., -i name=value -i flag=true).",
|
|
62
62
|
),
|
|
63
63
|
) -> None:
|
|
64
|
+
project_dir = project_dir.absolute()
|
|
64
65
|
project_slurper = ProjectSlurper(project_dir, config_file)
|
|
65
66
|
if print:
|
|
66
67
|
logger.info("Pipeline steps:")
|
pypeline/steps/create_venv.py
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import json
|
|
1
3
|
import re
|
|
4
|
+
import shutil
|
|
2
5
|
import sys
|
|
6
|
+
import traceback
|
|
3
7
|
from dataclasses import dataclass
|
|
4
8
|
from enum import Enum, auto
|
|
5
9
|
from pathlib import Path
|
|
6
10
|
from typing import Any, ClassVar, Dict, List, Optional
|
|
7
11
|
|
|
8
12
|
from mashumaro import DataClassDictMixin
|
|
13
|
+
from mashumaro.mixins.json import DataClassJSONMixin
|
|
9
14
|
from py_app_dev.core.exceptions import UserNotificationException
|
|
10
15
|
from py_app_dev.core.logging import logger
|
|
11
16
|
|
|
@@ -20,7 +25,13 @@ from ..domain.pipeline import PipelineStep
|
|
|
20
25
|
class CreateVEnvConfig(DataClassDictMixin):
|
|
21
26
|
bootstrap_script: Optional[str] = None
|
|
22
27
|
python_executable: Optional[str] = None
|
|
28
|
+
# Bootstrap-specific configuration
|
|
23
29
|
package_manager: Optional[str] = None
|
|
30
|
+
python_version: Optional[str] = None
|
|
31
|
+
package_manager_args: Optional[List[str]] = None
|
|
32
|
+
bootstrap_packages: Optional[List[str]] = None
|
|
33
|
+
bootstrap_cache_dir: Optional[str] = None
|
|
34
|
+
venv_install_command: Optional[str] = None
|
|
24
35
|
|
|
25
36
|
|
|
26
37
|
class BootstrapScriptType(Enum):
|
|
@@ -28,6 +39,21 @@ class BootstrapScriptType(Enum):
|
|
|
28
39
|
INTERNAL = auto()
|
|
29
40
|
|
|
30
41
|
|
|
42
|
+
@dataclass
|
|
43
|
+
class CreateVEnvDeps(DataClassJSONMixin):
|
|
44
|
+
outputs: List[Path]
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def from_json_file(cls, file_path: Path) -> "CreateVEnvDeps":
|
|
48
|
+
try:
|
|
49
|
+
result = cls.from_dict(json.loads(file_path.read_text()))
|
|
50
|
+
except Exception as e:
|
|
51
|
+
output = io.StringIO()
|
|
52
|
+
traceback.print_exc(file=output)
|
|
53
|
+
raise UserNotificationException(output.getvalue()) from e
|
|
54
|
+
return result
|
|
55
|
+
|
|
56
|
+
|
|
31
57
|
class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
32
58
|
DEFAULT_PACKAGE_MANAGER = "uv>=0.6"
|
|
33
59
|
DEFAULT_PYTHON_EXECUTABLE = "python311"
|
|
@@ -44,11 +70,107 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
|
44
70
|
self.logger = logger.bind()
|
|
45
71
|
self.internal_bootstrap_script = get_bootstrap_script()
|
|
46
72
|
self.package_manager = self.user_config.package_manager if self.user_config.package_manager else self.DEFAULT_PACKAGE_MANAGER
|
|
47
|
-
self.python_executable = self.user_config.python_executable if self.user_config.python_executable else self.DEFAULT_PYTHON_EXECUTABLE
|
|
48
73
|
self.venv_dir = self.project_root_dir / ".venv"
|
|
49
74
|
|
|
75
|
+
@property
|
|
76
|
+
def has_bootstrap_config(self) -> bool:
|
|
77
|
+
"""Check if user provided any bootstrap-specific configuration."""
|
|
78
|
+
return any(
|
|
79
|
+
[
|
|
80
|
+
self.user_config.package_manager,
|
|
81
|
+
self.user_config.python_version,
|
|
82
|
+
self.user_config.package_manager_args,
|
|
83
|
+
self.user_config.bootstrap_packages,
|
|
84
|
+
self.user_config.bootstrap_cache_dir,
|
|
85
|
+
self.user_config.venv_install_command,
|
|
86
|
+
]
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def _find_python_executable(self, python_version: str) -> Optional[str]:
|
|
90
|
+
"""
|
|
91
|
+
Find Python executable based on version string.
|
|
92
|
+
|
|
93
|
+
Supports version formats:
|
|
94
|
+
- "3.11.5" or "3.11" -> tries python3.11, python311
|
|
95
|
+
- "3" -> tries python3
|
|
96
|
+
|
|
97
|
+
Always ignores patch version. No fallbacks to generic python.
|
|
98
|
+
|
|
99
|
+
Returns the first executable found in PATH, or None if not found.
|
|
100
|
+
"""
|
|
101
|
+
# Handle empty string
|
|
102
|
+
if not python_version:
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
# Parse version string and extract components
|
|
106
|
+
version_parts = python_version.split(".")
|
|
107
|
+
|
|
108
|
+
if len(version_parts) == 0:
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
major = version_parts[0]
|
|
112
|
+
|
|
113
|
+
# Determine candidates based on version format
|
|
114
|
+
candidates = []
|
|
115
|
+
|
|
116
|
+
if len(version_parts) >= 2:
|
|
117
|
+
# Has minor version (e.g., "3.11" or "3.11.5") - ignore patch
|
|
118
|
+
minor = version_parts[1]
|
|
119
|
+
major_minor = f"{major}.{minor}"
|
|
120
|
+
major_minor_no_dot = f"{major}{minor}"
|
|
121
|
+
|
|
122
|
+
candidates = [
|
|
123
|
+
f"python{major_minor}", # python3.11 (Linux/Mac preference)
|
|
124
|
+
f"python{major_minor_no_dot}", # python311 (Windows preference)
|
|
125
|
+
]
|
|
126
|
+
else:
|
|
127
|
+
# Only major version (e.g., "3")
|
|
128
|
+
candidates = [f"python{major}"]
|
|
129
|
+
|
|
130
|
+
# Try to find each candidate in PATH
|
|
131
|
+
for candidate in candidates:
|
|
132
|
+
executable_path = shutil.which(candidate)
|
|
133
|
+
if executable_path:
|
|
134
|
+
self.logger.debug(f"Found Python executable: {executable_path} (candidate: {candidate})")
|
|
135
|
+
return candidate
|
|
136
|
+
|
|
137
|
+
# No fallback - return None if specific version not found
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def python_executable(self) -> str:
|
|
142
|
+
"""
|
|
143
|
+
Get python executable to use.
|
|
144
|
+
|
|
145
|
+
Priority:
|
|
146
|
+
1. User-specified python_executable config
|
|
147
|
+
2. Auto-detect from python_version config
|
|
148
|
+
3. Current Python interpreter (sys.executable)
|
|
149
|
+
"""
|
|
150
|
+
# Priority 1: User explicitly specified executable
|
|
151
|
+
if self.user_config.python_executable:
|
|
152
|
+
return self.user_config.python_executable
|
|
153
|
+
|
|
154
|
+
# Priority 2: Auto-detect from python_version
|
|
155
|
+
if self.user_config.python_version:
|
|
156
|
+
found_executable = self._find_python_executable(self.user_config.python_version)
|
|
157
|
+
if found_executable:
|
|
158
|
+
return found_executable
|
|
159
|
+
# If version specified but not found, fail with helpful error
|
|
160
|
+
raise UserNotificationException(
|
|
161
|
+
f"Could not find Python {self.user_config.python_version} in PATH. Please install Python {self.user_config.python_version} or specify python_executable explicitly."
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Priority 3: Use current interpreter
|
|
165
|
+
return sys.executable
|
|
166
|
+
|
|
50
167
|
@property
|
|
51
168
|
def install_dirs(self) -> List[Path]:
|
|
169
|
+
deps_file = self.project_root_dir / ".venv" / "create-virtual-environment.deps.json"
|
|
170
|
+
if deps_file.exists():
|
|
171
|
+
deps = CreateVEnvDeps.from_json_file(deps_file)
|
|
172
|
+
if deps.outputs:
|
|
173
|
+
return deps.outputs
|
|
52
174
|
return [self.project_root_dir / dir for dir in [".venv/Scripts", ".venv/bin"] if (self.project_root_dir / dir).exists()]
|
|
53
175
|
|
|
54
176
|
@property
|
|
@@ -67,6 +189,10 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
|
67
189
|
def target_internal_bootstrap_script(self) -> Path:
|
|
68
190
|
return self.project_root_dir.joinpath(".bootstrap/bootstrap.py")
|
|
69
191
|
|
|
192
|
+
@property
|
|
193
|
+
def bootstrap_config_file(self) -> Path:
|
|
194
|
+
return self.project_root_dir / "bootstrap.json"
|
|
195
|
+
|
|
70
196
|
def get_name(self) -> str:
|
|
71
197
|
return self.__class__.__name__
|
|
72
198
|
|
|
@@ -74,6 +200,7 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
|
74
200
|
self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
|
|
75
201
|
|
|
76
202
|
if self.user_config.bootstrap_script:
|
|
203
|
+
# User provided a custom bootstrap script - run it directly
|
|
77
204
|
bootstrap_script = self.project_root_dir / self.user_config.bootstrap_script
|
|
78
205
|
if not bootstrap_script.exists():
|
|
79
206
|
raise UserNotificationException(f"Bootstrap script {bootstrap_script} does not exist.")
|
|
@@ -82,19 +209,43 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
|
82
209
|
cwd=self.project_root_dir,
|
|
83
210
|
).execute()
|
|
84
211
|
else:
|
|
212
|
+
# Use internal bootstrap script
|
|
85
213
|
skip_venv_creation = False
|
|
86
214
|
python_executable = Path(sys.executable).absolute()
|
|
87
215
|
if python_executable.is_relative_to(self.project_root_dir):
|
|
88
216
|
self.logger.info(f"Detected that the python executable '{python_executable}' is from the virtual environment. Skip updating the virtual environment.")
|
|
89
217
|
skip_venv_creation = True
|
|
90
218
|
|
|
91
|
-
#
|
|
219
|
+
# Create bootstrap.json with all configuration
|
|
220
|
+
bootstrap_config = {}
|
|
221
|
+
if self.user_config.package_manager:
|
|
222
|
+
bootstrap_config["python_package_manager"] = self.user_config.package_manager
|
|
223
|
+
if self.user_config.python_version:
|
|
224
|
+
bootstrap_config["python_version"] = self.user_config.python_version
|
|
225
|
+
if self.user_config.package_manager_args:
|
|
226
|
+
bootstrap_config["python_package_manager_args"] = self.user_config.package_manager_args
|
|
227
|
+
if self.user_config.bootstrap_packages:
|
|
228
|
+
bootstrap_config["bootstrap_packages"] = self.user_config.bootstrap_packages
|
|
229
|
+
if self.user_config.bootstrap_cache_dir:
|
|
230
|
+
bootstrap_config["bootstrap_cache_dir"] = self.user_config.bootstrap_cache_dir
|
|
231
|
+
if self.user_config.venv_install_command:
|
|
232
|
+
bootstrap_config["venv_install_command"] = self.user_config.venv_install_command
|
|
233
|
+
|
|
234
|
+
# Write bootstrap.json if any configuration is provided
|
|
235
|
+
if bootstrap_config:
|
|
236
|
+
self.bootstrap_config_file.write_text(json.dumps(bootstrap_config, indent=2))
|
|
237
|
+
self.logger.info(f"Created bootstrap configuration at {self.bootstrap_config_file}")
|
|
238
|
+
|
|
239
|
+
# Build bootstrap script arguments
|
|
92
240
|
bootstrap_args = [
|
|
93
241
|
"--project-dir",
|
|
94
242
|
self.project_root_dir.as_posix(),
|
|
95
|
-
"--package-manager",
|
|
96
|
-
f'"{self.package_manager}"',
|
|
97
243
|
]
|
|
244
|
+
|
|
245
|
+
# Always use --config if bootstrap.json exists
|
|
246
|
+
if self.bootstrap_config_file.exists():
|
|
247
|
+
bootstrap_args.extend(["--config", self.bootstrap_config_file.as_posix()])
|
|
248
|
+
|
|
98
249
|
if skip_venv_creation:
|
|
99
250
|
bootstrap_args.append("--skip-venv-creation")
|
|
100
251
|
|
|
@@ -114,12 +265,19 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
|
114
265
|
|
|
115
266
|
def get_inputs(self) -> List[Path]:
|
|
116
267
|
package_manager_relevant_file = self.SUPPORTED_PACKAGE_MANAGERS.get(self.package_manager_name, [])
|
|
117
|
-
|
|
268
|
+
inputs = [self.project_root_dir / file for file in package_manager_relevant_file]
|
|
269
|
+
# Include bootstrap.json if it exists
|
|
270
|
+
if self.bootstrap_config_file.exists():
|
|
271
|
+
inputs.append(self.bootstrap_config_file)
|
|
272
|
+
return inputs
|
|
118
273
|
|
|
119
274
|
def get_outputs(self) -> List[Path]:
|
|
120
275
|
outputs = [self.venv_dir]
|
|
121
276
|
if self.bootstrap_script_type == BootstrapScriptType.INTERNAL:
|
|
122
277
|
outputs.append(self.target_internal_bootstrap_script)
|
|
278
|
+
# Include bootstrap.json if it will be created
|
|
279
|
+
if self.has_bootstrap_config:
|
|
280
|
+
outputs.append(self.bootstrap_config_file)
|
|
123
281
|
return outputs
|
|
124
282
|
|
|
125
283
|
def get_config(self) -> Optional[dict[str, str]]:
|
|
@@ -133,4 +291,6 @@ class CreateVEnv(PipelineStep[ExecutionContext]):
|
|
|
133
291
|
self.execution_context.add_install_dirs(self.install_dirs)
|
|
134
292
|
|
|
135
293
|
def get_needs_dependency_management(self) -> bool:
|
|
136
|
-
return False
|
|
294
|
+
# Always return False - the bootstrap script handles dependency management internally
|
|
295
|
+
# via its Executor framework which checks input/output hashes and configuration changes
|
|
296
|
+
return False
|
pypeline/steps/scoop_install.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import json
|
|
3
|
+
import platform
|
|
3
4
|
import traceback
|
|
4
5
|
from dataclasses import dataclass, field
|
|
5
6
|
from pathlib import Path
|
|
@@ -69,6 +70,12 @@ class ScoopInstall(PipelineStep[ExecutionContext]):
|
|
|
69
70
|
|
|
70
71
|
def run(self) -> int:
|
|
71
72
|
self.logger.debug(f"Run {self.get_name()} step. Output dir: {self.output_dir}")
|
|
73
|
+
|
|
74
|
+
if platform.system() != "Windows":
|
|
75
|
+
self.logger.warning(f"ScoopInstall skipped on non-Windows platform ({platform.system()}).")
|
|
76
|
+
self.execution_info.to_json_file(self.execution_info_file)
|
|
77
|
+
return 0
|
|
78
|
+
|
|
72
79
|
installed_apps = create_scoop_wrapper().install(self.scoop_file)
|
|
73
80
|
self.logger.debug("Installed apps:")
|
|
74
81
|
for app in installed_apps:
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pypeline-runner
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.21.0
|
|
4
4
|
Summary: Configure and execute pipelines with Python (similar to GitHub workflows or Jenkins pipelines).
|
|
5
5
|
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
6
7
|
Author: cuinixam
|
|
7
8
|
Author-email: me@cuinixam.com
|
|
8
9
|
Requires-Python: >=3.10,<4.0
|
|
@@ -16,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
16
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
21
|
Classifier: Topic :: Software Development :: Libraries
|
|
20
22
|
Requires-Dist: py-app-dev (>=2.10,<3.0)
|
|
21
23
|
Requires-Dist: pyyaml (>=6.0,<7.0)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
pypeline/__init__.py,sha256=
|
|
1
|
+
pypeline/__init__.py,sha256=oQdy_VDKQZm6_UKnVHbkyIDnpqXxS3OREilFLTax8MU,23
|
|
2
2
|
pypeline/__run.py,sha256=TCdaX05Qm3g8T4QYryKB25Xxf0L5Km7hFOHe1mK9vI0,350
|
|
3
3
|
pypeline/bootstrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
-
pypeline/bootstrap/run.py,sha256
|
|
4
|
+
pypeline/bootstrap/run.py,sha256=-DP9os7i5QaxmYdp_kGnEAhMFCILv1YCRwiYxut3lEc,31394
|
|
5
5
|
pypeline/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
pypeline/domain/artifacts.py,sha256=5k7cVfHhLmvWXNuHKxXb9ca4Lxu0JytGQqazENCeKEU,1404
|
|
7
7
|
pypeline/domain/config.py,sha256=6vWdHi7B6MA7NGi9wWXQE-YhSg1COSRmc3b1ji6AdAk,2053
|
|
@@ -13,20 +13,20 @@ pypeline/kickstart/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
|
13
13
|
pypeline/kickstart/create.py,sha256=iaB8MMC7PinpPBwRmz3rWZuE-DRbsLh2NtvczYaVgi0,2133
|
|
14
14
|
pypeline/kickstart/templates/project/.gitignore,sha256=y8GJoVvRPez1LBokf1NaDOt2X1XtGwKFMF5yjA8AVS0,24
|
|
15
15
|
pypeline/kickstart/templates/project/pypeline.ps1,sha256=PjCJULG8XA3AHKbNt3oHrIgD04huvvpIue_gjSo3PMA,104
|
|
16
|
-
pypeline/kickstart/templates/project/pypeline.yaml,sha256=
|
|
17
|
-
pypeline/kickstart/templates/project/pyproject.toml,sha256=
|
|
16
|
+
pypeline/kickstart/templates/project/pypeline.yaml,sha256=KKqRqxH7emAuZI1FBC-ITL8glVi2tKux61WAnbY6Ks0,401
|
|
17
|
+
pypeline/kickstart/templates/project/pyproject.toml,sha256=7hAoK6BammBxxoolMdCkNx7qPSFFiFUkQN8oAbCf7Yk,271
|
|
18
18
|
pypeline/kickstart/templates/project/steps/my_step.py,sha256=b-JEwF9EyF4G6lgvkk3I2aT2wpD_zQ2fTiQrR6lWhs4,788
|
|
19
19
|
pypeline/kickstart/templates/project/west.yaml,sha256=ZfVym7M4yzzC-Nm0vESdhqNYs6EaJuMQWGJBht_i0b4,188
|
|
20
|
-
pypeline/main.py,sha256=
|
|
20
|
+
pypeline/main.py,sha256=k1CkeFGRvQ-zLv6C-AMLC2ed1iyFzDUdvEam3HLHy2E,4210
|
|
21
21
|
pypeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
pypeline/pypeline.py,sha256=mDKUnTuMDw8l-kSDJCHRNbn6zrxAfXhAIAqc5HyHd5M,8758
|
|
23
23
|
pypeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
-
pypeline/steps/create_venv.py,sha256=
|
|
24
|
+
pypeline/steps/create_venv.py,sha256=F3YpWsZAP1YWjlPRx6uAx1ru82vtZ8uOJ7p51A-S3Fs,12802
|
|
25
25
|
pypeline/steps/env_setup_script.py,sha256=DRDCNMUDiW2rzkgEs0FhQfA_-WjPzPLb_e9dGc-mjLg,2526
|
|
26
|
-
pypeline/steps/scoop_install.py,sha256=
|
|
26
|
+
pypeline/steps/scoop_install.py,sha256=2MhsJ0iPmL8ueQhI52sKjVY9fqzj5xOQweQ65C0onfE,4117
|
|
27
27
|
pypeline/steps/west_install.py,sha256=hPyr28ksdKsQ0tv0gMNytzupgk1IgjN9CpmaBdX5zps,1947
|
|
28
|
-
pypeline_runner-1.
|
|
29
|
-
pypeline_runner-1.
|
|
30
|
-
pypeline_runner-1.
|
|
31
|
-
pypeline_runner-1.
|
|
32
|
-
pypeline_runner-1.
|
|
28
|
+
pypeline_runner-1.21.0.dist-info/METADATA,sha256=NpqAR3b3WS5GQUnf_m70c9SYm7oAGNZ619-i3Lnt3K0,7659
|
|
29
|
+
pypeline_runner-1.21.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
30
|
+
pypeline_runner-1.21.0.dist-info/entry_points.txt,sha256=pe1u0uuhPI_yeQ0KjEw6jK-EvQfPcZwBSajgbAdKz1o,47
|
|
31
|
+
pypeline_runner-1.21.0.dist-info/licenses/LICENSE,sha256=sKxdoqSmW9ezvPvt0ZGJbneyA0SBcm0GiqzTv2jN230,1066
|
|
32
|
+
pypeline_runner-1.21.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|