truefoundry 0.11.12__py3-none-any.whl → 0.12.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/common/constants.py +8 -1
- truefoundry/common/utils.py +4 -0
- truefoundry/deploy/__init__.py +3 -0
- truefoundry/deploy/_autogen/models.py +192 -145
- truefoundry/deploy/builder/__init__.py +1 -0
- truefoundry/deploy/builder/builders/dockerfile.py +4 -6
- truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py +3 -6
- truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py +351 -84
- truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py +3 -7
- truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py +19 -25
- truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/__init__.py +3 -7
- truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py +19 -29
- truefoundry/deploy/builder/constants.py +12 -0
- truefoundry/deploy/builder/utils.py +223 -21
- truefoundry/deploy/lib/util.py +120 -0
- truefoundry/deploy/v2/lib/deploy.py +7 -1
- truefoundry/deploy/v2/lib/patched_models.py +60 -11
- truefoundry/deploy/v2/lib/source.py +6 -31
- {truefoundry-0.11.12.dist-info → truefoundry-0.12.0rc2.dist-info}/METADATA +2 -1
- {truefoundry-0.11.12.dist-info → truefoundry-0.12.0rc2.dist-info}/RECORD +22 -22
- {truefoundry-0.11.12.dist-info → truefoundry-0.12.0rc2.dist-info}/WHEEL +0 -0
- {truefoundry-0.11.12.dist-info → truefoundry-0.12.0rc2.dist-info}/entry_points.txt +0 -0
|
@@ -1,18 +1,15 @@
|
|
|
1
|
+
from typing import List, Optional
|
|
2
|
+
|
|
1
3
|
from mako.template import Template
|
|
2
4
|
|
|
3
5
|
from truefoundry.common.constants import ENV_VARS, PythonPackageManager
|
|
4
6
|
from truefoundry.deploy._autogen.models import TaskPySparkBuild
|
|
5
|
-
from truefoundry.deploy.builder.constants import (
|
|
6
|
-
PIP_CONF_BUILDKIT_SECRET_MOUNT,
|
|
7
|
-
UV_CONF_BUILDKIT_SECRET_MOUNT,
|
|
8
|
-
)
|
|
9
7
|
from truefoundry.deploy.builder.utils import (
|
|
10
8
|
generate_apt_install_command,
|
|
11
9
|
generate_pip_install_command,
|
|
10
|
+
generate_secret_mounts,
|
|
12
11
|
generate_uv_pip_install_command,
|
|
13
|
-
|
|
14
|
-
from truefoundry.deploy.v2.lib.patched_models import (
|
|
15
|
-
_resolve_requirements_path,
|
|
12
|
+
get_available_secrets,
|
|
16
13
|
)
|
|
17
14
|
|
|
18
15
|
# TODO[GW]: Switch to a non-root user inside the container
|
|
@@ -57,20 +54,18 @@ def get_additional_pip_packages(build_configuration: TaskPySparkBuild):
|
|
|
57
54
|
def generate_dockerfile_content(
|
|
58
55
|
build_configuration: TaskPySparkBuild,
|
|
59
56
|
package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,
|
|
60
|
-
|
|
57
|
+
docker_build_extra_args: Optional[List[str]] = None,
|
|
61
58
|
) -> str:
|
|
59
|
+
# Get available secrets from docker build extra args
|
|
60
|
+
available_secrets = set()
|
|
61
|
+
if docker_build_extra_args:
|
|
62
|
+
available_secrets = get_available_secrets(docker_build_extra_args)
|
|
63
|
+
|
|
62
64
|
# TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
|
|
63
|
-
requirements_path =
|
|
64
|
-
build_context_path="",
|
|
65
|
-
requirements_path=build_configuration.requirements_path,
|
|
66
|
-
)
|
|
65
|
+
requirements_path = build_configuration.requirements_path
|
|
67
66
|
requirements_destination_path = (
|
|
68
67
|
"/tmp/requirements.txt" if requirements_path else None
|
|
69
68
|
)
|
|
70
|
-
# if not build_configuration.python_version:
|
|
71
|
-
# raise ValueError(
|
|
72
|
-
# "`python_version` is required for `tfy-python-buildpack` builder"
|
|
73
|
-
# )
|
|
74
69
|
pip_packages = get_additional_pip_packages(build_configuration) + (
|
|
75
70
|
build_configuration.pip_packages or []
|
|
76
71
|
)
|
|
@@ -78,13 +73,13 @@ def generate_dockerfile_content(
|
|
|
78
73
|
python_packages_install_command = generate_pip_install_command(
|
|
79
74
|
requirements_path=requirements_destination_path,
|
|
80
75
|
pip_packages=pip_packages,
|
|
81
|
-
|
|
76
|
+
available_secrets=available_secrets,
|
|
82
77
|
)
|
|
83
78
|
elif package_manager == PythonPackageManager.UV.value:
|
|
84
79
|
python_packages_install_command = generate_uv_pip_install_command(
|
|
85
80
|
requirements_path=requirements_destination_path,
|
|
86
81
|
pip_packages=pip_packages,
|
|
87
|
-
|
|
82
|
+
available_secrets=available_secrets,
|
|
88
83
|
)
|
|
89
84
|
else:
|
|
90
85
|
raise ValueError(f"Unsupported package manager: {package_manager}")
|
|
@@ -101,17 +96,12 @@ def generate_dockerfile_content(
|
|
|
101
96
|
"python_packages_install_command": python_packages_install_command,
|
|
102
97
|
}
|
|
103
98
|
|
|
104
|
-
if
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
template_args["package_manager_config_secret_mount"] = (
|
|
111
|
-
UV_CONF_BUILDKIT_SECRET_MOUNT
|
|
112
|
-
)
|
|
113
|
-
else:
|
|
114
|
-
raise ValueError(f"Unsupported package manager: {package_manager}")
|
|
99
|
+
if available_secrets:
|
|
100
|
+
template_args["package_manager_config_secret_mount"] = generate_secret_mounts(
|
|
101
|
+
available_secrets=available_secrets,
|
|
102
|
+
python_dependencies_type="pip",
|
|
103
|
+
package_manager=package_manager,
|
|
104
|
+
)
|
|
115
105
|
else:
|
|
116
106
|
template_args["package_manager_config_secret_mount"] = ""
|
|
117
107
|
|
|
@@ -13,3 +13,15 @@ UV_CONF_BUILDKIT_SECRET_MOUNT = (
|
|
|
13
13
|
UV_CONF_SECRET_MOUNT_AS_ENV = (
|
|
14
14
|
f"UV_CONFIG_FILE=/run/secrets/{BUILDKIT_SECRET_MOUNT_UV_CONF_ID}"
|
|
15
15
|
)
|
|
16
|
+
|
|
17
|
+
BUILDKIT_SECRET_MOUNT_UV_ENV_ID = "uv.env"
|
|
18
|
+
UV_ENV_BUILDKIT_SECRET_MOUNT = (
|
|
19
|
+
f"--mount=type=secret,id={BUILDKIT_SECRET_MOUNT_UV_ENV_ID}"
|
|
20
|
+
)
|
|
21
|
+
UV_ENV_SECRET_MOUNT_AS_ENV = f". /run/secrets/{BUILDKIT_SECRET_MOUNT_UV_ENV_ID}"
|
|
22
|
+
|
|
23
|
+
BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID = "poetry.env"
|
|
24
|
+
POETRY_ENV_BUILDKIT_SECRET_MOUNT = (
|
|
25
|
+
f"--mount=type=secret,id={BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID}"
|
|
26
|
+
)
|
|
27
|
+
POETRY_ENV_SECRET_MOUNT_AS_ENV = f". /run/secrets/{BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID}"
|
|
@@ -1,12 +1,21 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import shlex
|
|
2
|
-
from typing import List, Optional
|
|
3
|
+
from typing import List, Optional, Set
|
|
3
4
|
|
|
4
5
|
from truefoundry.common.constants import ENV_VARS
|
|
5
6
|
from truefoundry.deploy.builder.constants import (
|
|
6
7
|
BUILDKIT_SECRET_MOUNT_PIP_CONF_ID,
|
|
8
|
+
BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID,
|
|
7
9
|
BUILDKIT_SECRET_MOUNT_UV_CONF_ID,
|
|
10
|
+
BUILDKIT_SECRET_MOUNT_UV_ENV_ID,
|
|
11
|
+
PIP_CONF_BUILDKIT_SECRET_MOUNT,
|
|
8
12
|
PIP_CONF_SECRET_MOUNT_AS_ENV,
|
|
13
|
+
POETRY_ENV_BUILDKIT_SECRET_MOUNT,
|
|
14
|
+
POETRY_ENV_SECRET_MOUNT_AS_ENV,
|
|
15
|
+
UV_CONF_BUILDKIT_SECRET_MOUNT,
|
|
9
16
|
UV_CONF_SECRET_MOUNT_AS_ENV,
|
|
17
|
+
UV_ENV_BUILDKIT_SECRET_MOUNT,
|
|
18
|
+
UV_ENV_SECRET_MOUNT_AS_ENV,
|
|
10
19
|
)
|
|
11
20
|
|
|
12
21
|
|
|
@@ -28,29 +37,129 @@ def _get_id_from_buildkit_secret_value(value: str) -> Optional[str]:
|
|
|
28
37
|
|
|
29
38
|
def has_python_package_manager_conf_secret(docker_build_extra_args: List[str]) -> bool:
|
|
30
39
|
args = [arg.strip() for arg in docker_build_extra_args]
|
|
31
|
-
for i, arg in enumerate(
|
|
40
|
+
for i, arg in enumerate(args):
|
|
32
41
|
if (
|
|
33
42
|
arg == "--secret"
|
|
34
43
|
and i + 1 < len(args)
|
|
35
44
|
and (
|
|
36
45
|
_get_id_from_buildkit_secret_value(args[i + 1])
|
|
37
|
-
in (
|
|
46
|
+
in (
|
|
47
|
+
BUILDKIT_SECRET_MOUNT_PIP_CONF_ID,
|
|
48
|
+
BUILDKIT_SECRET_MOUNT_UV_CONF_ID,
|
|
49
|
+
BUILDKIT_SECRET_MOUNT_UV_ENV_ID,
|
|
50
|
+
BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID,
|
|
51
|
+
)
|
|
38
52
|
)
|
|
39
53
|
):
|
|
40
54
|
return True
|
|
41
55
|
return False
|
|
42
56
|
|
|
43
57
|
|
|
58
|
+
def get_available_secrets(docker_build_extra_args: List[str]) -> Set[str]:
|
|
59
|
+
available_secrets = set()
|
|
60
|
+
args = [arg.strip() for arg in docker_build_extra_args]
|
|
61
|
+
for i, arg in enumerate(args):
|
|
62
|
+
if arg == "--secret" and i + 1 < len(args):
|
|
63
|
+
secret_id = _get_id_from_buildkit_secret_value(args[i + 1])
|
|
64
|
+
if secret_id:
|
|
65
|
+
available_secrets.add(secret_id)
|
|
66
|
+
return available_secrets
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def generate_secret_mounts(
|
|
70
|
+
available_secrets: Set[str], python_dependencies_type: str, package_manager: str
|
|
71
|
+
) -> str:
|
|
72
|
+
mounts = []
|
|
73
|
+
|
|
74
|
+
if python_dependencies_type == "pip":
|
|
75
|
+
if (
|
|
76
|
+
package_manager == "pip"
|
|
77
|
+
and BUILDKIT_SECRET_MOUNT_PIP_CONF_ID in available_secrets
|
|
78
|
+
):
|
|
79
|
+
mounts.append(PIP_CONF_BUILDKIT_SECRET_MOUNT)
|
|
80
|
+
elif (
|
|
81
|
+
package_manager == "uv"
|
|
82
|
+
and BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets
|
|
83
|
+
):
|
|
84
|
+
mounts.append(UV_CONF_BUILDKIT_SECRET_MOUNT)
|
|
85
|
+
elif python_dependencies_type == "uv":
|
|
86
|
+
if BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets:
|
|
87
|
+
mounts.append(UV_CONF_BUILDKIT_SECRET_MOUNT)
|
|
88
|
+
if BUILDKIT_SECRET_MOUNT_UV_ENV_ID in available_secrets:
|
|
89
|
+
mounts.append(UV_ENV_BUILDKIT_SECRET_MOUNT)
|
|
90
|
+
elif python_dependencies_type == "poetry":
|
|
91
|
+
if BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID in available_secrets:
|
|
92
|
+
mounts.append(POETRY_ENV_BUILDKIT_SECRET_MOUNT)
|
|
93
|
+
|
|
94
|
+
return " ".join(mounts)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def generate_secret_env_commands(
|
|
98
|
+
available_secrets: Set[str], python_dependencies_type: str, package_manager: str
|
|
99
|
+
) -> List[str]:
|
|
100
|
+
env_commands = []
|
|
101
|
+
|
|
102
|
+
if python_dependencies_type == "pip":
|
|
103
|
+
if (
|
|
104
|
+
package_manager == "pip"
|
|
105
|
+
and BUILDKIT_SECRET_MOUNT_PIP_CONF_ID in available_secrets
|
|
106
|
+
):
|
|
107
|
+
env_commands.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
|
|
108
|
+
elif (
|
|
109
|
+
package_manager == "uv"
|
|
110
|
+
and BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets
|
|
111
|
+
):
|
|
112
|
+
env_commands.append(UV_CONF_SECRET_MOUNT_AS_ENV)
|
|
113
|
+
elif python_dependencies_type == "uv":
|
|
114
|
+
if BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets:
|
|
115
|
+
env_commands.append(UV_CONF_SECRET_MOUNT_AS_ENV)
|
|
116
|
+
if BUILDKIT_SECRET_MOUNT_UV_ENV_ID in available_secrets:
|
|
117
|
+
env_commands.append(UV_ENV_SECRET_MOUNT_AS_ENV)
|
|
118
|
+
elif python_dependencies_type == "poetry":
|
|
119
|
+
if BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID in available_secrets:
|
|
120
|
+
env_commands.append(POETRY_ENV_SECRET_MOUNT_AS_ENV)
|
|
121
|
+
|
|
122
|
+
return env_commands
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def generate_shell_command_with_secrets(
|
|
126
|
+
env_commands: List[str], command: List[str]
|
|
127
|
+
) -> str:
|
|
128
|
+
"""Generate a shell command that properly handles source commands and environment variables."""
|
|
129
|
+
if not env_commands:
|
|
130
|
+
return shlex.join(command)
|
|
131
|
+
|
|
132
|
+
# Separate source commands from env var assignments
|
|
133
|
+
source_commands = [
|
|
134
|
+
cmd for cmd in env_commands if cmd.startswith(". ") or cmd.startswith("source ")
|
|
135
|
+
]
|
|
136
|
+
env_assignments = [
|
|
137
|
+
cmd
|
|
138
|
+
for cmd in env_commands
|
|
139
|
+
if not cmd.startswith(". ") and not cmd.startswith("source ")
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
if source_commands:
|
|
143
|
+
# If we have source commands, join them with the command
|
|
144
|
+
shell_command_parts = source_commands.copy()
|
|
145
|
+
|
|
146
|
+
# Add environment variable assignments and command together
|
|
147
|
+
if env_assignments:
|
|
148
|
+
final_command = shlex.join(env_assignments + command)
|
|
149
|
+
else:
|
|
150
|
+
final_command = shlex.join(command)
|
|
151
|
+
|
|
152
|
+
shell_command_parts.append(final_command)
|
|
153
|
+
return " && ".join(shell_command_parts)
|
|
154
|
+
|
|
155
|
+
return shlex.join(env_assignments + command)
|
|
156
|
+
|
|
157
|
+
|
|
44
158
|
def generate_pip_install_command(
|
|
45
159
|
requirements_path: Optional[str],
|
|
46
160
|
pip_packages: Optional[List[str]],
|
|
47
|
-
|
|
161
|
+
available_secrets: Optional[Set[str]] = None,
|
|
48
162
|
) -> Optional[str]:
|
|
49
|
-
upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
|
|
50
|
-
envs = []
|
|
51
|
-
if mount_pip_conf_secret:
|
|
52
|
-
envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
|
|
53
|
-
|
|
54
163
|
command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
|
|
55
164
|
args = []
|
|
56
165
|
if requirements_path:
|
|
@@ -63,27 +172,37 @@ def generate_pip_install_command(
|
|
|
63
172
|
if not args:
|
|
64
173
|
return None
|
|
65
174
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
175
|
+
secret_env_commands = []
|
|
176
|
+
if available_secrets:
|
|
177
|
+
secret_env_commands = generate_secret_env_commands(
|
|
178
|
+
available_secrets, python_dependencies_type="pip", package_manager="pip"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
final_pip_install_command = generate_shell_command_with_secrets(
|
|
182
|
+
secret_env_commands, command + args
|
|
69
183
|
)
|
|
70
|
-
|
|
184
|
+
|
|
185
|
+
return final_pip_install_command
|
|
71
186
|
|
|
72
187
|
|
|
73
188
|
def generate_uv_pip_install_command(
|
|
74
189
|
requirements_path: Optional[str],
|
|
75
190
|
pip_packages: Optional[List[str]],
|
|
76
|
-
|
|
191
|
+
available_secrets: Optional[Set[str]] = None,
|
|
77
192
|
) -> Optional[str]:
|
|
78
|
-
|
|
79
|
-
|
|
193
|
+
uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_REPO}:{ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_TAG},source=/uv,target=/usr/local/bin/uv"
|
|
194
|
+
|
|
80
195
|
envs = [
|
|
81
196
|
"UV_LINK_MODE=copy",
|
|
82
197
|
"UV_PYTHON_DOWNLOADS=never",
|
|
83
198
|
"UV_INDEX_STRATEGY=unsafe-best-match",
|
|
84
199
|
]
|
|
85
|
-
|
|
86
|
-
|
|
200
|
+
|
|
201
|
+
secret_env_commands = []
|
|
202
|
+
if available_secrets:
|
|
203
|
+
secret_env_commands = generate_secret_env_commands(
|
|
204
|
+
available_secrets, python_dependencies_type="pip", package_manager="uv"
|
|
205
|
+
)
|
|
87
206
|
|
|
88
207
|
command = ["uv", "pip", "install", "--no-cache-dir"]
|
|
89
208
|
|
|
@@ -99,9 +218,10 @@ def generate_uv_pip_install_command(
|
|
|
99
218
|
if not args:
|
|
100
219
|
return None
|
|
101
220
|
|
|
102
|
-
uv_pip_install_command =
|
|
103
|
-
|
|
104
|
-
|
|
221
|
+
uv_pip_install_command = generate_shell_command_with_secrets(
|
|
222
|
+
secret_env_commands, envs + command + args
|
|
223
|
+
)
|
|
224
|
+
final_docker_run_command = " ".join([uv_mount, uv_pip_install_command])
|
|
105
225
|
|
|
106
226
|
return final_docker_run_command
|
|
107
227
|
|
|
@@ -118,3 +238,85 @@ def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[
|
|
|
118
238
|
return " && ".join(
|
|
119
239
|
[apt_update_command, apt_install_command, clear_apt_lists_command]
|
|
120
240
|
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def generate_command_to_install_from_uv_lock(
|
|
244
|
+
sync_options: Optional[str],
|
|
245
|
+
uv_version: Optional[str],
|
|
246
|
+
install_project: bool = False,
|
|
247
|
+
available_secrets: Optional[Set[str]] = None,
|
|
248
|
+
):
|
|
249
|
+
uv_image_uri = f"{ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_REPO}:{uv_version if uv_version is not None else ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_TAG}"
|
|
250
|
+
uv_mount = f"--mount=from={uv_image_uri},source=/uv,target=/usr/local/bin/uv"
|
|
251
|
+
|
|
252
|
+
envs = [
|
|
253
|
+
"UV_LINK_MODE=copy",
|
|
254
|
+
"UV_PYTHON_DOWNLOADS=never",
|
|
255
|
+
"UV_INDEX_STRATEGY=unsafe-best-match",
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
secret_env_commands = []
|
|
259
|
+
if available_secrets:
|
|
260
|
+
secret_env_commands = generate_secret_env_commands(
|
|
261
|
+
available_secrets, python_dependencies_type="uv", package_manager="uv"
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
command = ["uv", "sync"]
|
|
265
|
+
sync_options_list = shlex.split(sync_options or "")
|
|
266
|
+
if "--active" not in sync_options_list:
|
|
267
|
+
sync_options_list.append("--active")
|
|
268
|
+
|
|
269
|
+
if not install_project and "--no-install-project" not in sync_options_list:
|
|
270
|
+
sync_options_list.append("--no-install-project")
|
|
271
|
+
|
|
272
|
+
command.extend(sync_options_list)
|
|
273
|
+
|
|
274
|
+
uv_sync_install_command = generate_shell_command_with_secrets(
|
|
275
|
+
secret_env_commands, envs + command
|
|
276
|
+
)
|
|
277
|
+
final_docker_run_command = " ".join([uv_mount, uv_sync_install_command])
|
|
278
|
+
|
|
279
|
+
return final_docker_run_command
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def generate_poetry_install_command(
|
|
283
|
+
install_options: Optional[str],
|
|
284
|
+
install_project: bool = False,
|
|
285
|
+
available_secrets: Optional[Set[str]] = None,
|
|
286
|
+
) -> Optional[str]:
|
|
287
|
+
command = ["poetry", "install"]
|
|
288
|
+
install_options_list = shlex.split(install_options or "")
|
|
289
|
+
|
|
290
|
+
if "--no-interaction" not in install_options_list:
|
|
291
|
+
command.append("--no-interaction")
|
|
292
|
+
|
|
293
|
+
if not install_project and "--no-root" not in install_options_list:
|
|
294
|
+
command.append("--no-root")
|
|
295
|
+
|
|
296
|
+
command.extend(install_options_list)
|
|
297
|
+
|
|
298
|
+
secret_env_commands = []
|
|
299
|
+
if available_secrets:
|
|
300
|
+
secret_env_commands = generate_secret_env_commands(
|
|
301
|
+
available_secrets=available_secrets,
|
|
302
|
+
python_dependencies_type="poetry",
|
|
303
|
+
package_manager="poetry",
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
poetry_install_cmd = generate_shell_command_with_secrets(
|
|
307
|
+
secret_env_commands, command
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
return poetry_install_cmd
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def check_whether_poetry_toml_exists(
|
|
314
|
+
build_context_path: str,
|
|
315
|
+
) -> bool:
|
|
316
|
+
required_filename = "poetry.toml"
|
|
317
|
+
possible_path = os.path.join(build_context_path, required_filename)
|
|
318
|
+
|
|
319
|
+
if os.path.isfile(possible_path):
|
|
320
|
+
return True
|
|
321
|
+
|
|
322
|
+
return False
|
truefoundry/deploy/lib/util.py
CHANGED
|
@@ -1,6 +1,17 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import re
|
|
2
3
|
from typing import Union
|
|
3
4
|
|
|
5
|
+
from truefoundry.common.utils import get_expanded_and_absolute_path
|
|
6
|
+
from truefoundry.deploy._autogen.models import (
|
|
7
|
+
UV,
|
|
8
|
+
Build,
|
|
9
|
+
DockerFileBuild,
|
|
10
|
+
Pip,
|
|
11
|
+
Poetry,
|
|
12
|
+
PythonBuild,
|
|
13
|
+
)
|
|
14
|
+
|
|
4
15
|
|
|
5
16
|
def get_application_fqn_from_deployment_fqn(deployment_fqn: str) -> str:
|
|
6
17
|
if not re.search(r":\d+$", deployment_fqn):
|
|
@@ -29,3 +40,112 @@ def find_list_paths(data, parent_key="", sep="."):
|
|
|
29
40
|
new_key = f"{parent_key}[{i}]"
|
|
30
41
|
list_paths.extend(find_list_paths(value, new_key, sep))
|
|
31
42
|
return list_paths
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _validate_file_path(
|
|
46
|
+
parent_path: str, relative_file_path: str, parent_dir_type: str
|
|
47
|
+
):
|
|
48
|
+
parent_abs = get_expanded_and_absolute_path(parent_path)
|
|
49
|
+
file_path_abs = get_expanded_and_absolute_path(
|
|
50
|
+
os.path.join(parent_abs, relative_file_path)
|
|
51
|
+
)
|
|
52
|
+
# Ensure the file path is actually inside the build context
|
|
53
|
+
outside_context = False
|
|
54
|
+
try:
|
|
55
|
+
# Use os.path.commonpath to check if file_path is inside build_context_abs
|
|
56
|
+
common_path = os.path.commonpath([parent_abs, file_path_abs])
|
|
57
|
+
outside_context = common_path != parent_abs
|
|
58
|
+
except ValueError:
|
|
59
|
+
# os.path.commonpath raises ValueError if paths are on different drives (Windows)
|
|
60
|
+
outside_context = True
|
|
61
|
+
|
|
62
|
+
if outside_context:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"Referenced file `{relative_file_path}` is outside the {parent_dir_type} `{parent_abs}`. "
|
|
65
|
+
f"It must exist in {parent_dir_type} `{parent_abs}`."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
if not os.path.exists(file_path_abs):
|
|
69
|
+
raise ValueError(
|
|
70
|
+
f"Referenced file `{relative_file_path}` not found. It must exist in {parent_dir_type} `{parent_abs}`."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def validate_dockerfile_build_paths(
|
|
75
|
+
dockerfile_build: DockerFileBuild, project_root_path: str
|
|
76
|
+
):
|
|
77
|
+
_validate_file_path(
|
|
78
|
+
parent_path=project_root_path,
|
|
79
|
+
relative_file_path=dockerfile_build.dockerfile_path,
|
|
80
|
+
parent_dir_type="project root",
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def validate_python_build_paths(python_build: PythonBuild, build_context_path: str):
|
|
85
|
+
if not python_build.python_dependencies:
|
|
86
|
+
# Old style flat requirements file
|
|
87
|
+
if python_build.requirements_path:
|
|
88
|
+
_validate_file_path(
|
|
89
|
+
parent_path=build_context_path,
|
|
90
|
+
relative_file_path=python_build.requirements_path,
|
|
91
|
+
parent_dir_type="build context",
|
|
92
|
+
)
|
|
93
|
+
return
|
|
94
|
+
|
|
95
|
+
if (
|
|
96
|
+
isinstance(python_build.python_dependencies, Pip)
|
|
97
|
+
and python_build.python_dependencies.requirements_path
|
|
98
|
+
):
|
|
99
|
+
_validate_file_path(
|
|
100
|
+
parent_path=build_context_path,
|
|
101
|
+
relative_file_path=python_build.python_dependencies.requirements_path,
|
|
102
|
+
parent_dir_type="build context",
|
|
103
|
+
)
|
|
104
|
+
elif isinstance(python_build.python_dependencies, UV):
|
|
105
|
+
_validate_file_path(
|
|
106
|
+
parent_path=build_context_path,
|
|
107
|
+
relative_file_path="pyproject.toml",
|
|
108
|
+
parent_dir_type="build context",
|
|
109
|
+
)
|
|
110
|
+
_validate_file_path(
|
|
111
|
+
parent_path=build_context_path,
|
|
112
|
+
relative_file_path="uv.lock",
|
|
113
|
+
parent_dir_type="build context",
|
|
114
|
+
)
|
|
115
|
+
elif isinstance(python_build.python_dependencies, Poetry):
|
|
116
|
+
_validate_file_path(
|
|
117
|
+
parent_path=build_context_path,
|
|
118
|
+
relative_file_path="pyproject.toml",
|
|
119
|
+
parent_dir_type="build context",
|
|
120
|
+
)
|
|
121
|
+
_validate_file_path(
|
|
122
|
+
parent_path=build_context_path,
|
|
123
|
+
relative_file_path="poetry.lock",
|
|
124
|
+
parent_dir_type="build context",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def validate_local_source_paths(component_name: str, build: Build):
|
|
129
|
+
source_dir = get_expanded_and_absolute_path(build.build_source.project_root_path)
|
|
130
|
+
if not os.path.exists(source_dir):
|
|
131
|
+
raise ValueError(
|
|
132
|
+
f"Project root path {source_dir!r} of component {component_name!r} does not exist"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
build_context_path = get_expanded_and_absolute_path(
|
|
136
|
+
os.path.join(source_dir, build.build_spec.build_context_path)
|
|
137
|
+
)
|
|
138
|
+
if not os.path.exists(build_context_path):
|
|
139
|
+
raise ValueError(
|
|
140
|
+
f"Build context path {build_context_path!r} "
|
|
141
|
+
f"of component {component_name!r} does not exist"
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
if isinstance(build.build_spec, DockerFileBuild):
|
|
145
|
+
validate_dockerfile_build_paths(
|
|
146
|
+
dockerfile_build=build.build_spec, project_root_path=source_dir
|
|
147
|
+
)
|
|
148
|
+
elif isinstance(build.build_spec, PythonBuild):
|
|
149
|
+
validate_python_build_paths(
|
|
150
|
+
python_build=build.build_spec, build_context_path=build_context_path
|
|
151
|
+
)
|
|
@@ -13,7 +13,10 @@ from truefoundry.deploy.lib.clients.servicefoundry_client import (
|
|
|
13
13
|
)
|
|
14
14
|
from truefoundry.deploy.lib.dao.workspace import get_workspace_by_fqn
|
|
15
15
|
from truefoundry.deploy.lib.model.entity import Deployment, DeploymentTransitionStatus
|
|
16
|
-
from truefoundry.deploy.lib.util import
|
|
16
|
+
from truefoundry.deploy.lib.util import (
|
|
17
|
+
get_application_fqn_from_deployment_fqn,
|
|
18
|
+
validate_local_source_paths,
|
|
19
|
+
)
|
|
17
20
|
from truefoundry.deploy.v2.lib.models import BuildResponse
|
|
18
21
|
from truefoundry.deploy.v2.lib.source import (
|
|
19
22
|
local_source_to_image,
|
|
@@ -45,6 +48,9 @@ def _handle_if_local_source(component: Component, workspace_fqn: str) -> Compone
|
|
|
45
48
|
and isinstance(component.image.build_source, autogen_models.LocalSource)
|
|
46
49
|
):
|
|
47
50
|
new_component = component.copy(deep=True)
|
|
51
|
+
validate_local_source_paths(
|
|
52
|
+
component_name=new_component.name, build=new_component.image
|
|
53
|
+
)
|
|
48
54
|
|
|
49
55
|
if new_component.image.build_source.local_build:
|
|
50
56
|
if not env_has_docker():
|
|
@@ -24,7 +24,7 @@ build=Build(
|
|
|
24
24
|
...
|
|
25
25
|
build_spec=PythonBuild(
|
|
26
26
|
...
|
|
27
|
-
requirements_path={requirements_txt_path!r}
|
|
27
|
+
python_dependencies=Pip(requirements_path={requirements_txt_path!r})
|
|
28
28
|
)
|
|
29
29
|
)
|
|
30
30
|
```
|
|
@@ -36,12 +36,45 @@ build:
|
|
|
36
36
|
type: build
|
|
37
37
|
build_spec:
|
|
38
38
|
type: tfy-python-buildpack
|
|
39
|
-
|
|
39
|
+
python_dependencies:
|
|
40
|
+
type: pip
|
|
41
|
+
requirements_path: {requirements_txt_path!r}
|
|
40
42
|
...
|
|
41
43
|
...
|
|
42
44
|
```
|
|
43
45
|
|
|
44
|
-
or set it to None if you don't want use any requirements file.
|
|
46
|
+
or set it to None if you don't want to use any requirements file.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
SPECS_UPGRADE_WARNING_MESSAGE_TEMPLATE = """\
|
|
50
|
+
The `requirements_path` and `pip_packages` fields are deprecated.
|
|
51
|
+
It is recommended to use the `python_dependencies` field instead which supports pip, uv and poetry.
|
|
52
|
+
Please use the following format:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
build=Build(
|
|
56
|
+
...
|
|
57
|
+
build_spec=PythonBuild(
|
|
58
|
+
...
|
|
59
|
+
python_dependencies=Pip(requirements_path={requirements_txt_path!r}, pip_packages={pip_packages!r})
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
OR
|
|
65
|
+
|
|
66
|
+
```yaml
|
|
67
|
+
build:
|
|
68
|
+
type: build
|
|
69
|
+
build_spec:
|
|
70
|
+
type: tfy-python-buildpack
|
|
71
|
+
python_dependencies:
|
|
72
|
+
type: pip
|
|
73
|
+
requirements_path: {requirements_txt_path!r}
|
|
74
|
+
pip_packages: {pip_packages!r}
|
|
75
|
+
...
|
|
76
|
+
...
|
|
77
|
+
```
|
|
45
78
|
"""
|
|
46
79
|
|
|
47
80
|
|
|
@@ -165,20 +198,24 @@ class PythonBuild(models.PythonBuild, PatchedModelBase):
|
|
|
165
198
|
build_context_path=values.get("build_context_path") or "./",
|
|
166
199
|
requirements_path=values.get("requirements_path"),
|
|
167
200
|
)
|
|
201
|
+
|
|
202
|
+
if (
|
|
203
|
+
values.get("requirements_path") or values.get("pip_packages")
|
|
204
|
+
) and not values.get("python_dependencies"):
|
|
205
|
+
warnings.warn(
|
|
206
|
+
SPECS_UPGRADE_WARNING_MESSAGE_TEMPLATE.format(
|
|
207
|
+
requirements_txt_path=values.get("requirements_path"),
|
|
208
|
+
pip_packages=values.get("pip_packages"),
|
|
209
|
+
),
|
|
210
|
+
category=TrueFoundryDeprecationWarning,
|
|
211
|
+
stacklevel=2,
|
|
212
|
+
)
|
|
168
213
|
return values
|
|
169
214
|
|
|
170
215
|
|
|
171
216
|
class SparkBuild(models.SparkBuild, PatchedModelBase):
|
|
172
217
|
type: Literal["tfy-spark-buildpack"] = "tfy-spark-buildpack"
|
|
173
218
|
|
|
174
|
-
@root_validator
|
|
175
|
-
def validate_values(cls, values):
|
|
176
|
-
_resolve_requirements_path(
|
|
177
|
-
build_context_path=values.get("build_context_path") or "./",
|
|
178
|
-
requirements_path=values.get("requirements_path"),
|
|
179
|
-
)
|
|
180
|
-
return values
|
|
181
|
-
|
|
182
219
|
|
|
183
220
|
class SparkImageBuild(models.SparkImageBuild, PatchedModelBase):
|
|
184
221
|
type: Literal["spark-image-build"] = "spark-image-build"
|
|
@@ -574,3 +611,15 @@ class SparkExecutorDynamicScaling(models.SparkExecutorDynamicScaling, PatchedMod
|
|
|
574
611
|
|
|
575
612
|
class TaskPySparkBuild(models.TaskPySparkBuild, PatchedModelBase):
|
|
576
613
|
type: Literal["task-pyspark-build"] = "task-pyspark-build"
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
class Pip(models.Pip, PatchedModelBase):
|
|
617
|
+
type: Literal["pip"] = "pip"
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
class UV(models.UV, PatchedModelBase):
|
|
621
|
+
type: Literal["uv"] = "uv"
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
class Poetry(models.Poetry, PatchedModelBase):
|
|
625
|
+
type: Literal["poetry"] = "poetry"
|