truefoundry 0.11.11rc1__py3-none-any.whl → 0.11.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/common/constants.py +1 -8
- truefoundry/common/storage_provider_utils.py +35 -10
- truefoundry/deploy/__init__.py +0 -3
- truefoundry/deploy/_autogen/models.py +144 -191
- truefoundry/deploy/builder/__init__.py +0 -1
- truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py +6 -3
- truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py +84 -347
- truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py +7 -3
- truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py +18 -18
- truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/__init__.py +7 -3
- truefoundry/deploy/builder/builders/tfy_task_pyspark_buildpack/dockerfile_template.py +18 -18
- truefoundry/deploy/builder/constants.py +0 -12
- truefoundry/deploy/builder/utils.py +21 -210
- truefoundry/deploy/lib/util.py +0 -35
- truefoundry/deploy/v2/lib/deploy.py +1 -3
- truefoundry/deploy/v2/lib/patched_models.py +3 -76
- truefoundry/deploy/v2/lib/source.py +0 -4
- truefoundry/ml/artifact/truefoundry_artifact_repo.py +25 -11
- truefoundry/ml/log_types/artifacts/artifact.py +3 -2
- truefoundry/ml/log_types/artifacts/model.py +1 -0
- {truefoundry-0.11.11rc1.dist-info → truefoundry-0.11.12.dist-info}/METADATA +1 -2
- {truefoundry-0.11.11rc1.dist-info → truefoundry-0.11.12.dist-info}/RECORD +24 -24
- {truefoundry-0.11.11rc1.dist-info → truefoundry-0.11.12.dist-info}/WHEEL +0 -0
- {truefoundry-0.11.11rc1.dist-info → truefoundry-0.11.12.dist-info}/entry_points.txt +0 -0
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
from typing import List, Optional
|
|
2
|
-
|
|
3
1
|
from mako.template import Template
|
|
4
2
|
|
|
5
3
|
from truefoundry.common.constants import ENV_VARS, PythonPackageManager
|
|
6
4
|
from truefoundry.deploy._autogen.models import TaskPySparkBuild
|
|
5
|
+
from truefoundry.deploy.builder.constants import (
|
|
6
|
+
PIP_CONF_BUILDKIT_SECRET_MOUNT,
|
|
7
|
+
UV_CONF_BUILDKIT_SECRET_MOUNT,
|
|
8
|
+
)
|
|
7
9
|
from truefoundry.deploy.builder.utils import (
|
|
8
10
|
generate_apt_install_command,
|
|
9
11
|
generate_pip_install_command,
|
|
10
|
-
generate_secret_mounts,
|
|
11
12
|
generate_uv_pip_install_command,
|
|
12
|
-
get_available_secrets,
|
|
13
13
|
)
|
|
14
14
|
from truefoundry.deploy.v2.lib.patched_models import (
|
|
15
15
|
_resolve_requirements_path,
|
|
@@ -57,13 +57,8 @@ def get_additional_pip_packages(build_configuration: TaskPySparkBuild):
|
|
|
57
57
|
def generate_dockerfile_content(
|
|
58
58
|
build_configuration: TaskPySparkBuild,
|
|
59
59
|
package_manager: str = ENV_VARS.TFY_PYTHON_BUILD_PACKAGE_MANAGER,
|
|
60
|
-
|
|
60
|
+
mount_python_package_manager_conf_secret: bool = False,
|
|
61
61
|
) -> str:
|
|
62
|
-
# Get available secrets from docker build extra args
|
|
63
|
-
available_secrets = set()
|
|
64
|
-
if docker_build_extra_args:
|
|
65
|
-
available_secrets = get_available_secrets(docker_build_extra_args)
|
|
66
|
-
|
|
67
62
|
# TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
|
|
68
63
|
requirements_path = _resolve_requirements_path(
|
|
69
64
|
build_context_path="",
|
|
@@ -83,13 +78,13 @@ def generate_dockerfile_content(
|
|
|
83
78
|
python_packages_install_command = generate_pip_install_command(
|
|
84
79
|
requirements_path=requirements_destination_path,
|
|
85
80
|
pip_packages=pip_packages,
|
|
86
|
-
|
|
81
|
+
mount_pip_conf_secret=mount_python_package_manager_conf_secret,
|
|
87
82
|
)
|
|
88
83
|
elif package_manager == PythonPackageManager.UV.value:
|
|
89
84
|
python_packages_install_command = generate_uv_pip_install_command(
|
|
90
85
|
requirements_path=requirements_destination_path,
|
|
91
86
|
pip_packages=pip_packages,
|
|
92
|
-
|
|
87
|
+
mount_uv_conf_secret=mount_python_package_manager_conf_secret,
|
|
93
88
|
)
|
|
94
89
|
else:
|
|
95
90
|
raise ValueError(f"Unsupported package manager: {package_manager}")
|
|
@@ -106,12 +101,17 @@ def generate_dockerfile_content(
|
|
|
106
101
|
"python_packages_install_command": python_packages_install_command,
|
|
107
102
|
}
|
|
108
103
|
|
|
109
|
-
if
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
104
|
+
if mount_python_package_manager_conf_secret:
|
|
105
|
+
if package_manager == PythonPackageManager.PIP.value:
|
|
106
|
+
template_args["package_manager_config_secret_mount"] = (
|
|
107
|
+
PIP_CONF_BUILDKIT_SECRET_MOUNT
|
|
108
|
+
)
|
|
109
|
+
elif package_manager == PythonPackageManager.UV.value:
|
|
110
|
+
template_args["package_manager_config_secret_mount"] = (
|
|
111
|
+
UV_CONF_BUILDKIT_SECRET_MOUNT
|
|
112
|
+
)
|
|
113
|
+
else:
|
|
114
|
+
raise ValueError(f"Unsupported package manager: {package_manager}")
|
|
115
115
|
else:
|
|
116
116
|
template_args["package_manager_config_secret_mount"] = ""
|
|
117
117
|
|
|
@@ -13,15 +13,3 @@ UV_CONF_BUILDKIT_SECRET_MOUNT = (
|
|
|
13
13
|
UV_CONF_SECRET_MOUNT_AS_ENV = (
|
|
14
14
|
f"UV_CONFIG_FILE=/run/secrets/{BUILDKIT_SECRET_MOUNT_UV_CONF_ID}"
|
|
15
15
|
)
|
|
16
|
-
|
|
17
|
-
BUILDKIT_SECRET_MOUNT_UV_ENV_ID = "uv.env"
|
|
18
|
-
UV_ENV_BUILDKIT_SECRET_MOUNT = (
|
|
19
|
-
f"--mount=type=secret,id={BUILDKIT_SECRET_MOUNT_UV_ENV_ID}"
|
|
20
|
-
)
|
|
21
|
-
UV_ENV_SECRET_MOUNT_AS_ENV = f". /run/secrets/{BUILDKIT_SECRET_MOUNT_UV_ENV_ID}"
|
|
22
|
-
|
|
23
|
-
BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID = "poetry.env"
|
|
24
|
-
POETRY_ENV_BUILDKIT_SECRET_MOUNT = (
|
|
25
|
-
f"--mount=type=secret,id={BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID}"
|
|
26
|
-
)
|
|
27
|
-
POETRY_ENV_SECRET_MOUNT_AS_ENV = f". /run/secrets/{BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID}"
|
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
import shlex
|
|
2
|
-
from typing import List, Optional
|
|
2
|
+
from typing import List, Optional
|
|
3
3
|
|
|
4
4
|
from truefoundry.common.constants import ENV_VARS
|
|
5
5
|
from truefoundry.deploy.builder.constants import (
|
|
6
6
|
BUILDKIT_SECRET_MOUNT_PIP_CONF_ID,
|
|
7
|
-
BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID,
|
|
8
7
|
BUILDKIT_SECRET_MOUNT_UV_CONF_ID,
|
|
9
|
-
BUILDKIT_SECRET_MOUNT_UV_ENV_ID,
|
|
10
|
-
PIP_CONF_BUILDKIT_SECRET_MOUNT,
|
|
11
8
|
PIP_CONF_SECRET_MOUNT_AS_ENV,
|
|
12
|
-
POETRY_ENV_BUILDKIT_SECRET_MOUNT,
|
|
13
|
-
POETRY_ENV_SECRET_MOUNT_AS_ENV,
|
|
14
|
-
UV_CONF_BUILDKIT_SECRET_MOUNT,
|
|
15
9
|
UV_CONF_SECRET_MOUNT_AS_ENV,
|
|
16
|
-
UV_ENV_BUILDKIT_SECRET_MOUNT,
|
|
17
|
-
UV_ENV_SECRET_MOUNT_AS_ENV,
|
|
18
10
|
)
|
|
19
11
|
|
|
20
12
|
|
|
@@ -36,129 +28,29 @@ def _get_id_from_buildkit_secret_value(value: str) -> Optional[str]:
|
|
|
36
28
|
|
|
37
29
|
def has_python_package_manager_conf_secret(docker_build_extra_args: List[str]) -> bool:
|
|
38
30
|
args = [arg.strip() for arg in docker_build_extra_args]
|
|
39
|
-
for i, arg in enumerate(
|
|
31
|
+
for i, arg in enumerate(docker_build_extra_args):
|
|
40
32
|
if (
|
|
41
33
|
arg == "--secret"
|
|
42
34
|
and i + 1 < len(args)
|
|
43
35
|
and (
|
|
44
36
|
_get_id_from_buildkit_secret_value(args[i + 1])
|
|
45
|
-
in (
|
|
46
|
-
BUILDKIT_SECRET_MOUNT_PIP_CONF_ID,
|
|
47
|
-
BUILDKIT_SECRET_MOUNT_UV_CONF_ID,
|
|
48
|
-
BUILDKIT_SECRET_MOUNT_UV_ENV_ID,
|
|
49
|
-
BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID,
|
|
50
|
-
)
|
|
37
|
+
in (BUILDKIT_SECRET_MOUNT_PIP_CONF_ID, BUILDKIT_SECRET_MOUNT_UV_CONF_ID)
|
|
51
38
|
)
|
|
52
39
|
):
|
|
53
40
|
return True
|
|
54
41
|
return False
|
|
55
42
|
|
|
56
43
|
|
|
57
|
-
def get_available_secrets(docker_build_extra_args: List[str]) -> Set[str]:
|
|
58
|
-
available_secrets = set()
|
|
59
|
-
args = [arg.strip() for arg in docker_build_extra_args]
|
|
60
|
-
for i, arg in enumerate(args):
|
|
61
|
-
if arg == "--secret" and i + 1 < len(args):
|
|
62
|
-
secret_id = _get_id_from_buildkit_secret_value(args[i + 1])
|
|
63
|
-
if secret_id:
|
|
64
|
-
available_secrets.add(secret_id)
|
|
65
|
-
return available_secrets
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def generate_secret_mounts(
|
|
69
|
-
available_secrets: Set[str], python_dependencies_type: str, package_manager: str
|
|
70
|
-
) -> str:
|
|
71
|
-
mounts = []
|
|
72
|
-
|
|
73
|
-
if python_dependencies_type == "pip":
|
|
74
|
-
if (
|
|
75
|
-
package_manager == "pip"
|
|
76
|
-
and BUILDKIT_SECRET_MOUNT_PIP_CONF_ID in available_secrets
|
|
77
|
-
):
|
|
78
|
-
mounts.append(PIP_CONF_BUILDKIT_SECRET_MOUNT)
|
|
79
|
-
elif (
|
|
80
|
-
package_manager == "uv"
|
|
81
|
-
and BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets
|
|
82
|
-
):
|
|
83
|
-
mounts.append(UV_CONF_BUILDKIT_SECRET_MOUNT)
|
|
84
|
-
elif python_dependencies_type == "uv":
|
|
85
|
-
if BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets:
|
|
86
|
-
mounts.append(UV_CONF_BUILDKIT_SECRET_MOUNT)
|
|
87
|
-
if BUILDKIT_SECRET_MOUNT_UV_ENV_ID in available_secrets:
|
|
88
|
-
mounts.append(UV_ENV_BUILDKIT_SECRET_MOUNT)
|
|
89
|
-
elif python_dependencies_type == "poetry":
|
|
90
|
-
if BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID in available_secrets:
|
|
91
|
-
mounts.append(POETRY_ENV_BUILDKIT_SECRET_MOUNT)
|
|
92
|
-
|
|
93
|
-
return " ".join(mounts)
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
def generate_secret_env_commands(
|
|
97
|
-
available_secrets: Set[str], python_dependencies_type: str, package_manager: str
|
|
98
|
-
) -> List[str]:
|
|
99
|
-
env_commands = []
|
|
100
|
-
|
|
101
|
-
if python_dependencies_type == "pip":
|
|
102
|
-
if (
|
|
103
|
-
package_manager == "pip"
|
|
104
|
-
and BUILDKIT_SECRET_MOUNT_PIP_CONF_ID in available_secrets
|
|
105
|
-
):
|
|
106
|
-
env_commands.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
|
|
107
|
-
elif (
|
|
108
|
-
package_manager == "uv"
|
|
109
|
-
and BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets
|
|
110
|
-
):
|
|
111
|
-
env_commands.append(UV_CONF_SECRET_MOUNT_AS_ENV)
|
|
112
|
-
elif python_dependencies_type == "uv":
|
|
113
|
-
if BUILDKIT_SECRET_MOUNT_UV_CONF_ID in available_secrets:
|
|
114
|
-
env_commands.append(UV_CONF_SECRET_MOUNT_AS_ENV)
|
|
115
|
-
if BUILDKIT_SECRET_MOUNT_UV_ENV_ID in available_secrets:
|
|
116
|
-
env_commands.append(UV_ENV_SECRET_MOUNT_AS_ENV)
|
|
117
|
-
elif python_dependencies_type == "poetry":
|
|
118
|
-
if BUILDKIT_SECRET_MOUNT_POETRY_ENV_ID in available_secrets:
|
|
119
|
-
env_commands.append(POETRY_ENV_SECRET_MOUNT_AS_ENV)
|
|
120
|
-
|
|
121
|
-
return env_commands
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
def generate_shell_command_with_secrets(
|
|
125
|
-
env_commands: List[str], command: List[str]
|
|
126
|
-
) -> str:
|
|
127
|
-
"""Generate a shell command that properly handles source commands and environment variables."""
|
|
128
|
-
if not env_commands:
|
|
129
|
-
return shlex.join(command)
|
|
130
|
-
|
|
131
|
-
# Separate source commands from env var assignments
|
|
132
|
-
source_commands = [
|
|
133
|
-
cmd for cmd in env_commands if cmd.startswith(". ") or cmd.startswith("source ")
|
|
134
|
-
]
|
|
135
|
-
env_assignments = [
|
|
136
|
-
cmd
|
|
137
|
-
for cmd in env_commands
|
|
138
|
-
if not cmd.startswith(". ") and not cmd.startswith("source ")
|
|
139
|
-
]
|
|
140
|
-
|
|
141
|
-
if source_commands:
|
|
142
|
-
# If we have source commands, join them with the command
|
|
143
|
-
shell_command_parts = source_commands.copy()
|
|
144
|
-
|
|
145
|
-
# Add environment variable assignments and command together
|
|
146
|
-
if env_assignments:
|
|
147
|
-
final_command = shlex.join(env_assignments + command)
|
|
148
|
-
else:
|
|
149
|
-
final_command = shlex.join(command)
|
|
150
|
-
|
|
151
|
-
shell_command_parts.append(final_command)
|
|
152
|
-
return " && ".join(shell_command_parts)
|
|
153
|
-
|
|
154
|
-
return shlex.join(env_assignments + command)
|
|
155
|
-
|
|
156
|
-
|
|
157
44
|
def generate_pip_install_command(
|
|
158
45
|
requirements_path: Optional[str],
|
|
159
46
|
pip_packages: Optional[List[str]],
|
|
160
|
-
|
|
47
|
+
mount_pip_conf_secret: bool = False,
|
|
161
48
|
) -> Optional[str]:
|
|
49
|
+
upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
|
|
50
|
+
envs = []
|
|
51
|
+
if mount_pip_conf_secret:
|
|
52
|
+
envs.append(PIP_CONF_SECRET_MOUNT_AS_ENV)
|
|
53
|
+
|
|
162
54
|
command = ["python", "-m", "pip", "install", "--use-pep517", "--no-cache-dir"]
|
|
163
55
|
args = []
|
|
164
56
|
if requirements_path:
|
|
@@ -171,37 +63,27 @@ def generate_pip_install_command(
|
|
|
171
63
|
if not args:
|
|
172
64
|
return None
|
|
173
65
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
available_secrets, python_dependencies_type="pip", package_manager="pip"
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
final_pip_install_command = generate_shell_command_with_secrets(
|
|
181
|
-
secret_env_commands, command + args
|
|
66
|
+
final_pip_install_command = shlex.join(envs + command + args)
|
|
67
|
+
final_docker_run_command = " && ".join(
|
|
68
|
+
[upgrade_pip_command, final_pip_install_command]
|
|
182
69
|
)
|
|
183
|
-
|
|
184
|
-
return final_pip_install_command
|
|
70
|
+
return final_docker_run_command
|
|
185
71
|
|
|
186
72
|
|
|
187
73
|
def generate_uv_pip_install_command(
|
|
188
74
|
requirements_path: Optional[str],
|
|
189
75
|
pip_packages: Optional[List[str]],
|
|
190
|
-
|
|
76
|
+
mount_uv_conf_secret: bool = False,
|
|
191
77
|
) -> Optional[str]:
|
|
192
|
-
|
|
193
|
-
|
|
78
|
+
upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
|
|
79
|
+
uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
|
|
194
80
|
envs = [
|
|
195
81
|
"UV_LINK_MODE=copy",
|
|
196
82
|
"UV_PYTHON_DOWNLOADS=never",
|
|
197
83
|
"UV_INDEX_STRATEGY=unsafe-best-match",
|
|
198
84
|
]
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
if available_secrets:
|
|
202
|
-
secret_env_commands = generate_secret_env_commands(
|
|
203
|
-
available_secrets, python_dependencies_type="pip", package_manager="uv"
|
|
204
|
-
)
|
|
85
|
+
if mount_uv_conf_secret:
|
|
86
|
+
envs.append(UV_CONF_SECRET_MOUNT_AS_ENV)
|
|
205
87
|
|
|
206
88
|
command = ["uv", "pip", "install", "--no-cache-dir"]
|
|
207
89
|
|
|
@@ -217,10 +99,9 @@ def generate_uv_pip_install_command(
|
|
|
217
99
|
if not args:
|
|
218
100
|
return None
|
|
219
101
|
|
|
220
|
-
uv_pip_install_command =
|
|
221
|
-
|
|
222
|
-
)
|
|
223
|
-
final_docker_run_command = " ".join([uv_mount, uv_pip_install_command])
|
|
102
|
+
uv_pip_install_command = shlex.join(envs + command + args)
|
|
103
|
+
shell_commands = " && ".join([upgrade_pip_command, uv_pip_install_command])
|
|
104
|
+
final_docker_run_command = " ".join([uv_mount, shell_commands])
|
|
224
105
|
|
|
225
106
|
return final_docker_run_command
|
|
226
107
|
|
|
@@ -237,73 +118,3 @@ def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[
|
|
|
237
118
|
return " && ".join(
|
|
238
119
|
[apt_update_command, apt_install_command, clear_apt_lists_command]
|
|
239
120
|
)
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
def generate_command_to_install_from_uv_lock(
|
|
243
|
-
sync_options: Optional[str],
|
|
244
|
-
uv_version: Optional[str],
|
|
245
|
-
install_project: bool = False,
|
|
246
|
-
available_secrets: Optional[Set[str]] = None,
|
|
247
|
-
):
|
|
248
|
-
uv_image_uri = f"{ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_REPO}:{uv_version if uv_version is not None else ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_TAG}"
|
|
249
|
-
uv_mount = f"--mount=from={uv_image_uri},source=/uv,target=/usr/local/bin/uv"
|
|
250
|
-
|
|
251
|
-
envs = [
|
|
252
|
-
"UV_LINK_MODE=copy",
|
|
253
|
-
"UV_PYTHON_DOWNLOADS=never",
|
|
254
|
-
"UV_INDEX_STRATEGY=unsafe-best-match",
|
|
255
|
-
]
|
|
256
|
-
|
|
257
|
-
secret_env_commands = []
|
|
258
|
-
if available_secrets:
|
|
259
|
-
secret_env_commands = generate_secret_env_commands(
|
|
260
|
-
available_secrets, python_dependencies_type="uv", package_manager="uv"
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
command = ["uv", "sync"]
|
|
264
|
-
sync_options_list = shlex.split(sync_options or "")
|
|
265
|
-
if "--active" not in sync_options_list:
|
|
266
|
-
sync_options_list.append("--active")
|
|
267
|
-
|
|
268
|
-
if not install_project and "--no-install-project" not in sync_options_list:
|
|
269
|
-
sync_options_list.append("--no-install-project")
|
|
270
|
-
|
|
271
|
-
command.extend(sync_options_list)
|
|
272
|
-
|
|
273
|
-
uv_sync_install_command = generate_shell_command_with_secrets(
|
|
274
|
-
secret_env_commands, envs + command
|
|
275
|
-
)
|
|
276
|
-
final_docker_run_command = " ".join([uv_mount, uv_sync_install_command])
|
|
277
|
-
|
|
278
|
-
return final_docker_run_command
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
def generate_poetry_install_command(
|
|
282
|
-
install_options: Optional[str],
|
|
283
|
-
install_project: bool = False,
|
|
284
|
-
available_secrets: Optional[Set[str]] = None,
|
|
285
|
-
) -> Optional[str]:
|
|
286
|
-
command = ["poetry", "install"]
|
|
287
|
-
install_options_list = shlex.split(install_options or "")
|
|
288
|
-
|
|
289
|
-
if "--no-interaction" not in install_options_list:
|
|
290
|
-
command.append("--no-interaction")
|
|
291
|
-
|
|
292
|
-
if not install_project and "--no-root" not in install_options_list:
|
|
293
|
-
command.append("--no-root")
|
|
294
|
-
|
|
295
|
-
command.extend(install_options_list)
|
|
296
|
-
|
|
297
|
-
secret_env_commands = []
|
|
298
|
-
if available_secrets:
|
|
299
|
-
secret_env_commands = generate_secret_env_commands(
|
|
300
|
-
available_secrets=available_secrets,
|
|
301
|
-
python_dependencies_type="poetry",
|
|
302
|
-
package_manager="poetry",
|
|
303
|
-
)
|
|
304
|
-
|
|
305
|
-
poetry_install_cmd = generate_shell_command_with_secrets(
|
|
306
|
-
secret_env_commands, command
|
|
307
|
-
)
|
|
308
|
-
|
|
309
|
-
return poetry_install_cmd
|
truefoundry/deploy/lib/util.py
CHANGED
|
@@ -1,11 +1,6 @@
|
|
|
1
|
-
import os
|
|
2
1
|
import re
|
|
3
2
|
from typing import Union
|
|
4
3
|
|
|
5
|
-
from truefoundry.deploy._autogen.models import (
|
|
6
|
-
PythonBuild,
|
|
7
|
-
)
|
|
8
|
-
|
|
9
4
|
|
|
10
5
|
def get_application_fqn_from_deployment_fqn(deployment_fqn: str) -> str:
|
|
11
6
|
if not re.search(r":\d+$", deployment_fqn):
|
|
@@ -34,33 +29,3 @@ def find_list_paths(data, parent_key="", sep="."):
|
|
|
34
29
|
new_key = f"{parent_key}[{i}]"
|
|
35
30
|
list_paths.extend(find_list_paths(value, new_key, sep))
|
|
36
31
|
return list_paths
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def _validate_file_path(project_root_path: str, file_path: str):
|
|
40
|
-
file_absolute_path = os.path.join(project_root_path, file_path)
|
|
41
|
-
if not os.path.exists(file_absolute_path):
|
|
42
|
-
raise FileNotFoundError(
|
|
43
|
-
f"file {file_path} not found. file path should be relative to your project root path: {project_root_path}."
|
|
44
|
-
)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def validate_paths(python_build: PythonBuild, source_dir: str):
|
|
48
|
-
if not python_build.python_dependencies:
|
|
49
|
-
return
|
|
50
|
-
|
|
51
|
-
if not os.path.exists(source_dir):
|
|
52
|
-
raise ValueError(f"project root path {source_dir!r} of does not exist")
|
|
53
|
-
if (
|
|
54
|
-
python_build.python_dependencies.type == "pip"
|
|
55
|
-
and python_build.python_dependencies.requirements_path
|
|
56
|
-
):
|
|
57
|
-
_validate_file_path(
|
|
58
|
-
source_dir,
|
|
59
|
-
python_build.python_dependencies.requirements_path,
|
|
60
|
-
)
|
|
61
|
-
if python_build.python_dependencies.type == "uv":
|
|
62
|
-
_validate_file_path(source_dir, "uv.lock")
|
|
63
|
-
_validate_file_path(source_dir, "pyproject.toml")
|
|
64
|
-
if python_build.python_dependencies.type == "poetry":
|
|
65
|
-
_validate_file_path(source_dir, "pyproject.toml")
|
|
66
|
-
_validate_file_path(source_dir, "poetry.lock")
|
|
@@ -13,9 +13,7 @@ from truefoundry.deploy.lib.clients.servicefoundry_client import (
|
|
|
13
13
|
)
|
|
14
14
|
from truefoundry.deploy.lib.dao.workspace import get_workspace_by_fqn
|
|
15
15
|
from truefoundry.deploy.lib.model.entity import Deployment, DeploymentTransitionStatus
|
|
16
|
-
from truefoundry.deploy.lib.util import
|
|
17
|
-
get_application_fqn_from_deployment_fqn,
|
|
18
|
-
)
|
|
16
|
+
from truefoundry.deploy.lib.util import get_application_fqn_from_deployment_fqn
|
|
19
17
|
from truefoundry.deploy.v2.lib.models import BuildResponse
|
|
20
18
|
from truefoundry.deploy.v2.lib.source import (
|
|
21
19
|
local_source_to_image,
|
|
@@ -24,7 +24,7 @@ build=Build(
|
|
|
24
24
|
...
|
|
25
25
|
build_spec=PythonBuild(
|
|
26
26
|
...
|
|
27
|
-
|
|
27
|
+
requirements_path={requirements_txt_path!r}
|
|
28
28
|
)
|
|
29
29
|
)
|
|
30
30
|
```
|
|
@@ -36,44 +36,12 @@ build:
|
|
|
36
36
|
type: build
|
|
37
37
|
build_spec:
|
|
38
38
|
type: tfy-python-buildpack
|
|
39
|
-
|
|
40
|
-
type: pip
|
|
41
|
-
requirements_path: {requirements_txt_path!r}
|
|
39
|
+
requirements_path: {requirements_txt_path!r}
|
|
42
40
|
...
|
|
43
41
|
...
|
|
44
42
|
```
|
|
45
43
|
|
|
46
|
-
or set it to None if you don't want
|
|
47
|
-
"""
|
|
48
|
-
|
|
49
|
-
SPECS_UPGRADE_WARNING_MESSAGE_TEMPLATE = """\
|
|
50
|
-
The `requirements_path` and `pip_packages` fields are deprecated and will be removed in a future release.
|
|
51
|
-
Please use the `python_dependencies` field instead. Please use the following format:
|
|
52
|
-
|
|
53
|
-
```python
|
|
54
|
-
build=Build(
|
|
55
|
-
...
|
|
56
|
-
build_spec=PythonBuild(
|
|
57
|
-
...
|
|
58
|
-
python_dependencies=Pip(requirements_path={requirements_txt_path!r}, pip_packages={pip_packages!r})
|
|
59
|
-
)
|
|
60
|
-
)
|
|
61
|
-
```
|
|
62
|
-
|
|
63
|
-
OR
|
|
64
|
-
|
|
65
|
-
```yaml
|
|
66
|
-
build:
|
|
67
|
-
type: build
|
|
68
|
-
build_spec:
|
|
69
|
-
type: tfy-python-buildpack
|
|
70
|
-
python_dependencies:
|
|
71
|
-
type: pip
|
|
72
|
-
requirements_path: {requirements_txt_path!r}
|
|
73
|
-
pip_packages: {pip_packages!r}
|
|
74
|
-
...
|
|
75
|
-
...
|
|
76
|
-
```
|
|
44
|
+
or set it to None if you don't want use any requirements file.
|
|
77
45
|
"""
|
|
78
46
|
|
|
79
47
|
|
|
@@ -106,18 +74,6 @@ def _resolve_requirements_path(
|
|
|
106
74
|
return None
|
|
107
75
|
|
|
108
76
|
|
|
109
|
-
def check_whether_poetry_toml_exists(
|
|
110
|
-
build_context_path: str,
|
|
111
|
-
) -> bool:
|
|
112
|
-
required_filename = "poetry.toml"
|
|
113
|
-
possible_path = os.path.join(build_context_path, required_filename)
|
|
114
|
-
|
|
115
|
-
if os.path.isfile(possible_path):
|
|
116
|
-
return True
|
|
117
|
-
|
|
118
|
-
return False
|
|
119
|
-
|
|
120
|
-
|
|
121
77
|
class CUDAVersion(str, enum.Enum):
|
|
122
78
|
CUDA_11_0_CUDNN8 = "11.0-cudnn8"
|
|
123
79
|
CUDA_11_1_CUDNN8 = "11.1-cudnn8"
|
|
@@ -209,23 +165,6 @@ class PythonBuild(models.PythonBuild, PatchedModelBase):
|
|
|
209
165
|
build_context_path=values.get("build_context_path") or "./",
|
|
210
166
|
requirements_path=values.get("requirements_path"),
|
|
211
167
|
)
|
|
212
|
-
|
|
213
|
-
if not values.get("python_dependencies"):
|
|
214
|
-
warnings.warn(
|
|
215
|
-
SPECS_UPGRADE_WARNING_MESSAGE_TEMPLATE.format(
|
|
216
|
-
requirements_txt_path=values.get("requirements_path"),
|
|
217
|
-
pip_packages=values.get("pip_packages"),
|
|
218
|
-
),
|
|
219
|
-
category=TrueFoundryDeprecationWarning,
|
|
220
|
-
stacklevel=2,
|
|
221
|
-
)
|
|
222
|
-
values["python_dependencies"] = Pip(
|
|
223
|
-
type="pip",
|
|
224
|
-
requirements_path=values.get("requirements_path"),
|
|
225
|
-
pip_packages=values.get("pip_packages"),
|
|
226
|
-
)
|
|
227
|
-
values.pop("pip_packages", None)
|
|
228
|
-
values.pop("requirements_path", None)
|
|
229
168
|
return values
|
|
230
169
|
|
|
231
170
|
|
|
@@ -635,15 +574,3 @@ class SparkExecutorDynamicScaling(models.SparkExecutorDynamicScaling, PatchedMod
|
|
|
635
574
|
|
|
636
575
|
class TaskPySparkBuild(models.TaskPySparkBuild, PatchedModelBase):
|
|
637
576
|
type: Literal["task-pyspark-build"] = "task-pyspark-build"
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
class Pip(models.Pip, PatchedModelBase):
|
|
641
|
-
type: Literal["pip"] = "pip"
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
class UV(models.UV, PatchedModelBase):
|
|
645
|
-
type: Literal["uv"] = "uv"
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
class Poetry(models.Poetry, PatchedModelBase):
|
|
649
|
-
type: Literal["poetry"] = "poetry"
|
|
@@ -21,7 +21,6 @@ from truefoundry.deploy.lib.clients.servicefoundry_client import (
|
|
|
21
21
|
ServiceFoundryServiceClient,
|
|
22
22
|
)
|
|
23
23
|
from truefoundry.deploy.lib.dao import workspace as workspace_lib
|
|
24
|
-
from truefoundry.deploy.lib.util import validate_paths
|
|
25
24
|
from truefoundry.deploy.v2.lib.patched_models import Image, RemoteSource
|
|
26
25
|
from truefoundry.logger import logger
|
|
27
26
|
|
|
@@ -210,9 +209,6 @@ def local_source_to_image(
|
|
|
210
209
|
f"of component {component_name!r} does not exist"
|
|
211
210
|
)
|
|
212
211
|
|
|
213
|
-
if isinstance(build.build_spec, models.PythonBuild):
|
|
214
|
-
validate_paths(build.build_spec, source_dir)
|
|
215
|
-
|
|
216
212
|
client = ServiceFoundryServiceClient()
|
|
217
213
|
|
|
218
214
|
workspace = workspace_lib.get_workspace_by_fqn(workspace_fqn=workspace_fqn)
|
|
@@ -40,6 +40,7 @@ from truefoundry.common.storage_provider_utils import (
|
|
|
40
40
|
azure_multi_part_upload,
|
|
41
41
|
decide_file_parts,
|
|
42
42
|
s3_compatible_multipart_upload,
|
|
43
|
+
truncate_path_for_progress,
|
|
43
44
|
)
|
|
44
45
|
from truefoundry.ml._autogen.client import ( # type: ignore[attr-defined]
|
|
45
46
|
ApiClient,
|
|
@@ -125,14 +126,14 @@ def _signed_url_upload_file(
|
|
|
125
126
|
augmented_raise_for_status(response, exception_class=MlFoundryException) # type: ignore
|
|
126
127
|
return
|
|
127
128
|
|
|
128
|
-
|
|
129
|
-
f"[green]
|
|
129
|
+
task_id = progress_bar.add_task(
|
|
130
|
+
f"[green]⬆ {truncate_path_for_progress(local_file, 64, relpath=True)}",
|
|
131
|
+
start=True,
|
|
132
|
+
visible=True,
|
|
130
133
|
)
|
|
131
134
|
|
|
132
135
|
def callback(length):
|
|
133
|
-
progress_bar.update(
|
|
134
|
-
task_progress_bar, advance=length, total=os.stat(local_file).st_size
|
|
135
|
-
)
|
|
136
|
+
progress_bar.update(task_id, advance=length, total=os.stat(local_file).st_size)
|
|
136
137
|
if abort_event and abort_event.is_set():
|
|
137
138
|
raise Exception("aborting upload")
|
|
138
139
|
|
|
@@ -147,6 +148,9 @@ def _signed_url_upload_file(
|
|
|
147
148
|
) as response:
|
|
148
149
|
augmented_raise_for_status(response, exception_class=MlFoundryException) # type: ignore
|
|
149
150
|
|
|
151
|
+
if progress_bar is not None:
|
|
152
|
+
progress_bar.refresh()
|
|
153
|
+
|
|
150
154
|
|
|
151
155
|
def _download_file_using_http_uri(
|
|
152
156
|
http_uri,
|
|
@@ -167,7 +171,11 @@ def _download_file_using_http_uri(
|
|
|
167
171
|
exception_class=MlFoundryException, # type: ignore
|
|
168
172
|
) as response:
|
|
169
173
|
augmented_raise_for_status(response, exception_class=MlFoundryException) # type: ignore
|
|
170
|
-
file_size = int(response.headers.get("Content-Length",
|
|
174
|
+
file_size = int(response.headers.get("Content-Length", -1))
|
|
175
|
+
if file_size == 0 and callback:
|
|
176
|
+
# special case for empty files
|
|
177
|
+
callback(1, 1)
|
|
178
|
+
file_size = file_size if file_size > 0 else 0
|
|
171
179
|
with open(download_path, "wb") as output_file:
|
|
172
180
|
for chunk in response.iter_content(chunk_size=chunk_size):
|
|
173
181
|
if callback:
|
|
@@ -658,15 +666,18 @@ class MlFoundryArtifactsRepository:
|
|
|
658
666
|
logger.info("Downloading %s to %s", remote_file_path, local_path)
|
|
659
667
|
|
|
660
668
|
if progress_bar is not None:
|
|
661
|
-
|
|
662
|
-
f"[green]
|
|
669
|
+
task_id = progress_bar.add_task(
|
|
670
|
+
f"[green]⬇ {truncate_path_for_progress(remote_file_path, 64)}",
|
|
671
|
+
start=True,
|
|
672
|
+
visible=True,
|
|
663
673
|
)
|
|
664
674
|
|
|
665
|
-
def callback(
|
|
675
|
+
def callback(chunk_size: int, total_file_size: int):
|
|
676
|
+
nonlocal task_id
|
|
666
677
|
if progress_bar is not None:
|
|
667
678
|
progress_bar.update(
|
|
668
|
-
|
|
669
|
-
advance=
|
|
679
|
+
task_id,
|
|
680
|
+
advance=chunk_size,
|
|
670
681
|
total=total_file_size,
|
|
671
682
|
)
|
|
672
683
|
if abort_event and abort_event.is_set():
|
|
@@ -677,6 +688,9 @@ class MlFoundryArtifactsRepository:
|
|
|
677
688
|
download_path=local_path,
|
|
678
689
|
callback=callback,
|
|
679
690
|
)
|
|
691
|
+
|
|
692
|
+
if progress_bar is not None:
|
|
693
|
+
progress_bar.refresh()
|
|
680
694
|
logger.debug("Downloaded %s to %s", remote_file_path, local_path)
|
|
681
695
|
|
|
682
696
|
def _download_artifact(
|
|
@@ -347,10 +347,11 @@ class ArtifactVersion:
|
|
|
347
347
|
artifact_version.download(path="<your-desired-download-path>")
|
|
348
348
|
```
|
|
349
349
|
"""
|
|
350
|
-
|
|
350
|
+
download_info = self._download(
|
|
351
351
|
path=path, overwrite=overwrite, progress=progress
|
|
352
352
|
)
|
|
353
|
-
|
|
353
|
+
logger.info("Downloaded artifact contents to %s", download_info.download_dir)
|
|
354
|
+
return download_info.download_dir
|
|
354
355
|
|
|
355
356
|
def delete(self) -> bool:
|
|
356
357
|
"""
|