truefoundry 0.5.2__py3-none-any.whl → 0.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/__init__.py +10 -1
- truefoundry/autodeploy/agents/base.py +1 -1
- truefoundry/autodeploy/agents/developer.py +1 -1
- truefoundry/autodeploy/agents/project_identifier.py +1 -1
- truefoundry/autodeploy/agents/tester.py +1 -1
- truefoundry/autodeploy/cli.py +2 -2
- truefoundry/autodeploy/tools/base.py +2 -1
- truefoundry/autodeploy/tools/commit.py +1 -1
- truefoundry/autodeploy/tools/docker_build.py +1 -1
- truefoundry/autodeploy/tools/docker_run.py +1 -1
- truefoundry/autodeploy/tools/file_type_counts.py +1 -1
- truefoundry/autodeploy/tools/list_files.py +1 -1
- truefoundry/autodeploy/tools/read_file.py +1 -2
- truefoundry/autodeploy/tools/send_request.py +1 -1
- truefoundry/autodeploy/tools/write_file.py +1 -1
- truefoundry/cli/util.py +12 -3
- truefoundry/common/auth_service_client.py +7 -4
- truefoundry/common/constants.py +3 -0
- truefoundry/common/credential_provider.py +7 -8
- truefoundry/common/exceptions.py +11 -7
- truefoundry/common/request_utils.py +96 -14
- truefoundry/common/servicefoundry_client.py +31 -29
- truefoundry/common/session.py +93 -0
- truefoundry/common/storage_provider_utils.py +331 -0
- truefoundry/common/utils.py +9 -9
- truefoundry/common/warnings.py +21 -0
- truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py +8 -21
- truefoundry/deploy/cli/commands/deploy_command.py +4 -4
- truefoundry/deploy/lib/clients/servicefoundry_client.py +13 -14
- truefoundry/deploy/lib/dao/application.py +2 -2
- truefoundry/deploy/lib/dao/workspace.py +1 -1
- truefoundry/deploy/lib/session.py +1 -1
- truefoundry/deploy/v2/lib/deploy.py +2 -2
- truefoundry/deploy/v2/lib/deploy_workflow.py +1 -1
- truefoundry/deploy/v2/lib/patched_models.py +70 -4
- truefoundry/deploy/v2/lib/source.py +2 -1
- truefoundry/ml/artifact/truefoundry_artifact_repo.py +33 -297
- truefoundry/ml/autogen/client/__init__.py +3 -0
- truefoundry/ml/autogen/client/api/mlfoundry_artifacts_api.py +149 -0
- truefoundry/ml/autogen/client/models/__init__.py +3 -0
- truefoundry/ml/autogen/client/models/artifact_version_manifest.py +25 -1
- truefoundry/ml/autogen/client/models/get_artifact_version_aliases_response_dto.py +67 -0
- truefoundry/ml/autogen/client/models/model_version_manifest.py +18 -0
- truefoundry/ml/autogen/client_README.md +2 -0
- truefoundry/ml/autogen/entities/artifacts.py +7 -1
- truefoundry/ml/clients/servicefoundry_client.py +36 -15
- truefoundry/ml/exceptions.py +2 -1
- truefoundry/ml/log_types/artifacts/artifact.py +37 -4
- truefoundry/ml/log_types/artifacts/model.py +51 -10
- truefoundry/ml/log_types/artifacts/utils.py +2 -2
- truefoundry/ml/mlfoundry_api.py +6 -38
- truefoundry/ml/mlfoundry_run.py +6 -15
- truefoundry/ml/model_framework.py +5 -3
- truefoundry/ml/session.py +69 -97
- truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py +42 -9
- truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py +126 -7
- {truefoundry-0.5.2.dist-info → truefoundry-0.5.3.dist-info}/METADATA +2 -2
- {truefoundry-0.5.2.dist-info → truefoundry-0.5.3.dist-info}/RECORD +60 -59
- {truefoundry-0.5.2.dist-info → truefoundry-0.5.3.dist-info}/WHEEL +1 -1
- truefoundry/deploy/lib/auth/servicefoundry_session.py +0 -61
- truefoundry/ml/clients/entities.py +0 -8
- truefoundry/ml/clients/utils.py +0 -122
- {truefoundry-0.5.2.dist-info → truefoundry-0.5.3.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
from truefoundry.common.credential_provider import (
|
|
7
|
+
CredentialProvider,
|
|
8
|
+
EnvCredentialProvider,
|
|
9
|
+
FileCredentialProvider,
|
|
10
|
+
)
|
|
11
|
+
from truefoundry.common.entities import UserInfo
|
|
12
|
+
from truefoundry.common.utils import relogin_error_message
|
|
13
|
+
from truefoundry.logger import logger
|
|
14
|
+
|
|
15
|
+
SESSION_LOCK = threading.RLock()
|
|
16
|
+
ACTIVE_SESSION: Optional["Session"] = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Session:
|
|
20
|
+
def __init__(self) -> None:
|
|
21
|
+
self._closed = False
|
|
22
|
+
self._cred_provider: Optional[CredentialProvider] = self._get_cred_provider()
|
|
23
|
+
self._user_info: Optional[UserInfo] = self._cred_provider.token.to_user_info()
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def _get_cred_provider() -> CredentialProvider:
|
|
27
|
+
final_cred_provider = None
|
|
28
|
+
for cred_provider in [EnvCredentialProvider, FileCredentialProvider]:
|
|
29
|
+
if cred_provider.can_provide():
|
|
30
|
+
final_cred_provider = cred_provider()
|
|
31
|
+
break
|
|
32
|
+
if final_cred_provider is None:
|
|
33
|
+
raise Exception(
|
|
34
|
+
relogin_error_message(
|
|
35
|
+
"No active session found. Perhaps you are not logged in?",
|
|
36
|
+
)
|
|
37
|
+
)
|
|
38
|
+
return final_cred_provider
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def new(cls) -> "Session":
|
|
42
|
+
global ACTIVE_SESSION
|
|
43
|
+
with SESSION_LOCK:
|
|
44
|
+
new_session = cls()
|
|
45
|
+
if ACTIVE_SESSION and ACTIVE_SESSION == new_session:
|
|
46
|
+
return ACTIVE_SESSION
|
|
47
|
+
|
|
48
|
+
if ACTIVE_SESSION:
|
|
49
|
+
ACTIVE_SESSION.close()
|
|
50
|
+
|
|
51
|
+
ACTIVE_SESSION = new_session
|
|
52
|
+
logger.info(
|
|
53
|
+
"Logged in to %r as %r (%s)",
|
|
54
|
+
new_session.tfy_host,
|
|
55
|
+
new_session.user_info.user_id,
|
|
56
|
+
new_session.user_info.email or new_session.user_info.user_type.value,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
return ACTIVE_SESSION
|
|
60
|
+
|
|
61
|
+
def close(self):
|
|
62
|
+
self._closed = True
|
|
63
|
+
self._user_info = None
|
|
64
|
+
self._cred_provider = None
|
|
65
|
+
|
|
66
|
+
def _assert_not_closed(self):
|
|
67
|
+
if self._closed:
|
|
68
|
+
raise Exception("This session has been deactivated.")
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def access_token(self) -> str:
|
|
72
|
+
assert self._cred_provider is not None
|
|
73
|
+
return self._cred_provider.token.access_token
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def tfy_host(self) -> str:
|
|
77
|
+
assert self._cred_provider is not None
|
|
78
|
+
return self._cred_provider.tfy_host
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def user_info(self) -> UserInfo:
|
|
82
|
+
self._assert_not_closed()
|
|
83
|
+
assert self._user_info is not None
|
|
84
|
+
return self._user_info
|
|
85
|
+
|
|
86
|
+
def __eq__(self, other: object) -> bool:
|
|
87
|
+
if not isinstance(other, Session):
|
|
88
|
+
return False
|
|
89
|
+
return (
|
|
90
|
+
type(self._cred_provider) == type(other._cred_provider) # noqa: E721
|
|
91
|
+
and self.user_info == other.user_info
|
|
92
|
+
and self.tfy_host == other.tfy_host
|
|
93
|
+
)
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import math
|
|
3
|
+
import mmap
|
|
4
|
+
import os
|
|
5
|
+
from concurrent.futures import FIRST_EXCEPTION, Future, ThreadPoolExecutor, wait
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from threading import Event
|
|
8
|
+
from typing import List, NamedTuple, Optional
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
from rich.progress import Progress
|
|
12
|
+
from tqdm.utils import CallbackIOWrapper
|
|
13
|
+
|
|
14
|
+
from truefoundry.common.constants import ENV_VARS
|
|
15
|
+
from truefoundry.common.exceptions import HttpRequestException
|
|
16
|
+
from truefoundry.common.request_utils import (
|
|
17
|
+
augmented_raise_for_status,
|
|
18
|
+
cloud_storage_http_request,
|
|
19
|
+
)
|
|
20
|
+
from truefoundry.pydantic_v1 import BaseModel
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger("truefoundry")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class MultiPartUploadStorageProvider(str, Enum):
|
|
26
|
+
S3_COMPATIBLE = "S3_COMPATIBLE"
|
|
27
|
+
AZURE_BLOB = "AZURE_BLOB"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SignedURL(BaseModel):
|
|
31
|
+
signed_url: str
|
|
32
|
+
path: Optional[str] = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MultiPartUpload(BaseModel):
|
|
36
|
+
storage_provider: MultiPartUploadStorageProvider
|
|
37
|
+
part_signed_urls: List[SignedURL]
|
|
38
|
+
s3_compatible_upload_id: Optional[str] = None
|
|
39
|
+
azure_blob_block_ids: Optional[List[str]] = None
|
|
40
|
+
finalize_signed_url: SignedURL
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _FileMultiPartInfo(NamedTuple):
|
|
44
|
+
num_parts: int
|
|
45
|
+
part_size: int
|
|
46
|
+
file_size: int
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class _PartNumberEtag(NamedTuple):
|
|
50
|
+
part_number: int
|
|
51
|
+
etag: str
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
_MIN_BYTES_REQUIRED_FOR_MULTIPART = 100 * 1024 * 1024
|
|
55
|
+
# GCP/S3 Maximum number of parts per upload 10,000
|
|
56
|
+
# Maximum number of blocks in a block blob 50,000 blocks
|
|
57
|
+
# TODO: This number is artificially limited now. Later
|
|
58
|
+
# we will ask for parts signed URI in batches rather than in a single
|
|
59
|
+
# API Calls:
|
|
60
|
+
# Create Multipart Upload (Returns maximum number of parts, size limit of
|
|
61
|
+
# a single part, upload id for s3 etc )
|
|
62
|
+
# Get me signed uris for first 500 parts
|
|
63
|
+
# Upload 500 parts
|
|
64
|
+
# Get me signed uris for the next 500 parts
|
|
65
|
+
# Upload 500 parts
|
|
66
|
+
# ...
|
|
67
|
+
# Finalize the Multipart upload using the finalize signed url returned
|
|
68
|
+
# by Create Multipart Upload or get a new one.
|
|
69
|
+
_MAX_NUM_PARTS_FOR_MULTIPART = 1000
|
|
70
|
+
# Azure Maximum size of a block in a block blob 4000 MiB
|
|
71
|
+
# GCP/S3 Maximum size of an individual part in a multipart upload 5 GiB
|
|
72
|
+
_MAX_PART_SIZE_BYTES_FOR_MULTIPART = 4 * 1024 * 1024 * 1000
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _CallbackIOWrapperForMultiPartUpload(CallbackIOWrapper):
|
|
76
|
+
def __init__(self, callback, stream, method, length: int):
|
|
77
|
+
self.wrapper_setattr("_length", length)
|
|
78
|
+
super().__init__(callback, stream, method)
|
|
79
|
+
|
|
80
|
+
def __len__(self):
|
|
81
|
+
return self.wrapper_getattr("_length")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _align_part_size_with_mmap_allocation_granularity(part_size: int) -> int:
|
|
85
|
+
modulo = part_size % mmap.ALLOCATIONGRANULARITY
|
|
86
|
+
if modulo == 0:
|
|
87
|
+
return part_size
|
|
88
|
+
|
|
89
|
+
part_size += mmap.ALLOCATIONGRANULARITY - modulo
|
|
90
|
+
return part_size
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# Can not be less than 5 * 1024 * 1024
|
|
94
|
+
_PART_SIZE_BYTES_FOR_MULTIPART = _align_part_size_with_mmap_allocation_granularity(
|
|
95
|
+
10 * 1024 * 1024
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def decide_file_parts(
|
|
100
|
+
file_path: str,
|
|
101
|
+
multipart_upload_allowed: bool = not ENV_VARS.TFY_ARTIFACTS_DISABLE_MULTIPART_UPLOAD,
|
|
102
|
+
min_file_size_bytes_for_multipart: int = _MIN_BYTES_REQUIRED_FOR_MULTIPART,
|
|
103
|
+
) -> _FileMultiPartInfo:
|
|
104
|
+
file_size = os.path.getsize(file_path)
|
|
105
|
+
if not multipart_upload_allowed or file_size < min_file_size_bytes_for_multipart:
|
|
106
|
+
return _FileMultiPartInfo(1, part_size=file_size, file_size=file_size)
|
|
107
|
+
|
|
108
|
+
ideal_num_parts = math.ceil(file_size / _PART_SIZE_BYTES_FOR_MULTIPART)
|
|
109
|
+
if ideal_num_parts <= _MAX_NUM_PARTS_FOR_MULTIPART:
|
|
110
|
+
return _FileMultiPartInfo(
|
|
111
|
+
ideal_num_parts,
|
|
112
|
+
part_size=_PART_SIZE_BYTES_FOR_MULTIPART,
|
|
113
|
+
file_size=file_size,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
part_size_when_using_max_parts = math.ceil(file_size / _MAX_NUM_PARTS_FOR_MULTIPART)
|
|
117
|
+
part_size_when_using_max_parts = _align_part_size_with_mmap_allocation_granularity(
|
|
118
|
+
part_size_when_using_max_parts
|
|
119
|
+
)
|
|
120
|
+
if part_size_when_using_max_parts > _MAX_PART_SIZE_BYTES_FOR_MULTIPART:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"file {file_path!r} is too big for upload. Multipart chunk"
|
|
123
|
+
f" size {part_size_when_using_max_parts} is higher"
|
|
124
|
+
f" than {_MAX_PART_SIZE_BYTES_FOR_MULTIPART}"
|
|
125
|
+
)
|
|
126
|
+
num_parts = math.ceil(file_size / part_size_when_using_max_parts)
|
|
127
|
+
return _FileMultiPartInfo(
|
|
128
|
+
num_parts, part_size=part_size_when_using_max_parts, file_size=file_size
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _get_s3_compatible_completion_body(multi_parts: List[_PartNumberEtag]) -> str:
|
|
133
|
+
body = "<CompleteMultipartUpload>\n"
|
|
134
|
+
for part in multi_parts:
|
|
135
|
+
body += " <Part>\n"
|
|
136
|
+
body += f" <PartNumber>{part.part_number}</PartNumber>\n"
|
|
137
|
+
body += f" <ETag>{part.etag}</ETag>\n"
|
|
138
|
+
body += " </Part>\n"
|
|
139
|
+
body += "</CompleteMultipartUpload>"
|
|
140
|
+
return body
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _get_azure_blob_completion_body(block_ids: List[str]) -> str:
|
|
144
|
+
body = "<BlockList>\n"
|
|
145
|
+
for block_id in block_ids:
|
|
146
|
+
body += f"<Uncommitted>{block_id}</Uncommitted> "
|
|
147
|
+
body += "</BlockList>"
|
|
148
|
+
return body
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _file_part_upload(
|
|
152
|
+
url: str,
|
|
153
|
+
file_path: str,
|
|
154
|
+
seek: int,
|
|
155
|
+
length: int,
|
|
156
|
+
file_size: int,
|
|
157
|
+
abort_event: Optional[Event] = None,
|
|
158
|
+
method: str = "put",
|
|
159
|
+
exception_class=HttpRequestException,
|
|
160
|
+
):
|
|
161
|
+
def callback(*_, **__):
|
|
162
|
+
if abort_event and abort_event.is_set():
|
|
163
|
+
raise Exception("aborting upload")
|
|
164
|
+
|
|
165
|
+
with open(file_path, "rb") as file:
|
|
166
|
+
with mmap.mmap(
|
|
167
|
+
file.fileno(),
|
|
168
|
+
length=min(file_size - seek, length),
|
|
169
|
+
offset=seek,
|
|
170
|
+
access=mmap.ACCESS_READ,
|
|
171
|
+
) as mapped_file:
|
|
172
|
+
wrapped_file = _CallbackIOWrapperForMultiPartUpload(
|
|
173
|
+
callback, mapped_file, "read", len(mapped_file)
|
|
174
|
+
)
|
|
175
|
+
with cloud_storage_http_request(
|
|
176
|
+
method=method,
|
|
177
|
+
url=url,
|
|
178
|
+
data=wrapped_file,
|
|
179
|
+
exception_class=exception_class,
|
|
180
|
+
) as response:
|
|
181
|
+
augmented_raise_for_status(response, exception_class=exception_class)
|
|
182
|
+
return response
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def s3_compatible_multipart_upload(
|
|
186
|
+
multipart_upload: MultiPartUpload,
|
|
187
|
+
local_file: str,
|
|
188
|
+
multipart_info: _FileMultiPartInfo,
|
|
189
|
+
executor: ThreadPoolExecutor,
|
|
190
|
+
progress_bar: Optional[Progress] = None,
|
|
191
|
+
abort_event: Optional[Event] = None,
|
|
192
|
+
exception_class=HttpRequestException,
|
|
193
|
+
):
|
|
194
|
+
abort_event = abort_event or Event()
|
|
195
|
+
parts = []
|
|
196
|
+
|
|
197
|
+
if progress_bar:
|
|
198
|
+
multi_part_upload_progress = progress_bar.add_task(
|
|
199
|
+
f"[green]Uploading {local_file}:", start=True
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def upload(part_number: int, seek: int) -> None:
|
|
203
|
+
logger.debug(
|
|
204
|
+
"Uploading part %d/%d of %s",
|
|
205
|
+
part_number,
|
|
206
|
+
multipart_info.num_parts,
|
|
207
|
+
local_file,
|
|
208
|
+
)
|
|
209
|
+
response = _file_part_upload(
|
|
210
|
+
url=multipart_upload.part_signed_urls[part_number].signed_url,
|
|
211
|
+
file_path=local_file,
|
|
212
|
+
seek=seek,
|
|
213
|
+
length=multipart_info.part_size,
|
|
214
|
+
file_size=multipart_info.file_size,
|
|
215
|
+
abort_event=abort_event,
|
|
216
|
+
exception_class=exception_class,
|
|
217
|
+
)
|
|
218
|
+
logger.debug(
|
|
219
|
+
"Uploaded part %d/%d of %s",
|
|
220
|
+
part_number,
|
|
221
|
+
multipart_info.num_parts,
|
|
222
|
+
local_file,
|
|
223
|
+
)
|
|
224
|
+
if progress_bar:
|
|
225
|
+
progress_bar.update(
|
|
226
|
+
multi_part_upload_progress,
|
|
227
|
+
advance=multipart_info.part_size,
|
|
228
|
+
total=multipart_info.file_size,
|
|
229
|
+
)
|
|
230
|
+
etag = response.headers["ETag"]
|
|
231
|
+
parts.append(_PartNumberEtag(etag=etag, part_number=part_number + 1))
|
|
232
|
+
|
|
233
|
+
futures: List[Future] = []
|
|
234
|
+
for part_number, seek in enumerate(
|
|
235
|
+
range(0, multipart_info.file_size, multipart_info.part_size)
|
|
236
|
+
):
|
|
237
|
+
future = executor.submit(upload, part_number=part_number, seek=seek)
|
|
238
|
+
futures.append(future)
|
|
239
|
+
|
|
240
|
+
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
|
241
|
+
if len(not_done) > 0:
|
|
242
|
+
abort_event.set()
|
|
243
|
+
for future in not_done:
|
|
244
|
+
future.cancel()
|
|
245
|
+
for future in done:
|
|
246
|
+
if future.exception() is not None:
|
|
247
|
+
raise future.exception()
|
|
248
|
+
|
|
249
|
+
logger.debug("Finalizing multipart upload of %s", local_file)
|
|
250
|
+
parts = sorted(parts, key=lambda part: part.part_number)
|
|
251
|
+
response = requests.post(
|
|
252
|
+
multipart_upload.finalize_signed_url.signed_url,
|
|
253
|
+
data=_get_s3_compatible_completion_body(parts),
|
|
254
|
+
timeout=2 * 60,
|
|
255
|
+
)
|
|
256
|
+
response.raise_for_status()
|
|
257
|
+
logger.debug("Multipart upload of %s completed", local_file)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def azure_multi_part_upload(
|
|
261
|
+
multipart_upload: MultiPartUpload,
|
|
262
|
+
local_file: str,
|
|
263
|
+
multipart_info: _FileMultiPartInfo,
|
|
264
|
+
executor: ThreadPoolExecutor,
|
|
265
|
+
progress_bar: Optional[Progress] = None,
|
|
266
|
+
abort_event: Optional[Event] = None,
|
|
267
|
+
exception_class=HttpRequestException,
|
|
268
|
+
):
|
|
269
|
+
abort_event = abort_event or Event()
|
|
270
|
+
|
|
271
|
+
if progress_bar:
|
|
272
|
+
multi_part_upload_progress = progress_bar.add_task(
|
|
273
|
+
f"[green]Uploading {local_file}:", start=True
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
def upload(part_number: int, seek: int):
|
|
277
|
+
logger.debug(
|
|
278
|
+
"Uploading part %d/%d of %s",
|
|
279
|
+
part_number,
|
|
280
|
+
multipart_info.num_parts,
|
|
281
|
+
local_file,
|
|
282
|
+
)
|
|
283
|
+
_file_part_upload(
|
|
284
|
+
url=multipart_upload.part_signed_urls[part_number].signed_url,
|
|
285
|
+
file_path=local_file,
|
|
286
|
+
seek=seek,
|
|
287
|
+
length=multipart_info.part_size,
|
|
288
|
+
file_size=multipart_info.file_size,
|
|
289
|
+
abort_event=abort_event,
|
|
290
|
+
exception_class=exception_class,
|
|
291
|
+
)
|
|
292
|
+
if progress_bar:
|
|
293
|
+
progress_bar.update(
|
|
294
|
+
multi_part_upload_progress,
|
|
295
|
+
advance=multipart_info.part_size,
|
|
296
|
+
total=multipart_info.file_size,
|
|
297
|
+
)
|
|
298
|
+
logger.debug(
|
|
299
|
+
"Uploaded part %d/%d of %s",
|
|
300
|
+
part_number,
|
|
301
|
+
multipart_info.num_parts,
|
|
302
|
+
local_file,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
futures: List[Future] = []
|
|
306
|
+
for part_number, seek in enumerate(
|
|
307
|
+
range(0, multipart_info.file_size, multipart_info.part_size)
|
|
308
|
+
):
|
|
309
|
+
future = executor.submit(upload, part_number=part_number, seek=seek)
|
|
310
|
+
futures.append(future)
|
|
311
|
+
|
|
312
|
+
done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
|
|
313
|
+
if len(not_done) > 0:
|
|
314
|
+
abort_event.set()
|
|
315
|
+
for future in not_done:
|
|
316
|
+
future.cancel()
|
|
317
|
+
for future in done:
|
|
318
|
+
if future.exception() is not None:
|
|
319
|
+
raise future.exception()
|
|
320
|
+
|
|
321
|
+
logger.debug("Finalizing multipart upload of %s", local_file)
|
|
322
|
+
if multipart_upload.azure_blob_block_ids:
|
|
323
|
+
response = requests.put(
|
|
324
|
+
multipart_upload.finalize_signed_url.signed_url,
|
|
325
|
+
data=_get_azure_blob_completion_body(
|
|
326
|
+
block_ids=multipart_upload.azure_blob_block_ids
|
|
327
|
+
),
|
|
328
|
+
timeout=2 * 60,
|
|
329
|
+
)
|
|
330
|
+
response.raise_for_status()
|
|
331
|
+
logger.debug("Multipart upload of %s completed", local_file)
|
truefoundry/common/utils.py
CHANGED
|
@@ -37,25 +37,25 @@ class _TFYServersConfig(BaseSettings):
|
|
|
37
37
|
mlfoundry_server_url: str
|
|
38
38
|
|
|
39
39
|
@classmethod
|
|
40
|
-
def
|
|
41
|
-
|
|
40
|
+
def from_tfy_host(cls, tfy_host: str) -> "_TFYServersConfig":
|
|
41
|
+
tfy_host = tfy_host.strip("/")
|
|
42
42
|
return cls(
|
|
43
|
-
tenant_host=urlparse(
|
|
44
|
-
servicefoundry_server_url=urljoin(
|
|
45
|
-
mlfoundry_server_url=urljoin(
|
|
43
|
+
tenant_host=urlparse(tfy_host).netloc,
|
|
44
|
+
servicefoundry_server_url=urljoin(tfy_host, API_SERVER_RELATIVE_PATH),
|
|
45
|
+
mlfoundry_server_url=urljoin(tfy_host, MLFOUNDRY_SERVER_RELATIVE_PATH),
|
|
46
46
|
)
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
_tfy_servers_config = None
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
def get_tfy_servers_config(
|
|
52
|
+
def get_tfy_servers_config(tfy_host: str) -> _TFYServersConfig:
|
|
53
53
|
global _tfy_servers_config
|
|
54
54
|
if _tfy_servers_config is None:
|
|
55
55
|
if ENV_VARS.TFY_CLI_LOCAL_DEV_MODE:
|
|
56
56
|
_tfy_servers_config = _TFYServersConfig() # type: ignore[call-arg]
|
|
57
57
|
else:
|
|
58
|
-
_tfy_servers_config = _TFYServersConfig.
|
|
58
|
+
_tfy_servers_config = _TFYServersConfig.from_tfy_host(tfy_host)
|
|
59
59
|
return _tfy_servers_config
|
|
60
60
|
|
|
61
61
|
|
|
@@ -106,11 +106,11 @@ def validate_tfy_host(tfy_host: str) -> None:
|
|
|
106
106
|
|
|
107
107
|
|
|
108
108
|
def resolve_tfy_host(tfy_host: Optional[str] = None) -> str:
|
|
109
|
-
|
|
109
|
+
tfy_host = tfy_host or ENV_VARS.TFY_HOST
|
|
110
|
+
if not tfy_host:
|
|
110
111
|
raise ValueError(
|
|
111
112
|
f"Either `host` should be provided using `--host <value>`, or `{TFY_HOST_ENV_KEY}` env must be set"
|
|
112
113
|
)
|
|
113
|
-
tfy_host = tfy_host or ENV_VARS.TFY_HOST
|
|
114
114
|
tfy_host = tfy_host.strip("/")
|
|
115
115
|
validate_tfy_host(tfy_host)
|
|
116
116
|
return tfy_host
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class TrueFoundryDeprecationWarning(DeprecationWarning):
|
|
5
|
+
pass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def surface_truefoundry_deprecation_warnings() -> None:
|
|
9
|
+
"""Unmute TrueFoundry deprecation warnings."""
|
|
10
|
+
warnings.filterwarnings(
|
|
11
|
+
"default",
|
|
12
|
+
category=TrueFoundryDeprecationWarning,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def suppress_truefoundry_deprecation_warnings() -> None:
|
|
17
|
+
"""Mute TrueFoundry deprecation warnings."""
|
|
18
|
+
warnings.filterwarnings(
|
|
19
|
+
"ignore",
|
|
20
|
+
category=TrueFoundryDeprecationWarning,
|
|
21
|
+
)
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import os
|
|
2
1
|
import shlex
|
|
3
2
|
from typing import Dict, List, Optional
|
|
4
3
|
|
|
@@ -12,7 +11,10 @@ from truefoundry.deploy.builder.constants import (
|
|
|
12
11
|
UV_CONF_BUILDKIT_SECRET_MOUNT,
|
|
13
12
|
UV_CONF_SECRET_MOUNT_AS_ENV,
|
|
14
13
|
)
|
|
15
|
-
from truefoundry.deploy.v2.lib.patched_models import
|
|
14
|
+
from truefoundry.deploy.v2.lib.patched_models import (
|
|
15
|
+
CUDAVersion,
|
|
16
|
+
_resolve_requirements_path,
|
|
17
|
+
)
|
|
16
18
|
|
|
17
19
|
# TODO (chiragjn): Switch to a non-root user inside the container
|
|
18
20
|
|
|
@@ -80,23 +82,6 @@ CUDA_VERSION_TO_IMAGE_TAG: Dict[str, str] = {
|
|
|
80
82
|
}
|
|
81
83
|
|
|
82
84
|
|
|
83
|
-
def resolve_requirements_txt_path(build_configuration: PythonBuild) -> Optional[str]:
|
|
84
|
-
if build_configuration.requirements_path:
|
|
85
|
-
return build_configuration.requirements_path
|
|
86
|
-
|
|
87
|
-
# TODO: what if there is a requirements.txt but user does not wants us to use it.
|
|
88
|
-
possible_requirements_txt_path = os.path.join(
|
|
89
|
-
build_configuration.build_context_path, "requirements.txt"
|
|
90
|
-
)
|
|
91
|
-
|
|
92
|
-
if os.path.isfile(possible_requirements_txt_path):
|
|
93
|
-
return os.path.relpath(
|
|
94
|
-
possible_requirements_txt_path, start=build_configuration.build_context_path
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
return None
|
|
98
|
-
|
|
99
|
-
|
|
100
85
|
def generate_apt_install_command(apt_packages: Optional[List[str]]) -> Optional[str]:
|
|
101
86
|
packages_list = None
|
|
102
87
|
if apt_packages:
|
|
@@ -148,7 +133,6 @@ def generate_uv_pip_install_command(
|
|
|
148
133
|
upgrade_pip_command = "python -m pip install -U pip setuptools wheel"
|
|
149
134
|
uv_mount = f"--mount=from={ENV_VARS.TFY_PYTHON_BUILD_UV_IMAGE_URI},source=/uv,target=/usr/local/bin/uv"
|
|
150
135
|
envs = [
|
|
151
|
-
"UV_SYSTEM_PYTHON=true",
|
|
152
136
|
"UV_LINK_MODE=copy",
|
|
153
137
|
"UV_PYTHON_DOWNLOADS=never",
|
|
154
138
|
"UV_INDEX_STRATEGY=unsafe-best-match",
|
|
@@ -183,7 +167,10 @@ def generate_dockerfile_content(
|
|
|
183
167
|
mount_python_package_manager_conf_secret: bool = False,
|
|
184
168
|
) -> str:
|
|
185
169
|
# TODO (chiragjn): Handle recursive references to other requirements files e.g. `-r requirements-gpu.txt`
|
|
186
|
-
requirements_path =
|
|
170
|
+
requirements_path = _resolve_requirements_path(
|
|
171
|
+
build_context_path=build_configuration.build_context_path,
|
|
172
|
+
requirements_path=build_configuration.requirements_path,
|
|
173
|
+
)
|
|
187
174
|
requirements_destination_path = (
|
|
188
175
|
"/tmp/requirements.txt" if requirements_path else None
|
|
189
176
|
)
|
|
@@ -61,11 +61,11 @@ def deploy_command(
|
|
|
61
61
|
):
|
|
62
62
|
if ctx.invoked_subcommand is not None:
|
|
63
63
|
return
|
|
64
|
-
from truefoundry.
|
|
64
|
+
from truefoundry.common.session import Session
|
|
65
65
|
from truefoundry.deploy.v2.lib.deployable_patched_models import Application
|
|
66
66
|
|
|
67
67
|
try:
|
|
68
|
-
_ =
|
|
68
|
+
_ = Session.new()
|
|
69
69
|
except Exception as e:
|
|
70
70
|
raise ClickException(message=str(e)) from e
|
|
71
71
|
|
|
@@ -126,10 +126,10 @@ def deploy_command(
|
|
|
126
126
|
)
|
|
127
127
|
@handle_exception_wrapper
|
|
128
128
|
def deploy_workflow_command(name: str, file: str, workspace_fqn: str):
|
|
129
|
-
from truefoundry.
|
|
129
|
+
from truefoundry.common.session import Session
|
|
130
130
|
|
|
131
131
|
try:
|
|
132
|
-
_ =
|
|
132
|
+
_ = Session.new()
|
|
133
133
|
except Exception as e:
|
|
134
134
|
raise ClickException(message=str(e)) from e
|
|
135
135
|
|
|
@@ -23,9 +23,9 @@ from truefoundry.common.servicefoundry_client import (
|
|
|
23
23
|
check_min_cli_version,
|
|
24
24
|
session_with_retries,
|
|
25
25
|
)
|
|
26
|
+
from truefoundry.common.session import Session
|
|
26
27
|
from truefoundry.deploy.auto_gen import models as auto_gen_models
|
|
27
28
|
from truefoundry.deploy.io.output_callback import OutputCallBack
|
|
28
|
-
from truefoundry.deploy.lib.auth.servicefoundry_session import ServiceFoundrySession
|
|
29
29
|
from truefoundry.deploy.lib.model.entity import (
|
|
30
30
|
Application,
|
|
31
31
|
CreateDockerRepositoryResponse,
|
|
@@ -77,17 +77,16 @@ def _upload_packaged_code(metadata, package_file):
|
|
|
77
77
|
|
|
78
78
|
|
|
79
79
|
class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
self._session: Optional[ServiceFoundrySession] = None
|
|
80
|
+
def __init__(self, init_session: bool = True, tfy_host: Optional[str] = None):
|
|
81
|
+
self._session: Optional[Session] = None
|
|
83
82
|
if init_session:
|
|
84
|
-
if
|
|
85
|
-
logger.warning("Passed
|
|
86
|
-
self._session =
|
|
87
|
-
|
|
88
|
-
elif not
|
|
89
|
-
raise Exception("Neither session, not
|
|
90
|
-
super().__init__(
|
|
83
|
+
if tfy_host:
|
|
84
|
+
logger.warning(f"Passed tfy_host {tfy_host!r} will be ignored")
|
|
85
|
+
self._session = Session.new()
|
|
86
|
+
tfy_host = self._session.tfy_host
|
|
87
|
+
elif not tfy_host:
|
|
88
|
+
raise Exception("Neither session, not tfy_host provided")
|
|
89
|
+
super().__init__(tfy_host=tfy_host)
|
|
91
90
|
|
|
92
91
|
def _get_header(self):
|
|
93
92
|
if not self._session:
|
|
@@ -249,11 +248,11 @@ class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
|
|
|
249
248
|
}
|
|
250
249
|
logger.debug(json.dumps(data))
|
|
251
250
|
url = f"{self._api_server_url}/{VERSION_PREFIX}/deployment"
|
|
252
|
-
|
|
251
|
+
response = session_with_retries().post(
|
|
253
252
|
url, json=data, headers=self._get_header()
|
|
254
253
|
)
|
|
255
|
-
|
|
256
|
-
return Deployment.parse_obj(
|
|
254
|
+
response_data = request_handling(response)
|
|
255
|
+
return Deployment.parse_obj(response_data["deployment"])
|
|
257
256
|
|
|
258
257
|
def _get_log_print_line(self, log: dict):
|
|
259
258
|
timestamp = int(log["time"]) / 1e6
|
|
@@ -180,7 +180,7 @@ def trigger_job(
|
|
|
180
180
|
params=params if params else None,
|
|
181
181
|
)
|
|
182
182
|
jobRunName = result.jobRunName
|
|
183
|
-
previous_runs_url = f"{client.
|
|
183
|
+
previous_runs_url = f"{client.tfy_host.strip('/')}/deployments/{application_info.id}?tab=previousRuns"
|
|
184
184
|
logger.info(
|
|
185
185
|
f"{message}.\n"
|
|
186
186
|
f"You can check the status of your job run at {previous_runs_url} with jobRunName: {jobRunName}"
|
|
@@ -259,6 +259,6 @@ def trigger_workflow(application_fqn: str, inputs: Optional[Dict[str, Any]] = No
|
|
|
259
259
|
)
|
|
260
260
|
logger.info(f"Started Execution for Workflow: {application_info.name}")
|
|
261
261
|
executions_page = (
|
|
262
|
-
f"{client.
|
|
262
|
+
f"{client.tfy_host.strip('/')}/deployments/{application_info.id}?tab=executions"
|
|
263
263
|
)
|
|
264
264
|
logger.info(f"You can check the executions at {executions_page}")
|
|
@@ -32,7 +32,7 @@ def create_workspace(
|
|
|
32
32
|
resources=workspace_resources,
|
|
33
33
|
)
|
|
34
34
|
|
|
35
|
-
url = f"{client.
|
|
35
|
+
url = f"{client.tfy_host.strip('/')}/workspaces"
|
|
36
36
|
logger.info(
|
|
37
37
|
"You can find your workspace: '%s' on the dashboard: %s", workspace.name, url
|
|
38
38
|
)
|
|
@@ -68,7 +68,7 @@ def login(
|
|
|
68
68
|
if api_key:
|
|
69
69
|
token = Token(access_token=api_key, refresh_token=None)
|
|
70
70
|
else:
|
|
71
|
-
auth_service = AuthServiceClient.
|
|
71
|
+
auth_service = AuthServiceClient.from_tfy_host(tfy_host=host)
|
|
72
72
|
# interactive login
|
|
73
73
|
token = _login_with_device_code(base_url=host, auth_service=auth_service)
|
|
74
74
|
|