truefoundry 0.11.9rc1__py3-none-any.whl → 0.11.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/common/storage_provider_utils.py +35 -10
- truefoundry/deploy/_autogen/models.py +39 -24
- truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py +3 -1
- truefoundry/deploy/cli/commands/patch_command.py +12 -2
- truefoundry/deploy/v2/lib/patched_models.py +9 -0
- truefoundry/ml/artifact/truefoundry_artifact_repo.py +29 -12
- truefoundry/ml/log_types/artifacts/artifact.py +3 -2
- truefoundry/ml/log_types/artifacts/model.py +1 -0
- {truefoundry-0.11.9rc1.dist-info → truefoundry-0.11.11.dist-info}/METADATA +1 -1
- {truefoundry-0.11.9rc1.dist-info → truefoundry-0.11.11.dist-info}/RECORD +12 -12
- {truefoundry-0.11.9rc1.dist-info → truefoundry-0.11.11.dist-info}/WHEEL +0 -0
- {truefoundry-0.11.9rc1.dist-info → truefoundry-0.11.11.dist-info}/entry_points.txt +0 -0
|
@@ -22,6 +22,23 @@ from truefoundry.pydantic_v1 import BaseModel
|
|
|
22
22
|
logger = logging.getLogger("truefoundry")
|
|
23
23
|
|
|
24
24
|
|
|
25
|
+
def truncate_path_for_progress(
|
|
26
|
+
path: str, max_length: int, relpath: bool = False
|
|
27
|
+
) -> str:
|
|
28
|
+
if relpath:
|
|
29
|
+
path = os.path.relpath(path)
|
|
30
|
+
if len(path) <= max_length:
|
|
31
|
+
return path
|
|
32
|
+
parts = path.split(os.sep)
|
|
33
|
+
result = parts[-1] # start with filename
|
|
34
|
+
i = len(parts) - 2
|
|
35
|
+
# keep prepending directories until adding more would exceed max_len - 3
|
|
36
|
+
while i >= 0 and len(result) + len(parts[i]) + 1 <= max_length - 3:
|
|
37
|
+
result = parts[i] + os.sep + result
|
|
38
|
+
i -= 1
|
|
39
|
+
return "..." + os.sep + result
|
|
40
|
+
|
|
41
|
+
|
|
25
42
|
class MultiPartUploadStorageProvider(str, Enum):
|
|
26
43
|
S3_COMPATIBLE = "S3_COMPATIBLE"
|
|
27
44
|
AZURE_BLOB = "AZURE_BLOB"
|
|
@@ -182,7 +199,7 @@ def _file_part_upload(
|
|
|
182
199
|
return response
|
|
183
200
|
|
|
184
201
|
|
|
185
|
-
def s3_compatible_multipart_upload(
|
|
202
|
+
def s3_compatible_multipart_upload( # noqa: C901
|
|
186
203
|
multipart_upload: MultiPartUpload,
|
|
187
204
|
local_file: str,
|
|
188
205
|
multipart_info: _FileMultiPartInfo,
|
|
@@ -190,13 +207,15 @@ def s3_compatible_multipart_upload(
|
|
|
190
207
|
progress_bar: Optional[Progress] = None,
|
|
191
208
|
abort_event: Optional[Event] = None,
|
|
192
209
|
exception_class=HttpRequestException,
|
|
193
|
-
):
|
|
210
|
+
) -> None:
|
|
194
211
|
abort_event = abort_event or Event()
|
|
195
212
|
parts = []
|
|
196
213
|
|
|
197
|
-
if progress_bar:
|
|
214
|
+
if progress_bar is not None:
|
|
198
215
|
multi_part_upload_progress = progress_bar.add_task(
|
|
199
|
-
f"[green]
|
|
216
|
+
f"[green]⬆ {truncate_path_for_progress(local_file, 64, relpath=True)}",
|
|
217
|
+
start=True,
|
|
218
|
+
visible=True,
|
|
200
219
|
)
|
|
201
220
|
|
|
202
221
|
def upload(part_number: int, seek: int) -> None:
|
|
@@ -221,7 +240,7 @@ def s3_compatible_multipart_upload(
|
|
|
221
240
|
multipart_info.num_parts,
|
|
222
241
|
local_file,
|
|
223
242
|
)
|
|
224
|
-
if progress_bar:
|
|
243
|
+
if progress_bar is not None:
|
|
225
244
|
progress_bar.update(
|
|
226
245
|
multi_part_upload_progress,
|
|
227
246
|
advance=multipart_info.part_size,
|
|
@@ -254,10 +273,12 @@ def s3_compatible_multipart_upload(
|
|
|
254
273
|
timeout=2 * 60,
|
|
255
274
|
)
|
|
256
275
|
response.raise_for_status()
|
|
276
|
+
if progress_bar is not None:
|
|
277
|
+
progress_bar.refresh()
|
|
257
278
|
logger.debug("Multipart upload of %s completed", local_file)
|
|
258
279
|
|
|
259
280
|
|
|
260
|
-
def azure_multi_part_upload(
|
|
281
|
+
def azure_multi_part_upload( # noqa: C901
|
|
261
282
|
multipart_upload: MultiPartUpload,
|
|
262
283
|
local_file: str,
|
|
263
284
|
multipart_info: _FileMultiPartInfo,
|
|
@@ -265,12 +286,14 @@ def azure_multi_part_upload(
|
|
|
265
286
|
progress_bar: Optional[Progress] = None,
|
|
266
287
|
abort_event: Optional[Event] = None,
|
|
267
288
|
exception_class=HttpRequestException,
|
|
268
|
-
):
|
|
289
|
+
) -> None:
|
|
269
290
|
abort_event = abort_event or Event()
|
|
270
291
|
|
|
271
|
-
if progress_bar:
|
|
292
|
+
if progress_bar is not None:
|
|
272
293
|
multi_part_upload_progress = progress_bar.add_task(
|
|
273
|
-
f"[green]
|
|
294
|
+
f"[green]⬆ {truncate_path_for_progress(local_file, 64, relpath=True)}",
|
|
295
|
+
start=True,
|
|
296
|
+
visible=True,
|
|
274
297
|
)
|
|
275
298
|
|
|
276
299
|
def upload(part_number: int, seek: int):
|
|
@@ -289,7 +312,7 @@ def azure_multi_part_upload(
|
|
|
289
312
|
abort_event=abort_event,
|
|
290
313
|
exception_class=exception_class,
|
|
291
314
|
)
|
|
292
|
-
if progress_bar:
|
|
315
|
+
if progress_bar is not None:
|
|
293
316
|
progress_bar.update(
|
|
294
317
|
multi_part_upload_progress,
|
|
295
318
|
advance=multipart_info.part_size,
|
|
@@ -328,4 +351,6 @@ def azure_multi_part_upload(
|
|
|
328
351
|
timeout=2 * 60,
|
|
329
352
|
)
|
|
330
353
|
response.raise_for_status()
|
|
354
|
+
if progress_bar is not None:
|
|
355
|
+
progress_bar.refresh()
|
|
331
356
|
logger.debug("Multipart upload of %s completed", local_file)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# generated by datamodel-codegen:
|
|
2
2
|
# filename: application.json
|
|
3
|
-
# timestamp: 2025-
|
|
3
|
+
# timestamp: 2025-09-09T12:09:08+00:00
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
@@ -504,34 +504,49 @@ class Profile(str, Enum):
|
|
|
504
504
|
"""
|
|
505
505
|
Name of the MIG profile to use. One of the following based on gpu type
|
|
506
506
|
Please refer to https://docs.nvidia.com/datacenter/tesla/mig-user-guide/#supported-mig-profiles for more details
|
|
507
|
-
|
|
508
|
-
A100
|
|
509
|
-
H100
|
|
510
|
-
H100
|
|
507
|
+
A30 - [1g.6gb, 2g.12gb, 4g.24gb]
|
|
508
|
+
A100 40 GB - [1g.5gb, 1g.10gb, 2g.10gb, 3g.20gb, 4g.20gb, 7g.40gb]
|
|
509
|
+
A100 80 GB / H100 80 GB - [1g.10gb, 1g.20gb, 2g.20gb, 3g.40gb, 4g.40gb, 7g.80gb]
|
|
510
|
+
H100 94 GB - [1g.12gb, 1g.24gb, 2g.24gb, 3g.47gb, 4g.47gb, 7g.94gb]
|
|
511
|
+
H100 96 GB - [1g.12gb, 1g.24gb, 2g.24gb, 3g.48gb, 4g.48gb, 7g.96gb]
|
|
511
512
|
H200 141 GB - [1g.18gb, 1g.35gb, 2g.35gb, 3g.71gb, 4g.71gb]
|
|
513
|
+
B200 180 GB - [1g.23gb, 1g.45gb, 2g.45gb, 3g.90gb, 4g.90gb, 7g.180gb]
|
|
512
514
|
"""
|
|
513
515
|
|
|
516
|
+
field_1g_6gb = "1g.6gb"
|
|
517
|
+
field_2g_12gb = "2g.12gb"
|
|
514
518
|
field_1g_5gb = "1g.5gb"
|
|
515
519
|
field_1g_10gb = "1g.10gb"
|
|
516
|
-
field_1g_12gb = "1g.12gb"
|
|
517
|
-
field_1g_18gb = "1g.18gb"
|
|
518
|
-
field_1g_20gb = "1g.20gb"
|
|
519
|
-
field_1g_24gb = "1g.24gb"
|
|
520
|
-
field_1g_35gb = "1g.35gb"
|
|
521
520
|
field_2g_10gb = "2g.10gb"
|
|
522
|
-
field_2g_20gb = "2g.20gb"
|
|
523
|
-
field_2g_24gb = "2g.24gb"
|
|
524
|
-
field_2g_35gb = "2g.35gb"
|
|
525
521
|
field_3g_20gb = "3g.20gb"
|
|
526
|
-
field_3g_40gb = "3g.40gb"
|
|
527
|
-
field_3g_47gb = "3g.47gb"
|
|
528
|
-
field_3g_48gb = "3g.48gb"
|
|
529
|
-
field_3g_71gb = "3g.71gb"
|
|
530
522
|
field_4g_20gb = "4g.20gb"
|
|
523
|
+
field_1g_20gb = "1g.20gb"
|
|
524
|
+
field_2g_20gb = "2g.20gb"
|
|
525
|
+
field_3g_40gb = "3g.40gb"
|
|
531
526
|
field_4g_40gb = "4g.40gb"
|
|
527
|
+
field_1g_12gb = "1g.12gb"
|
|
528
|
+
field_1g_24gb = "1g.24gb"
|
|
529
|
+
field_2g_24gb = "2g.24gb"
|
|
530
|
+
field_3g_47gb = "3g.47gb"
|
|
532
531
|
field_4g_47gb = "4g.47gb"
|
|
532
|
+
field_3g_48gb = "3g.48gb"
|
|
533
533
|
field_4g_48gb = "4g.48gb"
|
|
534
|
+
field_1g_18gb = "1g.18gb"
|
|
535
|
+
field_1g_35gb = "1g.35gb"
|
|
536
|
+
field_2g_35gb = "2g.35gb"
|
|
537
|
+
field_3g_71gb = "3g.71gb"
|
|
534
538
|
field_4g_71gb = "4g.71gb"
|
|
539
|
+
field_1g_23gb = "1g.23gb"
|
|
540
|
+
field_1g_45gb = "1g.45gb"
|
|
541
|
+
field_2g_45gb = "2g.45gb"
|
|
542
|
+
field_3g_90gb = "3g.90gb"
|
|
543
|
+
field_4g_90gb = "4g.90gb"
|
|
544
|
+
field_4g_24gb = "4g.24gb"
|
|
545
|
+
field_7g_40gb = "7g.40gb"
|
|
546
|
+
field_7g_80gb = "7g.80gb"
|
|
547
|
+
field_7g_94gb = "7g.94gb"
|
|
548
|
+
field_7g_96gb = "7g.96gb"
|
|
549
|
+
field_7g_180gb = "7g.180gb"
|
|
535
550
|
|
|
536
551
|
|
|
537
552
|
class NvidiaMIGGPU(BaseModel):
|
|
@@ -542,7 +557,7 @@ class NvidiaMIGGPU(BaseModel):
|
|
|
542
557
|
)
|
|
543
558
|
profile: Profile = Field(
|
|
544
559
|
...,
|
|
545
|
-
description="Name of the MIG profile to use. One of the following based on gpu type\nPlease refer to https://docs.nvidia.com/datacenter/tesla/mig-user-guide/#supported-mig-profiles for more details\nA100 40 GB - [1g.5gb, 1g.10gb, 2g.10gb, 3g.20gb, 4g.20gb]\nA100 80 GB / H100 80 GB - [1g.10gb, 1g.20gb, 2g.20gb, 3g.40gb, 4g.40gb]\nH100 94 GB - [1g.12gb, 1g.24gb, 2g.24gb, 3g.47gb, 4g.47gb]\nH100 96 GB - [1g.12gb, 1g.24gb, 2g.24gb, 3g.48gb, 4g.48gb]\nH200 141 GB - [1g.18gb, 1g.35gb, 2g.35gb, 3g.71gb, 4g.71gb]",
|
|
560
|
+
description="Name of the MIG profile to use. One of the following based on gpu type\nPlease refer to https://docs.nvidia.com/datacenter/tesla/mig-user-guide/#supported-mig-profiles for more details\nA30 - [1g.6gb, 2g.12gb, 4g.24gb]\nA100 40 GB - [1g.5gb, 1g.10gb, 2g.10gb, 3g.20gb, 4g.20gb, 7g.40gb]\nA100 80 GB / H100 80 GB - [1g.10gb, 1g.20gb, 2g.20gb, 3g.40gb, 4g.40gb, 7g.80gb]\nH100 94 GB - [1g.12gb, 1g.24gb, 2g.24gb, 3g.47gb, 4g.47gb, 7g.94gb]\nH100 96 GB - [1g.12gb, 1g.24gb, 2g.24gb, 3g.48gb, 4g.48gb, 7g.96gb]\nH200 141 GB - [1g.18gb, 1g.35gb, 2g.35gb, 3g.71gb, 4g.71gb]\nB200 180 GB - [1g.23gb, 1g.45gb, 2g.45gb, 3g.90gb, 4g.90gb, 7g.180gb]",
|
|
546
561
|
)
|
|
547
562
|
|
|
548
563
|
|
|
@@ -552,7 +567,7 @@ class NvidiaTimeslicingGPU(BaseModel):
|
|
|
552
567
|
None,
|
|
553
568
|
description="Name of the Nvidia GPU. One of [P4, P100, V100, T4, A10G, A100_40GB, A100_80GB]\nThis field is required for Node Selector and can be ignored in Nodepool Selector.\nOne instance of the card contains the following amount of memory -\nP4: 8 GB, P100: 16 GB, V100: 16 GB, T4: 16 GB, A10G: 24 GB, A100_40GB: 40GB, A100_80GB: 80 GB",
|
|
554
569
|
)
|
|
555
|
-
gpu_memory: conint(ge=1, le=
|
|
570
|
+
gpu_memory: conint(ge=1, le=400000) = Field(
|
|
556
571
|
...,
|
|
557
572
|
description="Amount of GPU memory (in MB) to allocate. Please note, this limit is not being enforced today but will be in future. Applications are expected to operate in co-opertative mode",
|
|
558
573
|
)
|
|
@@ -681,11 +696,11 @@ class Resources(BaseModel):
|
|
|
681
696
|
0.5,
|
|
682
697
|
description="CPU limit beyond which the usage cannot be exceeded. 1 CPU means 1 CPU core. Fractional CPU can be requested\nlike `0.5`. CPU limit should be >= cpu request.",
|
|
683
698
|
)
|
|
684
|
-
memory_request: conint(ge=1, le=
|
|
699
|
+
memory_request: conint(ge=1, le=4000000) = Field(
|
|
685
700
|
200,
|
|
686
701
|
description="Requested memory which determines the minimum cost incurred. The unit of memory is in megabytes(MB).\nSo 1 means 1 MB and 2000 means 2GB.",
|
|
687
702
|
)
|
|
688
|
-
memory_limit: conint(ge=1, le=
|
|
703
|
+
memory_limit: conint(ge=1, le=4000000) = Field(
|
|
689
704
|
500,
|
|
690
705
|
description="Memory limit after which the application will be killed with an OOM error. The unit of memory is\nin megabytes(MB). So 1 means 1 MB and 2000 means 2GB. MemoryLimit should be greater than memory request.",
|
|
691
706
|
)
|
|
@@ -697,7 +712,7 @@ class Resources(BaseModel):
|
|
|
697
712
|
2000,
|
|
698
713
|
description="Disk storage limit. The unit of memory is in megabytes(MB). Exceeding this limit will result in eviction.\nIt should be greater than the request. This is ephemeral storage and will be wiped out on pod restarts or eviction",
|
|
699
714
|
)
|
|
700
|
-
shared_memory_size: Optional[conint(ge=64, le=
|
|
715
|
+
shared_memory_size: Optional[conint(ge=64, le=4000000)] = Field(
|
|
701
716
|
None,
|
|
702
717
|
description="Define the shared memory requirements for your workload. Machine learning libraries like Pytorch can use Shared Memory\nfor inter-process communication. If you use this, we will mount a `tmpfs` backed volume at the `/dev/shm` directory.\nAny usage will also count against the workload's memory limit (`resources.memory_limit`) along with your workload's memory usage.\nIf the overall usage goes above `resources.memory_limit` the user process may get killed.\nShared Memory Size cannot be more than the defined Memory Limit for the workload.",
|
|
703
718
|
)
|
|
@@ -1236,7 +1251,7 @@ class ContainerTaskConfig(BaseModel):
|
|
|
1236
1251
|
description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
|
|
1237
1252
|
)
|
|
1238
1253
|
resources: Optional[Resources] = None
|
|
1239
|
-
mounts: Optional[List[VolumeMount]] = Field(
|
|
1254
|
+
mounts: Optional[List[Union[SecretMount, StringDataMount, VolumeMount]]] = Field(
|
|
1240
1255
|
None, description="Configure data to be mounted to Workflow pod(s) as a volume."
|
|
1241
1256
|
)
|
|
1242
1257
|
service_account: Optional[str] = Field(None, description="")
|
|
@@ -1636,7 +1651,7 @@ class PySparkTaskConfig(BaseModel):
|
|
|
1636
1651
|
None,
|
|
1637
1652
|
description="Configure environment variables to be injected in the task either as plain text or secrets. [Docs](https://docs.truefoundry.com/docs/env-variables)",
|
|
1638
1653
|
)
|
|
1639
|
-
mounts: Optional[List[
|
|
1654
|
+
mounts: Optional[List[VolumeMount]] = Field(
|
|
1640
1655
|
None, description="Configure data to be mounted to Workflow pod(s) as a volume."
|
|
1641
1656
|
)
|
|
1642
1657
|
service_account: Optional[str] = Field(None, description="")
|
|
@@ -80,7 +80,9 @@ CUDA_VERSION_TO_IMAGE_TAG: Dict[str, str] = {
|
|
|
80
80
|
# From 12.4+ onwards, the image tags drop the cudnn version
|
|
81
81
|
CUDAVersion.CUDA_12_4_CUDNN9.value: "12.4.1-cudnn-runtime-ubuntu22.04",
|
|
82
82
|
CUDAVersion.CUDA_12_5_CUDNN9.value: "12.5.1-cudnn-runtime-ubuntu22.04",
|
|
83
|
-
CUDAVersion.CUDA_12_6_CUDNN9.value: "12.6.
|
|
83
|
+
CUDAVersion.CUDA_12_6_CUDNN9.value: "12.6.3-cudnn-runtime-ubuntu22.04",
|
|
84
|
+
CUDAVersion.CUDA_12_8_CUDNN9.value: "12.8.1-cudnn-runtime-ubuntu22.04",
|
|
85
|
+
CUDAVersion.CUDA_12_9_CUDNN9.value: "12.9.1-cudnn-runtime-ubuntu22.04",
|
|
84
86
|
}
|
|
85
87
|
|
|
86
88
|
|
|
@@ -58,9 +58,19 @@ def patch_command(
|
|
|
58
58
|
]
|
|
59
59
|
if output_file:
|
|
60
60
|
with open(output_file, "w") as fd:
|
|
61
|
-
subprocess.run(
|
|
61
|
+
p = subprocess.run(
|
|
62
|
+
yq_command, stdout=fd, stderr=subprocess.PIPE, check=False
|
|
63
|
+
)
|
|
64
|
+
if p.returncode != 0:
|
|
65
|
+
stderr = p.stderr.decode("UTF-8")
|
|
66
|
+
raise Exception(f"Failed to patch yaml file: {stderr}")
|
|
62
67
|
else:
|
|
63
|
-
p = subprocess.run(
|
|
68
|
+
p = subprocess.run(
|
|
69
|
+
yq_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False
|
|
70
|
+
)
|
|
71
|
+
if p.returncode != 0:
|
|
72
|
+
stderr = p.stderr.decode("UTF-8")
|
|
73
|
+
raise Exception(f"Failed to patch yaml file: {stderr}")
|
|
64
74
|
output = p.stdout.decode("UTF-8")
|
|
65
75
|
callback = RichOutputCallBack()
|
|
66
76
|
callback.print_line(output)
|
|
@@ -91,6 +91,8 @@ class CUDAVersion(str, enum.Enum):
|
|
|
91
91
|
CUDA_12_4_CUDNN9 = "12.4-cudnn9"
|
|
92
92
|
CUDA_12_5_CUDNN9 = "12.5-cudnn9"
|
|
93
93
|
CUDA_12_6_CUDNN9 = "12.6-cudnn9"
|
|
94
|
+
CUDA_12_8_CUDNN9 = "12.8-cudnn9"
|
|
95
|
+
CUDA_12_9_CUDNN9 = "12.9-cudnn9"
|
|
94
96
|
|
|
95
97
|
|
|
96
98
|
class GPUType(str, enum.Enum):
|
|
@@ -99,12 +101,19 @@ class GPUType(str, enum.Enum):
|
|
|
99
101
|
V100 = "V100"
|
|
100
102
|
T4 = "T4"
|
|
101
103
|
A10G = "A10G"
|
|
104
|
+
A10_4GB = "A10_4GB"
|
|
105
|
+
A10_8GB = "A10_8GB"
|
|
106
|
+
A10_12GB = "A10_12GB"
|
|
107
|
+
A10_24GB = "A10_24GB"
|
|
102
108
|
A100_40GB = "A100_40GB"
|
|
103
109
|
A100_80GB = "A100_80GB"
|
|
104
110
|
L4 = "L4"
|
|
111
|
+
L40S = "L40S"
|
|
105
112
|
H100_80GB = "H100_80GB"
|
|
106
113
|
H100_94GB = "H100_94GB"
|
|
114
|
+
H100_96GB = "H100_96GB"
|
|
107
115
|
H200 = "H200"
|
|
116
|
+
B200 = "B200"
|
|
108
117
|
|
|
109
118
|
|
|
110
119
|
class TPUType(str, enum.Enum):
|
|
@@ -40,6 +40,7 @@ from truefoundry.common.storage_provider_utils import (
|
|
|
40
40
|
azure_multi_part_upload,
|
|
41
41
|
decide_file_parts,
|
|
42
42
|
s3_compatible_multipart_upload,
|
|
43
|
+
truncate_path_for_progress,
|
|
43
44
|
)
|
|
44
45
|
from truefoundry.ml._autogen.client import ( # type: ignore[attr-defined]
|
|
45
46
|
ApiClient,
|
|
@@ -125,14 +126,14 @@ def _signed_url_upload_file(
|
|
|
125
126
|
augmented_raise_for_status(response, exception_class=MlFoundryException) # type: ignore
|
|
126
127
|
return
|
|
127
128
|
|
|
128
|
-
|
|
129
|
-
f"[green]
|
|
129
|
+
task_id = progress_bar.add_task(
|
|
130
|
+
f"[green]⬆ {truncate_path_for_progress(local_file, 64, relpath=True)}",
|
|
131
|
+
start=True,
|
|
132
|
+
visible=True,
|
|
130
133
|
)
|
|
131
134
|
|
|
132
135
|
def callback(length):
|
|
133
|
-
progress_bar.update(
|
|
134
|
-
task_progress_bar, advance=length, total=os.stat(local_file).st_size
|
|
135
|
-
)
|
|
136
|
+
progress_bar.update(task_id, advance=length, total=os.stat(local_file).st_size)
|
|
136
137
|
if abort_event and abort_event.is_set():
|
|
137
138
|
raise Exception("aborting upload")
|
|
138
139
|
|
|
@@ -147,6 +148,9 @@ def _signed_url_upload_file(
|
|
|
147
148
|
) as response:
|
|
148
149
|
augmented_raise_for_status(response, exception_class=MlFoundryException) # type: ignore
|
|
149
150
|
|
|
151
|
+
if progress_bar is not None:
|
|
152
|
+
progress_bar.refresh()
|
|
153
|
+
|
|
150
154
|
|
|
151
155
|
def _download_file_using_http_uri(
|
|
152
156
|
http_uri,
|
|
@@ -167,11 +171,18 @@ def _download_file_using_http_uri(
|
|
|
167
171
|
exception_class=MlFoundryException, # type: ignore
|
|
168
172
|
) as response:
|
|
169
173
|
augmented_raise_for_status(response, exception_class=MlFoundryException) # type: ignore
|
|
170
|
-
file_size = int(response.headers.get("Content-Length",
|
|
174
|
+
file_size = int(response.headers.get("Content-Length", -1))
|
|
175
|
+
if file_size == 0 and callback:
|
|
176
|
+
# special case for empty files
|
|
177
|
+
callback(1, 1)
|
|
178
|
+
file_size = file_size if file_size > 0 else 0
|
|
171
179
|
with open(download_path, "wb") as output_file:
|
|
172
180
|
for chunk in response.iter_content(chunk_size=chunk_size):
|
|
173
181
|
if callback:
|
|
174
182
|
callback(len(chunk), file_size)
|
|
183
|
+
print(
|
|
184
|
+
f"Sent callback for {download_path} {len(chunk)} / {file_size}"
|
|
185
|
+
)
|
|
175
186
|
if not chunk:
|
|
176
187
|
break
|
|
177
188
|
output_file.write(chunk)
|
|
@@ -371,7 +382,7 @@ class MlFoundryArtifactsRepository:
|
|
|
371
382
|
is MultiPartUploadStorageProvider.AZURE_BLOB
|
|
372
383
|
):
|
|
373
384
|
azure_multi_part_upload(
|
|
374
|
-
multipart_upload=MultiPartUpload.parse_obj(multipart_upload.
|
|
385
|
+
multipart_upload=MultiPartUpload.parse_obj(multipart_upload.dict()),
|
|
375
386
|
local_file=local_file,
|
|
376
387
|
executor=executor,
|
|
377
388
|
multipart_info=multipart_info,
|
|
@@ -658,15 +669,18 @@ class MlFoundryArtifactsRepository:
|
|
|
658
669
|
logger.info("Downloading %s to %s", remote_file_path, local_path)
|
|
659
670
|
|
|
660
671
|
if progress_bar is not None:
|
|
661
|
-
|
|
662
|
-
f"[green]
|
|
672
|
+
task_id = progress_bar.add_task(
|
|
673
|
+
f"[green]⬇ {truncate_path_for_progress(remote_file_path, 64)}",
|
|
674
|
+
start=True,
|
|
675
|
+
visible=True,
|
|
663
676
|
)
|
|
664
677
|
|
|
665
|
-
def callback(
|
|
678
|
+
def callback(chunk_size: int, total_file_size: int):
|
|
679
|
+
nonlocal task_id
|
|
666
680
|
if progress_bar is not None:
|
|
667
681
|
progress_bar.update(
|
|
668
|
-
|
|
669
|
-
advance=
|
|
682
|
+
task_id,
|
|
683
|
+
advance=chunk_size,
|
|
670
684
|
total=total_file_size,
|
|
671
685
|
)
|
|
672
686
|
if abort_event and abort_event.is_set():
|
|
@@ -677,6 +691,9 @@ class MlFoundryArtifactsRepository:
|
|
|
677
691
|
download_path=local_path,
|
|
678
692
|
callback=callback,
|
|
679
693
|
)
|
|
694
|
+
|
|
695
|
+
if progress_bar is not None:
|
|
696
|
+
progress_bar.refresh()
|
|
680
697
|
logger.debug("Downloaded %s to %s", remote_file_path, local_path)
|
|
681
698
|
|
|
682
699
|
def _download_artifact(
|
|
@@ -347,10 +347,11 @@ class ArtifactVersion:
|
|
|
347
347
|
artifact_version.download(path="<your-desired-download-path>")
|
|
348
348
|
```
|
|
349
349
|
"""
|
|
350
|
-
|
|
350
|
+
download_info = self._download(
|
|
351
351
|
path=path, overwrite=overwrite, progress=progress
|
|
352
352
|
)
|
|
353
|
-
|
|
353
|
+
logger.info("Downloaded artifact contents to %s", download_info.download_dir)
|
|
354
|
+
return download_info.download_dir
|
|
354
355
|
|
|
355
356
|
def delete(self) -> bool:
|
|
356
357
|
"""
|
|
@@ -48,13 +48,13 @@ truefoundry/common/exceptions.py,sha256=jkU0N7hV_P-EhXeud4I5vuB9glXXZSWPf8LcH04m
|
|
|
48
48
|
truefoundry/common/request_utils.py,sha256=e9qrAQ1MutU7JALDKcucmNd0KQEVBqgW3yx0w1zeHIU,5700
|
|
49
49
|
truefoundry/common/servicefoundry_client.py,sha256=2fYhdVPSvLXz5C5tosOq86JD8WM3IRUIy1VO9deDxZI,3340
|
|
50
50
|
truefoundry/common/session.py,sha256=d9l3TEBpqVP4mr4mTGY1qVxc815skzMlNNdw14otg34,2923
|
|
51
|
-
truefoundry/common/storage_provider_utils.py,sha256=
|
|
51
|
+
truefoundry/common/storage_provider_utils.py,sha256=jX9maCtWusZx63324cmPxHiGeIluZZzbDaJ1QYTIhAw,11856
|
|
52
52
|
truefoundry/common/types.py,sha256=BMJFCsR1lPJAw66IQBSvLyV4I6o_x5oj78gVsUa9si8,188
|
|
53
53
|
truefoundry/common/utils.py,sha256=P0FuAadoJGdpieUORLSN-PiFnkyoGO-K2cS4OPITBWg,6714
|
|
54
54
|
truefoundry/common/warnings.py,sha256=xDMhR_-ZGC40Ycaj6nlFb5MYPexn8WbKCHd4FlflTXQ,705
|
|
55
55
|
truefoundry/deploy/__init__.py,sha256=sP-6Nv-_uV2o3knWcNSGV07j_Hkq0lfUkfZffBg-Hfo,2874
|
|
56
56
|
truefoundry/deploy/python_deploy_codegen.py,sha256=WwP6bIzFoLpF7J2Bgef2HMSIeefJ8TWtSv4hXNycEzQ,8872
|
|
57
|
-
truefoundry/deploy/_autogen/models.py,sha256=
|
|
57
|
+
truefoundry/deploy/_autogen/models.py,sha256=d1FjNfni3L3Az8d0zjT51N-TWSrW22GFC2wq8hFVoRo,76655
|
|
58
58
|
truefoundry/deploy/builder/__init__.py,sha256=VR07ZB7ziONEBbVgg1JdRTWY7t4qJjJTMhc2VodXYdA,5036
|
|
59
59
|
truefoundry/deploy/builder/constants.py,sha256=amUkHoHvVKzGv0v_knfiioRuKiJM0V0xW0diERgWiI0,508
|
|
60
60
|
truefoundry/deploy/builder/docker_service.py,sha256=sm7GWeIqyrKaZpxskdLejZlsxcZnM3BTDJr6orvPN4E,3948
|
|
@@ -64,7 +64,7 @@ truefoundry/deploy/builder/builders/dockerfile.py,sha256=XMbMlPUTMPCyaHl7jJQY1OD
|
|
|
64
64
|
truefoundry/deploy/builder/builders/tfy_notebook_buildpack/__init__.py,sha256=RGWGqY8xOF7vycUPJd10N7ZzahWv24lO0anrOPtLuDU,1796
|
|
65
65
|
truefoundry/deploy/builder/builders/tfy_notebook_buildpack/dockerfile_template.py,sha256=rQgdvKmAT9HArVW4TAG5yd2QTKRs3S5LJ9RQbc_EkHE,2518
|
|
66
66
|
truefoundry/deploy/builder/builders/tfy_python_buildpack/__init__.py,sha256=_fjqHKn80qKi68SAMMALge7_A6e1sTsQWichw8uoGIw,2025
|
|
67
|
-
truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=
|
|
67
|
+
truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py,sha256=bpYW8DN-fgRw85Vo3kSpigbjEkX4lELoI_iBXHmg_4Q,6856
|
|
68
68
|
truefoundry/deploy/builder/builders/tfy_spark_buildpack/__init__.py,sha256=NEPlM6_vTVxp4ITa18B8DBbgYCn1q5d8be21lbgu5oY,2888
|
|
69
69
|
truefoundry/deploy/builder/builders/tfy_spark_buildpack/dockerfile_template.py,sha256=nMJJfxjy8R7BZK89KicerQQwKLspUSJ3kerWZI3hFxk,4571
|
|
70
70
|
truefoundry/deploy/builder/builders/tfy_spark_buildpack/tfy_execute_notebook.py,sha256=-D37Zjy2SBt3RHxonPEpR1_LR0W7vTSM1kQ1S-fdK-I,6363
|
|
@@ -84,7 +84,7 @@ truefoundry/deploy/cli/commands/login_command.py,sha256=GdFrH2zgFFrSZi35p6MPDkQi
|
|
|
84
84
|
truefoundry/deploy/cli/commands/logout_command.py,sha256=u3kfrEp0ETbrz40KjD4GCC3XEZ5YRAlrca_Df4U_mk0,536
|
|
85
85
|
truefoundry/deploy/cli/commands/logs_command.py,sha256=osl2z5VaIceB9sYa6GtwsuyAPZKcw-A0oVEt3g1f62Q,4140
|
|
86
86
|
truefoundry/deploy/cli/commands/patch_application_command.py,sha256=aRTHu2OmxQd7j9iE0RavsFCkCILp0rGh4DJO51Oij5I,2591
|
|
87
|
-
truefoundry/deploy/cli/commands/patch_command.py,sha256=
|
|
87
|
+
truefoundry/deploy/cli/commands/patch_command.py,sha256=F4M6aWzx1e7TKkDjvYr9S7A-S0vqkdVlMj0N_KO4_TI,2083
|
|
88
88
|
truefoundry/deploy/cli/commands/terminate_comand.py,sha256=UKhOdbAej8ubX3q44vpLrOotAcvH4vHpRZJQrRf_AfM,1077
|
|
89
89
|
truefoundry/deploy/cli/commands/trigger_command.py,sha256=-FZy_XnFsexH5SOWzDc4Dj9fTwmdjene_EaLjJPmG2c,5119
|
|
90
90
|
truefoundry/deploy/cli/commands/utils.py,sha256=mIMYbHuAxnT0yz_0PU8LDC9sAZPU_xURZFMOrGoasuc,3694
|
|
@@ -119,7 +119,7 @@ truefoundry/deploy/v2/lib/deploy.py,sha256=Ltm7cpIW14IbmEsR3EAIeWQUch2Z6HLej7heu
|
|
|
119
119
|
truefoundry/deploy/v2/lib/deploy_workflow.py,sha256=G5BzMIbap8pgDX1eY-TITruUxQdkKhYtBmRwLL6lDeY,14342
|
|
120
120
|
truefoundry/deploy/v2/lib/deployable_patched_models.py,sha256=mUi-OjPf7bc8rzfrPLdFb79LKuDq7F36RxL4V-AXebs,6830
|
|
121
121
|
truefoundry/deploy/v2/lib/models.py,sha256=ogc1UYs1Z2nBdGSKCrde9sk8d0GxFKMkem99uqO5CmM,1148
|
|
122
|
-
truefoundry/deploy/v2/lib/patched_models.py,sha256=
|
|
122
|
+
truefoundry/deploy/v2/lib/patched_models.py,sha256=IFTIdAjXSG93f6ZgGC3gC2lXAsWq5p-ZqJGCmQ2Br5o,17149
|
|
123
123
|
truefoundry/deploy/v2/lib/source.py,sha256=d6-8_6Zn5koBglqrBrY6ZLG_7yyPuLdyEmK4iZTw6xY,9405
|
|
124
124
|
truefoundry/ml/__init__.py,sha256=EEEHV7w58Krpo_W9Chd8Y3TdItfFO3LI6j6Izqc4-P8,2219
|
|
125
125
|
truefoundry/ml/constants.py,sha256=vDq72d4C9FSWqr9MMdjgTF4TuyNFApvo_6RVsSeAjB4,2837
|
|
@@ -349,7 +349,7 @@ truefoundry/ml/_autogen/models/schema.py,sha256=a_bp42MMPUbwO3407m0UW2W8EOhnxZXf
|
|
|
349
349
|
truefoundry/ml/_autogen/models/signature.py,sha256=rBjpxUIsEeWM0sIyYG5uCJB18DKHR4k5yZw8TzuoP48,4987
|
|
350
350
|
truefoundry/ml/_autogen/models/utils.py,sha256=c7RtSLXhOLcP8rjuUtfnMdaKVTZvvbsmw98gPAkAFrs,24371
|
|
351
351
|
truefoundry/ml/artifact/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
352
|
-
truefoundry/ml/artifact/truefoundry_artifact_repo.py,sha256=
|
|
352
|
+
truefoundry/ml/artifact/truefoundry_artifact_repo.py,sha256=ITZbhvzwa8NbN66K7V9iufzZHVguuLYn1pHCDSE-P-o,36273
|
|
353
353
|
truefoundry/ml/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
354
354
|
truefoundry/ml/cli/cli.py,sha256=MwpY7z_NEeJE_XIP7XbZELjNeu2vpMmohttHCKDRk54,335
|
|
355
355
|
truefoundry/ml/cli/utils.py,sha256=j6_mZ4Spn114mz3P4QQ8jx0tmorXIuyQnHXVUSDvZi4,1035
|
|
@@ -365,11 +365,11 @@ truefoundry/ml/log_types/plot.py,sha256=LDh4uy6z2P_a2oPM2lc85c0lt8utVvunohzeMawF
|
|
|
365
365
|
truefoundry/ml/log_types/pydantic_base.py,sha256=eBlw_AEyAz4iJKDP4zgJOCFWcldwQqpf7FADW1jzIQY,272
|
|
366
366
|
truefoundry/ml/log_types/utils.py,sha256=xjJ21jdPScvFmw3TbVh5NCzbzJwaqiXJyiiT4xxX1EI,335
|
|
367
367
|
truefoundry/ml/log_types/artifacts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
368
|
-
truefoundry/ml/log_types/artifacts/artifact.py,sha256=
|
|
368
|
+
truefoundry/ml/log_types/artifacts/artifact.py,sha256=OXJDaJZLGE8jA-6A8pdApfpbz1x3O-8rIMWUkG_8hYA,19940
|
|
369
369
|
truefoundry/ml/log_types/artifacts/constants.py,sha256=7blG13UXSkZVZkN8EDFqaH-a_ZonJA6pI0gwJwkIZ-s,1298
|
|
370
370
|
truefoundry/ml/log_types/artifacts/dataset.py,sha256=OgWIoT59AhMw8P01FfvUKbJ3EL6HQf_Xw8X4E3Ff5Sg,13172
|
|
371
371
|
truefoundry/ml/log_types/artifacts/general_artifact.py,sha256=yr-SQ2fhUR_sE1MB5zoHHYpGC8tizH_-t3lhsxCAULU,2747
|
|
372
|
-
truefoundry/ml/log_types/artifacts/model.py,sha256=
|
|
372
|
+
truefoundry/ml/log_types/artifacts/model.py,sha256=j4Gf0TKpi-JCOBqCdkH40pkpDLwvSk04KgQn2sK5fZI,24911
|
|
373
373
|
truefoundry/ml/log_types/artifacts/utils.py,sha256=INZhhzl6OaD6qFAyxaLJAhXhFd6wKNujMj-lq2v8p4Q,9340
|
|
374
374
|
truefoundry/ml/log_types/image/__init__.py,sha256=fcOq8yQnNj1rkLcPeIjLXBpdA1WIeiPsXOlAAvMxx7M,76
|
|
375
375
|
truefoundry/ml/log_types/image/constants.py,sha256=wLtGEOA4T5fZHSlOXPuNDLX3lpbCtwlvGKPFk_1fah0,255
|
|
@@ -387,7 +387,7 @@ truefoundry/workflow/remote_filesystem/__init__.py,sha256=LQ95ViEjJ7Ts4JcCGOxMPs
|
|
|
387
387
|
truefoundry/workflow/remote_filesystem/logger.py,sha256=em2l7D6sw7xTLDP0kQSLpgfRRCLpN14Qw85TN7ujQcE,1022
|
|
388
388
|
truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py,sha256=xcT0wQmQlgzcj0nP3tJopyFSVWT1uv3nhiTIuwfXYeg,12342
|
|
389
389
|
truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py,sha256=nSGPZu0Gyd_jz0KsEE-7w_BmnTD8CVF1S8cUJoxaCbc,13305
|
|
390
|
-
truefoundry-0.11.
|
|
391
|
-
truefoundry-0.11.
|
|
392
|
-
truefoundry-0.11.
|
|
393
|
-
truefoundry-0.11.
|
|
390
|
+
truefoundry-0.11.11.dist-info/METADATA,sha256=KvW4uvzJ44zwy8dOpOK_NfMp3IS2h3Z44gG5OOowTHI,2761
|
|
391
|
+
truefoundry-0.11.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
392
|
+
truefoundry-0.11.11.dist-info/entry_points.txt,sha256=xVjn7RMN-MW2-9f7YU-bBdlZSvvrwzhpX1zmmRmsNPU,98
|
|
393
|
+
truefoundry-0.11.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|