truefoundry 0.4.4rc7__py3-none-any.whl → 0.4.4rc9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/common/constants.py +7 -12
- truefoundry/common/tfy_signed_url_client.py +84 -86
- truefoundry/common/tfy_signed_url_fs.py +16 -8
- truefoundry/workflow/__init__.py +8 -3
- {truefoundry-0.4.4rc7.dist-info → truefoundry-0.4.4rc9.dist-info}/METADATA +1 -1
- {truefoundry-0.4.4rc7.dist-info → truefoundry-0.4.4rc9.dist-info}/RECORD +8 -8
- {truefoundry-0.4.4rc7.dist-info → truefoundry-0.4.4rc9.dist-info}/WHEEL +0 -0
- {truefoundry-0.4.4rc7.dist-info → truefoundry-0.4.4rc9.dist-info}/entry_points.txt +0 -0
truefoundry/common/constants.py
CHANGED
|
@@ -13,12 +13,6 @@ TFY_API_KEY_ENV_KEY = "TFY_API_KEY"
|
|
|
13
13
|
|
|
14
14
|
TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY = "TFY_INTERNAL_SIGNED_URL_SERVER_HOST"
|
|
15
15
|
TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN_ENV_KEY = "TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN"
|
|
16
|
-
TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL_ENV_KEY = (
|
|
17
|
-
"TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL"
|
|
18
|
-
)
|
|
19
|
-
TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT_ENV_KEY = (
|
|
20
|
-
"TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT"
|
|
21
|
-
)
|
|
22
16
|
|
|
23
17
|
|
|
24
18
|
class TrueFoundrySdkEnv(BaseSettings):
|
|
@@ -26,18 +20,19 @@ class TrueFoundrySdkEnv(BaseSettings):
|
|
|
26
20
|
# Never expect the user to set these values
|
|
27
21
|
TFY_HOST: Optional[str] = Field(default=None, env=TFY_HOST_ENV_KEY)
|
|
28
22
|
TFY_API_KEY: Optional[SecretStr] = Field(default=None, env=TFY_API_KEY_ENV_KEY)
|
|
23
|
+
|
|
24
|
+
# Internal Signed URL Server
|
|
29
25
|
TFY_INTERNAL_SIGNED_URL_SERVER_HOST: Optional[str] = Field(
|
|
30
26
|
default=None, env=TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY
|
|
31
27
|
)
|
|
32
28
|
TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN: Optional[str] = Field(
|
|
33
29
|
default=None, env=TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN_ENV_KEY
|
|
34
30
|
)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
) # 5 seconds
|
|
31
|
+
TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT: int = 5 # default: 5 seconds
|
|
32
|
+
TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL: int = 3600 # default: 1 hour
|
|
33
|
+
TFY_INTERNAL_SIGNED_URL_REQUEST_TIMEOUT: int = 3600 # default: 1 hour
|
|
34
|
+
|
|
35
|
+
# Artifacts
|
|
41
36
|
TFY_ARTIFACTS_DOWNLOAD_CHUNK_SIZE_BYTES: int = 100 * 1000 * 1000
|
|
42
37
|
TFY_ARTIFACTS_DOWNLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
|
|
43
38
|
TFY_ARTIFACTS_UPLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
|
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
# file: client.py
|
|
2
2
|
from enum import Enum
|
|
3
|
-
from typing import Any, Dict, List, Optional
|
|
3
|
+
from typing import Any, Dict, List, Optional, Union
|
|
4
4
|
from urllib.parse import urlencode, urljoin
|
|
5
5
|
|
|
6
6
|
from requests.exceptions import RequestException
|
|
7
7
|
|
|
8
|
-
from truefoundry.common.constants import
|
|
8
|
+
from truefoundry.common.constants import (
|
|
9
|
+
ENV_VARS,
|
|
10
|
+
TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY,
|
|
11
|
+
TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN_ENV_KEY,
|
|
12
|
+
)
|
|
9
13
|
from truefoundry.common.request_utils import requests_retry_session
|
|
10
14
|
from truefoundry.common.utils import log_time
|
|
15
|
+
from truefoundry.logger import logger
|
|
11
16
|
from truefoundry.pydantic_v1 import BaseModel, Field
|
|
12
17
|
|
|
13
|
-
|
|
14
|
-
# TODO: Move these constants to the constants module
|
|
15
|
-
REQUEST_TIMEOUT = 3600
|
|
16
|
-
|
|
18
|
+
LOG_PREFIX = "[tfy][fs]"
|
|
17
19
|
DEFAULT_TTL = ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL
|
|
18
20
|
MAX_TIMEOUT = ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT
|
|
21
|
+
REQUEST_TIMEOUT = ENV_VARS.TFY_INTERNAL_SIGNED_URL_REQUEST_TIMEOUT
|
|
19
22
|
|
|
20
23
|
|
|
21
24
|
class SignedURLAPIResponseDto(BaseModel):
|
|
@@ -34,9 +37,9 @@ class SignedURLExistsAPIResponseDto(BaseModel):
|
|
|
34
37
|
|
|
35
38
|
class FileInfo(BaseModel):
|
|
36
39
|
path: str
|
|
37
|
-
|
|
40
|
+
is_directory: bool = Field(..., alias="isDirectory")
|
|
38
41
|
bytes: Optional[int] = None
|
|
39
|
-
|
|
42
|
+
signed_url: Optional[str] = None
|
|
40
43
|
|
|
41
44
|
|
|
42
45
|
class PagedList(BaseModel):
|
|
@@ -62,51 +65,46 @@ class SignedURLClient:
|
|
|
62
65
|
base_url: Optional[str] = None,
|
|
63
66
|
token: Optional[str] = None,
|
|
64
67
|
ttl: int = DEFAULT_TTL,
|
|
68
|
+
max_retries: int = 3,
|
|
69
|
+
retry_backoff_factor: float = 0.3,
|
|
65
70
|
):
|
|
66
|
-
"""
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
Args:
|
|
70
|
-
base_url (str): Base URL of the signed URL server (optional).
|
|
71
|
-
token (str): Token for authentication (optional).
|
|
72
|
-
ttl (int): Default time-to-live for signed URLs in seconds.
|
|
73
|
-
"""
|
|
71
|
+
"""Initialize the SignedURLClient."""
|
|
72
|
+
# Set the base URL and token from the environment variables
|
|
74
73
|
self.base_url = base_url or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_HOST
|
|
75
74
|
self.token = token or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN
|
|
75
|
+
self.signed_url_server_headers = {
|
|
76
|
+
"Authorization": f"Bearer {self.token}",
|
|
77
|
+
"Content-Type": "application/json",
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if not self.base_url or not self.token:
|
|
81
|
+
raise ValueError(
|
|
82
|
+
f"Set `base_url` and `token` or define env vars "
|
|
83
|
+
f"{TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY} and "
|
|
84
|
+
f"{TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN_ENV_KEY}"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Set the TTL, max retries, and backoff factor
|
|
76
88
|
self.ttl = ttl
|
|
77
|
-
self.max_retries: int =
|
|
78
|
-
self.retry_backoff_factor: float =
|
|
89
|
+
self.max_retries: int = max_retries
|
|
90
|
+
self.retry_backoff_factor: float = retry_backoff_factor
|
|
79
91
|
self.session = requests_retry_session(
|
|
80
92
|
retries=self.max_retries, backoff_factor=self.retry_backoff_factor
|
|
81
93
|
)
|
|
82
94
|
|
|
83
|
-
self.headers = {
|
|
84
|
-
"Authorization": f"Bearer {self.token}",
|
|
85
|
-
"Content-Type": "application/json",
|
|
86
|
-
}
|
|
87
|
-
|
|
88
95
|
@log_time
|
|
89
96
|
def _make_request(
|
|
90
|
-
self,
|
|
97
|
+
self,
|
|
98
|
+
endpoint: str,
|
|
99
|
+
method: str = "GET",
|
|
100
|
+
payload: Optional[Dict] = None,
|
|
101
|
+
headers: Optional[Dict] = None,
|
|
91
102
|
) -> Dict:
|
|
92
|
-
"""
|
|
93
|
-
Internal method to handle requests to the signed URL server.
|
|
94
|
-
|
|
95
|
-
Args:
|
|
96
|
-
endpoint (str): The endpoint for the request.
|
|
97
|
-
method (str): HTTP method (GET, POST, etc.).
|
|
98
|
-
payload (Dict, optional): JSON payload for the request.
|
|
99
|
-
|
|
100
|
-
Returns:
|
|
101
|
-
Dict: JSON response from the server.
|
|
102
|
-
|
|
103
|
-
Raises:
|
|
104
|
-
RuntimeError: For network errors or invalid responses.
|
|
105
|
-
"""
|
|
103
|
+
"""Internal method to handle requests to the signed URL server."""
|
|
106
104
|
url = urljoin(self.base_url, endpoint)
|
|
107
105
|
try:
|
|
108
106
|
response = self.session.request(
|
|
109
|
-
method, url, headers=
|
|
107
|
+
method, url, headers=headers, json=payload, timeout=MAX_TIMEOUT
|
|
110
108
|
)
|
|
111
109
|
response.raise_for_status()
|
|
112
110
|
return response.json()
|
|
@@ -114,19 +112,30 @@ class SignedURLClient:
|
|
|
114
112
|
raise RuntimeError(f"Error during request to {url}: {e}") from e
|
|
115
113
|
|
|
116
114
|
def _make_server_api_call(
|
|
117
|
-
self,
|
|
115
|
+
self,
|
|
116
|
+
endpoint: SignedURLServerEndpoint,
|
|
117
|
+
params: Optional[Dict] = None,
|
|
118
|
+
headers: Optional[Dict] = None,
|
|
118
119
|
) -> Dict:
|
|
119
120
|
"""Get a signed URL for the specified operation and URI."""
|
|
120
121
|
query_string = urlencode(params or {})
|
|
121
122
|
endpoint_with_params = f"{endpoint.value}?{query_string}"
|
|
122
|
-
return self._make_request(
|
|
123
|
+
return self._make_request(
|
|
124
|
+
endpoint=endpoint_with_params, method="GET", payload=None, headers=headers
|
|
125
|
+
)
|
|
123
126
|
|
|
124
127
|
@log_time
|
|
125
128
|
def _upload_data(self, signed_url: str, data: Any) -> None:
|
|
126
|
-
"""
|
|
129
|
+
"""
|
|
130
|
+
Upload data to the specified storage path using a signed URL.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
signed_url: str: The signed URL to upload the data to.
|
|
134
|
+
data: Bytes or IO: The data to upload.
|
|
135
|
+
"""
|
|
127
136
|
try:
|
|
128
137
|
response = self.session.put(
|
|
129
|
-
signed_url,
|
|
138
|
+
url=signed_url,
|
|
130
139
|
headers={"Content-Type": "application/octet-stream"},
|
|
131
140
|
data=data,
|
|
132
141
|
timeout=REQUEST_TIMEOUT,
|
|
@@ -140,6 +149,7 @@ class SignedURLClient:
|
|
|
140
149
|
signed_object = self._make_server_api_call(
|
|
141
150
|
endpoint=SignedURLServerEndpoint.WRITE,
|
|
142
151
|
params={"uri": storage_uri, "expiryInSeconds": self.ttl},
|
|
152
|
+
headers=self.signed_url_server_headers,
|
|
143
153
|
)
|
|
144
154
|
pre_signed_object_dto = SignedURLAPIResponseDto.parse_obj(signed_object)
|
|
145
155
|
self._upload_data(pre_signed_object_dto.signed_url, data)
|
|
@@ -148,10 +158,11 @@ class SignedURLClient:
|
|
|
148
158
|
@log_time
|
|
149
159
|
def upload(self, file_path: str, storage_uri: str) -> str:
|
|
150
160
|
"""Upload a file to the specified storage path using a signed URL."""
|
|
151
|
-
|
|
161
|
+
logger.info(f"{LOG_PREFIX} Uploading {file_path} to {storage_uri}")
|
|
152
162
|
response = self._make_server_api_call(
|
|
153
163
|
endpoint=SignedURLServerEndpoint.WRITE,
|
|
154
164
|
params={"uri": storage_uri, "expiryInSeconds": self.ttl},
|
|
165
|
+
headers=self.signed_url_server_headers,
|
|
155
166
|
)
|
|
156
167
|
pre_signed_object_dto = SignedURLAPIResponseDto.parse_obj(response)
|
|
157
168
|
with open(file_path, "rb") as file:
|
|
@@ -162,62 +173,43 @@ class SignedURLClient:
|
|
|
162
173
|
def _download_file(
|
|
163
174
|
self, signed_url: str, local_path: Optional[str] = None
|
|
164
175
|
) -> Optional[bytes]:
|
|
165
|
-
"""Common method to download a file using a signed URL.
|
|
166
|
-
|
|
167
|
-
Args:
|
|
168
|
-
signed_url (str): The signed URL to download from.
|
|
169
|
-
local_path (Optional[str]): The local path to save the file. If None, the file will not be saved.
|
|
170
|
-
|
|
171
|
-
Returns:
|
|
172
|
-
Optional[bytes]: The content of the downloaded file if local_path is None; otherwise, None.
|
|
173
|
-
"""
|
|
176
|
+
"""Common method to download a file using a signed URL."""
|
|
174
177
|
try:
|
|
175
178
|
response = self.session.get(
|
|
176
|
-
signed_url,
|
|
179
|
+
signed_url,
|
|
180
|
+
stream=True,
|
|
181
|
+
headers={"Content-Type": "application/octet-stream"},
|
|
182
|
+
timeout=REQUEST_TIMEOUT,
|
|
177
183
|
)
|
|
178
184
|
response.raise_for_status()
|
|
179
185
|
if local_path:
|
|
180
186
|
with open(local_path, "wb") as file:
|
|
181
187
|
for chunk in response.iter_content(chunk_size=8192):
|
|
182
188
|
file.write(chunk)
|
|
183
|
-
return None
|
|
184
|
-
return response.content
|
|
185
|
-
|
|
189
|
+
return None
|
|
190
|
+
return response.content
|
|
186
191
|
except RequestException as e:
|
|
187
192
|
raise RuntimeError(f"Failed to download file from {signed_url}: {e}") from e
|
|
188
193
|
|
|
189
194
|
@log_time
|
|
190
195
|
def download(self, storage_uri: str, local_path: str) -> Optional[str]:
|
|
191
|
-
"""Download a file from the specified storage path to a local path using a signed URL.
|
|
192
|
-
|
|
193
|
-
Args:
|
|
194
|
-
storage_uri (str): The storage URI to download the file from.
|
|
195
|
-
local_path (str): The local path to save the downloaded file.
|
|
196
|
-
|
|
197
|
-
Returns:
|
|
198
|
-
Optional[str]: The local path if successful; None if the file is saved.
|
|
199
|
-
"""
|
|
200
|
-
print(f"Dowloading {storage_uri} to {local_path}")
|
|
196
|
+
"""Download a file from the specified storage path to a local path using a signed URL."""
|
|
197
|
+
logger.info(f"{LOG_PREFIX} Downloading {storage_uri} to {local_path}")
|
|
201
198
|
response = self._make_server_api_call(
|
|
202
199
|
endpoint=SignedURLServerEndpoint.READ,
|
|
203
200
|
params={"uri": storage_uri, "expiryInSeconds": self.ttl},
|
|
201
|
+
headers=self.signed_url_server_headers,
|
|
204
202
|
)
|
|
205
203
|
presigned_object = SignedURLAPIResponseDto.parse_obj(response)
|
|
206
204
|
self._download_file(presigned_object.signed_url, local_path)
|
|
207
205
|
return local_path
|
|
208
206
|
|
|
209
207
|
def download_to_bytes(self, storage_uri: str) -> bytes:
|
|
210
|
-
"""Download a file from the specified storage path and return it as bytes.
|
|
211
|
-
|
|
212
|
-
Args:
|
|
213
|
-
storage_uri (str): The storage URI to download the file from.
|
|
214
|
-
|
|
215
|
-
Returns:
|
|
216
|
-
bytes: The content of the downloaded file.
|
|
217
|
-
"""
|
|
208
|
+
"""Download a file from the specified storage path and return it as bytes."""
|
|
218
209
|
response = self._make_server_api_call(
|
|
219
210
|
endpoint=SignedURLServerEndpoint.READ,
|
|
220
211
|
params={"uri": storage_uri, "expiryInSeconds": self.ttl},
|
|
212
|
+
headers=self.signed_url_server_headers,
|
|
221
213
|
)
|
|
222
214
|
presigned_object = SignedURLAPIResponseDto.parse_obj(response)
|
|
223
215
|
return self._download_file(presigned_object.signed_url)
|
|
@@ -225,22 +217,29 @@ class SignedURLClient:
|
|
|
225
217
|
def exists(self, uri: str) -> bool:
|
|
226
218
|
"""Check if a file exists at the specified path."""
|
|
227
219
|
response = self._make_server_api_call(
|
|
228
|
-
endpoint=SignedURLServerEndpoint.EXISTS,
|
|
220
|
+
endpoint=SignedURLServerEndpoint.EXISTS,
|
|
221
|
+
params={"uri": uri},
|
|
222
|
+
headers=self.signed_url_server_headers,
|
|
229
223
|
)
|
|
230
|
-
|
|
231
|
-
return response_obj.exists
|
|
224
|
+
return SignedURLExistsAPIResponseDto.parse_obj(response).exists
|
|
232
225
|
|
|
233
226
|
def is_directory(self, uri: str) -> bool:
|
|
234
227
|
"""Check if the specified URI is a directory."""
|
|
235
228
|
response = self._make_server_api_call(
|
|
236
|
-
endpoint=SignedURLServerEndpoint.IS_DIRECTORY,
|
|
229
|
+
endpoint=SignedURLServerEndpoint.IS_DIRECTORY,
|
|
230
|
+
params={"path": uri},
|
|
231
|
+
headers=self.signed_url_server_headers,
|
|
237
232
|
)
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
233
|
+
is_directory = SignedURLIsDirectoryAPIResponseDto.parse_obj(
|
|
234
|
+
response
|
|
235
|
+
).is_directory
|
|
236
|
+
logger.info(f"{LOG_PREFIX} Path {uri} is a directory: {is_directory}")
|
|
237
|
+
return is_directory
|
|
241
238
|
|
|
242
239
|
@log_time
|
|
243
|
-
def list_files(
|
|
240
|
+
def list_files(
|
|
241
|
+
self, path: str, detail: bool = False, max_results: int = 1000
|
|
242
|
+
) -> Union[List[FileInfo], List[str]]:
|
|
244
243
|
"""List files in the specified directory."""
|
|
245
244
|
token = ""
|
|
246
245
|
items: List[FileInfo] = []
|
|
@@ -249,6 +248,7 @@ class SignedURLClient:
|
|
|
249
248
|
response = self._make_server_api_call(
|
|
250
249
|
endpoint=SignedURLServerEndpoint.LIST_FILES,
|
|
251
250
|
params={"path": path, "maxResults": max_results, "pageToken": token},
|
|
251
|
+
headers=self.signed_url_server_headers,
|
|
252
252
|
)
|
|
253
253
|
response_obj = PagedList.parse_obj(response)
|
|
254
254
|
items.extend(response_obj.items)
|
|
@@ -257,6 +257,4 @@ class SignedURLClient:
|
|
|
257
257
|
break
|
|
258
258
|
|
|
259
259
|
# Return the items or paths based on the detail flag
|
|
260
|
-
if detail
|
|
261
|
-
return items
|
|
262
|
-
return [item.path for item in items]
|
|
260
|
+
return items if detail else [item.path for item in items]
|
|
@@ -13,7 +13,9 @@ from truefoundry.common.utils import log_time
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class SignedURLFileSystem(AbstractFileSystem):
|
|
16
|
-
def __init__(
|
|
16
|
+
def __init__(
|
|
17
|
+
self, base_url: Optional[str] = None, token: Optional[str] = None, **kwargs
|
|
18
|
+
):
|
|
17
19
|
super().__init__()
|
|
18
20
|
base_url = base_url or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_HOST
|
|
19
21
|
token = token or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN
|
|
@@ -21,7 +23,7 @@ class SignedURLFileSystem(AbstractFileSystem):
|
|
|
21
23
|
|
|
22
24
|
@log_time
|
|
23
25
|
def exists(self, path, **kwargs):
|
|
24
|
-
"""
|
|
26
|
+
"""Check if a file exists at the given path."""
|
|
25
27
|
return self.client.exists(path)
|
|
26
28
|
|
|
27
29
|
@log_time
|
|
@@ -36,8 +38,8 @@ class SignedURLFileSystem(AbstractFileSystem):
|
|
|
36
38
|
):
|
|
37
39
|
"""Get file(s) to local"""
|
|
38
40
|
# TODO: Add support for ThreadPoolExecutor here
|
|
39
|
-
# TODO: Do a proper error handling here
|
|
40
41
|
# TODO: Add support for async download
|
|
42
|
+
# TODO: Do a proper error handling here
|
|
41
43
|
if self.isdir(rpath):
|
|
42
44
|
if not recursive:
|
|
43
45
|
raise ValueError(
|
|
@@ -50,7 +52,7 @@ class SignedURLFileSystem(AbstractFileSystem):
|
|
|
50
52
|
for file_info in files:
|
|
51
53
|
file_path = file_info.path.rstrip("/").rsplit("/")[-1]
|
|
52
54
|
|
|
53
|
-
is_directory = file_info.
|
|
55
|
+
is_directory = file_info.is_directory
|
|
54
56
|
# Construct the relative path for local download
|
|
55
57
|
relative_path = rpath.rstrip("/") + "/" + file_path
|
|
56
58
|
target_local_path = lpath.rstrip("/") + "/" + file_path
|
|
@@ -69,16 +71,20 @@ class SignedURLFileSystem(AbstractFileSystem):
|
|
|
69
71
|
**kwargs,
|
|
70
72
|
)
|
|
71
73
|
else:
|
|
72
|
-
# Download the file
|
|
73
74
|
self.client.download(
|
|
74
75
|
storage_uri=relative_path, local_path=target_local_path
|
|
75
76
|
)
|
|
76
77
|
|
|
77
78
|
else:
|
|
79
|
+
# Ensure the directory exists first
|
|
78
80
|
target_local_path = lpath
|
|
79
81
|
if target_local_path.endswith("/"):
|
|
80
|
-
|
|
81
|
-
target_local_path =
|
|
82
|
+
# If it ends with "/", it's a directory, so create the directory first
|
|
83
|
+
target_local_path = os.path.join(
|
|
84
|
+
target_local_path, rpath.rsplit("/", 1)[-1]
|
|
85
|
+
)
|
|
86
|
+
# Create the directory for the target file path (common for both cases)
|
|
87
|
+
Path(os.path.dirname(target_local_path)).mkdir(parents=True, exist_ok=True)
|
|
82
88
|
self.client.download(storage_uri=rpath, local_path=target_local_path)
|
|
83
89
|
|
|
84
90
|
@log_time
|
|
@@ -124,7 +130,8 @@ class SignedURLFileSystem(AbstractFileSystem):
|
|
|
124
130
|
)
|
|
125
131
|
return None
|
|
126
132
|
else:
|
|
127
|
-
|
|
133
|
+
if rpath.endswith("/"):
|
|
134
|
+
rpath = os.path.join(rpath, local_path.name)
|
|
128
135
|
return self.client.upload(file_path=lpath, storage_uri=rpath)
|
|
129
136
|
|
|
130
137
|
@log_time
|
|
@@ -187,6 +194,7 @@ class SignedURLFileSystem(AbstractFileSystem):
|
|
|
187
194
|
class SignedURLBufferedFile(AbstractBufferedFile):
|
|
188
195
|
"""
|
|
189
196
|
Buffered file implementation for Signed URL-based file system.
|
|
197
|
+
# TODO: Need to test this implementation
|
|
190
198
|
"""
|
|
191
199
|
|
|
192
200
|
def __init__(
|
truefoundry/workflow/__init__.py
CHANGED
|
@@ -7,6 +7,7 @@ except ImportError:
|
|
|
7
7
|
from flytekit import conditional
|
|
8
8
|
from flytekit.types.directory import FlyteDirectory
|
|
9
9
|
|
|
10
|
+
from truefoundry.common.constants import ENV_VARS
|
|
10
11
|
from truefoundry.common.tfy_signed_url_fs import SignedURLFileSystem
|
|
11
12
|
from truefoundry.deploy.v2.lib.patched_models import (
|
|
12
13
|
ContainerTaskConfig,
|
|
@@ -34,6 +35,10 @@ __all__ = [
|
|
|
34
35
|
"PythonTaskConfig",
|
|
35
36
|
"ExecutionConfig",
|
|
36
37
|
]
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# Register the SignedURLFileSystem implementation for fsspec
|
|
41
|
+
if ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_HOST:
|
|
42
|
+
fsspec.register_implementation("s3", SignedURLFileSystem, clobber=True)
|
|
43
|
+
fsspec.register_implementation("gs", SignedURLFileSystem, clobber=True)
|
|
44
|
+
fsspec.register_implementation("abfs", SignedURLFileSystem, clobber=True)
|
|
@@ -27,15 +27,15 @@ truefoundry/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
27
27
|
truefoundry/cli/__main__.py,sha256=-NkhYlT3mC5MhtekueKAvCw-sWvguj0LJRpXWzvvFjc,727
|
|
28
28
|
truefoundry/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
29
|
truefoundry/common/auth_service_client.py,sha256=tZOa0NdATnItsMeTnEnUeTZQIgUJtpU-nvLdWtB4Px8,7978
|
|
30
|
-
truefoundry/common/constants.py,sha256=
|
|
30
|
+
truefoundry/common/constants.py,sha256=y8T5S-CoFKubGbSrHXq0VCgr2O3RiEbNV5KP4NDNqhk,2089
|
|
31
31
|
truefoundry/common/credential_file_manager.py,sha256=1yEk1Zm2xS4G0VDFwKSZ4w0VUrcPWQ1nJnoBaz9xyKA,4251
|
|
32
32
|
truefoundry/common/credential_provider.py,sha256=Aht7hFLsnyRgMR34dRbzln7dor0WYSeA8ej8ApNmnKM,4148
|
|
33
33
|
truefoundry/common/entities.py,sha256=8O-EGPk4PKqnyoFMKUTxISCU19rz0KBnfRDJU695DhY,3797
|
|
34
34
|
truefoundry/common/exceptions.py,sha256=ePpiQ_zmWe4e94gOgeMiyP_AZnKwjEBfyXsB5ScGYcI,329
|
|
35
35
|
truefoundry/common/request_utils.py,sha256=5xw4YGUcMf71Ncal3OfFCa-PoWDIvG3hYGCDa4Da4OI,2854
|
|
36
36
|
truefoundry/common/servicefoundry_client.py,sha256=2fxmgCM-ckFHpnm6n_mL-5Z8RWN_q-dYVvFC29bkYSg,3120
|
|
37
|
-
truefoundry/common/tfy_signed_url_client.py,sha256=
|
|
38
|
-
truefoundry/common/tfy_signed_url_fs.py,sha256=
|
|
37
|
+
truefoundry/common/tfy_signed_url_client.py,sha256=wa9pfZIUquXFE9EdHDOJXjqqU1uJr1PkvI_7vnOjZ9M,9904
|
|
38
|
+
truefoundry/common/tfy_signed_url_fs.py,sha256=Vc7Il077nZIGbDBQ8aTPNtJVcypYaoZHr13vV00REko,8486
|
|
39
39
|
truefoundry/common/utils.py,sha256=vacYbVTEvcEY4niH4C9Wb_aCeaGgh2un3_ApAt556VQ,3621
|
|
40
40
|
truefoundry/deploy/__init__.py,sha256=ugawKF2G02EmEXX35oZ2tec12d9oWN28Sf6mtGGIERY,2281
|
|
41
41
|
truefoundry/deploy/auto_gen/models.py,sha256=4MaxkG2_5Wg6avaZRlK0D4JiVEM5rk3NU0BCiTx8VyU,82477
|
|
@@ -331,7 +331,7 @@ truefoundry/ml/session.py,sha256=F83GTC5WwGBjnJ69Ct8MqMnlutYc56JCc6YhEY1Wl-A,539
|
|
|
331
331
|
truefoundry/ml/validation_utils.py,sha256=XBSUd9OoyriWJpT3M5LKz17iWY3yVMr3hM5vdaVjtf0,12082
|
|
332
332
|
truefoundry/pydantic_v1.py,sha256=jSuhGtz0Mbk1qYu8jJ1AcnIDK4oxUsdhALc4spqstmM,345
|
|
333
333
|
truefoundry/version.py,sha256=bqiT4Q-VWrTC6P4qfK43mez-Ppf-smWfrl6DcwV7mrw,137
|
|
334
|
-
truefoundry/workflow/__init__.py,sha256=
|
|
334
|
+
truefoundry/workflow/__init__.py,sha256=c4daVkQE269b69lZRiRrRn4abAfIhVDN_4Na5MsFfmk,1414
|
|
335
335
|
truefoundry/workflow/container_task.py,sha256=8arieePsX4__OnG337hOtCiNgJwtKJJCsZcmFmCBJtk,402
|
|
336
336
|
truefoundry/workflow/example/deploy.sh,sha256=wfbPRrCi04WYRqCf4g-Xo12uWbcqPD6G_Tz0lV0jU_U,60
|
|
337
337
|
truefoundry/workflow/example/hello_world_package/workflow.py,sha256=IkRKfPY5BcvLPo_PVuNbZKK9PPJ93LRkzb1a3RKQYOw,435
|
|
@@ -342,7 +342,7 @@ truefoundry/workflow/map_task.py,sha256=2m3qGXQ90k9LdS45q8dqCCECc3qr8t2m_LMCVd1m
|
|
|
342
342
|
truefoundry/workflow/python_task.py,sha256=SRXRLC4vdBqGjhkwuaY39LEWN6iPCpJAuW17URRdWTY,1128
|
|
343
343
|
truefoundry/workflow/task.py,sha256=ToitYiKcNzFCtOVQwz1W8sRjbR97eVS7vQBdbgUQtKg,1779
|
|
344
344
|
truefoundry/workflow/workflow.py,sha256=WaTqUjhwfAXDWu4E5ehuwAxrCbDJkoAf1oWmR2E9Qy0,4575
|
|
345
|
-
truefoundry-0.4.
|
|
346
|
-
truefoundry-0.4.
|
|
347
|
-
truefoundry-0.4.
|
|
348
|
-
truefoundry-0.4.
|
|
345
|
+
truefoundry-0.4.4rc9.dist-info/METADATA,sha256=yhVxkhDM-AIf-DK8mdAV-Y4PmPgBpIJ3ZVdDmje1VD0,3101
|
|
346
|
+
truefoundry-0.4.4rc9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
347
|
+
truefoundry-0.4.4rc9.dist-info/entry_points.txt,sha256=TXvUxQkI6zmqJuycPsyxEIMr3oqfDjgrWj0m_9X12x4,95
|
|
348
|
+
truefoundry-0.4.4rc9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|