ml-dash 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +37 -63
- ml_dash/auth/token_storage.py +267 -226
- ml_dash/auto_start.py +30 -30
- ml_dash/cli.py +16 -2
- ml_dash/cli_commands/api.py +165 -0
- ml_dash/cli_commands/download.py +757 -667
- ml_dash/cli_commands/list.py +146 -13
- ml_dash/cli_commands/login.py +190 -183
- ml_dash/cli_commands/profile.py +92 -0
- ml_dash/cli_commands/upload.py +1291 -1141
- ml_dash/client.py +122 -34
- ml_dash/config.py +119 -119
- ml_dash/experiment.py +1242 -995
- ml_dash/files.py +1051 -340
- ml_dash/log.py +7 -7
- ml_dash/metric.py +359 -100
- ml_dash/params.py +6 -6
- ml_dash/remote_auto_start.py +20 -17
- ml_dash/run.py +231 -0
- ml_dash/snowflake.py +173 -0
- ml_dash/storage.py +1051 -1079
- {ml_dash-0.6.0.dist-info → ml_dash-0.6.2.dist-info}/METADATA +45 -20
- ml_dash-0.6.2.dist-info/RECORD +33 -0
- ml_dash-0.6.0.dist-info/RECORD +0 -29
- {ml_dash-0.6.0.dist-info → ml_dash-0.6.2.dist-info}/WHEEL +0 -0
- {ml_dash-0.6.0.dist-info → ml_dash-0.6.2.dist-info}/entry_points.txt +0 -0
ml_dash/files.py
CHANGED
|
@@ -6,6 +6,7 @@ Provides fluent API for file upload, download, list, and delete operations.
|
|
|
6
6
|
|
|
7
7
|
import hashlib
|
|
8
8
|
import mimetypes
|
|
9
|
+
import fnmatch
|
|
9
10
|
from typing import Dict, Any, List, Optional, Union, TYPE_CHECKING
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
|
|
@@ -18,38 +19,69 @@ class FileBuilder:
|
|
|
18
19
|
Fluent interface for file operations.
|
|
19
20
|
|
|
20
21
|
Usage:
|
|
21
|
-
# Upload file
|
|
22
|
-
|
|
22
|
+
# Upload existing file from disk
|
|
23
|
+
dxp.files("models").upload("./local_model.pt")
|
|
24
|
+
dxp.files("checkpoints").upload("./model.pt", to="latest.pt")
|
|
25
|
+
|
|
26
|
+
# Save objects as files (using keyword dir argument)
|
|
27
|
+
dxp.files(dir="models").save_torch(model, to="checkpoint.pt")
|
|
28
|
+
dxp.files(dir="configs").save_json({"lr": 0.001}, to="config.json")
|
|
29
|
+
dxp.files(dir="data").save_blob(b"binary data", to="data.bin")
|
|
23
30
|
|
|
24
31
|
# List files
|
|
25
|
-
files = experiment.files().list()
|
|
26
|
-
files = experiment.files(
|
|
32
|
+
files = experiment.files(dir="some/location").list()
|
|
33
|
+
files = experiment.files(dir="models").list()
|
|
27
34
|
|
|
28
35
|
# Download file
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
#
|
|
33
|
-
experiment.files(
|
|
36
|
+
dxp.files("some.text").download()
|
|
37
|
+
dxp.files("some.text").download(to="./model.pt")
|
|
38
|
+
|
|
39
|
+
# Download files via glob pattern
|
|
40
|
+
file_paths = experiment.files("images").list("*.png")
|
|
41
|
+
dxp.files("images").download("*.png")
|
|
42
|
+
|
|
43
|
+
# Delete files
|
|
44
|
+
dxp.files("some.text").delete()
|
|
45
|
+
|
|
46
|
+
# Check if file exists
|
|
47
|
+
if dxp.files("config.json").exists():
|
|
48
|
+
config = dxp.files("config.json").read_text()
|
|
49
|
+
|
|
50
|
+
# Read file content as text
|
|
51
|
+
config_yaml = dxp.files("configs/view.yaml").read_text()
|
|
52
|
+
|
|
53
|
+
Specific Save Methods:
|
|
54
|
+
experiment.files.save_text("content", to="view.yaml")
|
|
55
|
+
experiment.files.save_json(dict(hey="yo"), to="config.json")
|
|
56
|
+
experiment.files.save_blob(b"xxx", to="data.bin")
|
|
57
|
+
experiment.files.save_torch(model, to="model.pt")
|
|
58
|
+
experiment.files.save_pkl(data, to="data.pkl")
|
|
59
|
+
experiment.files.save_fig(fig, to="plot.png")
|
|
60
|
+
experiment.files.save_video(frames, to="video.mp4")
|
|
34
61
|
"""
|
|
35
62
|
|
|
36
|
-
def __init__(self, experiment: 'Experiment', **kwargs):
|
|
63
|
+
def __init__(self, experiment: 'Experiment', path: Optional[str] = None, **kwargs):
|
|
37
64
|
"""
|
|
38
65
|
Initialize file builder.
|
|
39
66
|
|
|
40
67
|
Args:
|
|
41
68
|
experiment: Parent experiment instance
|
|
42
|
-
|
|
43
|
-
-
|
|
44
|
-
-
|
|
69
|
+
path: File path or prefix for operations. Can be:
|
|
70
|
+
- A prefix/directory (e.g., "checkpoints", "/models")
|
|
71
|
+
- A file path (e.g., "some.text", "images/photo.png")
|
|
72
|
+
**kwargs: Additional file operation parameters (for backwards compatibility)
|
|
73
|
+
- file_path: Path to file to upload (deprecated, use save(to=))
|
|
74
|
+
- prefix: Logical path prefix (deprecated, use path argument)
|
|
45
75
|
- description: Optional description
|
|
46
76
|
- tags: Optional list of tags
|
|
47
77
|
- bindrs: Optional list of bindrs
|
|
48
78
|
- metadata: Optional metadata dict
|
|
49
79
|
- file_id: File ID for download/delete/update operations
|
|
50
|
-
- dest_path: Destination path for download
|
|
80
|
+
- dest_path: Destination path for download (deprecated, use download(to=))
|
|
51
81
|
"""
|
|
52
82
|
self._experiment = experiment
|
|
83
|
+
self._path = path
|
|
84
|
+
# Backwards compatibility
|
|
53
85
|
self._file_path = kwargs.get('file_path')
|
|
54
86
|
self._prefix = kwargs.get('prefix', '/')
|
|
55
87
|
self._description = kwargs.get('description')
|
|
@@ -59,9 +91,30 @@ class FileBuilder:
|
|
|
59
91
|
self._file_id = kwargs.get('file_id')
|
|
60
92
|
self._dest_path = kwargs.get('dest_path')
|
|
61
93
|
|
|
62
|
-
|
|
94
|
+
# If path is provided, determine if it's a file or prefix
|
|
95
|
+
if path:
|
|
96
|
+
# Normalize path
|
|
97
|
+
path = path.lstrip('/')
|
|
98
|
+
self._normalized_path = '/' + path if not path.startswith('/') else path
|
|
99
|
+
|
|
100
|
+
def upload(
|
|
101
|
+
self,
|
|
102
|
+
fpath: str,
|
|
103
|
+
*,
|
|
104
|
+
to: Optional[str] = None,
|
|
105
|
+
description: Optional[str] = None,
|
|
106
|
+
tags: Optional[List[str]] = None,
|
|
107
|
+
metadata: Optional[Dict[str, Any]] = None
|
|
108
|
+
) -> Dict[str, Any]:
|
|
63
109
|
"""
|
|
64
|
-
Upload
|
|
110
|
+
Upload an existing file from disk.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
fpath: Path to existing file to upload (required)
|
|
114
|
+
to: Optional new filename (uses original filename if not provided)
|
|
115
|
+
description: Optional description
|
|
116
|
+
tags: Optional list of tags
|
|
117
|
+
metadata: Optional metadata dict
|
|
65
118
|
|
|
66
119
|
Returns:
|
|
67
120
|
File metadata dict with id, path, filename, checksum, etc.
|
|
@@ -72,8 +125,19 @@ class FileBuilder:
|
|
|
72
125
|
ValueError: If file size exceeds 100GB limit
|
|
73
126
|
|
|
74
127
|
Examples:
|
|
75
|
-
|
|
76
|
-
|
|
128
|
+
# Upload with original filename
|
|
129
|
+
dxp.files("models").upload("./local_model.pt")
|
|
130
|
+
|
|
131
|
+
# Upload with new filename
|
|
132
|
+
dxp.files("models").upload("./local_model.pt", to="remote_model.pt")
|
|
133
|
+
|
|
134
|
+
# Upload with metadata
|
|
135
|
+
dxp.files("checkpoints").upload(
|
|
136
|
+
"./checkpoint.pt",
|
|
137
|
+
to="epoch_10.pt",
|
|
138
|
+
description="Best model at epoch 10",
|
|
139
|
+
tags=["best", "training"]
|
|
140
|
+
)
|
|
77
141
|
"""
|
|
78
142
|
if not self._experiment._is_open:
|
|
79
143
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
@@ -81,47 +145,312 @@ class FileBuilder:
|
|
|
81
145
|
if self._experiment._write_protected:
|
|
82
146
|
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
83
147
|
|
|
84
|
-
if not
|
|
85
|
-
raise ValueError("
|
|
148
|
+
if not fpath:
|
|
149
|
+
raise ValueError("fpath is required")
|
|
150
|
+
|
|
151
|
+
# Use provided values or fall back to constructor values
|
|
152
|
+
desc = description if description is not None else self._description
|
|
153
|
+
file_tags = tags if tags is not None else self._tags
|
|
154
|
+
file_metadata = metadata if metadata is not None else self._metadata
|
|
155
|
+
|
|
156
|
+
# Determine prefix from path
|
|
157
|
+
prefix = self._prefix
|
|
158
|
+
if self._path:
|
|
159
|
+
prefix = '/' + self._path.lstrip('/')
|
|
160
|
+
|
|
161
|
+
return self._save_file(
|
|
162
|
+
fpath=fpath,
|
|
163
|
+
prefix=prefix,
|
|
164
|
+
description=desc,
|
|
165
|
+
tags=file_tags,
|
|
166
|
+
metadata=file_metadata,
|
|
167
|
+
to=to
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def save(
|
|
171
|
+
self,
|
|
172
|
+
content: Any = None,
|
|
173
|
+
*,
|
|
174
|
+
to: Optional[str] = None,
|
|
175
|
+
description: Optional[str] = None,
|
|
176
|
+
tags: Optional[List[str]] = None,
|
|
177
|
+
metadata: Optional[Dict[str, Any]] = None
|
|
178
|
+
) -> Dict[str, Any]:
|
|
179
|
+
"""
|
|
180
|
+
Unified save method that handles different content types.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
content: Content to save - can be:
|
|
184
|
+
- str (file path): Uploads existing file
|
|
185
|
+
- bytes: Saves as binary blob (requires 'to' parameter)
|
|
186
|
+
- dict/list: Saves as JSON (requires 'to' parameter)
|
|
187
|
+
- None: Uses file_path from constructor (backwards compatibility)
|
|
188
|
+
to: Target filename (required for bytes/dict/list, optional for file paths)
|
|
189
|
+
description: Optional description
|
|
190
|
+
tags: Optional list of tags
|
|
191
|
+
metadata: Optional metadata dict
|
|
86
192
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
193
|
+
Returns:
|
|
194
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
195
|
+
"""
|
|
196
|
+
# Override builder metadata if provided
|
|
197
|
+
if description is not None:
|
|
198
|
+
self._description = description
|
|
199
|
+
if tags is not None:
|
|
200
|
+
self._tags = tags
|
|
201
|
+
if metadata is not None:
|
|
202
|
+
self._metadata = metadata
|
|
203
|
+
|
|
204
|
+
# Backwards compatibility: use file_path from constructor if no content provided
|
|
205
|
+
if content is None:
|
|
206
|
+
if self._file_path:
|
|
207
|
+
content = self._file_path
|
|
208
|
+
else:
|
|
209
|
+
raise ValueError("No content provided and no file_path set in constructor")
|
|
210
|
+
|
|
211
|
+
# Check if content is a file path
|
|
212
|
+
if isinstance(content, str) and Path(content).exists():
|
|
213
|
+
return self.upload(content, to=to)
|
|
214
|
+
|
|
215
|
+
# Check if content is bytes
|
|
216
|
+
if isinstance(content, bytes):
|
|
217
|
+
if not to:
|
|
218
|
+
raise ValueError("'to' parameter is required when saving bytes")
|
|
219
|
+
return self.save_blob(content, to=to)
|
|
220
|
+
|
|
221
|
+
# Check if content is dict or list (save as JSON)
|
|
222
|
+
if isinstance(content, (dict, list)):
|
|
223
|
+
if not to:
|
|
224
|
+
raise ValueError("'to' parameter is required when saving dict/list")
|
|
225
|
+
return self.save_json(content, to=to)
|
|
226
|
+
|
|
227
|
+
raise ValueError(f"Unsupported content type: {type(content)}. Expected str (file path), bytes, dict, or list.")
|
|
228
|
+
|
|
229
|
+
def _save_file(
|
|
230
|
+
self,
|
|
231
|
+
fpath: str,
|
|
232
|
+
prefix: str,
|
|
233
|
+
description: Optional[str],
|
|
234
|
+
tags: Optional[List[str]],
|
|
235
|
+
metadata: Optional[Dict[str, Any]],
|
|
236
|
+
to: Optional[str] = None
|
|
237
|
+
) -> Dict[str, Any]:
|
|
238
|
+
"""Internal method to save an existing file."""
|
|
239
|
+
fpath_obj = Path(fpath)
|
|
240
|
+
if not fpath_obj.exists():
|
|
241
|
+
raise ValueError(f"File not found: {fpath}")
|
|
90
242
|
|
|
91
|
-
if not
|
|
92
|
-
raise ValueError(f"Path is not a file: {
|
|
243
|
+
if not fpath_obj.is_file():
|
|
244
|
+
raise ValueError(f"Path is not a file: {fpath}")
|
|
93
245
|
|
|
94
246
|
# Check file size (max 100GB)
|
|
95
|
-
file_size =
|
|
247
|
+
file_size = fpath_obj.stat().st_size
|
|
96
248
|
MAX_FILE_SIZE = 100 * 1024 * 1024 * 1024 # 100GB in bytes
|
|
97
249
|
if file_size > MAX_FILE_SIZE:
|
|
98
250
|
raise ValueError(f"File size ({file_size} bytes) exceeds 100GB limit")
|
|
99
251
|
|
|
100
252
|
# Compute checksum
|
|
101
|
-
checksum = compute_sha256(str(
|
|
253
|
+
checksum = compute_sha256(str(fpath_obj))
|
|
102
254
|
|
|
103
255
|
# Detect MIME type
|
|
104
|
-
content_type = get_mime_type(str(
|
|
256
|
+
content_type = get_mime_type(str(fpath_obj))
|
|
105
257
|
|
|
106
|
-
# Get filename
|
|
107
|
-
filename =
|
|
258
|
+
# Get filename (use provided 'to' or original)
|
|
259
|
+
filename = to if to else fpath_obj.name
|
|
108
260
|
|
|
109
261
|
# Upload through experiment
|
|
110
262
|
return self._experiment._upload_file(
|
|
111
|
-
file_path=str(
|
|
112
|
-
prefix=
|
|
263
|
+
file_path=str(fpath_obj),
|
|
264
|
+
prefix=prefix,
|
|
113
265
|
filename=filename,
|
|
114
|
-
description=
|
|
115
|
-
tags=
|
|
116
|
-
metadata=
|
|
266
|
+
description=description,
|
|
267
|
+
tags=tags or [],
|
|
268
|
+
metadata=metadata,
|
|
117
269
|
checksum=checksum,
|
|
118
270
|
content_type=content_type,
|
|
119
271
|
size_bytes=file_size
|
|
120
272
|
)
|
|
121
273
|
|
|
122
|
-
def
|
|
274
|
+
def _save_bytes(
|
|
275
|
+
self,
|
|
276
|
+
data: bytes,
|
|
277
|
+
filename: str,
|
|
278
|
+
prefix: str,
|
|
279
|
+
description: Optional[str],
|
|
280
|
+
tags: Optional[List[str]],
|
|
281
|
+
metadata: Optional[Dict[str, Any]]
|
|
282
|
+
) -> Dict[str, Any]:
|
|
283
|
+
"""Save bytes data to a file."""
|
|
284
|
+
import tempfile
|
|
285
|
+
import os
|
|
286
|
+
|
|
287
|
+
temp_dir = tempfile.mkdtemp()
|
|
288
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
289
|
+
# Create parent directories if filename contains path
|
|
290
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
291
|
+
try:
|
|
292
|
+
with open(temp_path, 'wb') as f:
|
|
293
|
+
f.write(data)
|
|
294
|
+
return self._save_file(
|
|
295
|
+
fpath=temp_path,
|
|
296
|
+
prefix=prefix,
|
|
297
|
+
description=description,
|
|
298
|
+
tags=tags,
|
|
299
|
+
metadata=metadata
|
|
300
|
+
)
|
|
301
|
+
finally:
|
|
302
|
+
try:
|
|
303
|
+
os.unlink(temp_path)
|
|
304
|
+
os.rmdir(temp_dir)
|
|
305
|
+
except Exception:
|
|
306
|
+
pass
|
|
307
|
+
|
|
308
|
+
def _save_json(
|
|
309
|
+
self,
|
|
310
|
+
content: Any,
|
|
311
|
+
filename: str,
|
|
312
|
+
prefix: str,
|
|
313
|
+
description: Optional[str],
|
|
314
|
+
tags: Optional[List[str]],
|
|
315
|
+
metadata: Optional[Dict[str, Any]]
|
|
316
|
+
) -> Dict[str, Any]:
|
|
317
|
+
"""Save JSON content to a file."""
|
|
318
|
+
import json
|
|
319
|
+
import tempfile
|
|
320
|
+
import os
|
|
321
|
+
|
|
322
|
+
temp_dir = tempfile.mkdtemp()
|
|
323
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
324
|
+
# Create parent directories if filename contains path
|
|
325
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
326
|
+
try:
|
|
327
|
+
with open(temp_path, 'w') as f:
|
|
328
|
+
json.dump(content, f, indent=2)
|
|
329
|
+
return self._save_file(
|
|
330
|
+
fpath=temp_path,
|
|
331
|
+
prefix=prefix,
|
|
332
|
+
description=description,
|
|
333
|
+
tags=tags,
|
|
334
|
+
metadata=metadata
|
|
335
|
+
)
|
|
336
|
+
finally:
|
|
337
|
+
try:
|
|
338
|
+
os.unlink(temp_path)
|
|
339
|
+
os.rmdir(temp_dir)
|
|
340
|
+
except Exception:
|
|
341
|
+
pass
|
|
342
|
+
|
|
343
|
+
def _save_torch(
|
|
344
|
+
self,
|
|
345
|
+
model: Any,
|
|
346
|
+
filename: str,
|
|
347
|
+
prefix: str,
|
|
348
|
+
description: Optional[str],
|
|
349
|
+
tags: Optional[List[str]],
|
|
350
|
+
metadata: Optional[Dict[str, Any]]
|
|
351
|
+
) -> Dict[str, Any]:
|
|
352
|
+
"""Save PyTorch model to a file."""
|
|
353
|
+
import tempfile
|
|
354
|
+
import os
|
|
355
|
+
import torch
|
|
356
|
+
|
|
357
|
+
temp_dir = tempfile.mkdtemp()
|
|
358
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
359
|
+
# Create parent directories if filename contains path
|
|
360
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
361
|
+
try:
|
|
362
|
+
torch.save(model, temp_path)
|
|
363
|
+
return self._save_file(
|
|
364
|
+
fpath=temp_path,
|
|
365
|
+
prefix=prefix,
|
|
366
|
+
description=description,
|
|
367
|
+
tags=tags,
|
|
368
|
+
metadata=metadata
|
|
369
|
+
)
|
|
370
|
+
finally:
|
|
371
|
+
try:
|
|
372
|
+
os.unlink(temp_path)
|
|
373
|
+
os.rmdir(temp_dir)
|
|
374
|
+
except Exception:
|
|
375
|
+
pass
|
|
376
|
+
|
|
377
|
+
def _save_fig(
|
|
378
|
+
self,
|
|
379
|
+
fig: Any,
|
|
380
|
+
filename: str,
|
|
381
|
+
prefix: str,
|
|
382
|
+
description: Optional[str],
|
|
383
|
+
tags: Optional[List[str]],
|
|
384
|
+
metadata: Optional[Dict[str, Any]],
|
|
385
|
+
**kwargs
|
|
386
|
+
) -> Dict[str, Any]:
|
|
387
|
+
"""Save matplotlib figure to a file."""
|
|
388
|
+
import tempfile
|
|
389
|
+
import os
|
|
390
|
+
import matplotlib.pyplot as plt
|
|
391
|
+
|
|
392
|
+
temp_dir = tempfile.mkdtemp()
|
|
393
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
394
|
+
# Create parent directories if filename contains path
|
|
395
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
396
|
+
try:
|
|
397
|
+
fig.savefig(temp_path, **kwargs)
|
|
398
|
+
plt.close(fig)
|
|
399
|
+
return self._save_file(
|
|
400
|
+
fpath=temp_path,
|
|
401
|
+
prefix=prefix,
|
|
402
|
+
description=description,
|
|
403
|
+
tags=tags,
|
|
404
|
+
metadata=metadata
|
|
405
|
+
)
|
|
406
|
+
finally:
|
|
407
|
+
try:
|
|
408
|
+
os.unlink(temp_path)
|
|
409
|
+
os.rmdir(temp_dir)
|
|
410
|
+
except Exception:
|
|
411
|
+
pass
|
|
412
|
+
|
|
413
|
+
def _save_pickle(
|
|
414
|
+
self,
|
|
415
|
+
content: Any,
|
|
416
|
+
filename: str,
|
|
417
|
+
prefix: str,
|
|
418
|
+
description: Optional[str],
|
|
419
|
+
tags: Optional[List[str]],
|
|
420
|
+
metadata: Optional[Dict[str, Any]]
|
|
421
|
+
) -> Dict[str, Any]:
|
|
422
|
+
"""Save Python object to a pickle file."""
|
|
423
|
+
import pickle
|
|
424
|
+
import tempfile
|
|
425
|
+
import os
|
|
426
|
+
|
|
427
|
+
temp_dir = tempfile.mkdtemp()
|
|
428
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
429
|
+
# Create parent directories if filename contains path
|
|
430
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
431
|
+
try:
|
|
432
|
+
with open(temp_path, 'wb') as f:
|
|
433
|
+
pickle.dump(content, f)
|
|
434
|
+
return self._save_file(
|
|
435
|
+
fpath=temp_path,
|
|
436
|
+
prefix=prefix,
|
|
437
|
+
description=description,
|
|
438
|
+
tags=tags,
|
|
439
|
+
metadata=metadata
|
|
440
|
+
)
|
|
441
|
+
finally:
|
|
442
|
+
try:
|
|
443
|
+
os.unlink(temp_path)
|
|
444
|
+
os.rmdir(temp_dir)
|
|
445
|
+
except Exception:
|
|
446
|
+
pass
|
|
447
|
+
|
|
448
|
+
def list(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
123
449
|
"""
|
|
124
|
-
List files with optional
|
|
450
|
+
List files with optional glob pattern filtering.
|
|
451
|
+
|
|
452
|
+
Args:
|
|
453
|
+
pattern: Optional glob pattern to filter files (e.g., "*.png", "model_*.pt")
|
|
125
454
|
|
|
126
455
|
Returns:
|
|
127
456
|
List of file metadata dicts
|
|
@@ -131,62 +460,169 @@ class FileBuilder:
|
|
|
131
460
|
|
|
132
461
|
Examples:
|
|
133
462
|
files = experiment.files().list() # All files
|
|
134
|
-
files = experiment.files(
|
|
135
|
-
files = experiment.files(
|
|
463
|
+
files = experiment.files("/models").list() # Files in /models prefix
|
|
464
|
+
files = experiment.files("images").list("*.png") # PNG files in images
|
|
465
|
+
files = experiment.files().list("**/*.pt") # All .pt files
|
|
136
466
|
"""
|
|
137
467
|
if not self._experiment._is_open:
|
|
138
468
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
139
469
|
|
|
140
|
-
|
|
141
|
-
|
|
470
|
+
# Determine prefix filter - support both new (path) and old (prefix) API
|
|
471
|
+
prefix = None
|
|
472
|
+
if self._path:
|
|
473
|
+
prefix = '/' + self._path.lstrip('/')
|
|
474
|
+
elif self._prefix and self._prefix != '/':
|
|
475
|
+
prefix = self._prefix
|
|
476
|
+
|
|
477
|
+
# Get all files matching prefix
|
|
478
|
+
files = self._experiment._list_files(
|
|
479
|
+
prefix=prefix,
|
|
142
480
|
tags=self._tags if self._tags else None
|
|
143
481
|
)
|
|
144
482
|
|
|
145
|
-
|
|
483
|
+
# Apply glob pattern if provided
|
|
484
|
+
if pattern:
|
|
485
|
+
pattern = pattern.lstrip('/')
|
|
486
|
+
filtered = []
|
|
487
|
+
for f in files:
|
|
488
|
+
filename = f.get('filename', '')
|
|
489
|
+
full_path = f.get('path', '/').rstrip('/') + '/' + filename
|
|
490
|
+
full_path = full_path.lstrip('/')
|
|
491
|
+
|
|
492
|
+
# Match against filename or full path
|
|
493
|
+
if fnmatch.fnmatch(filename, pattern) or fnmatch.fnmatch(full_path, pattern):
|
|
494
|
+
filtered.append(f)
|
|
495
|
+
return filtered
|
|
496
|
+
|
|
497
|
+
return files
|
|
498
|
+
|
|
499
|
+
def download(
|
|
500
|
+
self,
|
|
501
|
+
pattern: Optional[str] = None,
|
|
502
|
+
*,
|
|
503
|
+
to: Optional[str] = None
|
|
504
|
+
) -> Union[str, List[str]]:
|
|
146
505
|
"""
|
|
147
|
-
Download file with automatic checksum verification.
|
|
506
|
+
Download file(s) with automatic checksum verification.
|
|
148
507
|
|
|
149
|
-
|
|
508
|
+
Args:
|
|
509
|
+
pattern: Optional glob pattern for batch download (e.g., "*.png")
|
|
510
|
+
to: Destination path. For single files, this is the file path.
|
|
511
|
+
For patterns, this is the destination directory.
|
|
150
512
|
|
|
151
513
|
Returns:
|
|
152
|
-
Path to downloaded file
|
|
514
|
+
For single file: Path to downloaded file
|
|
515
|
+
For pattern: List of paths to downloaded files
|
|
153
516
|
|
|
154
517
|
Raises:
|
|
155
518
|
RuntimeError: If experiment is not open
|
|
156
|
-
ValueError: If
|
|
157
|
-
ValueError: If checksum verification fails
|
|
519
|
+
ValueError: If file not found or checksum verification fails
|
|
158
520
|
|
|
159
521
|
Examples:
|
|
160
|
-
# Download
|
|
522
|
+
# Download single file
|
|
523
|
+
path = experiment.files("model.pt").download()
|
|
524
|
+
path = experiment.files("model.pt").download(to="./local_model.pt")
|
|
525
|
+
|
|
526
|
+
# Download by file ID (backwards compatibility)
|
|
161
527
|
path = experiment.files(file_id="123").download()
|
|
162
528
|
|
|
163
|
-
# Download
|
|
164
|
-
|
|
529
|
+
# Download multiple files matching pattern
|
|
530
|
+
paths = experiment.files("images").download("*.png")
|
|
531
|
+
paths = experiment.files("images").download("*.png", to="./local_images")
|
|
165
532
|
"""
|
|
166
533
|
if not self._experiment._is_open:
|
|
167
534
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
168
535
|
|
|
169
|
-
|
|
170
|
-
|
|
536
|
+
# If file_id is set (backwards compatibility)
|
|
537
|
+
if self._file_id:
|
|
538
|
+
return self._experiment._download_file(
|
|
539
|
+
file_id=self._file_id,
|
|
540
|
+
dest_path=to or self._dest_path
|
|
541
|
+
)
|
|
171
542
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
543
|
+
# If pattern is provided, download multiple files
|
|
544
|
+
if pattern:
|
|
545
|
+
files = self.list(pattern)
|
|
546
|
+
if not files:
|
|
547
|
+
raise ValueError(f"No files found matching pattern: {pattern}")
|
|
548
|
+
|
|
549
|
+
downloaded = []
|
|
550
|
+
dest_dir = Path(to) if to else Path('.')
|
|
551
|
+
if to and not dest_dir.exists():
|
|
552
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
553
|
+
|
|
554
|
+
for f in files:
|
|
555
|
+
file_id = f.get('id')
|
|
556
|
+
filename = f.get('filename', 'file')
|
|
557
|
+
dest_path = str(dest_dir / filename)
|
|
558
|
+
path = self._experiment._download_file(
|
|
559
|
+
file_id=file_id,
|
|
560
|
+
dest_path=dest_path
|
|
561
|
+
)
|
|
562
|
+
downloaded.append(path)
|
|
563
|
+
|
|
564
|
+
return downloaded
|
|
565
|
+
|
|
566
|
+
# Download single file by path
|
|
567
|
+
if self._path:
|
|
568
|
+
# Find file by path
|
|
569
|
+
files = self._experiment._list_files(prefix=None, tags=None)
|
|
570
|
+
matching = []
|
|
571
|
+
search_path = self._path.lstrip('/')
|
|
572
|
+
|
|
573
|
+
for f in files:
|
|
574
|
+
filename = f.get('filename', '')
|
|
575
|
+
prefix = f.get('path', '/').lstrip('/')
|
|
576
|
+
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
577
|
+
full_path = full_path.lstrip('/')
|
|
578
|
+
|
|
579
|
+
if full_path == search_path or filename == search_path:
|
|
580
|
+
matching.append(f)
|
|
581
|
+
|
|
582
|
+
if not matching:
|
|
583
|
+
raise ValueError(f"File not found: {self._path}")
|
|
584
|
+
|
|
585
|
+
if len(matching) > 1:
|
|
586
|
+
# If multiple matches, prefer exact path match
|
|
587
|
+
exact = [f for f in matching if
|
|
588
|
+
(f.get('path', '/').lstrip('/').rstrip('/') + '/' + f.get('filename', '')).lstrip('/') == search_path]
|
|
589
|
+
if exact:
|
|
590
|
+
matching = exact[:1]
|
|
591
|
+
else:
|
|
592
|
+
matching = matching[:1]
|
|
593
|
+
|
|
594
|
+
file_info = matching[0]
|
|
595
|
+
return self._experiment._download_file(
|
|
596
|
+
file_id=file_info['id'],
|
|
597
|
+
dest_path=to
|
|
598
|
+
)
|
|
599
|
+
|
|
600
|
+
raise ValueError("No file path or pattern specified")
|
|
176
601
|
|
|
177
|
-
def delete(self) -> Dict[str, Any]:
|
|
602
|
+
def delete(self, pattern: Optional[str] = None) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
|
|
178
603
|
"""
|
|
179
|
-
Delete file (soft delete).
|
|
604
|
+
Delete file(s) (soft delete).
|
|
605
|
+
|
|
606
|
+
Args:
|
|
607
|
+
pattern: Optional glob pattern for batch delete (e.g., "*.png")
|
|
180
608
|
|
|
181
609
|
Returns:
|
|
182
|
-
Dict with id and deletedAt timestamp
|
|
610
|
+
For single file: Dict with id and deletedAt timestamp
|
|
611
|
+
For pattern: List of deletion results
|
|
183
612
|
|
|
184
613
|
Raises:
|
|
185
614
|
RuntimeError: If experiment is not open or write-protected
|
|
186
|
-
ValueError: If
|
|
615
|
+
ValueError: If file not found
|
|
187
616
|
|
|
188
617
|
Examples:
|
|
618
|
+
# Delete single file
|
|
619
|
+
result = experiment.files("some.text").delete()
|
|
620
|
+
|
|
621
|
+
# Delete by file ID (backwards compatibility)
|
|
189
622
|
result = experiment.files(file_id="123").delete()
|
|
623
|
+
|
|
624
|
+
# Delete multiple files matching pattern
|
|
625
|
+
results = experiment.files("images").delete("*.png")
|
|
190
626
|
"""
|
|
191
627
|
if not self._experiment._is_open:
|
|
192
628
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
@@ -194,10 +630,52 @@ class FileBuilder:
|
|
|
194
630
|
if self._experiment._write_protected:
|
|
195
631
|
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
196
632
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
633
|
+
# If file_id is set (backwards compatibility)
|
|
634
|
+
if self._file_id:
|
|
635
|
+
return self._experiment._delete_file(file_id=self._file_id)
|
|
636
|
+
|
|
637
|
+
# If pattern is provided, delete multiple files
|
|
638
|
+
if pattern:
|
|
639
|
+
files = self.list(pattern)
|
|
640
|
+
if not files:
|
|
641
|
+
raise ValueError(f"No files found matching pattern: {pattern}")
|
|
642
|
+
|
|
643
|
+
results = []
|
|
644
|
+
for f in files:
|
|
645
|
+
file_id = f.get('id')
|
|
646
|
+
result = self._experiment._delete_file(file_id=file_id)
|
|
647
|
+
results.append(result)
|
|
648
|
+
return results
|
|
649
|
+
|
|
650
|
+
# Delete single file by path
|
|
651
|
+
if self._path:
|
|
652
|
+
files = self._experiment._list_files(prefix=None, tags=None)
|
|
653
|
+
matching = []
|
|
654
|
+
search_path = self._path.lstrip('/')
|
|
655
|
+
|
|
656
|
+
for f in files:
|
|
657
|
+
filename = f.get('filename', '')
|
|
658
|
+
prefix = f.get('path', '/').lstrip('/')
|
|
659
|
+
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
660
|
+
full_path = full_path.lstrip('/')
|
|
661
|
+
|
|
662
|
+
if full_path == search_path or filename == search_path:
|
|
663
|
+
matching.append(f)
|
|
664
|
+
|
|
665
|
+
if not matching:
|
|
666
|
+
raise ValueError(f"File not found: {self._path}")
|
|
667
|
+
|
|
668
|
+
# Delete all matching files
|
|
669
|
+
if len(matching) == 1:
|
|
670
|
+
return self._experiment._delete_file(file_id=matching[0]['id'])
|
|
671
|
+
|
|
672
|
+
results = []
|
|
673
|
+
for f in matching:
|
|
674
|
+
result = self._experiment._delete_file(file_id=f['id'])
|
|
675
|
+
results.append(result)
|
|
676
|
+
return results
|
|
677
|
+
|
|
678
|
+
raise ValueError("No file path or pattern specified")
|
|
201
679
|
|
|
202
680
|
def update(self) -> Dict[str, Any]:
|
|
203
681
|
"""
|
|
@@ -234,271 +712,222 @@ class FileBuilder:
|
|
|
234
712
|
metadata=self._metadata
|
|
235
713
|
)
|
|
236
714
|
|
|
237
|
-
|
|
715
|
+
# Convenience methods for specific file types
|
|
716
|
+
|
|
717
|
+
def save_json(
|
|
718
|
+
self,
|
|
719
|
+
content: Any,
|
|
720
|
+
*,
|
|
721
|
+
to: str
|
|
722
|
+
) -> Dict[str, Any]:
|
|
238
723
|
"""
|
|
239
724
|
Save JSON content to a file.
|
|
240
725
|
|
|
241
726
|
Args:
|
|
242
727
|
content: Content to save as JSON (dict, list, or any JSON-serializable object)
|
|
243
|
-
|
|
728
|
+
to: Target filename
|
|
244
729
|
|
|
245
730
|
Returns:
|
|
246
731
|
File metadata dict with id, path, filename, checksum, etc.
|
|
247
732
|
|
|
248
|
-
Raises:
|
|
249
|
-
RuntimeError: If experiment is not open or write-protected
|
|
250
|
-
ValueError: If content is not JSON-serializable
|
|
251
|
-
|
|
252
733
|
Examples:
|
|
253
734
|
config = {"model": "resnet50", "lr": 0.001}
|
|
254
|
-
result =
|
|
735
|
+
result = dxp.files("configs").save_json(config, to="config.json")
|
|
255
736
|
"""
|
|
256
|
-
|
|
257
|
-
import tempfile
|
|
258
|
-
import os
|
|
259
|
-
|
|
260
|
-
if not self._experiment._is_open:
|
|
261
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
737
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
262
738
|
|
|
263
|
-
if
|
|
264
|
-
|
|
739
|
+
# Extract path component from 'to' if present (e.g., "configs/settings.json")
|
|
740
|
+
import os
|
|
741
|
+
to_dirname = os.path.dirname(to)
|
|
742
|
+
to_filename = os.path.basename(to)
|
|
743
|
+
if to_dirname:
|
|
744
|
+
# Merge the path component into prefix
|
|
745
|
+
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
746
|
+
|
|
747
|
+
return self._save_json(
|
|
748
|
+
content=content,
|
|
749
|
+
filename=to_filename,
|
|
750
|
+
prefix=prefix,
|
|
751
|
+
description=self._description,
|
|
752
|
+
tags=self._tags,
|
|
753
|
+
metadata=self._metadata
|
|
754
|
+
)
|
|
265
755
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
try:
|
|
270
|
-
# Write JSON content to temp file
|
|
271
|
-
with open(temp_path, 'w') as f:
|
|
272
|
-
json.dump(content, f, indent=2)
|
|
756
|
+
def save_text(self, content: str, *, to: str) -> Dict[str, Any]:
|
|
757
|
+
"""
|
|
758
|
+
Save text content to a file.
|
|
273
759
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
760
|
+
Args:
|
|
761
|
+
content: Text content to save
|
|
762
|
+
to: Target filename
|
|
277
763
|
|
|
278
|
-
|
|
279
|
-
|
|
764
|
+
Returns:
|
|
765
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
280
766
|
|
|
281
|
-
|
|
282
|
-
|
|
767
|
+
Examples:
|
|
768
|
+
result = experiment.files().save_text("Hello, world!", to="greeting.txt")
|
|
769
|
+
result = experiment.files("configs").save_text(yaml_content, to="view.yaml")
|
|
770
|
+
"""
|
|
771
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
283
772
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
773
|
+
# Extract path component from 'to' if present
|
|
774
|
+
import os
|
|
775
|
+
to_dirname = os.path.dirname(to)
|
|
776
|
+
to_filename = os.path.basename(to)
|
|
777
|
+
if to_dirname:
|
|
778
|
+
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
779
|
+
|
|
780
|
+
return self._save_bytes(
|
|
781
|
+
data=content.encode('utf-8'),
|
|
782
|
+
filename=to_filename,
|
|
783
|
+
prefix=prefix,
|
|
784
|
+
description=self._description,
|
|
785
|
+
tags=self._tags,
|
|
786
|
+
metadata=self._metadata
|
|
787
|
+
)
|
|
292
788
|
|
|
293
|
-
def
|
|
789
|
+
def save_blob(self, data: bytes, *, to: str) -> Dict[str, Any]:
|
|
294
790
|
"""
|
|
295
|
-
Save
|
|
791
|
+
Save binary data to a file.
|
|
296
792
|
|
|
297
793
|
Args:
|
|
298
|
-
|
|
299
|
-
|
|
794
|
+
data: Binary data to save
|
|
795
|
+
to: Target filename
|
|
300
796
|
|
|
301
797
|
Returns:
|
|
302
798
|
File metadata dict with id, path, filename, checksum, etc.
|
|
303
799
|
|
|
304
|
-
Raises:
|
|
305
|
-
RuntimeError: If experiment is not open or write-protected
|
|
306
|
-
ImportError: If torch is not installed
|
|
307
|
-
ValueError: If model cannot be saved
|
|
308
|
-
|
|
309
800
|
Examples:
|
|
310
|
-
|
|
311
|
-
model = torch.nn.Linear(10, 5)
|
|
312
|
-
result = experiment.files(prefix="/models").save_torch(model, "model.pt")
|
|
313
|
-
|
|
314
|
-
# Or save state dict
|
|
315
|
-
result = experiment.files(prefix="/models").save_torch(model.state_dict(), "model.pth")
|
|
801
|
+
result = experiment.files("data").save_blob(binary_data, to="model.bin")
|
|
316
802
|
"""
|
|
317
|
-
|
|
318
|
-
import os
|
|
803
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
319
804
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
805
|
+
# Extract path component from 'to' if present
|
|
806
|
+
import os
|
|
807
|
+
to_dirname = os.path.dirname(to)
|
|
808
|
+
to_filename = os.path.basename(to)
|
|
809
|
+
if to_dirname:
|
|
810
|
+
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
811
|
+
|
|
812
|
+
return self._save_bytes(
|
|
813
|
+
data=data,
|
|
814
|
+
filename=to_filename,
|
|
815
|
+
prefix=prefix,
|
|
816
|
+
description=self._description,
|
|
817
|
+
tags=self._tags,
|
|
818
|
+
metadata=self._metadata
|
|
819
|
+
)
|
|
330
820
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
821
|
+
def save_torch(
|
|
822
|
+
self,
|
|
823
|
+
model: Any,
|
|
824
|
+
*,
|
|
825
|
+
to: str
|
|
826
|
+
) -> Dict[str, Any]:
|
|
827
|
+
"""
|
|
828
|
+
Save PyTorch model to a file.
|
|
334
829
|
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
830
|
+
Args:
|
|
831
|
+
model: PyTorch model or state dict to save
|
|
832
|
+
to: Target filename
|
|
338
833
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
self._file_path = temp_path
|
|
834
|
+
Returns:
|
|
835
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
342
836
|
|
|
343
|
-
|
|
344
|
-
result =
|
|
837
|
+
Examples:
|
|
838
|
+
result = dxp.files("models").save_torch(model, to="model.pt")
|
|
839
|
+
result = dxp.files("models").save_torch(model.state_dict(), to="weights.pth")
|
|
840
|
+
"""
|
|
345
841
|
|
|
346
|
-
|
|
347
|
-
self._file_path = original_file_path
|
|
842
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
348
843
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
844
|
+
return self._save_torch(
|
|
845
|
+
model=model,
|
|
846
|
+
filename=to,
|
|
847
|
+
prefix=prefix,
|
|
848
|
+
description=self._description,
|
|
849
|
+
tags=self._tags,
|
|
850
|
+
metadata=self._metadata
|
|
851
|
+
)
|
|
357
852
|
|
|
358
|
-
def save_pkl(
|
|
853
|
+
def save_pkl(
|
|
854
|
+
self,
|
|
855
|
+
content: Any,
|
|
856
|
+
*,
|
|
857
|
+
to: str
|
|
858
|
+
) -> Dict[str, Any]:
|
|
359
859
|
"""
|
|
360
860
|
Save Python object to a pickle file.
|
|
361
861
|
|
|
362
862
|
Args:
|
|
363
863
|
content: Python object to pickle (must be pickle-serializable)
|
|
364
|
-
|
|
864
|
+
to: Target filename
|
|
365
865
|
|
|
366
866
|
Returns:
|
|
367
867
|
File metadata dict with id, path, filename, checksum, etc.
|
|
368
868
|
|
|
369
|
-
Raises:
|
|
370
|
-
RuntimeError: If experiment is not open or write-protected
|
|
371
|
-
ValueError: If content cannot be pickled
|
|
372
|
-
|
|
373
869
|
Examples:
|
|
374
870
|
data = {"model": "resnet50", "weights": np.array([1, 2, 3])}
|
|
375
|
-
result =
|
|
376
|
-
|
|
377
|
-
# Or save any Python object
|
|
378
|
-
result = experiment.files(prefix="/models").save_pkl(trained_model, "model.pickle")
|
|
871
|
+
result = dxp.files("data").save_pkl(data, to="data.pkl")
|
|
379
872
|
"""
|
|
380
|
-
|
|
381
|
-
import tempfile
|
|
382
|
-
import os
|
|
383
|
-
|
|
384
|
-
if not self._experiment._is_open:
|
|
385
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
386
|
-
|
|
387
|
-
if self._experiment._write_protected:
|
|
388
|
-
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
873
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
389
874
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
# Save using existing save() method
|
|
399
|
-
original_file_path = self._file_path
|
|
400
|
-
self._file_path = temp_path
|
|
401
|
-
|
|
402
|
-
# Upload and get result
|
|
403
|
-
result = self.save()
|
|
404
|
-
|
|
405
|
-
# Restore original file_path
|
|
406
|
-
self._file_path = original_file_path
|
|
407
|
-
|
|
408
|
-
return result
|
|
409
|
-
finally:
|
|
410
|
-
# Clean up temp file and directory
|
|
411
|
-
try:
|
|
412
|
-
os.unlink(temp_path)
|
|
413
|
-
os.rmdir(temp_dir)
|
|
414
|
-
except Exception:
|
|
415
|
-
pass
|
|
875
|
+
return self._save_pickle(
|
|
876
|
+
content=content,
|
|
877
|
+
filename=to,
|
|
878
|
+
prefix=prefix,
|
|
879
|
+
description=self._description,
|
|
880
|
+
tags=self._tags,
|
|
881
|
+
metadata=self._metadata
|
|
882
|
+
)
|
|
416
883
|
|
|
417
|
-
def save_fig(
|
|
884
|
+
def save_fig(
|
|
885
|
+
self,
|
|
886
|
+
fig: Optional[Any] = None,
|
|
887
|
+
*,
|
|
888
|
+
to: str,
|
|
889
|
+
**kwargs
|
|
890
|
+
) -> Dict[str, Any]:
|
|
418
891
|
"""
|
|
419
892
|
Save matplotlib figure to a file.
|
|
420
893
|
|
|
421
894
|
Args:
|
|
422
895
|
fig: Matplotlib figure object. If None, uses plt.gcf() (current figure)
|
|
423
|
-
|
|
424
|
-
**kwargs: Additional arguments passed to fig.savefig()
|
|
425
|
-
- dpi: Resolution (int or 'figure')
|
|
426
|
-
- transparent: Make background transparent (bool)
|
|
427
|
-
- bbox_inches: 'tight' to auto-crop (str or Bbox)
|
|
428
|
-
- quality: JPEG quality 0-100 (int)
|
|
429
|
-
- format: Override format detection (str)
|
|
896
|
+
to: Target filename
|
|
897
|
+
**kwargs: Additional arguments passed to fig.savefig()
|
|
430
898
|
|
|
431
899
|
Returns:
|
|
432
900
|
File metadata dict with id, path, filename, checksum, etc.
|
|
433
901
|
|
|
434
|
-
Raises:
|
|
435
|
-
RuntimeError: If experiment not open or write-protected
|
|
436
|
-
ImportError: If matplotlib not installed
|
|
437
|
-
|
|
438
902
|
Examples:
|
|
439
|
-
import matplotlib.pyplot as plt
|
|
440
|
-
|
|
441
|
-
# Use current figure
|
|
442
903
|
plt.plot([1, 2, 3], [1, 4, 9])
|
|
443
|
-
result =
|
|
444
|
-
|
|
445
|
-
# Specify figure explicitly
|
|
446
|
-
fig, ax = plt.subplots()
|
|
447
|
-
ax.plot([1, 2, 3])
|
|
448
|
-
result = experiment.files(prefix="/plots").save_fig(fig=fig, file_name="plot.pdf", dpi=150)
|
|
904
|
+
result = dxp.files("plots").save_fig(to="plot.png")
|
|
449
905
|
"""
|
|
450
|
-
import tempfile
|
|
451
|
-
import os
|
|
452
|
-
|
|
453
906
|
try:
|
|
454
907
|
import matplotlib.pyplot as plt
|
|
455
908
|
except ImportError:
|
|
456
909
|
raise ImportError("Matplotlib is not installed. Install it with: pip install matplotlib")
|
|
457
910
|
|
|
458
|
-
if not self._experiment._is_open:
|
|
459
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
460
|
-
|
|
461
|
-
if self._experiment._write_protected:
|
|
462
|
-
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
463
|
-
|
|
464
|
-
# Get figure
|
|
465
911
|
if fig is None:
|
|
466
912
|
fig = plt.gcf()
|
|
467
913
|
|
|
468
|
-
|
|
469
|
-
temp_dir = tempfile.mkdtemp()
|
|
470
|
-
temp_path = os.path.join(temp_dir, file_name)
|
|
471
|
-
|
|
472
|
-
try:
|
|
473
|
-
# Save figure to temp file
|
|
474
|
-
fig.savefig(temp_path, **kwargs)
|
|
475
|
-
|
|
476
|
-
# Close figure to prevent memory leaks
|
|
477
|
-
plt.close(fig)
|
|
478
|
-
|
|
479
|
-
# Save using existing save() method
|
|
480
|
-
original_file_path = self._file_path
|
|
481
|
-
self._file_path = temp_path
|
|
482
|
-
|
|
483
|
-
# Upload and get result
|
|
484
|
-
result = self.save()
|
|
914
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
485
915
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
except Exception:
|
|
496
|
-
pass
|
|
916
|
+
return self._save_fig(
|
|
917
|
+
fig=fig,
|
|
918
|
+
filename=to,
|
|
919
|
+
prefix=prefix,
|
|
920
|
+
description=self._description,
|
|
921
|
+
tags=self._tags,
|
|
922
|
+
metadata=self._metadata,
|
|
923
|
+
**kwargs
|
|
924
|
+
)
|
|
497
925
|
|
|
498
926
|
def save_video(
|
|
499
927
|
self,
|
|
500
928
|
frame_stack: Union[List, Any],
|
|
501
|
-
|
|
929
|
+
*,
|
|
930
|
+
to: str,
|
|
502
931
|
fps: int = 20,
|
|
503
932
|
**imageio_kwargs
|
|
504
933
|
) -> Dict[str, Any]:
|
|
@@ -506,42 +935,17 @@ class FileBuilder:
|
|
|
506
935
|
Save video frame stack to a file.
|
|
507
936
|
|
|
508
937
|
Args:
|
|
509
|
-
frame_stack: List of numpy arrays or stacked array
|
|
510
|
-
|
|
938
|
+
frame_stack: List of numpy arrays or stacked array
|
|
939
|
+
to: Target filename
|
|
511
940
|
fps: Frames per second (default: 20)
|
|
512
|
-
**imageio_kwargs: Additional arguments passed to imageio
|
|
513
|
-
- codec: Video codec (e.g., 'libx264', 'h264')
|
|
514
|
-
- quality: Quality level (int, higher is better)
|
|
515
|
-
- pixelformat: Pixel format (e.g., 'yuv420p')
|
|
516
|
-
- macro_block_size: Macro block size for encoding
|
|
941
|
+
**imageio_kwargs: Additional arguments passed to imageio
|
|
517
942
|
|
|
518
943
|
Returns:
|
|
519
944
|
File metadata dict with id, path, filename, checksum, etc.
|
|
520
945
|
|
|
521
|
-
Raises:
|
|
522
|
-
RuntimeError: If experiment not open or write-protected
|
|
523
|
-
ImportError: If imageio or scikit-image not installed
|
|
524
|
-
ValueError: If frame_stack is empty or invalid format
|
|
525
|
-
|
|
526
946
|
Examples:
|
|
527
|
-
import numpy as np
|
|
528
|
-
|
|
529
|
-
# Grayscale frames (float values 0-1)
|
|
530
947
|
frames = [np.random.rand(480, 640) for _ in range(30)]
|
|
531
|
-
result =
|
|
532
|
-
|
|
533
|
-
# RGB frames with custom FPS
|
|
534
|
-
frames = [np.random.rand(480, 640, 3) for _ in range(60)]
|
|
535
|
-
result = experiment.files(prefix="/videos").save_video(frames, "output.mp4", fps=30)
|
|
536
|
-
|
|
537
|
-
# Save as GIF
|
|
538
|
-
frames = [np.random.rand(200, 200) for _ in range(20)]
|
|
539
|
-
result = experiment.files(prefix="/videos").save_video(frames, "animation.gif")
|
|
540
|
-
|
|
541
|
-
# With custom codec and quality
|
|
542
|
-
result = experiment.files(prefix="/videos").save_video(
|
|
543
|
-
frames, "output.mp4", fps=30, codec='libx264', quality=8
|
|
544
|
-
)
|
|
948
|
+
result = dxp.files("videos").save_video(frames, to="output.mp4")
|
|
545
949
|
"""
|
|
546
950
|
import tempfile
|
|
547
951
|
import os
|
|
@@ -556,51 +960,37 @@ class FileBuilder:
|
|
|
556
960
|
except ImportError:
|
|
557
961
|
raise ImportError("scikit-image is not installed. Install it with: pip install scikit-image")
|
|
558
962
|
|
|
559
|
-
if not self._experiment._is_open:
|
|
560
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
561
|
-
|
|
562
|
-
if self._experiment._write_protected:
|
|
563
|
-
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
564
|
-
|
|
565
963
|
# Validate frame_stack
|
|
566
964
|
try:
|
|
567
|
-
# Handle both list and numpy array
|
|
568
965
|
if len(frame_stack) == 0:
|
|
569
966
|
raise ValueError("frame_stack is empty")
|
|
570
967
|
except TypeError:
|
|
571
968
|
raise ValueError("frame_stack must be a list or numpy array")
|
|
572
969
|
|
|
573
|
-
|
|
970
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
971
|
+
|
|
574
972
|
temp_dir = tempfile.mkdtemp()
|
|
575
|
-
temp_path = os.path.join(temp_dir,
|
|
973
|
+
temp_path = os.path.join(temp_dir, to)
|
|
974
|
+
# Create parent directories if filename contains path
|
|
975
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
576
976
|
|
|
577
977
|
try:
|
|
578
|
-
# Convert frames to uint8 format (handles float32/64, grayscale, RGB, etc.)
|
|
579
|
-
# img_as_ubyte automatically scales [0.0, 1.0] floats to [0, 255] uint8
|
|
580
978
|
frames_ubyte = img_as_ubyte(frame_stack)
|
|
581
|
-
|
|
582
|
-
# Encode video to temp file
|
|
583
979
|
try:
|
|
584
980
|
iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
|
|
585
981
|
except iio.core.NeedDownloadError:
|
|
586
|
-
# Auto-download FFmpeg if not available
|
|
587
982
|
import imageio.plugins.ffmpeg
|
|
588
983
|
imageio.plugins.ffmpeg.download()
|
|
589
984
|
iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
|
|
590
985
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
# Restore original file_path
|
|
599
|
-
self._file_path = original_file_path
|
|
600
|
-
|
|
601
|
-
return result
|
|
986
|
+
return self._save_file(
|
|
987
|
+
fpath=temp_path,
|
|
988
|
+
prefix=prefix,
|
|
989
|
+
description=self._description,
|
|
990
|
+
tags=self._tags,
|
|
991
|
+
metadata=self._metadata
|
|
992
|
+
)
|
|
602
993
|
finally:
|
|
603
|
-
# Clean up temp file and directory
|
|
604
994
|
try:
|
|
605
995
|
os.unlink(temp_path)
|
|
606
996
|
os.rmdir(temp_dir)
|
|
@@ -611,31 +1001,16 @@ class FileBuilder:
|
|
|
611
1001
|
"""
|
|
612
1002
|
Duplicate an existing file to a new path within the same experiment.
|
|
613
1003
|
|
|
614
|
-
Useful for checkpoint rotation patterns where you save versioned checkpoints
|
|
615
|
-
and maintain a "latest" or "best" pointer.
|
|
616
|
-
|
|
617
1004
|
Args:
|
|
618
1005
|
source: Source file - either file ID (str) or metadata dict with 'id' key
|
|
619
1006
|
to: Target path like "models/latest.pt" or "/checkpoints/best.pt"
|
|
620
1007
|
|
|
621
1008
|
Returns:
|
|
622
|
-
File metadata dict for the duplicated file
|
|
623
|
-
|
|
624
|
-
Raises:
|
|
625
|
-
RuntimeError: If experiment is not open or write-protected
|
|
626
|
-
ValueError: If source file not found or target path invalid
|
|
1009
|
+
File metadata dict for the duplicated file
|
|
627
1010
|
|
|
628
1011
|
Examples:
|
|
629
|
-
|
|
630
|
-
dxp.files().duplicate("file-id-123", to="models/latest.pt")
|
|
631
|
-
|
|
632
|
-
# Using metadata dict from save_torch
|
|
633
|
-
snapshot = dxp.files(prefix="/models").save_torch(model, f"model_{epoch:05d}.pt")
|
|
1012
|
+
snapshot = dxp.files("models").save_torch(model, to=f"model_{epoch:05d}.pt")
|
|
634
1013
|
dxp.files().duplicate(snapshot, to="models/latest.pt")
|
|
635
|
-
|
|
636
|
-
# Checkpoint rotation pattern
|
|
637
|
-
snap = dxp.files(prefix="/checkpoints").save_torch(model, f"model_{epoch:05d}.pt")
|
|
638
|
-
dxp.files().duplicate(snap, to="checkpoints/best.pt")
|
|
639
1014
|
"""
|
|
640
1015
|
import tempfile
|
|
641
1016
|
import os
|
|
@@ -669,34 +1044,127 @@ class FileBuilder:
|
|
|
669
1044
|
if not target_filename:
|
|
670
1045
|
raise ValueError(f"Invalid target path '{to}': must include filename")
|
|
671
1046
|
|
|
672
|
-
# Download source file to temp location
|
|
673
1047
|
temp_dir = tempfile.mkdtemp()
|
|
674
1048
|
temp_path = os.path.join(temp_dir, target_filename)
|
|
675
1049
|
|
|
676
1050
|
try:
|
|
677
|
-
# Download the source file
|
|
678
1051
|
downloaded_path = self._experiment._download_file(
|
|
679
1052
|
file_id=source_id,
|
|
680
1053
|
dest_path=temp_path
|
|
681
1054
|
)
|
|
682
1055
|
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
1056
|
+
return self._save_file(
|
|
1057
|
+
fpath=downloaded_path,
|
|
1058
|
+
prefix=target_prefix,
|
|
1059
|
+
description=self._description,
|
|
1060
|
+
tags=self._tags,
|
|
1061
|
+
metadata=self._metadata
|
|
1062
|
+
)
|
|
1063
|
+
finally:
|
|
1064
|
+
try:
|
|
1065
|
+
if os.path.exists(temp_path):
|
|
1066
|
+
os.unlink(temp_path)
|
|
1067
|
+
os.rmdir(temp_dir)
|
|
1068
|
+
except Exception:
|
|
1069
|
+
pass
|
|
1070
|
+
|
|
1071
|
+
def exists(self) -> bool:
|
|
1072
|
+
"""
|
|
1073
|
+
Check if a file exists at the specified path.
|
|
1074
|
+
|
|
1075
|
+
Returns:
|
|
1076
|
+
True if file exists, False otherwise
|
|
1077
|
+
|
|
1078
|
+
Raises:
|
|
1079
|
+
RuntimeError: If experiment is not open
|
|
1080
|
+
ValueError: If no file path specified
|
|
1081
|
+
|
|
1082
|
+
Examples:
|
|
1083
|
+
# Check if file exists
|
|
1084
|
+
if dxp.files("models/checkpoint.pt").exists():
|
|
1085
|
+
print("File exists!")
|
|
1086
|
+
|
|
1087
|
+
# Check before downloading
|
|
1088
|
+
if not dxp.files("config.json").exists():
|
|
1089
|
+
raise FileNotFoundError("Config file missing")
|
|
1090
|
+
"""
|
|
1091
|
+
if not self._experiment._is_open:
|
|
1092
|
+
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1093
|
+
|
|
1094
|
+
if not self._path:
|
|
1095
|
+
raise ValueError("No file path specified. Use: experiment.files('path/to/file').exists()")
|
|
1096
|
+
|
|
1097
|
+
# Try to find the file
|
|
1098
|
+
try:
|
|
1099
|
+
files = self._experiment._list_files(prefix=None, tags=None)
|
|
1100
|
+
search_path = self._path.lstrip('/')
|
|
1101
|
+
|
|
1102
|
+
for f in files:
|
|
1103
|
+
filename = f.get('filename', '')
|
|
1104
|
+
prefix = f.get('path', '/').lstrip('/')
|
|
1105
|
+
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
1106
|
+
full_path = full_path.lstrip('/')
|
|
1107
|
+
|
|
1108
|
+
# Check if this file matches (by full path or just filename)
|
|
1109
|
+
if full_path == search_path or filename == search_path:
|
|
1110
|
+
# Make sure it's not deleted
|
|
1111
|
+
if f.get('deletedAt') is None:
|
|
1112
|
+
return True
|
|
1113
|
+
|
|
1114
|
+
return False
|
|
1115
|
+
except Exception:
|
|
1116
|
+
return False
|
|
1117
|
+
|
|
1118
|
+
def read_text(self, encoding: str = 'utf-8') -> str:
|
|
1119
|
+
"""
|
|
1120
|
+
Download file and read its content as text.
|
|
1121
|
+
|
|
1122
|
+
Args:
|
|
1123
|
+
encoding: Text encoding to use (default: 'utf-8')
|
|
1124
|
+
|
|
1125
|
+
Returns:
|
|
1126
|
+
File content as string
|
|
1127
|
+
|
|
1128
|
+
Raises:
|
|
1129
|
+
RuntimeError: If experiment is not open
|
|
1130
|
+
ValueError: If file not found or no path specified
|
|
1131
|
+
UnicodeDecodeError: If file cannot be decoded with specified encoding
|
|
1132
|
+
|
|
1133
|
+
Examples:
|
|
1134
|
+
# Read configuration file
|
|
1135
|
+
config_yaml = dxp.files("configs/view.yaml").read_text()
|
|
1136
|
+
|
|
1137
|
+
# Read log file
|
|
1138
|
+
logs = dxp.files("logs/training.log").read_text()
|
|
1139
|
+
|
|
1140
|
+
# Read with different encoding
|
|
1141
|
+
content = dxp.files("data.txt").read_text(encoding='latin-1')
|
|
1142
|
+
"""
|
|
1143
|
+
import tempfile
|
|
1144
|
+
import os
|
|
1145
|
+
|
|
1146
|
+
if not self._experiment._is_open:
|
|
1147
|
+
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1148
|
+
|
|
1149
|
+
if not self._path:
|
|
1150
|
+
raise ValueError("No file path specified. Use: experiment.files('path/to/file').read_text()")
|
|
686
1151
|
|
|
687
|
-
|
|
688
|
-
|
|
1152
|
+
# Create temporary file for download
|
|
1153
|
+
temp_dir = tempfile.mkdtemp()
|
|
1154
|
+
temp_filename = Path(self._path).name
|
|
1155
|
+
temp_path = os.path.join(temp_dir, temp_filename)
|
|
689
1156
|
|
|
690
|
-
|
|
691
|
-
|
|
1157
|
+
try:
|
|
1158
|
+
# Download the file
|
|
1159
|
+
downloaded_path = self.download(to=temp_path)
|
|
692
1160
|
|
|
693
|
-
#
|
|
694
|
-
|
|
695
|
-
|
|
1161
|
+
# Read content as text
|
|
1162
|
+
with open(downloaded_path, 'r', encoding=encoding) as f:
|
|
1163
|
+
content = f.read()
|
|
696
1164
|
|
|
697
|
-
return
|
|
1165
|
+
return content
|
|
698
1166
|
finally:
|
|
699
|
-
# Clean up
|
|
1167
|
+
# Clean up temporary file
|
|
700
1168
|
try:
|
|
701
1169
|
if os.path.exists(temp_path):
|
|
702
1170
|
os.unlink(temp_path)
|
|
@@ -705,6 +1173,249 @@ class FileBuilder:
|
|
|
705
1173
|
pass
|
|
706
1174
|
|
|
707
1175
|
|
|
1176
|
+
class FilesAccessor:
|
|
1177
|
+
"""
|
|
1178
|
+
Accessor that enables both callable and attribute-style access to file operations.
|
|
1179
|
+
|
|
1180
|
+
This allows:
|
|
1181
|
+
dxp.files("models") # Returns FileBuilder
|
|
1182
|
+
dxp.files(dir="models") # Keyword argument form
|
|
1183
|
+
experiment.files.upload(...) # Direct method call
|
|
1184
|
+
experiment.files.download(...) # Direct method call
|
|
1185
|
+
"""
|
|
1186
|
+
|
|
1187
|
+
def __init__(self, experiment: 'Experiment'):
|
|
1188
|
+
self._experiment = experiment
|
|
1189
|
+
self._builder = FileBuilder(experiment)
|
|
1190
|
+
|
|
1191
|
+
def __call__(self, dir: Optional[str] = None, **kwargs) -> FileBuilder:
|
|
1192
|
+
"""
|
|
1193
|
+
Create a FileBuilder with the given directory.
|
|
1194
|
+
|
|
1195
|
+
Supports flexible argument styles:
|
|
1196
|
+
- Positional: files("models")
|
|
1197
|
+
- Keyword: files(dir="models")
|
|
1198
|
+
- No args (root): files()
|
|
1199
|
+
|
|
1200
|
+
Args:
|
|
1201
|
+
dir: Directory/path for file operations
|
|
1202
|
+
**kwargs: Additional FileBuilder options
|
|
1203
|
+
|
|
1204
|
+
Returns:
|
|
1205
|
+
FileBuilder instance
|
|
1206
|
+
"""
|
|
1207
|
+
return FileBuilder(self._experiment, path=dir, **kwargs)
|
|
1208
|
+
|
|
1209
|
+
# Direct methods that don't require a path first
|
|
1210
|
+
|
|
1211
|
+
def upload(
|
|
1212
|
+
self,
|
|
1213
|
+
fpath: str,
|
|
1214
|
+
*,
|
|
1215
|
+
to: Optional[str] = None,
|
|
1216
|
+
**kwargs
|
|
1217
|
+
) -> Dict[str, Any]:
|
|
1218
|
+
"""
|
|
1219
|
+
Upload a file directly without specifying a path prefix first.
|
|
1220
|
+
|
|
1221
|
+
Args:
|
|
1222
|
+
fpath: Path to existing file to upload (required)
|
|
1223
|
+
to: Optional destination path, e.g., "models/model.pt" or "renamed.pt"
|
|
1224
|
+
**kwargs: Additional FileBuilder options
|
|
1225
|
+
|
|
1226
|
+
Returns:
|
|
1227
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
1228
|
+
|
|
1229
|
+
Examples:
|
|
1230
|
+
# Upload with original filename
|
|
1231
|
+
experiment.files.upload("./model.pt")
|
|
1232
|
+
|
|
1233
|
+
# Upload with destination path
|
|
1234
|
+
experiment.files.upload("./model.pt", to="models/model.pt")
|
|
1235
|
+
|
|
1236
|
+
# Upload with metadata
|
|
1237
|
+
experiment.files.upload("./model.pt", to="best.pt", description="Best model")
|
|
1238
|
+
"""
|
|
1239
|
+
# Parse 'to' to extract prefix and filename if provided
|
|
1240
|
+
if to:
|
|
1241
|
+
to_path = to.lstrip('/')
|
|
1242
|
+
if '/' in to_path:
|
|
1243
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1244
|
+
prefix = '/' + prefix
|
|
1245
|
+
else:
|
|
1246
|
+
prefix = '/'
|
|
1247
|
+
filename = to_path
|
|
1248
|
+
return FileBuilder(self._experiment, path=prefix, **kwargs).upload(fpath, to=filename)
|
|
1249
|
+
|
|
1250
|
+
# No prefix, just upload with original or specified filename
|
|
1251
|
+
return FileBuilder(self._experiment, **kwargs).upload(fpath)
|
|
1252
|
+
|
|
1253
|
+
def download(
|
|
1254
|
+
self,
|
|
1255
|
+
path: str,
|
|
1256
|
+
*,
|
|
1257
|
+
to: Optional[str] = None
|
|
1258
|
+
) -> Union[str, List[str]]:
|
|
1259
|
+
"""
|
|
1260
|
+
Download file(s) directly.
|
|
1261
|
+
|
|
1262
|
+
Examples:
|
|
1263
|
+
experiment.files.download("model.pt")
|
|
1264
|
+
experiment.files.download("images/*.png", to="local_images")
|
|
1265
|
+
"""
|
|
1266
|
+
path = path.lstrip('/')
|
|
1267
|
+
|
|
1268
|
+
# Check if path contains glob pattern
|
|
1269
|
+
if '*' in path or '?' in path:
|
|
1270
|
+
# Extract prefix and pattern
|
|
1271
|
+
if '/' in path:
|
|
1272
|
+
parts = path.split('/')
|
|
1273
|
+
# Find where the pattern starts
|
|
1274
|
+
prefix_parts = []
|
|
1275
|
+
pattern_parts = []
|
|
1276
|
+
in_pattern = False
|
|
1277
|
+
for part in parts:
|
|
1278
|
+
if '*' in part or '?' in part:
|
|
1279
|
+
in_pattern = True
|
|
1280
|
+
if in_pattern:
|
|
1281
|
+
pattern_parts.append(part)
|
|
1282
|
+
else:
|
|
1283
|
+
prefix_parts.append(part)
|
|
1284
|
+
|
|
1285
|
+
prefix = '/'.join(prefix_parts) if prefix_parts else None
|
|
1286
|
+
pattern = '/'.join(pattern_parts)
|
|
1287
|
+
else:
|
|
1288
|
+
prefix = None
|
|
1289
|
+
pattern = path
|
|
1290
|
+
|
|
1291
|
+
return FileBuilder(self._experiment, path=prefix).download(pattern, to=to)
|
|
1292
|
+
|
|
1293
|
+
# Single file download
|
|
1294
|
+
return FileBuilder(self._experiment, path=path).download(to=to)
|
|
1295
|
+
|
|
1296
|
+
def delete(self, path: str) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
|
|
1297
|
+
"""
|
|
1298
|
+
Delete file(s) directly.
|
|
1299
|
+
|
|
1300
|
+
Examples:
|
|
1301
|
+
experiment.files.delete("some.text")
|
|
1302
|
+
experiment.files.delete("images/*.png")
|
|
1303
|
+
"""
|
|
1304
|
+
path = path.lstrip('/')
|
|
1305
|
+
|
|
1306
|
+
# Check if path contains glob pattern
|
|
1307
|
+
if '*' in path or '?' in path:
|
|
1308
|
+
# Extract prefix and pattern
|
|
1309
|
+
if '/' in path:
|
|
1310
|
+
parts = path.split('/')
|
|
1311
|
+
prefix_parts = []
|
|
1312
|
+
pattern_parts = []
|
|
1313
|
+
in_pattern = False
|
|
1314
|
+
for part in parts:
|
|
1315
|
+
if '*' in part or '?' in part:
|
|
1316
|
+
in_pattern = True
|
|
1317
|
+
if in_pattern:
|
|
1318
|
+
pattern_parts.append(part)
|
|
1319
|
+
else:
|
|
1320
|
+
prefix_parts.append(part)
|
|
1321
|
+
|
|
1322
|
+
prefix = '/'.join(prefix_parts) if prefix_parts else None
|
|
1323
|
+
pattern = '/'.join(pattern_parts)
|
|
1324
|
+
else:
|
|
1325
|
+
prefix = None
|
|
1326
|
+
pattern = path
|
|
1327
|
+
|
|
1328
|
+
return FileBuilder(self._experiment, path=prefix).delete(pattern)
|
|
1329
|
+
|
|
1330
|
+
# Single file delete
|
|
1331
|
+
return FileBuilder(self._experiment, path=path).delete()
|
|
1332
|
+
|
|
1333
|
+
def list(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
1334
|
+
"""
|
|
1335
|
+
List files directly.
|
|
1336
|
+
|
|
1337
|
+
Examples:
|
|
1338
|
+
files = experiment.files.list()
|
|
1339
|
+
files = experiment.files.list("*.pt")
|
|
1340
|
+
"""
|
|
1341
|
+
return FileBuilder(self._experiment).list(pattern)
|
|
1342
|
+
|
|
1343
|
+
def save_text(self, content: str, *, to: str) -> Dict[str, Any]:
|
|
1344
|
+
"""
|
|
1345
|
+
Save text content to a file.
|
|
1346
|
+
|
|
1347
|
+
Examples:
|
|
1348
|
+
experiment.files.save_text("content", to="view.yaml")
|
|
1349
|
+
"""
|
|
1350
|
+
to_path = to.lstrip('/')
|
|
1351
|
+
if '/' in to_path:
|
|
1352
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1353
|
+
prefix = '/' + prefix
|
|
1354
|
+
else:
|
|
1355
|
+
prefix = '/'
|
|
1356
|
+
filename = to_path
|
|
1357
|
+
return FileBuilder(self._experiment, path=prefix).save_text(content, to=filename)
|
|
1358
|
+
|
|
1359
|
+
def save_json(self, content: Any, *, to: str) -> Dict[str, Any]:
|
|
1360
|
+
"""
|
|
1361
|
+
Save JSON content to a file.
|
|
1362
|
+
|
|
1363
|
+
Examples:
|
|
1364
|
+
experiment.files.save_json({"key": "value"}, to="config.json")
|
|
1365
|
+
"""
|
|
1366
|
+
to_path = to.lstrip('/')
|
|
1367
|
+
if '/' in to_path:
|
|
1368
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1369
|
+
prefix = '/' + prefix
|
|
1370
|
+
else:
|
|
1371
|
+
prefix = '/'
|
|
1372
|
+
filename = to_path
|
|
1373
|
+
return FileBuilder(self._experiment, path=prefix).save_json(content, to=filename)
|
|
1374
|
+
|
|
1375
|
+
def save_blob(self, data: bytes, *, to: str) -> Dict[str, Any]:
|
|
1376
|
+
"""
|
|
1377
|
+
Save binary data to a file.
|
|
1378
|
+
|
|
1379
|
+
Examples:
|
|
1380
|
+
experiment.files.save_blob(b"data", to="data.bin")
|
|
1381
|
+
"""
|
|
1382
|
+
to_path = to.lstrip('/')
|
|
1383
|
+
if '/' in to_path:
|
|
1384
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1385
|
+
prefix = '/' + prefix
|
|
1386
|
+
else:
|
|
1387
|
+
prefix = '/'
|
|
1388
|
+
filename = to_path
|
|
1389
|
+
return FileBuilder(self._experiment, path=prefix).save_blob(data, to=filename)
|
|
1390
|
+
|
|
1391
|
+
|
|
1392
|
+
class BindrsBuilder:
|
|
1393
|
+
"""
|
|
1394
|
+
Fluent interface for bindr (collection) operations.
|
|
1395
|
+
|
|
1396
|
+
Usage:
|
|
1397
|
+
file_paths = experiment.bindrs("some-bindr").list()
|
|
1398
|
+
"""
|
|
1399
|
+
|
|
1400
|
+
def __init__(self, experiment: 'Experiment', bindr_name: str):
|
|
1401
|
+
self._experiment = experiment
|
|
1402
|
+
self._bindr_name = bindr_name
|
|
1403
|
+
|
|
1404
|
+
def list(self) -> List[Dict[str, Any]]:
|
|
1405
|
+
"""
|
|
1406
|
+
List files in this bindr.
|
|
1407
|
+
|
|
1408
|
+
Returns:
|
|
1409
|
+
List of file metadata dicts belonging to this bindr
|
|
1410
|
+
"""
|
|
1411
|
+
if not self._experiment._is_open:
|
|
1412
|
+
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1413
|
+
|
|
1414
|
+
# Get all files and filter by bindr
|
|
1415
|
+
all_files = self._experiment._list_files(prefix=None, tags=None)
|
|
1416
|
+
return [f for f in all_files if self._bindr_name in f.get('bindrs', [])]
|
|
1417
|
+
|
|
1418
|
+
|
|
708
1419
|
def compute_sha256(file_path: str) -> str:
|
|
709
1420
|
"""
|
|
710
1421
|
Compute SHA256 checksum of a file.
|