ml-dash 0.6.1__py3-none-any.whl → 0.6.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +2 -0
- ml_dash/experiment.py +83 -36
- ml_dash/files.py +932 -336
- ml_dash/run.py +85 -0
- ml_dash/storage.py +4 -2
- {ml_dash-0.6.1.dist-info → ml_dash-0.6.2rc1.dist-info}/METADATA +42 -15
- {ml_dash-0.6.1.dist-info → ml_dash-0.6.2rc1.dist-info}/RECORD +9 -8
- {ml_dash-0.6.1.dist-info → ml_dash-0.6.2rc1.dist-info}/WHEEL +1 -1
- {ml_dash-0.6.1.dist-info → ml_dash-0.6.2rc1.dist-info}/entry_points.txt +0 -0
ml_dash/files.py
CHANGED
|
@@ -6,6 +6,7 @@ Provides fluent API for file upload, download, list, and delete operations.
|
|
|
6
6
|
|
|
7
7
|
import hashlib
|
|
8
8
|
import mimetypes
|
|
9
|
+
import fnmatch
|
|
9
10
|
from typing import Dict, Any, List, Optional, Union, TYPE_CHECKING
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
|
|
@@ -19,37 +20,51 @@ class FileBuilder:
|
|
|
19
20
|
|
|
20
21
|
Usage:
|
|
21
22
|
# Upload file
|
|
22
|
-
experiment.files(
|
|
23
|
+
experiment.files("checkpoints").save(net, to="checkpoint.pt")
|
|
23
24
|
|
|
24
25
|
# List files
|
|
25
|
-
files = experiment.files().list()
|
|
26
|
-
files = experiment.files(
|
|
26
|
+
files = experiment.files("/some/location").list()
|
|
27
|
+
files = experiment.files("/models").list()
|
|
27
28
|
|
|
28
29
|
# Download file
|
|
29
|
-
experiment.files(
|
|
30
|
-
experiment.files(
|
|
30
|
+
experiment.files("some.text").download()
|
|
31
|
+
experiment.files("some.text").download(to="./model.pt")
|
|
31
32
|
|
|
32
|
-
#
|
|
33
|
-
experiment.files(
|
|
33
|
+
# Download Files via Glob Pattern
|
|
34
|
+
file_paths = experiment.files("images").list("*.png")
|
|
35
|
+
experiment.files("images").download("*.png")
|
|
36
|
+
|
|
37
|
+
# Delete files
|
|
38
|
+
experiment.files("some.text").delete()
|
|
39
|
+
|
|
40
|
+
Specific File Types:
|
|
41
|
+
dxp.files.save_text("content", to="view.yaml")
|
|
42
|
+
dxp.files.save_json(dict(hey="yo"), to="config.json")
|
|
43
|
+
dxp.files.save_blob(b"xxx", to="data.bin")
|
|
34
44
|
"""
|
|
35
45
|
|
|
36
|
-
def __init__(self, experiment: 'Experiment', **kwargs):
|
|
46
|
+
def __init__(self, experiment: 'Experiment', path: Optional[str] = None, **kwargs):
|
|
37
47
|
"""
|
|
38
48
|
Initialize file builder.
|
|
39
49
|
|
|
40
50
|
Args:
|
|
41
51
|
experiment: Parent experiment instance
|
|
42
|
-
|
|
43
|
-
-
|
|
44
|
-
-
|
|
52
|
+
path: File path or prefix for operations. Can be:
|
|
53
|
+
- A prefix/directory (e.g., "checkpoints", "/models")
|
|
54
|
+
- A file path (e.g., "some.text", "images/photo.png")
|
|
55
|
+
**kwargs: Additional file operation parameters (for backwards compatibility)
|
|
56
|
+
- file_path: Path to file to upload (deprecated, use save(to=))
|
|
57
|
+
- prefix: Logical path prefix (deprecated, use path argument)
|
|
45
58
|
- description: Optional description
|
|
46
59
|
- tags: Optional list of tags
|
|
47
60
|
- bindrs: Optional list of bindrs
|
|
48
61
|
- metadata: Optional metadata dict
|
|
49
62
|
- file_id: File ID for download/delete/update operations
|
|
50
|
-
- dest_path: Destination path for download
|
|
63
|
+
- dest_path: Destination path for download (deprecated, use download(to=))
|
|
51
64
|
"""
|
|
52
65
|
self._experiment = experiment
|
|
66
|
+
self._path = path
|
|
67
|
+
# Backwards compatibility
|
|
53
68
|
self._file_path = kwargs.get('file_path')
|
|
54
69
|
self._prefix = kwargs.get('prefix', '/')
|
|
55
70
|
self._description = kwargs.get('description')
|
|
@@ -59,9 +74,37 @@ class FileBuilder:
|
|
|
59
74
|
self._file_id = kwargs.get('file_id')
|
|
60
75
|
self._dest_path = kwargs.get('dest_path')
|
|
61
76
|
|
|
62
|
-
|
|
77
|
+
# If path is provided, determine if it's a file or prefix
|
|
78
|
+
if path:
|
|
79
|
+
# Normalize path
|
|
80
|
+
path = path.lstrip('/')
|
|
81
|
+
self._normalized_path = '/' + path if not path.startswith('/') else path
|
|
82
|
+
|
|
83
|
+
def save(
|
|
84
|
+
self,
|
|
85
|
+
obj: Optional[Any] = None,
|
|
86
|
+
*,
|
|
87
|
+
to: Optional[str] = None,
|
|
88
|
+
description: Optional[str] = None,
|
|
89
|
+
tags: Optional[List[str]] = None,
|
|
90
|
+
metadata: Optional[Dict[str, Any]] = None
|
|
91
|
+
) -> Dict[str, Any]:
|
|
63
92
|
"""
|
|
64
|
-
Upload and save
|
|
93
|
+
Upload and save a file or object.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
obj: Object to save. Can be:
|
|
97
|
+
- None: Uses file_path from constructor (backwards compatibility)
|
|
98
|
+
- str: Path to an existing file
|
|
99
|
+
- bytes: Binary data to save
|
|
100
|
+
- dict/list: JSON-serializable data
|
|
101
|
+
- PyTorch model/state_dict: Saved with torch.save()
|
|
102
|
+
- matplotlib figure: Saved as image
|
|
103
|
+
- Any picklable object: Saved with pickle
|
|
104
|
+
to: Target filename (required when obj is not a file path)
|
|
105
|
+
description: Optional description (overrides constructor)
|
|
106
|
+
tags: Optional list of tags (overrides constructor)
|
|
107
|
+
metadata: Optional metadata dict (overrides constructor)
|
|
65
108
|
|
|
66
109
|
Returns:
|
|
67
110
|
File metadata dict with id, path, filename, checksum, etc.
|
|
@@ -72,8 +115,19 @@ class FileBuilder:
|
|
|
72
115
|
ValueError: If file size exceeds 100GB limit
|
|
73
116
|
|
|
74
117
|
Examples:
|
|
75
|
-
|
|
76
|
-
|
|
118
|
+
# Save existing file
|
|
119
|
+
experiment.files("models").save("./model.pt")
|
|
120
|
+
experiment.files("models").save(to="model.pt") # copies from self._file_path
|
|
121
|
+
|
|
122
|
+
# Save PyTorch model
|
|
123
|
+
experiment.files("checkpoints").save(model, to="checkpoint.pt")
|
|
124
|
+
experiment.files("checkpoints").save(model.state_dict(), to="weights.pt")
|
|
125
|
+
|
|
126
|
+
# Save dict as JSON
|
|
127
|
+
experiment.files("configs").save({"lr": 0.001}, to="config.json")
|
|
128
|
+
|
|
129
|
+
# Save bytes
|
|
130
|
+
experiment.files("data").save(b"binary data", to="data.bin")
|
|
77
131
|
"""
|
|
78
132
|
if not self._experiment._is_open:
|
|
79
133
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
@@ -81,47 +135,326 @@ class FileBuilder:
|
|
|
81
135
|
if self._experiment._write_protected:
|
|
82
136
|
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
83
137
|
|
|
84
|
-
|
|
85
|
-
|
|
138
|
+
# Use provided values or fall back to constructor values
|
|
139
|
+
desc = description if description is not None else self._description
|
|
140
|
+
file_tags = tags if tags is not None else self._tags
|
|
141
|
+
file_metadata = metadata if metadata is not None else self._metadata
|
|
142
|
+
|
|
143
|
+
# Determine prefix from path
|
|
144
|
+
prefix = self._prefix
|
|
145
|
+
if self._path:
|
|
146
|
+
prefix = '/' + self._path.lstrip('/')
|
|
147
|
+
|
|
148
|
+
# Handle different object types
|
|
149
|
+
if obj is None:
|
|
150
|
+
# Backwards compatibility: use file_path from constructor
|
|
151
|
+
if not self._file_path:
|
|
152
|
+
raise ValueError("No file or object provided. Pass a file path or object to save().")
|
|
153
|
+
return self._save_file(
|
|
154
|
+
file_path=self._file_path,
|
|
155
|
+
prefix=prefix,
|
|
156
|
+
description=desc,
|
|
157
|
+
tags=file_tags,
|
|
158
|
+
metadata=file_metadata
|
|
159
|
+
)
|
|
86
160
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
161
|
+
if isinstance(obj, str) and Path(obj).exists():
|
|
162
|
+
# obj is a path to an existing file
|
|
163
|
+
return self._save_file(
|
|
164
|
+
file_path=obj,
|
|
165
|
+
prefix=prefix,
|
|
166
|
+
description=desc,
|
|
167
|
+
tags=file_tags,
|
|
168
|
+
metadata=file_metadata
|
|
169
|
+
)
|
|
90
170
|
|
|
91
|
-
if
|
|
92
|
-
|
|
171
|
+
if isinstance(obj, bytes):
|
|
172
|
+
# Save bytes directly
|
|
173
|
+
if not to:
|
|
174
|
+
raise ValueError("'to' parameter is required when saving bytes")
|
|
175
|
+
return self._save_bytes(
|
|
176
|
+
data=obj,
|
|
177
|
+
filename=to,
|
|
178
|
+
prefix=prefix,
|
|
179
|
+
description=desc,
|
|
180
|
+
tags=file_tags,
|
|
181
|
+
metadata=file_metadata
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
if isinstance(obj, (dict, list)):
|
|
185
|
+
# Try JSON first
|
|
186
|
+
if not to:
|
|
187
|
+
raise ValueError("'to' parameter is required when saving dict/list")
|
|
188
|
+
return self._save_json(
|
|
189
|
+
content=obj,
|
|
190
|
+
filename=to,
|
|
191
|
+
prefix=prefix,
|
|
192
|
+
description=desc,
|
|
193
|
+
tags=file_tags,
|
|
194
|
+
metadata=file_metadata
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
# Check for PyTorch model
|
|
198
|
+
try:
|
|
199
|
+
import torch
|
|
200
|
+
if isinstance(obj, (torch.nn.Module, dict)) or hasattr(obj, 'state_dict'):
|
|
201
|
+
if not to:
|
|
202
|
+
raise ValueError("'to' parameter is required when saving PyTorch model")
|
|
203
|
+
return self._save_torch(
|
|
204
|
+
model=obj,
|
|
205
|
+
filename=to,
|
|
206
|
+
prefix=prefix,
|
|
207
|
+
description=desc,
|
|
208
|
+
tags=file_tags,
|
|
209
|
+
metadata=file_metadata
|
|
210
|
+
)
|
|
211
|
+
except ImportError:
|
|
212
|
+
pass
|
|
213
|
+
|
|
214
|
+
# Check for matplotlib figure
|
|
215
|
+
try:
|
|
216
|
+
import matplotlib.pyplot as plt
|
|
217
|
+
from matplotlib.figure import Figure
|
|
218
|
+
if isinstance(obj, Figure):
|
|
219
|
+
if not to:
|
|
220
|
+
raise ValueError("'to' parameter is required when saving matplotlib figure")
|
|
221
|
+
return self._save_fig(
|
|
222
|
+
fig=obj,
|
|
223
|
+
filename=to,
|
|
224
|
+
prefix=prefix,
|
|
225
|
+
description=desc,
|
|
226
|
+
tags=file_tags,
|
|
227
|
+
metadata=file_metadata
|
|
228
|
+
)
|
|
229
|
+
except ImportError:
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
# Fall back to pickle
|
|
233
|
+
if not to:
|
|
234
|
+
raise ValueError("'to' parameter is required when saving object")
|
|
235
|
+
return self._save_pickle(
|
|
236
|
+
content=obj,
|
|
237
|
+
filename=to,
|
|
238
|
+
prefix=prefix,
|
|
239
|
+
description=desc,
|
|
240
|
+
tags=file_tags,
|
|
241
|
+
metadata=file_metadata
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
def _save_file(
|
|
245
|
+
self,
|
|
246
|
+
file_path: str,
|
|
247
|
+
prefix: str,
|
|
248
|
+
description: Optional[str],
|
|
249
|
+
tags: Optional[List[str]],
|
|
250
|
+
metadata: Optional[Dict[str, Any]]
|
|
251
|
+
) -> Dict[str, Any]:
|
|
252
|
+
"""Internal method to save an existing file."""
|
|
253
|
+
file_path_obj = Path(file_path)
|
|
254
|
+
if not file_path_obj.exists():
|
|
255
|
+
raise ValueError(f"File not found: {file_path}")
|
|
256
|
+
|
|
257
|
+
if not file_path_obj.is_file():
|
|
258
|
+
raise ValueError(f"Path is not a file: {file_path}")
|
|
93
259
|
|
|
94
260
|
# Check file size (max 100GB)
|
|
95
|
-
file_size =
|
|
261
|
+
file_size = file_path_obj.stat().st_size
|
|
96
262
|
MAX_FILE_SIZE = 100 * 1024 * 1024 * 1024 # 100GB in bytes
|
|
97
263
|
if file_size > MAX_FILE_SIZE:
|
|
98
264
|
raise ValueError(f"File size ({file_size} bytes) exceeds 100GB limit")
|
|
99
265
|
|
|
100
266
|
# Compute checksum
|
|
101
|
-
checksum = compute_sha256(str(
|
|
267
|
+
checksum = compute_sha256(str(file_path_obj))
|
|
102
268
|
|
|
103
269
|
# Detect MIME type
|
|
104
|
-
content_type = get_mime_type(str(
|
|
270
|
+
content_type = get_mime_type(str(file_path_obj))
|
|
105
271
|
|
|
106
272
|
# Get filename
|
|
107
|
-
filename =
|
|
273
|
+
filename = file_path_obj.name
|
|
108
274
|
|
|
109
275
|
# Upload through experiment
|
|
110
276
|
return self._experiment._upload_file(
|
|
111
|
-
file_path=str(
|
|
112
|
-
prefix=
|
|
277
|
+
file_path=str(file_path_obj),
|
|
278
|
+
prefix=prefix,
|
|
113
279
|
filename=filename,
|
|
114
|
-
description=
|
|
115
|
-
tags=
|
|
116
|
-
metadata=
|
|
280
|
+
description=description,
|
|
281
|
+
tags=tags or [],
|
|
282
|
+
metadata=metadata,
|
|
117
283
|
checksum=checksum,
|
|
118
284
|
content_type=content_type,
|
|
119
285
|
size_bytes=file_size
|
|
120
286
|
)
|
|
121
287
|
|
|
122
|
-
def
|
|
288
|
+
def _save_bytes(
|
|
289
|
+
self,
|
|
290
|
+
data: bytes,
|
|
291
|
+
filename: str,
|
|
292
|
+
prefix: str,
|
|
293
|
+
description: Optional[str],
|
|
294
|
+
tags: Optional[List[str]],
|
|
295
|
+
metadata: Optional[Dict[str, Any]]
|
|
296
|
+
) -> Dict[str, Any]:
|
|
297
|
+
"""Save bytes data to a file."""
|
|
298
|
+
import tempfile
|
|
299
|
+
import os
|
|
300
|
+
|
|
301
|
+
temp_dir = tempfile.mkdtemp()
|
|
302
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
303
|
+
try:
|
|
304
|
+
with open(temp_path, 'wb') as f:
|
|
305
|
+
f.write(data)
|
|
306
|
+
return self._save_file(
|
|
307
|
+
file_path=temp_path,
|
|
308
|
+
prefix=prefix,
|
|
309
|
+
description=description,
|
|
310
|
+
tags=tags,
|
|
311
|
+
metadata=metadata
|
|
312
|
+
)
|
|
313
|
+
finally:
|
|
314
|
+
try:
|
|
315
|
+
os.unlink(temp_path)
|
|
316
|
+
os.rmdir(temp_dir)
|
|
317
|
+
except Exception:
|
|
318
|
+
pass
|
|
319
|
+
|
|
320
|
+
def _save_json(
|
|
321
|
+
self,
|
|
322
|
+
content: Any,
|
|
323
|
+
filename: str,
|
|
324
|
+
prefix: str,
|
|
325
|
+
description: Optional[str],
|
|
326
|
+
tags: Optional[List[str]],
|
|
327
|
+
metadata: Optional[Dict[str, Any]]
|
|
328
|
+
) -> Dict[str, Any]:
|
|
329
|
+
"""Save JSON content to a file."""
|
|
330
|
+
import json
|
|
331
|
+
import tempfile
|
|
332
|
+
import os
|
|
333
|
+
|
|
334
|
+
temp_dir = tempfile.mkdtemp()
|
|
335
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
336
|
+
try:
|
|
337
|
+
with open(temp_path, 'w') as f:
|
|
338
|
+
json.dump(content, f, indent=2)
|
|
339
|
+
return self._save_file(
|
|
340
|
+
file_path=temp_path,
|
|
341
|
+
prefix=prefix,
|
|
342
|
+
description=description,
|
|
343
|
+
tags=tags,
|
|
344
|
+
metadata=metadata
|
|
345
|
+
)
|
|
346
|
+
finally:
|
|
347
|
+
try:
|
|
348
|
+
os.unlink(temp_path)
|
|
349
|
+
os.rmdir(temp_dir)
|
|
350
|
+
except Exception:
|
|
351
|
+
pass
|
|
352
|
+
|
|
353
|
+
def _save_torch(
|
|
354
|
+
self,
|
|
355
|
+
model: Any,
|
|
356
|
+
filename: str,
|
|
357
|
+
prefix: str,
|
|
358
|
+
description: Optional[str],
|
|
359
|
+
tags: Optional[List[str]],
|
|
360
|
+
metadata: Optional[Dict[str, Any]]
|
|
361
|
+
) -> Dict[str, Any]:
|
|
362
|
+
"""Save PyTorch model to a file."""
|
|
363
|
+
import tempfile
|
|
364
|
+
import os
|
|
365
|
+
import torch
|
|
366
|
+
|
|
367
|
+
temp_dir = tempfile.mkdtemp()
|
|
368
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
369
|
+
try:
|
|
370
|
+
torch.save(model, temp_path)
|
|
371
|
+
return self._save_file(
|
|
372
|
+
file_path=temp_path,
|
|
373
|
+
prefix=prefix,
|
|
374
|
+
description=description,
|
|
375
|
+
tags=tags,
|
|
376
|
+
metadata=metadata
|
|
377
|
+
)
|
|
378
|
+
finally:
|
|
379
|
+
try:
|
|
380
|
+
os.unlink(temp_path)
|
|
381
|
+
os.rmdir(temp_dir)
|
|
382
|
+
except Exception:
|
|
383
|
+
pass
|
|
384
|
+
|
|
385
|
+
def _save_fig(
|
|
386
|
+
self,
|
|
387
|
+
fig: Any,
|
|
388
|
+
filename: str,
|
|
389
|
+
prefix: str,
|
|
390
|
+
description: Optional[str],
|
|
391
|
+
tags: Optional[List[str]],
|
|
392
|
+
metadata: Optional[Dict[str, Any]],
|
|
393
|
+
**kwargs
|
|
394
|
+
) -> Dict[str, Any]:
|
|
395
|
+
"""Save matplotlib figure to a file."""
|
|
396
|
+
import tempfile
|
|
397
|
+
import os
|
|
398
|
+
import matplotlib.pyplot as plt
|
|
399
|
+
|
|
400
|
+
temp_dir = tempfile.mkdtemp()
|
|
401
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
402
|
+
try:
|
|
403
|
+
fig.savefig(temp_path, **kwargs)
|
|
404
|
+
plt.close(fig)
|
|
405
|
+
return self._save_file(
|
|
406
|
+
file_path=temp_path,
|
|
407
|
+
prefix=prefix,
|
|
408
|
+
description=description,
|
|
409
|
+
tags=tags,
|
|
410
|
+
metadata=metadata
|
|
411
|
+
)
|
|
412
|
+
finally:
|
|
413
|
+
try:
|
|
414
|
+
os.unlink(temp_path)
|
|
415
|
+
os.rmdir(temp_dir)
|
|
416
|
+
except Exception:
|
|
417
|
+
pass
|
|
418
|
+
|
|
419
|
+
def _save_pickle(
|
|
420
|
+
self,
|
|
421
|
+
content: Any,
|
|
422
|
+
filename: str,
|
|
423
|
+
prefix: str,
|
|
424
|
+
description: Optional[str],
|
|
425
|
+
tags: Optional[List[str]],
|
|
426
|
+
metadata: Optional[Dict[str, Any]]
|
|
427
|
+
) -> Dict[str, Any]:
|
|
428
|
+
"""Save Python object to a pickle file."""
|
|
429
|
+
import pickle
|
|
430
|
+
import tempfile
|
|
431
|
+
import os
|
|
432
|
+
|
|
433
|
+
temp_dir = tempfile.mkdtemp()
|
|
434
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
435
|
+
try:
|
|
436
|
+
with open(temp_path, 'wb') as f:
|
|
437
|
+
pickle.dump(content, f)
|
|
438
|
+
return self._save_file(
|
|
439
|
+
file_path=temp_path,
|
|
440
|
+
prefix=prefix,
|
|
441
|
+
description=description,
|
|
442
|
+
tags=tags,
|
|
443
|
+
metadata=metadata
|
|
444
|
+
)
|
|
445
|
+
finally:
|
|
446
|
+
try:
|
|
447
|
+
os.unlink(temp_path)
|
|
448
|
+
os.rmdir(temp_dir)
|
|
449
|
+
except Exception:
|
|
450
|
+
pass
|
|
451
|
+
|
|
452
|
+
def list(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
123
453
|
"""
|
|
124
|
-
List files with optional
|
|
454
|
+
List files with optional glob pattern filtering.
|
|
455
|
+
|
|
456
|
+
Args:
|
|
457
|
+
pattern: Optional glob pattern to filter files (e.g., "*.png", "model_*.pt")
|
|
125
458
|
|
|
126
459
|
Returns:
|
|
127
460
|
List of file metadata dicts
|
|
@@ -131,62 +464,169 @@ class FileBuilder:
|
|
|
131
464
|
|
|
132
465
|
Examples:
|
|
133
466
|
files = experiment.files().list() # All files
|
|
134
|
-
files = experiment.files(
|
|
135
|
-
files = experiment.files(
|
|
467
|
+
files = experiment.files("/models").list() # Files in /models prefix
|
|
468
|
+
files = experiment.files("images").list("*.png") # PNG files in images
|
|
469
|
+
files = experiment.files().list("**/*.pt") # All .pt files
|
|
136
470
|
"""
|
|
137
471
|
if not self._experiment._is_open:
|
|
138
472
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
139
473
|
|
|
140
|
-
|
|
141
|
-
|
|
474
|
+
# Determine prefix filter - support both new (path) and old (prefix) API
|
|
475
|
+
prefix = None
|
|
476
|
+
if self._path:
|
|
477
|
+
prefix = '/' + self._path.lstrip('/')
|
|
478
|
+
elif self._prefix and self._prefix != '/':
|
|
479
|
+
prefix = self._prefix
|
|
480
|
+
|
|
481
|
+
# Get all files matching prefix
|
|
482
|
+
files = self._experiment._list_files(
|
|
483
|
+
prefix=prefix,
|
|
142
484
|
tags=self._tags if self._tags else None
|
|
143
485
|
)
|
|
144
486
|
|
|
145
|
-
|
|
487
|
+
# Apply glob pattern if provided
|
|
488
|
+
if pattern:
|
|
489
|
+
pattern = pattern.lstrip('/')
|
|
490
|
+
filtered = []
|
|
491
|
+
for f in files:
|
|
492
|
+
filename = f.get('filename', '')
|
|
493
|
+
full_path = f.get('path', '/').rstrip('/') + '/' + filename
|
|
494
|
+
full_path = full_path.lstrip('/')
|
|
495
|
+
|
|
496
|
+
# Match against filename or full path
|
|
497
|
+
if fnmatch.fnmatch(filename, pattern) or fnmatch.fnmatch(full_path, pattern):
|
|
498
|
+
filtered.append(f)
|
|
499
|
+
return filtered
|
|
500
|
+
|
|
501
|
+
return files
|
|
502
|
+
|
|
503
|
+
def download(
|
|
504
|
+
self,
|
|
505
|
+
pattern: Optional[str] = None,
|
|
506
|
+
*,
|
|
507
|
+
to: Optional[str] = None
|
|
508
|
+
) -> Union[str, List[str]]:
|
|
146
509
|
"""
|
|
147
|
-
Download file with automatic checksum verification.
|
|
510
|
+
Download file(s) with automatic checksum verification.
|
|
148
511
|
|
|
149
|
-
|
|
512
|
+
Args:
|
|
513
|
+
pattern: Optional glob pattern for batch download (e.g., "*.png")
|
|
514
|
+
to: Destination path. For single files, this is the file path.
|
|
515
|
+
For patterns, this is the destination directory.
|
|
150
516
|
|
|
151
517
|
Returns:
|
|
152
|
-
Path to downloaded file
|
|
518
|
+
For single file: Path to downloaded file
|
|
519
|
+
For pattern: List of paths to downloaded files
|
|
153
520
|
|
|
154
521
|
Raises:
|
|
155
522
|
RuntimeError: If experiment is not open
|
|
156
|
-
ValueError: If
|
|
157
|
-
ValueError: If checksum verification fails
|
|
523
|
+
ValueError: If file not found or checksum verification fails
|
|
158
524
|
|
|
159
525
|
Examples:
|
|
160
|
-
# Download
|
|
526
|
+
# Download single file
|
|
527
|
+
path = experiment.files("model.pt").download()
|
|
528
|
+
path = experiment.files("model.pt").download(to="./local_model.pt")
|
|
529
|
+
|
|
530
|
+
# Download by file ID (backwards compatibility)
|
|
161
531
|
path = experiment.files(file_id="123").download()
|
|
162
532
|
|
|
163
|
-
# Download
|
|
164
|
-
|
|
533
|
+
# Download multiple files matching pattern
|
|
534
|
+
paths = experiment.files("images").download("*.png")
|
|
535
|
+
paths = experiment.files("images").download("*.png", to="./local_images")
|
|
165
536
|
"""
|
|
166
537
|
if not self._experiment._is_open:
|
|
167
538
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
168
539
|
|
|
169
|
-
|
|
170
|
-
|
|
540
|
+
# If file_id is set (backwards compatibility)
|
|
541
|
+
if self._file_id:
|
|
542
|
+
return self._experiment._download_file(
|
|
543
|
+
file_id=self._file_id,
|
|
544
|
+
dest_path=to or self._dest_path
|
|
545
|
+
)
|
|
171
546
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
547
|
+
# If pattern is provided, download multiple files
|
|
548
|
+
if pattern:
|
|
549
|
+
files = self.list(pattern)
|
|
550
|
+
if not files:
|
|
551
|
+
raise ValueError(f"No files found matching pattern: {pattern}")
|
|
552
|
+
|
|
553
|
+
downloaded = []
|
|
554
|
+
dest_dir = Path(to) if to else Path('.')
|
|
555
|
+
if to and not dest_dir.exists():
|
|
556
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
557
|
+
|
|
558
|
+
for f in files:
|
|
559
|
+
file_id = f.get('id')
|
|
560
|
+
filename = f.get('filename', 'file')
|
|
561
|
+
dest_path = str(dest_dir / filename)
|
|
562
|
+
path = self._experiment._download_file(
|
|
563
|
+
file_id=file_id,
|
|
564
|
+
dest_path=dest_path
|
|
565
|
+
)
|
|
566
|
+
downloaded.append(path)
|
|
567
|
+
|
|
568
|
+
return downloaded
|
|
569
|
+
|
|
570
|
+
# Download single file by path
|
|
571
|
+
if self._path:
|
|
572
|
+
# Find file by path
|
|
573
|
+
files = self._experiment._list_files(prefix=None, tags=None)
|
|
574
|
+
matching = []
|
|
575
|
+
search_path = self._path.lstrip('/')
|
|
576
|
+
|
|
577
|
+
for f in files:
|
|
578
|
+
filename = f.get('filename', '')
|
|
579
|
+
prefix = f.get('path', '/').lstrip('/')
|
|
580
|
+
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
581
|
+
full_path = full_path.lstrip('/')
|
|
582
|
+
|
|
583
|
+
if full_path == search_path or filename == search_path:
|
|
584
|
+
matching.append(f)
|
|
585
|
+
|
|
586
|
+
if not matching:
|
|
587
|
+
raise ValueError(f"File not found: {self._path}")
|
|
588
|
+
|
|
589
|
+
if len(matching) > 1:
|
|
590
|
+
# If multiple matches, prefer exact path match
|
|
591
|
+
exact = [f for f in matching if
|
|
592
|
+
(f.get('path', '/').lstrip('/').rstrip('/') + '/' + f.get('filename', '')).lstrip('/') == search_path]
|
|
593
|
+
if exact:
|
|
594
|
+
matching = exact[:1]
|
|
595
|
+
else:
|
|
596
|
+
matching = matching[:1]
|
|
597
|
+
|
|
598
|
+
file_info = matching[0]
|
|
599
|
+
return self._experiment._download_file(
|
|
600
|
+
file_id=file_info['id'],
|
|
601
|
+
dest_path=to
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
raise ValueError("No file path or pattern specified")
|
|
176
605
|
|
|
177
|
-
def delete(self) -> Dict[str, Any]:
|
|
606
|
+
def delete(self, pattern: Optional[str] = None) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
|
|
178
607
|
"""
|
|
179
|
-
Delete file (soft delete).
|
|
608
|
+
Delete file(s) (soft delete).
|
|
609
|
+
|
|
610
|
+
Args:
|
|
611
|
+
pattern: Optional glob pattern for batch delete (e.g., "*.png")
|
|
180
612
|
|
|
181
613
|
Returns:
|
|
182
|
-
Dict with id and deletedAt timestamp
|
|
614
|
+
For single file: Dict with id and deletedAt timestamp
|
|
615
|
+
For pattern: List of deletion results
|
|
183
616
|
|
|
184
617
|
Raises:
|
|
185
618
|
RuntimeError: If experiment is not open or write-protected
|
|
186
|
-
ValueError: If
|
|
619
|
+
ValueError: If file not found
|
|
187
620
|
|
|
188
621
|
Examples:
|
|
622
|
+
# Delete single file
|
|
623
|
+
result = experiment.files("some.text").delete()
|
|
624
|
+
|
|
625
|
+
# Delete by file ID (backwards compatibility)
|
|
189
626
|
result = experiment.files(file_id="123").delete()
|
|
627
|
+
|
|
628
|
+
# Delete multiple files matching pattern
|
|
629
|
+
results = experiment.files("images").delete("*.png")
|
|
190
630
|
"""
|
|
191
631
|
if not self._experiment._is_open:
|
|
192
632
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
@@ -194,10 +634,52 @@ class FileBuilder:
|
|
|
194
634
|
if self._experiment._write_protected:
|
|
195
635
|
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
196
636
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
637
|
+
# If file_id is set (backwards compatibility)
|
|
638
|
+
if self._file_id:
|
|
639
|
+
return self._experiment._delete_file(file_id=self._file_id)
|
|
640
|
+
|
|
641
|
+
# If pattern is provided, delete multiple files
|
|
642
|
+
if pattern:
|
|
643
|
+
files = self.list(pattern)
|
|
644
|
+
if not files:
|
|
645
|
+
raise ValueError(f"No files found matching pattern: {pattern}")
|
|
646
|
+
|
|
647
|
+
results = []
|
|
648
|
+
for f in files:
|
|
649
|
+
file_id = f.get('id')
|
|
650
|
+
result = self._experiment._delete_file(file_id=file_id)
|
|
651
|
+
results.append(result)
|
|
652
|
+
return results
|
|
653
|
+
|
|
654
|
+
# Delete single file by path
|
|
655
|
+
if self._path:
|
|
656
|
+
files = self._experiment._list_files(prefix=None, tags=None)
|
|
657
|
+
matching = []
|
|
658
|
+
search_path = self._path.lstrip('/')
|
|
659
|
+
|
|
660
|
+
for f in files:
|
|
661
|
+
filename = f.get('filename', '')
|
|
662
|
+
prefix = f.get('path', '/').lstrip('/')
|
|
663
|
+
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
664
|
+
full_path = full_path.lstrip('/')
|
|
665
|
+
|
|
666
|
+
if full_path == search_path or filename == search_path:
|
|
667
|
+
matching.append(f)
|
|
668
|
+
|
|
669
|
+
if not matching:
|
|
670
|
+
raise ValueError(f"File not found: {self._path}")
|
|
671
|
+
|
|
672
|
+
# Delete all matching files
|
|
673
|
+
if len(matching) == 1:
|
|
674
|
+
return self._experiment._delete_file(file_id=matching[0]['id'])
|
|
675
|
+
|
|
676
|
+
results = []
|
|
677
|
+
for f in matching:
|
|
678
|
+
result = self._experiment._delete_file(file_id=f['id'])
|
|
679
|
+
results.append(result)
|
|
680
|
+
return results
|
|
681
|
+
|
|
682
|
+
raise ValueError("No file path or pattern specified")
|
|
201
683
|
|
|
202
684
|
def update(self) -> Dict[str, Any]:
|
|
203
685
|
"""
|
|
@@ -234,271 +716,230 @@ class FileBuilder:
|
|
|
234
716
|
metadata=self._metadata
|
|
235
717
|
)
|
|
236
718
|
|
|
237
|
-
|
|
719
|
+
# Convenience methods for specific file types
|
|
720
|
+
|
|
721
|
+
def save_json(
|
|
722
|
+
self,
|
|
723
|
+
content: Any,
|
|
724
|
+
file_name: Optional[str] = None,
|
|
725
|
+
*,
|
|
726
|
+
to: Optional[str] = None
|
|
727
|
+
) -> Dict[str, Any]:
|
|
238
728
|
"""
|
|
239
729
|
Save JSON content to a file.
|
|
240
730
|
|
|
241
731
|
Args:
|
|
242
732
|
content: Content to save as JSON (dict, list, or any JSON-serializable object)
|
|
243
|
-
file_name: Name of the file to create
|
|
733
|
+
file_name: Name of the file to create (deprecated, use 'to')
|
|
734
|
+
to: Target filename (preferred)
|
|
244
735
|
|
|
245
736
|
Returns:
|
|
246
737
|
File metadata dict with id, path, filename, checksum, etc.
|
|
247
738
|
|
|
248
|
-
Raises:
|
|
249
|
-
RuntimeError: If experiment is not open or write-protected
|
|
250
|
-
ValueError: If content is not JSON-serializable
|
|
251
|
-
|
|
252
739
|
Examples:
|
|
253
740
|
config = {"model": "resnet50", "lr": 0.001}
|
|
254
|
-
result = experiment.files(
|
|
741
|
+
result = experiment.files("configs").save_json(config, to="config.json")
|
|
255
742
|
"""
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
743
|
+
filename = to or file_name
|
|
744
|
+
if not filename:
|
|
745
|
+
raise ValueError("'to' parameter is required")
|
|
259
746
|
|
|
260
|
-
if
|
|
261
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
747
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
262
748
|
|
|
263
|
-
|
|
264
|
-
|
|
749
|
+
return self._save_json(
|
|
750
|
+
content=content,
|
|
751
|
+
filename=filename,
|
|
752
|
+
prefix=prefix,
|
|
753
|
+
description=self._description,
|
|
754
|
+
tags=self._tags,
|
|
755
|
+
metadata=self._metadata
|
|
756
|
+
)
|
|
265
757
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
try:
|
|
270
|
-
# Write JSON content to temp file
|
|
271
|
-
with open(temp_path, 'w') as f:
|
|
272
|
-
json.dump(content, f, indent=2)
|
|
758
|
+
def save_text(self, content: str, *, to: str) -> Dict[str, Any]:
|
|
759
|
+
"""
|
|
760
|
+
Save text content to a file.
|
|
273
761
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
762
|
+
Args:
|
|
763
|
+
content: Text content to save
|
|
764
|
+
to: Target filename
|
|
277
765
|
|
|
278
|
-
|
|
279
|
-
|
|
766
|
+
Returns:
|
|
767
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
280
768
|
|
|
281
|
-
|
|
282
|
-
|
|
769
|
+
Examples:
|
|
770
|
+
result = experiment.files().save_text("Hello, world!", to="greeting.txt")
|
|
771
|
+
result = experiment.files("configs").save_text(yaml_content, to="view.yaml")
|
|
772
|
+
"""
|
|
773
|
+
if not to:
|
|
774
|
+
raise ValueError("'to' parameter is required")
|
|
283
775
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
776
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
777
|
+
|
|
778
|
+
return self._save_bytes(
|
|
779
|
+
data=content.encode('utf-8'),
|
|
780
|
+
filename=to,
|
|
781
|
+
prefix=prefix,
|
|
782
|
+
description=self._description,
|
|
783
|
+
tags=self._tags,
|
|
784
|
+
metadata=self._metadata
|
|
785
|
+
)
|
|
292
786
|
|
|
293
|
-
def
|
|
787
|
+
def save_blob(self, data: bytes, *, to: str) -> Dict[str, Any]:
|
|
294
788
|
"""
|
|
295
|
-
Save
|
|
789
|
+
Save binary data to a file.
|
|
296
790
|
|
|
297
791
|
Args:
|
|
298
|
-
|
|
299
|
-
|
|
792
|
+
data: Binary data to save
|
|
793
|
+
to: Target filename
|
|
300
794
|
|
|
301
795
|
Returns:
|
|
302
796
|
File metadata dict with id, path, filename, checksum, etc.
|
|
303
797
|
|
|
304
|
-
Raises:
|
|
305
|
-
RuntimeError: If experiment is not open or write-protected
|
|
306
|
-
ImportError: If torch is not installed
|
|
307
|
-
ValueError: If model cannot be saved
|
|
308
|
-
|
|
309
798
|
Examples:
|
|
310
|
-
|
|
311
|
-
model = torch.nn.Linear(10, 5)
|
|
312
|
-
result = experiment.files(prefix="/models").save_torch(model, "model.pt")
|
|
313
|
-
|
|
314
|
-
# Or save state dict
|
|
315
|
-
result = experiment.files(prefix="/models").save_torch(model.state_dict(), "model.pth")
|
|
799
|
+
result = experiment.files("data").save_blob(binary_data, to="model.bin")
|
|
316
800
|
"""
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
try:
|
|
321
|
-
import torch
|
|
322
|
-
except ImportError:
|
|
323
|
-
raise ImportError("PyTorch is not installed. Install it with: pip install torch")
|
|
801
|
+
if not to:
|
|
802
|
+
raise ValueError("'to' parameter is required")
|
|
324
803
|
|
|
325
|
-
if
|
|
326
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
804
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
327
805
|
|
|
328
|
-
|
|
329
|
-
|
|
806
|
+
return self._save_bytes(
|
|
807
|
+
data=data,
|
|
808
|
+
filename=to,
|
|
809
|
+
prefix=prefix,
|
|
810
|
+
description=self._description,
|
|
811
|
+
tags=self._tags,
|
|
812
|
+
metadata=self._metadata
|
|
813
|
+
)
|
|
330
814
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
815
|
+
def save_torch(
|
|
816
|
+
self,
|
|
817
|
+
model: Any,
|
|
818
|
+
file_name: Optional[str] = None,
|
|
819
|
+
*,
|
|
820
|
+
to: Optional[str] = None
|
|
821
|
+
) -> Dict[str, Any]:
|
|
822
|
+
"""
|
|
823
|
+
Save PyTorch model to a file.
|
|
334
824
|
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
825
|
+
Args:
|
|
826
|
+
model: PyTorch model or state dict to save
|
|
827
|
+
file_name: Name of the file to create (deprecated, use 'to')
|
|
828
|
+
to: Target filename (preferred)
|
|
338
829
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
self._file_path = temp_path
|
|
830
|
+
Returns:
|
|
831
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
342
832
|
|
|
343
|
-
|
|
344
|
-
result =
|
|
833
|
+
Examples:
|
|
834
|
+
result = experiment.files("models").save_torch(model, to="model.pt")
|
|
835
|
+
result = experiment.files("models").save_torch(model.state_dict(), to="weights.pth")
|
|
836
|
+
"""
|
|
837
|
+
filename = to or file_name
|
|
838
|
+
if not filename:
|
|
839
|
+
raise ValueError("'to' parameter is required")
|
|
345
840
|
|
|
346
|
-
|
|
347
|
-
self._file_path = original_file_path
|
|
841
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
348
842
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
843
|
+
return self._save_torch(
|
|
844
|
+
model=model,
|
|
845
|
+
filename=filename,
|
|
846
|
+
prefix=prefix,
|
|
847
|
+
description=self._description,
|
|
848
|
+
tags=self._tags,
|
|
849
|
+
metadata=self._metadata
|
|
850
|
+
)
|
|
357
851
|
|
|
358
|
-
def save_pkl(
|
|
852
|
+
def save_pkl(
|
|
853
|
+
self,
|
|
854
|
+
content: Any,
|
|
855
|
+
file_name: Optional[str] = None,
|
|
856
|
+
*,
|
|
857
|
+
to: Optional[str] = None
|
|
858
|
+
) -> Dict[str, Any]:
|
|
359
859
|
"""
|
|
360
860
|
Save Python object to a pickle file.
|
|
361
861
|
|
|
362
862
|
Args:
|
|
363
863
|
content: Python object to pickle (must be pickle-serializable)
|
|
364
|
-
file_name: Name of the file to create (
|
|
864
|
+
file_name: Name of the file to create (deprecated, use 'to')
|
|
865
|
+
to: Target filename (preferred)
|
|
365
866
|
|
|
366
867
|
Returns:
|
|
367
868
|
File metadata dict with id, path, filename, checksum, etc.
|
|
368
869
|
|
|
369
|
-
Raises:
|
|
370
|
-
RuntimeError: If experiment is not open or write-protected
|
|
371
|
-
ValueError: If content cannot be pickled
|
|
372
|
-
|
|
373
870
|
Examples:
|
|
374
871
|
data = {"model": "resnet50", "weights": np.array([1, 2, 3])}
|
|
375
|
-
result = experiment.files(
|
|
376
|
-
|
|
377
|
-
# Or save any Python object
|
|
378
|
-
result = experiment.files(prefix="/models").save_pkl(trained_model, "model.pickle")
|
|
872
|
+
result = experiment.files("data").save_pkl(data, to="data.pkl")
|
|
379
873
|
"""
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
if not self._experiment._is_open:
|
|
385
|
-
raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
|
|
874
|
+
filename = to or file_name
|
|
875
|
+
if not filename:
|
|
876
|
+
raise ValueError("'to' parameter is required")
|
|
386
877
|
|
|
387
|
-
if self.
|
|
388
|
-
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
878
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
389
879
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
# Save using existing save() method
|
|
399
|
-
original_file_path = self._file_path
|
|
400
|
-
self._file_path = temp_path
|
|
401
|
-
|
|
402
|
-
# Upload and get result
|
|
403
|
-
result = self.save()
|
|
404
|
-
|
|
405
|
-
# Restore original file_path
|
|
406
|
-
self._file_path = original_file_path
|
|
407
|
-
|
|
408
|
-
return result
|
|
409
|
-
finally:
|
|
410
|
-
# Clean up temp file and directory
|
|
411
|
-
try:
|
|
412
|
-
os.unlink(temp_path)
|
|
413
|
-
os.rmdir(temp_dir)
|
|
414
|
-
except Exception:
|
|
415
|
-
pass
|
|
880
|
+
return self._save_pickle(
|
|
881
|
+
content=content,
|
|
882
|
+
filename=filename,
|
|
883
|
+
prefix=prefix,
|
|
884
|
+
description=self._description,
|
|
885
|
+
tags=self._tags,
|
|
886
|
+
metadata=self._metadata
|
|
887
|
+
)
|
|
416
888
|
|
|
417
|
-
def save_fig(
|
|
889
|
+
def save_fig(
|
|
890
|
+
self,
|
|
891
|
+
fig: Optional[Any] = None,
|
|
892
|
+
file_name: Optional[str] = None,
|
|
893
|
+
*,
|
|
894
|
+
to: Optional[str] = None,
|
|
895
|
+
**kwargs
|
|
896
|
+
) -> Dict[str, Any]:
|
|
418
897
|
"""
|
|
419
898
|
Save matplotlib figure to a file.
|
|
420
899
|
|
|
421
900
|
Args:
|
|
422
901
|
fig: Matplotlib figure object. If None, uses plt.gcf() (current figure)
|
|
423
|
-
file_name: Name of file to create (
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
- transparent: Make background transparent (bool)
|
|
427
|
-
- bbox_inches: 'tight' to auto-crop (str or Bbox)
|
|
428
|
-
- quality: JPEG quality 0-100 (int)
|
|
429
|
-
- format: Override format detection (str)
|
|
902
|
+
file_name: Name of file to create (deprecated, use 'to')
|
|
903
|
+
to: Target filename (preferred)
|
|
904
|
+
**kwargs: Additional arguments passed to fig.savefig()
|
|
430
905
|
|
|
431
906
|
Returns:
|
|
432
907
|
File metadata dict with id, path, filename, checksum, etc.
|
|
433
908
|
|
|
434
|
-
Raises:
|
|
435
|
-
RuntimeError: If experiment not open or write-protected
|
|
436
|
-
ImportError: If matplotlib not installed
|
|
437
|
-
|
|
438
909
|
Examples:
|
|
439
|
-
import matplotlib.pyplot as plt
|
|
440
|
-
|
|
441
|
-
# Use current figure
|
|
442
910
|
plt.plot([1, 2, 3], [1, 4, 9])
|
|
443
|
-
result = experiment.files(
|
|
444
|
-
|
|
445
|
-
# Specify figure explicitly
|
|
446
|
-
fig, ax = plt.subplots()
|
|
447
|
-
ax.plot([1, 2, 3])
|
|
448
|
-
result = experiment.files(prefix="/plots").save_fig(fig=fig, file_name="plot.pdf", dpi=150)
|
|
911
|
+
result = experiment.files("plots").save_fig(to="plot.png")
|
|
449
912
|
"""
|
|
450
|
-
import tempfile
|
|
451
|
-
import os
|
|
452
|
-
|
|
453
913
|
try:
|
|
454
914
|
import matplotlib.pyplot as plt
|
|
455
915
|
except ImportError:
|
|
456
916
|
raise ImportError("Matplotlib is not installed. Install it with: pip install matplotlib")
|
|
457
917
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
if self._experiment._write_protected:
|
|
462
|
-
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
918
|
+
filename = to or file_name
|
|
919
|
+
if not filename:
|
|
920
|
+
raise ValueError("'to' parameter is required")
|
|
463
921
|
|
|
464
|
-
# Get figure
|
|
465
922
|
if fig is None:
|
|
466
923
|
fig = plt.gcf()
|
|
467
924
|
|
|
468
|
-
|
|
469
|
-
temp_dir = tempfile.mkdtemp()
|
|
470
|
-
temp_path = os.path.join(temp_dir, file_name)
|
|
471
|
-
|
|
472
|
-
try:
|
|
473
|
-
# Save figure to temp file
|
|
474
|
-
fig.savefig(temp_path, **kwargs)
|
|
475
|
-
|
|
476
|
-
# Close figure to prevent memory leaks
|
|
477
|
-
plt.close(fig)
|
|
478
|
-
|
|
479
|
-
# Save using existing save() method
|
|
480
|
-
original_file_path = self._file_path
|
|
481
|
-
self._file_path = temp_path
|
|
482
|
-
|
|
483
|
-
# Upload and get result
|
|
484
|
-
result = self.save()
|
|
485
|
-
|
|
486
|
-
# Restore original file_path
|
|
487
|
-
self._file_path = original_file_path
|
|
925
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
488
926
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
927
|
+
return self._save_fig(
|
|
928
|
+
fig=fig,
|
|
929
|
+
filename=filename,
|
|
930
|
+
prefix=prefix,
|
|
931
|
+
description=self._description,
|
|
932
|
+
tags=self._tags,
|
|
933
|
+
metadata=self._metadata,
|
|
934
|
+
**kwargs
|
|
935
|
+
)
|
|
497
936
|
|
|
498
937
|
def save_video(
|
|
499
938
|
self,
|
|
500
939
|
frame_stack: Union[List, Any],
|
|
501
|
-
file_name: str,
|
|
940
|
+
file_name: Optional[str] = None,
|
|
941
|
+
*,
|
|
942
|
+
to: Optional[str] = None,
|
|
502
943
|
fps: int = 20,
|
|
503
944
|
**imageio_kwargs
|
|
504
945
|
) -> Dict[str, Any]:
|
|
@@ -506,42 +947,18 @@ class FileBuilder:
|
|
|
506
947
|
Save video frame stack to a file.
|
|
507
948
|
|
|
508
949
|
Args:
|
|
509
|
-
frame_stack: List of numpy arrays or stacked array
|
|
510
|
-
file_name: Name of file to create (
|
|
950
|
+
frame_stack: List of numpy arrays or stacked array
|
|
951
|
+
file_name: Name of file to create (deprecated, use 'to')
|
|
952
|
+
to: Target filename (preferred)
|
|
511
953
|
fps: Frames per second (default: 20)
|
|
512
|
-
**imageio_kwargs: Additional arguments passed to imageio
|
|
513
|
-
- codec: Video codec (e.g., 'libx264', 'h264')
|
|
514
|
-
- quality: Quality level (int, higher is better)
|
|
515
|
-
- pixelformat: Pixel format (e.g., 'yuv420p')
|
|
516
|
-
- macro_block_size: Macro block size for encoding
|
|
954
|
+
**imageio_kwargs: Additional arguments passed to imageio
|
|
517
955
|
|
|
518
956
|
Returns:
|
|
519
957
|
File metadata dict with id, path, filename, checksum, etc.
|
|
520
958
|
|
|
521
|
-
Raises:
|
|
522
|
-
RuntimeError: If experiment not open or write-protected
|
|
523
|
-
ImportError: If imageio or scikit-image not installed
|
|
524
|
-
ValueError: If frame_stack is empty or invalid format
|
|
525
|
-
|
|
526
959
|
Examples:
|
|
527
|
-
import numpy as np
|
|
528
|
-
|
|
529
|
-
# Grayscale frames (float values 0-1)
|
|
530
960
|
frames = [np.random.rand(480, 640) for _ in range(30)]
|
|
531
|
-
result = experiment.files(
|
|
532
|
-
|
|
533
|
-
# RGB frames with custom FPS
|
|
534
|
-
frames = [np.random.rand(480, 640, 3) for _ in range(60)]
|
|
535
|
-
result = experiment.files(prefix="/videos").save_video(frames, "output.mp4", fps=30)
|
|
536
|
-
|
|
537
|
-
# Save as GIF
|
|
538
|
-
frames = [np.random.rand(200, 200) for _ in range(20)]
|
|
539
|
-
result = experiment.files(prefix="/videos").save_video(frames, "animation.gif")
|
|
540
|
-
|
|
541
|
-
# With custom codec and quality
|
|
542
|
-
result = experiment.files(prefix="/videos").save_video(
|
|
543
|
-
frames, "output.mp4", fps=30, codec='libx264', quality=8
|
|
544
|
-
)
|
|
961
|
+
result = experiment.files("videos").save_video(frames, to="output.mp4")
|
|
545
962
|
"""
|
|
546
963
|
import tempfile
|
|
547
964
|
import os
|
|
@@ -556,51 +973,39 @@ class FileBuilder:
|
|
|
556
973
|
except ImportError:
|
|
557
974
|
raise ImportError("scikit-image is not installed. Install it with: pip install scikit-image")
|
|
558
975
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
if self._experiment._write_protected:
|
|
563
|
-
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
976
|
+
filename = to or file_name
|
|
977
|
+
if not filename:
|
|
978
|
+
raise ValueError("'to' parameter is required")
|
|
564
979
|
|
|
565
980
|
# Validate frame_stack
|
|
566
981
|
try:
|
|
567
|
-
# Handle both list and numpy array
|
|
568
982
|
if len(frame_stack) == 0:
|
|
569
983
|
raise ValueError("frame_stack is empty")
|
|
570
984
|
except TypeError:
|
|
571
985
|
raise ValueError("frame_stack must be a list or numpy array")
|
|
572
986
|
|
|
573
|
-
|
|
987
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
988
|
+
|
|
574
989
|
temp_dir = tempfile.mkdtemp()
|
|
575
|
-
temp_path = os.path.join(temp_dir,
|
|
990
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
576
991
|
|
|
577
992
|
try:
|
|
578
|
-
# Convert frames to uint8 format (handles float32/64, grayscale, RGB, etc.)
|
|
579
|
-
# img_as_ubyte automatically scales [0.0, 1.0] floats to [0, 255] uint8
|
|
580
993
|
frames_ubyte = img_as_ubyte(frame_stack)
|
|
581
|
-
|
|
582
|
-
# Encode video to temp file
|
|
583
994
|
try:
|
|
584
995
|
iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
|
|
585
996
|
except iio.core.NeedDownloadError:
|
|
586
|
-
# Auto-download FFmpeg if not available
|
|
587
997
|
import imageio.plugins.ffmpeg
|
|
588
998
|
imageio.plugins.ffmpeg.download()
|
|
589
999
|
iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
|
|
590
1000
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
# Restore original file_path
|
|
599
|
-
self._file_path = original_file_path
|
|
600
|
-
|
|
601
|
-
return result
|
|
1001
|
+
return self._save_file(
|
|
1002
|
+
file_path=temp_path,
|
|
1003
|
+
prefix=prefix,
|
|
1004
|
+
description=self._description,
|
|
1005
|
+
tags=self._tags,
|
|
1006
|
+
metadata=self._metadata
|
|
1007
|
+
)
|
|
602
1008
|
finally:
|
|
603
|
-
# Clean up temp file and directory
|
|
604
1009
|
try:
|
|
605
1010
|
os.unlink(temp_path)
|
|
606
1011
|
os.rmdir(temp_dir)
|
|
@@ -611,31 +1016,16 @@ class FileBuilder:
|
|
|
611
1016
|
"""
|
|
612
1017
|
Duplicate an existing file to a new path within the same experiment.
|
|
613
1018
|
|
|
614
|
-
Useful for checkpoint rotation patterns where you save versioned checkpoints
|
|
615
|
-
and maintain a "latest" or "best" pointer.
|
|
616
|
-
|
|
617
1019
|
Args:
|
|
618
1020
|
source: Source file - either file ID (str) or metadata dict with 'id' key
|
|
619
1021
|
to: Target path like "models/latest.pt" or "/checkpoints/best.pt"
|
|
620
1022
|
|
|
621
1023
|
Returns:
|
|
622
|
-
File metadata dict for the duplicated file
|
|
623
|
-
|
|
624
|
-
Raises:
|
|
625
|
-
RuntimeError: If experiment is not open or write-protected
|
|
626
|
-
ValueError: If source file not found or target path invalid
|
|
1024
|
+
File metadata dict for the duplicated file
|
|
627
1025
|
|
|
628
1026
|
Examples:
|
|
629
|
-
|
|
630
|
-
dxp.files().duplicate("file-id-123", to="models/latest.pt")
|
|
631
|
-
|
|
632
|
-
# Using metadata dict from save_torch
|
|
633
|
-
snapshot = dxp.files(prefix="/models").save_torch(model, f"model_{epoch:05d}.pt")
|
|
1027
|
+
snapshot = dxp.files("models").save_torch(model, to=f"model_{epoch:05d}.pt")
|
|
634
1028
|
dxp.files().duplicate(snapshot, to="models/latest.pt")
|
|
635
|
-
|
|
636
|
-
# Checkpoint rotation pattern
|
|
637
|
-
snap = dxp.files(prefix="/checkpoints").save_torch(model, f"model_{epoch:05d}.pt")
|
|
638
|
-
dxp.files().duplicate(snap, to="checkpoints/best.pt")
|
|
639
1029
|
"""
|
|
640
1030
|
import tempfile
|
|
641
1031
|
import os
|
|
@@ -669,34 +1059,23 @@ class FileBuilder:
|
|
|
669
1059
|
if not target_filename:
|
|
670
1060
|
raise ValueError(f"Invalid target path '{to}': must include filename")
|
|
671
1061
|
|
|
672
|
-
# Download source file to temp location
|
|
673
1062
|
temp_dir = tempfile.mkdtemp()
|
|
674
1063
|
temp_path = os.path.join(temp_dir, target_filename)
|
|
675
1064
|
|
|
676
1065
|
try:
|
|
677
|
-
# Download the source file
|
|
678
1066
|
downloaded_path = self._experiment._download_file(
|
|
679
1067
|
file_id=source_id,
|
|
680
1068
|
dest_path=temp_path
|
|
681
1069
|
)
|
|
682
1070
|
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
# Upload and get result
|
|
691
|
-
result = self.save()
|
|
692
|
-
|
|
693
|
-
# Restore original values
|
|
694
|
-
self._file_path = original_file_path
|
|
695
|
-
self._prefix = original_prefix
|
|
696
|
-
|
|
697
|
-
return result
|
|
1071
|
+
return self._save_file(
|
|
1072
|
+
file_path=downloaded_path,
|
|
1073
|
+
prefix=target_prefix,
|
|
1074
|
+
description=self._description,
|
|
1075
|
+
tags=self._tags,
|
|
1076
|
+
metadata=self._metadata
|
|
1077
|
+
)
|
|
698
1078
|
finally:
|
|
699
|
-
# Clean up temp file and directory
|
|
700
1079
|
try:
|
|
701
1080
|
if os.path.exists(temp_path):
|
|
702
1081
|
os.unlink(temp_path)
|
|
@@ -705,6 +1084,223 @@ class FileBuilder:
|
|
|
705
1084
|
pass
|
|
706
1085
|
|
|
707
1086
|
|
|
1087
|
+
class FilesAccessor:
|
|
1088
|
+
"""
|
|
1089
|
+
Accessor that enables both callable and attribute-style access to file operations.
|
|
1090
|
+
|
|
1091
|
+
This allows:
|
|
1092
|
+
experiment.files("path") # Returns FileBuilder
|
|
1093
|
+
experiment.files.save(...) # Direct method call
|
|
1094
|
+
experiment.files.download(...) # Direct method call
|
|
1095
|
+
"""
|
|
1096
|
+
|
|
1097
|
+
def __init__(self, experiment: 'Experiment'):
|
|
1098
|
+
self._experiment = experiment
|
|
1099
|
+
self._builder = FileBuilder(experiment)
|
|
1100
|
+
|
|
1101
|
+
def __call__(self, path: Optional[str] = None, **kwargs) -> FileBuilder:
|
|
1102
|
+
"""Create a FileBuilder with the given path."""
|
|
1103
|
+
return FileBuilder(self._experiment, path=path, **kwargs)
|
|
1104
|
+
|
|
1105
|
+
# Direct methods that don't require a path first
|
|
1106
|
+
|
|
1107
|
+
def save(
|
|
1108
|
+
self,
|
|
1109
|
+
obj: Optional[Any] = None,
|
|
1110
|
+
*,
|
|
1111
|
+
to: Optional[str] = None,
|
|
1112
|
+
**kwargs
|
|
1113
|
+
) -> Dict[str, Any]:
|
|
1114
|
+
"""
|
|
1115
|
+
Save a file directly.
|
|
1116
|
+
|
|
1117
|
+
Examples:
|
|
1118
|
+
experiment.files.save("./model.pt")
|
|
1119
|
+
experiment.files.save(model, to="checkpoints/model.pt")
|
|
1120
|
+
"""
|
|
1121
|
+
# Parse 'to' to extract prefix and filename
|
|
1122
|
+
if to:
|
|
1123
|
+
to_path = to.lstrip('/')
|
|
1124
|
+
if '/' in to_path:
|
|
1125
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1126
|
+
prefix = '/' + prefix
|
|
1127
|
+
else:
|
|
1128
|
+
prefix = '/'
|
|
1129
|
+
filename = to_path
|
|
1130
|
+
return FileBuilder(self._experiment, path=prefix, **kwargs).save(obj, to=filename)
|
|
1131
|
+
|
|
1132
|
+
if isinstance(obj, str) and Path(obj).exists():
|
|
1133
|
+
# obj is a file path, extract prefix from it
|
|
1134
|
+
return FileBuilder(self._experiment, **kwargs).save(obj)
|
|
1135
|
+
|
|
1136
|
+
raise ValueError("'to' parameter is required when not saving an existing file path")
|
|
1137
|
+
|
|
1138
|
+
def download(
|
|
1139
|
+
self,
|
|
1140
|
+
path: str,
|
|
1141
|
+
*,
|
|
1142
|
+
to: Optional[str] = None
|
|
1143
|
+
) -> Union[str, List[str]]:
|
|
1144
|
+
"""
|
|
1145
|
+
Download file(s) directly.
|
|
1146
|
+
|
|
1147
|
+
Examples:
|
|
1148
|
+
experiment.files.download("model.pt")
|
|
1149
|
+
experiment.files.download("images/*.png", to="local_images")
|
|
1150
|
+
"""
|
|
1151
|
+
path = path.lstrip('/')
|
|
1152
|
+
|
|
1153
|
+
# Check if path contains glob pattern
|
|
1154
|
+
if '*' in path or '?' in path:
|
|
1155
|
+
# Extract prefix and pattern
|
|
1156
|
+
if '/' in path:
|
|
1157
|
+
parts = path.split('/')
|
|
1158
|
+
# Find where the pattern starts
|
|
1159
|
+
prefix_parts = []
|
|
1160
|
+
pattern_parts = []
|
|
1161
|
+
in_pattern = False
|
|
1162
|
+
for part in parts:
|
|
1163
|
+
if '*' in part or '?' in part:
|
|
1164
|
+
in_pattern = True
|
|
1165
|
+
if in_pattern:
|
|
1166
|
+
pattern_parts.append(part)
|
|
1167
|
+
else:
|
|
1168
|
+
prefix_parts.append(part)
|
|
1169
|
+
|
|
1170
|
+
prefix = '/'.join(prefix_parts) if prefix_parts else None
|
|
1171
|
+
pattern = '/'.join(pattern_parts)
|
|
1172
|
+
else:
|
|
1173
|
+
prefix = None
|
|
1174
|
+
pattern = path
|
|
1175
|
+
|
|
1176
|
+
return FileBuilder(self._experiment, path=prefix).download(pattern, to=to)
|
|
1177
|
+
|
|
1178
|
+
# Single file download
|
|
1179
|
+
return FileBuilder(self._experiment, path=path).download(to=to)
|
|
1180
|
+
|
|
1181
|
+
def delete(self, path: str) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
|
|
1182
|
+
"""
|
|
1183
|
+
Delete file(s) directly.
|
|
1184
|
+
|
|
1185
|
+
Examples:
|
|
1186
|
+
experiment.files.delete("some.text")
|
|
1187
|
+
experiment.files.delete("images/*.png")
|
|
1188
|
+
"""
|
|
1189
|
+
path = path.lstrip('/')
|
|
1190
|
+
|
|
1191
|
+
# Check if path contains glob pattern
|
|
1192
|
+
if '*' in path or '?' in path:
|
|
1193
|
+
# Extract prefix and pattern
|
|
1194
|
+
if '/' in path:
|
|
1195
|
+
parts = path.split('/')
|
|
1196
|
+
prefix_parts = []
|
|
1197
|
+
pattern_parts = []
|
|
1198
|
+
in_pattern = False
|
|
1199
|
+
for part in parts:
|
|
1200
|
+
if '*' in part or '?' in part:
|
|
1201
|
+
in_pattern = True
|
|
1202
|
+
if in_pattern:
|
|
1203
|
+
pattern_parts.append(part)
|
|
1204
|
+
else:
|
|
1205
|
+
prefix_parts.append(part)
|
|
1206
|
+
|
|
1207
|
+
prefix = '/'.join(prefix_parts) if prefix_parts else None
|
|
1208
|
+
pattern = '/'.join(pattern_parts)
|
|
1209
|
+
else:
|
|
1210
|
+
prefix = None
|
|
1211
|
+
pattern = path
|
|
1212
|
+
|
|
1213
|
+
return FileBuilder(self._experiment, path=prefix).delete(pattern)
|
|
1214
|
+
|
|
1215
|
+
# Single file delete
|
|
1216
|
+
return FileBuilder(self._experiment, path=path).delete()
|
|
1217
|
+
|
|
1218
|
+
def list(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
1219
|
+
"""
|
|
1220
|
+
List files directly.
|
|
1221
|
+
|
|
1222
|
+
Examples:
|
|
1223
|
+
files = experiment.files.list()
|
|
1224
|
+
files = experiment.files.list("*.pt")
|
|
1225
|
+
"""
|
|
1226
|
+
return FileBuilder(self._experiment).list(pattern)
|
|
1227
|
+
|
|
1228
|
+
def save_text(self, content: str, *, to: str) -> Dict[str, Any]:
|
|
1229
|
+
"""
|
|
1230
|
+
Save text content to a file.
|
|
1231
|
+
|
|
1232
|
+
Examples:
|
|
1233
|
+
experiment.files.save_text("content", to="view.yaml")
|
|
1234
|
+
"""
|
|
1235
|
+
to_path = to.lstrip('/')
|
|
1236
|
+
if '/' in to_path:
|
|
1237
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1238
|
+
prefix = '/' + prefix
|
|
1239
|
+
else:
|
|
1240
|
+
prefix = '/'
|
|
1241
|
+
filename = to_path
|
|
1242
|
+
return FileBuilder(self._experiment, path=prefix).save_text(content, to=filename)
|
|
1243
|
+
|
|
1244
|
+
def save_json(self, content: Any, *, to: str) -> Dict[str, Any]:
|
|
1245
|
+
"""
|
|
1246
|
+
Save JSON content to a file.
|
|
1247
|
+
|
|
1248
|
+
Examples:
|
|
1249
|
+
experiment.files.save_json({"key": "value"}, to="config.json")
|
|
1250
|
+
"""
|
|
1251
|
+
to_path = to.lstrip('/')
|
|
1252
|
+
if '/' in to_path:
|
|
1253
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1254
|
+
prefix = '/' + prefix
|
|
1255
|
+
else:
|
|
1256
|
+
prefix = '/'
|
|
1257
|
+
filename = to_path
|
|
1258
|
+
return FileBuilder(self._experiment, path=prefix).save_json(content, to=filename)
|
|
1259
|
+
|
|
1260
|
+
def save_blob(self, data: bytes, *, to: str) -> Dict[str, Any]:
|
|
1261
|
+
"""
|
|
1262
|
+
Save binary data to a file.
|
|
1263
|
+
|
|
1264
|
+
Examples:
|
|
1265
|
+
experiment.files.save_blob(b"data", to="data.bin")
|
|
1266
|
+
"""
|
|
1267
|
+
to_path = to.lstrip('/')
|
|
1268
|
+
if '/' in to_path:
|
|
1269
|
+
prefix, filename = to_path.rsplit('/', 1)
|
|
1270
|
+
prefix = '/' + prefix
|
|
1271
|
+
else:
|
|
1272
|
+
prefix = '/'
|
|
1273
|
+
filename = to_path
|
|
1274
|
+
return FileBuilder(self._experiment, path=prefix).save_blob(data, to=filename)
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
class BindrsBuilder:
|
|
1278
|
+
"""
|
|
1279
|
+
Fluent interface for bindr (collection) operations.
|
|
1280
|
+
|
|
1281
|
+
Usage:
|
|
1282
|
+
file_paths = experiment.bindrs("some-bindr").list()
|
|
1283
|
+
"""
|
|
1284
|
+
|
|
1285
|
+
def __init__(self, experiment: 'Experiment', bindr_name: str):
|
|
1286
|
+
self._experiment = experiment
|
|
1287
|
+
self._bindr_name = bindr_name
|
|
1288
|
+
|
|
1289
|
+
def list(self) -> List[Dict[str, Any]]:
|
|
1290
|
+
"""
|
|
1291
|
+
List files in this bindr.
|
|
1292
|
+
|
|
1293
|
+
Returns:
|
|
1294
|
+
List of file metadata dicts belonging to this bindr
|
|
1295
|
+
"""
|
|
1296
|
+
if not self._experiment._is_open:
|
|
1297
|
+
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1298
|
+
|
|
1299
|
+
# Get all files and filter by bindr
|
|
1300
|
+
all_files = self._experiment._list_files(prefix=None, tags=None)
|
|
1301
|
+
return [f for f in all_files if self._bindr_name in f.get('bindrs', [])]
|
|
1302
|
+
|
|
1303
|
+
|
|
708
1304
|
def compute_sha256(file_path: str) -> str:
|
|
709
1305
|
"""
|
|
710
1306
|
Compute SHA256 checksum of a file.
|