ml-dash 0.6.2__py3-none-any.whl → 0.6.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +64 -36
- ml_dash/auth/token_storage.py +226 -267
- ml_dash/auto_start.py +15 -28
- ml_dash/cli.py +2 -16
- ml_dash/cli_commands/download.py +667 -757
- ml_dash/cli_commands/list.py +13 -146
- ml_dash/cli_commands/login.py +183 -190
- ml_dash/cli_commands/upload.py +1141 -1291
- ml_dash/client.py +6 -79
- ml_dash/config.py +119 -119
- ml_dash/experiment.py +1034 -1234
- ml_dash/files.py +224 -339
- ml_dash/log.py +7 -7
- ml_dash/metric.py +100 -359
- ml_dash/params.py +6 -6
- ml_dash/remote_auto_start.py +17 -20
- ml_dash/run.py +65 -211
- ml_dash/storage.py +1081 -1051
- {ml_dash-0.6.2.dist-info → ml_dash-0.6.2rc1.dist-info}/METADATA +14 -12
- ml_dash-0.6.2rc1.dist-info/RECORD +30 -0
- {ml_dash-0.6.2.dist-info → ml_dash-0.6.2rc1.dist-info}/WHEEL +1 -1
- ml_dash/cli_commands/api.py +0 -165
- ml_dash/cli_commands/profile.py +0 -92
- ml_dash/snowflake.py +0 -173
- ml_dash-0.6.2.dist-info/RECORD +0 -33
- {ml_dash-0.6.2.dist-info → ml_dash-0.6.2rc1.dist-info}/entry_points.txt +0 -0
ml_dash/files.py
CHANGED
|
@@ -19,45 +19,28 @@ class FileBuilder:
|
|
|
19
19
|
Fluent interface for file operations.
|
|
20
20
|
|
|
21
21
|
Usage:
|
|
22
|
-
# Upload
|
|
23
|
-
|
|
24
|
-
dxp.files("checkpoints").upload("./model.pt", to="latest.pt")
|
|
25
|
-
|
|
26
|
-
# Save objects as files (using keyword dir argument)
|
|
27
|
-
dxp.files(dir="models").save_torch(model, to="checkpoint.pt")
|
|
28
|
-
dxp.files(dir="configs").save_json({"lr": 0.001}, to="config.json")
|
|
29
|
-
dxp.files(dir="data").save_blob(b"binary data", to="data.bin")
|
|
22
|
+
# Upload file
|
|
23
|
+
experiment.files("checkpoints").save(net, to="checkpoint.pt")
|
|
30
24
|
|
|
31
25
|
# List files
|
|
32
|
-
files = experiment.files(
|
|
33
|
-
files = experiment.files(
|
|
26
|
+
files = experiment.files("/some/location").list()
|
|
27
|
+
files = experiment.files("/models").list()
|
|
34
28
|
|
|
35
29
|
# Download file
|
|
36
|
-
|
|
37
|
-
|
|
30
|
+
experiment.files("some.text").download()
|
|
31
|
+
experiment.files("some.text").download(to="./model.pt")
|
|
38
32
|
|
|
39
|
-
# Download
|
|
33
|
+
# Download Files via Glob Pattern
|
|
40
34
|
file_paths = experiment.files("images").list("*.png")
|
|
41
|
-
|
|
35
|
+
experiment.files("images").download("*.png")
|
|
42
36
|
|
|
43
37
|
# Delete files
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
# Read file content as text
|
|
51
|
-
config_yaml = dxp.files("configs/view.yaml").read_text()
|
|
52
|
-
|
|
53
|
-
Specific Save Methods:
|
|
54
|
-
experiment.files.save_text("content", to="view.yaml")
|
|
55
|
-
experiment.files.save_json(dict(hey="yo"), to="config.json")
|
|
56
|
-
experiment.files.save_blob(b"xxx", to="data.bin")
|
|
57
|
-
experiment.files.save_torch(model, to="model.pt")
|
|
58
|
-
experiment.files.save_pkl(data, to="data.pkl")
|
|
59
|
-
experiment.files.save_fig(fig, to="plot.png")
|
|
60
|
-
experiment.files.save_video(frames, to="video.mp4")
|
|
38
|
+
experiment.files("some.text").delete()
|
|
39
|
+
|
|
40
|
+
Specific File Types:
|
|
41
|
+
dxp.files.save_text("content", to="view.yaml")
|
|
42
|
+
dxp.files.save_json(dict(hey="yo"), to="config.json")
|
|
43
|
+
dxp.files.save_blob(b"xxx", to="data.bin")
|
|
61
44
|
"""
|
|
62
45
|
|
|
63
46
|
def __init__(self, experiment: 'Experiment', path: Optional[str] = None, **kwargs):
|
|
@@ -97,9 +80,9 @@ class FileBuilder:
|
|
|
97
80
|
path = path.lstrip('/')
|
|
98
81
|
self._normalized_path = '/' + path if not path.startswith('/') else path
|
|
99
82
|
|
|
100
|
-
def
|
|
83
|
+
def save(
|
|
101
84
|
self,
|
|
102
|
-
|
|
85
|
+
obj: Optional[Any] = None,
|
|
103
86
|
*,
|
|
104
87
|
to: Optional[str] = None,
|
|
105
88
|
description: Optional[str] = None,
|
|
@@ -107,14 +90,21 @@ class FileBuilder:
|
|
|
107
90
|
metadata: Optional[Dict[str, Any]] = None
|
|
108
91
|
) -> Dict[str, Any]:
|
|
109
92
|
"""
|
|
110
|
-
Upload
|
|
93
|
+
Upload and save a file or object.
|
|
111
94
|
|
|
112
95
|
Args:
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
96
|
+
obj: Object to save. Can be:
|
|
97
|
+
- None: Uses file_path from constructor (backwards compatibility)
|
|
98
|
+
- str: Path to an existing file
|
|
99
|
+
- bytes: Binary data to save
|
|
100
|
+
- dict/list: JSON-serializable data
|
|
101
|
+
- PyTorch model/state_dict: Saved with torch.save()
|
|
102
|
+
- matplotlib figure: Saved as image
|
|
103
|
+
- Any picklable object: Saved with pickle
|
|
104
|
+
to: Target filename (required when obj is not a file path)
|
|
105
|
+
description: Optional description (overrides constructor)
|
|
106
|
+
tags: Optional list of tags (overrides constructor)
|
|
107
|
+
metadata: Optional metadata dict (overrides constructor)
|
|
118
108
|
|
|
119
109
|
Returns:
|
|
120
110
|
File metadata dict with id, path, filename, checksum, etc.
|
|
@@ -125,19 +115,19 @@ class FileBuilder:
|
|
|
125
115
|
ValueError: If file size exceeds 100GB limit
|
|
126
116
|
|
|
127
117
|
Examples:
|
|
128
|
-
#
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
)
|
|
118
|
+
# Save existing file
|
|
119
|
+
experiment.files("models").save("./model.pt")
|
|
120
|
+
experiment.files("models").save(to="model.pt") # copies from self._file_path
|
|
121
|
+
|
|
122
|
+
# Save PyTorch model
|
|
123
|
+
experiment.files("checkpoints").save(model, to="checkpoint.pt")
|
|
124
|
+
experiment.files("checkpoints").save(model.state_dict(), to="weights.pt")
|
|
125
|
+
|
|
126
|
+
# Save dict as JSON
|
|
127
|
+
experiment.files("configs").save({"lr": 0.001}, to="config.json")
|
|
128
|
+
|
|
129
|
+
# Save bytes
|
|
130
|
+
experiment.files("data").save(b"binary data", to="data.bin")
|
|
141
131
|
"""
|
|
142
132
|
if not self._experiment._is_open:
|
|
143
133
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
@@ -145,9 +135,6 @@ class FileBuilder:
|
|
|
145
135
|
if self._experiment._write_protected:
|
|
146
136
|
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
147
137
|
|
|
148
|
-
if not fpath:
|
|
149
|
-
raise ValueError("fpath is required")
|
|
150
|
-
|
|
151
138
|
# Use provided values or fall back to constructor values
|
|
152
139
|
desc = description if description is not None else self._description
|
|
153
140
|
file_tags = tags if tags is not None else self._tags
|
|
@@ -158,109 +145,136 @@ class FileBuilder:
|
|
|
158
145
|
if self._path:
|
|
159
146
|
prefix = '/' + self._path.lstrip('/')
|
|
160
147
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
*,
|
|
174
|
-
to: Optional[str] = None,
|
|
175
|
-
description: Optional[str] = None,
|
|
176
|
-
tags: Optional[List[str]] = None,
|
|
177
|
-
metadata: Optional[Dict[str, Any]] = None
|
|
178
|
-
) -> Dict[str, Any]:
|
|
179
|
-
"""
|
|
180
|
-
Unified save method that handles different content types.
|
|
181
|
-
|
|
182
|
-
Args:
|
|
183
|
-
content: Content to save - can be:
|
|
184
|
-
- str (file path): Uploads existing file
|
|
185
|
-
- bytes: Saves as binary blob (requires 'to' parameter)
|
|
186
|
-
- dict/list: Saves as JSON (requires 'to' parameter)
|
|
187
|
-
- None: Uses file_path from constructor (backwards compatibility)
|
|
188
|
-
to: Target filename (required for bytes/dict/list, optional for file paths)
|
|
189
|
-
description: Optional description
|
|
190
|
-
tags: Optional list of tags
|
|
191
|
-
metadata: Optional metadata dict
|
|
192
|
-
|
|
193
|
-
Returns:
|
|
194
|
-
File metadata dict with id, path, filename, checksum, etc.
|
|
195
|
-
"""
|
|
196
|
-
# Override builder metadata if provided
|
|
197
|
-
if description is not None:
|
|
198
|
-
self._description = description
|
|
199
|
-
if tags is not None:
|
|
200
|
-
self._tags = tags
|
|
201
|
-
if metadata is not None:
|
|
202
|
-
self._metadata = metadata
|
|
203
|
-
|
|
204
|
-
# Backwards compatibility: use file_path from constructor if no content provided
|
|
205
|
-
if content is None:
|
|
206
|
-
if self._file_path:
|
|
207
|
-
content = self._file_path
|
|
208
|
-
else:
|
|
209
|
-
raise ValueError("No content provided and no file_path set in constructor")
|
|
148
|
+
# Handle different object types
|
|
149
|
+
if obj is None:
|
|
150
|
+
# Backwards compatibility: use file_path from constructor
|
|
151
|
+
if not self._file_path:
|
|
152
|
+
raise ValueError("No file or object provided. Pass a file path or object to save().")
|
|
153
|
+
return self._save_file(
|
|
154
|
+
file_path=self._file_path,
|
|
155
|
+
prefix=prefix,
|
|
156
|
+
description=desc,
|
|
157
|
+
tags=file_tags,
|
|
158
|
+
metadata=file_metadata
|
|
159
|
+
)
|
|
210
160
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
return self.
|
|
161
|
+
if isinstance(obj, str) and Path(obj).exists():
|
|
162
|
+
# obj is a path to an existing file
|
|
163
|
+
return self._save_file(
|
|
164
|
+
file_path=obj,
|
|
165
|
+
prefix=prefix,
|
|
166
|
+
description=desc,
|
|
167
|
+
tags=file_tags,
|
|
168
|
+
metadata=file_metadata
|
|
169
|
+
)
|
|
214
170
|
|
|
215
|
-
|
|
216
|
-
|
|
171
|
+
if isinstance(obj, bytes):
|
|
172
|
+
# Save bytes directly
|
|
217
173
|
if not to:
|
|
218
174
|
raise ValueError("'to' parameter is required when saving bytes")
|
|
219
|
-
return self.
|
|
175
|
+
return self._save_bytes(
|
|
176
|
+
data=obj,
|
|
177
|
+
filename=to,
|
|
178
|
+
prefix=prefix,
|
|
179
|
+
description=desc,
|
|
180
|
+
tags=file_tags,
|
|
181
|
+
metadata=file_metadata
|
|
182
|
+
)
|
|
220
183
|
|
|
221
|
-
|
|
222
|
-
|
|
184
|
+
if isinstance(obj, (dict, list)):
|
|
185
|
+
# Try JSON first
|
|
223
186
|
if not to:
|
|
224
187
|
raise ValueError("'to' parameter is required when saving dict/list")
|
|
225
|
-
return self.
|
|
188
|
+
return self._save_json(
|
|
189
|
+
content=obj,
|
|
190
|
+
filename=to,
|
|
191
|
+
prefix=prefix,
|
|
192
|
+
description=desc,
|
|
193
|
+
tags=file_tags,
|
|
194
|
+
metadata=file_metadata
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
# Check for PyTorch model
|
|
198
|
+
try:
|
|
199
|
+
import torch
|
|
200
|
+
if isinstance(obj, (torch.nn.Module, dict)) or hasattr(obj, 'state_dict'):
|
|
201
|
+
if not to:
|
|
202
|
+
raise ValueError("'to' parameter is required when saving PyTorch model")
|
|
203
|
+
return self._save_torch(
|
|
204
|
+
model=obj,
|
|
205
|
+
filename=to,
|
|
206
|
+
prefix=prefix,
|
|
207
|
+
description=desc,
|
|
208
|
+
tags=file_tags,
|
|
209
|
+
metadata=file_metadata
|
|
210
|
+
)
|
|
211
|
+
except ImportError:
|
|
212
|
+
pass
|
|
226
213
|
|
|
227
|
-
|
|
214
|
+
# Check for matplotlib figure
|
|
215
|
+
try:
|
|
216
|
+
import matplotlib.pyplot as plt
|
|
217
|
+
from matplotlib.figure import Figure
|
|
218
|
+
if isinstance(obj, Figure):
|
|
219
|
+
if not to:
|
|
220
|
+
raise ValueError("'to' parameter is required when saving matplotlib figure")
|
|
221
|
+
return self._save_fig(
|
|
222
|
+
fig=obj,
|
|
223
|
+
filename=to,
|
|
224
|
+
prefix=prefix,
|
|
225
|
+
description=desc,
|
|
226
|
+
tags=file_tags,
|
|
227
|
+
metadata=file_metadata
|
|
228
|
+
)
|
|
229
|
+
except ImportError:
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
# Fall back to pickle
|
|
233
|
+
if not to:
|
|
234
|
+
raise ValueError("'to' parameter is required when saving object")
|
|
235
|
+
return self._save_pickle(
|
|
236
|
+
content=obj,
|
|
237
|
+
filename=to,
|
|
238
|
+
prefix=prefix,
|
|
239
|
+
description=desc,
|
|
240
|
+
tags=file_tags,
|
|
241
|
+
metadata=file_metadata
|
|
242
|
+
)
|
|
228
243
|
|
|
229
244
|
def _save_file(
|
|
230
245
|
self,
|
|
231
|
-
|
|
246
|
+
file_path: str,
|
|
232
247
|
prefix: str,
|
|
233
248
|
description: Optional[str],
|
|
234
249
|
tags: Optional[List[str]],
|
|
235
|
-
metadata: Optional[Dict[str, Any]]
|
|
236
|
-
to: Optional[str] = None
|
|
250
|
+
metadata: Optional[Dict[str, Any]]
|
|
237
251
|
) -> Dict[str, Any]:
|
|
238
252
|
"""Internal method to save an existing file."""
|
|
239
|
-
|
|
240
|
-
if not
|
|
241
|
-
raise ValueError(f"File not found: {
|
|
253
|
+
file_path_obj = Path(file_path)
|
|
254
|
+
if not file_path_obj.exists():
|
|
255
|
+
raise ValueError(f"File not found: {file_path}")
|
|
242
256
|
|
|
243
|
-
if not
|
|
244
|
-
raise ValueError(f"Path is not a file: {
|
|
257
|
+
if not file_path_obj.is_file():
|
|
258
|
+
raise ValueError(f"Path is not a file: {file_path}")
|
|
245
259
|
|
|
246
260
|
# Check file size (max 100GB)
|
|
247
|
-
file_size =
|
|
261
|
+
file_size = file_path_obj.stat().st_size
|
|
248
262
|
MAX_FILE_SIZE = 100 * 1024 * 1024 * 1024 # 100GB in bytes
|
|
249
263
|
if file_size > MAX_FILE_SIZE:
|
|
250
264
|
raise ValueError(f"File size ({file_size} bytes) exceeds 100GB limit")
|
|
251
265
|
|
|
252
266
|
# Compute checksum
|
|
253
|
-
checksum = compute_sha256(str(
|
|
267
|
+
checksum = compute_sha256(str(file_path_obj))
|
|
254
268
|
|
|
255
269
|
# Detect MIME type
|
|
256
|
-
content_type = get_mime_type(str(
|
|
270
|
+
content_type = get_mime_type(str(file_path_obj))
|
|
257
271
|
|
|
258
|
-
# Get filename
|
|
259
|
-
filename =
|
|
272
|
+
# Get filename
|
|
273
|
+
filename = file_path_obj.name
|
|
260
274
|
|
|
261
275
|
# Upload through experiment
|
|
262
276
|
return self._experiment._upload_file(
|
|
263
|
-
file_path=str(
|
|
277
|
+
file_path=str(file_path_obj),
|
|
264
278
|
prefix=prefix,
|
|
265
279
|
filename=filename,
|
|
266
280
|
description=description,
|
|
@@ -286,13 +300,11 @@ class FileBuilder:
|
|
|
286
300
|
|
|
287
301
|
temp_dir = tempfile.mkdtemp()
|
|
288
302
|
temp_path = os.path.join(temp_dir, filename)
|
|
289
|
-
# Create parent directories if filename contains path
|
|
290
|
-
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
291
303
|
try:
|
|
292
304
|
with open(temp_path, 'wb') as f:
|
|
293
305
|
f.write(data)
|
|
294
306
|
return self._save_file(
|
|
295
|
-
|
|
307
|
+
file_path=temp_path,
|
|
296
308
|
prefix=prefix,
|
|
297
309
|
description=description,
|
|
298
310
|
tags=tags,
|
|
@@ -321,13 +333,11 @@ class FileBuilder:
|
|
|
321
333
|
|
|
322
334
|
temp_dir = tempfile.mkdtemp()
|
|
323
335
|
temp_path = os.path.join(temp_dir, filename)
|
|
324
|
-
# Create parent directories if filename contains path
|
|
325
|
-
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
326
336
|
try:
|
|
327
337
|
with open(temp_path, 'w') as f:
|
|
328
338
|
json.dump(content, f, indent=2)
|
|
329
339
|
return self._save_file(
|
|
330
|
-
|
|
340
|
+
file_path=temp_path,
|
|
331
341
|
prefix=prefix,
|
|
332
342
|
description=description,
|
|
333
343
|
tags=tags,
|
|
@@ -356,12 +366,10 @@ class FileBuilder:
|
|
|
356
366
|
|
|
357
367
|
temp_dir = tempfile.mkdtemp()
|
|
358
368
|
temp_path = os.path.join(temp_dir, filename)
|
|
359
|
-
# Create parent directories if filename contains path
|
|
360
|
-
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
361
369
|
try:
|
|
362
370
|
torch.save(model, temp_path)
|
|
363
371
|
return self._save_file(
|
|
364
|
-
|
|
372
|
+
file_path=temp_path,
|
|
365
373
|
prefix=prefix,
|
|
366
374
|
description=description,
|
|
367
375
|
tags=tags,
|
|
@@ -391,13 +399,11 @@ class FileBuilder:
|
|
|
391
399
|
|
|
392
400
|
temp_dir = tempfile.mkdtemp()
|
|
393
401
|
temp_path = os.path.join(temp_dir, filename)
|
|
394
|
-
# Create parent directories if filename contains path
|
|
395
|
-
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
396
402
|
try:
|
|
397
403
|
fig.savefig(temp_path, **kwargs)
|
|
398
404
|
plt.close(fig)
|
|
399
405
|
return self._save_file(
|
|
400
|
-
|
|
406
|
+
file_path=temp_path,
|
|
401
407
|
prefix=prefix,
|
|
402
408
|
description=description,
|
|
403
409
|
tags=tags,
|
|
@@ -426,13 +432,11 @@ class FileBuilder:
|
|
|
426
432
|
|
|
427
433
|
temp_dir = tempfile.mkdtemp()
|
|
428
434
|
temp_path = os.path.join(temp_dir, filename)
|
|
429
|
-
# Create parent directories if filename contains path
|
|
430
|
-
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
431
435
|
try:
|
|
432
436
|
with open(temp_path, 'wb') as f:
|
|
433
437
|
pickle.dump(content, f)
|
|
434
438
|
return self._save_file(
|
|
435
|
-
|
|
439
|
+
file_path=temp_path,
|
|
436
440
|
prefix=prefix,
|
|
437
441
|
description=description,
|
|
438
442
|
tags=tags,
|
|
@@ -717,36 +721,34 @@ class FileBuilder:
|
|
|
717
721
|
def save_json(
|
|
718
722
|
self,
|
|
719
723
|
content: Any,
|
|
724
|
+
file_name: Optional[str] = None,
|
|
720
725
|
*,
|
|
721
|
-
to: str
|
|
726
|
+
to: Optional[str] = None
|
|
722
727
|
) -> Dict[str, Any]:
|
|
723
728
|
"""
|
|
724
729
|
Save JSON content to a file.
|
|
725
730
|
|
|
726
731
|
Args:
|
|
727
732
|
content: Content to save as JSON (dict, list, or any JSON-serializable object)
|
|
728
|
-
|
|
733
|
+
file_name: Name of the file to create (deprecated, use 'to')
|
|
734
|
+
to: Target filename (preferred)
|
|
729
735
|
|
|
730
736
|
Returns:
|
|
731
737
|
File metadata dict with id, path, filename, checksum, etc.
|
|
732
738
|
|
|
733
739
|
Examples:
|
|
734
740
|
config = {"model": "resnet50", "lr": 0.001}
|
|
735
|
-
result =
|
|
741
|
+
result = experiment.files("configs").save_json(config, to="config.json")
|
|
736
742
|
"""
|
|
737
|
-
|
|
743
|
+
filename = to or file_name
|
|
744
|
+
if not filename:
|
|
745
|
+
raise ValueError("'to' parameter is required")
|
|
738
746
|
|
|
739
|
-
|
|
740
|
-
import os
|
|
741
|
-
to_dirname = os.path.dirname(to)
|
|
742
|
-
to_filename = os.path.basename(to)
|
|
743
|
-
if to_dirname:
|
|
744
|
-
# Merge the path component into prefix
|
|
745
|
-
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
747
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
746
748
|
|
|
747
749
|
return self._save_json(
|
|
748
750
|
content=content,
|
|
749
|
-
filename=
|
|
751
|
+
filename=filename,
|
|
750
752
|
prefix=prefix,
|
|
751
753
|
description=self._description,
|
|
752
754
|
tags=self._tags,
|
|
@@ -768,18 +770,14 @@ class FileBuilder:
|
|
|
768
770
|
result = experiment.files().save_text("Hello, world!", to="greeting.txt")
|
|
769
771
|
result = experiment.files("configs").save_text(yaml_content, to="view.yaml")
|
|
770
772
|
"""
|
|
771
|
-
|
|
773
|
+
if not to:
|
|
774
|
+
raise ValueError("'to' parameter is required")
|
|
772
775
|
|
|
773
|
-
|
|
774
|
-
import os
|
|
775
|
-
to_dirname = os.path.dirname(to)
|
|
776
|
-
to_filename = os.path.basename(to)
|
|
777
|
-
if to_dirname:
|
|
778
|
-
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
776
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
779
777
|
|
|
780
778
|
return self._save_bytes(
|
|
781
779
|
data=content.encode('utf-8'),
|
|
782
|
-
filename=
|
|
780
|
+
filename=to,
|
|
783
781
|
prefix=prefix,
|
|
784
782
|
description=self._description,
|
|
785
783
|
tags=self._tags,
|
|
@@ -800,18 +798,14 @@ class FileBuilder:
|
|
|
800
798
|
Examples:
|
|
801
799
|
result = experiment.files("data").save_blob(binary_data, to="model.bin")
|
|
802
800
|
"""
|
|
803
|
-
|
|
801
|
+
if not to:
|
|
802
|
+
raise ValueError("'to' parameter is required")
|
|
804
803
|
|
|
805
|
-
|
|
806
|
-
import os
|
|
807
|
-
to_dirname = os.path.dirname(to)
|
|
808
|
-
to_filename = os.path.basename(to)
|
|
809
|
-
if to_dirname:
|
|
810
|
-
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
804
|
+
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
811
805
|
|
|
812
806
|
return self._save_bytes(
|
|
813
807
|
data=data,
|
|
814
|
-
filename=
|
|
808
|
+
filename=to,
|
|
815
809
|
prefix=prefix,
|
|
816
810
|
description=self._description,
|
|
817
811
|
tags=self._tags,
|
|
@@ -821,29 +815,34 @@ class FileBuilder:
|
|
|
821
815
|
def save_torch(
|
|
822
816
|
self,
|
|
823
817
|
model: Any,
|
|
818
|
+
file_name: Optional[str] = None,
|
|
824
819
|
*,
|
|
825
|
-
to: str
|
|
820
|
+
to: Optional[str] = None
|
|
826
821
|
) -> Dict[str, Any]:
|
|
827
822
|
"""
|
|
828
823
|
Save PyTorch model to a file.
|
|
829
824
|
|
|
830
825
|
Args:
|
|
831
826
|
model: PyTorch model or state dict to save
|
|
832
|
-
|
|
827
|
+
file_name: Name of the file to create (deprecated, use 'to')
|
|
828
|
+
to: Target filename (preferred)
|
|
833
829
|
|
|
834
830
|
Returns:
|
|
835
831
|
File metadata dict with id, path, filename, checksum, etc.
|
|
836
832
|
|
|
837
833
|
Examples:
|
|
838
|
-
result =
|
|
839
|
-
result =
|
|
834
|
+
result = experiment.files("models").save_torch(model, to="model.pt")
|
|
835
|
+
result = experiment.files("models").save_torch(model.state_dict(), to="weights.pth")
|
|
840
836
|
"""
|
|
837
|
+
filename = to or file_name
|
|
838
|
+
if not filename:
|
|
839
|
+
raise ValueError("'to' parameter is required")
|
|
841
840
|
|
|
842
841
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
843
842
|
|
|
844
843
|
return self._save_torch(
|
|
845
844
|
model=model,
|
|
846
|
-
filename=
|
|
845
|
+
filename=filename,
|
|
847
846
|
prefix=prefix,
|
|
848
847
|
description=self._description,
|
|
849
848
|
tags=self._tags,
|
|
@@ -853,28 +852,34 @@ class FileBuilder:
|
|
|
853
852
|
def save_pkl(
|
|
854
853
|
self,
|
|
855
854
|
content: Any,
|
|
855
|
+
file_name: Optional[str] = None,
|
|
856
856
|
*,
|
|
857
|
-
to: str
|
|
857
|
+
to: Optional[str] = None
|
|
858
858
|
) -> Dict[str, Any]:
|
|
859
859
|
"""
|
|
860
860
|
Save Python object to a pickle file.
|
|
861
861
|
|
|
862
862
|
Args:
|
|
863
863
|
content: Python object to pickle (must be pickle-serializable)
|
|
864
|
-
|
|
864
|
+
file_name: Name of the file to create (deprecated, use 'to')
|
|
865
|
+
to: Target filename (preferred)
|
|
865
866
|
|
|
866
867
|
Returns:
|
|
867
868
|
File metadata dict with id, path, filename, checksum, etc.
|
|
868
869
|
|
|
869
870
|
Examples:
|
|
870
871
|
data = {"model": "resnet50", "weights": np.array([1, 2, 3])}
|
|
871
|
-
result =
|
|
872
|
+
result = experiment.files("data").save_pkl(data, to="data.pkl")
|
|
872
873
|
"""
|
|
874
|
+
filename = to or file_name
|
|
875
|
+
if not filename:
|
|
876
|
+
raise ValueError("'to' parameter is required")
|
|
877
|
+
|
|
873
878
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
874
879
|
|
|
875
880
|
return self._save_pickle(
|
|
876
881
|
content=content,
|
|
877
|
-
filename=
|
|
882
|
+
filename=filename,
|
|
878
883
|
prefix=prefix,
|
|
879
884
|
description=self._description,
|
|
880
885
|
tags=self._tags,
|
|
@@ -884,8 +889,9 @@ class FileBuilder:
|
|
|
884
889
|
def save_fig(
|
|
885
890
|
self,
|
|
886
891
|
fig: Optional[Any] = None,
|
|
892
|
+
file_name: Optional[str] = None,
|
|
887
893
|
*,
|
|
888
|
-
to: str,
|
|
894
|
+
to: Optional[str] = None,
|
|
889
895
|
**kwargs
|
|
890
896
|
) -> Dict[str, Any]:
|
|
891
897
|
"""
|
|
@@ -893,7 +899,8 @@ class FileBuilder:
|
|
|
893
899
|
|
|
894
900
|
Args:
|
|
895
901
|
fig: Matplotlib figure object. If None, uses plt.gcf() (current figure)
|
|
896
|
-
|
|
902
|
+
file_name: Name of file to create (deprecated, use 'to')
|
|
903
|
+
to: Target filename (preferred)
|
|
897
904
|
**kwargs: Additional arguments passed to fig.savefig()
|
|
898
905
|
|
|
899
906
|
Returns:
|
|
@@ -901,13 +908,17 @@ class FileBuilder:
|
|
|
901
908
|
|
|
902
909
|
Examples:
|
|
903
910
|
plt.plot([1, 2, 3], [1, 4, 9])
|
|
904
|
-
result =
|
|
911
|
+
result = experiment.files("plots").save_fig(to="plot.png")
|
|
905
912
|
"""
|
|
906
913
|
try:
|
|
907
914
|
import matplotlib.pyplot as plt
|
|
908
915
|
except ImportError:
|
|
909
916
|
raise ImportError("Matplotlib is not installed. Install it with: pip install matplotlib")
|
|
910
917
|
|
|
918
|
+
filename = to or file_name
|
|
919
|
+
if not filename:
|
|
920
|
+
raise ValueError("'to' parameter is required")
|
|
921
|
+
|
|
911
922
|
if fig is None:
|
|
912
923
|
fig = plt.gcf()
|
|
913
924
|
|
|
@@ -915,7 +926,7 @@ class FileBuilder:
|
|
|
915
926
|
|
|
916
927
|
return self._save_fig(
|
|
917
928
|
fig=fig,
|
|
918
|
-
filename=
|
|
929
|
+
filename=filename,
|
|
919
930
|
prefix=prefix,
|
|
920
931
|
description=self._description,
|
|
921
932
|
tags=self._tags,
|
|
@@ -926,8 +937,9 @@ class FileBuilder:
|
|
|
926
937
|
def save_video(
|
|
927
938
|
self,
|
|
928
939
|
frame_stack: Union[List, Any],
|
|
940
|
+
file_name: Optional[str] = None,
|
|
929
941
|
*,
|
|
930
|
-
to: str,
|
|
942
|
+
to: Optional[str] = None,
|
|
931
943
|
fps: int = 20,
|
|
932
944
|
**imageio_kwargs
|
|
933
945
|
) -> Dict[str, Any]:
|
|
@@ -936,7 +948,8 @@ class FileBuilder:
|
|
|
936
948
|
|
|
937
949
|
Args:
|
|
938
950
|
frame_stack: List of numpy arrays or stacked array
|
|
939
|
-
|
|
951
|
+
file_name: Name of file to create (deprecated, use 'to')
|
|
952
|
+
to: Target filename (preferred)
|
|
940
953
|
fps: Frames per second (default: 20)
|
|
941
954
|
**imageio_kwargs: Additional arguments passed to imageio
|
|
942
955
|
|
|
@@ -945,7 +958,7 @@ class FileBuilder:
|
|
|
945
958
|
|
|
946
959
|
Examples:
|
|
947
960
|
frames = [np.random.rand(480, 640) for _ in range(30)]
|
|
948
|
-
result =
|
|
961
|
+
result = experiment.files("videos").save_video(frames, to="output.mp4")
|
|
949
962
|
"""
|
|
950
963
|
import tempfile
|
|
951
964
|
import os
|
|
@@ -960,6 +973,10 @@ class FileBuilder:
|
|
|
960
973
|
except ImportError:
|
|
961
974
|
raise ImportError("scikit-image is not installed. Install it with: pip install scikit-image")
|
|
962
975
|
|
|
976
|
+
filename = to or file_name
|
|
977
|
+
if not filename:
|
|
978
|
+
raise ValueError("'to' parameter is required")
|
|
979
|
+
|
|
963
980
|
# Validate frame_stack
|
|
964
981
|
try:
|
|
965
982
|
if len(frame_stack) == 0:
|
|
@@ -970,9 +987,7 @@ class FileBuilder:
|
|
|
970
987
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
971
988
|
|
|
972
989
|
temp_dir = tempfile.mkdtemp()
|
|
973
|
-
temp_path = os.path.join(temp_dir,
|
|
974
|
-
# Create parent directories if filename contains path
|
|
975
|
-
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
990
|
+
temp_path = os.path.join(temp_dir, filename)
|
|
976
991
|
|
|
977
992
|
try:
|
|
978
993
|
frames_ubyte = img_as_ubyte(frame_stack)
|
|
@@ -984,7 +999,7 @@ class FileBuilder:
|
|
|
984
999
|
iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
|
|
985
1000
|
|
|
986
1001
|
return self._save_file(
|
|
987
|
-
|
|
1002
|
+
file_path=temp_path,
|
|
988
1003
|
prefix=prefix,
|
|
989
1004
|
description=self._description,
|
|
990
1005
|
tags=self._tags,
|
|
@@ -1054,7 +1069,7 @@ class FileBuilder:
|
|
|
1054
1069
|
)
|
|
1055
1070
|
|
|
1056
1071
|
return self._save_file(
|
|
1057
|
-
|
|
1072
|
+
file_path=downloaded_path,
|
|
1058
1073
|
prefix=target_prefix,
|
|
1059
1074
|
description=self._description,
|
|
1060
1075
|
tags=self._tags,
|
|
@@ -1068,119 +1083,14 @@ class FileBuilder:
|
|
|
1068
1083
|
except Exception:
|
|
1069
1084
|
pass
|
|
1070
1085
|
|
|
1071
|
-
def exists(self) -> bool:
|
|
1072
|
-
"""
|
|
1073
|
-
Check if a file exists at the specified path.
|
|
1074
|
-
|
|
1075
|
-
Returns:
|
|
1076
|
-
True if file exists, False otherwise
|
|
1077
|
-
|
|
1078
|
-
Raises:
|
|
1079
|
-
RuntimeError: If experiment is not open
|
|
1080
|
-
ValueError: If no file path specified
|
|
1081
|
-
|
|
1082
|
-
Examples:
|
|
1083
|
-
# Check if file exists
|
|
1084
|
-
if dxp.files("models/checkpoint.pt").exists():
|
|
1085
|
-
print("File exists!")
|
|
1086
|
-
|
|
1087
|
-
# Check before downloading
|
|
1088
|
-
if not dxp.files("config.json").exists():
|
|
1089
|
-
raise FileNotFoundError("Config file missing")
|
|
1090
|
-
"""
|
|
1091
|
-
if not self._experiment._is_open:
|
|
1092
|
-
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1093
|
-
|
|
1094
|
-
if not self._path:
|
|
1095
|
-
raise ValueError("No file path specified. Use: experiment.files('path/to/file').exists()")
|
|
1096
|
-
|
|
1097
|
-
# Try to find the file
|
|
1098
|
-
try:
|
|
1099
|
-
files = self._experiment._list_files(prefix=None, tags=None)
|
|
1100
|
-
search_path = self._path.lstrip('/')
|
|
1101
|
-
|
|
1102
|
-
for f in files:
|
|
1103
|
-
filename = f.get('filename', '')
|
|
1104
|
-
prefix = f.get('path', '/').lstrip('/')
|
|
1105
|
-
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
1106
|
-
full_path = full_path.lstrip('/')
|
|
1107
|
-
|
|
1108
|
-
# Check if this file matches (by full path or just filename)
|
|
1109
|
-
if full_path == search_path or filename == search_path:
|
|
1110
|
-
# Make sure it's not deleted
|
|
1111
|
-
if f.get('deletedAt') is None:
|
|
1112
|
-
return True
|
|
1113
|
-
|
|
1114
|
-
return False
|
|
1115
|
-
except Exception:
|
|
1116
|
-
return False
|
|
1117
|
-
|
|
1118
|
-
def read_text(self, encoding: str = 'utf-8') -> str:
|
|
1119
|
-
"""
|
|
1120
|
-
Download file and read its content as text.
|
|
1121
|
-
|
|
1122
|
-
Args:
|
|
1123
|
-
encoding: Text encoding to use (default: 'utf-8')
|
|
1124
|
-
|
|
1125
|
-
Returns:
|
|
1126
|
-
File content as string
|
|
1127
|
-
|
|
1128
|
-
Raises:
|
|
1129
|
-
RuntimeError: If experiment is not open
|
|
1130
|
-
ValueError: If file not found or no path specified
|
|
1131
|
-
UnicodeDecodeError: If file cannot be decoded with specified encoding
|
|
1132
|
-
|
|
1133
|
-
Examples:
|
|
1134
|
-
# Read configuration file
|
|
1135
|
-
config_yaml = dxp.files("configs/view.yaml").read_text()
|
|
1136
|
-
|
|
1137
|
-
# Read log file
|
|
1138
|
-
logs = dxp.files("logs/training.log").read_text()
|
|
1139
|
-
|
|
1140
|
-
# Read with different encoding
|
|
1141
|
-
content = dxp.files("data.txt").read_text(encoding='latin-1')
|
|
1142
|
-
"""
|
|
1143
|
-
import tempfile
|
|
1144
|
-
import os
|
|
1145
|
-
|
|
1146
|
-
if not self._experiment._is_open:
|
|
1147
|
-
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1148
|
-
|
|
1149
|
-
if not self._path:
|
|
1150
|
-
raise ValueError("No file path specified. Use: experiment.files('path/to/file').read_text()")
|
|
1151
|
-
|
|
1152
|
-
# Create temporary file for download
|
|
1153
|
-
temp_dir = tempfile.mkdtemp()
|
|
1154
|
-
temp_filename = Path(self._path).name
|
|
1155
|
-
temp_path = os.path.join(temp_dir, temp_filename)
|
|
1156
|
-
|
|
1157
|
-
try:
|
|
1158
|
-
# Download the file
|
|
1159
|
-
downloaded_path = self.download(to=temp_path)
|
|
1160
|
-
|
|
1161
|
-
# Read content as text
|
|
1162
|
-
with open(downloaded_path, 'r', encoding=encoding) as f:
|
|
1163
|
-
content = f.read()
|
|
1164
|
-
|
|
1165
|
-
return content
|
|
1166
|
-
finally:
|
|
1167
|
-
# Clean up temporary file
|
|
1168
|
-
try:
|
|
1169
|
-
if os.path.exists(temp_path):
|
|
1170
|
-
os.unlink(temp_path)
|
|
1171
|
-
os.rmdir(temp_dir)
|
|
1172
|
-
except Exception:
|
|
1173
|
-
pass
|
|
1174
|
-
|
|
1175
1086
|
|
|
1176
1087
|
class FilesAccessor:
|
|
1177
1088
|
"""
|
|
1178
1089
|
Accessor that enables both callable and attribute-style access to file operations.
|
|
1179
1090
|
|
|
1180
1091
|
This allows:
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
experiment.files.upload(...) # Direct method call
|
|
1092
|
+
experiment.files("path") # Returns FileBuilder
|
|
1093
|
+
experiment.files.save(...) # Direct method call
|
|
1184
1094
|
experiment.files.download(...) # Direct method call
|
|
1185
1095
|
"""
|
|
1186
1096
|
|
|
@@ -1188,55 +1098,27 @@ class FilesAccessor:
|
|
|
1188
1098
|
self._experiment = experiment
|
|
1189
1099
|
self._builder = FileBuilder(experiment)
|
|
1190
1100
|
|
|
1191
|
-
def __call__(self,
|
|
1192
|
-
"""
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
Supports flexible argument styles:
|
|
1196
|
-
- Positional: files("models")
|
|
1197
|
-
- Keyword: files(dir="models")
|
|
1198
|
-
- No args (root): files()
|
|
1199
|
-
|
|
1200
|
-
Args:
|
|
1201
|
-
dir: Directory/path for file operations
|
|
1202
|
-
**kwargs: Additional FileBuilder options
|
|
1203
|
-
|
|
1204
|
-
Returns:
|
|
1205
|
-
FileBuilder instance
|
|
1206
|
-
"""
|
|
1207
|
-
return FileBuilder(self._experiment, path=dir, **kwargs)
|
|
1101
|
+
def __call__(self, path: Optional[str] = None, **kwargs) -> FileBuilder:
|
|
1102
|
+
"""Create a FileBuilder with the given path."""
|
|
1103
|
+
return FileBuilder(self._experiment, path=path, **kwargs)
|
|
1208
1104
|
|
|
1209
1105
|
# Direct methods that don't require a path first
|
|
1210
1106
|
|
|
1211
|
-
def
|
|
1107
|
+
def save(
|
|
1212
1108
|
self,
|
|
1213
|
-
|
|
1109
|
+
obj: Optional[Any] = None,
|
|
1214
1110
|
*,
|
|
1215
1111
|
to: Optional[str] = None,
|
|
1216
1112
|
**kwargs
|
|
1217
1113
|
) -> Dict[str, Any]:
|
|
1218
1114
|
"""
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
Args:
|
|
1222
|
-
fpath: Path to existing file to upload (required)
|
|
1223
|
-
to: Optional destination path, e.g., "models/model.pt" or "renamed.pt"
|
|
1224
|
-
**kwargs: Additional FileBuilder options
|
|
1225
|
-
|
|
1226
|
-
Returns:
|
|
1227
|
-
File metadata dict with id, path, filename, checksum, etc.
|
|
1115
|
+
Save a file directly.
|
|
1228
1116
|
|
|
1229
1117
|
Examples:
|
|
1230
|
-
|
|
1231
|
-
experiment.files.
|
|
1232
|
-
|
|
1233
|
-
# Upload with destination path
|
|
1234
|
-
experiment.files.upload("./model.pt", to="models/model.pt")
|
|
1235
|
-
|
|
1236
|
-
# Upload with metadata
|
|
1237
|
-
experiment.files.upload("./model.pt", to="best.pt", description="Best model")
|
|
1118
|
+
experiment.files.save("./model.pt")
|
|
1119
|
+
experiment.files.save(model, to="checkpoints/model.pt")
|
|
1238
1120
|
"""
|
|
1239
|
-
# Parse 'to' to extract prefix and filename
|
|
1121
|
+
# Parse 'to' to extract prefix and filename
|
|
1240
1122
|
if to:
|
|
1241
1123
|
to_path = to.lstrip('/')
|
|
1242
1124
|
if '/' in to_path:
|
|
@@ -1245,10 +1127,13 @@ class FilesAccessor:
|
|
|
1245
1127
|
else:
|
|
1246
1128
|
prefix = '/'
|
|
1247
1129
|
filename = to_path
|
|
1248
|
-
return FileBuilder(self._experiment, path=prefix, **kwargs).
|
|
1130
|
+
return FileBuilder(self._experiment, path=prefix, **kwargs).save(obj, to=filename)
|
|
1131
|
+
|
|
1132
|
+
if isinstance(obj, str) and Path(obj).exists():
|
|
1133
|
+
# obj is a file path, extract prefix from it
|
|
1134
|
+
return FileBuilder(self._experiment, **kwargs).save(obj)
|
|
1249
1135
|
|
|
1250
|
-
|
|
1251
|
-
return FileBuilder(self._experiment, **kwargs).upload(fpath)
|
|
1136
|
+
raise ValueError("'to' parameter is required when not saving an existing file path")
|
|
1252
1137
|
|
|
1253
1138
|
def download(
|
|
1254
1139
|
self,
|