ml-dash 0.6.2rc1__py3-none-any.whl → 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +36 -64
- ml_dash/auth/token_storage.py +267 -226
- ml_dash/auto_start.py +28 -15
- ml_dash/cli.py +16 -2
- ml_dash/cli_commands/api.py +165 -0
- ml_dash/cli_commands/download.py +757 -667
- ml_dash/cli_commands/list.py +146 -13
- ml_dash/cli_commands/login.py +190 -183
- ml_dash/cli_commands/profile.py +92 -0
- ml_dash/cli_commands/upload.py +1291 -1141
- ml_dash/client.py +79 -6
- ml_dash/config.py +119 -119
- ml_dash/experiment.py +1234 -1034
- ml_dash/files.py +339 -224
- ml_dash/log.py +7 -7
- ml_dash/metric.py +359 -100
- ml_dash/params.py +6 -6
- ml_dash/remote_auto_start.py +20 -17
- ml_dash/run.py +211 -65
- ml_dash/snowflake.py +173 -0
- ml_dash/storage.py +1051 -1081
- {ml_dash-0.6.2rc1.dist-info → ml_dash-0.6.3.dist-info}/METADATA +12 -14
- ml_dash-0.6.3.dist-info/RECORD +33 -0
- {ml_dash-0.6.2rc1.dist-info → ml_dash-0.6.3.dist-info}/WHEEL +1 -1
- ml_dash-0.6.2rc1.dist-info/RECORD +0 -30
- {ml_dash-0.6.2rc1.dist-info → ml_dash-0.6.3.dist-info}/entry_points.txt +0 -0
ml_dash/files.py
CHANGED
|
@@ -19,28 +19,45 @@ class FileBuilder:
|
|
|
19
19
|
Fluent interface for file operations.
|
|
20
20
|
|
|
21
21
|
Usage:
|
|
22
|
-
# Upload file
|
|
23
|
-
|
|
22
|
+
# Upload existing file from disk
|
|
23
|
+
dxp.files("models").upload("./local_model.pt")
|
|
24
|
+
dxp.files("checkpoints").upload("./model.pt", to="latest.pt")
|
|
25
|
+
|
|
26
|
+
# Save objects as files (using keyword dir argument)
|
|
27
|
+
dxp.files(dir="models").save_torch(model, to="checkpoint.pt")
|
|
28
|
+
dxp.files(dir="configs").save_json({"lr": 0.001}, to="config.json")
|
|
29
|
+
dxp.files(dir="data").save_blob(b"binary data", to="data.bin")
|
|
24
30
|
|
|
25
31
|
# List files
|
|
26
|
-
files = experiment.files("
|
|
27
|
-
files = experiment.files("
|
|
32
|
+
files = experiment.files(dir="some/location").list()
|
|
33
|
+
files = experiment.files(dir="models").list()
|
|
28
34
|
|
|
29
35
|
# Download file
|
|
30
|
-
|
|
31
|
-
|
|
36
|
+
dxp.files("some.text").download()
|
|
37
|
+
dxp.files("some.text").download(to="./model.pt")
|
|
32
38
|
|
|
33
|
-
# Download
|
|
39
|
+
# Download files via glob pattern
|
|
34
40
|
file_paths = experiment.files("images").list("*.png")
|
|
35
|
-
|
|
41
|
+
dxp.files("images").download("*.png")
|
|
36
42
|
|
|
37
43
|
# Delete files
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
dxp.files
|
|
42
|
-
|
|
43
|
-
|
|
44
|
+
dxp.files("some.text").delete()
|
|
45
|
+
|
|
46
|
+
# Check if file exists
|
|
47
|
+
if dxp.files("config.json").exists():
|
|
48
|
+
config = dxp.files("config.json").read_text()
|
|
49
|
+
|
|
50
|
+
# Read file content as text
|
|
51
|
+
config_yaml = dxp.files("configs/view.yaml").read_text()
|
|
52
|
+
|
|
53
|
+
Specific Save Methods:
|
|
54
|
+
experiment.files.save_text("content", to="view.yaml")
|
|
55
|
+
experiment.files.save_json(dict(hey="yo"), to="config.json")
|
|
56
|
+
experiment.files.save_blob(b"xxx", to="data.bin")
|
|
57
|
+
experiment.files.save_torch(model, to="model.pt")
|
|
58
|
+
experiment.files.save_pkl(data, to="data.pkl")
|
|
59
|
+
experiment.files.save_fig(fig, to="plot.png")
|
|
60
|
+
experiment.files.save_video(frames, to="video.mp4")
|
|
44
61
|
"""
|
|
45
62
|
|
|
46
63
|
def __init__(self, experiment: 'Experiment', path: Optional[str] = None, **kwargs):
|
|
@@ -80,9 +97,9 @@ class FileBuilder:
|
|
|
80
97
|
path = path.lstrip('/')
|
|
81
98
|
self._normalized_path = '/' + path if not path.startswith('/') else path
|
|
82
99
|
|
|
83
|
-
def
|
|
100
|
+
def upload(
|
|
84
101
|
self,
|
|
85
|
-
|
|
102
|
+
fpath: str,
|
|
86
103
|
*,
|
|
87
104
|
to: Optional[str] = None,
|
|
88
105
|
description: Optional[str] = None,
|
|
@@ -90,21 +107,14 @@ class FileBuilder:
|
|
|
90
107
|
metadata: Optional[Dict[str, Any]] = None
|
|
91
108
|
) -> Dict[str, Any]:
|
|
92
109
|
"""
|
|
93
|
-
Upload
|
|
110
|
+
Upload an existing file from disk.
|
|
94
111
|
|
|
95
112
|
Args:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
- PyTorch model/state_dict: Saved with torch.save()
|
|
102
|
-
- matplotlib figure: Saved as image
|
|
103
|
-
- Any picklable object: Saved with pickle
|
|
104
|
-
to: Target filename (required when obj is not a file path)
|
|
105
|
-
description: Optional description (overrides constructor)
|
|
106
|
-
tags: Optional list of tags (overrides constructor)
|
|
107
|
-
metadata: Optional metadata dict (overrides constructor)
|
|
113
|
+
fpath: Path to existing file to upload (required)
|
|
114
|
+
to: Optional new filename (uses original filename if not provided)
|
|
115
|
+
description: Optional description
|
|
116
|
+
tags: Optional list of tags
|
|
117
|
+
metadata: Optional metadata dict
|
|
108
118
|
|
|
109
119
|
Returns:
|
|
110
120
|
File metadata dict with id, path, filename, checksum, etc.
|
|
@@ -115,19 +125,19 @@ class FileBuilder:
|
|
|
115
125
|
ValueError: If file size exceeds 100GB limit
|
|
116
126
|
|
|
117
127
|
Examples:
|
|
118
|
-
#
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
128
|
+
# Upload with original filename
|
|
129
|
+
dxp.files("models").upload("./local_model.pt")
|
|
130
|
+
|
|
131
|
+
# Upload with new filename
|
|
132
|
+
dxp.files("models").upload("./local_model.pt", to="remote_model.pt")
|
|
133
|
+
|
|
134
|
+
# Upload with metadata
|
|
135
|
+
dxp.files("checkpoints").upload(
|
|
136
|
+
"./checkpoint.pt",
|
|
137
|
+
to="epoch_10.pt",
|
|
138
|
+
description="Best model at epoch 10",
|
|
139
|
+
tags=["best", "training"]
|
|
140
|
+
)
|
|
131
141
|
"""
|
|
132
142
|
if not self._experiment._is_open:
|
|
133
143
|
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
@@ -135,6 +145,9 @@ class FileBuilder:
|
|
|
135
145
|
if self._experiment._write_protected:
|
|
136
146
|
raise RuntimeError("Experiment is write-protected and cannot be modified.")
|
|
137
147
|
|
|
148
|
+
if not fpath:
|
|
149
|
+
raise ValueError("fpath is required")
|
|
150
|
+
|
|
138
151
|
# Use provided values or fall back to constructor values
|
|
139
152
|
desc = description if description is not None else self._description
|
|
140
153
|
file_tags = tags if tags is not None else self._tags
|
|
@@ -145,136 +158,109 @@ class FileBuilder:
|
|
|
145
158
|
if self._path:
|
|
146
159
|
prefix = '/' + self._path.lstrip('/')
|
|
147
160
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
description=desc,
|
|
157
|
-
tags=file_tags,
|
|
158
|
-
metadata=file_metadata
|
|
159
|
-
)
|
|
161
|
+
return self._save_file(
|
|
162
|
+
fpath=fpath,
|
|
163
|
+
prefix=prefix,
|
|
164
|
+
description=desc,
|
|
165
|
+
tags=file_tags,
|
|
166
|
+
metadata=file_metadata,
|
|
167
|
+
to=to
|
|
168
|
+
)
|
|
160
169
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
+
def save(
|
|
171
|
+
self,
|
|
172
|
+
content: Any = None,
|
|
173
|
+
*,
|
|
174
|
+
to: Optional[str] = None,
|
|
175
|
+
description: Optional[str] = None,
|
|
176
|
+
tags: Optional[List[str]] = None,
|
|
177
|
+
metadata: Optional[Dict[str, Any]] = None
|
|
178
|
+
) -> Dict[str, Any]:
|
|
179
|
+
"""
|
|
180
|
+
Unified save method that handles different content types.
|
|
170
181
|
|
|
171
|
-
|
|
172
|
-
|
|
182
|
+
Args:
|
|
183
|
+
content: Content to save - can be:
|
|
184
|
+
- str (file path): Uploads existing file
|
|
185
|
+
- bytes: Saves as binary blob (requires 'to' parameter)
|
|
186
|
+
- dict/list: Saves as JSON (requires 'to' parameter)
|
|
187
|
+
- None: Uses file_path from constructor (backwards compatibility)
|
|
188
|
+
to: Target filename (required for bytes/dict/list, optional for file paths)
|
|
189
|
+
description: Optional description
|
|
190
|
+
tags: Optional list of tags
|
|
191
|
+
metadata: Optional metadata dict
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
195
|
+
"""
|
|
196
|
+
# Override builder metadata if provided
|
|
197
|
+
if description is not None:
|
|
198
|
+
self._description = description
|
|
199
|
+
if tags is not None:
|
|
200
|
+
self._tags = tags
|
|
201
|
+
if metadata is not None:
|
|
202
|
+
self._metadata = metadata
|
|
203
|
+
|
|
204
|
+
# Backwards compatibility: use file_path from constructor if no content provided
|
|
205
|
+
if content is None:
|
|
206
|
+
if self._file_path:
|
|
207
|
+
content = self._file_path
|
|
208
|
+
else:
|
|
209
|
+
raise ValueError("No content provided and no file_path set in constructor")
|
|
210
|
+
|
|
211
|
+
# Check if content is a file path
|
|
212
|
+
if isinstance(content, str) and Path(content).exists():
|
|
213
|
+
return self.upload(content, to=to)
|
|
214
|
+
|
|
215
|
+
# Check if content is bytes
|
|
216
|
+
if isinstance(content, bytes):
|
|
173
217
|
if not to:
|
|
174
218
|
raise ValueError("'to' parameter is required when saving bytes")
|
|
175
|
-
return self.
|
|
176
|
-
data=obj,
|
|
177
|
-
filename=to,
|
|
178
|
-
prefix=prefix,
|
|
179
|
-
description=desc,
|
|
180
|
-
tags=file_tags,
|
|
181
|
-
metadata=file_metadata
|
|
182
|
-
)
|
|
219
|
+
return self.save_blob(content, to=to)
|
|
183
220
|
|
|
184
|
-
if
|
|
185
|
-
|
|
221
|
+
# Check if content is dict or list (save as JSON)
|
|
222
|
+
if isinstance(content, (dict, list)):
|
|
186
223
|
if not to:
|
|
187
224
|
raise ValueError("'to' parameter is required when saving dict/list")
|
|
188
|
-
return self.
|
|
189
|
-
content=obj,
|
|
190
|
-
filename=to,
|
|
191
|
-
prefix=prefix,
|
|
192
|
-
description=desc,
|
|
193
|
-
tags=file_tags,
|
|
194
|
-
metadata=file_metadata
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
# Check for PyTorch model
|
|
198
|
-
try:
|
|
199
|
-
import torch
|
|
200
|
-
if isinstance(obj, (torch.nn.Module, dict)) or hasattr(obj, 'state_dict'):
|
|
201
|
-
if not to:
|
|
202
|
-
raise ValueError("'to' parameter is required when saving PyTorch model")
|
|
203
|
-
return self._save_torch(
|
|
204
|
-
model=obj,
|
|
205
|
-
filename=to,
|
|
206
|
-
prefix=prefix,
|
|
207
|
-
description=desc,
|
|
208
|
-
tags=file_tags,
|
|
209
|
-
metadata=file_metadata
|
|
210
|
-
)
|
|
211
|
-
except ImportError:
|
|
212
|
-
pass
|
|
225
|
+
return self.save_json(content, to=to)
|
|
213
226
|
|
|
214
|
-
|
|
215
|
-
try:
|
|
216
|
-
import matplotlib.pyplot as plt
|
|
217
|
-
from matplotlib.figure import Figure
|
|
218
|
-
if isinstance(obj, Figure):
|
|
219
|
-
if not to:
|
|
220
|
-
raise ValueError("'to' parameter is required when saving matplotlib figure")
|
|
221
|
-
return self._save_fig(
|
|
222
|
-
fig=obj,
|
|
223
|
-
filename=to,
|
|
224
|
-
prefix=prefix,
|
|
225
|
-
description=desc,
|
|
226
|
-
tags=file_tags,
|
|
227
|
-
metadata=file_metadata
|
|
228
|
-
)
|
|
229
|
-
except ImportError:
|
|
230
|
-
pass
|
|
231
|
-
|
|
232
|
-
# Fall back to pickle
|
|
233
|
-
if not to:
|
|
234
|
-
raise ValueError("'to' parameter is required when saving object")
|
|
235
|
-
return self._save_pickle(
|
|
236
|
-
content=obj,
|
|
237
|
-
filename=to,
|
|
238
|
-
prefix=prefix,
|
|
239
|
-
description=desc,
|
|
240
|
-
tags=file_tags,
|
|
241
|
-
metadata=file_metadata
|
|
242
|
-
)
|
|
227
|
+
raise ValueError(f"Unsupported content type: {type(content)}. Expected str (file path), bytes, dict, or list.")
|
|
243
228
|
|
|
244
229
|
def _save_file(
|
|
245
230
|
self,
|
|
246
|
-
|
|
231
|
+
fpath: str,
|
|
247
232
|
prefix: str,
|
|
248
233
|
description: Optional[str],
|
|
249
234
|
tags: Optional[List[str]],
|
|
250
|
-
metadata: Optional[Dict[str, Any]]
|
|
235
|
+
metadata: Optional[Dict[str, Any]],
|
|
236
|
+
to: Optional[str] = None
|
|
251
237
|
) -> Dict[str, Any]:
|
|
252
238
|
"""Internal method to save an existing file."""
|
|
253
|
-
|
|
254
|
-
if not
|
|
255
|
-
raise ValueError(f"File not found: {
|
|
239
|
+
fpath_obj = Path(fpath)
|
|
240
|
+
if not fpath_obj.exists():
|
|
241
|
+
raise ValueError(f"File not found: {fpath}")
|
|
256
242
|
|
|
257
|
-
if not
|
|
258
|
-
raise ValueError(f"Path is not a file: {
|
|
243
|
+
if not fpath_obj.is_file():
|
|
244
|
+
raise ValueError(f"Path is not a file: {fpath}")
|
|
259
245
|
|
|
260
246
|
# Check file size (max 100GB)
|
|
261
|
-
file_size =
|
|
247
|
+
file_size = fpath_obj.stat().st_size
|
|
262
248
|
MAX_FILE_SIZE = 100 * 1024 * 1024 * 1024 # 100GB in bytes
|
|
263
249
|
if file_size > MAX_FILE_SIZE:
|
|
264
250
|
raise ValueError(f"File size ({file_size} bytes) exceeds 100GB limit")
|
|
265
251
|
|
|
266
252
|
# Compute checksum
|
|
267
|
-
checksum = compute_sha256(str(
|
|
253
|
+
checksum = compute_sha256(str(fpath_obj))
|
|
268
254
|
|
|
269
255
|
# Detect MIME type
|
|
270
|
-
content_type = get_mime_type(str(
|
|
256
|
+
content_type = get_mime_type(str(fpath_obj))
|
|
271
257
|
|
|
272
|
-
# Get filename
|
|
273
|
-
filename =
|
|
258
|
+
# Get filename (use provided 'to' or original)
|
|
259
|
+
filename = to if to else fpath_obj.name
|
|
274
260
|
|
|
275
261
|
# Upload through experiment
|
|
276
262
|
return self._experiment._upload_file(
|
|
277
|
-
file_path=str(
|
|
263
|
+
file_path=str(fpath_obj),
|
|
278
264
|
prefix=prefix,
|
|
279
265
|
filename=filename,
|
|
280
266
|
description=description,
|
|
@@ -300,11 +286,13 @@ class FileBuilder:
|
|
|
300
286
|
|
|
301
287
|
temp_dir = tempfile.mkdtemp()
|
|
302
288
|
temp_path = os.path.join(temp_dir, filename)
|
|
289
|
+
# Create parent directories if filename contains path
|
|
290
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
303
291
|
try:
|
|
304
292
|
with open(temp_path, 'wb') as f:
|
|
305
293
|
f.write(data)
|
|
306
294
|
return self._save_file(
|
|
307
|
-
|
|
295
|
+
fpath=temp_path,
|
|
308
296
|
prefix=prefix,
|
|
309
297
|
description=description,
|
|
310
298
|
tags=tags,
|
|
@@ -333,11 +321,13 @@ class FileBuilder:
|
|
|
333
321
|
|
|
334
322
|
temp_dir = tempfile.mkdtemp()
|
|
335
323
|
temp_path = os.path.join(temp_dir, filename)
|
|
324
|
+
# Create parent directories if filename contains path
|
|
325
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
336
326
|
try:
|
|
337
327
|
with open(temp_path, 'w') as f:
|
|
338
328
|
json.dump(content, f, indent=2)
|
|
339
329
|
return self._save_file(
|
|
340
|
-
|
|
330
|
+
fpath=temp_path,
|
|
341
331
|
prefix=prefix,
|
|
342
332
|
description=description,
|
|
343
333
|
tags=tags,
|
|
@@ -366,10 +356,12 @@ class FileBuilder:
|
|
|
366
356
|
|
|
367
357
|
temp_dir = tempfile.mkdtemp()
|
|
368
358
|
temp_path = os.path.join(temp_dir, filename)
|
|
359
|
+
# Create parent directories if filename contains path
|
|
360
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
369
361
|
try:
|
|
370
362
|
torch.save(model, temp_path)
|
|
371
363
|
return self._save_file(
|
|
372
|
-
|
|
364
|
+
fpath=temp_path,
|
|
373
365
|
prefix=prefix,
|
|
374
366
|
description=description,
|
|
375
367
|
tags=tags,
|
|
@@ -399,11 +391,13 @@ class FileBuilder:
|
|
|
399
391
|
|
|
400
392
|
temp_dir = tempfile.mkdtemp()
|
|
401
393
|
temp_path = os.path.join(temp_dir, filename)
|
|
394
|
+
# Create parent directories if filename contains path
|
|
395
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
402
396
|
try:
|
|
403
397
|
fig.savefig(temp_path, **kwargs)
|
|
404
398
|
plt.close(fig)
|
|
405
399
|
return self._save_file(
|
|
406
|
-
|
|
400
|
+
fpath=temp_path,
|
|
407
401
|
prefix=prefix,
|
|
408
402
|
description=description,
|
|
409
403
|
tags=tags,
|
|
@@ -432,11 +426,13 @@ class FileBuilder:
|
|
|
432
426
|
|
|
433
427
|
temp_dir = tempfile.mkdtemp()
|
|
434
428
|
temp_path = os.path.join(temp_dir, filename)
|
|
429
|
+
# Create parent directories if filename contains path
|
|
430
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
435
431
|
try:
|
|
436
432
|
with open(temp_path, 'wb') as f:
|
|
437
433
|
pickle.dump(content, f)
|
|
438
434
|
return self._save_file(
|
|
439
|
-
|
|
435
|
+
fpath=temp_path,
|
|
440
436
|
prefix=prefix,
|
|
441
437
|
description=description,
|
|
442
438
|
tags=tags,
|
|
@@ -721,34 +717,36 @@ class FileBuilder:
|
|
|
721
717
|
def save_json(
|
|
722
718
|
self,
|
|
723
719
|
content: Any,
|
|
724
|
-
file_name: Optional[str] = None,
|
|
725
720
|
*,
|
|
726
|
-
to:
|
|
721
|
+
to: str
|
|
727
722
|
) -> Dict[str, Any]:
|
|
728
723
|
"""
|
|
729
724
|
Save JSON content to a file.
|
|
730
725
|
|
|
731
726
|
Args:
|
|
732
727
|
content: Content to save as JSON (dict, list, or any JSON-serializable object)
|
|
733
|
-
|
|
734
|
-
to: Target filename (preferred)
|
|
728
|
+
to: Target filename
|
|
735
729
|
|
|
736
730
|
Returns:
|
|
737
731
|
File metadata dict with id, path, filename, checksum, etc.
|
|
738
732
|
|
|
739
733
|
Examples:
|
|
740
734
|
config = {"model": "resnet50", "lr": 0.001}
|
|
741
|
-
result =
|
|
735
|
+
result = dxp.files("configs").save_json(config, to="config.json")
|
|
742
736
|
"""
|
|
743
|
-
filename = to or file_name
|
|
744
|
-
if not filename:
|
|
745
|
-
raise ValueError("'to' parameter is required")
|
|
746
|
-
|
|
747
737
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
748
738
|
|
|
739
|
+
# Extract path component from 'to' if present (e.g., "configs/settings.json")
|
|
740
|
+
import os
|
|
741
|
+
to_dirname = os.path.dirname(to)
|
|
742
|
+
to_filename = os.path.basename(to)
|
|
743
|
+
if to_dirname:
|
|
744
|
+
# Merge the path component into prefix
|
|
745
|
+
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
746
|
+
|
|
749
747
|
return self._save_json(
|
|
750
748
|
content=content,
|
|
751
|
-
filename=
|
|
749
|
+
filename=to_filename,
|
|
752
750
|
prefix=prefix,
|
|
753
751
|
description=self._description,
|
|
754
752
|
tags=self._tags,
|
|
@@ -770,14 +768,18 @@ class FileBuilder:
|
|
|
770
768
|
result = experiment.files().save_text("Hello, world!", to="greeting.txt")
|
|
771
769
|
result = experiment.files("configs").save_text(yaml_content, to="view.yaml")
|
|
772
770
|
"""
|
|
773
|
-
if not to:
|
|
774
|
-
raise ValueError("'to' parameter is required")
|
|
775
|
-
|
|
776
771
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
777
772
|
|
|
773
|
+
# Extract path component from 'to' if present
|
|
774
|
+
import os
|
|
775
|
+
to_dirname = os.path.dirname(to)
|
|
776
|
+
to_filename = os.path.basename(to)
|
|
777
|
+
if to_dirname:
|
|
778
|
+
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
779
|
+
|
|
778
780
|
return self._save_bytes(
|
|
779
781
|
data=content.encode('utf-8'),
|
|
780
|
-
filename=
|
|
782
|
+
filename=to_filename,
|
|
781
783
|
prefix=prefix,
|
|
782
784
|
description=self._description,
|
|
783
785
|
tags=self._tags,
|
|
@@ -798,14 +800,18 @@ class FileBuilder:
|
|
|
798
800
|
Examples:
|
|
799
801
|
result = experiment.files("data").save_blob(binary_data, to="model.bin")
|
|
800
802
|
"""
|
|
801
|
-
if not to:
|
|
802
|
-
raise ValueError("'to' parameter is required")
|
|
803
|
-
|
|
804
803
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
805
804
|
|
|
805
|
+
# Extract path component from 'to' if present
|
|
806
|
+
import os
|
|
807
|
+
to_dirname = os.path.dirname(to)
|
|
808
|
+
to_filename = os.path.basename(to)
|
|
809
|
+
if to_dirname:
|
|
810
|
+
prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
|
|
811
|
+
|
|
806
812
|
return self._save_bytes(
|
|
807
813
|
data=data,
|
|
808
|
-
filename=
|
|
814
|
+
filename=to_filename,
|
|
809
815
|
prefix=prefix,
|
|
810
816
|
description=self._description,
|
|
811
817
|
tags=self._tags,
|
|
@@ -815,34 +821,29 @@ class FileBuilder:
|
|
|
815
821
|
def save_torch(
|
|
816
822
|
self,
|
|
817
823
|
model: Any,
|
|
818
|
-
file_name: Optional[str] = None,
|
|
819
824
|
*,
|
|
820
|
-
to:
|
|
825
|
+
to: str
|
|
821
826
|
) -> Dict[str, Any]:
|
|
822
827
|
"""
|
|
823
828
|
Save PyTorch model to a file.
|
|
824
829
|
|
|
825
830
|
Args:
|
|
826
831
|
model: PyTorch model or state dict to save
|
|
827
|
-
|
|
828
|
-
to: Target filename (preferred)
|
|
832
|
+
to: Target filename
|
|
829
833
|
|
|
830
834
|
Returns:
|
|
831
835
|
File metadata dict with id, path, filename, checksum, etc.
|
|
832
836
|
|
|
833
837
|
Examples:
|
|
834
|
-
result =
|
|
835
|
-
result =
|
|
838
|
+
result = dxp.files("models").save_torch(model, to="model.pt")
|
|
839
|
+
result = dxp.files("models").save_torch(model.state_dict(), to="weights.pth")
|
|
836
840
|
"""
|
|
837
|
-
filename = to or file_name
|
|
838
|
-
if not filename:
|
|
839
|
-
raise ValueError("'to' parameter is required")
|
|
840
841
|
|
|
841
842
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
842
843
|
|
|
843
844
|
return self._save_torch(
|
|
844
845
|
model=model,
|
|
845
|
-
filename=
|
|
846
|
+
filename=to,
|
|
846
847
|
prefix=prefix,
|
|
847
848
|
description=self._description,
|
|
848
849
|
tags=self._tags,
|
|
@@ -852,34 +853,28 @@ class FileBuilder:
|
|
|
852
853
|
def save_pkl(
|
|
853
854
|
self,
|
|
854
855
|
content: Any,
|
|
855
|
-
file_name: Optional[str] = None,
|
|
856
856
|
*,
|
|
857
|
-
to:
|
|
857
|
+
to: str
|
|
858
858
|
) -> Dict[str, Any]:
|
|
859
859
|
"""
|
|
860
860
|
Save Python object to a pickle file.
|
|
861
861
|
|
|
862
862
|
Args:
|
|
863
863
|
content: Python object to pickle (must be pickle-serializable)
|
|
864
|
-
|
|
865
|
-
to: Target filename (preferred)
|
|
864
|
+
to: Target filename
|
|
866
865
|
|
|
867
866
|
Returns:
|
|
868
867
|
File metadata dict with id, path, filename, checksum, etc.
|
|
869
868
|
|
|
870
869
|
Examples:
|
|
871
870
|
data = {"model": "resnet50", "weights": np.array([1, 2, 3])}
|
|
872
|
-
result =
|
|
871
|
+
result = dxp.files("data").save_pkl(data, to="data.pkl")
|
|
873
872
|
"""
|
|
874
|
-
filename = to or file_name
|
|
875
|
-
if not filename:
|
|
876
|
-
raise ValueError("'to' parameter is required")
|
|
877
|
-
|
|
878
873
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
879
874
|
|
|
880
875
|
return self._save_pickle(
|
|
881
876
|
content=content,
|
|
882
|
-
filename=
|
|
877
|
+
filename=to,
|
|
883
878
|
prefix=prefix,
|
|
884
879
|
description=self._description,
|
|
885
880
|
tags=self._tags,
|
|
@@ -889,9 +884,8 @@ class FileBuilder:
|
|
|
889
884
|
def save_fig(
|
|
890
885
|
self,
|
|
891
886
|
fig: Optional[Any] = None,
|
|
892
|
-
file_name: Optional[str] = None,
|
|
893
887
|
*,
|
|
894
|
-
to:
|
|
888
|
+
to: str,
|
|
895
889
|
**kwargs
|
|
896
890
|
) -> Dict[str, Any]:
|
|
897
891
|
"""
|
|
@@ -899,8 +893,7 @@ class FileBuilder:
|
|
|
899
893
|
|
|
900
894
|
Args:
|
|
901
895
|
fig: Matplotlib figure object. If None, uses plt.gcf() (current figure)
|
|
902
|
-
|
|
903
|
-
to: Target filename (preferred)
|
|
896
|
+
to: Target filename
|
|
904
897
|
**kwargs: Additional arguments passed to fig.savefig()
|
|
905
898
|
|
|
906
899
|
Returns:
|
|
@@ -908,17 +901,13 @@ class FileBuilder:
|
|
|
908
901
|
|
|
909
902
|
Examples:
|
|
910
903
|
plt.plot([1, 2, 3], [1, 4, 9])
|
|
911
|
-
result =
|
|
904
|
+
result = dxp.files("plots").save_fig(to="plot.png")
|
|
912
905
|
"""
|
|
913
906
|
try:
|
|
914
907
|
import matplotlib.pyplot as plt
|
|
915
908
|
except ImportError:
|
|
916
909
|
raise ImportError("Matplotlib is not installed. Install it with: pip install matplotlib")
|
|
917
910
|
|
|
918
|
-
filename = to or file_name
|
|
919
|
-
if not filename:
|
|
920
|
-
raise ValueError("'to' parameter is required")
|
|
921
|
-
|
|
922
911
|
if fig is None:
|
|
923
912
|
fig = plt.gcf()
|
|
924
913
|
|
|
@@ -926,7 +915,7 @@ class FileBuilder:
|
|
|
926
915
|
|
|
927
916
|
return self._save_fig(
|
|
928
917
|
fig=fig,
|
|
929
|
-
filename=
|
|
918
|
+
filename=to,
|
|
930
919
|
prefix=prefix,
|
|
931
920
|
description=self._description,
|
|
932
921
|
tags=self._tags,
|
|
@@ -937,9 +926,8 @@ class FileBuilder:
|
|
|
937
926
|
def save_video(
|
|
938
927
|
self,
|
|
939
928
|
frame_stack: Union[List, Any],
|
|
940
|
-
file_name: Optional[str] = None,
|
|
941
929
|
*,
|
|
942
|
-
to:
|
|
930
|
+
to: str,
|
|
943
931
|
fps: int = 20,
|
|
944
932
|
**imageio_kwargs
|
|
945
933
|
) -> Dict[str, Any]:
|
|
@@ -948,8 +936,7 @@ class FileBuilder:
|
|
|
948
936
|
|
|
949
937
|
Args:
|
|
950
938
|
frame_stack: List of numpy arrays or stacked array
|
|
951
|
-
|
|
952
|
-
to: Target filename (preferred)
|
|
939
|
+
to: Target filename
|
|
953
940
|
fps: Frames per second (default: 20)
|
|
954
941
|
**imageio_kwargs: Additional arguments passed to imageio
|
|
955
942
|
|
|
@@ -958,7 +945,7 @@ class FileBuilder:
|
|
|
958
945
|
|
|
959
946
|
Examples:
|
|
960
947
|
frames = [np.random.rand(480, 640) for _ in range(30)]
|
|
961
|
-
result =
|
|
948
|
+
result = dxp.files("videos").save_video(frames, to="output.mp4")
|
|
962
949
|
"""
|
|
963
950
|
import tempfile
|
|
964
951
|
import os
|
|
@@ -973,10 +960,6 @@ class FileBuilder:
|
|
|
973
960
|
except ImportError:
|
|
974
961
|
raise ImportError("scikit-image is not installed. Install it with: pip install scikit-image")
|
|
975
962
|
|
|
976
|
-
filename = to or file_name
|
|
977
|
-
if not filename:
|
|
978
|
-
raise ValueError("'to' parameter is required")
|
|
979
|
-
|
|
980
963
|
# Validate frame_stack
|
|
981
964
|
try:
|
|
982
965
|
if len(frame_stack) == 0:
|
|
@@ -987,7 +970,9 @@ class FileBuilder:
|
|
|
987
970
|
prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
|
|
988
971
|
|
|
989
972
|
temp_dir = tempfile.mkdtemp()
|
|
990
|
-
temp_path = os.path.join(temp_dir,
|
|
973
|
+
temp_path = os.path.join(temp_dir, to)
|
|
974
|
+
# Create parent directories if filename contains path
|
|
975
|
+
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
|
991
976
|
|
|
992
977
|
try:
|
|
993
978
|
frames_ubyte = img_as_ubyte(frame_stack)
|
|
@@ -999,7 +984,7 @@ class FileBuilder:
|
|
|
999
984
|
iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
|
|
1000
985
|
|
|
1001
986
|
return self._save_file(
|
|
1002
|
-
|
|
987
|
+
fpath=temp_path,
|
|
1003
988
|
prefix=prefix,
|
|
1004
989
|
description=self._description,
|
|
1005
990
|
tags=self._tags,
|
|
@@ -1069,7 +1054,7 @@ class FileBuilder:
|
|
|
1069
1054
|
)
|
|
1070
1055
|
|
|
1071
1056
|
return self._save_file(
|
|
1072
|
-
|
|
1057
|
+
fpath=downloaded_path,
|
|
1073
1058
|
prefix=target_prefix,
|
|
1074
1059
|
description=self._description,
|
|
1075
1060
|
tags=self._tags,
|
|
@@ -1083,14 +1068,119 @@ class FileBuilder:
|
|
|
1083
1068
|
except Exception:
|
|
1084
1069
|
pass
|
|
1085
1070
|
|
|
1071
|
+
def exists(self) -> bool:
|
|
1072
|
+
"""
|
|
1073
|
+
Check if a file exists at the specified path.
|
|
1074
|
+
|
|
1075
|
+
Returns:
|
|
1076
|
+
True if file exists, False otherwise
|
|
1077
|
+
|
|
1078
|
+
Raises:
|
|
1079
|
+
RuntimeError: If experiment is not open
|
|
1080
|
+
ValueError: If no file path specified
|
|
1081
|
+
|
|
1082
|
+
Examples:
|
|
1083
|
+
# Check if file exists
|
|
1084
|
+
if dxp.files("models/checkpoint.pt").exists():
|
|
1085
|
+
print("File exists!")
|
|
1086
|
+
|
|
1087
|
+
# Check before downloading
|
|
1088
|
+
if not dxp.files("config.json").exists():
|
|
1089
|
+
raise FileNotFoundError("Config file missing")
|
|
1090
|
+
"""
|
|
1091
|
+
if not self._experiment._is_open:
|
|
1092
|
+
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1093
|
+
|
|
1094
|
+
if not self._path:
|
|
1095
|
+
raise ValueError("No file path specified. Use: experiment.files('path/to/file').exists()")
|
|
1096
|
+
|
|
1097
|
+
# Try to find the file
|
|
1098
|
+
try:
|
|
1099
|
+
files = self._experiment._list_files(prefix=None, tags=None)
|
|
1100
|
+
search_path = self._path.lstrip('/')
|
|
1101
|
+
|
|
1102
|
+
for f in files:
|
|
1103
|
+
filename = f.get('filename', '')
|
|
1104
|
+
prefix = f.get('path', '/').lstrip('/')
|
|
1105
|
+
full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
|
|
1106
|
+
full_path = full_path.lstrip('/')
|
|
1107
|
+
|
|
1108
|
+
# Check if this file matches (by full path or just filename)
|
|
1109
|
+
if full_path == search_path or filename == search_path:
|
|
1110
|
+
# Make sure it's not deleted
|
|
1111
|
+
if f.get('deletedAt') is None:
|
|
1112
|
+
return True
|
|
1113
|
+
|
|
1114
|
+
return False
|
|
1115
|
+
except Exception:
|
|
1116
|
+
return False
|
|
1117
|
+
|
|
1118
|
+
def read_text(self, encoding: str = 'utf-8') -> str:
|
|
1119
|
+
"""
|
|
1120
|
+
Download file and read its content as text.
|
|
1121
|
+
|
|
1122
|
+
Args:
|
|
1123
|
+
encoding: Text encoding to use (default: 'utf-8')
|
|
1124
|
+
|
|
1125
|
+
Returns:
|
|
1126
|
+
File content as string
|
|
1127
|
+
|
|
1128
|
+
Raises:
|
|
1129
|
+
RuntimeError: If experiment is not open
|
|
1130
|
+
ValueError: If file not found or no path specified
|
|
1131
|
+
UnicodeDecodeError: If file cannot be decoded with specified encoding
|
|
1132
|
+
|
|
1133
|
+
Examples:
|
|
1134
|
+
# Read configuration file
|
|
1135
|
+
config_yaml = dxp.files("configs/view.yaml").read_text()
|
|
1136
|
+
|
|
1137
|
+
# Read log file
|
|
1138
|
+
logs = dxp.files("logs/training.log").read_text()
|
|
1139
|
+
|
|
1140
|
+
# Read with different encoding
|
|
1141
|
+
content = dxp.files("data.txt").read_text(encoding='latin-1')
|
|
1142
|
+
"""
|
|
1143
|
+
import tempfile
|
|
1144
|
+
import os
|
|
1145
|
+
|
|
1146
|
+
if not self._experiment._is_open:
|
|
1147
|
+
raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
|
|
1148
|
+
|
|
1149
|
+
if not self._path:
|
|
1150
|
+
raise ValueError("No file path specified. Use: experiment.files('path/to/file').read_text()")
|
|
1151
|
+
|
|
1152
|
+
# Create temporary file for download
|
|
1153
|
+
temp_dir = tempfile.mkdtemp()
|
|
1154
|
+
temp_filename = Path(self._path).name
|
|
1155
|
+
temp_path = os.path.join(temp_dir, temp_filename)
|
|
1156
|
+
|
|
1157
|
+
try:
|
|
1158
|
+
# Download the file
|
|
1159
|
+
downloaded_path = self.download(to=temp_path)
|
|
1160
|
+
|
|
1161
|
+
# Read content as text
|
|
1162
|
+
with open(downloaded_path, 'r', encoding=encoding) as f:
|
|
1163
|
+
content = f.read()
|
|
1164
|
+
|
|
1165
|
+
return content
|
|
1166
|
+
finally:
|
|
1167
|
+
# Clean up temporary file
|
|
1168
|
+
try:
|
|
1169
|
+
if os.path.exists(temp_path):
|
|
1170
|
+
os.unlink(temp_path)
|
|
1171
|
+
os.rmdir(temp_dir)
|
|
1172
|
+
except Exception:
|
|
1173
|
+
pass
|
|
1174
|
+
|
|
1086
1175
|
|
|
1087
1176
|
class FilesAccessor:
|
|
1088
1177
|
"""
|
|
1089
1178
|
Accessor that enables both callable and attribute-style access to file operations.
|
|
1090
1179
|
|
|
1091
1180
|
This allows:
|
|
1092
|
-
|
|
1093
|
-
|
|
1181
|
+
dxp.files("models") # Returns FileBuilder
|
|
1182
|
+
dxp.files(dir="models") # Keyword argument form
|
|
1183
|
+
experiment.files.upload(...) # Direct method call
|
|
1094
1184
|
experiment.files.download(...) # Direct method call
|
|
1095
1185
|
"""
|
|
1096
1186
|
|
|
@@ -1098,27 +1188,55 @@ class FilesAccessor:
|
|
|
1098
1188
|
self._experiment = experiment
|
|
1099
1189
|
self._builder = FileBuilder(experiment)
|
|
1100
1190
|
|
|
1101
|
-
def __call__(self,
|
|
1102
|
-
"""
|
|
1103
|
-
|
|
1191
|
+
def __call__(self, dir: Optional[str] = None, **kwargs) -> FileBuilder:
|
|
1192
|
+
"""
|
|
1193
|
+
Create a FileBuilder with the given directory.
|
|
1194
|
+
|
|
1195
|
+
Supports flexible argument styles:
|
|
1196
|
+
- Positional: files("models")
|
|
1197
|
+
- Keyword: files(dir="models")
|
|
1198
|
+
- No args (root): files()
|
|
1199
|
+
|
|
1200
|
+
Args:
|
|
1201
|
+
dir: Directory/path for file operations
|
|
1202
|
+
**kwargs: Additional FileBuilder options
|
|
1203
|
+
|
|
1204
|
+
Returns:
|
|
1205
|
+
FileBuilder instance
|
|
1206
|
+
"""
|
|
1207
|
+
return FileBuilder(self._experiment, path=dir, **kwargs)
|
|
1104
1208
|
|
|
1105
1209
|
# Direct methods that don't require a path first
|
|
1106
1210
|
|
|
1107
|
-
def
|
|
1211
|
+
def upload(
|
|
1108
1212
|
self,
|
|
1109
|
-
|
|
1213
|
+
fpath: str,
|
|
1110
1214
|
*,
|
|
1111
1215
|
to: Optional[str] = None,
|
|
1112
1216
|
**kwargs
|
|
1113
1217
|
) -> Dict[str, Any]:
|
|
1114
1218
|
"""
|
|
1115
|
-
|
|
1219
|
+
Upload a file directly without specifying a path prefix first.
|
|
1220
|
+
|
|
1221
|
+
Args:
|
|
1222
|
+
fpath: Path to existing file to upload (required)
|
|
1223
|
+
to: Optional destination path, e.g., "models/model.pt" or "renamed.pt"
|
|
1224
|
+
**kwargs: Additional FileBuilder options
|
|
1225
|
+
|
|
1226
|
+
Returns:
|
|
1227
|
+
File metadata dict with id, path, filename, checksum, etc.
|
|
1116
1228
|
|
|
1117
1229
|
Examples:
|
|
1118
|
-
|
|
1119
|
-
experiment.files.
|
|
1230
|
+
# Upload with original filename
|
|
1231
|
+
experiment.files.upload("./model.pt")
|
|
1232
|
+
|
|
1233
|
+
# Upload with destination path
|
|
1234
|
+
experiment.files.upload("./model.pt", to="models/model.pt")
|
|
1235
|
+
|
|
1236
|
+
# Upload with metadata
|
|
1237
|
+
experiment.files.upload("./model.pt", to="best.pt", description="Best model")
|
|
1120
1238
|
"""
|
|
1121
|
-
# Parse 'to' to extract prefix and filename
|
|
1239
|
+
# Parse 'to' to extract prefix and filename if provided
|
|
1122
1240
|
if to:
|
|
1123
1241
|
to_path = to.lstrip('/')
|
|
1124
1242
|
if '/' in to_path:
|
|
@@ -1127,13 +1245,10 @@ class FilesAccessor:
|
|
|
1127
1245
|
else:
|
|
1128
1246
|
prefix = '/'
|
|
1129
1247
|
filename = to_path
|
|
1130
|
-
return FileBuilder(self._experiment, path=prefix, **kwargs).
|
|
1131
|
-
|
|
1132
|
-
if isinstance(obj, str) and Path(obj).exists():
|
|
1133
|
-
# obj is a file path, extract prefix from it
|
|
1134
|
-
return FileBuilder(self._experiment, **kwargs).save(obj)
|
|
1248
|
+
return FileBuilder(self._experiment, path=prefix, **kwargs).upload(fpath, to=filename)
|
|
1135
1249
|
|
|
1136
|
-
|
|
1250
|
+
# No prefix, just upload with original or specified filename
|
|
1251
|
+
return FileBuilder(self._experiment, **kwargs).upload(fpath)
|
|
1137
1252
|
|
|
1138
1253
|
def download(
|
|
1139
1254
|
self,
|