ml-dash 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ml_dash/files.py CHANGED
@@ -6,6 +6,7 @@ Provides fluent API for file upload, download, list, and delete operations.
6
6
 
7
7
  import hashlib
8
8
  import mimetypes
9
+ import fnmatch
9
10
  from typing import Dict, Any, List, Optional, Union, TYPE_CHECKING
10
11
  from pathlib import Path
11
12
 
@@ -18,38 +19,69 @@ class FileBuilder:
18
19
  Fluent interface for file operations.
19
20
 
20
21
  Usage:
21
- # Upload file
22
- experiment.files(file_path="./model.pt", prefix="/models").save()
22
+ # Upload existing file from disk
23
+ dxp.files("models").upload("./local_model.pt")
24
+ dxp.files("checkpoints").upload("./model.pt", to="latest.pt")
25
+
26
+ # Save objects as files (using keyword dir argument)
27
+ dxp.files(dir="models").save_torch(model, to="checkpoint.pt")
28
+ dxp.files(dir="configs").save_json({"lr": 0.001}, to="config.json")
29
+ dxp.files(dir="data").save_blob(b"binary data", to="data.bin")
23
30
 
24
31
  # List files
25
- files = experiment.files().list()
26
- files = experiment.files(prefix="/models").list()
32
+ files = experiment.files(dir="some/location").list()
33
+ files = experiment.files(dir="models").list()
27
34
 
28
35
  # Download file
29
- experiment.files(file_id="123").download()
30
- experiment.files(file_id="123", dest_path="./model.pt").download()
31
-
32
- # Delete file
33
- experiment.files(file_id="123").delete()
36
+ dxp.files("some.text").download()
37
+ dxp.files("some.text").download(to="./model.pt")
38
+
39
+ # Download files via glob pattern
40
+ file_paths = experiment.files("images").list("*.png")
41
+ dxp.files("images").download("*.png")
42
+
43
+ # Delete files
44
+ dxp.files("some.text").delete()
45
+
46
+ # Check if file exists
47
+ if dxp.files("config.json").exists():
48
+ config = dxp.files("config.json").read_text()
49
+
50
+ # Read file content as text
51
+ config_yaml = dxp.files("configs/view.yaml").read_text()
52
+
53
+ Specific Save Methods:
54
+ experiment.files.save_text("content", to="view.yaml")
55
+ experiment.files.save_json(dict(hey="yo"), to="config.json")
56
+ experiment.files.save_blob(b"xxx", to="data.bin")
57
+ experiment.files.save_torch(model, to="model.pt")
58
+ experiment.files.save_pkl(data, to="data.pkl")
59
+ experiment.files.save_fig(fig, to="plot.png")
60
+ experiment.files.save_video(frames, to="video.mp4")
34
61
  """
35
62
 
36
- def __init__(self, experiment: 'Experiment', **kwargs):
63
+ def __init__(self, experiment: 'Experiment', path: Optional[str] = None, **kwargs):
37
64
  """
38
65
  Initialize file builder.
39
66
 
40
67
  Args:
41
68
  experiment: Parent experiment instance
42
- **kwargs: File operation parameters
43
- - file_path: Path to file to upload
44
- - prefix: Logical path prefix (default: "/")
69
+ path: File path or prefix for operations. Can be:
70
+ - A prefix/directory (e.g., "checkpoints", "/models")
71
+ - A file path (e.g., "some.text", "images/photo.png")
72
+ **kwargs: Additional file operation parameters (for backwards compatibility)
73
+ - file_path: Path to file to upload (deprecated, use save(to=))
74
+ - prefix: Logical path prefix (deprecated, use path argument)
45
75
  - description: Optional description
46
76
  - tags: Optional list of tags
47
77
  - bindrs: Optional list of bindrs
48
78
  - metadata: Optional metadata dict
49
79
  - file_id: File ID for download/delete/update operations
50
- - dest_path: Destination path for download
80
+ - dest_path: Destination path for download (deprecated, use download(to=))
51
81
  """
52
82
  self._experiment = experiment
83
+ self._path = path
84
+ # Backwards compatibility
53
85
  self._file_path = kwargs.get('file_path')
54
86
  self._prefix = kwargs.get('prefix', '/')
55
87
  self._description = kwargs.get('description')
@@ -59,9 +91,30 @@ class FileBuilder:
59
91
  self._file_id = kwargs.get('file_id')
60
92
  self._dest_path = kwargs.get('dest_path')
61
93
 
62
- def save(self) -> Dict[str, Any]:
94
+ # If path is provided, determine if it's a file or prefix
95
+ if path:
96
+ # Normalize path
97
+ path = path.lstrip('/')
98
+ self._normalized_path = '/' + path if not path.startswith('/') else path
99
+
100
+ def upload(
101
+ self,
102
+ fpath: str,
103
+ *,
104
+ to: Optional[str] = None,
105
+ description: Optional[str] = None,
106
+ tags: Optional[List[str]] = None,
107
+ metadata: Optional[Dict[str, Any]] = None
108
+ ) -> Dict[str, Any]:
63
109
  """
64
- Upload and save the file.
110
+ Upload an existing file from disk.
111
+
112
+ Args:
113
+ fpath: Path to existing file to upload (required)
114
+ to: Optional new filename (uses original filename if not provided)
115
+ description: Optional description
116
+ tags: Optional list of tags
117
+ metadata: Optional metadata dict
65
118
 
66
119
  Returns:
67
120
  File metadata dict with id, path, filename, checksum, etc.
@@ -72,8 +125,19 @@ class FileBuilder:
72
125
  ValueError: If file size exceeds 100GB limit
73
126
 
74
127
  Examples:
75
- result = experiment.files(file_path="./model.pt", prefix="/models").save()
76
- # Returns: {"id": "123", "path": "/models", "filename": "model.pt", ...}
128
+ # Upload with original filename
129
+ dxp.files("models").upload("./local_model.pt")
130
+
131
+ # Upload with new filename
132
+ dxp.files("models").upload("./local_model.pt", to="remote_model.pt")
133
+
134
+ # Upload with metadata
135
+ dxp.files("checkpoints").upload(
136
+ "./checkpoint.pt",
137
+ to="epoch_10.pt",
138
+ description="Best model at epoch 10",
139
+ tags=["best", "training"]
140
+ )
77
141
  """
78
142
  if not self._experiment._is_open:
79
143
  raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
@@ -81,47 +145,312 @@ class FileBuilder:
81
145
  if self._experiment._write_protected:
82
146
  raise RuntimeError("Experiment is write-protected and cannot be modified.")
83
147
 
84
- if not self._file_path:
85
- raise ValueError("file_path is required for save() operation")
148
+ if not fpath:
149
+ raise ValueError("fpath is required")
150
+
151
+ # Use provided values or fall back to constructor values
152
+ desc = description if description is not None else self._description
153
+ file_tags = tags if tags is not None else self._tags
154
+ file_metadata = metadata if metadata is not None else self._metadata
155
+
156
+ # Determine prefix from path
157
+ prefix = self._prefix
158
+ if self._path:
159
+ prefix = '/' + self._path.lstrip('/')
160
+
161
+ return self._save_file(
162
+ fpath=fpath,
163
+ prefix=prefix,
164
+ description=desc,
165
+ tags=file_tags,
166
+ metadata=file_metadata,
167
+ to=to
168
+ )
169
+
170
+ def save(
171
+ self,
172
+ content: Any = None,
173
+ *,
174
+ to: Optional[str] = None,
175
+ description: Optional[str] = None,
176
+ tags: Optional[List[str]] = None,
177
+ metadata: Optional[Dict[str, Any]] = None
178
+ ) -> Dict[str, Any]:
179
+ """
180
+ Unified save method that handles different content types.
181
+
182
+ Args:
183
+ content: Content to save - can be:
184
+ - str (file path): Uploads existing file
185
+ - bytes: Saves as binary blob (requires 'to' parameter)
186
+ - dict/list: Saves as JSON (requires 'to' parameter)
187
+ - None: Uses file_path from constructor (backwards compatibility)
188
+ to: Target filename (required for bytes/dict/list, optional for file paths)
189
+ description: Optional description
190
+ tags: Optional list of tags
191
+ metadata: Optional metadata dict
86
192
 
87
- file_path = Path(self._file_path)
88
- if not file_path.exists():
89
- raise ValueError(f"File not found: {self._file_path}")
193
+ Returns:
194
+ File metadata dict with id, path, filename, checksum, etc.
195
+ """
196
+ # Override builder metadata if provided
197
+ if description is not None:
198
+ self._description = description
199
+ if tags is not None:
200
+ self._tags = tags
201
+ if metadata is not None:
202
+ self._metadata = metadata
203
+
204
+ # Backwards compatibility: use file_path from constructor if no content provided
205
+ if content is None:
206
+ if self._file_path:
207
+ content = self._file_path
208
+ else:
209
+ raise ValueError("No content provided and no file_path set in constructor")
210
+
211
+ # Check if content is a file path
212
+ if isinstance(content, str) and Path(content).exists():
213
+ return self.upload(content, to=to)
214
+
215
+ # Check if content is bytes
216
+ if isinstance(content, bytes):
217
+ if not to:
218
+ raise ValueError("'to' parameter is required when saving bytes")
219
+ return self.save_blob(content, to=to)
220
+
221
+ # Check if content is dict or list (save as JSON)
222
+ if isinstance(content, (dict, list)):
223
+ if not to:
224
+ raise ValueError("'to' parameter is required when saving dict/list")
225
+ return self.save_json(content, to=to)
226
+
227
+ raise ValueError(f"Unsupported content type: {type(content)}. Expected str (file path), bytes, dict, or list.")
228
+
229
+ def _save_file(
230
+ self,
231
+ fpath: str,
232
+ prefix: str,
233
+ description: Optional[str],
234
+ tags: Optional[List[str]],
235
+ metadata: Optional[Dict[str, Any]],
236
+ to: Optional[str] = None
237
+ ) -> Dict[str, Any]:
238
+ """Internal method to save an existing file."""
239
+ fpath_obj = Path(fpath)
240
+ if not fpath_obj.exists():
241
+ raise ValueError(f"File not found: {fpath}")
90
242
 
91
- if not file_path.is_file():
92
- raise ValueError(f"Path is not a file: {self._file_path}")
243
+ if not fpath_obj.is_file():
244
+ raise ValueError(f"Path is not a file: {fpath}")
93
245
 
94
246
  # Check file size (max 100GB)
95
- file_size = file_path.stat().st_size
247
+ file_size = fpath_obj.stat().st_size
96
248
  MAX_FILE_SIZE = 100 * 1024 * 1024 * 1024 # 100GB in bytes
97
249
  if file_size > MAX_FILE_SIZE:
98
250
  raise ValueError(f"File size ({file_size} bytes) exceeds 100GB limit")
99
251
 
100
252
  # Compute checksum
101
- checksum = compute_sha256(str(file_path))
253
+ checksum = compute_sha256(str(fpath_obj))
102
254
 
103
255
  # Detect MIME type
104
- content_type = get_mime_type(str(file_path))
256
+ content_type = get_mime_type(str(fpath_obj))
105
257
 
106
- # Get filename
107
- filename = file_path.name
258
+ # Get filename (use provided 'to' or original)
259
+ filename = to if to else fpath_obj.name
108
260
 
109
261
  # Upload through experiment
110
262
  return self._experiment._upload_file(
111
- file_path=str(file_path),
112
- prefix=self._prefix,
263
+ file_path=str(fpath_obj),
264
+ prefix=prefix,
113
265
  filename=filename,
114
- description=self._description,
115
- tags=self._tags,
116
- metadata=self._metadata,
266
+ description=description,
267
+ tags=tags or [],
268
+ metadata=metadata,
117
269
  checksum=checksum,
118
270
  content_type=content_type,
119
271
  size_bytes=file_size
120
272
  )
121
273
 
122
- def list(self) -> List[Dict[str, Any]]:
274
+ def _save_bytes(
275
+ self,
276
+ data: bytes,
277
+ filename: str,
278
+ prefix: str,
279
+ description: Optional[str],
280
+ tags: Optional[List[str]],
281
+ metadata: Optional[Dict[str, Any]]
282
+ ) -> Dict[str, Any]:
283
+ """Save bytes data to a file."""
284
+ import tempfile
285
+ import os
286
+
287
+ temp_dir = tempfile.mkdtemp()
288
+ temp_path = os.path.join(temp_dir, filename)
289
+ # Create parent directories if filename contains path
290
+ os.makedirs(os.path.dirname(temp_path), exist_ok=True)
291
+ try:
292
+ with open(temp_path, 'wb') as f:
293
+ f.write(data)
294
+ return self._save_file(
295
+ fpath=temp_path,
296
+ prefix=prefix,
297
+ description=description,
298
+ tags=tags,
299
+ metadata=metadata
300
+ )
301
+ finally:
302
+ try:
303
+ os.unlink(temp_path)
304
+ os.rmdir(temp_dir)
305
+ except Exception:
306
+ pass
307
+
308
+ def _save_json(
309
+ self,
310
+ content: Any,
311
+ filename: str,
312
+ prefix: str,
313
+ description: Optional[str],
314
+ tags: Optional[List[str]],
315
+ metadata: Optional[Dict[str, Any]]
316
+ ) -> Dict[str, Any]:
317
+ """Save JSON content to a file."""
318
+ import json
319
+ import tempfile
320
+ import os
321
+
322
+ temp_dir = tempfile.mkdtemp()
323
+ temp_path = os.path.join(temp_dir, filename)
324
+ # Create parent directories if filename contains path
325
+ os.makedirs(os.path.dirname(temp_path), exist_ok=True)
326
+ try:
327
+ with open(temp_path, 'w') as f:
328
+ json.dump(content, f, indent=2)
329
+ return self._save_file(
330
+ fpath=temp_path,
331
+ prefix=prefix,
332
+ description=description,
333
+ tags=tags,
334
+ metadata=metadata
335
+ )
336
+ finally:
337
+ try:
338
+ os.unlink(temp_path)
339
+ os.rmdir(temp_dir)
340
+ except Exception:
341
+ pass
342
+
343
+ def _save_torch(
344
+ self,
345
+ model: Any,
346
+ filename: str,
347
+ prefix: str,
348
+ description: Optional[str],
349
+ tags: Optional[List[str]],
350
+ metadata: Optional[Dict[str, Any]]
351
+ ) -> Dict[str, Any]:
352
+ """Save PyTorch model to a file."""
353
+ import tempfile
354
+ import os
355
+ import torch
356
+
357
+ temp_dir = tempfile.mkdtemp()
358
+ temp_path = os.path.join(temp_dir, filename)
359
+ # Create parent directories if filename contains path
360
+ os.makedirs(os.path.dirname(temp_path), exist_ok=True)
361
+ try:
362
+ torch.save(model, temp_path)
363
+ return self._save_file(
364
+ fpath=temp_path,
365
+ prefix=prefix,
366
+ description=description,
367
+ tags=tags,
368
+ metadata=metadata
369
+ )
370
+ finally:
371
+ try:
372
+ os.unlink(temp_path)
373
+ os.rmdir(temp_dir)
374
+ except Exception:
375
+ pass
376
+
377
+ def _save_fig(
378
+ self,
379
+ fig: Any,
380
+ filename: str,
381
+ prefix: str,
382
+ description: Optional[str],
383
+ tags: Optional[List[str]],
384
+ metadata: Optional[Dict[str, Any]],
385
+ **kwargs
386
+ ) -> Dict[str, Any]:
387
+ """Save matplotlib figure to a file."""
388
+ import tempfile
389
+ import os
390
+ import matplotlib.pyplot as plt
391
+
392
+ temp_dir = tempfile.mkdtemp()
393
+ temp_path = os.path.join(temp_dir, filename)
394
+ # Create parent directories if filename contains path
395
+ os.makedirs(os.path.dirname(temp_path), exist_ok=True)
396
+ try:
397
+ fig.savefig(temp_path, **kwargs)
398
+ plt.close(fig)
399
+ return self._save_file(
400
+ fpath=temp_path,
401
+ prefix=prefix,
402
+ description=description,
403
+ tags=tags,
404
+ metadata=metadata
405
+ )
406
+ finally:
407
+ try:
408
+ os.unlink(temp_path)
409
+ os.rmdir(temp_dir)
410
+ except Exception:
411
+ pass
412
+
413
+ def _save_pickle(
414
+ self,
415
+ content: Any,
416
+ filename: str,
417
+ prefix: str,
418
+ description: Optional[str],
419
+ tags: Optional[List[str]],
420
+ metadata: Optional[Dict[str, Any]]
421
+ ) -> Dict[str, Any]:
422
+ """Save Python object to a pickle file."""
423
+ import pickle
424
+ import tempfile
425
+ import os
426
+
427
+ temp_dir = tempfile.mkdtemp()
428
+ temp_path = os.path.join(temp_dir, filename)
429
+ # Create parent directories if filename contains path
430
+ os.makedirs(os.path.dirname(temp_path), exist_ok=True)
431
+ try:
432
+ with open(temp_path, 'wb') as f:
433
+ pickle.dump(content, f)
434
+ return self._save_file(
435
+ fpath=temp_path,
436
+ prefix=prefix,
437
+ description=description,
438
+ tags=tags,
439
+ metadata=metadata
440
+ )
441
+ finally:
442
+ try:
443
+ os.unlink(temp_path)
444
+ os.rmdir(temp_dir)
445
+ except Exception:
446
+ pass
447
+
448
+ def list(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
123
449
  """
124
- List files with optional filters.
450
+ List files with optional glob pattern filtering.
451
+
452
+ Args:
453
+ pattern: Optional glob pattern to filter files (e.g., "*.png", "model_*.pt")
125
454
 
126
455
  Returns:
127
456
  List of file metadata dicts
@@ -131,62 +460,169 @@ class FileBuilder:
131
460
 
132
461
  Examples:
133
462
  files = experiment.files().list() # All files
134
- files = experiment.files(prefix="/models").list() # Filter by prefix
135
- files = experiment.files(tags=["checkpoint"]).list() # Filter by tags
463
+ files = experiment.files("/models").list() # Files in /models prefix
464
+ files = experiment.files("images").list("*.png") # PNG files in images
465
+ files = experiment.files().list("**/*.pt") # All .pt files
136
466
  """
137
467
  if not self._experiment._is_open:
138
468
  raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
139
469
 
140
- return self._experiment._list_files(
141
- prefix=self._prefix if self._prefix != '/' else None,
470
+ # Determine prefix filter - support both new (path) and old (prefix) API
471
+ prefix = None
472
+ if self._path:
473
+ prefix = '/' + self._path.lstrip('/')
474
+ elif self._prefix and self._prefix != '/':
475
+ prefix = self._prefix
476
+
477
+ # Get all files matching prefix
478
+ files = self._experiment._list_files(
479
+ prefix=prefix,
142
480
  tags=self._tags if self._tags else None
143
481
  )
144
482
 
145
- def download(self) -> str:
483
+ # Apply glob pattern if provided
484
+ if pattern:
485
+ pattern = pattern.lstrip('/')
486
+ filtered = []
487
+ for f in files:
488
+ filename = f.get('filename', '')
489
+ full_path = f.get('path', '/').rstrip('/') + '/' + filename
490
+ full_path = full_path.lstrip('/')
491
+
492
+ # Match against filename or full path
493
+ if fnmatch.fnmatch(filename, pattern) or fnmatch.fnmatch(full_path, pattern):
494
+ filtered.append(f)
495
+ return filtered
496
+
497
+ return files
498
+
499
+ def download(
500
+ self,
501
+ pattern: Optional[str] = None,
502
+ *,
503
+ to: Optional[str] = None
504
+ ) -> Union[str, List[str]]:
146
505
  """
147
- Download file with automatic checksum verification.
506
+ Download file(s) with automatic checksum verification.
148
507
 
149
- If dest_path not provided, downloads to current directory with original filename.
508
+ Args:
509
+ pattern: Optional glob pattern for batch download (e.g., "*.png")
510
+ to: Destination path. For single files, this is the file path.
511
+ For patterns, this is the destination directory.
150
512
 
151
513
  Returns:
152
- Path to downloaded file
514
+ For single file: Path to downloaded file
515
+ For pattern: List of paths to downloaded files
153
516
 
154
517
  Raises:
155
518
  RuntimeError: If experiment is not open
156
- ValueError: If file_id not provided
157
- ValueError: If checksum verification fails
519
+ ValueError: If file not found or checksum verification fails
158
520
 
159
521
  Examples:
160
- # Download to current directory with original filename
522
+ # Download single file
523
+ path = experiment.files("model.pt").download()
524
+ path = experiment.files("model.pt").download(to="./local_model.pt")
525
+
526
+ # Download by file ID (backwards compatibility)
161
527
  path = experiment.files(file_id="123").download()
162
528
 
163
- # Download to custom path
164
- path = experiment.files(file_id="123", dest_path="./model.pt").download()
529
+ # Download multiple files matching pattern
530
+ paths = experiment.files("images").download("*.png")
531
+ paths = experiment.files("images").download("*.png", to="./local_images")
165
532
  """
166
533
  if not self._experiment._is_open:
167
534
  raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
168
535
 
169
- if not self._file_id:
170
- raise ValueError("file_id is required for download() operation")
536
+ # If file_id is set (backwards compatibility)
537
+ if self._file_id:
538
+ return self._experiment._download_file(
539
+ file_id=self._file_id,
540
+ dest_path=to or self._dest_path
541
+ )
171
542
 
172
- return self._experiment._download_file(
173
- file_id=self._file_id,
174
- dest_path=self._dest_path
175
- )
543
+ # If pattern is provided, download multiple files
544
+ if pattern:
545
+ files = self.list(pattern)
546
+ if not files:
547
+ raise ValueError(f"No files found matching pattern: {pattern}")
548
+
549
+ downloaded = []
550
+ dest_dir = Path(to) if to else Path('.')
551
+ if to and not dest_dir.exists():
552
+ dest_dir.mkdir(parents=True, exist_ok=True)
553
+
554
+ for f in files:
555
+ file_id = f.get('id')
556
+ filename = f.get('filename', 'file')
557
+ dest_path = str(dest_dir / filename)
558
+ path = self._experiment._download_file(
559
+ file_id=file_id,
560
+ dest_path=dest_path
561
+ )
562
+ downloaded.append(path)
563
+
564
+ return downloaded
565
+
566
+ # Download single file by path
567
+ if self._path:
568
+ # Find file by path
569
+ files = self._experiment._list_files(prefix=None, tags=None)
570
+ matching = []
571
+ search_path = self._path.lstrip('/')
572
+
573
+ for f in files:
574
+ filename = f.get('filename', '')
575
+ prefix = f.get('path', '/').lstrip('/')
576
+ full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
577
+ full_path = full_path.lstrip('/')
578
+
579
+ if full_path == search_path or filename == search_path:
580
+ matching.append(f)
581
+
582
+ if not matching:
583
+ raise ValueError(f"File not found: {self._path}")
584
+
585
+ if len(matching) > 1:
586
+ # If multiple matches, prefer exact path match
587
+ exact = [f for f in matching if
588
+ (f.get('path', '/').lstrip('/').rstrip('/') + '/' + f.get('filename', '')).lstrip('/') == search_path]
589
+ if exact:
590
+ matching = exact[:1]
591
+ else:
592
+ matching = matching[:1]
593
+
594
+ file_info = matching[0]
595
+ return self._experiment._download_file(
596
+ file_id=file_info['id'],
597
+ dest_path=to
598
+ )
599
+
600
+ raise ValueError("No file path or pattern specified")
176
601
 
177
- def delete(self) -> Dict[str, Any]:
602
+ def delete(self, pattern: Optional[str] = None) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
178
603
  """
179
- Delete file (soft delete).
604
+ Delete file(s) (soft delete).
605
+
606
+ Args:
607
+ pattern: Optional glob pattern for batch delete (e.g., "*.png")
180
608
 
181
609
  Returns:
182
- Dict with id and deletedAt timestamp
610
+ For single file: Dict with id and deletedAt timestamp
611
+ For pattern: List of deletion results
183
612
 
184
613
  Raises:
185
614
  RuntimeError: If experiment is not open or write-protected
186
- ValueError: If file_id not provided
615
+ ValueError: If file not found
187
616
 
188
617
  Examples:
618
+ # Delete single file
619
+ result = experiment.files("some.text").delete()
620
+
621
+ # Delete by file ID (backwards compatibility)
189
622
  result = experiment.files(file_id="123").delete()
623
+
624
+ # Delete multiple files matching pattern
625
+ results = experiment.files("images").delete("*.png")
190
626
  """
191
627
  if not self._experiment._is_open:
192
628
  raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
@@ -194,10 +630,52 @@ class FileBuilder:
194
630
  if self._experiment._write_protected:
195
631
  raise RuntimeError("Experiment is write-protected and cannot be modified.")
196
632
 
197
- if not self._file_id:
198
- raise ValueError("file_id is required for delete() operation")
199
-
200
- return self._experiment._delete_file(file_id=self._file_id)
633
+ # If file_id is set (backwards compatibility)
634
+ if self._file_id:
635
+ return self._experiment._delete_file(file_id=self._file_id)
636
+
637
+ # If pattern is provided, delete multiple files
638
+ if pattern:
639
+ files = self.list(pattern)
640
+ if not files:
641
+ raise ValueError(f"No files found matching pattern: {pattern}")
642
+
643
+ results = []
644
+ for f in files:
645
+ file_id = f.get('id')
646
+ result = self._experiment._delete_file(file_id=file_id)
647
+ results.append(result)
648
+ return results
649
+
650
+ # Delete single file by path
651
+ if self._path:
652
+ files = self._experiment._list_files(prefix=None, tags=None)
653
+ matching = []
654
+ search_path = self._path.lstrip('/')
655
+
656
+ for f in files:
657
+ filename = f.get('filename', '')
658
+ prefix = f.get('path', '/').lstrip('/')
659
+ full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
660
+ full_path = full_path.lstrip('/')
661
+
662
+ if full_path == search_path or filename == search_path:
663
+ matching.append(f)
664
+
665
+ if not matching:
666
+ raise ValueError(f"File not found: {self._path}")
667
+
668
+ # Delete all matching files
669
+ if len(matching) == 1:
670
+ return self._experiment._delete_file(file_id=matching[0]['id'])
671
+
672
+ results = []
673
+ for f in matching:
674
+ result = self._experiment._delete_file(file_id=f['id'])
675
+ results.append(result)
676
+ return results
677
+
678
+ raise ValueError("No file path or pattern specified")
201
679
 
202
680
  def update(self) -> Dict[str, Any]:
203
681
  """
@@ -234,271 +712,222 @@ class FileBuilder:
234
712
  metadata=self._metadata
235
713
  )
236
714
 
237
- def save_json(self, content: Any, file_name: str) -> Dict[str, Any]:
715
+ # Convenience methods for specific file types
716
+
717
+ def save_json(
718
+ self,
719
+ content: Any,
720
+ *,
721
+ to: str
722
+ ) -> Dict[str, Any]:
238
723
  """
239
724
  Save JSON content to a file.
240
725
 
241
726
  Args:
242
727
  content: Content to save as JSON (dict, list, or any JSON-serializable object)
243
- file_name: Name of the file to create
728
+ to: Target filename
244
729
 
245
730
  Returns:
246
731
  File metadata dict with id, path, filename, checksum, etc.
247
732
 
248
- Raises:
249
- RuntimeError: If experiment is not open or write-protected
250
- ValueError: If content is not JSON-serializable
251
-
252
733
  Examples:
253
734
  config = {"model": "resnet50", "lr": 0.001}
254
- result = experiment.files(prefix="/configs").save_json(config, "config.json")
735
+ result = dxp.files("configs").save_json(config, to="config.json")
255
736
  """
256
- import json
257
- import tempfile
258
- import os
259
-
260
- if not self._experiment._is_open:
261
- raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
737
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
262
738
 
263
- if self._experiment._write_protected:
264
- raise RuntimeError("Experiment is write-protected and cannot be modified.")
739
+ # Extract path component from 'to' if present (e.g., "configs/settings.json")
740
+ import os
741
+ to_dirname = os.path.dirname(to)
742
+ to_filename = os.path.basename(to)
743
+ if to_dirname:
744
+ # Merge the path component into prefix
745
+ prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
746
+
747
+ return self._save_json(
748
+ content=content,
749
+ filename=to_filename,
750
+ prefix=prefix,
751
+ description=self._description,
752
+ tags=self._tags,
753
+ metadata=self._metadata
754
+ )
265
755
 
266
- # Create temporary file with desired filename
267
- temp_dir = tempfile.mkdtemp()
268
- temp_path = os.path.join(temp_dir, file_name)
269
- try:
270
- # Write JSON content to temp file
271
- with open(temp_path, 'w') as f:
272
- json.dump(content, f, indent=2)
756
+ def save_text(self, content: str, *, to: str) -> Dict[str, Any]:
757
+ """
758
+ Save text content to a file.
273
759
 
274
- # Save using existing save() method
275
- original_file_path = self._file_path
276
- self._file_path = temp_path
760
+ Args:
761
+ content: Text content to save
762
+ to: Target filename
277
763
 
278
- # Upload and get result
279
- result = self.save()
764
+ Returns:
765
+ File metadata dict with id, path, filename, checksum, etc.
280
766
 
281
- # Restore original file_path
282
- self._file_path = original_file_path
767
+ Examples:
768
+ result = experiment.files().save_text("Hello, world!", to="greeting.txt")
769
+ result = experiment.files("configs").save_text(yaml_content, to="view.yaml")
770
+ """
771
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
283
772
 
284
- return result
285
- finally:
286
- # Clean up temp file and directory
287
- try:
288
- os.unlink(temp_path)
289
- os.rmdir(temp_dir)
290
- except Exception:
291
- pass
773
+ # Extract path component from 'to' if present
774
+ import os
775
+ to_dirname = os.path.dirname(to)
776
+ to_filename = os.path.basename(to)
777
+ if to_dirname:
778
+ prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
779
+
780
+ return self._save_bytes(
781
+ data=content.encode('utf-8'),
782
+ filename=to_filename,
783
+ prefix=prefix,
784
+ description=self._description,
785
+ tags=self._tags,
786
+ metadata=self._metadata
787
+ )
292
788
 
293
- def save_torch(self, model: Any, file_name: str) -> Dict[str, Any]:
789
+ def save_blob(self, data: bytes, *, to: str) -> Dict[str, Any]:
294
790
  """
295
- Save PyTorch model to a file.
791
+ Save binary data to a file.
296
792
 
297
793
  Args:
298
- model: PyTorch model or state dict to save
299
- file_name: Name of the file to create (should end with .pt or .pth)
794
+ data: Binary data to save
795
+ to: Target filename
300
796
 
301
797
  Returns:
302
798
  File metadata dict with id, path, filename, checksum, etc.
303
799
 
304
- Raises:
305
- RuntimeError: If experiment is not open or write-protected
306
- ImportError: If torch is not installed
307
- ValueError: If model cannot be saved
308
-
309
800
  Examples:
310
- import torch
311
- model = torch.nn.Linear(10, 5)
312
- result = experiment.files(prefix="/models").save_torch(model, "model.pt")
313
-
314
- # Or save state dict
315
- result = experiment.files(prefix="/models").save_torch(model.state_dict(), "model.pth")
801
+ result = experiment.files("data").save_blob(binary_data, to="model.bin")
316
802
  """
317
- import tempfile
318
- import os
803
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
319
804
 
320
- try:
321
- import torch
322
- except ImportError:
323
- raise ImportError("PyTorch is not installed. Install it with: pip install torch")
324
-
325
- if not self._experiment._is_open:
326
- raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
327
-
328
- if self._experiment._write_protected:
329
- raise RuntimeError("Experiment is write-protected and cannot be modified.")
805
+ # Extract path component from 'to' if present
806
+ import os
807
+ to_dirname = os.path.dirname(to)
808
+ to_filename = os.path.basename(to)
809
+ if to_dirname:
810
+ prefix = prefix.rstrip('/') + '/' + to_dirname.lstrip('/')
811
+
812
+ return self._save_bytes(
813
+ data=data,
814
+ filename=to_filename,
815
+ prefix=prefix,
816
+ description=self._description,
817
+ tags=self._tags,
818
+ metadata=self._metadata
819
+ )
330
820
 
331
- # Create temporary file with desired filename
332
- temp_dir = tempfile.mkdtemp()
333
- temp_path = os.path.join(temp_dir, file_name)
821
+ def save_torch(
822
+ self,
823
+ model: Any,
824
+ *,
825
+ to: str
826
+ ) -> Dict[str, Any]:
827
+ """
828
+ Save PyTorch model to a file.
334
829
 
335
- try:
336
- # Save model to temp file
337
- torch.save(model, temp_path)
830
+ Args:
831
+ model: PyTorch model or state dict to save
832
+ to: Target filename
338
833
 
339
- # Save using existing save() method
340
- original_file_path = self._file_path
341
- self._file_path = temp_path
834
+ Returns:
835
+ File metadata dict with id, path, filename, checksum, etc.
342
836
 
343
- # Upload and get result
344
- result = self.save()
837
+ Examples:
838
+ result = dxp.files("models").save_torch(model, to="model.pt")
839
+ result = dxp.files("models").save_torch(model.state_dict(), to="weights.pth")
840
+ """
345
841
 
346
- # Restore original file_path
347
- self._file_path = original_file_path
842
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
348
843
 
349
- return result
350
- finally:
351
- # Clean up temp file and directory
352
- try:
353
- os.unlink(temp_path)
354
- os.rmdir(temp_dir)
355
- except Exception:
356
- pass
844
+ return self._save_torch(
845
+ model=model,
846
+ filename=to,
847
+ prefix=prefix,
848
+ description=self._description,
849
+ tags=self._tags,
850
+ metadata=self._metadata
851
+ )
357
852
 
358
- def save_pkl(self, content: Any, file_name: str) -> Dict[str, Any]:
853
+ def save_pkl(
854
+ self,
855
+ content: Any,
856
+ *,
857
+ to: str
858
+ ) -> Dict[str, Any]:
359
859
  """
360
860
  Save Python object to a pickle file.
361
861
 
362
862
  Args:
363
863
  content: Python object to pickle (must be pickle-serializable)
364
- file_name: Name of the file to create (should end with .pkl or .pickle)
864
+ to: Target filename
365
865
 
366
866
  Returns:
367
867
  File metadata dict with id, path, filename, checksum, etc.
368
868
 
369
- Raises:
370
- RuntimeError: If experiment is not open or write-protected
371
- ValueError: If content cannot be pickled
372
-
373
869
  Examples:
374
870
  data = {"model": "resnet50", "weights": np.array([1, 2, 3])}
375
- result = experiment.files(prefix="/data").save_pkl(data, "data.pkl")
376
-
377
- # Or save any Python object
378
- result = experiment.files(prefix="/models").save_pkl(trained_model, "model.pickle")
871
+ result = dxp.files("data").save_pkl(data, to="data.pkl")
379
872
  """
380
- import pickle
381
- import tempfile
382
- import os
383
-
384
- if not self._experiment._is_open:
385
- raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
386
-
387
- if self._experiment._write_protected:
388
- raise RuntimeError("Experiment is write-protected and cannot be modified.")
873
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
389
874
 
390
- # Create temporary file with desired filename
391
- temp_dir = tempfile.mkdtemp()
392
- temp_path = os.path.join(temp_dir, file_name)
393
- try:
394
- # Write pickled content to temp file
395
- with open(temp_path, 'wb') as f:
396
- pickle.dump(content, f)
397
-
398
- # Save using existing save() method
399
- original_file_path = self._file_path
400
- self._file_path = temp_path
401
-
402
- # Upload and get result
403
- result = self.save()
404
-
405
- # Restore original file_path
406
- self._file_path = original_file_path
407
-
408
- return result
409
- finally:
410
- # Clean up temp file and directory
411
- try:
412
- os.unlink(temp_path)
413
- os.rmdir(temp_dir)
414
- except Exception:
415
- pass
875
+ return self._save_pickle(
876
+ content=content,
877
+ filename=to,
878
+ prefix=prefix,
879
+ description=self._description,
880
+ tags=self._tags,
881
+ metadata=self._metadata
882
+ )
416
883
 
417
- def save_fig(self, fig: Optional[Any] = None, file_name: str = "figure.png", **kwargs) -> Dict[str, Any]:
884
+ def save_fig(
885
+ self,
886
+ fig: Optional[Any] = None,
887
+ *,
888
+ to: str,
889
+ **kwargs
890
+ ) -> Dict[str, Any]:
418
891
  """
419
892
  Save matplotlib figure to a file.
420
893
 
421
894
  Args:
422
895
  fig: Matplotlib figure object. If None, uses plt.gcf() (current figure)
423
- file_name: Name of file to create (extension determines format: .png, .pdf, .svg, .jpg)
424
- **kwargs: Additional arguments passed to fig.savefig():
425
- - dpi: Resolution (int or 'figure')
426
- - transparent: Make background transparent (bool)
427
- - bbox_inches: 'tight' to auto-crop (str or Bbox)
428
- - quality: JPEG quality 0-100 (int)
429
- - format: Override format detection (str)
896
+ to: Target filename
897
+ **kwargs: Additional arguments passed to fig.savefig()
430
898
 
431
899
  Returns:
432
900
  File metadata dict with id, path, filename, checksum, etc.
433
901
 
434
- Raises:
435
- RuntimeError: If experiment not open or write-protected
436
- ImportError: If matplotlib not installed
437
-
438
902
  Examples:
439
- import matplotlib.pyplot as plt
440
-
441
- # Use current figure
442
903
  plt.plot([1, 2, 3], [1, 4, 9])
443
- result = experiment.files(prefix="/plots").save_fig(file_name="plot.png")
444
-
445
- # Specify figure explicitly
446
- fig, ax = plt.subplots()
447
- ax.plot([1, 2, 3])
448
- result = experiment.files(prefix="/plots").save_fig(fig=fig, file_name="plot.pdf", dpi=150)
904
+ result = dxp.files("plots").save_fig(to="plot.png")
449
905
  """
450
- import tempfile
451
- import os
452
-
453
906
  try:
454
907
  import matplotlib.pyplot as plt
455
908
  except ImportError:
456
909
  raise ImportError("Matplotlib is not installed. Install it with: pip install matplotlib")
457
910
 
458
- if not self._experiment._is_open:
459
- raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
460
-
461
- if self._experiment._write_protected:
462
- raise RuntimeError("Experiment is write-protected and cannot be modified.")
463
-
464
- # Get figure
465
911
  if fig is None:
466
912
  fig = plt.gcf()
467
913
 
468
- # Create temporary file with desired filename
469
- temp_dir = tempfile.mkdtemp()
470
- temp_path = os.path.join(temp_dir, file_name)
471
-
472
- try:
473
- # Save figure to temp file
474
- fig.savefig(temp_path, **kwargs)
475
-
476
- # Close figure to prevent memory leaks
477
- plt.close(fig)
478
-
479
- # Save using existing save() method
480
- original_file_path = self._file_path
481
- self._file_path = temp_path
482
-
483
- # Upload and get result
484
- result = self.save()
914
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
485
915
 
486
- # Restore original file_path
487
- self._file_path = original_file_path
488
-
489
- return result
490
- finally:
491
- # Clean up temp file and directory
492
- try:
493
- os.unlink(temp_path)
494
- os.rmdir(temp_dir)
495
- except Exception:
496
- pass
916
+ return self._save_fig(
917
+ fig=fig,
918
+ filename=to,
919
+ prefix=prefix,
920
+ description=self._description,
921
+ tags=self._tags,
922
+ metadata=self._metadata,
923
+ **kwargs
924
+ )
497
925
 
498
926
  def save_video(
499
927
  self,
500
928
  frame_stack: Union[List, Any],
501
- file_name: str,
929
+ *,
930
+ to: str,
502
931
  fps: int = 20,
503
932
  **imageio_kwargs
504
933
  ) -> Dict[str, Any]:
@@ -506,42 +935,17 @@ class FileBuilder:
506
935
  Save video frame stack to a file.
507
936
 
508
937
  Args:
509
- frame_stack: List of numpy arrays or stacked array (shape: [N, H, W] or [N, H, W, C])
510
- file_name: Name of file to create (extension determines format: .mp4, .gif, .avi, .webm)
938
+ frame_stack: List of numpy arrays or stacked array
939
+ to: Target filename
511
940
  fps: Frames per second (default: 20)
512
- **imageio_kwargs: Additional arguments passed to imageio.v3.imwrite():
513
- - codec: Video codec (e.g., 'libx264', 'h264')
514
- - quality: Quality level (int, higher is better)
515
- - pixelformat: Pixel format (e.g., 'yuv420p')
516
- - macro_block_size: Macro block size for encoding
941
+ **imageio_kwargs: Additional arguments passed to imageio
517
942
 
518
943
  Returns:
519
944
  File metadata dict with id, path, filename, checksum, etc.
520
945
 
521
- Raises:
522
- RuntimeError: If experiment not open or write-protected
523
- ImportError: If imageio or scikit-image not installed
524
- ValueError: If frame_stack is empty or invalid format
525
-
526
946
  Examples:
527
- import numpy as np
528
-
529
- # Grayscale frames (float values 0-1)
530
947
  frames = [np.random.rand(480, 640) for _ in range(30)]
531
- result = experiment.files(prefix="/videos").save_video(frames, "output.mp4")
532
-
533
- # RGB frames with custom FPS
534
- frames = [np.random.rand(480, 640, 3) for _ in range(60)]
535
- result = experiment.files(prefix="/videos").save_video(frames, "output.mp4", fps=30)
536
-
537
- # Save as GIF
538
- frames = [np.random.rand(200, 200) for _ in range(20)]
539
- result = experiment.files(prefix="/videos").save_video(frames, "animation.gif")
540
-
541
- # With custom codec and quality
542
- result = experiment.files(prefix="/videos").save_video(
543
- frames, "output.mp4", fps=30, codec='libx264', quality=8
544
- )
948
+ result = dxp.files("videos").save_video(frames, to="output.mp4")
545
949
  """
546
950
  import tempfile
547
951
  import os
@@ -556,51 +960,37 @@ class FileBuilder:
556
960
  except ImportError:
557
961
  raise ImportError("scikit-image is not installed. Install it with: pip install scikit-image")
558
962
 
559
- if not self._experiment._is_open:
560
- raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
561
-
562
- if self._experiment._write_protected:
563
- raise RuntimeError("Experiment is write-protected and cannot be modified.")
564
-
565
963
  # Validate frame_stack
566
964
  try:
567
- # Handle both list and numpy array
568
965
  if len(frame_stack) == 0:
569
966
  raise ValueError("frame_stack is empty")
570
967
  except TypeError:
571
968
  raise ValueError("frame_stack must be a list or numpy array")
572
969
 
573
- # Create temporary file with desired filename
970
+ prefix = '/' + self._path.lstrip('/') if self._path else self._prefix
971
+
574
972
  temp_dir = tempfile.mkdtemp()
575
- temp_path = os.path.join(temp_dir, file_name)
973
+ temp_path = os.path.join(temp_dir, to)
974
+ # Create parent directories if filename contains path
975
+ os.makedirs(os.path.dirname(temp_path), exist_ok=True)
576
976
 
577
977
  try:
578
- # Convert frames to uint8 format (handles float32/64, grayscale, RGB, etc.)
579
- # img_as_ubyte automatically scales [0.0, 1.0] floats to [0, 255] uint8
580
978
  frames_ubyte = img_as_ubyte(frame_stack)
581
-
582
- # Encode video to temp file
583
979
  try:
584
980
  iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
585
981
  except iio.core.NeedDownloadError:
586
- # Auto-download FFmpeg if not available
587
982
  import imageio.plugins.ffmpeg
588
983
  imageio.plugins.ffmpeg.download()
589
984
  iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
590
985
 
591
- # Save using existing save() method
592
- original_file_path = self._file_path
593
- self._file_path = temp_path
594
-
595
- # Upload and get result
596
- result = self.save()
597
-
598
- # Restore original file_path
599
- self._file_path = original_file_path
600
-
601
- return result
986
+ return self._save_file(
987
+ fpath=temp_path,
988
+ prefix=prefix,
989
+ description=self._description,
990
+ tags=self._tags,
991
+ metadata=self._metadata
992
+ )
602
993
  finally:
603
- # Clean up temp file and directory
604
994
  try:
605
995
  os.unlink(temp_path)
606
996
  os.rmdir(temp_dir)
@@ -611,31 +1001,16 @@ class FileBuilder:
611
1001
  """
612
1002
  Duplicate an existing file to a new path within the same experiment.
613
1003
 
614
- Useful for checkpoint rotation patterns where you save versioned checkpoints
615
- and maintain a "latest" or "best" pointer.
616
-
617
1004
  Args:
618
1005
  source: Source file - either file ID (str) or metadata dict with 'id' key
619
1006
  to: Target path like "models/latest.pt" or "/checkpoints/best.pt"
620
1007
 
621
1008
  Returns:
622
- File metadata dict for the duplicated file with id, path, filename, checksum, etc.
623
-
624
- Raises:
625
- RuntimeError: If experiment is not open or write-protected
626
- ValueError: If source file not found or target path invalid
1009
+ File metadata dict for the duplicated file
627
1010
 
628
1011
  Examples:
629
- # Using file ID
630
- dxp.files().duplicate("file-id-123", to="models/latest.pt")
631
-
632
- # Using metadata dict from save_torch
633
- snapshot = dxp.files(prefix="/models").save_torch(model, f"model_{epoch:05d}.pt")
1012
+ snapshot = dxp.files("models").save_torch(model, to=f"model_{epoch:05d}.pt")
634
1013
  dxp.files().duplicate(snapshot, to="models/latest.pt")
635
-
636
- # Checkpoint rotation pattern
637
- snap = dxp.files(prefix="/checkpoints").save_torch(model, f"model_{epoch:05d}.pt")
638
- dxp.files().duplicate(snap, to="checkpoints/best.pt")
639
1014
  """
640
1015
  import tempfile
641
1016
  import os
@@ -669,34 +1044,127 @@ class FileBuilder:
669
1044
  if not target_filename:
670
1045
  raise ValueError(f"Invalid target path '{to}': must include filename")
671
1046
 
672
- # Download source file to temp location
673
1047
  temp_dir = tempfile.mkdtemp()
674
1048
  temp_path = os.path.join(temp_dir, target_filename)
675
1049
 
676
1050
  try:
677
- # Download the source file
678
1051
  downloaded_path = self._experiment._download_file(
679
1052
  file_id=source_id,
680
1053
  dest_path=temp_path
681
1054
  )
682
1055
 
683
- # Save to new location using existing save() method
684
- original_file_path = self._file_path
685
- original_prefix = self._prefix
1056
+ return self._save_file(
1057
+ fpath=downloaded_path,
1058
+ prefix=target_prefix,
1059
+ description=self._description,
1060
+ tags=self._tags,
1061
+ metadata=self._metadata
1062
+ )
1063
+ finally:
1064
+ try:
1065
+ if os.path.exists(temp_path):
1066
+ os.unlink(temp_path)
1067
+ os.rmdir(temp_dir)
1068
+ except Exception:
1069
+ pass
1070
+
1071
+ def exists(self) -> bool:
1072
+ """
1073
+ Check if a file exists at the specified path.
1074
+
1075
+ Returns:
1076
+ True if file exists, False otherwise
1077
+
1078
+ Raises:
1079
+ RuntimeError: If experiment is not open
1080
+ ValueError: If no file path specified
1081
+
1082
+ Examples:
1083
+ # Check if file exists
1084
+ if dxp.files("models/checkpoint.pt").exists():
1085
+ print("File exists!")
1086
+
1087
+ # Check before downloading
1088
+ if not dxp.files("config.json").exists():
1089
+ raise FileNotFoundError("Config file missing")
1090
+ """
1091
+ if not self._experiment._is_open:
1092
+ raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
1093
+
1094
+ if not self._path:
1095
+ raise ValueError("No file path specified. Use: experiment.files('path/to/file').exists()")
1096
+
1097
+ # Try to find the file
1098
+ try:
1099
+ files = self._experiment._list_files(prefix=None, tags=None)
1100
+ search_path = self._path.lstrip('/')
1101
+
1102
+ for f in files:
1103
+ filename = f.get('filename', '')
1104
+ prefix = f.get('path', '/').lstrip('/')
1105
+ full_path = prefix.rstrip('/') + '/' + filename if prefix else filename
1106
+ full_path = full_path.lstrip('/')
1107
+
1108
+ # Check if this file matches (by full path or just filename)
1109
+ if full_path == search_path or filename == search_path:
1110
+ # Make sure it's not deleted
1111
+ if f.get('deletedAt') is None:
1112
+ return True
1113
+
1114
+ return False
1115
+ except Exception:
1116
+ return False
1117
+
1118
+ def read_text(self, encoding: str = 'utf-8') -> str:
1119
+ """
1120
+ Download file and read its content as text.
1121
+
1122
+ Args:
1123
+ encoding: Text encoding to use (default: 'utf-8')
1124
+
1125
+ Returns:
1126
+ File content as string
1127
+
1128
+ Raises:
1129
+ RuntimeError: If experiment is not open
1130
+ ValueError: If file not found or no path specified
1131
+ UnicodeDecodeError: If file cannot be decoded with specified encoding
1132
+
1133
+ Examples:
1134
+ # Read configuration file
1135
+ config_yaml = dxp.files("configs/view.yaml").read_text()
1136
+
1137
+ # Read log file
1138
+ logs = dxp.files("logs/training.log").read_text()
1139
+
1140
+ # Read with different encoding
1141
+ content = dxp.files("data.txt").read_text(encoding='latin-1')
1142
+ """
1143
+ import tempfile
1144
+ import os
1145
+
1146
+ if not self._experiment._is_open:
1147
+ raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
1148
+
1149
+ if not self._path:
1150
+ raise ValueError("No file path specified. Use: experiment.files('path/to/file').read_text()")
686
1151
 
687
- self._file_path = downloaded_path
688
- self._prefix = target_prefix
1152
+ # Create temporary file for download
1153
+ temp_dir = tempfile.mkdtemp()
1154
+ temp_filename = Path(self._path).name
1155
+ temp_path = os.path.join(temp_dir, temp_filename)
689
1156
 
690
- # Upload and get result
691
- result = self.save()
1157
+ try:
1158
+ # Download the file
1159
+ downloaded_path = self.download(to=temp_path)
692
1160
 
693
- # Restore original values
694
- self._file_path = original_file_path
695
- self._prefix = original_prefix
1161
+ # Read content as text
1162
+ with open(downloaded_path, 'r', encoding=encoding) as f:
1163
+ content = f.read()
696
1164
 
697
- return result
1165
+ return content
698
1166
  finally:
699
- # Clean up temp file and directory
1167
+ # Clean up temporary file
700
1168
  try:
701
1169
  if os.path.exists(temp_path):
702
1170
  os.unlink(temp_path)
@@ -705,6 +1173,249 @@ class FileBuilder:
705
1173
  pass
706
1174
 
707
1175
 
1176
+ class FilesAccessor:
1177
+ """
1178
+ Accessor that enables both callable and attribute-style access to file operations.
1179
+
1180
+ This allows:
1181
+ dxp.files("models") # Returns FileBuilder
1182
+ dxp.files(dir="models") # Keyword argument form
1183
+ experiment.files.upload(...) # Direct method call
1184
+ experiment.files.download(...) # Direct method call
1185
+ """
1186
+
1187
+ def __init__(self, experiment: 'Experiment'):
1188
+ self._experiment = experiment
1189
+ self._builder = FileBuilder(experiment)
1190
+
1191
+ def __call__(self, dir: Optional[str] = None, **kwargs) -> FileBuilder:
1192
+ """
1193
+ Create a FileBuilder with the given directory.
1194
+
1195
+ Supports flexible argument styles:
1196
+ - Positional: files("models")
1197
+ - Keyword: files(dir="models")
1198
+ - No args (root): files()
1199
+
1200
+ Args:
1201
+ dir: Directory/path for file operations
1202
+ **kwargs: Additional FileBuilder options
1203
+
1204
+ Returns:
1205
+ FileBuilder instance
1206
+ """
1207
+ return FileBuilder(self._experiment, path=dir, **kwargs)
1208
+
1209
+ # Direct methods that don't require a path first
1210
+
1211
+ def upload(
1212
+ self,
1213
+ fpath: str,
1214
+ *,
1215
+ to: Optional[str] = None,
1216
+ **kwargs
1217
+ ) -> Dict[str, Any]:
1218
+ """
1219
+ Upload a file directly without specifying a path prefix first.
1220
+
1221
+ Args:
1222
+ fpath: Path to existing file to upload (required)
1223
+ to: Optional destination path, e.g., "models/model.pt" or "renamed.pt"
1224
+ **kwargs: Additional FileBuilder options
1225
+
1226
+ Returns:
1227
+ File metadata dict with id, path, filename, checksum, etc.
1228
+
1229
+ Examples:
1230
+ # Upload with original filename
1231
+ experiment.files.upload("./model.pt")
1232
+
1233
+ # Upload with destination path
1234
+ experiment.files.upload("./model.pt", to="models/model.pt")
1235
+
1236
+ # Upload with metadata
1237
+ experiment.files.upload("./model.pt", to="best.pt", description="Best model")
1238
+ """
1239
+ # Parse 'to' to extract prefix and filename if provided
1240
+ if to:
1241
+ to_path = to.lstrip('/')
1242
+ if '/' in to_path:
1243
+ prefix, filename = to_path.rsplit('/', 1)
1244
+ prefix = '/' + prefix
1245
+ else:
1246
+ prefix = '/'
1247
+ filename = to_path
1248
+ return FileBuilder(self._experiment, path=prefix, **kwargs).upload(fpath, to=filename)
1249
+
1250
+ # No prefix, just upload with original or specified filename
1251
+ return FileBuilder(self._experiment, **kwargs).upload(fpath)
1252
+
1253
+ def download(
1254
+ self,
1255
+ path: str,
1256
+ *,
1257
+ to: Optional[str] = None
1258
+ ) -> Union[str, List[str]]:
1259
+ """
1260
+ Download file(s) directly.
1261
+
1262
+ Examples:
1263
+ experiment.files.download("model.pt")
1264
+ experiment.files.download("images/*.png", to="local_images")
1265
+ """
1266
+ path = path.lstrip('/')
1267
+
1268
+ # Check if path contains glob pattern
1269
+ if '*' in path or '?' in path:
1270
+ # Extract prefix and pattern
1271
+ if '/' in path:
1272
+ parts = path.split('/')
1273
+ # Find where the pattern starts
1274
+ prefix_parts = []
1275
+ pattern_parts = []
1276
+ in_pattern = False
1277
+ for part in parts:
1278
+ if '*' in part or '?' in part:
1279
+ in_pattern = True
1280
+ if in_pattern:
1281
+ pattern_parts.append(part)
1282
+ else:
1283
+ prefix_parts.append(part)
1284
+
1285
+ prefix = '/'.join(prefix_parts) if prefix_parts else None
1286
+ pattern = '/'.join(pattern_parts)
1287
+ else:
1288
+ prefix = None
1289
+ pattern = path
1290
+
1291
+ return FileBuilder(self._experiment, path=prefix).download(pattern, to=to)
1292
+
1293
+ # Single file download
1294
+ return FileBuilder(self._experiment, path=path).download(to=to)
1295
+
1296
+ def delete(self, path: str) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
1297
+ """
1298
+ Delete file(s) directly.
1299
+
1300
+ Examples:
1301
+ experiment.files.delete("some.text")
1302
+ experiment.files.delete("images/*.png")
1303
+ """
1304
+ path = path.lstrip('/')
1305
+
1306
+ # Check if path contains glob pattern
1307
+ if '*' in path or '?' in path:
1308
+ # Extract prefix and pattern
1309
+ if '/' in path:
1310
+ parts = path.split('/')
1311
+ prefix_parts = []
1312
+ pattern_parts = []
1313
+ in_pattern = False
1314
+ for part in parts:
1315
+ if '*' in part or '?' in part:
1316
+ in_pattern = True
1317
+ if in_pattern:
1318
+ pattern_parts.append(part)
1319
+ else:
1320
+ prefix_parts.append(part)
1321
+
1322
+ prefix = '/'.join(prefix_parts) if prefix_parts else None
1323
+ pattern = '/'.join(pattern_parts)
1324
+ else:
1325
+ prefix = None
1326
+ pattern = path
1327
+
1328
+ return FileBuilder(self._experiment, path=prefix).delete(pattern)
1329
+
1330
+ # Single file delete
1331
+ return FileBuilder(self._experiment, path=path).delete()
1332
+
1333
+ def list(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
1334
+ """
1335
+ List files directly.
1336
+
1337
+ Examples:
1338
+ files = experiment.files.list()
1339
+ files = experiment.files.list("*.pt")
1340
+ """
1341
+ return FileBuilder(self._experiment).list(pattern)
1342
+
1343
+ def save_text(self, content: str, *, to: str) -> Dict[str, Any]:
1344
+ """
1345
+ Save text content to a file.
1346
+
1347
+ Examples:
1348
+ experiment.files.save_text("content", to="view.yaml")
1349
+ """
1350
+ to_path = to.lstrip('/')
1351
+ if '/' in to_path:
1352
+ prefix, filename = to_path.rsplit('/', 1)
1353
+ prefix = '/' + prefix
1354
+ else:
1355
+ prefix = '/'
1356
+ filename = to_path
1357
+ return FileBuilder(self._experiment, path=prefix).save_text(content, to=filename)
1358
+
1359
+ def save_json(self, content: Any, *, to: str) -> Dict[str, Any]:
1360
+ """
1361
+ Save JSON content to a file.
1362
+
1363
+ Examples:
1364
+ experiment.files.save_json({"key": "value"}, to="config.json")
1365
+ """
1366
+ to_path = to.lstrip('/')
1367
+ if '/' in to_path:
1368
+ prefix, filename = to_path.rsplit('/', 1)
1369
+ prefix = '/' + prefix
1370
+ else:
1371
+ prefix = '/'
1372
+ filename = to_path
1373
+ return FileBuilder(self._experiment, path=prefix).save_json(content, to=filename)
1374
+
1375
+ def save_blob(self, data: bytes, *, to: str) -> Dict[str, Any]:
1376
+ """
1377
+ Save binary data to a file.
1378
+
1379
+ Examples:
1380
+ experiment.files.save_blob(b"data", to="data.bin")
1381
+ """
1382
+ to_path = to.lstrip('/')
1383
+ if '/' in to_path:
1384
+ prefix, filename = to_path.rsplit('/', 1)
1385
+ prefix = '/' + prefix
1386
+ else:
1387
+ prefix = '/'
1388
+ filename = to_path
1389
+ return FileBuilder(self._experiment, path=prefix).save_blob(data, to=filename)
1390
+
1391
+
1392
+ class BindrsBuilder:
1393
+ """
1394
+ Fluent interface for bindr (collection) operations.
1395
+
1396
+ Usage:
1397
+ file_paths = experiment.bindrs("some-bindr").list()
1398
+ """
1399
+
1400
+ def __init__(self, experiment: 'Experiment', bindr_name: str):
1401
+ self._experiment = experiment
1402
+ self._bindr_name = bindr_name
1403
+
1404
+ def list(self) -> List[Dict[str, Any]]:
1405
+ """
1406
+ List files in this bindr.
1407
+
1408
+ Returns:
1409
+ List of file metadata dicts belonging to this bindr
1410
+ """
1411
+ if not self._experiment._is_open:
1412
+ raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
1413
+
1414
+ # Get all files and filter by bindr
1415
+ all_files = self._experiment._list_files(prefix=None, tags=None)
1416
+ return [f for f in all_files if self._bindr_name in f.get('bindrs', [])]
1417
+
1418
+
708
1419
  def compute_sha256(file_path: str) -> str:
709
1420
  """
710
1421
  Compute SHA256 checksum of a file.