ml-dash 0.5.2__tar.gz → 0.5.6__tar.gz

{ml_dash-0.5.2 → ml_dash-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ml-dash
-Version: 0.5.2
+Version: 0.5.6
 Summary: ML experiment tracking and data storage
 Keywords: machine-learning,experiment-tracking,mlops,data-storage
 Author: Ge Yang, Tom Tao
@@ -38,6 +38,9 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: pyjwt>=2.8.0
+Requires-Dist: imageio>=2.31.0
+Requires-Dist: imageio-ffmpeg>=0.4.9
+Requires-Dist: scikit-image>=0.21.0
 Requires-Dist: pytest>=8.0.0 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23.0 ; extra == 'dev'
 Requires-Dist: sphinx>=7.2.0 ; extra == 'dev'

{ml_dash-0.5.2 → ml_dash-0.5.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ml-dash"
-version = "0.5.2"
+version = "0.5.6"
 description = "ML experiment tracking and data storage"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,6 +26,9 @@ classifiers = [
 dependencies = [
     "httpx>=0.27.0",
     "pyjwt>=2.8.0",
+    "imageio>=2.31.0",
+    "imageio-ffmpeg>=0.4.9",
+    "scikit-image>=0.21.0",
 ]
 
 [project.optional-dependencies]

{ml_dash-0.5.2 → ml_dash-0.5.6}/src/ml_dash/experiment.py

@@ -460,7 +460,7 @@ class Experiment:
                 timestamp=log_entry["timestamp"]
             )
 
-    def file(self, **kwargs) -> FileBuilder:
+    def files(self, **kwargs) -> FileBuilder:
         """
         Get a FileBuilder for fluent file operations.
 
@@ -472,17 +472,17 @@ class Experiment:
 
         Examples:
             # Upload file
-            experiment.file(file_path="./model.pt", prefix="/models").save()
+            experiment.files(file_path="./model.pt", prefix="/models").save()
 
             # List files
-            files = experiment.file().list()
-            files = experiment.file(prefix="/models").list()
+            files = experiment.files().list()
+            files = experiment.files(prefix="/models").list()
 
             # Download file
-            experiment.file(file_id="123").download()
+            experiment.files(file_id="123").download()
 
             # Delete file
-            experiment.file(file_id="123").delete()
+            experiment.files(file_id="123").delete()
         """
         if not self._is_open:
             raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
@@ -791,7 +791,7 @@ class Experiment:
 
     def _append_to_metric(
         self,
-        name: str,
+        name: Optional[str],
         data: Dict[str, Any],
         description: Optional[str],
         tags: Optional[List[str]],
@@ -801,7 +801,7 @@ class Experiment:
         Internal method to append a single data point to a metric.
 
         Args:
-            name: Metric name
+            name: Metric name (can be None for unnamed metrics)
             data: Data point (flexible schema)
             description: Optional metric description
             tags: Optional tags
@@ -839,7 +839,7 @@ class Experiment:
 
     def _append_batch_to_metric(
         self,
-        name: str,
+        name: Optional[str],
         data_points: List[Dict[str, Any]],
         description: Optional[str],
         tags: Optional[List[str]],
@@ -849,7 +849,7 @@ class Experiment:
         Internal method to append multiple data points to a metric.
 
         Args:
-            name: Metric name
+            name: Metric name (can be None for unnamed metrics)
             data_points: List of data points
             description: Optional metric description
             tags: Optional tags

{ml_dash-0.5.2 → ml_dash-0.5.6}/src/ml_dash/files.py

@@ -6,7 +6,7 @@ Provides fluent API for file upload, download, list, and delete operations.
 
 import hashlib
 import mimetypes
-from typing import Dict, Any, List, Optional, TYPE_CHECKING
+from typing import Dict, Any, List, Optional, Union, TYPE_CHECKING
 from pathlib import Path
 
 if TYPE_CHECKING:
@@ -19,18 +19,18 @@ class FileBuilder:
 
     Usage:
         # Upload file
-        experiment.file(file_path="./model.pt", prefix="/models").save()
+        experiment.files(file_path="./model.pt", prefix="/models").save()
 
         # List files
-        files = experiment.file().list()
-        files = experiment.file(prefix="/models").list()
+        files = experiment.files().list()
+        files = experiment.files(prefix="/models").list()
 
         # Download file
-        experiment.file(file_id="123").download()
-        experiment.file(file_id="123", dest_path="./model.pt").download()
+        experiment.files(file_id="123").download()
+        experiment.files(file_id="123", dest_path="./model.pt").download()
 
         # Delete file
-        experiment.file(file_id="123").delete()
+        experiment.files(file_id="123").delete()
     """
 
     def __init__(self, experiment: 'Experiment', **kwargs):
@@ -72,7 +72,7 @@ class FileBuilder:
             ValueError: If file size exceeds 5GB limit
 
         Examples:
-            result = experiment.file(file_path="./model.pt", prefix="/models").save()
+            result = experiment.files(file_path="./model.pt", prefix="/models").save()
             # Returns: {"id": "123", "path": "/models", "filename": "model.pt", ...}
         """
         if not self._experiment._is_open:
@@ -130,9 +130,9 @@ class FileBuilder:
             RuntimeError: If experiment is not open
 
         Examples:
-            files = experiment.file().list()  # All files
-            files = experiment.file(prefix="/models").list()  # Filter by prefix
-            files = experiment.file(tags=["checkpoint"]).list()  # Filter by tags
+            files = experiment.files().list()  # All files
+            files = experiment.files(prefix="/models").list()  # Filter by prefix
+            files = experiment.files(tags=["checkpoint"]).list()  # Filter by tags
         """
         if not self._experiment._is_open:
             raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
@@ -158,10 +158,10 @@ class FileBuilder:
 
         Examples:
             # Download to current directory with original filename
-            path = experiment.file(file_id="123").download()
+            path = experiment.files(file_id="123").download()
 
             # Download to custom path
-            path = experiment.file(file_id="123", dest_path="./model.pt").download()
+            path = experiment.files(file_id="123", dest_path="./model.pt").download()
         """
         if not self._experiment._is_open:
             raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
@@ -186,7 +186,7 @@ class FileBuilder:
             ValueError: If file_id not provided
 
         Examples:
-            result = experiment.file(file_id="123").delete()
+            result = experiment.files(file_id="123").delete()
         """
         if not self._experiment._is_open:
             raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
@@ -211,7 +211,7 @@ class FileBuilder:
             ValueError: If file_id not provided
 
         Examples:
-            result = experiment.file(
+            result = experiment.files(
                 file_id="123",
                 description="Updated description",
                 tags=["new", "tags"],
@@ -251,7 +251,7 @@ class FileBuilder:
 
         Examples:
             config = {"model": "resnet50", "lr": 0.001}
-            result = experiment.file(prefix="/configs").save_json(config, "config.json")
+            result = experiment.files(prefix="/configs").save_json(config, "config.json")
         """
         import json
         import tempfile
@@ -263,11 +263,12 @@ class FileBuilder:
         if self._experiment._write_protected:
             raise RuntimeError("Experiment is write-protected and cannot be modified.")
 
-        # Create temporary file
-        temp_fd, temp_path = tempfile.mkstemp(suffix='.json', text=True)
+        # Create temporary file with desired filename
+        temp_dir = tempfile.mkdtemp()
+        temp_path = os.path.join(temp_dir, file_name)
         try:
             # Write JSON content to temp file
-            with os.fdopen(temp_fd, 'w') as f:
+            with open(temp_path, 'w') as f:
                 json.dump(content, f, indent=2)
 
             # Save using existing save() method
@@ -282,9 +283,10 @@ class FileBuilder:
 
             return result
         finally:
-            # Clean up temp file
+            # Clean up temp file and directory
             try:
                 os.unlink(temp_path)
+                os.rmdir(temp_dir)
             except Exception:
                 pass
 
@@ -307,10 +309,10 @@ class FileBuilder:
         Examples:
             import torch
             model = torch.nn.Linear(10, 5)
-            result = experiment.file(prefix="/models").save_torch(model, "model.pt")
+            result = experiment.files(prefix="/models").save_torch(model, "model.pt")
 
             # Or save state dict
-            result = experiment.file(prefix="/models").save_torch(model.state_dict(), "model.pth")
+            result = experiment.files(prefix="/models").save_torch(model.state_dict(), "model.pth")
         """
         import tempfile
         import os
@@ -326,9 +328,9 @@ class FileBuilder:
         if self._experiment._write_protected:
             raise RuntimeError("Experiment is write-protected and cannot be modified.")
 
-        # Create temporary file
-        temp_fd, temp_path = tempfile.mkstemp(suffix='.pt')
-        os.close(temp_fd)  # Close the file descriptor
+        # Create temporary file with desired filename
+        temp_dir = tempfile.mkdtemp()
+        temp_path = os.path.join(temp_dir, file_name)
 
         try:
             # Save model to temp file
@@ -346,9 +348,262 @@ class FileBuilder:
 
             return result
         finally:
-            # Clean up temp file
+            # Clean up temp file and directory
             try:
                 os.unlink(temp_path)
+                os.rmdir(temp_dir)
+            except Exception:
+                pass
+
+    def save_pkl(self, content: Any, file_name: str) -> Dict[str, Any]:
+        """
+        Save Python object to a pickle file.
+
+        Args:
+            content: Python object to pickle (must be pickle-serializable)
+            file_name: Name of the file to create (should end with .pkl or .pickle)
+
+        Returns:
+            File metadata dict with id, path, filename, checksum, etc.
+
+        Raises:
+            RuntimeError: If experiment is not open or write-protected
+            ValueError: If content cannot be pickled
+
+        Examples:
+            data = {"model": "resnet50", "weights": np.array([1, 2, 3])}
+            result = experiment.files(prefix="/data").save_pkl(data, "data.pkl")
+
+            # Or save any Python object
+            result = experiment.files(prefix="/models").save_pkl(trained_model, "model.pickle")
+        """
+        import pickle
+        import tempfile
+        import os
+
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
+
+        if self._experiment._write_protected:
+            raise RuntimeError("Experiment is write-protected and cannot be modified.")
+
+        # Create temporary file with desired filename
+        temp_dir = tempfile.mkdtemp()
+        temp_path = os.path.join(temp_dir, file_name)
+        try:
+            # Write pickled content to temp file
+            with open(temp_path, 'wb') as f:
+                pickle.dump(content, f)
+
+            # Save using existing save() method
+            original_file_path = self._file_path
+            self._file_path = temp_path
+
+            # Upload and get result
+            result = self.save()
+
+            # Restore original file_path
+            self._file_path = original_file_path
+
+            return result
+        finally:
+            # Clean up temp file and directory
+            try:
+                os.unlink(temp_path)
+                os.rmdir(temp_dir)
+            except Exception:
+                pass
+
+    def save_fig(self, fig: Optional[Any] = None, file_name: str = "figure.png", **kwargs) -> Dict[str, Any]:
+        """
+        Save matplotlib figure to a file.
+
+        Args:
+            fig: Matplotlib figure object. If None, uses plt.gcf() (current figure)
+            file_name: Name of file to create (extension determines format: .png, .pdf, .svg, .jpg)
+            **kwargs: Additional arguments passed to fig.savefig():
+                - dpi: Resolution (int or 'figure')
+                - transparent: Make background transparent (bool)
+                - bbox_inches: 'tight' to auto-crop (str or Bbox)
+                - quality: JPEG quality 0-100 (int)
+                - format: Override format detection (str)
+
+        Returns:
+            File metadata dict with id, path, filename, checksum, etc.
+
+        Raises:
+            RuntimeError: If experiment not open or write-protected
+            ImportError: If matplotlib not installed
+
+        Examples:
+            import matplotlib.pyplot as plt
+
+            # Use current figure
+            plt.plot([1, 2, 3], [1, 4, 9])
+            result = experiment.files(prefix="/plots").save_fig(file_name="plot.png")
+
+            # Specify figure explicitly
+            fig, ax = plt.subplots()
+            ax.plot([1, 2, 3])
+            result = experiment.files(prefix="/plots").save_fig(fig=fig, file_name="plot.pdf", dpi=150)
+        """
+        import tempfile
+        import os
+
+        try:
+            import matplotlib.pyplot as plt
+        except ImportError:
+            raise ImportError("Matplotlib is not installed. Install it with: pip install matplotlib")
+
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
+
+        if self._experiment._write_protected:
+            raise RuntimeError("Experiment is write-protected and cannot be modified.")
+
+        # Get figure
+        if fig is None:
+            fig = plt.gcf()
+
+        # Create temporary file with desired filename
+        temp_dir = tempfile.mkdtemp()
+        temp_path = os.path.join(temp_dir, file_name)
+
+        try:
+            # Save figure to temp file
+            fig.savefig(temp_path, **kwargs)
+
+            # Close figure to prevent memory leaks
+            plt.close(fig)
+
+            # Save using existing save() method
+            original_file_path = self._file_path
+            self._file_path = temp_path
+
+            # Upload and get result
+            result = self.save()
+
+            # Restore original file_path
+            self._file_path = original_file_path
+
+            return result
+        finally:
+            # Clean up temp file and directory
+            try:
+                os.unlink(temp_path)
+                os.rmdir(temp_dir)
+            except Exception:
+                pass
+
+    def save_video(
+        self,
+        frame_stack: Union[List, Any],
+        file_name: str,
+        fps: int = 20,
+        **imageio_kwargs
+    ) -> Dict[str, Any]:
+        """
+        Save video frame stack to a file.
+
+        Args:
+            frame_stack: List of numpy arrays or stacked array (shape: [N, H, W] or [N, H, W, C])
+            file_name: Name of file to create (extension determines format: .mp4, .gif, .avi, .webm)
+            fps: Frames per second (default: 20)
+            **imageio_kwargs: Additional arguments passed to imageio.v3.imwrite():
+                - codec: Video codec (e.g., 'libx264', 'h264')
+                - quality: Quality level (int, higher is better)
+                - pixelformat: Pixel format (e.g., 'yuv420p')
+                - macro_block_size: Macro block size for encoding
+
+        Returns:
+            File metadata dict with id, path, filename, checksum, etc.
+
+        Raises:
+            RuntimeError: If experiment not open or write-protected
+            ImportError: If imageio or scikit-image not installed
+            ValueError: If frame_stack is empty or invalid format
+
+        Examples:
+            import numpy as np
+
+            # Grayscale frames (float values 0-1)
+            frames = [np.random.rand(480, 640) for _ in range(30)]
+            result = experiment.files(prefix="/videos").save_video(frames, "output.mp4")
+
+            # RGB frames with custom FPS
+            frames = [np.random.rand(480, 640, 3) for _ in range(60)]
+            result = experiment.files(prefix="/videos").save_video(frames, "output.mp4", fps=30)
+
+            # Save as GIF
+            frames = [np.random.rand(200, 200) for _ in range(20)]
+            result = experiment.files(prefix="/videos").save_video(frames, "animation.gif")
+
+            # With custom codec and quality
+            result = experiment.files(prefix="/videos").save_video(
+                frames, "output.mp4", fps=30, codec='libx264', quality=8
+            )
+        """
+        import tempfile
+        import os
+
+        try:
+            import imageio.v3 as iio
+        except ImportError:
+            raise ImportError("imageio is not installed. Install it with: pip install imageio imageio-ffmpeg")
+
+        try:
+            from skimage import img_as_ubyte
+        except ImportError:
+            raise ImportError("scikit-image is not installed. Install it with: pip install scikit-image")
+
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.run.start() or context manager.")
+
+        if self._experiment._write_protected:
+            raise RuntimeError("Experiment is write-protected and cannot be modified.")
+
+        # Validate frame_stack
+        try:
+            # Handle both list and numpy array
+            if len(frame_stack) == 0:
+                raise ValueError("frame_stack is empty")
+        except TypeError:
+            raise ValueError("frame_stack must be a list or numpy array")
+
+        # Create temporary file with desired filename
+        temp_dir = tempfile.mkdtemp()
+        temp_path = os.path.join(temp_dir, file_name)
+
+        try:
+            # Convert frames to uint8 format (handles float32/64, grayscale, RGB, etc.)
+            # img_as_ubyte automatically scales [0.0, 1.0] floats to [0, 255] uint8
+            frames_ubyte = img_as_ubyte(frame_stack)
+
+            # Encode video to temp file
+            try:
+                iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
+            except iio.core.NeedDownloadError:
+                # Auto-download FFmpeg if not available
+                import imageio.plugins.ffmpeg
+                imageio.plugins.ffmpeg.download()
+                iio.imwrite(temp_path, frames_ubyte, fps=fps, **imageio_kwargs)
+
+            # Save using existing save() method
+            original_file_path = self._file_path
+            self._file_path = temp_path
+
+            # Upload and get result
+            result = self.save()
+
+            # Restore original file_path
+            self._file_path = original_file_path
+
+            return result
+        finally:
+            # Clean up temp file and directory
+            try:
+                os.unlink(temp_path)
+                os.rmdir(temp_dir)
             except Exception:
                 pass
 

{ml_dash-0.5.2 → ml_dash-0.5.6}/src/ml_dash/metric.py

@@ -15,16 +15,20 @@ class MetricsManager:
     """
     Manager for metric operations that supports both named and unnamed usage.
 
-    Supports two usage patterns:
-    1. Named: experiment.metrics("loss").append(value=0.5, step=1)
-    2. Unnamed: experiment.metrics.append(name="loss", value=0.5, step=1)
+    Supports three usage patterns:
+    1. Named via call: experiment.metrics("loss").append(value=0.5, step=1)
+    2. Named via argument: experiment.metrics.append(name="loss", value=0.5, step=1)
+    3. Unnamed: experiment.metrics.append(value=0.5, step=1)  # name=None
 
     Usage:
-        # With explicit metric name
+        # With explicit metric name (via call)
         experiment.metrics("train_loss").append(value=0.5, step=100)
 
-        # Without specifying name upfront (name in append call)
+        # With explicit metric name (via argument)
         experiment.metrics.append(name="train_loss", value=0.5, step=100)
+
+        # Without name (uses None as metric name)
+        experiment.metrics.append(value=0.5, step=100)
     """
 
     def __init__(self, experiment: 'Experiment'):
@@ -55,12 +59,12 @@ class MetricsManager:
         """
         return MetricBuilder(self._experiment, name, description, tags, metadata)
 
-    def append(self, name: str, data: Optional[Dict[str, Any]] = None, **kwargs) -> Dict[str, Any]:
+    def append(self, name: Optional[str] = None, data: Optional[Dict[str, Any]] = None, **kwargs) -> Dict[str, Any]:
         """
-        Append a data point to a metric (name specified in call).
+        Append a data point to a metric (name can be optional).
 
         Args:
-            name: Metric name
+            name: Metric name (optional, can be None for unnamed metrics)
             data: Data dict (alternative to kwargs)
             **kwargs: Data as keyword arguments
 
@@ -69,13 +73,14 @@ class MetricsManager:
 
         Examples:
             experiment.metrics.append(name="loss", value=0.5, step=1)
+            experiment.metrics.append(value=0.5, step=1)  # name=None
             experiment.metrics.append(name="loss", data={"value": 0.5, "step": 1})
         """
         if data is None:
             data = kwargs
         return self._experiment._append_to_metric(name, data, None, None, None)
 
-    def append_batch(self, name: str, data_points: List[Dict[str, Any]],
+    def append_batch(self, name: Optional[str] = None, data_points: Optional[List[Dict[str, Any]]] = None,
                      description: Optional[str] = None,
                      tags: Optional[List[str]] = None,
                      metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
@@ -83,7 +88,7 @@ class MetricsManager:
         Append multiple data points to a metric.
 
         Args:
-            name: Metric name
+            name: Metric name (optional, can be None for unnamed metrics)
             data_points: List of data point dicts
             description: Optional metric description
             tags: Optional tags for categorization
@@ -100,7 +105,15 @@ class MetricsManager:
                     {"value": 0.4, "step": 2}
                 ]
             )
+            experiment.metrics.append_batch(
+                data_points=[
+                    {"value": 0.5, "step": 1},
+                    {"value": 0.4, "step": 2}
+                ]
+            )  # name=None
         """
+        if data_points is None:
+            data_points = []
         return self._experiment._append_batch_to_metric(name, data_points, description, tags, metadata)
 
 

{ml_dash-0.5.2 → ml_dash-0.5.6}/src/ml_dash/storage.py

@@ -636,7 +636,7 @@ class LocalStorage:
         self,
         project: str,
         experiment: str,
-        metric_name: str,
+        metric_name: Optional[str],
         data: Dict[str, Any],
         description: Optional[str] = None,
         tags: Optional[List[str]] = None,
@@ -653,7 +653,7 @@ class LocalStorage:
         Args:
             project: Project name
             experiment: Experiment name
-            metric_name: Metric name
+            metric_name: Metric name (None for unnamed metrics)
             data: Data point (flexible schema)
             description: Optional metric description
             tags: Optional tags
@@ -666,7 +666,9 @@ class LocalStorage:
         metrics_dir = experiment_dir / "metrics"
         metrics_dir.mkdir(parents=True, exist_ok=True)
 
-        metric_dir = metrics_dir / metric_name
+        # Convert None to string for directory name
+        dir_name = str(metric_name) if metric_name is not None else "None"
+        metric_dir = metrics_dir / dir_name
         metric_dir.mkdir(exist_ok=True)
 
         data_file = metric_dir / "data.jsonl"
@@ -720,7 +722,7 @@ class LocalStorage:
         self,
         project: str,
         experiment: str,
-        metric_name: str,
+        metric_name: Optional[str],
         data_points: List[Dict[str, Any]],
         description: Optional[str] = None,
         tags: Optional[List[str]] = None,
@@ -732,7 +734,7 @@ class LocalStorage:
         Args:
             project: Project name
             experiment: Experiment name
-            metric_name: Metric name
+            metric_name: Metric name (None for unnamed metrics)
             data_points: List of data points
             description: Optional metric description
             tags: Optional tags
@@ -745,7 +747,9 @@ class LocalStorage:
         metrics_dir = experiment_dir / "metrics"
         metrics_dir.mkdir(parents=True, exist_ok=True)
 
-        metric_dir = metrics_dir / metric_name
+        # Convert None to string for directory name
+        dir_name = str(metric_name) if metric_name is not None else "None"
+        metric_dir = metrics_dir / dir_name
         metric_dir.mkdir(exist_ok=True)
 
         data_file = metric_dir / "data.jsonl"