ml-dash 0.6.3__tar.gz → 0.6.10__tar.gz

{ml_dash-0.6.3 → ml_dash-0.6.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ml-dash
-Version: 0.6.3
+Version: 0.6.10
 Summary: ML experiment tracking and data storage
 Keywords: machine-learning,experiment-tracking,mlops,data-storage
 Author: Ge Yang, Tom Tao
@@ -68,10 +68,11 @@ Description-Content-Type: text/markdown
 
 # ML-Dash
 
-A simple and flexible SDK for ML experiment tracking and data storage.
+A simple and flexible SDK for ML experiment tracking and data storage with background buffering for high-performance training.
 
 ## Features
 
+### Core Features
 - **Three Usage Styles**: Pre-configured singleton (dxp), context manager, or direct instantiation
 - **Dual Operation Modes**: Remote (API server) or local (filesystem)
 - **OAuth2 Authentication**: Secure device flow authentication for CLI and SDK
@@ -82,6 +83,13 @@ A simple and flexible SDK for ML experiment tracking and data storage.
 - **Rich Metadata**: Tags, bindrs, descriptions, and custom metadata support
 - **Simple API**: Minimal configuration, maximum flexibility
 
+### Performance Features (New in 0.6.7)
+- **Background Buffering**: Non-blocking I/O operations eliminate training interruptions
+- **Automatic Batching**: Time-based (5s) and size-based (100 items) flush triggers
+- **Track API**: Time-series data tracking for robotics, RL, and sequential experiments
+- **Numpy Image Support**: Direct saving of numpy arrays as PNG/JPEG images
+- **Parallel Uploads**: ThreadPoolExecutor for efficient file uploads
+
 ## Installation
 
 <table>
@@ -93,14 +101,14 @@ A simple and flexible SDK for ML experiment tracking and data storage.
 <td>
 
 ```bash
-uv add ml-dash==0.6.2rc1
+uv add ml-dash
 ```
 
 </td>
 <td>
 
 ```bash
-pip install ml-dash==0.6.2rc1
+pip install ml-dash
 ```
 
 </td>
@@ -159,7 +167,75 @@ with Experiment(
 
 ```
 
-See [docs/getting-started.md](docs/getting-started.md) for more examples.
+## New Features in 0.6.7
+
+### 🚀 Background Buffering (Non-blocking I/O)
+
+All write operations are now buffered and executed in background threads:
+
+```python
+with Experiment("my-project/exp").run as experiment:
+    for i in range(10000):
+        # Non-blocking! Returns immediately
+        experiment.log(f"Step {i}")
+        experiment.metrics("train").log(loss=loss, accuracy=acc)
+        experiment.files("frames").save_image(frame, to=f"frame_{i}.jpg")
+
+    # All data automatically flushed when context exits
+```
+
+Configure buffering via environment variables:
+```bash
+export ML_DASH_BUFFER_ENABLED=true
+export ML_DASH_FLUSH_INTERVAL=5.0
+export ML_DASH_LOG_BATCH_SIZE=100
+```
+
+### 📊 Track API (Time-Series Data)
+
+Perfect for robotics, RL, and sequential experiments:
+
+```python
+with Experiment("robotics/training").run as experiment:
+    for step in range(1000):
+        # Track robot position over time
+        experiment.track("robot/position").append({
+            "step": step,
+            "x": position[0],
+            "y": position[1],
+            "z": position[2]
+        })
+
+        # Track control signals
+        experiment.track("robot/control").append({
+            "step": step,
+            "motor1": ctrl[0],
+            "motor2": ctrl[1]
+        })
+```
+
+### 🖼️ Numpy Image Support
+
+Save numpy arrays directly as images (PNG/JPEG):
+
+```python
+import numpy as np
+
+with Experiment("vision/training").run as experiment:
+    # From MuJoCo, OpenCV, PIL, etc.
+    pixels = renderer.render()  # numpy array
+
+    # Save as PNG (lossless)
+    experiment.files("frames").save_image(pixels, to="frame.png")
+
+    # Save as JPEG with quality control
+    experiment.files("frames").save_image(pixels, to="frame.jpg", quality=85)
+
+    # Auto-detection also works
+    experiment.files("frames").save(pixels, to="frame.jpg")
+```
+
+See [CHANGELOG.md](CHANGELOG.md) for complete release notes.
 
 ## Development Setup
 

{ml_dash-0.6.3 → ml_dash-0.6.10}/README.md

@@ -1,9 +1,10 @@
 # ML-Dash
 
-A simple and flexible SDK for ML experiment tracking and data storage.
+A simple and flexible SDK for ML experiment tracking and data storage with background buffering for high-performance training.
 
 ## Features
 
+### Core Features
 - **Three Usage Styles**: Pre-configured singleton (dxp), context manager, or direct instantiation
 - **Dual Operation Modes**: Remote (API server) or local (filesystem)
 - **OAuth2 Authentication**: Secure device flow authentication for CLI and SDK
@@ -14,6 +15,13 @@ A simple and flexible SDK for ML experiment tracking and data storage.
 - **Rich Metadata**: Tags, bindrs, descriptions, and custom metadata support
 - **Simple API**: Minimal configuration, maximum flexibility
 
+### Performance Features (New in 0.6.7)
+- **Background Buffering**: Non-blocking I/O operations eliminate training interruptions
+- **Automatic Batching**: Time-based (5s) and size-based (100 items) flush triggers
+- **Track API**: Time-series data tracking for robotics, RL, and sequential experiments
+- **Numpy Image Support**: Direct saving of numpy arrays as PNG/JPEG images
+- **Parallel Uploads**: ThreadPoolExecutor for efficient file uploads
+
 ## Installation
 
 <table>
@@ -25,14 +33,14 @@ A simple and flexible SDK for ML experiment tracking and data storage.
 <td>
 
 ```bash
-uv add ml-dash==0.6.2rc1
+uv add ml-dash
 ```
 
 </td>
 <td>
 
 ```bash
-pip install ml-dash==0.6.2rc1
+pip install ml-dash
 ```
 
 </td>
@@ -91,7 +99,75 @@ with Experiment(
 
 ```
 
-See [docs/getting-started.md](docs/getting-started.md) for more examples.
+## New Features in 0.6.7
+
+### 🚀 Background Buffering (Non-blocking I/O)
+
+All write operations are now buffered and executed in background threads:
+
+```python
+with Experiment("my-project/exp").run as experiment:
+    for i in range(10000):
+        # Non-blocking! Returns immediately
+        experiment.log(f"Step {i}")
+        experiment.metrics("train").log(loss=loss, accuracy=acc)
+        experiment.files("frames").save_image(frame, to=f"frame_{i}.jpg")
+
+    # All data automatically flushed when context exits
+```
+
+Configure buffering via environment variables:
+```bash
+export ML_DASH_BUFFER_ENABLED=true
+export ML_DASH_FLUSH_INTERVAL=5.0
+export ML_DASH_LOG_BATCH_SIZE=100
+```
+
+### 📊 Track API (Time-Series Data)
+
+Perfect for robotics, RL, and sequential experiments:
+
+```python
+with Experiment("robotics/training").run as experiment:
+    for step in range(1000):
+        # Track robot position over time
+        experiment.track("robot/position").append({
+            "step": step,
+            "x": position[0],
+            "y": position[1],
+            "z": position[2]
+        })
+
+        # Track control signals
+        experiment.track("robot/control").append({
+            "step": step,
+            "motor1": ctrl[0],
+            "motor2": ctrl[1]
+        })
+```
+
+### 🖼️ Numpy Image Support
+
+Save numpy arrays directly as images (PNG/JPEG):
+
+```python
+import numpy as np
+
+with Experiment("vision/training").run as experiment:
+    # From MuJoCo, OpenCV, PIL, etc.
+    pixels = renderer.render()  # numpy array
+
+    # Save as PNG (lossless)
+    experiment.files("frames").save_image(pixels, to="frame.png")
+
+    # Save as JPEG with quality control
+    experiment.files("frames").save_image(pixels, to="frame.jpg", quality=85)
+
+    # Auto-detection also works
+    experiment.files("frames").save(pixels, to="frame.jpg")
+```
+
+See [CHANGELOG.md](CHANGELOG.md) for complete release notes.
 
 ## Development Setup
 

{ml_dash-0.6.3 → ml_dash-0.6.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ml-dash"
-version = "0.6.3"
+version = "0.6.10"
 description = "ML experiment tracking and data storage"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -60,6 +60,9 @@ dev = [
     "mypy>=1.9.0",
 ]
 
+[tool.ruff]
+indent-width = 2
+
 [tool.uv]
 
 [build-system]

ml_dash-0.6.10/src/ml_dash/__init__.py

@@ -0,0 +1,105 @@
+"""
+ML-Dash Python SDK
+
+A simple and flexible SDK for ML experiment metricing and data storage.
+
+Prefix format: {owner}/{project}/path.../[name]
+  - owner: First segment (e.g., your username)
+  - project: Second segment (e.g., project name)
+  - path: Remaining segments form the folder structure
+  - name: Derived from last segment (may be a seed/id)
+
+Usage:
+
+    from ml_dash import Experiment
+
+    # Local mode - explicit configuration
+    with Experiment(
+        prefix="ge/my-project/experiments/exp1",
+        dash_root=".dash"
+    ).run as exp:
+        exp.log("Training started")
+        exp.params.set(lr=0.001)
+        exp.metrics("train").log(loss=0.5, step=0)
+
+    # Default: Remote mode (defaults to https://api.dash.ml)
+    with Experiment(prefix="ge/my-project/experiments/exp1").run as exp:
+        exp.log("Training started")
+        exp.params.set(lr=0.001)
+        exp.metrics("train").log(loss=0.5, step=0)
+
+    # Decorator style
+    from ml_dash import ml_dash_experiment
+
+    @ml_dash_experiment(prefix="ge/my-project/experiments/exp1")
+    def train_model(exp):
+        exp.log("Training started")
+"""
+
+from .client import RemoteClient, userinfo
+from .experiment import Experiment, OperationMode, ml_dash_experiment
+from .log import LogBuilder, LogLevel
+from .params import ParametersBuilder
+from .run import RUN
+from .storage import LocalStorage
+
+__version__ = "0.6.10"
+
+# Minimum version required - blocks older versions
+MINIMUM_REQUIRED_VERSION = "0.6.10"
+
+
+def _check_version_compatibility():
+    """
+    Enforce minimum version requirement.
+
+    Raises ImportError if installed version is below minimum required version.
+    This ensures users have the latest features (userinfo, namespace auto-detection, etc.)
+    """
+    try:
+        from packaging import version
+    except ImportError:
+        # If packaging is not available, skip check
+        # (unlikely since it's a common dependency)
+        return
+
+    current = version.parse(__version__)
+    minimum = version.parse(MINIMUM_REQUIRED_VERSION)
+
+    if current < minimum:
+        raise ImportError(
+            f"\n"
+            f"{'=' * 80}\n"
+            f"ERROR: ml-dash version {__version__} is too old!\n"
+            f"{'=' * 80}\n"
+            f"\n"
+            f"This version of ml-dash ({__version__}) is no longer supported.\n"
+            f"Minimum required version: {MINIMUM_REQUIRED_VERSION}\n"
+            f"\n"
+            f"Please upgrade to the latest version:\n"
+            f"\n"
+            f"  pip install --upgrade ml-dash\n"
+            f"\n"
+            f"Or install specific version:\n"
+            f"\n"
+            f"  pip install ml-dash>={MINIMUM_REQUIRED_VERSION}\n"
+            f"\n"
+            f"{'=' * 80}\n"
+        )
+
+
+# Enforce version check on import
+_check_version_compatibility()
+
+__all__ = [
+  "Experiment",
+  "ml_dash_experiment",
+  "OperationMode",
+  "RemoteClient",
+  "LocalStorage",
+  "LogLevel",
+  "LogBuilder",
+  "ParametersBuilder",
+  "RUN",
+  "userinfo",
+]

{ml_dash-0.6.3 → ml_dash-0.6.10}/src/ml_dash/auth/token_storage.py

@@ -292,12 +292,3 @@ def decode_jwt_payload(token: str) -> dict:
     return {}
 
 
-def get_jwt_user():
-  # Load token
-  storage = get_token_storage()
-  token = storage.load("ml-dash-token")
-
-  if token:
-    user = decode_jwt_payload(token)
-    return user
-  return None

{ml_dash-0.6.3 → ml_dash-0.6.10}/src/ml_dash/auto_start.py

@@ -31,21 +31,33 @@ import atexit
 # Token is auto-loaded from storage when first used
 # If not authenticated, operations will fail with AuthenticationError
 # Prefix format: {owner}/{project}/path...
-# Using getpass to get current user as owner for local convenience
 import getpass
 from datetime import datetime
 
-from .auth.token_storage import get_jwt_user
 from .experiment import Experiment
 
-_user = get_jwt_user()
-# Fallback to system username if not authenticated
-_username = _user["username"] if _user else getpass.getuser()
+# Get username for dxp namespace
+# Note: We use userinfo for fresh data (recommended approach)
+# Falls back to system username if not authenticated
+try:
+    from .client import userinfo
+    _username = userinfo.username or getpass.getuser()
+except Exception:
+    # If userinfo fails (e.g., no network), fall back to system user
+    _username = getpass.getuser()
+
 _now = datetime.now()
 
+# Create pre-configured singleton experiment in REMOTE mode
+# - dash_url=True: Use default remote server (https://api.dash.ml)
+# - dash_root=None: Remote-only mode (no local storage)
+# - user: Uses authenticated username from userinfo (fresh from server)
+# - Token is auto-loaded from storage when first used
+# - If not authenticated, operations will fail with AuthenticationError
 dxp = Experiment(
-  prefix=f"{_username}/scratch/{_now:%Y-%m-%d/%H%M%S}",
-  dash_url="https://api.dash.ml",
+    user=_username,      # Use authenticated username for namespace
+    dash_url=True,       # Use remote API (https://api.dash.ml)
+    dash_root=None,      # Remote-only mode (no local .dash/)
 )