ml-dash 0.0.17__py3-none-any.whl → 0.2.1__py3-none-any.whl

ml_dash/files.py

@@ -0,0 +1,313 @@
+"""
+Files module for ML-Dash SDK.
+
+Provides fluent API for file upload, download, list, and delete operations.
+"""
+
+import hashlib
+import mimetypes
+from typing import Dict, Any, List, Optional, TYPE_CHECKING
+from pathlib import Path
+
+if TYPE_CHECKING:
+    from .experiment import Experiment
+
+
+class FileBuilder:
+    """
+    Fluent interface for file operations.
+
+    Usage:
+        # Upload file
+        experiment.file(file_path="./model.pt", prefix="/models").save()
+
+        # List files
+        files = experiment.file().list()
+        files = experiment.file(prefix="/models").list()
+
+        # Download file
+        experiment.file(file_id="123").download()
+        experiment.file(file_id="123", dest_path="./model.pt").download()
+
+        # Delete file
+        experiment.file(file_id="123").delete()
+    """
+
+    def __init__(self, experiment: 'Experiment', **kwargs):
+        """
+        Initialize file builder.
+
+        Args:
+            experiment: Parent experiment instance
+            **kwargs: File operation parameters
+                - file_path: Path to file to upload
+                - prefix: Logical path prefix (default: "/")
+                - description: Optional description
+                - tags: Optional list of tags
+                - metadata: Optional metadata dict
+                - file_id: File ID for download/delete/update operations
+                - dest_path: Destination path for download
+        """
+        self._experiment = experiment
+        self._file_path = kwargs.get('file_path')
+        self._prefix = kwargs.get('prefix', '/')
+        self._description = kwargs.get('description')
+        self._tags = kwargs.get('tags', [])
+        self._metadata = kwargs.get('metadata')
+        self._file_id = kwargs.get('file_id')
+        self._dest_path = kwargs.get('dest_path')
+
+    def save(self) -> Dict[str, Any]:
+        """
+        Upload and save the file.
+
+        Returns:
+            File metadata dict with id, path, filename, checksum, etc.
+
+        Raises:
+            RuntimeError: If experiment is not open or write-protected
+            ValueError: If file_path not provided or file doesn't exist
+            ValueError: If file size exceeds 5GB limit
+
+        Examples:
+            result = experiment.file(file_path="./model.pt", prefix="/models").save()
+            # Returns: {"id": "123", "path": "/models", "filename": "model.pt", ...}
+        """
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
+
+        if self._experiment.write_protected:
+            raise RuntimeError("Experiment is write-protected and cannot be modified.")
+
+        if not self._file_path:
+            raise ValueError("file_path is required for save() operation")
+
+        file_path = Path(self._file_path)
+        if not file_path.exists():
+            raise ValueError(f"File not found: {self._file_path}")
+
+        if not file_path.is_file():
+            raise ValueError(f"Path is not a file: {self._file_path}")
+
+        # Check file size (max 5GB)
+        file_size = file_path.stat().st_size
+        MAX_FILE_SIZE = 5 * 1024 * 1024 * 1024  # 5GB in bytes
+        if file_size > MAX_FILE_SIZE:
+            raise ValueError(f"File size ({file_size} bytes) exceeds 5GB limit")
+
+        # Compute checksum
+        checksum = compute_sha256(str(file_path))
+
+        # Detect MIME type
+        content_type = get_mime_type(str(file_path))
+
+        # Get filename
+        filename = file_path.name
+
+        # Upload through experiment
+        return self._experiment._upload_file(
+            file_path=str(file_path),
+            prefix=self._prefix,
+            filename=filename,
+            description=self._description,
+            tags=self._tags,
+            metadata=self._metadata,
+            checksum=checksum,
+            content_type=content_type,
+            size_bytes=file_size
+        )
+
+    def list(self) -> List[Dict[str, Any]]:
+        """
+        List files with optional filters.
+
+        Returns:
+            List of file metadata dicts
+
+        Raises:
+            RuntimeError: If experiment is not open
+
+        Examples:
+            files = experiment.file().list()  # All files
+            files = experiment.file(prefix="/models").list()  # Filter by prefix
+            files = experiment.file(tags=["checkpoint"]).list()  # Filter by tags
+        """
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
+
+        return self._experiment._list_files(
+            prefix=self._prefix if self._prefix != '/' else None,
+            tags=self._tags if self._tags else None
+        )
+
+    def download(self) -> str:
+        """
+        Download file with automatic checksum verification.
+
+        If dest_path not provided, downloads to current directory with original filename.
+
+        Returns:
+            Path to downloaded file
+
+        Raises:
+            RuntimeError: If experiment is not open
+            ValueError: If file_id not provided
+            ValueError: If checksum verification fails
+
+        Examples:
+            # Download to current directory with original filename
+            path = experiment.file(file_id="123").download()
+
+            # Download to custom path
+            path = experiment.file(file_id="123", dest_path="./model.pt").download()
+        """
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
+
+        if not self._file_id:
+            raise ValueError("file_id is required for download() operation")
+
+        return self._experiment._download_file(
+            file_id=self._file_id,
+            dest_path=self._dest_path
+        )
+
+    def delete(self) -> Dict[str, Any]:
+        """
+        Delete file (soft delete).
+
+        Returns:
+            Dict with id and deletedAt timestamp
+
+        Raises:
+            RuntimeError: If experiment is not open or write-protected
+            ValueError: If file_id not provided
+
+        Examples:
+            result = experiment.file(file_id="123").delete()
+        """
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
+
+        if self._experiment.write_protected:
+            raise RuntimeError("Experiment is write-protected and cannot be modified.")
+
+        if not self._file_id:
+            raise ValueError("file_id is required for delete() operation")
+
+        return self._experiment._delete_file(file_id=self._file_id)
+
+    def update(self) -> Dict[str, Any]:
+        """
+        Update file metadata (description, tags, metadata).
+
+        Returns:
+            Updated file metadata dict
+
+        Raises:
+            RuntimeError: If experiment is not open or write-protected
+            ValueError: If file_id not provided
+
+        Examples:
+            result = experiment.file(
+                file_id="123",
+                description="Updated description",
+                tags=["new", "tags"],
+                metadata={"updated": True}
+            ).update()
+        """
+        if not self._experiment._is_open:
+            raise RuntimeError("Experiment not open. Use experiment.open() or context manager.")
+
+        if self._experiment.write_protected:
+            raise RuntimeError("Experiment is write-protected and cannot be modified.")
+
+        if not self._file_id:
+            raise ValueError("file_id is required for update() operation")
+
+        return self._experiment._update_file(
+            file_id=self._file_id,
+            description=self._description,
+            tags=self._tags,
+            metadata=self._metadata
+        )
+
+
+def compute_sha256(file_path: str) -> str:
+    """
+    Compute SHA256 checksum of a file.
+
+    Args:
+        file_path: Path to file
+
+    Returns:
+        Hex-encoded SHA256 checksum
+
+    Examples:
+        checksum = compute_sha256("./model.pt")
+        # Returns: "abc123def456..."
+    """
+    sha256_hash = hashlib.sha256()
+
+    with open(file_path, "rb") as f:
+        # Read file in chunks to handle large files
+        for byte_block in iter(lambda: f.read(8192), b""):
+            sha256_hash.update(byte_block)
+
+    return sha256_hash.hexdigest()
+
+
+def get_mime_type(file_path: str) -> str:
+    """
+    Detect MIME type of a file.
+
+    Args:
+        file_path: Path to file
+
+    Returns:
+        MIME type string (default: "application/octet-stream")
+
+    Examples:
+        mime_type = get_mime_type("./model.pt")
+        # Returns: "application/octet-stream"
+
+        mime_type = get_mime_type("./image.png")
+        # Returns: "image/png"
+    """
+    mime_type, _ = mimetypes.guess_type(file_path)
+    return mime_type or "application/octet-stream"
+
+
+def verify_checksum(file_path: str, expected_checksum: str) -> bool:
+    """
+    Verify SHA256 checksum of a file.
+
+    Args:
+        file_path: Path to file
+        expected_checksum: Expected SHA256 checksum (hex-encoded)
+
+    Returns:
+        True if checksum matches, False otherwise
+
+    Examples:
+        is_valid = verify_checksum("./model.pt", "abc123...")
+    """
+    actual_checksum = compute_sha256(file_path)
+    return actual_checksum == expected_checksum
+
+
+def generate_snowflake_id() -> str:
+    """
+    Generate a simple Snowflake-like ID for local mode.
+
+    Not a true Snowflake ID, but provides unique IDs for local storage.
+
+    Returns:
+        String representation of generated ID
+    """
+    import time
+    import random
+
+    timestamp = int(time.time() * 1000)
+    random_bits = random.randint(0, 4095)
+    return str((timestamp << 12) | random_bits)

ml_dash/log.py

@@ -0,0 +1,181 @@
+"""
+Log API for ML-Dash SDK.
+
+Provides fluent interface for structured logging with validation.
+"""
+
+from typing import Optional, Dict, Any, TYPE_CHECKING
+from datetime import datetime
+from enum import Enum
+
+if TYPE_CHECKING:
+    from .experiment import Experiment
+
+
+class LogLevel(Enum):
+    """
+    Standard log levels for ML-Dash.
+
+    Supported levels:
+    - INFO: Informational messages
+    - WARN: Warning messages
+    - ERROR: Error messages
+    - DEBUG: Debug messages
+    - FATAL: Fatal error messages
+    """
+    INFO = "info"
+    WARN = "warn"
+    ERROR = "error"
+    DEBUG = "debug"
+    FATAL = "fatal"
+
+    @classmethod
+    def validate(cls, level: str) -> str:
+        """
+        Validate and normalize log level.
+
+        Args:
+            level: Log level string (case-insensitive)
+
+        Returns:
+            Normalized log level string (lowercase)
+
+        Raises:
+            ValueError: If level is not one of the 5 standard levels
+
+        Example:
+            >>> LogLevel.validate("INFO")
+            "info"
+            >>> LogLevel.validate("invalid")
+            ValueError: Invalid log level: 'invalid'. Must be one of: info, warn, error, debug, fatal
+        """
+        level_lower = level.lower()
+        try:
+            return cls[level_lower.upper()].value
+        except KeyError:
+            valid_levels = ", ".join([l.value for l in cls])
+            raise ValueError(
+                f"Invalid log level: '{level}'. "
+                f"Must be one of: {valid_levels}"
+            )
+
+
+class LogBuilder:
+    """
+    Fluent builder for creating log entries.
+
+    This class is returned by Experiment.log() when no message is provided.
+    It allows for a fluent API style where metadata is set first, then
+    the log level method is called to write the log.
+
+    Example:
+        experiment.log(metadata={"epoch": 1}).info("Training started")
+        experiment.log().error("Failed", error_code=500)
+    """
+
+    def __init__(self, experiment: 'Experiment', metadata: Optional[Dict[str, Any]] = None):
+        """
+        Initialize LogBuilder.
+
+        Args:
+            experiment: Parent Experiment instance
+            metadata: Optional metadata dict from log() call
+        """
+        self._experiment = experiment
+        self._metadata = metadata
+
+    def info(self, message: str, **extra_metadata) -> None:
+        """
+        Write info level log.
+
+        Args:
+            message: Log message
+            **extra_metadata: Additional metadata as keyword arguments
+
+        Example:
+            experiment.log().info("Training started")
+            experiment.log().info("Epoch complete", epoch=1, loss=0.5)
+        """
+        self._write(LogLevel.INFO.value, message, extra_metadata)
+
+    def warn(self, message: str, **extra_metadata) -> None:
+        """
+        Write warning level log.
+
+        Args:
+            message: Log message
+            **extra_metadata: Additional metadata as keyword arguments
+
+        Example:
+            experiment.log().warn("High loss detected", loss=1.5)
+        """
+        self._write(LogLevel.WARN.value, message, extra_metadata)
+
+    def error(self, message: str, **extra_metadata) -> None:
+        """
+        Write error level log.
+
+        Args:
+            message: Log message
+            **extra_metadata: Additional metadata as keyword arguments
+
+        Example:
+            experiment.log().error("Failed to save", path="/models/checkpoint.pth")
+        """
+        self._write(LogLevel.ERROR.value, message, extra_metadata)
+
+    def debug(self, message: str, **extra_metadata) -> None:
+        """
+        Write debug level log.
+
+        Args:
+            message: Log message
+            **extra_metadata: Additional metadata as keyword arguments
+
+        Example:
+            experiment.log().debug("Memory usage", memory_mb=2500)
+        """
+        self._write(LogLevel.DEBUG.value, message, extra_metadata)
+
+    def fatal(self, message: str, **extra_metadata) -> None:
+        """
+        Write fatal level log.
+
+        Args:
+            message: Log message
+            **extra_metadata: Additional metadata as keyword arguments
+
+        Example:
+            experiment.log().fatal("Unrecoverable error", exit_code=1)
+        """
+        self._write(LogLevel.FATAL.value, message, extra_metadata)
+
+    def _write(self, level: str, message: str, extra_metadata: Dict[str, Any]) -> None:
+        """
+        Internal: Execute the actual log write.
+
+        Merges metadata from log() call with metadata from level method,
+        then writes immediately (no buffering).
+
+        Args:
+            level: Log level (already validated)
+            message: Log message
+            extra_metadata: Additional metadata from level method kwargs
+        """
+        # Merge metadata from log() call with metadata from level method
+        if self._metadata and extra_metadata:
+            final_metadata = {**self._metadata, **extra_metadata}
+        elif self._metadata:
+            final_metadata = self._metadata
+        elif extra_metadata:
+            final_metadata = extra_metadata
+        else:
+            final_metadata = None
+
+        # Write immediately (no buffering)
+        self._experiment._write_log(
+            message=message,
+            level=level,
+            metadata=final_metadata,
+            timestamp=None
+        )

ml_dash/metric.py

@@ -0,0 +1,186 @@
+"""
+Metric API - Time-series data metricing for ML experiments.
+
+Metrics are used for storing continuous data series like training metrics,
+validation losses, system measurements, etc.
+"""
+
+from typing import Dict, Any, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .experiment import Experiment
+
+
+class MetricBuilder:
+    """
+    Builder for metric operations.
+
+    Provides fluent API for appending, reading, and querying metric data.
+
+    Usage:
+        # Append single data point
+        experiment.metric(name="train_loss").append(value=0.5, step=100)
+
+        # Append batch
+        experiment.metric(name="train_loss").append_batch([
+            {"value": 0.5, "step": 100},
+            {"value": 0.45, "step": 101}
+        ])
+
+        # Read data
+        data = experiment.metric(name="train_loss").read(start_index=0, limit=100)
+
+        # Get statistics
+        stats = experiment.metric(name="train_loss").stats()
+    """
+
+    def __init__(self, experiment: 'Experiment', name: str, description: Optional[str] = None,
+                 tags: Optional[List[str]] = None, metadata: Optional[Dict[str, Any]] = None):
+        """
+        Initialize MetricBuilder.
+
+        Args:
+            experiment: Parent Experiment instance
+            name: Metric name (unique within experiment)
+            description: Optional metric description
+            tags: Optional tags for categorization
+            metadata: Optional structured metadata (units, type, etc.)
+        """
+        self._experiment = experiment
+        self._name = name
+        self._description = description
+        self._tags = tags
+        self._metadata = metadata
+
+    def append(self, **kwargs) -> 'MetricBuilder':
+        """
+        Append a single data point to the metric.
+
+        The data point can have any structure - common patterns:
+        - {value: 0.5, step: 100}
+        - {loss: 0.3, accuracy: 0.92, epoch: 5}
+        - {timestamp: "...", temperature: 25.5, humidity: 60}
+
+        Args:
+            **kwargs: Data point fields (flexible schema)
+
+        Returns:
+            Dict with metricId, index, bufferedDataPoints, chunkSize
+
+        Example:
+            result = experiment.metric(name="train_loss").append(value=0.5, step=100, epoch=1)
+            print(f"Appended at index {result['index']}")
+        """
+        result = self._experiment._append_to_metric(
+            name=self._name,
+            data=kwargs,
+            description=self._description,
+            tags=self._tags,
+            metadata=self._metadata
+        )
+        return result
+
+    def append_batch(self, data_points: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """
+        Append multiple data points in batch (more efficient than multiple append calls).
+
+        Args:
+            data_points: List of data point dicts
+
+        Returns:
+            Dict with metricId, startIndex, endIndex, count, bufferedDataPoints, chunkSize
+
+        Example:
+            result = experiment.metric(name="metrics").append_batch([
+                {"loss": 0.5, "acc": 0.8, "step": 1},
+                {"loss": 0.4, "acc": 0.85, "step": 2},
+                {"loss": 0.3, "acc": 0.9, "step": 3}
+            ])
+            print(f"Appended {result['count']} points")
+        """
+        if not data_points:
+            raise ValueError("data_points cannot be empty")
+
+        result = self._experiment._append_batch_to_metric(
+            name=self._name,
+            data_points=data_points,
+            description=self._description,
+            tags=self._tags,
+            metadata=self._metadata
+        )
+        return result
+
+    def read(self, start_index: int = 0, limit: int = 1000) -> Dict[str, Any]:
+        """
+        Read data points from the metric by index range.
+
+        Args:
+            start_index: Starting index (inclusive, default 0)
+            limit: Maximum number of points to read (default 1000, max 10000)
+
+        Returns:
+            Dict with keys:
+            - data: List of {index: str, data: dict, createdAt: str}
+            - startIndex: Starting index
+            - endIndex: Ending index
+            - total: Number of points returned
+            - hasMore: Whether more data exists beyond this range
+
+        Example:
+            result = experiment.metric(name="train_loss").read(start_index=0, limit=100)
+            for point in result['data']:
+                print(f"Index {point['index']}: {point['data']}")
+        """
+        return self._experiment._read_metric_data(
+            name=self._name,
+            start_index=start_index,
+            limit=limit
+        )
+
+    def stats(self) -> Dict[str, Any]:
+        """
+        Get metric statistics and metadata.
+
+        Returns:
+            Dict with metric info:
+            - metricId: Unique metric ID
+            - name: Metric name
+            - description: Metric description (if set)
+            - tags: Tags list
+            - metadata: User metadata
+            - totalDataPoints: Total points (buffered + chunked)
+            - bufferedDataPoints: Points in MongoDB (hot storage)
+            - chunkedDataPoints: Points in S3 (cold storage)
+            - totalChunks: Number of chunks in S3
+            - chunkSize: Chunking threshold
+            - firstDataAt: Timestamp of first point (if data has timestamp)
+            - lastDataAt: Timestamp of last point (if data has timestamp)
+            - createdAt: Metric creation time
+            - updatedAt: Last update time
+
+        Example:
+            stats = experiment.metric(name="train_loss").stats()
+            print(f"Total points: {stats['totalDataPoints']}")
+            print(f"Buffered: {stats['bufferedDataPoints']}, Chunked: {stats['chunkedDataPoints']}")
+        """
+        return self._experiment._get_metric_stats(name=self._name)
+
+    def list_all(self) -> List[Dict[str, Any]]:
+        """
+        List all metrics in the experiment.
+
+        Returns:
+            List of metric summaries with keys:
+            - metricId: Unique metric ID
+            - name: Metric name
+            - description: Metric description
+            - tags: Tags list
+            - totalDataPoints: Total data points
+            - createdAt: Creation timestamp
+
+        Example:
+            metrics = experiment.metric().list_all()
+            for metric in metrics:
+                print(f"{metric['name']}: {metric['totalDataPoints']} points")
+        """
+        return self._experiment._list_metrics()