ml-dash 0.6.5__py3-none-any.whl → 0.6.7__py3-none-any.whl

ml_dash/experiment.py

@@ -11,13 +11,14 @@ import functools
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Union, Unpack
+from typing import Any, Callable, Dict, List, Optional, Union
 
+from .buffer import BackgroundBufferManager, BufferConfig
 from .client import RemoteClient
 from .files import BindrsBuilder, FilesAccessor
 from .log import LogBuilder, LogLevel
 from .params import ParametersBuilder
-from .run import RUN
+from .run import RUN, requires_open
 from .storage import LocalStorage
 
 
@@ -60,140 +61,6 @@ class OperationMode(Enum):
   HYBRID = "hybrid"  # Future: sync local to remote
 
 
-class RunManager:
-  """
-  Lifecycle manager for experiments.
-
-  Supports three usage patterns:
-  1. Method calls: experiment.run.start(), experiment.run.complete()
-  2. Context manager: with Experiment(...).run as exp:
-  3. Decorator: @exp.run or @Experiment(...).run
-  """
-
-  def __init__(self, experiment: "Experiment"):
-    """
-    Initialize RunManager.
-
-    Args:
-        experiment: Parent Experiment instance
-    """
-    self._experiment = experiment
-
-  def start(self) -> "Experiment":
-    """
-    Start the experiment (sets status to RUNNING).
-
-    Returns:
-        The experiment instance for chaining
-    """
-    return self._experiment._open()
-
-  def complete(self) -> None:
-    """Mark experiment as completed (status: COMPLETED)."""
-    self._experiment._close(status="COMPLETED")
-
-  def fail(self) -> None:
-    """Mark experiment as failed (status: FAILED)."""
-    self._experiment._close(status="FAILED")
-
-  def cancel(self) -> None:
-    """Mark experiment as cancelled (status: CANCELLED)."""
-    self._experiment._close(status="CANCELLED")
-
-  @property
-  def prefix(self) -> Optional[str]:
-    """
-    Get the current folder prefix for this experiment.
-
-    Returns:
-        Current folder prefix path or None
-
-    Example:
-        current_prefix = exp.run.prefix
-    """
-    return self._experiment._folder_path
-
-  @prefix.setter
-  def prefix(self, value: Optional[str]) -> None:
-    """
-    Set the folder prefix for this experiment before initialization.
-
-    This can ONLY be set before the experiment is started (initialized).
-    Once the experiment is opened, the prefix cannot be changed.
-
-    Supports template variables:
-    - {EXP.name} - Experiment name
-    - {EXP.id} - Experiment ID
-
-    Args:
-        value: Folder prefix path with optional template variables
-               (e.g., "ge/myproject/{EXP.name}" or None)
-
-    Raises:
-        RuntimeError: If experiment is already initialized/open
-
-    Examples:
-        from ml_dash import dxp
-
-        # Static folder
-        dxp.run.prefix = "ge/myproject/experiments/resnet"
-
-        # Template with experiment name
-        dxp.run.prefix = "ge/iclr_2024/{EXP.name}"
-
-        # Now start the experiment
-        with dxp.run:
-            dxp.params.set(lr=0.001)
-    """
-    if self._experiment._is_open:
-      raise RuntimeError(
-        "Cannot change prefix after experiment is initialized. "
-        "Set prefix before calling start() or entering 'with' block."
-      )
-
-    if value:
-      # Sync EXP with this experiment's values
-      RUN.name = self._experiment.name
-      RUN.description = self._experiment.description
-      # Generate id/timestamp if not already set
-      if RUN.id is None:
-        RUN._init_run()
-      # Format with EXP - use helper to expand properties correctly
-      value = _expand_exp_template(value)
-
-    # Update the folder on the experiment
-    self._experiment._folder_path = value
-
-  def __enter__(self) -> "Experiment":
-    """Context manager entry - starts the experiment."""
-    return self.start()
-
-  def __exit__(self, exc_type, exc_val, exc_tb):
-    """Context manager exit - completes or fails the experiment."""
-    if exc_type is not None:
-      self.fail()
-    else:
-      self.complete()
-    return False
-
-  def __call__(self, func: Callable) -> Callable:
-    """
-    Decorator support for wrapping functions with experiment lifecycle.
-
-    Usage:
-        @exp.run
-        def train(exp):
-            exp.log("Training...")
-    """
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-      with self as exp:
-        return func(exp, *args, **kwargs)
-
-    return wrapper
-
-
 class Experiment:
   """
   ML-Dash experiment for metricing experiments.
@@ -231,6 +98,28 @@ class Experiment:
       ...
   """
 
+  run: RUN
+  """
+  Get the RunManager for lifecycle operations.
+
+  Usage:
+      # Method calls
+      experiment.run.start()
+      experiment.run.complete()
+
+      # Context manager
+      with Experiment(...).run as exp:
+          exp.log("Training...")
+
+      # Decorator
+      @experiment.run
+      def train(exp):
+          exp.log("Training...")
+
+  Returns:
+      RunManager instance
+  """
+
   def __init__(
     self,
     prefix: Optional[str] = None,
@@ -251,7 +140,7 @@ class Experiment:
     # Internal parameters
     _write_protected: bool = False,
     # The rest of the params go directly to populate the RUN object.
-    **run_params: Unpack[RUN],
+    **run_params,
   ):
     """
     Initialize an ML-Dash experiment.
@@ -278,7 +167,6 @@ class Experiment:
         - dash_url + dash_root: Hybrid mode (local + remote)
         - dash_url + dash_root=None: Remote-only mode
     """
-    import os
     import warnings
 
     # Handle backward compatibility
@@ -286,7 +174,7 @@ class Experiment:
       warnings.warn(
         "Parameter 'remote' is deprecated. Use 'dash_url' instead.",
         DeprecationWarning,
-        stacklevel=2
+        stacklevel=2,
       )
       if dash_url is None:
         dash_url = remote
@@ -295,28 +183,15 @@ class Experiment:
       warnings.warn(
         "Parameter 'local_path' is deprecated. Use 'dash_root' instead.",
         DeprecationWarning,
-        stacklevel=2
+        stacklevel=2,
       )
       if dash_root == ".dash":  # Only override if dash_root is default
         dash_root = local_path
 
-    # Resolve prefix from environment variable if not provided
-    self._folder_path = prefix or os.getenv("DASH_PREFIX")
+    if prefix:
+      run_params["prefix"] = prefix
 
-    if not self._folder_path:
-      raise ValueError("prefix (or DASH_PREFIX env var) must be provided")
-
-    # Parse prefix: {owner}/{project}/path.../[name]
-    parts = self._folder_path.strip("/").split("/")
-    if len(parts) < 2:
-      raise ValueError(
-        f"prefix must have at least owner/project: got '{self._folder_path}'"
-      )
-
-    self.owner = parts[0]
-    self.project = parts[1]
-    # Name is the last segment (may be a seed/id, not always a meaningful name)
-    self.name = parts[-1] if len(parts) > 2 else parts[1]
+    self.run = RUN(_experiment=self, **run_params)
 
     self.readme = readme
     self.tags = tags
@@ -324,11 +199,6 @@ class Experiment:
     self._write_protected = _write_protected
     self.metadata = metadata
 
-    # Initialize RUN with experiment values
-    RUN.name = self.name
-    if readme:
-      RUN.readme = readme
-
     # Determine operation mode
     # dash_root defaults to ".dash", dash_url defaults to None
     if dash_url and dash_root:
@@ -339,21 +209,24 @@ class Experiment:
       self.mode = OperationMode.LOCAL
 
     # Initialize backend
-    self._client: Optional[RemoteClient] = None
-    self._storage: Optional[LocalStorage] = None
     self._experiment_id: Optional[str] = None
     self._experiment_data: Optional[Dict[str, Any]] = None
     self._is_open = False
     self._metrics_manager: Optional["MetricsManager"] = None  # Cached metrics manager
+    self._tracks_manager: Optional["TracksManager"] = None  # Cached tracks manager
+
+    # Initialize buffer manager
+    self._buffer_config = BufferConfig.from_env()
+    self._buffer_manager: Optional[BackgroundBufferManager] = None
 
     if self.mode in (OperationMode.REMOTE, OperationMode.HYBRID):
-      # RemoteClient will auto-load token from ~/.dash/token.enc
+      # RemoteClient will autoload token from ~/.dash/token.enc
       # Use RUN.api_url if dash_url=True (boolean), otherwise use the provided URL
       api_url = RUN.api_url if dash_url is True else dash_url
-      self._client = RemoteClient(base_url=api_url, namespace=self.owner)
+      self.run._client = RemoteClient(base_url=api_url, namespace=self.run.owner)
 
     if self.mode in (OperationMode.LOCAL, OperationMode.HYBRID):
-      self._storage = LocalStorage(root_path=Path(dash_root))
+      self.run._storage = LocalStorage(root_path=Path(dash_root))
 
   def _open(self) -> "Experiment":
     """
@@ -365,16 +238,16 @@ class Experiment:
     if self._is_open:
       return self
 
-    if self._client:
+    if self.run._client:
       # Remote mode: create/update experiment via API
       try:
-        response = self._client.create_or_update_experiment(
-          project=self.project,
-          name=self.name,
+        response = self.run._client.create_or_update_experiment(
+          project=self.run.project,
+          name=self.run.name,
           description=self.readme,
           tags=self.tags,
           bindrs=self._bindrs_list,
-          prefix=self._folder_path,
+          prefix=self.run._folder_path,
           write_protected=self._write_protected,
           metadata=self.metadata,
         )
@@ -387,13 +260,13 @@ class Experiment:
 
           console = Console()
           console.print(
-            f"[dim]✓ Experiment started: [bold]{self.name}[/bold] (project: {self.project})[/dim]\n"
+            f"[dim]✓ Experiment started: [bold]{self.run.name}[/bold] (project: {self.run.project})[/dim]\n"
             f"[dim]View your data, statistics, and plots online at:[/dim] "
             f"[link=https://dash.ml]https://dash.ml[/link]"
           )
         except ImportError:
           # Fallback if rich is not available
-          print(f"✓ Experiment started: {self.name} (project: {self.project})")
+          print(f"✓ Experiment started: {self.run.name} (project: {self.run.project})")
           print("View your data at: https://dash.ml")
 
       except Exception as e:
@@ -446,18 +319,22 @@ class Experiment:
           # Re-raise other exceptions
           raise
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: create experiment directory structure
-      self._storage.create_experiment(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      self.run._storage.create_experiment(
+        project=self.run.project,
+        prefix=self.run._folder_path,
         description=self.readme,
         tags=self.tags,
         bindrs=self._bindrs_list,
         metadata=self.metadata,
       )
 
+    # Start background buffer
+    if self._buffer_config.buffer_enabled:
+      self._buffer_manager = BackgroundBufferManager(self, self._buffer_config)
+      self._buffer_manager.start()
+
     self._is_open = True
     return self
 
@@ -468,17 +345,24 @@ class Experiment:
     Args:
         status: Status to set - "COMPLETED" (default), "FAILED", or "CANCELLED"
     """
-    if not self._is_open:
-      return
-
-    # Flush any pending writes
-    if self._storage:
-      self._storage.flush()
+    # if not self._is_open:
+    #   return
+    #
+    # note-ge: do NOT flush because the upload will be async. we will NEVER reuse
+    # experiment objects.
+    # # Flush any pending writes
+    # if self.run._storage:
+    #   self.run._storage.flush()
+
+    # Flush and stop buffer BEFORE status update
+    # Waits indefinitely for all data to be flushed (important for large files)
+    if self._buffer_manager:
+      self._buffer_manager.stop()
 
     # Update experiment status in remote mode
-    if self._client and self._experiment_id:
+    if self.run._client and self._experiment_id:
       try:
-        self._client.update_experiment_status(
+        self.run._client.update_experiment_status(
           experiment_id=self._experiment_id, status=status
         )
 
@@ -499,14 +383,14 @@ class Experiment:
           console = Console()
           console.print(
             f"[{status_color}]{status_emoji} Experiment {status.lower()}: "
-            f"[bold]{self.name}[/bold] (project: {self.project})[/{status_color}]\n"
+            f"[bold]{self.run.name}[/bold] (project: {self.run.project})[/{status_color}]\n"
             f"[dim]View results, statistics, and plots online at:[/dim] "
             f"[link=https://dash.ml]https://dash.ml[/link]"
           )
         except ImportError:
           # Fallback if rich is not available
           print(
-            f"{status_emoji} Experiment {status.lower()}: {self.name} (project: {self.project})"
+            f"{status_emoji} Experiment {status.lower()}: {self.run.name} (project: {self.run.project})"
           )
           print("View results at: https://dash.ml")
 
@@ -516,35 +400,8 @@ class Experiment:
 
     self._is_open = False
 
-    # Reset RUN for next experiment
-    # TODO: RUN._reset() - method doesn't exist
-    # RUN._reset()
-
-  @property
-  def run(self) -> RunManager:
-    """
-    Get the RunManager for lifecycle operations.
-
-    Usage:
-        # Method calls
-        experiment.run.start()
-        experiment.run.complete()
-
-        # Context manager
-        with Experiment(...).run as exp:
-            exp.log("Training...")
-
-        # Decorator
-        @experiment.run
-        def train(exp):
-            exp.log("Training...")
-
-    Returns:
-        RunManager instance
-    """
-    return RunManager(self)
-
   @property
+  @requires_open
   def params(self) -> ParametersBuilder:
     """
     Get a ParametersBuilder for parameter operations.
@@ -562,17 +419,10 @@ class Experiment:
     Raises:
         RuntimeError: If experiment is not open
     """
-    if not self._is_open:
-      raise RuntimeError(
-        "Experiment not started. Use 'with experiment.run:' or call experiment.run.start() first.\n"
-        "Example:\n"
-        "  with dxp.run:\n"
-        "      dxp.params.set(lr=0.001)"
-      )
-
     return ParametersBuilder(self)
 
   @property
+  @requires_open
   def logs(self) -> LogBuilder:
     """
     Get a LogBuilder for fluent-style logging.
@@ -592,16 +442,9 @@ class Experiment:
         exp.logs.warn("GPU memory low", memory_available="1GB")
         exp.logs.debug("Debug info", step=100)
     """
-    if not self._is_open:
-      raise RuntimeError(
-        "Experiment not started. Use 'with experiment.run:' or call experiment.run.start() first.\n"
-        "Example:\n"
-        "  with dxp.run:\n"
-        "      dxp.logs.info('Training started')"
-      )
-
     return LogBuilder(self, metadata=None)
 
+  @requires_open
   def log(
     self,
     message: Optional[str] = None,
@@ -638,22 +481,16 @@ class Experiment:
         RuntimeError: If experiment is not open
         ValueError: If log level is invalid
     """
-    if not self._is_open:
-      raise RuntimeError(
-        "Experiment not started. Use 'with experiment.run:' or call experiment.run.start() first.\n"
-        "Example:\n"
-        "  with dxp.run:\n"
-        "      dxp.logs.info('Training started')"
-      )
 
     # Fluent mode: return LogBuilder (deprecated)
     if message is None:
       import warnings
+
       warnings.warn(
         "Using exp.log() without a message is deprecated. "
         "Use exp.logs.info('message') instead.",
         DeprecationWarning,
-        stacklevel=2
+        stacklevel=2,
       )
       combined_metadata = {**(metadata or {}), **extra_metadata}
       return LogBuilder(self, combined_metadata if combined_metadata else None)
@@ -679,8 +516,8 @@ class Experiment:
     timestamp: Optional[datetime],
   ) -> None:
     """
-    Internal method to write a log entry immediately.
-    No buffering - writes directly to storage/remote AND stdout/stderr.
+    Internal method to write a log entry.
+    Uses buffering if enabled, otherwise writes directly.
 
     Args:
         message: Log message
@@ -688,55 +525,59 @@ class Experiment:
         metadata: Optional metadata dict
         timestamp: Optional custom timestamp (defaults to now)
     """
-    log_entry = {
-      "timestamp": (timestamp or datetime.utcnow()).isoformat() + "Z",
-      "level": level,
-      "message": message,
-    }
-
-    if metadata:
-      log_entry["metadata"] = metadata
-
-    # Mirror to stdout/stderr before writing to storage
+    # Print to console immediately (user visibility)
     self._print_log(message, level, metadata)
 
-    # Write immediately (no buffering)
-    if self._client:
-      # Remote mode: send to API (wrapped in array for batch API)
-      try:
-        self._client.create_log_entries(
-          experiment_id=self._experiment_id,
-          logs=[log_entry],  # Single log in array
-        )
-      except Exception as e:
-        # Log warning but don't crash training
-        import warnings
-        warnings.warn(
-          f"Failed to write log to remote server: {e}. Training will continue.",
-          RuntimeWarning,
-          stacklevel=4
-        )
-        # Fall through to local storage if available
+    # Buffer or write immediately
+    if self._buffer_manager and self._buffer_config.buffer_enabled:
+      self._buffer_manager.buffer_log(message, level, metadata, timestamp)
+    else:
+      # Immediate write (backward compatibility)
+      log_entry = {
+        "timestamp": (timestamp or datetime.utcnow()).isoformat() + "Z",
+        "level": level,
+        "message": message,
+      }
+
+      if metadata:
+        log_entry["metadata"] = metadata
+
+      if self.run._client:
+        # Remote mode: send to API (wrapped in array for batch API)
+        try:
+          self.run._client.create_log_entries(
+            experiment_id=self._experiment_id,
+            logs=[log_entry],  # Single log in array
+          )
+        except Exception as e:
+          # Log warning but don't crash training
+          import warnings
+
+          warnings.warn(
+            f"Failed to write log to remote server: {e}. Training will continue.",
+            RuntimeWarning,
+            stacklevel=4,
+          )
+          # Fall through to local storage if available
 
-    if self._storage:
-      # Local mode: write to file immediately
-      try:
-        self._storage.write_log(
-          owner=self.owner,
-          project=self.project,
-          prefix=self._folder_path,
-          message=log_entry["message"],
-          level=log_entry["level"],
-          metadata=log_entry.get("metadata"),
-          timestamp=log_entry["timestamp"],
-        )
-      except Exception as e:
-        import warnings
-        warnings.warn(
-          f"Failed to write log to local storage: {e}",
-          RuntimeWarning,
-          stacklevel=4
-        )
+      if self.run._storage:
+        # Local mode: write to file immediately
+        try:
+          self.run._storage.write_log(
+            owner=self.run.owner,
+            project=self.run.project,
+            prefix=self.run._folder_path,
+            message=log_entry["message"],
+            level=log_entry["level"],
+            metadata=log_entry.get("metadata"),
+            timestamp=log_entry["timestamp"],
+          )
+        except Exception as e:
+          import warnings
+
+          warnings.warn(
+            f"Failed to write log to local storage: {e}", RuntimeWarning, stacklevel=4
+          )
 
   def _print_log(
     self, message: str, level: str, metadata: Optional[Dict[str, Any]]
@@ -773,6 +614,7 @@ class Experiment:
       print(formatted_message, file=sys.stdout)
 
   @property
+  @requires_open
   def files(self) -> FilesAccessor:
     """
     Get a FilesAccessor for fluent file operations.
@@ -813,16 +655,9 @@ class Experiment:
         dxp.files.save_json(dict(hey="yo"), to="config.json")
         dxp.files.save_blob(b"xxx", to="data.bin")
     """
-    if not self._is_open:
-      raise RuntimeError(
-        "Experiment not started. Use 'with experiment.run:' or call experiment.run.start() first.\n"
-        "Example:\n"
-        "  with dxp.run:\n"
-        "      dxp.files('path').upload()"
-      )
-
     return FilesAccessor(self)
 
+  @requires_open
   def bindrs(self, bindr_name: str) -> BindrsBuilder:
     """
     Get a BindrsBuilder for working with file collections (bindrs).
@@ -845,14 +680,6 @@ class Experiment:
     Note:
         This is a placeholder for future bindr functionality.
     """
-    if not self._is_open:
-      raise RuntimeError(
-        "Experiment not started. Use 'with experiment.run:' or call experiment.run.start() first.\n"
-        "Example:\n"
-        "  with dxp.run:\n"
-        "      files = dxp.bindrs('my-bindr').list()"
-      )
-
     return BindrsBuilder(self, bindr_name)
 
   def _upload_file(
@@ -869,6 +696,7 @@ class Experiment:
   ) -> Dict[str, Any]:
     """
     Internal method to upload a file.
+    Uses buffering if enabled, otherwise uploads directly.
 
     Args:
         file_path: Local file path
@@ -882,43 +710,52 @@ class Experiment:
         size_bytes: File size in bytes
 
     Returns:
-        File metadata dict
+        File metadata dict (or pending status if buffering)
     """
-    result = None
-
-    if self._client:
-      # Remote mode: upload to API
-      result = self._client.upload_file(
-        experiment_id=self._experiment_id,
-        file_path=file_path,
-        prefix=prefix,
-        filename=filename,
-        description=description,
-        tags=tags,
-        metadata=metadata,
-        checksum=checksum,
-        content_type=content_type,
-        size_bytes=size_bytes,
+    # Buffer or upload immediately
+    if self._buffer_manager and self._buffer_config.buffer_enabled:
+      self._buffer_manager.buffer_file(
+        file_path, prefix, filename, description, tags, metadata,
+        checksum, content_type, size_bytes
       )
+      return {"id": "pending", "status": "queued"}
+    else:
+      # Immediate upload (backward compatibility)
+      result = None
 
-    if self._storage:
-      # Local mode: copy to local storage
-      result = self._storage.write_file(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
-        file_path=file_path,
-        path=prefix,
-        filename=filename,
-        description=description,
-        tags=tags,
-        metadata=metadata,
-        checksum=checksum,
-        content_type=content_type,
-        size_bytes=size_bytes,
-      )
+      if self.run._client:
+        # Remote mode: upload to API
+        result = self.run._client.upload_file(
+          experiment_id=self._experiment_id,
+          file_path=file_path,
+          prefix=prefix,
+          filename=filename,
+          description=description,
+          tags=tags,
+          metadata=metadata,
+          checksum=checksum,
+          content_type=content_type,
+          size_bytes=size_bytes,
+        )
 
-    return result
+      if self.run._storage:
+        # Local mode: copy to local storage
+        result = self.run._storage.write_file(
+          owner=self.run.owner,
+          project=self.run.project,
+          prefix=self.run._folder_path,
+          file_path=file_path,
+          path=prefix,
+          filename=filename,
+          description=description,
+          tags=tags,
+          metadata=metadata,
+          checksum=checksum,
+          content_type=content_type,
+          size_bytes=size_bytes,
+        )
+
+      return result
 
   def _list_files(
     self, prefix: Optional[str] = None, tags: Optional[List[str]] = None
@@ -935,18 +772,18 @@ class Experiment:
     """
     files = []
 
-    if self._client:
+    if self.run._client:
       # Remote mode: fetch from API
-      files = self._client.list_files(
+      files = self.run._client.list_files(
         experiment_id=self._experiment_id, prefix=prefix, tags=tags
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: read from metadata file
-      files = self._storage.list_files(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      files = self.run._storage.list_files(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         path_prefix=prefix,
         tags=tags,
       )
@@ -964,18 +801,18 @@ class Experiment:
     Returns:
         Path to downloaded file
     """
-    if self._client:
+    if self.run._client:
       # Remote mode: download from API
-      return self._client.download_file(
+      return self.run._client.download_file(
         experiment_id=self._experiment_id, file_id=file_id, dest_path=dest_path
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: copy from local storage
-      return self._storage.read_file(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      return self.run._storage.read_file(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         file_id=file_id,
         dest_path=dest_path,
       )
@@ -994,18 +831,18 @@ class Experiment:
     """
     result = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: delete via API
-      result = self._client.delete_file(
+      result = self.run._client.delete_file(
         experiment_id=self._experiment_id, file_id=file_id
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: soft delete in metadata
-      result = self._storage.delete_file(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      result = self.run._storage.delete_file(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         file_id=file_id,
       )
 
@@ -1032,9 +869,9 @@ class Experiment:
     """
     result = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: update via API
-      result = self._client.update_file(
+      result = self.run._client.update_file(
         experiment_id=self._experiment_id,
         file_id=file_id,
         description=description,
@@ -1042,12 +879,12 @@ class Experiment:
         metadata=metadata,
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: update in metadata file
-      result = self._storage.update_file_metadata(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      result = self.run._storage.update_file_metadata(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         file_id=file_id,
         description=description,
         tags=tags,
@@ -1063,18 +900,18 @@ class Experiment:
     Args:
         flattened_params: Already-flattened parameter dict with dot notation
     """
-    if self._client:
+    if self.run._client:
       # Remote mode: send to API
-      self._client.set_parameters(
+      self.run._client.set_parameters(
         experiment_id=self._experiment_id, data=flattened_params
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: write to file
-      self._storage.write_parameters(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      self.run._storage.write_parameters(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         data=flattened_params,
       )
 
@@ -1087,23 +924,24 @@ class Experiment:
     """
     params = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: fetch from API
       try:
-        params = self._client.get_parameters(experiment_id=self._experiment_id)
+        params = self.run._client.get_parameters(experiment_id=self._experiment_id)
       except Exception:
         # Parameters don't exist yet
         params = None
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: read from file
-      params = self._storage.read_parameters(
-        owner=self.owner, project=self.project, prefix=self._folder_path
+      params = self.run._storage.read_parameters(
+        owner=self.run.owner, project=self.run.project, prefix=self.run._folder_path
       )
 
     return params
 
   @property
+  @requires_open
   def metrics(self) -> "MetricsManager":
     """
     Get a MetricsManager for metric operations.
@@ -1139,17 +977,55 @@ class Experiment:
     """
     from .metric import MetricsManager
 
-    if not self._is_open:
-      raise RuntimeError(
-        "Cannot use metrics on closed experiment. "
-        "Use 'with Experiment(...).run as experiment:' or call experiment.run.start() first."
-      )
-
     # Cache the MetricsManager instance to preserve MetricBuilder cache across calls
     if self._metrics_manager is None:
       self._metrics_manager = MetricsManager(self)
     return self._metrics_manager
 
+  @property
+  @requires_open
+  def tracks(self) -> "TracksManager":
+    """
+    Get a TracksManager for timestamped track operations.
+
+    Supports topic-based logging with automatic timestamp merging:
+    - experiment.tracks("robot/position").append(q=[0.1, 0.2], _ts=0.0)
+    - experiment.tracks.flush()  # Flush all topics
+    - experiment.tracks("robot/position").flush()  # Flush specific topic
+
+    Returns:
+        TracksManager instance
+
+    Raises:
+        RuntimeError: If experiment is not open
+
+    Examples:
+        # Log track data with timestamp
+        experiment.tracks("robot/position").append(
+            q=[0.1, -0.22, 0.45],
+            e=[0.5, 0.0, 0.6],
+            _ts=2.0
+        )
+
+        # Entries with same timestamp are automatically merged
+        experiment.tracks("camera/rgb").append(frame_id=0, _ts=0.0)
+        experiment.tracks("camera/rgb").append(path="frame_0.png", _ts=0.0)
+
+        # Read track data
+        data = experiment.tracks("robot/position").read(format="json")
+
+        # Download in different formats
+        jsonl = experiment.tracks("robot/position").read(format="jsonl")
+        parquet = experiment.tracks("robot/position").read(format="parquet")
+        mocap = experiment.tracks("robot/position").read(format="mocap")
+    """
+    from .track import TracksManager
+
+    # Cache the TracksManager instance to preserve TrackBuilder cache across calls
+    if self._tracks_manager is None:
+      self._tracks_manager = TracksManager(self)
+    return self._tracks_manager
+
   def _append_to_metric(
     self,
     name: Optional[str],
@@ -1160,6 +1036,7 @@ class Experiment:
   ) -> Optional[Dict[str, Any]]:
     """
     Internal method to append a single data point to a metric.
+    Uses buffering if enabled, otherwise writes directly.
 
     Args:
         name: Metric name (can be None for unnamed metrics)
@@ -1169,56 +1046,125 @@ class Experiment:
         metadata: Optional metadata
 
     Returns:
-        Dict with metricId, index, bufferedDataPoints, chunkSize or None if all backends fail
+        Dict with metricId, index, bufferedDataPoints, chunkSize or None if buffering enabled/all backends fail
     """
-    result = None
+    # Buffer or write immediately
+    if self._buffer_manager and self._buffer_config.buffer_enabled:
+      self._buffer_manager.buffer_metric(name, data, description, tags, metadata)
+      return None  # No immediate response when buffering
+    else:
+      # Immediate write (backward compatibility)
+      result = None
 
-    if self._client:
-      # Remote mode: append via API
-      try:
-        result = self._client.append_to_metric(
-          experiment_id=self._experiment_id,
-          metric_name=name,
-          data=data,
-          description=description,
-          tags=tags,
-          metadata=metadata,
-        )
-      except Exception as e:
-        # Log warning but don't crash training
-        import warnings
-        metric_display = f"'{name}'" if name else "unnamed metric"
-        warnings.warn(
-          f"Failed to log {metric_display} to remote server: {e}. "
-          f"Training will continue.",
-          RuntimeWarning,
-          stacklevel=3
-        )
-        # Fall through to local storage if available
+      if self.run._client:
+        # Remote mode: append via API
+        try:
+          result = self.run._client.append_to_metric(
+            experiment_id=self._experiment_id,
+            metric_name=name,
+            data=data,
+            description=description,
+            tags=tags,
+            metadata=metadata,
+          )
+        except Exception as e:
+          # Log warning but don't crash training
+          import warnings
+
+          metric_display = f"'{name}'" if name else "unnamed metric"
+          warnings.warn(
+            f"Failed to log {metric_display} to remote server: {e}. "
+            f"Training will continue.",
+            RuntimeWarning,
+            stacklevel=3,
+          )
+          # Fall through to local storage if available
 
-    if self._storage:
-      # Local mode: append to local storage
-      try:
-        result = self._storage.append_to_metric(
-          owner=self.owner,
-          project=self.project,
-          prefix=self._folder_path,
-          metric_name=name,
-          data=data,
-          description=description,
-          tags=tags,
-          metadata=metadata,
-        )
-      except Exception as e:
-        import warnings
-        metric_display = f"'{name}'" if name else "unnamed metric"
-        warnings.warn(
-          f"Failed to log {metric_display} to local storage: {e}",
-          RuntimeWarning,
-          stacklevel=3
-        )
+      if self.run._storage:
+        # Local mode: append to local storage
+        try:
+          result = self.run._storage.append_to_metric(
+            owner=self.run.owner,
+            project=self.run.project,
+            prefix=self.run._folder_path,
+            metric_name=name,
+            data=data,
+            description=description,
+            tags=tags,
+            metadata=metadata,
+          )
+        except Exception as e:
+          import warnings
+
+          metric_display = f"'{name}'" if name else "unnamed metric"
+          warnings.warn(
+            f"Failed to log {metric_display} to local storage: {e}",
+            RuntimeWarning,
+            stacklevel=3,
+          )
 
-    return result
+      return result
+
+  def _write_track(
+    self,
+    topic: str,
+    timestamp: float,
+    data: Dict[str, Any],
+  ) -> None:
+    """
+    Internal method to write a track entry with timestamp.
+    Uses buffering with timestamp-based merging if enabled.
+
+    Args:
+        topic: Track topic (e.g., "robot/position")
+        timestamp: Entry timestamp
+        data: Data fields
+
+    Note:
+        Entries with the same timestamp are automatically merged in the buffer.
+    """
+    # Buffer or write immediately
+    if self._buffer_manager and self._buffer_config.buffer_enabled:
+      self._buffer_manager.buffer_track(topic, timestamp, data)
+    else:
+      # Immediate write (no buffering)
+      if self.run._client:
+        # Remote mode: append via API
+        try:
+          self.run._client.append_batch_to_track(
+            experiment_id=self._experiment_id,
+            topic=topic,
+            entries=[{"timestamp": timestamp, **data}],
+          )
+        except Exception as e:
+          # Log warning but don't crash training
+          import warnings
+
+          warnings.warn(
+            f"Failed to log track '{topic}' to remote server: {e}. "
+            f"Training will continue.",
+            RuntimeWarning,
+            stacklevel=3,
+          )
+
+      if self.run._storage:
+        # Local mode: append to local storage
+        try:
+          self.run._storage.append_batch_to_track(
+            owner=self.run.owner,
+            project=self.run.project,
+            prefix=self.run._folder_path,
+            topic=topic,
+            entries=[{"timestamp": timestamp, **data}],
+          )
+        except Exception as e:
+          import warnings
+
+          warnings.warn(
+            f"Failed to log track '{topic}' to local storage: {e}",
+            RuntimeWarning,
+            stacklevel=3,
+          )
 
   def _append_batch_to_metric(
     self,
@@ -1243,10 +1189,10 @@ class Experiment:
     """
     result = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: append batch via API
       try:
-        result = self._client.append_batch_to_metric(
+        result = self.run._client.append_batch_to_metric(
           experiment_id=self._experiment_id,
           metric_name=name,
           data_points=data_points,
@@ -1257,22 +1203,23 @@ class Experiment:
       except Exception as e:
         # Log warning but don't crash training
         import warnings
+
         metric_display = f"'{name}'" if name else "unnamed metric"
         warnings.warn(
           f"Failed to log batch to {metric_display} on remote server: {e}. "
           f"Training will continue.",
           RuntimeWarning,
-          stacklevel=3
+          stacklevel=3,
         )
         # Fall through to local storage if available
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: append batch to local storage
       try:
-        result = self._storage.append_batch_to_metric(
-          owner=self.owner,
-          project=self.project,
-          prefix=self._folder_path,
+        result = self.run._storage.append_batch_to_metric(
+          owner=self.run.owner,
+          project=self.run.project,
+          prefix=self.run._folder_path,
           metric_name=name,
           data_points=data_points,
           description=description,
@@ -1281,11 +1228,12 @@ class Experiment:
         )
       except Exception as e:
         import warnings
+
         metric_display = f"'{name}'" if name else "unnamed metric"
         warnings.warn(
           f"Failed to log batch to {metric_display} in local storage: {e}",
           RuntimeWarning,
-          stacklevel=3
+          stacklevel=3,
         )
 
     return result
@@ -1306,21 +1254,21 @@ class Experiment:
     """
     result = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: read via API
-      result = self._client.read_metric_data(
+      result = self.run._client.read_metric_data(
         experiment_id=self._experiment_id,
         metric_name=name,
         start_index=start_index,
         limit=limit,
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: read from local storage
-      result = self._storage.read_metric_data(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      result = self.run._storage.read_metric_data(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         metric_name=name,
         start_index=start_index,
         limit=limit,
@@ -1340,18 +1288,18 @@ class Experiment:
     """
     result = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: get stats via API
-      result = self._client.get_metric_stats(
+      result = self.run._client.get_metric_stats(
         experiment_id=self._experiment_id, metric_name=name
       )
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: get stats from local storage
-      result = self._storage.get_metric_stats(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
+      result = self.run._storage.get_metric_stats(
+        owner=self.run.owner,
+        project=self.run.project,
+        prefix=self.run._folder_path,
         metric_name=name,
       )
 
@@ -1366,18 +1314,104 @@ class Experiment:
     """
     result = None
 
-    if self._client:
+    if self.run._client:
       # Remote mode: list via API
-      result = self._client.list_metrics(experiment_id=self._experiment_id)
+      result = self.run._client.list_metrics(experiment_id=self._experiment_id)
 
-    if self._storage:
+    if self.run._storage:
       # Local mode: list from local storage
-      result = self._storage.list_metrics(
-        owner=self.owner, project=self.project, prefix=self._folder_path
+      result = self.run._storage.list_metrics(
+        owner=self.run.owner, project=self.run.project, prefix=self.run._folder_path
       )
 
     return result or []
 
+  @property
+  def owner(self) -> Optional[str]:
+    """Get the owner (first segment of prefix)."""
+    return self.run.owner
+
+  @owner.setter
+  def owner(self, value: str) -> None:
+    """Set the owner."""
+    self.run.owner = value
+
+  @property
+  def project(self) -> Optional[str]:
+    """Get the project (second segment of prefix or RUN.project)."""
+    return self.run.project
+
+  @project.setter
+  def project(self, value: str) -> None:
+    """Set the project."""
+    self.run.project = value
+
+  @property
+  def name(self) -> Optional[str]:
+    """Get the experiment name (last segment of prefix)."""
+    return self.run.name
+
+  @name.setter
+  def name(self, value: str) -> None:
+    """Set the name."""
+    self.run.name = value
+
+  @property
+  def _folder_path(self) -> Optional[str]:
+    """Get the full folder path (same as prefix)."""
+    return self.run._folder_path
+
+  @_folder_path.setter
+  def _folder_path(self, value: str) -> None:
+    """Set the full folder path and re-parse into components."""
+    self.run._folder_path = value
+    self.run.prefix = value
+    # Re-parse prefix into components
+    if value:
+      parts = value.strip("/").split("/")
+      if len(parts) >= 2:
+        self.run.owner = parts[0]
+        self.run.project = parts[1]
+        self.run.name = parts[-1] if len(parts) > 2 else parts[1]
+
+  @property
+  def _client(self):
+    """Get the remote client."""
+    return self.run._client
+
+  @_client.setter
+  def _client(self, value) -> None:
+    """Set the remote client."""
+    self.run._client = value
+
+  @property
+  def _storage(self):
+    """Get the local storage."""
+    return self.run._storage
+
+  @_storage.setter
+  def _storage(self, value) -> None:
+    """Set the local storage."""
+    self.run._storage = value
+
+  def flush(self) -> None:
+    """
+    Manually flush all buffered data.
+
+    Forces immediate flush of all queued logs, metrics, and files.
+    Waits for all file uploads to complete.
+
+    Examples:
+        with Experiment("my-project/exp").run as exp:
+            for epoch in range(100):
+                exp.metrics("train").log(loss=loss)
+
+            exp.flush()  # Ensure metrics written before checkpoint
+            torch.save(model, "model.pt")
+    """
+    if self._buffer_manager:
+      self._buffer_manager.flush_all()
+
   @property
   def id(self) -> Optional[str]:
     """Get the experiment ID (only available after open in remote mode)."""