ml-dash 0.6.11__tar.gz → 0.6.13__tar.gz

{ml_dash-0.6.11 → ml_dash-0.6.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ml-dash
-Version: 0.6.11
+Version: 0.6.13
 Summary: ML experiment tracking and data storage
 Keywords: machine-learning,experiment-tracking,mlops,data-storage
 Author: Ge Yang, Tom Tao

{ml_dash-0.6.11 → ml_dash-0.6.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ml-dash"
-version = "0.6.11"
+version = "0.6.13"
 description = "ML experiment tracking and data storage"
 readme = "README.md"
 requires-python = ">=3.9"

{ml_dash-0.6.11 → ml_dash-0.6.13}/src/ml_dash/__init__.py

@@ -43,18 +43,19 @@ from .params import ParametersBuilder
 from .run import RUN
 from .storage import LocalStorage
 
-__version__ = "0.6.10"
+__version__ = "0.6.13"
 
-# Minimum version required - blocks older versions
-MINIMUM_REQUIRED_VERSION = "0.6.10"
+# Required version - MUST match exactly (blocks all older versions)
+# Update this with EVERY release to force users to upgrade
+REQUIRED_VERSION = "0.6.13"
 
 
 def _check_version_compatibility():
     """
-    Enforce minimum version requirement.
+    Enforce strict version requirement.
 
-    Raises ImportError if installed version is below minimum required version.
-    This ensures users have the latest features (userinfo, namespace auto-detection, etc.)
+    Raises ImportError if installed version doesn't match the required version.
+    This ensures all users are on the latest version with newest features and bug fixes.
     """
     try:
         from packaging import version
@@ -64,25 +65,26 @@ def _check_version_compatibility():
         return
 
     current = version.parse(__version__)
-    minimum = version.parse(MINIMUM_REQUIRED_VERSION)
+    required = version.parse(REQUIRED_VERSION)
 
-    if current < minimum:
+    if current < required:
         raise ImportError(
             f"\n"
             f"{'=' * 80}\n"
-            f"ERROR: ml-dash version {__version__} is too old!\n"
+            f"ERROR: ml-dash version {__version__} is outdated!\n"
             f"{'=' * 80}\n"
             f"\n"
-            f"This version of ml-dash ({__version__}) is no longer supported.\n"
-            f"Minimum required version: {MINIMUM_REQUIRED_VERSION}\n"
+            f"Your installed version ({__version__}) is no longer supported.\n"
+            f"Required version: {REQUIRED_VERSION}\n"
             f"\n"
             f"Please upgrade to the latest version:\n"
             f"\n"
             f"  pip install --upgrade ml-dash\n"
             f"\n"
-            f"Or install specific version:\n"
+            f"Or with uv:\n"
             f"\n"
-            f"  pip install ml-dash>={MINIMUM_REQUIRED_VERSION}\n"
+            f"  uv pip install --upgrade ml-dash\n"
+            f"  uv sync --upgrade-package ml-dash\n"
             f"\n"
             f"{'=' * 80}\n"
         )

{ml_dash-0.6.11 → ml_dash-0.6.13}/src/ml_dash/buffer.py

@@ -340,18 +340,21 @@ class BackgroundBufferManager:
             if items:
                 print(f"[ML-Dash] Flushing {', '.join(items)}...", flush=True)
 
-        # Flush logs immediately
-        self._flush_logs()
+        # Flush logs immediately (loop until empty)
+        while not self._log_queue.empty():
+            self._flush_logs()
 
-        # Flush all metrics immediately
+        # Flush all metrics immediately (loop until empty for each metric)
         for metric_name in list(self._metric_queues.keys()):
-            self._flush_metric(metric_name)
+            while not self._metric_queues[metric_name].empty():
+                self._flush_metric(metric_name)
 
         # Flush all tracks immediately
         self.flush_tracks()
 
-        # Flush files immediately
-        self._flush_files()
+        # Flush files immediately (loop until empty)
+        while not self._file_queue.empty():
+            self._flush_files()
 
         if log_count > 0 or metric_count > 0 or track_count > 0 or file_count > 0:
             print("[ML-Dash] ✓ Flush complete", flush=True)

{ml_dash-0.6.11 → ml_dash-0.6.13}/src/ml_dash/cli_commands/upload.py

@@ -67,7 +67,7 @@ class UploadResult:
 class UploadState:
   """Tracks upload state for resume functionality."""
 
-  local_path: str
+  dash_root: str
   remote_url: str
   completed_experiments: List[str] = field(
     default_factory=list
@@ -79,7 +79,7 @@ class UploadState:
   def to_dict(self) -> Dict[str, Any]:
     """Convert to dictionary for JSON serialization."""
     return {
-      "local_path": self.local_path,
+      "dash_root": self.dash_root,
       "remote_url": self.remote_url,
       "completed_experiments": self.completed_experiments,
       "failed_experiments": self.failed_experiments,
@@ -91,7 +91,7 @@ class UploadState:
   def from_dict(cls, data: Dict[str, Any]) -> "UploadState":
     """Create from dictionary."""
     return cls(
-      local_path=data["local_path"],
+      dash_root=data["dash_root"],
       remote_url=data["remote_url"],
       completed_experiments=data.get("completed_experiments", []),
       failed_experiments=data.get("failed_experiments", []),
@@ -265,18 +265,18 @@ def add_parser(subparsers) -> argparse.ArgumentParser:
 
 
 def discover_experiments(
-  local_path: Path,
+  dash_root: Path,
   project_filter: Optional[str] = None,
   experiment_filter: Optional[str] = None,
 ) -> List[ExperimentInfo]:
   """
   Discover experiments in local storage directory.
 
-  Supports both flat (local_path/project/experiment) and folder-based
-  (local_path/folder/project/experiment) hierarchies.
+  Supports both flat (dash_root/project/experiment) and folder-based
+  (dash_root/folder/project/experiment) hierarchies.
 
   Args:
-      local_path: Root path of local storage
+      dash_root: Root path of local storage
       project_filter: Either a simple project name (e.g., "proj1") or a glob
                      pattern for the full path (e.g., "tom/*/exp*"). If the
                      filter contains '/', '*', or '?', it's treated as a glob
@@ -289,15 +289,15 @@ def discover_experiments(
   """
   import fnmatch
 
-  local_path = Path(local_path)
+  dash_root = Path(dash_root)
 
-  if not local_path.exists():
+  if not dash_root.exists():
     return []
 
   experiments = []
 
   # Find all experiment.json files recursively
-  for exp_json in local_path.rglob("*/experiment.json"):
+  for exp_json in dash_root.rglob("*/experiment.json"):
     exp_dir = exp_json.parent
 
     # Read prefix from experiment.json first
@@ -313,7 +313,7 @@ def discover_experiments(
     # This handles nested folders correctly
     # Prefix format: owner/project/folder.../experiment
     try:
-      relative_path = exp_dir.relative_to(local_path)
+      relative_path = exp_dir.relative_to(dash_root)
       full_relative_path = str(relative_path)
 
       if prefix:
@@ -1217,7 +1217,7 @@ def cmd_upload(args: argparse.Namespace) -> int:
       Exit code (0 for success, 1 for error)
   """
   # Handle track upload if --tracks is specified
-  if args.tracks:
+  if getattr(args, 'tracks', False):
     return cmd_upload_track(args)
 
   # Load config
@@ -1234,9 +1234,9 @@ def cmd_upload(args: argparse.Namespace) -> int:
   api_key = args.api_key or config.api_key
 
   # Discover experiments
-  local_path = Path(args.path)
-  if not local_path.exists():
-    console.print(f"[red]Error:[/red] Local storage path does not exist: {local_path}")
+  dash_root = Path(args.path)
+  if not dash_root.exists():
+    console.print(f"[red]Error:[/red] Local storage path does not exist: {dash_root}")
     return 1
 
   # Handle state file for resume functionality
@@ -1247,7 +1247,7 @@ def cmd_upload(args: argparse.Namespace) -> int:
     upload_state = UploadState.load(state_file)
     if upload_state:
       # Validate state matches current upload
-      if upload_state.local_path != str(local_path.absolute()):
+      if upload_state.dash_root != str(dash_root.absolute()):
         console.print(
           "[yellow]Warning:[/yellow] State file local path doesn't match. Starting fresh upload."
         )
@@ -1273,13 +1273,13 @@ def cmd_upload(args: argparse.Namespace) -> int:
   # Create new state if not resuming
   if not upload_state:
     upload_state = UploadState(
-      local_path=str(local_path.absolute()),
+      dash_root=str(dash_root.absolute()),
       remote_url=remote_url,
     )
 
-  console.print(f"[bold]Scanning local storage:[/bold] {local_path.absolute()}")
+  console.print(f"[bold]Scanning local storage:[/bold] {dash_root.absolute()}")
   experiments = discover_experiments(
-    local_path,
+    dash_root,
     project_filter=args.pref,  # Using --prefix/-p argument
     experiment_filter=None,
   )
@@ -1398,7 +1398,7 @@ def cmd_upload(args: argparse.Namespace) -> int:
 
   # Initialize remote client and local storage
   remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
-  local_storage = LocalStorage(root_path=local_path)
+  local_storage = LocalStorage(root_path=dash_root)
 
   # Upload experiments with progress tracking
   console.print(f"\n[bold]Uploading to:[/bold] {remote_url}")

{ml_dash-0.6.11 → ml_dash-0.6.13}/src/ml_dash/client.py

@@ -369,7 +369,7 @@ class RemoteClient:
             raise ValueError(f"Project '{project_slug}' not found in namespace '{self.namespace}'")
 
         # Delete using project-specific endpoint
-        response = self._client.delete(f"/projects/{project_id}")
+        response = self._client.delete(f"projects/{project_id}")
         response.raise_for_status()
         return response.json()
 
@@ -463,7 +463,7 @@ class RemoteClient:
                     if not project_id:
                         # Create the project first since we need its ID for folders
                         project_response = self._client.post(
-                            f"/namespaces/{self.namespace}/nodes",
+                            f"namespaces/{self.namespace}/nodes",
                             json={
                                 "type": "PROJECT",
                                 "name": project,
@@ -483,7 +483,7 @@ class RemoteClient:
                             # Create folder (server handles upsert)
                             # NOTE: Do NOT pass experimentId for project-level folders
                             folder_response = self._client.post(
-                                f"/namespaces/{self.namespace}/nodes",
+                                f"namespaces/{self.namespace}/nodes",
                                 json={
                                     "type": "FOLDER",
                                     "projectId": project_id,
@@ -525,7 +525,7 @@ class RemoteClient:
 
         # Call unified node creation API
         response = self._client.post(
-            f"/namespaces/{self.namespace}/nodes",
+            f"namespaces/{self.namespace}/nodes",
             json=payload,
         )
         response.raise_for_status()
@@ -565,7 +565,7 @@ class RemoteClient:
         payload = {"status": status}
 
         response = self._client.patch(
-            f"/nodes/{node_id}",
+            f"nodes/{node_id}",
             json=payload,
         )
         response.raise_for_status()
@@ -602,7 +602,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.post(
-            f"/experiments/{experiment_id}/logs",
+            f"experiments/{experiment_id}/logs",
             json={"logs": logs}
         )
         response.raise_for_status()
@@ -638,7 +638,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.post(
-            f"/experiments/{experiment_id}/parameters",
+            f"experiments/{experiment_id}/parameters",
             json={"data": data}
         )
         response.raise_for_status()
@@ -658,7 +658,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails or parameters don't exist
         """
-        response = self._client.get(f"/experiments/{experiment_id}/parameters")
+        response = self._client.get(f"experiments/{experiment_id}/parameters")
         response.raise_for_status()
         result = response.json()
         return result.get("data", {})
@@ -935,7 +935,7 @@ class RemoteClient:
 
                     # Create folder (server will return existing if duplicate)
                     folder_response = self._client.post(
-                        f"/namespaces/{self.namespace}/nodes",
+                        f"namespaces/{self.namespace}/nodes",
                         json={
                             "type": "FOLDER",
                             "projectId": project_id,
@@ -974,7 +974,7 @@ class RemoteClient:
 
         # Call unified node creation API
         response = self._client.post(
-            f"/namespaces/{self.namespace}/nodes",
+            f"namespaces/{self.namespace}/nodes",
             files=files,
             data=data
         )
@@ -1092,7 +1092,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         # file_id is actually the node ID in the new system
-        response = self._client.get(f"/nodes/{file_id}")
+        response = self._client.get(f"nodes/{file_id}")
         response.raise_for_status()
         return response.json()
 
@@ -1127,7 +1127,7 @@ class RemoteClient:
             dest_path = filename
 
         # Download file using node API
-        response = self._client.get(f"/nodes/{file_id}/download")
+        response = self._client.get(f"nodes/{file_id}/download")
         response.raise_for_status()
 
         # Write to file
@@ -1159,7 +1159,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        response = self._client.delete(f"/nodes/{file_id}")
+        response = self._client.delete(f"nodes/{file_id}")
         response.raise_for_status()
         return response.json()
 
@@ -1196,7 +1196,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.patch(
-            f"/nodes/{file_id}",
+            f"nodes/{file_id}",
             json=payload
         )
         response.raise_for_status()
@@ -1237,7 +1237,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/append",
+            f"experiments/{experiment_id}/metrics/{metric_name}/append",
             json=payload
         )
         response.raise_for_status()
@@ -1278,7 +1278,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/append-batch",
+            f"experiments/{experiment_id}/metrics/{metric_name}/append-batch",
             json=payload
         )
         response.raise_for_status()
@@ -1307,7 +1307,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/data",
+            f"experiments/{experiment_id}/metrics/{metric_name}/data",
             params={"startIndex": start_index, "limit": limit}
         )
         response.raise_for_status()
@@ -1332,7 +1332,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/stats"
+            f"experiments/{experiment_id}/metrics/{metric_name}/stats"
         )
         response.raise_for_status()
         return response.json()
@@ -1353,7 +1353,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        response = self._client.get(f"/experiments/{experiment_id}/metrics")
+        response = self._client.get(f"experiments/{experiment_id}/metrics")
         response.raise_for_status()
         return response.json()["metrics"]
 
@@ -1668,7 +1668,7 @@ class RemoteClient:
         expected_checksum = file_metadata.get("physicalFile", {}).get("checksum")
 
         # Stream download using node API
-        with self._client.stream("GET", f"/nodes/{file_id}/download") as response:
+        with self._client.stream("GET", f"nodes/{file_id}/download") as response:
             response.raise_for_status()
 
             with open(dest_path, "wb") as f:
@@ -1736,7 +1736,7 @@ class RemoteClient:
         if search is not None:
             params["search"] = search
 
-        response = self._client.get(f"/experiments/{experiment_id}/logs", params=params)
+        response = self._client.get(f"experiments/{experiment_id}/logs", params=params)
         response.raise_for_status()
         return response.json()
 
@@ -1774,7 +1774,7 @@ class RemoteClient:
             params["bufferOnly"] = "true"
 
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/data",
+            f"experiments/{experiment_id}/metrics/{metric_name}/data",
             params=params
         )
         response.raise_for_status()
@@ -1801,7 +1801,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/chunks/{chunk_number}"
+            f"experiments/{experiment_id}/metrics/{metric_name}/chunks/{chunk_number}"
         )
         response.raise_for_status()
         return response.json()
@@ -1845,7 +1845,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/tracks",
+            f"experiments/{experiment_id}/tracks",
             json=payload,
         )
         response.raise_for_status()
@@ -1878,7 +1878,7 @@ class RemoteClient:
         topic_encoded = urllib.parse.quote(topic, safe='')
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/tracks/{topic_encoded}/append",
+            f"experiments/{experiment_id}/tracks/{topic_encoded}/append",
             json={"timestamp": timestamp, "data": data},
         )
         response.raise_for_status()
@@ -1912,7 +1912,7 @@ class RemoteClient:
         serialized_entries = [_serialize_value(entry) for entry in entries]
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/tracks/{topic_encoded}/append_batch",
+            f"experiments/{experiment_id}/tracks/{topic_encoded}/append_batch",
             json={"entries": serialized_entries},
         )
         response.raise_for_status()
@@ -1957,7 +1957,7 @@ class RemoteClient:
             params["columns"] = ",".join(columns)
 
         response = self._client.get(
-            f"/experiments/{experiment_id}/tracks/{topic_encoded}/data",
+            f"experiments/{experiment_id}/tracks/{topic_encoded}/data",
             params=params,
         )
         response.raise_for_status()
@@ -1990,7 +1990,7 @@ class RemoteClient:
             params["topic"] = topic_filter
 
         response = self._client.get(
-            f"/experiments/{experiment_id}/tracks",
+            f"experiments/{experiment_id}/tracks",
             params=params,
         )
         response.raise_for_status()

{ml_dash-0.6.11 → ml_dash-0.6.13}/src/ml_dash/run.py

@@ -158,6 +158,16 @@ class RUN:
   now = datetime.now()
   """Timestamp at import time. Does not change during the session."""
 
+  @property
+  def date(self) -> str:
+    """Date string in YYYYMMDD format."""
+    return self.now.strftime("%Y%m%d")
+
+  @property
+  def datetime_str(self) -> str:
+    """DateTime string in YYYYMMDD.HHMMSS format."""
+    return self.now.strftime("%Y%m%d.%H%M%S")
+
   timestamp: str = None
   """Timestamp created at instantiation"""
 
@@ -277,6 +287,61 @@ class RUN:
         # self.name is the last segment
         self.name = parts[-1] if len(parts) > 2 else parts[1]
 
+  def __setattr__(self, name: str, value):
+    """
+    Intercept attribute setting to expand {EXP.attr} templates in prefix.
+
+    When prefix is set, expands any {EXP.name}, {EXP.id}, {EXP.date}, etc. templates
+    using current instance's attributes. Also syncs back to class-level RUN attributes.
+    """
+    # Prevent prefix changes after experiment has started
+    if name == "prefix" and isinstance(value, str):
+      experiment = getattr(self, "_experiment", None)
+      if experiment is not None and getattr(experiment, "_is_open", False):
+        raise RuntimeError(
+          "Cannot change prefix after experiment has been initialized. "
+          "Set prefix before calling experiment.run.start() or entering the context manager."
+        )
+
+    # Expand templates if setting prefix
+    if name == "prefix" and isinstance(value, str):
+      # Check if value contains {EXP. templates
+      if "{EXP." in value:
+        import re
+
+        def replace_match(match):
+          attr_name = match.group(1)
+          # Special handling for id - generate if needed
+          if attr_name == "id" and not getattr(self, "id", None):
+            from ml_dash.snowflake import generate_id
+            object.__setattr__(self, "id", generate_id())
+
+          # Get attribute, raising error if not found
+          try:
+            attr_value = getattr(self, attr_name)
+            if attr_value is None:
+              raise AttributeError(f"Attribute '{attr_name}' is None")
+            return str(attr_value)
+          except AttributeError:
+            raise AttributeError(f"RUN has no attribute '{attr_name}'")
+
+        # Match {EXP.attr_name} pattern
+        pattern = r"\{EXP\.(\w+)\}"
+        value = re.sub(pattern, replace_match, value)
+
+      # Always update _folder_path when prefix changes
+      object.__setattr__(self, "_folder_path", value)
+
+      # Parse and update owner, project, name from new prefix
+      parts = value.strip("/").split("/")
+      if len(parts) >= 2:
+        object.__setattr__(self, "owner", parts[0])
+        object.__setattr__(self, "project", parts[1])
+        object.__setattr__(self, "name", parts[-1] if len(parts) > 2 else parts[1])
+
+    # Use object.__setattr__ to set the value
+    object.__setattr__(self, name, value)
+
   def start(self) -> "Experiment":
     """
     Start the experiment (sets status to RUNNING).