ml-dash 0.6.11__py3-none-any.whl → 0.6.12__py3-none-any.whl

ml_dash/buffer.py

@@ -340,18 +340,21 @@ class BackgroundBufferManager:
             if items:
                 print(f"[ML-Dash] Flushing {', '.join(items)}...", flush=True)
 
-        # Flush logs immediately
-        self._flush_logs()
+        # Flush logs immediately (loop until empty)
+        while not self._log_queue.empty():
+            self._flush_logs()
 
-        # Flush all metrics immediately
+        # Flush all metrics immediately (loop until empty for each metric)
         for metric_name in list(self._metric_queues.keys()):
-            self._flush_metric(metric_name)
+            while not self._metric_queues[metric_name].empty():
+                self._flush_metric(metric_name)
 
         # Flush all tracks immediately
         self.flush_tracks()
 
-        # Flush files immediately
-        self._flush_files()
+        # Flush files immediately (loop until empty)
+        while not self._file_queue.empty():
+            self._flush_files()
 
         if log_count > 0 or metric_count > 0 or track_count > 0 or file_count > 0:
             print("[ML-Dash] ✓ Flush complete", flush=True)

ml_dash/cli_commands/upload.py

@@ -67,7 +67,7 @@ class UploadResult:
 class UploadState:
   """Tracks upload state for resume functionality."""
 
-  local_path: str
+  dash_root: str
   remote_url: str
   completed_experiments: List[str] = field(
     default_factory=list
@@ -79,7 +79,7 @@ class UploadState:
   def to_dict(self) -> Dict[str, Any]:
     """Convert to dictionary for JSON serialization."""
     return {
-      "local_path": self.local_path,
+      "dash_root": self.dash_root,
       "remote_url": self.remote_url,
       "completed_experiments": self.completed_experiments,
       "failed_experiments": self.failed_experiments,
@@ -91,7 +91,7 @@ class UploadState:
   def from_dict(cls, data: Dict[str, Any]) -> "UploadState":
     """Create from dictionary."""
     return cls(
-      local_path=data["local_path"],
+      dash_root=data["dash_root"],
       remote_url=data["remote_url"],
       completed_experiments=data.get("completed_experiments", []),
       failed_experiments=data.get("failed_experiments", []),
@@ -265,18 +265,18 @@ def add_parser(subparsers) -> argparse.ArgumentParser:
 
 
 def discover_experiments(
-  local_path: Path,
+  dash_root: Path,
   project_filter: Optional[str] = None,
   experiment_filter: Optional[str] = None,
 ) -> List[ExperimentInfo]:
   """
   Discover experiments in local storage directory.
 
-  Supports both flat (local_path/project/experiment) and folder-based
-  (local_path/folder/project/experiment) hierarchies.
+  Supports both flat (dash_root/project/experiment) and folder-based
+  (dash_root/folder/project/experiment) hierarchies.
 
   Args:
-      local_path: Root path of local storage
+      dash_root: Root path of local storage
       project_filter: Either a simple project name (e.g., "proj1") or a glob
                      pattern for the full path (e.g., "tom/*/exp*"). If the
                      filter contains '/', '*', or '?', it's treated as a glob
@@ -289,15 +289,15 @@ def discover_experiments(
   """
   import fnmatch
 
-  local_path = Path(local_path)
+  dash_root = Path(dash_root)
 
-  if not local_path.exists():
+  if not dash_root.exists():
     return []
 
   experiments = []
 
   # Find all experiment.json files recursively
-  for exp_json in local_path.rglob("*/experiment.json"):
+  for exp_json in dash_root.rglob("*/experiment.json"):
     exp_dir = exp_json.parent
 
     # Read prefix from experiment.json first
@@ -313,7 +313,7 @@ def discover_experiments(
     # This handles nested folders correctly
     # Prefix format: owner/project/folder.../experiment
     try:
-      relative_path = exp_dir.relative_to(local_path)
+      relative_path = exp_dir.relative_to(dash_root)
       full_relative_path = str(relative_path)
 
       if prefix:
@@ -1234,9 +1234,9 @@ def cmd_upload(args: argparse.Namespace) -> int:
   api_key = args.api_key or config.api_key
 
   # Discover experiments
-  local_path = Path(args.path)
-  if not local_path.exists():
-    console.print(f"[red]Error:[/red] Local storage path does not exist: {local_path}")
+  dash_root = Path(args.path)
+  if not dash_root.exists():
+    console.print(f"[red]Error:[/red] Local storage path does not exist: {dash_root}")
     return 1
 
   # Handle state file for resume functionality
@@ -1247,7 +1247,7 @@ def cmd_upload(args: argparse.Namespace) -> int:
     upload_state = UploadState.load(state_file)
     if upload_state:
       # Validate state matches current upload
-      if upload_state.local_path != str(local_path.absolute()):
+      if upload_state.dash_root != str(dash_root.absolute()):
         console.print(
           "[yellow]Warning:[/yellow] State file local path doesn't match. Starting fresh upload."
         )
@@ -1273,13 +1273,13 @@ def cmd_upload(args: argparse.Namespace) -> int:
   # Create new state if not resuming
   if not upload_state:
     upload_state = UploadState(
-      local_path=str(local_path.absolute()),
+      dash_root=str(dash_root.absolute()),
       remote_url=remote_url,
     )
 
-  console.print(f"[bold]Scanning local storage:[/bold] {local_path.absolute()}")
+  console.print(f"[bold]Scanning local storage:[/bold] {dash_root.absolute()}")
   experiments = discover_experiments(
-    local_path,
+    dash_root,
     project_filter=args.pref,  # Using --prefix/-p argument
     experiment_filter=None,
   )
@@ -1398,7 +1398,7 @@ def cmd_upload(args: argparse.Namespace) -> int:
 
   # Initialize remote client and local storage
   remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
-  local_storage = LocalStorage(root_path=local_path)
+  local_storage = LocalStorage(root_path=dash_root)
 
   # Upload experiments with progress tracking
   console.print(f"\n[bold]Uploading to:[/bold] {remote_url}")

ml_dash/client.py

@@ -369,7 +369,7 @@ class RemoteClient:
             raise ValueError(f"Project '{project_slug}' not found in namespace '{self.namespace}'")
 
         # Delete using project-specific endpoint
-        response = self._client.delete(f"/projects/{project_id}")
+        response = self._client.delete(f"projects/{project_id}")
         response.raise_for_status()
         return response.json()
 
@@ -463,7 +463,7 @@ class RemoteClient:
                     if not project_id:
                         # Create the project first since we need its ID for folders
                         project_response = self._client.post(
-                            f"/namespaces/{self.namespace}/nodes",
+                            f"namespaces/{self.namespace}/nodes",
                             json={
                                 "type": "PROJECT",
                                 "name": project,
@@ -483,7 +483,7 @@ class RemoteClient:
                             # Create folder (server handles upsert)
                             # NOTE: Do NOT pass experimentId for project-level folders
                             folder_response = self._client.post(
-                                f"/namespaces/{self.namespace}/nodes",
+                                f"namespaces/{self.namespace}/nodes",
                                 json={
                                     "type": "FOLDER",
                                     "projectId": project_id,
@@ -525,7 +525,7 @@ class RemoteClient:
 
         # Call unified node creation API
         response = self._client.post(
-            f"/namespaces/{self.namespace}/nodes",
+            f"namespaces/{self.namespace}/nodes",
             json=payload,
         )
         response.raise_for_status()
@@ -565,7 +565,7 @@ class RemoteClient:
         payload = {"status": status}
 
         response = self._client.patch(
-            f"/nodes/{node_id}",
+            f"nodes/{node_id}",
             json=payload,
         )
         response.raise_for_status()
@@ -602,7 +602,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.post(
-            f"/experiments/{experiment_id}/logs",
+            f"experiments/{experiment_id}/logs",
             json={"logs": logs}
         )
         response.raise_for_status()
@@ -638,7 +638,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.post(
-            f"/experiments/{experiment_id}/parameters",
+            f"experiments/{experiment_id}/parameters",
             json={"data": data}
         )
         response.raise_for_status()
@@ -658,7 +658,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails or parameters don't exist
         """
-        response = self._client.get(f"/experiments/{experiment_id}/parameters")
+        response = self._client.get(f"experiments/{experiment_id}/parameters")
         response.raise_for_status()
         result = response.json()
         return result.get("data", {})
@@ -935,7 +935,7 @@ class RemoteClient:
 
                     # Create folder (server will return existing if duplicate)
                     folder_response = self._client.post(
-                        f"/namespaces/{self.namespace}/nodes",
+                        f"namespaces/{self.namespace}/nodes",
                         json={
                             "type": "FOLDER",
                             "projectId": project_id,
@@ -974,7 +974,7 @@ class RemoteClient:
 
         # Call unified node creation API
         response = self._client.post(
-            f"/namespaces/{self.namespace}/nodes",
+            f"namespaces/{self.namespace}/nodes",
             files=files,
             data=data
         )
@@ -1092,7 +1092,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         # file_id is actually the node ID in the new system
-        response = self._client.get(f"/nodes/{file_id}")
+        response = self._client.get(f"nodes/{file_id}")
         response.raise_for_status()
         return response.json()
 
@@ -1127,7 +1127,7 @@ class RemoteClient:
             dest_path = filename
 
         # Download file using node API
-        response = self._client.get(f"/nodes/{file_id}/download")
+        response = self._client.get(f"nodes/{file_id}/download")
         response.raise_for_status()
 
         # Write to file
@@ -1159,7 +1159,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        response = self._client.delete(f"/nodes/{file_id}")
+        response = self._client.delete(f"nodes/{file_id}")
         response.raise_for_status()
         return response.json()
 
@@ -1196,7 +1196,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.patch(
-            f"/nodes/{file_id}",
+            f"nodes/{file_id}",
             json=payload
         )
         response.raise_for_status()
@@ -1237,7 +1237,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/append",
+            f"experiments/{experiment_id}/metrics/{metric_name}/append",
             json=payload
         )
         response.raise_for_status()
@@ -1278,7 +1278,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/append-batch",
+            f"experiments/{experiment_id}/metrics/{metric_name}/append-batch",
             json=payload
         )
         response.raise_for_status()
@@ -1307,7 +1307,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/data",
+            f"experiments/{experiment_id}/metrics/{metric_name}/data",
             params={"startIndex": start_index, "limit": limit}
         )
         response.raise_for_status()
@@ -1332,7 +1332,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/stats"
+            f"experiments/{experiment_id}/metrics/{metric_name}/stats"
         )
         response.raise_for_status()
         return response.json()
@@ -1353,7 +1353,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        response = self._client.get(f"/experiments/{experiment_id}/metrics")
+        response = self._client.get(f"experiments/{experiment_id}/metrics")
         response.raise_for_status()
         return response.json()["metrics"]
 
@@ -1668,7 +1668,7 @@ class RemoteClient:
         expected_checksum = file_metadata.get("physicalFile", {}).get("checksum")
 
         # Stream download using node API
-        with self._client.stream("GET", f"/nodes/{file_id}/download") as response:
+        with self._client.stream("GET", f"nodes/{file_id}/download") as response:
             response.raise_for_status()
 
             with open(dest_path, "wb") as f:
@@ -1736,7 +1736,7 @@ class RemoteClient:
         if search is not None:
             params["search"] = search
 
-        response = self._client.get(f"/experiments/{experiment_id}/logs", params=params)
+        response = self._client.get(f"experiments/{experiment_id}/logs", params=params)
         response.raise_for_status()
         return response.json()
 
@@ -1774,7 +1774,7 @@ class RemoteClient:
             params["bufferOnly"] = "true"
 
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/data",
+            f"experiments/{experiment_id}/metrics/{metric_name}/data",
             params=params
         )
         response.raise_for_status()
@@ -1801,7 +1801,7 @@ class RemoteClient:
             httpx.HTTPStatusError: If request fails
         """
         response = self._client.get(
-            f"/experiments/{experiment_id}/metrics/{metric_name}/chunks/{chunk_number}"
+            f"experiments/{experiment_id}/metrics/{metric_name}/chunks/{chunk_number}"
         )
         response.raise_for_status()
         return response.json()
@@ -1845,7 +1845,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/tracks",
+            f"experiments/{experiment_id}/tracks",
             json=payload,
         )
         response.raise_for_status()
@@ -1878,7 +1878,7 @@ class RemoteClient:
         topic_encoded = urllib.parse.quote(topic, safe='')
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/tracks/{topic_encoded}/append",
+            f"experiments/{experiment_id}/tracks/{topic_encoded}/append",
             json={"timestamp": timestamp, "data": data},
         )
         response.raise_for_status()
@@ -1912,7 +1912,7 @@ class RemoteClient:
         serialized_entries = [_serialize_value(entry) for entry in entries]
 
         response = self._client.post(
-            f"/experiments/{experiment_id}/tracks/{topic_encoded}/append_batch",
+            f"experiments/{experiment_id}/tracks/{topic_encoded}/append_batch",
             json={"entries": serialized_entries},
         )
         response.raise_for_status()
@@ -1957,7 +1957,7 @@ class RemoteClient:
             params["columns"] = ",".join(columns)
 
         response = self._client.get(
-            f"/experiments/{experiment_id}/tracks/{topic_encoded}/data",
+            f"experiments/{experiment_id}/tracks/{topic_encoded}/data",
             params=params,
         )
         response.raise_for_status()
@@ -1990,7 +1990,7 @@ class RemoteClient:
             params["topic"] = topic_filter
 
         response = self._client.get(
-            f"/experiments/{experiment_id}/tracks",
+            f"experiments/{experiment_id}/tracks",
             params=params,
         )
         response.raise_for_status()

{ml_dash-0.6.11.dist-info → ml_dash-0.6.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ml-dash
-Version: 0.6.11
+Version: 0.6.12
 Summary: ML experiment tracking and data storage
 Keywords: machine-learning,experiment-tracking,mlops,data-storage
 Author: Ge Yang, Tom Tao

{ml_dash-0.6.11.dist-info → ml_dash-0.6.12.dist-info}/RECORD

@@ -6,7 +6,7 @@ ml_dash/auth/device_secret.py,sha256=qUsz6M9S1GEIukvmz57eJEp57srSx74O4MU9mZEeDlE
 ml_dash/auth/exceptions.py,sha256=IeBwUzoaTyFtPwd4quFOIel49inIzuabe_ChEeEXEWI,725
 ml_dash/auth/token_storage.py,sha256=9YQXGrn41UVyc1wUvZYbTYLzxSt5NGOyNFNjeX28bjA,7976
 ml_dash/auto_start.py,sha256=mYNjLGI2kyylIfOX5wGOR74gb9UlXg1n5OUQu7aw5SE,2412
-ml_dash/buffer.py,sha256=i4-PZ703_yuKJPAXpmWBGm8jHAAdIBqiA0NIZQEc3wo,26201
+ml_dash/buffer.py,sha256=R3QaD55s1an3tJvzyXfrEuBxeJsN4o4QNdE__BoIr9w,26437
 ml_dash/cli.py,sha256=Vd0taM5MQrhvxqL2KQhklZ00wxZLdWl6Qw1IPkjPNuw,2897
 ml_dash/cli_commands/__init__.py,sha256=bjAmV7MsW-bhtW_4SnLJ0Cfkt9h82vMDC8ebW1Ke8KE,38
 ml_dash/cli_commands/api.py,sha256=NekZEJGWNpIfB6YrsrOw7kw7rZKjVudwgJWPZIy6ANQ,4535
@@ -17,8 +17,8 @@ ml_dash/cli_commands/login.py,sha256=zX-urtUrfzg2qOGtKNYQgj6UloN9kzj4zEO6h_xwuNs
 ml_dash/cli_commands/logout.py,sha256=lTUUNyRXqvo61qNkCd4KBrPUujDAHnNqsHkU6bHie0U,1332
 ml_dash/cli_commands/profile.py,sha256=PoRO1XA4bnOINptj4AO0SyNDBADeryPJBfgC74327e4,5997
 ml_dash/cli_commands/remove.py,sha256=AtDlUWkNeGcnZWN0Wbg6XoyYhFHkCFMPdxsGA33v38c,5325
-ml_dash/cli_commands/upload.py,sha256=SSUfXC3qoNpoFvPM_ia-ing_N50LNiAvMy9op6FM9Ew,49664
-ml_dash/client.py,sha256=5mNdoPfMsUaL-3TqjC1DuxvNLS_6kHIO1X42JVkbmCg,66636
+ml_dash/cli_commands/upload.py,sha256=8i1ZVkQQ7VBzGV3GxlW-71GaETtTdUshexII2GTz5JM,49640
+ml_dash/client.py,sha256=sQokhvofq56JxpmUBVUsfjjn5qKqyUXM9F3uxsUbUbA,66608
 ml_dash/config.py,sha256=oz2xvoBh2X_xUXWr92cPD5nFxXMT5LxVNypv5B5O0fA,3116
 ml_dash/experiment.py,sha256=JqNYsoAOsvqIRMVl6gHFodlZ79b0X2rUFGz6NRixKic,43726
 ml_dash/files.py,sha256=tGJCTxPfd9vmfvIEqstZjzLvqmHzMZffPXHz0jU9bYU,54441
@@ -31,7 +31,7 @@ ml_dash/run.py,sha256=Hlt_YHaN95TC3TDejzLjFmW9EWyKWi6ruibw-eiPm2U,8833
 ml_dash/snowflake.py,sha256=14rEpRU5YltsmmmZW0EMUy_hdv5S5ME9gWVtmdmwfiU,4917
 ml_dash/storage.py,sha256=x1W-dK6wQY36-LVOJ4kA8Dn07ObNQuIErQWJ3b0PoGY,44910
 ml_dash/track.py,sha256=Dfg1ZnmKZ_FlE5ZfG8Qld_wN4RIMs3nrOxrxwf3thiY,8164
-ml_dash-0.6.11.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
-ml_dash-0.6.11.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
-ml_dash-0.6.11.dist-info/METADATA,sha256=4xuJSa68MYT0HDlOzowdRFOHC61ASJC9AN2Sg-zri54,9536
-ml_dash-0.6.11.dist-info/RECORD,,
+ml_dash-0.6.12.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
+ml_dash-0.6.12.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
+ml_dash-0.6.12.dist-info/METADATA,sha256=u31huYFuxz_NMIuFiaUyE373GXMRbYCzpBB61D6KFu4,9536
+ml_dash-0.6.12.dist-info/RECORD,,