ml-dash 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

ml_dash/__init__.py

@@ -43,7 +43,7 @@ from .params import ParametersBuilder
 from .run import RUN
 from .storage import LocalStorage
 
-__version__ = "0.6.3"
+__version__ = "0.6.4"
 
 __all__ = [
   "Experiment",

ml_dash/cli_commands/api.py

@@ -56,6 +56,12 @@ Notes:
     type=str,
     help="ML-Dash server URL (default: https://api.dash.ml)",
   )
+  parser.add_argument(
+    "--namespace",
+    type=str,
+    required=True,
+    help="Namespace to use for queries (required)",
+  )
 
 
 def extract_path(data, path: str):
@@ -131,9 +137,12 @@ def cmd_api(args) -> int:
   # Get remote URL
   remote_url = args.dash_url or config.remote_url or "https://api.dash.ml"
 
+  # Get namespace
+  namespace = args.namespace
+
   try:
     # Initialize client
-    client = RemoteClient(base_url=remote_url)
+    client = RemoteClient(base_url=remote_url, namespace=namespace)
 
     # Determine query type and build query
     if args.mutation:

ml_dash/cli_commands/download.py

@@ -240,7 +240,9 @@ class ExperimentDownloader:
     """Get thread-local remote client for safe concurrent access."""
     if not hasattr(self._thread_local, "client"):
       self._thread_local.client = RemoteClient(
-        base_url=self.remote.base_url, api_key=self.remote.api_key
+        base_url=self.remote.base_url,
+        namespace=self.remote.namespace,
+        api_key=self.remote.api_key
       )
     return self._thread_local.client
 
@@ -630,8 +632,23 @@ def cmd_download(args: argparse.Namespace) -> int:
     console.print("[red]Error:[/red] --dash-url is required (or set in config)")
     return 1
 
+  # Extract namespace from project argument
+  namespace = None
+  if args.project:
+    # Parse namespace from project filter (format: "owner/project" or "owner/project/exp")
+    project_parts = args.project.strip("/").split("/")
+    if len(project_parts) >= 2:  # Has at least "owner/project"
+      namespace = project_parts[0]
+
+  if not namespace:
+    console.print(
+      "[red]Error:[/red] --project must be in format 'namespace/project' or 'namespace/project/exp'"
+    )
+    console.print("Example: ml-dash download --project alice/my-project")
+    return 1
+
   # Initialize clients (RemoteClient will auto-load token if api_key is None)
-  remote_client = RemoteClient(base_url=remote_url, api_key=api_key)
+  remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
   local_storage = LocalStorage(root_path=Path(args.path))
 
   # Load or create state

ml_dash/cli_commands/list.py

@@ -260,9 +260,26 @@ def cmd_list(args: argparse.Namespace) -> int:
     # Get API key (command line > config > auto-loaded from storage)
     api_key = args.api_key or config.api_key
 
+    # Extract namespace from project argument
+    namespace = None
+    if args.project:
+        # Parse namespace from project filter (format: "namespace/project")
+        project_parts = args.project.strip("/").split("/")
+        # For simple patterns without '/', treat as project-only pattern
+        if '/' in args.project and len(project_parts) >= 2:
+            namespace = project_parts[0]
+
+    if not namespace:
+        console.print(
+            "[red]Error:[/red] --project must be in format 'namespace/project'"
+        )
+        console.print("Example: ml-dash list --project alice/my-project")
+        console.print("Or use glob patterns: ml-dash list --project alice/proj-*")
+        return 1
+
     # Create remote client
     try:
-        remote_client = RemoteClient(base_url=remote_url, api_key=api_key)
+        remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
     except Exception as e:
         console.print(f"[red]Error connecting to remote:[/red] {e}")
         return 1

ml_dash/cli_commands/upload.py

@@ -632,7 +632,9 @@ class ExperimentUploader:
       # Create a new client for this thread
       # Use graphql_base_url (without /api) since RemoteClient.__init__ will add /api
       self._thread_local.client = RemoteClient(
-        base_url=self.remote.graphql_base_url, api_key=self.remote.api_key
+        base_url=self.remote.graphql_base_url,
+        namespace=self.remote.namespace,
+        api_key=self.remote.api_key
       )
     return self._thread_local.client
 
@@ -1231,8 +1233,27 @@ def cmd_upload(args: argparse.Namespace) -> int:
     f"[green]{len(valid_experiments)} experiment(s) ready to upload[/green]"
   )
 
+  # Extract namespace from target or first experiment
+  namespace = None
+  if args.target:
+    # Parse namespace from target prefix (format: "owner/project/...")
+    target_parts = args.target.strip("/").split("/")
+    if len(target_parts) >= 1:
+      namespace = target_parts[0]
+  if not namespace and valid_experiments:
+    # Parse namespace from first experiment's prefix
+    first_prefix = valid_experiments[0].prefix
+    if first_prefix:
+      prefix_parts = first_prefix.strip("/").split("/")
+      if len(prefix_parts) >= 1:
+        namespace = prefix_parts[0]
+
+  if not namespace:
+    console.print("[red]Error:[/red] Could not determine namespace from experiments or target")
+    return 1
+
   # Initialize remote client and local storage
-  remote_client = RemoteClient(base_url=remote_url, api_key=api_key)
+  remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
   local_storage = LocalStorage(root_path=local_path)
 
   # Upload experiments with progress tracking

ml_dash/client.py

@@ -9,12 +9,13 @@ import httpx
 class RemoteClient:
     """Client for communicating with ML-Dash server."""
 
-    def __init__(self, base_url: str, api_key: Optional[str] = None):
+    def __init__(self, base_url: str, namespace: str, api_key: Optional[str] = None):
         """
         Initialize remote client.
 
         Args:
             base_url: Base URL of ML-Dash server (e.g., "http://localhost:3000")
+            namespace: Namespace slug (e.g., "my-namespace")
             api_key: JWT token for authentication (optional - auto-loads from storage if not provided)
 
         Note:
@@ -27,6 +28,9 @@ class RemoteClient:
         # Add /api prefix to base URL for REST API calls
         self.base_url = base_url.rstrip("/") + "/api"
 
+        # Store namespace
+        self.namespace = namespace
+
         # If no api_key provided, try to load from storage
         if not api_key:
             from .auth.token_storage import get_token_storage
@@ -37,6 +41,7 @@ class RemoteClient:
         self.api_key = api_key
         self._rest_client = None
         self._gql_client = None
+        self._id_cache: Dict[str, str] = {}  # Cache for slug -> ID mappings
 
     def _ensure_authenticated(self):
         """Check if authenticated, raise error if not."""
@@ -77,6 +82,83 @@ class RemoteClient:
             )
         return self._gql_client
 
+    def _get_project_id(self, project_slug: str) -> Optional[str]:
+        """
+        Resolve project ID from slug using GraphQL.
+
+        Args:
+            project_slug: Project slug
+
+        Returns:
+            Project ID (Snowflake ID) if found, None if not found
+            When None is returned, the server will auto-create the project
+        """
+        cache_key = f"project:{self.namespace}:{project_slug}"
+        if cache_key in self._id_cache:
+            return self._id_cache[cache_key]
+
+        query = """
+        query GetProject($namespace: String!) {
+          namespace(slug: $namespace) {
+            projects {
+              id
+              slug
+            }
+          }
+        }
+        """
+        result = self.graphql_query(query, {
+            "namespace": self.namespace
+        })
+
+        namespace_data = result.get("namespace")
+        if namespace_data is None:
+            raise ValueError(f"Namespace '{self.namespace}' not found. Please check the namespace exists on the server.")
+
+        projects = namespace_data.get("projects", [])
+        for project in projects:
+            if project["slug"] == project_slug:
+                project_id = project["id"]
+                self._id_cache[cache_key] = project_id
+                return project_id
+
+        # Project not found - return None to let server auto-create it
+        return None
+
+    def _get_experiment_node_id(self, experiment_id: str) -> str:
+        """
+        Resolve node ID from experiment ID using GraphQL.
+
+        Args:
+            experiment_id: Experiment ID
+
+        Returns:
+            Node ID
+
+        Raises:
+            ValueError: If experiment node not found
+        """
+        cache_key = f"exp_node:{experiment_id}"
+        if cache_key in self._id_cache:
+            return self._id_cache[cache_key]
+
+        query = """
+        query GetExperimentNode($experimentId: ID!) {
+          experimentNode(experimentId: $experimentId) {
+            id
+          }
+        }
+        """
+        result = self.graphql_query(query, {"experimentId": experiment_id})
+
+        node = result.get("experimentNode")
+        if not node:
+            raise ValueError(f"No node found for experiment ID '{experiment_id}'")
+
+        node_id = node["id"]
+        self._id_cache[cache_key] = node_id
+        return node_id
+
     def create_or_update_experiment(
         self,
         project: str,
@@ -89,28 +171,99 @@ class RemoteClient:
         metadata: Optional[Dict[str, Any]] = None,
     ) -> Dict[str, Any]:
         """
-        Create or update an experiment.
+        Create or update an experiment using unified node API.
 
         Args:
-            project: Project name
-            name: Experiment name (last segment of prefix)
+            project: Project slug
+            name: Experiment name
             description: Optional description
             tags: Optional list of tags
             bindrs: Optional list of bindrs
-            prefix: Full prefix path sent to backend for folder hierarchy creation
+            prefix: Full prefix path (ignored in new API - use folders instead)
             write_protected: If True, experiment becomes immutable
             metadata: Optional metadata dict
 
         Returns:
-            Response dict with experiment, project, and namespace data
+            Response dict with experiment, node, and project data
+            Note: Project will be auto-created if it doesn't exist
 
         Raises:
             httpx.HTTPStatusError: If request fails
         """
+        # Resolve project ID from slug (returns None if not found)
+        project_id = self._get_project_id(project)
+
+        # Parse prefix to create folder hierarchy for experiment
+        # prefix format: "namespace/project/folder1/folder2/experiment_name"
+        # We need to create folders: folder1 -> folder2 and place experiment under folder2
+        parent_id = "ROOT"
+
+        if prefix:
+            # Parse prefix to extract folder path
+            parts = prefix.strip('/').split('/')
+            # parts: [namespace, project, folder1, folder2, ..., experiment_name]
+
+            if len(parts) >= 3:
+                # We have at least namespace/project/something
+                # Extract folder parts (everything between project and experiment name)
+                # Skip namespace (parts[0]) and project (parts[1])
+                # Skip experiment name (parts[-1])
+                folder_parts = parts[2:-1] if len(parts) > 3 else []
+
+                if folder_parts:
+                    # Ensure we have a project_id for folder creation
+                    if not project_id:
+                        # Create the project first since we need its ID for folders
+                        project_response = self._client.post(
+                            f"/namespaces/{self.namespace}/nodes",
+                            json={
+                                "type": "PROJECT",
+                                "name": project,
+                                "slug": project,
+                            }
+                        )
+                        project_response.raise_for_status()
+                        project_data = project_response.json()
+                        project_id = project_data.get("project", {}).get("id")
+
+                    if project_id:
+                        # Create folder hierarchy
+                        current_parent_id = "ROOT"
+                        for folder_name in folder_parts:
+                            if not folder_name:
+                                continue
+                            # Create folder (server handles upsert)
+                            # NOTE: Do NOT pass experimentId for project-level folders
+                            folder_response = self._client.post(
+                                f"/namespaces/{self.namespace}/nodes",
+                                json={
+                                    "type": "FOLDER",
+                                    "projectId": project_id,
+                                    "parentId": current_parent_id,
+                                    "name": folder_name
+                                    # experimentId intentionally omitted - these are project-level folders
+                                }
+                            )
+                            folder_response.raise_for_status()
+                            folder_data = folder_response.json()
+                            current_parent_id = folder_data.get("node", {}).get("id")
+
+                        # Update parent_id for experiment
+                        parent_id = current_parent_id
+
+        # Build payload for unified node API
         payload = {
+            "type": "EXPERIMENT",
             "name": name,
+            "parentId": parent_id,
         }
 
+        # Send projectId if available, otherwise projectSlug (server will auto-create)
+        if project_id:
+            payload["projectId"] = project_id
+        else:
+            payload["projectSlug"] = project
+
         if description is not None:
             payload["description"] = description
         if tags is not None:
@@ -121,15 +274,22 @@ class RemoteClient:
             payload["writeProtected"] = write_protected
         if metadata is not None:
             payload["metadata"] = metadata
-        if prefix is not None:
-            payload["prefix"] = prefix
 
+        # Call unified node creation API
         response = self._client.post(
-            f"/projects/{project}/experiments",
+            f"/namespaces/{self.namespace}/nodes",
             json=payload,
         )
         response.raise_for_status()
-        return response.json()
+        result = response.json()
+
+        # Cache the experiment node ID mapping
+        if "experiment" in result and "node" in result:
+            exp_id = result["experiment"]["id"]
+            node_id = result["node"]["id"]
+            self._id_cache[f"exp_node:{exp_id}"] = node_id
+
+        return result
 
     def update_experiment_status(
         self,
@@ -137,24 +297,27 @@ class RemoteClient:
         status: str,
     ) -> Dict[str, Any]:
         """
-        Update experiment status.
+        Update experiment status using unified node API.
 
         Args:
             experiment_id: Experiment ID
             status: Status value - "RUNNING" | "COMPLETED" | "FAILED" | "CANCELLED"
 
         Returns:
-            Response dict with updated experiment data
+            Response dict with updated node data
 
         Raises:
             httpx.HTTPStatusError: If request fails
+            ValueError: If experiment node not found
         """
-        payload = {
-            "status": status,
-        }
+        # Resolve node ID from experiment ID
+        node_id = self._get_experiment_node_id(experiment_id)
+
+        # Update node with new status
+        payload = {"status": status}
 
         response = self._client.patch(
-            f"/experiments/{experiment_id}/status",
+            f"/nodes/{node_id}",
             json=payload,
         )
         response.raise_for_status()
@@ -263,15 +426,20 @@ class RemoteClient:
         metadata: Optional[Dict[str, Any]],
         checksum: str,
         content_type: str,
-        size_bytes: int
+        size_bytes: int,
+        project_id: Optional[str] = None,
+        parent_id: str = "ROOT"
     ) -> Dict[str, Any]:
         """
-        Upload a file to an experiment.
+        Upload a file to an experiment using unified node API.
 
         Args:
             experiment_id: Experiment ID (Snowflake ID)
             file_path: Local file path
-            prefix: Logical path prefix
+            prefix: Logical path prefix for folder structure (e.g., "models/checkpoints")
+                   Will create nested folders automatically. May include namespace/project
+                   parts which will be stripped automatically (e.g., "ns/proj/folder1/folder2"
+                   will create folders: folder1 -> folder2)
             filename: Original filename
             description: Optional description
             tags: Optional tags
@@ -279,23 +447,274 @@ class RemoteClient:
             checksum: SHA256 checksum
             content_type: MIME type
             size_bytes: File size in bytes
+            project_id: Project ID (optional - will be resolved from experiment if not provided)
+            parent_id: Parent node ID (folder) or "ROOT" for root level.
+                      If prefix is provided, folders will be created under this parent.
 
         Returns:
-            File metadata dict
+            Response dict with node and physicalFile data
 
         Raises:
             httpx.HTTPStatusError: If request fails
         """
+        # If project_id not provided, need to resolve it from experiment
+        # For now, assuming we have it or it will be queried separately
+        if project_id is None:
+            # Query experiment to get project ID
+            query = """
+            query GetExperimentProject($experimentId: ID!) {
+              experimentById(id: $experimentId) {
+                projectId
+              }
+            }
+            """
+            result = self.graphql_query(query, {"experimentId": experiment_id})
+            project_id = result.get("experimentById", {}).get("projectId")
+            if not project_id:
+                raise ValueError(f"Could not resolve project ID for experiment {experiment_id}")
+
+        # Resolve experiment node ID (files should be children of the experiment node, not ROOT)
+        # Check cache first, otherwise query
+        experiment_node_id = self._id_cache.get(f"exp_node:{experiment_id}")
+        if not experiment_node_id:
+            # Query to get the experiment node ID
+            query = """
+            query GetExperimentNode($experimentId: ID!) {
+              experimentById(id: $experimentId) {
+                id
+              }
+            }
+            """
+            # Note: experimentById returns the Experiment record, not the Node
+            # We need to find the Node with type=EXPERIMENT and experimentId=experiment_id
+            # Use the project nodes query instead
+            query = """
+            query GetExperimentNode($projectId: ID!, $experimentId: ID!) {
+              project(id: $projectId) {
+                nodes(parentId: null, maxDepth: 10) {
+                  id
+                  type
+                  experimentId
+                  children {
+                    id
+                    type
+                    experimentId
+                    children {
+                      id
+                      type
+                      experimentId
+                    }
+                  }
+                }
+              }
+            }
+            """
+            result = self.graphql_query(query, {"projectId": project_id, "experimentId": experiment_id})
+
+            # Find the experiment node
+            def find_experiment_node(nodes, exp_id):
+                for node in nodes:
+                    if node.get("type") == "EXPERIMENT" and node.get("experimentId") == exp_id:
+                        return node.get("id")
+                    if node.get("children"):
+                        found = find_experiment_node(node["children"], exp_id)
+                        if found:
+                            return found
+                return None
+
+            project_nodes = result.get("project", {}).get("nodes", [])
+            experiment_node_id = find_experiment_node(project_nodes, experiment_id)
+
+            if experiment_node_id:
+                # Cache it for future uploads
+                self._id_cache[f"exp_node:{experiment_id}"] = experiment_node_id
+            else:
+                # Fallback to ROOT if we can't find the experiment node
+                # This might happen for old experiments or legacy data
+                experiment_node_id = "ROOT"
+
+        # Get experiment node path to strip from prefix
+        # When we use experiment_node_id as parent, we need to strip the experiment's
+        # folder path from the prefix to avoid creating duplicate folders
+        # We'll cache this in the id_cache to avoid repeated queries
+        cache_key = f"exp_folder_path:{experiment_id}"
+        experiment_folder_path = self._id_cache.get(cache_key)
+
+        if experiment_folder_path is None and experiment_node_id != "ROOT":
+            # Query experiment to get its project info for the GraphQL query
+            exp_query = """
+            query GetExpInfo($experimentId: ID!) {
+              experimentById(id: $experimentId) {
+                project {
+                  slug
+                  namespace {
+                    slug
+                  }
+                }
+              }
+            }
+            """
+            exp_result = self.graphql_query(exp_query, {"experimentId": experiment_id})
+            project_slug = exp_result.get("experimentById", {}).get("project", {}).get("slug")
+            namespace_slug = exp_result.get("experimentById", {}).get("project", {}).get("namespace", {}).get("slug")
+
+            if project_slug and namespace_slug:
+                # Query to get the experiment node's path
+                # This includes all ancestor folders up to the experiment
+                query = """
+                query GetExperimentPath($namespaceSlug: String!, $projectSlug: String!) {
+                  project(namespaceSlug: $namespaceSlug, projectSlug: $projectSlug) {
+                    nodes(parentId: null, maxDepth: 10) {
+                      id
+                      name
+                      type
+                      experimentId
+                      parentId
+                      children {
+                        id
+                        name
+                        type
+                        experimentId
+                        parentId
+                        children {
+                          id
+                          name
+                          type
+                          experimentId
+                          parentId
+                        }
+                      }
+                    }
+                  }
+                }
+                """
+                result = self.graphql_query(query, {"namespaceSlug": namespace_slug, "projectSlug": project_slug})
+
+                # Build path to experiment node
+                def find_node_path(nodes, target_id, current_path=None):
+                    if current_path is None:
+                        current_path = []
+                    for node in nodes:
+                        new_path = current_path + [node.get("name")]
+                        if node.get("id") == target_id:
+                            return new_path
+                        if node.get("children"):
+                            found = find_node_path(node["children"], target_id, new_path)
+                            if found:
+                                return found
+                    return None
+
+                project_nodes = result.get("project", {}).get("nodes", [])
+                path_parts = find_node_path(project_nodes, experiment_node_id)
+                if path_parts:
+                    # IMPORTANT: Don't include the experiment node's name itself
+                    # We want the path TO the experiment's parent folder, not the experiment
+                    # E.g., if path is ["examples", "exp-name"], we want "examples"
+                    if len(path_parts) > 1:
+                        experiment_folder_path = "/".join(path_parts[:-1])
+                    else:
+                        # Experiment is at root level, no parent folders
+                        experiment_folder_path = ""
+                    # Cache it
+                    self._id_cache[cache_key] = experiment_folder_path
+                else:
+                    # Couldn't find path, set empty string to avoid re-querying
+                    experiment_folder_path = ""
+                    self._id_cache[cache_key] = experiment_folder_path
+
+        # Use experiment node ID as the parent for file uploads
+        # Files and folders should be children of the experiment node
+        if parent_id == "ROOT" and experiment_node_id != "ROOT":
+            parent_id = experiment_node_id
+
+        # Parse prefix to create folder hierarchy
+        # prefix like "models/checkpoints" should create folders: models -> checkpoints
+        # NOTE: The prefix may contain namespace/project parts (e.g., "ns/proj/folder1/folder2")
+        # We need to strip the namespace and project parts since we're already in an experiment context
+        if prefix and prefix != '/' and prefix.strip():
+            # Clean and normalize prefix
+            prefix = prefix.strip('/')
+
+            # Try to detect and strip namespace/project from prefix
+            # Common patterns: "namespace/project/folders..." or just "folders..."
+            # Since we're in experiment context, we already know the namespace and project
+            # Check if prefix starts with namespace
+            if prefix.startswith(self.namespace + '/'):
+                # Strip namespace
+                prefix = prefix[len(self.namespace) + 1:]
+
+                # Now check if it starts with project slug/name
+                # We need to query the experiment to get the project info
+                query = """
+                query GetExperimentProject($experimentId: ID!) {
+                  experimentById(id: $experimentId) {
+                    project {
+                      slug
+                      name
+                    }
+                  }
+                }
+                """
+                exp_result = self.graphql_query(query, {"experimentId": experiment_id})
+                project_info = exp_result.get("experimentById", {}).get("project", {})
+                project_slug = project_info.get("slug", "")
+                project_name = project_info.get("name", "")
+
+                # Try to strip project slug or name
+                if project_slug and prefix.startswith(project_slug + '/'):
+                    prefix = prefix[len(project_slug) + 1:]
+                elif project_name and prefix.startswith(project_name + '/'):
+                    prefix = prefix[len(project_name) + 1:]
+
+            # Strip experiment folder path from prefix since we're using experiment node as parent
+            # For example: if prefix is "examples/exp1/models" and experiment is at "examples/exp1",
+            # strip "examples/exp1/" to get "models"
+            if experiment_folder_path and prefix.startswith(experiment_folder_path + '/'):
+                prefix = prefix[len(experiment_folder_path) + 1:]
+            elif experiment_folder_path and prefix == experiment_folder_path:
+                # Prefix is exactly the experiment path, no subfolders
+                prefix = ""
+
+            if prefix:
+                folder_parts = prefix.split('/')
+                current_parent_id = parent_id
+
+                # Create or find each folder in the hierarchy
+                # Server handles upsert - will return existing folder if it exists
+                for folder_name in folder_parts:
+                    if not folder_name:  # Skip empty parts
+                        continue
+
+                    # Create folder (server will return existing if duplicate)
+                    folder_response = self._client.post(
+                        f"/namespaces/{self.namespace}/nodes",
+                        json={
+                            "type": "FOLDER",
+                            "projectId": project_id,
+                            "experimentId": experiment_id,
+                            "parentId": current_parent_id,
+                            "name": folder_name
+                        }
+                    )
+                    folder_response.raise_for_status()
+                    folder_data = folder_response.json()
+                    current_parent_id = folder_data.get("node", {}).get("id")
+
+                # Update parent_id to the final folder in the hierarchy
+                parent_id = current_parent_id
+
         # Prepare multipart form data
-        # Read file content first (httpx needs content, not file handle)
         with open(file_path, "rb") as f:
             file_content = f.read()
 
         files = {"file": (filename, file_content, content_type)}
         data = {
-            "prefix": prefix,
+            "type": "FILE",
+            "projectId": project_id,
+            "experimentId": experiment_id,
+            "parentId": parent_id,
+            "name": filename,
             "checksum": checksum,
-            "sizeBytes": str(size_bytes),
         }
         if description:
             data["description"] = description
@@ -305,15 +724,53 @@ class RemoteClient:
             import json
             data["metadata"] = json.dumps(metadata)
 
-        # httpx will automatically set multipart/form-data content-type
+        # Call unified node creation API
         response = self._client.post(
-            f"/experiments/{experiment_id}/files",
+            f"/namespaces/{self.namespace}/nodes",
             files=files,
             data=data
         )
 
         response.raise_for_status()
-        return response.json()
+        result = response.json()
+
+        # Transform unified node response to expected file metadata format
+        # The server returns {node: {...}, physicalFile: {...}}
+        # We need to flatten it to match the expected format
+        node = result.get("node", {})
+        physical_file = result.get("physicalFile", {})
+
+        # Convert BigInt IDs and sizeBytes from string back to appropriate types
+        # Node ID should remain as string for consistency
+        node_id = node.get("id")
+        if isinstance(node_id, (int, float)):
+            # If it was deserialized as a number, convert to string to preserve full precision
+            node_id = str(int(node_id))
+
+        size_bytes = physical_file.get("sizeBytes")
+        if isinstance(size_bytes, str):
+            size_bytes = int(size_bytes)
+
+        # Use experimentId from node, not the parameter (which might be a path string)
+        experiment_id_from_node = node.get("experimentId")
+        if isinstance(experiment_id_from_node, (int, float)):
+            experiment_id_from_node = str(int(experiment_id_from_node))
+
+        return {
+            "id": node_id,
+            "experimentId": experiment_id_from_node or experiment_id,
+            "path": prefix,  # Use prefix as path for backward compatibility
+            "filename": filename,
+            "description": node.get("description"),
+            "tags": node.get("tags", []),
+            "contentType": physical_file.get("contentType"),
+            "sizeBytes": size_bytes,
+            "checksum": physical_file.get("checksum"),
+            "metadata": node.get("metadata"),
+            "uploadedAt": node.get("createdAt"),
+            "updatedAt": node.get("updatedAt"),
+            "deletedAt": node.get("deletedAt"),
+        }
 
     def list_files(
         self,
@@ -322,48 +779,72 @@ class RemoteClient:
         tags: Optional[List[str]] = None
     ) -> List[Dict[str, Any]]:
         """
-        List files in an experiment.
+        List files in an experiment using GraphQL.
 
         Args:
             experiment_id: Experiment ID (Snowflake ID)
-            prefix: Optional prefix filter
+            prefix: Optional prefix filter (DEPRECATED - filtering not supported in new API)
             tags: Optional tags filter
 
         Returns:
-            List of file metadata dicts
+            List of file node dicts
 
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        params = {}
-        if prefix:
-            params["prefix"] = prefix
+        query = """
+        query ListExperimentFiles($experimentId: ID!) {
+          experimentById(id: $experimentId) {
+            files {
+              id
+              name
+              description
+              tags
+              metadata
+              createdAt
+              pPath
+              physicalFile {
+                id
+                filename
+                contentType
+                sizeBytes
+                checksum
+                s3Url
+              }
+            }
+          }
+        }
+        """
+        result = self.graphql_query(query, {"experimentId": experiment_id})
+        files = result.get("experimentById", {}).get("files", [])
+
+        # Apply client-side filtering if tags specified
         if tags:
-            params["tags"] = ",".join(tags)
+            filtered_files = []
+            for file in files:
+                file_tags = file.get("tags", [])
+                if any(tag in file_tags for tag in tags):
+                    filtered_files.append(file)
+            return filtered_files
 
-        response = self._client.get(
-            f"/experiments/{experiment_id}/files",
-            params=params
-        )
-        response.raise_for_status()
-        result = response.json()
-        return result.get("files", [])
+        return files
 
     def get_file(self, experiment_id: str, file_id: str) -> Dict[str, Any]:
         """
-        Get file metadata.
+        Get file metadata using unified node API.
 
         Args:
-            experiment_id: Experiment ID (Snowflake ID)
-            file_id: File ID (Snowflake ID)
+            experiment_id: Experiment ID (DEPRECATED - not used in new API)
+            file_id: File node ID (Snowflake ID)
 
         Returns:
-            File metadata dict
+            Node metadata dict
 
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        response = self._client.get(f"/experiments/{experiment_id}/files/{file_id}")
+        # file_id is actually the node ID in the new system
+        response = self._client.get(f"/nodes/{file_id}")
         response.raise_for_status()
         return response.json()
 
@@ -374,11 +855,11 @@ class RemoteClient:
         dest_path: Optional[str] = None
     ) -> str:
         """
-        Download a file from a experiment.
+        Download a file using unified node API.
 
         Args:
-            experiment_id: Experiment ID (Snowflake ID)
-            file_id: File ID (Snowflake ID)
+            experiment_id: Experiment ID (DEPRECATED - not used in new API)
+            file_id: File node ID (Snowflake ID)
             dest_path: Optional destination path (defaults to original filename)
 
         Returns:
@@ -390,40 +871,39 @@ class RemoteClient:
         """
         # Get file metadata first to get filename and checksum
         file_metadata = self.get_file(experiment_id, file_id)
-        filename = file_metadata["filename"]
-        expected_checksum = file_metadata["checksum"]
+        filename = file_metadata.get("name") or file_metadata.get("physicalFile", {}).get("filename")
+        expected_checksum = file_metadata.get("physicalFile", {}).get("checksum")
 
         # Determine destination path
         if dest_path is None:
             dest_path = filename
 
-        # Download file
-        response = self._client.get(
-            f"/experiments/{experiment_id}/files/{file_id}/download"
-        )
+        # Download file using node API
+        response = self._client.get(f"/nodes/{file_id}/download")
         response.raise_for_status()
 
         # Write to file
         with open(dest_path, "wb") as f:
             f.write(response.content)
 
-        # Verify checksum
-        from .files import verify_checksum
-        if not verify_checksum(dest_path, expected_checksum):
-            # Delete corrupted file
-            import os
-            os.remove(dest_path)
-            raise ValueError(f"Checksum verification failed for file {file_id}")
+        # Verify checksum if available
+        if expected_checksum:
+            from .files import verify_checksum
+            if not verify_checksum(dest_path, expected_checksum):
+                # Delete corrupted file
+                import os
+                os.remove(dest_path)
+                raise ValueError(f"Checksum verification failed for file {file_id}")
 
         return dest_path
 
     def delete_file(self, experiment_id: str, file_id: str) -> Dict[str, Any]:
         """
-        Delete a file (soft delete).
+        Delete a file using unified node API (soft delete).
 
         Args:
-            experiment_id: Experiment ID (Snowflake ID)
-            file_id: File ID (Snowflake ID)
+            experiment_id: Experiment ID (DEPRECATED - not used in new API)
+            file_id: File node ID (Snowflake ID)
 
         Returns:
             Dict with id and deletedAt
@@ -431,7 +911,7 @@ class RemoteClient:
         Raises:
             httpx.HTTPStatusError: If request fails
         """
-        response = self._client.delete(f"/experiments/{experiment_id}/files/{file_id}")
+        response = self._client.delete(f"/nodes/{file_id}")
         response.raise_for_status()
         return response.json()
 
@@ -444,17 +924,17 @@ class RemoteClient:
         metadata: Optional[Dict[str, Any]] = None
     ) -> Dict[str, Any]:
         """
-        Update file metadata.
+        Update file metadata using unified node API.
 
         Args:
-            experiment_id: Experiment ID (Snowflake ID)
-            file_id: File ID (Snowflake ID)
+            experiment_id: Experiment ID (DEPRECATED - not used in new API)
+            file_id: File node ID (Snowflake ID)
             description: Optional description
             tags: Optional tags
             metadata: Optional metadata
 
         Returns:
-            Updated file metadata dict
+            Updated node metadata dict
 
         Raises:
             httpx.HTTPStatusError: If request fails
@@ -468,7 +948,7 @@ class RemoteClient:
             payload["metadata"] = metadata
 
         response = self._client.patch(
-            f"/experiments/{experiment_id}/files/{file_id}",
+            f"/nodes/{file_id}",
             json=payload
         )
         response.raise_for_status()
@@ -654,7 +1134,8 @@ class RemoteClient:
         if "errors" in result:
             raise Exception(f"GraphQL errors: {result['errors']}")
 
-        return result.get("data", {})
+        # Handle case where data is explicitly null in response
+        return result.get("data") or {}
 
     def list_projects_graphql(self) -> List[Dict[str, Any]]:
         """
@@ -905,11 +1386,11 @@ class RemoteClient:
         self, experiment_id: str, file_id: str, dest_path: str
     ) -> str:
         """
-        Download a file with streaming for large files.
+        Download a file with streaming for large files using unified node API.
 
         Args:
-            experiment_id: Experiment ID (Snowflake ID)
-            file_id: File ID (Snowflake ID)
+            experiment_id: Experiment ID (DEPRECATED - not used in new API)
+            file_id: File node ID (Snowflake ID)
             dest_path: Destination path to save file
 
         Returns:
@@ -921,22 +1402,23 @@ class RemoteClient:
         """
         # Get metadata first for checksum
         file_metadata = self.get_file(experiment_id, file_id)
-        expected_checksum = file_metadata["checksum"]
+        expected_checksum = file_metadata.get("physicalFile", {}).get("checksum")
 
-        # Stream download
-        with self._client.stream("GET", f"/experiments/{experiment_id}/files/{file_id}/download") as response:
+        # Stream download using node API
+        with self._client.stream("GET", f"/nodes/{file_id}/download") as response:
             response.raise_for_status()
 
             with open(dest_path, "wb") as f:
                 for chunk in response.iter_bytes(chunk_size=8192):
                     f.write(chunk)
 
-        # Verify checksum
-        from .files import verify_checksum
-        if not verify_checksum(dest_path, expected_checksum):
-            import os
-            os.remove(dest_path)
-            raise ValueError(f"Checksum verification failed for file {file_id}")
+        # Verify checksum if available
+        if expected_checksum:
+            from .files import verify_checksum
+            if not verify_checksum(dest_path, expected_checksum):
+                import os
+                os.remove(dest_path)
+                raise ValueError(f"Checksum verification failed for file {file_id}")
 
         return dest_path
 

ml_dash/experiment.py

@@ -350,7 +350,7 @@ class Experiment:
       # RemoteClient will auto-load token from ~/.dash/token.enc
       # Use RUN.api_url if dash_url=True (boolean), otherwise use the provided URL
       api_url = RUN.api_url if dash_url is True else dash_url
-      self._client = RemoteClient(base_url=api_url)
+      self._client = RemoteClient(base_url=api_url, namespace=self.owner)
 
     if self.mode in (OperationMode.LOCAL, OperationMode.HYBRID):
       self._storage = LocalStorage(root_path=Path(dash_root))
@@ -703,22 +703,40 @@ class Experiment:
     # Write immediately (no buffering)
     if self._client:
       # Remote mode: send to API (wrapped in array for batch API)
-      self._client.create_log_entries(
-        experiment_id=self._experiment_id,
-        logs=[log_entry],  # Single log in array
-      )
+      try:
+        self._client.create_log_entries(
+          experiment_id=self._experiment_id,
+          logs=[log_entry],  # Single log in array
+        )
+      except Exception as e:
+        # Log warning but don't crash training
+        import warnings
+        warnings.warn(
+          f"Failed to write log to remote server: {e}. Training will continue.",
+          RuntimeWarning,
+          stacklevel=4
+        )
+        # Fall through to local storage if available
 
     if self._storage:
       # Local mode: write to file immediately
-      self._storage.write_log(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
-        message=log_entry["message"],
-        level=log_entry["level"],
-        metadata=log_entry.get("metadata"),
-        timestamp=log_entry["timestamp"],
-      )
+      try:
+        self._storage.write_log(
+          owner=self.owner,
+          project=self.project,
+          prefix=self._folder_path,
+          message=log_entry["message"],
+          level=log_entry["level"],
+          metadata=log_entry.get("metadata"),
+          timestamp=log_entry["timestamp"],
+        )
+      except Exception as e:
+        import warnings
+        warnings.warn(
+          f"Failed to write log to local storage: {e}",
+          RuntimeWarning,
+          stacklevel=4
+        )
 
   def _print_log(
     self, message: str, level: str, metadata: Optional[Dict[str, Any]]
@@ -1139,7 +1157,7 @@ class Experiment:
     description: Optional[str],
     tags: Optional[List[str]],
     metadata: Optional[Dict[str, Any]],
-  ) -> Dict[str, Any]:
+  ) -> Optional[Dict[str, Any]]:
     """
     Internal method to append a single data point to a metric.
 
@@ -1151,33 +1169,54 @@ class Experiment:
         metadata: Optional metadata
 
     Returns:
-        Dict with metricId, index, bufferedDataPoints, chunkSize
+        Dict with metricId, index, bufferedDataPoints, chunkSize or None if all backends fail
     """
     result = None
 
     if self._client:
       # Remote mode: append via API
-      result = self._client.append_to_metric(
-        experiment_id=self._experiment_id,
-        metric_name=name,
-        data=data,
-        description=description,
-        tags=tags,
-        metadata=metadata,
-      )
+      try:
+        result = self._client.append_to_metric(
+          experiment_id=self._experiment_id,
+          metric_name=name,
+          data=data,
+          description=description,
+          tags=tags,
+          metadata=metadata,
+        )
+      except Exception as e:
+        # Log warning but don't crash training
+        import warnings
+        metric_display = f"'{name}'" if name else "unnamed metric"
+        warnings.warn(
+          f"Failed to log {metric_display} to remote server: {e}. "
+          f"Training will continue.",
+          RuntimeWarning,
+          stacklevel=3
+        )
+        # Fall through to local storage if available
 
     if self._storage:
       # Local mode: append to local storage
-      result = self._storage.append_to_metric(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
-        metric_name=name,
-        data=data,
-        description=description,
-        tags=tags,
-        metadata=metadata,
-      )
+      try:
+        result = self._storage.append_to_metric(
+          owner=self.owner,
+          project=self.project,
+          prefix=self._folder_path,
+          metric_name=name,
+          data=data,
+          description=description,
+          tags=tags,
+          metadata=metadata,
+        )
+      except Exception as e:
+        import warnings
+        metric_display = f"'{name}'" if name else "unnamed metric"
+        warnings.warn(
+          f"Failed to log {metric_display} to local storage: {e}",
+          RuntimeWarning,
+          stacklevel=3
+        )
 
     return result
 
@@ -1188,7 +1227,7 @@ class Experiment:
     description: Optional[str],
     tags: Optional[List[str]],
     metadata: Optional[Dict[str, Any]],
-  ) -> Dict[str, Any]:
+  ) -> Optional[Dict[str, Any]]:
     """
     Internal method to append multiple data points to a metric.
 
@@ -1200,33 +1239,54 @@ class Experiment:
         metadata: Optional metadata
 
     Returns:
-        Dict with metricId, startIndex, endIndex, count
+        Dict with metricId, startIndex, endIndex, count or None if all backends fail
     """
     result = None
 
     if self._client:
       # Remote mode: append batch via API
-      result = self._client.append_batch_to_metric(
-        experiment_id=self._experiment_id,
-        metric_name=name,
-        data_points=data_points,
-        description=description,
-        tags=tags,
-        metadata=metadata,
-      )
+      try:
+        result = self._client.append_batch_to_metric(
+          experiment_id=self._experiment_id,
+          metric_name=name,
+          data_points=data_points,
+          description=description,
+          tags=tags,
+          metadata=metadata,
+        )
+      except Exception as e:
+        # Log warning but don't crash training
+        import warnings
+        metric_display = f"'{name}'" if name else "unnamed metric"
+        warnings.warn(
+          f"Failed to log batch to {metric_display} on remote server: {e}. "
+          f"Training will continue.",
+          RuntimeWarning,
+          stacklevel=3
+        )
+        # Fall through to local storage if available
 
     if self._storage:
       # Local mode: append batch to local storage
-      result = self._storage.append_batch_to_metric(
-        owner=self.owner,
-        project=self.project,
-        prefix=self._folder_path,
-        metric_name=name,
-        data_points=data_points,
-        description=description,
-        tags=tags,
-        metadata=metadata,
-      )
+      try:
+        result = self._storage.append_batch_to_metric(
+          owner=self.owner,
+          project=self.project,
+          prefix=self._folder_path,
+          metric_name=name,
+          data_points=data_points,
+          description=description,
+          tags=tags,
+          metadata=metadata,
+        )
+      except Exception as e:
+        import warnings
+        metric_display = f"'{name}'" if name else "unnamed metric"
+        warnings.warn(
+          f"Failed to log batch to {metric_display} in local storage: {e}",
+          RuntimeWarning,
+          stacklevel=3
+        )
 
     return result
 

{ml_dash-0.6.3.dist-info → ml_dash-0.6.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ml-dash
-Version: 0.6.3
+Version: 0.6.5
 Summary: ML experiment tracking and data storage
 Keywords: machine-learning,experiment-tracking,mlops,data-storage
 Author: Ge Yang, Tom Tao

{ml_dash-0.6.3.dist-info → ml_dash-0.6.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ml_dash/__init__.py,sha256=D5p0zXPS1M1dfD3_jT_NSYI_XwbB_7Q9ZOKN80BUY78,1583
+ml_dash/__init__.py,sha256=XJym_-vgqFVwyAD-VsPZF9WWepTZ4w-Lwui5ns1gmJI,1583
 ml_dash/auth/__init__.py,sha256=3lwM-Y8UBHPU1gFW2JNpmXlPVTnkGudWLKNFFKulQfo,1200
 ml_dash/auth/constants.py,sha256=ku4QzQUMNjvyJwjy7AUdywMAZd59jXSxNHZxDiagUWU,280
 ml_dash/auth/device_flow.py,sha256=DQOdPNlZCuU1umZOA_A6WXdRM3zWphnyo9IntToBl_A,7921
@@ -8,16 +8,16 @@ ml_dash/auth/token_storage.py,sha256=L18W8J7D1LlCDlY3Q32l0RXeNh0o7YVDQeeGYm64Dgw
 ml_dash/auto_start.py,sha256=62_eZG1qBNAwu6AXduTSo4niCVZ27X52ZK0WEr3yS1o,1812
 ml_dash/cli.py,sha256=BoaBulcqnM88XuV5BQEx_-AQAXJAYSJqpvnHggEII_I,2559
 ml_dash/cli_commands/__init__.py,sha256=bjAmV7MsW-bhtW_4SnLJ0Cfkt9h82vMDC8ebW1Ke8KE,38
-ml_dash/cli_commands/api.py,sha256=tgHB3pvSYv36_RbxsAtiEfjtivnIn7NjdHq0AL2QQGo,4335
-ml_dash/cli_commands/download.py,sha256=ZnRhaDLIM28Dri4-YHLU1fBwC9AAvNoiuut3pkdBhJU,27422
-ml_dash/cli_commands/list.py,sha256=9dK0UbNTvysGM5c8Mkb5XfFNkhMIhtjIP1v9BFo-5ew,15400
+ml_dash/cli_commands/api.py,sha256=NekZEJGWNpIfB6YrsrOw7kw7rZKjVudwgJWPZIy6ANQ,4535
+ml_dash/cli_commands/download.py,sha256=LeZXjQSEPIxZALuo90fj8RHjFWIbtGPE0F625sD3cU8,28054
+ml_dash/cli_commands/list.py,sha256=oc_yJXFhsvGgr3JedG2j7747yX69Qc546geIi4DQ54k,16129
 ml_dash/cli_commands/login.py,sha256=zX-urtUrfzg2qOGtKNYQgj6UloN9kzj4zEO6h_xwuNs,6782
 ml_dash/cli_commands/logout.py,sha256=lTUUNyRXqvo61qNkCd4KBrPUujDAHnNqsHkU6bHie0U,1332
 ml_dash/cli_commands/profile.py,sha256=BaSM6BAN3YM4tw95iKV_nypKZxwsB3PoAAejQcYip5E,2351
-ml_dash/cli_commands/upload.py,sha256=Ch1pWC4rU3M9P52Ne_gAlkE7yz4WZKgZlRBG3hpy9_4,44059
-ml_dash/client.py,sha256=TEk-Vt323wBpDPPwX-fFFS7IVF7hS3aBDxn9lewbpls,31455
+ml_dash/cli_commands/upload.py,sha256=_607CcGjvjnwTgGzyxHaDG0qDAlSLlpZDoq6Sy-3paQ,44828
+ml_dash/client.py,sha256=0j4mgr7u9MPIkBL7LR4EqxYYGkFDOSMY6KizF2aNSGA,52848
 ml_dash/config.py,sha256=oz2xvoBh2X_xUXWr92cPD5nFxXMT5LxVNypv5B5O0fA,3116
-ml_dash/experiment.py,sha256=DsEl4q7EksfBApOjd1q4ncX6COSC7Hv2bCeFPbeELC8,39218
+ml_dash/experiment.py,sha256=1uDCKNDlgGkKoogao3sEFz1sUhmiRvX3ZPGoQ7H3ozE,41361
 ml_dash/files.py,sha256=bihUHKpdknytLGuGgkcvhh585nziZrvYjiHl6rHnoD0,49227
 ml_dash/log.py,sha256=E-DLg0vejVLLEyShJ_r0LneDMI0XU7XTH5iKWYJe9jI,5298
 ml_dash/metric.py,sha256=ghD1jnuv6dbjV1Jlo7q0mx9UEzpdto2Y1-oDWrSfg04,25809
@@ -27,7 +27,7 @@ ml_dash/remote_auto_start.py,sha256=5fvQDHv1CWEKFb6WAa5_uyEInwV_SvotXjOO_6i6ZKE,
 ml_dash/run.py,sha256=C0quTLZXKDAlwstzEiJ75CWCX1pwYrmtMZH3z-ia6Pw,6310
 ml_dash/snowflake.py,sha256=14rEpRU5YltsmmmZW0EMUy_hdv5S5ME9gWVtmdmwfiU,4917
 ml_dash/storage.py,sha256=9mG42pvvWkkracbjCr9Xdp890Nm4XSxL7_JeFbBe28g,33020
-ml_dash-0.6.3.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
-ml_dash-0.6.3.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
-ml_dash-0.6.3.dist-info/METADATA,sha256=CUfPUXV3i0CUsxrq80yvbXUe0s8xJUPfLzD6jLozEQ8,7203
-ml_dash-0.6.3.dist-info/RECORD,,
+ml_dash-0.6.5.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
+ml_dash-0.6.5.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
+ml_dash-0.6.5.dist-info/METADATA,sha256=a9v8BibbvXdWfXDTTGWVsLWWTA6iE2DMMwUOZjYH-J8,7203
+ml_dash-0.6.5.dist-info/RECORD,,