ml-dash 0.6.6__py3-none-any.whl → 0.6.9__py3-none-any.whl

ml_dash/cli_commands/download.py

@@ -615,6 +615,158 @@ class ExperimentDownloader:
       return {"success": False, "error": str(e), "bytes": 0}
 
 
+# ============================================================================
+# Track Download Command
+# ============================================================================
+
+
+def cmd_download_track(args: argparse.Namespace) -> int:
+  """Download track data from remote server."""
+  # Load configuration
+  config = Config()
+  remote_url = args.dash_url or config.remote_url
+  api_key = args.api_key or config.api_key
+
+  # Validate inputs
+  if not remote_url:
+    console.print("[red]Error:[/red] --dash-url is required (or set in config)")
+    return 1
+
+  if not args.tracks:
+    console.print("[red]Error:[/red] Track path is required")
+    return 1
+
+  # Parse track path: namespace/project/experiment/topic...
+  # The challenge is determining where experiment ends and topic begins
+  # Strategy: Try to find the experiment by checking server, working backwards from the end
+  track_path = args.tracks.strip("/")
+  parts = track_path.split("/")
+
+  if len(parts) < 4:
+    console.print(
+      "[red]Error:[/red] Track path must be in format: "
+      "'namespace/project/experiment/topic'"
+    )
+    console.print("Examples:")
+    console.print("  geyang/project/experiment/position")
+    console.print("  geyang/project/experiment/robot/position")
+    console.print("  geyang/project/folder/experiment/robot/camera/position")
+    return 1
+
+  # Extract namespace and project (always first 2 parts)
+  namespace = parts[0]
+  project = parts[1]
+
+  # Try to find the experiment by iterating backwards
+  # We assume experiment is a single name (not a path), and topic comes after it
+  experiment_name = None
+  topic = None
+
+  # Initialize client early to test experiment existence
+  try:
+    remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
+  except Exception as e:
+    console.print(f"[red]Error:[/red] Failed to connect to server: {e}")
+    return 1
+
+  # Try different split points: experiment could be at index 2, 3, 4, etc.
+  # Topic is everything after the experiment
+  for exp_idx in range(2, len(parts) - 1):  # Experiment must leave at least 1 part for topic
+    potential_exp_name = parts[exp_idx]
+    potential_topic = "/".join(parts[exp_idx + 1:])
+
+    # Try to fetch this experiment from server
+    try:
+      exp_data = remote_client.get_experiment_graphql(project, potential_exp_name, namespace)
+      if exp_data:
+        # Found the experiment!
+        experiment_name = potential_exp_name
+        topic = potential_topic
+        break
+    except Exception:
+      # This experiment doesn't exist, try next index
+      continue
+
+  if not experiment_name or not topic:
+    console.print(
+      f"[red]Error:[/red] Could not find valid experiment in path: {track_path}"
+    )
+    console.print("\nTried the following experiment names:")
+    for exp_idx in range(2, len(parts) - 1):
+      console.print(f"  - {parts[exp_idx]}")
+    console.print(f"\nMake sure the experiment exists in project '{project}'")
+    return 1
+
+  console.print(f"[bold]Downloading track data...[/bold]")
+  console.print(f"  Namespace: {namespace}")
+  console.print(f"  Project: {project}")
+  console.print(f"  Experiment: {experiment_name}")
+  console.print(f"  Topic: {topic}")
+  console.print(f"  Format: {args.format}")
+
+  try:
+    # Get experiment ID (we already validated it exists during path parsing)
+    exp_data = remote_client.get_experiment_graphql(project, experiment_name, namespace)
+    experiment_id = exp_data["id"]
+
+    # Download track data
+    console.print(f"\n[cyan]Fetching track data from server...[/cyan]")
+    track_data = remote_client.get_track_data(
+      experiment_id=experiment_id,
+      topic=topic,
+      format=args.format,
+    )
+
+    # Determine output filename
+    if args.output:
+      output_path = Path(args.output)
+    else:
+      # Generate filename from topic and format
+      safe_topic = topic.replace("/", "_")
+      extension = {
+        "json": "json",
+        "jsonl": "jsonl",
+        "parquet": "parquet",
+        "mcap": "mcap",
+      }.get(args.format, args.format)
+      output_path = Path(f"{safe_topic}.{extension}")
+
+    # Write to file
+    if args.format in ("jsonl", "parquet", "mcap"):
+      # Binary data
+      output_path.write_bytes(track_data)
+    else:
+      # JSON data
+      if isinstance(track_data, dict):
+        output_path.write_text(json.dumps(track_data, indent=2))
+      else:
+        output_path.write_text(str(track_data))
+
+    # Show success message
+    file_size = output_path.stat().st_size
+    console.print(
+      f"\n[green]✓ Track data downloaded successfully[/green]"
+    )
+    console.print(f"  Output: {output_path}")
+    console.print(f"  Size: {_format_bytes(file_size)}")
+    console.print(f"  Format: {args.format}")
+
+    # Show entry count if available
+    if args.format == "json" and isinstance(track_data, dict):
+      entry_count = track_data.get("count", len(track_data.get("entries", [])))
+      if entry_count:
+        console.print(f"  Entries: {entry_count}")
+
+    return 0
+
+  except Exception as e:
+    console.print(f"[red]Error downloading track data:[/red] {e}")
+    if args.verbose:
+      import traceback
+      console.print(traceback.format_exc())
+    return 1
+
+
 # ============================================================================
 # Main Command
 # ============================================================================
@@ -622,6 +774,10 @@ class ExperimentDownloader:
 
 def cmd_download(args: argparse.Namespace) -> int:
   """Execute download command."""
+  # Handle track download if --tracks is specified
+  if args.tracks:
+    return cmd_download_track(args)
+
   # Load configuration
   config = Config()
   remote_url = args.dash_url or config.remote_url
@@ -800,6 +956,27 @@ def add_parser(subparsers):
     help="Local storage directory (default: ./.dash)",
   )
 
+  # Track download mode
+  parser.add_argument(
+    "--tracks",
+    type=str,
+    help="Download track data from path (e.g., 'namespace/project/exp/robot/position')",
+  )
+  parser.add_argument(
+    "-f",
+    "--format",
+    type=str,
+    choices=["json", "jsonl", "parquet", "mcap"],
+    default="jsonl",
+    help="Track export format (default: jsonl)",
+  )
+  parser.add_argument(
+    "-o",
+    "--output",
+    type=str,
+    help="Output file path (default: auto-generated from topic)",
+  )
+
   # Remote configuration
   parser.add_argument(
     "--dash-url",

ml_dash/cli_commands/list.py

@@ -18,6 +18,101 @@ from ..config import Config
 console = Console()
 
 
+def list_tracks(
+    remote_client: RemoteClient,
+    experiment_path: str,
+    topic_filter: Optional[str] = None,
+    output_json: bool = False,
+    verbose: bool = False
+) -> int:
+    """
+    List tracks in an experiment.
+
+    Args:
+        remote_client: Remote API client
+        experiment_path: Experiment path (namespace/project/experiment)
+        topic_filter: Optional topic filter (e.g., "robot/*")
+        output_json: Output as JSON
+        verbose: Show verbose output
+
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    # Parse experiment path
+    parts = experiment_path.strip("/").split("/")
+    if len(parts) < 3:
+        console.print("[red]Error:[/red] Experiment path must be 'namespace/project/experiment'")
+        return 1
+
+    namespace = parts[0]
+    project = parts[1]
+    experiment = parts[2]
+
+    try:
+        # Get experiment ID
+        exp_data = remote_client.get_experiment_graphql(project, experiment)
+        if not exp_data:
+            console.print(f"[red]Error:[/red] Experiment '{experiment}' not found in project '{project}'")
+            return 1
+
+        experiment_id = exp_data["id"]
+
+        # List tracks
+        tracks = remote_client.list_tracks(experiment_id, topic_filter)
+
+        if output_json:
+            # JSON output
+            output = {
+                "tracks": tracks,
+                "count": len(tracks),
+                "experiment": experiment_path
+            }
+            console.print(json.dumps(output, indent=2))
+            return 0
+
+        # Human-readable output
+        if not tracks:
+            console.print(f"[yellow]No tracks found[/yellow]")
+            return 0
+
+        console.print(f"\n[bold]Tracks in {experiment_path}[/bold]\n")
+
+        # Create table
+        table = Table(box=box.ROUNDED)
+        table.add_column("Topic", style="cyan", no_wrap=True)
+        table.add_column("Entries", justify="right")
+        table.add_column("Columns", style="dim")
+        table.add_column("Time Range", style="dim")
+
+        for track in tracks:
+            topic = track["topic"]
+            entries = str(track.get("totalEntries", 0))
+            columns = ", ".join(track.get("columns", [])[:5])
+            if len(track.get("columns", [])) > 5:
+                columns += f", ... (+{len(track['columns']) - 5})"
+
+            first_ts = track.get("firstTimestamp")
+            last_ts = track.get("lastTimestamp")
+            if first_ts is not None and last_ts is not None:
+                time_range = f"{first_ts:.3f} - {last_ts:.3f}"
+            else:
+                time_range = "N/A"
+
+            table.add_row(topic, entries, columns, time_range)
+
+        console.print(table)
+        console.print(f"\n[dim]Total tracks: {len(tracks)}[/dim]\n")
+
+        return 0
+
+    except Exception as e:
+        console.print(f"[red]Error listing tracks:[/red] {e}")
+        if verbose:
+            import traceback
+            console.print(traceback.format_exc())
+        return 1
+
+
 def _format_timestamp(iso_timestamp: str) -> str:
     """Format ISO timestamp as human-readable relative time."""
     try:
@@ -248,6 +343,45 @@ def cmd_list(args: argparse.Namespace) -> int:
     Returns:
         Exit code (0 for success, 1 for error)
     """
+    # Handle track listing if --tracks is specified
+    if args.tracks:
+        # Load config
+        config = Config()
+        remote_url = args.dash_url or config.remote_url
+        api_key = args.api_key or config.api_key
+
+        if not remote_url:
+            console.print("[red]Error:[/red] --dash-url is required (or set in config)")
+            return 1
+
+        if not args.project:
+            console.print("[red]Error:[/red] --project is required for listing tracks")
+            console.print("Example: ml-dash list --tracks --project namespace/project/experiment")
+            return 1
+
+        # Extract namespace from project path
+        parts = args.project.strip("/").split("/")
+        if len(parts) < 3:
+            console.print("[red]Error:[/red] For tracks, --project must be 'namespace/project/experiment'")
+            return 1
+
+        namespace = parts[0]
+
+        # Create remote client
+        try:
+            remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
+        except Exception as e:
+            console.print(f"[red]Error connecting to remote:[/red] {e}")
+            return 1
+
+        return list_tracks(
+            remote_client=remote_client,
+            experiment_path=args.project,
+            topic_filter=args.topic_filter,
+            output_json=args.json,
+            verbose=args.verbose
+        )
+
     # Load config
     config = Config()
 
@@ -463,6 +597,18 @@ def add_parser(subparsers) -> None:
                        help="Filter experiments by status")
     parser.add_argument("--tags", type=str, help="Filter experiments by tags (comma-separated)")
 
+    # Track listing mode
+    parser.add_argument(
+        "--tracks",
+        action="store_true",
+        help="List tracks in experiment (requires --project as 'namespace/project/experiment')"
+    )
+    parser.add_argument(
+        "--topic-filter",
+        type=str,
+        help="Filter tracks by topic (e.g., 'robot/*')"
+    )
+
     # Output options
     parser.add_argument("--json", action="store_true", help="Output as JSON")
     parser.add_argument("--detailed", action="store_true", help="Show detailed information")

ml_dash/cli_commands/profile.py

@@ -1,6 +1,7 @@
 """Profile command for ml-dash CLI - shows current user and configuration."""
 
 import json
+import time
 
 from rich.console import Console
 from rich.panel import Panel
@@ -22,6 +23,83 @@ def add_parser(subparsers):
     action="store_true",
     help="Output as JSON",
   )
+  parser.add_argument(
+    "--refresh",
+    action="store_true",
+    help="Fetch fresh profile from server (not from cached token)",
+  )
+
+
+def _fetch_fresh_profile(remote_url: str, token: str) -> dict:
+  """Fetch fresh user profile from the API server.
+
+  Args:
+      remote_url: API server URL
+      token: JWT authentication token
+
+  Returns:
+      User profile dict with username, email, name, etc.
+  """
+  try:
+    from ml_dash.client import RemoteClient
+
+    client = RemoteClient(remote_url, api_key=token)
+
+    # Query for full user profile
+    query = """
+    query GetUserProfile {
+      me {
+        id
+        username
+        name
+        email
+      }
+    }
+    """
+
+    result = client.graphql_query(query)
+    me = result.get("me", {})
+
+    if me:
+      return {
+        "sub": me.get("id"),
+        "username": me.get("username"),
+        "name": me.get("name"),
+        "email": me.get("email"),
+      }
+  except Exception as e:
+    # If API call fails, return None to fall back to token decoding
+    return None
+
+  return None
+
+
+def _check_token_expiration(token_payload: dict) -> tuple[bool, str]:
+  """Check if token is expired or close to expiring.
+
+  Args:
+      token_payload: Decoded JWT payload
+
+  Returns:
+      Tuple of (is_expired, message)
+  """
+  exp = token_payload.get("exp")
+  if not exp:
+    return False, None
+
+  current_time = int(time.time())
+  time_left = exp - current_time
+
+  if time_left < 0:
+    return True, "[red]Token expired[/red]"
+  elif time_left < 86400:  # Less than 1 day
+    hours_left = time_left // 3600
+    return False, f"[yellow]Token expires in {hours_left} hours[/yellow]"
+  else:
+    days_left = time_left // 86400
+    return False, f"Expires in {days_left} days"
+
+  return False, None
 
 
 def cmd_profile(args) -> int:
@@ -42,7 +120,33 @@ def cmd_profile(args) -> int:
 
   if token:
     info["authenticated"] = True
-    info["user"] = decode_jwt_payload(token)
+
+    # Decode token payload for initial data and expiration check
+    token_payload = decode_jwt_payload(token)
+
+    # Check token expiration
+    is_expired, expiry_message = _check_token_expiration(token_payload)
+
+    if is_expired:
+      info["authenticated"] = False
+      info["error"] = "Token expired. Please run 'ml-dash login' to re-authenticate."
+    else:
+      # Fetch fresh profile from server if requested, or fall back to token
+      if args.refresh:
+        fresh_profile = _fetch_fresh_profile(config.remote_url, token)
+        if fresh_profile:
+          info["user"] = fresh_profile
+          info["source"] = "server"
+        else:
+          info["user"] = token_payload
+          info["source"] = "token"
+          info["warning"] = "Could not fetch fresh profile from server, using cached token data"
+      else:
+        info["user"] = token_payload
+        info["source"] = "token"
+
+      if expiry_message:
+        info["token_status"] = expiry_message
 
   if args.json:
     console.print_json(json.dumps(info))
@@ -50,10 +154,11 @@ def cmd_profile(args) -> int:
 
   # Rich display
   if not info["authenticated"]:
+    error_msg = info.get("error", "Not authenticated")
     console.print(
       Panel(
         f"[bold cyan]OS Username:[/bold cyan]  {info.get('local_user')}\n\n"
-        "[yellow]Not authenticated[/yellow]\n\n"
+        f"[yellow]{error_msg}[/yellow]\n\n"
         "Run [cyan]ml-dash login[/cyan] to authenticate.",
         title="[bold]ML-Dash Info[/bold]",
         border_style="yellow",
@@ -66,7 +171,6 @@ def cmd_profile(args) -> int:
   table.add_column("Key", style="bold cyan")
   table.add_column("Value")
 
-  # table.add_row("OS Username", info.get("local_user"))
   user = info.get("user", {})
   if user.get("username"):
     table.add_row("Username", user["username"])
@@ -78,12 +182,43 @@ def cmd_profile(args) -> int:
   if user.get("email"):
     table.add_row("Email", user["email"])
   table.add_row("Remote", info.get("remote_url") or "https://api.dash.ml")
-  if info.get("token_expires"):
-    table.add_row("Token Expires", info["token_expires"])
+
+  # Show token status (expiration)
+  if info.get("token_status"):
+    table.add_row("Token Status", info["token_status"])
+
+  # Show data source
+  source = info.get("source", "token")
+  if source == "server":
+    table.add_row("Data Source", "[green]Server (Fresh)[/green]")
+  else:
+    table.add_row("Data Source", "[yellow]Token (Cached)[/yellow]")
+
+  # Show warning if any
+  warning_text = None
+  if info.get("warning"):
+    warning_text = f"\n[yellow]⚠ {info['warning']}[/yellow]"
+
+  # Show tip for refreshing
+  if source == "token":
+    tip_text = "\n[dim]Tip: Use --refresh to fetch fresh data from server[/dim]"
+  else:
+    tip_text = None
+
+  # Build panel content
+  panel_content = table
+  if warning_text or tip_text:
+    from rich.console import Group
+    items = [table]
+    if warning_text:
+      items.append(warning_text)
+    if tip_text:
+      items.append(tip_text)
+    panel_content = Group(*items)
 
   console.print(
     Panel(
-      table,
+      panel_content,
       title="[bold green]✓ Authenticated[/bold green]",
       border_style="green",
     )

ml_dash/cli_commands/upload.py

@@ -148,6 +148,18 @@ def add_parser(subparsers) -> argparse.ArgumentParser:
     help="JWT token for authentication (optional - auto-loads from 'ml-dash login' if not provided)",
   )
 
+  # Track upload mode
+  parser.add_argument(
+    "--tracks",
+    type=str,
+    help="Upload track data file (e.g., robot_position.jsonl). Requires --remote-path.",
+  )
+  parser.add_argument(
+    "--remote-path",
+    type=str,
+    help="Remote path for track (e.g., 'namespace/project/exp/robot/position')",
+  )
+
   """
   
   cd .dash/geyang
@@ -1079,6 +1091,121 @@ class ExperimentUploader:
     return total_uploaded
 
 
+def cmd_upload_track(args: argparse.Namespace) -> int:
+  """Upload track data file to remote server."""
+  from datetime import datetime
+
+  # Load config
+  config = Config()
+  remote_url = args.dash_url or config.remote_url
+  api_key = args.api_key or config.api_key
+
+  if not remote_url:
+    console.print("[red]Error:[/red] --dash-url is required (or set in config)")
+    return 1
+
+  if not args.tracks or not args.remote_path:
+    console.print("[red]Error:[/red] Both --tracks and --remote-path are required for track upload")
+    console.print("Usage: ml-dash upload --tracks <local-file> --remote-path namespace/project/exp/topic")
+    return 1
+
+  # Parse local file path
+  local_file = Path(args.tracks)
+  if not local_file.exists():
+    console.print(f"[red]Error:[/red] File not found: {local_file}")
+    return 1
+
+  # Parse remote path: namespace/project/.../experiment/topic
+  remote_path = args.remote_path.strip("/")
+  parts = remote_path.split("/")
+
+  if len(parts) < 4:
+    console.print(
+      "[red]Error:[/red] Remote path must be: 'namespace/project/experiment/topic'"
+    )
+    console.print("Example: geyang/project/exp1/robot/position")
+    return 1
+
+  namespace = parts[0]
+  project = parts[1]
+  experiment_name = parts[-2]  # Second to last is experiment
+  topic = "/".join(parts[-1:])  # Last part is topic (could be multi-level)
+
+  console.print(f"[bold]Uploading track data...[/bold]")
+  console.print(f"  Local file: {local_file}")
+  console.print(f"  Namespace: {namespace}")
+  console.print(f"  Project: {project}")
+  console.print(f"  Experiment: {experiment_name}")
+  console.print(f"  Topic: {topic}")
+
+  try:
+    # Initialize remote client
+    remote_client = RemoteClient(base_url=remote_url, namespace=namespace, api_key=api_key)
+
+    # Get experiment ID
+    exp_data = remote_client.get_experiment_graphql(project, experiment_name)
+    if not exp_data:
+      console.print(
+        f"[red]Error:[/red] Experiment '{experiment_name}' not found in project '{project}'"
+      )
+      return 1
+
+    experiment_id = exp_data["id"]
+
+    # Read local file (assume JSONL format)
+    console.print(f"\n[cyan]Reading local file...[/cyan]")
+    entries = []
+
+    with open(local_file, 'r') as f:
+      for line_num, line in enumerate(f, 1):
+        if line.strip():
+          try:
+            entry = json.loads(line)
+            if "timestamp" not in entry:
+              console.print(f"[yellow]Warning:[/yellow] Line {line_num} missing timestamp, skipping")
+              continue
+            entries.append(entry)
+          except json.JSONDecodeError as e:
+            console.print(f"[yellow]Warning:[/yellow] Line {line_num} invalid JSON: {e}")
+            continue
+
+    if not entries:
+      console.print("[red]Error:[/red] No valid entries found in file")
+      return 1
+
+    console.print(f"  Found {len(entries)} entries")
+
+    # Upload in batches
+    console.print(f"\n[cyan]Uploading to server...[/cyan]")
+    batch_size = 1000
+    total_uploaded = 0
+
+    for i in range(0, len(entries), batch_size):
+      batch = entries[i:i + batch_size]
+      remote_client.append_batch_to_track(
+        experiment_id=experiment_id,
+        topic=topic,
+        entries=batch
+      )
+      total_uploaded += len(batch)
+      console.print(f"  Uploaded {total_uploaded}/{len(entries)} entries")
+
+    # Success
+    console.print(f"\n[green]✓ Track data uploaded successfully[/green]")
+    console.print(f"  Total entries: {total_uploaded}")
+    console.print(f"  Topic: {topic}")
+    console.print(f"  Experiment: {namespace}/{project}/{experiment_name}")
+
+    return 0
+
+  except Exception as e:
+    console.print(f"[red]Error uploading track data:[/red] {e}")
+    if args.verbose:
+      import traceback
+      console.print(traceback.format_exc())
+    return 1
+
+
 def cmd_upload(args: argparse.Namespace) -> int:
   """
   Execute upload command.
@@ -1089,6 +1216,10 @@ def cmd_upload(args: argparse.Namespace) -> int:
   Returns:
       Exit code (0 for success, 1 for error)
   """
+  # Handle track upload if --tracks is specified
+  if args.tracks:
+    return cmd_upload_track(args)
+
   # Load config
   config = Config()