ml-dash 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +37 -63
- ml_dash/auth/token_storage.py +267 -226
- ml_dash/auto_start.py +30 -30
- ml_dash/cli.py +16 -2
- ml_dash/cli_commands/api.py +165 -0
- ml_dash/cli_commands/download.py +757 -667
- ml_dash/cli_commands/list.py +146 -13
- ml_dash/cli_commands/login.py +190 -183
- ml_dash/cli_commands/profile.py +92 -0
- ml_dash/cli_commands/upload.py +1291 -1141
- ml_dash/client.py +122 -34
- ml_dash/config.py +119 -119
- ml_dash/experiment.py +1242 -995
- ml_dash/files.py +1051 -340
- ml_dash/log.py +7 -7
- ml_dash/metric.py +359 -100
- ml_dash/params.py +6 -6
- ml_dash/remote_auto_start.py +20 -17
- ml_dash/run.py +231 -0
- ml_dash/snowflake.py +173 -0
- ml_dash/storage.py +1051 -1079
- {ml_dash-0.6.0.dist-info → ml_dash-0.6.2.dist-info}/METADATA +45 -20
- ml_dash-0.6.2.dist-info/RECORD +33 -0
- ml_dash-0.6.0.dist-info/RECORD +0 -29
- {ml_dash-0.6.0.dist-info → ml_dash-0.6.2.dist-info}/WHEEL +0 -0
- {ml_dash-0.6.0.dist-info → ml_dash-0.6.2.dist-info}/entry_points.txt +0 -0
ml_dash/auto_start.py
CHANGED
|
@@ -9,56 +9,56 @@ Usage:
|
|
|
9
9
|
# First, authenticate
|
|
10
10
|
# $ ml-dash login
|
|
11
11
|
|
|
12
|
-
from ml_dash import dxp
|
|
12
|
+
from ml_dash.auto_start import dxp
|
|
13
13
|
|
|
14
14
|
# Use with statement (recommended)
|
|
15
15
|
with dxp.run:
|
|
16
|
-
dxp.log(
|
|
16
|
+
dxp.log("Hello from dxp!", level="info")
|
|
17
17
|
dxp.params.set(lr=0.001)
|
|
18
|
-
dxp.metrics("
|
|
18
|
+
dxp.metrics("train").log(loss=0.5, step=0)
|
|
19
19
|
# Automatically completes on exit from with block
|
|
20
20
|
|
|
21
21
|
# Or start/complete manually
|
|
22
22
|
dxp.run.start()
|
|
23
|
-
dxp.log(
|
|
23
|
+
dxp.log("Training...", level="info")
|
|
24
24
|
dxp.run.complete()
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
27
|
import atexit
|
|
28
|
-
from .experiment import Experiment
|
|
29
|
-
from .auth.token_storage import get_token_storage
|
|
30
|
-
from .auth.exceptions import AuthenticationError
|
|
31
|
-
|
|
32
|
-
# Check if user is authenticated
|
|
33
|
-
_storage = get_token_storage()
|
|
34
|
-
_token = _storage.load("ml-dash-token")
|
|
35
|
-
|
|
36
|
-
if not _token:
|
|
37
|
-
raise AuthenticationError(
|
|
38
|
-
"Not authenticated. Please run 'ml-dash login' to authenticate before using dxp.\n\n"
|
|
39
|
-
"To login:\n"
|
|
40
|
-
" ml-dash login\n\n"
|
|
41
|
-
"Or use Experiment() with explicit api_key parameter."
|
|
42
|
-
)
|
|
43
28
|
|
|
44
29
|
# Create pre-configured singleton experiment in remote mode
|
|
45
30
|
# Uses default remote server (https://api.dash.ml)
|
|
46
|
-
# Token is auto-loaded from storage
|
|
31
|
+
# Token is auto-loaded from storage when first used
|
|
32
|
+
# If not authenticated, operations will fail with AuthenticationError
|
|
33
|
+
# Prefix format: {owner}/{project}/path...
|
|
34
|
+
# Using getpass to get current user as owner for local convenience
|
|
35
|
+
import getpass
|
|
36
|
+
from datetime import datetime
|
|
37
|
+
|
|
38
|
+
from .auth.token_storage import get_jwt_user
|
|
39
|
+
from .experiment import Experiment
|
|
40
|
+
|
|
41
|
+
_user = get_jwt_user()
|
|
42
|
+
# Fallback to system username if not authenticated
|
|
43
|
+
_username = _user["username"] if _user else getpass.getuser()
|
|
44
|
+
_now = datetime.now()
|
|
45
|
+
|
|
47
46
|
dxp = Experiment(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
remote="https://api.dash.ml",
|
|
47
|
+
prefix=f"{_username}/scratch/{_now:%Y-%m-%d/%H%M%S}",
|
|
48
|
+
dash_url="https://api.dash.ml",
|
|
51
49
|
)
|
|
52
50
|
|
|
51
|
+
|
|
53
52
|
# Register cleanup handler to complete experiment on Python exit (if still open)
|
|
54
53
|
def _cleanup():
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
54
|
+
"""Complete the dxp experiment on exit if still open."""
|
|
55
|
+
if dxp._is_open:
|
|
56
|
+
try:
|
|
57
|
+
dxp.run.complete()
|
|
58
|
+
except Exception:
|
|
59
|
+
# Silently ignore errors during cleanup
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
62
|
|
|
63
63
|
atexit.register(_cleanup)
|
|
64
64
|
|
ml_dash/cli.py
CHANGED
|
@@ -9,7 +9,11 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
9
9
|
"""Create the main CLI argument parser."""
|
|
10
10
|
parser = argparse.ArgumentParser(
|
|
11
11
|
prog="ml-dash",
|
|
12
|
-
description=
|
|
12
|
+
description=(
|
|
13
|
+
"ML-Dash: ML experiment tracking and data storage CLI\n\n"
|
|
14
|
+
"View your experiments, statistics, and plots online at:\n"
|
|
15
|
+
" https://dash.ml\n"
|
|
16
|
+
),
|
|
13
17
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
14
18
|
)
|
|
15
19
|
|
|
@@ -21,11 +25,15 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
21
25
|
)
|
|
22
26
|
|
|
23
27
|
# Import and add command parsers
|
|
24
|
-
from .cli_commands import upload, download, list as list_cmd, login, logout
|
|
28
|
+
from .cli_commands import upload, download, list as list_cmd, login, logout, profile, api
|
|
25
29
|
|
|
26
30
|
# Authentication commands
|
|
27
31
|
login.add_parser(subparsers)
|
|
28
32
|
logout.add_parser(subparsers)
|
|
33
|
+
profile.add_parser(subparsers)
|
|
34
|
+
|
|
35
|
+
# API commands
|
|
36
|
+
api.add_parser(subparsers)
|
|
29
37
|
|
|
30
38
|
# Data commands
|
|
31
39
|
upload.add_parser(subparsers)
|
|
@@ -60,6 +68,9 @@ def main(argv: Optional[List[str]] = None) -> int:
|
|
|
60
68
|
elif args.command == "logout":
|
|
61
69
|
from .cli_commands import logout
|
|
62
70
|
return logout.cmd_logout(args)
|
|
71
|
+
elif args.command == "profile":
|
|
72
|
+
from .cli_commands import profile
|
|
73
|
+
return profile.cmd_profile(args)
|
|
63
74
|
elif args.command == "upload":
|
|
64
75
|
from .cli_commands import upload
|
|
65
76
|
return upload.cmd_upload(args)
|
|
@@ -69,6 +80,9 @@ def main(argv: Optional[List[str]] = None) -> int:
|
|
|
69
80
|
elif args.command == "list":
|
|
70
81
|
from .cli_commands import list as list_cmd
|
|
71
82
|
return list_cmd.cmd_list(args)
|
|
83
|
+
elif args.command == "api":
|
|
84
|
+
from .cli_commands import api
|
|
85
|
+
return api.cmd_api(args)
|
|
72
86
|
|
|
73
87
|
# Unknown command (shouldn't happen due to subparsers)
|
|
74
88
|
parser.print_help()
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""API command for ml-dash CLI - send GraphQL queries to the server."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
|
|
8
|
+
from ml_dash.client import RemoteClient
|
|
9
|
+
from ml_dash.config import config
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def add_parser(subparsers):
|
|
13
|
+
"""Add api command parser."""
|
|
14
|
+
parser = subparsers.add_parser(
|
|
15
|
+
"api",
|
|
16
|
+
help="Send GraphQL queries to ml-dash server",
|
|
17
|
+
description="""Send GraphQL queries to the ml-dash server.
|
|
18
|
+
|
|
19
|
+
Examples:
|
|
20
|
+
# Query current user
|
|
21
|
+
ml-dash api --query "me { username name email }"
|
|
22
|
+
|
|
23
|
+
# Query with arguments (single quotes auto-converted to double)
|
|
24
|
+
ml-dash api --query "user(title: 'hello') { id title }"
|
|
25
|
+
|
|
26
|
+
# Extract specific field with jq-like syntax
|
|
27
|
+
ml-dash api --query "me { username }" --jq ".data.me.username"
|
|
28
|
+
|
|
29
|
+
# Mutation to update username
|
|
30
|
+
ml-dash api --mutation "updateUser(username: 'newname') { username }"
|
|
31
|
+
|
|
32
|
+
Notes:
|
|
33
|
+
- Single quotes are auto-converted to double quotes for GraphQL
|
|
34
|
+
- Use --jq for dot-path extraction (built-in, no deps)
|
|
35
|
+
""",
|
|
36
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
37
|
+
)
|
|
38
|
+
group = parser.add_mutually_exclusive_group(required=True)
|
|
39
|
+
group.add_argument(
|
|
40
|
+
"--query", "-q",
|
|
41
|
+
metavar="QUERY",
|
|
42
|
+
help="GraphQL query string",
|
|
43
|
+
)
|
|
44
|
+
group.add_argument(
|
|
45
|
+
"--mutation", "-m",
|
|
46
|
+
metavar="MUTATION",
|
|
47
|
+
help="GraphQL mutation string",
|
|
48
|
+
)
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--jq",
|
|
51
|
+
metavar="PATH",
|
|
52
|
+
help="Extract value using dot-path (e.g., .data.me.username)",
|
|
53
|
+
)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--dash-url",
|
|
56
|
+
type=str,
|
|
57
|
+
help="ML-Dash server URL (default: https://api.dash.ml)",
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def extract_path(data, path: str):
|
|
62
|
+
"""Extract value from nested dict using dot-path notation.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
data: Nested dict/list structure
|
|
66
|
+
path: Dot-separated path (e.g., ".data.me.username")
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
Extracted value
|
|
70
|
+
|
|
71
|
+
Examples:
|
|
72
|
+
>>> extract_path({"data": {"me": {"username": "ge"}}}, ".data.me.username")
|
|
73
|
+
"ge"
|
|
74
|
+
"""
|
|
75
|
+
for key in path.lstrip(".").split("."):
|
|
76
|
+
if key:
|
|
77
|
+
if isinstance(data, dict):
|
|
78
|
+
data = data[key]
|
|
79
|
+
elif isinstance(data, list):
|
|
80
|
+
data = data[int(key)]
|
|
81
|
+
else:
|
|
82
|
+
raise KeyError(f"Cannot access '{key}' on {type(data).__name__}")
|
|
83
|
+
return data
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def fix_quotes(query: str) -> str:
|
|
87
|
+
"""Convert single quotes to double quotes for GraphQL.
|
|
88
|
+
|
|
89
|
+
GraphQL requires double quotes for strings. This allows users to write
|
|
90
|
+
queries with single quotes for shell convenience.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
query: GraphQL query string with possible single quotes
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
Query with single quotes converted to double quotes
|
|
97
|
+
"""
|
|
98
|
+
# Simple conversion - assumes single quotes are for strings
|
|
99
|
+
# This handles: user(title: 'hello') -> user(title: "hello")
|
|
100
|
+
return query.replace("'", '"')
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def build_query(query: str, is_mutation: bool) -> str:
|
|
104
|
+
"""Build complete GraphQL query string.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
query: Query or mutation body
|
|
108
|
+
is_mutation: Whether this is a mutation
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
Complete GraphQL query string
|
|
112
|
+
"""
|
|
113
|
+
query = query.strip()
|
|
114
|
+
query = fix_quotes(query)
|
|
115
|
+
|
|
116
|
+
# If already properly formatted, return as-is
|
|
117
|
+
if query.startswith("{") or query.startswith("mutation") or query.startswith("query"):
|
|
118
|
+
return query
|
|
119
|
+
|
|
120
|
+
# Wrap appropriately
|
|
121
|
+
if is_mutation:
|
|
122
|
+
return "mutation { " + query + " }"
|
|
123
|
+
else:
|
|
124
|
+
return "{ " + query + " }"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def cmd_api(args) -> int:
|
|
128
|
+
"""Execute api command."""
|
|
129
|
+
console = Console()
|
|
130
|
+
|
|
131
|
+
# Get remote URL
|
|
132
|
+
remote_url = args.dash_url or config.remote_url or "https://api.dash.ml"
|
|
133
|
+
|
|
134
|
+
try:
|
|
135
|
+
# Initialize client
|
|
136
|
+
client = RemoteClient(base_url=remote_url)
|
|
137
|
+
|
|
138
|
+
# Determine query type and build query
|
|
139
|
+
if args.mutation:
|
|
140
|
+
query = build_query(args.mutation, is_mutation=True)
|
|
141
|
+
else:
|
|
142
|
+
query = build_query(args.query, is_mutation=False)
|
|
143
|
+
|
|
144
|
+
# Execute GraphQL query
|
|
145
|
+
result = client.graphql_query(query)
|
|
146
|
+
|
|
147
|
+
# Apply jq path extraction if specified
|
|
148
|
+
if args.jq:
|
|
149
|
+
try:
|
|
150
|
+
result = extract_path(result, args.jq)
|
|
151
|
+
except (KeyError, IndexError, TypeError) as e:
|
|
152
|
+
console.print(f"[red]Error extracting path '{args.jq}': {e}[/red]")
|
|
153
|
+
return 1
|
|
154
|
+
|
|
155
|
+
# Output result
|
|
156
|
+
if isinstance(result, (dict, list)):
|
|
157
|
+
console.print_json(json.dumps(result))
|
|
158
|
+
else:
|
|
159
|
+
console.print(json.dumps(result))
|
|
160
|
+
|
|
161
|
+
return 0
|
|
162
|
+
|
|
163
|
+
except Exception as e:
|
|
164
|
+
console.print(f"[red]Error: {e}[/red]")
|
|
165
|
+
return 1
|