ml-dash 0.6.7__py3-none-any.whl → 0.6.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +49 -2
- ml_dash/auth/token_storage.py +0 -9
- ml_dash/auto_start.py +21 -6
- ml_dash/cli_commands/profile.py +133 -6
- ml_dash/client.py +127 -0
- ml_dash/run.py +1 -1
- {ml_dash-0.6.7.dist-info → ml_dash-0.6.10.dist-info}/METADATA +81 -5
- {ml_dash-0.6.7.dist-info → ml_dash-0.6.10.dist-info}/RECORD +10 -10
- {ml_dash-0.6.7.dist-info → ml_dash-0.6.10.dist-info}/WHEEL +2 -2
- {ml_dash-0.6.7.dist-info → ml_dash-0.6.10.dist-info}/entry_points.txt +0 -0
ml_dash/__init__.py
CHANGED
|
@@ -36,14 +36,60 @@ Usage:
|
|
|
36
36
|
exp.log("Training started")
|
|
37
37
|
"""
|
|
38
38
|
|
|
39
|
-
from .client import RemoteClient
|
|
39
|
+
from .client import RemoteClient, userinfo
|
|
40
40
|
from .experiment import Experiment, OperationMode, ml_dash_experiment
|
|
41
41
|
from .log import LogBuilder, LogLevel
|
|
42
42
|
from .params import ParametersBuilder
|
|
43
43
|
from .run import RUN
|
|
44
44
|
from .storage import LocalStorage
|
|
45
45
|
|
|
46
|
-
__version__ = "0.6.
|
|
46
|
+
__version__ = "0.6.10"
|
|
47
|
+
|
|
48
|
+
# Minimum version required - blocks older versions
|
|
49
|
+
MINIMUM_REQUIRED_VERSION = "0.6.10"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _check_version_compatibility():
|
|
53
|
+
"""
|
|
54
|
+
Enforce minimum version requirement.
|
|
55
|
+
|
|
56
|
+
Raises ImportError if installed version is below minimum required version.
|
|
57
|
+
This ensures users have the latest features (userinfo, namespace auto-detection, etc.)
|
|
58
|
+
"""
|
|
59
|
+
try:
|
|
60
|
+
from packaging import version
|
|
61
|
+
except ImportError:
|
|
62
|
+
# If packaging is not available, skip check
|
|
63
|
+
# (unlikely since it's a common dependency)
|
|
64
|
+
return
|
|
65
|
+
|
|
66
|
+
current = version.parse(__version__)
|
|
67
|
+
minimum = version.parse(MINIMUM_REQUIRED_VERSION)
|
|
68
|
+
|
|
69
|
+
if current < minimum:
|
|
70
|
+
raise ImportError(
|
|
71
|
+
f"\n"
|
|
72
|
+
f"{'=' * 80}\n"
|
|
73
|
+
f"ERROR: ml-dash version {__version__} is too old!\n"
|
|
74
|
+
f"{'=' * 80}\n"
|
|
75
|
+
f"\n"
|
|
76
|
+
f"This version of ml-dash ({__version__}) is no longer supported.\n"
|
|
77
|
+
f"Minimum required version: {MINIMUM_REQUIRED_VERSION}\n"
|
|
78
|
+
f"\n"
|
|
79
|
+
f"Please upgrade to the latest version:\n"
|
|
80
|
+
f"\n"
|
|
81
|
+
f" pip install --upgrade ml-dash\n"
|
|
82
|
+
f"\n"
|
|
83
|
+
f"Or install specific version:\n"
|
|
84
|
+
f"\n"
|
|
85
|
+
f" pip install ml-dash>={MINIMUM_REQUIRED_VERSION}\n"
|
|
86
|
+
f"\n"
|
|
87
|
+
f"{'=' * 80}\n"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# Enforce version check on import
|
|
92
|
+
_check_version_compatibility()
|
|
47
93
|
|
|
48
94
|
__all__ = [
|
|
49
95
|
"Experiment",
|
|
@@ -55,4 +101,5 @@ __all__ = [
|
|
|
55
101
|
"LogBuilder",
|
|
56
102
|
"ParametersBuilder",
|
|
57
103
|
"RUN",
|
|
104
|
+
"userinfo",
|
|
58
105
|
]
|
ml_dash/auth/token_storage.py
CHANGED
|
@@ -292,12 +292,3 @@ def decode_jwt_payload(token: str) -> dict:
|
|
|
292
292
|
return {}
|
|
293
293
|
|
|
294
294
|
|
|
295
|
-
def get_jwt_user():
|
|
296
|
-
# Load token
|
|
297
|
-
storage = get_token_storage()
|
|
298
|
-
token = storage.load("ml-dash-token")
|
|
299
|
-
|
|
300
|
-
if token:
|
|
301
|
-
user = decode_jwt_payload(token)
|
|
302
|
-
return user
|
|
303
|
-
return None
|
ml_dash/auto_start.py
CHANGED
|
@@ -31,19 +31,34 @@ import atexit
|
|
|
31
31
|
# Token is auto-loaded from storage when first used
|
|
32
32
|
# If not authenticated, operations will fail with AuthenticationError
|
|
33
33
|
# Prefix format: {owner}/{project}/path...
|
|
34
|
-
# Using getpass to get current user as owner for local convenience
|
|
35
34
|
import getpass
|
|
36
35
|
from datetime import datetime
|
|
37
36
|
|
|
38
|
-
from .auth.token_storage import get_jwt_user
|
|
39
37
|
from .experiment import Experiment
|
|
40
38
|
|
|
41
|
-
|
|
42
|
-
#
|
|
43
|
-
|
|
39
|
+
# Get username for dxp namespace
|
|
40
|
+
# Note: We use userinfo for fresh data (recommended approach)
|
|
41
|
+
# Falls back to system username if not authenticated
|
|
42
|
+
try:
|
|
43
|
+
from .client import userinfo
|
|
44
|
+
_username = userinfo.username or getpass.getuser()
|
|
45
|
+
except Exception:
|
|
46
|
+
# If userinfo fails (e.g., no network), fall back to system user
|
|
47
|
+
_username = getpass.getuser()
|
|
48
|
+
|
|
44
49
|
_now = datetime.now()
|
|
45
50
|
|
|
46
|
-
|
|
51
|
+
# Create pre-configured singleton experiment in REMOTE mode
|
|
52
|
+
# - dash_url=True: Use default remote server (https://api.dash.ml)
|
|
53
|
+
# - dash_root=None: Remote-only mode (no local storage)
|
|
54
|
+
# - user: Uses authenticated username from userinfo (fresh from server)
|
|
55
|
+
# - Token is auto-loaded from storage when first used
|
|
56
|
+
# - If not authenticated, operations will fail with AuthenticationError
|
|
57
|
+
dxp = Experiment(
|
|
58
|
+
user=_username, # Use authenticated username for namespace
|
|
59
|
+
dash_url=True, # Use remote API (https://api.dash.ml)
|
|
60
|
+
dash_root=None, # Remote-only mode (no local .dash/)
|
|
61
|
+
)
|
|
47
62
|
|
|
48
63
|
|
|
49
64
|
# Register cleanup handler to complete experiment on Python exit (if still open)
|
ml_dash/cli_commands/profile.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Profile command for ml-dash CLI - shows current user and configuration."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
+
import time
|
|
4
5
|
|
|
5
6
|
from rich.console import Console
|
|
6
7
|
from rich.panel import Panel
|
|
@@ -22,6 +23,73 @@ def add_parser(subparsers):
|
|
|
22
23
|
action="store_true",
|
|
23
24
|
help="Output as JSON",
|
|
24
25
|
)
|
|
26
|
+
parser.add_argument(
|
|
27
|
+
"--cached",
|
|
28
|
+
action="store_true",
|
|
29
|
+
help="Use cached token data (default: fetch fresh from server)",
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _fetch_fresh_profile(remote_url: str, token: str) -> dict:
|
|
34
|
+
"""Fetch fresh user profile from the API server.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
remote_url: API server URL
|
|
38
|
+
token: JWT authentication token
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
User profile dict with username, email, name, etc.
|
|
42
|
+
"""
|
|
43
|
+
try:
|
|
44
|
+
from ml_dash.client import RemoteClient
|
|
45
|
+
|
|
46
|
+
client = RemoteClient(remote_url, api_key=token)
|
|
47
|
+
|
|
48
|
+
# Use the new get_current_user() method
|
|
49
|
+
user_data = client.get_current_user()
|
|
50
|
+
|
|
51
|
+
if user_data:
|
|
52
|
+
return {
|
|
53
|
+
"sub": user_data.get("id"),
|
|
54
|
+
"username": user_data.get("username"),
|
|
55
|
+
"name": user_data.get("name"),
|
|
56
|
+
"email": user_data.get("email"),
|
|
57
|
+
"given_name": user_data.get("given_name"),
|
|
58
|
+
"family_name": user_data.get("family_name"),
|
|
59
|
+
}
|
|
60
|
+
except Exception as e:
|
|
61
|
+
# If API call fails, return None to fall back to token decoding
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _check_token_expiration(token_payload: dict) -> tuple[bool, str]:
|
|
68
|
+
"""Check if token is expired or close to expiring.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
token_payload: Decoded JWT payload
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
Tuple of (is_expired, message)
|
|
75
|
+
"""
|
|
76
|
+
exp = token_payload.get("exp")
|
|
77
|
+
if not exp:
|
|
78
|
+
return False, None
|
|
79
|
+
|
|
80
|
+
current_time = int(time.time())
|
|
81
|
+
time_left = exp - current_time
|
|
82
|
+
|
|
83
|
+
if time_left < 0:
|
|
84
|
+
return True, "[red]Token expired[/red]"
|
|
85
|
+
elif time_left < 86400: # Less than 1 day
|
|
86
|
+
hours_left = time_left // 3600
|
|
87
|
+
return False, f"[yellow]Token expires in {hours_left} hours[/yellow]"
|
|
88
|
+
else:
|
|
89
|
+
days_left = time_left // 86400
|
|
90
|
+
return False, f"Expires in {days_left} days"
|
|
91
|
+
|
|
92
|
+
return False, None
|
|
25
93
|
|
|
26
94
|
|
|
27
95
|
def cmd_profile(args) -> int:
|
|
@@ -42,7 +110,35 @@ def cmd_profile(args) -> int:
|
|
|
42
110
|
|
|
43
111
|
if token:
|
|
44
112
|
info["authenticated"] = True
|
|
45
|
-
|
|
113
|
+
|
|
114
|
+
# Decode token payload for initial data and expiration check
|
|
115
|
+
token_payload = decode_jwt_payload(token)
|
|
116
|
+
|
|
117
|
+
# Check token expiration
|
|
118
|
+
is_expired, expiry_message = _check_token_expiration(token_payload)
|
|
119
|
+
|
|
120
|
+
if is_expired:
|
|
121
|
+
info["authenticated"] = False
|
|
122
|
+
info["error"] = "Token expired. Please run 'ml-dash login' to re-authenticate."
|
|
123
|
+
else:
|
|
124
|
+
# Fetch fresh profile from server by default, use cached token only if --cached flag is set
|
|
125
|
+
if args.cached:
|
|
126
|
+
# Use cached token data
|
|
127
|
+
info["user"] = token_payload
|
|
128
|
+
info["source"] = "token"
|
|
129
|
+
else:
|
|
130
|
+
# Fetch fresh data from server (default behavior)
|
|
131
|
+
fresh_profile = _fetch_fresh_profile(config.remote_url, token)
|
|
132
|
+
if fresh_profile:
|
|
133
|
+
info["user"] = fresh_profile
|
|
134
|
+
info["source"] = "server"
|
|
135
|
+
else:
|
|
136
|
+
info["user"] = token_payload
|
|
137
|
+
info["source"] = "token"
|
|
138
|
+
info["warning"] = "Could not fetch fresh profile from server, using cached token data"
|
|
139
|
+
|
|
140
|
+
if expiry_message:
|
|
141
|
+
info["token_status"] = expiry_message
|
|
46
142
|
|
|
47
143
|
if args.json:
|
|
48
144
|
console.print_json(json.dumps(info))
|
|
@@ -50,10 +146,11 @@ def cmd_profile(args) -> int:
|
|
|
50
146
|
|
|
51
147
|
# Rich display
|
|
52
148
|
if not info["authenticated"]:
|
|
149
|
+
error_msg = info.get("error", "Not authenticated")
|
|
53
150
|
console.print(
|
|
54
151
|
Panel(
|
|
55
152
|
f"[bold cyan]OS Username:[/bold cyan] {info.get('local_user')}\n\n"
|
|
56
|
-
"[yellow]
|
|
153
|
+
f"[yellow]{error_msg}[/yellow]\n\n"
|
|
57
154
|
"Run [cyan]ml-dash login[/cyan] to authenticate.",
|
|
58
155
|
title="[bold]ML-Dash Info[/bold]",
|
|
59
156
|
border_style="yellow",
|
|
@@ -66,7 +163,6 @@ def cmd_profile(args) -> int:
|
|
|
66
163
|
table.add_column("Key", style="bold cyan")
|
|
67
164
|
table.add_column("Value")
|
|
68
165
|
|
|
69
|
-
# table.add_row("OS Username", info.get("local_user"))
|
|
70
166
|
user = info.get("user", {})
|
|
71
167
|
if user.get("username"):
|
|
72
168
|
table.add_row("Username", user["username"])
|
|
@@ -78,12 +174,43 @@ def cmd_profile(args) -> int:
|
|
|
78
174
|
if user.get("email"):
|
|
79
175
|
table.add_row("Email", user["email"])
|
|
80
176
|
table.add_row("Remote", info.get("remote_url") or "https://api.dash.ml")
|
|
81
|
-
|
|
82
|
-
|
|
177
|
+
|
|
178
|
+
# Show token status (expiration)
|
|
179
|
+
if info.get("token_status"):
|
|
180
|
+
table.add_row("Token Status", info["token_status"])
|
|
181
|
+
|
|
182
|
+
# Show data source
|
|
183
|
+
source = info.get("source", "token")
|
|
184
|
+
if source == "server":
|
|
185
|
+
table.add_row("Data Source", "[green]Server (Fresh)[/green]")
|
|
186
|
+
else:
|
|
187
|
+
table.add_row("Data Source", "[yellow]Token (Cached)[/yellow]")
|
|
188
|
+
|
|
189
|
+
# Show warning if any
|
|
190
|
+
warning_text = None
|
|
191
|
+
if info.get("warning"):
|
|
192
|
+
warning_text = f"\n[yellow]⚠ {info['warning']}[/yellow]"
|
|
193
|
+
|
|
194
|
+
# Show tip for using cached data
|
|
195
|
+
if source == "server":
|
|
196
|
+
tip_text = "\n[dim]Tip: Use --cached to use cached token data (faster but may be outdated)[/dim]"
|
|
197
|
+
else:
|
|
198
|
+
tip_text = None
|
|
199
|
+
|
|
200
|
+
# Build panel content
|
|
201
|
+
panel_content = table
|
|
202
|
+
if warning_text or tip_text:
|
|
203
|
+
from rich.console import Group
|
|
204
|
+
items = [table]
|
|
205
|
+
if warning_text:
|
|
206
|
+
items.append(warning_text)
|
|
207
|
+
if tip_text:
|
|
208
|
+
items.append(tip_text)
|
|
209
|
+
panel_content = Group(*items)
|
|
83
210
|
|
|
84
211
|
console.print(
|
|
85
212
|
Panel(
|
|
86
|
-
|
|
213
|
+
panel_content,
|
|
87
214
|
title="[bold green]✓ Authenticated[/bold green]",
|
|
88
215
|
border_style="green",
|
|
89
216
|
)
|
ml_dash/client.py
CHANGED
|
@@ -6,6 +6,95 @@ from typing import Optional, Dict, Any, List
|
|
|
6
6
|
import httpx
|
|
7
7
|
|
|
8
8
|
|
|
9
|
+
class UserInfo:
|
|
10
|
+
"""
|
|
11
|
+
Singleton user info object that fetches current user from API server.
|
|
12
|
+
|
|
13
|
+
Fetches user info from API server on first access (lazy loading).
|
|
14
|
+
This queries the API for fresh user data, ensuring up-to-date information.
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
>>> from ml_dash import userinfo
|
|
18
|
+
>>> if userinfo.username:
|
|
19
|
+
... print(f"Namespace: {userinfo.username}")
|
|
20
|
+
... print(f"Email: {userinfo.email}")
|
|
21
|
+
... print(f"Project: {userinfo.username}/my-project")
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
self._data = None
|
|
26
|
+
self._fetched = False
|
|
27
|
+
|
|
28
|
+
def _fetch(self):
|
|
29
|
+
"""Fetch user info from API server (lazy loading)."""
|
|
30
|
+
if self._fetched:
|
|
31
|
+
return
|
|
32
|
+
|
|
33
|
+
self._fetched = True
|
|
34
|
+
try:
|
|
35
|
+
client = RemoteClient("https://api.dash.ml")
|
|
36
|
+
self._data = client.get_current_user()
|
|
37
|
+
except Exception:
|
|
38
|
+
self._data = None
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def username(self) -> Optional[str]:
|
|
42
|
+
"""Username (namespace) - e.g., 'tom_tao_e4c2c9'"""
|
|
43
|
+
self._fetch()
|
|
44
|
+
return self._data.get("username") if self._data else None
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def email(self) -> Optional[str]:
|
|
48
|
+
"""User email"""
|
|
49
|
+
self._fetch()
|
|
50
|
+
return self._data.get("email") if self._data else None
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def name(self) -> Optional[str]:
|
|
54
|
+
"""Full name"""
|
|
55
|
+
self._fetch()
|
|
56
|
+
return self._data.get("name") if self._data else None
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def given_name(self) -> Optional[str]:
|
|
60
|
+
"""First/given name"""
|
|
61
|
+
self._fetch()
|
|
62
|
+
return self._data.get("given_name") if self._data else None
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def family_name(self) -> Optional[str]:
|
|
66
|
+
"""Last/family name"""
|
|
67
|
+
self._fetch()
|
|
68
|
+
return self._data.get("family_name") if self._data else None
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def picture(self) -> Optional[str]:
|
|
72
|
+
"""Profile picture URL"""
|
|
73
|
+
self._fetch()
|
|
74
|
+
return self._data.get("picture") if self._data else None
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def id(self) -> Optional[str]:
|
|
78
|
+
"""User ID"""
|
|
79
|
+
self._fetch()
|
|
80
|
+
return self._data.get("id") if self._data else None
|
|
81
|
+
|
|
82
|
+
def __bool__(self) -> bool:
|
|
83
|
+
"""Return True if user is authenticated and data was fetched successfully."""
|
|
84
|
+
self._fetch()
|
|
85
|
+
return self._data is not None
|
|
86
|
+
|
|
87
|
+
def __repr__(self) -> str:
|
|
88
|
+
self._fetch()
|
|
89
|
+
if self._data:
|
|
90
|
+
return f"UserInfo(username='{self.username}', email='{self.email}')"
|
|
91
|
+
return "UserInfo(not authenticated)"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# Create singleton instance
|
|
95
|
+
userinfo = UserInfo()
|
|
96
|
+
|
|
97
|
+
|
|
9
98
|
def _serialize_value(value: Any) -> Any:
|
|
10
99
|
"""
|
|
11
100
|
Convert value to JSON-serializable format.
|
|
@@ -140,6 +229,44 @@ class RemoteClient:
|
|
|
140
229
|
except Exception:
|
|
141
230
|
return None
|
|
142
231
|
|
|
232
|
+
def get_current_user(self) -> Optional[Dict[str, Any]]:
|
|
233
|
+
"""
|
|
234
|
+
Get current authenticated user's info from server.
|
|
235
|
+
|
|
236
|
+
This queries the API server for fresh user data, ensuring up-to-date information.
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
User info dict with keys: username, email, name, given_name, family_name, picture
|
|
240
|
+
Returns None if not authenticated or if query fails
|
|
241
|
+
|
|
242
|
+
Example:
|
|
243
|
+
>>> client = RemoteClient("https://api.dash.ml")
|
|
244
|
+
>>> user = client.get_current_user()
|
|
245
|
+
>>> print(user["username"]) # e.g., "tom_tao_e4c2c9"
|
|
246
|
+
>>> print(user["email"]) # e.g., "user@example.com"
|
|
247
|
+
"""
|
|
248
|
+
try:
|
|
249
|
+
self._ensure_authenticated()
|
|
250
|
+
|
|
251
|
+
# Query server for current user's complete profile
|
|
252
|
+
query = """
|
|
253
|
+
query GetCurrentUser {
|
|
254
|
+
me {
|
|
255
|
+
id
|
|
256
|
+
username
|
|
257
|
+
email
|
|
258
|
+
name
|
|
259
|
+
given_name
|
|
260
|
+
family_name
|
|
261
|
+
picture
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
"""
|
|
265
|
+
result = self.graphql_query(query)
|
|
266
|
+
return result.get("me")
|
|
267
|
+
except Exception:
|
|
268
|
+
return None
|
|
269
|
+
|
|
143
270
|
def _ensure_authenticated(self):
|
|
144
271
|
"""Check if authenticated, raise error if not."""
|
|
145
272
|
if not self.api_key:
|
ml_dash/run.py
CHANGED
|
@@ -223,7 +223,7 @@ class RUN:
|
|
|
223
223
|
# experiments/vision/resnet/train.py
|
|
224
224
|
from ml_dash import RUN
|
|
225
225
|
|
|
226
|
-
RUN
|
|
226
|
+
RUN(entry=__file__)
|
|
227
227
|
# Result: RUN.prefix = "vision/resnet", RUN.name = "resnet"
|
|
228
228
|
"""
|
|
229
229
|
# Use provided entry or try to auto-detect from caller
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: ml-dash
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.10
|
|
4
4
|
Summary: ML experiment tracking and data storage
|
|
5
5
|
Keywords: machine-learning,experiment-tracking,mlops,data-storage
|
|
6
6
|
Author: Ge Yang, Tom Tao
|
|
@@ -68,10 +68,11 @@ Description-Content-Type: text/markdown
|
|
|
68
68
|
|
|
69
69
|
# ML-Dash
|
|
70
70
|
|
|
71
|
-
A simple and flexible SDK for ML experiment tracking and data storage.
|
|
71
|
+
A simple and flexible SDK for ML experiment tracking and data storage with background buffering for high-performance training.
|
|
72
72
|
|
|
73
73
|
## Features
|
|
74
74
|
|
|
75
|
+
### Core Features
|
|
75
76
|
- **Three Usage Styles**: Pre-configured singleton (dxp), context manager, or direct instantiation
|
|
76
77
|
- **Dual Operation Modes**: Remote (API server) or local (filesystem)
|
|
77
78
|
- **OAuth2 Authentication**: Secure device flow authentication for CLI and SDK
|
|
@@ -82,6 +83,13 @@ A simple and flexible SDK for ML experiment tracking and data storage.
|
|
|
82
83
|
- **Rich Metadata**: Tags, bindrs, descriptions, and custom metadata support
|
|
83
84
|
- **Simple API**: Minimal configuration, maximum flexibility
|
|
84
85
|
|
|
86
|
+
### Performance Features (New in 0.6.7)
|
|
87
|
+
- **Background Buffering**: Non-blocking I/O operations eliminate training interruptions
|
|
88
|
+
- **Automatic Batching**: Time-based (5s) and size-based (100 items) flush triggers
|
|
89
|
+
- **Track API**: Time-series data tracking for robotics, RL, and sequential experiments
|
|
90
|
+
- **Numpy Image Support**: Direct saving of numpy arrays as PNG/JPEG images
|
|
91
|
+
- **Parallel Uploads**: ThreadPoolExecutor for efficient file uploads
|
|
92
|
+
|
|
85
93
|
## Installation
|
|
86
94
|
|
|
87
95
|
<table>
|
|
@@ -93,14 +101,14 @@ A simple and flexible SDK for ML experiment tracking and data storage.
|
|
|
93
101
|
<td>
|
|
94
102
|
|
|
95
103
|
```bash
|
|
96
|
-
uv add ml-dash
|
|
104
|
+
uv add ml-dash
|
|
97
105
|
```
|
|
98
106
|
|
|
99
107
|
</td>
|
|
100
108
|
<td>
|
|
101
109
|
|
|
102
110
|
```bash
|
|
103
|
-
pip install ml-dash
|
|
111
|
+
pip install ml-dash
|
|
104
112
|
```
|
|
105
113
|
|
|
106
114
|
</td>
|
|
@@ -159,7 +167,75 @@ with Experiment(
|
|
|
159
167
|
|
|
160
168
|
```
|
|
161
169
|
|
|
162
|
-
|
|
170
|
+
## New Features in 0.6.7
|
|
171
|
+
|
|
172
|
+
### 🚀 Background Buffering (Non-blocking I/O)
|
|
173
|
+
|
|
174
|
+
All write operations are now buffered and executed in background threads:
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
with Experiment("my-project/exp").run as experiment:
|
|
178
|
+
for i in range(10000):
|
|
179
|
+
# Non-blocking! Returns immediately
|
|
180
|
+
experiment.log(f"Step {i}")
|
|
181
|
+
experiment.metrics("train").log(loss=loss, accuracy=acc)
|
|
182
|
+
experiment.files("frames").save_image(frame, to=f"frame_{i}.jpg")
|
|
183
|
+
|
|
184
|
+
# All data automatically flushed when context exits
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Configure buffering via environment variables:
|
|
188
|
+
```bash
|
|
189
|
+
export ML_DASH_BUFFER_ENABLED=true
|
|
190
|
+
export ML_DASH_FLUSH_INTERVAL=5.0
|
|
191
|
+
export ML_DASH_LOG_BATCH_SIZE=100
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### 📊 Track API (Time-Series Data)
|
|
195
|
+
|
|
196
|
+
Perfect for robotics, RL, and sequential experiments:
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
with Experiment("robotics/training").run as experiment:
|
|
200
|
+
for step in range(1000):
|
|
201
|
+
# Track robot position over time
|
|
202
|
+
experiment.track("robot/position").append({
|
|
203
|
+
"step": step,
|
|
204
|
+
"x": position[0],
|
|
205
|
+
"y": position[1],
|
|
206
|
+
"z": position[2]
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
# Track control signals
|
|
210
|
+
experiment.track("robot/control").append({
|
|
211
|
+
"step": step,
|
|
212
|
+
"motor1": ctrl[0],
|
|
213
|
+
"motor2": ctrl[1]
|
|
214
|
+
})
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
### 🖼️ Numpy Image Support
|
|
218
|
+
|
|
219
|
+
Save numpy arrays directly as images (PNG/JPEG):
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
import numpy as np
|
|
223
|
+
|
|
224
|
+
with Experiment("vision/training").run as experiment:
|
|
225
|
+
# From MuJoCo, OpenCV, PIL, etc.
|
|
226
|
+
pixels = renderer.render() # numpy array
|
|
227
|
+
|
|
228
|
+
# Save as PNG (lossless)
|
|
229
|
+
experiment.files("frames").save_image(pixels, to="frame.png")
|
|
230
|
+
|
|
231
|
+
# Save as JPEG with quality control
|
|
232
|
+
experiment.files("frames").save_image(pixels, to="frame.jpg", quality=85)
|
|
233
|
+
|
|
234
|
+
# Auto-detection also works
|
|
235
|
+
experiment.files("frames").save(pixels, to="frame.jpg")
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
See [CHANGELOG.md](CHANGELOG.md) for complete release notes.
|
|
163
239
|
|
|
164
240
|
## Development Setup
|
|
165
241
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
ml_dash/__init__.py,sha256=
|
|
1
|
+
ml_dash/__init__.py,sha256=bsn9dwFrkXyZNcm6A4pAKS3ow1Q-2C5Z1jpak-at0Wo,2995
|
|
2
2
|
ml_dash/auth/__init__.py,sha256=3lwM-Y8UBHPU1gFW2JNpmXlPVTnkGudWLKNFFKulQfo,1200
|
|
3
3
|
ml_dash/auth/constants.py,sha256=ku4QzQUMNjvyJwjy7AUdywMAZd59jXSxNHZxDiagUWU,280
|
|
4
4
|
ml_dash/auth/device_flow.py,sha256=DQOdPNlZCuU1umZOA_A6WXdRM3zWphnyo9IntToBl_A,7921
|
|
5
5
|
ml_dash/auth/device_secret.py,sha256=qUsz6M9S1GEIukvmz57eJEp57srSx74O4MU9mZEeDlE,1158
|
|
6
6
|
ml_dash/auth/exceptions.py,sha256=IeBwUzoaTyFtPwd4quFOIel49inIzuabe_ChEeEXEWI,725
|
|
7
|
-
ml_dash/auth/token_storage.py,sha256=
|
|
8
|
-
ml_dash/auto_start.py,sha256=
|
|
7
|
+
ml_dash/auth/token_storage.py,sha256=9YQXGrn41UVyc1wUvZYbTYLzxSt5NGOyNFNjeX28bjA,7976
|
|
8
|
+
ml_dash/auto_start.py,sha256=mYNjLGI2kyylIfOX5wGOR74gb9UlXg1n5OUQu7aw5SE,2412
|
|
9
9
|
ml_dash/buffer.py,sha256=i4-PZ703_yuKJPAXpmWBGm8jHAAdIBqiA0NIZQEc3wo,26201
|
|
10
10
|
ml_dash/cli.py,sha256=X8LsQA8Wfa1XuXsbvePaGo6NYer7f8CNzy33VT3jrqg,2740
|
|
11
11
|
ml_dash/cli_commands/__init__.py,sha256=bjAmV7MsW-bhtW_4SnLJ0Cfkt9h82vMDC8ebW1Ke8KE,38
|
|
@@ -15,9 +15,9 @@ ml_dash/cli_commands/download.py,sha256=Jw-ZeVH8SL9t2yRNCwSmQ0qUOzI4_iWGWt7SwAfM
|
|
|
15
15
|
ml_dash/cli_commands/list.py,sha256=H442wOAcWYtDwq6BS7lpZbkKhqfTXBCHGctbw8zT1Zw,20841
|
|
16
16
|
ml_dash/cli_commands/login.py,sha256=zX-urtUrfzg2qOGtKNYQgj6UloN9kzj4zEO6h_xwuNs,6782
|
|
17
17
|
ml_dash/cli_commands/logout.py,sha256=lTUUNyRXqvo61qNkCd4KBrPUujDAHnNqsHkU6bHie0U,1332
|
|
18
|
-
ml_dash/cli_commands/profile.py,sha256=
|
|
18
|
+
ml_dash/cli_commands/profile.py,sha256=PoRO1XA4bnOINptj4AO0SyNDBADeryPJBfgC74327e4,5997
|
|
19
19
|
ml_dash/cli_commands/upload.py,sha256=SSUfXC3qoNpoFvPM_ia-ing_N50LNiAvMy9op6FM9Ew,49664
|
|
20
|
-
ml_dash/client.py,sha256=
|
|
20
|
+
ml_dash/client.py,sha256=WgdQRwI9OzEB9dtBtkjFOWNP1t1jj7wr4gvcDUYzZBM,65806
|
|
21
21
|
ml_dash/config.py,sha256=oz2xvoBh2X_xUXWr92cPD5nFxXMT5LxVNypv5B5O0fA,3116
|
|
22
22
|
ml_dash/experiment.py,sha256=sq5Bu-vcKhzTQlUsI6SzIHUZyUkxn7ZSMp_AbhFe4Bw,43462
|
|
23
23
|
ml_dash/files.py,sha256=tGJCTxPfd9vmfvIEqstZjzLvqmHzMZffPXHz0jU9bYU,54441
|
|
@@ -26,11 +26,11 @@ ml_dash/metric.py,sha256=ghD1jnuv6dbjV1Jlo7q0mx9UEzpdto2Y1-oDWrSfg04,25809
|
|
|
26
26
|
ml_dash/params.py,sha256=pPFvknJAJX5uhckzjO1r-HNnKbQFFKDISFmOXNET5eY,9046
|
|
27
27
|
ml_dash/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
28
|
ml_dash/remote_auto_start.py,sha256=5fvQDHv1CWEKFb6WAa5_uyEInwV_SvotXjOO_6i6ZKE,1687
|
|
29
|
-
ml_dash/run.py,sha256=
|
|
29
|
+
ml_dash/run.py,sha256=Hlt_YHaN95TC3TDejzLjFmW9EWyKWi6ruibw-eiPm2U,8833
|
|
30
30
|
ml_dash/snowflake.py,sha256=14rEpRU5YltsmmmZW0EMUy_hdv5S5ME9gWVtmdmwfiU,4917
|
|
31
31
|
ml_dash/storage.py,sha256=x1W-dK6wQY36-LVOJ4kA8Dn07ObNQuIErQWJ3b0PoGY,44910
|
|
32
32
|
ml_dash/track.py,sha256=Dfg1ZnmKZ_FlE5ZfG8Qld_wN4RIMs3nrOxrxwf3thiY,8164
|
|
33
|
-
ml_dash-0.6.
|
|
34
|
-
ml_dash-0.6.
|
|
35
|
-
ml_dash-0.6.
|
|
36
|
-
ml_dash-0.6.
|
|
33
|
+
ml_dash-0.6.10.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
|
|
34
|
+
ml_dash-0.6.10.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
|
|
35
|
+
ml_dash-0.6.10.dist-info/METADATA,sha256=z9L0TSRTGJUSZ1gDYNR3Pj5IW3oh0AYeMFjEplnhFYA,9536
|
|
36
|
+
ml_dash-0.6.10.dist-info/RECORD,,
|
|
File without changes
|