ml-dash 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +85 -0
- ml_dash/auth/__init__.py +51 -0
- ml_dash/auth/constants.py +10 -0
- ml_dash/auth/device_flow.py +237 -0
- ml_dash/auth/device_secret.py +49 -0
- ml_dash/auth/exceptions.py +31 -0
- ml_dash/auth/token_storage.py +262 -0
- ml_dash/auto_start.py +52 -0
- ml_dash/cli.py +79 -0
- ml_dash/cli_commands/__init__.py +1 -0
- ml_dash/cli_commands/download.py +769 -0
- ml_dash/cli_commands/list.py +319 -0
- ml_dash/cli_commands/login.py +225 -0
- ml_dash/cli_commands/logout.py +54 -0
- ml_dash/cli_commands/upload.py +1248 -0
- ml_dash/client.py +1003 -0
- ml_dash/config.py +133 -0
- ml_dash/experiment.py +1116 -0
- ml_dash/files.py +785 -0
- ml_dash/log.py +181 -0
- ml_dash/metric.py +481 -0
- ml_dash/params.py +277 -0
- ml_dash/py.typed +0 -0
- ml_dash/remote_auto_start.py +55 -0
- ml_dash/storage.py +1127 -0
- ml_dash-0.6.1.dist-info/METADATA +248 -0
- ml_dash-0.6.1.dist-info/RECORD +29 -0
- ml_dash-0.6.1.dist-info/WHEEL +4 -0
- ml_dash-0.6.1.dist-info/entry_points.txt +3 -0
ml_dash/__init__.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ML-Dash Python SDK
|
|
3
|
+
|
|
4
|
+
A simple and flexible SDK for ML experiment metricing and data storage.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
|
|
8
|
+
# Quickest - dxp (pre-configured remote singleton)
|
|
9
|
+
# Requires: ml-dash login
|
|
10
|
+
from ml_dash import dxp
|
|
11
|
+
|
|
12
|
+
with dxp.run:
|
|
13
|
+
dxp.params.set(lr=0.001)
|
|
14
|
+
dxp.log().info("Training started")
|
|
15
|
+
# Auto-completes on exit from with block
|
|
16
|
+
|
|
17
|
+
# Local mode - explicit configuration
|
|
18
|
+
from ml_dash import Experiment
|
|
19
|
+
|
|
20
|
+
with Experiment(
|
|
21
|
+
name="my-experiment",
|
|
22
|
+
project="my-project",
|
|
23
|
+
local_path=".ml-dash"
|
|
24
|
+
) as experiment:
|
|
25
|
+
experiment.log("Training started")
|
|
26
|
+
experiment.params.set(lr=0.001)
|
|
27
|
+
experiment.metrics("loss").append(step=0, value=0.5)
|
|
28
|
+
|
|
29
|
+
# Remote mode - explicit configuration
|
|
30
|
+
with Experiment(
|
|
31
|
+
name="my-experiment",
|
|
32
|
+
project="my-project",
|
|
33
|
+
remote="https://api.dash.ml",
|
|
34
|
+
api_key="your-jwt-token"
|
|
35
|
+
) as experiment:
|
|
36
|
+
experiment.log("Training started")
|
|
37
|
+
|
|
38
|
+
# Decorator style
|
|
39
|
+
from ml_dash import ml_dash_experiment
|
|
40
|
+
|
|
41
|
+
@ml_dash_experiment(
|
|
42
|
+
name="my-experiment",
|
|
43
|
+
project="my-project"
|
|
44
|
+
)
|
|
45
|
+
def train_model(experiment):
|
|
46
|
+
experiment.log("Training started")
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from .experiment import Experiment, ml_dash_experiment, OperationMode, RunManager
|
|
50
|
+
from .client import RemoteClient
|
|
51
|
+
from .storage import LocalStorage
|
|
52
|
+
from .log import LogLevel, LogBuilder
|
|
53
|
+
from .params import ParametersBuilder
|
|
54
|
+
from .auto_start import dxp
|
|
55
|
+
|
|
56
|
+
__version__ = "0.1.0"
|
|
57
|
+
|
|
58
|
+
__all__ = [
|
|
59
|
+
"Experiment",
|
|
60
|
+
"ml_dash_experiment",
|
|
61
|
+
"OperationMode",
|
|
62
|
+
"RunManager",
|
|
63
|
+
"RemoteClient",
|
|
64
|
+
"LocalStorage",
|
|
65
|
+
"LogLevel",
|
|
66
|
+
"LogBuilder",
|
|
67
|
+
"ParametersBuilder",
|
|
68
|
+
"dxp",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
# Hidden for now - rdxp (remote auto-start singleton)
|
|
72
|
+
# Will be exposed in a future release
|
|
73
|
+
#
|
|
74
|
+
# # Lazy-load rdxp to avoid auto-connecting to server on package import
|
|
75
|
+
# _rdxp = None
|
|
76
|
+
#
|
|
77
|
+
# def __getattr__(name):
|
|
78
|
+
# """Lazy-load rdxp only when accessed."""
|
|
79
|
+
# if name == "rdxp":
|
|
80
|
+
# global _rdxp
|
|
81
|
+
# if _rdxp is None:
|
|
82
|
+
# from .remote_auto_start import rdxp as _loaded_rdxp
|
|
83
|
+
# _rdxp = _loaded_rdxp
|
|
84
|
+
# return _rdxp
|
|
85
|
+
# raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
ml_dash/auth/__init__.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Authentication module for ml-dash."""
|
|
2
|
+
|
|
3
|
+
from .constants import VUER_AUTH_URL, CLIENT_ID, DEFAULT_SCOPE
|
|
4
|
+
from .device_flow import DeviceFlowClient, DeviceFlowResponse
|
|
5
|
+
from .device_secret import (
|
|
6
|
+
generate_device_secret,
|
|
7
|
+
hash_device_secret,
|
|
8
|
+
get_or_create_device_secret,
|
|
9
|
+
)
|
|
10
|
+
from .token_storage import (
|
|
11
|
+
TokenStorage,
|
|
12
|
+
KeyringStorage,
|
|
13
|
+
EncryptedFileStorage,
|
|
14
|
+
PlaintextFileStorage,
|
|
15
|
+
get_token_storage,
|
|
16
|
+
)
|
|
17
|
+
from .exceptions import (
|
|
18
|
+
AuthenticationError,
|
|
19
|
+
NotAuthenticatedError,
|
|
20
|
+
DeviceCodeExpiredError,
|
|
21
|
+
AuthorizationDeniedError,
|
|
22
|
+
TokenExchangeError,
|
|
23
|
+
StorageError,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
# Constants
|
|
28
|
+
"VUER_AUTH_URL",
|
|
29
|
+
"CLIENT_ID",
|
|
30
|
+
"DEFAULT_SCOPE",
|
|
31
|
+
# Device Flow
|
|
32
|
+
"DeviceFlowClient",
|
|
33
|
+
"DeviceFlowResponse",
|
|
34
|
+
# Device Secret
|
|
35
|
+
"generate_device_secret",
|
|
36
|
+
"hash_device_secret",
|
|
37
|
+
"get_or_create_device_secret",
|
|
38
|
+
# Token Storage
|
|
39
|
+
"TokenStorage",
|
|
40
|
+
"KeyringStorage",
|
|
41
|
+
"EncryptedFileStorage",
|
|
42
|
+
"PlaintextFileStorage",
|
|
43
|
+
"get_token_storage",
|
|
44
|
+
# Exceptions
|
|
45
|
+
"AuthenticationError",
|
|
46
|
+
"NotAuthenticatedError",
|
|
47
|
+
"DeviceCodeExpiredError",
|
|
48
|
+
"AuthorizationDeniedError",
|
|
49
|
+
"TokenExchangeError",
|
|
50
|
+
"StorageError",
|
|
51
|
+
]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Authentication constants for ml-dash."""
|
|
2
|
+
|
|
3
|
+
# Vuer-auth server URL
|
|
4
|
+
VUER_AUTH_URL = "https://auth.vuer.ai"
|
|
5
|
+
|
|
6
|
+
# OAuth client ID for ml-dash
|
|
7
|
+
CLIENT_ID = "ml-dash-client"
|
|
8
|
+
|
|
9
|
+
# Default OAuth scopes (no offline_access since we get permanent tokens)
|
|
10
|
+
DEFAULT_SCOPE = "openid profile email"
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Device authorization flow (RFC 8628) client for ml-dash."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from .constants import VUER_AUTH_URL, CLIENT_ID, DEFAULT_SCOPE
|
|
10
|
+
from .device_secret import hash_device_secret
|
|
11
|
+
from .exceptions import (
|
|
12
|
+
DeviceCodeExpiredError,
|
|
13
|
+
AuthorizationDeniedError,
|
|
14
|
+
TokenExchangeError,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class DeviceFlowResponse:
|
|
20
|
+
"""Response from device flow initiation."""
|
|
21
|
+
|
|
22
|
+
user_code: str
|
|
23
|
+
device_code: str
|
|
24
|
+
verification_uri: str
|
|
25
|
+
verification_uri_complete: str
|
|
26
|
+
expires_in: int
|
|
27
|
+
interval: int
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class DeviceFlowClient:
|
|
31
|
+
"""Client for OAuth 2.0 Device Authorization Flow (RFC 8628)."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, device_secret: str, ml_dash_server_url: str):
|
|
34
|
+
"""Initialize device flow client.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
device_secret: Persistent device secret for this client
|
|
38
|
+
ml_dash_server_url: ML-Dash server URL for token exchange
|
|
39
|
+
"""
|
|
40
|
+
self.device_secret = device_secret
|
|
41
|
+
self.ml_dash_server_url = ml_dash_server_url.rstrip("/")
|
|
42
|
+
|
|
43
|
+
def start_device_flow(self, scope: str = DEFAULT_SCOPE) -> DeviceFlowResponse:
|
|
44
|
+
"""Initiate device authorization flow with vuer-auth.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
scope: OAuth scopes to request
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
DeviceFlowResponse with user code and verification URI
|
|
51
|
+
|
|
52
|
+
Raises:
|
|
53
|
+
httpx.HTTPError: If request fails
|
|
54
|
+
"""
|
|
55
|
+
response = httpx.post(
|
|
56
|
+
f"{VUER_AUTH_URL}/api/device-flow/start",
|
|
57
|
+
json={
|
|
58
|
+
"client_id": CLIENT_ID,
|
|
59
|
+
"scope": scope,
|
|
60
|
+
"device_secret_hash": hash_device_secret(self.device_secret),
|
|
61
|
+
},
|
|
62
|
+
timeout=10.0,
|
|
63
|
+
)
|
|
64
|
+
response.raise_for_status()
|
|
65
|
+
data = response.json()
|
|
66
|
+
|
|
67
|
+
return DeviceFlowResponse(
|
|
68
|
+
user_code=data["user_code"],
|
|
69
|
+
device_code=data.get("device_code", ""),
|
|
70
|
+
verification_uri=data["verification_uri"],
|
|
71
|
+
verification_uri_complete=data.get(
|
|
72
|
+
"verification_uri_complete",
|
|
73
|
+
f"{data['verification_uri']}?code={data['user_code'].replace('-', '')}"
|
|
74
|
+
),
|
|
75
|
+
expires_in=data.get("expires_in", 600),
|
|
76
|
+
interval=data.get("interval", 5),
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
def poll_for_token(
|
|
80
|
+
self,
|
|
81
|
+
max_attempts: int = 120,
|
|
82
|
+
progress_callback: Optional[callable] = None,
|
|
83
|
+
) -> str:
|
|
84
|
+
"""Poll vuer-auth for authorization completion.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
max_attempts: Maximum polling attempts (default: 120 = 10 minutes at 5s intervals)
|
|
88
|
+
progress_callback: Optional callback(elapsed_seconds) for progress updates
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
Vuer-auth access token (JWT)
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
DeviceCodeExpiredError: If device code expires
|
|
95
|
+
AuthorizationDeniedError: If user denies authorization
|
|
96
|
+
TimeoutError: If polling times out
|
|
97
|
+
"""
|
|
98
|
+
device_secret_hash = hash_device_secret(self.device_secret)
|
|
99
|
+
|
|
100
|
+
for attempt in range(max_attempts):
|
|
101
|
+
elapsed = attempt * 5 # 5 second intervals
|
|
102
|
+
|
|
103
|
+
if progress_callback:
|
|
104
|
+
progress_callback(elapsed)
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
response = httpx.post(
|
|
108
|
+
f"{VUER_AUTH_URL}/api/device-flow/poll",
|
|
109
|
+
json={
|
|
110
|
+
"client_id": CLIENT_ID,
|
|
111
|
+
"device_secret_hash": device_secret_hash,
|
|
112
|
+
},
|
|
113
|
+
timeout=10.0,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if response.status_code == 200:
|
|
117
|
+
# Authorization successful
|
|
118
|
+
data = response.json()
|
|
119
|
+
return data["access_token"]
|
|
120
|
+
|
|
121
|
+
# Check error responses
|
|
122
|
+
error_data = response.json()
|
|
123
|
+
error = error_data.get("error")
|
|
124
|
+
|
|
125
|
+
if error == "authorization_pending":
|
|
126
|
+
# Still waiting for user authorization
|
|
127
|
+
time.sleep(5)
|
|
128
|
+
continue
|
|
129
|
+
elif error == "expired_token":
|
|
130
|
+
raise DeviceCodeExpiredError(
|
|
131
|
+
"Device code expired. Please run 'ml-dash login' again."
|
|
132
|
+
)
|
|
133
|
+
elif error == "access_denied":
|
|
134
|
+
raise AuthorizationDeniedError(
|
|
135
|
+
"User denied authorization request."
|
|
136
|
+
)
|
|
137
|
+
elif error == "slow_down":
|
|
138
|
+
# Server requests slower polling
|
|
139
|
+
time.sleep(10)
|
|
140
|
+
continue
|
|
141
|
+
else:
|
|
142
|
+
# Unknown error
|
|
143
|
+
raise TokenExchangeError(f"Device flow error: {error}")
|
|
144
|
+
|
|
145
|
+
except httpx.HTTPError as e:
|
|
146
|
+
# Network error, retry
|
|
147
|
+
time.sleep(5)
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
raise TimeoutError(
|
|
151
|
+
"Authorization timed out after 10 minutes. Please run 'ml-dash login' again."
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
def exchange_token(self, vuer_auth_token: str) -> str:
|
|
155
|
+
"""Exchange vuer-auth token for ml-dash permanent token.
|
|
156
|
+
|
|
157
|
+
This calls the ml-dash server's token exchange endpoint.
|
|
158
|
+
The server will:
|
|
159
|
+
1. Decode the vuer-auth JWT
|
|
160
|
+
2. Validate signature and expiry
|
|
161
|
+
3. Extract username from claims
|
|
162
|
+
4. Generate a permanent ml-dash token for that username
|
|
163
|
+
5. Return the ml-dash token
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
vuer_auth_token: Temporary vuer-auth JWT access token
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
Permanent ml-dash token string
|
|
170
|
+
|
|
171
|
+
Raises:
|
|
172
|
+
TokenExchangeError: If exchange fails
|
|
173
|
+
httpx.HTTPError: If request fails
|
|
174
|
+
"""
|
|
175
|
+
try:
|
|
176
|
+
response = httpx.post(
|
|
177
|
+
f"{self.ml_dash_server_url}/api/auth/exchange",
|
|
178
|
+
headers={"Authorization": f"Bearer {vuer_auth_token}"},
|
|
179
|
+
timeout=10.0,
|
|
180
|
+
)
|
|
181
|
+
response.raise_for_status()
|
|
182
|
+
data = response.json()
|
|
183
|
+
|
|
184
|
+
ml_dash_token = data.get("ml_dash_token")
|
|
185
|
+
if not ml_dash_token:
|
|
186
|
+
raise TokenExchangeError(
|
|
187
|
+
"Server response missing ml_dash_token field"
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
return ml_dash_token
|
|
191
|
+
|
|
192
|
+
except httpx.HTTPStatusError as e:
|
|
193
|
+
if e.response.status_code == 401:
|
|
194
|
+
raise TokenExchangeError(
|
|
195
|
+
"Vuer-auth token invalid or expired. Please try logging in again."
|
|
196
|
+
)
|
|
197
|
+
elif e.response.status_code == 404:
|
|
198
|
+
raise TokenExchangeError(
|
|
199
|
+
"Token exchange endpoint not found. "
|
|
200
|
+
"Please ensure ml-dash server is up to date."
|
|
201
|
+
)
|
|
202
|
+
else:
|
|
203
|
+
raise TokenExchangeError(
|
|
204
|
+
f"Token exchange failed: {e.response.status_code} {e.response.text}"
|
|
205
|
+
)
|
|
206
|
+
except httpx.HTTPError as e:
|
|
207
|
+
raise TokenExchangeError(f"Network error during token exchange: {e}")
|
|
208
|
+
|
|
209
|
+
def authenticate(
|
|
210
|
+
self, progress_callback: Optional[callable] = None
|
|
211
|
+
) -> str:
|
|
212
|
+
"""Complete full device authorization flow.
|
|
213
|
+
|
|
214
|
+
This is a convenience method that:
|
|
215
|
+
1. Starts device flow with vuer-auth
|
|
216
|
+
2. Polls for authorization
|
|
217
|
+
3. Exchanges vuer-auth token for ml-dash token
|
|
218
|
+
|
|
219
|
+
Args:
|
|
220
|
+
progress_callback: Optional callback(elapsed_seconds) for progress updates
|
|
221
|
+
|
|
222
|
+
Returns:
|
|
223
|
+
Permanent ml-dash token string
|
|
224
|
+
|
|
225
|
+
Raises:
|
|
226
|
+
Various authentication exceptions
|
|
227
|
+
"""
|
|
228
|
+
# Step 1: Start device flow
|
|
229
|
+
flow = self.start_device_flow()
|
|
230
|
+
|
|
231
|
+
# Step 2: Poll for authorization (caller should display user_code)
|
|
232
|
+
vuer_auth_token = self.poll_for_token(progress_callback=progress_callback)
|
|
233
|
+
|
|
234
|
+
# Step 3: Exchange for ml-dash token
|
|
235
|
+
ml_dash_token = self.exchange_token(vuer_auth_token)
|
|
236
|
+
|
|
237
|
+
return ml_dash_token
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Device secret generation and management for ml-dash."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import secrets
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from ml_dash.config import Config
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def generate_device_secret() -> str:
|
|
12
|
+
"""Generate a cryptographically secure device secret.
|
|
13
|
+
|
|
14
|
+
Returns:
|
|
15
|
+
A 64-character hexadecimal string (256 bits of entropy)
|
|
16
|
+
"""
|
|
17
|
+
return secrets.token_hex(32) # 32 bytes = 256 bits
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def hash_device_secret(secret: str) -> str:
|
|
21
|
+
"""Hash device secret using SHA256.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
secret: The device secret to hash
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
SHA256 hash as hexadecimal string
|
|
28
|
+
"""
|
|
29
|
+
return hashlib.sha256(secret.encode()).hexdigest()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_or_create_device_secret(config: "Config") -> str:
|
|
33
|
+
"""Load device secret from config or generate a new one.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
config: ML-Dash configuration object
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
Device secret string
|
|
40
|
+
"""
|
|
41
|
+
device_secret = config.device_secret
|
|
42
|
+
|
|
43
|
+
if not device_secret:
|
|
44
|
+
# Generate new device secret
|
|
45
|
+
device_secret = generate_device_secret()
|
|
46
|
+
config.set("device_secret", device_secret)
|
|
47
|
+
config.save()
|
|
48
|
+
|
|
49
|
+
return device_secret
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Custom exceptions for ml-dash authentication."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AuthenticationError(Exception):
|
|
5
|
+
"""Base exception for authentication errors."""
|
|
6
|
+
pass
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class NotAuthenticatedError(AuthenticationError):
|
|
10
|
+
"""Raised when user is not authenticated."""
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DeviceCodeExpiredError(AuthenticationError):
|
|
15
|
+
"""Raised when device code expires before authorization."""
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AuthorizationDeniedError(AuthenticationError):
|
|
20
|
+
"""Raised when user denies authorization request."""
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TokenExchangeError(AuthenticationError):
|
|
25
|
+
"""Raised when token exchange with ml-dash server fails."""
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class StorageError(Exception):
|
|
30
|
+
"""Base exception for token storage errors."""
|
|
31
|
+
pass
|