ml-dash 0.6.2__py3-none-any.whl → 0.6.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_dash/__init__.py +64 -36
- ml_dash/auth/token_storage.py +226 -267
- ml_dash/auto_start.py +15 -28
- ml_dash/cli.py +2 -16
- ml_dash/cli_commands/download.py +667 -757
- ml_dash/cli_commands/list.py +13 -146
- ml_dash/cli_commands/login.py +183 -190
- ml_dash/cli_commands/upload.py +1141 -1291
- ml_dash/client.py +6 -79
- ml_dash/config.py +119 -119
- ml_dash/experiment.py +1034 -1234
- ml_dash/files.py +224 -339
- ml_dash/log.py +7 -7
- ml_dash/metric.py +100 -359
- ml_dash/params.py +6 -6
- ml_dash/remote_auto_start.py +17 -20
- ml_dash/run.py +65 -211
- ml_dash/storage.py +1081 -1051
- {ml_dash-0.6.2.dist-info → ml_dash-0.6.2rc1.dist-info}/METADATA +14 -12
- ml_dash-0.6.2rc1.dist-info/RECORD +30 -0
- {ml_dash-0.6.2.dist-info → ml_dash-0.6.2rc1.dist-info}/WHEEL +1 -1
- ml_dash/cli_commands/api.py +0 -165
- ml_dash/cli_commands/profile.py +0 -92
- ml_dash/snowflake.py +0 -173
- ml_dash-0.6.2.dist-info/RECORD +0 -33
- {ml_dash-0.6.2.dist-info → ml_dash-0.6.2rc1.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: ml-dash
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.2rc1
|
|
4
4
|
Summary: ML experiment tracking and data storage
|
|
5
5
|
Keywords: machine-learning,experiment-tracking,mlops,data-storage
|
|
6
6
|
Author: Ge Yang, Tom Tao
|
|
@@ -43,7 +43,7 @@ Requires-Dist: imageio-ffmpeg>=0.4.9
|
|
|
43
43
|
Requires-Dist: scikit-image>=0.21.0
|
|
44
44
|
Requires-Dist: rich>=13.0.0
|
|
45
45
|
Requires-Dist: cryptography>=42.0.0
|
|
46
|
-
Requires-Dist: params-proto
|
|
46
|
+
Requires-Dist: params-proto>=3.0.0rc12
|
|
47
47
|
Requires-Dist: keyring>=25.0.0 ; extra == 'auth'
|
|
48
48
|
Requires-Dist: qrcode>=8.0.0 ; extra == 'auth'
|
|
49
49
|
Requires-Dist: pytest>=8.0.0 ; extra == 'dev'
|
|
@@ -122,16 +122,16 @@ This opens your browser for secure OAuth2 authentication. Your credentials are s
|
|
|
122
122
|
#### Option A: Use the Pre-configured Singleton (Easiest)
|
|
123
123
|
|
|
124
124
|
```python
|
|
125
|
-
from ml_dash
|
|
125
|
+
from ml_dash import dxp
|
|
126
126
|
|
|
127
127
|
# Start experiment (uploads to https://api.dash.ml by default)
|
|
128
128
|
with dxp.run:
|
|
129
|
-
dxp.log("Training started"
|
|
129
|
+
dxp.log().info("Training started")
|
|
130
130
|
dxp.params.set(learning_rate=0.001, batch_size=32)
|
|
131
131
|
|
|
132
132
|
for epoch in range(10):
|
|
133
133
|
loss = train_one_epoch()
|
|
134
|
-
dxp.metrics("
|
|
134
|
+
dxp.metrics("loss").append(value=loss, epoch=epoch)
|
|
135
135
|
```
|
|
136
136
|
|
|
137
137
|
#### Option B: Create Your Own Experiment
|
|
@@ -140,11 +140,12 @@ with dxp.run:
|
|
|
140
140
|
from ml_dash import Experiment
|
|
141
141
|
|
|
142
142
|
with Experiment(
|
|
143
|
-
|
|
144
|
-
|
|
143
|
+
name="my-experiment",
|
|
144
|
+
project="my-project",
|
|
145
|
+
remote="https://api.dash.ml" # token auto-loaded
|
|
145
146
|
).run as experiment:
|
|
146
|
-
|
|
147
|
-
|
|
147
|
+
experiment.log().info("Hello!")
|
|
148
|
+
experiment.params.set(lr=0.001)
|
|
148
149
|
```
|
|
149
150
|
|
|
150
151
|
#### Option C: Local Mode (No Authentication Required)
|
|
@@ -153,10 +154,11 @@ with Experiment(
|
|
|
153
154
|
from ml_dash import Experiment
|
|
154
155
|
|
|
155
156
|
with Experiment(
|
|
156
|
-
|
|
157
|
+
name="my-experiment",
|
|
158
|
+
project="my-project",
|
|
159
|
+
local_path=".ml-dash"
|
|
157
160
|
).run as experiment:
|
|
158
|
-
|
|
159
|
-
|
|
161
|
+
experiment.log().info("Running locally")
|
|
160
162
|
```
|
|
161
163
|
|
|
162
164
|
See [docs/getting-started.md](docs/getting-started.md) for more examples.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
ml_dash/__init__.py,sha256=JJlSut9U_nixMsZxaqj9QQK0J3WUckGis7ouOQegWmw,2260
|
|
2
|
+
ml_dash/auth/__init__.py,sha256=3lwM-Y8UBHPU1gFW2JNpmXlPVTnkGudWLKNFFKulQfo,1200
|
|
3
|
+
ml_dash/auth/constants.py,sha256=ku4QzQUMNjvyJwjy7AUdywMAZd59jXSxNHZxDiagUWU,280
|
|
4
|
+
ml_dash/auth/device_flow.py,sha256=DQOdPNlZCuU1umZOA_A6WXdRM3zWphnyo9IntToBl_A,7921
|
|
5
|
+
ml_dash/auth/device_secret.py,sha256=qUsz6M9S1GEIukvmz57eJEp57srSx74O4MU9mZEeDlE,1158
|
|
6
|
+
ml_dash/auth/exceptions.py,sha256=IeBwUzoaTyFtPwd4quFOIel49inIzuabe_ChEeEXEWI,725
|
|
7
|
+
ml_dash/auth/token_storage.py,sha256=vuWDAhQPdjUwNTJAdoWP-EUDudIJVTNdWkWnvgS4Qgc,8217
|
|
8
|
+
ml_dash/auto_start.py,sha256=5_SIc0jyZEptGKuF5Lu910RnKcmVh03HM6QrrFST8WA,1440
|
|
9
|
+
ml_dash/cli.py,sha256=r55bNa3T_guMnSl2dwsj0WF9Ln-vOGsBmbrooMKYhRI,2103
|
|
10
|
+
ml_dash/cli_commands/__init__.py,sha256=bjAmV7MsW-bhtW_4SnLJ0Cfkt9h82vMDC8ebW1Ke8KE,38
|
|
11
|
+
ml_dash/cli_commands/download.py,sha256=jOUx2aejA_Hz4ttuL8ubFY3pI4j39fYtdn-sZyrYCrE,28674
|
|
12
|
+
ml_dash/cli_commands/list.py,sha256=0G0XMJ8cH8_k37IEjwq20t6obT-tKxZPTxr221UF8KA,9757
|
|
13
|
+
ml_dash/cli_commands/login.py,sha256=0v8UPj_Y-uYmNDfc8Zv6xddFJPx3_VgVNeESYO700b0,7173
|
|
14
|
+
ml_dash/cli_commands/logout.py,sha256=lTUUNyRXqvo61qNkCd4KBrPUujDAHnNqsHkU6bHie0U,1332
|
|
15
|
+
ml_dash/cli_commands/upload.py,sha256=4O9iZOQDOiiBBsoDG1Cr8BqeBGSZZSfc3i-5FERjFtU,46439
|
|
16
|
+
ml_dash/client.py,sha256=0jM3HzNyFy-QBN7ZLXQTuiIuiTuh4K1Zdldb4Eny0NY,29531
|
|
17
|
+
ml_dash/config.py,sha256=aNqX_HT2Yo-BzjFexQ97gQK9xOiBUwuJJ65dKM3oJjs,3481
|
|
18
|
+
ml_dash/experiment.py,sha256=4mbJ6nul-k5Il1z0zjX2mmw56TbhWkeW4KyUa5DHvcU,35410
|
|
19
|
+
ml_dash/files.py,sha256=OCcGI2RHV5iO5BSWObCpz-GVJ2-wGFAgTqGGhDSesbQ,45168
|
|
20
|
+
ml_dash/log.py,sha256=0yXaNnFwYeBI3tRLHX3kkqWRpg0MbSGwmgjnOfsElCk,5350
|
|
21
|
+
ml_dash/metric.py,sha256=vz3YwO1YBWZ797l7TzD5m9WwNBBXlrATyRRzSti8QS0,17194
|
|
22
|
+
ml_dash/params.py,sha256=S4wHQfv0EQRXRrFbKLuEKTQOLp5eqnALcIk1pZNroBM,9124
|
|
23
|
+
ml_dash/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
24
|
+
ml_dash/remote_auto_start.py,sha256=BAsTu41e9SboeSCn9N2vn4ftoYeUVklcAcNM45HJVGA,1610
|
|
25
|
+
ml_dash/run.py,sha256=JwLigo2trdkwVrv-eCP4q9enSxLzla8jXeHXd6ZvNFk,2449
|
|
26
|
+
ml_dash/storage.py,sha256=iaFpH-2eK-Vd_b1lPpOQHxJI9GGkehB9g7pT7-r3nu4,39553
|
|
27
|
+
ml_dash-0.6.2rc1.dist-info/WHEEL,sha256=ELhySV62sOro8I5wRaLaF3TWxhBpkcDkdZUdAYLy_Hk,78
|
|
28
|
+
ml_dash-0.6.2rc1.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
|
|
29
|
+
ml_dash-0.6.2rc1.dist-info/METADATA,sha256=d_EDyVjjcNTdhItUZYayP--dzrIchFsIOOBJ4aWrKSU,7206
|
|
30
|
+
ml_dash-0.6.2rc1.dist-info/RECORD,,
|
ml_dash/cli_commands/api.py
DELETED
|
@@ -1,165 +0,0 @@
|
|
|
1
|
-
"""API command for ml-dash CLI - send GraphQL queries to the server."""
|
|
2
|
-
|
|
3
|
-
import argparse
|
|
4
|
-
import json
|
|
5
|
-
|
|
6
|
-
from rich.console import Console
|
|
7
|
-
|
|
8
|
-
from ml_dash.client import RemoteClient
|
|
9
|
-
from ml_dash.config import config
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def add_parser(subparsers):
|
|
13
|
-
"""Add api command parser."""
|
|
14
|
-
parser = subparsers.add_parser(
|
|
15
|
-
"api",
|
|
16
|
-
help="Send GraphQL queries to ml-dash server",
|
|
17
|
-
description="""Send GraphQL queries to the ml-dash server.
|
|
18
|
-
|
|
19
|
-
Examples:
|
|
20
|
-
# Query current user
|
|
21
|
-
ml-dash api --query "me { username name email }"
|
|
22
|
-
|
|
23
|
-
# Query with arguments (single quotes auto-converted to double)
|
|
24
|
-
ml-dash api --query "user(title: 'hello') { id title }"
|
|
25
|
-
|
|
26
|
-
# Extract specific field with jq-like syntax
|
|
27
|
-
ml-dash api --query "me { username }" --jq ".data.me.username"
|
|
28
|
-
|
|
29
|
-
# Mutation to update username
|
|
30
|
-
ml-dash api --mutation "updateUser(username: 'newname') { username }"
|
|
31
|
-
|
|
32
|
-
Notes:
|
|
33
|
-
- Single quotes are auto-converted to double quotes for GraphQL
|
|
34
|
-
- Use --jq for dot-path extraction (built-in, no deps)
|
|
35
|
-
""",
|
|
36
|
-
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
37
|
-
)
|
|
38
|
-
group = parser.add_mutually_exclusive_group(required=True)
|
|
39
|
-
group.add_argument(
|
|
40
|
-
"--query", "-q",
|
|
41
|
-
metavar="QUERY",
|
|
42
|
-
help="GraphQL query string",
|
|
43
|
-
)
|
|
44
|
-
group.add_argument(
|
|
45
|
-
"--mutation", "-m",
|
|
46
|
-
metavar="MUTATION",
|
|
47
|
-
help="GraphQL mutation string",
|
|
48
|
-
)
|
|
49
|
-
parser.add_argument(
|
|
50
|
-
"--jq",
|
|
51
|
-
metavar="PATH",
|
|
52
|
-
help="Extract value using dot-path (e.g., .data.me.username)",
|
|
53
|
-
)
|
|
54
|
-
parser.add_argument(
|
|
55
|
-
"--dash-url",
|
|
56
|
-
type=str,
|
|
57
|
-
help="ML-Dash server URL (default: https://api.dash.ml)",
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def extract_path(data, path: str):
|
|
62
|
-
"""Extract value from nested dict using dot-path notation.
|
|
63
|
-
|
|
64
|
-
Args:
|
|
65
|
-
data: Nested dict/list structure
|
|
66
|
-
path: Dot-separated path (e.g., ".data.me.username")
|
|
67
|
-
|
|
68
|
-
Returns:
|
|
69
|
-
Extracted value
|
|
70
|
-
|
|
71
|
-
Examples:
|
|
72
|
-
>>> extract_path({"data": {"me": {"username": "ge"}}}, ".data.me.username")
|
|
73
|
-
"ge"
|
|
74
|
-
"""
|
|
75
|
-
for key in path.lstrip(".").split("."):
|
|
76
|
-
if key:
|
|
77
|
-
if isinstance(data, dict):
|
|
78
|
-
data = data[key]
|
|
79
|
-
elif isinstance(data, list):
|
|
80
|
-
data = data[int(key)]
|
|
81
|
-
else:
|
|
82
|
-
raise KeyError(f"Cannot access '{key}' on {type(data).__name__}")
|
|
83
|
-
return data
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def fix_quotes(query: str) -> str:
|
|
87
|
-
"""Convert single quotes to double quotes for GraphQL.
|
|
88
|
-
|
|
89
|
-
GraphQL requires double quotes for strings. This allows users to write
|
|
90
|
-
queries with single quotes for shell convenience.
|
|
91
|
-
|
|
92
|
-
Args:
|
|
93
|
-
query: GraphQL query string with possible single quotes
|
|
94
|
-
|
|
95
|
-
Returns:
|
|
96
|
-
Query with single quotes converted to double quotes
|
|
97
|
-
"""
|
|
98
|
-
# Simple conversion - assumes single quotes are for strings
|
|
99
|
-
# This handles: user(title: 'hello') -> user(title: "hello")
|
|
100
|
-
return query.replace("'", '"')
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def build_query(query: str, is_mutation: bool) -> str:
|
|
104
|
-
"""Build complete GraphQL query string.
|
|
105
|
-
|
|
106
|
-
Args:
|
|
107
|
-
query: Query or mutation body
|
|
108
|
-
is_mutation: Whether this is a mutation
|
|
109
|
-
|
|
110
|
-
Returns:
|
|
111
|
-
Complete GraphQL query string
|
|
112
|
-
"""
|
|
113
|
-
query = query.strip()
|
|
114
|
-
query = fix_quotes(query)
|
|
115
|
-
|
|
116
|
-
# If already properly formatted, return as-is
|
|
117
|
-
if query.startswith("{") or query.startswith("mutation") or query.startswith("query"):
|
|
118
|
-
return query
|
|
119
|
-
|
|
120
|
-
# Wrap appropriately
|
|
121
|
-
if is_mutation:
|
|
122
|
-
return "mutation { " + query + " }"
|
|
123
|
-
else:
|
|
124
|
-
return "{ " + query + " }"
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def cmd_api(args) -> int:
|
|
128
|
-
"""Execute api command."""
|
|
129
|
-
console = Console()
|
|
130
|
-
|
|
131
|
-
# Get remote URL
|
|
132
|
-
remote_url = args.dash_url or config.remote_url or "https://api.dash.ml"
|
|
133
|
-
|
|
134
|
-
try:
|
|
135
|
-
# Initialize client
|
|
136
|
-
client = RemoteClient(base_url=remote_url)
|
|
137
|
-
|
|
138
|
-
# Determine query type and build query
|
|
139
|
-
if args.mutation:
|
|
140
|
-
query = build_query(args.mutation, is_mutation=True)
|
|
141
|
-
else:
|
|
142
|
-
query = build_query(args.query, is_mutation=False)
|
|
143
|
-
|
|
144
|
-
# Execute GraphQL query
|
|
145
|
-
result = client.graphql_query(query)
|
|
146
|
-
|
|
147
|
-
# Apply jq path extraction if specified
|
|
148
|
-
if args.jq:
|
|
149
|
-
try:
|
|
150
|
-
result = extract_path(result, args.jq)
|
|
151
|
-
except (KeyError, IndexError, TypeError) as e:
|
|
152
|
-
console.print(f"[red]Error extracting path '{args.jq}': {e}[/red]")
|
|
153
|
-
return 1
|
|
154
|
-
|
|
155
|
-
# Output result
|
|
156
|
-
if isinstance(result, (dict, list)):
|
|
157
|
-
console.print_json(json.dumps(result))
|
|
158
|
-
else:
|
|
159
|
-
console.print(json.dumps(result))
|
|
160
|
-
|
|
161
|
-
return 0
|
|
162
|
-
|
|
163
|
-
except Exception as e:
|
|
164
|
-
console.print(f"[red]Error: {e}[/red]")
|
|
165
|
-
return 1
|
ml_dash/cli_commands/profile.py
DELETED
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
"""Profile command for ml-dash CLI - shows current user and configuration."""
|
|
2
|
-
|
|
3
|
-
import json
|
|
4
|
-
|
|
5
|
-
from rich.console import Console
|
|
6
|
-
from rich.panel import Panel
|
|
7
|
-
from rich.table import Table
|
|
8
|
-
|
|
9
|
-
from ml_dash.auth.token_storage import decode_jwt_payload, get_token_storage
|
|
10
|
-
from ml_dash.config import config
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def add_parser(subparsers):
|
|
14
|
-
"""Add profile command parser."""
|
|
15
|
-
parser = subparsers.add_parser(
|
|
16
|
-
"profile",
|
|
17
|
-
help="Show current user profile",
|
|
18
|
-
description="Display the current authenticated user profile and configuration.",
|
|
19
|
-
)
|
|
20
|
-
parser.add_argument(
|
|
21
|
-
"--json",
|
|
22
|
-
action="store_true",
|
|
23
|
-
help="Output as JSON",
|
|
24
|
-
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def cmd_profile(args) -> int:
|
|
28
|
-
"""Execute info command."""
|
|
29
|
-
console = Console()
|
|
30
|
-
|
|
31
|
-
# Load token
|
|
32
|
-
storage = get_token_storage()
|
|
33
|
-
token = storage.load("ml-dash-token")
|
|
34
|
-
|
|
35
|
-
import getpass
|
|
36
|
-
|
|
37
|
-
info = {
|
|
38
|
-
"authenticated": False,
|
|
39
|
-
"remote_url": config.remote_url,
|
|
40
|
-
"local_user": getpass.getuser(),
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
if token:
|
|
44
|
-
info["authenticated"] = True
|
|
45
|
-
info["user"] = decode_jwt_payload(token)
|
|
46
|
-
|
|
47
|
-
if args.json:
|
|
48
|
-
console.print_json(json.dumps(info))
|
|
49
|
-
return 0
|
|
50
|
-
|
|
51
|
-
# Rich display
|
|
52
|
-
if not info["authenticated"]:
|
|
53
|
-
console.print(
|
|
54
|
-
Panel(
|
|
55
|
-
f"[bold cyan]OS Username:[/bold cyan] {info.get('local_user')}\n\n"
|
|
56
|
-
"[yellow]Not authenticated[/yellow]\n\n"
|
|
57
|
-
"Run [cyan]ml-dash login[/cyan] to authenticate.",
|
|
58
|
-
title="[bold]ML-Dash Info[/bold]",
|
|
59
|
-
border_style="yellow",
|
|
60
|
-
)
|
|
61
|
-
)
|
|
62
|
-
return 0
|
|
63
|
-
|
|
64
|
-
# Build info table
|
|
65
|
-
table = Table(show_header=False, box=None, padding=(0, 2))
|
|
66
|
-
table.add_column("Key", style="bold cyan")
|
|
67
|
-
table.add_column("Value")
|
|
68
|
-
|
|
69
|
-
# table.add_row("OS Username", info.get("local_user"))
|
|
70
|
-
user = info.get("user", {})
|
|
71
|
-
if user.get("username"):
|
|
72
|
-
table.add_row("Username", user["username"])
|
|
73
|
-
else:
|
|
74
|
-
table.add_row("Username", "[red]Unavailable[/red]")
|
|
75
|
-
if user.get("sub"):
|
|
76
|
-
table.add_row("User ID", user["sub"])
|
|
77
|
-
table.add_row("Name", user.get("name") or "Unknown")
|
|
78
|
-
if user.get("email"):
|
|
79
|
-
table.add_row("Email", user["email"])
|
|
80
|
-
table.add_row("Remote", info.get("remote_url") or "https://api.dash.ml")
|
|
81
|
-
if info.get("token_expires"):
|
|
82
|
-
table.add_row("Token Expires", info["token_expires"])
|
|
83
|
-
|
|
84
|
-
console.print(
|
|
85
|
-
Panel(
|
|
86
|
-
table,
|
|
87
|
-
title="[bold green]✓ Authenticated[/bold green]",
|
|
88
|
-
border_style="green",
|
|
89
|
-
)
|
|
90
|
-
)
|
|
91
|
-
|
|
92
|
-
return 0
|
ml_dash/snowflake.py
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Snowflake ID generator for ML-Dash.
|
|
3
|
-
|
|
4
|
-
Snowflake IDs are 64-bit unique identifiers with the following structure:
|
|
5
|
-
- 1 bit: unused (always 0)
|
|
6
|
-
- 41 bits: timestamp in milliseconds since custom epoch
|
|
7
|
-
- 10 bits: worker/machine ID (0-1023)
|
|
8
|
-
- 12 bits: sequence number (0-4095)
|
|
9
|
-
|
|
10
|
-
This provides:
|
|
11
|
-
- Unique IDs across distributed systems
|
|
12
|
-
- Time-sortable (newer IDs are larger)
|
|
13
|
-
- ~69 years of IDs from custom epoch
|
|
14
|
-
- Up to 4096 IDs per millisecond per worker
|
|
15
|
-
"""
|
|
16
|
-
|
|
17
|
-
import time
|
|
18
|
-
import threading
|
|
19
|
-
import os
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class SnowflakeIDGenerator:
|
|
23
|
-
"""
|
|
24
|
-
Thread-safe Snowflake ID generator.
|
|
25
|
-
|
|
26
|
-
Based on Twitter's Snowflake algorithm.
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
# Custom epoch: 2024-01-01 00:00:00 UTC (in milliseconds)
|
|
30
|
-
EPOCH = 1704067200000
|
|
31
|
-
|
|
32
|
-
# Bit lengths
|
|
33
|
-
TIMESTAMP_BITS = 41
|
|
34
|
-
WORKER_BITS = 10
|
|
35
|
-
SEQUENCE_BITS = 12
|
|
36
|
-
|
|
37
|
-
# Max values
|
|
38
|
-
MAX_WORKER_ID = (1 << WORKER_BITS) - 1 # 1023
|
|
39
|
-
MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1 # 4095
|
|
40
|
-
|
|
41
|
-
# Bit shifts
|
|
42
|
-
TIMESTAMP_SHIFT = WORKER_BITS + SEQUENCE_BITS # 22
|
|
43
|
-
WORKER_SHIFT = SEQUENCE_BITS # 12
|
|
44
|
-
|
|
45
|
-
def __init__(self, worker_id: int = None):
|
|
46
|
-
"""
|
|
47
|
-
Initialize Snowflake ID generator.
|
|
48
|
-
|
|
49
|
-
Args:
|
|
50
|
-
worker_id: Worker/machine ID (0-1023). If None, derived from process ID.
|
|
51
|
-
"""
|
|
52
|
-
if worker_id is None:
|
|
53
|
-
# Derive from process ID
|
|
54
|
-
worker_id = os.getpid() & self.MAX_WORKER_ID
|
|
55
|
-
|
|
56
|
-
if not 0 <= worker_id <= self.MAX_WORKER_ID:
|
|
57
|
-
raise ValueError(f"worker_id must be between 0 and {self.MAX_WORKER_ID}")
|
|
58
|
-
|
|
59
|
-
self.worker_id = worker_id
|
|
60
|
-
self.sequence = 0
|
|
61
|
-
self.last_timestamp = -1
|
|
62
|
-
self.lock = threading.Lock()
|
|
63
|
-
|
|
64
|
-
def _current_millis(self) -> int:
|
|
65
|
-
"""Get current timestamp in milliseconds since custom epoch."""
|
|
66
|
-
return int(time.time() * 1000) - self.EPOCH
|
|
67
|
-
|
|
68
|
-
def _wait_next_millis(self, last_timestamp: int) -> int:
|
|
69
|
-
"""Wait until next millisecond."""
|
|
70
|
-
timestamp = self._current_millis()
|
|
71
|
-
while timestamp <= last_timestamp:
|
|
72
|
-
timestamp = self._current_millis()
|
|
73
|
-
return timestamp
|
|
74
|
-
|
|
75
|
-
def generate(self) -> int:
|
|
76
|
-
"""
|
|
77
|
-
Generate a new Snowflake ID.
|
|
78
|
-
|
|
79
|
-
Returns:
|
|
80
|
-
A unique 64-bit integer ID
|
|
81
|
-
|
|
82
|
-
Raises:
|
|
83
|
-
RuntimeError: If clock moves backwards
|
|
84
|
-
"""
|
|
85
|
-
with self.lock:
|
|
86
|
-
timestamp = self._current_millis()
|
|
87
|
-
|
|
88
|
-
# Check for clock moving backwards
|
|
89
|
-
if timestamp < self.last_timestamp:
|
|
90
|
-
raise RuntimeError(
|
|
91
|
-
f"Clock moved backwards. Refusing to generate ID. "
|
|
92
|
-
f"Last: {self.last_timestamp}, Current: {timestamp}"
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
if timestamp == self.last_timestamp:
|
|
96
|
-
# Same millisecond - increment sequence
|
|
97
|
-
self.sequence = (self.sequence + 1) & self.MAX_SEQUENCE
|
|
98
|
-
if self.sequence == 0:
|
|
99
|
-
# Sequence overflow - wait for next millisecond
|
|
100
|
-
timestamp = self._wait_next_millis(self.last_timestamp)
|
|
101
|
-
else:
|
|
102
|
-
# New millisecond - reset sequence
|
|
103
|
-
self.sequence = 0
|
|
104
|
-
|
|
105
|
-
self.last_timestamp = timestamp
|
|
106
|
-
|
|
107
|
-
# Construct the ID
|
|
108
|
-
snowflake_id = (
|
|
109
|
-
(timestamp << self.TIMESTAMP_SHIFT) |
|
|
110
|
-
(self.worker_id << self.WORKER_SHIFT) |
|
|
111
|
-
self.sequence
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
return snowflake_id
|
|
115
|
-
|
|
116
|
-
def parse(self, snowflake_id: int) -> dict:
|
|
117
|
-
"""
|
|
118
|
-
Parse a Snowflake ID into its components.
|
|
119
|
-
|
|
120
|
-
Args:
|
|
121
|
-
snowflake_id: The Snowflake ID to parse
|
|
122
|
-
|
|
123
|
-
Returns:
|
|
124
|
-
Dictionary with timestamp, worker_id, and sequence
|
|
125
|
-
"""
|
|
126
|
-
timestamp = (snowflake_id >> self.TIMESTAMP_SHIFT) + self.EPOCH
|
|
127
|
-
worker_id = (snowflake_id >> self.WORKER_SHIFT) & self.MAX_WORKER_ID
|
|
128
|
-
sequence = snowflake_id & self.MAX_SEQUENCE
|
|
129
|
-
|
|
130
|
-
return {
|
|
131
|
-
"timestamp": timestamp,
|
|
132
|
-
"timestamp_ms": timestamp,
|
|
133
|
-
"worker_id": worker_id,
|
|
134
|
-
"sequence": sequence,
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
# Global singleton instance
|
|
139
|
-
_generator = None
|
|
140
|
-
_generator_lock = threading.Lock()
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
def get_generator() -> SnowflakeIDGenerator:
|
|
144
|
-
"""Get or create the global Snowflake ID generator instance."""
|
|
145
|
-
global _generator
|
|
146
|
-
if _generator is None:
|
|
147
|
-
with _generator_lock:
|
|
148
|
-
if _generator is None:
|
|
149
|
-
_generator = SnowflakeIDGenerator()
|
|
150
|
-
return _generator
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
def generate_id() -> int:
|
|
154
|
-
"""
|
|
155
|
-
Generate a new Snowflake ID using the global generator.
|
|
156
|
-
|
|
157
|
-
Returns:
|
|
158
|
-
A unique 64-bit integer ID
|
|
159
|
-
"""
|
|
160
|
-
return get_generator().generate()
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
def parse_id(snowflake_id: int) -> dict:
|
|
164
|
-
"""
|
|
165
|
-
Parse a Snowflake ID into its components.
|
|
166
|
-
|
|
167
|
-
Args:
|
|
168
|
-
snowflake_id: The Snowflake ID to parse
|
|
169
|
-
|
|
170
|
-
Returns:
|
|
171
|
-
Dictionary with timestamp, worker_id, and sequence
|
|
172
|
-
"""
|
|
173
|
-
return get_generator().parse(snowflake_id)
|
ml_dash-0.6.2.dist-info/RECORD
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
ml_dash/__init__.py,sha256=JjKe0VyPxtWJGZU4a7YGxON6jkcrovDowUGKg9mzmNw,1583
|
|
2
|
-
ml_dash/auth/__init__.py,sha256=3lwM-Y8UBHPU1gFW2JNpmXlPVTnkGudWLKNFFKulQfo,1200
|
|
3
|
-
ml_dash/auth/constants.py,sha256=ku4QzQUMNjvyJwjy7AUdywMAZd59jXSxNHZxDiagUWU,280
|
|
4
|
-
ml_dash/auth/device_flow.py,sha256=DQOdPNlZCuU1umZOA_A6WXdRM3zWphnyo9IntToBl_A,7921
|
|
5
|
-
ml_dash/auth/device_secret.py,sha256=qUsz6M9S1GEIukvmz57eJEp57srSx74O4MU9mZEeDlE,1158
|
|
6
|
-
ml_dash/auth/exceptions.py,sha256=IeBwUzoaTyFtPwd4quFOIel49inIzuabe_ChEeEXEWI,725
|
|
7
|
-
ml_dash/auth/token_storage.py,sha256=L18W8J7D1LlCDlY3Q32l0RXeNh0o7YVDQeeGYm64Dgw,8163
|
|
8
|
-
ml_dash/auto_start.py,sha256=62_eZG1qBNAwu6AXduTSo4niCVZ27X52ZK0WEr3yS1o,1812
|
|
9
|
-
ml_dash/cli.py,sha256=BoaBulcqnM88XuV5BQEx_-AQAXJAYSJqpvnHggEII_I,2559
|
|
10
|
-
ml_dash/cli_commands/__init__.py,sha256=bjAmV7MsW-bhtW_4SnLJ0Cfkt9h82vMDC8ebW1Ke8KE,38
|
|
11
|
-
ml_dash/cli_commands/api.py,sha256=tgHB3pvSYv36_RbxsAtiEfjtivnIn7NjdHq0AL2QQGo,4335
|
|
12
|
-
ml_dash/cli_commands/download.py,sha256=ZnRhaDLIM28Dri4-YHLU1fBwC9AAvNoiuut3pkdBhJU,27422
|
|
13
|
-
ml_dash/cli_commands/list.py,sha256=9dK0UbNTvysGM5c8Mkb5XfFNkhMIhtjIP1v9BFo-5ew,15400
|
|
14
|
-
ml_dash/cli_commands/login.py,sha256=zX-urtUrfzg2qOGtKNYQgj6UloN9kzj4zEO6h_xwuNs,6782
|
|
15
|
-
ml_dash/cli_commands/logout.py,sha256=lTUUNyRXqvo61qNkCd4KBrPUujDAHnNqsHkU6bHie0U,1332
|
|
16
|
-
ml_dash/cli_commands/profile.py,sha256=BaSM6BAN3YM4tw95iKV_nypKZxwsB3PoAAejQcYip5E,2351
|
|
17
|
-
ml_dash/cli_commands/upload.py,sha256=Ch1pWC4rU3M9P52Ne_gAlkE7yz4WZKgZlRBG3hpy9_4,44059
|
|
18
|
-
ml_dash/client.py,sha256=TEk-Vt323wBpDPPwX-fFFS7IVF7hS3aBDxn9lewbpls,31455
|
|
19
|
-
ml_dash/config.py,sha256=oz2xvoBh2X_xUXWr92cPD5nFxXMT5LxVNypv5B5O0fA,3116
|
|
20
|
-
ml_dash/experiment.py,sha256=DsEl4q7EksfBApOjd1q4ncX6COSC7Hv2bCeFPbeELC8,39218
|
|
21
|
-
ml_dash/files.py,sha256=bihUHKpdknytLGuGgkcvhh585nziZrvYjiHl6rHnoD0,49227
|
|
22
|
-
ml_dash/log.py,sha256=E-DLg0vejVLLEyShJ_r0LneDMI0XU7XTH5iKWYJe9jI,5298
|
|
23
|
-
ml_dash/metric.py,sha256=ghD1jnuv6dbjV1Jlo7q0mx9UEzpdto2Y1-oDWrSfg04,25809
|
|
24
|
-
ml_dash/params.py,sha256=pPFvknJAJX5uhckzjO1r-HNnKbQFFKDISFmOXNET5eY,9046
|
|
25
|
-
ml_dash/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
-
ml_dash/remote_auto_start.py,sha256=5fvQDHv1CWEKFb6WAa5_uyEInwV_SvotXjOO_6i6ZKE,1687
|
|
27
|
-
ml_dash/run.py,sha256=C0quTLZXKDAlwstzEiJ75CWCX1pwYrmtMZH3z-ia6Pw,6310
|
|
28
|
-
ml_dash/snowflake.py,sha256=14rEpRU5YltsmmmZW0EMUy_hdv5S5ME9gWVtmdmwfiU,4917
|
|
29
|
-
ml_dash/storage.py,sha256=9mG42pvvWkkracbjCr9Xdp890Nm4XSxL7_JeFbBe28g,33020
|
|
30
|
-
ml_dash-0.6.2.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
|
|
31
|
-
ml_dash-0.6.2.dist-info/entry_points.txt,sha256=dYs2EHX1uRNO7AQGNnVaJJpgiy0Z9q7tiy4fHSyaf3Q,46
|
|
32
|
-
ml_dash-0.6.2.dist-info/METADATA,sha256=HvCgXKi2TsP0-K1J5KECG64Nuzs_OnbUCVLBFMT6w2Y,7207
|
|
33
|
-
ml_dash-0.6.2.dist-info/RECORD,,
|
|
File without changes
|