ml-dash 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ml_dash/__init__.py CHANGED
@@ -1,14 +1,58 @@
1
- """ML-Logger: A minimal, local-first experiment tracking library."""
1
+ """
2
+ ML-Dash Python SDK
2
3
 
3
- from .run import Experiment
4
- from .ml_logger import ML_Logger, LogLevel
5
- from .job_logger import JobLogger
4
+ A simple and flexible SDK for ML experiment metricing and data storage.
6
5
 
7
- __version__ = "0.4.0"
6
+ Usage:
7
+
8
+ # Remote mode (API server)
9
+ from ml_dash import Experiment
10
+
11
+ with Experiment(
12
+ name="my-experiment",
13
+ project="my-project",
14
+ remote="http://localhost:3000",
15
+ api_key="your-jwt-token"
16
+ ) as experiment:
17
+ experiment.log("Training started")
18
+ experiment.metric("loss", {"step": 0, "value": 0.5})
19
+
20
+ # Local mode (filesystem)
21
+ with Experiment(
22
+ name="my-experiment",
23
+ project="my-project",
24
+ local_path=".ml-dash"
25
+ ) as experiment:
26
+ experiment.log("Training started")
27
+
28
+ # Decorator style
29
+ from ml_dash import ml_dash_experiment
30
+
31
+ @ml_dash_experiment(
32
+ name="my-experiment",
33
+ project="my-project",
34
+ remote="http://localhost:3000",
35
+ api_key="your-jwt-token"
36
+ )
37
+ def train_model(experiment):
38
+ experiment.log("Training started")
39
+ """
40
+
41
+ from .experiment import Experiment, ml_dash_experiment, OperationMode
42
+ from .client import RemoteClient
43
+ from .storage import LocalStorage
44
+ from .log import LogLevel, LogBuilder
45
+ from .params import ParametersBuilder
46
+
47
+ __version__ = "0.1.0"
8
48
 
9
49
  __all__ = [
10
50
  "Experiment",
11
- "ML_Logger",
51
+ "ml_dash_experiment",
52
+ "OperationMode",
53
+ "RemoteClient",
54
+ "LocalStorage",
12
55
  "LogLevel",
13
- "JobLogger",
56
+ "LogBuilder",
57
+ "ParametersBuilder",
14
58
  ]
ml_dash/client.py ADDED
@@ -0,0 +1,562 @@
1
+ """
2
+ Remote API client for ML-Dash server.
3
+ """
4
+
5
+ from typing import Optional, Dict, Any, List
6
+ import httpx
7
+
8
+
9
+ class RemoteClient:
10
+ """Client for communicating with ML-Dash server."""
11
+
12
+ def __init__(self, base_url: str, api_key: str):
13
+ """
14
+ Initialize remote client.
15
+
16
+ Args:
17
+ base_url: Base URL of ML-Dash server (e.g., "http://localhost:3000")
18
+ api_key: JWT token for authentication
19
+ """
20
+ self.base_url = base_url.rstrip("/")
21
+ self.api_key = api_key
22
+ self._client = httpx.Client(
23
+ base_url=self.base_url,
24
+ headers={
25
+ "Authorization": f"Bearer {api_key}",
26
+ # Note: Don't set Content-Type here as default
27
+ # It will be set per-request (json or multipart)
28
+ },
29
+ timeout=30.0,
30
+ )
31
+
32
+ def create_or_update_experiment(
33
+ self,
34
+ project: str,
35
+ name: str,
36
+ description: Optional[str] = None,
37
+ tags: Optional[List[str]] = None,
38
+ folder: Optional[str] = None,
39
+ write_protected: bool = False,
40
+ metadata: Optional[Dict[str, Any]] = None,
41
+ ) -> Dict[str, Any]:
42
+ """
43
+ Create or update an experiment.
44
+
45
+ Args:
46
+ project: Project name
47
+ name: Experiment name
48
+ description: Optional description
49
+ tags: Optional list of tags
50
+ folder: Optional folder path
51
+ write_protected: If True, experiment becomes immutable
52
+ metadata: Optional metadata dict
53
+
54
+ Returns:
55
+ Response dict with experiment, project, folder, and namespace data
56
+
57
+ Raises:
58
+ httpx.HTTPStatusError: If request fails
59
+ """
60
+ payload = {
61
+ "name": name,
62
+ }
63
+
64
+ if description is not None:
65
+ payload["description"] = description
66
+ if tags is not None:
67
+ payload["tags"] = tags
68
+ if folder is not None:
69
+ payload["folder"] = folder
70
+ if write_protected:
71
+ payload["writeProtected"] = write_protected
72
+ if metadata is not None:
73
+ payload["metadata"] = metadata
74
+
75
+ response = self._client.post(
76
+ f"/projects/{project}/experiments",
77
+ json=payload,
78
+ )
79
+ response.raise_for_status()
80
+ return response.json()
81
+
82
+ def create_log_entries(
83
+ self,
84
+ experiment_id: str,
85
+ logs: List[Dict[str, Any]]
86
+ ) -> Dict[str, Any]:
87
+ """
88
+ Create log entries in batch.
89
+
90
+ Supports both single log and multiple logs via array.
91
+
92
+ Args:
93
+ experiment_id: Experiment ID (Snowflake ID)
94
+ logs: List of log entries, each with fields:
95
+ - timestamp: ISO 8601 string
96
+ - level: "info"|"warn"|"error"|"debug"|"fatal"
97
+ - message: Log message string
98
+ - metadata: Optional dict
99
+
100
+ Returns:
101
+ Response dict:
102
+ {
103
+ "created": 1,
104
+ "startSequence": 42,
105
+ "endSequence": 42,
106
+ "experimentId": "123456789"
107
+ }
108
+
109
+ Raises:
110
+ httpx.HTTPStatusError: If request fails
111
+ """
112
+ response = self._client.post(
113
+ f"/experiments/{experiment_id}/logs",
114
+ json={"logs": logs}
115
+ )
116
+ response.raise_for_status()
117
+ return response.json()
118
+
119
+ def set_parameters(
120
+ self,
121
+ experiment_id: str,
122
+ data: Dict[str, Any]
123
+ ) -> Dict[str, Any]:
124
+ """
125
+ Set/merge parameters for an experiment.
126
+
127
+ Always merges with existing parameters (upsert behavior).
128
+
129
+ Args:
130
+ experiment_id: Experiment ID (Snowflake ID)
131
+ data: Flattened parameter dict with dot notation
132
+ Example: {"model.lr": 0.001, "model.batch_size": 32}
133
+
134
+ Returns:
135
+ Response dict:
136
+ {
137
+ "id": "snowflake_id",
138
+ "experimentId": "experiment_id",
139
+ "data": {...},
140
+ "version": 2,
141
+ "createdAt": "...",
142
+ "updatedAt": "..."
143
+ }
144
+
145
+ Raises:
146
+ httpx.HTTPStatusError: If request fails
147
+ """
148
+ response = self._client.post(
149
+ f"/experiments/{experiment_id}/parameters",
150
+ json={"data": data}
151
+ )
152
+ response.raise_for_status()
153
+ return response.json()
154
+
155
+ def get_parameters(self, experiment_id: str) -> Dict[str, Any]:
156
+ """
157
+ Get parameters for an experiment.
158
+
159
+ Args:
160
+ experiment_id: Experiment ID (Snowflake ID)
161
+
162
+ Returns:
163
+ Flattened parameter dict with dot notation
164
+ Example: {"model.lr": 0.001, "model.batch_size": 32}
165
+
166
+ Raises:
167
+ httpx.HTTPStatusError: If request fails or parameters don't exist
168
+ """
169
+ response = self._client.get(f"/experiments/{experiment_id}/parameters")
170
+ response.raise_for_status()
171
+ result = response.json()
172
+ return result.get("data", {})
173
+
174
+ def upload_file(
175
+ self,
176
+ experiment_id: str,
177
+ file_path: str,
178
+ prefix: str,
179
+ filename: str,
180
+ description: Optional[str],
181
+ tags: Optional[List[str]],
182
+ metadata: Optional[Dict[str, Any]],
183
+ checksum: str,
184
+ content_type: str,
185
+ size_bytes: int
186
+ ) -> Dict[str, Any]:
187
+ """
188
+ Upload a file to an experiment.
189
+
190
+ Args:
191
+ experiment_id: Experiment ID (Snowflake ID)
192
+ file_path: Local file path
193
+ prefix: Logical path prefix
194
+ filename: Original filename
195
+ description: Optional description
196
+ tags: Optional tags
197
+ metadata: Optional metadata
198
+ checksum: SHA256 checksum
199
+ content_type: MIME type
200
+ size_bytes: File size in bytes
201
+
202
+ Returns:
203
+ File metadata dict
204
+
205
+ Raises:
206
+ httpx.HTTPStatusError: If request fails
207
+ """
208
+ # Prepare multipart form data
209
+ # Read file content first (httpx needs content, not file handle)
210
+ with open(file_path, "rb") as f:
211
+ file_content = f.read()
212
+
213
+ files = {"file": (filename, file_content, content_type)}
214
+ data = {
215
+ "prefix": prefix,
216
+ "checksum": checksum,
217
+ "sizeBytes": str(size_bytes),
218
+ }
219
+ if description:
220
+ data["description"] = description
221
+ if tags:
222
+ data["tags"] = ",".join(tags)
223
+ if metadata:
224
+ import json
225
+ data["metadata"] = json.dumps(metadata)
226
+
227
+ # httpx will automatically set multipart/form-data content-type
228
+ response = self._client.post(
229
+ f"/experiments/{experiment_id}/files",
230
+ files=files,
231
+ data=data
232
+ )
233
+
234
+ response.raise_for_status()
235
+ return response.json()
236
+
237
+ def list_files(
238
+ self,
239
+ experiment_id: str,
240
+ prefix: Optional[str] = None,
241
+ tags: Optional[List[str]] = None
242
+ ) -> List[Dict[str, Any]]:
243
+ """
244
+ List files in an experiment.
245
+
246
+ Args:
247
+ experiment_id: Experiment ID (Snowflake ID)
248
+ prefix: Optional prefix filter
249
+ tags: Optional tags filter
250
+
251
+ Returns:
252
+ List of file metadata dicts
253
+
254
+ Raises:
255
+ httpx.HTTPStatusError: If request fails
256
+ """
257
+ params = {}
258
+ if prefix:
259
+ params["prefix"] = prefix
260
+ if tags:
261
+ params["tags"] = ",".join(tags)
262
+
263
+ response = self._client.get(
264
+ f"/experiments/{experiment_id}/files",
265
+ params=params
266
+ )
267
+ response.raise_for_status()
268
+ result = response.json()
269
+ return result.get("files", [])
270
+
271
+ def get_file(self, experiment_id: str, file_id: str) -> Dict[str, Any]:
272
+ """
273
+ Get file metadata.
274
+
275
+ Args:
276
+ experiment_id: Experiment ID (Snowflake ID)
277
+ file_id: File ID (Snowflake ID)
278
+
279
+ Returns:
280
+ File metadata dict
281
+
282
+ Raises:
283
+ httpx.HTTPStatusError: If request fails
284
+ """
285
+ response = self._client.get(f"/experiments/{experiment_id}/files/{file_id}")
286
+ response.raise_for_status()
287
+ return response.json()
288
+
289
+ def download_file(
290
+ self,
291
+ experiment_id: str,
292
+ file_id: str,
293
+ dest_path: Optional[str] = None
294
+ ) -> str:
295
+ """
296
+ Download a file from a experiment.
297
+
298
+ Args:
299
+ experiment_id: Experiment ID (Snowflake ID)
300
+ file_id: File ID (Snowflake ID)
301
+ dest_path: Optional destination path (defaults to original filename)
302
+
303
+ Returns:
304
+ Path to downloaded file
305
+
306
+ Raises:
307
+ httpx.HTTPStatusError: If request fails
308
+ ValueError: If checksum verification fails
309
+ """
310
+ # Get file metadata first to get filename and checksum
311
+ file_metadata = self.get_file(experiment_id, file_id)
312
+ filename = file_metadata["filename"]
313
+ expected_checksum = file_metadata["checksum"]
314
+
315
+ # Determine destination path
316
+ if dest_path is None:
317
+ dest_path = filename
318
+
319
+ # Download file
320
+ response = self._client.get(
321
+ f"/experiments/{experiment_id}/files/{file_id}/download"
322
+ )
323
+ response.raise_for_status()
324
+
325
+ # Write to file
326
+ with open(dest_path, "wb") as f:
327
+ f.write(response.content)
328
+
329
+ # Verify checksum
330
+ from .files import verify_checksum
331
+ if not verify_checksum(dest_path, expected_checksum):
332
+ # Delete corrupted file
333
+ import os
334
+ os.remove(dest_path)
335
+ raise ValueError(f"Checksum verification failed for file {file_id}")
336
+
337
+ return dest_path
338
+
339
+ def delete_file(self, experiment_id: str, file_id: str) -> Dict[str, Any]:
340
+ """
341
+ Delete a file (soft delete).
342
+
343
+ Args:
344
+ experiment_id: Experiment ID (Snowflake ID)
345
+ file_id: File ID (Snowflake ID)
346
+
347
+ Returns:
348
+ Dict with id and deletedAt
349
+
350
+ Raises:
351
+ httpx.HTTPStatusError: If request fails
352
+ """
353
+ response = self._client.delete(f"/experiments/{experiment_id}/files/{file_id}")
354
+ response.raise_for_status()
355
+ return response.json()
356
+
357
+ def update_file(
358
+ self,
359
+ experiment_id: str,
360
+ file_id: str,
361
+ description: Optional[str] = None,
362
+ tags: Optional[List[str]] = None,
363
+ metadata: Optional[Dict[str, Any]] = None
364
+ ) -> Dict[str, Any]:
365
+ """
366
+ Update file metadata.
367
+
368
+ Args:
369
+ experiment_id: Experiment ID (Snowflake ID)
370
+ file_id: File ID (Snowflake ID)
371
+ description: Optional description
372
+ tags: Optional tags
373
+ metadata: Optional metadata
374
+
375
+ Returns:
376
+ Updated file metadata dict
377
+
378
+ Raises:
379
+ httpx.HTTPStatusError: If request fails
380
+ """
381
+ payload = {}
382
+ if description is not None:
383
+ payload["description"] = description
384
+ if tags is not None:
385
+ payload["tags"] = tags
386
+ if metadata is not None:
387
+ payload["metadata"] = metadata
388
+
389
+ response = self._client.patch(
390
+ f"/experiments/{experiment_id}/files/{file_id}",
391
+ json=payload
392
+ )
393
+ response.raise_for_status()
394
+ return response.json()
395
+
396
+ def append_to_metric(
397
+ self,
398
+ experiment_id: str,
399
+ metric_name: str,
400
+ data: Dict[str, Any],
401
+ description: Optional[str] = None,
402
+ tags: Optional[List[str]] = None,
403
+ metadata: Optional[Dict[str, Any]] = None
404
+ ) -> Dict[str, Any]:
405
+ """
406
+ Append a single data point to a metric.
407
+
408
+ Args:
409
+ experiment_id: Experiment ID (Snowflake ID)
410
+ metric_name: Metric name (unique within experiment)
411
+ data: Data point (flexible schema)
412
+ description: Optional metric description
413
+ tags: Optional tags
414
+ metadata: Optional metadata
415
+
416
+ Returns:
417
+ Dict with metricId, index, bufferedDataPoints, chunkSize
418
+
419
+ Raises:
420
+ httpx.HTTPStatusError: If request fails
421
+ """
422
+ payload = {"data": data}
423
+ if description:
424
+ payload["description"] = description
425
+ if tags:
426
+ payload["tags"] = tags
427
+ if metadata:
428
+ payload["metadata"] = metadata
429
+
430
+ response = self._client.post(
431
+ f"/experiments/{experiment_id}/metrics/{metric_name}/append",
432
+ json=payload
433
+ )
434
+ response.raise_for_status()
435
+ return response.json()
436
+
437
+ def append_batch_to_metric(
438
+ self,
439
+ experiment_id: str,
440
+ metric_name: str,
441
+ data_points: List[Dict[str, Any]],
442
+ description: Optional[str] = None,
443
+ tags: Optional[List[str]] = None,
444
+ metadata: Optional[Dict[str, Any]] = None
445
+ ) -> Dict[str, Any]:
446
+ """
447
+ Append multiple data points to a metric in batch.
448
+
449
+ Args:
450
+ experiment_id: Experiment ID (Snowflake ID)
451
+ metric_name: Metric name (unique within experiment)
452
+ data_points: List of data points
453
+ description: Optional metric description
454
+ tags: Optional tags
455
+ metadata: Optional metadata
456
+
457
+ Returns:
458
+ Dict with metricId, startIndex, endIndex, count, bufferedDataPoints, chunkSize
459
+
460
+ Raises:
461
+ httpx.HTTPStatusError: If request fails
462
+ """
463
+ payload = {"dataPoints": data_points}
464
+ if description:
465
+ payload["description"] = description
466
+ if tags:
467
+ payload["tags"] = tags
468
+ if metadata:
469
+ payload["metadata"] = metadata
470
+
471
+ response = self._client.post(
472
+ f"/experiments/{experiment_id}/metrics/{metric_name}/append-batch",
473
+ json=payload
474
+ )
475
+ response.raise_for_status()
476
+ return response.json()
477
+
478
+ def read_metric_data(
479
+ self,
480
+ experiment_id: str,
481
+ metric_name: str,
482
+ start_index: int = 0,
483
+ limit: int = 1000
484
+ ) -> Dict[str, Any]:
485
+ """
486
+ Read data points from a metric.
487
+
488
+ Args:
489
+ experiment_id: Experiment ID (Snowflake ID)
490
+ metric_name: Metric name
491
+ start_index: Starting index (default 0)
492
+ limit: Max points to read (default 1000, max 10000)
493
+
494
+ Returns:
495
+ Dict with data, startIndex, endIndex, total, hasMore
496
+
497
+ Raises:
498
+ httpx.HTTPStatusError: If request fails
499
+ """
500
+ response = self._client.get(
501
+ f"/experiments/{experiment_id}/metrics/{metric_name}/data",
502
+ params={"startIndex": start_index, "limit": limit}
503
+ )
504
+ response.raise_for_status()
505
+ return response.json()
506
+
507
+ def get_metric_stats(
508
+ self,
509
+ experiment_id: str,
510
+ metric_name: str
511
+ ) -> Dict[str, Any]:
512
+ """
513
+ Get metric statistics and metadata.
514
+
515
+ Args:
516
+ experiment_id: Experiment ID (Snowflake ID)
517
+ metric_name: Metric name
518
+
519
+ Returns:
520
+ Dict with metric stats (totalDataPoints, bufferedDataPoints, etc.)
521
+
522
+ Raises:
523
+ httpx.HTTPStatusError: If request fails
524
+ """
525
+ response = self._client.get(
526
+ f"/experiments/{experiment_id}/metrics/{metric_name}/stats"
527
+ )
528
+ response.raise_for_status()
529
+ return response.json()
530
+
531
+ def list_metrics(
532
+ self,
533
+ experiment_id: str
534
+ ) -> List[Dict[str, Any]]:
535
+ """
536
+ List all metrics in an experiment.
537
+
538
+ Args:
539
+ experiment_id: Experiment ID (Snowflake ID)
540
+
541
+ Returns:
542
+ List of metric summaries
543
+
544
+ Raises:
545
+ httpx.HTTPStatusError: If request fails
546
+ """
547
+ response = self._client.get(f"/experiments/{experiment_id}/metrics")
548
+ response.raise_for_status()
549
+ return response.json()["metrics"]
550
+
551
+ def close(self):
552
+ """Close the HTTP client."""
553
+ self._client.close()
554
+
555
+ def __enter__(self):
556
+ """Context manager entry."""
557
+ return self
558
+
559
+ def __exit__(self, exc_type, exc_val, exc_tb):
560
+ """Context manager exit."""
561
+ self.close()
562
+ return False