maarg 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maarg/__init__.py +36 -0
- maarg/_capture.py +132 -0
- maarg/_convenience.py +104 -0
- maarg/_models.py +35 -0
- maarg/_query.py +101 -0
- maarg/_tracking.py +155 -0
- maarg/storage/__init__.py +2 -0
- maarg/storage/_base.py +52 -0
- maarg/storage/_sqlite.py +160 -0
- maarg-0.2.0.dist-info/METADATA +290 -0
- maarg-0.2.0.dist-info/RECORD +14 -0
- maarg-0.2.0.dist-info/WHEEL +5 -0
- maarg-0.2.0.dist-info/licenses/LICENSE +21 -0
- maarg-0.2.0.dist-info/top_level.txt +1 -0
maarg/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
maarg — zero-instrumentation experiment tracking for Python.
|
|
3
|
+
|
|
4
|
+
from maarg import track
|
|
5
|
+
|
|
6
|
+
@track
|
|
7
|
+
def train_model(learning_rate, epochs):
|
|
8
|
+
return {"accuracy": 0.95}
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
__version__ = version("maarg")
|
|
15
|
+
except PackageNotFoundError:
|
|
16
|
+
# Fallback for uninstalled local development mode
|
|
17
|
+
__version__ = "0.2.0"
|
|
18
|
+
|
|
19
|
+
from maarg._convenience import best_run, filter_runs, get_runs, top_n
|
|
20
|
+
from maarg._models import Run
|
|
21
|
+
from maarg._query import compare
|
|
22
|
+
from maarg._tracking import track
|
|
23
|
+
from maarg.storage._base import StorageBackend as StorageBackend
|
|
24
|
+
from maarg.storage._sqlite import SQLiteStorage as SQLiteStorage
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Run",
|
|
28
|
+
"SQLiteStorage",
|
|
29
|
+
"StorageBackend",
|
|
30
|
+
"best_run",
|
|
31
|
+
"compare",
|
|
32
|
+
"filter_runs",
|
|
33
|
+
"get_runs",
|
|
34
|
+
"top_n",
|
|
35
|
+
"track",
|
|
36
|
+
]
|
maarg/_capture.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Input filtering logic for maarg.
|
|
3
|
+
|
|
4
|
+
Decides which of a function's arguments are simple "settings" worth
|
|
5
|
+
logging automatically, versus large/complex objects (datasets, models,
|
|
6
|
+
file handles) that should be skipped.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
# Defaults — overridable per-call via the @track decorator later.
|
|
16
|
+
DEFAULT_MAX_SCALAR_BYTES = 1000
|
|
17
|
+
DEFAULT_MAX_COLLECTION_LENGTH = 20
|
|
18
|
+
|
|
19
|
+
_SCALAR_TYPES = (int, float, bool, str, type(None))
|
|
20
|
+
_COLLECTION_TYPES = (list, tuple, dict)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def is_loggable(
|
|
24
|
+
value: Any,
|
|
25
|
+
max_scalar_bytes: int = DEFAULT_MAX_SCALAR_BYTES,
|
|
26
|
+
max_collection_length: int = DEFAULT_MAX_COLLECTION_LENGTH,
|
|
27
|
+
) -> bool:
|
|
28
|
+
"""
|
|
29
|
+
Decide whether `value` is simple enough to log automatically.
|
|
30
|
+
|
|
31
|
+
Scalars (int, float, bool, str, None) are loggable if they're under
|
|
32
|
+
`max_scalar_bytes` in size. Collections (list, tuple, dict) are
|
|
33
|
+
loggable if they have at most `max_collection_length` items AND
|
|
34
|
+
every item inside them is itself loggable (checked recursively).
|
|
35
|
+
Anything else (DataFrames, arrays, custom objects, file handles,
|
|
36
|
+
etc.) is never loggable.
|
|
37
|
+
"""
|
|
38
|
+
if isinstance(value, _SCALAR_TYPES):
|
|
39
|
+
return sys.getsizeof(value) <= max_scalar_bytes
|
|
40
|
+
|
|
41
|
+
if isinstance(value, _COLLECTION_TYPES):
|
|
42
|
+
if len(value) > max_collection_length:
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
items = value.values() if isinstance(value, dict) else value
|
|
46
|
+
return all(
|
|
47
|
+
is_loggable(item, max_scalar_bytes, max_collection_length)
|
|
48
|
+
for item in items
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# DataFrames, ndarrays, custom classes, file handles, etc.
|
|
52
|
+
return False
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def filter_inputs(
|
|
56
|
+
args: dict[str, Any],
|
|
57
|
+
max_scalar_bytes: int = DEFAULT_MAX_SCALAR_BYTES,
|
|
58
|
+
max_collection_length: int = DEFAULT_MAX_COLLECTION_LENGTH,
|
|
59
|
+
) -> dict[str, Any]:
|
|
60
|
+
"""
|
|
61
|
+
Given a dict of {param_name: value} (e.g. from a function's bound
|
|
62
|
+
arguments), return only the entries that pass `is_loggable`.
|
|
63
|
+
"""
|
|
64
|
+
return {
|
|
65
|
+
name: value
|
|
66
|
+
for name, value in args.items()
|
|
67
|
+
if is_loggable(value, max_scalar_bytes, max_collection_length)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ── Output splitting ─────────────────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
OTHER_REPR_MAX_LEN = 200
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _is_matplotlib_figure(value: Any) -> bool:
|
|
78
|
+
"""Check for a matplotlib Figure without hard-depending on matplotlib."""
|
|
79
|
+
try:
|
|
80
|
+
import matplotlib.figure
|
|
81
|
+
return isinstance(value, matplotlib.figure.Figure)
|
|
82
|
+
except ImportError:
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _save_figure(fig: Any, name: str, artifacts_dir: str | Path) -> str:
|
|
87
|
+
"""Save a figure to `artifacts_dir/<name>.png` and return the path."""
|
|
88
|
+
artifacts_dir = Path(artifacts_dir)
|
|
89
|
+
artifacts_dir.mkdir(parents=True, exist_ok=True)
|
|
90
|
+
path = artifacts_dir / f"{name}.png"
|
|
91
|
+
fig.savefig(path)
|
|
92
|
+
return str(path)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def split_output(
|
|
96
|
+
output: Any,
|
|
97
|
+
artifacts_dir: str | Path,
|
|
98
|
+
) -> tuple[dict[str, float], list[dict[str, str]], dict[str, Any]]:
|
|
99
|
+
"""
|
|
100
|
+
Classify a function's return value into (metrics, artifacts, other).
|
|
101
|
+
|
|
102
|
+
- A dict return value is walked key by key, classifying each value.
|
|
103
|
+
- A non-dict return value is treated as a single item named "result".
|
|
104
|
+
|
|
105
|
+
Classification rules, in order:
|
|
106
|
+
- bool -> other (never treated as a number)
|
|
107
|
+
- int / float -> metrics
|
|
108
|
+
- a known artifact type (currently: matplotlib Figure) -> saved to
|
|
109
|
+
disk under `artifacts_dir`, recorded in artifacts
|
|
110
|
+
- str / None -> other, stored as-is
|
|
111
|
+
- anything else (unrecognized object) -> other, as a truncated repr
|
|
112
|
+
"""
|
|
113
|
+
metrics: dict[str, float] = {}
|
|
114
|
+
artifacts: list[dict[str, str]] = []
|
|
115
|
+
other: dict[str, Any] = {}
|
|
116
|
+
|
|
117
|
+
items = output.items() if isinstance(output, dict) else [("result", output)]
|
|
118
|
+
|
|
119
|
+
for name, value in items:
|
|
120
|
+
if isinstance(value, bool):
|
|
121
|
+
other[name] = value
|
|
122
|
+
elif isinstance(value, (int, float)):
|
|
123
|
+
metrics[name] = value
|
|
124
|
+
elif _is_matplotlib_figure(value):
|
|
125
|
+
path = _save_figure(value, name, artifacts_dir)
|
|
126
|
+
artifacts.append({"name": name, "path": path, "type": "chart"})
|
|
127
|
+
elif isinstance(value, (str, type(None))):
|
|
128
|
+
other[name] = value
|
|
129
|
+
else:
|
|
130
|
+
other[name] = repr(value)[:OTHER_REPR_MAX_LEN]
|
|
131
|
+
|
|
132
|
+
return metrics, artifacts, other
|
maarg/_convenience.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Convenience layer for maarg's public API.
|
|
3
|
+
|
|
4
|
+
maarg._query's functions are deliberately pure — they operate only on an
|
|
5
|
+
already-fetched list[Run], with no storage dependency, so they stay
|
|
6
|
+
trivially testable with hand-built Run lists. The wrappers here add the
|
|
7
|
+
"just fetch it for me" convenience most people actually want at the top
|
|
8
|
+
level, without touching that pure core: if `runs` isn't given, fetch from
|
|
9
|
+
storage first, then delegate to the pure implementation.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
|
|
16
|
+
from maarg._models import Run
|
|
17
|
+
from maarg._query import best_run as _pure_best_run
|
|
18
|
+
from maarg._query import filter_runs as _pure_filter_runs
|
|
19
|
+
from maarg._query import top_n as _pure_top_n
|
|
20
|
+
from maarg._tracking import _get_default_storage
|
|
21
|
+
from maarg.storage._base import StorageBackend
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_runs(
|
|
25
|
+
function: str | None = None,
|
|
26
|
+
experiment: str | None = None,
|
|
27
|
+
*,
|
|
28
|
+
storage: StorageBackend | None = None,
|
|
29
|
+
) -> list[Run]:
|
|
30
|
+
"""
|
|
31
|
+
Fetch runs from storage, optionally filtered by function and/or experiment.
|
|
32
|
+
|
|
33
|
+
Defaults to the same storage backend @track uses when none is given.
|
|
34
|
+
"""
|
|
35
|
+
backend = storage if storage is not None else _get_default_storage()
|
|
36
|
+
|
|
37
|
+
if function is not None:
|
|
38
|
+
runs = backend.list_by_function(function)
|
|
39
|
+
if experiment is not None:
|
|
40
|
+
runs = [r for r in runs if r.experiment == experiment]
|
|
41
|
+
return runs
|
|
42
|
+
|
|
43
|
+
if experiment is not None:
|
|
44
|
+
return backend.list_by_experiment(experiment)
|
|
45
|
+
|
|
46
|
+
return backend.list_all()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def top_n(
|
|
50
|
+
runs: Sequence[Run] | None = None,
|
|
51
|
+
*,
|
|
52
|
+
metric: str,
|
|
53
|
+
n: int = 5,
|
|
54
|
+
higher_is_better: bool = True,
|
|
55
|
+
only_successful: bool = True,
|
|
56
|
+
function: str | None = None,
|
|
57
|
+
experiment: str | None = None,
|
|
58
|
+
storage: StorageBackend | None = None,
|
|
59
|
+
) -> list[Run]:
|
|
60
|
+
"""Like the pure top_n, but fetches from storage when `runs` isn't given."""
|
|
61
|
+
if runs is None:
|
|
62
|
+
runs = get_runs(function=function, experiment=experiment, storage=storage)
|
|
63
|
+
return _pure_top_n(
|
|
64
|
+
runs, metric, n=n,
|
|
65
|
+
higher_is_better=higher_is_better,
|
|
66
|
+
only_successful=only_successful,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def best_run(
|
|
71
|
+
runs: Sequence[Run] | None = None,
|
|
72
|
+
*,
|
|
73
|
+
metric: str,
|
|
74
|
+
higher_is_better: bool = True,
|
|
75
|
+
only_successful: bool = True,
|
|
76
|
+
function: str | None = None,
|
|
77
|
+
experiment: str | None = None,
|
|
78
|
+
storage: StorageBackend | None = None,
|
|
79
|
+
) -> Run | None:
|
|
80
|
+
"""Like the pure best_run, but fetches from storage when `runs` isn't given."""
|
|
81
|
+
if runs is None:
|
|
82
|
+
runs = get_runs(function=function, experiment=experiment, storage=storage)
|
|
83
|
+
return _pure_best_run(
|
|
84
|
+
runs, metric,
|
|
85
|
+
higher_is_better=higher_is_better,
|
|
86
|
+
only_successful=only_successful,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def filter_runs(
|
|
91
|
+
runs: Sequence[Run] | None = None,
|
|
92
|
+
*,
|
|
93
|
+
only_successful: bool = True,
|
|
94
|
+
experiment: str | None = None,
|
|
95
|
+
function: str | None = None,
|
|
96
|
+
storage: StorageBackend | None = None,
|
|
97
|
+
**inputs,
|
|
98
|
+
) -> list[Run]:
|
|
99
|
+
"""Like the pure filter_runs, but fetches from storage when `runs` isn't given."""
|
|
100
|
+
if runs is None:
|
|
101
|
+
runs = get_runs(function=function, experiment=experiment, storage=storage)
|
|
102
|
+
return _pure_filter_runs(
|
|
103
|
+
runs, only_successful=only_successful, experiment=experiment, **inputs
|
|
104
|
+
)
|
maarg/_models.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core data model for maarg.
|
|
3
|
+
|
|
4
|
+
A Run is an immutable record of one execution of a tracked function:
|
|
5
|
+
what it was called with, what it produced, and when.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import uuid
|
|
11
|
+
from dataclasses import asdict, dataclass, field
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class Run:
|
|
18
|
+
"""An immutable record of a single tracked function execution."""
|
|
19
|
+
|
|
20
|
+
function: str
|
|
21
|
+
experiment: str
|
|
22
|
+
inputs: dict[str, Any]
|
|
23
|
+
metrics: dict[str, float] = field(default_factory=dict)
|
|
24
|
+
artifacts: list[dict[str, str]] = field(default_factory=list)
|
|
25
|
+
other: dict[str, Any] = field(default_factory=dict)
|
|
26
|
+
duration_sec: float = 0.0
|
|
27
|
+
|
|
28
|
+
run_id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
|
29
|
+
timestamp: str = field(
|
|
30
|
+
default_factory=lambda: datetime.now(timezone.utc).isoformat()
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def to_dict(self) -> dict[str, Any]:
|
|
34
|
+
"""Serialize this run to a plain dict (e.g. for JSON/SQLite storage)."""
|
|
35
|
+
return asdict(self)
|
maarg/_query.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Query and comparison utilities for maarg.
|
|
3
|
+
|
|
4
|
+
These are pure functions operating on already-fetched lists of Run
|
|
5
|
+
objects — deliberately decoupled from StorageBackend, so this layer
|
|
6
|
+
stays trivially testable (hand-built Run lists, no database involved)
|
|
7
|
+
and independent of which backend produced the data.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
|
|
14
|
+
from maarg._models import Run
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _is_failed(run: Run) -> bool:
|
|
18
|
+
"""A run is failed only if explicitly marked so; absence means success."""
|
|
19
|
+
return run.other.get("status") == "failed"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def filter_runs(
|
|
23
|
+
runs: Sequence[Run],
|
|
24
|
+
*,
|
|
25
|
+
only_successful: bool = True,
|
|
26
|
+
experiment: str | None = None,
|
|
27
|
+
**inputs,
|
|
28
|
+
) -> list[Run]:
|
|
29
|
+
"""
|
|
30
|
+
Filter runs by success status, experiment, and/or exact input values.
|
|
31
|
+
|
|
32
|
+
`only_successful=True` (the default) excludes runs marked failed.
|
|
33
|
+
Any extra keyword arguments are treated as exact-match filters against
|
|
34
|
+
a run's `inputs` dict, e.g. filter_runs(runs, learning_rate=0.01).
|
|
35
|
+
"""
|
|
36
|
+
result = []
|
|
37
|
+
for r in runs:
|
|
38
|
+
if only_successful and _is_failed(r):
|
|
39
|
+
continue
|
|
40
|
+
if experiment is not None and r.experiment != experiment:
|
|
41
|
+
continue
|
|
42
|
+
if any(r.inputs.get(k) != v for k, v in inputs.items()):
|
|
43
|
+
continue
|
|
44
|
+
result.append(r)
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def top_n(
|
|
49
|
+
runs: Sequence[Run],
|
|
50
|
+
metric: str,
|
|
51
|
+
n: int = 5,
|
|
52
|
+
*,
|
|
53
|
+
higher_is_better: bool = True,
|
|
54
|
+
only_successful: bool = True,
|
|
55
|
+
) -> list[Run]:
|
|
56
|
+
"""
|
|
57
|
+
Rank the top N runs by a metric. Runs missing that metric (or where
|
|
58
|
+
it isn't numeric) are silently excluded, not treated as an error.
|
|
59
|
+
"""
|
|
60
|
+
candidates = filter_runs(runs, only_successful=only_successful)
|
|
61
|
+
valid = [
|
|
62
|
+
r for r in candidates
|
|
63
|
+
if isinstance(r.metrics.get(metric), (int, float))
|
|
64
|
+
]
|
|
65
|
+
return sorted(
|
|
66
|
+
valid,
|
|
67
|
+
key=lambda r: r.metrics[metric],
|
|
68
|
+
reverse=higher_is_better,
|
|
69
|
+
)[:n]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def best_run(
|
|
73
|
+
runs: Sequence[Run],
|
|
74
|
+
metric: str,
|
|
75
|
+
*,
|
|
76
|
+
higher_is_better: bool = True,
|
|
77
|
+
only_successful: bool = True,
|
|
78
|
+
) -> Run | None:
|
|
79
|
+
"""Return the single best run for a metric, or None if none qualify."""
|
|
80
|
+
results = top_n(
|
|
81
|
+
runs, metric, n=1,
|
|
82
|
+
higher_is_better=higher_is_better,
|
|
83
|
+
only_successful=only_successful,
|
|
84
|
+
)
|
|
85
|
+
return results[0] if results else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def compare(runs: Sequence[Run]) -> dict:
|
|
89
|
+
"""Build a side-by-side view of inputs, metrics, and status across runs."""
|
|
90
|
+
if not runs:
|
|
91
|
+
return {"run_ids": [], "inputs": {}, "metrics": {}, "status": []}
|
|
92
|
+
|
|
93
|
+
all_input_keys = {k for r in runs for k in r.inputs}
|
|
94
|
+
all_metric_keys = {k for r in runs for k in r.metrics}
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
"run_ids": [r.run_id for r in runs],
|
|
98
|
+
"inputs": {k: [r.inputs.get(k) for r in runs] for k in sorted(all_input_keys)},
|
|
99
|
+
"metrics": {k: [r.metrics.get(k) for r in runs] for k in sorted(all_metric_keys)},
|
|
100
|
+
"status": ["failed" if _is_failed(r) else "success" for r in runs],
|
|
101
|
+
}
|
maarg/_tracking.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Decorator module for maarg.
|
|
3
|
+
|
|
4
|
+
Provides the @track decorator to automatically inspect function parameters,
|
|
5
|
+
filter inputs, time execution, split outputs into metrics and artifacts,
|
|
6
|
+
and persist the resulting Run to a storage backend.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import functools
|
|
12
|
+
import inspect
|
|
13
|
+
import time
|
|
14
|
+
import uuid
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Callable, TypeVar, overload
|
|
17
|
+
|
|
18
|
+
from maarg._capture import (
|
|
19
|
+
DEFAULT_MAX_COLLECTION_LENGTH,
|
|
20
|
+
DEFAULT_MAX_SCALAR_BYTES,
|
|
21
|
+
filter_inputs,
|
|
22
|
+
split_output,
|
|
23
|
+
)
|
|
24
|
+
from maarg._models import Run
|
|
25
|
+
from maarg.storage._base import StorageBackend
|
|
26
|
+
from maarg.storage._sqlite import SQLiteStorage
|
|
27
|
+
|
|
28
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
29
|
+
|
|
30
|
+
_DEFAULT_STORAGE: StorageBackend | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _get_default_storage() -> StorageBackend:
|
|
34
|
+
"""Lazy singleton initializer for the default SQLite storage backend."""
|
|
35
|
+
global _DEFAULT_STORAGE
|
|
36
|
+
if _DEFAULT_STORAGE is None:
|
|
37
|
+
_DEFAULT_STORAGE = SQLiteStorage()
|
|
38
|
+
return _DEFAULT_STORAGE
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@overload
|
|
42
|
+
def track(_func: F) -> F: ...
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@overload
|
|
46
|
+
def track(
|
|
47
|
+
_func: None = None,
|
|
48
|
+
*,
|
|
49
|
+
experiment: str | None = None,
|
|
50
|
+
storage: StorageBackend | None = None,
|
|
51
|
+
artifacts_dir: str | Path = Path(".maarg") / "artifacts",
|
|
52
|
+
max_scalar_bytes: int = DEFAULT_MAX_SCALAR_BYTES,
|
|
53
|
+
max_collection_length: int = DEFAULT_MAX_COLLECTION_LENGTH,
|
|
54
|
+
) -> Callable[[F], F]: ...
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def track(
|
|
58
|
+
_func: F | None = None,
|
|
59
|
+
*,
|
|
60
|
+
experiment: str | None = None,
|
|
61
|
+
storage: StorageBackend | None = None,
|
|
62
|
+
artifacts_dir: str | Path = Path(".maarg") / "artifacts",
|
|
63
|
+
max_scalar_bytes: int = DEFAULT_MAX_SCALAR_BYTES,
|
|
64
|
+
max_collection_length: int = DEFAULT_MAX_COLLECTION_LENGTH,
|
|
65
|
+
) -> Any:
|
|
66
|
+
"""
|
|
67
|
+
Decorator to track function execution, parameters, metrics, and artifacts.
|
|
68
|
+
|
|
69
|
+
Usage:
|
|
70
|
+
@track
|
|
71
|
+
def train(lr=0.01, epochs=10):
|
|
72
|
+
return {"accuracy": 0.95}
|
|
73
|
+
|
|
74
|
+
@track(experiment="resnet-50", max_scalar_bytes=2000)
|
|
75
|
+
def evaluate(model, dataset):
|
|
76
|
+
return {"loss": 0.12}
|
|
77
|
+
|
|
78
|
+
`experiment` defaults to the wrapped function's own name if not given.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
def decorator(func: F) -> F:
|
|
82
|
+
# Resolved once, at decoration time — the function's name and
|
|
83
|
+
# signature never change between calls, so no need to redo this
|
|
84
|
+
# work on every invocation.
|
|
85
|
+
resolved_experiment = experiment if experiment is not None else func.__name__
|
|
86
|
+
sig = inspect.signature(func)
|
|
87
|
+
|
|
88
|
+
@functools.wraps(func)
|
|
89
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
90
|
+
# Storage IS resolved at call time (not here), so runtime
|
|
91
|
+
# changes to the default backend take effect on already-
|
|
92
|
+
# decorated functions, and per-test overrides stay isolated.
|
|
93
|
+
backend = storage if storage is not None else _get_default_storage()
|
|
94
|
+
|
|
95
|
+
# 1. Generate run_id early so artifact subdirectories are predictable
|
|
96
|
+
run_id = str(uuid.uuid4())
|
|
97
|
+
run_artifacts_dir = Path(artifacts_dir) / run_id
|
|
98
|
+
|
|
99
|
+
# 2. Bind parameter values including defaults
|
|
100
|
+
bound = sig.bind(*args, **kwargs)
|
|
101
|
+
bound.apply_defaults()
|
|
102
|
+
|
|
103
|
+
# 3. Filter bound inputs
|
|
104
|
+
filtered_inputs = filter_inputs(
|
|
105
|
+
bound.arguments,
|
|
106
|
+
max_scalar_bytes=max_scalar_bytes,
|
|
107
|
+
max_collection_length=max_collection_length,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# 4. Execute wrapped function and handle exceptions
|
|
111
|
+
start_time = time.perf_counter()
|
|
112
|
+
try:
|
|
113
|
+
result = func(*args, **kwargs)
|
|
114
|
+
except Exception as exc:
|
|
115
|
+
duration_sec = time.perf_counter() - start_time
|
|
116
|
+
failed_run = Run(
|
|
117
|
+
run_id=run_id,
|
|
118
|
+
function=func.__name__,
|
|
119
|
+
experiment=resolved_experiment,
|
|
120
|
+
inputs=filtered_inputs,
|
|
121
|
+
duration_sec=duration_sec,
|
|
122
|
+
other={
|
|
123
|
+
"status": "failed",
|
|
124
|
+
"error_type": type(exc).__name__,
|
|
125
|
+
"error_message": str(exc),
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
backend.save(failed_run)
|
|
129
|
+
raise
|
|
130
|
+
|
|
131
|
+
duration_sec = time.perf_counter() - start_time
|
|
132
|
+
|
|
133
|
+
# 5. Split output into metrics, artifacts, and other metadata
|
|
134
|
+
metrics, artifacts, other = split_output(result, run_artifacts_dir)
|
|
135
|
+
|
|
136
|
+
# 6. Save successful Run
|
|
137
|
+
successful_run = Run(
|
|
138
|
+
run_id=run_id,
|
|
139
|
+
function=func.__name__,
|
|
140
|
+
experiment=resolved_experiment,
|
|
141
|
+
inputs=filtered_inputs,
|
|
142
|
+
metrics=metrics,
|
|
143
|
+
artifacts=artifacts,
|
|
144
|
+
other=other,
|
|
145
|
+
duration_sec=duration_sec,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
backend.save(successful_run)
|
|
149
|
+
return result
|
|
150
|
+
|
|
151
|
+
return wrapper # type: ignore[return-value]
|
|
152
|
+
|
|
153
|
+
if _func is None:
|
|
154
|
+
return decorator
|
|
155
|
+
return decorator(_func)
|
maarg/storage/_base.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Abstract storage interface for maarg.
|
|
3
|
+
|
|
4
|
+
Any storage backend (SQLite, JSON files, a future Postgres backend, etc.)
|
|
5
|
+
must implement this interface. Code elsewhere in maarg (the decorator,
|
|
6
|
+
the query layer, the CLI) depends only on this contract — never on any
|
|
7
|
+
specific backend's internals.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from abc import ABC, abstractmethod
|
|
13
|
+
|
|
14
|
+
from maarg._models import Run
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class StorageBackend(ABC):
|
|
18
|
+
"""Contract that every maarg storage backend must implement."""
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def save(self, run: Run) -> None:
|
|
22
|
+
"""Persist a completed Run."""
|
|
23
|
+
raise NotImplementedError
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def get_by_id(self, run_id: str) -> Run | None:
|
|
27
|
+
"""Retrieve a single Run by its ID, or None if it doesn't exist."""
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def list_by_function(self, function: str) -> list[Run]:
|
|
32
|
+
"""
|
|
33
|
+
Return all Runs recorded for a given function name,
|
|
34
|
+
newest-first by timestamp.
|
|
35
|
+
"""
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
|
|
38
|
+
@abstractmethod
|
|
39
|
+
def list_by_experiment(self, experiment: str) -> list[Run]:
|
|
40
|
+
"""
|
|
41
|
+
Return all Runs recorded under a given experiment label,
|
|
42
|
+
newest-first by timestamp.
|
|
43
|
+
"""
|
|
44
|
+
raise NotImplementedError
|
|
45
|
+
|
|
46
|
+
@abstractmethod
|
|
47
|
+
def list_all(self) -> list[Run]:
|
|
48
|
+
"""
|
|
49
|
+
Return every Run this backend has stored,
|
|
50
|
+
newest-first by timestamp.
|
|
51
|
+
"""
|
|
52
|
+
raise NotImplementedError
|
maarg/storage/_sqlite.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""
|
|
2
|
+
SQLite implementation of the maarg storage backend.
|
|
3
|
+
|
|
4
|
+
Stores each Run as one row in a `runs` table. Complex fields (inputs,
|
|
5
|
+
metrics, artifacts, other) are stored as JSON-serialized text, since
|
|
6
|
+
SQLite has no native nested-object column type.
|
|
7
|
+
|
|
8
|
+
Connections are opened fresh for each method call rather than held
|
|
9
|
+
open for the lifetime of this object — simplest correct behavior for
|
|
10
|
+
v1, since we're not yet designing for concurrent/multi-process access.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import sqlite3
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from maarg._models import Run
|
|
20
|
+
from maarg.storage._base import StorageBackend
|
|
21
|
+
|
|
22
|
+
DEFAULT_DB_PATH = Path(".maarg") / "runs.db"
|
|
23
|
+
|
|
24
|
+
# Explicit column order, used consistently for CREATE TABLE, INSERT, and
|
|
25
|
+
# SELECT — avoids relying on "SELECT *" matching table order by accident.
|
|
26
|
+
_COLUMNS = (
|
|
27
|
+
"run_id", "function", "experiment", "timestamp", "duration_sec",
|
|
28
|
+
"inputs", "metrics", "artifacts", "other",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
_CREATE_TABLE_SQL = """
|
|
32
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
33
|
+
run_id TEXT PRIMARY KEY,
|
|
34
|
+
function TEXT NOT NULL,
|
|
35
|
+
experiment TEXT NOT NULL,
|
|
36
|
+
timestamp TEXT NOT NULL,
|
|
37
|
+
duration_sec REAL NOT NULL,
|
|
38
|
+
inputs TEXT NOT NULL,
|
|
39
|
+
metrics TEXT NOT NULL,
|
|
40
|
+
artifacts TEXT NOT NULL,
|
|
41
|
+
other TEXT NOT NULL
|
|
42
|
+
)
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
_SELECT_COLUMNS_SQL = ", ".join(_COLUMNS)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class SQLiteStorage(StorageBackend):
|
|
49
|
+
"""SQLite-backed implementation of StorageBackend."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, db_path: str | Path = DEFAULT_DB_PATH):
|
|
52
|
+
self.db_path = Path(db_path)
|
|
53
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
54
|
+
self._init_schema()
|
|
55
|
+
|
|
56
|
+
def _connect(self) -> sqlite3.Connection:
|
|
57
|
+
return sqlite3.connect(self.db_path)
|
|
58
|
+
|
|
59
|
+
def _init_schema(self) -> None:
|
|
60
|
+
conn = self._connect()
|
|
61
|
+
try:
|
|
62
|
+
conn.execute(_CREATE_TABLE_SQL)
|
|
63
|
+
conn.commit()
|
|
64
|
+
finally:
|
|
65
|
+
conn.close()
|
|
66
|
+
|
|
67
|
+
# ── Serialization helpers ────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
@staticmethod
|
|
70
|
+
def _run_to_row(run: Run) -> tuple:
|
|
71
|
+
return (
|
|
72
|
+
run.run_id,
|
|
73
|
+
run.function,
|
|
74
|
+
run.experiment,
|
|
75
|
+
run.timestamp,
|
|
76
|
+
run.duration_sec,
|
|
77
|
+
json.dumps(run.inputs),
|
|
78
|
+
json.dumps(run.metrics),
|
|
79
|
+
json.dumps(run.artifacts),
|
|
80
|
+
json.dumps(run.other),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
@staticmethod
|
|
84
|
+
def _row_to_run(row: tuple) -> Run:
|
|
85
|
+
(run_id, function, experiment, timestamp, duration_sec,
|
|
86
|
+
inputs_json, metrics_json, artifacts_json, other_json) = row
|
|
87
|
+
return Run(
|
|
88
|
+
run_id=run_id,
|
|
89
|
+
function=function,
|
|
90
|
+
experiment=experiment,
|
|
91
|
+
timestamp=timestamp,
|
|
92
|
+
duration_sec=duration_sec,
|
|
93
|
+
inputs=json.loads(inputs_json),
|
|
94
|
+
metrics=json.loads(metrics_json),
|
|
95
|
+
artifacts=json.loads(artifacts_json),
|
|
96
|
+
other=json.loads(other_json),
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# ── StorageBackend implementation ────────────────────────────────
|
|
100
|
+
|
|
101
|
+
def save(self, run: Run) -> None:
|
|
102
|
+
conn = self._connect()
|
|
103
|
+
try:
|
|
104
|
+
placeholders = ", ".join("?" * len(_COLUMNS))
|
|
105
|
+
conn.execute(
|
|
106
|
+
f"INSERT INTO runs ({_SELECT_COLUMNS_SQL}) VALUES ({placeholders})",
|
|
107
|
+
self._run_to_row(run),
|
|
108
|
+
)
|
|
109
|
+
conn.commit()
|
|
110
|
+
finally:
|
|
111
|
+
conn.close()
|
|
112
|
+
|
|
113
|
+
def get_by_id(self, run_id: str) -> Run | None:
|
|
114
|
+
conn = self._connect()
|
|
115
|
+
try:
|
|
116
|
+
cursor = conn.execute(
|
|
117
|
+
f"SELECT {_SELECT_COLUMNS_SQL} FROM runs WHERE run_id = ?",
|
|
118
|
+
(run_id,),
|
|
119
|
+
)
|
|
120
|
+
row = cursor.fetchone()
|
|
121
|
+
finally:
|
|
122
|
+
conn.close()
|
|
123
|
+
return self._row_to_run(row) if row is not None else None
|
|
124
|
+
|
|
125
|
+
def list_by_function(self, function: str) -> list[Run]:
|
|
126
|
+
conn = self._connect()
|
|
127
|
+
try:
|
|
128
|
+
cursor = conn.execute(
|
|
129
|
+
f"SELECT {_SELECT_COLUMNS_SQL} FROM runs "
|
|
130
|
+
f"WHERE function = ? ORDER BY timestamp DESC",
|
|
131
|
+
(function,),
|
|
132
|
+
)
|
|
133
|
+
rows = cursor.fetchall()
|
|
134
|
+
finally:
|
|
135
|
+
conn.close()
|
|
136
|
+
return [self._row_to_run(row) for row in rows]
|
|
137
|
+
|
|
138
|
+
def list_by_experiment(self, experiment: str) -> list[Run]:
|
|
139
|
+
conn = self._connect()
|
|
140
|
+
try:
|
|
141
|
+
cursor = conn.execute(
|
|
142
|
+
f"SELECT {_SELECT_COLUMNS_SQL} FROM runs "
|
|
143
|
+
f"WHERE experiment = ? ORDER BY timestamp DESC",
|
|
144
|
+
(experiment,),
|
|
145
|
+
)
|
|
146
|
+
rows = cursor.fetchall()
|
|
147
|
+
finally:
|
|
148
|
+
conn.close()
|
|
149
|
+
return [self._row_to_run(row) for row in rows]
|
|
150
|
+
|
|
151
|
+
def list_all(self) -> list[Run]:
|
|
152
|
+
conn = self._connect()
|
|
153
|
+
try:
|
|
154
|
+
cursor = conn.execute(
|
|
155
|
+
f"SELECT {_SELECT_COLUMNS_SQL} FROM runs ORDER BY timestamp DESC"
|
|
156
|
+
)
|
|
157
|
+
rows = cursor.fetchall()
|
|
158
|
+
finally:
|
|
159
|
+
conn.close()
|
|
160
|
+
return [self._row_to_run(row) for row in rows]
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: maarg
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Zero-instrumentation experiment tracking for Python — a decorator that auto-captures inputs and outputs, no logging calls required.
|
|
5
|
+
Author: Moazzam Matin
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Moazzam-Matin/maarg
|
|
8
|
+
Project-URL: Repository, https://github.com/Moazzam-Matin/maarg
|
|
9
|
+
Project-URL: Issues, https://github.com/Moazzam-Matin/maarg/issues
|
|
10
|
+
Keywords: experiment-tracking,mlops,machine-learning,decorator,introspection
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff; extra == "dev"
|
|
30
|
+
Requires-Dist: build; extra == "dev"
|
|
31
|
+
Requires-Dist: twine; extra == "dev"
|
|
32
|
+
Provides-Extra: plotting
|
|
33
|
+
Requires-Dist: matplotlib; extra == "plotting"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
[](https://github.com/Moazzam-Matin/maarg/actions)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
[](https://www.python.org/)
|
|
39
|
+
[](https://test.pypi.org/project/maarg/)
|
|
40
|
+
|
|
41
|
+
<p align="center">
|
|
42
|
+
<picture>
|
|
43
|
+
<source media="(prefers-color-scheme: dark)" srcset="docs/assets/maarg_logo_dark.svg">
|
|
44
|
+
<img alt="maarg - Zero-Boilerplate Experiment Tracking"
|
|
45
|
+
src="docs/assets/maarg_logo.svg "width="320">
|
|
46
|
+
</picture>
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
<h3 align="center">Experiment tracking with zero logging code.</h3>
|
|
50
|
+
|
|
51
|
+
<p align="center">
|
|
52
|
+
Put <code>@track</code> on a function. Every execution—arguments,
|
|
53
|
+
returns, metrics, execution timing, and failures—is automatically saved
|
|
54
|
+
to local storage for instant querying.
|
|
55
|
+
</p>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## How It Works
|
|
60
|
+
|
|
61
|
+
`maarg` sits transparently at function boundaries. It reads signature parameter defaults and runtime return payloads without requiring explicit parameter or metric logging statements inside your function logic.
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
```text
|
|
65
|
+
┌────────────────────────┐
|
|
66
|
+
│ @track decorated fn │ ──► (Intercepts arguments & execution context)
|
|
67
|
+
└───────────┬────────────┘
|
|
68
|
+
│
|
|
69
|
+
▼
|
|
70
|
+
┌────────────────────────┐
|
|
71
|
+
│ Function Execution │ ──► (Captures return dict / scalars / figures)
|
|
72
|
+
└───────────┬────────────┘
|
|
73
|
+
│
|
|
74
|
+
▼
|
|
75
|
+
┌────────────────────────┐
|
|
76
|
+
│ SQLite Persistence │ ──► Saves to .maarg/runs.db (or custom backend)
|
|
77
|
+
└───────────┬────────────┘
|
|
78
|
+
│
|
|
79
|
+
▼
|
|
80
|
+
┌────────────────────────┐
|
|
81
|
+
│ Query & Analysis API │ ──► maarg.get_runs() ──► top_n() / filter_runs()
|
|
82
|
+
└────────────────────────┘
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## Quickstart
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from maarg import track, get_runs, top_n
|
|
91
|
+
|
|
92
|
+
@track(experiment="learning-rate-sweep")
|
|
93
|
+
def fit(learning_rate, epochs=100):
|
|
94
|
+
w = 0.0
|
|
95
|
+
for _ in range(epochs):
|
|
96
|
+
grad = sum(2 * (w * x - 3 * x) * x for x in range(1, 6)) / 5
|
|
97
|
+
w -= learning_rate * grad
|
|
98
|
+
return {"error": abs(w - 3)}
|
|
99
|
+
|
|
100
|
+
# Run experiments across hyperparameters
|
|
101
|
+
for lr in (0.001, 0.003, 0.01):
|
|
102
|
+
fit(learning_rate=lr)
|
|
103
|
+
|
|
104
|
+
# Query top 3 runs directly from default storage
|
|
105
|
+
for run in top_n(get_runs(), "error", n=3, higher_is_better=False):
|
|
106
|
+
print(f"lr={run.inputs['learning_rate']:<6} epochs={run.inputs['epochs']} error={run.metrics['error']:.2e}")
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
```text
|
|
110
|
+
lr=0.01 epochs=100 error=4.86e-11
|
|
111
|
+
lr=0.003 epochs=100 error=3.25e-03
|
|
112
|
+
lr=0.001 epochs=100 error=3.24e-01
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## Project & Storage Structure
|
|
118
|
+
|
|
119
|
+
`maarg` enforces a clean public package API while automatically managing runtime tracking databases and artifact outputs.
|
|
120
|
+
|
|
121
|
+
### Repository Layout
|
|
122
|
+
```text
|
|
123
|
+
maarg/
|
|
124
|
+
├── .github/
|
|
125
|
+
│ └── workflows/
|
|
126
|
+
│ └── ci.yaml
|
|
127
|
+
├── docs/
|
|
128
|
+
├── src/
|
|
129
|
+
│ └── maarg/
|
|
130
|
+
│ ├── storage/ # Storage backends package
|
|
131
|
+
│ │ ├── __init__.py
|
|
132
|
+
│ │ ├── _base.py # StorageBackend base interface
|
|
133
|
+
│ │ └── _sqlite.py # SQLiteStorage implementation
|
|
134
|
+
│ ├── __init__.py # Public API exports (track, get_runs, top_n, etc.)
|
|
135
|
+
│ ├── _capture.py # Value parsing & scalar payload truncation
|
|
136
|
+
│ ├── _convenience.py # get_runs() wrapper & top-level defaults
|
|
137
|
+
│ ├── _models.py # Core Run and Storage schema dataclasses
|
|
138
|
+
│ ├── _query.py # Pure analytical query engine (top_n, filter_runs)
|
|
139
|
+
│ └── _tracking.py # @track decorator implementation
|
|
140
|
+
├── tests/ # Full test suite matching internal modules
|
|
141
|
+
│ ├── test_capture.py
|
|
142
|
+
│ ├── test_convenience.py
|
|
143
|
+
│ ├── test_models.py
|
|
144
|
+
│ ├── test_query.py
|
|
145
|
+
│ ├── test_storage.py
|
|
146
|
+
│ └── test_tracking.py
|
|
147
|
+
├── LICENSE
|
|
148
|
+
├── PLANNING.md
|
|
149
|
+
├── pyproject.toml
|
|
150
|
+
└── README.md
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### Runtime Storage Directory (`.maarg/`)
|
|
154
|
+
When you execute tracked functions, `maarg` initializes a local directory relative to your working workspace:
|
|
155
|
+
|
|
156
|
+
```text
|
|
157
|
+
your_project/
|
|
158
|
+
├── .maarg/
|
|
159
|
+
│ ├── runs.db # Local SQLite database containing experiment runs
|
|
160
|
+
│ └── artifacts/ # Generated PNG plots & exported binary files
|
|
161
|
+
│ └── <run_id>/
|
|
162
|
+
│ └── figure_1.png
|
|
163
|
+
├── train.py
|
|
164
|
+
└── notebook.ipynb
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## Installation
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
pip install maarg
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Supports Python 3.9+ with zero required external server dependencies. To automatically capture Matplotlib plots into `.maarg/artifacts/`, install with plotting support:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
pip install "maarg[plotting]"
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
|
|
183
|
+
## Querying Runs
|
|
184
|
+
|
|
185
|
+
Query functions operate as pure functions on collections of `Run` objects. You can fetch runs effortlessly using the top-level `get_runs()` helper or pass custom storage backends explicitly.
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
from maarg import get_runs, filter_runs, top_n, best_run, compare
|
|
189
|
+
|
|
190
|
+
# Fetch runs from default local storage (.maarg/runs.db)
|
|
191
|
+
runs = get_runs()
|
|
192
|
+
|
|
193
|
+
# Optionally scope by experiment
|
|
194
|
+
exp_runs = get_runs(experiment="learning-rate-sweep")
|
|
195
|
+
|
|
196
|
+
# Identify top performers
|
|
197
|
+
best = best_run(runs, "error", higher_is_better=False)
|
|
198
|
+
top_3 = top_n(runs, "error", n=3, higher_is_better=False)
|
|
199
|
+
|
|
200
|
+
# Filter by input configuration
|
|
201
|
+
specific = filter_runs(runs, learning_rate=0.01)
|
|
202
|
+
|
|
203
|
+
# Tabulate run comparisons
|
|
204
|
+
comparison = compare(top_3)
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
By default, failed runs are filtered out of ranking queries (`only_successful=True`). Pass `only_successful=False` to include failed executions.
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## What Gets Recorded
|
|
212
|
+
|
|
213
|
+
| Field | Description |
|
|
214
|
+
| --- | --- |
|
|
215
|
+
| `run_id` | Unique UUID generated automatically per call |
|
|
216
|
+
| `timestamp` | ISO 8601 UTC timestamp of call execution |
|
|
217
|
+
| `function` | Name of the decorated function |
|
|
218
|
+
| `experiment` | Experiment grouping label (defaults to function name) |
|
|
219
|
+
| `inputs` | Captured function call parameters (including defaults) |
|
|
220
|
+
| `metrics` | Numeric dictionary outputs returned by the function |
|
|
221
|
+
| `artifacts` | Saved files/figures stored as `{name, path, type}` |
|
|
222
|
+
| `other` | Unclassified outputs, strings, booleans, or truncated `repr()` representations |
|
|
223
|
+
| `duration_sec` | Execution duration in seconds |
|
|
224
|
+
|
|
225
|
+
### Value Safety & Limits
|
|
226
|
+
- **Inputs:** Simple scalar values (numbers, strings, booleans) and collections with ≤ 20 elements or ≤ 1000 bytes are recorded. Large arrays, dataframes, or complex objects are automatically skipped to avoid database bloat.
|
|
227
|
+
- **Outputs:** Dictionary return values with numeric scalars become `metrics`. Returned Matplotlib figures are serialized to PNG artifacts inside `.maarg/artifacts/<run_id>/`.
|
|
228
|
+
- **Failures:** Exceptions are caught, recorded with `other["status"] = "failed"` along with the exception class and traceback message, and then re-raised unchanged.
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
|
|
232
|
+
## Configuration
|
|
233
|
+
|
|
234
|
+
Pass optional controls directly to the `@track` decorator:
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
from maarg import track
|
|
238
|
+
from maarg.storage import SQLiteStorage
|
|
239
|
+
|
|
240
|
+
@track(
|
|
241
|
+
experiment="hyperparameter-sweep",
|
|
242
|
+
storage=SQLiteStorage("results/custom_experiment.db"),
|
|
243
|
+
artifacts_dir="results/artifacts"
|
|
244
|
+
)
|
|
245
|
+
def train(lr, batch_size):
|
|
246
|
+
...
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
| Option | Default | Description |
|
|
250
|
+
| --- | --- | --- |
|
|
251
|
+
| `experiment` | Function name | Label for grouping related runs |
|
|
252
|
+
| `storage` | SQLite at `.maarg/runs.db` | Target storage engine instance |
|
|
253
|
+
| `artifacts_dir` | `.maarg/artifacts` | Directory path for stored plots/files |
|
|
254
|
+
| `max_scalar_bytes` | `1000` | Max byte size allowed for individual scalar inputs |
|
|
255
|
+
| `max_collection_length` | `20` | Max items allowed in recorded input lists/dicts |
|
|
256
|
+
|
|
257
|
+
---
|
|
258
|
+
|
|
259
|
+
## Custom Storage Backends
|
|
260
|
+
|
|
261
|
+
You can define custom storage targets by subclassing `StorageBackend` and implementing `save`, `get_by_id`, `list_by_function`, `list_by_experiment`, and `list_all`:
|
|
262
|
+
|
|
263
|
+
```python
|
|
264
|
+
from maarg.storage import StorageBackend
|
|
265
|
+
|
|
266
|
+
class CustomStorage(StorageBackend):
|
|
267
|
+
# Implement persistence methods
|
|
268
|
+
...
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## Roadmap
|
|
274
|
+
|
|
275
|
+
- Command-line interface (CLI) for browsing and inspecting runs directly in the terminal
|
|
276
|
+
- Event hooks triggered on run completion (e.g., Slack or webhook notifications)
|
|
277
|
+
- Web dashboard extension package
|
|
278
|
+
- Additional remote storage backends
|
|
279
|
+
|
|
280
|
+
---
|
|
281
|
+
|
|
282
|
+
## About the Name
|
|
283
|
+
|
|
284
|
+
*maarg* (मार्ग) is Hindi for "path" or "route".
|
|
285
|
+
|
|
286
|
+
---
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
maarg/__init__.py,sha256=71hYo6y0Pj-SjDzd2oo1jzOoLI7rmW24L8Ysue_dUYQ,895
|
|
2
|
+
maarg/_capture.py,sha256=p7IdBfMnv7sHbvHXmML0-OFtIc9vSvniRBrWKDsV5nU,4572
|
|
3
|
+
maarg/_convenience.py,sha256=myUvthKTz3ik_DOs9RM8DmmkuOVrCx5QTYQWTPMMnAg,3403
|
|
4
|
+
maarg/_models.py,sha256=6Q6SUUiagwGPlBBUTB0AHFtGc3H35W8w-KGl_qy152Y,1061
|
|
5
|
+
maarg/_query.py,sha256=hOUo3OQ2xmjxzSirYJEmXoxXFQAbOsnQgoYnj5lcRII,3128
|
|
6
|
+
maarg/_tracking.py,sha256=uVhO2xVbvA0qw8kHmmxM8wmxX0Pg-0-ixd96Z7T-qDc,5176
|
|
7
|
+
maarg/storage/__init__.py,sha256=XsJr7ThbheYKnYea5BHs7YnwUl-SmnncVNQztjANgaw,130
|
|
8
|
+
maarg/storage/_base.py,sha256=MsYIcnK7WHWyO1ld0THiphFCoD_RS_7K_7cMLVQIxAM,1539
|
|
9
|
+
maarg/storage/_sqlite.py,sha256=wdNuAOvULRa9bJAdvGZO_inxaC2he1VJkxrxlpybq5Y,5281
|
|
10
|
+
maarg-0.2.0.dist-info/licenses/LICENSE,sha256=QwTgP1Y04UJcLderk7sRfryP097IGELxbCb_28oy8hg,1091
|
|
11
|
+
maarg-0.2.0.dist-info/METADATA,sha256=Td7VEgVlggV4tVCTSOGS3hCa0L8U2mcTaBKj1zLPU_k,10804
|
|
12
|
+
maarg-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
13
|
+
maarg-0.2.0.dist-info/top_level.txt,sha256=885AA4XNSkyPoGc9r9yjpL-BZvzsBztY06NHH2EMLM4,6
|
|
14
|
+
maarg-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Moazzam Matin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
maarg
|