openrunner-sdk 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/PKG-INFO +1 -1
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/huggingface.py +10 -2
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/lightning.py +6 -1
- openrunner_sdk-0.4.2/openrunner/integration/pytorch.py +95 -0
- openrunner_sdk-0.4.2/openrunner/integration/sklearn.py +431 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/run.py +31 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/pyproject.toml +1 -1
- openrunner_sdk-0.4.0/openrunner/integration/pytorch.py +0 -49
- openrunner_sdk-0.4.0/openrunner/integration/sklearn.py +0 -203
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/.gitignore +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/=6.0 +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/=8.1 +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/README.md +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/__init__.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/api_client.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/artifact.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/buffer.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/cache.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/cli.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/config.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/evaluation.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/git_info.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/__init__.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/fastai.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/keras.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/langchain.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/xgboost.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/launch.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/media.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/offline.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/prompt.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/query_api.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/scorers.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/sender.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/settings.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/summary.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/sweep.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/system_metrics.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/trace.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/wandb_compat/__init__.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/wandb_compat/_shim.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/__init__.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/conftest.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_alert.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_aliases.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_api_client.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_artifact.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_buffer.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_cache.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_cli.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_config.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_evaluation.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_finish.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_git_info.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_init.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_fastai.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_huggingface.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_keras.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_langchain.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_lightning.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_pytorch.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_sklearn.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_xgboost.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_launch.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_log.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_media.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_offline.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_offline_sync.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_query_api.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_resume.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_sender.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_summary.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_sweep.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_system_metrics.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_trace.py +0 -0
- {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_wandb_compat.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openrunner-sdk
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
|
|
5
5
|
Project-URL: Homepage, https://github.com/jqueguiner/openrunner
|
|
6
6
|
Project-URL: Repository, https://github.com/jqueguiner/openrunner
|
|
@@ -52,9 +52,17 @@ class OpenRunnerCallback(TrainerCallback):
|
|
|
52
52
|
control: TrainerControl,
|
|
53
53
|
**kwargs,
|
|
54
54
|
) -> None:
|
|
55
|
-
"""Start a new run if one isn't already active.
|
|
55
|
+
"""Start a new run if one isn't already active.
|
|
56
|
+
|
|
57
|
+
If the user has already called ``openrunner.init()``, merges
|
|
58
|
+
training args into the existing config instead of creating
|
|
59
|
+
a new run.
|
|
60
|
+
"""
|
|
56
61
|
if openrunner._active_run is None:
|
|
57
|
-
openrunner.init(
|
|
62
|
+
openrunner.init(config=args.to_dict())
|
|
63
|
+
else:
|
|
64
|
+
# Merge trainer args into existing run config
|
|
65
|
+
openrunner.config.update(args.to_dict())
|
|
58
66
|
|
|
59
67
|
def on_log(
|
|
60
68
|
self,
|
|
@@ -68,7 +68,12 @@ class OpenRunnerLogger(Logger):
|
|
|
68
68
|
|
|
69
69
|
@property
|
|
70
70
|
def experiment(self) -> Any:
|
|
71
|
-
"""Return the underlying Run object (Lightning Logger protocol).
|
|
71
|
+
"""Return the underlying Run object (Lightning Logger protocol).
|
|
72
|
+
|
|
73
|
+
Auto-initialises the run on first access so that
|
|
74
|
+
``logger.experiment.config["key"] = val`` works before training.
|
|
75
|
+
"""
|
|
76
|
+
self._ensure_run()
|
|
72
77
|
return self._run
|
|
73
78
|
|
|
74
79
|
# -- Core methods -------------------------------------------------------
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""PyTorch integration for OpenRunner.
|
|
2
|
+
|
|
3
|
+
Provides a ``log_gradients`` utility that logs gradient norms for all
|
|
4
|
+
model parameters via ``openrunner.log()``.
|
|
5
|
+
|
|
6
|
+
Usage::
|
|
7
|
+
|
|
8
|
+
from openrunner.integration.pytorch import log_gradients
|
|
9
|
+
|
|
10
|
+
for batch in dataloader:
|
|
11
|
+
loss = model(batch)
|
|
12
|
+
loss.backward()
|
|
13
|
+
log_gradients(model, step=step)
|
|
14
|
+
optimizer.step()
|
|
15
|
+
|
|
16
|
+
Requires: ``pip install openrunner[pytorch]``
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
import torch # noqa: F401
|
|
23
|
+
except ImportError:
|
|
24
|
+
raise ImportError(
|
|
25
|
+
"PyTorch integration requires torch. "
|
|
26
|
+
"Install with: pip install openrunner[pytorch]"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
import openrunner
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def watch(
|
|
33
|
+
model: torch.nn.Module,
|
|
34
|
+
log: str = "gradients",
|
|
35
|
+
log_freq: int = 100,
|
|
36
|
+
log_graph: bool = False,
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Automatically log gradients and/or parameters of a model.
|
|
39
|
+
|
|
40
|
+
Registers a backward hook that logs gradient norms every
|
|
41
|
+
``log_freq`` backward passes. Similar to ``wandb.watch()``.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
model: A PyTorch ``nn.Module`` to watch.
|
|
45
|
+
log: What to log -- ``"gradients"``, ``"parameters"``, or ``"all"``.
|
|
46
|
+
log_freq: Log every N backward passes.
|
|
47
|
+
log_graph: Ignored (kept for W&B API compatibility).
|
|
48
|
+
"""
|
|
49
|
+
_counter = {"n": 0}
|
|
50
|
+
log_grads = log in ("gradients", "all")
|
|
51
|
+
log_params = log in ("parameters", "all")
|
|
52
|
+
|
|
53
|
+
def _hook(module: torch.nn.Module, grad_input, grad_output) -> None:
|
|
54
|
+
_counter["n"] += 1
|
|
55
|
+
if _counter["n"] % log_freq != 0:
|
|
56
|
+
return
|
|
57
|
+
step = _counter["n"]
|
|
58
|
+
if log_grads:
|
|
59
|
+
for name, param in module.named_parameters():
|
|
60
|
+
if param.grad is not None:
|
|
61
|
+
openrunner.log(
|
|
62
|
+
{f"gradients/{name}.norm": param.grad.norm().item()},
|
|
63
|
+
step=step,
|
|
64
|
+
commit=False,
|
|
65
|
+
)
|
|
66
|
+
if log_params:
|
|
67
|
+
for name, param in module.named_parameters():
|
|
68
|
+
openrunner.log(
|
|
69
|
+
{f"parameters/{name}.norm": param.norm().item()},
|
|
70
|
+
step=step,
|
|
71
|
+
commit=False,
|
|
72
|
+
)
|
|
73
|
+
openrunner.log({}, step=step, commit=True)
|
|
74
|
+
|
|
75
|
+
model.register_full_backward_hook(_hook)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def log_gradients(model: torch.nn.Module, step: int | None = None) -> None:
|
|
79
|
+
"""Log gradient norms for all model parameters.
|
|
80
|
+
|
|
81
|
+
For each named parameter that has a gradient, logs the L2 norm
|
|
82
|
+
as ``gradients/{name}``. Parameters whose ``.grad`` is ``None``
|
|
83
|
+
(e.g., frozen layers) are silently skipped.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
model: A PyTorch ``nn.Module`` whose ``.backward()`` has been called.
|
|
87
|
+
step: Optional explicit step value passed to ``openrunner.log()``.
|
|
88
|
+
"""
|
|
89
|
+
for name, param in model.named_parameters():
|
|
90
|
+
if param.grad is not None:
|
|
91
|
+
norm = param.grad.norm().item()
|
|
92
|
+
openrunner.log({f"gradients/{name}": norm}, step=step, commit=False)
|
|
93
|
+
|
|
94
|
+
# Commit the step after all gradient norms have been logged.
|
|
95
|
+
openrunner.log({}, step=step, commit=True)
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Scikit-learn integration for OpenRunner.
|
|
2
|
+
|
|
3
|
+
Provides utility functions to log scikit-learn model information,
|
|
4
|
+
classification reports, confusion matrices, and feature importances
|
|
5
|
+
to an active OpenRunner run.
|
|
6
|
+
|
|
7
|
+
Usage::
|
|
8
|
+
|
|
9
|
+
from openrunner.integration.sklearn import (
|
|
10
|
+
log_model,
|
|
11
|
+
log_classification_report,
|
|
12
|
+
log_confusion_matrix,
|
|
13
|
+
log_feature_importance,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
model = RandomForestClassifier().fit(X_train, y_train)
|
|
17
|
+
log_model(model)
|
|
18
|
+
log_classification_report(y_test, y_pred)
|
|
19
|
+
log_confusion_matrix(y_test, y_pred)
|
|
20
|
+
log_feature_importance(model, feature_names=["f1", "f2", "f3"])
|
|
21
|
+
|
|
22
|
+
Requires: ``pip install openrunner[sklearn]``
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from typing import Any, Sequence
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
import sklearn # noqa: F401
|
|
31
|
+
except ImportError:
|
|
32
|
+
raise ImportError(
|
|
33
|
+
"Scikit-learn integration requires scikit-learn. "
|
|
34
|
+
"Install with: pip install openrunner[sklearn]"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
import openrunner
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def log_model(model: Any, name: str | None = None) -> None:
|
|
41
|
+
"""Log model hyperparameters via ``get_params()``.
|
|
42
|
+
|
|
43
|
+
Logs all parameters returned by the model's ``get_params()`` method
|
|
44
|
+
into the run config under ``model/`` prefix.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
model: A fitted scikit-learn estimator.
|
|
48
|
+
name: Optional model name stored in config.
|
|
49
|
+
"""
|
|
50
|
+
if openrunner._active_run is None:
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
params = model.get_params()
|
|
54
|
+
prefixed = {f"model/{k}": v for k, v in params.items() if _is_loggable(v)}
|
|
55
|
+
if name:
|
|
56
|
+
prefixed["model/name"] = name
|
|
57
|
+
prefixed["model/class"] = type(model).__name__
|
|
58
|
+
if prefixed:
|
|
59
|
+
openrunner.config.update(prefixed)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def log_classification_report(
|
|
63
|
+
y_true: Sequence,
|
|
64
|
+
y_pred: Sequence,
|
|
65
|
+
labels: Sequence | None = None,
|
|
66
|
+
target_names: Sequence[str] | None = None,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Log a classification report as a Table.
|
|
69
|
+
|
|
70
|
+
Uses ``sklearn.metrics.classification_report`` to compute precision,
|
|
71
|
+
recall, f1-score and support, then logs the result as an
|
|
72
|
+
``openrunner.Table``.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
y_true: Ground-truth labels.
|
|
76
|
+
y_pred: Predicted labels.
|
|
77
|
+
labels: Optional list of label indices to include.
|
|
78
|
+
target_names: Optional display names for each label.
|
|
79
|
+
"""
|
|
80
|
+
if openrunner._active_run is None:
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
from sklearn.metrics import classification_report as _cr
|
|
84
|
+
|
|
85
|
+
report = _cr(
|
|
86
|
+
y_true,
|
|
87
|
+
y_pred,
|
|
88
|
+
labels=labels,
|
|
89
|
+
target_names=target_names,
|
|
90
|
+
output_dict=True,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
columns = ["class", "precision", "recall", "f1-score", "support"]
|
|
94
|
+
data: list[list[Any]] = []
|
|
95
|
+
for cls_name, metrics in report.items():
|
|
96
|
+
if isinstance(metrics, dict):
|
|
97
|
+
data.append([
|
|
98
|
+
cls_name,
|
|
99
|
+
round(metrics.get("precision", 0), 4),
|
|
100
|
+
round(metrics.get("recall", 0), 4),
|
|
101
|
+
round(metrics.get("f1-score", 0), 4),
|
|
102
|
+
metrics.get("support", 0),
|
|
103
|
+
])
|
|
104
|
+
else:
|
|
105
|
+
# scalar like "accuracy"
|
|
106
|
+
data.append([cls_name, metrics, None, None, None])
|
|
107
|
+
|
|
108
|
+
table = openrunner.Table(columns=columns, data=data)
|
|
109
|
+
openrunner.log({"classification_report": table})
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def log_confusion_matrix(
|
|
113
|
+
y_true: Sequence,
|
|
114
|
+
y_pred: Sequence,
|
|
115
|
+
labels: Sequence | None = None,
|
|
116
|
+
class_names: Sequence[str] | None = None,
|
|
117
|
+
) -> None:
|
|
118
|
+
"""Log a confusion matrix as an Image (matplotlib heatmap).
|
|
119
|
+
|
|
120
|
+
Generates a heatmap using ``matplotlib`` and logs it as an
|
|
121
|
+
``openrunner.Image``.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
y_true: Ground-truth labels.
|
|
125
|
+
y_pred: Predicted labels.
|
|
126
|
+
labels: Optional list of label values for ordering.
|
|
127
|
+
class_names: Optional display names for the axes.
|
|
128
|
+
"""
|
|
129
|
+
if openrunner._active_run is None:
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
from sklearn.metrics import confusion_matrix as _cm
|
|
133
|
+
|
|
134
|
+
try:
|
|
135
|
+
import matplotlib
|
|
136
|
+
matplotlib.use("Agg")
|
|
137
|
+
import matplotlib.pyplot as plt
|
|
138
|
+
except ImportError:
|
|
139
|
+
raise ImportError(
|
|
140
|
+
"log_confusion_matrix requires matplotlib. "
|
|
141
|
+
"Install with: pip install matplotlib"
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
cm = _cm(y_true, y_pred, labels=labels)
|
|
145
|
+
|
|
146
|
+
fig, ax = plt.subplots(figsize=(6, 5))
|
|
147
|
+
im = ax.imshow(cm, interpolation="nearest", cmap="Blues")
|
|
148
|
+
fig.colorbar(im, ax=ax)
|
|
149
|
+
|
|
150
|
+
if class_names is not None:
|
|
151
|
+
tick_marks = list(range(len(class_names)))
|
|
152
|
+
ax.set_xticks(tick_marks)
|
|
153
|
+
ax.set_xticklabels(class_names, rotation=45, ha="right")
|
|
154
|
+
ax.set_yticks(tick_marks)
|
|
155
|
+
ax.set_yticklabels(class_names)
|
|
156
|
+
|
|
157
|
+
ax.set_xlabel("Predicted")
|
|
158
|
+
ax.set_ylabel("True")
|
|
159
|
+
ax.set_title("Confusion Matrix")
|
|
160
|
+
fig.tight_layout()
|
|
161
|
+
|
|
162
|
+
openrunner.log({"confusion_matrix": openrunner.Image(fig)})
|
|
163
|
+
plt.close(fig)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def log_feature_importance(
|
|
167
|
+
model: Any,
|
|
168
|
+
feature_names: Sequence[str] | None = None,
|
|
169
|
+
) -> None:
|
|
170
|
+
"""Log feature importances as a Table.
|
|
171
|
+
|
|
172
|
+
Extracts ``feature_importances_`` from the model and logs them
|
|
173
|
+
as a sorted ``openrunner.Table`` (most important first).
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
model: A fitted scikit-learn estimator with
|
|
177
|
+
``feature_importances_`` attribute.
|
|
178
|
+
feature_names: Optional list of feature names. If not
|
|
179
|
+
provided, uses ``feature_0``, ``feature_1``, etc.
|
|
180
|
+
"""
|
|
181
|
+
if openrunner._active_run is None:
|
|
182
|
+
return
|
|
183
|
+
|
|
184
|
+
if not hasattr(model, "feature_importances_"):
|
|
185
|
+
return
|
|
186
|
+
|
|
187
|
+
importances = model.feature_importances_
|
|
188
|
+
if feature_names is None:
|
|
189
|
+
feature_names = [f"feature_{i}" for i in range(len(importances))]
|
|
190
|
+
|
|
191
|
+
# Sort by importance descending
|
|
192
|
+
pairs = sorted(
|
|
193
|
+
zip(feature_names, importances),
|
|
194
|
+
key=lambda x: x[1],
|
|
195
|
+
reverse=True,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
columns = ["feature", "importance"]
|
|
199
|
+
data = [[name, round(float(imp), 6)] for name, imp in pairs]
|
|
200
|
+
|
|
201
|
+
table = openrunner.Table(columns=columns, data=data)
|
|
202
|
+
openrunner.log({"feature_importance": table})
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def log_learning_curve(
|
|
206
|
+
model: Any,
|
|
207
|
+
X: Any,
|
|
208
|
+
y: Any,
|
|
209
|
+
cv: int = 5,
|
|
210
|
+
scoring: str = "accuracy",
|
|
211
|
+
) -> None:
|
|
212
|
+
"""Log a learning curve as an Image.
|
|
213
|
+
|
|
214
|
+
Uses ``sklearn.model_selection.learning_curve`` and plots train/val
|
|
215
|
+
scores vs training set size.
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
model: A scikit-learn estimator.
|
|
219
|
+
X: Training features.
|
|
220
|
+
y: Training labels.
|
|
221
|
+
cv: Number of cross-validation folds.
|
|
222
|
+
scoring: Scoring metric name.
|
|
223
|
+
"""
|
|
224
|
+
if openrunner._active_run is None:
|
|
225
|
+
return
|
|
226
|
+
|
|
227
|
+
from sklearn.model_selection import learning_curve as _lc
|
|
228
|
+
|
|
229
|
+
try:
|
|
230
|
+
import matplotlib
|
|
231
|
+
matplotlib.use("Agg")
|
|
232
|
+
import matplotlib.pyplot as plt
|
|
233
|
+
except ImportError:
|
|
234
|
+
raise ImportError(
|
|
235
|
+
"log_learning_curve requires matplotlib. "
|
|
236
|
+
"Install with: pip install matplotlib"
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
import numpy as np
|
|
240
|
+
|
|
241
|
+
train_sizes, train_scores, val_scores = _lc(
|
|
242
|
+
model, X, y, cv=cv, scoring=scoring,
|
|
243
|
+
train_sizes=np.linspace(0.1, 1.0, 8),
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
fig, ax = plt.subplots(figsize=(6, 4))
|
|
247
|
+
ax.plot(train_sizes, train_scores.mean(axis=1), "o-", label="Train")
|
|
248
|
+
ax.fill_between(
|
|
249
|
+
train_sizes,
|
|
250
|
+
train_scores.mean(axis=1) - train_scores.std(axis=1),
|
|
251
|
+
train_scores.mean(axis=1) + train_scores.std(axis=1),
|
|
252
|
+
alpha=0.15,
|
|
253
|
+
)
|
|
254
|
+
ax.plot(train_sizes, val_scores.mean(axis=1), "o-", label="Validation")
|
|
255
|
+
ax.fill_between(
|
|
256
|
+
train_sizes,
|
|
257
|
+
val_scores.mean(axis=1) - val_scores.std(axis=1),
|
|
258
|
+
val_scores.mean(axis=1) + val_scores.std(axis=1),
|
|
259
|
+
alpha=0.15,
|
|
260
|
+
)
|
|
261
|
+
ax.set_xlabel("Training Set Size")
|
|
262
|
+
ax.set_ylabel(scoring.capitalize())
|
|
263
|
+
ax.set_title("Learning Curve")
|
|
264
|
+
ax.legend(loc="best")
|
|
265
|
+
fig.tight_layout()
|
|
266
|
+
|
|
267
|
+
openrunner.log({"learning_curve": openrunner.Image(fig)})
|
|
268
|
+
plt.close(fig)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def log_roc(
|
|
272
|
+
y_true: Sequence,
|
|
273
|
+
y_probas: Any,
|
|
274
|
+
labels: Sequence[str] | None = None,
|
|
275
|
+
) -> None:
|
|
276
|
+
"""Log ROC curves as an Image.
|
|
277
|
+
|
|
278
|
+
Computes per-class ROC curves using one-vs-rest strategy and
|
|
279
|
+
logs the plot as an ``openrunner.Image``.
|
|
280
|
+
|
|
281
|
+
Args:
|
|
282
|
+
y_true: Ground-truth labels.
|
|
283
|
+
y_probas: Predicted probabilities (shape: n_samples x n_classes).
|
|
284
|
+
labels: Optional class names for the legend.
|
|
285
|
+
"""
|
|
286
|
+
if openrunner._active_run is None:
|
|
287
|
+
return
|
|
288
|
+
|
|
289
|
+
from sklearn.metrics import roc_curve, auc
|
|
290
|
+
from sklearn.preprocessing import label_binarize
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
import matplotlib
|
|
294
|
+
matplotlib.use("Agg")
|
|
295
|
+
import matplotlib.pyplot as plt
|
|
296
|
+
except ImportError:
|
|
297
|
+
raise ImportError(
|
|
298
|
+
"log_roc requires matplotlib. Install with: pip install matplotlib"
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
import numpy as np
|
|
302
|
+
|
|
303
|
+
classes = np.unique(y_true)
|
|
304
|
+
n_classes = len(classes)
|
|
305
|
+
|
|
306
|
+
if n_classes == 2:
|
|
307
|
+
# Binary classification
|
|
308
|
+
fpr, tpr, _ = roc_curve(y_true, y_probas[:, 1] if y_probas.ndim > 1 else y_probas)
|
|
309
|
+
roc_auc = auc(fpr, tpr)
|
|
310
|
+
fig, ax = plt.subplots(figsize=(6, 5))
|
|
311
|
+
label_name = labels[1] if labels and len(labels) > 1 else "Positive"
|
|
312
|
+
ax.plot(fpr, tpr, label=f"{label_name} (AUC={roc_auc:.3f})")
|
|
313
|
+
else:
|
|
314
|
+
# Multi-class: one-vs-rest
|
|
315
|
+
y_bin = label_binarize(y_true, classes=classes)
|
|
316
|
+
fig, ax = plt.subplots(figsize=(6, 5))
|
|
317
|
+
for i in range(n_classes):
|
|
318
|
+
fpr, tpr, _ = roc_curve(y_bin[:, i], y_probas[:, i])
|
|
319
|
+
roc_auc = auc(fpr, tpr)
|
|
320
|
+
name = labels[i] if labels and i < len(labels) else str(classes[i])
|
|
321
|
+
ax.plot(fpr, tpr, label=f"{name} (AUC={roc_auc:.3f})")
|
|
322
|
+
|
|
323
|
+
ax.plot([0, 1], [0, 1], "k--", alpha=0.3)
|
|
324
|
+
ax.set_xlabel("False Positive Rate")
|
|
325
|
+
ax.set_ylabel("True Positive Rate")
|
|
326
|
+
ax.set_title("ROC Curve")
|
|
327
|
+
ax.legend(loc="lower right", fontsize=9)
|
|
328
|
+
fig.tight_layout()
|
|
329
|
+
|
|
330
|
+
openrunner.log({"roc_curve": openrunner.Image(fig)})
|
|
331
|
+
plt.close(fig)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def log_precision_recall(
|
|
335
|
+
y_true: Sequence,
|
|
336
|
+
y_probas: Any,
|
|
337
|
+
labels: Sequence[str] | None = None,
|
|
338
|
+
) -> None:
|
|
339
|
+
"""Log precision-recall curves as an Image.
|
|
340
|
+
|
|
341
|
+
Args:
|
|
342
|
+
y_true: Ground-truth labels.
|
|
343
|
+
y_probas: Predicted probabilities (shape: n_samples x n_classes).
|
|
344
|
+
labels: Optional class names for the legend.
|
|
345
|
+
"""
|
|
346
|
+
if openrunner._active_run is None:
|
|
347
|
+
return
|
|
348
|
+
|
|
349
|
+
from sklearn.metrics import precision_recall_curve, average_precision_score
|
|
350
|
+
from sklearn.preprocessing import label_binarize
|
|
351
|
+
|
|
352
|
+
try:
|
|
353
|
+
import matplotlib
|
|
354
|
+
matplotlib.use("Agg")
|
|
355
|
+
import matplotlib.pyplot as plt
|
|
356
|
+
except ImportError:
|
|
357
|
+
raise ImportError(
|
|
358
|
+
"log_precision_recall requires matplotlib. "
|
|
359
|
+
"Install with: pip install matplotlib"
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
import numpy as np
|
|
363
|
+
|
|
364
|
+
classes = np.unique(y_true)
|
|
365
|
+
n_classes = len(classes)
|
|
366
|
+
|
|
367
|
+
fig, ax = plt.subplots(figsize=(6, 5))
|
|
368
|
+
|
|
369
|
+
if n_classes == 2:
|
|
370
|
+
probs = y_probas[:, 1] if y_probas.ndim > 1 else y_probas
|
|
371
|
+
precision, recall, _ = precision_recall_curve(y_true, probs)
|
|
372
|
+
ap = average_precision_score(y_true, probs)
|
|
373
|
+
name = labels[1] if labels and len(labels) > 1 else "Positive"
|
|
374
|
+
ax.plot(recall, precision, label=f"{name} (AP={ap:.3f})")
|
|
375
|
+
else:
|
|
376
|
+
y_bin = label_binarize(y_true, classes=classes)
|
|
377
|
+
for i in range(n_classes):
|
|
378
|
+
precision, recall, _ = precision_recall_curve(y_bin[:, i], y_probas[:, i])
|
|
379
|
+
ap = average_precision_score(y_bin[:, i], y_probas[:, i])
|
|
380
|
+
name = labels[i] if labels and i < len(labels) else str(classes[i])
|
|
381
|
+
ax.plot(recall, precision, label=f"{name} (AP={ap:.3f})")
|
|
382
|
+
|
|
383
|
+
ax.set_xlabel("Recall")
|
|
384
|
+
ax.set_ylabel("Precision")
|
|
385
|
+
ax.set_title("Precision-Recall Curve")
|
|
386
|
+
ax.legend(loc="best", fontsize=9)
|
|
387
|
+
fig.tight_layout()
|
|
388
|
+
|
|
389
|
+
openrunner.log({"precision_recall": openrunner.Image(fig)})
|
|
390
|
+
plt.close(fig)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def log_class_proportions(
|
|
394
|
+
y_train: Sequence,
|
|
395
|
+
y_test: Sequence,
|
|
396
|
+
labels: Sequence[str] | None = None,
|
|
397
|
+
) -> None:
|
|
398
|
+
"""Log class distribution as a Table.
|
|
399
|
+
|
|
400
|
+
Args:
|
|
401
|
+
y_train: Training labels.
|
|
402
|
+
y_test: Test labels.
|
|
403
|
+
labels: Optional class names.
|
|
404
|
+
"""
|
|
405
|
+
if openrunner._active_run is None:
|
|
406
|
+
return
|
|
407
|
+
|
|
408
|
+
import numpy as np
|
|
409
|
+
|
|
410
|
+
classes = np.unique(np.concatenate([y_train, y_test]))
|
|
411
|
+
columns = ["class", "train_count", "test_count", "train_%", "test_%"]
|
|
412
|
+
data = []
|
|
413
|
+
for i, cls in enumerate(classes):
|
|
414
|
+
name = labels[i] if labels and i < len(labels) else str(cls)
|
|
415
|
+
train_ct = int(np.sum(np.array(y_train) == cls))
|
|
416
|
+
test_ct = int(np.sum(np.array(y_test) == cls))
|
|
417
|
+
data.append([
|
|
418
|
+
name,
|
|
419
|
+
train_ct,
|
|
420
|
+
test_ct,
|
|
421
|
+
round(train_ct / len(y_train) * 100, 1),
|
|
422
|
+
round(test_ct / len(y_test) * 100, 1),
|
|
423
|
+
])
|
|
424
|
+
|
|
425
|
+
table = openrunner.Table(columns=columns, data=data)
|
|
426
|
+
openrunner.log({"class_proportions": table})
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _is_loggable(value: Any) -> bool:
|
|
430
|
+
"""Check if a value can be logged as a metric (numeric or string)."""
|
|
431
|
+
return isinstance(value, (int, float, str, bool))
|
|
@@ -83,6 +83,29 @@ class _TeeWriter:
|
|
|
83
83
|
return getattr(self._original, name)
|
|
84
84
|
|
|
85
85
|
|
|
86
|
+
class _LogCaptureHandler(logging.Handler):
|
|
87
|
+
"""Captures Python logging output and sends it to OpenRunner console logs.
|
|
88
|
+
|
|
89
|
+
This catches output from frameworks that use `logging` instead of `print()`
|
|
90
|
+
(HuggingFace Transformers, PyTorch Lightning, etc.)
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(self, sender) -> None:
|
|
94
|
+
super().__init__()
|
|
95
|
+
self._sender = sender
|
|
96
|
+
# Don't capture openrunner's own logs to avoid recursion
|
|
97
|
+
self.addFilter(lambda record: not record.name.startswith("openrunner"))
|
|
98
|
+
|
|
99
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
100
|
+
try:
|
|
101
|
+
msg = self.format(record)
|
|
102
|
+
ts = datetime.now(timezone.utc).isoformat()
|
|
103
|
+
stream = "stderr" if record.levelno >= logging.WARNING else "stdout"
|
|
104
|
+
self._sender.add_console_line(ts, stream, msg)
|
|
105
|
+
except Exception:
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
|
|
86
109
|
def _generate_run_id() -> str:
|
|
87
110
|
"""Generate a W&B-style 8-character run ID."""
|
|
88
111
|
return "".join(secrets.choice(_ALPHABET) for _ in range(8))
|
|
@@ -226,6 +249,10 @@ class Run:
|
|
|
226
249
|
sys.stdout = _TeeWriter(sys.stdout, "stdout", self._sender)
|
|
227
250
|
sys.stderr = _TeeWriter(sys.stderr, "stderr", self._sender)
|
|
228
251
|
|
|
252
|
+
# Also capture Python logging output (used by HuggingFace, PyTorch, etc.)
|
|
253
|
+
self._log_handler = _LogCaptureHandler(self._sender)
|
|
254
|
+
logging.root.addHandler(self._log_handler)
|
|
255
|
+
|
|
229
256
|
# -- Properties -----------------------------------------------------------
|
|
230
257
|
|
|
231
258
|
@property
|
|
@@ -668,6 +695,10 @@ class Run:
|
|
|
668
695
|
exit_code: Optional exit code for the run.
|
|
669
696
|
quiet: If True, suppress the finish message.
|
|
670
697
|
"""
|
|
698
|
+
# Remove logging handler
|
|
699
|
+
if hasattr(self, "_log_handler"):
|
|
700
|
+
logging.root.removeHandler(self._log_handler)
|
|
701
|
+
|
|
671
702
|
# Restore original stdout/stderr before stopping sender
|
|
672
703
|
if hasattr(self, "_original_stdout"):
|
|
673
704
|
# Flush any remaining captured output
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
"""PyTorch integration for OpenRunner.
|
|
2
|
-
|
|
3
|
-
Provides a ``log_gradients`` utility that logs gradient norms for all
|
|
4
|
-
model parameters via ``openrunner.log()``.
|
|
5
|
-
|
|
6
|
-
Usage::
|
|
7
|
-
|
|
8
|
-
from openrunner.integration.pytorch import log_gradients
|
|
9
|
-
|
|
10
|
-
for batch in dataloader:
|
|
11
|
-
loss = model(batch)
|
|
12
|
-
loss.backward()
|
|
13
|
-
log_gradients(model, step=step)
|
|
14
|
-
optimizer.step()
|
|
15
|
-
|
|
16
|
-
Requires: ``pip install openrunner[pytorch]``
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
try:
|
|
22
|
-
import torch # noqa: F401
|
|
23
|
-
except ImportError:
|
|
24
|
-
raise ImportError(
|
|
25
|
-
"PyTorch integration requires torch. "
|
|
26
|
-
"Install with: pip install openrunner[pytorch]"
|
|
27
|
-
)
|
|
28
|
-
|
|
29
|
-
import openrunner
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def log_gradients(model: torch.nn.Module, step: int | None = None) -> None:
|
|
33
|
-
"""Log gradient norms for all model parameters.
|
|
34
|
-
|
|
35
|
-
For each named parameter that has a gradient, logs the L2 norm
|
|
36
|
-
as ``gradients/{name}``. Parameters whose ``.grad`` is ``None``
|
|
37
|
-
(e.g., frozen layers) are silently skipped.
|
|
38
|
-
|
|
39
|
-
Args:
|
|
40
|
-
model: A PyTorch ``nn.Module`` whose ``.backward()`` has been called.
|
|
41
|
-
step: Optional explicit step value passed to ``openrunner.log()``.
|
|
42
|
-
"""
|
|
43
|
-
for name, param in model.named_parameters():
|
|
44
|
-
if param.grad is not None:
|
|
45
|
-
norm = param.grad.norm().item()
|
|
46
|
-
openrunner.log({f"gradients/{name}": norm}, step=step, commit=False)
|
|
47
|
-
|
|
48
|
-
# Commit the step after all gradient norms have been logged.
|
|
49
|
-
openrunner.log({}, step=step, commit=True)
|
|
@@ -1,203 +0,0 @@
|
|
|
1
|
-
"""Scikit-learn integration for OpenRunner.
|
|
2
|
-
|
|
3
|
-
Provides utility functions to log scikit-learn model information,
|
|
4
|
-
classification reports, confusion matrices, and feature importances
|
|
5
|
-
to an active OpenRunner run.
|
|
6
|
-
|
|
7
|
-
Usage::
|
|
8
|
-
|
|
9
|
-
from openrunner.integration.sklearn import (
|
|
10
|
-
log_model,
|
|
11
|
-
log_classification_report,
|
|
12
|
-
log_confusion_matrix,
|
|
13
|
-
log_feature_importance,
|
|
14
|
-
)
|
|
15
|
-
|
|
16
|
-
model = RandomForestClassifier().fit(X_train, y_train)
|
|
17
|
-
log_model(model)
|
|
18
|
-
log_classification_report(y_test, y_pred)
|
|
19
|
-
log_confusion_matrix(y_test, y_pred)
|
|
20
|
-
log_feature_importance(model, feature_names=["f1", "f2", "f3"])
|
|
21
|
-
|
|
22
|
-
Requires: ``pip install openrunner[sklearn]``
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
from __future__ import annotations
|
|
26
|
-
|
|
27
|
-
from typing import Any, Sequence
|
|
28
|
-
|
|
29
|
-
try:
|
|
30
|
-
import sklearn # noqa: F401
|
|
31
|
-
except ImportError:
|
|
32
|
-
raise ImportError(
|
|
33
|
-
"Scikit-learn integration requires scikit-learn. "
|
|
34
|
-
"Install with: pip install openrunner[sklearn]"
|
|
35
|
-
)
|
|
36
|
-
|
|
37
|
-
import openrunner
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def log_model(model: Any) -> None:
|
|
41
|
-
"""Log model hyperparameters via ``get_params()``.
|
|
42
|
-
|
|
43
|
-
Logs all parameters returned by the model's ``get_params()`` method
|
|
44
|
-
as a flat dict under ``model/`` prefix.
|
|
45
|
-
|
|
46
|
-
Args:
|
|
47
|
-
model: A fitted scikit-learn estimator.
|
|
48
|
-
"""
|
|
49
|
-
if openrunner._active_run is None:
|
|
50
|
-
return
|
|
51
|
-
|
|
52
|
-
params = model.get_params()
|
|
53
|
-
prefixed = {f"model/{k}": v for k, v in params.items() if _is_loggable(v)}
|
|
54
|
-
if prefixed:
|
|
55
|
-
openrunner.log(prefixed)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def log_classification_report(
|
|
59
|
-
y_true: Sequence,
|
|
60
|
-
y_pred: Sequence,
|
|
61
|
-
labels: Sequence | None = None,
|
|
62
|
-
target_names: Sequence[str] | None = None,
|
|
63
|
-
) -> None:
|
|
64
|
-
"""Log a classification report as a Table.
|
|
65
|
-
|
|
66
|
-
Uses ``sklearn.metrics.classification_report`` to compute precision,
|
|
67
|
-
recall, f1-score and support, then logs the result as an
|
|
68
|
-
``openrunner.Table``.
|
|
69
|
-
|
|
70
|
-
Args:
|
|
71
|
-
y_true: Ground-truth labels.
|
|
72
|
-
y_pred: Predicted labels.
|
|
73
|
-
labels: Optional list of label indices to include.
|
|
74
|
-
target_names: Optional display names for each label.
|
|
75
|
-
"""
|
|
76
|
-
if openrunner._active_run is None:
|
|
77
|
-
return
|
|
78
|
-
|
|
79
|
-
from sklearn.metrics import classification_report as _cr
|
|
80
|
-
|
|
81
|
-
report = _cr(
|
|
82
|
-
y_true,
|
|
83
|
-
y_pred,
|
|
84
|
-
labels=labels,
|
|
85
|
-
target_names=target_names,
|
|
86
|
-
output_dict=True,
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
columns = ["class", "precision", "recall", "f1-score", "support"]
|
|
90
|
-
data: list[list[Any]] = []
|
|
91
|
-
for cls_name, metrics in report.items():
|
|
92
|
-
if isinstance(metrics, dict):
|
|
93
|
-
data.append([
|
|
94
|
-
cls_name,
|
|
95
|
-
round(metrics.get("precision", 0), 4),
|
|
96
|
-
round(metrics.get("recall", 0), 4),
|
|
97
|
-
round(metrics.get("f1-score", 0), 4),
|
|
98
|
-
metrics.get("support", 0),
|
|
99
|
-
])
|
|
100
|
-
else:
|
|
101
|
-
# scalar like "accuracy"
|
|
102
|
-
data.append([cls_name, metrics, None, None, None])
|
|
103
|
-
|
|
104
|
-
table = openrunner.Table(columns=columns, data=data)
|
|
105
|
-
openrunner.log({"classification_report": table})
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
def log_confusion_matrix(
|
|
109
|
-
y_true: Sequence,
|
|
110
|
-
y_pred: Sequence,
|
|
111
|
-
labels: Sequence | None = None,
|
|
112
|
-
class_names: Sequence[str] | None = None,
|
|
113
|
-
) -> None:
|
|
114
|
-
"""Log a confusion matrix as an Image (matplotlib heatmap).
|
|
115
|
-
|
|
116
|
-
Generates a heatmap using ``matplotlib`` and logs it as an
|
|
117
|
-
``openrunner.Image``.
|
|
118
|
-
|
|
119
|
-
Args:
|
|
120
|
-
y_true: Ground-truth labels.
|
|
121
|
-
y_pred: Predicted labels.
|
|
122
|
-
labels: Optional list of label values for ordering.
|
|
123
|
-
class_names: Optional display names for the axes.
|
|
124
|
-
"""
|
|
125
|
-
if openrunner._active_run is None:
|
|
126
|
-
return
|
|
127
|
-
|
|
128
|
-
from sklearn.metrics import confusion_matrix as _cm
|
|
129
|
-
|
|
130
|
-
try:
|
|
131
|
-
import matplotlib
|
|
132
|
-
matplotlib.use("Agg")
|
|
133
|
-
import matplotlib.pyplot as plt
|
|
134
|
-
except ImportError:
|
|
135
|
-
raise ImportError(
|
|
136
|
-
"log_confusion_matrix requires matplotlib. "
|
|
137
|
-
"Install with: pip install matplotlib"
|
|
138
|
-
)
|
|
139
|
-
|
|
140
|
-
cm = _cm(y_true, y_pred, labels=labels)
|
|
141
|
-
|
|
142
|
-
fig, ax = plt.subplots(figsize=(6, 5))
|
|
143
|
-
im = ax.imshow(cm, interpolation="nearest", cmap="Blues")
|
|
144
|
-
fig.colorbar(im, ax=ax)
|
|
145
|
-
|
|
146
|
-
if class_names is not None:
|
|
147
|
-
tick_marks = list(range(len(class_names)))
|
|
148
|
-
ax.set_xticks(tick_marks)
|
|
149
|
-
ax.set_xticklabels(class_names, rotation=45, ha="right")
|
|
150
|
-
ax.set_yticks(tick_marks)
|
|
151
|
-
ax.set_yticklabels(class_names)
|
|
152
|
-
|
|
153
|
-
ax.set_xlabel("Predicted")
|
|
154
|
-
ax.set_ylabel("True")
|
|
155
|
-
ax.set_title("Confusion Matrix")
|
|
156
|
-
fig.tight_layout()
|
|
157
|
-
|
|
158
|
-
openrunner.log({"confusion_matrix": openrunner.Image(fig)})
|
|
159
|
-
plt.close(fig)
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
def log_feature_importance(
|
|
163
|
-
model: Any,
|
|
164
|
-
feature_names: Sequence[str] | None = None,
|
|
165
|
-
) -> None:
|
|
166
|
-
"""Log feature importances as a Table.
|
|
167
|
-
|
|
168
|
-
Extracts ``feature_importances_`` from the model and logs them
|
|
169
|
-
as a sorted ``openrunner.Table`` (most important first).
|
|
170
|
-
|
|
171
|
-
Args:
|
|
172
|
-
model: A fitted scikit-learn estimator with
|
|
173
|
-
``feature_importances_`` attribute.
|
|
174
|
-
feature_names: Optional list of feature names. If not
|
|
175
|
-
provided, uses ``feature_0``, ``feature_1``, etc.
|
|
176
|
-
"""
|
|
177
|
-
if openrunner._active_run is None:
|
|
178
|
-
return
|
|
179
|
-
|
|
180
|
-
if not hasattr(model, "feature_importances_"):
|
|
181
|
-
return
|
|
182
|
-
|
|
183
|
-
importances = model.feature_importances_
|
|
184
|
-
if feature_names is None:
|
|
185
|
-
feature_names = [f"feature_{i}" for i in range(len(importances))]
|
|
186
|
-
|
|
187
|
-
# Sort by importance descending
|
|
188
|
-
pairs = sorted(
|
|
189
|
-
zip(feature_names, importances),
|
|
190
|
-
key=lambda x: x[1],
|
|
191
|
-
reverse=True,
|
|
192
|
-
)
|
|
193
|
-
|
|
194
|
-
columns = ["feature", "importance"]
|
|
195
|
-
data = [[name, round(float(imp), 6)] for name, imp in pairs]
|
|
196
|
-
|
|
197
|
-
table = openrunner.Table(columns=columns, data=data)
|
|
198
|
-
openrunner.log({"feature_importance": table})
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def _is_loggable(value: Any) -> bool:
|
|
202
|
-
"""Check if a value can be logged as a metric (numeric or string)."""
|
|
203
|
-
return isinstance(value, (int, float, str, bool))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|