openrunner-sdk 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/PKG-INFO +1 -1
  2. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/huggingface.py +10 -2
  3. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/lightning.py +6 -1
  4. openrunner_sdk-0.4.2/openrunner/integration/pytorch.py +95 -0
  5. openrunner_sdk-0.4.2/openrunner/integration/sklearn.py +431 -0
  6. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/run.py +31 -0
  7. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/pyproject.toml +1 -1
  8. openrunner_sdk-0.4.0/openrunner/integration/pytorch.py +0 -49
  9. openrunner_sdk-0.4.0/openrunner/integration/sklearn.py +0 -203
  10. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/.gitignore +0 -0
  11. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/=6.0 +0 -0
  12. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/=8.1 +0 -0
  13. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/README.md +0 -0
  14. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/__init__.py +0 -0
  15. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/api_client.py +0 -0
  16. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/artifact.py +0 -0
  17. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/buffer.py +0 -0
  18. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/cache.py +0 -0
  19. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/cli.py +0 -0
  20. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/config.py +0 -0
  21. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/evaluation.py +0 -0
  22. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/git_info.py +0 -0
  23. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/__init__.py +0 -0
  24. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/fastai.py +0 -0
  25. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/keras.py +0 -0
  26. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/langchain.py +0 -0
  27. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/integration/xgboost.py +0 -0
  28. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/launch.py +0 -0
  29. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/media.py +0 -0
  30. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/offline.py +0 -0
  31. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/prompt.py +0 -0
  32. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/query_api.py +0 -0
  33. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/scorers.py +0 -0
  34. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/sender.py +0 -0
  35. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/settings.py +0 -0
  36. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/summary.py +0 -0
  37. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/sweep.py +0 -0
  38. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/system_metrics.py +0 -0
  39. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/trace.py +0 -0
  40. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/wandb_compat/__init__.py +0 -0
  41. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/openrunner/wandb_compat/_shim.py +0 -0
  42. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/__init__.py +0 -0
  43. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/conftest.py +0 -0
  44. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_alert.py +0 -0
  45. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_aliases.py +0 -0
  46. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_api_client.py +0 -0
  47. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_artifact.py +0 -0
  48. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_buffer.py +0 -0
  49. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_cache.py +0 -0
  50. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_cli.py +0 -0
  51. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_config.py +0 -0
  52. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_evaluation.py +0 -0
  53. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_finish.py +0 -0
  54. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_git_info.py +0 -0
  55. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_init.py +0 -0
  56. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_fastai.py +0 -0
  57. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_huggingface.py +0 -0
  58. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_keras.py +0 -0
  59. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_langchain.py +0 -0
  60. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_lightning.py +0 -0
  61. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_pytorch.py +0 -0
  62. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_sklearn.py +0 -0
  63. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_integration_xgboost.py +0 -0
  64. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_launch.py +0 -0
  65. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_log.py +0 -0
  66. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_media.py +0 -0
  67. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_offline.py +0 -0
  68. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_offline_sync.py +0 -0
  69. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_query_api.py +0 -0
  70. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_resume.py +0 -0
  71. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_sender.py +0 -0
  72. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_summary.py +0 -0
  73. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_sweep.py +0 -0
  74. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_system_metrics.py +0 -0
  75. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_trace.py +0 -0
  76. {openrunner_sdk-0.4.0 → openrunner_sdk-0.4.2}/tests/test_wandb_compat.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: openrunner-sdk
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
5
5
  Project-URL: Homepage, https://github.com/jqueguiner/openrunner
6
6
  Project-URL: Repository, https://github.com/jqueguiner/openrunner
@@ -52,9 +52,17 @@ class OpenRunnerCallback(TrainerCallback):
52
52
  control: TrainerControl,
53
53
  **kwargs,
54
54
  ) -> None:
55
- """Start a new run if one isn't already active."""
55
+ """Start a new run if one isn't already active.
56
+
57
+ If the user has already called ``openrunner.init()``, merges
58
+ training args into the existing config instead of creating
59
+ a new run.
60
+ """
56
61
  if openrunner._active_run is None:
57
- openrunner.init(project=args.output_dir, config=args.to_dict())
62
+ openrunner.init(config=args.to_dict())
63
+ else:
64
+ # Merge trainer args into existing run config
65
+ openrunner.config.update(args.to_dict())
58
66
 
59
67
  def on_log(
60
68
  self,
@@ -68,7 +68,12 @@ class OpenRunnerLogger(Logger):
68
68
 
69
69
  @property
70
70
  def experiment(self) -> Any:
71
- """Return the underlying Run object (Lightning Logger protocol)."""
71
+ """Return the underlying Run object (Lightning Logger protocol).
72
+
73
+ Auto-initialises the run on first access so that
74
+ ``logger.experiment.config["key"] = val`` works before training.
75
+ """
76
+ self._ensure_run()
72
77
  return self._run
73
78
 
74
79
  # -- Core methods -------------------------------------------------------
@@ -0,0 +1,95 @@
1
+ """PyTorch integration for OpenRunner.
2
+
3
+ Provides a ``log_gradients`` utility that logs gradient norms for all
4
+ model parameters via ``openrunner.log()``.
5
+
6
+ Usage::
7
+
8
+ from openrunner.integration.pytorch import log_gradients
9
+
10
+ for batch in dataloader:
11
+ loss = model(batch)
12
+ loss.backward()
13
+ log_gradients(model, step=step)
14
+ optimizer.step()
15
+
16
+ Requires: ``pip install openrunner[pytorch]``
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ try:
22
+ import torch # noqa: F401
23
+ except ImportError:
24
+ raise ImportError(
25
+ "PyTorch integration requires torch. "
26
+ "Install with: pip install openrunner[pytorch]"
27
+ )
28
+
29
+ import openrunner
30
+
31
+
32
+ def watch(
33
+ model: torch.nn.Module,
34
+ log: str = "gradients",
35
+ log_freq: int = 100,
36
+ log_graph: bool = False,
37
+ ) -> None:
38
+ """Automatically log gradients and/or parameters of a model.
39
+
40
+ Registers a backward hook that logs gradient norms every
41
+ ``log_freq`` backward passes. Similar to ``wandb.watch()``.
42
+
43
+ Args:
44
+ model: A PyTorch ``nn.Module`` to watch.
45
+ log: What to log -- ``"gradients"``, ``"parameters"``, or ``"all"``.
46
+ log_freq: Log every N backward passes.
47
+ log_graph: Ignored (kept for W&B API compatibility).
48
+ """
49
+ _counter = {"n": 0}
50
+ log_grads = log in ("gradients", "all")
51
+ log_params = log in ("parameters", "all")
52
+
53
+ def _hook(module: torch.nn.Module, grad_input, grad_output) -> None:
54
+ _counter["n"] += 1
55
+ if _counter["n"] % log_freq != 0:
56
+ return
57
+ step = _counter["n"]
58
+ if log_grads:
59
+ for name, param in module.named_parameters():
60
+ if param.grad is not None:
61
+ openrunner.log(
62
+ {f"gradients/{name}.norm": param.grad.norm().item()},
63
+ step=step,
64
+ commit=False,
65
+ )
66
+ if log_params:
67
+ for name, param in module.named_parameters():
68
+ openrunner.log(
69
+ {f"parameters/{name}.norm": param.norm().item()},
70
+ step=step,
71
+ commit=False,
72
+ )
73
+ openrunner.log({}, step=step, commit=True)
74
+
75
+ model.register_full_backward_hook(_hook)
76
+
77
+
78
+ def log_gradients(model: torch.nn.Module, step: int | None = None) -> None:
79
+ """Log gradient norms for all model parameters.
80
+
81
+ For each named parameter that has a gradient, logs the L2 norm
82
+ as ``gradients/{name}``. Parameters whose ``.grad`` is ``None``
83
+ (e.g., frozen layers) are silently skipped.
84
+
85
+ Args:
86
+ model: A PyTorch ``nn.Module`` whose ``.backward()`` has been called.
87
+ step: Optional explicit step value passed to ``openrunner.log()``.
88
+ """
89
+ for name, param in model.named_parameters():
90
+ if param.grad is not None:
91
+ norm = param.grad.norm().item()
92
+ openrunner.log({f"gradients/{name}": norm}, step=step, commit=False)
93
+
94
+ # Commit the step after all gradient norms have been logged.
95
+ openrunner.log({}, step=step, commit=True)
@@ -0,0 +1,431 @@
1
+ """Scikit-learn integration for OpenRunner.
2
+
3
+ Provides utility functions to log scikit-learn model information,
4
+ classification reports, confusion matrices, and feature importances
5
+ to an active OpenRunner run.
6
+
7
+ Usage::
8
+
9
+ from openrunner.integration.sklearn import (
10
+ log_model,
11
+ log_classification_report,
12
+ log_confusion_matrix,
13
+ log_feature_importance,
14
+ )
15
+
16
+ model = RandomForestClassifier().fit(X_train, y_train)
17
+ log_model(model)
18
+ log_classification_report(y_test, y_pred)
19
+ log_confusion_matrix(y_test, y_pred)
20
+ log_feature_importance(model, feature_names=["f1", "f2", "f3"])
21
+
22
+ Requires: ``pip install openrunner[sklearn]``
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from typing import Any, Sequence
28
+
29
+ try:
30
+ import sklearn # noqa: F401
31
+ except ImportError:
32
+ raise ImportError(
33
+ "Scikit-learn integration requires scikit-learn. "
34
+ "Install with: pip install openrunner[sklearn]"
35
+ )
36
+
37
+ import openrunner
38
+
39
+
40
+ def log_model(model: Any, name: str | None = None) -> None:
41
+ """Log model hyperparameters via ``get_params()``.
42
+
43
+ Logs all parameters returned by the model's ``get_params()`` method
44
+ into the run config under ``model/`` prefix.
45
+
46
+ Args:
47
+ model: A fitted scikit-learn estimator.
48
+ name: Optional model name stored in config.
49
+ """
50
+ if openrunner._active_run is None:
51
+ return
52
+
53
+ params = model.get_params()
54
+ prefixed = {f"model/{k}": v for k, v in params.items() if _is_loggable(v)}
55
+ if name:
56
+ prefixed["model/name"] = name
57
+ prefixed["model/class"] = type(model).__name__
58
+ if prefixed:
59
+ openrunner.config.update(prefixed)
60
+
61
+
62
+ def log_classification_report(
63
+ y_true: Sequence,
64
+ y_pred: Sequence,
65
+ labels: Sequence | None = None,
66
+ target_names: Sequence[str] | None = None,
67
+ ) -> None:
68
+ """Log a classification report as a Table.
69
+
70
+ Uses ``sklearn.metrics.classification_report`` to compute precision,
71
+ recall, f1-score and support, then logs the result as an
72
+ ``openrunner.Table``.
73
+
74
+ Args:
75
+ y_true: Ground-truth labels.
76
+ y_pred: Predicted labels.
77
+ labels: Optional list of label indices to include.
78
+ target_names: Optional display names for each label.
79
+ """
80
+ if openrunner._active_run is None:
81
+ return
82
+
83
+ from sklearn.metrics import classification_report as _cr
84
+
85
+ report = _cr(
86
+ y_true,
87
+ y_pred,
88
+ labels=labels,
89
+ target_names=target_names,
90
+ output_dict=True,
91
+ )
92
+
93
+ columns = ["class", "precision", "recall", "f1-score", "support"]
94
+ data: list[list[Any]] = []
95
+ for cls_name, metrics in report.items():
96
+ if isinstance(metrics, dict):
97
+ data.append([
98
+ cls_name,
99
+ round(metrics.get("precision", 0), 4),
100
+ round(metrics.get("recall", 0), 4),
101
+ round(metrics.get("f1-score", 0), 4),
102
+ metrics.get("support", 0),
103
+ ])
104
+ else:
105
+ # scalar like "accuracy"
106
+ data.append([cls_name, metrics, None, None, None])
107
+
108
+ table = openrunner.Table(columns=columns, data=data)
109
+ openrunner.log({"classification_report": table})
110
+
111
+
112
+ def log_confusion_matrix(
113
+ y_true: Sequence,
114
+ y_pred: Sequence,
115
+ labels: Sequence | None = None,
116
+ class_names: Sequence[str] | None = None,
117
+ ) -> None:
118
+ """Log a confusion matrix as an Image (matplotlib heatmap).
119
+
120
+ Generates a heatmap using ``matplotlib`` and logs it as an
121
+ ``openrunner.Image``.
122
+
123
+ Args:
124
+ y_true: Ground-truth labels.
125
+ y_pred: Predicted labels.
126
+ labels: Optional list of label values for ordering.
127
+ class_names: Optional display names for the axes.
128
+ """
129
+ if openrunner._active_run is None:
130
+ return
131
+
132
+ from sklearn.metrics import confusion_matrix as _cm
133
+
134
+ try:
135
+ import matplotlib
136
+ matplotlib.use("Agg")
137
+ import matplotlib.pyplot as plt
138
+ except ImportError:
139
+ raise ImportError(
140
+ "log_confusion_matrix requires matplotlib. "
141
+ "Install with: pip install matplotlib"
142
+ )
143
+
144
+ cm = _cm(y_true, y_pred, labels=labels)
145
+
146
+ fig, ax = plt.subplots(figsize=(6, 5))
147
+ im = ax.imshow(cm, interpolation="nearest", cmap="Blues")
148
+ fig.colorbar(im, ax=ax)
149
+
150
+ if class_names is not None:
151
+ tick_marks = list(range(len(class_names)))
152
+ ax.set_xticks(tick_marks)
153
+ ax.set_xticklabels(class_names, rotation=45, ha="right")
154
+ ax.set_yticks(tick_marks)
155
+ ax.set_yticklabels(class_names)
156
+
157
+ ax.set_xlabel("Predicted")
158
+ ax.set_ylabel("True")
159
+ ax.set_title("Confusion Matrix")
160
+ fig.tight_layout()
161
+
162
+ openrunner.log({"confusion_matrix": openrunner.Image(fig)})
163
+ plt.close(fig)
164
+
165
+
166
+ def log_feature_importance(
167
+ model: Any,
168
+ feature_names: Sequence[str] | None = None,
169
+ ) -> None:
170
+ """Log feature importances as a Table.
171
+
172
+ Extracts ``feature_importances_`` from the model and logs them
173
+ as a sorted ``openrunner.Table`` (most important first).
174
+
175
+ Args:
176
+ model: A fitted scikit-learn estimator with
177
+ ``feature_importances_`` attribute.
178
+ feature_names: Optional list of feature names. If not
179
+ provided, uses ``feature_0``, ``feature_1``, etc.
180
+ """
181
+ if openrunner._active_run is None:
182
+ return
183
+
184
+ if not hasattr(model, "feature_importances_"):
185
+ return
186
+
187
+ importances = model.feature_importances_
188
+ if feature_names is None:
189
+ feature_names = [f"feature_{i}" for i in range(len(importances))]
190
+
191
+ # Sort by importance descending
192
+ pairs = sorted(
193
+ zip(feature_names, importances),
194
+ key=lambda x: x[1],
195
+ reverse=True,
196
+ )
197
+
198
+ columns = ["feature", "importance"]
199
+ data = [[name, round(float(imp), 6)] for name, imp in pairs]
200
+
201
+ table = openrunner.Table(columns=columns, data=data)
202
+ openrunner.log({"feature_importance": table})
203
+
204
+
205
+ def log_learning_curve(
206
+ model: Any,
207
+ X: Any,
208
+ y: Any,
209
+ cv: int = 5,
210
+ scoring: str = "accuracy",
211
+ ) -> None:
212
+ """Log a learning curve as an Image.
213
+
214
+ Uses ``sklearn.model_selection.learning_curve`` and plots train/val
215
+ scores vs training set size.
216
+
217
+ Args:
218
+ model: A scikit-learn estimator.
219
+ X: Training features.
220
+ y: Training labels.
221
+ cv: Number of cross-validation folds.
222
+ scoring: Scoring metric name.
223
+ """
224
+ if openrunner._active_run is None:
225
+ return
226
+
227
+ from sklearn.model_selection import learning_curve as _lc
228
+
229
+ try:
230
+ import matplotlib
231
+ matplotlib.use("Agg")
232
+ import matplotlib.pyplot as plt
233
+ except ImportError:
234
+ raise ImportError(
235
+ "log_learning_curve requires matplotlib. "
236
+ "Install with: pip install matplotlib"
237
+ )
238
+
239
+ import numpy as np
240
+
241
+ train_sizes, train_scores, val_scores = _lc(
242
+ model, X, y, cv=cv, scoring=scoring,
243
+ train_sizes=np.linspace(0.1, 1.0, 8),
244
+ )
245
+
246
+ fig, ax = plt.subplots(figsize=(6, 4))
247
+ ax.plot(train_sizes, train_scores.mean(axis=1), "o-", label="Train")
248
+ ax.fill_between(
249
+ train_sizes,
250
+ train_scores.mean(axis=1) - train_scores.std(axis=1),
251
+ train_scores.mean(axis=1) + train_scores.std(axis=1),
252
+ alpha=0.15,
253
+ )
254
+ ax.plot(train_sizes, val_scores.mean(axis=1), "o-", label="Validation")
255
+ ax.fill_between(
256
+ train_sizes,
257
+ val_scores.mean(axis=1) - val_scores.std(axis=1),
258
+ val_scores.mean(axis=1) + val_scores.std(axis=1),
259
+ alpha=0.15,
260
+ )
261
+ ax.set_xlabel("Training Set Size")
262
+ ax.set_ylabel(scoring.capitalize())
263
+ ax.set_title("Learning Curve")
264
+ ax.legend(loc="best")
265
+ fig.tight_layout()
266
+
267
+ openrunner.log({"learning_curve": openrunner.Image(fig)})
268
+ plt.close(fig)
269
+
270
+
271
+ def log_roc(
272
+ y_true: Sequence,
273
+ y_probas: Any,
274
+ labels: Sequence[str] | None = None,
275
+ ) -> None:
276
+ """Log ROC curves as an Image.
277
+
278
+ Computes per-class ROC curves using one-vs-rest strategy and
279
+ logs the plot as an ``openrunner.Image``.
280
+
281
+ Args:
282
+ y_true: Ground-truth labels.
283
+ y_probas: Predicted probabilities (shape: n_samples x n_classes).
284
+ labels: Optional class names for the legend.
285
+ """
286
+ if openrunner._active_run is None:
287
+ return
288
+
289
+ from sklearn.metrics import roc_curve, auc
290
+ from sklearn.preprocessing import label_binarize
291
+
292
+ try:
293
+ import matplotlib
294
+ matplotlib.use("Agg")
295
+ import matplotlib.pyplot as plt
296
+ except ImportError:
297
+ raise ImportError(
298
+ "log_roc requires matplotlib. Install with: pip install matplotlib"
299
+ )
300
+
301
+ import numpy as np
302
+
303
+ classes = np.unique(y_true)
304
+ n_classes = len(classes)
305
+
306
+ if n_classes == 2:
307
+ # Binary classification
308
+ fpr, tpr, _ = roc_curve(y_true, y_probas[:, 1] if y_probas.ndim > 1 else y_probas)
309
+ roc_auc = auc(fpr, tpr)
310
+ fig, ax = plt.subplots(figsize=(6, 5))
311
+ label_name = labels[1] if labels and len(labels) > 1 else "Positive"
312
+ ax.plot(fpr, tpr, label=f"{label_name} (AUC={roc_auc:.3f})")
313
+ else:
314
+ # Multi-class: one-vs-rest
315
+ y_bin = label_binarize(y_true, classes=classes)
316
+ fig, ax = plt.subplots(figsize=(6, 5))
317
+ for i in range(n_classes):
318
+ fpr, tpr, _ = roc_curve(y_bin[:, i], y_probas[:, i])
319
+ roc_auc = auc(fpr, tpr)
320
+ name = labels[i] if labels and i < len(labels) else str(classes[i])
321
+ ax.plot(fpr, tpr, label=f"{name} (AUC={roc_auc:.3f})")
322
+
323
+ ax.plot([0, 1], [0, 1], "k--", alpha=0.3)
324
+ ax.set_xlabel("False Positive Rate")
325
+ ax.set_ylabel("True Positive Rate")
326
+ ax.set_title("ROC Curve")
327
+ ax.legend(loc="lower right", fontsize=9)
328
+ fig.tight_layout()
329
+
330
+ openrunner.log({"roc_curve": openrunner.Image(fig)})
331
+ plt.close(fig)
332
+
333
+
334
+ def log_precision_recall(
335
+ y_true: Sequence,
336
+ y_probas: Any,
337
+ labels: Sequence[str] | None = None,
338
+ ) -> None:
339
+ """Log precision-recall curves as an Image.
340
+
341
+ Args:
342
+ y_true: Ground-truth labels.
343
+ y_probas: Predicted probabilities (shape: n_samples x n_classes).
344
+ labels: Optional class names for the legend.
345
+ """
346
+ if openrunner._active_run is None:
347
+ return
348
+
349
+ from sklearn.metrics import precision_recall_curve, average_precision_score
350
+ from sklearn.preprocessing import label_binarize
351
+
352
+ try:
353
+ import matplotlib
354
+ matplotlib.use("Agg")
355
+ import matplotlib.pyplot as plt
356
+ except ImportError:
357
+ raise ImportError(
358
+ "log_precision_recall requires matplotlib. "
359
+ "Install with: pip install matplotlib"
360
+ )
361
+
362
+ import numpy as np
363
+
364
+ classes = np.unique(y_true)
365
+ n_classes = len(classes)
366
+
367
+ fig, ax = plt.subplots(figsize=(6, 5))
368
+
369
+ if n_classes == 2:
370
+ probs = y_probas[:, 1] if y_probas.ndim > 1 else y_probas
371
+ precision, recall, _ = precision_recall_curve(y_true, probs)
372
+ ap = average_precision_score(y_true, probs)
373
+ name = labels[1] if labels and len(labels) > 1 else "Positive"
374
+ ax.plot(recall, precision, label=f"{name} (AP={ap:.3f})")
375
+ else:
376
+ y_bin = label_binarize(y_true, classes=classes)
377
+ for i in range(n_classes):
378
+ precision, recall, _ = precision_recall_curve(y_bin[:, i], y_probas[:, i])
379
+ ap = average_precision_score(y_bin[:, i], y_probas[:, i])
380
+ name = labels[i] if labels and i < len(labels) else str(classes[i])
381
+ ax.plot(recall, precision, label=f"{name} (AP={ap:.3f})")
382
+
383
+ ax.set_xlabel("Recall")
384
+ ax.set_ylabel("Precision")
385
+ ax.set_title("Precision-Recall Curve")
386
+ ax.legend(loc="best", fontsize=9)
387
+ fig.tight_layout()
388
+
389
+ openrunner.log({"precision_recall": openrunner.Image(fig)})
390
+ plt.close(fig)
391
+
392
+
393
+ def log_class_proportions(
394
+ y_train: Sequence,
395
+ y_test: Sequence,
396
+ labels: Sequence[str] | None = None,
397
+ ) -> None:
398
+ """Log class distribution as a Table.
399
+
400
+ Args:
401
+ y_train: Training labels.
402
+ y_test: Test labels.
403
+ labels: Optional class names.
404
+ """
405
+ if openrunner._active_run is None:
406
+ return
407
+
408
+ import numpy as np
409
+
410
+ classes = np.unique(np.concatenate([y_train, y_test]))
411
+ columns = ["class", "train_count", "test_count", "train_%", "test_%"]
412
+ data = []
413
+ for i, cls in enumerate(classes):
414
+ name = labels[i] if labels and i < len(labels) else str(cls)
415
+ train_ct = int(np.sum(np.array(y_train) == cls))
416
+ test_ct = int(np.sum(np.array(y_test) == cls))
417
+ data.append([
418
+ name,
419
+ train_ct,
420
+ test_ct,
421
+ round(train_ct / len(y_train) * 100, 1),
422
+ round(test_ct / len(y_test) * 100, 1),
423
+ ])
424
+
425
+ table = openrunner.Table(columns=columns, data=data)
426
+ openrunner.log({"class_proportions": table})
427
+
428
+
429
+ def _is_loggable(value: Any) -> bool:
430
+ """Check if a value can be logged as a metric (numeric or string)."""
431
+ return isinstance(value, (int, float, str, bool))
@@ -83,6 +83,29 @@ class _TeeWriter:
83
83
  return getattr(self._original, name)
84
84
 
85
85
 
86
+ class _LogCaptureHandler(logging.Handler):
87
+ """Captures Python logging output and sends it to OpenRunner console logs.
88
+
89
+ This catches output from frameworks that use `logging` instead of `print()`
90
+ (HuggingFace Transformers, PyTorch Lightning, etc.)
91
+ """
92
+
93
+ def __init__(self, sender) -> None:
94
+ super().__init__()
95
+ self._sender = sender
96
+ # Don't capture openrunner's own logs to avoid recursion
97
+ self.addFilter(lambda record: not record.name.startswith("openrunner"))
98
+
99
+ def emit(self, record: logging.LogRecord) -> None:
100
+ try:
101
+ msg = self.format(record)
102
+ ts = datetime.now(timezone.utc).isoformat()
103
+ stream = "stderr" if record.levelno >= logging.WARNING else "stdout"
104
+ self._sender.add_console_line(ts, stream, msg)
105
+ except Exception:
106
+ pass
107
+
108
+
86
109
  def _generate_run_id() -> str:
87
110
  """Generate a W&B-style 8-character run ID."""
88
111
  return "".join(secrets.choice(_ALPHABET) for _ in range(8))
@@ -226,6 +249,10 @@ class Run:
226
249
  sys.stdout = _TeeWriter(sys.stdout, "stdout", self._sender)
227
250
  sys.stderr = _TeeWriter(sys.stderr, "stderr", self._sender)
228
251
 
252
+ # Also capture Python logging output (used by HuggingFace, PyTorch, etc.)
253
+ self._log_handler = _LogCaptureHandler(self._sender)
254
+ logging.root.addHandler(self._log_handler)
255
+
229
256
  # -- Properties -----------------------------------------------------------
230
257
 
231
258
  @property
@@ -668,6 +695,10 @@ class Run:
668
695
  exit_code: Optional exit code for the run.
669
696
  quiet: If True, suppress the finish message.
670
697
  """
698
+ # Remove logging handler
699
+ if hasattr(self, "_log_handler"):
700
+ logging.root.removeHandler(self._log_handler)
701
+
671
702
  # Restore original stdout/stderr before stopping sender
672
703
  if hasattr(self, "_original_stdout"):
673
704
  # Flush any remaining captured output
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "openrunner-sdk"
3
- version = "0.4.0"
3
+ version = "0.4.2"
4
4
  description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
5
5
  readme = "README.md"
6
6
  license = {text = "MIT"}
@@ -1,49 +0,0 @@
1
- """PyTorch integration for OpenRunner.
2
-
3
- Provides a ``log_gradients`` utility that logs gradient norms for all
4
- model parameters via ``openrunner.log()``.
5
-
6
- Usage::
7
-
8
- from openrunner.integration.pytorch import log_gradients
9
-
10
- for batch in dataloader:
11
- loss = model(batch)
12
- loss.backward()
13
- log_gradients(model, step=step)
14
- optimizer.step()
15
-
16
- Requires: ``pip install openrunner[pytorch]``
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- try:
22
- import torch # noqa: F401
23
- except ImportError:
24
- raise ImportError(
25
- "PyTorch integration requires torch. "
26
- "Install with: pip install openrunner[pytorch]"
27
- )
28
-
29
- import openrunner
30
-
31
-
32
- def log_gradients(model: torch.nn.Module, step: int | None = None) -> None:
33
- """Log gradient norms for all model parameters.
34
-
35
- For each named parameter that has a gradient, logs the L2 norm
36
- as ``gradients/{name}``. Parameters whose ``.grad`` is ``None``
37
- (e.g., frozen layers) are silently skipped.
38
-
39
- Args:
40
- model: A PyTorch ``nn.Module`` whose ``.backward()`` has been called.
41
- step: Optional explicit step value passed to ``openrunner.log()``.
42
- """
43
- for name, param in model.named_parameters():
44
- if param.grad is not None:
45
- norm = param.grad.norm().item()
46
- openrunner.log({f"gradients/{name}": norm}, step=step, commit=False)
47
-
48
- # Commit the step after all gradient norms have been logged.
49
- openrunner.log({}, step=step, commit=True)
@@ -1,203 +0,0 @@
1
- """Scikit-learn integration for OpenRunner.
2
-
3
- Provides utility functions to log scikit-learn model information,
4
- classification reports, confusion matrices, and feature importances
5
- to an active OpenRunner run.
6
-
7
- Usage::
8
-
9
- from openrunner.integration.sklearn import (
10
- log_model,
11
- log_classification_report,
12
- log_confusion_matrix,
13
- log_feature_importance,
14
- )
15
-
16
- model = RandomForestClassifier().fit(X_train, y_train)
17
- log_model(model)
18
- log_classification_report(y_test, y_pred)
19
- log_confusion_matrix(y_test, y_pred)
20
- log_feature_importance(model, feature_names=["f1", "f2", "f3"])
21
-
22
- Requires: ``pip install openrunner[sklearn]``
23
- """
24
-
25
- from __future__ import annotations
26
-
27
- from typing import Any, Sequence
28
-
29
- try:
30
- import sklearn # noqa: F401
31
- except ImportError:
32
- raise ImportError(
33
- "Scikit-learn integration requires scikit-learn. "
34
- "Install with: pip install openrunner[sklearn]"
35
- )
36
-
37
- import openrunner
38
-
39
-
40
- def log_model(model: Any) -> None:
41
- """Log model hyperparameters via ``get_params()``.
42
-
43
- Logs all parameters returned by the model's ``get_params()`` method
44
- as a flat dict under ``model/`` prefix.
45
-
46
- Args:
47
- model: A fitted scikit-learn estimator.
48
- """
49
- if openrunner._active_run is None:
50
- return
51
-
52
- params = model.get_params()
53
- prefixed = {f"model/{k}": v for k, v in params.items() if _is_loggable(v)}
54
- if prefixed:
55
- openrunner.log(prefixed)
56
-
57
-
58
- def log_classification_report(
59
- y_true: Sequence,
60
- y_pred: Sequence,
61
- labels: Sequence | None = None,
62
- target_names: Sequence[str] | None = None,
63
- ) -> None:
64
- """Log a classification report as a Table.
65
-
66
- Uses ``sklearn.metrics.classification_report`` to compute precision,
67
- recall, f1-score and support, then logs the result as an
68
- ``openrunner.Table``.
69
-
70
- Args:
71
- y_true: Ground-truth labels.
72
- y_pred: Predicted labels.
73
- labels: Optional list of label indices to include.
74
- target_names: Optional display names for each label.
75
- """
76
- if openrunner._active_run is None:
77
- return
78
-
79
- from sklearn.metrics import classification_report as _cr
80
-
81
- report = _cr(
82
- y_true,
83
- y_pred,
84
- labels=labels,
85
- target_names=target_names,
86
- output_dict=True,
87
- )
88
-
89
- columns = ["class", "precision", "recall", "f1-score", "support"]
90
- data: list[list[Any]] = []
91
- for cls_name, metrics in report.items():
92
- if isinstance(metrics, dict):
93
- data.append([
94
- cls_name,
95
- round(metrics.get("precision", 0), 4),
96
- round(metrics.get("recall", 0), 4),
97
- round(metrics.get("f1-score", 0), 4),
98
- metrics.get("support", 0),
99
- ])
100
- else:
101
- # scalar like "accuracy"
102
- data.append([cls_name, metrics, None, None, None])
103
-
104
- table = openrunner.Table(columns=columns, data=data)
105
- openrunner.log({"classification_report": table})
106
-
107
-
108
- def log_confusion_matrix(
109
- y_true: Sequence,
110
- y_pred: Sequence,
111
- labels: Sequence | None = None,
112
- class_names: Sequence[str] | None = None,
113
- ) -> None:
114
- """Log a confusion matrix as an Image (matplotlib heatmap).
115
-
116
- Generates a heatmap using ``matplotlib`` and logs it as an
117
- ``openrunner.Image``.
118
-
119
- Args:
120
- y_true: Ground-truth labels.
121
- y_pred: Predicted labels.
122
- labels: Optional list of label values for ordering.
123
- class_names: Optional display names for the axes.
124
- """
125
- if openrunner._active_run is None:
126
- return
127
-
128
- from sklearn.metrics import confusion_matrix as _cm
129
-
130
- try:
131
- import matplotlib
132
- matplotlib.use("Agg")
133
- import matplotlib.pyplot as plt
134
- except ImportError:
135
- raise ImportError(
136
- "log_confusion_matrix requires matplotlib. "
137
- "Install with: pip install matplotlib"
138
- )
139
-
140
- cm = _cm(y_true, y_pred, labels=labels)
141
-
142
- fig, ax = plt.subplots(figsize=(6, 5))
143
- im = ax.imshow(cm, interpolation="nearest", cmap="Blues")
144
- fig.colorbar(im, ax=ax)
145
-
146
- if class_names is not None:
147
- tick_marks = list(range(len(class_names)))
148
- ax.set_xticks(tick_marks)
149
- ax.set_xticklabels(class_names, rotation=45, ha="right")
150
- ax.set_yticks(tick_marks)
151
- ax.set_yticklabels(class_names)
152
-
153
- ax.set_xlabel("Predicted")
154
- ax.set_ylabel("True")
155
- ax.set_title("Confusion Matrix")
156
- fig.tight_layout()
157
-
158
- openrunner.log({"confusion_matrix": openrunner.Image(fig)})
159
- plt.close(fig)
160
-
161
-
162
- def log_feature_importance(
163
- model: Any,
164
- feature_names: Sequence[str] | None = None,
165
- ) -> None:
166
- """Log feature importances as a Table.
167
-
168
- Extracts ``feature_importances_`` from the model and logs them
169
- as a sorted ``openrunner.Table`` (most important first).
170
-
171
- Args:
172
- model: A fitted scikit-learn estimator with
173
- ``feature_importances_`` attribute.
174
- feature_names: Optional list of feature names. If not
175
- provided, uses ``feature_0``, ``feature_1``, etc.
176
- """
177
- if openrunner._active_run is None:
178
- return
179
-
180
- if not hasattr(model, "feature_importances_"):
181
- return
182
-
183
- importances = model.feature_importances_
184
- if feature_names is None:
185
- feature_names = [f"feature_{i}" for i in range(len(importances))]
186
-
187
- # Sort by importance descending
188
- pairs = sorted(
189
- zip(feature_names, importances),
190
- key=lambda x: x[1],
191
- reverse=True,
192
- )
193
-
194
- columns = ["feature", "importance"]
195
- data = [[name, round(float(imp), 6)] for name, imp in pairs]
196
-
197
- table = openrunner.Table(columns=columns, data=data)
198
- openrunner.log({"feature_importance": table})
199
-
200
-
201
- def _is_loggable(value: Any) -> bool:
202
- """Check if a value can be logged as a metric (numeric or string)."""
203
- return isinstance(value, (int, float, str, bool))
File without changes
File without changes
File without changes