cnn-classifier-toolkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. cnn_classifier/__init__.py +5 -0
  2. cnn_classifier/_version.py +1 -0
  3. cnn_classifier/cli.py +1085 -0
  4. cnn_classifier/config/__init__.py +17 -0
  5. cnn_classifier/config/loader.py +102 -0
  6. cnn_classifier/config/presets/default.yaml +47 -0
  7. cnn_classifier/config/resolve.py +108 -0
  8. cnn_classifier/config/schema.py +251 -0
  9. cnn_classifier/core/__init__.py +19 -0
  10. cnn_classifier/core/device.py +50 -0
  11. cnn_classifier/core/errors.py +21 -0
  12. cnn_classifier/core/hashing.py +18 -0
  13. cnn_classifier/core/registry.py +46 -0
  14. cnn_classifier/core/seeding.py +37 -0
  15. cnn_classifier/core/types.py +97 -0
  16. cnn_classifier/data/__init__.py +22 -0
  17. cnn_classifier/data/adapters/__init__.py +19 -0
  18. cnn_classifier/data/adapters/_common.py +241 -0
  19. cnn_classifier/data/adapters/image_folder.py +65 -0
  20. cnn_classifier/data/adapters/manifest.py +62 -0
  21. cnn_classifier/data/adapters/numpy.py +96 -0
  22. cnn_classifier/data/adapters/registry.py +29 -0
  23. cnn_classifier/data/adapters/synthetic.py +114 -0
  24. cnn_classifier/data/adapters/torchvision.py +192 -0
  25. cnn_classifier/data/base.py +17 -0
  26. cnn_classifier/data/contract.py +28 -0
  27. cnn_classifier/data/image_io.py +116 -0
  28. cnn_classifier/data/inspection.py +111 -0
  29. cnn_classifier/data/loaders.py +79 -0
  30. cnn_classifier/data/manifest.py +98 -0
  31. cnn_classifier/data/splits.py +136 -0
  32. cnn_classifier/data/stats.py +106 -0
  33. cnn_classifier/data/transforms.py +192 -0
  34. cnn_classifier/diagnostics/__init__.py +22 -0
  35. cnn_classifier/diagnostics/activations.py +56 -0
  36. cnn_classifier/diagnostics/architecture.py +172 -0
  37. cnn_classifier/diagnostics/determinism.py +25 -0
  38. cnn_classifier/diagnostics/gradients.py +32 -0
  39. cnn_classifier/diagnostics/sanity.py +49 -0
  40. cnn_classifier/diagnostics/tracing.py +56 -0
  41. cnn_classifier/evaluation/__init__.py +53 -0
  42. cnn_classifier/evaluation/bootstrap.py +96 -0
  43. cnn_classifier/evaluation/calibration.py +74 -0
  44. cnn_classifier/evaluation/compare.py +111 -0
  45. cnn_classifier/evaluation/error_analysis.py +61 -0
  46. cnn_classifier/evaluation/evaluator.py +143 -0
  47. cnn_classifier/evaluation/latency.py +58 -0
  48. cnn_classifier/evaluation/metrics.py +94 -0
  49. cnn_classifier/evaluation/predict.py +64 -0
  50. cnn_classifier/models/__init__.py +50 -0
  51. cnn_classifier/models/base.py +62 -0
  52. cnn_classifier/models/residual_cnn.py +138 -0
  53. cnn_classifier/models/simple_cnn.py +55 -0
  54. cnn_classifier/models/summary.py +39 -0
  55. cnn_classifier/models/torchvision.py +46 -0
  56. cnn_classifier/models/transfer.py +193 -0
  57. cnn_classifier/runs/__init__.py +6 -0
  58. cnn_classifier/runs/artifacts.py +50 -0
  59. cnn_classifier/runs/context.py +130 -0
  60. cnn_classifier/tracking/__init__.py +34 -0
  61. cnn_classifier/tracking/base.py +16 -0
  62. cnn_classifier/tracking/composite.py +20 -0
  63. cnn_classifier/tracking/console.py +14 -0
  64. cnn_classifier/tracking/csv.py +19 -0
  65. cnn_classifier/tracking/tensorboard.py +22 -0
  66. cnn_classifier/training/__init__.py +37 -0
  67. cnn_classifier/training/amp.py +51 -0
  68. cnn_classifier/training/callbacks.py +229 -0
  69. cnn_classifier/training/checkpoint.py +151 -0
  70. cnn_classifier/training/losses.py +67 -0
  71. cnn_classifier/training/optim.py +40 -0
  72. cnn_classifier/training/schedulers.py +53 -0
  73. cnn_classifier/training/state.py +35 -0
  74. cnn_classifier/training/trainer.py +357 -0
  75. cnn_classifier/viz/__init__.py +42 -0
  76. cnn_classifier/viz/architecture.py +276 -0
  77. cnn_classifier/viz/compare.py +36 -0
  78. cnn_classifier/viz/data.py +56 -0
  79. cnn_classifier/viz/evaluation.py +134 -0
  80. cnn_classifier/viz/features.py +122 -0
  81. cnn_classifier/viz/report.py +821 -0
  82. cnn_classifier/viz/style.py +69 -0
  83. cnn_classifier/viz/templates/report.html.j2 +355 -0
  84. cnn_classifier/viz/training.py +83 -0
  85. cnn_classifier_toolkit-0.1.0.dist-info/METADATA +249 -0
  86. cnn_classifier_toolkit-0.1.0.dist-info/RECORD +89 -0
  87. cnn_classifier_toolkit-0.1.0.dist-info/WHEEL +4 -0
  88. cnn_classifier_toolkit-0.1.0.dist-info/entry_points.txt +2 -0
  89. cnn_classifier_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,821 @@
1
+ """Self-contained experiment report generated only from saved artifacts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import csv
7
+ import json
8
+ import mimetypes
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ import numpy as np
14
+ import yaml
15
+ from jinja2 import Environment, FileSystemLoader, select_autoescape
16
+
17
+ _TEMPLATE_DIR = Path(__file__).parent / "templates"
18
+ _TEMPLATE_NAME = "report.html.j2"
19
+
20
+ _CONFIG_SECTIONS = (
21
+ "experiment",
22
+ "data",
23
+ "model",
24
+ "training",
25
+ "evaluation",
26
+ "tracking",
27
+ "report",
28
+ "runtime",
29
+ )
30
+
31
+ _HISTORY_LABELS = {
32
+ "epoch": "Epoch",
33
+ "train_loss": "Train loss",
34
+ "val_loss": "Validation loss",
35
+ "train_accuracy": "Train accuracy",
36
+ "val_accuracy": "Validation accuracy",
37
+ "lr": "Learning rate",
38
+ }
39
+
40
+ _FIGURE_CAPTIONS = {
41
+ "training_history": "Training history",
42
+ "confusion_matrix": "Confusion matrix",
43
+ "per_class": "Per-class metrics",
44
+ "calibration": "Calibration",
45
+ "errors": "Misclassifications",
46
+ "error_gallery": "Misclassifications",
47
+ "model_card": "Model card",
48
+ "model_summary": "Model summary",
49
+ "dataset_overview": "Dataset overview",
50
+ "augmentation_gallery": "Augmentation samples",
51
+ "comparison": "Model comparison",
52
+ }
53
+
54
+ _EVALUATION_CANDIDATES = (
55
+ "evaluation.json",
56
+ "test_evaluation.json",
57
+ "evaluation/test.json",
58
+ )
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class _Figure:
63
+ name: str
64
+ caption: str
65
+ data_uri: str
66
+
67
+
68
+ def _read_json(path: Path) -> dict[str, Any] | None:
69
+ if not path.is_file():
70
+ return None
71
+ try:
72
+ payload = json.loads(path.read_text(encoding="utf-8"))
73
+ except (OSError, json.JSONDecodeError):
74
+ return None
75
+ return payload if isinstance(payload, dict) else None
76
+
77
+
78
+ def _read_yaml(path: Path) -> dict[str, Any] | None:
79
+ if not path.is_file():
80
+ return None
81
+ try:
82
+ payload = yaml.safe_load(path.read_text(encoding="utf-8"))
83
+ except (OSError, yaml.YAMLError):
84
+ return None
85
+ return payload if isinstance(payload, dict) else None
86
+
87
+
88
+ def _read_history(path: Path) -> list[dict[str, float]]:
89
+ if not path.is_file():
90
+ return []
91
+ rows: list[dict[str, float]] = []
92
+ try:
93
+ with path.open(encoding="utf-8", newline="") as handle:
94
+ for row in csv.DictReader(handle):
95
+ parsed: dict[str, float] = {}
96
+ for key, value in row.items():
97
+ if key is None or value is None or value == "":
98
+ continue
99
+ try:
100
+ parsed[key] = float(value)
101
+ except ValueError:
102
+ continue
103
+ if parsed:
104
+ rows.append(parsed)
105
+ except OSError:
106
+ return []
107
+ return rows
108
+
109
+
110
+ def _read_metrics_jsonl(path: Path) -> list[dict[str, float]]:
111
+ if not path.is_file():
112
+ return []
113
+ rows: list[dict[str, float]] = []
114
+ try:
115
+ for line in path.read_text(encoding="utf-8").splitlines():
116
+ if not line.strip():
117
+ continue
118
+ try:
119
+ payload = json.loads(line)
120
+ except json.JSONDecodeError:
121
+ continue
122
+ if not isinstance(payload, dict):
123
+ continue
124
+ parsed = {
125
+ str(key): float(value)
126
+ for key, value in payload.items()
127
+ if isinstance(value, (int, float)) and not isinstance(value, bool)
128
+ }
129
+ if parsed:
130
+ rows.append(parsed)
131
+ except OSError:
132
+ return []
133
+ return rows
134
+
135
+
136
+ def _read_predictions(run_dir: Path) -> list[dict[str, str]]:
137
+ directory = run_dir / "predictions"
138
+ if not directory.is_dir():
139
+ return []
140
+ rows: list[dict[str, str]] = []
141
+ for path in sorted(directory.glob("*.npz")):
142
+ try:
143
+ with np.load(path) as archive:
144
+ labels = archive.get("y_true", None)
145
+ predictions = archive.get("y_pred", None)
146
+ samples = len(labels) if labels is not None else None
147
+ if samples is None and archive.files:
148
+ samples = len(archive[archive.files[0]])
149
+ accuracy = (
150
+ float(np.mean(labels == predictions))
151
+ if labels is not None
152
+ and predictions is not None
153
+ and len(labels) == len(predictions)
154
+ and len(labels) > 0
155
+ else None
156
+ )
157
+ except (OSError, ValueError, KeyError):
158
+ continue
159
+ rows.append(
160
+ {
161
+ "name": path.name,
162
+ "samples": _format(samples),
163
+ "accuracy": _format_metric(accuracy) if accuracy is not None else "—",
164
+ }
165
+ )
166
+ return rows
167
+
168
+
169
+ def _format(value: Any) -> str:
170
+ if value is None:
171
+ return "—"
172
+ if isinstance(value, bool):
173
+ return "yes" if value else "no"
174
+ if isinstance(value, float):
175
+ return f"{value:.4g}"
176
+ if isinstance(value, int):
177
+ return f"{value:,}"
178
+ if isinstance(value, (dict, list, tuple)):
179
+ return json.dumps(value, ensure_ascii=False, sort_keys=True)
180
+ return str(value)
181
+
182
+
183
+ def _format_metric(value: Any) -> str:
184
+ if value is None:
185
+ return "—"
186
+ if isinstance(value, int):
187
+ return f"{value:,}"
188
+ try:
189
+ return f"{float(value):.3f}"
190
+ except (TypeError, ValueError):
191
+ return str(value)
192
+
193
+
194
+ def _flatten(prefix: str, payload: Any) -> list[tuple[str, str]]:
195
+ rows: list[tuple[str, str]] = []
196
+ if isinstance(payload, dict):
197
+ for key, value in payload.items():
198
+ label = f"{prefix}.{key}" if prefix else str(key)
199
+ if isinstance(value, dict):
200
+ rows.extend(_flatten(label, value))
201
+ else:
202
+ rows.append((label, _format(value)))
203
+ elif isinstance(payload, list):
204
+ rows.append((prefix or "value", _format(payload)))
205
+ else:
206
+ rows.append((prefix or "value", _format(payload)))
207
+ return rows
208
+
209
+
210
+ def _config_sections(config: dict[str, Any] | None) -> list[dict[str, Any]]:
211
+ if not config:
212
+ return []
213
+ sections: list[dict[str, Any]] = []
214
+ for name in _CONFIG_SECTIONS:
215
+ value = config.get(name)
216
+ if not value:
217
+ continue
218
+ rows = _flatten("", value)
219
+ sections.append({"title": name.replace("_", " ").title(), "rows": rows})
220
+ extras = [
221
+ (key, value)
222
+ for key, value in config.items()
223
+ if key not in _CONFIG_SECTIONS and key != "defaults"
224
+ ]
225
+ if extras:
226
+ sections.append(
227
+ {
228
+ "title": "Other",
229
+ "rows": _flatten("", dict(extras)),
230
+ }
231
+ )
232
+ return sections
233
+
234
+
235
+ def _contract_rows(contract: dict[str, Any] | None) -> list[tuple[str, str]]:
236
+ if not contract:
237
+ return []
238
+ rows: list[tuple[str, str]] = []
239
+ for key, value in contract.items():
240
+ if key == "label_map" and isinstance(value, dict):
241
+ rows.append(
242
+ ("label_map", ", ".join(f"{name}={index}" for name, index in value.items()))
243
+ )
244
+ elif key == "normalization" and isinstance(value, dict):
245
+ mean = value.get("mean")
246
+ std = value.get("std")
247
+ rows.append(("normalization.mean", _format(mean)))
248
+ rows.append(("normalization.std", _format(std)))
249
+ rows.append(("normalization.source", _format(value.get("source"))))
250
+ else:
251
+ rows.append((key, _format(value)))
252
+ return rows
253
+
254
+
255
+ def _split_section(
256
+ fingerprint: dict[str, Any] | None,
257
+ ) -> tuple[list[dict[str, str]], list[str]]:
258
+ if not fingerprint:
259
+ return [], ["Split integrity: not evaluated (no fingerprint artifact found)."]
260
+ rows: list[dict[str, str]] = []
261
+ splits = fingerprint.get("splits")
262
+ if isinstance(splits, dict):
263
+ for name, details in splits.items():
264
+ if isinstance(details, dict):
265
+ rows.append(
266
+ {
267
+ "split": str(name),
268
+ "count": _format(details.get("count")),
269
+ "fingerprint": _format(details.get("fingerprint")),
270
+ }
271
+ )
272
+ else:
273
+ rows.append({"split": str(name), "count": _format(details), "fingerprint": "—"})
274
+ notes: list[str] = []
275
+ for key in ("split_mode", "group_overlap", "duplicate_overlap", "dataset"):
276
+ if key in fingerprint:
277
+ notes.append(f"{key.replace('_', ' ').capitalize()}: {_format(fingerprint[key])}")
278
+ if not rows:
279
+ rows = [
280
+ {"split": key, "count": _format(value), "fingerprint": "—"}
281
+ for key, value in fingerprint.items()
282
+ ]
283
+ notes = []
284
+ return rows, notes
285
+
286
+
287
+ def _model_section(
288
+ summary: dict[str, Any] | None,
289
+ ) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
290
+ if not summary:
291
+ return [], []
292
+ modules = summary.get("modules")
293
+ rows: list[dict[str, str]] = []
294
+ if isinstance(modules, list):
295
+ for module in modules[:24]:
296
+ if isinstance(module, dict):
297
+ rows.append(
298
+ {
299
+ "name": _format(module.get("name")),
300
+ "type": _format(module.get("type")),
301
+ "params": _format(module.get("params")),
302
+ }
303
+ )
304
+ facts = [
305
+ {"label": "Total parameters", "value": _format(summary.get("total_params"))},
306
+ {"label": "Trainable parameters", "value": _format(summary.get("trainable_params"))},
307
+ {"label": "Input shape", "value": _format(summary.get("input_shape"))},
308
+ {"label": "Output shape", "value": _format(summary.get("output_shape"))},
309
+ ]
310
+ return facts, rows
311
+
312
+
313
+ def _history_section(
314
+ history: list[dict[str, float]], summary: dict[str, Any] | None
315
+ ) -> dict[str, Any]:
316
+ if not history:
317
+ return {
318
+ "rows": [],
319
+ "columns": [],
320
+ "best_epoch": None,
321
+ "best_val_accuracy": None,
322
+ "epochs_run": None,
323
+ }
324
+ columns = [
325
+ (key, label) for key, label in _HISTORY_LABELS.items() if any(key in row for row in history)
326
+ ]
327
+ rows = []
328
+ for row in history:
329
+ rows.append(
330
+ {
331
+ key: (
332
+ str(int(row[key])) if key == "epoch" and key in row else _format(row.get(key))
333
+ )
334
+ for key, _ in columns
335
+ }
336
+ )
337
+ best_epoch: Any = summary.get("best_epoch") if summary else None
338
+ best_accuracy: Any = summary.get("best_metric") if summary else None
339
+ if best_accuracy is None and any("val_accuracy" in row for row in history):
340
+ best_row = max(history, key=lambda item: item.get("val_accuracy", float("-inf")))
341
+ best_accuracy = best_row.get("val_accuracy")
342
+ best_epoch = int(best_row.get("epoch", 0))
343
+ return {
344
+ "rows": rows,
345
+ "columns": [{"key": key, "label": label} for key, label in columns],
346
+ "best_epoch": best_epoch,
347
+ "best_val_accuracy": best_accuracy,
348
+ "epochs_run": len(history),
349
+ }
350
+
351
+
352
+ def _find_evaluation(run_dir: Path) -> dict[str, Any] | None:
353
+ for candidate in _EVALUATION_CANDIDATES:
354
+ payload = _read_json(run_dir / candidate)
355
+ if payload is not None:
356
+ return payload
357
+ return None
358
+
359
+
360
+ def _evaluation_sections(
361
+ evaluation: dict[str, Any] | None,
362
+ contract: dict[str, Any] | None,
363
+ ) -> dict[str, Any]:
364
+ if not evaluation:
365
+ return {
366
+ "available": False,
367
+ "metrics": [],
368
+ "per_class": [],
369
+ "confusion": None,
370
+ "class_names": [],
371
+ "confidence": None,
372
+ "latency": [],
373
+ "split": None,
374
+ }
375
+ metrics = evaluation.get("metrics")
376
+ if not isinstance(metrics, dict):
377
+ metrics = evaluation
378
+ rows = [
379
+ {"label": "Accuracy", "value": _format_metric(metrics.get("accuracy"))},
380
+ {"label": "Top-k accuracy", "value": _format_metric(metrics.get("top_k_accuracy"))},
381
+ {"label": "Macro F1", "value": _format_metric(metrics.get("macro_f1"))},
382
+ {"label": "NLL", "value": _format_metric(metrics.get("nll"))},
383
+ {"label": "ECE", "value": _format_metric(metrics.get("ece"))},
384
+ {"label": "MCE", "value": _format_metric(metrics.get("mce"))},
385
+ {"label": "Brier score", "value": _format_metric(metrics.get("brier"))},
386
+ {"label": "Support", "value": _format_metric(metrics.get("support"))},
387
+ ]
388
+ calibration = metrics.get("calibration")
389
+ if isinstance(calibration, dict):
390
+ rows[4] = {"label": "ECE", "value": _format_metric(calibration.get("ece"))}
391
+ rows[5] = {"label": "MCE", "value": _format_metric(calibration.get("mce"))}
392
+ rows[6] = {"label": "Brier score", "value": _format_metric(calibration.get("brier"))}
393
+ rows = [row for row in rows if row["value"] != "—"]
394
+
395
+ per_class_rows: list[dict[str, str]] = []
396
+ per_class = metrics.get("per_class")
397
+ if isinstance(per_class, dict):
398
+ for name, values in per_class.items():
399
+ if isinstance(values, dict):
400
+ per_class_rows.append(
401
+ {
402
+ "name": str(name),
403
+ "precision": _format_metric(values.get("precision")),
404
+ "recall": _format_metric(values.get("recall")),
405
+ "f1": _format_metric(values.get("f1")),
406
+ "support": _format_metric(values.get("support")),
407
+ }
408
+ )
409
+
410
+ confusion = metrics.get("confusion_matrix")
411
+ if not isinstance(confusion, list):
412
+ confusion = None
413
+ names = []
414
+ if contract and isinstance(contract.get("class_names"), list):
415
+ names = [str(name) for name in contract["class_names"]]
416
+ elif per_class:
417
+ names = [row["name"] for row in per_class_rows]
418
+ if confusion and not names:
419
+ names = [str(index) for index in range(len(confusion))]
420
+
421
+ confidence = None
422
+ intervals = evaluation.get("confidence_intervals")
423
+ if isinstance(intervals, dict) and isinstance(intervals.get("accuracy"), dict):
424
+ interval = intervals["accuracy"]
425
+ confidence = (
426
+ f"{_format_metric(interval.get('confidence'))} CI "
427
+ f"[{_format_metric(interval.get('low'))}, {_format_metric(interval.get('high'))}]"
428
+ )
429
+ elif isinstance(evaluation.get("accuracy_ci"), list):
430
+ low, high = evaluation["accuracy_ci"]
431
+ confidence = f"95% CI [{_format_metric(low)}, {_format_metric(high)}]"
432
+
433
+ latency_rows: list[dict[str, str]] = []
434
+ latency = evaluation.get("latency")
435
+ if isinstance(latency, dict):
436
+ for key, label in (
437
+ ("mean_ms", "Mean latency (ms)"),
438
+ ("p50_ms", "p50 latency (ms)"),
439
+ ("p95_ms", "p95 latency (ms)"),
440
+ ("throughput", "Throughput (images/s)"),
441
+ ):
442
+ if key in latency:
443
+ latency_rows.append({"label": label, "value": _format_metric(latency[key])})
444
+
445
+ return {
446
+ "available": True,
447
+ "metrics": rows,
448
+ "per_class": per_class_rows,
449
+ "confusion": confusion,
450
+ "class_names": names,
451
+ "confidence": confidence,
452
+ "latency": latency_rows,
453
+ "split": evaluation.get("split"),
454
+ }
455
+
456
+
457
+ def _collect_figures(run_dir: Path) -> list[_Figure]:
458
+ directory = run_dir / "figures"
459
+ if not directory.is_dir():
460
+ return []
461
+ by_stem: dict[str, dict[str, Path]] = {}
462
+ for path in sorted(directory.iterdir()):
463
+ suffix = path.suffix.lower()
464
+ if suffix in {".png", ".svg"}:
465
+ by_stem.setdefault(path.stem, {})[suffix] = path
466
+ figures: list[_Figure] = []
467
+ for stem in sorted(by_stem):
468
+ candidates = by_stem[stem]
469
+ selected = candidates.get(".png") or candidates.get(".svg")
470
+ if selected is None:
471
+ continue
472
+ mime = mimetypes.guess_type(selected.name)[0] or "application/octet-stream"
473
+ encoded = base64.b64encode(selected.read_bytes()).decode("ascii")
474
+ figures.append(
475
+ _Figure(
476
+ name=selected.name,
477
+ caption=_FIGURE_CAPTIONS.get(stem, stem.replace("_", " ").capitalize()),
478
+ data_uri=f"data:{mime};base64,{encoded}",
479
+ )
480
+ )
481
+ return figures
482
+
483
+
484
+ def _runtime_rows(
485
+ environment: dict[str, Any] | None,
486
+ summary: dict[str, Any] | None,
487
+ git: dict[str, Any] | None,
488
+ ) -> list[dict[str, str]]:
489
+ rows: list[dict[str, str]] = []
490
+ if environment:
491
+ for key, label in (
492
+ ("python", "Python"),
493
+ ("torch", "PyTorch"),
494
+ ("torchvision", "TorchVision"),
495
+ ("platform", "Platform"),
496
+ ("device", "Device"),
497
+ ):
498
+ if key in environment:
499
+ rows.append({"label": label, "value": _format(environment[key])})
500
+ if summary:
501
+ if "device" in summary:
502
+ rows.append({"label": "Training device", "value": _format(summary["device"])})
503
+ if "config_hash" in summary:
504
+ rows.append({"label": "Config hash", "value": _format(summary["config_hash"])})
505
+ if "epochs_run" in summary:
506
+ rows.append({"label": "Epochs run", "value": _format(summary["epochs_run"])})
507
+ if git:
508
+ if git.get("commit"):
509
+ dirty = " (dirty)" if git.get("dirty") else ""
510
+ rows.append({"label": "Git commit", "value": f"{git['commit']}{dirty}"})
511
+ if git.get("branch"):
512
+ rows.append({"label": "Git branch", "value": _format(git["branch"])})
513
+ return rows
514
+
515
+
516
+ def _repo_root(run_dir: Path) -> Path | None:
517
+ for parent in (run_dir.resolve(), *run_dir.resolve().parents):
518
+ if (parent / "pyproject.toml").is_file():
519
+ return parent
520
+ return None
521
+
522
+
523
+ def _sanitize_paths(value: Any, root: Path) -> Any:
524
+ if isinstance(value, str):
525
+ return value.replace(str(root), ".")
526
+ if isinstance(value, dict):
527
+ return {key: _sanitize_paths(item, root) for key, item in value.items()}
528
+ if isinstance(value, list):
529
+ return [_sanitize_paths(item, root) for item in value]
530
+ if isinstance(value, tuple):
531
+ return tuple(_sanitize_paths(item, root) for item in value)
532
+ return value
533
+
534
+
535
+ def _reproduction_commands(run_dir: Path) -> list[str]:
536
+ root = _repo_root(run_dir)
537
+ display = str(run_dir.resolve().relative_to(root)) if root is not None else run_dir.name
538
+ return [
539
+ f"cnncls train -c <config.yaml> --runs-dir runs # produced {run_dir.name}",
540
+ f"cnncls evaluate --run {display} --checkpoint {display}/checkpoints/best.pt",
541
+ f"cnncls report --run {display}",
542
+ f"cnncls compare {display} <other-run>",
543
+ ]
544
+
545
+
546
+ def _build_context(run_dir: Path) -> dict[str, Any]:
547
+ config = _read_yaml(run_dir / "config.resolved.yaml")
548
+ root = _repo_root(run_dir)
549
+ if root is not None:
550
+ config = _sanitize_paths(config, root)
551
+ environment = _read_json(run_dir / "environment.json")
552
+ status = _read_json(run_dir / "status.json")
553
+ summary = _read_json(run_dir / "summary.json")
554
+ contract = _read_json(run_dir / "data" / "model_contract.json")
555
+ fingerprint = _read_json(run_dir / "data" / "fingerprint.json")
556
+ model_summary = _read_json(run_dir / "model_summary.json")
557
+ git = _read_json(run_dir / "git.json")
558
+ evaluation = _find_evaluation(run_dir)
559
+ history = _read_history(run_dir / "history.csv")
560
+ if not history:
561
+ history = _read_metrics_jsonl(run_dir / "metrics.jsonl")
562
+ prediction_rows = _read_predictions(run_dir)
563
+ split_rows, split_notes = _split_section(fingerprint)
564
+ model_facts, module_rows = _model_section(model_summary)
565
+ history_data = _history_section(history, summary)
566
+ evaluation_data = _evaluation_sections(evaluation, contract)
567
+ figures = _collect_figures(run_dir)
568
+ figure_groups: dict[str, list[dict[str, str]]] = {
569
+ "training": [],
570
+ "confusion": [],
571
+ "calibration": [],
572
+ "errors": [],
573
+ "other": [],
574
+ }
575
+ for figure in figures:
576
+ payload = {
577
+ "name": figure.name,
578
+ "caption": figure.caption,
579
+ "data_uri": figure.data_uri,
580
+ }
581
+ if "training_history" in figure.name:
582
+ figure_groups["training"].append(payload)
583
+ elif "confusion_matrix" in figure.name:
584
+ figure_groups["confusion"].append(payload)
585
+ elif "calibration" in figure.name:
586
+ figure_groups["calibration"].append(payload)
587
+ elif "error" in figure.name:
588
+ figure_groups["errors"].append(payload)
589
+ else:
590
+ figure_groups["other"].append(payload)
591
+
592
+ experiment = config.get("experiment") if config else None
593
+ run_name = (
594
+ str(experiment.get("name"))
595
+ if isinstance(experiment, dict) and experiment.get("name")
596
+ else run_dir.name
597
+ )
598
+ cards: list[dict[str, str]] = []
599
+ if evaluation_data["available"]:
600
+ metrics = evaluation.get("metrics") if evaluation else {}
601
+ if not isinstance(metrics, dict):
602
+ metrics = evaluation or {}
603
+ cards.append({"label": "Test accuracy", "value": _format_metric(metrics.get("accuracy"))})
604
+ cards.append({"label": "Macro F1", "value": _format_metric(metrics.get("macro_f1"))})
605
+ if evaluation_data["confidence"]:
606
+ cards.append({"label": "Accuracy interval", "value": evaluation_data["confidence"]})
607
+ if model_summary and model_summary.get("total_params") is not None:
608
+ cards.append({"label": "Parameters", "value": _format(model_summary["total_params"])})
609
+ if history_data["best_val_accuracy"] is not None:
610
+ cards.append(
611
+ {
612
+ "label": "Best validation accuracy",
613
+ "value": _format_metric(history_data["best_val_accuracy"]),
614
+ }
615
+ )
616
+ if evaluation_data["latency"]:
617
+ cards.append(
618
+ {
619
+ "label": "Latency",
620
+ "value": next(
621
+ row["value"] for row in evaluation_data["latency"] if "Mean" in row["label"]
622
+ )
623
+ + " ms",
624
+ }
625
+ )
626
+ cards = [card for card in cards if card["value"] not in {"—", ""}]
627
+
628
+ return {
629
+ "title": f"Experiment report — {run_name}",
630
+ "run_name": run_name,
631
+ "run_directory": (
632
+ str(run_dir.resolve().relative_to(root)) if root is not None else str(run_dir)
633
+ ),
634
+ "status": _format(status.get("status")) if status else "unknown",
635
+ "config_hash": _format(summary.get("config_hash")) if summary else "—",
636
+ "cards": cards,
637
+ "config_sections": _config_sections(config),
638
+ "contract_rows": _contract_rows(contract),
639
+ "split_rows": split_rows,
640
+ "split_notes": split_notes,
641
+ "model_facts": model_facts,
642
+ "module_rows": module_rows,
643
+ "history": history_data,
644
+ "evaluation": evaluation_data,
645
+ "figures": [
646
+ {"name": figure.name, "caption": figure.caption, "data_uri": figure.data_uri}
647
+ for figure in figures
648
+ ],
649
+ "figure_groups": figure_groups,
650
+ "prediction_rows": prediction_rows,
651
+ "runtime_rows": _runtime_rows(environment, summary, git),
652
+ "reproduction": _reproduction_commands(run_dir),
653
+ }
654
+
655
+
656
+ def build_markdown_report(run_dir: Path) -> str:
657
+ """Return a Markdown report with the same numbers as the HTML report."""
658
+ context = _build_context(run_dir)
659
+ lines: list[str] = [
660
+ f"# Experiment report — {context['run_name']}",
661
+ "",
662
+ f"- Run directory: `{context['run_directory']}`",
663
+ f"- Status: {context['status']}",
664
+ f"- Config hash: {context['config_hash']}",
665
+ "",
666
+ "## Metrics",
667
+ "",
668
+ ]
669
+ evaluation = context["evaluation"]
670
+ if evaluation["available"]:
671
+ lines.extend(["| Metric | Value |", "| --- | --- |"])
672
+ lines.extend(f"| {row['label']} | {row['value']} |" for row in evaluation["metrics"])
673
+ if evaluation["confidence"]:
674
+ lines.append(f"| Accuracy interval | {evaluation['confidence']} |")
675
+ else:
676
+ lines.append("Test metrics: not evaluated (no evaluation artifact found).")
677
+ if context["prediction_rows"]:
678
+ lines.extend(
679
+ [
680
+ "",
681
+ "| Prediction archive | Samples | Saved accuracy |",
682
+ "| --- | --- | --- |",
683
+ ]
684
+ )
685
+ lines.extend(
686
+ f"| {row['name']} | {row['samples']} | {row['accuracy']} |"
687
+ for row in context["prediction_rows"]
688
+ )
689
+ else:
690
+ lines.extend(["", "Saved predictions: not evaluated."])
691
+ lines.extend(["", "## Configuration", ""])
692
+ if context["config_sections"]:
693
+ for section in context["config_sections"]:
694
+ lines.append(f"### {section['title']}")
695
+ lines.append("")
696
+ lines.extend(f"- `{key}`: {value}" for key, value in section["rows"])
697
+ lines.append("")
698
+ else:
699
+ lines.append("Configuration: not evaluated (no resolved config found).")
700
+ lines.append("")
701
+ lines.extend(["## Data contract", ""])
702
+ if context["contract_rows"]:
703
+ lines.extend(f"- `{key}`: {value}" for key, value in context["contract_rows"])
704
+ else:
705
+ lines.append("Data contract: not evaluated.")
706
+ lines.extend(["", "## Split integrity", ""])
707
+ if context["split_rows"]:
708
+ lines.extend(["| Split | Count | Fingerprint |", "| --- | --- | --- |"])
709
+ lines.extend(
710
+ f"| {row['split']} | {row['count']} | {row['fingerprint']} |"
711
+ for row in context["split_rows"]
712
+ )
713
+ lines.extend(f"\n{note}" for note in context["split_notes"])
714
+ else:
715
+ lines.extend(context["split_notes"] or ["Split integrity: not evaluated."])
716
+ lines.extend(["", "## Model", ""])
717
+ if context["model_facts"]:
718
+ lines.extend(f"- {row['label']}: {row['value']}" for row in context["model_facts"])
719
+ if context["module_rows"]:
720
+ lines.extend(["", "| Module | Type | Parameters |", "| --- | --- | --- |"])
721
+ lines.extend(
722
+ f"| {row['name']} | {row['type']} | {row['params']} |"
723
+ for row in context["module_rows"]
724
+ )
725
+ else:
726
+ lines.append("Model summary: not evaluated.")
727
+ history = context["history"]
728
+ lines.extend(["", "## Training", ""])
729
+ if history["rows"]:
730
+ headers = [column["label"] for column in history["columns"]]
731
+ lines.append(f"Epochs run: {history['epochs_run']}")
732
+ if history["best_val_accuracy"] is not None:
733
+ lines.append(
734
+ f"Best validation accuracy: {_format_metric(history['best_val_accuracy'])} "
735
+ f"(epoch {_format(history['best_epoch'])})"
736
+ )
737
+ lines.append("")
738
+ lines.extend(
739
+ [
740
+ "| " + " | ".join(headers) + " |",
741
+ "| " + " | ".join("---" for _ in headers) + " |",
742
+ ]
743
+ )
744
+ lines.extend(
745
+ "| " + " | ".join(row[column["key"]] for column in history["columns"]) + " |"
746
+ for row in history["rows"]
747
+ )
748
+ else:
749
+ lines.append("Training history: not evaluated.")
750
+ lines.extend(["", "## Test metrics", ""])
751
+ if evaluation["per_class"]:
752
+ lines.extend(
753
+ ["| Class | Precision | Recall | F1 | Support |", "| --- | --- | --- | --- | --- |"]
754
+ )
755
+ lines.extend(
756
+ f"| {row['name']} | {row['precision']} | {row['recall']} | "
757
+ f"{row['f1']} | {row['support']} |"
758
+ for row in evaluation["per_class"]
759
+ )
760
+ else:
761
+ lines.append("Per-class metrics: not evaluated.")
762
+ lines.extend(["", "## Calibration and error analysis", ""])
763
+ calibration_figures = [
764
+ figure for figure in context["figures"] if "calibration" in figure["name"]
765
+ ]
766
+ error_figures = [figure for figure in context["figures"] if "error" in figure["name"]]
767
+ if calibration_figures:
768
+ lines.extend(
769
+ f"![{figure['caption']}](figures/{figure['name']})" for figure in calibration_figures
770
+ )
771
+ else:
772
+ lines.append("Calibration figure: not evaluated.")
773
+ if error_figures:
774
+ lines.extend(
775
+ f"![{figure['caption']}](figures/{figure['name']})" for figure in error_figures
776
+ )
777
+ else:
778
+ lines.append("Error gallery: not evaluated.")
779
+ lines.extend(["", "## Figures", ""])
780
+ other_figures = [
781
+ figure
782
+ for figure in context["figures"]
783
+ if not any(
784
+ marker in figure["name"]
785
+ for marker in ("training_history", "confusion_matrix", "calibration", "error")
786
+ )
787
+ ]
788
+ if other_figures:
789
+ lines.extend(
790
+ f"![{figure['caption']}](figures/{figure['name']})" for figure in other_figures
791
+ )
792
+ else:
793
+ lines.append("Additional figures: not evaluated.")
794
+ lines.extend(["", "## Runtime", ""])
795
+ if context["runtime_rows"]:
796
+ lines.extend(f"- {row['label']}: {row['value']}" for row in context["runtime_rows"])
797
+ else:
798
+ lines.append("Runtime: not evaluated.")
799
+ if evaluation["latency"]:
800
+ lines.extend(f"- {row['label']}: {row['value']}" for row in evaluation["latency"])
801
+ lines.extend(["", "## Reproduction", "", "```bash"])
802
+ lines.extend(context["reproduction"])
803
+ lines.extend(["```", ""])
804
+ return "\n".join(lines)
805
+
806
+
807
+ def build_report(run_dir: Path, output_path: Path | None = None) -> Path:
808
+ """Render a self-contained HTML report from saved run artifacts."""
809
+ context = _build_context(run_dir)
810
+ environment = Environment(
811
+ loader=FileSystemLoader(str(_TEMPLATE_DIR)),
812
+ autoescape=select_autoescape(["html", "xml"]),
813
+ trim_blocks=True,
814
+ lstrip_blocks=True,
815
+ )
816
+ template = environment.get_template(_TEMPLATE_NAME)
817
+ html = template.render(**context)
818
+ destination = output_path or (run_dir / "report.html")
819
+ destination.parent.mkdir(parents=True, exist_ok=True)
820
+ destination.write_text(html, encoding="utf-8")
821
+ return destination