syndset 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
syndset/__init__.py ADDED
@@ -0,0 +1,94 @@
1
+ """syndset: Diagnostic battery and synthetic evaluation toolkit for PyTorch neural architectures."""
2
+
3
+ __version__ = "0.2.0"
4
+
5
+ from syndset.audit import (
6
+ audit,
7
+ audit_detection,
8
+ audit_llm,
9
+ audit_segmentation,
10
+ audit_tabular,
11
+ audit_timeseries,
12
+ audit_vision,
13
+ )
14
+ from syndset.diagnostics import (
15
+ check_curvature_sharpness,
16
+ check_dead_units,
17
+ check_effective_rank,
18
+ check_gradient_flow,
19
+ check_gradient_snr,
20
+ check_initialization_scale,
21
+ check_numerical_stability,
22
+ check_overfit_capacity,
23
+ check_permutation_equivariance,
24
+ )
25
+ from syndset.domains import (
26
+ SHAPE_NAMES,
27
+ AssociativeRecallDataset,
28
+ AutoregressiveLagDataset,
29
+ ConcentricHyperspheresDataset,
30
+ CumulativeParityDataset,
31
+ DyckLanguageDataset,
32
+ HarmonicSuperpositionDataset,
33
+ IllConditionedRegressionDataset,
34
+ InductionDataset,
35
+ MarkovLanguageDataset,
36
+ MultiQueryAssociativeRecallDataset,
37
+ NumerosityDataset,
38
+ SelectiveCopyDataset,
39
+ SparseXORDataset,
40
+ SpatialInvarianceDataset,
41
+ SpatialRelationDataset,
42
+ SyntheticDataset,
43
+ SyntheticDetectionDataset,
44
+ SyntheticSegmentationDataset,
45
+ SyntheticShapesDataset,
46
+ TextureVsShapeDataset,
47
+ TwoSpiralsDataset,
48
+ rasterize_shape,
49
+ )
50
+ from syndset.report import AuditReport
51
+
52
+ __all__ = [
53
+ "__version__",
54
+ "audit",
55
+ "audit_llm",
56
+ "audit_detection",
57
+ "audit_segmentation",
58
+ "audit_vision",
59
+ "audit_tabular",
60
+ "audit_timeseries",
61
+ "AuditReport",
62
+ "check_gradient_flow",
63
+ "check_gradient_snr",
64
+ "check_overfit_capacity",
65
+ "check_effective_rank",
66
+ "check_dead_units",
67
+ "check_numerical_stability",
68
+ "check_initialization_scale",
69
+ "check_curvature_sharpness",
70
+ "check_permutation_equivariance",
71
+ "SyntheticDataset",
72
+ "AssociativeRecallDataset",
73
+ "MultiQueryAssociativeRecallDataset",
74
+ "InductionDataset",
75
+ "MarkovLanguageDataset",
76
+ "SelectiveCopyDataset",
77
+ "CumulativeParityDataset",
78
+ "DyckLanguageDataset",
79
+ "SyntheticShapesDataset",
80
+ "SyntheticSegmentationDataset",
81
+ "SyntheticDetectionDataset",
82
+ "NumerosityDataset",
83
+ "SpatialRelationDataset",
84
+ "TextureVsShapeDataset",
85
+ "SpatialInvarianceDataset",
86
+ "ConcentricHyperspheresDataset",
87
+ "SparseXORDataset",
88
+ "IllConditionedRegressionDataset",
89
+ "TwoSpiralsDataset",
90
+ "HarmonicSuperpositionDataset",
91
+ "AutoregressiveLagDataset",
92
+ "SHAPE_NAMES",
93
+ "rasterize_shape",
94
+ ]
syndset/audit.py ADDED
@@ -0,0 +1,424 @@
1
+ """Top-level architecture audit orchestrator."""
2
+
3
+ import torch.nn as nn
4
+
5
+ from syndset.diagnostics.capacity import check_overfit_capacity
6
+ from syndset.diagnostics.gradient import check_gradient_flow
7
+ from syndset.diagnostics.representation import check_dead_units, check_effective_rank
8
+ from syndset.diagnostics.stability import check_numerical_stability
9
+ from syndset.domains.llm import AssociativeRecallDataset
10
+ from syndset.domains.tabular import ConcentricHyperspheresDataset
11
+ from syndset.domains.timeseries import HarmonicSuperpositionDataset
12
+ from syndset.domains.vision import (
13
+ SyntheticDetectionDataset,
14
+ SyntheticSegmentationDataset,
15
+ SyntheticShapesDataset,
16
+ )
17
+ from syndset.report import AuditReport
18
+
19
+
20
+ def audit(
21
+ model,
22
+ sample_input,
23
+ sample_target=None,
24
+ loss_fn=None,
25
+ run_overfit=True,
26
+ run_rank=True,
27
+ run_stability=True,
28
+ model_name=None,
29
+ ):
30
+ """Runs a comprehensive battery of diagnostic health checks on a neural architecture.
31
+
32
+ Executes gradient flow verification, representation collapse checks,
33
+ numerical stability tests, and single-batch capacity tests in seconds.
34
+ Aggregates findings into an `AuditReport` scorecard.
35
+
36
+ Args:
37
+ model: The `torch.nn.Module` to evaluate.
38
+ sample_input: Sample tensor or dict/tuple matching model forward inputs.
39
+ Must have a batch dimension (e.g. shape `(B, ...)`).
40
+ sample_target: Optional target tensor corresponding to `sample_input`.
41
+ Required for overfit capacity probe when `run_overfit=True`.
42
+ loss_fn: Optional loss function callable `loss_fn(outputs, targets)`. If `None`,
43
+ defaults to cross-entropy for 1D integer targets or MSE loss otherwise.
44
+ run_overfit: Whether to run single-batch memorization test (default: True).
45
+ run_rank: Whether to compute effective rank and dead units (default: True).
46
+ run_stability: Whether to check half precision (FP16/BF16) and NaN stability (default: True).
47
+ model_name: Optional string name for the model in report summary. If `None`,
48
+ uses `model.__class__.__name__`.
49
+
50
+ Returns:
51
+ An `AuditReport` instance containing detailed findings and scorecard.
52
+ Call `report.print_summary()` to view findings or `report.is_healthy()` to inspect status.
53
+
54
+ Example:
55
+ >>> import torch
56
+ >>> import torch.nn as nn
57
+ >>> import syndset as syn
58
+ >>> model = nn.Sequential(nn.Linear(16, 32), nn.ReLU(), nn.Linear(32, 2))
59
+ >>> inputs = torch.randn(8, 16)
60
+ >>> targets = torch.randint(0, 2, (8,))
61
+ >>> report = syn.audit(model, inputs, targets, model_name="MLP")
62
+ >>> report.print_summary()
63
+ """
64
+ name = model_name or model.__class__.__name__
65
+ report = AuditReport(model_name=name)
66
+
67
+ # 1. Gradient Flow Probe
68
+ grad_res = check_gradient_flow(model, sample_input, target=sample_target, loss_fn=loss_fn)
69
+ num_zeros = len(grad_res["zero_grad_params"])
70
+ summary_text = (f"Norm: {grad_res['global_norm']:.3f}, "
71
+ f"Vanishing ratio: {grad_res['vanishing_ratio']:.2e}, "
72
+ f"Zero-grad params: {num_zeros}")
73
+ report.add_check("Gradient Flow", grad_res["status"], summary_text, grad_res["issues"])
74
+
75
+ # 2. Representation & Rank Probes
76
+ if run_rank:
77
+ rank_res = check_effective_rank(model, sample_input)
78
+ num_collapsed = len(rank_res["collapsed_layers"])
79
+ rank_summary = f"{len(rank_res['layer_ranks'])} layers monitored, {num_collapsed} collapsed"
80
+ report.add_check("Effective Rank", rank_res["status"], rank_summary, rank_res["issues"])
81
+
82
+ dead_res = check_dead_units(model, sample_input)
83
+ num_dead_layers = len(dead_res["problematic_layers"])
84
+ dead_summary = f"{num_dead_layers} layers with >30% dead units"
85
+ report.add_check("Dead Units", dead_res["status"], dead_summary, dead_res["issues"])
86
+
87
+ # 3. Numerical Stability Probe
88
+ if run_stability:
89
+ stab_res = check_numerical_stability(model, sample_input)
90
+ half_err = stab_res["half_precision_relative_error"]
91
+ err_str = f"{half_err:.2%}" if half_err is not None else "N/A"
92
+ stab_summary = f"Max act: {stab_res['max_abs_activation']:.2f}, Half diff: {err_str}"
93
+ report.add_check("Numerical Stability", stab_res["status"], stab_summary, stab_res["issues"])
94
+
95
+ # 4. Capacity & Single-Batch Overfit Probe
96
+ if run_overfit and sample_target is not None:
97
+ cap_res = check_overfit_capacity(model, sample_input, sample_target, loss_fn=loss_fn)
98
+ steps_val = cap_res["steps_to_converge"]
99
+ steps_str = f"in {steps_val} steps" if cap_res["converged"] else "not converged"
100
+ cap_summary = f"Final loss: {cap_res['final_loss']:.4f} ({steps_str})"
101
+ report.add_check("Overfit Capacity", cap_res["status"], cap_summary, cap_res["issues"])
102
+
103
+ return report
104
+
105
+
106
+ def audit_llm(model, num_pairs=8, vocab_size=64, batch_size=8, seed=42):
107
+ """Audits an autoregressive or sequence model on synthetic associative recall.
108
+
109
+ Instantiates an `AssociativeRecallDataset` and runs diagnostic probes (gradient flow,
110
+ effective rank, dead units, numerical stability, and single-batch capacity).
111
+
112
+ Args:
113
+ model: The sequence `torch.nn.Module` to evaluate. Must accept input token ID
114
+ tensors of shape `(batch_size, 2 * num_pairs + 1)` with dtype `torch.long`.
115
+ num_pairs: Number of distinct (key, value) pairs per sequence (default: 8).
116
+ vocab_size: Total vocabulary size partitioned between keys and values (default: 64).
117
+ Must satisfy `vocab_size >= 2 * num_pairs`.
118
+ batch_size: Batch size of synthetic sequences to audit (default: 8).
119
+ seed: Random seed for reproducible generation (default: 42).
120
+
121
+ Returns:
122
+ An `AuditReport` summarizing model health on the sequence task.
123
+
124
+ Example:
125
+ >>> import torch.nn as nn
126
+ >>> import syndset as syn
127
+ >>> class SimpleSeqModel(nn.Module):
128
+ ... def __init__(self):
129
+ ... super().__init__()
130
+ ... self.emb = nn.Embedding(64, 32)
131
+ ... self.gru = nn.GRU(32, 32, batch_first=True)
132
+ ... self.head = nn.Linear(32, 64)
133
+ ... def forward(self, x):
134
+ ... out, _ = self.gru(self.emb(x))
135
+ ... return self.head(out[:, -1, :])
136
+ >>> model = SimpleSeqModel()
137
+ >>> report = syn.audit_llm(model, num_pairs=8, vocab_size=64)
138
+ >>> report.print_summary()
139
+ """
140
+ dataset = AssociativeRecallDataset(num_samples=batch_size,
141
+ num_pairs=num_pairs,
142
+ vocab_size=vocab_size,
143
+ seed=seed)
144
+ inputs, targets = dataset[:batch_size]
145
+ return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (LLM Task)")
146
+
147
+
148
+ def audit_vision(model, img_size=32, channels=1, num_classes=None, batch_size=8, seed=42):
149
+ """Audits a vision classifier architecture on procedural geometric shapes.
150
+
151
+ Instantiates a `SyntheticShapesDataset` across geometric classes (circles,
152
+ squares, triangles, crosses, rectangles, ellipses, annuli, diamonds) and executes
153
+ gradient flow, representation rank, numerical stability, and overfit probes.
154
+
155
+ Args:
156
+ model: The vision `torch.nn.Module` to evaluate. Accepts image tensors of shape
157
+ `(batch_size, channels, img_size, img_size)` and outputs logits of shape
158
+ `(batch_size, num_classes)`.
159
+ img_size: Image height and width in pixels (default: 32).
160
+ channels: Image channels (default: 1).
161
+ num_classes: Number of shape classes in `[1, 8]`. If `None`, attempts to infer
162
+ from `model.num_classes` or `model.classifier.out_features`, defaulting to 8.
163
+ batch_size: Batch size for audit (default: 8).
164
+ seed: Random seed for reproducible generation (default: 42).
165
+
166
+ Returns:
167
+ An `AuditReport` summarizing model health on synthetic vision inputs.
168
+
169
+ Example:
170
+ >>> import torch.nn as nn
171
+ >>> import syndset as syn
172
+ >>> model = nn.Sequential(
173
+ ... nn.Conv2d(1, 16, 3, padding=1),
174
+ ... nn.ReLU(),
175
+ ... nn.AdaptiveAvgPool2d((1, 1)),
176
+ ... nn.Flatten(),
177
+ ... nn.Linear(16, 8)
178
+ ... )
179
+ >>> report = syn.audit_vision(model, img_size=32, channels=1, num_classes=8)
180
+ >>> report.print_summary()
181
+ """
182
+ if num_classes is None:
183
+ num_classes = getattr(model, "num_classes", None)
184
+ classifier = getattr(model, "classifier", None)
185
+ if num_classes is None and classifier is not None and hasattr(classifier, "out_features"):
186
+ num_classes = classifier.out_features
187
+ if num_classes is None:
188
+ num_classes = 8
189
+
190
+ dataset = SyntheticShapesDataset(num_samples=batch_size,
191
+ img_size=img_size,
192
+ channels=channels,
193
+ num_classes=num_classes,
194
+ seed=seed)
195
+ inputs, targets = dataset[:batch_size]
196
+ return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (Vision Task)")
197
+
198
+
199
+ def audit_tabular(model, dim=16, num_classes=3, batch_size=16, seed=42):
200
+ """Audits a feedforward / tabular architecture on concentric hyperspheres.
201
+
202
+ Instantiates a `ConcentricHyperspheresDataset` to evaluate non-linear representation
203
+ capacity, gradient flow, and numerical stability on non-linearly separable manifold shells.
204
+
205
+ Args:
206
+ model: The tabular `torch.nn.Module` to evaluate. Accepts float feature tensors
207
+ of shape `(batch_size, dim)` and outputs logits of shape `(batch_size, num_classes)`.
208
+ dim: Dimensionality of input feature vectors (default: 16).
209
+ num_classes: Number of concentric spherical shells (default: 3).
210
+ batch_size: Batch size for audit (default: 16).
211
+ seed: Random seed for reproducible generation (default: 42).
212
+
213
+ Returns:
214
+ An `AuditReport` summarizing model health on non-linear manifold inputs.
215
+
216
+ Example:
217
+ >>> import torch.nn as nn
218
+ >>> import syndset as syn
219
+ >>> model = nn.Sequential(
220
+ ... nn.Linear(16, 32),
221
+ ... nn.ReLU(),
222
+ ... nn.Linear(32, 3)
223
+ ... )
224
+ >>> report = syn.audit_tabular(model, dim=16, num_classes=3)
225
+ >>> report.print_summary()
226
+ """
227
+ dataset = ConcentricHyperspheresDataset(num_samples=batch_size,
228
+ num_classes=num_classes,
229
+ dim=dim,
230
+ seed=seed)
231
+ inputs, targets = dataset[:batch_size]
232
+ return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (Tabular Task)")
233
+
234
+
235
+ def audit_timeseries(model, seq_len=32, batch_size=8, seed=42):
236
+ """Audits a time series or recurrent architecture on harmonic waveforms.
237
+
238
+ Instantiates a `HarmonicSuperpositionDataset` with multi-frequency Fourier sums
239
+ and checks whether the model preserves gradient flow, avoids representation collapse,
240
+ and fits temporal sequences.
241
+
242
+ Args:
243
+ model: The time-series `torch.nn.Module` to evaluate. Accepts input tensors
244
+ of shape `(batch_size, seq_len - 1, 1)` and outputs forecasted tensors of
245
+ seq_len: Number of time steps in generated waveforms (default: 32).
246
+ Model input sequence length is `seq_len - 1`.
247
+ batch_size: Batch size for audit (default: 8).
248
+ seed: Random seed for reproducible generation (default: 42).
249
+
250
+ Returns:
251
+ An `AuditReport` summarizing model health on spectral wave inputs.
252
+
253
+ Example:
254
+ >>> import torch.nn as nn
255
+ >>> import syndset as syn
256
+ >>> class SimpleForecaster(nn.Module):
257
+ ... def __init__(self):
258
+ ... super().__init__()
259
+ ... self.rnn = nn.GRU(1, 16, batch_first=True)
260
+ ... self.head = nn.Linear(16, 1)
261
+ ... def forward(self, x):
262
+ ... out, _ = self.rnn(x)
263
+ ... return self.head(out)
264
+ >>> model = SimpleForecaster()
265
+ >>> report = syn.audit_timeseries(model, seq_len=32)
266
+ >>> report.print_summary()
267
+ """
268
+ dataset = HarmonicSuperpositionDataset(num_samples=batch_size, seq_len=seq_len, seed=seed)
269
+ inputs, targets = dataset[:batch_size]
270
+ return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (Time Series Task)")
271
+
272
+
273
+ def audit_segmentation(
274
+ model,
275
+ img_size=32,
276
+ channels=1,
277
+ num_classes=None,
278
+ num_objects=3,
279
+ batch_size=8,
280
+ seed=42,
281
+ ):
282
+ """Audits a dense segmentation model (e.g. U-Net, FPN) on procedural pixel masks.
283
+
284
+ Instantiates a `SyntheticSegmentationDataset` and evaluates encoder-decoder gradient
285
+ propagation, representation rank across bottleneck and skip layers, numerical stability,
286
+ and single-batch pixel-level overfit capacity using `nn.CrossEntropyLoss`.
287
+
288
+ Args:
289
+ model: The segmentation `torch.nn.Module` to evaluate. Must accept inputs of shape
290
+ `(batch_size, channels, img_size, img_size)` and output logits of shape
291
+ `(batch_size, num_classes + 1, img_size, img_size)`.
292
+ img_size: Image height and width in pixels (default: 32).
293
+ num_classes: Number of foreground shape classes in `[1, 8]`
294
+ (default: `model.out_channels - 1` or 8).
295
+ Total segmentation classes will be `num_classes + 1` (class 0 is background).
296
+ num_objects: Number of shapes placed per image canvas (default: 3).
297
+ batch_size: Batch size for audit (default: 8).
298
+ seed: Random seed for reproducible generation (default: 42).
299
+
300
+ Returns:
301
+ An `AuditReport` summarizing model health on the semantic segmentation task.
302
+
303
+ Example:
304
+ >>> import torch.nn as nn
305
+ >>> import syndset as syn
306
+ >>> class SimpleUNet(nn.Module):
307
+ ... def __init__(self, in_c=1, out_c=9):
308
+ ... super().__init__()
309
+ ... self.enc = nn.Conv2d(in_c, 16, 3, padding=1)
310
+ ... self.dec = nn.Conv2d(16, out_c, 3, padding=1)
311
+ ... def forward(self, x):
312
+ ... return self.dec(nn.functional.relu(self.enc(x)))
313
+ >>> model = SimpleUNet()
314
+ >>> report = syn.audit_segmentation(model, img_size=32, channels=1, num_classes=8)
315
+ >>> report.print_summary()
316
+ """
317
+ if num_classes is None:
318
+ num_classes = getattr(model, "num_classes", None)
319
+ if num_classes is None:
320
+ out_channels = getattr(model, "out_channels", None)
321
+ if out_channels is not None:
322
+ num_classes = max(1, out_channels - 1)
323
+ if num_classes is None:
324
+ num_classes = 8
325
+
326
+ dataset = SyntheticSegmentationDataset(
327
+ num_samples=batch_size,
328
+ img_size=img_size,
329
+ channels=channels,
330
+ num_classes=num_classes,
331
+ num_objects=num_objects,
332
+ seed=seed,
333
+ )
334
+ inputs, targets = dataset[:batch_size]
335
+ return audit(
336
+ model,
337
+ inputs,
338
+ targets,
339
+ loss_fn=nn.CrossEntropyLoss(),
340
+ model_name=f"{model.__class__.__name__} (Segmentation Task)",
341
+ )
342
+
343
+
344
+ def audit_detection(
345
+ model,
346
+ img_size=32,
347
+ channels=1,
348
+ num_objects=3,
349
+ num_classes=8,
350
+ batch_size=8,
351
+ loss_fn=None,
352
+ seed=42,
353
+ ):
354
+ """Audits an object detection or coordinate regression model on procedural boxes.
355
+
356
+ Instantiates a `SyntheticDetectionDataset` and evaluates coordinate regression
357
+ gradient propagation, representation rank across backbone/pyramid layers, numerical
358
+ stability, and single-batch bounding box overfit capacity.
359
+
360
+ Args:
361
+ model: The detector `torch.nn.Module` to evaluate. Accepts input tensors of shape
362
+ `(batch_size, channels, img_size, img_size)` and must output predicted bounding
363
+ boxes of shape `(batch_size, num_objects, 4)` in normalized `[cx, cy, w, h]` format
364
+ (or tuple/dict containing box predictions).
365
+ img_size: Image height and width in pixels (default: 32).
366
+ channels: Image channels (default: 1).
367
+ num_objects: Number of objects per image (default: 3).
368
+ num_classes: Number of geometric shape classes in `[1, 8]` (default: 8).
369
+ batch_size: Batch size for audit (default: 8).
370
+ loss_fn: Optional loss function callable `loss_fn(preds, targets)`. If `None`,
371
+ defaults to Smooth L1 loss on normalized box coordinates.
372
+ seed: Random seed for reproducible generation (default: 42).
373
+
374
+ Returns:
375
+ An `AuditReport` summarizing model health on the object detection task.
376
+
377
+ Example:
378
+ >>> import torch.nn as nn
379
+ >>> import syndset as syn
380
+ >>> class SimpleDetector(nn.Module):
381
+ ... def __init__(self, num_objects=3):
382
+ ... super().__init__()
383
+ ... self.conv = nn.Conv2d(1, 16, 3, padding=1)
384
+ ... self.pool = nn.AdaptiveAvgPool2d((1, 1))
385
+ ... self.head = nn.Linear(16, num_objects * 4)
386
+ ... self.num_objects = num_objects
387
+ ... def forward(self, x):
388
+ ... h = self.pool(nn.functional.relu(self.conv(x))).flatten(1)
389
+ ... return torch.sigmoid(self.head(h)).view(-1, self.num_objects, 4)
390
+ >>> model = SimpleDetector()
391
+ >>> report = syn.audit_detection(model, img_size=32, channels=1, num_objects=3)
392
+ >>> report.print_summary()
393
+ """
394
+ dataset = SyntheticDetectionDataset(
395
+ num_samples=batch_size,
396
+ img_size=img_size,
397
+ channels=channels,
398
+ num_classes=num_classes,
399
+ num_objects=num_objects,
400
+ seed=seed,
401
+ )
402
+ inputs = dataset.images[:batch_size]
403
+ target_boxes = dataset.boxes[:batch_size]
404
+
405
+ if loss_fn is None:
406
+
407
+ def default_detection_loss(preds, targets):
408
+ if isinstance(preds, (tuple, list)):
409
+ preds = preds[0]
410
+ elif isinstance(preds, dict):
411
+ preds = preds.get("boxes", preds.get("pred_boxes", next(iter(preds.values()))))
412
+ if preds.dim() == 2 and preds.shape[1] == targets.numel() // targets.shape[0]:
413
+ preds = preds.view_as(targets)
414
+ return nn.functional.smooth_l1_loss(preds, targets)
415
+
416
+ loss_fn = default_detection_loss
417
+
418
+ return audit(
419
+ model,
420
+ inputs,
421
+ target_boxes,
422
+ loss_fn=loss_fn,
423
+ model_name=f"{model.__class__.__name__} (Detection Task)",
424
+ )
@@ -0,0 +1,21 @@
1
+ """Architecture diagnostics and health verification probes."""
2
+
3
+ from syndset.diagnostics.capacity import check_overfit_capacity
4
+ from syndset.diagnostics.curvature import check_curvature_sharpness
5
+ from syndset.diagnostics.gradient import check_gradient_flow, check_gradient_snr
6
+ from syndset.diagnostics.initialization import check_initialization_scale
7
+ from syndset.diagnostics.permutation import check_permutation_equivariance
8
+ from syndset.diagnostics.representation import check_dead_units, check_effective_rank
9
+ from syndset.diagnostics.stability import check_numerical_stability
10
+
11
+ __all__ = [
12
+ "check_overfit_capacity",
13
+ "check_gradient_flow",
14
+ "check_gradient_snr",
15
+ "check_dead_units",
16
+ "check_effective_rank",
17
+ "check_numerical_stability",
18
+ "check_initialization_scale",
19
+ "check_curvature_sharpness",
20
+ "check_permutation_equivariance",
21
+ ]
@@ -0,0 +1,151 @@
1
+ """Capacity and single-batch memorization probes for neural networks."""
2
+
3
+ import copy
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+ import torch.optim as optim
8
+
9
+
10
+ def _extract_primary_tensor(output):
11
+ """Extracts the primary floating-point tensor from various model return structures.
12
+
13
+ Args:
14
+ output: Model output, which could be a Tensor, tuple/list, or object with .logits.
15
+
16
+ Returns:
17
+ A torch.Tensor if found, or None.
18
+ """
19
+ if isinstance(output, torch.Tensor):
20
+ return output
21
+ if isinstance(output, (tuple, list)) and len(output) > 0 and isinstance(output[0], torch.Tensor):
22
+ return output[0]
23
+ if hasattr(output, "logits"):
24
+ return output.logits
25
+ return None
26
+
27
+
28
+ def check_overfit_capacity(
29
+ model,
30
+ inputs,
31
+ targets,
32
+ loss_fn=None,
33
+ num_steps=100,
34
+ learning_rate=1e-3,
35
+ target_loss=1e-3,
36
+ optimizer_cls=optim.AdamW,
37
+ ):
38
+ """Tests whether the model has enough parameter capacity to memorize a single batch.
39
+
40
+ In deep learning engineering, a model that cannot drive loss on a small batch
41
+ to near zero almost always has a structural defect (e.g. Broken skip connections,
42
+ mismatched activation scales, or disconnected parameters).
43
+
44
+ This test works on an isolated deep copy of the model weights so the original
45
+ model parameters remain completely untouched.
46
+
47
+ Args:
48
+ model: The torch.nn.Module to evaluate.
49
+ inputs: A torch.Tensor or tuple/dict of tensors for the model.
50
+ targets: A torch.Tensor representing the expected target labels or values.
51
+ loss_fn: Optional loss function. If None, automatically selects CrossEntropyLoss
52
+ for integer labels or MSELoss for float targets.
53
+ num_steps: Maximum number of optimization steps to attempt (default: 100).
54
+ learning_rate: Learning rate for the optimizer (default: 1e-3).
55
+ target_loss: The loss threshold below which the model is deemed to have
56
+ successfully memorized the batch (default: 1e-3).
57
+ optimizer_cls: Optimizer class to use (default: torch.optim.AdamW).
58
+
59
+ Returns:
60
+ A dictionary containing:
61
+ - 'status': One of 'passed', 'slow', or 'failed'.
62
+ - 'converged': Boolean indicating if target_loss was achieved.
63
+ - 'steps_to_converge': The step index where target_loss was reached, or None.
64
+ - 'initial_loss': Loss at step 0.
65
+ - 'final_loss': Loss at the final optimization step.
66
+ - 'loss_history': List of loss values recorded across steps.
67
+ - 'issues': Human-readable notes or warnings.
68
+ """
69
+ # Select default loss function if omitted
70
+ if loss_fn is None:
71
+ if targets.dtype in (torch.long, torch.int64):
72
+ loss_fn = nn.CrossEntropyLoss()
73
+ else:
74
+ loss_fn = nn.MSELoss()
75
+
76
+ # Create an isolated clone of the model to avoid mutating caller's weights
77
+ cloned_model = copy.deepcopy(model)
78
+ cloned_model.train()
79
+
80
+ optimizer = optimizer_cls(cloned_model.parameters(), lr=learning_rate)
81
+ loss_history = []
82
+ converged = False
83
+ steps_to_converge = None
84
+
85
+ for step in range(num_steps):
86
+ optimizer.zero_grad()
87
+
88
+ if isinstance(inputs, dict):
89
+ outputs = cloned_model(**inputs)
90
+ elif isinstance(inputs, (tuple, list)):
91
+ outputs = cloned_model(*inputs)
92
+ else:
93
+ outputs = cloned_model(inputs)
94
+
95
+ pred = _extract_primary_tensor(outputs)
96
+ loss = loss_fn(pred if pred is not None else outputs, targets)
97
+ loss_val = float(loss.item())
98
+ loss_history.append(loss_val)
99
+
100
+ if torch.isnan(loss) or torch.isinf(loss):
101
+ return {
102
+ "status": "failed",
103
+ "converged": False,
104
+ "steps_to_converge": None,
105
+ "initial_loss": loss_history[0],
106
+ "final_loss": loss_val,
107
+ "loss_history": loss_history,
108
+ "issues": [f"Loss exploded to {loss_val} at step {step}."],
109
+ }
110
+
111
+ if loss_val <= target_loss and not converged:
112
+ converged = True
113
+ steps_to_converge = step + 1
114
+ break
115
+
116
+ loss.backward()
117
+ optimizer.step()
118
+
119
+ initial_loss = loss_history[0]
120
+ final_loss = loss_history[-1]
121
+ issues = []
122
+
123
+ if converged:
124
+ if steps_to_converge <= num_steps // 3:
125
+ status = "passed"
126
+ else:
127
+ status = "slow"
128
+ issues.append(f"Memorized batch slowly (took {steps_to_converge}/{num_steps} steps). "
129
+ "Check learning rate or gradient scaling.")
130
+ else:
131
+ # Check if loss at least improved
132
+ loss_ratio = final_loss / (initial_loss + 1e-10)
133
+ if loss_ratio > 0.8:
134
+ status = "failed"
135
+ issues.append(
136
+ f"Failed to memorize batch (final {final_loss:.4f} vs initial {initial_loss:.4f}). "
137
+ "The model may lack capacity or have dead forward pathways.")
138
+ else:
139
+ status = "slow"
140
+ issues.append(f"Loss decreased from {initial_loss:.4f} to {final_loss:.4f} but did not reach "
141
+ f"the target threshold of {target_loss} within {num_steps} steps.")
142
+
143
+ return {
144
+ "status": status,
145
+ "converged": converged,
146
+ "steps_to_converge": steps_to_converge,
147
+ "initial_loss": initial_loss,
148
+ "final_loss": final_loss,
149
+ "loss_history": loss_history,
150
+ "issues": issues,
151
+ }