syndset 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syndset/__init__.py +94 -0
- syndset/audit.py +424 -0
- syndset/diagnostics/__init__.py +21 -0
- syndset/diagnostics/capacity.py +151 -0
- syndset/diagnostics/curvature.py +140 -0
- syndset/diagnostics/gradient.py +228 -0
- syndset/diagnostics/initialization.py +112 -0
- syndset/diagnostics/permutation.py +123 -0
- syndset/diagnostics/representation.py +184 -0
- syndset/diagnostics/stability.py +179 -0
- syndset/domains/__init__.py +59 -0
- syndset/domains/base.py +50 -0
- syndset/domains/llm.py +1044 -0
- syndset/domains/tabular.py +457 -0
- syndset/domains/timeseries.py +242 -0
- syndset/domains/vision.py +1284 -0
- syndset/report.py +101 -0
- syndset/utils/__init__.py +1 -0
- syndset/utils/hooks.py +145 -0
- syndset/utils/stats.py +121 -0
- syndset-0.2.0.dist-info/METADATA +2043 -0
- syndset-0.2.0.dist-info/RECORD +25 -0
- syndset-0.2.0.dist-info/WHEEL +5 -0
- syndset-0.2.0.dist-info/licenses/LICENSE +21 -0
- syndset-0.2.0.dist-info/top_level.txt +1 -0
syndset/__init__.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""syndset: Diagnostic battery and synthetic evaluation toolkit for PyTorch neural architectures."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.2.0"
|
|
4
|
+
|
|
5
|
+
from syndset.audit import (
|
|
6
|
+
audit,
|
|
7
|
+
audit_detection,
|
|
8
|
+
audit_llm,
|
|
9
|
+
audit_segmentation,
|
|
10
|
+
audit_tabular,
|
|
11
|
+
audit_timeseries,
|
|
12
|
+
audit_vision,
|
|
13
|
+
)
|
|
14
|
+
from syndset.diagnostics import (
|
|
15
|
+
check_curvature_sharpness,
|
|
16
|
+
check_dead_units,
|
|
17
|
+
check_effective_rank,
|
|
18
|
+
check_gradient_flow,
|
|
19
|
+
check_gradient_snr,
|
|
20
|
+
check_initialization_scale,
|
|
21
|
+
check_numerical_stability,
|
|
22
|
+
check_overfit_capacity,
|
|
23
|
+
check_permutation_equivariance,
|
|
24
|
+
)
|
|
25
|
+
from syndset.domains import (
|
|
26
|
+
SHAPE_NAMES,
|
|
27
|
+
AssociativeRecallDataset,
|
|
28
|
+
AutoregressiveLagDataset,
|
|
29
|
+
ConcentricHyperspheresDataset,
|
|
30
|
+
CumulativeParityDataset,
|
|
31
|
+
DyckLanguageDataset,
|
|
32
|
+
HarmonicSuperpositionDataset,
|
|
33
|
+
IllConditionedRegressionDataset,
|
|
34
|
+
InductionDataset,
|
|
35
|
+
MarkovLanguageDataset,
|
|
36
|
+
MultiQueryAssociativeRecallDataset,
|
|
37
|
+
NumerosityDataset,
|
|
38
|
+
SelectiveCopyDataset,
|
|
39
|
+
SparseXORDataset,
|
|
40
|
+
SpatialInvarianceDataset,
|
|
41
|
+
SpatialRelationDataset,
|
|
42
|
+
SyntheticDataset,
|
|
43
|
+
SyntheticDetectionDataset,
|
|
44
|
+
SyntheticSegmentationDataset,
|
|
45
|
+
SyntheticShapesDataset,
|
|
46
|
+
TextureVsShapeDataset,
|
|
47
|
+
TwoSpiralsDataset,
|
|
48
|
+
rasterize_shape,
|
|
49
|
+
)
|
|
50
|
+
from syndset.report import AuditReport
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"__version__",
|
|
54
|
+
"audit",
|
|
55
|
+
"audit_llm",
|
|
56
|
+
"audit_detection",
|
|
57
|
+
"audit_segmentation",
|
|
58
|
+
"audit_vision",
|
|
59
|
+
"audit_tabular",
|
|
60
|
+
"audit_timeseries",
|
|
61
|
+
"AuditReport",
|
|
62
|
+
"check_gradient_flow",
|
|
63
|
+
"check_gradient_snr",
|
|
64
|
+
"check_overfit_capacity",
|
|
65
|
+
"check_effective_rank",
|
|
66
|
+
"check_dead_units",
|
|
67
|
+
"check_numerical_stability",
|
|
68
|
+
"check_initialization_scale",
|
|
69
|
+
"check_curvature_sharpness",
|
|
70
|
+
"check_permutation_equivariance",
|
|
71
|
+
"SyntheticDataset",
|
|
72
|
+
"AssociativeRecallDataset",
|
|
73
|
+
"MultiQueryAssociativeRecallDataset",
|
|
74
|
+
"InductionDataset",
|
|
75
|
+
"MarkovLanguageDataset",
|
|
76
|
+
"SelectiveCopyDataset",
|
|
77
|
+
"CumulativeParityDataset",
|
|
78
|
+
"DyckLanguageDataset",
|
|
79
|
+
"SyntheticShapesDataset",
|
|
80
|
+
"SyntheticSegmentationDataset",
|
|
81
|
+
"SyntheticDetectionDataset",
|
|
82
|
+
"NumerosityDataset",
|
|
83
|
+
"SpatialRelationDataset",
|
|
84
|
+
"TextureVsShapeDataset",
|
|
85
|
+
"SpatialInvarianceDataset",
|
|
86
|
+
"ConcentricHyperspheresDataset",
|
|
87
|
+
"SparseXORDataset",
|
|
88
|
+
"IllConditionedRegressionDataset",
|
|
89
|
+
"TwoSpiralsDataset",
|
|
90
|
+
"HarmonicSuperpositionDataset",
|
|
91
|
+
"AutoregressiveLagDataset",
|
|
92
|
+
"SHAPE_NAMES",
|
|
93
|
+
"rasterize_shape",
|
|
94
|
+
]
|
syndset/audit.py
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
"""Top-level architecture audit orchestrator."""
|
|
2
|
+
|
|
3
|
+
import torch.nn as nn
|
|
4
|
+
|
|
5
|
+
from syndset.diagnostics.capacity import check_overfit_capacity
|
|
6
|
+
from syndset.diagnostics.gradient import check_gradient_flow
|
|
7
|
+
from syndset.diagnostics.representation import check_dead_units, check_effective_rank
|
|
8
|
+
from syndset.diagnostics.stability import check_numerical_stability
|
|
9
|
+
from syndset.domains.llm import AssociativeRecallDataset
|
|
10
|
+
from syndset.domains.tabular import ConcentricHyperspheresDataset
|
|
11
|
+
from syndset.domains.timeseries import HarmonicSuperpositionDataset
|
|
12
|
+
from syndset.domains.vision import (
|
|
13
|
+
SyntheticDetectionDataset,
|
|
14
|
+
SyntheticSegmentationDataset,
|
|
15
|
+
SyntheticShapesDataset,
|
|
16
|
+
)
|
|
17
|
+
from syndset.report import AuditReport
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def audit(
|
|
21
|
+
model,
|
|
22
|
+
sample_input,
|
|
23
|
+
sample_target=None,
|
|
24
|
+
loss_fn=None,
|
|
25
|
+
run_overfit=True,
|
|
26
|
+
run_rank=True,
|
|
27
|
+
run_stability=True,
|
|
28
|
+
model_name=None,
|
|
29
|
+
):
|
|
30
|
+
"""Runs a comprehensive battery of diagnostic health checks on a neural architecture.
|
|
31
|
+
|
|
32
|
+
Executes gradient flow verification, representation collapse checks,
|
|
33
|
+
numerical stability tests, and single-batch capacity tests in seconds.
|
|
34
|
+
Aggregates findings into an `AuditReport` scorecard.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
model: The `torch.nn.Module` to evaluate.
|
|
38
|
+
sample_input: Sample tensor or dict/tuple matching model forward inputs.
|
|
39
|
+
Must have a batch dimension (e.g. shape `(B, ...)`).
|
|
40
|
+
sample_target: Optional target tensor corresponding to `sample_input`.
|
|
41
|
+
Required for overfit capacity probe when `run_overfit=True`.
|
|
42
|
+
loss_fn: Optional loss function callable `loss_fn(outputs, targets)`. If `None`,
|
|
43
|
+
defaults to cross-entropy for 1D integer targets or MSE loss otherwise.
|
|
44
|
+
run_overfit: Whether to run single-batch memorization test (default: True).
|
|
45
|
+
run_rank: Whether to compute effective rank and dead units (default: True).
|
|
46
|
+
run_stability: Whether to check half precision (FP16/BF16) and NaN stability (default: True).
|
|
47
|
+
model_name: Optional string name for the model in report summary. If `None`,
|
|
48
|
+
uses `model.__class__.__name__`.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
An `AuditReport` instance containing detailed findings and scorecard.
|
|
52
|
+
Call `report.print_summary()` to view findings or `report.is_healthy()` to inspect status.
|
|
53
|
+
|
|
54
|
+
Example:
|
|
55
|
+
>>> import torch
|
|
56
|
+
>>> import torch.nn as nn
|
|
57
|
+
>>> import syndset as syn
|
|
58
|
+
>>> model = nn.Sequential(nn.Linear(16, 32), nn.ReLU(), nn.Linear(32, 2))
|
|
59
|
+
>>> inputs = torch.randn(8, 16)
|
|
60
|
+
>>> targets = torch.randint(0, 2, (8,))
|
|
61
|
+
>>> report = syn.audit(model, inputs, targets, model_name="MLP")
|
|
62
|
+
>>> report.print_summary()
|
|
63
|
+
"""
|
|
64
|
+
name = model_name or model.__class__.__name__
|
|
65
|
+
report = AuditReport(model_name=name)
|
|
66
|
+
|
|
67
|
+
# 1. Gradient Flow Probe
|
|
68
|
+
grad_res = check_gradient_flow(model, sample_input, target=sample_target, loss_fn=loss_fn)
|
|
69
|
+
num_zeros = len(grad_res["zero_grad_params"])
|
|
70
|
+
summary_text = (f"Norm: {grad_res['global_norm']:.3f}, "
|
|
71
|
+
f"Vanishing ratio: {grad_res['vanishing_ratio']:.2e}, "
|
|
72
|
+
f"Zero-grad params: {num_zeros}")
|
|
73
|
+
report.add_check("Gradient Flow", grad_res["status"], summary_text, grad_res["issues"])
|
|
74
|
+
|
|
75
|
+
# 2. Representation & Rank Probes
|
|
76
|
+
if run_rank:
|
|
77
|
+
rank_res = check_effective_rank(model, sample_input)
|
|
78
|
+
num_collapsed = len(rank_res["collapsed_layers"])
|
|
79
|
+
rank_summary = f"{len(rank_res['layer_ranks'])} layers monitored, {num_collapsed} collapsed"
|
|
80
|
+
report.add_check("Effective Rank", rank_res["status"], rank_summary, rank_res["issues"])
|
|
81
|
+
|
|
82
|
+
dead_res = check_dead_units(model, sample_input)
|
|
83
|
+
num_dead_layers = len(dead_res["problematic_layers"])
|
|
84
|
+
dead_summary = f"{num_dead_layers} layers with >30% dead units"
|
|
85
|
+
report.add_check("Dead Units", dead_res["status"], dead_summary, dead_res["issues"])
|
|
86
|
+
|
|
87
|
+
# 3. Numerical Stability Probe
|
|
88
|
+
if run_stability:
|
|
89
|
+
stab_res = check_numerical_stability(model, sample_input)
|
|
90
|
+
half_err = stab_res["half_precision_relative_error"]
|
|
91
|
+
err_str = f"{half_err:.2%}" if half_err is not None else "N/A"
|
|
92
|
+
stab_summary = f"Max act: {stab_res['max_abs_activation']:.2f}, Half diff: {err_str}"
|
|
93
|
+
report.add_check("Numerical Stability", stab_res["status"], stab_summary, stab_res["issues"])
|
|
94
|
+
|
|
95
|
+
# 4. Capacity & Single-Batch Overfit Probe
|
|
96
|
+
if run_overfit and sample_target is not None:
|
|
97
|
+
cap_res = check_overfit_capacity(model, sample_input, sample_target, loss_fn=loss_fn)
|
|
98
|
+
steps_val = cap_res["steps_to_converge"]
|
|
99
|
+
steps_str = f"in {steps_val} steps" if cap_res["converged"] else "not converged"
|
|
100
|
+
cap_summary = f"Final loss: {cap_res['final_loss']:.4f} ({steps_str})"
|
|
101
|
+
report.add_check("Overfit Capacity", cap_res["status"], cap_summary, cap_res["issues"])
|
|
102
|
+
|
|
103
|
+
return report
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def audit_llm(model, num_pairs=8, vocab_size=64, batch_size=8, seed=42):
|
|
107
|
+
"""Audits an autoregressive or sequence model on synthetic associative recall.
|
|
108
|
+
|
|
109
|
+
Instantiates an `AssociativeRecallDataset` and runs diagnostic probes (gradient flow,
|
|
110
|
+
effective rank, dead units, numerical stability, and single-batch capacity).
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
model: The sequence `torch.nn.Module` to evaluate. Must accept input token ID
|
|
114
|
+
tensors of shape `(batch_size, 2 * num_pairs + 1)` with dtype `torch.long`.
|
|
115
|
+
num_pairs: Number of distinct (key, value) pairs per sequence (default: 8).
|
|
116
|
+
vocab_size: Total vocabulary size partitioned between keys and values (default: 64).
|
|
117
|
+
Must satisfy `vocab_size >= 2 * num_pairs`.
|
|
118
|
+
batch_size: Batch size of synthetic sequences to audit (default: 8).
|
|
119
|
+
seed: Random seed for reproducible generation (default: 42).
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
An `AuditReport` summarizing model health on the sequence task.
|
|
123
|
+
|
|
124
|
+
Example:
|
|
125
|
+
>>> import torch.nn as nn
|
|
126
|
+
>>> import syndset as syn
|
|
127
|
+
>>> class SimpleSeqModel(nn.Module):
|
|
128
|
+
... def __init__(self):
|
|
129
|
+
... super().__init__()
|
|
130
|
+
... self.emb = nn.Embedding(64, 32)
|
|
131
|
+
... self.gru = nn.GRU(32, 32, batch_first=True)
|
|
132
|
+
... self.head = nn.Linear(32, 64)
|
|
133
|
+
... def forward(self, x):
|
|
134
|
+
... out, _ = self.gru(self.emb(x))
|
|
135
|
+
... return self.head(out[:, -1, :])
|
|
136
|
+
>>> model = SimpleSeqModel()
|
|
137
|
+
>>> report = syn.audit_llm(model, num_pairs=8, vocab_size=64)
|
|
138
|
+
>>> report.print_summary()
|
|
139
|
+
"""
|
|
140
|
+
dataset = AssociativeRecallDataset(num_samples=batch_size,
|
|
141
|
+
num_pairs=num_pairs,
|
|
142
|
+
vocab_size=vocab_size,
|
|
143
|
+
seed=seed)
|
|
144
|
+
inputs, targets = dataset[:batch_size]
|
|
145
|
+
return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (LLM Task)")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def audit_vision(model, img_size=32, channels=1, num_classes=None, batch_size=8, seed=42):
|
|
149
|
+
"""Audits a vision classifier architecture on procedural geometric shapes.
|
|
150
|
+
|
|
151
|
+
Instantiates a `SyntheticShapesDataset` across geometric classes (circles,
|
|
152
|
+
squares, triangles, crosses, rectangles, ellipses, annuli, diamonds) and executes
|
|
153
|
+
gradient flow, representation rank, numerical stability, and overfit probes.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
model: The vision `torch.nn.Module` to evaluate. Accepts image tensors of shape
|
|
157
|
+
`(batch_size, channels, img_size, img_size)` and outputs logits of shape
|
|
158
|
+
`(batch_size, num_classes)`.
|
|
159
|
+
img_size: Image height and width in pixels (default: 32).
|
|
160
|
+
channels: Image channels (default: 1).
|
|
161
|
+
num_classes: Number of shape classes in `[1, 8]`. If `None`, attempts to infer
|
|
162
|
+
from `model.num_classes` or `model.classifier.out_features`, defaulting to 8.
|
|
163
|
+
batch_size: Batch size for audit (default: 8).
|
|
164
|
+
seed: Random seed for reproducible generation (default: 42).
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
An `AuditReport` summarizing model health on synthetic vision inputs.
|
|
168
|
+
|
|
169
|
+
Example:
|
|
170
|
+
>>> import torch.nn as nn
|
|
171
|
+
>>> import syndset as syn
|
|
172
|
+
>>> model = nn.Sequential(
|
|
173
|
+
... nn.Conv2d(1, 16, 3, padding=1),
|
|
174
|
+
... nn.ReLU(),
|
|
175
|
+
... nn.AdaptiveAvgPool2d((1, 1)),
|
|
176
|
+
... nn.Flatten(),
|
|
177
|
+
... nn.Linear(16, 8)
|
|
178
|
+
... )
|
|
179
|
+
>>> report = syn.audit_vision(model, img_size=32, channels=1, num_classes=8)
|
|
180
|
+
>>> report.print_summary()
|
|
181
|
+
"""
|
|
182
|
+
if num_classes is None:
|
|
183
|
+
num_classes = getattr(model, "num_classes", None)
|
|
184
|
+
classifier = getattr(model, "classifier", None)
|
|
185
|
+
if num_classes is None and classifier is not None and hasattr(classifier, "out_features"):
|
|
186
|
+
num_classes = classifier.out_features
|
|
187
|
+
if num_classes is None:
|
|
188
|
+
num_classes = 8
|
|
189
|
+
|
|
190
|
+
dataset = SyntheticShapesDataset(num_samples=batch_size,
|
|
191
|
+
img_size=img_size,
|
|
192
|
+
channels=channels,
|
|
193
|
+
num_classes=num_classes,
|
|
194
|
+
seed=seed)
|
|
195
|
+
inputs, targets = dataset[:batch_size]
|
|
196
|
+
return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (Vision Task)")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def audit_tabular(model, dim=16, num_classes=3, batch_size=16, seed=42):
|
|
200
|
+
"""Audits a feedforward / tabular architecture on concentric hyperspheres.
|
|
201
|
+
|
|
202
|
+
Instantiates a `ConcentricHyperspheresDataset` to evaluate non-linear representation
|
|
203
|
+
capacity, gradient flow, and numerical stability on non-linearly separable manifold shells.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
model: The tabular `torch.nn.Module` to evaluate. Accepts float feature tensors
|
|
207
|
+
of shape `(batch_size, dim)` and outputs logits of shape `(batch_size, num_classes)`.
|
|
208
|
+
dim: Dimensionality of input feature vectors (default: 16).
|
|
209
|
+
num_classes: Number of concentric spherical shells (default: 3).
|
|
210
|
+
batch_size: Batch size for audit (default: 16).
|
|
211
|
+
seed: Random seed for reproducible generation (default: 42).
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
An `AuditReport` summarizing model health on non-linear manifold inputs.
|
|
215
|
+
|
|
216
|
+
Example:
|
|
217
|
+
>>> import torch.nn as nn
|
|
218
|
+
>>> import syndset as syn
|
|
219
|
+
>>> model = nn.Sequential(
|
|
220
|
+
... nn.Linear(16, 32),
|
|
221
|
+
... nn.ReLU(),
|
|
222
|
+
... nn.Linear(32, 3)
|
|
223
|
+
... )
|
|
224
|
+
>>> report = syn.audit_tabular(model, dim=16, num_classes=3)
|
|
225
|
+
>>> report.print_summary()
|
|
226
|
+
"""
|
|
227
|
+
dataset = ConcentricHyperspheresDataset(num_samples=batch_size,
|
|
228
|
+
num_classes=num_classes,
|
|
229
|
+
dim=dim,
|
|
230
|
+
seed=seed)
|
|
231
|
+
inputs, targets = dataset[:batch_size]
|
|
232
|
+
return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (Tabular Task)")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def audit_timeseries(model, seq_len=32, batch_size=8, seed=42):
|
|
236
|
+
"""Audits a time series or recurrent architecture on harmonic waveforms.
|
|
237
|
+
|
|
238
|
+
Instantiates a `HarmonicSuperpositionDataset` with multi-frequency Fourier sums
|
|
239
|
+
and checks whether the model preserves gradient flow, avoids representation collapse,
|
|
240
|
+
and fits temporal sequences.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
model: The time-series `torch.nn.Module` to evaluate. Accepts input tensors
|
|
244
|
+
of shape `(batch_size, seq_len - 1, 1)` and outputs forecasted tensors of
|
|
245
|
+
seq_len: Number of time steps in generated waveforms (default: 32).
|
|
246
|
+
Model input sequence length is `seq_len - 1`.
|
|
247
|
+
batch_size: Batch size for audit (default: 8).
|
|
248
|
+
seed: Random seed for reproducible generation (default: 42).
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
An `AuditReport` summarizing model health on spectral wave inputs.
|
|
252
|
+
|
|
253
|
+
Example:
|
|
254
|
+
>>> import torch.nn as nn
|
|
255
|
+
>>> import syndset as syn
|
|
256
|
+
>>> class SimpleForecaster(nn.Module):
|
|
257
|
+
... def __init__(self):
|
|
258
|
+
... super().__init__()
|
|
259
|
+
... self.rnn = nn.GRU(1, 16, batch_first=True)
|
|
260
|
+
... self.head = nn.Linear(16, 1)
|
|
261
|
+
... def forward(self, x):
|
|
262
|
+
... out, _ = self.rnn(x)
|
|
263
|
+
... return self.head(out)
|
|
264
|
+
>>> model = SimpleForecaster()
|
|
265
|
+
>>> report = syn.audit_timeseries(model, seq_len=32)
|
|
266
|
+
>>> report.print_summary()
|
|
267
|
+
"""
|
|
268
|
+
dataset = HarmonicSuperpositionDataset(num_samples=batch_size, seq_len=seq_len, seed=seed)
|
|
269
|
+
inputs, targets = dataset[:batch_size]
|
|
270
|
+
return audit(model, inputs, targets, model_name=f"{model.__class__.__name__} (Time Series Task)")
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def audit_segmentation(
|
|
274
|
+
model,
|
|
275
|
+
img_size=32,
|
|
276
|
+
channels=1,
|
|
277
|
+
num_classes=None,
|
|
278
|
+
num_objects=3,
|
|
279
|
+
batch_size=8,
|
|
280
|
+
seed=42,
|
|
281
|
+
):
|
|
282
|
+
"""Audits a dense segmentation model (e.g. U-Net, FPN) on procedural pixel masks.
|
|
283
|
+
|
|
284
|
+
Instantiates a `SyntheticSegmentationDataset` and evaluates encoder-decoder gradient
|
|
285
|
+
propagation, representation rank across bottleneck and skip layers, numerical stability,
|
|
286
|
+
and single-batch pixel-level overfit capacity using `nn.CrossEntropyLoss`.
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
model: The segmentation `torch.nn.Module` to evaluate. Must accept inputs of shape
|
|
290
|
+
`(batch_size, channels, img_size, img_size)` and output logits of shape
|
|
291
|
+
`(batch_size, num_classes + 1, img_size, img_size)`.
|
|
292
|
+
img_size: Image height and width in pixels (default: 32).
|
|
293
|
+
num_classes: Number of foreground shape classes in `[1, 8]`
|
|
294
|
+
(default: `model.out_channels - 1` or 8).
|
|
295
|
+
Total segmentation classes will be `num_classes + 1` (class 0 is background).
|
|
296
|
+
num_objects: Number of shapes placed per image canvas (default: 3).
|
|
297
|
+
batch_size: Batch size for audit (default: 8).
|
|
298
|
+
seed: Random seed for reproducible generation (default: 42).
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
An `AuditReport` summarizing model health on the semantic segmentation task.
|
|
302
|
+
|
|
303
|
+
Example:
|
|
304
|
+
>>> import torch.nn as nn
|
|
305
|
+
>>> import syndset as syn
|
|
306
|
+
>>> class SimpleUNet(nn.Module):
|
|
307
|
+
... def __init__(self, in_c=1, out_c=9):
|
|
308
|
+
... super().__init__()
|
|
309
|
+
... self.enc = nn.Conv2d(in_c, 16, 3, padding=1)
|
|
310
|
+
... self.dec = nn.Conv2d(16, out_c, 3, padding=1)
|
|
311
|
+
... def forward(self, x):
|
|
312
|
+
... return self.dec(nn.functional.relu(self.enc(x)))
|
|
313
|
+
>>> model = SimpleUNet()
|
|
314
|
+
>>> report = syn.audit_segmentation(model, img_size=32, channels=1, num_classes=8)
|
|
315
|
+
>>> report.print_summary()
|
|
316
|
+
"""
|
|
317
|
+
if num_classes is None:
|
|
318
|
+
num_classes = getattr(model, "num_classes", None)
|
|
319
|
+
if num_classes is None:
|
|
320
|
+
out_channels = getattr(model, "out_channels", None)
|
|
321
|
+
if out_channels is not None:
|
|
322
|
+
num_classes = max(1, out_channels - 1)
|
|
323
|
+
if num_classes is None:
|
|
324
|
+
num_classes = 8
|
|
325
|
+
|
|
326
|
+
dataset = SyntheticSegmentationDataset(
|
|
327
|
+
num_samples=batch_size,
|
|
328
|
+
img_size=img_size,
|
|
329
|
+
channels=channels,
|
|
330
|
+
num_classes=num_classes,
|
|
331
|
+
num_objects=num_objects,
|
|
332
|
+
seed=seed,
|
|
333
|
+
)
|
|
334
|
+
inputs, targets = dataset[:batch_size]
|
|
335
|
+
return audit(
|
|
336
|
+
model,
|
|
337
|
+
inputs,
|
|
338
|
+
targets,
|
|
339
|
+
loss_fn=nn.CrossEntropyLoss(),
|
|
340
|
+
model_name=f"{model.__class__.__name__} (Segmentation Task)",
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def audit_detection(
|
|
345
|
+
model,
|
|
346
|
+
img_size=32,
|
|
347
|
+
channels=1,
|
|
348
|
+
num_objects=3,
|
|
349
|
+
num_classes=8,
|
|
350
|
+
batch_size=8,
|
|
351
|
+
loss_fn=None,
|
|
352
|
+
seed=42,
|
|
353
|
+
):
|
|
354
|
+
"""Audits an object detection or coordinate regression model on procedural boxes.
|
|
355
|
+
|
|
356
|
+
Instantiates a `SyntheticDetectionDataset` and evaluates coordinate regression
|
|
357
|
+
gradient propagation, representation rank across backbone/pyramid layers, numerical
|
|
358
|
+
stability, and single-batch bounding box overfit capacity.
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
model: The detector `torch.nn.Module` to evaluate. Accepts input tensors of shape
|
|
362
|
+
`(batch_size, channels, img_size, img_size)` and must output predicted bounding
|
|
363
|
+
boxes of shape `(batch_size, num_objects, 4)` in normalized `[cx, cy, w, h]` format
|
|
364
|
+
(or tuple/dict containing box predictions).
|
|
365
|
+
img_size: Image height and width in pixels (default: 32).
|
|
366
|
+
channels: Image channels (default: 1).
|
|
367
|
+
num_objects: Number of objects per image (default: 3).
|
|
368
|
+
num_classes: Number of geometric shape classes in `[1, 8]` (default: 8).
|
|
369
|
+
batch_size: Batch size for audit (default: 8).
|
|
370
|
+
loss_fn: Optional loss function callable `loss_fn(preds, targets)`. If `None`,
|
|
371
|
+
defaults to Smooth L1 loss on normalized box coordinates.
|
|
372
|
+
seed: Random seed for reproducible generation (default: 42).
|
|
373
|
+
|
|
374
|
+
Returns:
|
|
375
|
+
An `AuditReport` summarizing model health on the object detection task.
|
|
376
|
+
|
|
377
|
+
Example:
|
|
378
|
+
>>> import torch.nn as nn
|
|
379
|
+
>>> import syndset as syn
|
|
380
|
+
>>> class SimpleDetector(nn.Module):
|
|
381
|
+
... def __init__(self, num_objects=3):
|
|
382
|
+
... super().__init__()
|
|
383
|
+
... self.conv = nn.Conv2d(1, 16, 3, padding=1)
|
|
384
|
+
... self.pool = nn.AdaptiveAvgPool2d((1, 1))
|
|
385
|
+
... self.head = nn.Linear(16, num_objects * 4)
|
|
386
|
+
... self.num_objects = num_objects
|
|
387
|
+
... def forward(self, x):
|
|
388
|
+
... h = self.pool(nn.functional.relu(self.conv(x))).flatten(1)
|
|
389
|
+
... return torch.sigmoid(self.head(h)).view(-1, self.num_objects, 4)
|
|
390
|
+
>>> model = SimpleDetector()
|
|
391
|
+
>>> report = syn.audit_detection(model, img_size=32, channels=1, num_objects=3)
|
|
392
|
+
>>> report.print_summary()
|
|
393
|
+
"""
|
|
394
|
+
dataset = SyntheticDetectionDataset(
|
|
395
|
+
num_samples=batch_size,
|
|
396
|
+
img_size=img_size,
|
|
397
|
+
channels=channels,
|
|
398
|
+
num_classes=num_classes,
|
|
399
|
+
num_objects=num_objects,
|
|
400
|
+
seed=seed,
|
|
401
|
+
)
|
|
402
|
+
inputs = dataset.images[:batch_size]
|
|
403
|
+
target_boxes = dataset.boxes[:batch_size]
|
|
404
|
+
|
|
405
|
+
if loss_fn is None:
|
|
406
|
+
|
|
407
|
+
def default_detection_loss(preds, targets):
|
|
408
|
+
if isinstance(preds, (tuple, list)):
|
|
409
|
+
preds = preds[0]
|
|
410
|
+
elif isinstance(preds, dict):
|
|
411
|
+
preds = preds.get("boxes", preds.get("pred_boxes", next(iter(preds.values()))))
|
|
412
|
+
if preds.dim() == 2 and preds.shape[1] == targets.numel() // targets.shape[0]:
|
|
413
|
+
preds = preds.view_as(targets)
|
|
414
|
+
return nn.functional.smooth_l1_loss(preds, targets)
|
|
415
|
+
|
|
416
|
+
loss_fn = default_detection_loss
|
|
417
|
+
|
|
418
|
+
return audit(
|
|
419
|
+
model,
|
|
420
|
+
inputs,
|
|
421
|
+
target_boxes,
|
|
422
|
+
loss_fn=loss_fn,
|
|
423
|
+
model_name=f"{model.__class__.__name__} (Detection Task)",
|
|
424
|
+
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Architecture diagnostics and health verification probes."""
|
|
2
|
+
|
|
3
|
+
from syndset.diagnostics.capacity import check_overfit_capacity
|
|
4
|
+
from syndset.diagnostics.curvature import check_curvature_sharpness
|
|
5
|
+
from syndset.diagnostics.gradient import check_gradient_flow, check_gradient_snr
|
|
6
|
+
from syndset.diagnostics.initialization import check_initialization_scale
|
|
7
|
+
from syndset.diagnostics.permutation import check_permutation_equivariance
|
|
8
|
+
from syndset.diagnostics.representation import check_dead_units, check_effective_rank
|
|
9
|
+
from syndset.diagnostics.stability import check_numerical_stability
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"check_overfit_capacity",
|
|
13
|
+
"check_gradient_flow",
|
|
14
|
+
"check_gradient_snr",
|
|
15
|
+
"check_dead_units",
|
|
16
|
+
"check_effective_rank",
|
|
17
|
+
"check_numerical_stability",
|
|
18
|
+
"check_initialization_scale",
|
|
19
|
+
"check_curvature_sharpness",
|
|
20
|
+
"check_permutation_equivariance",
|
|
21
|
+
]
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Capacity and single-batch memorization probes for neural networks."""
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn as nn
|
|
7
|
+
import torch.optim as optim
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _extract_primary_tensor(output):
|
|
11
|
+
"""Extracts the primary floating-point tensor from various model return structures.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
output: Model output, which could be a Tensor, tuple/list, or object with .logits.
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
A torch.Tensor if found, or None.
|
|
18
|
+
"""
|
|
19
|
+
if isinstance(output, torch.Tensor):
|
|
20
|
+
return output
|
|
21
|
+
if isinstance(output, (tuple, list)) and len(output) > 0 and isinstance(output[0], torch.Tensor):
|
|
22
|
+
return output[0]
|
|
23
|
+
if hasattr(output, "logits"):
|
|
24
|
+
return output.logits
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def check_overfit_capacity(
|
|
29
|
+
model,
|
|
30
|
+
inputs,
|
|
31
|
+
targets,
|
|
32
|
+
loss_fn=None,
|
|
33
|
+
num_steps=100,
|
|
34
|
+
learning_rate=1e-3,
|
|
35
|
+
target_loss=1e-3,
|
|
36
|
+
optimizer_cls=optim.AdamW,
|
|
37
|
+
):
|
|
38
|
+
"""Tests whether the model has enough parameter capacity to memorize a single batch.
|
|
39
|
+
|
|
40
|
+
In deep learning engineering, a model that cannot drive loss on a small batch
|
|
41
|
+
to near zero almost always has a structural defect (e.g. Broken skip connections,
|
|
42
|
+
mismatched activation scales, or disconnected parameters).
|
|
43
|
+
|
|
44
|
+
This test works on an isolated deep copy of the model weights so the original
|
|
45
|
+
model parameters remain completely untouched.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
model: The torch.nn.Module to evaluate.
|
|
49
|
+
inputs: A torch.Tensor or tuple/dict of tensors for the model.
|
|
50
|
+
targets: A torch.Tensor representing the expected target labels or values.
|
|
51
|
+
loss_fn: Optional loss function. If None, automatically selects CrossEntropyLoss
|
|
52
|
+
for integer labels or MSELoss for float targets.
|
|
53
|
+
num_steps: Maximum number of optimization steps to attempt (default: 100).
|
|
54
|
+
learning_rate: Learning rate for the optimizer (default: 1e-3).
|
|
55
|
+
target_loss: The loss threshold below which the model is deemed to have
|
|
56
|
+
successfully memorized the batch (default: 1e-3).
|
|
57
|
+
optimizer_cls: Optimizer class to use (default: torch.optim.AdamW).
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
A dictionary containing:
|
|
61
|
+
- 'status': One of 'passed', 'slow', or 'failed'.
|
|
62
|
+
- 'converged': Boolean indicating if target_loss was achieved.
|
|
63
|
+
- 'steps_to_converge': The step index where target_loss was reached, or None.
|
|
64
|
+
- 'initial_loss': Loss at step 0.
|
|
65
|
+
- 'final_loss': Loss at the final optimization step.
|
|
66
|
+
- 'loss_history': List of loss values recorded across steps.
|
|
67
|
+
- 'issues': Human-readable notes or warnings.
|
|
68
|
+
"""
|
|
69
|
+
# Select default loss function if omitted
|
|
70
|
+
if loss_fn is None:
|
|
71
|
+
if targets.dtype in (torch.long, torch.int64):
|
|
72
|
+
loss_fn = nn.CrossEntropyLoss()
|
|
73
|
+
else:
|
|
74
|
+
loss_fn = nn.MSELoss()
|
|
75
|
+
|
|
76
|
+
# Create an isolated clone of the model to avoid mutating caller's weights
|
|
77
|
+
cloned_model = copy.deepcopy(model)
|
|
78
|
+
cloned_model.train()
|
|
79
|
+
|
|
80
|
+
optimizer = optimizer_cls(cloned_model.parameters(), lr=learning_rate)
|
|
81
|
+
loss_history = []
|
|
82
|
+
converged = False
|
|
83
|
+
steps_to_converge = None
|
|
84
|
+
|
|
85
|
+
for step in range(num_steps):
|
|
86
|
+
optimizer.zero_grad()
|
|
87
|
+
|
|
88
|
+
if isinstance(inputs, dict):
|
|
89
|
+
outputs = cloned_model(**inputs)
|
|
90
|
+
elif isinstance(inputs, (tuple, list)):
|
|
91
|
+
outputs = cloned_model(*inputs)
|
|
92
|
+
else:
|
|
93
|
+
outputs = cloned_model(inputs)
|
|
94
|
+
|
|
95
|
+
pred = _extract_primary_tensor(outputs)
|
|
96
|
+
loss = loss_fn(pred if pred is not None else outputs, targets)
|
|
97
|
+
loss_val = float(loss.item())
|
|
98
|
+
loss_history.append(loss_val)
|
|
99
|
+
|
|
100
|
+
if torch.isnan(loss) or torch.isinf(loss):
|
|
101
|
+
return {
|
|
102
|
+
"status": "failed",
|
|
103
|
+
"converged": False,
|
|
104
|
+
"steps_to_converge": None,
|
|
105
|
+
"initial_loss": loss_history[0],
|
|
106
|
+
"final_loss": loss_val,
|
|
107
|
+
"loss_history": loss_history,
|
|
108
|
+
"issues": [f"Loss exploded to {loss_val} at step {step}."],
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if loss_val <= target_loss and not converged:
|
|
112
|
+
converged = True
|
|
113
|
+
steps_to_converge = step + 1
|
|
114
|
+
break
|
|
115
|
+
|
|
116
|
+
loss.backward()
|
|
117
|
+
optimizer.step()
|
|
118
|
+
|
|
119
|
+
initial_loss = loss_history[0]
|
|
120
|
+
final_loss = loss_history[-1]
|
|
121
|
+
issues = []
|
|
122
|
+
|
|
123
|
+
if converged:
|
|
124
|
+
if steps_to_converge <= num_steps // 3:
|
|
125
|
+
status = "passed"
|
|
126
|
+
else:
|
|
127
|
+
status = "slow"
|
|
128
|
+
issues.append(f"Memorized batch slowly (took {steps_to_converge}/{num_steps} steps). "
|
|
129
|
+
"Check learning rate or gradient scaling.")
|
|
130
|
+
else:
|
|
131
|
+
# Check if loss at least improved
|
|
132
|
+
loss_ratio = final_loss / (initial_loss + 1e-10)
|
|
133
|
+
if loss_ratio > 0.8:
|
|
134
|
+
status = "failed"
|
|
135
|
+
issues.append(
|
|
136
|
+
f"Failed to memorize batch (final {final_loss:.4f} vs initial {initial_loss:.4f}). "
|
|
137
|
+
"The model may lack capacity or have dead forward pathways.")
|
|
138
|
+
else:
|
|
139
|
+
status = "slow"
|
|
140
|
+
issues.append(f"Loss decreased from {initial_loss:.4f} to {final_loss:.4f} but did not reach "
|
|
141
|
+
f"the target threshold of {target_loss} within {num_steps} steps.")
|
|
142
|
+
|
|
143
|
+
return {
|
|
144
|
+
"status": status,
|
|
145
|
+
"converged": converged,
|
|
146
|
+
"steps_to_converge": steps_to_converge,
|
|
147
|
+
"initial_loss": initial_loss,
|
|
148
|
+
"final_loss": final_loss,
|
|
149
|
+
"loss_history": loss_history,
|
|
150
|
+
"issues": issues,
|
|
151
|
+
}
|