silentbug 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
silentbug/session.py ADDED
@@ -0,0 +1,794 @@
1
+ """The tracker: one TorchFunctionMode plus a few global hooks.
2
+
3
+ How it works, in short:
4
+
5
+ * Every torch call made by user code passes through ``_Mode.__torch_function__``.
6
+ There we look at the shapes going into broadcasting ops and losses, and we
7
+ carry a bitmask on every tensor saying which trainable parameters it was
8
+ computed from (bit i = parameter i). When a tensor that came from parameters
9
+ goes into an op and comes out without a gradient (``.detach()``, ``.data``,
10
+ ``.item()``, ``torch.no_grad()``, ``argmax`` ...) we record that cut, with the
11
+ line that did it.
12
+ * Hooks on every parameter record which ones receive a gradient, and whether it
13
+ is ever non-zero.
14
+ * Global module and optimizer hooks find the models and optimizers on their own,
15
+ so the user only wraps their code.
16
+
17
+ After the run, parameters that never got a gradient are matched to the cut (or
18
+ the missing call) that explains it.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import weakref
24
+ from dataclasses import dataclass
25
+
26
+ import torch
27
+ import torch.nn as nn
28
+ from torch.overrides import TorchFunctionMode
29
+ from torch.utils.weak import WeakTensorKeyDictionary
30
+
31
+ try:
32
+ from torch.utils._pytree import tree_leaves
33
+ except Exception: # pragma: no cover
34
+ def tree_leaves(x):
35
+ if isinstance(x, (list, tuple)):
36
+ return [z for y in x for z in tree_leaves(y)]
37
+ if isinstance(x, dict):
38
+ return [z for y in x.values() for z in tree_leaves(y)]
39
+ return [x]
40
+
41
+ from . import explain
42
+ from ._where import Where, torch_caller, where
43
+ from .report import BUG, CHECK, Finding, Report
44
+
45
+ # ---------------------------------------------------------------- op tables
46
+
47
+ _REFLECTED = {"radd", "rsub", "rmul", "rtruediv", "rdiv", "rfloordiv", "rpow", "rmod", "rand", "ror", "rxor", "rmatmul"}
48
+ _ALIASES = {
49
+ "truediv": "div", "true_divide": "div", "divide": "div", "multiply": "mul", "subtract": "sub",
50
+ "floordiv": "floor_divide", "mod": "remainder", "and": "logical_and", "or": "logical_or",
51
+ "xor": "logical_xor", "greater": "gt", "less": "lt", "greater_equal": "ge", "less_equal": "le",
52
+ "not_equal": "ne",
53
+ }
54
+ BROADCAST_OPS = {
55
+ "add", "sub", "mul", "div", "floor_divide", "pow", "rsub", "maximum", "minimum", "fmax", "fmin",
56
+ "atan2", "hypot", "remainder", "fmod", "eq", "ne", "lt", "le", "gt", "ge", "logical_and",
57
+ "logical_or", "logical_xor", "bitwise_and", "bitwise_or", "where", "copysign", "xlogy", "lerp",
58
+ "addcmul", "addcdiv",
59
+ }
60
+ SYMBOL = {"add": "+", "sub": "-", "rsub": "-", "mul": "*", "div": "/", "pow": "**", "eq": "==", "ne": "!=",
61
+ "lt": "<", "le": "<=", "gt": ">", "ge": ">=", "floor_divide": "//"}
62
+ LOSS_SHAPE = {"mse_loss", "l1_loss", "smooth_l1_loss", "huber_loss", "soft_margin_loss", "poisson_nll_loss", "kl_div"}
63
+ ACTIVATIONS = {"softmax": "softmax", "sigmoid": "sigmoid", "log_softmax": "log_softmax"}
64
+ ACT_LOSSES = {"cross_entropy", "nll_loss", "binary_cross_entropy_with_logits", "kl_div"}
65
+ # Ops that keep the values (so a softmax stays a softmax through them).
66
+ PASS_THROUGH = {
67
+ "view", "reshape", "squeeze", "unsqueeze", "flatten", "contiguous", "float", "double", "half", "bfloat16",
68
+ "to", "clone", "transpose", "permute", "t", "getitem", "view_as", "reshape_as", "type", "attr:T", "cuda", "cpu",
69
+ }
70
+ MIXING = {"matmul", "bmm", "mm", "einsum"}
71
+ # Ops that show the author shaped the tensor on purpose for broadcasting.
72
+ SHAPING = {"unsqueeze", "expand", "expand_as", "broadcast_to", "atleast_2d", "atleast_3d"}
73
+ # Ops whose derivative is zero almost everywhere.
74
+ ZERO_DERIV = {"round", "floor", "ceil", "trunc", "sign", "sgn", "fix", "heaviside"}
75
+ # Calls that turn a tensor into a Python value or NumPy array.
76
+ VALUE_FUNCS = {"item", "tolist", "numpy", "__float__", "__int__", "__index__", "__array__", "__complex__"}
77
+ NEW_TENSOR = {"tensor", "as_tensor", "from_numpy", "new_tensor", "Tensor"}
78
+
79
+ EVAL_SENSITIVE = (nn.modules.dropout._DropoutNd, nn.modules.batchnorm._BatchNorm)
80
+
81
+ _CUT_PRIORITY = ["detach", "data", "value", "new_tensor", "no_grad", "inference_mode", "requires_grad_false", "int", "other"]
82
+
83
+
84
+ def _name(func) -> str:
85
+ n = getattr(func, "__name__", None)
86
+ if n is None:
87
+ return str(func)
88
+ if n == "__get__":
89
+ owner = getattr(func, "__self__", None)
90
+ return "attr:" + str(getattr(owner, "__name__", "?"))
91
+ return n
92
+
93
+
94
+ def _base(name: str) -> str:
95
+ """Map __radd__, __iadd__, add_, true_divide ... onto one op name."""
96
+ n = name
97
+ if n.startswith("__") and n.endswith("__"):
98
+ n = n[2:-2]
99
+ if n in _REFLECTED:
100
+ n = n[1:]
101
+ elif n.startswith("i") and (n[1:] in BROADCAST_OPS or n[1:] in _ALIASES):
102
+ n = n[1:]
103
+ elif n.endswith("_") and not n.endswith("__"):
104
+ n = n[:-1]
105
+ return _ALIASES.get(n, n)
106
+
107
+
108
+ def _is_inplace(name: str) -> bool:
109
+ if name.startswith("__"):
110
+ return name in ("__iadd__", "__isub__", "__imul__", "__itruediv__")
111
+ return name.endswith("_") and not name.startswith("attr:")
112
+
113
+
114
+ def _shape(t: torch.Tensor) -> tuple[int, ...]:
115
+ return tuple(int(s) for s in t.shape)
116
+
117
+
118
+ def crossed_broadcast(sa: tuple[int, ...], sb: tuple[int, ...]):
119
+ """Return (out_shape, same_size) when both operands get expanded and their ranks differ, else None.
120
+
121
+ (N,1) with (N,) -> (N,N) is the classic case: a size-N axis of one operand is
122
+ crossed with a size-N axis of the other. same_size is True when that happens.
123
+ """
124
+ if len(sa) == len(sb):
125
+ return None
126
+ n = max(len(sa), len(sb))
127
+ a = (1,) * (n - len(sa)) + sa
128
+ b = (1,) * (n - len(sb)) + sb
129
+ a_exp, b_exp = [], [] # sizes each operand gets stretched to
130
+ for x, y in zip(a, b):
131
+ if x == 1 and y > 1:
132
+ a_exp.append(y)
133
+ elif y == 1 and x > 1:
134
+ b_exp.append(x)
135
+ elif x != y:
136
+ return None # would raise anyway
137
+ if not a_exp or not b_exp:
138
+ return None
139
+ out = tuple(max(x, y) for x, y in zip(a, b))
140
+ same = bool(set(a_exp) & set(b_exp))
141
+ return out, same
142
+
143
+
144
+ # ---------------------------------------------------------------- records
145
+
146
+ @dataclass
147
+ class _Cut:
148
+ mask: int
149
+ kind: str
150
+ op: str
151
+ where: Where | None
152
+ caller: Where | None
153
+ order: int
154
+
155
+
156
+ class _Done(BaseException):
157
+ """Raised to end a `silentbug run` after the requested number of steps."""
158
+
159
+
160
+ class _Mode(TorchFunctionMode):
161
+ def __init__(self, session: "Session"):
162
+ super().__init__()
163
+ self.s = session
164
+
165
+ def __torch_function__(self, func, types, args=(), kwargs=None):
166
+ kwargs = kwargs or {}
167
+ out = func(*args, **kwargs)
168
+ s = self.s
169
+ if not s._busy:
170
+ s._busy = True
171
+ try:
172
+ s._on_op(func, args, kwargs, out)
173
+ except Exception as e: # never break the user's program
174
+ s._internal_errors.append(repr(e))
175
+ finally:
176
+ s._busy = False
177
+ return out
178
+
179
+
180
+ class Session:
181
+ """Watches a stretch of training code. Use `silentbug.check()` rather than this directly."""
182
+
183
+ def __init__(self, *models: nn.Module, optimizer=None, max_steps: int | None = None, stop_at_limit: bool = False):
184
+ self.report = Report()
185
+ self._explicit_models = list(models)
186
+ self._optimizers: list[weakref.ref] = []
187
+ if optimizer is not None:
188
+ for o in optimizer if isinstance(optimizer, (list, tuple)) else [optimizer]:
189
+ self._add_opt(o)
190
+ self.max_steps = max_steps
191
+ self.stop_at_limit = stop_at_limit
192
+
193
+ self._busy = False
194
+ self._active = False
195
+ self._done = False
196
+ self._internal_errors: list[str] = []
197
+ self._order = 0
198
+
199
+ # taint
200
+ self._masks = WeakTensorKeyDictionary() # tensor -> int bitmask of parameters it depends on (with grad)
201
+ self._pbit = WeakTensorKeyDictionary() # leaf tensor -> bit
202
+ self._bit_param: list[weakref.ref] = []
203
+ self._first_use: dict[int, tuple[Where | None, Where | None]] = {}
204
+ self._first_mask = 0 # bits that already have a first use
205
+ self._frozen_use = WeakTensorKeyDictionary() # frozen nn.Parameter -> first line that read it
206
+ self._used_mask = 0
207
+ self._cuts: list[_Cut] = []
208
+ self._cut_cover: dict[str, int] = {}
209
+ self._zero_ops: list[_Cut] = []
210
+ self._zero_cover = 0
211
+
212
+ # tags
213
+ self._act = WeakTensorKeyDictionary() # tensor -> (activation, Where)
214
+ self._shaped = WeakTensorKeyDictionary() # tensor -> True
215
+
216
+ # gradient bookkeeping, by id(param)
217
+ self._hooked: dict[int, object] = {}
218
+ self._got_grad: set[int] = set()
219
+ self._nonzero_grad: set[int] = set()
220
+ self._stepped: set[int] = set()
221
+ self._dirty: set[int] = set() # holds a gradient carried over from before the last step
222
+ self._data_of = WeakTensorKeyDictionary() # p.data tensor -> weakref to the parameter p
223
+ self._param_refs: dict[int, weakref.ref] = {}
224
+
225
+ # modules
226
+ self._called: dict[int, weakref.ref] = {}
227
+ self._eval_mods: list[tuple[weakref.ref, Where | None, Where | None]] = []
228
+ self._eval_seen: set[int] = set()
229
+
230
+ self._seen_findings: set = set()
231
+ self._handles: list = []
232
+ self._mode = _Mode(self)
233
+
234
+ # ------------------------------------------------------------ lifecycle
235
+
236
+ def start(self) -> "Session":
237
+ if self._active:
238
+ return self
239
+ self._active = True
240
+ from torch.nn.modules.module import register_module_forward_pre_hook
241
+ from torch.optim.optimizer import register_optimizer_step_post_hook, register_optimizer_step_pre_hook
242
+
243
+ self._handles.append(register_module_forward_pre_hook(self._module_pre))
244
+ self._handles.append(register_optimizer_step_pre_hook(self._opt_pre))
245
+ self._handles.append(register_optimizer_step_post_hook(self._opt_post))
246
+ for m in self._explicit_models:
247
+ self._hook_params(m)
248
+ self._mode.__enter__()
249
+ return self
250
+
251
+ def stop(self) -> Report:
252
+ if not self._active:
253
+ return self.report
254
+ self._active = False
255
+ try:
256
+ self._mode.__exit__(None, None, None)
257
+ except Exception:
258
+ pass
259
+ for h in self._handles:
260
+ h.remove()
261
+ self._handles.clear()
262
+ self._busy = True
263
+ try:
264
+ self._finalize()
265
+ finally:
266
+ for h in self._hooked.values():
267
+ try:
268
+ h.remove()
269
+ except Exception:
270
+ pass
271
+ self._hooked.clear()
272
+ self._busy = False
273
+ return self.report
274
+
275
+ def __enter__(self) -> Report:
276
+ self.start()
277
+ return self.report
278
+
279
+ def __exit__(self, *exc) -> bool:
280
+ self.stop()
281
+ return False
282
+
283
+ # ------------------------------------------------------------ helpers
284
+
285
+ def _add(self, f: Finding) -> None:
286
+ k = f.key()
287
+ if k in self._seen_findings:
288
+ return
289
+ self._seen_findings.add(k)
290
+ self.report.findings.append(f)
291
+
292
+ def _add_opt(self, opt) -> None:
293
+ if not any(r() is opt for r in self._optimizers):
294
+ self._optimizers.append(weakref.ref(opt))
295
+
296
+ def _hook_param(self, p: torch.Tensor) -> None:
297
+ pid = id(p)
298
+ if pid in self._hooked or not p.requires_grad:
299
+ return
300
+ self._param_refs[pid] = weakref.ref(p)
301
+
302
+ def hook(grad, pid=pid, pref=weakref.ref(p)):
303
+ if self._busy or not self._active or grad is None:
304
+ return None
305
+ self._busy = True
306
+ try:
307
+ self._got_grad.add(pid)
308
+ if pid not in self._nonzero_grad and bool(grad.detach().ne(0).any()):
309
+ self._nonzero_grad.add(pid)
310
+ p = pref()
311
+ old = None if p is None else p.grad
312
+ old_nonzero = old is not None and bool(old.detach().ne(0).any())
313
+ if pid in self._stepped:
314
+ # first backward since this parameter was stepped: old gradient should be gone
315
+ self._stepped.discard(pid)
316
+ if old_nonzero:
317
+ self._dirty.add(pid)
318
+ elif not old_nonzero:
319
+ self._dirty.discard(pid)
320
+ except Exception as e:
321
+ self._internal_errors.append(repr(e))
322
+ finally:
323
+ self._busy = False
324
+ return None
325
+
326
+ self._hooked[pid] = p.register_hook(hook)
327
+
328
+ def _hook_params(self, m: nn.Module) -> None:
329
+ for p in m.parameters():
330
+ self._hook_param(p)
331
+
332
+ def _mask_of(self, t: torch.Tensor) -> int:
333
+ m = self._masks.get(t)
334
+ if m is not None:
335
+ return m
336
+ b = self._pbit.get(t)
337
+ if b is None:
338
+ if t.grad_fn is not None or not t.is_leaf:
339
+ return 0
340
+ b = len(self._bit_param)
341
+ self._pbit[t] = b
342
+ self._bit_param.append(weakref.ref(t))
343
+ self._hook_param(t)
344
+ return 1 << b
345
+
346
+ # ------------------------------------------------------------ the op hook
347
+
348
+ def _on_op(self, func, args, kwargs, out) -> None:
349
+ name = _name(func)
350
+ if name == "backward":
351
+ self.report.backwards += 1
352
+ return
353
+ if name.startswith("attr:") and name != "attr:data" and name != "attr:T":
354
+ return
355
+ base = _base(name)
356
+ leaves = tree_leaves((args, kwargs))
357
+ ins = [t for t in leaves if isinstance(t, torch.Tensor)]
358
+ if not ins:
359
+ return
360
+ self._order += 1
361
+ if name == "attr:data" or _is_inplace(name):
362
+ self._manual_update(name, args, out)
363
+
364
+ if base in BROADCAST_OPS and len(ins) >= 2:
365
+ self._check_broadcast(base, ins)
366
+ if base in LOSS_SHAPE:
367
+ self._check_loss_shape(base, args, kwargs)
368
+ if base in ACT_LOSSES or base in ("softmax", "log_softmax"):
369
+ self._check_act_loss(base, args, kwargs)
370
+
371
+ outs = [o for o in tree_leaves(out) if isinstance(o, torch.Tensor)] if out is not None else []
372
+
373
+ # gradient taint
374
+ live = 0
375
+ for t in ins:
376
+ if t.requires_grad:
377
+ live |= self._mask_of(t)
378
+ elif (isinstance(t, nn.Parameter) and outs and not name.endswith("_") and torch.is_grad_enabled()
379
+ and t not in self._frozen_use): # a real read, not requires_grad_(False) or an in-place init
380
+ self._frozen_use[t] = where(3)
381
+ if live:
382
+ self._used_mask |= live
383
+ # first real use: an op that builds a gradient-carrying tensor from the parameter
384
+ # (not nn.init reading .dim() / .size() or filling it under no_grad)
385
+ fresh = live & ~self._first_mask
386
+ if fresh and any(o.requires_grad and o not in self._pbit for o in outs):
387
+ w = where(3)
388
+ self._first_mask |= fresh
389
+ while fresh:
390
+ low = fresh & -fresh
391
+ self._first_use[low.bit_length() - 1] = w
392
+ fresh ^= low
393
+ if not outs:
394
+ if base in VALUE_FUNCS or name in VALUE_FUNCS:
395
+ self._cut(live, "value", name)
396
+ else:
397
+ for o in outs:
398
+ if o.requires_grad:
399
+ if o in self._pbit:
400
+ continue
401
+ prev = self._masks.get(o)
402
+ self._masks[o] = live if prev is None else (prev | live)
403
+ else:
404
+ self._cut(live, self._cut_kind(name, base, o), name)
405
+ if base in ZERO_DERIV and any(o.requires_grad for o in outs):
406
+ if live & ~self._zero_cover:
407
+ w, c = where(2)
408
+ self._zero_ops.append(_Cut(live, "zero", base, w, c, self._order))
409
+ self._zero_cover |= live
410
+
411
+ # tags for the next ops
412
+ if outs:
413
+ o = outs[0]
414
+ if base in ACTIVATIONS:
415
+ self._act[o] = (ACTIVATIONS[base], where(2)[0])
416
+ elif base in PASS_THROUGH and ins[0] in self._act and o is not ins[0]:
417
+ self._act[o] = self._act[ins[0]]
418
+ elif base in MIXING and len(ins) >= 2:
419
+ # probabilities @ constant one-hot labels is still a probability mix (matching networks);
420
+ # attention (probabilities @ trainable values) is not, so require the other side to be constant
421
+ a, b = ins[0], ins[1]
422
+ for p, q in ((a, b), (b, a)):
423
+ tag = self._act.get(p)
424
+ if tag and tag[0] in ("softmax", "softmax-mix") and not q.requires_grad:
425
+ self._act[o] = ("softmax-mix", tag[1])
426
+ break
427
+ shaped = base in SHAPING or (
428
+ base == "getitem" and len(args) > 1 and (args[1] is None or (isinstance(args[1], tuple) and any(i is None for i in args[1])))
429
+ )
430
+ if shaped:
431
+ self._shaped[o] = True
432
+
433
+ @staticmethod
434
+ def _cut_kind(name: str, base: str, o: torch.Tensor) -> str:
435
+ if torch.is_inference_mode_enabled():
436
+ return "inference_mode"
437
+ if not torch.is_grad_enabled():
438
+ return "no_grad"
439
+ if base in ("detach",):
440
+ return "detach"
441
+ if name == "attr:data":
442
+ return "data"
443
+ if base == "requires_grad":
444
+ return "requires_grad_false"
445
+ if base in NEW_TENSOR:
446
+ return "new_tensor"
447
+ if not (o.is_floating_point() or o.is_complex()):
448
+ return "int"
449
+ return "other"
450
+
451
+ def _cut(self, mask: int, kind: str, op: str) -> None:
452
+ # torch's own bookkeeping (DDP syncing parameters, optimizers, clip_grad_norm_ ...)
453
+ # detaches on purpose; only cuts made by the user's code (or other libraries) count.
454
+ # no_grad / inference_mode are the user's choice when the op is torch's layer code
455
+ # (F.linear inside nn.Linear), but not inside DDP, optimizers or nn.utils.
456
+ by = torch_caller()
457
+ if by == "infra" or (by == "torch" and kind not in ("no_grad", "inference_mode")):
458
+ return
459
+ cover = self._cut_cover.get(kind, 0)
460
+ if not mask & ~cover:
461
+ return
462
+ self._cut_cover[kind] = cover | mask
463
+ w, c = where(3)
464
+ self._cuts.append(_Cut(mask, kind, op, w, c, self._order))
465
+
466
+ # ------------------------------------------------------------ shape and loss checks
467
+
468
+ def _check_broadcast(self, base: str, ins: list[torch.Tensor]) -> None:
469
+ ts = [t for t in ins[:3] if t.dim() > 0 and t.numel() > 1]
470
+ for i in range(len(ts)):
471
+ for j in range(i + 1, len(ts)):
472
+ a, b = ts[i], ts[j]
473
+ if a in self._shaped or b in self._shaped:
474
+ continue
475
+ sa, sb = _shape(a), _shape(b)
476
+ r = crossed_broadcast(sa, sb)
477
+ if r is None:
478
+ continue
479
+ outs, same = r
480
+ w, c = where(3)
481
+ self._add(explain.broadcast(base, SYMBOL.get(base), sa, sb, outs, same, w, c))
482
+
483
+ def _check_loss_shape(self, base: str, args, kwargs) -> None:
484
+ x = args[0] if len(args) > 0 else kwargs.get("input")
485
+ y = args[1] if len(args) > 1 else kwargs.get("target")
486
+ if not (isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor)):
487
+ return
488
+ sx, sy = _shape(x), _shape(y)
489
+ if sx == sy:
490
+ return
491
+ try:
492
+ so = tuple(torch.broadcast_shapes(sx, sy))
493
+ except RuntimeError:
494
+ return
495
+ w, c = where(3)
496
+ self._add(explain.loss_shape(base, sx, sy, so, w, c))
497
+
498
+ def _check_act_loss(self, base: str, args, kwargs) -> None:
499
+ x = args[0] if len(args) > 0 else kwargs.get("input")
500
+ if not isinstance(x, torch.Tensor):
501
+ return
502
+ tag = self._act.get(x)
503
+ if tag is None:
504
+ return
505
+ act, act_where = tag
506
+ f = explain.activation_into_loss(base, act, act_where, *where(3))
507
+ if f is not None:
508
+ self._add(f)
509
+
510
+ # ------------------------------------------------------------ module / optimizer hooks
511
+
512
+ def _module_pre(self, module: nn.Module, inputs) -> None:
513
+ if self._busy or not self._active:
514
+ return
515
+ # no optimizer (weights updated by hand): count backward passes as steps instead
516
+ if (self.max_steps is not None and self.report.steps == 0 and not self._optimizers
517
+ and self.report.backwards >= self.max_steps):
518
+ self.stop()
519
+ if self.stop_at_limit:
520
+ raise _Done()
521
+ return
522
+ self._busy = True
523
+ try:
524
+ mid = id(module)
525
+ if mid not in self._called:
526
+ self._called[mid] = weakref.ref(module)
527
+ self._hook_params(module)
528
+ if torch.is_grad_enabled() and isinstance(module, EVAL_SENSITIVE):
529
+ if not module.training and mid not in self._eval_seen:
530
+ self._eval_seen.add(mid)
531
+ self._eval_mods.append((weakref.ref(module), *where(2)))
532
+ if module.training and isinstance(module, nn.modules.batchnorm._BatchNorm):
533
+ x = inputs[0] if inputs else None
534
+ if isinstance(x, torch.Tensor) and x.dim() > 2 and x.shape[0] == 1:
535
+ self._add(explain.batchnorm_batch1(_shape(x), *where(2)))
536
+ except Exception as e:
537
+ self._internal_errors.append(repr(e))
538
+ finally:
539
+ self._busy = False
540
+
541
+ def _param_of(self, t) -> torch.Tensor | None:
542
+ if not isinstance(t, torch.Tensor):
543
+ return None
544
+ r = self._param_refs.get(id(t))
545
+ if r is not None and r() is t:
546
+ return t
547
+ r = self._data_of.get(t)
548
+ return r() if r is not None else None
549
+
550
+ def _manual_update(self, name: str, args, out) -> None:
551
+ """Weights updated by hand (p.data.add_(...), p -= lr * p.grad under no_grad)."""
552
+ p = self._param_of(args[0]) if args else None
553
+ if p is None:
554
+ return
555
+ if name == "attr:data":
556
+ if isinstance(out, torch.Tensor):
557
+ self._data_of[out] = weakref.ref(p)
558
+ return
559
+ if p.grad is None: # initialisation, not an update
560
+ return
561
+ pid = id(p)
562
+ if pid in self._dirty:
563
+ self._dirty.discard(pid)
564
+ self._add(explain.grad_not_cleared(*where(4), manual=True))
565
+ self._stepped.add(pid)
566
+
567
+ def _opt_pre(self, opt, args, kwargs) -> None:
568
+ if not self._active:
569
+ return
570
+ self._add_opt(opt)
571
+ stale = [p for g in opt.param_groups for p in g["params"] if id(p) in self._dirty and p.grad is not None]
572
+ if stale:
573
+ for p in stale:
574
+ self._dirty.discard(id(p))
575
+ self._add(explain.grad_not_cleared(*where(2)))
576
+ self._busy = True # do not trace the update itself
577
+
578
+ def _opt_post(self, opt, args, kwargs) -> None:
579
+ if not self._active:
580
+ return
581
+ self._busy = False
582
+ self.report.steps += 1
583
+ for g in opt.param_groups:
584
+ for p in g["params"]:
585
+ self._stepped.add(id(p))
586
+ if self.max_steps is not None and self.report.steps >= self.max_steps:
587
+ self.stop()
588
+ if self.stop_at_limit:
589
+ raise _Done()
590
+
591
+ # ------------------------------------------------------------ analysis
592
+
593
+ def _models(self) -> list[nn.Module]:
594
+ mods = [r() for r in self._called.values()]
595
+ mods = [m for m in mods if m is not None] + [m for m in self._explicit_models if m is not None]
596
+ # torch.compile wraps the model in an OptimizedModule; report the original module's names
597
+ mods = [getattr(m, "_orig_mod", m) if isinstance(getattr(m, "_orig_mod", None), nn.Module) else m
598
+ for m in mods]
599
+ uniq: dict[int, nn.Module] = {}
600
+ for m in mods:
601
+ uniq.setdefault(id(m), m)
602
+ inner: set[int] = set()
603
+ for m in uniq.values():
604
+ for c in m.modules():
605
+ if c is not m:
606
+ inner.add(id(c))
607
+ roots = [m for mid, m in uniq.items() if mid not in inner and any(True for _ in m.parameters())]
608
+ return roots
609
+
610
+ def _finalize(self) -> None:
611
+ rep = self.report
612
+ models = self._models()
613
+ names: dict[int, str] = {}
614
+ owner: dict[int, nn.Module] = {}
615
+ counts: dict[str, int] = {}
616
+ for m in models:
617
+ counts[type(m).__name__] = counts.get(type(m).__name__, 0) + 1
618
+ seen_cls: dict[str, int] = {}
619
+ for m in models:
620
+ cls = type(m).__name__
621
+ label = cls
622
+ if counts[cls] > 1:
623
+ seen_cls[cls] = seen_cls.get(cls, 0) + 1
624
+ label = f"{cls}#{seen_cls[cls]}"
625
+ rep.models.append(label)
626
+ prefix = f"{label}." if len(models) > 1 else ""
627
+ for n, p in m.named_parameters(remove_duplicate=True):
628
+ names.setdefault(id(p), prefix + n)
629
+ owner.setdefault(id(p), m)
630
+
631
+ opts = [o for o in (r() for r in self._optimizers) if o is not None]
632
+ in_opt: set[int] = set()
633
+ opt_params: list[torch.Tensor] = []
634
+ for o in opts:
635
+ for gi, g in enumerate(o.param_groups):
636
+ for i, p in enumerate(g["params"]):
637
+ in_opt.add(id(p))
638
+ opt_params.append(p)
639
+ names.setdefault(id(p), f"optimizer group {gi} param {i} {tuple(p.shape)}")
640
+
641
+ # Which models are being trained?
642
+ trained: list[nn.Module] = []
643
+ for m in models:
644
+ ps = [p for p in m.parameters() if p.requires_grad]
645
+ if not ps:
646
+ continue
647
+ if opts:
648
+ if any(id(p) in in_opt for p in ps):
649
+ trained.append(m)
650
+ elif any(id(p) in self._got_grad for p in ps):
651
+ trained.append(m)
652
+
653
+ # optimizer entries that no model holds and that never got a gradient: replaced parameters
654
+ model_ids = {id(p) for m in models for p in m.parameters()}
655
+ stale = ([p for p in opt_params if id(p) not in model_ids and id(p) not in self._got_grad]
656
+ if opts and models else [])
657
+ stale_ids = {id(p) for p in stale}
658
+
659
+ candidates: dict[int, torch.Tensor] = {}
660
+ for m in trained:
661
+ for p in m.parameters():
662
+ if p.requires_grad:
663
+ candidates[id(p)] = p
664
+ for p in opt_params:
665
+ if p.requires_grad and id(p) not in stale_ids:
666
+ candidates[id(p)] = p
667
+
668
+ # parameters left out of every optimizer
669
+ if opts:
670
+ missing = [p for m in trained for p in m.parameters() if p.requires_grad and id(p) not in in_opt]
671
+ uniq = list({id(p): p for p in missing}.values())
672
+ if stale:
673
+ # the parameters the model now uses are the replacements of the stale ones
674
+ rep.findings.append(explain.stale_optimizer(len(stale), sum(p.numel() for p in stale),
675
+ [names.get(id(p), "?") for p in uniq]))
676
+ elif uniq:
677
+ rep.findings.append(explain.not_in_optimizer([names.get(id(p), "?") for p in uniq],
678
+ sum(p.numel() for p in uniq)))
679
+
680
+ # dead and zero-gradient parameters
681
+ if rep.backwards == 0 and self._got_grad:
682
+ rep.backwards = 1
683
+ if rep.backwards > 0 and self._got_grad:
684
+ groups: dict[tuple, list[str]] = {}
685
+ group_info: dict[tuple, tuple] = {}
686
+ for pid, p in candidates.items():
687
+ if pid in self._nonzero_grad:
688
+ continue
689
+ nm = names.get(pid, f"tensor {tuple(p.shape)}")
690
+ bit = self._pbit.get(p)
691
+ if pid in self._got_grad:
692
+ ev = self._first_event(self._zero_ops, bit)
693
+ key = ("zero", ev.op if ev else None, ev.where if ev else None)
694
+ group_info[key] = ("zero", ev, None)
695
+ elif bit is None or (bit not in self._first_use and self._first_event(self._cuts, bit) is None):
696
+ mod_name = nm.rsplit(".", 1)[0] if "." in nm else ""
697
+ sub = self._submodule(owner.get(pid), nm, names)
698
+ called = sub is not None and id(sub) in self._called
699
+ # name the outermost module that never ran (the skipped wrapper, not its pieces)
700
+ parts, bypass = nm.split(".")[:-1], None
701
+ for k in range(1, len(parts) + 1):
702
+ sm = self._submodule(owner.get(pid), ".".join(parts[:k]) + ".x", names)
703
+ if sm is not None and id(sm) not in self._called:
704
+ mod_name, called = ".".join(parts[:k]), False
705
+ bypass = self._bypass_use(sm)
706
+ break
707
+ key = ("unused", mod_name, called)
708
+ group_info[key] = ("unused", mod_name, called, bypass)
709
+ else:
710
+ ev = self._first_event(self._cuts, bit)
711
+ if ev is not None:
712
+ key = ("cut", ev.kind, ev.where, ev.op)
713
+ group_info[key] = ("cut", ev, None)
714
+ else:
715
+ fu = self._first_use.get(bit, (None, None))
716
+ key = ("orphan", fu[0])
717
+ group_info[key] = ("orphan", fu, None)
718
+ groups.setdefault(key, []).append(nm)
719
+ for key, plist in groups.items():
720
+ kind, a, b, *rest = group_info[key]
721
+ trained_by_opt = bool(opts)
722
+ if kind == "zero":
723
+ rep.findings.append(explain.zero_grad(plist, a))
724
+ elif kind == "unused":
725
+ w, c = rest[0] if rest and rest[0] else (None, None)
726
+ rep.findings.append(explain.unused(plist, a, b, trained_by_opt, w, c))
727
+ elif kind == "cut":
728
+ rep.findings.append(explain.cut(plist, a))
729
+ else:
730
+ rep.findings.append(explain.orphan(plist, *a))
731
+
732
+ # eval mode during training
733
+ if self._eval_mods and trained:
734
+ trained_ids = {id(c) for m in trained for c in m.modules()}
735
+ hits = [(r(), w, c) for r, w, c in self._eval_mods if r() is not None and id(r()) in trained_ids]
736
+ if hits:
737
+ whole = any(not m.training for m in trained)
738
+ rep.findings.append(explain.eval_mode([type(m).__name__ for m, _, _ in hits], whole, hits[0][1], hits[0][2]))
739
+
740
+ if self._internal_errors:
741
+ rep.findings.append(Finding("internal", CHECK, "silentbug hit an internal error and may have missed things",
742
+ "; ".join(sorted(set(self._internal_errors))[:3]),
743
+ "please report this at the project issue tracker"))
744
+
745
+ def _bypass_use(self, module: nn.Module) -> tuple[Where | None, Where | None] | None:
746
+ """Where a parameter of a module that never ran was read directly (skipping its forward)."""
747
+ for p in module.parameters():
748
+ if p in self._frozen_use:
749
+ return self._frozen_use[p]
750
+ b = self._pbit.get(p)
751
+ if b is not None and b in self._first_use:
752
+ return self._first_use[b]
753
+ return None
754
+
755
+ @staticmethod
756
+ def _submodule(root: nn.Module | None, pname: str, names: dict) -> nn.Module | None:
757
+ if root is None or "." not in pname:
758
+ return root
759
+ path = pname.split(".")[:-1]
760
+ if len(path) and path[0] == f"{type(root).__name__}" or (path and "#" in path[0]):
761
+ path = path[1:]
762
+ m = root
763
+ for part in path:
764
+ m = getattr(m, part, None)
765
+ if m is None:
766
+ return None
767
+ return m if isinstance(m, nn.Module) else None
768
+
769
+ @staticmethod
770
+ def _first_event(events: list[_Cut], bit: int | None) -> _Cut | None:
771
+ if bit is None:
772
+ return None
773
+ hits = [e for e in events if e.mask >> bit & 1]
774
+ if not hits:
775
+ return None
776
+ rank = {k: i for i, k in enumerate(_CUT_PRIORITY)}
777
+ hits.sort(key=lambda e: (rank.get(e.kind, 99), e.order))
778
+ return hits[0]
779
+
780
+
781
+ def check(*models: nn.Module, optimizer=None) -> Session:
782
+ """Watch the code inside the `with` block for silent training bugs.
783
+
784
+ with silentbug.check() as report:
785
+ for step in range(3):
786
+ loss = loss_fn(model(x), y)
787
+ loss.backward()
788
+ opt.step(); opt.zero_grad()
789
+ print(report)
790
+
791
+ Models and optimizers are found on their own. Pass them in only if the model
792
+ is never called inside the block.
793
+ """
794
+ return Session(*models, optimizer=optimizer)