silentbug 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- silentbug/__init__.py +7 -0
- silentbug/__main__.py +3 -0
- silentbug/_where.py +100 -0
- silentbug/cli.py +80 -0
- silentbug/explain.py +225 -0
- silentbug/report.py +105 -0
- silentbug/session.py +794 -0
- silentbug-0.1.0.dist-info/METADATA +183 -0
- silentbug-0.1.0.dist-info/RECORD +13 -0
- silentbug-0.1.0.dist-info/WHEEL +5 -0
- silentbug-0.1.0.dist-info/entry_points.txt +2 -0
- silentbug-0.1.0.dist-info/licenses/LICENSE +21 -0
- silentbug-0.1.0.dist-info/top_level.txt +1 -0
silentbug/session.py
ADDED
|
@@ -0,0 +1,794 @@
|
|
|
1
|
+
"""The tracker: one TorchFunctionMode plus a few global hooks.
|
|
2
|
+
|
|
3
|
+
How it works, in short:
|
|
4
|
+
|
|
5
|
+
* Every torch call made by user code passes through ``_Mode.__torch_function__``.
|
|
6
|
+
There we look at the shapes going into broadcasting ops and losses, and we
|
|
7
|
+
carry a bitmask on every tensor saying which trainable parameters it was
|
|
8
|
+
computed from (bit i = parameter i). When a tensor that came from parameters
|
|
9
|
+
goes into an op and comes out without a gradient (``.detach()``, ``.data``,
|
|
10
|
+
``.item()``, ``torch.no_grad()``, ``argmax`` ...) we record that cut, with the
|
|
11
|
+
line that did it.
|
|
12
|
+
* Hooks on every parameter record which ones receive a gradient, and whether it
|
|
13
|
+
is ever non-zero.
|
|
14
|
+
* Global module and optimizer hooks find the models and optimizers on their own,
|
|
15
|
+
so the user only wraps their code.
|
|
16
|
+
|
|
17
|
+
After the run, parameters that never got a gradient are matched to the cut (or
|
|
18
|
+
the missing call) that explains it.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import weakref
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
|
|
26
|
+
import torch
|
|
27
|
+
import torch.nn as nn
|
|
28
|
+
from torch.overrides import TorchFunctionMode
|
|
29
|
+
from torch.utils.weak import WeakTensorKeyDictionary
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
from torch.utils._pytree import tree_leaves
|
|
33
|
+
except Exception: # pragma: no cover
|
|
34
|
+
def tree_leaves(x):
|
|
35
|
+
if isinstance(x, (list, tuple)):
|
|
36
|
+
return [z for y in x for z in tree_leaves(y)]
|
|
37
|
+
if isinstance(x, dict):
|
|
38
|
+
return [z for y in x.values() for z in tree_leaves(y)]
|
|
39
|
+
return [x]
|
|
40
|
+
|
|
41
|
+
from . import explain
|
|
42
|
+
from ._where import Where, torch_caller, where
|
|
43
|
+
from .report import BUG, CHECK, Finding, Report
|
|
44
|
+
|
|
45
|
+
# ---------------------------------------------------------------- op tables
|
|
46
|
+
|
|
47
|
+
_REFLECTED = {"radd", "rsub", "rmul", "rtruediv", "rdiv", "rfloordiv", "rpow", "rmod", "rand", "ror", "rxor", "rmatmul"}
|
|
48
|
+
_ALIASES = {
|
|
49
|
+
"truediv": "div", "true_divide": "div", "divide": "div", "multiply": "mul", "subtract": "sub",
|
|
50
|
+
"floordiv": "floor_divide", "mod": "remainder", "and": "logical_and", "or": "logical_or",
|
|
51
|
+
"xor": "logical_xor", "greater": "gt", "less": "lt", "greater_equal": "ge", "less_equal": "le",
|
|
52
|
+
"not_equal": "ne",
|
|
53
|
+
}
|
|
54
|
+
BROADCAST_OPS = {
|
|
55
|
+
"add", "sub", "mul", "div", "floor_divide", "pow", "rsub", "maximum", "minimum", "fmax", "fmin",
|
|
56
|
+
"atan2", "hypot", "remainder", "fmod", "eq", "ne", "lt", "le", "gt", "ge", "logical_and",
|
|
57
|
+
"logical_or", "logical_xor", "bitwise_and", "bitwise_or", "where", "copysign", "xlogy", "lerp",
|
|
58
|
+
"addcmul", "addcdiv",
|
|
59
|
+
}
|
|
60
|
+
SYMBOL = {"add": "+", "sub": "-", "rsub": "-", "mul": "*", "div": "/", "pow": "**", "eq": "==", "ne": "!=",
|
|
61
|
+
"lt": "<", "le": "<=", "gt": ">", "ge": ">=", "floor_divide": "//"}
|
|
62
|
+
LOSS_SHAPE = {"mse_loss", "l1_loss", "smooth_l1_loss", "huber_loss", "soft_margin_loss", "poisson_nll_loss", "kl_div"}
|
|
63
|
+
ACTIVATIONS = {"softmax": "softmax", "sigmoid": "sigmoid", "log_softmax": "log_softmax"}
|
|
64
|
+
ACT_LOSSES = {"cross_entropy", "nll_loss", "binary_cross_entropy_with_logits", "kl_div"}
|
|
65
|
+
# Ops that keep the values (so a softmax stays a softmax through them).
|
|
66
|
+
PASS_THROUGH = {
|
|
67
|
+
"view", "reshape", "squeeze", "unsqueeze", "flatten", "contiguous", "float", "double", "half", "bfloat16",
|
|
68
|
+
"to", "clone", "transpose", "permute", "t", "getitem", "view_as", "reshape_as", "type", "attr:T", "cuda", "cpu",
|
|
69
|
+
}
|
|
70
|
+
MIXING = {"matmul", "bmm", "mm", "einsum"}
|
|
71
|
+
# Ops that show the author shaped the tensor on purpose for broadcasting.
|
|
72
|
+
SHAPING = {"unsqueeze", "expand", "expand_as", "broadcast_to", "atleast_2d", "atleast_3d"}
|
|
73
|
+
# Ops whose derivative is zero almost everywhere.
|
|
74
|
+
ZERO_DERIV = {"round", "floor", "ceil", "trunc", "sign", "sgn", "fix", "heaviside"}
|
|
75
|
+
# Calls that turn a tensor into a Python value or NumPy array.
|
|
76
|
+
VALUE_FUNCS = {"item", "tolist", "numpy", "__float__", "__int__", "__index__", "__array__", "__complex__"}
|
|
77
|
+
NEW_TENSOR = {"tensor", "as_tensor", "from_numpy", "new_tensor", "Tensor"}
|
|
78
|
+
|
|
79
|
+
EVAL_SENSITIVE = (nn.modules.dropout._DropoutNd, nn.modules.batchnorm._BatchNorm)
|
|
80
|
+
|
|
81
|
+
_CUT_PRIORITY = ["detach", "data", "value", "new_tensor", "no_grad", "inference_mode", "requires_grad_false", "int", "other"]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _name(func) -> str:
|
|
85
|
+
n = getattr(func, "__name__", None)
|
|
86
|
+
if n is None:
|
|
87
|
+
return str(func)
|
|
88
|
+
if n == "__get__":
|
|
89
|
+
owner = getattr(func, "__self__", None)
|
|
90
|
+
return "attr:" + str(getattr(owner, "__name__", "?"))
|
|
91
|
+
return n
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _base(name: str) -> str:
|
|
95
|
+
"""Map __radd__, __iadd__, add_, true_divide ... onto one op name."""
|
|
96
|
+
n = name
|
|
97
|
+
if n.startswith("__") and n.endswith("__"):
|
|
98
|
+
n = n[2:-2]
|
|
99
|
+
if n in _REFLECTED:
|
|
100
|
+
n = n[1:]
|
|
101
|
+
elif n.startswith("i") and (n[1:] in BROADCAST_OPS or n[1:] in _ALIASES):
|
|
102
|
+
n = n[1:]
|
|
103
|
+
elif n.endswith("_") and not n.endswith("__"):
|
|
104
|
+
n = n[:-1]
|
|
105
|
+
return _ALIASES.get(n, n)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _is_inplace(name: str) -> bool:
|
|
109
|
+
if name.startswith("__"):
|
|
110
|
+
return name in ("__iadd__", "__isub__", "__imul__", "__itruediv__")
|
|
111
|
+
return name.endswith("_") and not name.startswith("attr:")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _shape(t: torch.Tensor) -> tuple[int, ...]:
|
|
115
|
+
return tuple(int(s) for s in t.shape)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def crossed_broadcast(sa: tuple[int, ...], sb: tuple[int, ...]):
|
|
119
|
+
"""Return (out_shape, same_size) when both operands get expanded and their ranks differ, else None.
|
|
120
|
+
|
|
121
|
+
(N,1) with (N,) -> (N,N) is the classic case: a size-N axis of one operand is
|
|
122
|
+
crossed with a size-N axis of the other. same_size is True when that happens.
|
|
123
|
+
"""
|
|
124
|
+
if len(sa) == len(sb):
|
|
125
|
+
return None
|
|
126
|
+
n = max(len(sa), len(sb))
|
|
127
|
+
a = (1,) * (n - len(sa)) + sa
|
|
128
|
+
b = (1,) * (n - len(sb)) + sb
|
|
129
|
+
a_exp, b_exp = [], [] # sizes each operand gets stretched to
|
|
130
|
+
for x, y in zip(a, b):
|
|
131
|
+
if x == 1 and y > 1:
|
|
132
|
+
a_exp.append(y)
|
|
133
|
+
elif y == 1 and x > 1:
|
|
134
|
+
b_exp.append(x)
|
|
135
|
+
elif x != y:
|
|
136
|
+
return None # would raise anyway
|
|
137
|
+
if not a_exp or not b_exp:
|
|
138
|
+
return None
|
|
139
|
+
out = tuple(max(x, y) for x, y in zip(a, b))
|
|
140
|
+
same = bool(set(a_exp) & set(b_exp))
|
|
141
|
+
return out, same
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ---------------------------------------------------------------- records
|
|
145
|
+
|
|
146
|
+
@dataclass
|
|
147
|
+
class _Cut:
|
|
148
|
+
mask: int
|
|
149
|
+
kind: str
|
|
150
|
+
op: str
|
|
151
|
+
where: Where | None
|
|
152
|
+
caller: Where | None
|
|
153
|
+
order: int
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class _Done(BaseException):
|
|
157
|
+
"""Raised to end a `silentbug run` after the requested number of steps."""
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class _Mode(TorchFunctionMode):
|
|
161
|
+
def __init__(self, session: "Session"):
|
|
162
|
+
super().__init__()
|
|
163
|
+
self.s = session
|
|
164
|
+
|
|
165
|
+
def __torch_function__(self, func, types, args=(), kwargs=None):
|
|
166
|
+
kwargs = kwargs or {}
|
|
167
|
+
out = func(*args, **kwargs)
|
|
168
|
+
s = self.s
|
|
169
|
+
if not s._busy:
|
|
170
|
+
s._busy = True
|
|
171
|
+
try:
|
|
172
|
+
s._on_op(func, args, kwargs, out)
|
|
173
|
+
except Exception as e: # never break the user's program
|
|
174
|
+
s._internal_errors.append(repr(e))
|
|
175
|
+
finally:
|
|
176
|
+
s._busy = False
|
|
177
|
+
return out
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class Session:
|
|
181
|
+
"""Watches a stretch of training code. Use `silentbug.check()` rather than this directly."""
|
|
182
|
+
|
|
183
|
+
def __init__(self, *models: nn.Module, optimizer=None, max_steps: int | None = None, stop_at_limit: bool = False):
|
|
184
|
+
self.report = Report()
|
|
185
|
+
self._explicit_models = list(models)
|
|
186
|
+
self._optimizers: list[weakref.ref] = []
|
|
187
|
+
if optimizer is not None:
|
|
188
|
+
for o in optimizer if isinstance(optimizer, (list, tuple)) else [optimizer]:
|
|
189
|
+
self._add_opt(o)
|
|
190
|
+
self.max_steps = max_steps
|
|
191
|
+
self.stop_at_limit = stop_at_limit
|
|
192
|
+
|
|
193
|
+
self._busy = False
|
|
194
|
+
self._active = False
|
|
195
|
+
self._done = False
|
|
196
|
+
self._internal_errors: list[str] = []
|
|
197
|
+
self._order = 0
|
|
198
|
+
|
|
199
|
+
# taint
|
|
200
|
+
self._masks = WeakTensorKeyDictionary() # tensor -> int bitmask of parameters it depends on (with grad)
|
|
201
|
+
self._pbit = WeakTensorKeyDictionary() # leaf tensor -> bit
|
|
202
|
+
self._bit_param: list[weakref.ref] = []
|
|
203
|
+
self._first_use: dict[int, tuple[Where | None, Where | None]] = {}
|
|
204
|
+
self._first_mask = 0 # bits that already have a first use
|
|
205
|
+
self._frozen_use = WeakTensorKeyDictionary() # frozen nn.Parameter -> first line that read it
|
|
206
|
+
self._used_mask = 0
|
|
207
|
+
self._cuts: list[_Cut] = []
|
|
208
|
+
self._cut_cover: dict[str, int] = {}
|
|
209
|
+
self._zero_ops: list[_Cut] = []
|
|
210
|
+
self._zero_cover = 0
|
|
211
|
+
|
|
212
|
+
# tags
|
|
213
|
+
self._act = WeakTensorKeyDictionary() # tensor -> (activation, Where)
|
|
214
|
+
self._shaped = WeakTensorKeyDictionary() # tensor -> True
|
|
215
|
+
|
|
216
|
+
# gradient bookkeeping, by id(param)
|
|
217
|
+
self._hooked: dict[int, object] = {}
|
|
218
|
+
self._got_grad: set[int] = set()
|
|
219
|
+
self._nonzero_grad: set[int] = set()
|
|
220
|
+
self._stepped: set[int] = set()
|
|
221
|
+
self._dirty: set[int] = set() # holds a gradient carried over from before the last step
|
|
222
|
+
self._data_of = WeakTensorKeyDictionary() # p.data tensor -> weakref to the parameter p
|
|
223
|
+
self._param_refs: dict[int, weakref.ref] = {}
|
|
224
|
+
|
|
225
|
+
# modules
|
|
226
|
+
self._called: dict[int, weakref.ref] = {}
|
|
227
|
+
self._eval_mods: list[tuple[weakref.ref, Where | None, Where | None]] = []
|
|
228
|
+
self._eval_seen: set[int] = set()
|
|
229
|
+
|
|
230
|
+
self._seen_findings: set = set()
|
|
231
|
+
self._handles: list = []
|
|
232
|
+
self._mode = _Mode(self)
|
|
233
|
+
|
|
234
|
+
# ------------------------------------------------------------ lifecycle
|
|
235
|
+
|
|
236
|
+
def start(self) -> "Session":
|
|
237
|
+
if self._active:
|
|
238
|
+
return self
|
|
239
|
+
self._active = True
|
|
240
|
+
from torch.nn.modules.module import register_module_forward_pre_hook
|
|
241
|
+
from torch.optim.optimizer import register_optimizer_step_post_hook, register_optimizer_step_pre_hook
|
|
242
|
+
|
|
243
|
+
self._handles.append(register_module_forward_pre_hook(self._module_pre))
|
|
244
|
+
self._handles.append(register_optimizer_step_pre_hook(self._opt_pre))
|
|
245
|
+
self._handles.append(register_optimizer_step_post_hook(self._opt_post))
|
|
246
|
+
for m in self._explicit_models:
|
|
247
|
+
self._hook_params(m)
|
|
248
|
+
self._mode.__enter__()
|
|
249
|
+
return self
|
|
250
|
+
|
|
251
|
+
def stop(self) -> Report:
|
|
252
|
+
if not self._active:
|
|
253
|
+
return self.report
|
|
254
|
+
self._active = False
|
|
255
|
+
try:
|
|
256
|
+
self._mode.__exit__(None, None, None)
|
|
257
|
+
except Exception:
|
|
258
|
+
pass
|
|
259
|
+
for h in self._handles:
|
|
260
|
+
h.remove()
|
|
261
|
+
self._handles.clear()
|
|
262
|
+
self._busy = True
|
|
263
|
+
try:
|
|
264
|
+
self._finalize()
|
|
265
|
+
finally:
|
|
266
|
+
for h in self._hooked.values():
|
|
267
|
+
try:
|
|
268
|
+
h.remove()
|
|
269
|
+
except Exception:
|
|
270
|
+
pass
|
|
271
|
+
self._hooked.clear()
|
|
272
|
+
self._busy = False
|
|
273
|
+
return self.report
|
|
274
|
+
|
|
275
|
+
def __enter__(self) -> Report:
|
|
276
|
+
self.start()
|
|
277
|
+
return self.report
|
|
278
|
+
|
|
279
|
+
def __exit__(self, *exc) -> bool:
|
|
280
|
+
self.stop()
|
|
281
|
+
return False
|
|
282
|
+
|
|
283
|
+
# ------------------------------------------------------------ helpers
|
|
284
|
+
|
|
285
|
+
def _add(self, f: Finding) -> None:
|
|
286
|
+
k = f.key()
|
|
287
|
+
if k in self._seen_findings:
|
|
288
|
+
return
|
|
289
|
+
self._seen_findings.add(k)
|
|
290
|
+
self.report.findings.append(f)
|
|
291
|
+
|
|
292
|
+
def _add_opt(self, opt) -> None:
|
|
293
|
+
if not any(r() is opt for r in self._optimizers):
|
|
294
|
+
self._optimizers.append(weakref.ref(opt))
|
|
295
|
+
|
|
296
|
+
def _hook_param(self, p: torch.Tensor) -> None:
|
|
297
|
+
pid = id(p)
|
|
298
|
+
if pid in self._hooked or not p.requires_grad:
|
|
299
|
+
return
|
|
300
|
+
self._param_refs[pid] = weakref.ref(p)
|
|
301
|
+
|
|
302
|
+
def hook(grad, pid=pid, pref=weakref.ref(p)):
|
|
303
|
+
if self._busy or not self._active or grad is None:
|
|
304
|
+
return None
|
|
305
|
+
self._busy = True
|
|
306
|
+
try:
|
|
307
|
+
self._got_grad.add(pid)
|
|
308
|
+
if pid not in self._nonzero_grad and bool(grad.detach().ne(0).any()):
|
|
309
|
+
self._nonzero_grad.add(pid)
|
|
310
|
+
p = pref()
|
|
311
|
+
old = None if p is None else p.grad
|
|
312
|
+
old_nonzero = old is not None and bool(old.detach().ne(0).any())
|
|
313
|
+
if pid in self._stepped:
|
|
314
|
+
# first backward since this parameter was stepped: old gradient should be gone
|
|
315
|
+
self._stepped.discard(pid)
|
|
316
|
+
if old_nonzero:
|
|
317
|
+
self._dirty.add(pid)
|
|
318
|
+
elif not old_nonzero:
|
|
319
|
+
self._dirty.discard(pid)
|
|
320
|
+
except Exception as e:
|
|
321
|
+
self._internal_errors.append(repr(e))
|
|
322
|
+
finally:
|
|
323
|
+
self._busy = False
|
|
324
|
+
return None
|
|
325
|
+
|
|
326
|
+
self._hooked[pid] = p.register_hook(hook)
|
|
327
|
+
|
|
328
|
+
def _hook_params(self, m: nn.Module) -> None:
|
|
329
|
+
for p in m.parameters():
|
|
330
|
+
self._hook_param(p)
|
|
331
|
+
|
|
332
|
+
def _mask_of(self, t: torch.Tensor) -> int:
|
|
333
|
+
m = self._masks.get(t)
|
|
334
|
+
if m is not None:
|
|
335
|
+
return m
|
|
336
|
+
b = self._pbit.get(t)
|
|
337
|
+
if b is None:
|
|
338
|
+
if t.grad_fn is not None or not t.is_leaf:
|
|
339
|
+
return 0
|
|
340
|
+
b = len(self._bit_param)
|
|
341
|
+
self._pbit[t] = b
|
|
342
|
+
self._bit_param.append(weakref.ref(t))
|
|
343
|
+
self._hook_param(t)
|
|
344
|
+
return 1 << b
|
|
345
|
+
|
|
346
|
+
# ------------------------------------------------------------ the op hook
|
|
347
|
+
|
|
348
|
+
def _on_op(self, func, args, kwargs, out) -> None:
|
|
349
|
+
name = _name(func)
|
|
350
|
+
if name == "backward":
|
|
351
|
+
self.report.backwards += 1
|
|
352
|
+
return
|
|
353
|
+
if name.startswith("attr:") and name != "attr:data" and name != "attr:T":
|
|
354
|
+
return
|
|
355
|
+
base = _base(name)
|
|
356
|
+
leaves = tree_leaves((args, kwargs))
|
|
357
|
+
ins = [t for t in leaves if isinstance(t, torch.Tensor)]
|
|
358
|
+
if not ins:
|
|
359
|
+
return
|
|
360
|
+
self._order += 1
|
|
361
|
+
if name == "attr:data" or _is_inplace(name):
|
|
362
|
+
self._manual_update(name, args, out)
|
|
363
|
+
|
|
364
|
+
if base in BROADCAST_OPS and len(ins) >= 2:
|
|
365
|
+
self._check_broadcast(base, ins)
|
|
366
|
+
if base in LOSS_SHAPE:
|
|
367
|
+
self._check_loss_shape(base, args, kwargs)
|
|
368
|
+
if base in ACT_LOSSES or base in ("softmax", "log_softmax"):
|
|
369
|
+
self._check_act_loss(base, args, kwargs)
|
|
370
|
+
|
|
371
|
+
outs = [o for o in tree_leaves(out) if isinstance(o, torch.Tensor)] if out is not None else []
|
|
372
|
+
|
|
373
|
+
# gradient taint
|
|
374
|
+
live = 0
|
|
375
|
+
for t in ins:
|
|
376
|
+
if t.requires_grad:
|
|
377
|
+
live |= self._mask_of(t)
|
|
378
|
+
elif (isinstance(t, nn.Parameter) and outs and not name.endswith("_") and torch.is_grad_enabled()
|
|
379
|
+
and t not in self._frozen_use): # a real read, not requires_grad_(False) or an in-place init
|
|
380
|
+
self._frozen_use[t] = where(3)
|
|
381
|
+
if live:
|
|
382
|
+
self._used_mask |= live
|
|
383
|
+
# first real use: an op that builds a gradient-carrying tensor from the parameter
|
|
384
|
+
# (not nn.init reading .dim() / .size() or filling it under no_grad)
|
|
385
|
+
fresh = live & ~self._first_mask
|
|
386
|
+
if fresh and any(o.requires_grad and o not in self._pbit for o in outs):
|
|
387
|
+
w = where(3)
|
|
388
|
+
self._first_mask |= fresh
|
|
389
|
+
while fresh:
|
|
390
|
+
low = fresh & -fresh
|
|
391
|
+
self._first_use[low.bit_length() - 1] = w
|
|
392
|
+
fresh ^= low
|
|
393
|
+
if not outs:
|
|
394
|
+
if base in VALUE_FUNCS or name in VALUE_FUNCS:
|
|
395
|
+
self._cut(live, "value", name)
|
|
396
|
+
else:
|
|
397
|
+
for o in outs:
|
|
398
|
+
if o.requires_grad:
|
|
399
|
+
if o in self._pbit:
|
|
400
|
+
continue
|
|
401
|
+
prev = self._masks.get(o)
|
|
402
|
+
self._masks[o] = live if prev is None else (prev | live)
|
|
403
|
+
else:
|
|
404
|
+
self._cut(live, self._cut_kind(name, base, o), name)
|
|
405
|
+
if base in ZERO_DERIV and any(o.requires_grad for o in outs):
|
|
406
|
+
if live & ~self._zero_cover:
|
|
407
|
+
w, c = where(2)
|
|
408
|
+
self._zero_ops.append(_Cut(live, "zero", base, w, c, self._order))
|
|
409
|
+
self._zero_cover |= live
|
|
410
|
+
|
|
411
|
+
# tags for the next ops
|
|
412
|
+
if outs:
|
|
413
|
+
o = outs[0]
|
|
414
|
+
if base in ACTIVATIONS:
|
|
415
|
+
self._act[o] = (ACTIVATIONS[base], where(2)[0])
|
|
416
|
+
elif base in PASS_THROUGH and ins[0] in self._act and o is not ins[0]:
|
|
417
|
+
self._act[o] = self._act[ins[0]]
|
|
418
|
+
elif base in MIXING and len(ins) >= 2:
|
|
419
|
+
# probabilities @ constant one-hot labels is still a probability mix (matching networks);
|
|
420
|
+
# attention (probabilities @ trainable values) is not, so require the other side to be constant
|
|
421
|
+
a, b = ins[0], ins[1]
|
|
422
|
+
for p, q in ((a, b), (b, a)):
|
|
423
|
+
tag = self._act.get(p)
|
|
424
|
+
if tag and tag[0] in ("softmax", "softmax-mix") and not q.requires_grad:
|
|
425
|
+
self._act[o] = ("softmax-mix", tag[1])
|
|
426
|
+
break
|
|
427
|
+
shaped = base in SHAPING or (
|
|
428
|
+
base == "getitem" and len(args) > 1 and (args[1] is None or (isinstance(args[1], tuple) and any(i is None for i in args[1])))
|
|
429
|
+
)
|
|
430
|
+
if shaped:
|
|
431
|
+
self._shaped[o] = True
|
|
432
|
+
|
|
433
|
+
@staticmethod
|
|
434
|
+
def _cut_kind(name: str, base: str, o: torch.Tensor) -> str:
|
|
435
|
+
if torch.is_inference_mode_enabled():
|
|
436
|
+
return "inference_mode"
|
|
437
|
+
if not torch.is_grad_enabled():
|
|
438
|
+
return "no_grad"
|
|
439
|
+
if base in ("detach",):
|
|
440
|
+
return "detach"
|
|
441
|
+
if name == "attr:data":
|
|
442
|
+
return "data"
|
|
443
|
+
if base == "requires_grad":
|
|
444
|
+
return "requires_grad_false"
|
|
445
|
+
if base in NEW_TENSOR:
|
|
446
|
+
return "new_tensor"
|
|
447
|
+
if not (o.is_floating_point() or o.is_complex()):
|
|
448
|
+
return "int"
|
|
449
|
+
return "other"
|
|
450
|
+
|
|
451
|
+
def _cut(self, mask: int, kind: str, op: str) -> None:
|
|
452
|
+
# torch's own bookkeeping (DDP syncing parameters, optimizers, clip_grad_norm_ ...)
|
|
453
|
+
# detaches on purpose; only cuts made by the user's code (or other libraries) count.
|
|
454
|
+
# no_grad / inference_mode are the user's choice when the op is torch's layer code
|
|
455
|
+
# (F.linear inside nn.Linear), but not inside DDP, optimizers or nn.utils.
|
|
456
|
+
by = torch_caller()
|
|
457
|
+
if by == "infra" or (by == "torch" and kind not in ("no_grad", "inference_mode")):
|
|
458
|
+
return
|
|
459
|
+
cover = self._cut_cover.get(kind, 0)
|
|
460
|
+
if not mask & ~cover:
|
|
461
|
+
return
|
|
462
|
+
self._cut_cover[kind] = cover | mask
|
|
463
|
+
w, c = where(3)
|
|
464
|
+
self._cuts.append(_Cut(mask, kind, op, w, c, self._order))
|
|
465
|
+
|
|
466
|
+
# ------------------------------------------------------------ shape and loss checks
|
|
467
|
+
|
|
468
|
+
def _check_broadcast(self, base: str, ins: list[torch.Tensor]) -> None:
|
|
469
|
+
ts = [t for t in ins[:3] if t.dim() > 0 and t.numel() > 1]
|
|
470
|
+
for i in range(len(ts)):
|
|
471
|
+
for j in range(i + 1, len(ts)):
|
|
472
|
+
a, b = ts[i], ts[j]
|
|
473
|
+
if a in self._shaped or b in self._shaped:
|
|
474
|
+
continue
|
|
475
|
+
sa, sb = _shape(a), _shape(b)
|
|
476
|
+
r = crossed_broadcast(sa, sb)
|
|
477
|
+
if r is None:
|
|
478
|
+
continue
|
|
479
|
+
outs, same = r
|
|
480
|
+
w, c = where(3)
|
|
481
|
+
self._add(explain.broadcast(base, SYMBOL.get(base), sa, sb, outs, same, w, c))
|
|
482
|
+
|
|
483
|
+
def _check_loss_shape(self, base: str, args, kwargs) -> None:
|
|
484
|
+
x = args[0] if len(args) > 0 else kwargs.get("input")
|
|
485
|
+
y = args[1] if len(args) > 1 else kwargs.get("target")
|
|
486
|
+
if not (isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor)):
|
|
487
|
+
return
|
|
488
|
+
sx, sy = _shape(x), _shape(y)
|
|
489
|
+
if sx == sy:
|
|
490
|
+
return
|
|
491
|
+
try:
|
|
492
|
+
so = tuple(torch.broadcast_shapes(sx, sy))
|
|
493
|
+
except RuntimeError:
|
|
494
|
+
return
|
|
495
|
+
w, c = where(3)
|
|
496
|
+
self._add(explain.loss_shape(base, sx, sy, so, w, c))
|
|
497
|
+
|
|
498
|
+
def _check_act_loss(self, base: str, args, kwargs) -> None:
|
|
499
|
+
x = args[0] if len(args) > 0 else kwargs.get("input")
|
|
500
|
+
if not isinstance(x, torch.Tensor):
|
|
501
|
+
return
|
|
502
|
+
tag = self._act.get(x)
|
|
503
|
+
if tag is None:
|
|
504
|
+
return
|
|
505
|
+
act, act_where = tag
|
|
506
|
+
f = explain.activation_into_loss(base, act, act_where, *where(3))
|
|
507
|
+
if f is not None:
|
|
508
|
+
self._add(f)
|
|
509
|
+
|
|
510
|
+
# ------------------------------------------------------------ module / optimizer hooks
|
|
511
|
+
|
|
512
|
+
def _module_pre(self, module: nn.Module, inputs) -> None:
|
|
513
|
+
if self._busy or not self._active:
|
|
514
|
+
return
|
|
515
|
+
# no optimizer (weights updated by hand): count backward passes as steps instead
|
|
516
|
+
if (self.max_steps is not None and self.report.steps == 0 and not self._optimizers
|
|
517
|
+
and self.report.backwards >= self.max_steps):
|
|
518
|
+
self.stop()
|
|
519
|
+
if self.stop_at_limit:
|
|
520
|
+
raise _Done()
|
|
521
|
+
return
|
|
522
|
+
self._busy = True
|
|
523
|
+
try:
|
|
524
|
+
mid = id(module)
|
|
525
|
+
if mid not in self._called:
|
|
526
|
+
self._called[mid] = weakref.ref(module)
|
|
527
|
+
self._hook_params(module)
|
|
528
|
+
if torch.is_grad_enabled() and isinstance(module, EVAL_SENSITIVE):
|
|
529
|
+
if not module.training and mid not in self._eval_seen:
|
|
530
|
+
self._eval_seen.add(mid)
|
|
531
|
+
self._eval_mods.append((weakref.ref(module), *where(2)))
|
|
532
|
+
if module.training and isinstance(module, nn.modules.batchnorm._BatchNorm):
|
|
533
|
+
x = inputs[0] if inputs else None
|
|
534
|
+
if isinstance(x, torch.Tensor) and x.dim() > 2 and x.shape[0] == 1:
|
|
535
|
+
self._add(explain.batchnorm_batch1(_shape(x), *where(2)))
|
|
536
|
+
except Exception as e:
|
|
537
|
+
self._internal_errors.append(repr(e))
|
|
538
|
+
finally:
|
|
539
|
+
self._busy = False
|
|
540
|
+
|
|
541
|
+
def _param_of(self, t) -> torch.Tensor | None:
|
|
542
|
+
if not isinstance(t, torch.Tensor):
|
|
543
|
+
return None
|
|
544
|
+
r = self._param_refs.get(id(t))
|
|
545
|
+
if r is not None and r() is t:
|
|
546
|
+
return t
|
|
547
|
+
r = self._data_of.get(t)
|
|
548
|
+
return r() if r is not None else None
|
|
549
|
+
|
|
550
|
+
def _manual_update(self, name: str, args, out) -> None:
|
|
551
|
+
"""Weights updated by hand (p.data.add_(...), p -= lr * p.grad under no_grad)."""
|
|
552
|
+
p = self._param_of(args[0]) if args else None
|
|
553
|
+
if p is None:
|
|
554
|
+
return
|
|
555
|
+
if name == "attr:data":
|
|
556
|
+
if isinstance(out, torch.Tensor):
|
|
557
|
+
self._data_of[out] = weakref.ref(p)
|
|
558
|
+
return
|
|
559
|
+
if p.grad is None: # initialisation, not an update
|
|
560
|
+
return
|
|
561
|
+
pid = id(p)
|
|
562
|
+
if pid in self._dirty:
|
|
563
|
+
self._dirty.discard(pid)
|
|
564
|
+
self._add(explain.grad_not_cleared(*where(4), manual=True))
|
|
565
|
+
self._stepped.add(pid)
|
|
566
|
+
|
|
567
|
+
def _opt_pre(self, opt, args, kwargs) -> None:
|
|
568
|
+
if not self._active:
|
|
569
|
+
return
|
|
570
|
+
self._add_opt(opt)
|
|
571
|
+
stale = [p for g in opt.param_groups for p in g["params"] if id(p) in self._dirty and p.grad is not None]
|
|
572
|
+
if stale:
|
|
573
|
+
for p in stale:
|
|
574
|
+
self._dirty.discard(id(p))
|
|
575
|
+
self._add(explain.grad_not_cleared(*where(2)))
|
|
576
|
+
self._busy = True # do not trace the update itself
|
|
577
|
+
|
|
578
|
+
def _opt_post(self, opt, args, kwargs) -> None:
|
|
579
|
+
if not self._active:
|
|
580
|
+
return
|
|
581
|
+
self._busy = False
|
|
582
|
+
self.report.steps += 1
|
|
583
|
+
for g in opt.param_groups:
|
|
584
|
+
for p in g["params"]:
|
|
585
|
+
self._stepped.add(id(p))
|
|
586
|
+
if self.max_steps is not None and self.report.steps >= self.max_steps:
|
|
587
|
+
self.stop()
|
|
588
|
+
if self.stop_at_limit:
|
|
589
|
+
raise _Done()
|
|
590
|
+
|
|
591
|
+
# ------------------------------------------------------------ analysis
|
|
592
|
+
|
|
593
|
+
def _models(self) -> list[nn.Module]:
|
|
594
|
+
mods = [r() for r in self._called.values()]
|
|
595
|
+
mods = [m for m in mods if m is not None] + [m for m in self._explicit_models if m is not None]
|
|
596
|
+
# torch.compile wraps the model in an OptimizedModule; report the original module's names
|
|
597
|
+
mods = [getattr(m, "_orig_mod", m) if isinstance(getattr(m, "_orig_mod", None), nn.Module) else m
|
|
598
|
+
for m in mods]
|
|
599
|
+
uniq: dict[int, nn.Module] = {}
|
|
600
|
+
for m in mods:
|
|
601
|
+
uniq.setdefault(id(m), m)
|
|
602
|
+
inner: set[int] = set()
|
|
603
|
+
for m in uniq.values():
|
|
604
|
+
for c in m.modules():
|
|
605
|
+
if c is not m:
|
|
606
|
+
inner.add(id(c))
|
|
607
|
+
roots = [m for mid, m in uniq.items() if mid not in inner and any(True for _ in m.parameters())]
|
|
608
|
+
return roots
|
|
609
|
+
|
|
610
|
+
def _finalize(self) -> None:
|
|
611
|
+
rep = self.report
|
|
612
|
+
models = self._models()
|
|
613
|
+
names: dict[int, str] = {}
|
|
614
|
+
owner: dict[int, nn.Module] = {}
|
|
615
|
+
counts: dict[str, int] = {}
|
|
616
|
+
for m in models:
|
|
617
|
+
counts[type(m).__name__] = counts.get(type(m).__name__, 0) + 1
|
|
618
|
+
seen_cls: dict[str, int] = {}
|
|
619
|
+
for m in models:
|
|
620
|
+
cls = type(m).__name__
|
|
621
|
+
label = cls
|
|
622
|
+
if counts[cls] > 1:
|
|
623
|
+
seen_cls[cls] = seen_cls.get(cls, 0) + 1
|
|
624
|
+
label = f"{cls}#{seen_cls[cls]}"
|
|
625
|
+
rep.models.append(label)
|
|
626
|
+
prefix = f"{label}." if len(models) > 1 else ""
|
|
627
|
+
for n, p in m.named_parameters(remove_duplicate=True):
|
|
628
|
+
names.setdefault(id(p), prefix + n)
|
|
629
|
+
owner.setdefault(id(p), m)
|
|
630
|
+
|
|
631
|
+
opts = [o for o in (r() for r in self._optimizers) if o is not None]
|
|
632
|
+
in_opt: set[int] = set()
|
|
633
|
+
opt_params: list[torch.Tensor] = []
|
|
634
|
+
for o in opts:
|
|
635
|
+
for gi, g in enumerate(o.param_groups):
|
|
636
|
+
for i, p in enumerate(g["params"]):
|
|
637
|
+
in_opt.add(id(p))
|
|
638
|
+
opt_params.append(p)
|
|
639
|
+
names.setdefault(id(p), f"optimizer group {gi} param {i} {tuple(p.shape)}")
|
|
640
|
+
|
|
641
|
+
# Which models are being trained?
|
|
642
|
+
trained: list[nn.Module] = []
|
|
643
|
+
for m in models:
|
|
644
|
+
ps = [p for p in m.parameters() if p.requires_grad]
|
|
645
|
+
if not ps:
|
|
646
|
+
continue
|
|
647
|
+
if opts:
|
|
648
|
+
if any(id(p) in in_opt for p in ps):
|
|
649
|
+
trained.append(m)
|
|
650
|
+
elif any(id(p) in self._got_grad for p in ps):
|
|
651
|
+
trained.append(m)
|
|
652
|
+
|
|
653
|
+
# optimizer entries that no model holds and that never got a gradient: replaced parameters
|
|
654
|
+
model_ids = {id(p) for m in models for p in m.parameters()}
|
|
655
|
+
stale = ([p for p in opt_params if id(p) not in model_ids and id(p) not in self._got_grad]
|
|
656
|
+
if opts and models else [])
|
|
657
|
+
stale_ids = {id(p) for p in stale}
|
|
658
|
+
|
|
659
|
+
candidates: dict[int, torch.Tensor] = {}
|
|
660
|
+
for m in trained:
|
|
661
|
+
for p in m.parameters():
|
|
662
|
+
if p.requires_grad:
|
|
663
|
+
candidates[id(p)] = p
|
|
664
|
+
for p in opt_params:
|
|
665
|
+
if p.requires_grad and id(p) not in stale_ids:
|
|
666
|
+
candidates[id(p)] = p
|
|
667
|
+
|
|
668
|
+
# parameters left out of every optimizer
|
|
669
|
+
if opts:
|
|
670
|
+
missing = [p for m in trained for p in m.parameters() if p.requires_grad and id(p) not in in_opt]
|
|
671
|
+
uniq = list({id(p): p for p in missing}.values())
|
|
672
|
+
if stale:
|
|
673
|
+
# the parameters the model now uses are the replacements of the stale ones
|
|
674
|
+
rep.findings.append(explain.stale_optimizer(len(stale), sum(p.numel() for p in stale),
|
|
675
|
+
[names.get(id(p), "?") for p in uniq]))
|
|
676
|
+
elif uniq:
|
|
677
|
+
rep.findings.append(explain.not_in_optimizer([names.get(id(p), "?") for p in uniq],
|
|
678
|
+
sum(p.numel() for p in uniq)))
|
|
679
|
+
|
|
680
|
+
# dead and zero-gradient parameters
|
|
681
|
+
if rep.backwards == 0 and self._got_grad:
|
|
682
|
+
rep.backwards = 1
|
|
683
|
+
if rep.backwards > 0 and self._got_grad:
|
|
684
|
+
groups: dict[tuple, list[str]] = {}
|
|
685
|
+
group_info: dict[tuple, tuple] = {}
|
|
686
|
+
for pid, p in candidates.items():
|
|
687
|
+
if pid in self._nonzero_grad:
|
|
688
|
+
continue
|
|
689
|
+
nm = names.get(pid, f"tensor {tuple(p.shape)}")
|
|
690
|
+
bit = self._pbit.get(p)
|
|
691
|
+
if pid in self._got_grad:
|
|
692
|
+
ev = self._first_event(self._zero_ops, bit)
|
|
693
|
+
key = ("zero", ev.op if ev else None, ev.where if ev else None)
|
|
694
|
+
group_info[key] = ("zero", ev, None)
|
|
695
|
+
elif bit is None or (bit not in self._first_use and self._first_event(self._cuts, bit) is None):
|
|
696
|
+
mod_name = nm.rsplit(".", 1)[0] if "." in nm else ""
|
|
697
|
+
sub = self._submodule(owner.get(pid), nm, names)
|
|
698
|
+
called = sub is not None and id(sub) in self._called
|
|
699
|
+
# name the outermost module that never ran (the skipped wrapper, not its pieces)
|
|
700
|
+
parts, bypass = nm.split(".")[:-1], None
|
|
701
|
+
for k in range(1, len(parts) + 1):
|
|
702
|
+
sm = self._submodule(owner.get(pid), ".".join(parts[:k]) + ".x", names)
|
|
703
|
+
if sm is not None and id(sm) not in self._called:
|
|
704
|
+
mod_name, called = ".".join(parts[:k]), False
|
|
705
|
+
bypass = self._bypass_use(sm)
|
|
706
|
+
break
|
|
707
|
+
key = ("unused", mod_name, called)
|
|
708
|
+
group_info[key] = ("unused", mod_name, called, bypass)
|
|
709
|
+
else:
|
|
710
|
+
ev = self._first_event(self._cuts, bit)
|
|
711
|
+
if ev is not None:
|
|
712
|
+
key = ("cut", ev.kind, ev.where, ev.op)
|
|
713
|
+
group_info[key] = ("cut", ev, None)
|
|
714
|
+
else:
|
|
715
|
+
fu = self._first_use.get(bit, (None, None))
|
|
716
|
+
key = ("orphan", fu[0])
|
|
717
|
+
group_info[key] = ("orphan", fu, None)
|
|
718
|
+
groups.setdefault(key, []).append(nm)
|
|
719
|
+
for key, plist in groups.items():
|
|
720
|
+
kind, a, b, *rest = group_info[key]
|
|
721
|
+
trained_by_opt = bool(opts)
|
|
722
|
+
if kind == "zero":
|
|
723
|
+
rep.findings.append(explain.zero_grad(plist, a))
|
|
724
|
+
elif kind == "unused":
|
|
725
|
+
w, c = rest[0] if rest and rest[0] else (None, None)
|
|
726
|
+
rep.findings.append(explain.unused(plist, a, b, trained_by_opt, w, c))
|
|
727
|
+
elif kind == "cut":
|
|
728
|
+
rep.findings.append(explain.cut(plist, a))
|
|
729
|
+
else:
|
|
730
|
+
rep.findings.append(explain.orphan(plist, *a))
|
|
731
|
+
|
|
732
|
+
# eval mode during training
|
|
733
|
+
if self._eval_mods and trained:
|
|
734
|
+
trained_ids = {id(c) for m in trained for c in m.modules()}
|
|
735
|
+
hits = [(r(), w, c) for r, w, c in self._eval_mods if r() is not None and id(r()) in trained_ids]
|
|
736
|
+
if hits:
|
|
737
|
+
whole = any(not m.training for m in trained)
|
|
738
|
+
rep.findings.append(explain.eval_mode([type(m).__name__ for m, _, _ in hits], whole, hits[0][1], hits[0][2]))
|
|
739
|
+
|
|
740
|
+
if self._internal_errors:
|
|
741
|
+
rep.findings.append(Finding("internal", CHECK, "silentbug hit an internal error and may have missed things",
|
|
742
|
+
"; ".join(sorted(set(self._internal_errors))[:3]),
|
|
743
|
+
"please report this at the project issue tracker"))
|
|
744
|
+
|
|
745
|
+
def _bypass_use(self, module: nn.Module) -> tuple[Where | None, Where | None] | None:
|
|
746
|
+
"""Where a parameter of a module that never ran was read directly (skipping its forward)."""
|
|
747
|
+
for p in module.parameters():
|
|
748
|
+
if p in self._frozen_use:
|
|
749
|
+
return self._frozen_use[p]
|
|
750
|
+
b = self._pbit.get(p)
|
|
751
|
+
if b is not None and b in self._first_use:
|
|
752
|
+
return self._first_use[b]
|
|
753
|
+
return None
|
|
754
|
+
|
|
755
|
+
@staticmethod
|
|
756
|
+
def _submodule(root: nn.Module | None, pname: str, names: dict) -> nn.Module | None:
|
|
757
|
+
if root is None or "." not in pname:
|
|
758
|
+
return root
|
|
759
|
+
path = pname.split(".")[:-1]
|
|
760
|
+
if len(path) and path[0] == f"{type(root).__name__}" or (path and "#" in path[0]):
|
|
761
|
+
path = path[1:]
|
|
762
|
+
m = root
|
|
763
|
+
for part in path:
|
|
764
|
+
m = getattr(m, part, None)
|
|
765
|
+
if m is None:
|
|
766
|
+
return None
|
|
767
|
+
return m if isinstance(m, nn.Module) else None
|
|
768
|
+
|
|
769
|
+
@staticmethod
|
|
770
|
+
def _first_event(events: list[_Cut], bit: int | None) -> _Cut | None:
|
|
771
|
+
if bit is None:
|
|
772
|
+
return None
|
|
773
|
+
hits = [e for e in events if e.mask >> bit & 1]
|
|
774
|
+
if not hits:
|
|
775
|
+
return None
|
|
776
|
+
rank = {k: i for i, k in enumerate(_CUT_PRIORITY)}
|
|
777
|
+
hits.sort(key=lambda e: (rank.get(e.kind, 99), e.order))
|
|
778
|
+
return hits[0]
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def check(*models: nn.Module, optimizer=None) -> Session:
|
|
782
|
+
"""Watch the code inside the `with` block for silent training bugs.
|
|
783
|
+
|
|
784
|
+
with silentbug.check() as report:
|
|
785
|
+
for step in range(3):
|
|
786
|
+
loss = loss_fn(model(x), y)
|
|
787
|
+
loss.backward()
|
|
788
|
+
opt.step(); opt.zero_grad()
|
|
789
|
+
print(report)
|
|
790
|
+
|
|
791
|
+
Models and optimizers are found on their own. Pass them in only if the model
|
|
792
|
+
is never called inside the block.
|
|
793
|
+
"""
|
|
794
|
+
return Session(*models, optimizer=optimizer)
|