loop-autopsy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loop_autopsy/__init__.py +57 -0
- loop_autopsy/config.py +109 -0
- loop_autopsy/detectors.py +279 -0
- loop_autopsy/growth.py +204 -0
- loop_autopsy/heatmap.py +171 -0
- loop_autopsy/magics.py +65 -0
- loop_autopsy/monitor.py +166 -0
- loop_autopsy/report.py +207 -0
- loop_autopsy/transformer.py +215 -0
- loop_autopsy-0.1.0.dist-info/METADATA +230 -0
- loop_autopsy-0.1.0.dist-info/RECORD +14 -0
- loop_autopsy-0.1.0.dist-info/WHEEL +5 -0
- loop_autopsy-0.1.0.dist-info/licenses/LICENSE +22 -0
- loop_autopsy-0.1.0.dist-info/top_level.txt +1 -0
loop_autopsy/__init__.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""loop-autopsy: stops runaway loops in Jupyter and shows the evidence for why
|
|
2
|
+
they never ended.
|
|
3
|
+
|
|
4
|
+
Usage in a notebook::
|
|
5
|
+
|
|
6
|
+
%load_ext loop_autopsy
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
|
|
13
|
+
from .config import Finding, LoopStopped, Settings # noqa: F401
|
|
14
|
+
|
|
15
|
+
_LOADED: dict = {}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def load_ipython_extension(ip) -> None:
|
|
19
|
+
from .heatmap import CellHealth
|
|
20
|
+
from .magics import AutopsyMagics
|
|
21
|
+
from .monitor import Monitor
|
|
22
|
+
from .report import display_report
|
|
23
|
+
from .transformer import RUNTIME_NAME, LoopTransformer
|
|
24
|
+
|
|
25
|
+
if id(ip) in _LOADED:
|
|
26
|
+
print("loop-autopsy is already loaded. Use %autopsy help for options.")
|
|
27
|
+
return
|
|
28
|
+
settings = Settings()
|
|
29
|
+
monitor = Monitor(ip, settings)
|
|
30
|
+
transformer = LoopTransformer(monitor.register, lambda: settings.max_watched, monitor.is_paused)
|
|
31
|
+
health = CellHealth(ip, settings, monitor.stats, show=display_report)
|
|
32
|
+
|
|
33
|
+
ip.user_ns[RUNTIME_NAME] = monitor
|
|
34
|
+
ip.user_ns_hidden[RUNTIME_NAME] = monitor # keep it out of %who / %whos
|
|
35
|
+
ip.ast_transformers.append(transformer)
|
|
36
|
+
ip.events.register("pre_run_cell", monitor.pre_run_cell)
|
|
37
|
+
health.register()
|
|
38
|
+
monitor.install_exception_handler()
|
|
39
|
+
ip.register_magics(AutopsyMagics(ip, monitor))
|
|
40
|
+
_LOADED[id(ip)] = (monitor, transformer, health)
|
|
41
|
+
print(f"loop-autopsy {__version__} is watching your loops. Type %autopsy help for options.")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def unload_ipython_extension(ip) -> None:
|
|
45
|
+
entry = _LOADED.pop(id(ip), None)
|
|
46
|
+
if entry is None:
|
|
47
|
+
return
|
|
48
|
+
monitor, transformer, health = entry
|
|
49
|
+
monitor.enabled = False # functions instrumented earlier keep working, unwatched
|
|
50
|
+
if transformer in ip.ast_transformers:
|
|
51
|
+
ip.ast_transformers.remove(transformer)
|
|
52
|
+
try:
|
|
53
|
+
ip.events.unregister("pre_run_cell", monitor.pre_run_cell)
|
|
54
|
+
except ValueError:
|
|
55
|
+
pass
|
|
56
|
+
health.unregister()
|
|
57
|
+
monitor.remove_exception_handler()
|
loop_autopsy/config.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import deque
|
|
4
|
+
from dataclasses import dataclass, field, fields
|
|
5
|
+
from typing import Any, Optional, Tuple
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class LoopStopped(BaseException):
|
|
9
|
+
"""Raised inside a watched loop to stop it.
|
|
10
|
+
|
|
11
|
+
It derives from BaseException (like KeyboardInterrupt) so that a normal
|
|
12
|
+
``except Exception:`` block inside the student's loop cannot swallow it.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, finding: "Finding"):
|
|
16
|
+
super().__init__(finding.title)
|
|
17
|
+
self.finding = finding
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Finding:
|
|
22
|
+
"""What a detection rule proved about a loop."""
|
|
23
|
+
|
|
24
|
+
kind: str # stuck | cycle | wrong_direction | jumped_over | growing_iterable | memory | time_limit
|
|
25
|
+
title: str
|
|
26
|
+
explanation: str
|
|
27
|
+
hint: str
|
|
28
|
+
culprits: Tuple[str, ...] = ()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class LoopInfo:
|
|
33
|
+
"""Static facts about one loop, collected when the cell is transformed."""
|
|
34
|
+
|
|
35
|
+
loop_id: int
|
|
36
|
+
kind: str # "while" | "for"
|
|
37
|
+
lineno: int
|
|
38
|
+
header: str
|
|
39
|
+
test_src: str = ""
|
|
40
|
+
cond_names: Tuple[str, ...] = ()
|
|
41
|
+
watched: Tuple[str, ...] = ()
|
|
42
|
+
compare: Optional[tuple] = None # (left, op, right); side = ("name", n) | ("const", v)
|
|
43
|
+
iter_name: Optional[str] = None
|
|
44
|
+
receivers: Tuple[str, ...] = () # names used as ``name.method(...)``; checked at run time
|
|
45
|
+
repeat_safe: bool = True
|
|
46
|
+
waits_for_user: bool = False
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class LoopState:
|
|
51
|
+
"""Run-time record of one execution of a loop."""
|
|
52
|
+
|
|
53
|
+
info: LoopInfo
|
|
54
|
+
started: float
|
|
55
|
+
replay_rows: int = 6
|
|
56
|
+
rounds: int = 0
|
|
57
|
+
checks: int = 0
|
|
58
|
+
stride: int = 1
|
|
59
|
+
next_check: int = 1
|
|
60
|
+
last_check: float = 0.0
|
|
61
|
+
history: deque = field(default_factory=deque)
|
|
62
|
+
scratch: dict = field(default_factory=dict)
|
|
63
|
+
stopped: Optional[Finding] = None
|
|
64
|
+
|
|
65
|
+
def __post_init__(self):
|
|
66
|
+
self.history = deque(maxlen=max(1, self.replay_rows))
|
|
67
|
+
self.last_check = self.started
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class Settings:
|
|
72
|
+
"""User-adjustable settings (``%autopsy config key=value``)."""
|
|
73
|
+
|
|
74
|
+
time_limit: float = 60.0
|
|
75
|
+
direction_after: int = 500
|
|
76
|
+
jump_after: int = 100
|
|
77
|
+
growth_after: int = 2000
|
|
78
|
+
memory_stop: float = 0.85
|
|
79
|
+
memory_warn: float = 0.75
|
|
80
|
+
max_output_lines: int = 1000
|
|
81
|
+
replay_rows: int = 6
|
|
82
|
+
max_watched: int = 25
|
|
83
|
+
heatmap: str = "auto" # on | off | auto
|
|
84
|
+
time_yellow: float = 1.0
|
|
85
|
+
time_red: float = 10.0
|
|
86
|
+
mem_yellow: float = 0.50
|
|
87
|
+
mem_red: float = 0.80
|
|
88
|
+
|
|
89
|
+
def update(self, key: str, raw: str) -> Any:
|
|
90
|
+
"""Set ``key`` from a string such as ``"30"``; returns the new value."""
|
|
91
|
+
names = {f.name: f for f in fields(self)}
|
|
92
|
+
if key not in names:
|
|
93
|
+
raise KeyError(f"unknown setting {key!r}; choose from: {', '.join(names)}")
|
|
94
|
+
current = getattr(self, key)
|
|
95
|
+
if isinstance(current, str):
|
|
96
|
+
value: Any = raw.strip().strip("'\"").lower()
|
|
97
|
+
if key == "heatmap" and value not in ("on", "off", "auto"):
|
|
98
|
+
raise ValueError("heatmap must be on, off or auto")
|
|
99
|
+
elif isinstance(current, int) and not isinstance(current, bool):
|
|
100
|
+
value = int(float(raw))
|
|
101
|
+
else:
|
|
102
|
+
value = float(raw)
|
|
103
|
+
if isinstance(value, (int, float)) and value < 0:
|
|
104
|
+
raise ValueError(f"{key} cannot be negative")
|
|
105
|
+
setattr(self, key, value)
|
|
106
|
+
return value
|
|
107
|
+
|
|
108
|
+
def as_dict(self) -> dict:
|
|
109
|
+
return {f.name: getattr(self, f.name) for f in fields(self)}
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Each ``check_*`` function receives the loop's static ``LoopInfo``, its run-time
|
|
3
|
+
``LoopState`` and ``values`` (the watched variables right now) and returns a
|
|
4
|
+
``Finding`` when it can prove the loop will not end, otherwise ``None``.
|
|
5
|
+
Private bookkeeping lives in ``state.scratch``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections import deque
|
|
11
|
+
from decimal import Decimal
|
|
12
|
+
from fractions import Fraction
|
|
13
|
+
from typing import Any, Dict, Optional
|
|
14
|
+
|
|
15
|
+
from .config import Finding, LoopInfo, LoopState, Settings
|
|
16
|
+
|
|
17
|
+
OPAQUE = object() # "cannot see inside this value"
|
|
18
|
+
SCALARS = (bool, int, float, complex, Fraction, Decimal)
|
|
19
|
+
SMALL = 50 # containers up to this size are fingerprinted by content
|
|
20
|
+
MAX_STORED = 50_000 # fingerprints kept per loop run
|
|
21
|
+
FULL_BUDGET = 1_000_000 # elements visited by one full fingerprint
|
|
22
|
+
# Methods on these types only change the object itself, which is fingerprinted.
|
|
23
|
+
BUILTIN_RECEIVERS = (str, bytes, list, dict, set, frozenset, tuple, deque, bytearray, int, float, complex)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# --------------------------------------------------------------------------- fingerprints
|
|
27
|
+
def cheap_fp(value: Any, depth: int = 0) -> Any:
|
|
28
|
+
"""Fast fingerprint. Equal states always give equal results; different
|
|
29
|
+
states *may* collide, which the full fingerprint then rules out."""
|
|
30
|
+
t = type(value)
|
|
31
|
+
if value is None or t in SCALARS:
|
|
32
|
+
return value
|
|
33
|
+
if t in (str, bytes):
|
|
34
|
+
return value if len(value) <= 64 else (t.__name__, len(value), hash(value))
|
|
35
|
+
if t is range:
|
|
36
|
+
return ("range", value.start, value.stop, value.step)
|
|
37
|
+
if depth >= 3:
|
|
38
|
+
return (t.__name__, id(value)) if t in (list, dict, set, deque, bytearray, tuple, frozenset) else OPAQUE
|
|
39
|
+
if t in (tuple, frozenset, list, set, deque, bytearray) and len(value) <= SMALL:
|
|
40
|
+
parts = tuple(cheap_fp(x, depth + 1) for x in value)
|
|
41
|
+
if any(p is OPAQUE for p in parts):
|
|
42
|
+
return OPAQUE
|
|
43
|
+
return (t.__name__, id(value) if t not in (tuple, frozenset) else 0) + parts
|
|
44
|
+
if t is dict and len(value) <= SMALL:
|
|
45
|
+
parts = tuple((cheap_fp(k, depth + 1), cheap_fp(v, depth + 1)) for k, v in value.items())
|
|
46
|
+
if any(p is OPAQUE for pair in parts for p in pair):
|
|
47
|
+
return OPAQUE
|
|
48
|
+
return ("dict", id(value)) + parts
|
|
49
|
+
if t in (tuple, frozenset, list, set, deque, bytearray, dict):
|
|
50
|
+
return (t.__name__, id(value), len(value))
|
|
51
|
+
return OPAQUE
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def full_fp(value: Any, budget: Optional[list] = None, depth: int = 0) -> Any:
|
|
55
|
+
"""Complete, content-based fingerprint (hashable), or OPAQUE."""
|
|
56
|
+
if budget is None:
|
|
57
|
+
budget = [FULL_BUDGET]
|
|
58
|
+
t = type(value)
|
|
59
|
+
if value is None or t in SCALARS or t in (str, bytes):
|
|
60
|
+
return value
|
|
61
|
+
if t is range:
|
|
62
|
+
return ("range", value.start, value.stop, value.step)
|
|
63
|
+
if depth > 20:
|
|
64
|
+
return OPAQUE # e.g. a list that contains itself
|
|
65
|
+
if t in (list, tuple, deque, bytearray, set, frozenset, dict):
|
|
66
|
+
budget[0] -= len(value)
|
|
67
|
+
if budget[0] < 0:
|
|
68
|
+
return OPAQUE
|
|
69
|
+
if t is dict:
|
|
70
|
+
parts = []
|
|
71
|
+
for k, v in value.items():
|
|
72
|
+
fk, fv = full_fp(k, budget, depth + 1), full_fp(v, budget, depth + 1)
|
|
73
|
+
if fk is OPAQUE or fv is OPAQUE:
|
|
74
|
+
return OPAQUE
|
|
75
|
+
parts.append((fk, fv))
|
|
76
|
+
return ("dict", tuple(parts))
|
|
77
|
+
parts = []
|
|
78
|
+
for x in value:
|
|
79
|
+
fx = full_fp(x, budget, depth + 1)
|
|
80
|
+
if fx is OPAQUE:
|
|
81
|
+
return OPAQUE
|
|
82
|
+
parts.append(fx)
|
|
83
|
+
if t in (set, frozenset):
|
|
84
|
+
return (t.__name__, frozenset(parts))
|
|
85
|
+
return (t.__name__, tuple(parts))
|
|
86
|
+
return OPAQUE
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _show(value: Any, width: int = 40) -> str:
|
|
90
|
+
text = repr(value)
|
|
91
|
+
return text if len(text) <= width else text[: width - 3] + "..."
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _numeric(x: Any) -> bool:
|
|
95
|
+
return isinstance(x, (int, float, Fraction, Decimal)) and not isinstance(x, bool)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# --------------------------------------------------------------------------- R1 / R2
|
|
99
|
+
def check_repeat(info: LoopInfo, state: LoopState, values: Dict[str, Any]) -> Optional[Finding]:
|
|
100
|
+
"""R1 stuck / R2 cycle: the loop returned to an exact earlier state."""
|
|
101
|
+
s = state.scratch
|
|
102
|
+
if info.kind != "while" or not info.repeat_safe or s.get("rep_off"):
|
|
103
|
+
return None
|
|
104
|
+
for name in info.receivers: # e.g. random.random(): hidden state, cannot prove anything
|
|
105
|
+
if type(values.get(name, OPAQUE)) not in BUILTIN_RECEIVERS:
|
|
106
|
+
s["rep_off"] = "calls"
|
|
107
|
+
return None
|
|
108
|
+
cheap = []
|
|
109
|
+
for name, value in values.items():
|
|
110
|
+
fp = cheap_fp(value)
|
|
111
|
+
if fp is OPAQUE:
|
|
112
|
+
s["rep_off"] = True
|
|
113
|
+
return None
|
|
114
|
+
cheap.append((name, fp))
|
|
115
|
+
try:
|
|
116
|
+
key = hash(tuple(cheap))
|
|
117
|
+
except TypeError:
|
|
118
|
+
s["rep_off"] = True
|
|
119
|
+
return None
|
|
120
|
+
seen = s.setdefault("rep_seen", {})
|
|
121
|
+
if key not in seen:
|
|
122
|
+
if len(seen) < MAX_STORED:
|
|
123
|
+
seen[key] = state.rounds
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
full = tuple((n, full_fp(v)) for n, v in values.items())
|
|
127
|
+
if any(fp is OPAQUE for _, fp in full):
|
|
128
|
+
s["rep_off"] = True
|
|
129
|
+
return None
|
|
130
|
+
try:
|
|
131
|
+
fkey = hash(full)
|
|
132
|
+
except TypeError:
|
|
133
|
+
s["rep_off"] = True
|
|
134
|
+
return None
|
|
135
|
+
full_seen = s.setdefault("rep_full", {})
|
|
136
|
+
if fkey not in full_seen:
|
|
137
|
+
if len(full_seen) < MAX_STORED:
|
|
138
|
+
full_seen[fkey] = state.rounds
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
if state.stride > 1:
|
|
142
|
+
# Sampled checks cannot tell the exact cycle length: re-measure every round.
|
|
143
|
+
s["rep_refine"] = True
|
|
144
|
+
state.stride = 1
|
|
145
|
+
state.next_check = state.rounds + 1
|
|
146
|
+
s["rep_seen"], s["rep_full"] = {}, {}
|
|
147
|
+
return None
|
|
148
|
+
distance = state.rounds - full_seen[fkey]
|
|
149
|
+
return _stuck_finding(info, values) if distance <= 1 else _cycle_finding(info, values, distance)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _cond_values(info: LoopInfo, values: Dict[str, Any]) -> str:
|
|
153
|
+
return ", ".join(f"`{n} = {_show(values[n])}`" for n in info.cond_names if n in values)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _stuck_finding(info: LoopInfo, values: Dict[str, Any]) -> Finding:
|
|
157
|
+
cond = info.test_src or "True"
|
|
158
|
+
if info.cond_names:
|
|
159
|
+
names = ", ".join(f"`{n}`" for n in info.cond_names)
|
|
160
|
+
explanation = (
|
|
161
|
+
f"Nothing changed between rounds: {_cond_values(info, values)} stayed exactly the same, "
|
|
162
|
+
f"so the condition `{cond}` is True every time and the loop can never end."
|
|
163
|
+
)
|
|
164
|
+
example = ""
|
|
165
|
+
if info.compare and info.compare[0][0] == "name":
|
|
166
|
+
var, op = info.compare[0][1], info.compare[1]
|
|
167
|
+
if op in ("<", "<="):
|
|
168
|
+
example = f" (for example `{var} += 1`)"
|
|
169
|
+
elif op in (">", ">="):
|
|
170
|
+
example = f" (for example `{var} -= 1`)"
|
|
171
|
+
hint = f"Add a line inside the loop that changes {names} so that `{cond}` eventually becomes False{example}."
|
|
172
|
+
else:
|
|
173
|
+
explanation = (
|
|
174
|
+
f"The condition `{cond}` never changes and nothing inside the loop changes either, "
|
|
175
|
+
f"so every round is identical and no `break` is ever reached."
|
|
176
|
+
)
|
|
177
|
+
hint = "Add a `break` that runs when the work is done, or use a condition that can become False."
|
|
178
|
+
return Finding("stuck", "stuck loop", explanation, hint, tuple(info.cond_names))
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _cycle_finding(info: LoopInfo, values: Dict[str, Any], k: int) -> Finding:
|
|
182
|
+
cond = info.test_src or "True"
|
|
183
|
+
shown = _cond_values(info, values)
|
|
184
|
+
explanation = (
|
|
185
|
+
f"Every {k} rounds the loop comes back to exactly the same values"
|
|
186
|
+
f"{' (' + shown + ')' if shown else ''}. "
|
|
187
|
+
f"Nothing else can change them, so it will repeat this {k}-round cycle forever."
|
|
188
|
+
)
|
|
189
|
+
hint = f"Look at the replay table: the values go round in a circle instead of moving towards making `{cond}` False."
|
|
190
|
+
return Finding("cycle", f"loop stuck in a {k}-round cycle", explanation, hint, tuple(info.cond_names))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# --------------------------------------------------------------------------- R3 / R4
|
|
194
|
+
def _side_value(side, values):
|
|
195
|
+
kind, data = side
|
|
196
|
+
return data if kind == "const" else values.get(data)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def check_direction(info: LoopInfo, state: LoopState, values: Dict[str, Any],
|
|
200
|
+
settings: Settings) -> Optional[Finding]:
|
|
201
|
+
"""R3 wrong direction and R4 jumped over, for simple comparisons."""
|
|
202
|
+
if info.kind != "while" or not info.compare:
|
|
203
|
+
return None
|
|
204
|
+
left, op, right = info.compare
|
|
205
|
+
a, b = _side_value(left, values), _side_value(right, values)
|
|
206
|
+
if not (_numeric(a) and _numeric(b)):
|
|
207
|
+
return None
|
|
208
|
+
s = state.scratch
|
|
209
|
+
moving = left if left[0] == "name" else right # the variable side we talk about
|
|
210
|
+
var = moving[1]
|
|
211
|
+
target = b if moving is left else a
|
|
212
|
+
now_val = values.get(var)
|
|
213
|
+
|
|
214
|
+
if op == "!=":
|
|
215
|
+
gap = b - a
|
|
216
|
+
prev = s.get("jump_prev")
|
|
217
|
+
if prev is not None and (prev > 0) != (gap > 0):
|
|
218
|
+
s["jump_crossed"] = (s.get("jump_prev_val"), now_val)
|
|
219
|
+
s["jump_streak"] = 0
|
|
220
|
+
elif s.get("jump_crossed") and prev is not None and abs(gap) > abs(prev):
|
|
221
|
+
s["jump_streak"] = s.get("jump_streak", 0) + 1
|
|
222
|
+
elif s.get("jump_crossed"):
|
|
223
|
+
s["jump_streak"] = 0
|
|
224
|
+
s["jump_prev"], s["jump_prev_val"] = gap, now_val
|
|
225
|
+
if s.get("jump_crossed") and s.get("jump_streak", 0) >= settings.jump_after:
|
|
226
|
+
before, after = s["jump_crossed"]
|
|
227
|
+
return Finding(
|
|
228
|
+
"jumped_over", "loop jumped over its stop value",
|
|
229
|
+
f"`{info.test_src}` only becomes False when `{var}` is exactly {_show(target)}. "
|
|
230
|
+
f"`{var}` went from {_show(before)} to {_show(after)}, jumping over {_show(target)}, "
|
|
231
|
+
f"and has moved further away ever since (now {_show(now_val)}).",
|
|
232
|
+
f"Use `<` or `>` instead of `!=` in the condition, or change the step so `{var}` "
|
|
233
|
+
f"lands exactly on {_show(target)}.",
|
|
234
|
+
(var,),
|
|
235
|
+
)
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
gap = (b - a) if op in ("<", "<=") else (a - b)
|
|
239
|
+
prev = s.get("dir_prev")
|
|
240
|
+
if prev is not None and gap > prev:
|
|
241
|
+
if s.get("dir_streak", 0) == 0:
|
|
242
|
+
s["dir_start"] = s.get("dir_prev_val")
|
|
243
|
+
s["dir_streak"] = s.get("dir_streak", 0) + 1
|
|
244
|
+
else:
|
|
245
|
+
s["dir_streak"] = 0
|
|
246
|
+
s["dir_prev"], s["dir_prev_val"] = gap, now_val
|
|
247
|
+
if s.get("dir_streak", 0) >= settings.direction_after:
|
|
248
|
+
return Finding(
|
|
249
|
+
"wrong_direction", "loop moving the wrong way",
|
|
250
|
+
f"The loop stops when `{info.test_src}` becomes False, which needs `{var}` to reach {_show(target)}. "
|
|
251
|
+
f"But over the last {s['dir_streak']:,} checks `{var}` went from {_show(s.get('dir_start'))} "
|
|
252
|
+
f"to {_show(now_val)}, moving further away from {_show(target)} every time.",
|
|
253
|
+
f"Check the line that updates `{var}`: should it go the other way (for example `+=` instead of `-=`)?",
|
|
254
|
+
(var,),
|
|
255
|
+
)
|
|
256
|
+
return None
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
# --------------------------------------------------------------------------- R7
|
|
260
|
+
def check_time(info: LoopInfo, state: LoopState, settings: Settings, now: float) -> Optional[Finding]:
|
|
261
|
+
"""R7: honest fallback when nothing could be proven."""
|
|
262
|
+
if settings.time_limit <= 0 or info.waits_for_user:
|
|
263
|
+
return None
|
|
264
|
+
elapsed = now - state.started
|
|
265
|
+
if elapsed < settings.time_limit:
|
|
266
|
+
return None
|
|
267
|
+
reason = ""
|
|
268
|
+
off = state.scratch.get("rep_off")
|
|
269
|
+
if info.kind == "while" and (not info.repeat_safe or off == "calls"):
|
|
270
|
+
reason = " (it calls functions whose effects loop-autopsy cannot check)"
|
|
271
|
+
elif info.kind == "while" and off:
|
|
272
|
+
reason = " (it uses values loop-autopsy cannot look inside)"
|
|
273
|
+
return Finding(
|
|
274
|
+
"time_limit", "loop ran too long",
|
|
275
|
+
f"This loop has been running for {elapsed:.0f} s ({state.rounds:,} rounds). loop-autopsy could not prove "
|
|
276
|
+
f"that it is stuck{reason}. It might just be slow, or stuck in a way the rules cannot see.",
|
|
277
|
+
"Use the replay table to see whether the values move towards the stop condition. "
|
|
278
|
+
f"To allow longer loops: `%autopsy config time_limit={int(settings.time_limit * 5)}`.",
|
|
279
|
+
)
|
loop_autopsy/growth.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Memory helpers and rules R5 (growing iterable) and R6 (memory).
|
|
2
|
+
|
|
3
|
+
No third-party dependency is required; psutil is used only if installed.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
11
|
+
|
|
12
|
+
from .config import Finding, LoopInfo, LoopState, Settings
|
|
13
|
+
|
|
14
|
+
_UNLIMITED = 1 << 60 # cgroup files use huge numbers for "no limit"
|
|
15
|
+
_limit_cache: List[Optional[int]] = []
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# --------------------------------------------------------------------------- memory readings
|
|
19
|
+
def format_bytes(n: Optional[float]) -> str:
|
|
20
|
+
if n is None:
|
|
21
|
+
return "?"
|
|
22
|
+
n = float(n)
|
|
23
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
24
|
+
if abs(n) < 1024 or unit == "TB":
|
|
25
|
+
return f"{n:.0f} {unit}" if unit in ("B", "KB") else f"{n:.1f} {unit}"
|
|
26
|
+
n /= 1024
|
|
27
|
+
return f"{n:.1f} TB" # pragma: no cover
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _read_int(path: str) -> Optional[int]:
|
|
31
|
+
try:
|
|
32
|
+
with open(path) as f:
|
|
33
|
+
text = f.read().strip()
|
|
34
|
+
return None if text in ("", "max") else int(text)
|
|
35
|
+
except (OSError, ValueError):
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def memory_limit(refresh: bool = False) -> Optional[int]:
|
|
40
|
+
"""Bytes this kernel may use: MEM_LIMIT -> cgroup v2 -> cgroup v1 -> physical RAM."""
|
|
41
|
+
if _limit_cache and not refresh:
|
|
42
|
+
return _limit_cache[0]
|
|
43
|
+
limit: Optional[int] = None
|
|
44
|
+
env = os.environ.get("MEM_LIMIT")
|
|
45
|
+
if env:
|
|
46
|
+
try:
|
|
47
|
+
limit = int(float(env))
|
|
48
|
+
except ValueError:
|
|
49
|
+
limit = None
|
|
50
|
+
if limit is None:
|
|
51
|
+
for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
|
|
52
|
+
value = _read_int(path)
|
|
53
|
+
if value and value < _UNLIMITED:
|
|
54
|
+
limit = value
|
|
55
|
+
break
|
|
56
|
+
if limit is None:
|
|
57
|
+
try:
|
|
58
|
+
import psutil # type: ignore
|
|
59
|
+
|
|
60
|
+
limit = int(psutil.virtual_memory().total)
|
|
61
|
+
except Exception:
|
|
62
|
+
try:
|
|
63
|
+
limit = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
|
|
64
|
+
except (ValueError, OSError, AttributeError):
|
|
65
|
+
limit = None
|
|
66
|
+
_limit_cache[:] = [limit]
|
|
67
|
+
return limit
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def current_rss() -> Optional[int]:
|
|
71
|
+
"""Resident memory of this process in bytes (best available method)."""
|
|
72
|
+
try:
|
|
73
|
+
with open("/proc/self/statm") as f:
|
|
74
|
+
pages = int(f.read().split()[1])
|
|
75
|
+
return pages * os.sysconf("SC_PAGE_SIZE")
|
|
76
|
+
except (OSError, ValueError, IndexError, AttributeError):
|
|
77
|
+
pass
|
|
78
|
+
try:
|
|
79
|
+
import psutil # type: ignore
|
|
80
|
+
|
|
81
|
+
return int(psutil.Process().memory_info().rss)
|
|
82
|
+
except Exception:
|
|
83
|
+
pass
|
|
84
|
+
try:
|
|
85
|
+
import resource
|
|
86
|
+
|
|
87
|
+
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
88
|
+
return peak if sys.platform == "darwin" else peak * 1024
|
|
89
|
+
except Exception: # pragma: no cover
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def memory_fraction() -> Optional[float]:
|
|
94
|
+
rss, limit = current_rss(), memory_limit()
|
|
95
|
+
if not rss or not limit:
|
|
96
|
+
return None
|
|
97
|
+
return rss / limit
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# --------------------------------------------------------------------------- sizes
|
|
101
|
+
def size_of(value: Any) -> int:
|
|
102
|
+
"""Approximate bytes used by ``value`` (pandas / NumPy aware)."""
|
|
103
|
+
try:
|
|
104
|
+
if hasattr(value, "memory_usage") and callable(value.memory_usage): # pandas
|
|
105
|
+
usage = value.memory_usage(deep=True)
|
|
106
|
+
return int(usage.sum() if hasattr(usage, "sum") else usage)
|
|
107
|
+
if hasattr(value, "nbytes") and isinstance(getattr(value, "nbytes"), int): # NumPy
|
|
108
|
+
return int(value.nbytes)
|
|
109
|
+
base = sys.getsizeof(value)
|
|
110
|
+
if isinstance(value, (list, tuple, set, frozenset)) and value:
|
|
111
|
+
sample = list(value)[:1000] if not isinstance(value, (list, tuple)) else value[:1000]
|
|
112
|
+
per_item = sum(sys.getsizeof(x) for x in sample) / len(sample)
|
|
113
|
+
return int(base + per_item * len(value))
|
|
114
|
+
if isinstance(value, dict) and value:
|
|
115
|
+
items = list(value.items())[:1000]
|
|
116
|
+
per_item = sum(sys.getsizeof(k) + sys.getsizeof(v) for k, v in items) / len(items)
|
|
117
|
+
return int(base + per_item * len(value))
|
|
118
|
+
return base
|
|
119
|
+
except Exception:
|
|
120
|
+
return 0
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def largest_variables(namespace: Dict[str, Any], top: int = 3) -> List[Tuple[str, int]]:
|
|
124
|
+
"""The ``top`` biggest user variables as ``(name, bytes)``."""
|
|
125
|
+
import types
|
|
126
|
+
|
|
127
|
+
sizes = []
|
|
128
|
+
for name, value in list(namespace.items()):
|
|
129
|
+
if name.startswith("_") or name in ("In", "Out", "exit", "quit", "get_ipython"):
|
|
130
|
+
continue
|
|
131
|
+
if isinstance(value, types.ModuleType) or callable(value):
|
|
132
|
+
continue
|
|
133
|
+
sizes.append((name, size_of(value)))
|
|
134
|
+
sizes.sort(key=lambda pair: pair[1], reverse=True)
|
|
135
|
+
return sizes[:top]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _lengths(values: Dict[str, Any]) -> Dict[str, int]:
|
|
139
|
+
out = {}
|
|
140
|
+
for name, value in values.items():
|
|
141
|
+
if isinstance(value, (list, dict, set, bytearray, str, bytes, tuple)) or hasattr(value, "__len__"):
|
|
142
|
+
try:
|
|
143
|
+
out[name] = len(value)
|
|
144
|
+
except Exception:
|
|
145
|
+
pass
|
|
146
|
+
return out
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def remember_start(state: LoopState, values: Dict[str, Any]) -> None:
|
|
150
|
+
"""Called on the first check of a loop run: remember container sizes."""
|
|
151
|
+
state.scratch["len0"] = _lengths(values)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# --------------------------------------------------------------------------- R5
|
|
155
|
+
def check_growing_iterable(info: LoopInfo, state: LoopState, values: Dict[str, Any],
|
|
156
|
+
settings: Settings) -> Optional[Finding]:
|
|
157
|
+
"""R5: ``for x in items`` while ``items`` keeps growing."""
|
|
158
|
+
name = info.iter_name
|
|
159
|
+
if info.kind != "for" or not name or name not in values:
|
|
160
|
+
return None
|
|
161
|
+
try:
|
|
162
|
+
now = len(values[name])
|
|
163
|
+
except TypeError:
|
|
164
|
+
return None
|
|
165
|
+
start = state.scratch.get("len0", {}).get(name)
|
|
166
|
+
if start is None or state.rounds < settings.growth_after:
|
|
167
|
+
return None
|
|
168
|
+
grew = now - start
|
|
169
|
+
if grew >= 0.9 * state.rounds:
|
|
170
|
+
return Finding(
|
|
171
|
+
"growing_iterable", "loop over a list that keeps growing",
|
|
172
|
+
f"You are looping over `{name}` while adding to it. It started with {start:,} items and now has "
|
|
173
|
+
f"{now:,}: it grows as fast as the loop reads it, so the loop never reaches the end.",
|
|
174
|
+
f"Loop over a copy (`for ... in {name}[:]:`) or put new items in a separate list.",
|
|
175
|
+
(name,),
|
|
176
|
+
)
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# --------------------------------------------------------------------------- R6
|
|
181
|
+
def check_memory(info: LoopInfo, state: LoopState, values: Dict[str, Any],
|
|
182
|
+
settings: Settings, fraction: Optional[float] = None) -> Optional[Finding]:
|
|
183
|
+
"""R6: stop before the kernel runs out of memory."""
|
|
184
|
+
if fraction is None:
|
|
185
|
+
fraction = memory_fraction()
|
|
186
|
+
if fraction is None or fraction < settings.memory_stop:
|
|
187
|
+
return None
|
|
188
|
+
start = state.scratch.get("len0", {})
|
|
189
|
+
now = _lengths(values)
|
|
190
|
+
growth = sorted(((now[n] - start.get(n, 0), n) for n in now), reverse=True)
|
|
191
|
+
culprit = growth[0] if growth and growth[0][0] > 0 else None
|
|
192
|
+
limit = memory_limit()
|
|
193
|
+
detail = ""
|
|
194
|
+
if culprit:
|
|
195
|
+
grew, name = culprit
|
|
196
|
+
detail = f" `{name}` grew from {start.get(name, 0):,} to {now[name]:,} items during this loop."
|
|
197
|
+
return Finding(
|
|
198
|
+
"memory", "loop is running out of memory",
|
|
199
|
+
f"Memory is at {fraction:.0%} of your {format_bytes(limit)} limit.{detail} "
|
|
200
|
+
f"The kernel would crash soon, so the loop was stopped first.",
|
|
201
|
+
"If the loop should end, check its stop condition. If you really need all that data, "
|
|
202
|
+
"process it in smaller chunks instead of keeping everything in one list.",
|
|
203
|
+
(culprit[1],) if culprit else (),
|
|
204
|
+
)
|