nnsight 0.5.0.dev8__tar.gz → 0.5.0.dev9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/PKG-INFO +1 -1
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/__init__.py +23 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/interleaver.py +115 -151
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/backwards.py +14 -18
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/iterator.py +7 -12
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight.egg-info/PKG-INFO +1 -1
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/.github/workflows/publish.yml +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/.github/workflows/python-app.yml +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/.gitignore +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/CHANGELOG.md +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/CODE_OF_CONDUCT.md +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/LICENSE +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/NNsight_Walkthrough.ipynb +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/NNsight_v0_3_guide.ipynb +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/NNsight_v0_4_guide.ipynb +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/README.md +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/conftest.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/pyproject.toml +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/pytest.ini +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/setup.cfg +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/setup.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/_c/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/_c/py_mount.c +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/config.yaml +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/backends/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/backends/base.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/backends/editing.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/backends/execution.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/backends/remote.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/batching.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/envoy.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/inject.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/serialization.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/base.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/editing.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/globals.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/invoker.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/tracer.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/intervention/tracing/util.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/log.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/base.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/diffusion.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/language.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/mixins/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/mixins/loadable.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/mixins/meta.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/mixins/remoteable.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/sampling.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/vllm.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/schema/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/schema/config.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/schema/request.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/schema/response.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/util.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight.egg-info/SOURCES.txt +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight.egg-info/dependency_links.txt +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight.egg-info/requires.txt +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight.egg-info/top_level.txt +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/tests/__init__.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/tests/test_diffusion.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/tests/test_lm.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/tests/test_tiny.py +0 -0
- {nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/tests/test_vllm.py +0 -0
|
@@ -95,6 +95,29 @@ def wrap_autocast(func):
|
|
|
95
95
|
DEFAULT_PATCHER.add(Patch(autocast, wrap_autocast(autocast.__init__), "__init__"))
|
|
96
96
|
|
|
97
97
|
|
|
98
|
+
from torch import Tensor
|
|
99
|
+
from .intervention.tracing.backwards import BackwardsTracer
|
|
100
|
+
from .intervention.tracing.base import WithBlockNotFoundError
|
|
101
|
+
|
|
102
|
+
def wrap_backward(func):
|
|
103
|
+
|
|
104
|
+
@wraps(func)
|
|
105
|
+
def inner(tensor: Tensor, *args, **kwargs):
|
|
106
|
+
|
|
107
|
+
try:
|
|
108
|
+
|
|
109
|
+
tracer = BackwardsTracer(tensor, func, *args, **kwargs)
|
|
110
|
+
|
|
111
|
+
except WithBlockNotFoundError:
|
|
112
|
+
|
|
113
|
+
return func(tensor, *args, **kwargs)
|
|
114
|
+
|
|
115
|
+
return tracer
|
|
116
|
+
|
|
117
|
+
return inner
|
|
118
|
+
|
|
119
|
+
DEFAULT_PATCHER.add(Patch(Tensor, wrap_backward(Tensor.backward), "backward"))
|
|
120
|
+
|
|
98
121
|
DEFAULT_PATCHER.__enter__()
|
|
99
122
|
|
|
100
123
|
|
|
@@ -10,8 +10,7 @@ from enum import Enum
|
|
|
10
10
|
from functools import wraps
|
|
11
11
|
from queue import Queue
|
|
12
12
|
from threading import Thread
|
|
13
|
-
from typing import
|
|
14
|
-
Union)
|
|
13
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
|
|
15
14
|
|
|
16
15
|
import torch
|
|
17
16
|
|
|
@@ -22,7 +21,6 @@ from .tracing.util import wrap_exception
|
|
|
22
21
|
|
|
23
22
|
if TYPE_CHECKING:
|
|
24
23
|
from .tracing.tracer import Cache, InterleavingTracer, Tracer
|
|
25
|
-
|
|
26
24
|
|
|
27
25
|
|
|
28
26
|
class Events(Enum):
|
|
@@ -32,7 +30,6 @@ class Events(Enum):
|
|
|
32
30
|
SWAP = "swap" # Request for a swap
|
|
33
31
|
END = "end" # Signal to end the execution
|
|
34
32
|
EXCEPTION = "exception" # Signal that an exception occurred
|
|
35
|
-
REGISTER = "register" # Signal that a mediator has been registered
|
|
36
33
|
SKIP = "skip" # Signal that an operation should be skipped
|
|
37
34
|
CONTINUE = "continue" # Signal that an operation should continue
|
|
38
35
|
|
|
@@ -50,10 +47,12 @@ class EarlyStopException(Exception):
|
|
|
50
47
|
|
|
51
48
|
pass
|
|
52
49
|
|
|
50
|
+
|
|
53
51
|
class SkipException(Exception):
|
|
54
52
|
"""
|
|
55
53
|
Exception raised to skip the execution of the model.
|
|
56
54
|
"""
|
|
55
|
+
|
|
57
56
|
def __init__(self, value: Any):
|
|
58
57
|
self.value = value
|
|
59
58
|
|
|
@@ -67,7 +66,13 @@ class Interleaver:
|
|
|
67
66
|
modification of intermediate values.
|
|
68
67
|
"""
|
|
69
68
|
|
|
70
|
-
def __init__(
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
invokers: List[Mediator],
|
|
72
|
+
tracer: InterleavingTracer,
|
|
73
|
+
batcher: Batcher = None,
|
|
74
|
+
user_cache: Optional[Cache] = None,
|
|
75
|
+
) -> None:
|
|
71
76
|
"""
|
|
72
77
|
Initialize an Interleaver with mediators.
|
|
73
78
|
|
|
@@ -76,7 +81,7 @@ class Interleaver:
|
|
|
76
81
|
"""
|
|
77
82
|
|
|
78
83
|
self.invokers = invokers
|
|
79
|
-
self.batcher = batcher
|
|
84
|
+
self.batcher = batcher if batcher is not None else Batcher()
|
|
80
85
|
self.tracer = tracer
|
|
81
86
|
self.mediators: Dict[str, Mediator] = {}
|
|
82
87
|
|
|
@@ -86,19 +91,18 @@ class Interleaver:
|
|
|
86
91
|
self.iteration_tracker = defaultdict(int)
|
|
87
92
|
|
|
88
93
|
self.user_cache: Optional[Cache] = user_cache
|
|
89
|
-
|
|
90
|
-
#TODO legacy?
|
|
94
|
+
|
|
95
|
+
# TODO legacy?
|
|
91
96
|
self.default_all = None
|
|
92
|
-
|
|
93
|
-
def iterate(self, provider:str):
|
|
94
|
-
|
|
97
|
+
|
|
98
|
+
def iterate(self, provider: str):
|
|
99
|
+
|
|
95
100
|
iteration = self.iteration_tracker[provider]
|
|
96
101
|
|
|
97
102
|
self.iteration_tracker[provider] += 1
|
|
98
103
|
|
|
99
104
|
return f"{provider}.i{iteration}"
|
|
100
105
|
|
|
101
|
-
|
|
102
106
|
def wrap(self, fn: Callable):
|
|
103
107
|
"""
|
|
104
108
|
Wrap a function to intercept inputs and outputs for intervention.
|
|
@@ -112,7 +116,7 @@ class Interleaver:
|
|
|
112
116
|
|
|
113
117
|
@wraps(fn)
|
|
114
118
|
def inner(module: torch.nn.Module, *args, **kwargs):
|
|
115
|
-
|
|
119
|
+
|
|
116
120
|
if torch.compiler.is_compiling():
|
|
117
121
|
return fn(module, *args, **kwargs)
|
|
118
122
|
|
|
@@ -120,9 +124,9 @@ class Interleaver:
|
|
|
120
124
|
return fn(module, *args, **kwargs)
|
|
121
125
|
|
|
122
126
|
provider = module.__path__
|
|
123
|
-
|
|
127
|
+
|
|
124
128
|
# Call our hook before the original __call__
|
|
125
|
-
|
|
129
|
+
|
|
126
130
|
try:
|
|
127
131
|
inputs = self.handle(self.iterate(f"{provider}.input"), (args, kwargs))
|
|
128
132
|
except SkipException as e:
|
|
@@ -130,9 +134,9 @@ class Interleaver:
|
|
|
130
134
|
else:
|
|
131
135
|
args, kwargs = inputs
|
|
132
136
|
value = fn(module, *args, **kwargs)
|
|
133
|
-
|
|
137
|
+
|
|
134
138
|
value = self.handle(self.iterate(f"{provider}.output"), value)
|
|
135
|
-
|
|
139
|
+
|
|
136
140
|
return value
|
|
137
141
|
|
|
138
142
|
return inner
|
|
@@ -221,31 +225,6 @@ class Interleaver:
|
|
|
221
225
|
|
|
222
226
|
return property(getter, setter)
|
|
223
227
|
|
|
224
|
-
def wrap_backward(self, fn: Callable):
|
|
225
|
-
"""
|
|
226
|
-
Wrap the backward method to intercept backpropagation.
|
|
227
|
-
|
|
228
|
-
Returns:
|
|
229
|
-
A wrapped version of the backward method
|
|
230
|
-
"""
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
def inner(tensor: torch.Tensor, *args, **kwargs):
|
|
234
|
-
|
|
235
|
-
from .tracing.backwards import BackwardsTracer
|
|
236
|
-
|
|
237
|
-
try:
|
|
238
|
-
|
|
239
|
-
tracer = BackwardsTracer(tensor, fn, self, *args, **kwargs)
|
|
240
|
-
|
|
241
|
-
except WithBlockNotFoundError:
|
|
242
|
-
|
|
243
|
-
return fn(tensor, *args, **kwargs)
|
|
244
|
-
|
|
245
|
-
return tracer
|
|
246
|
-
|
|
247
|
-
return inner
|
|
248
|
-
|
|
249
228
|
def check_cache_full(self):
|
|
250
229
|
"""
|
|
251
230
|
Print a warning if a module to be cached was missed.
|
|
@@ -255,15 +234,25 @@ class Interleaver:
|
|
|
255
234
|
if cache.modules:
|
|
256
235
|
if cache.include_inputs and cache.include_output:
|
|
257
236
|
for module in cache.modules:
|
|
258
|
-
if
|
|
259
|
-
|
|
237
|
+
if (
|
|
238
|
+
module not in cache.cache
|
|
239
|
+
or cache.cache[module].inputs is None
|
|
240
|
+
):
|
|
241
|
+
print(
|
|
242
|
+
"\033[33m"
|
|
243
|
+
+ "NNsight Warning: A module to be cached was missed! Consider defining the Cache before the module is called."
|
|
244
|
+
+ "\033[0m"
|
|
245
|
+
)
|
|
260
246
|
return
|
|
261
247
|
else:
|
|
262
248
|
if any(module not in cache.cache for module in cache.modules):
|
|
263
|
-
print(
|
|
249
|
+
print(
|
|
250
|
+
"\033[33m"
|
|
251
|
+
+ "NNsight Warning: A module to be cached was missed! Consider defining the Cache before the module is called."
|
|
252
|
+
+ "\033[0m"
|
|
253
|
+
)
|
|
264
254
|
return
|
|
265
255
|
|
|
266
|
-
|
|
267
256
|
def __enter__(self):
|
|
268
257
|
"""
|
|
269
258
|
Context manager entry point. Replaces torch.nn.Module.__call__ with wrapped version.
|
|
@@ -275,9 +264,6 @@ class Interleaver:
|
|
|
275
264
|
self.patcher = Patcher(
|
|
276
265
|
[
|
|
277
266
|
Patch(torch.nn.Module, self.wrap(torch.nn.Module.__call__), "__call__"),
|
|
278
|
-
Patch(
|
|
279
|
-
torch.Tensor, self.wrap_backward(torch.Tensor.backward), "backward"
|
|
280
|
-
),
|
|
281
267
|
]
|
|
282
268
|
)
|
|
283
269
|
|
|
@@ -322,25 +308,27 @@ class Interleaver:
|
|
|
322
308
|
# If any mediators are still waiting for their values for their events, they probably called an Envoy out of order
|
|
323
309
|
# Or their Envoy was not called.
|
|
324
310
|
for mediator in self.mediators.values():
|
|
325
|
-
|
|
311
|
+
|
|
326
312
|
if mediator.child is not None:
|
|
327
313
|
mediator = mediator.child
|
|
328
|
-
|
|
314
|
+
|
|
329
315
|
if not mediator.event_queue.empty():
|
|
330
316
|
requested_event, requester = mediator.event_queue.get()
|
|
331
|
-
|
|
317
|
+
|
|
332
318
|
if isinstance(requester, tuple):
|
|
333
319
|
requester = requester[0]
|
|
334
|
-
|
|
335
|
-
|
|
336
320
|
|
|
337
|
-
mediator.respond(
|
|
321
|
+
mediator.respond(
|
|
322
|
+
ValueError(
|
|
323
|
+
f"Execution complete but `{requester}` was not provided. Did you call an Envoy out of order? Investigate why this module was not called?"
|
|
324
|
+
)
|
|
325
|
+
)
|
|
338
326
|
mediator.wait()
|
|
339
|
-
|
|
327
|
+
|
|
340
328
|
if mediator.name.startswith("Iterator"):
|
|
341
329
|
try:
|
|
342
330
|
mediator.handle()
|
|
343
|
-
except ValueError as e:
|
|
331
|
+
except ValueError as e:
|
|
344
332
|
msg = f"Execution complete but `{requester}` was not provided. This was in an Iterator at iteration {mediator.iteration} so likely this iteration did not happen. If you were using `.iter[:]`, this is likely not an error."
|
|
345
333
|
warnings.warn(msg)
|
|
346
334
|
else:
|
|
@@ -364,15 +352,14 @@ class Interleaver:
|
|
|
364
352
|
The original or modified value
|
|
365
353
|
"""
|
|
366
354
|
|
|
367
|
-
|
|
368
355
|
old = self.batcher.current_value
|
|
369
|
-
|
|
356
|
+
|
|
370
357
|
self.batcher.current_value = value
|
|
371
358
|
|
|
372
359
|
batch_size = len(self.batcher.batch_groups)
|
|
373
360
|
skip_count = 0
|
|
374
361
|
skip_values = []
|
|
375
|
-
|
|
362
|
+
|
|
376
363
|
for mediator in self.invokers:
|
|
377
364
|
|
|
378
365
|
try:
|
|
@@ -381,28 +368,31 @@ class Interleaver:
|
|
|
381
368
|
skip_count += 1
|
|
382
369
|
skip_values.append(e.value)
|
|
383
370
|
|
|
384
|
-
if skip_count == batch_size:
|
|
371
|
+
if skip_count == batch_size and batch_size > 0:
|
|
372
|
+
|
|
385
373
|
def _swap(*args):
|
|
386
374
|
return torch.cat(args, dim=0)
|
|
387
375
|
|
|
388
376
|
skip_value = applyn(skip_values, _swap, torch.Tensor)
|
|
389
377
|
raise SkipException(skip_value)
|
|
390
378
|
elif skip_count > 0 and skip_count < batch_size:
|
|
391
|
-
raise ValueError(
|
|
392
|
-
|
|
379
|
+
raise ValueError(
|
|
380
|
+
f"A module skip must be applied to all the invokers defined in the tracer!"
|
|
381
|
+
)
|
|
382
|
+
|
|
393
383
|
value = self.batcher.current_value
|
|
394
|
-
|
|
384
|
+
|
|
395
385
|
self.batcher.current_value = old
|
|
396
386
|
|
|
397
|
-
if len(self.user_cache) > 0:
|
|
387
|
+
if self.user_cache is not None and len(self.user_cache) > 0:
|
|
398
388
|
for cache in self.user_cache:
|
|
399
389
|
cache.add(provider, value)
|
|
400
|
-
|
|
390
|
+
|
|
401
391
|
return value
|
|
402
392
|
|
|
403
393
|
def cancel(self):
|
|
404
394
|
"""Cancel all intervention threads."""
|
|
405
|
-
|
|
395
|
+
|
|
406
396
|
for mediator in list(self.mediators.values()):
|
|
407
397
|
mediator.cancel()
|
|
408
398
|
|
|
@@ -418,7 +408,6 @@ class Interleaver:
|
|
|
418
408
|
return self.mediators[threading.current_thread().name]
|
|
419
409
|
|
|
420
410
|
|
|
421
|
-
|
|
422
411
|
class Mediator:
|
|
423
412
|
"""
|
|
424
413
|
Mediates between the model execution and intervention functions.
|
|
@@ -432,6 +421,7 @@ class Mediator:
|
|
|
432
421
|
"""
|
|
433
422
|
Exception raised when interventions are defined out of order.
|
|
434
423
|
"""
|
|
424
|
+
|
|
435
425
|
pass
|
|
436
426
|
|
|
437
427
|
def __init__(
|
|
@@ -478,7 +468,7 @@ class Mediator:
|
|
|
478
468
|
|
|
479
469
|
self.thread = Thread(
|
|
480
470
|
target=self.intervention,
|
|
481
|
-
args=(self, self.info,
|
|
471
|
+
args=(self, self.info, *self.args),
|
|
482
472
|
daemon=True,
|
|
483
473
|
name=self.name,
|
|
484
474
|
)
|
|
@@ -505,7 +495,7 @@ class Mediator:
|
|
|
505
495
|
self.history.clear()
|
|
506
496
|
|
|
507
497
|
self.thread = None
|
|
508
|
-
|
|
498
|
+
|
|
509
499
|
self.state = None
|
|
510
500
|
|
|
511
501
|
if self.thread is not None and self.thread.is_alive():
|
|
@@ -540,23 +530,23 @@ class Mediator:
|
|
|
540
530
|
self.child = None
|
|
541
531
|
|
|
542
532
|
# Continue to handle parent event
|
|
533
|
+
else:
|
|
534
|
+
return
|
|
543
535
|
|
|
544
536
|
process = not self.event_queue.empty()
|
|
545
537
|
|
|
546
538
|
event = None
|
|
547
|
-
|
|
539
|
+
|
|
548
540
|
while process:
|
|
549
|
-
|
|
541
|
+
|
|
550
542
|
value = self.interleaver.batcher.current_value
|
|
551
543
|
|
|
552
544
|
event, data = self.event_queue.get()
|
|
553
|
-
|
|
545
|
+
|
|
554
546
|
if event == Events.VALUE:
|
|
555
547
|
process = self.handle_value_event(data, provider, value)
|
|
556
548
|
elif event == Events.SWAP:
|
|
557
549
|
process = self.handle_swap_event(provider, *data)
|
|
558
|
-
elif event == Events.REGISTER:
|
|
559
|
-
process = self.handle_register_event(*data)
|
|
560
550
|
elif event == Events.EXCEPTION:
|
|
561
551
|
process = self.handle_exception_event(data)
|
|
562
552
|
elif event == Events.SKIP:
|
|
@@ -579,8 +569,13 @@ class Mediator:
|
|
|
579
569
|
if len(self.user_cache) > 0 and provider is not None:
|
|
580
570
|
|
|
581
571
|
for cache in self.user_cache:
|
|
582
|
-
cache.add(
|
|
583
|
-
|
|
572
|
+
cache.add(
|
|
573
|
+
provider,
|
|
574
|
+
self.interleaver.batcher.narrow(
|
|
575
|
+
self.batch_group, self.interleaver.batcher.current_value
|
|
576
|
+
),
|
|
577
|
+
)
|
|
578
|
+
|
|
584
579
|
def handle_end_event(self):
|
|
585
580
|
"""
|
|
586
581
|
Handle an end event by stopping the mediator.
|
|
@@ -621,7 +616,7 @@ class Mediator:
|
|
|
621
616
|
|
|
622
617
|
return True
|
|
623
618
|
|
|
624
|
-
def handle_swap_event(self, provider: Any, requester: Any,
|
|
619
|
+
def handle_swap_event(self, provider: Any, requester: Any, swap_value: Any):
|
|
625
620
|
"""
|
|
626
621
|
Handle a swap event by swapping the value if the provider matches the requester.
|
|
627
622
|
|
|
@@ -634,9 +629,9 @@ class Mediator:
|
|
|
634
629
|
Boolean indicating whether to continue processing events
|
|
635
630
|
"""
|
|
636
631
|
if provider == requester:
|
|
637
|
-
|
|
632
|
+
|
|
638
633
|
self.interleaver.batcher.swap(self.batch_group, swap_value)
|
|
639
|
-
|
|
634
|
+
|
|
640
635
|
self.respond()
|
|
641
636
|
|
|
642
637
|
return True
|
|
@@ -675,17 +670,17 @@ class Mediator:
|
|
|
675
670
|
raise exception
|
|
676
671
|
|
|
677
672
|
return False
|
|
678
|
-
|
|
679
|
-
def handle_skip_event(self, provider:Any, requester: Any, value: Any):
|
|
680
|
-
|
|
673
|
+
|
|
674
|
+
def handle_skip_event(self, provider: Any, requester: Any, value: Any):
|
|
675
|
+
|
|
681
676
|
if provider == requester:
|
|
682
|
-
|
|
677
|
+
|
|
683
678
|
self.respond()
|
|
684
679
|
|
|
685
680
|
self.history.add(provider)
|
|
686
|
-
|
|
681
|
+
|
|
687
682
|
raise SkipException(value)
|
|
688
|
-
|
|
683
|
+
|
|
689
684
|
else:
|
|
690
685
|
if requester in self.history:
|
|
691
686
|
self.respond(
|
|
@@ -698,39 +693,8 @@ class Mediator:
|
|
|
698
693
|
else:
|
|
699
694
|
self.history.add(provider)
|
|
700
695
|
self.event_queue.put((Events.SKIP, (requester, value)))
|
|
701
|
-
|
|
702
|
-
return False
|
|
703
|
-
|
|
704
|
-
def handle_register_event(self, mediator: Mediator, fn: Optional[Callable] = None):
|
|
705
|
-
"""
|
|
706
|
-
Handle a register event by registering a new mediator.
|
|
707
|
-
|
|
708
|
-
Args:
|
|
709
|
-
mediator: The mediator to register
|
|
710
|
-
|
|
711
|
-
Returns:
|
|
712
|
-
Boolean indicating whether to continue processing events
|
|
713
|
-
"""
|
|
714
|
-
|
|
715
|
-
self.child = mediator
|
|
716
|
-
|
|
717
|
-
self.child.info.start_line += self.info.start_line - 1
|
|
718
696
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
self.response_queue.put(None)
|
|
722
|
-
|
|
723
|
-
if fn is not None:
|
|
724
|
-
|
|
725
|
-
fn()
|
|
726
|
-
|
|
727
|
-
self.response_queue.put(None)
|
|
728
|
-
|
|
729
|
-
self.wait()
|
|
730
|
-
|
|
731
|
-
self.handle()
|
|
732
|
-
|
|
733
|
-
return False
|
|
697
|
+
return False
|
|
734
698
|
|
|
735
699
|
def respond(self, value: Optional[Any] = None):
|
|
736
700
|
"""
|
|
@@ -762,9 +726,9 @@ class Mediator:
|
|
|
762
726
|
if frame and frame.f_code.co_filename.startswith("<nnsight"):
|
|
763
727
|
break
|
|
764
728
|
return frame
|
|
765
|
-
|
|
729
|
+
|
|
766
730
|
def iterate(self, requester: Any):
|
|
767
|
-
|
|
731
|
+
|
|
768
732
|
return f"{requester}.i{self.iteration}"
|
|
769
733
|
|
|
770
734
|
def push(self):
|
|
@@ -820,24 +784,10 @@ class Mediator:
|
|
|
820
784
|
The requested value
|
|
821
785
|
"""
|
|
822
786
|
|
|
823
|
-
|
|
824
787
|
value = self.send(Events.VALUE, requester)
|
|
825
788
|
|
|
826
789
|
return value
|
|
827
790
|
|
|
828
|
-
def register(self, mediator: Mediator, fn: Optional[Callable] = None):
|
|
829
|
-
|
|
830
|
-
self.send(Events.REGISTER, (mediator, fn))
|
|
831
|
-
|
|
832
|
-
self.response_queue.get()
|
|
833
|
-
|
|
834
|
-
if fn is not None:
|
|
835
|
-
self.event_queue.put((Events.CONTINUE, None))
|
|
836
|
-
|
|
837
|
-
self.response_queue.get()
|
|
838
|
-
|
|
839
|
-
self.pull()
|
|
840
|
-
|
|
841
791
|
def swap(self, requester: Any, value: Any):
|
|
842
792
|
"""
|
|
843
793
|
Set a value to swap during execution.
|
|
@@ -858,18 +808,26 @@ class Mediator:
|
|
|
858
808
|
iteration: The number of iterations
|
|
859
809
|
"""
|
|
860
810
|
|
|
861
|
-
|
|
811
|
+
def do_iteration(iter: int):
|
|
812
|
+
|
|
813
|
+
mediator.iteration = iter
|
|
814
|
+
mediator.args = list([mediator.iteration])
|
|
815
|
+
self.child = mediator
|
|
816
|
+
mediator.start(self.interleaver)
|
|
817
|
+
|
|
818
|
+
self.event_queue.put((Events.CONTINUE, None))
|
|
819
|
+
|
|
820
|
+
self.response_queue.get()
|
|
821
|
+
|
|
822
|
+
if isinstance(iteration, slice):
|
|
862
823
|
|
|
863
824
|
i = iteration.start if iteration.start is not None else self.iteration
|
|
864
|
-
|
|
825
|
+
|
|
865
826
|
stop = iteration.stop
|
|
866
827
|
|
|
867
828
|
while True:
|
|
868
829
|
|
|
869
|
-
|
|
870
|
-
mediator.args = list([mediator.iteration])
|
|
871
|
-
|
|
872
|
-
self.register(mediator)
|
|
830
|
+
do_iteration(i)
|
|
873
831
|
|
|
874
832
|
if self.interleaver.default_all is not None and stop is None:
|
|
875
833
|
stop = self.interleaver.default_all
|
|
@@ -878,23 +836,29 @@ class Mediator:
|
|
|
878
836
|
|
|
879
837
|
if stop is not None and i >= stop:
|
|
880
838
|
break
|
|
881
|
-
|
|
839
|
+
|
|
840
|
+
elif isinstance(iteration, list):
|
|
841
|
+
|
|
842
|
+
iteration.sort()
|
|
843
|
+
|
|
844
|
+
for i in iteration:
|
|
845
|
+
do_iteration(i)
|
|
846
|
+
|
|
882
847
|
elif isinstance(iteration, int):
|
|
883
848
|
|
|
884
|
-
|
|
885
|
-
|
|
849
|
+
do_iteration(iteration)
|
|
850
|
+
|
|
851
|
+
self.child = None
|
|
886
852
|
|
|
887
|
-
self.register(mediator)
|
|
888
|
-
|
|
889
853
|
def stop(self):
|
|
890
854
|
"""Stop the execution of the model by raising an EarlyStopException."""
|
|
891
855
|
|
|
892
856
|
self.push()
|
|
893
857
|
|
|
894
858
|
raise EarlyStopException()
|
|
895
|
-
|
|
859
|
+
|
|
896
860
|
def skip(self, requester: Any, value: Any):
|
|
897
|
-
|
|
861
|
+
|
|
898
862
|
self.send(Events.SKIP, (requester, value))
|
|
899
863
|
|
|
900
864
|
def end(self):
|
|
@@ -927,21 +891,21 @@ class Mediator:
|
|
|
927
891
|
|
|
928
892
|
def __getstate__(self):
|
|
929
893
|
"""Get the state of the mediator for serialization."""
|
|
930
|
-
|
|
894
|
+
|
|
931
895
|
return {
|
|
932
896
|
"name": self.name,
|
|
933
897
|
"info": self.info,
|
|
934
898
|
"batch_group": self.batch_group,
|
|
935
899
|
"intervention": self.intervention,
|
|
936
900
|
}
|
|
937
|
-
|
|
901
|
+
|
|
938
902
|
def __setstate__(self, state):
|
|
939
903
|
"""Set the state of the mediator for deserialization."""
|
|
940
904
|
self.name = state["name"]
|
|
941
905
|
self.info = state["info"]
|
|
942
906
|
self.batch_group = state["batch_group"]
|
|
943
907
|
self.intervention = state["intervention"]
|
|
944
|
-
|
|
908
|
+
|
|
945
909
|
self.event_queue = Queue()
|
|
946
910
|
self.response_queue = Queue()
|
|
947
911
|
|
|
@@ -951,4 +915,4 @@ class Mediator:
|
|
|
951
915
|
self.history = set()
|
|
952
916
|
self.user_cache: "Cache" = list()
|
|
953
917
|
self.iteration = 0
|
|
954
|
-
self.args = list()
|
|
918
|
+
self.args = list()
|
|
@@ -3,13 +3,9 @@ from typing import TYPE_CHECKING, Any, Callable
|
|
|
3
3
|
import torch
|
|
4
4
|
|
|
5
5
|
from ...util import Patch
|
|
6
|
-
from ..interleaver import Mediator
|
|
6
|
+
from ..interleaver import Mediator, Interleaver
|
|
7
7
|
from .invoker import Invoker
|
|
8
8
|
|
|
9
|
-
if TYPE_CHECKING:
|
|
10
|
-
from ..interleaver import Interleaver
|
|
11
|
-
else:
|
|
12
|
-
Interleaver = Any
|
|
13
9
|
|
|
14
10
|
|
|
15
11
|
class BackwardsMediator(Mediator):
|
|
@@ -30,7 +26,6 @@ class BackwardsTracer(Invoker):
|
|
|
30
26
|
self,
|
|
31
27
|
tensor: torch.Tensor,
|
|
32
28
|
fn: Callable,
|
|
33
|
-
interleaver: Interleaver,
|
|
34
29
|
*args,
|
|
35
30
|
**kwargs,
|
|
36
31
|
):
|
|
@@ -39,20 +34,21 @@ class BackwardsTracer(Invoker):
|
|
|
39
34
|
|
|
40
35
|
self.tensor = tensor
|
|
41
36
|
self.fn = fn
|
|
42
|
-
self.interleaver = interleaver
|
|
43
37
|
|
|
44
38
|
def execute(self, fn: Callable):
|
|
45
39
|
|
|
46
40
|
mediator = BackwardsMediator(fn, self.info)
|
|
47
41
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
42
|
+
|
|
43
|
+
interleaver = Interleaver([mediator], self)
|
|
44
|
+
grad_patch = Patch(torch.Tensor, interleaver.wrap_grad(), "grad")
|
|
45
|
+
|
|
46
|
+
try:
|
|
47
|
+
|
|
48
|
+
with interleaver:
|
|
49
|
+
interleaver.patcher.add(grad_patch)
|
|
50
|
+
interleaver(self.fn, self.tensor, *self.args, **self.kwargs)
|
|
51
|
+
self.push(interleaver.state)
|
|
52
|
+
|
|
53
|
+
finally:
|
|
54
|
+
interleaver.state.clear()
|
|
@@ -1,25 +1,20 @@
|
|
|
1
1
|
from typing import Callable, TYPE_CHECKING, Any, Union
|
|
2
|
-
from .
|
|
3
|
-
from ..interleaver import Mediator
|
|
2
|
+
from .base import Tracer
|
|
3
|
+
from ..interleaver import Interleaver, Mediator
|
|
4
4
|
from .util import try_catch
|
|
5
5
|
|
|
6
|
-
if TYPE_CHECKING:
|
|
7
|
-
from .tracer import InterleavingTracer
|
|
8
|
-
else:
|
|
9
|
-
InterleavingTracer = Any
|
|
10
|
-
|
|
11
6
|
class IteratorProxy:
|
|
12
7
|
|
|
13
|
-
def __init__(self, interleaver):
|
|
8
|
+
def __init__(self, interleaver: Interleaver):
|
|
14
9
|
self.interleaver = interleaver
|
|
15
10
|
|
|
16
11
|
def __getitem__(self, iteration: Union[int, slice]):
|
|
17
|
-
return IteratorTracer(iteration, self.interleaver
|
|
12
|
+
return IteratorTracer(iteration, self.interleaver)
|
|
18
13
|
|
|
19
|
-
class IteratorTracer(
|
|
14
|
+
class IteratorTracer(Tracer):
|
|
20
15
|
|
|
21
|
-
def __init__(self, iteration: Union[int, slice], interleaver
|
|
22
|
-
super().__init__(
|
|
16
|
+
def __init__(self, iteration: Union[int, slice], interleaver: Interleaver):
|
|
17
|
+
super().__init__()
|
|
23
18
|
|
|
24
19
|
self.interleaver = interleaver
|
|
25
20
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py
RENAMED
|
File without changes
|
{nnsight-0.5.0.dev8 → nnsight-0.5.0.dev9}/src/nnsight/modeling/vllm/model_runners/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|