nnsight 0.2.2.dev0__py3-none-any.whl → 0.2.3.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nnsight/__init__.py +17 -140
- nnsight/config.yaml +2 -1
- nnsight/contexts/Invoker.py +38 -58
- nnsight/contexts/Runner.py +22 -58
- nnsight/contexts/Tracer.py +79 -34
- nnsight/contexts/__init__.py +9 -11
- nnsight/envoy.py +323 -0
- nnsight/intervention.py +106 -148
- nnsight/models/LanguageModel.py +158 -37
- nnsight/models/Mamba.py +310 -44
- nnsight/models/NNsightModel.py +236 -314
- nnsight/models/UnifiedTransformer.py +30 -68
- nnsight/models/__init__.py +6 -18
- nnsight/models/mixins/Generation.py +47 -0
- nnsight/models/mixins/__init__.py +1 -0
- nnsight/nnsight.log +23 -0
- nnsight/pydantics/Config.py +19 -1
- nnsight/pydantics/Request.py +0 -4
- nnsight/pydantics/format/functions.py +2 -1
- nnsight/tracing/Graph.py +40 -156
- nnsight/tracing/Node.py +11 -26
- nnsight/tracing/Proxy.py +94 -45
- nnsight/util.py +19 -69
- {nnsight-0.2.2.dev0.dist-info → nnsight-0.2.3.dev0.dist-info}/METADATA +65 -132
- nnsight-0.2.3.dev0.dist-info/RECORD +51 -0
- nnsight/contexts/DirectInvoker.py +0 -37
- nnsight/editing/Editor.py +0 -29
- nnsight/editing/GraphEdit.py +0 -22
- nnsight/editing/WrapperModuleEdit.py +0 -22
- nnsight/editing/__init__.py +0 -1
- nnsight/models/DiffuserModel.py +0 -189
- nnsight/module.py +0 -248
- nnsight-0.2.2.dev0.dist-info/RECORD +0 -55
- {nnsight-0.2.2.dev0.dist-info → nnsight-0.2.3.dev0.dist-info}/LICENCE +0 -0
- {nnsight-0.2.2.dev0.dist-info → nnsight-0.2.3.dev0.dist-info}/WHEEL +0 -0
- {nnsight-0.2.2.dev0.dist-info → nnsight-0.2.3.dev0.dist-info}/top_level.txt +0 -0
nnsight/__init__.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from functools import wraps
|
|
1
2
|
import os
|
|
2
3
|
|
|
3
4
|
import yaml
|
|
@@ -10,174 +11,50 @@ with open(os.path.join(PATH, "config.yaml"), "r") as file:
|
|
|
10
11
|
CONFIG = ConfigModel(**yaml.safe_load(file))
|
|
11
12
|
|
|
12
13
|
from .logger import logger
|
|
13
|
-
from .models.
|
|
14
|
+
from .models.NNsightModel import NNsight
|
|
14
15
|
from .models.LanguageModel import LanguageModel
|
|
15
|
-
|
|
16
|
-
from .module import Module
|
|
16
|
+
|
|
17
17
|
from .patching import Patch, Patcher
|
|
18
18
|
from .tracing.Proxy import proxy_wrapper
|
|
19
19
|
|
|
20
20
|
logger.disabled = not CONFIG.APP.LOGGING
|
|
21
21
|
|
|
22
|
-
|
|
23
22
|
# Below do default patching:
|
|
24
23
|
DEFAULT_PATCHER = Patcher()
|
|
25
24
|
|
|
26
|
-
from functools import wraps
|
|
27
25
|
from inspect import getmembers, isfunction
|
|
28
26
|
|
|
29
27
|
import einops
|
|
30
|
-
import torch
|
|
31
28
|
|
|
32
29
|
for key, value in getmembers(einops.einops, isfunction):
|
|
33
30
|
DEFAULT_PATCHER.add(Patch(einops.einops, proxy_wrapper(value), key))
|
|
34
31
|
|
|
35
32
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
# Need to patch repeat_interleave to work with meta tensors
|
|
40
|
-
# Computes appropriate shape if meta. Otherwise just call repeat_interleave
|
|
41
|
-
def repeat_interleave_wrapper(fn):
|
|
42
|
-
@wraps(fn)
|
|
43
|
-
def repeat_interleave(
|
|
44
|
-
input: torch.Tensor, repeats: torch.LongTensor, dim=None, output_size=None
|
|
45
|
-
):
|
|
46
|
-
if input.device.type == "meta":
|
|
47
|
-
if not isinstance(repeats, torch.Tensor):
|
|
48
|
-
repeats = torch.LongTensor([repeats])
|
|
49
|
-
|
|
50
|
-
if dim is None:
|
|
51
|
-
input = input.flatten()
|
|
52
|
-
dim = 0
|
|
53
|
-
|
|
54
|
-
if repeats.dim() == 0 or (repeats.dim() == 1 and repeats.size(0) == 1):
|
|
55
|
-
repeats = repeats.reshape([1]).expand([input.size(dim)])
|
|
56
|
-
|
|
57
|
-
new_dim_size = repeats.cumsum(0)[-1].item()
|
|
58
|
-
new_output_shape = list(input.shape)
|
|
59
|
-
new_output_shape[dim] = new_dim_size
|
|
60
|
-
|
|
61
|
-
return torch.empty(new_output_shape, device="meta")
|
|
62
|
-
|
|
63
|
-
else:
|
|
64
|
-
return fn(input, repeats, dim=dim, output_size=output_size)
|
|
65
|
-
|
|
66
|
-
return repeat_interleave
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
DEFAULT_PATCHER.add(
|
|
70
|
-
Patch(
|
|
71
|
-
torch, repeat_interleave_wrapper(torch.repeat_interleave), "repeat_interleave"
|
|
72
|
-
)
|
|
73
|
-
)
|
|
74
|
-
|
|
33
|
+
from torch._subclasses.fake_tensor import FakeTensor
|
|
75
34
|
|
|
76
|
-
def noop_wrapper(fn):
|
|
77
|
-
@wraps(fn)
|
|
78
|
-
def noop(input: torch.Tensor, *args, **kwargs):
|
|
79
|
-
if input.device.type == "meta":
|
|
80
|
-
return input
|
|
81
|
-
|
|
82
|
-
else:
|
|
83
|
-
return fn(input, *args, **kwargs)
|
|
84
35
|
|
|
85
|
-
|
|
36
|
+
def _bool(self):
|
|
37
|
+
return True
|
|
86
38
|
|
|
87
39
|
|
|
88
|
-
DEFAULT_PATCHER.add(Patch(
|
|
40
|
+
DEFAULT_PATCHER.add(Patch(FakeTensor, _bool, "__bool__"))
|
|
89
41
|
|
|
90
42
|
|
|
91
|
-
def
|
|
43
|
+
def fake_tensor_new_wrapper(fn):
|
|
44
|
+
|
|
92
45
|
@wraps(fn)
|
|
93
|
-
def
|
|
94
|
-
|
|
95
|
-
|
|
46
|
+
def inner(cls, fake_mode, elem, device, constant=None):
|
|
47
|
+
|
|
48
|
+
if isinstance(elem, FakeTensor):
|
|
96
49
|
|
|
50
|
+
return elem
|
|
51
|
+
|
|
97
52
|
else:
|
|
98
|
-
return fn(input, num_classes=num_classes)
|
|
99
|
-
|
|
100
|
-
return onehot
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
DEFAULT_PATCHER.add(
|
|
104
|
-
Patch(torch.nn.functional, onehot_wrapper(torch.nn.functional.one_hot), "one_hot")
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
DEFAULT_PATCHER.add(Patch(torch.Tensor, noop_wrapper(torch.Tensor.tolist), "tolist"))
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def meta_nonzero(input: torch.Tensor, *args, as_tuple=False, **kwargs):
|
|
112
|
-
output = torch.zeros((input.numel(), input.ndim), device="meta", dtype=torch.long)
|
|
113
|
-
|
|
114
|
-
if as_tuple:
|
|
115
|
-
return tuple([output[:, i] for i in range(input.ndim)])
|
|
116
|
-
|
|
117
|
-
return output
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def meta_nonzero_wrapper(fn):
|
|
121
|
-
@wraps(fn)
|
|
122
|
-
def inner(input: torch.Tensor, *args, **kwargs):
|
|
123
|
-
if input.device.type == "meta":
|
|
124
|
-
return meta_nonzero(input, *args, **kwargs)
|
|
125
|
-
|
|
126
|
-
else:
|
|
127
|
-
return fn(input, *args, **kwargs)
|
|
128
53
|
|
|
54
|
+
return fn(cls, fake_mode, elem, device, constant=constant)
|
|
55
|
+
|
|
129
56
|
return inner
|
|
130
57
|
|
|
131
|
-
|
|
132
|
-
DEFAULT_PATCHER.add(
|
|
133
|
-
Patch(torch.Tensor, meta_nonzero_wrapper(torch.Tensor.nonzero), "nonzero")
|
|
134
|
-
)
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def meta_where_wrapper(fn):
|
|
138
|
-
@wraps(fn)
|
|
139
|
-
def where(input: torch.Tensor, *args, **kwargs):
|
|
140
|
-
if input.device.type == "meta":
|
|
141
|
-
if len(args) > 0:
|
|
142
|
-
dtype = (
|
|
143
|
-
args[0].dtype
|
|
144
|
-
if isinstance(args[0], torch.Tensor)
|
|
145
|
-
else type(args[0])
|
|
146
|
-
)
|
|
147
|
-
if isinstance(args[0], torch.Tensor):
|
|
148
|
-
return torch.zeros_like(torch.broadcast_tensors(input, args[0])[0], dtype=input.dtype, device="meta")
|
|
149
|
-
return torch.zeros_like(input, dtype=input.dtype, device="meta")
|
|
150
|
-
return meta_nonzero(input, as_tuple=True)
|
|
151
|
-
|
|
152
|
-
else:
|
|
153
|
-
return fn(input, *args, **kwargs)
|
|
154
|
-
|
|
155
|
-
return where
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
DEFAULT_PATCHER.add(Patch(torch, meta_where_wrapper(torch.where), "where"))
|
|
159
|
-
|
|
58
|
+
DEFAULT_PATCHER.add(Patch(FakeTensor, fake_tensor_new_wrapper(FakeTensor.__new__), "__new__"))
|
|
160
59
|
|
|
161
60
|
DEFAULT_PATCHER.__enter__()
|
|
162
|
-
|
|
163
|
-
from torch._meta_registrations import (_meta_lib_dont_use_me_use_register_meta,
|
|
164
|
-
aten, global_decomposition_table,
|
|
165
|
-
register_meta)
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
# Function which "activates" the most recent meta registered function.
|
|
169
|
-
def activate_recent_meta():
|
|
170
|
-
op_overload, fn = list(global_decomposition_table["meta"].items())[-1]
|
|
171
|
-
op_overload.py_impl(torch._C.DispatchKey.Meta)(fn)
|
|
172
|
-
_meta_lib_dont_use_me_use_register_meta.impl(op_overload, fn)
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
# Need to patch local_scalar_dense_meta for meta.
|
|
176
|
-
# In non-meta tensors this returns the singular value in tensors with one value
|
|
177
|
-
# In meta, lets just return 0
|
|
178
|
-
@register_meta(aten._local_scalar_dense)
|
|
179
|
-
def local_scalar_dense_meta(A):
|
|
180
|
-
return 0
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
activate_recent_meta()
|
nnsight/config.yaml
CHANGED
nnsight/contexts/Invoker.py
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import copy
|
|
3
4
|
from contextlib import AbstractContextManager
|
|
4
|
-
from typing import Any, Dict
|
|
5
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple
|
|
5
6
|
|
|
6
|
-
import
|
|
7
|
+
from torch._subclasses.fake_tensor import FakeCopyMode, FakeTensorMode
|
|
8
|
+
from torch.fx.experimental.symbolic_shapes import ShapeEnv
|
|
7
9
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
from .Tracer import Tracer
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
|
|
12
|
+
from .Tracer import Tracer
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class Invoker(AbstractContextManager):
|
|
@@ -15,62 +17,67 @@ class Invoker(AbstractContextManager):
|
|
|
15
17
|
|
|
16
18
|
Attributes:
|
|
17
19
|
tracer (nnsight.contexts.Tracer.Tracer): Tracer object to enter input and manage context.
|
|
18
|
-
|
|
19
|
-
scan (bool): If to
|
|
20
|
-
in order to update the potential sizes/dtypes of all module's inputs/outputs as well as validate things work correctly.
|
|
20
|
+
inputs (Tuple[Any]): Initially entered inputs, then post-processed inputs from model's ._prepare_inputs(...) method.
|
|
21
|
+
scan (bool): If to execute the model using `FakeTensor` in order to update the potential sizes/dtypes of all modules' Envoys' inputs/outputs as well as validate things work correctly.
|
|
21
22
|
Scanning is not free computation wise so you may want to turn this to false when running in a loop.
|
|
22
23
|
When making interventions, you made get shape errors if scan is false as it validates operations based on shapes so
|
|
23
24
|
for looped calls where shapes are consistent, you may want to have scan=True for the first loop. Defaults to True.
|
|
24
|
-
args (List[Any]): Positional arguments passed to the model's _prepare_inputs method.
|
|
25
25
|
kwargs (Dict[str,Any]): Keyword arguments passed to the model's _prepare_inputs method.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
28
|
def __init__(
|
|
29
29
|
self,
|
|
30
|
-
tracer: Tracer,
|
|
31
|
-
|
|
32
|
-
*args,
|
|
30
|
+
tracer: "Tracer",
|
|
31
|
+
*inputs: Tuple[Any],
|
|
33
32
|
scan: bool = True,
|
|
34
33
|
**kwargs,
|
|
35
34
|
) -> None:
|
|
35
|
+
|
|
36
36
|
self.tracer = tracer
|
|
37
|
-
self.
|
|
37
|
+
self.inputs = inputs
|
|
38
38
|
self.scan = scan
|
|
39
|
-
self.args = args
|
|
40
39
|
self.kwargs = kwargs
|
|
41
40
|
|
|
42
41
|
def __enter__(self) -> Invoker:
|
|
43
42
|
"""Enters a new invocation context with a given input.
|
|
44
43
|
|
|
45
|
-
Sets the generation_idx to 0.
|
|
46
44
|
Calls the model's _prepare_inputs method using the input and other arguments.
|
|
47
|
-
If scan is True, uses the model's
|
|
48
|
-
Gets a batched version of the post processed input using the model's _batched_inputs method to update the Tracer's
|
|
45
|
+
If scan is True, uses the model's ._execute method to update and validate module Envoy's inputs/outputs using a fake mode.
|
|
46
|
+
Gets a batched version of the post processed input using the model's ._batched_inputs method to update the Tracer's
|
|
49
47
|
current batch_size and batched_input.
|
|
50
48
|
|
|
51
49
|
Returns:
|
|
52
50
|
Invoker: Invoker.
|
|
53
51
|
"""
|
|
54
|
-
# Were in a new invocation so set generation_idx to 0,
|
|
55
|
-
self.tracer.generation_idx = 0
|
|
56
52
|
|
|
57
|
-
self.
|
|
58
|
-
|
|
53
|
+
self.tracer._invoker = self
|
|
54
|
+
|
|
55
|
+
self.inputs, batch_size = self.tracer._model._prepare_inputs(
|
|
56
|
+
*self.inputs, **self.kwargs
|
|
59
57
|
)
|
|
60
58
|
|
|
61
59
|
if self.scan:
|
|
62
|
-
self.tracer.
|
|
60
|
+
self.tracer._model._envoy._clear()
|
|
61
|
+
|
|
62
|
+
with FakeTensorMode(
|
|
63
|
+
allow_non_fake_inputs=True,
|
|
64
|
+
shape_env=ShapeEnv(assume_static_by_default=True),
|
|
65
|
+
) as fake_mode:
|
|
66
|
+
with FakeCopyMode(fake_mode):
|
|
67
|
+
self.tracer._model._execute(
|
|
68
|
+
*copy.deepcopy(self.inputs),
|
|
69
|
+
**copy.deepcopy(self.tracer._kwargs),
|
|
70
|
+
)
|
|
63
71
|
else:
|
|
64
|
-
|
|
65
|
-
if isinstance(module, Module):
|
|
66
|
-
module.clear()
|
|
72
|
+
self.tracer._model._envoy._reset()
|
|
67
73
|
|
|
68
|
-
self.tracer.
|
|
74
|
+
self.tracer._batch_start += self.tracer._batch_size
|
|
75
|
+
self.tracer._batch_size = batch_size
|
|
69
76
|
|
|
70
|
-
(
|
|
71
|
-
self.tracer.
|
|
72
|
-
self.
|
|
73
|
-
)
|
|
77
|
+
self.tracer._batched_input = self.tracer._model._batch_inputs(
|
|
78
|
+
self.tracer._batched_input,
|
|
79
|
+
*self.inputs,
|
|
80
|
+
)
|
|
74
81
|
|
|
75
82
|
return self
|
|
76
83
|
|
|
@@ -78,31 +85,4 @@ class Invoker(AbstractContextManager):
|
|
|
78
85
|
if isinstance(exc_val, BaseException):
|
|
79
86
|
raise exc_val
|
|
80
87
|
|
|
81
|
-
|
|
82
|
-
return self.tracer.graph.add(target=target, args=args, kwargs=kwargs)
|
|
83
|
-
|
|
84
|
-
def next(self, increment: int = 1) -> None:
|
|
85
|
-
"""Designates subsequent interventions should be applied to the next generation for multi-iteration generation runs.
|
|
86
|
-
|
|
87
|
-
Args:
|
|
88
|
-
increment (int): How many generation_idx to increment at once. Defaults to 1.
|
|
89
|
-
"""
|
|
90
|
-
# .next() increases which generation idx the interventions happen.
|
|
91
|
-
self.tracer.generation_idx += increment
|
|
92
|
-
|
|
93
|
-
for name, module in self.tracer.model.meta_model.named_modules():
|
|
94
|
-
if isinstance(module, Module):
|
|
95
|
-
module.clear()
|
|
96
|
-
|
|
97
|
-
def save_all(self) -> Dict[str, Proxy]:
|
|
98
|
-
"""Saves the output of all modules and returns a dictionary of [module_path -> save proxy]
|
|
99
|
-
|
|
100
|
-
Returns:
|
|
101
|
-
Dict[str, Proxy]: Dictionary of all modules saved, keyed by their module_path.
|
|
102
|
-
"""
|
|
103
|
-
result = {}
|
|
104
|
-
|
|
105
|
-
for name, module in self.tracer.model.meta_model.named_modules():
|
|
106
|
-
result[module.module_path] = module.output.save()
|
|
107
|
-
|
|
108
|
-
return result
|
|
88
|
+
self.tracer._invoker = None
|
nnsight/contexts/Runner.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import io
|
|
4
|
-
import pickle
|
|
5
4
|
import torch
|
|
6
5
|
import requests
|
|
7
6
|
import socketio
|
|
@@ -9,37 +8,13 @@ from tqdm import tqdm
|
|
|
9
8
|
|
|
10
9
|
from .. import CONFIG, pydantics
|
|
11
10
|
from ..logger import logger
|
|
12
|
-
from .Invoker import Invoker
|
|
13
11
|
from .Tracer import Tracer
|
|
14
12
|
|
|
15
13
|
|
|
16
14
|
class Runner(Tracer):
|
|
17
|
-
"""The Runner object manages the intervention tracing for a given model's
|
|
18
|
-
|
|
19
|
-
Examples:
|
|
20
|
-
|
|
21
|
-
Below example shows accessing the input and output of a gpt2 module, saving them during execution, and printing the resulting values after execution:
|
|
22
|
-
|
|
23
|
-
.. code-block:: python
|
|
24
|
-
|
|
25
|
-
with model.generate() as generator:
|
|
26
|
-
with generator.invoke('The Eiffel Tower is in the city of') as invoker:
|
|
27
|
-
hidden_states = model.lm_head.input.save()
|
|
28
|
-
logits = model.lm_head.output.save()
|
|
29
|
-
|
|
30
|
-
print(hidden_states.value)
|
|
31
|
-
print(logits.value)
|
|
32
|
-
|
|
33
|
-
Below example shows accessing the output of a gpt2 module and setting the values to zero:
|
|
34
|
-
|
|
35
|
-
.. code-block:: python
|
|
36
|
-
|
|
37
|
-
with model.forward() as runner:
|
|
38
|
-
with runner.invoke('The Eiffel Tower is in the city of') as invoker:
|
|
39
|
-
model.transformer.h[0].output[0] = 0
|
|
15
|
+
"""The Runner object manages the intervention tracing for a given model's _execute method locally or remotely.
|
|
40
16
|
|
|
41
17
|
Attributes:
|
|
42
|
-
generation (bool): If to use the _generation method of the model. Otherwise the _run_local method
|
|
43
18
|
remote (bool): If to use the remote NDIF server for execution of model and computation graph. (Assuming it's running/working)
|
|
44
19
|
blocking (bool): If when using the server option, to hang until job completion or return information you can use to retrieve the job result.
|
|
45
20
|
"""
|
|
@@ -47,18 +22,14 @@ class Runner(Tracer):
|
|
|
47
22
|
def __init__(
|
|
48
23
|
self,
|
|
49
24
|
*args,
|
|
50
|
-
generation: bool = False,
|
|
51
25
|
blocking: bool = True,
|
|
52
26
|
remote: bool = False,
|
|
53
|
-
remote_include_output: bool = False,
|
|
54
27
|
**kwargs,
|
|
55
28
|
) -> None:
|
|
56
29
|
super().__init__(*args, **kwargs)
|
|
57
30
|
|
|
58
|
-
self.generation = generation
|
|
59
31
|
self.remote = remote
|
|
60
32
|
self.blocking = blocking
|
|
61
|
-
self.remote_include_output = remote_include_output
|
|
62
33
|
|
|
63
34
|
def __enter__(self) -> Runner:
|
|
64
35
|
return self
|
|
@@ -67,32 +38,21 @@ class Runner(Tracer):
|
|
|
67
38
|
"""On exit, run and generate using the model whether locally or on the server."""
|
|
68
39
|
if isinstance(exc_val, BaseException):
|
|
69
40
|
raise exc_val
|
|
41
|
+
|
|
42
|
+
self._graph.tracing = False
|
|
43
|
+
|
|
70
44
|
if self.remote:
|
|
71
45
|
self.run_server()
|
|
72
46
|
else:
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def run_local(self):
|
|
76
|
-
"""Runs the local_model using it's chosen method."""
|
|
77
|
-
# Run the model and store the output.
|
|
78
|
-
self.output = self.model(
|
|
79
|
-
self.model._generation if self.generation else self.model._forward,
|
|
80
|
-
self.batched_input,
|
|
81
|
-
self.graph,
|
|
82
|
-
*self.args,
|
|
83
|
-
**self.kwargs,
|
|
84
|
-
)
|
|
47
|
+
super().__exit__(exc_type, exc_val, exc_tb)
|
|
85
48
|
|
|
86
49
|
def run_server(self):
|
|
87
50
|
# Create the pydantic object for the request.
|
|
88
51
|
request = pydantics.RequestModel(
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
intervention_graph=self.graph.nodes,
|
|
94
|
-
generation=self.generation,
|
|
95
|
-
include_output=self.remote_include_output,
|
|
52
|
+
kwargs=self._kwargs,
|
|
53
|
+
repo_id=self._model._model_key,
|
|
54
|
+
batched_input=self._batched_input,
|
|
55
|
+
intervention_graph=self._graph.nodes,
|
|
96
56
|
)
|
|
97
57
|
|
|
98
58
|
if self.blocking:
|
|
@@ -136,17 +96,16 @@ class Runner(Tracer):
|
|
|
136
96
|
result_bytes.seek(0)
|
|
137
97
|
|
|
138
98
|
# Decode bytes with pickle and then into pydantic object.
|
|
139
|
-
result = pydantics.ResultModel(
|
|
99
|
+
result = pydantics.ResultModel(
|
|
100
|
+
**torch.load(result_bytes, map_location="cpu")
|
|
101
|
+
)
|
|
140
102
|
|
|
141
103
|
# Close bytes
|
|
142
104
|
result_bytes.close()
|
|
143
105
|
|
|
144
106
|
# Set save data.
|
|
145
107
|
for name, value in result.saves.items():
|
|
146
|
-
self.
|
|
147
|
-
|
|
148
|
-
# Set output data.
|
|
149
|
-
self.output = result.output
|
|
108
|
+
self._graph.nodes[name].value = value
|
|
150
109
|
|
|
151
110
|
return True
|
|
152
111
|
# Or if there was some error.
|
|
@@ -173,8 +132,16 @@ class Runner(Tracer):
|
|
|
173
132
|
response = requests.post(
|
|
174
133
|
f"https://{CONFIG.API.HOST}/request",
|
|
175
134
|
json=request.model_dump(exclude=["id", "received"]),
|
|
135
|
+
headers={'ndif-api-key' : CONFIG.API.APIKEY}
|
|
176
136
|
)
|
|
177
|
-
|
|
137
|
+
|
|
138
|
+
if response.status_code == 200:
|
|
139
|
+
|
|
140
|
+
response = pydantics.ResponseModel(**response.json())
|
|
141
|
+
|
|
142
|
+
else:
|
|
143
|
+
|
|
144
|
+
raise Exception(response.reason)
|
|
178
145
|
|
|
179
146
|
print(response)
|
|
180
147
|
|
|
@@ -184,6 +151,3 @@ class Runner(Tracer):
|
|
|
184
151
|
|
|
185
152
|
def non_blocking_request(self, request: pydantics.RequestModel):
|
|
186
153
|
pass
|
|
187
|
-
|
|
188
|
-
def invoke(self, input, *args, **kwargs) -> Invoker:
|
|
189
|
-
return Invoker(self, input, *args, **kwargs)
|
nnsight/contexts/Tracer.py
CHANGED
|
@@ -1,65 +1,110 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
from
|
|
5
|
-
from typing import TYPE_CHECKING, Any, List
|
|
6
|
-
|
|
7
|
-
import torch
|
|
3
|
+
import weakref
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Callable, List, Tuple
|
|
8
5
|
|
|
9
6
|
from ..intervention import InterventionProxy
|
|
10
7
|
from ..tracing.Graph import Graph
|
|
8
|
+
from .Invoker import Invoker
|
|
11
9
|
|
|
12
10
|
if TYPE_CHECKING:
|
|
13
|
-
from ..models.NNsightModel import
|
|
11
|
+
from ..models.NNsightModel import NNsight
|
|
14
12
|
|
|
15
13
|
|
|
16
|
-
class Tracer
|
|
17
|
-
"""The Tracer class creates a :class:`nnsight.tracing.Graph.Graph` around the
|
|
14
|
+
class Tracer:
|
|
15
|
+
"""The Tracer class creates a :class:`nnsight.tracing.Graph.Graph` around the ._model of a :class:`nnsight.models.NNsightModel.NNsight` which tracks and manages the operations performed on the inputs and outputs of said model.
|
|
18
16
|
|
|
19
17
|
Attributes:
|
|
20
|
-
model (nnsight.models.NNsightModel.
|
|
21
|
-
graph (nnsight.tracing.Graph.Graph): Graph which operations performed on the input and output of
|
|
18
|
+
model (nnsight.models.NNsightModel.NNsight): nnsight Model object that ths context manager traces and executes.
|
|
19
|
+
graph (nnsight.tracing.Graph.Graph): Graph which traces operations performed on the input and output of modules' Envoys are added and later executed.
|
|
22
20
|
args (List[Any]): Positional arguments to be passed to function that executes the model.
|
|
23
21
|
kwargs (Dict[str,Any]): Keyword arguments to be passed to function that executes the model.
|
|
24
|
-
batch_size (int): Batch size of the most recent input. Used by
|
|
25
|
-
batch_start (int): Batch start of the most recent input. Used by
|
|
26
|
-
generation_idx (int): Current generation idx for multi-iteration generation. Used by Module to create input/output proxies.
|
|
22
|
+
batch_size (int): Batch size of the most recent input. Used by Envoy to create input/output proxies.
|
|
23
|
+
batch_start (int): Batch start of the most recent input. Used by Envoy to create input/output proxies.
|
|
27
24
|
batched_input Any: Batched version of all inputs involved in this Tracer.
|
|
28
|
-
output (Any): Output of execution after __exit__
|
|
29
25
|
"""
|
|
30
26
|
|
|
31
27
|
def __init__(
|
|
32
28
|
self,
|
|
33
|
-
model: "
|
|
34
|
-
*args,
|
|
29
|
+
model: "NNsight",
|
|
35
30
|
validate: bool = True,
|
|
36
31
|
**kwargs,
|
|
37
32
|
) -> None:
|
|
38
|
-
self.model = model
|
|
39
33
|
|
|
40
|
-
self.
|
|
41
|
-
|
|
34
|
+
self._model = model
|
|
35
|
+
|
|
36
|
+
self._kwargs = kwargs
|
|
42
37
|
|
|
43
|
-
self.
|
|
44
|
-
self.
|
|
38
|
+
self._graph = Graph(
|
|
39
|
+
self._model._envoy, proxy_class=model.proxy_class, validate=validate
|
|
45
40
|
)
|
|
46
41
|
|
|
47
|
-
self.
|
|
48
|
-
|
|
49
|
-
self.
|
|
50
|
-
self.
|
|
42
|
+
self._invoker: Invoker = None
|
|
43
|
+
|
|
44
|
+
self._batch_size: int = 0
|
|
45
|
+
self._batch_start: int = 0
|
|
46
|
+
|
|
47
|
+
self._batched_input: Any = None
|
|
48
|
+
|
|
49
|
+
# Module Envoys need to know about the current Tracer to create the correct proxies.
|
|
50
|
+
self._model._envoy._set_tracer(weakref.proxy(self))
|
|
51
51
|
|
|
52
|
-
|
|
52
|
+
def __getattr__(self, key: Any) -> Any:
|
|
53
|
+
"""Wrapper of .model._envoy's attributes to access module Envoy inputs and outputs.
|
|
53
54
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
Returns:
|
|
56
|
+
Any: Attribute.
|
|
57
|
+
"""
|
|
58
|
+
return getattr(self._model._envoy, key)
|
|
58
59
|
|
|
59
|
-
@abstractmethod
|
|
60
60
|
def __enter__(self) -> Tracer:
|
|
61
|
-
|
|
61
|
+
return self
|
|
62
62
|
|
|
63
|
-
@abstractmethod
|
|
64
63
|
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
|
65
|
-
|
|
64
|
+
if isinstance(exc_val, BaseException):
|
|
65
|
+
raise exc_val
|
|
66
|
+
|
|
67
|
+
self._graph.tracing = False
|
|
68
|
+
|
|
69
|
+
output = self._model.interleave(
|
|
70
|
+
self._model._execute,
|
|
71
|
+
self._graph,
|
|
72
|
+
*self._batched_input,
|
|
73
|
+
**self._kwargs,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def invoke(self, *inputs: Tuple[Any], **kwargs) -> Invoker:
|
|
77
|
+
"""Create an Invoker context dor a given input.
|
|
78
|
+
|
|
79
|
+
Raises:
|
|
80
|
+
Exception: If an Invoker context is already open
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
Invoker: Invoker.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
if self._invoker is not None:
|
|
87
|
+
|
|
88
|
+
raise Exception("Can't create an invoker context with one already open!")
|
|
89
|
+
|
|
90
|
+
return Invoker(self, *inputs, **kwargs)
|
|
91
|
+
|
|
92
|
+
def next(self, increment: int = 1) -> None:
|
|
93
|
+
"""Increments call_iter of all module Envoys. Useful when doing iterative/generative runs.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
increment (int): How many call_iter to increment at once. Defaults to 1.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
self._model._envoy.next(increment=increment, propagate=True)
|
|
100
|
+
|
|
101
|
+
def apply(self, target: Callable, *args, **kwargs) -> InterventionProxy:
|
|
102
|
+
"""Helper method to directly add a function to the intervention graph.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
target (Callable): Function to apply
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
InterventionProxy: Proxy of applying that function.
|
|
109
|
+
"""
|
|
110
|
+
return self._graph.add(target=target, args=args, kwargs=kwargs)
|