nnsight 0.5.0.dev13__tar.gz → 0.5.0.dev15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/PKG-INFO +1 -1
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/backends/remote.py +13 -6
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/batching.py +3 -1
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/envoy.py +17 -8
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/inject.py +6 -3
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/interleaver.py +25 -28
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/backwards.py +2 -4
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/base.py +26 -43
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/invoker.py +6 -2
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/iterator.py +4 -1
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/tracer.py +34 -13
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/util.py +57 -1
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/language.py +29 -34
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight.egg-info/PKG-INFO +1 -1
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/tests/test_lm.py +17 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/.github/workflows/publish.yml +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/.github/workflows/python-app.yml +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/.gitignore +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/CHANGELOG.md +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/CODE_OF_CONDUCT.md +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/LICENSE +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/NNsight_Walkthrough.ipynb +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/NNsight_v0_3_guide.ipynb +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/NNsight_v0_4_guide.ipynb +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/README.md +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/conftest.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/pyproject.toml +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/pytest.ini +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/setup.cfg +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/setup.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/_c/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/_c/py_mount.c +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/config.yaml +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/backends/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/backends/base.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/backends/editing.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/backends/execution.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/serialization.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/editing.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/intervention/tracing/globals.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/log.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/base.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/diffusion.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/mixins/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/mixins/loadable.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/mixins/meta.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/mixins/remoteable.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner copy.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/sampling copy.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/sampling.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/vllm.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/schema/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/schema/config.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/schema/request.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/schema/response.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/util.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight.egg-info/SOURCES.txt +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight.egg-info/dependency_links.txt +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight.egg-info/requires.txt +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight.egg-info/top_level.txt +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/tests/__init__.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/tests/test_diffusion.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/tests/test_tiny.py +0 -0
- {nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/tests/test_vllm.py +0 -0
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
4
|
import io
|
|
5
|
-
import
|
|
5
|
+
import os
|
|
6
6
|
import time
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
|
|
8
8
|
|
|
@@ -46,17 +46,20 @@ class RemoteBackend(Backend):
|
|
|
46
46
|
) -> None:
|
|
47
47
|
|
|
48
48
|
self.model_key = model_key
|
|
49
|
+
|
|
50
|
+
self.host = host or os.environ.get("NDIF_HOST", None) or CONFIG.API.HOST
|
|
51
|
+
self.api_key = api_key or os.environ.get("NDIF_API_KEY", None) or CONFIG.API.APIKEY
|
|
49
52
|
|
|
50
53
|
self.job_id = job_id or CONFIG.API.JOB_ID
|
|
51
54
|
self.ssl = CONFIG.API.SSL if ssl is None else ssl
|
|
52
55
|
self.zlib = CONFIG.API.ZLIB
|
|
53
|
-
self.api_key = api_key or CONFIG.API.APIKEY
|
|
54
56
|
self.blocking = blocking
|
|
55
57
|
self.callback = callback
|
|
56
|
-
|
|
57
|
-
self.host = host or CONFIG.API.HOST
|
|
58
|
+
|
|
58
59
|
self.address = f"http{'s' if self.ssl else ''}://{self.host}"
|
|
59
60
|
self.ws_address = f"ws{'s' if CONFIG.API.SSL else ''}://{self.host}"
|
|
61
|
+
|
|
62
|
+
self.job_status = None
|
|
60
63
|
|
|
61
64
|
def request(self, tracer: Tracer) -> Tuple[bytes, Dict[str, str]]:
|
|
62
65
|
|
|
@@ -77,7 +80,7 @@ class RemoteBackend(Backend):
|
|
|
77
80
|
|
|
78
81
|
return data, headers
|
|
79
82
|
|
|
80
|
-
def __call__(self, tracer):
|
|
83
|
+
def __call__(self, tracer = None):
|
|
81
84
|
|
|
82
85
|
if self.blocking:
|
|
83
86
|
|
|
@@ -89,8 +92,10 @@ class RemoteBackend(Backend):
|
|
|
89
92
|
# Otherwise we are getting the status / result of the existing job.
|
|
90
93
|
result = self.non_blocking_request(tracer)
|
|
91
94
|
|
|
92
|
-
if result is not None:
|
|
95
|
+
if tracer is not None and result is not None:
|
|
93
96
|
tracer.push(result)
|
|
97
|
+
|
|
98
|
+
return result
|
|
94
99
|
|
|
95
100
|
def handle_response(
|
|
96
101
|
self, response: ResponseModel, tracer: Optional[Tracer] = None
|
|
@@ -112,6 +117,8 @@ class RemoteBackend(Backend):
|
|
|
112
117
|
ResponseModel: ResponseModel.
|
|
113
118
|
"""
|
|
114
119
|
|
|
120
|
+
self.job_status = response.status
|
|
121
|
+
|
|
115
122
|
if response.status == ResponseModel.JobStatus.ERROR:
|
|
116
123
|
raise RemoteException(f"{response.description}\nRemote exception.")
|
|
117
124
|
|
|
@@ -46,7 +46,7 @@ class Batcher:
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
def batch(self, batchable: Batchable, *args, **kwargs):
|
|
49
|
+
def batch(self, batchable: Batchable, *args, **kwargs) -> Tuple[Any, Any]:
|
|
50
50
|
|
|
51
51
|
if args or kwargs:
|
|
52
52
|
|
|
@@ -75,6 +75,8 @@ class Batcher:
|
|
|
75
75
|
|
|
76
76
|
else:
|
|
77
77
|
self.batch_groups.append((-1, -1))
|
|
78
|
+
|
|
79
|
+
return args, kwargs
|
|
78
80
|
|
|
79
81
|
|
|
80
82
|
def narrow(self, batch_group: int, data:Any):
|
|
@@ -552,20 +552,27 @@ class Envoy(Batchable):
|
|
|
552
552
|
|
|
553
553
|
# TODO legacy
|
|
554
554
|
def session(self, *args, tracer_cls: Type[Tracer] = Tracer, **kwargs):
|
|
555
|
-
|
|
555
|
+
tracer = tracer_cls(*args, **kwargs)
|
|
556
|
+
setattr(tracer, "model", self)
|
|
557
|
+
return tracer
|
|
556
558
|
|
|
557
|
-
# TODO legacy
|
|
558
559
|
@property
|
|
559
560
|
@deprecated(message="Use `tracer.iter` instead.")
|
|
560
561
|
@trace_only
|
|
561
562
|
def iter(self):
|
|
562
563
|
return IteratorProxy(self._interleaver)
|
|
563
564
|
|
|
564
|
-
# TODO legacy
|
|
565
565
|
@deprecated(message="Use `tracer.all()` instead.")
|
|
566
566
|
@trace_only
|
|
567
567
|
def all(self):
|
|
568
568
|
return self.iter[:]
|
|
569
|
+
|
|
570
|
+
@deprecated(message="Use `tracer.next()` instead.")
|
|
571
|
+
@trace_only
|
|
572
|
+
def next(self, step: int = 1):
|
|
573
|
+
self._interleaver.current.iteration += step
|
|
574
|
+
|
|
575
|
+
return self
|
|
569
576
|
|
|
570
577
|
@trace_only
|
|
571
578
|
def skip(self, replacement: Any):
|
|
@@ -861,10 +868,12 @@ class Envoy(Batchable):
|
|
|
861
868
|
"""
|
|
862
869
|
Update the alias for this Envoy and its children.
|
|
863
870
|
"""
|
|
864
|
-
|
|
871
|
+
if alias is not None:
|
|
872
|
+
self._alias = Aliaser(alias)
|
|
873
|
+
self._alias.build(self)
|
|
865
874
|
|
|
866
|
-
|
|
867
|
-
|
|
875
|
+
for envoy in self._children:
|
|
876
|
+
envoy._update_alias(alias)
|
|
868
877
|
|
|
869
878
|
|
|
870
879
|
|
|
@@ -1377,9 +1386,9 @@ class EnvoySource:
|
|
|
1377
1386
|
|
|
1378
1387
|
|
|
1379
1388
|
|
|
1380
|
-
def
|
|
1389
|
+
def __getattribute__(self, name: str) -> Union[OperationEnvoy]:
|
|
1381
1390
|
|
|
1382
|
-
return
|
|
1391
|
+
return object.__getattribute__(self, name)
|
|
1383
1392
|
|
|
1384
1393
|
|
|
1385
1394
|
class Aliaser:
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import ast
|
|
2
|
-
import astor
|
|
3
|
-
import textwrap
|
|
4
|
-
from typing import Callable
|
|
5
2
|
import inspect
|
|
3
|
+
import textwrap
|
|
4
|
+
from builtins import compile, exec
|
|
6
5
|
from collections import defaultdict
|
|
6
|
+
from typing import Callable
|
|
7
|
+
|
|
8
|
+
import astor
|
|
9
|
+
|
|
7
10
|
|
|
8
11
|
class FunctionCallWrapper(ast.NodeTransformer):
|
|
9
12
|
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import ctypes
|
|
4
4
|
import inspect
|
|
5
|
+
import re
|
|
5
6
|
import threading
|
|
6
7
|
import time
|
|
7
8
|
import warnings
|
|
@@ -10,15 +11,15 @@ from enum import Enum
|
|
|
10
11
|
from functools import wraps
|
|
11
12
|
from queue import Queue
|
|
12
13
|
from threading import Thread
|
|
14
|
+
from types import FrameType
|
|
13
15
|
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
|
|
14
16
|
Union)
|
|
15
17
|
|
|
16
18
|
import torch
|
|
17
19
|
|
|
18
|
-
from ..util import
|
|
20
|
+
from ..util import applyn
|
|
19
21
|
from .batching import Batcher
|
|
20
|
-
from .tracing.
|
|
21
|
-
from .tracing.util import wrap_exception
|
|
22
|
+
from .tracing.util import wrap_exception, get_non_nnsight_frame, push_variables
|
|
22
23
|
|
|
23
24
|
if TYPE_CHECKING:
|
|
24
25
|
from .tracing.tracer import Cache, InterleavingTracer, Tracer
|
|
@@ -92,7 +93,6 @@ class Interleaver:
|
|
|
92
93
|
self.user_cache = user_cache
|
|
93
94
|
|
|
94
95
|
self.mediators: Dict[str, Mediator] = {}
|
|
95
|
-
self.state = dict()
|
|
96
96
|
self.iteration_tracker = defaultdict(int)
|
|
97
97
|
self.default_all = None
|
|
98
98
|
|
|
@@ -138,9 +138,10 @@ class Interleaver:
|
|
|
138
138
|
|
|
139
139
|
module.forward = skippable_forward
|
|
140
140
|
|
|
141
|
+
@torch._dynamo.disable
|
|
141
142
|
def input_hook(module: torch.nn.Module, args, kwargs):
|
|
142
143
|
|
|
143
|
-
if not self.interleaving
|
|
144
|
+
if not self.interleaving:
|
|
144
145
|
return args, kwargs
|
|
145
146
|
|
|
146
147
|
provider = module.__path__
|
|
@@ -158,9 +159,10 @@ class Interleaver:
|
|
|
158
159
|
|
|
159
160
|
module.register_forward_pre_hook(input_hook, with_kwargs=True, prepend=True)
|
|
160
161
|
|
|
162
|
+
@torch._dynamo.disable
|
|
161
163
|
def output_hook(module: torch.nn.Module, _, output: Any):
|
|
162
164
|
|
|
163
|
-
if not self.interleaving
|
|
165
|
+
if not self.interleaving:
|
|
164
166
|
return output
|
|
165
167
|
|
|
166
168
|
provider = module.__path__
|
|
@@ -304,7 +306,7 @@ class Interleaver:
|
|
|
304
306
|
return
|
|
305
307
|
|
|
306
308
|
### Provider Methods ###
|
|
307
|
-
|
|
309
|
+
|
|
308
310
|
def handle(
|
|
309
311
|
self,
|
|
310
312
|
provider: Optional[Any] = None,
|
|
@@ -474,8 +476,6 @@ class Mediator:
|
|
|
474
476
|
|
|
475
477
|
self.thread = None
|
|
476
478
|
|
|
477
|
-
self.state = None
|
|
478
|
-
|
|
479
479
|
if self.alive:
|
|
480
480
|
# TODO: cancel inactive threads at the end of the model's execution
|
|
481
481
|
self.response_queue.put(Cancelation())
|
|
@@ -566,11 +566,14 @@ class Mediator:
|
|
|
566
566
|
mediator = mediator.child
|
|
567
567
|
|
|
568
568
|
if mediator.name in participants:
|
|
569
|
-
|
|
569
|
+
|
|
570
|
+
mediator.respond()
|
|
571
|
+
|
|
570
572
|
mediator.handle(provider)
|
|
573
|
+
|
|
571
574
|
|
|
572
|
-
|
|
573
|
-
|
|
575
|
+
|
|
576
|
+
|
|
574
577
|
|
|
575
578
|
|
|
576
579
|
def handle_end_event(self):
|
|
@@ -708,20 +711,18 @@ class Mediator:
|
|
|
708
711
|
### Requester Methods ###
|
|
709
712
|
|
|
710
713
|
@property
|
|
711
|
-
def frame(self):
|
|
714
|
+
def frame(self) -> FrameType:
|
|
712
715
|
"""
|
|
713
716
|
Get the frame of the intervention function.
|
|
714
717
|
|
|
715
718
|
Returns:
|
|
716
719
|
The frame of the intervention function
|
|
717
720
|
"""
|
|
721
|
+
|
|
722
|
+
|
|
718
723
|
|
|
719
|
-
frame =
|
|
724
|
+
frame = get_non_nnsight_frame()
|
|
720
725
|
|
|
721
|
-
while frame:
|
|
722
|
-
frame = frame.f_back
|
|
723
|
-
if frame and frame.f_code.co_filename.startswith("<nnsight"):
|
|
724
|
-
break
|
|
725
726
|
return frame
|
|
726
727
|
|
|
727
728
|
def iterate(self, requester: Any):
|
|
@@ -730,21 +731,17 @@ class Mediator:
|
|
|
730
731
|
|
|
731
732
|
def push(self):
|
|
732
733
|
"""Push local variables to the interleaver state."""
|
|
733
|
-
|
|
734
|
+
|
|
735
|
+
state = {k: v for k, v in self.frame.f_locals.items() if not k.startswith("__nnsight")}
|
|
736
|
+
# this does not handle the case of a fn thats called in an invoker. this will push vars directly to where the invoke was called not the fn. really we need to grad the f_back of the <nnsight> frame. If its in threading.py, then we use info.frame
|
|
737
|
+
push_variables(self.info.frame, state)
|
|
734
738
|
|
|
735
739
|
def pull(self):
|
|
736
740
|
"""Pull variables from the interleaver state to the frame globals."""
|
|
737
741
|
|
|
738
|
-
for
|
|
739
|
-
if key.startswith("__nnsight"):
|
|
740
|
-
self.interleaver.state.pop(key)
|
|
741
|
-
|
|
742
|
-
self.frame.f_globals.update(self.interleaver.state)
|
|
743
|
-
self.frame.f_locals.update(self.interleaver.state)
|
|
742
|
+
state = {k: v for k, v in self.info.frame.f_locals.items() if not k.startswith("__nnsight") and k not in self.frame.f_locals}
|
|
744
743
|
|
|
745
|
-
|
|
746
|
-
ctypes.py_object(self.frame), ctypes.c_int(0)
|
|
747
|
-
)
|
|
744
|
+
push_variables(self.frame, state)
|
|
748
745
|
|
|
749
746
|
def send(self, event: Events, requester: Any):
|
|
750
747
|
"""
|
|
@@ -6,7 +6,6 @@ from ...util import Patch
|
|
|
6
6
|
from ..interleaver import Interleaver, Mediator
|
|
7
7
|
from .invoker import Invoker
|
|
8
8
|
|
|
9
|
-
|
|
10
9
|
def wrap_grad(interleaver: Interleaver):
|
|
11
10
|
"""
|
|
12
11
|
Create a hook for gradient intervention.
|
|
@@ -100,9 +99,8 @@ class BackwardsTracer(Invoker):
|
|
|
100
99
|
grad_patch.patch()
|
|
101
100
|
with interleaver:
|
|
102
101
|
self.fn(self.tensor, *self.args, **self.kwargs)
|
|
103
|
-
|
|
104
|
-
self.push(interleaver.state)
|
|
102
|
+
interleaver.check_dangling_mediators()
|
|
105
103
|
|
|
106
104
|
finally:
|
|
105
|
+
grad_patch.restore()
|
|
107
106
|
interleaver.cancel()
|
|
108
|
-
interleaver.state.clear()
|
|
@@ -9,7 +9,8 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
|
|
9
9
|
from ..backends.base import Backend
|
|
10
10
|
from ..backends.execution import ExecutionBackend
|
|
11
11
|
from .globals import Globals
|
|
12
|
-
from .util import
|
|
12
|
+
from .util import (TracingDeffermentException, get_non_nnsight_frame,
|
|
13
|
+
push_variables, suppress_all_output)
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class ExitTracingException(Exception):
|
|
@@ -29,12 +30,6 @@ class WithBlockNotFoundError(Exception):
|
|
|
29
30
|
|
|
30
31
|
pass
|
|
31
32
|
|
|
32
|
-
class TracingDeffermentException(Exception):
|
|
33
|
-
"""Exception raised when a tracing defferment is encountered.
|
|
34
|
-
|
|
35
|
-
This exception is used to indicate that a tracing defferment is encountered.
|
|
36
|
-
"""
|
|
37
|
-
pass
|
|
38
33
|
|
|
39
34
|
class Tracer:
|
|
40
35
|
"""
|
|
@@ -145,22 +140,7 @@ class Tracer:
|
|
|
145
140
|
the source code structure.
|
|
146
141
|
"""
|
|
147
142
|
# Find the frame outside of nnsight by walking up the call stack
|
|
148
|
-
frame =
|
|
149
|
-
|
|
150
|
-
while frame:
|
|
151
|
-
frame = frame.f_back
|
|
152
|
-
if frame:
|
|
153
|
-
filename = frame.f_code.co_filename
|
|
154
|
-
# Match if filename contains 'nnsight/tests' or 'nnsight\tests'
|
|
155
|
-
# OR if it does NOT contain '/nnsight/' or '\nnsight\'
|
|
156
|
-
|
|
157
|
-
if "__defer_capture__" in frame.f_locals:
|
|
158
|
-
raise TracingDeffermentException()
|
|
159
|
-
if (
|
|
160
|
-
re.search(r"[\\/]{1}nnsight[\\/]{1}tests", filename)
|
|
161
|
-
or not re.search(r"[\\/]{1}nnsight[\\/]", filename)
|
|
162
|
-
):
|
|
163
|
-
break
|
|
143
|
+
frame = get_non_nnsight_frame()
|
|
164
144
|
|
|
165
145
|
# Get source code lines from the appropriate location
|
|
166
146
|
start_line = frame.f_lineno
|
|
@@ -295,9 +275,12 @@ class Tracer:
|
|
|
295
275
|
# Wrap the captured code in a function definition with appropriate parameters
|
|
296
276
|
self.info.source = [
|
|
297
277
|
f"def __nnsight_tracer_{id(self)}__(__nnsight_tracer__, __nnsight_tracing_info__):\n",
|
|
278
|
+
" __nnsight_tracer__.pull()\n",
|
|
298
279
|
*self.info.source,
|
|
299
280
|
" __nnsight_tracer__.push()\n",
|
|
300
281
|
]
|
|
282
|
+
|
|
283
|
+
self.info.start_line -= 1
|
|
301
284
|
|
|
302
285
|
def execute(self, fn: Callable):
|
|
303
286
|
"""
|
|
@@ -339,31 +322,31 @@ class Tracer:
|
|
|
339
322
|
# Collect all non-nnsight variables from the frame
|
|
340
323
|
|
|
341
324
|
state = {k: v for k, v in state.items() if not k.startswith("__nnsight")}
|
|
325
|
+
|
|
326
|
+
if Globals.stack == 1 :
|
|
327
|
+
state = {k: v for k, v in state.items() if id(v) in Globals.saves}
|
|
342
328
|
|
|
343
|
-
|
|
344
|
-
# For dynamically generated code, update both globals and locals
|
|
345
|
-
frame.f_globals.update(state)
|
|
346
|
-
frame.f_locals.update(state)
|
|
347
|
-
|
|
348
|
-
# Ensure locals are properly synchronized with the frame
|
|
349
|
-
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
350
|
-
ctypes.py_object(frame), ctypes.c_int(0)
|
|
351
|
-
)
|
|
329
|
+
push_variables(frame, state)
|
|
352
330
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
continue
|
|
331
|
+
state.clear()
|
|
332
|
+
|
|
333
|
+
def pull(self):
|
|
334
|
+
|
|
335
|
+
frame = inspect.currentframe()
|
|
359
336
|
|
|
360
|
-
|
|
337
|
+
while frame:
|
|
338
|
+
frame = frame.f_back
|
|
339
|
+
if frame and frame.f_code.co_filename.startswith(
|
|
340
|
+
"<nnsight"
|
|
341
|
+
):
|
|
342
|
+
break
|
|
361
343
|
|
|
362
|
-
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
363
|
-
ctypes.py_object(frame), ctypes.c_int(0)
|
|
364
|
-
)
|
|
365
344
|
|
|
366
|
-
state.
|
|
345
|
+
state = self.info.frame.f_locals
|
|
346
|
+
|
|
347
|
+
state = {k: v for k, v in state.items() if not k.startswith("__nnsight")}
|
|
348
|
+
|
|
349
|
+
push_variables(frame, state)
|
|
367
350
|
|
|
368
351
|
def __enter__(self):
|
|
369
352
|
"""
|
|
@@ -48,6 +48,7 @@ class Invoker(Tracer):
|
|
|
48
48
|
"""
|
|
49
49
|
self.info.source = [
|
|
50
50
|
f"def __nnsight_tracer_{id(self)}__(__nnsight_mediator__, __nnsight_tracing_info__):\n",
|
|
51
|
+
" __nnsight_mediator__.pull()\n",
|
|
51
52
|
*try_catch(
|
|
52
53
|
self.info.source,
|
|
53
54
|
exception_source=["__nnsight_mediator__.exception(exception)\n"],
|
|
@@ -55,7 +56,7 @@ class Invoker(Tracer):
|
|
|
55
56
|
),
|
|
56
57
|
]
|
|
57
58
|
|
|
58
|
-
self.info.start_line -=
|
|
59
|
+
self.info.start_line -= 2
|
|
59
60
|
|
|
60
61
|
def execute(self, fn: Callable):
|
|
61
62
|
"""
|
|
@@ -67,7 +68,10 @@ class Invoker(Tracer):
|
|
|
67
68
|
Args:
|
|
68
69
|
fn: The compiled intervention function
|
|
69
70
|
"""
|
|
70
|
-
|
|
71
|
+
|
|
72
|
+
inputs = self.tracer.batcher.batch(self.tracer.model, *self.args, **self.kwargs)
|
|
73
|
+
|
|
74
|
+
self.inputs = inputs
|
|
71
75
|
|
|
72
76
|
mediator = Mediator(fn, self.info, batch_group=len(self.tracer.mediators))
|
|
73
77
|
|
|
@@ -36,6 +36,7 @@ class IteratorTracer(Tracer):
|
|
|
36
36
|
# Wrap the captured code in a function definition with appropriate parameters
|
|
37
37
|
self.info.source = [
|
|
38
38
|
f"def __nnsight_tracer_{id(self)}__(__nnsight_mediator__, __nnsight_tracing_info__, {iteration_var_name}):\n",
|
|
39
|
+
" __nnsight_mediator__.pull()\n",
|
|
39
40
|
*try_catch(
|
|
40
41
|
self.info.source,
|
|
41
42
|
exception_source=["__nnsight_mediator__.exception(exception)\n"],
|
|
@@ -43,9 +44,11 @@ class IteratorTracer(Tracer):
|
|
|
43
44
|
),
|
|
44
45
|
]
|
|
45
46
|
|
|
47
|
+
self.info.start_line -= 2
|
|
48
|
+
|
|
46
49
|
def execute(self, fn: Callable):
|
|
47
50
|
|
|
48
|
-
mediator = Mediator(fn, self.info)
|
|
51
|
+
mediator = Mediator(fn, self.info, batch_group=self.interleaver.current.batch_group)
|
|
49
52
|
mediator.name = "Iterator" + mediator.name
|
|
50
53
|
|
|
51
54
|
self.interleaver.current.iter(mediator, self.iteration)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import copy
|
|
2
|
+
import inspect
|
|
2
3
|
import re
|
|
3
4
|
from dataclasses import dataclass
|
|
4
5
|
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
|
|
@@ -256,6 +257,8 @@ class InterleavingTracer(Tracer):
|
|
|
256
257
|
self.batcher = Batcher(**kwargs)
|
|
257
258
|
|
|
258
259
|
self.user_cache: List[Cache] = list()
|
|
260
|
+
|
|
261
|
+
self._frame = None
|
|
259
262
|
|
|
260
263
|
super().__init__(*args, backend=backend)
|
|
261
264
|
|
|
@@ -297,20 +300,32 @@ class InterleavingTracer(Tracer):
|
|
|
297
300
|
|
|
298
301
|
# If positional arguments were passed directly to a tracer, assume one invoker
|
|
299
302
|
if self.args:
|
|
300
|
-
|
|
301
|
-
invoker
|
|
303
|
+
|
|
304
|
+
invoker = self.invoke(*self.args, _info=self.info.copy(), **self.kwargs)
|
|
302
305
|
|
|
303
306
|
invoker.__exit__(ExitTracingException, None, None)
|
|
304
307
|
|
|
305
|
-
self.info.source = ["
|
|
308
|
+
self.info.source = [f" {self.tracer_var_name}.mediators[0].info.frame = {self.tracer_var_name}.get_frame()\n"]
|
|
306
309
|
|
|
307
310
|
self.info.source = [
|
|
308
311
|
f"def __nnsight_tracer_{id(self)}__(__nnsight_tracing_info__,{self.tracer_var_name}):\n",
|
|
312
|
+
f" {self.tracer_var_name}.pull()\n",
|
|
309
313
|
*self.info.source,
|
|
310
|
-
f" {self.tracer_var_name}.
|
|
314
|
+
f" {self.tracer_var_name}.get_frame()\n",
|
|
311
315
|
]
|
|
316
|
+
|
|
317
|
+
self.info.start_line -= 1
|
|
312
318
|
|
|
313
319
|
self.args = tuple()
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def get_frame(self):
|
|
323
|
+
"""
|
|
324
|
+
Get the frame of the tracer.
|
|
325
|
+
"""
|
|
326
|
+
self._frame = inspect.currentframe().f_back
|
|
327
|
+
|
|
328
|
+
return self._frame
|
|
314
329
|
|
|
315
330
|
def execute(self, fn: Callable):
|
|
316
331
|
"""
|
|
@@ -322,7 +337,6 @@ class InterleavingTracer(Tracer):
|
|
|
322
337
|
Args:
|
|
323
338
|
fn: The compiled function to execute
|
|
324
339
|
"""
|
|
325
|
-
|
|
326
340
|
fn(self.info, self)
|
|
327
341
|
|
|
328
342
|
args = self.batcher.batched_args
|
|
@@ -334,13 +348,11 @@ class InterleavingTracer(Tracer):
|
|
|
334
348
|
interleaver = self.model._interleaver
|
|
335
349
|
interleaver.initialize(self.mediators, self, batcher=self.batcher, user_cache=self.user_cache)
|
|
336
350
|
|
|
337
|
-
|
|
338
|
-
|
|
351
|
+
self.model.interleave(self.fn, *args, **kwargs)
|
|
352
|
+
|
|
353
|
+
self.push(self._frame.f_locals)
|
|
339
354
|
|
|
340
|
-
|
|
341
|
-
finally:
|
|
342
|
-
interleaver.state.clear()
|
|
343
|
-
|
|
355
|
+
|
|
344
356
|
|
|
345
357
|
### Public API ####
|
|
346
358
|
|
|
@@ -370,6 +382,11 @@ class InterleavingTracer(Tracer):
|
|
|
370
382
|
|
|
371
383
|
def all(self):
|
|
372
384
|
return self.iter[:]
|
|
385
|
+
|
|
386
|
+
def next(self, step: int = 1):
|
|
387
|
+
self.model._interleaver.current.iteration += step
|
|
388
|
+
|
|
389
|
+
return self
|
|
373
390
|
|
|
374
391
|
def cache(
|
|
375
392
|
self,
|
|
@@ -443,11 +460,12 @@ class InterleavingTracer(Tracer):
|
|
|
443
460
|
def __getstate__(self):
|
|
444
461
|
"""Get the state of the tracer for serialization."""
|
|
445
462
|
state = super().__getstate__()
|
|
446
|
-
state["fn"] = self.fn.__name__
|
|
463
|
+
state["fn"] = self.fn if isinstance(self.fn, str) else self.fn.__name__
|
|
447
464
|
state["model"] = self.model
|
|
448
465
|
state["tracer_var_name"] = self.tracer_var_name
|
|
449
466
|
state["batcher"] = self.batcher
|
|
450
467
|
state["mediators"] = self.mediators
|
|
468
|
+
state["rename"] = self.model._alias.rename if self.model._alias is not None else None
|
|
451
469
|
|
|
452
470
|
return state
|
|
453
471
|
|
|
@@ -461,7 +479,8 @@ class InterleavingTracer(Tracer):
|
|
|
461
479
|
self.tracer_var_name = state["tracer_var_name"]
|
|
462
480
|
self.mediators = state["mediators"]
|
|
463
481
|
self.batcher = state["batcher"]
|
|
464
|
-
|
|
482
|
+
if state["rename"] is not None:
|
|
483
|
+
self.model._update_alias(state["rename"])
|
|
465
484
|
self.obj_var_name = None
|
|
466
485
|
self.user_cache = list()
|
|
467
486
|
|
|
@@ -538,11 +557,13 @@ class Barrier:
|
|
|
538
557
|
|
|
539
558
|
def __call__(self):
|
|
540
559
|
|
|
560
|
+
|
|
541
561
|
mediator = self.model._interleaver.current
|
|
542
562
|
|
|
543
563
|
self.participants.add(mediator.name)
|
|
544
564
|
|
|
545
565
|
if len(self.participants) == self.n_participants:
|
|
566
|
+
|
|
546
567
|
mediator.send(Events.BARRIER, self.participants)
|
|
547
568
|
self.participants = set()
|
|
548
569
|
else:
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
|
|
2
2
|
import contextlib
|
|
3
|
+
import ctypes
|
|
4
|
+
import inspect
|
|
3
5
|
import os
|
|
6
|
+
import re
|
|
4
7
|
import sys
|
|
5
8
|
from builtins import open
|
|
6
|
-
from
|
|
9
|
+
from types import FrameType
|
|
10
|
+
from typing import TYPE_CHECKING, Callable, Dict, List
|
|
7
11
|
|
|
8
12
|
if TYPE_CHECKING:
|
|
9
13
|
from .base import Tracer
|
|
@@ -83,6 +87,7 @@ def get_dependencies(fn:Callable):
|
|
|
83
87
|
|
|
84
88
|
from ... import CONFIG
|
|
85
89
|
|
|
90
|
+
|
|
86
91
|
class ExceptionWrapper(Exception):
|
|
87
92
|
"""
|
|
88
93
|
Wrapper for exceptions that provides additional details for tracer created code.
|
|
@@ -258,3 +263,54 @@ def wrap_exception(exception:Exception, info:"Tracer.Info"):
|
|
|
258
263
|
wrapped.__dict__.update(exception.__dict__)
|
|
259
264
|
|
|
260
265
|
return wrapped
|
|
266
|
+
|
|
267
|
+
class TracingDeffermentException(Exception):
|
|
268
|
+
"""Exception raised when a tracing defferment is encountered.
|
|
269
|
+
|
|
270
|
+
This exception is used to indicate that a tracing defferment is encountered.
|
|
271
|
+
"""
|
|
272
|
+
pass
|
|
273
|
+
|
|
274
|
+
def get_non_nnsight_frame() -> FrameType:
|
|
275
|
+
frame = inspect.currentframe()
|
|
276
|
+
|
|
277
|
+
while frame:
|
|
278
|
+
frame = frame.f_back
|
|
279
|
+
if frame:
|
|
280
|
+
filename = frame.f_code.co_filename
|
|
281
|
+
# Match if filename contains 'nnsight/tests' or 'nnsight\tests'
|
|
282
|
+
# OR if it does NOT contain '/nnsight/' or '\nnsight\'
|
|
283
|
+
|
|
284
|
+
if "__defer_capture__" in frame.f_locals:
|
|
285
|
+
raise TracingDeffermentException()
|
|
286
|
+
if (
|
|
287
|
+
re.search(r"[\\/]{1}nnsight[\\/]{1}tests", filename)
|
|
288
|
+
or not re.search(r"[\\/]{1}nnsight[\\/]", filename)
|
|
289
|
+
):
|
|
290
|
+
break
|
|
291
|
+
|
|
292
|
+
return frame
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def push_variables(frame:FrameType, variables:Dict):
|
|
296
|
+
|
|
297
|
+
is_generated_frame = frame.f_code.co_filename.startswith("<nnsight")
|
|
298
|
+
|
|
299
|
+
if is_generated_frame:
|
|
300
|
+
|
|
301
|
+
global_variables = {k: v for k, v in variables.items() if k not in frame.f_locals}
|
|
302
|
+
|
|
303
|
+
for key, value in global_variables.items():
|
|
304
|
+
frame.f_globals[key] = value
|
|
305
|
+
|
|
306
|
+
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
307
|
+
ctypes.py_object(frame), ctypes.c_int(0)
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
for key, value in variables.items():
|
|
311
|
+
frame.f_locals[key] = value
|
|
312
|
+
|
|
313
|
+
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
314
|
+
ctypes.py_object(frame), ctypes.c_int(0)
|
|
315
|
+
)
|
|
316
|
+
|
|
@@ -2,32 +2,16 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
Any,
|
|
8
|
-
Dict,
|
|
9
|
-
List,
|
|
10
|
-
Optional,
|
|
11
|
-
Tuple,
|
|
12
|
-
Type,
|
|
13
|
-
Union,
|
|
14
|
-
)
|
|
15
|
-
from huggingface_hub import constants
|
|
16
|
-
from huggingface_hub.file_download import repo_folder_name
|
|
5
|
+
import warnings
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
|
|
17
7
|
|
|
18
|
-
from nnsight import CONFIG
|
|
19
8
|
import torch
|
|
9
|
+
from huggingface_hub import constants
|
|
10
|
+
from huggingface_hub.file_download import repo_folder_name
|
|
20
11
|
from torch.nn.modules import Module
|
|
21
|
-
from transformers import (
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
AutoModelForCausalLM,
|
|
25
|
-
AutoTokenizer,
|
|
26
|
-
BatchEncoding,
|
|
27
|
-
PretrainedConfig,
|
|
28
|
-
PreTrainedModel,
|
|
29
|
-
PreTrainedTokenizer,
|
|
30
|
-
)
|
|
12
|
+
from transformers import (AutoConfig, AutoModel, AutoModelForCausalLM,
|
|
13
|
+
AutoTokenizer, BatchEncoding, PretrainedConfig,
|
|
14
|
+
PreTrainedModel, PreTrainedTokenizer)
|
|
31
15
|
from transformers.generation.utils import GenerationMixin
|
|
32
16
|
from transformers.models.auto import modeling_auto
|
|
33
17
|
from transformers.models.llama.configuration_llama import LlamaConfig
|
|
@@ -101,6 +85,7 @@ class LanguageModel(RemoteableMixin):
|
|
|
101
85
|
# If the user passed in a pre-loaded model, might be able to get repo id off of it.
|
|
102
86
|
# That way if they dont provide a tokenizer, we can load it for them later.
|
|
103
87
|
self.repo_id: str = args[0] if isinstance(args[0], str) else getattr(args[0], 'name_or_path', None)
|
|
88
|
+
self.revision: str = getattr(args[0], 'revision', 'main')
|
|
104
89
|
|
|
105
90
|
super().__init__(*args, **kwargs)
|
|
106
91
|
|
|
@@ -119,22 +104,26 @@ class LanguageModel(RemoteableMixin):
|
|
|
119
104
|
# Some transformer models compile on first generation. As of 0.5.0.dev7 this not not work with nnsight if fullgraph is True
|
|
120
105
|
def _patch_generation_config(self, model:torch.nn.Module):
|
|
121
106
|
|
|
122
|
-
if
|
|
107
|
+
if getattr(model, "generation_config", None) is not None:
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
warnings.filterwarnings("ignore", message="The CUDA Graph is empty")
|
|
123
111
|
|
|
124
112
|
generation_config = model.generation_config
|
|
125
113
|
|
|
126
|
-
compile_config = generation_config
|
|
114
|
+
compile_config = getattr(generation_config, "compile_config", None)
|
|
127
115
|
|
|
128
116
|
if compile_config is None:
|
|
129
117
|
|
|
130
|
-
from transformers.generation.configuration_utils import
|
|
118
|
+
from transformers.generation.configuration_utils import \
|
|
119
|
+
CompileConfig
|
|
131
120
|
|
|
132
121
|
compile_config = CompileConfig()
|
|
133
122
|
|
|
134
123
|
compile_config.fullgraph = False
|
|
135
124
|
compile_config.dynamic = True
|
|
136
125
|
|
|
137
|
-
generation_config
|
|
126
|
+
setattr(generation_config, "compile_config", compile_config)
|
|
138
127
|
|
|
139
128
|
def export_edits(self, name:Optional[str] = None, export_dir: Optional[str] = None, variant: str = '__default__'):
|
|
140
129
|
"""TODO
|
|
@@ -213,6 +202,7 @@ class LanguageModel(RemoteableMixin):
|
|
|
213
202
|
def _load_meta(
|
|
214
203
|
self,
|
|
215
204
|
repo_id: str,
|
|
205
|
+
revision:Optional[str] = "main",
|
|
216
206
|
tokenizer_kwargs: Optional[Dict[str, Any]] = {},
|
|
217
207
|
patch_llama_scan: bool = True,
|
|
218
208
|
**kwargs,
|
|
@@ -220,9 +210,11 @@ class LanguageModel(RemoteableMixin):
|
|
|
220
210
|
|
|
221
211
|
self.repo_id = repo_id
|
|
222
212
|
|
|
223
|
-
self.
|
|
213
|
+
self.revision = revision
|
|
224
214
|
|
|
225
|
-
self.
|
|
215
|
+
self._load_config(repo_id, revision=revision, **kwargs)
|
|
216
|
+
|
|
217
|
+
self._load_tokenizer(repo_id, revision=revision, **tokenizer_kwargs)
|
|
226
218
|
|
|
227
219
|
if (
|
|
228
220
|
patch_llama_scan
|
|
@@ -243,14 +235,15 @@ class LanguageModel(RemoteableMixin):
|
|
|
243
235
|
def _load(
|
|
244
236
|
self,
|
|
245
237
|
repo_id: str,
|
|
238
|
+
revision:Optional[str] = "main",
|
|
246
239
|
tokenizer_kwargs: Optional[Dict[str, Any]] = {},
|
|
247
240
|
patch_llama_scan: bool = True,
|
|
248
241
|
**kwargs,
|
|
249
242
|
) -> PreTrainedModel:
|
|
250
243
|
|
|
251
|
-
self._load_config(repo_id, **kwargs)
|
|
244
|
+
self._load_config(repo_id, revision=revision, **kwargs)
|
|
252
245
|
|
|
253
|
-
self._load_tokenizer(repo_id, **tokenizer_kwargs)
|
|
246
|
+
self._load_tokenizer(repo_id, revision=revision, **tokenizer_kwargs)
|
|
254
247
|
|
|
255
248
|
if (
|
|
256
249
|
patch_llama_scan
|
|
@@ -260,7 +253,7 @@ class LanguageModel(RemoteableMixin):
|
|
|
260
253
|
):
|
|
261
254
|
self.config.rope_scaling["rope_type"] = "llama3"
|
|
262
255
|
|
|
263
|
-
model = self.automodel.from_pretrained(repo_id, config=self.config, **kwargs)
|
|
256
|
+
model = self.automodel.from_pretrained(repo_id, config=self.config, revision=revision, **kwargs)
|
|
264
257
|
|
|
265
258
|
self.config = model.config
|
|
266
259
|
|
|
@@ -399,7 +392,7 @@ class LanguageModel(RemoteableMixin):
|
|
|
399
392
|
|
|
400
393
|
def _remoteable_model_key(self) -> str:
|
|
401
394
|
return json.dumps(
|
|
402
|
-
{"repo_id": self.repo_id} # , "torch_dtype": str(self._model.dtype)}
|
|
395
|
+
{"repo_id": self.repo_id, "revision": self.revision} # , "torch_dtype": str(self._model.dtype)}
|
|
403
396
|
)
|
|
404
397
|
|
|
405
398
|
@classmethod
|
|
@@ -409,7 +402,9 @@ class LanguageModel(RemoteableMixin):
|
|
|
409
402
|
|
|
410
403
|
repo_id = kwargs.pop("repo_id")
|
|
411
404
|
|
|
412
|
-
|
|
405
|
+
revision = kwargs.pop("revision", "main")
|
|
406
|
+
|
|
407
|
+
return LanguageModel(repo_id, revision=revision, **kwargs)
|
|
413
408
|
|
|
414
409
|
|
|
415
410
|
if TYPE_CHECKING:
|
|
@@ -872,6 +872,23 @@ def test_iter_and_skip(gpt2: nnsight.LanguageModel):
|
|
|
872
872
|
assert not torch.all(arr_gen[2] == 0)
|
|
873
873
|
|
|
874
874
|
|
|
875
|
+
@torch.no_grad()
|
|
876
|
+
@pytest.mark.iter
|
|
877
|
+
def test_iter_with_batched_interventions(gpt2: nnsight.LanguageModel, ET_prompt: str, MSG_prompt: str):
|
|
878
|
+
with gpt2.generate(max_new_tokens=3) as tracer:
|
|
879
|
+
with tracer.invoke(ET_prompt):
|
|
880
|
+
logits_1 = list().save()
|
|
881
|
+
with tracer.iter[:]:
|
|
882
|
+
logits_1.append(gpt2.lm_head.output[0][-1].argmax(dim=-1))
|
|
883
|
+
|
|
884
|
+
with tracer.invoke(MSG_prompt):
|
|
885
|
+
logits_2 = list().save()
|
|
886
|
+
with tracer.iter[0:3]:
|
|
887
|
+
logits_2.append(gpt2.lm_head.output[0][-1].argmax(dim=-1))
|
|
888
|
+
|
|
889
|
+
assert all([not torch.equal(logit_1, logit_2) for logit_1, logit_2 in zip(logits_1, logits_2)])
|
|
890
|
+
|
|
891
|
+
|
|
875
892
|
######################### CACHE #################################
|
|
876
893
|
|
|
877
894
|
@torch.no_grad()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/model_runners/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev13 → nnsight-0.5.0.dev15}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|