nnsight 0.5.0.dev11__tar.gz → 0.5.0.dev13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/PKG-INFO +1 -1
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/backends/remote.py +85 -20
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/envoy.py +10 -6
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/interleaver.py +42 -45
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/serialization.py +19 -12
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/tracer.py +56 -5
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/language.py +21 -3
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/mixins/remoteable.py +78 -2
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/schema/request.py +3 -11
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight.egg-info/PKG-INFO +1 -1
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/tests/test_lm.py +20 -13
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/.github/workflows/publish.yml +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/.github/workflows/python-app.yml +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/.gitignore +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/CHANGELOG.md +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/CODE_OF_CONDUCT.md +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/LICENSE +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/NNsight_Walkthrough.ipynb +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/NNsight_v0_3_guide.ipynb +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/NNsight_v0_4_guide.ipynb +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/README.md +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/conftest.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/pyproject.toml +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/pytest.ini +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/setup.cfg +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/setup.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/_c/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/_c/py_mount.c +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/config.yaml +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/backends/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/backends/base.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/backends/editing.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/backends/execution.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/batching.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/inject.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/backwards.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/base.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/editing.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/globals.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/invoker.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/iterator.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/intervention/tracing/util.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/log.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/base.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/diffusion.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/mixins/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/mixins/loadable.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/mixins/meta.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner copy.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/sampling copy.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/sampling.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/vllm.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/schema/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/schema/config.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/schema/response.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/util.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight.egg-info/SOURCES.txt +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight.egg-info/dependency_links.txt +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight.egg-info/requires.txt +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight.egg-info/top_level.txt +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/tests/__init__.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/tests/test_diffusion.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/tests/test_tiny.py +0 -0
- {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/tests/test_vllm.py +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import inspect
|
|
3
4
|
import io
|
|
4
5
|
import sys
|
|
5
6
|
import time
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
|
|
7
8
|
|
|
8
9
|
import requests
|
|
9
10
|
import socketio
|
|
@@ -11,16 +12,14 @@ import torch
|
|
|
11
12
|
from tqdm.auto import tqdm
|
|
12
13
|
|
|
13
14
|
from ... import __IPYTHON__, CONFIG, __version__
|
|
15
|
+
from ..._c.py_mount import mount, unmount
|
|
16
|
+
from ...intervention.serialization import load, save
|
|
14
17
|
from ...log import remote_logger
|
|
15
18
|
from ...schema.request import RequestModel
|
|
16
19
|
from ...schema.response import RESULT, ResponseModel
|
|
20
|
+
from ..tracing.tracer import Tracer
|
|
17
21
|
from .base import Backend
|
|
18
22
|
|
|
19
|
-
if TYPE_CHECKING:
|
|
20
|
-
from ..tracing.tracer import Tracer
|
|
21
|
-
else:
|
|
22
|
-
Tracer = Any
|
|
23
|
-
|
|
24
23
|
|
|
25
24
|
class RemoteException(Exception):
|
|
26
25
|
pass
|
|
@@ -129,8 +128,20 @@ class RemoteBackend(Backend):
|
|
|
129
128
|
else:
|
|
130
129
|
|
|
131
130
|
result = response.data
|
|
132
|
-
|
|
131
|
+
|
|
133
132
|
return result
|
|
133
|
+
|
|
134
|
+
elif response.status == ResponseModel.JobStatus.STREAM:
|
|
135
|
+
|
|
136
|
+
model = getattr(tracer, "model", None)
|
|
137
|
+
|
|
138
|
+
fn = load(response.data, model)
|
|
139
|
+
|
|
140
|
+
local_tracer = LocalTracer(_info=tracer.info)
|
|
141
|
+
|
|
142
|
+
local_tracer.execute(fn)
|
|
143
|
+
|
|
144
|
+
|
|
134
145
|
|
|
135
146
|
|
|
136
147
|
def submit_request(
|
|
@@ -262,6 +273,7 @@ class RemoteBackend(Backend):
|
|
|
262
273
|
response = self.submit_request(data, headers)
|
|
263
274
|
|
|
264
275
|
try:
|
|
276
|
+
LocalTracer.register(lambda data: self.stream_send(data, sio))
|
|
265
277
|
# Loop until
|
|
266
278
|
while True:
|
|
267
279
|
|
|
@@ -278,22 +290,28 @@ class RemoteBackend(Backend):
|
|
|
278
290
|
except Exception as e:
|
|
279
291
|
|
|
280
292
|
raise e
|
|
293
|
+
|
|
294
|
+
finally:
|
|
295
|
+
LocalTracer.deregister()
|
|
296
|
+
|
|
281
297
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
298
|
+
def stream_send(
|
|
299
|
+
self, values: Dict[int, Any], sio: socketio.SimpleClient
|
|
300
|
+
):
|
|
301
|
+
"""Upload some value to the remote service for some job id.
|
|
286
302
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
303
|
+
Args:
|
|
304
|
+
value (Any): Value to upload
|
|
305
|
+
job_id (str): Job id.
|
|
306
|
+
sio (socketio.SimpleClient): Connected websocket client.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
data = save(values)
|
|
292
310
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
311
|
+
sio.emit(
|
|
312
|
+
"stream_upload",
|
|
313
|
+
data=(data, self.job_id),
|
|
314
|
+
)
|
|
297
315
|
|
|
298
316
|
def non_blocking_request(self, tracer: Tracer):
|
|
299
317
|
"""Send intervention request to the remote service if request provided. Otherwise get job status.
|
|
@@ -338,3 +356,50 @@ class RemoteBackend(Backend):
|
|
|
338
356
|
CONFIG.save()
|
|
339
357
|
|
|
340
358
|
raise e
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class LocalTracer(Tracer):
|
|
362
|
+
|
|
363
|
+
_send: Callable = None
|
|
364
|
+
|
|
365
|
+
def __init__(self, *args, **kwargs):
|
|
366
|
+
super().__init__(*args, **kwargs)
|
|
367
|
+
|
|
368
|
+
self.remotes = set()
|
|
369
|
+
|
|
370
|
+
@classmethod
|
|
371
|
+
def register(cls, send_fn: Callable):
|
|
372
|
+
|
|
373
|
+
cls._send = send_fn
|
|
374
|
+
|
|
375
|
+
@classmethod
|
|
376
|
+
def deregister(cls):
|
|
377
|
+
|
|
378
|
+
cls._send = None
|
|
379
|
+
|
|
380
|
+
def _save_remote(self, obj: Any):
|
|
381
|
+
|
|
382
|
+
self.remotes.add(id(obj))
|
|
383
|
+
|
|
384
|
+
def execute(self, fn: Callable):
|
|
385
|
+
|
|
386
|
+
mount(self._save_remote, "remote")
|
|
387
|
+
|
|
388
|
+
fn(self, self.info)
|
|
389
|
+
|
|
390
|
+
unmount("remote")
|
|
391
|
+
|
|
392
|
+
return
|
|
393
|
+
|
|
394
|
+
def push(self):
|
|
395
|
+
|
|
396
|
+
# Find the frame where the traced code is executing
|
|
397
|
+
state_frame = inspect.currentframe().f_back
|
|
398
|
+
|
|
399
|
+
state = state_frame.f_locals
|
|
400
|
+
|
|
401
|
+
super().push(state)
|
|
402
|
+
|
|
403
|
+
state = {k:v for k,v in state.items() if id(v) in self.remotes}
|
|
404
|
+
|
|
405
|
+
LocalTracer._send(state)
|
|
@@ -5,14 +5,14 @@ import os
|
|
|
5
5
|
import warnings
|
|
6
6
|
from functools import wraps
|
|
7
7
|
from types import BuiltinFunctionType, BuiltinMethodType, FunctionType, MethodType
|
|
8
|
-
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type, Union
|
|
9
9
|
|
|
10
10
|
import torch
|
|
11
11
|
from torch.nn.modules.module import _addindent
|
|
12
12
|
|
|
13
13
|
from .. import CONFIG, base_deprecation_message, deprecated, util
|
|
14
14
|
from ..util import apply, Patch
|
|
15
|
-
|
|
15
|
+
|
|
16
16
|
from .batching import Batchable
|
|
17
17
|
from .inject import convert as inject
|
|
18
18
|
from .tracing.base import Tracer, WithBlockNotFoundError
|
|
@@ -386,7 +386,7 @@ class Envoy(Batchable):
|
|
|
386
386
|
|
|
387
387
|
#### Public methods ####
|
|
388
388
|
|
|
389
|
-
def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, **kwargs):
|
|
389
|
+
def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, tracer_cls: Type[InterleavingTracer] = InterleavingTracer, **kwargs):
|
|
390
390
|
"""
|
|
391
391
|
Create a tracer for this module.
|
|
392
392
|
|
|
@@ -418,7 +418,7 @@ class Envoy(Batchable):
|
|
|
418
418
|
fn = self.__call__
|
|
419
419
|
kwargs["hook"] = True
|
|
420
420
|
|
|
421
|
-
return
|
|
421
|
+
return tracer_cls(fn, self, *args, **kwargs)
|
|
422
422
|
|
|
423
423
|
def scan(self, *args, **kwargs):
|
|
424
424
|
"""
|
|
@@ -518,6 +518,8 @@ class Envoy(Batchable):
|
|
|
518
518
|
export_dir = os.path.expanduser(os.path.join(export_dir, name))
|
|
519
519
|
|
|
520
520
|
os.makedirs(export_dir, exist_ok=True)
|
|
521
|
+
|
|
522
|
+
from . import serialization
|
|
521
523
|
|
|
522
524
|
serialization.save(
|
|
523
525
|
self._default_mediators, os.path.join(export_dir, f"{variant}.dill")
|
|
@@ -539,6 +541,8 @@ class Envoy(Batchable):
|
|
|
539
541
|
export_dir = os.path.join(CONFIG.APP.CACHE_DIR, "exports")
|
|
540
542
|
|
|
541
543
|
export_dir = os.path.expanduser(os.path.join(export_dir, name))
|
|
544
|
+
|
|
545
|
+
from . import serialization
|
|
542
546
|
|
|
543
547
|
imported_mediators = serialization.load(
|
|
544
548
|
os.path.join(export_dir, f"{variant}.dill"), self
|
|
@@ -547,8 +551,8 @@ class Envoy(Batchable):
|
|
|
547
551
|
self._default_mediators.extend(imported_mediators)
|
|
548
552
|
|
|
549
553
|
# TODO legacy
|
|
550
|
-
def session(self, *args, **kwargs):
|
|
551
|
-
return
|
|
554
|
+
def session(self, *args, tracer_cls: Type[Tracer] = Tracer, **kwargs):
|
|
555
|
+
return tracer_cls(*args, **kwargs)
|
|
552
556
|
|
|
553
557
|
# TODO legacy
|
|
554
558
|
@property
|
|
@@ -10,7 +10,7 @@ from enum import Enum
|
|
|
10
10
|
from functools import wraps
|
|
11
11
|
from queue import Queue
|
|
12
12
|
from threading import Thread
|
|
13
|
-
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
|
|
13
|
+
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
|
|
14
14
|
Union)
|
|
15
15
|
|
|
16
16
|
import torch
|
|
@@ -32,7 +32,8 @@ class Events(Enum):
|
|
|
32
32
|
END = "end" # Signal to end the execution
|
|
33
33
|
EXCEPTION = "exception" # Signal that an exception occurred
|
|
34
34
|
SKIP = "skip" # Signal that an operation should be skipped
|
|
35
|
-
|
|
35
|
+
REGISTER = "register" # Signal that a child mediator should be registered
|
|
36
|
+
BARRIER = "barrier" # Signal that a barrier should be set
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
class Cancelation(Exception):
|
|
@@ -49,14 +50,6 @@ class EarlyStopException(Exception):
|
|
|
49
50
|
pass
|
|
50
51
|
|
|
51
52
|
|
|
52
|
-
class UnboundIteratorException(Exception):
|
|
53
|
-
"""
|
|
54
|
-
An .iterator was used with no ties to any module.
|
|
55
|
-
"""
|
|
56
|
-
|
|
57
|
-
pass
|
|
58
|
-
|
|
59
|
-
|
|
60
53
|
class SkipException(Exception):
|
|
61
54
|
"""
|
|
62
55
|
Exception raised to skip the execution of the model.
|
|
@@ -425,10 +418,11 @@ class Mediator:
|
|
|
425
418
|
self.batch_group = batch_group
|
|
426
419
|
self.event_queue = Queue()
|
|
427
420
|
self.response_queue = Queue()
|
|
421
|
+
|
|
422
|
+
self.child: Mediator = None
|
|
428
423
|
|
|
429
424
|
self.thread = None
|
|
430
425
|
self.interleaver = None
|
|
431
|
-
self.child: Mediator = None
|
|
432
426
|
self.history = set()
|
|
433
427
|
self.user_cache: List["Cache"] = list()
|
|
434
428
|
self.iteration = 0
|
|
@@ -496,7 +490,16 @@ class Mediator:
|
|
|
496
490
|
Returns:
|
|
497
491
|
The original or modified value
|
|
498
492
|
"""
|
|
499
|
-
|
|
493
|
+
|
|
494
|
+
if self.child is not None:
|
|
495
|
+
self.child.handle(provider)
|
|
496
|
+
|
|
497
|
+
if not self.child.alive:
|
|
498
|
+
self.child = None
|
|
499
|
+
self.respond()
|
|
500
|
+
else:
|
|
501
|
+
return
|
|
502
|
+
|
|
500
503
|
process = not self.event_queue.empty()
|
|
501
504
|
|
|
502
505
|
event = None
|
|
@@ -521,13 +524,12 @@ class Mediator:
|
|
|
521
524
|
for cache in self.user_cache:
|
|
522
525
|
cache.add(provider, e.value)
|
|
523
526
|
raise e
|
|
527
|
+
elif event == Events.REGISTER:
|
|
528
|
+
process = self.handle_register_event(provider, data)
|
|
529
|
+
elif event == Events.BARRIER:
|
|
530
|
+
process = self.handle_barrier_event(provider, data)
|
|
524
531
|
elif event == Events.END:
|
|
525
532
|
process = False
|
|
526
|
-
elif event == Events.CONTINUE:
|
|
527
|
-
process = False
|
|
528
|
-
# Putting it down here gives the child a chance to handle after the parent has already had one for this provider.
|
|
529
|
-
if self.handle_child(provider):
|
|
530
|
-
return
|
|
531
533
|
|
|
532
534
|
if event == Events.END:
|
|
533
535
|
self.handle_end_event()
|
|
@@ -543,28 +545,32 @@ class Mediator:
|
|
|
543
545
|
),
|
|
544
546
|
)
|
|
545
547
|
|
|
546
|
-
def
|
|
547
|
-
|
|
548
|
-
if self.child is not None:
|
|
549
|
-
|
|
550
|
-
self.child.handle(provider)
|
|
551
|
-
|
|
552
|
-
# If the child was canceled
|
|
553
|
-
if self.child.thread is None:
|
|
554
|
-
|
|
555
|
-
# Tell the parent (current mediator) to continue
|
|
556
|
-
self.response_queue.put(None)
|
|
548
|
+
def handle_register_event(self, provider: Any, child:Mediator) -> bool:
|
|
557
549
|
|
|
558
|
-
|
|
559
|
-
|
|
550
|
+
self.child = child
|
|
551
|
+
child.start(self.interleaver)
|
|
552
|
+
child.handle(provider)
|
|
553
|
+
|
|
554
|
+
return False
|
|
560
555
|
|
|
561
|
-
|
|
562
|
-
|
|
556
|
+
def handle_barrier_event(self, provider: Any, participants: Set[str]):
|
|
557
|
+
"""
|
|
558
|
+
Handle a barrier event by setting a barrier.
|
|
559
|
+
"""
|
|
560
|
+
|
|
561
|
+
if participants is not None:
|
|
562
|
+
|
|
563
|
+
for mediator in self.interleaver.invokers:
|
|
563
564
|
|
|
564
|
-
|
|
565
|
-
|
|
565
|
+
while mediator.child is not None:
|
|
566
|
+
mediator = mediator.child
|
|
567
|
+
|
|
568
|
+
if mediator.name in participants:
|
|
569
|
+
|
|
570
|
+
mediator.handle(provider)
|
|
571
|
+
|
|
572
|
+
mediator.respond()
|
|
566
573
|
|
|
567
|
-
return True
|
|
568
574
|
|
|
569
575
|
|
|
570
576
|
def handle_end_event(self):
|
|
@@ -803,15 +809,8 @@ class Mediator:
|
|
|
803
809
|
|
|
804
810
|
mediator.iteration = iter
|
|
805
811
|
mediator.args = list([mediator.iteration])
|
|
806
|
-
self.child = mediator
|
|
807
|
-
mediator.start(self.interleaver)
|
|
808
812
|
|
|
809
|
-
|
|
810
|
-
raise UnboundIteratorException("`.iter` encountered with no dependence on any module.\n`.iter` only makes sense in the context of iteration over mu;ltiple module exeecutions." )
|
|
811
|
-
|
|
812
|
-
self.event_queue.put((Events.CONTINUE, None))
|
|
813
|
-
|
|
814
|
-
self.response_queue.get()
|
|
813
|
+
self.send(Events.REGISTER, mediator)
|
|
815
814
|
|
|
816
815
|
if isinstance(iteration, slice):
|
|
817
816
|
|
|
@@ -842,8 +841,6 @@ class Mediator:
|
|
|
842
841
|
|
|
843
842
|
do_iteration(iteration)
|
|
844
843
|
|
|
845
|
-
self.child = None
|
|
846
|
-
|
|
847
844
|
def stop(self):
|
|
848
845
|
"""Stop the execution of the model by raising an EarlyStopException."""
|
|
849
846
|
|
|
@@ -1,24 +1,20 @@
|
|
|
1
|
+
import io
|
|
1
2
|
import pickle
|
|
3
|
+
from builtins import open
|
|
2
4
|
from types import FrameType
|
|
3
|
-
from typing import TYPE_CHECKING, Any, Optional
|
|
5
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
4
6
|
|
|
5
7
|
import dill
|
|
6
8
|
|
|
7
|
-
|
|
8
|
-
from .envoy import Envoy
|
|
9
|
-
else:
|
|
10
|
-
Envoy = Any
|
|
11
|
-
|
|
9
|
+
from .envoy import Envoy
|
|
12
10
|
|
|
13
11
|
|
|
14
12
|
class CustomDillPickler(dill.Pickler):
|
|
15
13
|
def persistent_id(self, obj):
|
|
16
14
|
|
|
17
|
-
from .envoy import Envoy
|
|
18
|
-
|
|
19
15
|
if isinstance(obj, Envoy):
|
|
20
16
|
return f"ENVOY:{obj.path}"
|
|
21
|
-
|
|
17
|
+
|
|
22
18
|
if isinstance(obj, FrameType):
|
|
23
19
|
return "FRAME"
|
|
24
20
|
|
|
@@ -43,13 +39,24 @@ class CustomDillUnpickler(dill.Unpickler):
|
|
|
43
39
|
raise pickle.UnpicklingError(f"Unknown persistent id: {pid}")
|
|
44
40
|
|
|
45
41
|
|
|
46
|
-
def save(obj: Any, path: str):
|
|
42
|
+
def save(obj: Any, path: Optional[str] = None):
|
|
43
|
+
|
|
44
|
+
dill.settings["recurse"] = True
|
|
45
|
+
|
|
46
|
+
if path is None:
|
|
47
|
+
file = io.BytesIO()
|
|
48
|
+
CustomDillPickler(file).dump(obj)
|
|
49
|
+
file.seek(0)
|
|
50
|
+
return file.read()
|
|
47
51
|
|
|
48
52
|
with open(path, "wb") as file:
|
|
49
53
|
CustomDillPickler(file).dump(obj)
|
|
50
54
|
|
|
51
55
|
|
|
52
|
-
def load(
|
|
56
|
+
def load(data: Union[str, bytes], model: Envoy, frame: Optional[FrameType] = None):
|
|
57
|
+
|
|
58
|
+
if isinstance(data, bytes):
|
|
59
|
+
return CustomDillUnpickler(io.BytesIO(data), model, frame).load()
|
|
53
60
|
|
|
54
|
-
with open(
|
|
61
|
+
with open(data, "rb") as file:
|
|
55
62
|
return CustomDillUnpickler(file, model, frame).load()
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import re
|
|
3
3
|
from dataclasses import dataclass
|
|
4
|
-
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional,
|
|
5
|
-
Union)
|
|
4
|
+
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
|
|
5
|
+
Tuple, Union)
|
|
6
6
|
|
|
7
7
|
import torch
|
|
8
8
|
from torch._subclasses.fake_tensor import FakeCopyMode, FakeTensorMode
|
|
@@ -11,7 +11,7 @@ from torch.fx.experimental.symbolic_shapes import ShapeEnv
|
|
|
11
11
|
from ... import util
|
|
12
12
|
from ..backends.base import Backend
|
|
13
13
|
from ..batching import Batcher
|
|
14
|
-
from ..interleaver import
|
|
14
|
+
from ..interleaver import Events, Mediator
|
|
15
15
|
from .base import ExitTracingException, Tracer
|
|
16
16
|
from .globals import Object
|
|
17
17
|
from .invoker import Invoker
|
|
@@ -19,7 +19,6 @@ from .iterator import IteratorProxy
|
|
|
19
19
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from ..envoy import Envoy
|
|
22
|
-
from ..interleaver import Mediator
|
|
23
22
|
else:
|
|
24
23
|
Envoy = Any
|
|
25
24
|
|
|
@@ -408,6 +407,36 @@ class InterleavingTracer(Tracer):
|
|
|
408
407
|
)
|
|
409
408
|
|
|
410
409
|
return self.model._interleaver.current.user_cache[-1].cache
|
|
410
|
+
|
|
411
|
+
def barrier(self, n_participants: int):
|
|
412
|
+
"""
|
|
413
|
+
nnsight barrier: A synchronization primitive for coordinating multiple concurrent invocations in nnsight.
|
|
414
|
+
|
|
415
|
+
This works similarly to a threading.Barrier, but is designed for use with nnsight's model tracing and intervention system.
|
|
416
|
+
A barrier allows you to pause execution in multiple parallel invocations until all participants have reached the barrier,
|
|
417
|
+
at which point all are released to continue. This is useful when you want to synchronize the execution of different
|
|
418
|
+
model runs, for example to ensure that all have reached a certain point (such as after embedding lookup) before
|
|
419
|
+
proceeding to the next stage (such as generation or intervention).
|
|
420
|
+
|
|
421
|
+
Example usage:
|
|
422
|
+
|
|
423
|
+
with gpt2.generate(max_new_tokens=3) as tracer:
|
|
424
|
+
barrier = tracer.barrier(2)
|
|
425
|
+
|
|
426
|
+
with tracer.invoke(MSG_prompt):
|
|
427
|
+
embeddings = gpt2.transformer.wte.output
|
|
428
|
+
barrier()
|
|
429
|
+
output1 = gpt2.generator.output.save()
|
|
430
|
+
|
|
431
|
+
with tracer.invoke("_ _ _ _ _ _ _ _ _"):
|
|
432
|
+
barrier()
|
|
433
|
+
gpt2.transformer.wte.output = embeddings
|
|
434
|
+
output2 = gpt2.generator.output.save()
|
|
435
|
+
|
|
436
|
+
In this example, both invocations will pause at the barrier until both have reached it, ensuring synchronization.
|
|
437
|
+
"""
|
|
438
|
+
|
|
439
|
+
return Barrier(self.model, n_participants)
|
|
411
440
|
|
|
412
441
|
### Serialization ###
|
|
413
442
|
|
|
@@ -495,4 +524,26 @@ class ScanningTracer(InterleavingTracer):
|
|
|
495
524
|
|
|
496
525
|
# Clean up hooks
|
|
497
526
|
for hook in hooks:
|
|
498
|
-
hook.remove()
|
|
527
|
+
hook.remove()
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
class Barrier:
|
|
532
|
+
|
|
533
|
+
def __init__(self, model: Envoy, n_participants: int):
|
|
534
|
+
|
|
535
|
+
self.model = model
|
|
536
|
+
self.n_participants = n_participants
|
|
537
|
+
self.participants: Set[str] = set()
|
|
538
|
+
|
|
539
|
+
def __call__(self):
|
|
540
|
+
|
|
541
|
+
mediator = self.model._interleaver.current
|
|
542
|
+
|
|
543
|
+
self.participants.add(mediator.name)
|
|
544
|
+
|
|
545
|
+
if len(self.participants) == self.n_participants:
|
|
546
|
+
mediator.send(Events.BARRIER, self.participants)
|
|
547
|
+
self.participants = set()
|
|
548
|
+
else:
|
|
549
|
+
mediator.send(Events.BARRIER, None)
|
|
@@ -63,6 +63,22 @@ class LanguageModel(RemoteableMixin):
|
|
|
63
63
|
"""
|
|
64
64
|
|
|
65
65
|
tokenizer: PreTrainedTokenizer
|
|
66
|
+
|
|
67
|
+
class Generator(WrapperModule):
|
|
68
|
+
|
|
69
|
+
class Streamer(WrapperModule):
|
|
70
|
+
|
|
71
|
+
def put(self, *args):
|
|
72
|
+
return self(*args)
|
|
73
|
+
|
|
74
|
+
def end(self):
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
def __init__(self) -> None:
|
|
78
|
+
|
|
79
|
+
super().__init__()
|
|
80
|
+
|
|
81
|
+
self.streamer = LanguageModel.Generator.Streamer()
|
|
66
82
|
|
|
67
83
|
def __init__(
|
|
68
84
|
self,
|
|
@@ -98,7 +114,7 @@ class LanguageModel(RemoteableMixin):
|
|
|
98
114
|
|
|
99
115
|
self.import_edits()
|
|
100
116
|
|
|
101
|
-
self.generator: Envoy =
|
|
117
|
+
self.generator: Envoy = LanguageModel.Generator()
|
|
102
118
|
|
|
103
119
|
# Some transformer models compile on first generation. As of 0.5.0.dev7 this not not work with nnsight if fullgraph is True
|
|
104
120
|
def _patch_generation_config(self, model:torch.nn.Module):
|
|
@@ -163,12 +179,14 @@ class LanguageModel(RemoteableMixin):
|
|
|
163
179
|
if max_new_tokens is not None and self._interleaver is not None:
|
|
164
180
|
self._interleaver.default_all = max_new_tokens
|
|
165
181
|
|
|
166
|
-
|
|
182
|
+
streamer = kwargs.pop("streamer", self.generator.streamer._module)
|
|
183
|
+
|
|
184
|
+
output = self._model.generate(*args, streamer=streamer, **kwargs)
|
|
167
185
|
|
|
168
186
|
if self._interleaver is not None:
|
|
169
187
|
self._interleaver.default_all = None
|
|
170
188
|
|
|
171
|
-
output = self.generator
|
|
189
|
+
output = self.generator(output, hook=True)
|
|
172
190
|
|
|
173
191
|
return output
|
|
174
192
|
|
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
import ctypes
|
|
2
|
+
import inspect
|
|
3
|
+
from types import FrameType
|
|
4
|
+
from typing import Any, Callable, Dict, Union
|
|
2
5
|
|
|
3
6
|
from typing_extensions import Self
|
|
4
7
|
|
|
5
8
|
from ...intervention.backends import Backend
|
|
6
9
|
from ...intervention.backends.remote import RemoteBackend
|
|
10
|
+
from ...intervention.serialization import load, save
|
|
11
|
+
from ...intervention.tracing.tracer import InterleavingTracer, Tracer
|
|
7
12
|
from ...util import from_import_path, to_import_path
|
|
8
13
|
from .meta import MetaMixin
|
|
9
14
|
|
|
@@ -31,9 +36,10 @@ class RemoteableMixin(MetaMixin):
|
|
|
31
36
|
return super().trace(
|
|
32
37
|
*inputs,
|
|
33
38
|
backend=backend,
|
|
39
|
+
tracer_cls=RemoteInterleavingTracer,
|
|
34
40
|
**kwargs,
|
|
35
41
|
)
|
|
36
|
-
|
|
42
|
+
|
|
37
43
|
def session(
|
|
38
44
|
self,
|
|
39
45
|
*inputs: Any,
|
|
@@ -55,6 +61,7 @@ class RemoteableMixin(MetaMixin):
|
|
|
55
61
|
return super().session(
|
|
56
62
|
*inputs,
|
|
57
63
|
backend=backend,
|
|
64
|
+
tracer_cls=RemoteTracer,
|
|
58
65
|
**kwargs,
|
|
59
66
|
)
|
|
60
67
|
|
|
@@ -78,3 +85,72 @@ class RemoteableMixin(MetaMixin):
|
|
|
78
85
|
type: RemoteableMixin = from_import_path(import_path)
|
|
79
86
|
|
|
80
87
|
return type._remoteable_from_model_key(model_key, **kwargs)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class StreamTracer(Tracer):
|
|
91
|
+
|
|
92
|
+
_send: Callable = None
|
|
93
|
+
_recv: Callable = None
|
|
94
|
+
|
|
95
|
+
def __init__(self, frame: FrameType, *args, **kwargs):
|
|
96
|
+
super().__init__(*args, **kwargs)
|
|
97
|
+
|
|
98
|
+
self.frame = frame
|
|
99
|
+
|
|
100
|
+
@classmethod
|
|
101
|
+
def register(cls, send_fn: Callable, recv_fn: Callable):
|
|
102
|
+
|
|
103
|
+
cls._send = send_fn
|
|
104
|
+
cls._recv = recv_fn
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
def deregister(cls):
|
|
108
|
+
|
|
109
|
+
cls._send = None
|
|
110
|
+
cls._recv = None
|
|
111
|
+
|
|
112
|
+
def execute(self, fn: Callable):
|
|
113
|
+
|
|
114
|
+
data = save(fn)
|
|
115
|
+
|
|
116
|
+
if self._send is None:
|
|
117
|
+
raise ValueError("No send function provided")
|
|
118
|
+
|
|
119
|
+
if self._recv is None:
|
|
120
|
+
raise ValueError("No recv function provided")
|
|
121
|
+
|
|
122
|
+
self._send(data)
|
|
123
|
+
|
|
124
|
+
data: Dict[str, Any] = load(self._recv(), None)
|
|
125
|
+
|
|
126
|
+
if self.frame.f_code.co_filename.startswith("<nnsight"):
|
|
127
|
+
# For dynamically generated code, update both globals and locals
|
|
128
|
+
self.frame.f_globals.update(data)
|
|
129
|
+
self.frame.f_locals.update(data)
|
|
130
|
+
|
|
131
|
+
# Ensure locals are properly synchronized with the frame
|
|
132
|
+
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
133
|
+
ctypes.py_object(self.frame), ctypes.c_int(0)
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
else:
|
|
137
|
+
# For regular files, just update locals
|
|
138
|
+
for key, value in data.items():
|
|
139
|
+
|
|
140
|
+
self.frame.f_locals[key] = value
|
|
141
|
+
|
|
142
|
+
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
143
|
+
ctypes.py_object(self.frame), ctypes.c_int(0)
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
class RemoteTracer(Tracer):
|
|
147
|
+
|
|
148
|
+
def local(self):
|
|
149
|
+
|
|
150
|
+
frame = inspect.currentframe().f_back
|
|
151
|
+
|
|
152
|
+
return StreamTracer(frame)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class RemoteInterleavingTracer(InterleavingTracer, RemoteTracer):
|
|
156
|
+
pass
|
|
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union
|
|
|
6
6
|
|
|
7
7
|
import dill
|
|
8
8
|
from pydantic import BaseModel, ConfigDict
|
|
9
|
-
from ..intervention.serialization import
|
|
9
|
+
from ..intervention.serialization import save, load
|
|
10
10
|
if TYPE_CHECKING:
|
|
11
11
|
from .. import NNsight
|
|
12
12
|
from ..intervention.tracing.tracer import Tracer
|
|
@@ -22,16 +22,8 @@ class RequestModel(BaseModel):
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def serialize(self, _zlib:bool) -> bytes:
|
|
25
|
-
|
|
26
|
-
dill.settings['recurse'] = True
|
|
27
25
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
CustomDillPickler(data).dump(self)
|
|
31
|
-
|
|
32
|
-
data.seek(0)
|
|
33
|
-
|
|
34
|
-
data = data.read()
|
|
26
|
+
data = save(self)
|
|
35
27
|
|
|
36
28
|
if _zlib:
|
|
37
29
|
|
|
@@ -50,7 +42,7 @@ class RequestModel(BaseModel):
|
|
|
50
42
|
|
|
51
43
|
data.seek(0)
|
|
52
44
|
|
|
53
|
-
request:RequestModel =
|
|
45
|
+
request:RequestModel = load(data.read(), model)
|
|
54
46
|
|
|
55
47
|
return request
|
|
56
48
|
|
|
@@ -165,15 +165,22 @@ def test_adhoc_module(gpt2: nnsight.LanguageModel):
|
|
|
165
165
|
def test_embeddings_set1(gpt2: nnsight.LanguageModel, MSG_prompt: str):
|
|
166
166
|
with gpt2.generate(
|
|
167
167
|
max_new_tokens=3
|
|
168
|
-
) as
|
|
169
|
-
|
|
168
|
+
) as tracer:
|
|
169
|
+
|
|
170
|
+
barrier = tracer.barrier(2)
|
|
171
|
+
|
|
172
|
+
with tracer.invoke(MSG_prompt):
|
|
170
173
|
embeddings = gpt2.transformer.wte.output
|
|
171
174
|
|
|
175
|
+
barrier()
|
|
176
|
+
|
|
172
177
|
|
|
173
178
|
output1 = gpt2.generator.output.save()
|
|
174
179
|
|
|
175
|
-
with
|
|
176
|
-
|
|
180
|
+
with tracer.invoke("_ _ _ _ _ _ _ _ _"):
|
|
181
|
+
|
|
182
|
+
barrier()
|
|
183
|
+
|
|
177
184
|
gpt2.transformer.wte.output = embeddings
|
|
178
185
|
|
|
179
186
|
output2 = gpt2.generator.output.save()
|
|
@@ -832,16 +839,16 @@ def test_batched_iter(gpt2: nnsight.LanguageModel, MSG_prompt: str):
|
|
|
832
839
|
assert gpt2.tokenizer.batch_decode(logits_1) == [" York", " City"]
|
|
833
840
|
assert gpt2.tokenizer.batch_decode(logits_2) == [" New", " York", " City"]
|
|
834
841
|
|
|
835
|
-
from nnsight.intervention.interleaver import UnboundIteratorException
|
|
836
|
-
@torch.no_grad()
|
|
837
|
-
@pytest.mark.iter
|
|
838
|
-
def test_one_iter(gpt2: nnsight.LanguageModel):
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
+
# from nnsight.intervention.interleaver import UnboundIteratorException
|
|
843
|
+
# @torch.no_grad()
|
|
844
|
+
# @pytest.mark.iter
|
|
845
|
+
# def test_one_iter(gpt2: nnsight.LanguageModel):
|
|
846
|
+
# with pytest.raises(UnboundIteratorException):
|
|
847
|
+
# with gpt2.generate("_", max_new_tokens=1) as tracer:
|
|
848
|
+
# arr_gen = list().save()
|
|
842
849
|
|
|
843
|
-
|
|
844
|
-
|
|
850
|
+
# with tracer.all():
|
|
851
|
+
# arr_gen.append(1)
|
|
845
852
|
|
|
846
853
|
|
|
847
854
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/model_runners/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev11 → nnsight-0.5.0.dev13}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|