nnsight 0.5.0.dev10__tar.gz → 0.5.0.dev12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/PKG-INFO +1 -1
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/remote.py +85 -20
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/envoy.py +14 -7
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/interleaver.py +55 -45
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/serialization.py +19 -12
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/tracer.py +55 -5
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/language.py +21 -3
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/remoteable.py +78 -2
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/request.py +3 -11
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/PKG-INFO +1 -1
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_lm.py +20 -13
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/.github/workflows/publish.yml +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/.github/workflows/python-app.yml +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/.gitignore +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/CHANGELOG.md +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/CODE_OF_CONDUCT.md +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/LICENSE +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/NNsight_Walkthrough.ipynb +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/NNsight_v0_3_guide.ipynb +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/NNsight_v0_4_guide.ipynb +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/README.md +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/conftest.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/pyproject.toml +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/pytest.ini +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/setup.cfg +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/setup.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/_c/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/_c/py_mount.c +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/config.yaml +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/base.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/editing.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/execution.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/batching.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/inject.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/backwards.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/base.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/editing.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/globals.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/invoker.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/iterator.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/util.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/log.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/base.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/diffusion.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/loadable.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/meta.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner copy.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/sampling copy.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/sampling.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/vllm.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/config.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/response.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/util.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/SOURCES.txt +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/dependency_links.txt +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/requires.txt +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/top_level.txt +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/__init__.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_diffusion.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_tiny.py +0 -0
- {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_vllm.py +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import inspect
|
|
3
4
|
import io
|
|
4
5
|
import sys
|
|
5
6
|
import time
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
|
|
7
8
|
|
|
8
9
|
import requests
|
|
9
10
|
import socketio
|
|
@@ -11,16 +12,14 @@ import torch
|
|
|
11
12
|
from tqdm.auto import tqdm
|
|
12
13
|
|
|
13
14
|
from ... import __IPYTHON__, CONFIG, __version__
|
|
15
|
+
from ..._c.py_mount import mount, unmount
|
|
16
|
+
from ...intervention.serialization import load, save
|
|
14
17
|
from ...log import remote_logger
|
|
15
18
|
from ...schema.request import RequestModel
|
|
16
19
|
from ...schema.response import RESULT, ResponseModel
|
|
20
|
+
from ..tracing.tracer import Tracer
|
|
17
21
|
from .base import Backend
|
|
18
22
|
|
|
19
|
-
if TYPE_CHECKING:
|
|
20
|
-
from ..tracing.tracer import Tracer
|
|
21
|
-
else:
|
|
22
|
-
Tracer = Any
|
|
23
|
-
|
|
24
23
|
|
|
25
24
|
class RemoteException(Exception):
|
|
26
25
|
pass
|
|
@@ -129,8 +128,20 @@ class RemoteBackend(Backend):
|
|
|
129
128
|
else:
|
|
130
129
|
|
|
131
130
|
result = response.data
|
|
132
|
-
|
|
131
|
+
|
|
133
132
|
return result
|
|
133
|
+
|
|
134
|
+
elif response.status == ResponseModel.JobStatus.STREAM:
|
|
135
|
+
|
|
136
|
+
model = getattr(tracer, "model", None)
|
|
137
|
+
|
|
138
|
+
fn = load(response.data, model)
|
|
139
|
+
|
|
140
|
+
local_tracer = LocalTracer(_info=tracer.info)
|
|
141
|
+
|
|
142
|
+
local_tracer.execute(fn)
|
|
143
|
+
|
|
144
|
+
|
|
134
145
|
|
|
135
146
|
|
|
136
147
|
def submit_request(
|
|
@@ -262,6 +273,7 @@ class RemoteBackend(Backend):
|
|
|
262
273
|
response = self.submit_request(data, headers)
|
|
263
274
|
|
|
264
275
|
try:
|
|
276
|
+
LocalTracer.register(lambda data: self.stream_send(data, sio))
|
|
265
277
|
# Loop until
|
|
266
278
|
while True:
|
|
267
279
|
|
|
@@ -278,22 +290,28 @@ class RemoteBackend(Backend):
|
|
|
278
290
|
except Exception as e:
|
|
279
291
|
|
|
280
292
|
raise e
|
|
293
|
+
|
|
294
|
+
finally:
|
|
295
|
+
LocalTracer.deregister()
|
|
296
|
+
|
|
281
297
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
298
|
+
def stream_send(
|
|
299
|
+
self, values: Dict[int, Any], sio: socketio.SimpleClient
|
|
300
|
+
):
|
|
301
|
+
"""Upload some value to the remote service for some job id.
|
|
286
302
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
303
|
+
Args:
|
|
304
|
+
value (Any): Value to upload
|
|
305
|
+
job_id (str): Job id.
|
|
306
|
+
sio (socketio.SimpleClient): Connected websocket client.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
data = save(values)
|
|
292
310
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
311
|
+
sio.emit(
|
|
312
|
+
"stream_upload",
|
|
313
|
+
data=(data, self.job_id),
|
|
314
|
+
)
|
|
297
315
|
|
|
298
316
|
def non_blocking_request(self, tracer: Tracer):
|
|
299
317
|
"""Send intervention request to the remote service if request provided. Otherwise get job status.
|
|
@@ -338,3 +356,50 @@ class RemoteBackend(Backend):
|
|
|
338
356
|
CONFIG.save()
|
|
339
357
|
|
|
340
358
|
raise e
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class LocalTracer(Tracer):
|
|
362
|
+
|
|
363
|
+
_send: Callable = None
|
|
364
|
+
|
|
365
|
+
def __init__(self, *args, **kwargs):
|
|
366
|
+
super().__init__(*args, **kwargs)
|
|
367
|
+
|
|
368
|
+
self.remotes = set()
|
|
369
|
+
|
|
370
|
+
@classmethod
|
|
371
|
+
def register(cls, send_fn: Callable):
|
|
372
|
+
|
|
373
|
+
cls._send = send_fn
|
|
374
|
+
|
|
375
|
+
@classmethod
|
|
376
|
+
def deregister(cls):
|
|
377
|
+
|
|
378
|
+
cls._send = None
|
|
379
|
+
|
|
380
|
+
def _save_remote(self, obj: Any):
|
|
381
|
+
|
|
382
|
+
self.remotes.add(id(obj))
|
|
383
|
+
|
|
384
|
+
def execute(self, fn: Callable):
|
|
385
|
+
|
|
386
|
+
mount(self._save_remote, "remote")
|
|
387
|
+
|
|
388
|
+
fn(self, self.info)
|
|
389
|
+
|
|
390
|
+
unmount("remote")
|
|
391
|
+
|
|
392
|
+
return
|
|
393
|
+
|
|
394
|
+
def push(self):
|
|
395
|
+
|
|
396
|
+
# Find the frame where the traced code is executing
|
|
397
|
+
state_frame = inspect.currentframe().f_back
|
|
398
|
+
|
|
399
|
+
state = state_frame.f_locals
|
|
400
|
+
|
|
401
|
+
super().push(state)
|
|
402
|
+
|
|
403
|
+
state = {k:v for k,v in state.items() if id(v) in self.remotes}
|
|
404
|
+
|
|
405
|
+
LocalTracer._send(state)
|
|
@@ -5,14 +5,14 @@ import os
|
|
|
5
5
|
import warnings
|
|
6
6
|
from functools import wraps
|
|
7
7
|
from types import BuiltinFunctionType, BuiltinMethodType, FunctionType, MethodType
|
|
8
|
-
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
|
|
8
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type, Union
|
|
9
9
|
|
|
10
10
|
import torch
|
|
11
11
|
from torch.nn.modules.module import _addindent
|
|
12
12
|
|
|
13
13
|
from .. import CONFIG, base_deprecation_message, deprecated, util
|
|
14
14
|
from ..util import apply, Patch
|
|
15
|
-
|
|
15
|
+
|
|
16
16
|
from .batching import Batchable
|
|
17
17
|
from .inject import convert as inject
|
|
18
18
|
from .tracing.base import Tracer, WithBlockNotFoundError
|
|
@@ -386,7 +386,7 @@ class Envoy(Batchable):
|
|
|
386
386
|
|
|
387
387
|
#### Public methods ####
|
|
388
388
|
|
|
389
|
-
def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, **kwargs):
|
|
389
|
+
def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, tracer_cls: Type[InterleavingTracer] = InterleavingTracer, **kwargs):
|
|
390
390
|
"""
|
|
391
391
|
Create a tracer for this module.
|
|
392
392
|
|
|
@@ -418,7 +418,7 @@ class Envoy(Batchable):
|
|
|
418
418
|
fn = self.__call__
|
|
419
419
|
kwargs["hook"] = True
|
|
420
420
|
|
|
421
|
-
return
|
|
421
|
+
return tracer_cls(fn, self, *args, **kwargs)
|
|
422
422
|
|
|
423
423
|
def scan(self, *args, **kwargs):
|
|
424
424
|
"""
|
|
@@ -518,6 +518,8 @@ class Envoy(Batchable):
|
|
|
518
518
|
export_dir = os.path.expanduser(os.path.join(export_dir, name))
|
|
519
519
|
|
|
520
520
|
os.makedirs(export_dir, exist_ok=True)
|
|
521
|
+
|
|
522
|
+
from . import serialization
|
|
521
523
|
|
|
522
524
|
serialization.save(
|
|
523
525
|
self._default_mediators, os.path.join(export_dir, f"{variant}.dill")
|
|
@@ -539,6 +541,8 @@ class Envoy(Batchable):
|
|
|
539
541
|
export_dir = os.path.join(CONFIG.APP.CACHE_DIR, "exports")
|
|
540
542
|
|
|
541
543
|
export_dir = os.path.expanduser(os.path.join(export_dir, name))
|
|
544
|
+
|
|
545
|
+
from . import serialization
|
|
542
546
|
|
|
543
547
|
imported_mediators = serialization.load(
|
|
544
548
|
os.path.join(export_dir, f"{variant}.dill"), self
|
|
@@ -547,8 +551,8 @@ class Envoy(Batchable):
|
|
|
547
551
|
self._default_mediators.extend(imported_mediators)
|
|
548
552
|
|
|
549
553
|
# TODO legacy
|
|
550
|
-
def session(self, *args, **kwargs):
|
|
551
|
-
return
|
|
554
|
+
def session(self, *args, tracer_cls: Type[Tracer] = Tracer, **kwargs):
|
|
555
|
+
return tracer_cls(*args, **kwargs)
|
|
552
556
|
|
|
553
557
|
# TODO legacy
|
|
554
558
|
@property
|
|
@@ -720,8 +724,11 @@ class Envoy(Batchable):
|
|
|
720
724
|
try:
|
|
721
725
|
with self._interleaver:
|
|
722
726
|
fn(*args, **kwargs)
|
|
723
|
-
|
|
727
|
+
|
|
724
728
|
self._interleaver.check_cache_full()
|
|
729
|
+
self._interleaver.check_dangling_mediators()
|
|
730
|
+
|
|
731
|
+
finally:
|
|
725
732
|
self._interleaver.cancel()
|
|
726
733
|
|
|
727
734
|
#### Private methods ####
|
|
@@ -10,7 +10,7 @@ from enum import Enum
|
|
|
10
10
|
from functools import wraps
|
|
11
11
|
from queue import Queue
|
|
12
12
|
from threading import Thread
|
|
13
|
-
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
|
|
13
|
+
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
|
|
14
14
|
Union)
|
|
15
15
|
|
|
16
16
|
import torch
|
|
@@ -32,7 +32,8 @@ class Events(Enum):
|
|
|
32
32
|
END = "end" # Signal to end the execution
|
|
33
33
|
EXCEPTION = "exception" # Signal that an exception occurred
|
|
34
34
|
SKIP = "skip" # Signal that an operation should be skipped
|
|
35
|
-
|
|
35
|
+
REGISTER = "register" # Signal that a child mediator should be registered
|
|
36
|
+
BARRIER = "barrier" # Signal that a barrier should be set
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
class Cancelation(Exception):
|
|
@@ -97,20 +98,15 @@ class Interleaver:
|
|
|
97
98
|
|
|
98
99
|
def cancel(self):
|
|
99
100
|
"""Cancel all intervention threads."""
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
self.check_dangling_mediators()
|
|
104
|
-
finally:
|
|
105
|
-
|
|
106
|
-
for mediator in list(self.mediators.values()):
|
|
107
|
-
mediator.cancel()
|
|
101
|
+
|
|
102
|
+
for mediator in list(self.mediators.values()):
|
|
103
|
+
mediator.cancel()
|
|
108
104
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
105
|
+
self.mediators = None
|
|
106
|
+
self.tracer = None
|
|
107
|
+
self.batcher = None
|
|
108
|
+
self.user_cache = None
|
|
109
|
+
self.invokers = None
|
|
114
110
|
|
|
115
111
|
|
|
116
112
|
def iterate(self, provider: str):
|
|
@@ -422,10 +418,11 @@ class Mediator:
|
|
|
422
418
|
self.batch_group = batch_group
|
|
423
419
|
self.event_queue = Queue()
|
|
424
420
|
self.response_queue = Queue()
|
|
421
|
+
|
|
422
|
+
self.child: Mediator = None
|
|
425
423
|
|
|
426
424
|
self.thread = None
|
|
427
425
|
self.interleaver = None
|
|
428
|
-
self.child: Mediator = None
|
|
429
426
|
self.history = set()
|
|
430
427
|
self.user_cache: List["Cache"] = list()
|
|
431
428
|
self.iteration = 0
|
|
@@ -493,33 +490,22 @@ class Mediator:
|
|
|
493
490
|
Returns:
|
|
494
491
|
The original or modified value
|
|
495
492
|
"""
|
|
496
|
-
|
|
493
|
+
|
|
497
494
|
if self.child is not None:
|
|
498
|
-
|
|
499
495
|
self.child.handle(provider)
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
if self.child.thread is None:
|
|
503
|
-
|
|
504
|
-
# Tell the parent (current mediator) to continue
|
|
505
|
-
self.response_queue.put(None)
|
|
506
|
-
|
|
507
|
-
# Then wipe the child
|
|
496
|
+
|
|
497
|
+
if not self.child.alive:
|
|
508
498
|
self.child = None
|
|
509
|
-
|
|
510
|
-
# Wait until the parent has an event
|
|
511
|
-
self.wait()
|
|
512
|
-
|
|
513
|
-
# Continue to handle parent event
|
|
499
|
+
self.respond()
|
|
514
500
|
else:
|
|
515
501
|
return
|
|
516
|
-
|
|
502
|
+
|
|
517
503
|
process = not self.event_queue.empty()
|
|
518
504
|
|
|
519
505
|
event = None
|
|
520
|
-
|
|
506
|
+
|
|
521
507
|
while process:
|
|
522
|
-
|
|
508
|
+
|
|
523
509
|
value = self.interleaver.batcher.current_value
|
|
524
510
|
|
|
525
511
|
event, data = self.event_queue.get()
|
|
@@ -538,10 +524,12 @@ class Mediator:
|
|
|
538
524
|
for cache in self.user_cache:
|
|
539
525
|
cache.add(provider, e.value)
|
|
540
526
|
raise e
|
|
527
|
+
elif event == Events.REGISTER:
|
|
528
|
+
process = self.handle_register_event(provider, data)
|
|
529
|
+
elif event == Events.BARRIER:
|
|
530
|
+
process = self.handle_barrier_event(provider, data)
|
|
541
531
|
elif event == Events.END:
|
|
542
532
|
process = False
|
|
543
|
-
elif event == Events.CONTINUE:
|
|
544
|
-
process = False
|
|
545
533
|
|
|
546
534
|
if event == Events.END:
|
|
547
535
|
self.handle_end_event()
|
|
@@ -556,6 +544,34 @@ class Mediator:
|
|
|
556
544
|
self.batch_group, self.interleaver.batcher.current_value
|
|
557
545
|
),
|
|
558
546
|
)
|
|
547
|
+
|
|
548
|
+
def handle_register_event(self, provider: Any, child:Mediator) -> bool:
|
|
549
|
+
|
|
550
|
+
self.child = child
|
|
551
|
+
child.start(self.interleaver)
|
|
552
|
+
child.handle(provider)
|
|
553
|
+
|
|
554
|
+
return False
|
|
555
|
+
|
|
556
|
+
def handle_barrier_event(self, provider: Any, participants: Set[str]):
|
|
557
|
+
"""
|
|
558
|
+
Handle a barrier event by setting a barrier.
|
|
559
|
+
"""
|
|
560
|
+
|
|
561
|
+
if participants is not None:
|
|
562
|
+
|
|
563
|
+
for mediator in self.interleaver.invokers:
|
|
564
|
+
|
|
565
|
+
while mediator.child is not None:
|
|
566
|
+
mediator = mediator.child
|
|
567
|
+
|
|
568
|
+
if mediator.name in participants:
|
|
569
|
+
|
|
570
|
+
mediator.handle(provider)
|
|
571
|
+
|
|
572
|
+
mediator.respond()
|
|
573
|
+
|
|
574
|
+
|
|
559
575
|
|
|
560
576
|
def handle_end_event(self):
|
|
561
577
|
"""
|
|
@@ -793,12 +809,8 @@ class Mediator:
|
|
|
793
809
|
|
|
794
810
|
mediator.iteration = iter
|
|
795
811
|
mediator.args = list([mediator.iteration])
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
self.event_queue.put((Events.CONTINUE, None))
|
|
800
|
-
|
|
801
|
-
self.response_queue.get()
|
|
812
|
+
|
|
813
|
+
self.send(Events.REGISTER, mediator)
|
|
802
814
|
|
|
803
815
|
if isinstance(iteration, slice):
|
|
804
816
|
|
|
@@ -809,7 +821,7 @@ class Mediator:
|
|
|
809
821
|
while True:
|
|
810
822
|
|
|
811
823
|
do_iteration(i)
|
|
812
|
-
|
|
824
|
+
|
|
813
825
|
if self.interleaver.default_all is not None and stop is None:
|
|
814
826
|
stop = self.interleaver.default_all
|
|
815
827
|
|
|
@@ -829,8 +841,6 @@ class Mediator:
|
|
|
829
841
|
|
|
830
842
|
do_iteration(iteration)
|
|
831
843
|
|
|
832
|
-
self.child = None
|
|
833
|
-
|
|
834
844
|
def stop(self):
|
|
835
845
|
"""Stop the execution of the model by raising an EarlyStopException."""
|
|
836
846
|
|
|
@@ -1,24 +1,20 @@
|
|
|
1
|
+
import io
|
|
1
2
|
import pickle
|
|
3
|
+
from builtins import open
|
|
2
4
|
from types import FrameType
|
|
3
|
-
from typing import TYPE_CHECKING, Any, Optional
|
|
5
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
4
6
|
|
|
5
7
|
import dill
|
|
6
8
|
|
|
7
|
-
|
|
8
|
-
from .envoy import Envoy
|
|
9
|
-
else:
|
|
10
|
-
Envoy = Any
|
|
11
|
-
|
|
9
|
+
from .envoy import Envoy
|
|
12
10
|
|
|
13
11
|
|
|
14
12
|
class CustomDillPickler(dill.Pickler):
|
|
15
13
|
def persistent_id(self, obj):
|
|
16
14
|
|
|
17
|
-
from .envoy import Envoy
|
|
18
|
-
|
|
19
15
|
if isinstance(obj, Envoy):
|
|
20
16
|
return f"ENVOY:{obj.path}"
|
|
21
|
-
|
|
17
|
+
|
|
22
18
|
if isinstance(obj, FrameType):
|
|
23
19
|
return "FRAME"
|
|
24
20
|
|
|
@@ -43,13 +39,24 @@ class CustomDillUnpickler(dill.Unpickler):
|
|
|
43
39
|
raise pickle.UnpicklingError(f"Unknown persistent id: {pid}")
|
|
44
40
|
|
|
45
41
|
|
|
46
|
-
def save(obj: Any, path: str):
|
|
42
|
+
def save(obj: Any, path: Optional[str] = None):
|
|
43
|
+
|
|
44
|
+
dill.settings["recurse"] = True
|
|
45
|
+
|
|
46
|
+
if path is None:
|
|
47
|
+
file = io.BytesIO()
|
|
48
|
+
CustomDillPickler(file).dump(obj)
|
|
49
|
+
file.seek(0)
|
|
50
|
+
return file.read()
|
|
47
51
|
|
|
48
52
|
with open(path, "wb") as file:
|
|
49
53
|
CustomDillPickler(file).dump(obj)
|
|
50
54
|
|
|
51
55
|
|
|
52
|
-
def load(
|
|
56
|
+
def load(data: Union[str, bytes], model: Envoy, frame: Optional[FrameType] = None):
|
|
57
|
+
|
|
58
|
+
if isinstance(data, bytes):
|
|
59
|
+
return CustomDillUnpickler(io.BytesIO(data), model, frame).load()
|
|
53
60
|
|
|
54
|
-
with open(
|
|
61
|
+
with open(data, "rb") as file:
|
|
55
62
|
return CustomDillUnpickler(file, model, frame).load()
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import re
|
|
3
3
|
from dataclasses import dataclass
|
|
4
|
-
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional,
|
|
5
|
-
Union)
|
|
4
|
+
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
|
|
5
|
+
Tuple, Union)
|
|
6
6
|
|
|
7
7
|
import torch
|
|
8
8
|
from torch._subclasses.fake_tensor import FakeCopyMode, FakeTensorMode
|
|
@@ -11,7 +11,7 @@ from torch.fx.experimental.symbolic_shapes import ShapeEnv
|
|
|
11
11
|
from ... import util
|
|
12
12
|
from ..backends.base import Backend
|
|
13
13
|
from ..batching import Batcher
|
|
14
|
-
from ..interleaver import
|
|
14
|
+
from ..interleaver import Events, Mediator
|
|
15
15
|
from .base import ExitTracingException, Tracer
|
|
16
16
|
from .globals import Object
|
|
17
17
|
from .invoker import Invoker
|
|
@@ -19,7 +19,6 @@ from .iterator import IteratorProxy
|
|
|
19
19
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from ..envoy import Envoy
|
|
22
|
-
from ..interleaver import Mediator
|
|
23
22
|
else:
|
|
24
23
|
Envoy = Any
|
|
25
24
|
|
|
@@ -408,6 +407,36 @@ class InterleavingTracer(Tracer):
|
|
|
408
407
|
)
|
|
409
408
|
|
|
410
409
|
return self.model._interleaver.current.user_cache[-1].cache
|
|
410
|
+
|
|
411
|
+
def barrier(self, n_participants: int):
|
|
412
|
+
"""
|
|
413
|
+
nnsight barrier: A synchronization primitive for coordinating multiple concurrent invocations in nnsight.
|
|
414
|
+
|
|
415
|
+
This works similarly to a threading.Barrier, but is designed for use with nnsight's model tracing and intervention system.
|
|
416
|
+
A barrier allows you to pause execution in multiple parallel invocations until all participants have reached the barrier,
|
|
417
|
+
at which point all are released to continue. This is useful when you want to synchronize the execution of different
|
|
418
|
+
model runs, for example to ensure that all have reached a certain point (such as after embedding lookup) before
|
|
419
|
+
proceeding to the next stage (such as generation or intervention).
|
|
420
|
+
|
|
421
|
+
Example usage:
|
|
422
|
+
|
|
423
|
+
with gpt2.generate(max_new_tokens=3) as tracer:
|
|
424
|
+
barrier = tracer.barrier(2)
|
|
425
|
+
|
|
426
|
+
with tracer.invoke(MSG_prompt):
|
|
427
|
+
embeddings = gpt2.transformer.wte.output
|
|
428
|
+
barrier()
|
|
429
|
+
output1 = gpt2.generator.output.save()
|
|
430
|
+
|
|
431
|
+
with tracer.invoke("_ _ _ _ _ _ _ _ _"):
|
|
432
|
+
barrier()
|
|
433
|
+
gpt2.transformer.wte.output = embeddings
|
|
434
|
+
output2 = gpt2.generator.output.save()
|
|
435
|
+
|
|
436
|
+
In this example, both invocations will pause at the barrier until both have reached it, ensuring synchronization.
|
|
437
|
+
"""
|
|
438
|
+
|
|
439
|
+
return Barrier(self.model, n_participants)
|
|
411
440
|
|
|
412
441
|
### Serialization ###
|
|
413
442
|
|
|
@@ -495,4 +524,25 @@ class ScanningTracer(InterleavingTracer):
|
|
|
495
524
|
|
|
496
525
|
# Clean up hooks
|
|
497
526
|
for hook in hooks:
|
|
498
|
-
hook.remove()
|
|
527
|
+
hook.remove()
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
class Barrier:
|
|
532
|
+
|
|
533
|
+
def __init__(self, model: Envoy, n_participants: int):
|
|
534
|
+
|
|
535
|
+
self.model = model
|
|
536
|
+
self.n_participants = n_participants
|
|
537
|
+
self.participants: Set[str] = set()
|
|
538
|
+
|
|
539
|
+
def __call__(self):
|
|
540
|
+
|
|
541
|
+
mediator = self.model._interleaver.current
|
|
542
|
+
|
|
543
|
+
self.participants.add(mediator.name)
|
|
544
|
+
|
|
545
|
+
if len(self.participants) == self.n_participants:
|
|
546
|
+
mediator.send(Events.BARRIER, self.participants)
|
|
547
|
+
else:
|
|
548
|
+
mediator.send(Events.BARRIER, None)
|
|
@@ -63,6 +63,22 @@ class LanguageModel(RemoteableMixin):
|
|
|
63
63
|
"""
|
|
64
64
|
|
|
65
65
|
tokenizer: PreTrainedTokenizer
|
|
66
|
+
|
|
67
|
+
class Generator(WrapperModule):
|
|
68
|
+
|
|
69
|
+
class Streamer(WrapperModule):
|
|
70
|
+
|
|
71
|
+
def put(self, *args):
|
|
72
|
+
return self(*args)
|
|
73
|
+
|
|
74
|
+
def end(self):
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
def __init__(self) -> None:
|
|
78
|
+
|
|
79
|
+
super().__init__()
|
|
80
|
+
|
|
81
|
+
self.streamer = LanguageModel.Generator.Streamer()
|
|
66
82
|
|
|
67
83
|
def __init__(
|
|
68
84
|
self,
|
|
@@ -98,7 +114,7 @@ class LanguageModel(RemoteableMixin):
|
|
|
98
114
|
|
|
99
115
|
self.import_edits()
|
|
100
116
|
|
|
101
|
-
self.generator: Envoy =
|
|
117
|
+
self.generator: Envoy = LanguageModel.Generator()
|
|
102
118
|
|
|
103
119
|
# Some transformer models compile on first generation. As of 0.5.0.dev7 this not not work with nnsight if fullgraph is True
|
|
104
120
|
def _patch_generation_config(self, model:torch.nn.Module):
|
|
@@ -163,12 +179,14 @@ class LanguageModel(RemoteableMixin):
|
|
|
163
179
|
if max_new_tokens is not None and self._interleaver is not None:
|
|
164
180
|
self._interleaver.default_all = max_new_tokens
|
|
165
181
|
|
|
166
|
-
|
|
182
|
+
streamer = kwargs.pop("streamer", self.generator.streamer._module)
|
|
183
|
+
|
|
184
|
+
output = self._model.generate(*args, streamer=streamer, **kwargs)
|
|
167
185
|
|
|
168
186
|
if self._interleaver is not None:
|
|
169
187
|
self._interleaver.default_all = None
|
|
170
188
|
|
|
171
|
-
output = self.generator
|
|
189
|
+
output = self.generator(output, hook=True)
|
|
172
190
|
|
|
173
191
|
return output
|
|
174
192
|
|
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
|
|
1
|
+
import ctypes
|
|
2
|
+
import inspect
|
|
3
|
+
from types import FrameType
|
|
4
|
+
from typing import Any, Callable, Dict, Union
|
|
2
5
|
|
|
3
6
|
from typing_extensions import Self
|
|
4
7
|
|
|
5
8
|
from ...intervention.backends import Backend
|
|
6
9
|
from ...intervention.backends.remote import RemoteBackend
|
|
10
|
+
from ...intervention.serialization import load, save
|
|
11
|
+
from ...intervention.tracing.tracer import InterleavingTracer, Tracer
|
|
7
12
|
from ...util import from_import_path, to_import_path
|
|
8
13
|
from .meta import MetaMixin
|
|
9
14
|
|
|
@@ -31,9 +36,10 @@ class RemoteableMixin(MetaMixin):
|
|
|
31
36
|
return super().trace(
|
|
32
37
|
*inputs,
|
|
33
38
|
backend=backend,
|
|
39
|
+
tracer_cls=RemoteInterleavingTracer,
|
|
34
40
|
**kwargs,
|
|
35
41
|
)
|
|
36
|
-
|
|
42
|
+
|
|
37
43
|
def session(
|
|
38
44
|
self,
|
|
39
45
|
*inputs: Any,
|
|
@@ -55,6 +61,7 @@ class RemoteableMixin(MetaMixin):
|
|
|
55
61
|
return super().session(
|
|
56
62
|
*inputs,
|
|
57
63
|
backend=backend,
|
|
64
|
+
tracer_cls=RemoteTracer,
|
|
58
65
|
**kwargs,
|
|
59
66
|
)
|
|
60
67
|
|
|
@@ -78,3 +85,72 @@ class RemoteableMixin(MetaMixin):
|
|
|
78
85
|
type: RemoteableMixin = from_import_path(import_path)
|
|
79
86
|
|
|
80
87
|
return type._remoteable_from_model_key(model_key, **kwargs)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class StreamTracer(Tracer):
|
|
91
|
+
|
|
92
|
+
_send: Callable = None
|
|
93
|
+
_recv: Callable = None
|
|
94
|
+
|
|
95
|
+
def __init__(self, frame: FrameType, *args, **kwargs):
|
|
96
|
+
super().__init__(*args, **kwargs)
|
|
97
|
+
|
|
98
|
+
self.frame = frame
|
|
99
|
+
|
|
100
|
+
@classmethod
|
|
101
|
+
def register(cls, send_fn: Callable, recv_fn: Callable):
|
|
102
|
+
|
|
103
|
+
cls._send = send_fn
|
|
104
|
+
cls._recv = recv_fn
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
def deregister(cls):
|
|
108
|
+
|
|
109
|
+
cls._send = None
|
|
110
|
+
cls._recv = None
|
|
111
|
+
|
|
112
|
+
def execute(self, fn: Callable):
|
|
113
|
+
|
|
114
|
+
data = save(fn)
|
|
115
|
+
|
|
116
|
+
if self._send is None:
|
|
117
|
+
raise ValueError("No send function provided")
|
|
118
|
+
|
|
119
|
+
if self._recv is None:
|
|
120
|
+
raise ValueError("No recv function provided")
|
|
121
|
+
|
|
122
|
+
self._send(data)
|
|
123
|
+
|
|
124
|
+
data: Dict[str, Any] = load(self._recv(), None)
|
|
125
|
+
|
|
126
|
+
if self.frame.f_code.co_filename.startswith("<nnsight"):
|
|
127
|
+
# For dynamically generated code, update both globals and locals
|
|
128
|
+
self.frame.f_globals.update(data)
|
|
129
|
+
self.frame.f_locals.update(data)
|
|
130
|
+
|
|
131
|
+
# Ensure locals are properly synchronized with the frame
|
|
132
|
+
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
133
|
+
ctypes.py_object(self.frame), ctypes.c_int(0)
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
else:
|
|
137
|
+
# For regular files, just update locals
|
|
138
|
+
for key, value in data.items():
|
|
139
|
+
|
|
140
|
+
self.frame.f_locals[key] = value
|
|
141
|
+
|
|
142
|
+
ctypes.pythonapi.PyFrame_LocalsToFast(
|
|
143
|
+
ctypes.py_object(self.frame), ctypes.c_int(0)
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
class RemoteTracer(Tracer):
|
|
147
|
+
|
|
148
|
+
def local(self):
|
|
149
|
+
|
|
150
|
+
frame = inspect.currentframe().f_back
|
|
151
|
+
|
|
152
|
+
return StreamTracer(frame)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class RemoteInterleavingTracer(InterleavingTracer, RemoteTracer):
|
|
156
|
+
pass
|
|
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union
|
|
|
6
6
|
|
|
7
7
|
import dill
|
|
8
8
|
from pydantic import BaseModel, ConfigDict
|
|
9
|
-
from ..intervention.serialization import
|
|
9
|
+
from ..intervention.serialization import save, load
|
|
10
10
|
if TYPE_CHECKING:
|
|
11
11
|
from .. import NNsight
|
|
12
12
|
from ..intervention.tracing.tracer import Tracer
|
|
@@ -22,16 +22,8 @@ class RequestModel(BaseModel):
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def serialize(self, _zlib:bool) -> bytes:
|
|
25
|
-
|
|
26
|
-
dill.settings['recurse'] = True
|
|
27
25
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
CustomDillPickler(data).dump(self)
|
|
31
|
-
|
|
32
|
-
data.seek(0)
|
|
33
|
-
|
|
34
|
-
data = data.read()
|
|
26
|
+
data = save(self)
|
|
35
27
|
|
|
36
28
|
if _zlib:
|
|
37
29
|
|
|
@@ -50,7 +42,7 @@ class RequestModel(BaseModel):
|
|
|
50
42
|
|
|
51
43
|
data.seek(0)
|
|
52
44
|
|
|
53
|
-
request:RequestModel =
|
|
45
|
+
request:RequestModel = load(data.read(), model)
|
|
54
46
|
|
|
55
47
|
return request
|
|
56
48
|
|
|
@@ -165,15 +165,22 @@ def test_adhoc_module(gpt2: nnsight.LanguageModel):
|
|
|
165
165
|
def test_embeddings_set1(gpt2: nnsight.LanguageModel, MSG_prompt: str):
|
|
166
166
|
with gpt2.generate(
|
|
167
167
|
max_new_tokens=3
|
|
168
|
-
) as
|
|
169
|
-
|
|
168
|
+
) as tracer:
|
|
169
|
+
|
|
170
|
+
barrier = tracer.barrier(2)
|
|
171
|
+
|
|
172
|
+
with tracer.invoke(MSG_prompt):
|
|
170
173
|
embeddings = gpt2.transformer.wte.output
|
|
171
174
|
|
|
175
|
+
barrier()
|
|
176
|
+
|
|
172
177
|
|
|
173
178
|
output1 = gpt2.generator.output.save()
|
|
174
179
|
|
|
175
|
-
with
|
|
176
|
-
|
|
180
|
+
with tracer.invoke("_ _ _ _ _ _ _ _ _"):
|
|
181
|
+
|
|
182
|
+
barrier()
|
|
183
|
+
|
|
177
184
|
gpt2.transformer.wte.output = embeddings
|
|
178
185
|
|
|
179
186
|
output2 = gpt2.generator.output.save()
|
|
@@ -832,17 +839,17 @@ def test_batched_iter(gpt2: nnsight.LanguageModel, MSG_prompt: str):
|
|
|
832
839
|
assert gpt2.tokenizer.batch_decode(logits_1) == [" York", " City"]
|
|
833
840
|
assert gpt2.tokenizer.batch_decode(logits_2) == [" New", " York", " City"]
|
|
834
841
|
|
|
842
|
+
# from nnsight.intervention.interleaver import UnboundIteratorException
|
|
843
|
+
# @torch.no_grad()
|
|
844
|
+
# @pytest.mark.iter
|
|
845
|
+
# def test_one_iter(gpt2: nnsight.LanguageModel):
|
|
846
|
+
# with pytest.raises(UnboundIteratorException):
|
|
847
|
+
# with gpt2.generate("_", max_new_tokens=1) as tracer:
|
|
848
|
+
# arr_gen = list().save()
|
|
835
849
|
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
def test_one_iter(gpt2: nnsight.LanguageModel):
|
|
839
|
-
with gpt2.generate("_", max_new_tokens=1) as tracer:
|
|
840
|
-
arr_gen = list().save()
|
|
841
|
-
|
|
842
|
-
with tracer.all():
|
|
843
|
-
arr_gen.append(1)
|
|
850
|
+
# with tracer.all():
|
|
851
|
+
# arr_gen.append(1)
|
|
844
852
|
|
|
845
|
-
assert len(arr_gen) == 1
|
|
846
853
|
|
|
847
854
|
|
|
848
855
|
@torch.no_grad()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|