nnsight 0.5.0.dev11__tar.gz → 0.5.0.dev12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/PKG-INFO +1 -1
  2. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/remote.py +85 -20
  3. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/envoy.py +10 -6
  4. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/interleaver.py +42 -45
  5. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/serialization.py +19 -12
  6. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/tracer.py +55 -5
  7. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/language.py +21 -3
  8. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/remoteable.py +78 -2
  9. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/schema/request.py +3 -11
  10. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/PKG-INFO +1 -1
  11. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/tests/test_lm.py +20 -13
  12. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/.github/workflows/publish.yml +0 -0
  13. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/.github/workflows/python-app.yml +0 -0
  14. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/.gitignore +0 -0
  15. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/CHANGELOG.md +0 -0
  16. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/CODE_OF_CONDUCT.md +0 -0
  17. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/LICENSE +0 -0
  18. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/NNsight_Walkthrough.ipynb +0 -0
  19. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/NNsight_v0_3_guide.ipynb +0 -0
  20. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/NNsight_v0_4_guide.ipynb +0 -0
  21. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/README.md +0 -0
  22. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/conftest.py +0 -0
  23. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/pyproject.toml +0 -0
  24. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/pytest.ini +0 -0
  25. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/setup.cfg +0 -0
  26. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/setup.py +0 -0
  27. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/__init__.py +0 -0
  28. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/_c/__init__.py +0 -0
  29. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/_c/py_mount.c +0 -0
  30. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/config.yaml +0 -0
  31. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/__init__.py +0 -0
  32. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/__init__.py +0 -0
  33. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/base.py +0 -0
  34. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/editing.py +0 -0
  35. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/execution.py +0 -0
  36. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/batching.py +0 -0
  37. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/inject.py +0 -0
  38. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/__init__.py +0 -0
  39. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/backwards.py +0 -0
  40. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/base.py +0 -0
  41. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/editing.py +0 -0
  42. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/globals.py +0 -0
  43. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/invoker.py +0 -0
  44. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/iterator.py +0 -0
  45. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/util.py +0 -0
  46. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/log.py +0 -0
  47. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/__init__.py +0 -0
  48. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/base.py +0 -0
  49. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/diffusion.py +0 -0
  50. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/__init__.py +0 -0
  51. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/loadable.py +0 -0
  52. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/meta.py +0 -0
  53. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/__init__.py +0 -0
  54. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner copy.py +0 -0
  55. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
  56. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
  57. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/sampling copy.py +0 -0
  58. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/sampling.py +0 -0
  59. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/vllm.py +0 -0
  60. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
  61. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
  62. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py +0 -0
  63. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/schema/__init__.py +0 -0
  64. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/schema/config.py +0 -0
  65. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/schema/response.py +0 -0
  66. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight/util.py +0 -0
  67. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/SOURCES.txt +0 -0
  68. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/dependency_links.txt +0 -0
  69. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/requires.txt +0 -0
  70. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/top_level.txt +0 -0
  71. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/tests/__init__.py +0 -0
  72. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/tests/test_diffusion.py +0 -0
  73. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/tests/test_tiny.py +0 -0
  74. {nnsight-0.5.0.dev11 → nnsight-0.5.0.dev12}/tests/test_vllm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nnsight
3
- Version: 0.5.0.dev11
3
+ Version: 0.5.0.dev12
4
4
  Summary: Package for interpreting and manipulating the internals of deep learning models.
5
5
  Author-email: Jaden Fiotto-Kaufman <jadenfk@outlook.com>
6
6
  License: MIT License
@@ -1,9 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import inspect
3
4
  import io
4
5
  import sys
5
6
  import time
6
- from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
7
+ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
7
8
 
8
9
  import requests
9
10
  import socketio
@@ -11,16 +12,14 @@ import torch
11
12
  from tqdm.auto import tqdm
12
13
 
13
14
  from ... import __IPYTHON__, CONFIG, __version__
15
+ from ..._c.py_mount import mount, unmount
16
+ from ...intervention.serialization import load, save
14
17
  from ...log import remote_logger
15
18
  from ...schema.request import RequestModel
16
19
  from ...schema.response import RESULT, ResponseModel
20
+ from ..tracing.tracer import Tracer
17
21
  from .base import Backend
18
22
 
19
- if TYPE_CHECKING:
20
- from ..tracing.tracer import Tracer
21
- else:
22
- Tracer = Any
23
-
24
23
 
25
24
  class RemoteException(Exception):
26
25
  pass
@@ -129,8 +128,20 @@ class RemoteBackend(Backend):
129
128
  else:
130
129
 
131
130
  result = response.data
132
-
131
+
133
132
  return result
133
+
134
+ elif response.status == ResponseModel.JobStatus.STREAM:
135
+
136
+ model = getattr(tracer, "model", None)
137
+
138
+ fn = load(response.data, model)
139
+
140
+ local_tracer = LocalTracer(_info=tracer.info)
141
+
142
+ local_tracer.execute(fn)
143
+
144
+
134
145
 
135
146
 
136
147
  def submit_request(
@@ -262,6 +273,7 @@ class RemoteBackend(Backend):
262
273
  response = self.submit_request(data, headers)
263
274
 
264
275
  try:
276
+ LocalTracer.register(lambda data: self.stream_send(data, sio))
265
277
  # Loop until
266
278
  while True:
267
279
 
@@ -278,22 +290,28 @@ class RemoteBackend(Backend):
278
290
  except Exception as e:
279
291
 
280
292
  raise e
293
+
294
+ finally:
295
+ LocalTracer.deregister()
296
+
281
297
 
282
- # def stream_send(
283
- # self, values: Dict[int, Any], job_id: str, sio: socketio.SimpleClient
284
- # ):
285
- # """Upload some value to the remote service for some job id.
298
+ def stream_send(
299
+ self, values: Dict[int, Any], sio: socketio.SimpleClient
300
+ ):
301
+ """Upload some value to the remote service for some job id.
286
302
 
287
- # Args:
288
- # value (Any): Value to upload
289
- # job_id (str): Job id.
290
- # sio (socketio.SimpleClient): Connected websocket client.
291
- # """
303
+ Args:
304
+ value (Any): Value to upload
305
+ job_id (str): Job id.
306
+ sio (socketio.SimpleClient): Connected websocket client.
307
+ """
308
+
309
+ data = save(values)
292
310
 
293
- # sio.emit(
294
- # "stream_upload",
295
- # data=(StreamValueModel.serialize(values, self.format, self.zlib), job_id),
296
- # )
311
+ sio.emit(
312
+ "stream_upload",
313
+ data=(data, self.job_id),
314
+ )
297
315
 
298
316
  def non_blocking_request(self, tracer: Tracer):
299
317
  """Send intervention request to the remote service if request provided. Otherwise get job status.
@@ -338,3 +356,50 @@ class RemoteBackend(Backend):
338
356
  CONFIG.save()
339
357
 
340
358
  raise e
359
+
360
+
361
+ class LocalTracer(Tracer):
362
+
363
+ _send: Callable = None
364
+
365
+ def __init__(self, *args, **kwargs):
366
+ super().__init__(*args, **kwargs)
367
+
368
+ self.remotes = set()
369
+
370
+ @classmethod
371
+ def register(cls, send_fn: Callable):
372
+
373
+ cls._send = send_fn
374
+
375
+ @classmethod
376
+ def deregister(cls):
377
+
378
+ cls._send = None
379
+
380
+ def _save_remote(self, obj: Any):
381
+
382
+ self.remotes.add(id(obj))
383
+
384
+ def execute(self, fn: Callable):
385
+
386
+ mount(self._save_remote, "remote")
387
+
388
+ fn(self, self.info)
389
+
390
+ unmount("remote")
391
+
392
+ return
393
+
394
+ def push(self):
395
+
396
+ # Find the frame where the traced code is executing
397
+ state_frame = inspect.currentframe().f_back
398
+
399
+ state = state_frame.f_locals
400
+
401
+ super().push(state)
402
+
403
+ state = {k:v for k,v in state.items() if id(v) in self.remotes}
404
+
405
+ LocalTracer._send(state)
@@ -5,14 +5,14 @@ import os
5
5
  import warnings
6
6
  from functools import wraps
7
7
  from types import BuiltinFunctionType, BuiltinMethodType, FunctionType, MethodType
8
- from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
8
+ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type, Union
9
9
 
10
10
  import torch
11
11
  from torch.nn.modules.module import _addindent
12
12
 
13
13
  from .. import CONFIG, base_deprecation_message, deprecated, util
14
14
  from ..util import apply, Patch
15
- from . import serialization
15
+
16
16
  from .batching import Batchable
17
17
  from .inject import convert as inject
18
18
  from .tracing.base import Tracer, WithBlockNotFoundError
@@ -386,7 +386,7 @@ class Envoy(Batchable):
386
386
 
387
387
  #### Public methods ####
388
388
 
389
- def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, **kwargs):
389
+ def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, tracer_cls: Type[InterleavingTracer] = InterleavingTracer, **kwargs):
390
390
  """
391
391
  Create a tracer for this module.
392
392
 
@@ -418,7 +418,7 @@ class Envoy(Batchable):
418
418
  fn = self.__call__
419
419
  kwargs["hook"] = True
420
420
 
421
- return InterleavingTracer(fn, self, *args, **kwargs)
421
+ return tracer_cls(fn, self, *args, **kwargs)
422
422
 
423
423
  def scan(self, *args, **kwargs):
424
424
  """
@@ -518,6 +518,8 @@ class Envoy(Batchable):
518
518
  export_dir = os.path.expanduser(os.path.join(export_dir, name))
519
519
 
520
520
  os.makedirs(export_dir, exist_ok=True)
521
+
522
+ from . import serialization
521
523
 
522
524
  serialization.save(
523
525
  self._default_mediators, os.path.join(export_dir, f"{variant}.dill")
@@ -539,6 +541,8 @@ class Envoy(Batchable):
539
541
  export_dir = os.path.join(CONFIG.APP.CACHE_DIR, "exports")
540
542
 
541
543
  export_dir = os.path.expanduser(os.path.join(export_dir, name))
544
+
545
+ from . import serialization
542
546
 
543
547
  imported_mediators = serialization.load(
544
548
  os.path.join(export_dir, f"{variant}.dill"), self
@@ -547,8 +551,8 @@ class Envoy(Batchable):
547
551
  self._default_mediators.extend(imported_mediators)
548
552
 
549
553
  # TODO legacy
550
- def session(self, *args, **kwargs):
551
- return Tracer(*args, **kwargs)
554
+ def session(self, *args, tracer_cls: Type[Tracer] = Tracer, **kwargs):
555
+ return tracer_cls(*args, **kwargs)
552
556
 
553
557
  # TODO legacy
554
558
  @property
@@ -10,7 +10,7 @@ from enum import Enum
10
10
  from functools import wraps
11
11
  from queue import Queue
12
12
  from threading import Thread
13
- from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
13
+ from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
14
14
  Union)
15
15
 
16
16
  import torch
@@ -32,7 +32,8 @@ class Events(Enum):
32
32
  END = "end" # Signal to end the execution
33
33
  EXCEPTION = "exception" # Signal that an exception occurred
34
34
  SKIP = "skip" # Signal that an operation should be skipped
35
- CONTINUE = "continue" # Signal that an operation should continue
35
+ REGISTER = "register" # Signal that a child mediator should be registered
36
+ BARRIER = "barrier" # Signal that a barrier should be set
36
37
 
37
38
 
38
39
  class Cancelation(Exception):
@@ -49,14 +50,6 @@ class EarlyStopException(Exception):
49
50
  pass
50
51
 
51
52
 
52
- class UnboundIteratorException(Exception):
53
- """
54
- An .iterator was used with no ties to any module.
55
- """
56
-
57
- pass
58
-
59
-
60
53
  class SkipException(Exception):
61
54
  """
62
55
  Exception raised to skip the execution of the model.
@@ -425,10 +418,11 @@ class Mediator:
425
418
  self.batch_group = batch_group
426
419
  self.event_queue = Queue()
427
420
  self.response_queue = Queue()
421
+
422
+ self.child: Mediator = None
428
423
 
429
424
  self.thread = None
430
425
  self.interleaver = None
431
- self.child: Mediator = None
432
426
  self.history = set()
433
427
  self.user_cache: List["Cache"] = list()
434
428
  self.iteration = 0
@@ -496,7 +490,16 @@ class Mediator:
496
490
  Returns:
497
491
  The original or modified value
498
492
  """
499
-
493
+
494
+ if self.child is not None:
495
+ self.child.handle(provider)
496
+
497
+ if not self.child.alive:
498
+ self.child = None
499
+ self.respond()
500
+ else:
501
+ return
502
+
500
503
  process = not self.event_queue.empty()
501
504
 
502
505
  event = None
@@ -521,13 +524,12 @@ class Mediator:
521
524
  for cache in self.user_cache:
522
525
  cache.add(provider, e.value)
523
526
  raise e
527
+ elif event == Events.REGISTER:
528
+ process = self.handle_register_event(provider, data)
529
+ elif event == Events.BARRIER:
530
+ process = self.handle_barrier_event(provider, data)
524
531
  elif event == Events.END:
525
532
  process = False
526
- elif event == Events.CONTINUE:
527
- process = False
528
- # Putting it down here gives the child a chance to handle after the parent has already had one for this provider.
529
- if self.handle_child(provider):
530
- return
531
533
 
532
534
  if event == Events.END:
533
535
  self.handle_end_event()
@@ -543,28 +545,32 @@ class Mediator:
543
545
  ),
544
546
  )
545
547
 
546
- def handle_child(self, provider: Any) -> bool:
547
-
548
- if self.child is not None:
549
-
550
- self.child.handle(provider)
551
-
552
- # If the child was canceled
553
- if self.child.thread is None:
554
-
555
- # Tell the parent (current mediator) to continue
556
- self.response_queue.put(None)
548
+ def handle_register_event(self, provider: Any, child:Mediator) -> bool:
557
549
 
558
- # Then wipe the child
559
- self.child = None
550
+ self.child = child
551
+ child.start(self.interleaver)
552
+ child.handle(provider)
553
+
554
+ return False
560
555
 
561
- # Wait until the parent has an event
562
- self.wait()
556
+ def handle_barrier_event(self, provider: Any, participants: Set[str]):
557
+ """
558
+ Handle a barrier event by setting a barrier.
559
+ """
560
+
561
+ if participants is not None:
562
+
563
+ for mediator in self.interleaver.invokers:
563
564
 
564
- # Give the parent a chance to handle the provider afer the child is done.
565
- self.handle(provider)
565
+ while mediator.child is not None:
566
+ mediator = mediator.child
567
+
568
+ if mediator.name in participants:
569
+
570
+ mediator.handle(provider)
571
+
572
+ mediator.respond()
566
573
 
567
- return True
568
574
 
569
575
 
570
576
  def handle_end_event(self):
@@ -803,15 +809,8 @@ class Mediator:
803
809
 
804
810
  mediator.iteration = iter
805
811
  mediator.args = list([mediator.iteration])
806
- self.child = mediator
807
- mediator.start(self.interleaver)
808
812
 
809
- if not self.child.alive:
810
- raise UnboundIteratorException("`.iter` encountered with no dependence on any module.\n`.iter` only makes sense in the context of iteration over mu;ltiple module exeecutions." )
811
-
812
- self.event_queue.put((Events.CONTINUE, None))
813
-
814
- self.response_queue.get()
813
+ self.send(Events.REGISTER, mediator)
815
814
 
816
815
  if isinstance(iteration, slice):
817
816
 
@@ -842,8 +841,6 @@ class Mediator:
842
841
 
843
842
  do_iteration(iteration)
844
843
 
845
- self.child = None
846
-
847
844
  def stop(self):
848
845
  """Stop the execution of the model by raising an EarlyStopException."""
849
846
 
@@ -1,24 +1,20 @@
1
+ import io
1
2
  import pickle
3
+ from builtins import open
2
4
  from types import FrameType
3
- from typing import TYPE_CHECKING, Any, Optional
5
+ from typing import TYPE_CHECKING, Any, Optional, Union
4
6
 
5
7
  import dill
6
8
 
7
- if TYPE_CHECKING:
8
- from .envoy import Envoy
9
- else:
10
- Envoy = Any
11
-
9
+ from .envoy import Envoy
12
10
 
13
11
 
14
12
  class CustomDillPickler(dill.Pickler):
15
13
  def persistent_id(self, obj):
16
14
 
17
- from .envoy import Envoy
18
-
19
15
  if isinstance(obj, Envoy):
20
16
  return f"ENVOY:{obj.path}"
21
-
17
+
22
18
  if isinstance(obj, FrameType):
23
19
  return "FRAME"
24
20
 
@@ -43,13 +39,24 @@ class CustomDillUnpickler(dill.Unpickler):
43
39
  raise pickle.UnpicklingError(f"Unknown persistent id: {pid}")
44
40
 
45
41
 
46
- def save(obj: Any, path: str):
42
+ def save(obj: Any, path: Optional[str] = None):
43
+
44
+ dill.settings["recurse"] = True
45
+
46
+ if path is None:
47
+ file = io.BytesIO()
48
+ CustomDillPickler(file).dump(obj)
49
+ file.seek(0)
50
+ return file.read()
47
51
 
48
52
  with open(path, "wb") as file:
49
53
  CustomDillPickler(file).dump(obj)
50
54
 
51
55
 
52
- def load(path: str, model: Envoy, frame: Optional[FrameType] = None):
56
+ def load(data: Union[str, bytes], model: Envoy, frame: Optional[FrameType] = None):
57
+
58
+ if isinstance(data, bytes):
59
+ return CustomDillUnpickler(io.BytesIO(data), model, frame).load()
53
60
 
54
- with open(path, "rb") as file:
61
+ with open(data, "rb") as file:
55
62
  return CustomDillUnpickler(file, model, frame).load()
@@ -1,8 +1,8 @@
1
1
  import copy
2
2
  import re
3
3
  from dataclasses import dataclass
4
- from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
5
- Union)
4
+ from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
5
+ Tuple, Union)
6
6
 
7
7
  import torch
8
8
  from torch._subclasses.fake_tensor import FakeCopyMode, FakeTensorMode
@@ -11,7 +11,7 @@ from torch.fx.experimental.symbolic_shapes import ShapeEnv
11
11
  from ... import util
12
12
  from ..backends.base import Backend
13
13
  from ..batching import Batcher
14
- from ..interleaver import Interleaver
14
+ from ..interleaver import Events, Mediator
15
15
  from .base import ExitTracingException, Tracer
16
16
  from .globals import Object
17
17
  from .invoker import Invoker
@@ -19,7 +19,6 @@ from .iterator import IteratorProxy
19
19
 
20
20
  if TYPE_CHECKING:
21
21
  from ..envoy import Envoy
22
- from ..interleaver import Mediator
23
22
  else:
24
23
  Envoy = Any
25
24
 
@@ -408,6 +407,36 @@ class InterleavingTracer(Tracer):
408
407
  )
409
408
 
410
409
  return self.model._interleaver.current.user_cache[-1].cache
410
+
411
+ def barrier(self, n_participants: int):
412
+ """
413
+ nnsight barrier: A synchronization primitive for coordinating multiple concurrent invocations in nnsight.
414
+
415
+ This works similarly to a threading.Barrier, but is designed for use with nnsight's model tracing and intervention system.
416
+ A barrier allows you to pause execution in multiple parallel invocations until all participants have reached the barrier,
417
+ at which point all are released to continue. This is useful when you want to synchronize the execution of different
418
+ model runs, for example to ensure that all have reached a certain point (such as after embedding lookup) before
419
+ proceeding to the next stage (such as generation or intervention).
420
+
421
+ Example usage:
422
+
423
+ with gpt2.generate(max_new_tokens=3) as tracer:
424
+ barrier = tracer.barrier(2)
425
+
426
+ with tracer.invoke(MSG_prompt):
427
+ embeddings = gpt2.transformer.wte.output
428
+ barrier()
429
+ output1 = gpt2.generator.output.save()
430
+
431
+ with tracer.invoke("_ _ _ _ _ _ _ _ _"):
432
+ barrier()
433
+ gpt2.transformer.wte.output = embeddings
434
+ output2 = gpt2.generator.output.save()
435
+
436
+ In this example, both invocations will pause at the barrier until both have reached it, ensuring synchronization.
437
+ """
438
+
439
+ return Barrier(self.model, n_participants)
411
440
 
412
441
  ### Serialization ###
413
442
 
@@ -495,4 +524,25 @@ class ScanningTracer(InterleavingTracer):
495
524
 
496
525
  # Clean up hooks
497
526
  for hook in hooks:
498
- hook.remove()
527
+ hook.remove()
528
+
529
+
530
+
531
+ class Barrier:
532
+
533
+ def __init__(self, model: Envoy, n_participants: int):
534
+
535
+ self.model = model
536
+ self.n_participants = n_participants
537
+ self.participants: Set[str] = set()
538
+
539
+ def __call__(self):
540
+
541
+ mediator = self.model._interleaver.current
542
+
543
+ self.participants.add(mediator.name)
544
+
545
+ if len(self.participants) == self.n_participants:
546
+ mediator.send(Events.BARRIER, self.participants)
547
+ else:
548
+ mediator.send(Events.BARRIER, None)
@@ -63,6 +63,22 @@ class LanguageModel(RemoteableMixin):
63
63
  """
64
64
 
65
65
  tokenizer: PreTrainedTokenizer
66
+
67
+ class Generator(WrapperModule):
68
+
69
+ class Streamer(WrapperModule):
70
+
71
+ def put(self, *args):
72
+ return self(*args)
73
+
74
+ def end(self):
75
+ pass
76
+
77
+ def __init__(self) -> None:
78
+
79
+ super().__init__()
80
+
81
+ self.streamer = LanguageModel.Generator.Streamer()
66
82
 
67
83
  def __init__(
68
84
  self,
@@ -98,7 +114,7 @@ class LanguageModel(RemoteableMixin):
98
114
 
99
115
  self.import_edits()
100
116
 
101
- self.generator: Envoy = WrapperModule()
117
+ self.generator: Envoy = LanguageModel.Generator()
102
118
 
103
119
  # Some transformer models compile on first generation. As of 0.5.0.dev7 this not not work with nnsight if fullgraph is True
104
120
  def _patch_generation_config(self, model:torch.nn.Module):
@@ -163,12 +179,14 @@ class LanguageModel(RemoteableMixin):
163
179
  if max_new_tokens is not None and self._interleaver is not None:
164
180
  self._interleaver.default_all = max_new_tokens
165
181
 
166
- output = self._model.generate(*args, **kwargs)
182
+ streamer = kwargs.pop("streamer", self.generator.streamer._module)
183
+
184
+ output = self._model.generate(*args, streamer=streamer, **kwargs)
167
185
 
168
186
  if self._interleaver is not None:
169
187
  self._interleaver.default_all = None
170
188
 
171
- output = self.generator._module(output)
189
+ output = self.generator(output, hook=True)
172
190
 
173
191
  return output
174
192
 
@@ -1,9 +1,14 @@
1
- from typing import Any, Dict, Union
1
+ import ctypes
2
+ import inspect
3
+ from types import FrameType
4
+ from typing import Any, Callable, Dict, Union
2
5
 
3
6
  from typing_extensions import Self
4
7
 
5
8
  from ...intervention.backends import Backend
6
9
  from ...intervention.backends.remote import RemoteBackend
10
+ from ...intervention.serialization import load, save
11
+ from ...intervention.tracing.tracer import InterleavingTracer, Tracer
7
12
  from ...util import from_import_path, to_import_path
8
13
  from .meta import MetaMixin
9
14
 
@@ -31,9 +36,10 @@ class RemoteableMixin(MetaMixin):
31
36
  return super().trace(
32
37
  *inputs,
33
38
  backend=backend,
39
+ tracer_cls=RemoteInterleavingTracer,
34
40
  **kwargs,
35
41
  )
36
-
42
+
37
43
  def session(
38
44
  self,
39
45
  *inputs: Any,
@@ -55,6 +61,7 @@ class RemoteableMixin(MetaMixin):
55
61
  return super().session(
56
62
  *inputs,
57
63
  backend=backend,
64
+ tracer_cls=RemoteTracer,
58
65
  **kwargs,
59
66
  )
60
67
 
@@ -78,3 +85,72 @@ class RemoteableMixin(MetaMixin):
78
85
  type: RemoteableMixin = from_import_path(import_path)
79
86
 
80
87
  return type._remoteable_from_model_key(model_key, **kwargs)
88
+
89
+
90
+ class StreamTracer(Tracer):
91
+
92
+ _send: Callable = None
93
+ _recv: Callable = None
94
+
95
+ def __init__(self, frame: FrameType, *args, **kwargs):
96
+ super().__init__(*args, **kwargs)
97
+
98
+ self.frame = frame
99
+
100
+ @classmethod
101
+ def register(cls, send_fn: Callable, recv_fn: Callable):
102
+
103
+ cls._send = send_fn
104
+ cls._recv = recv_fn
105
+
106
+ @classmethod
107
+ def deregister(cls):
108
+
109
+ cls._send = None
110
+ cls._recv = None
111
+
112
+ def execute(self, fn: Callable):
113
+
114
+ data = save(fn)
115
+
116
+ if self._send is None:
117
+ raise ValueError("No send function provided")
118
+
119
+ if self._recv is None:
120
+ raise ValueError("No recv function provided")
121
+
122
+ self._send(data)
123
+
124
+ data: Dict[str, Any] = load(self._recv(), None)
125
+
126
+ if self.frame.f_code.co_filename.startswith("<nnsight"):
127
+ # For dynamically generated code, update both globals and locals
128
+ self.frame.f_globals.update(data)
129
+ self.frame.f_locals.update(data)
130
+
131
+ # Ensure locals are properly synchronized with the frame
132
+ ctypes.pythonapi.PyFrame_LocalsToFast(
133
+ ctypes.py_object(self.frame), ctypes.c_int(0)
134
+ )
135
+
136
+ else:
137
+ # For regular files, just update locals
138
+ for key, value in data.items():
139
+
140
+ self.frame.f_locals[key] = value
141
+
142
+ ctypes.pythonapi.PyFrame_LocalsToFast(
143
+ ctypes.py_object(self.frame), ctypes.c_int(0)
144
+ )
145
+
146
+ class RemoteTracer(Tracer):
147
+
148
+ def local(self):
149
+
150
+ frame = inspect.currentframe().f_back
151
+
152
+ return StreamTracer(frame)
153
+
154
+
155
+ class RemoteInterleavingTracer(InterleavingTracer, RemoteTracer):
156
+ pass
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union
6
6
 
7
7
  import dill
8
8
  from pydantic import BaseModel, ConfigDict
9
- from ..intervention.serialization import CustomDillPickler, CustomDillUnpickler
9
+ from ..intervention.serialization import save, load
10
10
  if TYPE_CHECKING:
11
11
  from .. import NNsight
12
12
  from ..intervention.tracing.tracer import Tracer
@@ -22,16 +22,8 @@ class RequestModel(BaseModel):
22
22
 
23
23
 
24
24
  def serialize(self, _zlib:bool) -> bytes:
25
-
26
- dill.settings['recurse'] = True
27
25
 
28
- with io.BytesIO() as data:
29
-
30
- CustomDillPickler(data).dump(self)
31
-
32
- data.seek(0)
33
-
34
- data = data.read()
26
+ data = save(self)
35
27
 
36
28
  if _zlib:
37
29
 
@@ -50,7 +42,7 @@ class RequestModel(BaseModel):
50
42
 
51
43
  data.seek(0)
52
44
 
53
- request:RequestModel = CustomDillUnpickler(data, model, None).load()
45
+ request:RequestModel = load(data.read(), model)
54
46
 
55
47
  return request
56
48
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nnsight
3
- Version: 0.5.0.dev11
3
+ Version: 0.5.0.dev12
4
4
  Summary: Package for interpreting and manipulating the internals of deep learning models.
5
5
  Author-email: Jaden Fiotto-Kaufman <jadenfk@outlook.com>
6
6
  License: MIT License
@@ -165,15 +165,22 @@ def test_adhoc_module(gpt2: nnsight.LanguageModel):
165
165
  def test_embeddings_set1(gpt2: nnsight.LanguageModel, MSG_prompt: str):
166
166
  with gpt2.generate(
167
167
  max_new_tokens=3
168
- ) as generator:
169
- with generator.invoke(MSG_prompt) as invoker:
168
+ ) as tracer:
169
+
170
+ barrier = tracer.barrier(2)
171
+
172
+ with tracer.invoke(MSG_prompt):
170
173
  embeddings = gpt2.transformer.wte.output
171
174
 
175
+ barrier()
176
+
172
177
 
173
178
  output1 = gpt2.generator.output.save()
174
179
 
175
- with generator.invoke("_ _ _ _ _ _ _ _ _") as invoker:
176
- gpt2.transformer.wte.wait_for_output()
180
+ with tracer.invoke("_ _ _ _ _ _ _ _ _"):
181
+
182
+ barrier()
183
+
177
184
  gpt2.transformer.wte.output = embeddings
178
185
 
179
186
  output2 = gpt2.generator.output.save()
@@ -832,16 +839,16 @@ def test_batched_iter(gpt2: nnsight.LanguageModel, MSG_prompt: str):
832
839
  assert gpt2.tokenizer.batch_decode(logits_1) == [" York", " City"]
833
840
  assert gpt2.tokenizer.batch_decode(logits_2) == [" New", " York", " City"]
834
841
 
835
- from nnsight.intervention.interleaver import UnboundIteratorException
836
- @torch.no_grad()
837
- @pytest.mark.iter
838
- def test_one_iter(gpt2: nnsight.LanguageModel):
839
- with pytest.raises(UnboundIteratorException):
840
- with gpt2.generate("_", max_new_tokens=1) as tracer:
841
- arr_gen = list().save()
842
+ # from nnsight.intervention.interleaver import UnboundIteratorException
843
+ # @torch.no_grad()
844
+ # @pytest.mark.iter
845
+ # def test_one_iter(gpt2: nnsight.LanguageModel):
846
+ # with pytest.raises(UnboundIteratorException):
847
+ # with gpt2.generate("_", max_new_tokens=1) as tracer:
848
+ # arr_gen = list().save()
842
849
 
843
- with tracer.all():
844
- arr_gen.append(1)
850
+ # with tracer.all():
851
+ # arr_gen.append(1)
845
852
 
846
853
 
847
854
 
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes