nnsight 0.5.0.dev10__tar.gz → 0.5.0.dev12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/PKG-INFO +1 -1
  2. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/remote.py +85 -20
  3. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/envoy.py +14 -7
  4. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/interleaver.py +55 -45
  5. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/serialization.py +19 -12
  6. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/tracer.py +55 -5
  7. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/language.py +21 -3
  8. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/remoteable.py +78 -2
  9. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/request.py +3 -11
  10. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/PKG-INFO +1 -1
  11. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_lm.py +20 -13
  12. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/.github/workflows/publish.yml +0 -0
  13. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/.github/workflows/python-app.yml +0 -0
  14. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/.gitignore +0 -0
  15. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/CHANGELOG.md +0 -0
  16. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/CODE_OF_CONDUCT.md +0 -0
  17. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/LICENSE +0 -0
  18. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/NNsight_Walkthrough.ipynb +0 -0
  19. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/NNsight_v0_3_guide.ipynb +0 -0
  20. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/NNsight_v0_4_guide.ipynb +0 -0
  21. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/README.md +0 -0
  22. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/conftest.py +0 -0
  23. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/pyproject.toml +0 -0
  24. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/pytest.ini +0 -0
  25. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/setup.cfg +0 -0
  26. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/setup.py +0 -0
  27. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/__init__.py +0 -0
  28. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/_c/__init__.py +0 -0
  29. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/_c/py_mount.c +0 -0
  30. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/config.yaml +0 -0
  31. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/__init__.py +0 -0
  32. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/__init__.py +0 -0
  33. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/base.py +0 -0
  34. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/editing.py +0 -0
  35. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/backends/execution.py +0 -0
  36. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/batching.py +0 -0
  37. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/inject.py +0 -0
  38. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/__init__.py +0 -0
  39. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/backwards.py +0 -0
  40. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/base.py +0 -0
  41. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/editing.py +0 -0
  42. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/globals.py +0 -0
  43. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/invoker.py +0 -0
  44. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/iterator.py +0 -0
  45. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/intervention/tracing/util.py +0 -0
  46. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/log.py +0 -0
  47. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/__init__.py +0 -0
  48. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/base.py +0 -0
  49. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/diffusion.py +0 -0
  50. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/__init__.py +0 -0
  51. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/loadable.py +0 -0
  52. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/mixins/meta.py +0 -0
  53. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/__init__.py +0 -0
  54. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner copy.py +0 -0
  55. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
  56. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
  57. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/sampling copy.py +0 -0
  58. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/sampling.py +0 -0
  59. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/vllm.py +0 -0
  60. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
  61. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
  62. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py +0 -0
  63. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/__init__.py +0 -0
  64. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/config.py +0 -0
  65. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/schema/response.py +0 -0
  66. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight/util.py +0 -0
  67. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/SOURCES.txt +0 -0
  68. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/dependency_links.txt +0 -0
  69. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/requires.txt +0 -0
  70. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/src/nnsight.egg-info/top_level.txt +0 -0
  71. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/__init__.py +0 -0
  72. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_diffusion.py +0 -0
  73. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_tiny.py +0 -0
  74. {nnsight-0.5.0.dev10 → nnsight-0.5.0.dev12}/tests/test_vllm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nnsight
3
- Version: 0.5.0.dev10
3
+ Version: 0.5.0.dev12
4
4
  Summary: Package for interpreting and manipulating the internals of deep learning models.
5
5
  Author-email: Jaden Fiotto-Kaufman <jadenfk@outlook.com>
6
6
  License: MIT License
@@ -1,9 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import inspect
3
4
  import io
4
5
  import sys
5
6
  import time
6
- from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
7
+ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
7
8
 
8
9
  import requests
9
10
  import socketio
@@ -11,16 +12,14 @@ import torch
11
12
  from tqdm.auto import tqdm
12
13
 
13
14
  from ... import __IPYTHON__, CONFIG, __version__
15
+ from ..._c.py_mount import mount, unmount
16
+ from ...intervention.serialization import load, save
14
17
  from ...log import remote_logger
15
18
  from ...schema.request import RequestModel
16
19
  from ...schema.response import RESULT, ResponseModel
20
+ from ..tracing.tracer import Tracer
17
21
  from .base import Backend
18
22
 
19
- if TYPE_CHECKING:
20
- from ..tracing.tracer import Tracer
21
- else:
22
- Tracer = Any
23
-
24
23
 
25
24
  class RemoteException(Exception):
26
25
  pass
@@ -129,8 +128,20 @@ class RemoteBackend(Backend):
129
128
  else:
130
129
 
131
130
  result = response.data
132
-
131
+
133
132
  return result
133
+
134
+ elif response.status == ResponseModel.JobStatus.STREAM:
135
+
136
+ model = getattr(tracer, "model", None)
137
+
138
+ fn = load(response.data, model)
139
+
140
+ local_tracer = LocalTracer(_info=tracer.info)
141
+
142
+ local_tracer.execute(fn)
143
+
144
+
134
145
 
135
146
 
136
147
  def submit_request(
@@ -262,6 +273,7 @@ class RemoteBackend(Backend):
262
273
  response = self.submit_request(data, headers)
263
274
 
264
275
  try:
276
+ LocalTracer.register(lambda data: self.stream_send(data, sio))
265
277
  # Loop until
266
278
  while True:
267
279
 
@@ -278,22 +290,28 @@ class RemoteBackend(Backend):
278
290
  except Exception as e:
279
291
 
280
292
  raise e
293
+
294
+ finally:
295
+ LocalTracer.deregister()
296
+
281
297
 
282
- # def stream_send(
283
- # self, values: Dict[int, Any], job_id: str, sio: socketio.SimpleClient
284
- # ):
285
- # """Upload some value to the remote service for some job id.
298
+ def stream_send(
299
+ self, values: Dict[int, Any], sio: socketio.SimpleClient
300
+ ):
301
+ """Upload some value to the remote service for some job id.
286
302
 
287
- # Args:
288
- # value (Any): Value to upload
289
- # job_id (str): Job id.
290
- # sio (socketio.SimpleClient): Connected websocket client.
291
- # """
303
+ Args:
304
+ value (Any): Value to upload
305
+ job_id (str): Job id.
306
+ sio (socketio.SimpleClient): Connected websocket client.
307
+ """
308
+
309
+ data = save(values)
292
310
 
293
- # sio.emit(
294
- # "stream_upload",
295
- # data=(StreamValueModel.serialize(values, self.format, self.zlib), job_id),
296
- # )
311
+ sio.emit(
312
+ "stream_upload",
313
+ data=(data, self.job_id),
314
+ )
297
315
 
298
316
  def non_blocking_request(self, tracer: Tracer):
299
317
  """Send intervention request to the remote service if request provided. Otherwise get job status.
@@ -338,3 +356,50 @@ class RemoteBackend(Backend):
338
356
  CONFIG.save()
339
357
 
340
358
  raise e
359
+
360
+
361
+ class LocalTracer(Tracer):
362
+
363
+ _send: Callable = None
364
+
365
+ def __init__(self, *args, **kwargs):
366
+ super().__init__(*args, **kwargs)
367
+
368
+ self.remotes = set()
369
+
370
+ @classmethod
371
+ def register(cls, send_fn: Callable):
372
+
373
+ cls._send = send_fn
374
+
375
+ @classmethod
376
+ def deregister(cls):
377
+
378
+ cls._send = None
379
+
380
+ def _save_remote(self, obj: Any):
381
+
382
+ self.remotes.add(id(obj))
383
+
384
+ def execute(self, fn: Callable):
385
+
386
+ mount(self._save_remote, "remote")
387
+
388
+ fn(self, self.info)
389
+
390
+ unmount("remote")
391
+
392
+ return
393
+
394
+ def push(self):
395
+
396
+ # Find the frame where the traced code is executing
397
+ state_frame = inspect.currentframe().f_back
398
+
399
+ state = state_frame.f_locals
400
+
401
+ super().push(state)
402
+
403
+ state = {k:v for k,v in state.items() if id(v) in self.remotes}
404
+
405
+ LocalTracer._send(state)
@@ -5,14 +5,14 @@ import os
5
5
  import warnings
6
6
  from functools import wraps
7
7
  from types import BuiltinFunctionType, BuiltinMethodType, FunctionType, MethodType
8
- from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
8
+ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Type, Union
9
9
 
10
10
  import torch
11
11
  from torch.nn.modules.module import _addindent
12
12
 
13
13
  from .. import CONFIG, base_deprecation_message, deprecated, util
14
14
  from ..util import apply, Patch
15
- from . import serialization
15
+
16
16
  from .batching import Batchable
17
17
  from .inject import convert as inject
18
18
  from .tracing.base import Tracer, WithBlockNotFoundError
@@ -386,7 +386,7 @@ class Envoy(Batchable):
386
386
 
387
387
  #### Public methods ####
388
388
 
389
- def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, **kwargs):
389
+ def trace(self, *args, fn: Optional[Callable] = None, trace: bool = None, tracer_cls: Type[InterleavingTracer] = InterleavingTracer, **kwargs):
390
390
  """
391
391
  Create a tracer for this module.
392
392
 
@@ -418,7 +418,7 @@ class Envoy(Batchable):
418
418
  fn = self.__call__
419
419
  kwargs["hook"] = True
420
420
 
421
- return InterleavingTracer(fn, self, *args, **kwargs)
421
+ return tracer_cls(fn, self, *args, **kwargs)
422
422
 
423
423
  def scan(self, *args, **kwargs):
424
424
  """
@@ -518,6 +518,8 @@ class Envoy(Batchable):
518
518
  export_dir = os.path.expanduser(os.path.join(export_dir, name))
519
519
 
520
520
  os.makedirs(export_dir, exist_ok=True)
521
+
522
+ from . import serialization
521
523
 
522
524
  serialization.save(
523
525
  self._default_mediators, os.path.join(export_dir, f"{variant}.dill")
@@ -539,6 +541,8 @@ class Envoy(Batchable):
539
541
  export_dir = os.path.join(CONFIG.APP.CACHE_DIR, "exports")
540
542
 
541
543
  export_dir = os.path.expanduser(os.path.join(export_dir, name))
544
+
545
+ from . import serialization
542
546
 
543
547
  imported_mediators = serialization.load(
544
548
  os.path.join(export_dir, f"{variant}.dill"), self
@@ -547,8 +551,8 @@ class Envoy(Batchable):
547
551
  self._default_mediators.extend(imported_mediators)
548
552
 
549
553
  # TODO legacy
550
- def session(self, *args, **kwargs):
551
- return Tracer(*args, **kwargs)
554
+ def session(self, *args, tracer_cls: Type[Tracer] = Tracer, **kwargs):
555
+ return tracer_cls(*args, **kwargs)
552
556
 
553
557
  # TODO legacy
554
558
  @property
@@ -720,8 +724,11 @@ class Envoy(Batchable):
720
724
  try:
721
725
  with self._interleaver:
722
726
  fn(*args, **kwargs)
723
- finally:
727
+
724
728
  self._interleaver.check_cache_full()
729
+ self._interleaver.check_dangling_mediators()
730
+
731
+ finally:
725
732
  self._interleaver.cancel()
726
733
 
727
734
  #### Private methods ####
@@ -10,7 +10,7 @@ from enum import Enum
10
10
  from functools import wraps
11
11
  from queue import Queue
12
12
  from threading import Thread
13
- from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
13
+ from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
14
14
  Union)
15
15
 
16
16
  import torch
@@ -32,7 +32,8 @@ class Events(Enum):
32
32
  END = "end" # Signal to end the execution
33
33
  EXCEPTION = "exception" # Signal that an exception occurred
34
34
  SKIP = "skip" # Signal that an operation should be skipped
35
- CONTINUE = "continue" # Signal that an operation should continue
35
+ REGISTER = "register" # Signal that a child mediator should be registered
36
+ BARRIER = "barrier" # Signal that a barrier should be set
36
37
 
37
38
 
38
39
  class Cancelation(Exception):
@@ -97,20 +98,15 @@ class Interleaver:
97
98
 
98
99
  def cancel(self):
99
100
  """Cancel all intervention threads."""
100
-
101
- try:
102
- self.check_cache_full()
103
- self.check_dangling_mediators()
104
- finally:
105
-
106
- for mediator in list(self.mediators.values()):
107
- mediator.cancel()
101
+
102
+ for mediator in list(self.mediators.values()):
103
+ mediator.cancel()
108
104
 
109
- self.mediators = None
110
- self.tracer = None
111
- self.batcher = None
112
- self.user_cache = None
113
- self.invokers = None
105
+ self.mediators = None
106
+ self.tracer = None
107
+ self.batcher = None
108
+ self.user_cache = None
109
+ self.invokers = None
114
110
 
115
111
 
116
112
  def iterate(self, provider: str):
@@ -422,10 +418,11 @@ class Mediator:
422
418
  self.batch_group = batch_group
423
419
  self.event_queue = Queue()
424
420
  self.response_queue = Queue()
421
+
422
+ self.child: Mediator = None
425
423
 
426
424
  self.thread = None
427
425
  self.interleaver = None
428
- self.child: Mediator = None
429
426
  self.history = set()
430
427
  self.user_cache: List["Cache"] = list()
431
428
  self.iteration = 0
@@ -493,33 +490,22 @@ class Mediator:
493
490
  Returns:
494
491
  The original or modified value
495
492
  """
496
-
493
+
497
494
  if self.child is not None:
498
-
499
495
  self.child.handle(provider)
500
-
501
- # If the child was canceled
502
- if self.child.thread is None:
503
-
504
- # Tell the parent (current mediator) to continue
505
- self.response_queue.put(None)
506
-
507
- # Then wipe the child
496
+
497
+ if not self.child.alive:
508
498
  self.child = None
509
-
510
- # Wait until the parent has an event
511
- self.wait()
512
-
513
- # Continue to handle parent event
499
+ self.respond()
514
500
  else:
515
501
  return
516
-
502
+
517
503
  process = not self.event_queue.empty()
518
504
 
519
505
  event = None
520
-
506
+
521
507
  while process:
522
-
508
+
523
509
  value = self.interleaver.batcher.current_value
524
510
 
525
511
  event, data = self.event_queue.get()
@@ -538,10 +524,12 @@ class Mediator:
538
524
  for cache in self.user_cache:
539
525
  cache.add(provider, e.value)
540
526
  raise e
527
+ elif event == Events.REGISTER:
528
+ process = self.handle_register_event(provider, data)
529
+ elif event == Events.BARRIER:
530
+ process = self.handle_barrier_event(provider, data)
541
531
  elif event == Events.END:
542
532
  process = False
543
- elif event == Events.CONTINUE:
544
- process = False
545
533
 
546
534
  if event == Events.END:
547
535
  self.handle_end_event()
@@ -556,6 +544,34 @@ class Mediator:
556
544
  self.batch_group, self.interleaver.batcher.current_value
557
545
  ),
558
546
  )
547
+
548
+ def handle_register_event(self, provider: Any, child:Mediator) -> bool:
549
+
550
+ self.child = child
551
+ child.start(self.interleaver)
552
+ child.handle(provider)
553
+
554
+ return False
555
+
556
+ def handle_barrier_event(self, provider: Any, participants: Set[str]):
557
+ """
558
+ Handle a barrier event by setting a barrier.
559
+ """
560
+
561
+ if participants is not None:
562
+
563
+ for mediator in self.interleaver.invokers:
564
+
565
+ while mediator.child is not None:
566
+ mediator = mediator.child
567
+
568
+ if mediator.name in participants:
569
+
570
+ mediator.handle(provider)
571
+
572
+ mediator.respond()
573
+
574
+
559
575
 
560
576
  def handle_end_event(self):
561
577
  """
@@ -793,12 +809,8 @@ class Mediator:
793
809
 
794
810
  mediator.iteration = iter
795
811
  mediator.args = list([mediator.iteration])
796
- self.child = mediator
797
- mediator.start(self.interleaver)
798
-
799
- self.event_queue.put((Events.CONTINUE, None))
800
-
801
- self.response_queue.get()
812
+
813
+ self.send(Events.REGISTER, mediator)
802
814
 
803
815
  if isinstance(iteration, slice):
804
816
 
@@ -809,7 +821,7 @@ class Mediator:
809
821
  while True:
810
822
 
811
823
  do_iteration(i)
812
-
824
+
813
825
  if self.interleaver.default_all is not None and stop is None:
814
826
  stop = self.interleaver.default_all
815
827
 
@@ -829,8 +841,6 @@ class Mediator:
829
841
 
830
842
  do_iteration(iteration)
831
843
 
832
- self.child = None
833
-
834
844
  def stop(self):
835
845
  """Stop the execution of the model by raising an EarlyStopException."""
836
846
 
@@ -1,24 +1,20 @@
1
+ import io
1
2
  import pickle
3
+ from builtins import open
2
4
  from types import FrameType
3
- from typing import TYPE_CHECKING, Any, Optional
5
+ from typing import TYPE_CHECKING, Any, Optional, Union
4
6
 
5
7
  import dill
6
8
 
7
- if TYPE_CHECKING:
8
- from .envoy import Envoy
9
- else:
10
- Envoy = Any
11
-
9
+ from .envoy import Envoy
12
10
 
13
11
 
14
12
  class CustomDillPickler(dill.Pickler):
15
13
  def persistent_id(self, obj):
16
14
 
17
- from .envoy import Envoy
18
-
19
15
  if isinstance(obj, Envoy):
20
16
  return f"ENVOY:{obj.path}"
21
-
17
+
22
18
  if isinstance(obj, FrameType):
23
19
  return "FRAME"
24
20
 
@@ -43,13 +39,24 @@ class CustomDillUnpickler(dill.Unpickler):
43
39
  raise pickle.UnpicklingError(f"Unknown persistent id: {pid}")
44
40
 
45
41
 
46
- def save(obj: Any, path: str):
42
+ def save(obj: Any, path: Optional[str] = None):
43
+
44
+ dill.settings["recurse"] = True
45
+
46
+ if path is None:
47
+ file = io.BytesIO()
48
+ CustomDillPickler(file).dump(obj)
49
+ file.seek(0)
50
+ return file.read()
47
51
 
48
52
  with open(path, "wb") as file:
49
53
  CustomDillPickler(file).dump(obj)
50
54
 
51
55
 
52
- def load(path: str, model: Envoy, frame: Optional[FrameType] = None):
56
+ def load(data: Union[str, bytes], model: Envoy, frame: Optional[FrameType] = None):
57
+
58
+ if isinstance(data, bytes):
59
+ return CustomDillUnpickler(io.BytesIO(data), model, frame).load()
53
60
 
54
- with open(path, "rb") as file:
61
+ with open(data, "rb") as file:
55
62
  return CustomDillUnpickler(file, model, frame).load()
@@ -1,8 +1,8 @@
1
1
  import copy
2
2
  import re
3
3
  from dataclasses import dataclass
4
- from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
5
- Union)
4
+ from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
5
+ Tuple, Union)
6
6
 
7
7
  import torch
8
8
  from torch._subclasses.fake_tensor import FakeCopyMode, FakeTensorMode
@@ -11,7 +11,7 @@ from torch.fx.experimental.symbolic_shapes import ShapeEnv
11
11
  from ... import util
12
12
  from ..backends.base import Backend
13
13
  from ..batching import Batcher
14
- from ..interleaver import Interleaver
14
+ from ..interleaver import Events, Mediator
15
15
  from .base import ExitTracingException, Tracer
16
16
  from .globals import Object
17
17
  from .invoker import Invoker
@@ -19,7 +19,6 @@ from .iterator import IteratorProxy
19
19
 
20
20
  if TYPE_CHECKING:
21
21
  from ..envoy import Envoy
22
- from ..interleaver import Mediator
23
22
  else:
24
23
  Envoy = Any
25
24
 
@@ -408,6 +407,36 @@ class InterleavingTracer(Tracer):
408
407
  )
409
408
 
410
409
  return self.model._interleaver.current.user_cache[-1].cache
410
+
411
+ def barrier(self, n_participants: int):
412
+ """
413
+ nnsight barrier: A synchronization primitive for coordinating multiple concurrent invocations in nnsight.
414
+
415
+ This works similarly to a threading.Barrier, but is designed for use with nnsight's model tracing and intervention system.
416
+ A barrier allows you to pause execution in multiple parallel invocations until all participants have reached the barrier,
417
+ at which point all are released to continue. This is useful when you want to synchronize the execution of different
418
+ model runs, for example to ensure that all have reached a certain point (such as after embedding lookup) before
419
+ proceeding to the next stage (such as generation or intervention).
420
+
421
+ Example usage:
422
+
423
+ with gpt2.generate(max_new_tokens=3) as tracer:
424
+ barrier = tracer.barrier(2)
425
+
426
+ with tracer.invoke(MSG_prompt):
427
+ embeddings = gpt2.transformer.wte.output
428
+ barrier()
429
+ output1 = gpt2.generator.output.save()
430
+
431
+ with tracer.invoke("_ _ _ _ _ _ _ _ _"):
432
+ barrier()
433
+ gpt2.transformer.wte.output = embeddings
434
+ output2 = gpt2.generator.output.save()
435
+
436
+ In this example, both invocations will pause at the barrier until both have reached it, ensuring synchronization.
437
+ """
438
+
439
+ return Barrier(self.model, n_participants)
411
440
 
412
441
  ### Serialization ###
413
442
 
@@ -495,4 +524,25 @@ class ScanningTracer(InterleavingTracer):
495
524
 
496
525
  # Clean up hooks
497
526
  for hook in hooks:
498
- hook.remove()
527
+ hook.remove()
528
+
529
+
530
+
531
+ class Barrier:
532
+
533
+ def __init__(self, model: Envoy, n_participants: int):
534
+
535
+ self.model = model
536
+ self.n_participants = n_participants
537
+ self.participants: Set[str] = set()
538
+
539
+ def __call__(self):
540
+
541
+ mediator = self.model._interleaver.current
542
+
543
+ self.participants.add(mediator.name)
544
+
545
+ if len(self.participants) == self.n_participants:
546
+ mediator.send(Events.BARRIER, self.participants)
547
+ else:
548
+ mediator.send(Events.BARRIER, None)
@@ -63,6 +63,22 @@ class LanguageModel(RemoteableMixin):
63
63
  """
64
64
 
65
65
  tokenizer: PreTrainedTokenizer
66
+
67
+ class Generator(WrapperModule):
68
+
69
+ class Streamer(WrapperModule):
70
+
71
+ def put(self, *args):
72
+ return self(*args)
73
+
74
+ def end(self):
75
+ pass
76
+
77
+ def __init__(self) -> None:
78
+
79
+ super().__init__()
80
+
81
+ self.streamer = LanguageModel.Generator.Streamer()
66
82
 
67
83
  def __init__(
68
84
  self,
@@ -98,7 +114,7 @@ class LanguageModel(RemoteableMixin):
98
114
 
99
115
  self.import_edits()
100
116
 
101
- self.generator: Envoy = WrapperModule()
117
+ self.generator: Envoy = LanguageModel.Generator()
102
118
 
103
119
  # Some transformer models compile on first generation. As of 0.5.0.dev7 this not not work with nnsight if fullgraph is True
104
120
  def _patch_generation_config(self, model:torch.nn.Module):
@@ -163,12 +179,14 @@ class LanguageModel(RemoteableMixin):
163
179
  if max_new_tokens is not None and self._interleaver is not None:
164
180
  self._interleaver.default_all = max_new_tokens
165
181
 
166
- output = self._model.generate(*args, **kwargs)
182
+ streamer = kwargs.pop("streamer", self.generator.streamer._module)
183
+
184
+ output = self._model.generate(*args, streamer=streamer, **kwargs)
167
185
 
168
186
  if self._interleaver is not None:
169
187
  self._interleaver.default_all = None
170
188
 
171
- output = self.generator._module(output)
189
+ output = self.generator(output, hook=True)
172
190
 
173
191
  return output
174
192
 
@@ -1,9 +1,14 @@
1
- from typing import Any, Dict, Union
1
+ import ctypes
2
+ import inspect
3
+ from types import FrameType
4
+ from typing import Any, Callable, Dict, Union
2
5
 
3
6
  from typing_extensions import Self
4
7
 
5
8
  from ...intervention.backends import Backend
6
9
  from ...intervention.backends.remote import RemoteBackend
10
+ from ...intervention.serialization import load, save
11
+ from ...intervention.tracing.tracer import InterleavingTracer, Tracer
7
12
  from ...util import from_import_path, to_import_path
8
13
  from .meta import MetaMixin
9
14
 
@@ -31,9 +36,10 @@ class RemoteableMixin(MetaMixin):
31
36
  return super().trace(
32
37
  *inputs,
33
38
  backend=backend,
39
+ tracer_cls=RemoteInterleavingTracer,
34
40
  **kwargs,
35
41
  )
36
-
42
+
37
43
  def session(
38
44
  self,
39
45
  *inputs: Any,
@@ -55,6 +61,7 @@ class RemoteableMixin(MetaMixin):
55
61
  return super().session(
56
62
  *inputs,
57
63
  backend=backend,
64
+ tracer_cls=RemoteTracer,
58
65
  **kwargs,
59
66
  )
60
67
 
@@ -78,3 +85,72 @@ class RemoteableMixin(MetaMixin):
78
85
  type: RemoteableMixin = from_import_path(import_path)
79
86
 
80
87
  return type._remoteable_from_model_key(model_key, **kwargs)
88
+
89
+
90
+ class StreamTracer(Tracer):
91
+
92
+ _send: Callable = None
93
+ _recv: Callable = None
94
+
95
+ def __init__(self, frame: FrameType, *args, **kwargs):
96
+ super().__init__(*args, **kwargs)
97
+
98
+ self.frame = frame
99
+
100
+ @classmethod
101
+ def register(cls, send_fn: Callable, recv_fn: Callable):
102
+
103
+ cls._send = send_fn
104
+ cls._recv = recv_fn
105
+
106
+ @classmethod
107
+ def deregister(cls):
108
+
109
+ cls._send = None
110
+ cls._recv = None
111
+
112
+ def execute(self, fn: Callable):
113
+
114
+ data = save(fn)
115
+
116
+ if self._send is None:
117
+ raise ValueError("No send function provided")
118
+
119
+ if self._recv is None:
120
+ raise ValueError("No recv function provided")
121
+
122
+ self._send(data)
123
+
124
+ data: Dict[str, Any] = load(self._recv(), None)
125
+
126
+ if self.frame.f_code.co_filename.startswith("<nnsight"):
127
+ # For dynamically generated code, update both globals and locals
128
+ self.frame.f_globals.update(data)
129
+ self.frame.f_locals.update(data)
130
+
131
+ # Ensure locals are properly synchronized with the frame
132
+ ctypes.pythonapi.PyFrame_LocalsToFast(
133
+ ctypes.py_object(self.frame), ctypes.c_int(0)
134
+ )
135
+
136
+ else:
137
+ # For regular files, just update locals
138
+ for key, value in data.items():
139
+
140
+ self.frame.f_locals[key] = value
141
+
142
+ ctypes.pythonapi.PyFrame_LocalsToFast(
143
+ ctypes.py_object(self.frame), ctypes.c_int(0)
144
+ )
145
+
146
+ class RemoteTracer(Tracer):
147
+
148
+ def local(self):
149
+
150
+ frame = inspect.currentframe().f_back
151
+
152
+ return StreamTracer(frame)
153
+
154
+
155
+ class RemoteInterleavingTracer(InterleavingTracer, RemoteTracer):
156
+ pass
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Union
6
6
 
7
7
  import dill
8
8
  from pydantic import BaseModel, ConfigDict
9
- from ..intervention.serialization import CustomDillPickler, CustomDillUnpickler
9
+ from ..intervention.serialization import save, load
10
10
  if TYPE_CHECKING:
11
11
  from .. import NNsight
12
12
  from ..intervention.tracing.tracer import Tracer
@@ -22,16 +22,8 @@ class RequestModel(BaseModel):
22
22
 
23
23
 
24
24
  def serialize(self, _zlib:bool) -> bytes:
25
-
26
- dill.settings['recurse'] = True
27
25
 
28
- with io.BytesIO() as data:
29
-
30
- CustomDillPickler(data).dump(self)
31
-
32
- data.seek(0)
33
-
34
- data = data.read()
26
+ data = save(self)
35
27
 
36
28
  if _zlib:
37
29
 
@@ -50,7 +42,7 @@ class RequestModel(BaseModel):
50
42
 
51
43
  data.seek(0)
52
44
 
53
- request:RequestModel = CustomDillUnpickler(data, model, None).load()
45
+ request:RequestModel = load(data.read(), model)
54
46
 
55
47
  return request
56
48
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nnsight
3
- Version: 0.5.0.dev10
3
+ Version: 0.5.0.dev12
4
4
  Summary: Package for interpreting and manipulating the internals of deep learning models.
5
5
  Author-email: Jaden Fiotto-Kaufman <jadenfk@outlook.com>
6
6
  License: MIT License
@@ -165,15 +165,22 @@ def test_adhoc_module(gpt2: nnsight.LanguageModel):
165
165
  def test_embeddings_set1(gpt2: nnsight.LanguageModel, MSG_prompt: str):
166
166
  with gpt2.generate(
167
167
  max_new_tokens=3
168
- ) as generator:
169
- with generator.invoke(MSG_prompt) as invoker:
168
+ ) as tracer:
169
+
170
+ barrier = tracer.barrier(2)
171
+
172
+ with tracer.invoke(MSG_prompt):
170
173
  embeddings = gpt2.transformer.wte.output
171
174
 
175
+ barrier()
176
+
172
177
 
173
178
  output1 = gpt2.generator.output.save()
174
179
 
175
- with generator.invoke("_ _ _ _ _ _ _ _ _") as invoker:
176
- gpt2.transformer.wte.wait_for_output()
180
+ with tracer.invoke("_ _ _ _ _ _ _ _ _"):
181
+
182
+ barrier()
183
+
177
184
  gpt2.transformer.wte.output = embeddings
178
185
 
179
186
  output2 = gpt2.generator.output.save()
@@ -832,17 +839,17 @@ def test_batched_iter(gpt2: nnsight.LanguageModel, MSG_prompt: str):
832
839
  assert gpt2.tokenizer.batch_decode(logits_1) == [" York", " City"]
833
840
  assert gpt2.tokenizer.batch_decode(logits_2) == [" New", " York", " City"]
834
841
 
842
+ # from nnsight.intervention.interleaver import UnboundIteratorException
843
+ # @torch.no_grad()
844
+ # @pytest.mark.iter
845
+ # def test_one_iter(gpt2: nnsight.LanguageModel):
846
+ # with pytest.raises(UnboundIteratorException):
847
+ # with gpt2.generate("_", max_new_tokens=1) as tracer:
848
+ # arr_gen = list().save()
835
849
 
836
- @torch.no_grad()
837
- @pytest.mark.iter
838
- def test_one_iter(gpt2: nnsight.LanguageModel):
839
- with gpt2.generate("_", max_new_tokens=1) as tracer:
840
- arr_gen = list().save()
841
-
842
- with tracer.all():
843
- arr_gen.append(1)
850
+ # with tracer.all():
851
+ # arr_gen.append(1)
844
852
 
845
- assert len(arr_gen) == 1
846
853
 
847
854
 
848
855
  @torch.no_grad()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes