nnsight 0.5.0.dev12__tar.gz → 0.5.0.dev14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/PKG-INFO +1 -1
  2. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/batching.py +3 -1
  3. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/envoy.py +12 -5
  4. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/interleaver.py +25 -28
  5. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/backwards.py +2 -4
  6. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/base.py +26 -43
  7. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/invoker.py +6 -2
  8. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/iterator.py +4 -1
  9. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/tracer.py +31 -11
  10. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/util.py +57 -1
  11. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/language.py +29 -34
  12. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight.egg-info/PKG-INFO +1 -1
  13. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/tests/test_lm.py +17 -0
  14. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/.github/workflows/publish.yml +0 -0
  15. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/.github/workflows/python-app.yml +0 -0
  16. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/.gitignore +0 -0
  17. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/CHANGELOG.md +0 -0
  18. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/CODE_OF_CONDUCT.md +0 -0
  19. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/LICENSE +0 -0
  20. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/NNsight_Walkthrough.ipynb +0 -0
  21. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/NNsight_v0_3_guide.ipynb +0 -0
  22. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/NNsight_v0_4_guide.ipynb +0 -0
  23. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/README.md +0 -0
  24. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/conftest.py +0 -0
  25. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/pyproject.toml +0 -0
  26. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/pytest.ini +0 -0
  27. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/setup.cfg +0 -0
  28. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/setup.py +0 -0
  29. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/__init__.py +0 -0
  30. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/_c/__init__.py +0 -0
  31. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/_c/py_mount.c +0 -0
  32. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/config.yaml +0 -0
  33. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/__init__.py +0 -0
  34. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/backends/__init__.py +0 -0
  35. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/backends/base.py +0 -0
  36. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/backends/editing.py +0 -0
  37. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/backends/execution.py +0 -0
  38. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/backends/remote.py +0 -0
  39. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/inject.py +0 -0
  40. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/serialization.py +0 -0
  41. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/__init__.py +0 -0
  42. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/editing.py +0 -0
  43. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/intervention/tracing/globals.py +0 -0
  44. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/log.py +0 -0
  45. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/__init__.py +0 -0
  46. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/base.py +0 -0
  47. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/diffusion.py +0 -0
  48. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/mixins/__init__.py +0 -0
  49. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/mixins/loadable.py +0 -0
  50. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/mixins/meta.py +0 -0
  51. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/mixins/remoteable.py +0 -0
  52. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/__init__.py +0 -0
  53. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner copy.py +0 -0
  54. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/model_runners/GPUModelRunner.py +0 -0
  55. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/model_runners/__init__.py +0 -0
  56. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/sampling copy.py +0 -0
  57. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/sampling.py +0 -0
  58. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/vllm.py +0 -0
  59. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/workers/GPUWorker.py +0 -0
  60. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/workers/__init__.py +0 -0
  61. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/modeling/vllm/workers/gpu_input_batch.py +0 -0
  62. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/schema/__init__.py +0 -0
  63. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/schema/config.py +0 -0
  64. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/schema/request.py +0 -0
  65. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/schema/response.py +0 -0
  66. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight/util.py +0 -0
  67. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight.egg-info/SOURCES.txt +0 -0
  68. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight.egg-info/dependency_links.txt +0 -0
  69. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight.egg-info/requires.txt +0 -0
  70. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/src/nnsight.egg-info/top_level.txt +0 -0
  71. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/tests/__init__.py +0 -0
  72. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/tests/test_diffusion.py +0 -0
  73. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/tests/test_tiny.py +0 -0
  74. {nnsight-0.5.0.dev12 → nnsight-0.5.0.dev14}/tests/test_vllm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nnsight
3
- Version: 0.5.0.dev12
3
+ Version: 0.5.0.dev14
4
4
  Summary: Package for interpreting and manipulating the internals of deep learning models.
5
5
  Author-email: Jaden Fiotto-Kaufman <jadenfk@outlook.com>
6
6
  License: MIT License
@@ -46,7 +46,7 @@ class Batcher:
46
46
 
47
47
 
48
48
 
49
- def batch(self, batchable: Batchable, *args, **kwargs):
49
+ def batch(self, batchable: Batchable, *args, **kwargs) -> Tuple[Any, Any]:
50
50
 
51
51
  if args or kwargs:
52
52
 
@@ -75,6 +75,8 @@ class Batcher:
75
75
 
76
76
  else:
77
77
  self.batch_groups.append((-1, -1))
78
+
79
+ return args, kwargs
78
80
 
79
81
 
80
82
  def narrow(self, batch_group: int, data:Any):
@@ -552,20 +552,27 @@ class Envoy(Batchable):
552
552
 
553
553
  # TODO legacy
554
554
  def session(self, *args, tracer_cls: Type[Tracer] = Tracer, **kwargs):
555
- return tracer_cls(*args, **kwargs)
555
+ tracer = tracer_cls(*args, **kwargs)
556
+ setattr(tracer, "model", self)
557
+ return tracer
556
558
 
557
- # TODO legacy
558
559
  @property
559
560
  @deprecated(message="Use `tracer.iter` instead.")
560
561
  @trace_only
561
562
  def iter(self):
562
563
  return IteratorProxy(self._interleaver)
563
564
 
564
- # TODO legacy
565
565
  @deprecated(message="Use `tracer.all()` instead.")
566
566
  @trace_only
567
567
  def all(self):
568
568
  return self.iter[:]
569
+
570
+ @deprecated(message="Use `tracer.next()` instead.")
571
+ @trace_only
572
+ def next(self, step: int = 1):
573
+ self._interleaver.current.iteration += step
574
+
575
+ return self
569
576
 
570
577
  @trace_only
571
578
  def skip(self, replacement: Any):
@@ -1377,9 +1384,9 @@ class EnvoySource:
1377
1384
 
1378
1385
 
1379
1386
 
1380
- def __getattr__(self, name: str) -> Union[OperationEnvoy]:
1387
+ def __getattribute__(self, name: str) -> Union[OperationEnvoy]:
1381
1388
 
1382
- return super().__getattr__(name)
1389
+ return object.__getattribute__(self, name)
1383
1390
 
1384
1391
 
1385
1392
  class Aliaser:
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import ctypes
4
4
  import inspect
5
+ import re
5
6
  import threading
6
7
  import time
7
8
  import warnings
@@ -10,15 +11,15 @@ from enum import Enum
10
11
  from functools import wraps
11
12
  from queue import Queue
12
13
  from threading import Thread
14
+ from types import FrameType
13
15
  from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
14
16
  Union)
15
17
 
16
18
  import torch
17
19
 
18
- from ..util import Patch, Patcher, applyn
20
+ from ..util import applyn
19
21
  from .batching import Batcher
20
- from .tracing.base import WithBlockNotFoundError
21
- from .tracing.util import wrap_exception
22
+ from .tracing.util import wrap_exception, get_non_nnsight_frame, push_variables
22
23
 
23
24
  if TYPE_CHECKING:
24
25
  from .tracing.tracer import Cache, InterleavingTracer, Tracer
@@ -92,7 +93,6 @@ class Interleaver:
92
93
  self.user_cache = user_cache
93
94
 
94
95
  self.mediators: Dict[str, Mediator] = {}
95
- self.state = dict()
96
96
  self.iteration_tracker = defaultdict(int)
97
97
  self.default_all = None
98
98
 
@@ -138,9 +138,10 @@ class Interleaver:
138
138
 
139
139
  module.forward = skippable_forward
140
140
 
141
+ @torch._dynamo.disable
141
142
  def input_hook(module: torch.nn.Module, args, kwargs):
142
143
 
143
- if not self.interleaving or torch.compiler.is_compiling():
144
+ if not self.interleaving:
144
145
  return args, kwargs
145
146
 
146
147
  provider = module.__path__
@@ -158,9 +159,10 @@ class Interleaver:
158
159
 
159
160
  module.register_forward_pre_hook(input_hook, with_kwargs=True, prepend=True)
160
161
 
162
+ @torch._dynamo.disable
161
163
  def output_hook(module: torch.nn.Module, _, output: Any):
162
164
 
163
- if not self.interleaving or torch.compiler.is_compiling():
165
+ if not self.interleaving:
164
166
  return output
165
167
 
166
168
  provider = module.__path__
@@ -304,7 +306,7 @@ class Interleaver:
304
306
  return
305
307
 
306
308
  ### Provider Methods ###
307
- @torch._dynamo.disable
309
+
308
310
  def handle(
309
311
  self,
310
312
  provider: Optional[Any] = None,
@@ -474,8 +476,6 @@ class Mediator:
474
476
 
475
477
  self.thread = None
476
478
 
477
- self.state = None
478
-
479
479
  if self.alive:
480
480
  # TODO: cancel inactive threads at the end of the model's execution
481
481
  self.response_queue.put(Cancelation())
@@ -566,11 +566,14 @@ class Mediator:
566
566
  mediator = mediator.child
567
567
 
568
568
  if mediator.name in participants:
569
-
569
+
570
+ mediator.respond()
571
+
570
572
  mediator.handle(provider)
573
+
571
574
 
572
- mediator.respond()
573
-
575
+
576
+
574
577
 
575
578
 
576
579
  def handle_end_event(self):
@@ -708,20 +711,18 @@ class Mediator:
708
711
  ### Requester Methods ###
709
712
 
710
713
  @property
711
- def frame(self):
714
+ def frame(self) -> FrameType:
712
715
  """
713
716
  Get the frame of the intervention function.
714
717
 
715
718
  Returns:
716
719
  The frame of the intervention function
717
720
  """
721
+
722
+
718
723
 
719
- frame = inspect.currentframe()
724
+ frame = get_non_nnsight_frame()
720
725
 
721
- while frame:
722
- frame = frame.f_back
723
- if frame and frame.f_code.co_filename.startswith("<nnsight"):
724
- break
725
726
  return frame
726
727
 
727
728
  def iterate(self, requester: Any):
@@ -730,21 +731,17 @@ class Mediator:
730
731
 
731
732
  def push(self):
732
733
  """Push local variables to the interleaver state."""
733
- self.interleaver.state.update(self.frame.f_locals)
734
+
735
+ state = {k: v for k, v in self.frame.f_locals.items() if not k.startswith("__nnsight")}
736
+ # this does not handle the case of a fn thats called in an invoker. this will push vars directly to where the invoke was called not the fn. really we need to grad the f_back of the <nnsight> frame. If its in threading.py, then we use info.frame
737
+ push_variables(self.info.frame, state)
734
738
 
735
739
  def pull(self):
736
740
  """Pull variables from the interleaver state to the frame globals."""
737
741
 
738
- for key in list(self.interleaver.state.keys()):
739
- if key.startswith("__nnsight"):
740
- self.interleaver.state.pop(key)
741
-
742
- self.frame.f_globals.update(self.interleaver.state)
743
- self.frame.f_locals.update(self.interleaver.state)
742
+ state = {k: v for k, v in self.info.frame.f_locals.items() if not k.startswith("__nnsight") and k not in self.frame.f_locals}
744
743
 
745
- ctypes.pythonapi.PyFrame_LocalsToFast(
746
- ctypes.py_object(self.frame), ctypes.c_int(0)
747
- )
744
+ push_variables(self.frame, state)
748
745
 
749
746
  def send(self, event: Events, requester: Any):
750
747
  """
@@ -6,7 +6,6 @@ from ...util import Patch
6
6
  from ..interleaver import Interleaver, Mediator
7
7
  from .invoker import Invoker
8
8
 
9
-
10
9
  def wrap_grad(interleaver: Interleaver):
11
10
  """
12
11
  Create a hook for gradient intervention.
@@ -100,9 +99,8 @@ class BackwardsTracer(Invoker):
100
99
  grad_patch.patch()
101
100
  with interleaver:
102
101
  self.fn(self.tensor, *self.args, **self.kwargs)
103
- grad_patch.restore()
104
- self.push(interleaver.state)
102
+ interleaver.check_dangling_mediators()
105
103
 
106
104
  finally:
105
+ grad_patch.restore()
107
106
  interleaver.cancel()
108
- interleaver.state.clear()
@@ -9,7 +9,8 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
9
9
  from ..backends.base import Backend
10
10
  from ..backends.execution import ExecutionBackend
11
11
  from .globals import Globals
12
- from .util import suppress_all_output
12
+ from .util import (TracingDeffermentException, get_non_nnsight_frame,
13
+ push_variables, suppress_all_output)
13
14
 
14
15
 
15
16
  class ExitTracingException(Exception):
@@ -29,12 +30,6 @@ class WithBlockNotFoundError(Exception):
29
30
 
30
31
  pass
31
32
 
32
- class TracingDeffermentException(Exception):
33
- """Exception raised when a tracing defferment is encountered.
34
-
35
- This exception is used to indicate that a tracing defferment is encountered.
36
- """
37
- pass
38
33
 
39
34
  class Tracer:
40
35
  """
@@ -145,22 +140,7 @@ class Tracer:
145
140
  the source code structure.
146
141
  """
147
142
  # Find the frame outside of nnsight by walking up the call stack
148
- frame = inspect.currentframe()
149
-
150
- while frame:
151
- frame = frame.f_back
152
- if frame:
153
- filename = frame.f_code.co_filename
154
- # Match if filename contains 'nnsight/tests' or 'nnsight\tests'
155
- # OR if it does NOT contain '/nnsight/' or '\nnsight\'
156
-
157
- if "__defer_capture__" in frame.f_locals:
158
- raise TracingDeffermentException()
159
- if (
160
- re.search(r"[\\/]{1}nnsight[\\/]{1}tests", filename)
161
- or not re.search(r"[\\/]{1}nnsight[\\/]", filename)
162
- ):
163
- break
143
+ frame = get_non_nnsight_frame()
164
144
 
165
145
  # Get source code lines from the appropriate location
166
146
  start_line = frame.f_lineno
@@ -295,9 +275,12 @@ class Tracer:
295
275
  # Wrap the captured code in a function definition with appropriate parameters
296
276
  self.info.source = [
297
277
  f"def __nnsight_tracer_{id(self)}__(__nnsight_tracer__, __nnsight_tracing_info__):\n",
278
+ " __nnsight_tracer__.pull()\n",
298
279
  *self.info.source,
299
280
  " __nnsight_tracer__.push()\n",
300
281
  ]
282
+
283
+ self.info.start_line -= 1
301
284
 
302
285
  def execute(self, fn: Callable):
303
286
  """
@@ -339,31 +322,31 @@ class Tracer:
339
322
  # Collect all non-nnsight variables from the frame
340
323
 
341
324
  state = {k: v for k, v in state.items() if not k.startswith("__nnsight")}
325
+
326
+ if Globals.stack == 1 :
327
+ state = {k: v for k, v in state.items() if id(v) in Globals.saves}
342
328
 
343
- if frame.f_code.co_filename.startswith("<nnsight"):
344
- # For dynamically generated code, update both globals and locals
345
- frame.f_globals.update(state)
346
- frame.f_locals.update(state)
347
-
348
- # Ensure locals are properly synchronized with the frame
349
- ctypes.pythonapi.PyFrame_LocalsToFast(
350
- ctypes.py_object(frame), ctypes.c_int(0)
351
- )
329
+ push_variables(frame, state)
352
330
 
353
- else:
354
- # For regular files, just update locals
355
- for key, value in state.items():
356
-
357
- if Globals.stack == 1 and id(value) not in Globals.saves:
358
- continue
331
+ state.clear()
332
+
333
+ def pull(self):
334
+
335
+ frame = inspect.currentframe()
359
336
 
360
- frame.f_locals[key] = value
337
+ while frame:
338
+ frame = frame.f_back
339
+ if frame and frame.f_code.co_filename.startswith(
340
+ "<nnsight"
341
+ ):
342
+ break
361
343
 
362
- ctypes.pythonapi.PyFrame_LocalsToFast(
363
- ctypes.py_object(frame), ctypes.c_int(0)
364
- )
365
344
 
366
- state.clear()
345
+ state = self.info.frame.f_locals
346
+
347
+ state = {k: v for k, v in state.items() if not k.startswith("__nnsight")}
348
+
349
+ push_variables(frame, state)
367
350
 
368
351
  def __enter__(self):
369
352
  """
@@ -48,6 +48,7 @@ class Invoker(Tracer):
48
48
  """
49
49
  self.info.source = [
50
50
  f"def __nnsight_tracer_{id(self)}__(__nnsight_mediator__, __nnsight_tracing_info__):\n",
51
+ " __nnsight_mediator__.pull()\n",
51
52
  *try_catch(
52
53
  self.info.source,
53
54
  exception_source=["__nnsight_mediator__.exception(exception)\n"],
@@ -55,7 +56,7 @@ class Invoker(Tracer):
55
56
  ),
56
57
  ]
57
58
 
58
- self.info.start_line -= 1
59
+ self.info.start_line -= 2
59
60
 
60
61
  def execute(self, fn: Callable):
61
62
  """
@@ -67,7 +68,10 @@ class Invoker(Tracer):
67
68
  Args:
68
69
  fn: The compiled intervention function
69
70
  """
70
- self.tracer.batcher.batch(self.tracer.model, *self.args, **self.kwargs)
71
+
72
+ inputs = self.tracer.batcher.batch(self.tracer.model, *self.args, **self.kwargs)
73
+
74
+ self.inputs = inputs
71
75
 
72
76
  mediator = Mediator(fn, self.info, batch_group=len(self.tracer.mediators))
73
77
 
@@ -36,6 +36,7 @@ class IteratorTracer(Tracer):
36
36
  # Wrap the captured code in a function definition with appropriate parameters
37
37
  self.info.source = [
38
38
  f"def __nnsight_tracer_{id(self)}__(__nnsight_mediator__, __nnsight_tracing_info__, {iteration_var_name}):\n",
39
+ " __nnsight_mediator__.pull()\n",
39
40
  *try_catch(
40
41
  self.info.source,
41
42
  exception_source=["__nnsight_mediator__.exception(exception)\n"],
@@ -43,9 +44,11 @@ class IteratorTracer(Tracer):
43
44
  ),
44
45
  ]
45
46
 
47
+ self.info.start_line -= 2
48
+
46
49
  def execute(self, fn: Callable):
47
50
 
48
- mediator = Mediator(fn, self.info)
51
+ mediator = Mediator(fn, self.info, batch_group=self.interleaver.current.batch_group)
49
52
  mediator.name = "Iterator" + mediator.name
50
53
 
51
54
  self.interleaver.current.iter(mediator, self.iteration)
@@ -1,4 +1,5 @@
1
1
  import copy
2
+ import inspect
2
3
  import re
3
4
  from dataclasses import dataclass
4
5
  from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
@@ -256,6 +257,8 @@ class InterleavingTracer(Tracer):
256
257
  self.batcher = Batcher(**kwargs)
257
258
 
258
259
  self.user_cache: List[Cache] = list()
260
+
261
+ self._frame = None
259
262
 
260
263
  super().__init__(*args, backend=backend)
261
264
 
@@ -297,20 +300,32 @@ class InterleavingTracer(Tracer):
297
300
 
298
301
  # If positional arguments were passed directly to a tracer, assume one invoker
299
302
  if self.args:
300
- invoker = self.invoke(*self.args, **self.kwargs)
301
- invoker.info = self.info.copy()
303
+
304
+ invoker = self.invoke(*self.args, _info=self.info.copy(), **self.kwargs)
302
305
 
303
306
  invoker.__exit__(ExitTracingException, None, None)
304
307
 
305
- self.info.source = [" pass\n"]
308
+ self.info.source = [f" {self.tracer_var_name}.mediators[0].info.frame = {self.tracer_var_name}.get_frame()\n"]
306
309
 
307
310
  self.info.source = [
308
311
  f"def __nnsight_tracer_{id(self)}__(__nnsight_tracing_info__,{self.tracer_var_name}):\n",
312
+ f" {self.tracer_var_name}.pull()\n",
309
313
  *self.info.source,
310
- f" {self.tracer_var_name}.push()\n",
314
+ f" {self.tracer_var_name}.get_frame()\n",
311
315
  ]
316
+
317
+ self.info.start_line -= 1
312
318
 
313
319
  self.args = tuple()
320
+
321
+
322
+ def get_frame(self):
323
+ """
324
+ Get the frame of the tracer.
325
+ """
326
+ self._frame = inspect.currentframe().f_back
327
+
328
+ return self._frame
314
329
 
315
330
  def execute(self, fn: Callable):
316
331
  """
@@ -322,7 +337,6 @@ class InterleavingTracer(Tracer):
322
337
  Args:
323
338
  fn: The compiled function to execute
324
339
  """
325
-
326
340
  fn(self.info, self)
327
341
 
328
342
  args = self.batcher.batched_args
@@ -334,13 +348,11 @@ class InterleavingTracer(Tracer):
334
348
  interleaver = self.model._interleaver
335
349
  interleaver.initialize(self.mediators, self, batcher=self.batcher, user_cache=self.user_cache)
336
350
 
337
- try:
338
- self.model.interleave(self.fn, *args, **kwargs)
351
+ self.model.interleave(self.fn, *args, **kwargs)
352
+
353
+ self.push(self._frame.f_locals)
339
354
 
340
- self.push(interleaver.state)
341
- finally:
342
- interleaver.state.clear()
343
-
355
+
344
356
 
345
357
  ### Public API ####
346
358
 
@@ -370,6 +382,11 @@ class InterleavingTracer(Tracer):
370
382
 
371
383
  def all(self):
372
384
  return self.iter[:]
385
+
386
+ def next(self, step: int = 1):
387
+ self.model._interleaver.current.iteration += step
388
+
389
+ return self
373
390
 
374
391
  def cache(
375
392
  self,
@@ -538,11 +555,14 @@ class Barrier:
538
555
 
539
556
  def __call__(self):
540
557
 
558
+
541
559
  mediator = self.model._interleaver.current
542
560
 
543
561
  self.participants.add(mediator.name)
544
562
 
545
563
  if len(self.participants) == self.n_participants:
564
+
546
565
  mediator.send(Events.BARRIER, self.participants)
566
+ self.participants = set()
547
567
  else:
548
568
  mediator.send(Events.BARRIER, None)
@@ -1,9 +1,13 @@
1
1
 
2
2
  import contextlib
3
+ import ctypes
4
+ import inspect
3
5
  import os
6
+ import re
4
7
  import sys
5
8
  from builtins import open
6
- from typing import TYPE_CHECKING, Callable, List
9
+ from types import FrameType
10
+ from typing import TYPE_CHECKING, Callable, Dict, List
7
11
 
8
12
  if TYPE_CHECKING:
9
13
  from .base import Tracer
@@ -83,6 +87,7 @@ def get_dependencies(fn:Callable):
83
87
 
84
88
  from ... import CONFIG
85
89
 
90
+
86
91
  class ExceptionWrapper(Exception):
87
92
  """
88
93
  Wrapper for exceptions that provides additional details for tracer created code.
@@ -258,3 +263,54 @@ def wrap_exception(exception:Exception, info:"Tracer.Info"):
258
263
  wrapped.__dict__.update(exception.__dict__)
259
264
 
260
265
  return wrapped
266
+
267
+ class TracingDeffermentException(Exception):
268
+ """Exception raised when a tracing defferment is encountered.
269
+
270
+ This exception is used to indicate that a tracing defferment is encountered.
271
+ """
272
+ pass
273
+
274
+ def get_non_nnsight_frame() -> FrameType:
275
+ frame = inspect.currentframe()
276
+
277
+ while frame:
278
+ frame = frame.f_back
279
+ if frame:
280
+ filename = frame.f_code.co_filename
281
+ # Match if filename contains 'nnsight/tests' or 'nnsight\tests'
282
+ # OR if it does NOT contain '/nnsight/' or '\nnsight\'
283
+
284
+ if "__defer_capture__" in frame.f_locals:
285
+ raise TracingDeffermentException()
286
+ if (
287
+ re.search(r"[\\/]{1}nnsight[\\/]{1}tests", filename)
288
+ or not re.search(r"[\\/]{1}nnsight[\\/]", filename)
289
+ ):
290
+ break
291
+
292
+ return frame
293
+
294
+
295
+ def push_variables(frame:FrameType, variables:Dict):
296
+
297
+ is_generated_frame = frame.f_code.co_filename.startswith("<nnsight")
298
+
299
+ if is_generated_frame:
300
+
301
+ global_variables = {k: v for k, v in variables.items() if k not in frame.f_locals}
302
+
303
+ for key, value in global_variables.items():
304
+ frame.f_globals[key] = value
305
+
306
+ ctypes.pythonapi.PyFrame_LocalsToFast(
307
+ ctypes.py_object(frame), ctypes.c_int(0)
308
+ )
309
+
310
+ for key, value in variables.items():
311
+ frame.f_locals[key] = value
312
+
313
+ ctypes.pythonapi.PyFrame_LocalsToFast(
314
+ ctypes.py_object(frame), ctypes.c_int(0)
315
+ )
316
+
@@ -2,32 +2,16 @@ from __future__ import annotations
2
2
 
3
3
  import json
4
4
  import os
5
- from typing import (
6
- TYPE_CHECKING,
7
- Any,
8
- Dict,
9
- List,
10
- Optional,
11
- Tuple,
12
- Type,
13
- Union,
14
- )
15
- from huggingface_hub import constants
16
- from huggingface_hub.file_download import repo_folder_name
5
+ import warnings
6
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
17
7
 
18
- from nnsight import CONFIG
19
8
  import torch
9
+ from huggingface_hub import constants
10
+ from huggingface_hub.file_download import repo_folder_name
20
11
  from torch.nn.modules import Module
21
- from transformers import (
22
- AutoConfig,
23
- AutoModel,
24
- AutoModelForCausalLM,
25
- AutoTokenizer,
26
- BatchEncoding,
27
- PretrainedConfig,
28
- PreTrainedModel,
29
- PreTrainedTokenizer,
30
- )
12
+ from transformers import (AutoConfig, AutoModel, AutoModelForCausalLM,
13
+ AutoTokenizer, BatchEncoding, PretrainedConfig,
14
+ PreTrainedModel, PreTrainedTokenizer)
31
15
  from transformers.generation.utils import GenerationMixin
32
16
  from transformers.models.auto import modeling_auto
33
17
  from transformers.models.llama.configuration_llama import LlamaConfig
@@ -101,6 +85,7 @@ class LanguageModel(RemoteableMixin):
101
85
  # If the user passed in a pre-loaded model, might be able to get repo id off of it.
102
86
  # That way if they dont provide a tokenizer, we can load it for them later.
103
87
  self.repo_id: str = args[0] if isinstance(args[0], str) else getattr(args[0], 'name_or_path', None)
88
+ self.revision: str = getattr(args[0], 'revision', 'main')
104
89
 
105
90
  super().__init__(*args, **kwargs)
106
91
 
@@ -119,22 +104,26 @@ class LanguageModel(RemoteableMixin):
119
104
  # Some transformer models compile on first generation. As of 0.5.0.dev7 this not not work with nnsight if fullgraph is True
120
105
  def _patch_generation_config(self, model:torch.nn.Module):
121
106
 
122
- if hasattr(model, "generation_config"):
107
+ if getattr(model, "generation_config", None) is not None:
108
+
109
+
110
+ warnings.filterwarnings("ignore", message="The CUDA Graph is empty")
123
111
 
124
112
  generation_config = model.generation_config
125
113
 
126
- compile_config = generation_config.compile_config
114
+ compile_config = getattr(generation_config, "compile_config", None)
127
115
 
128
116
  if compile_config is None:
129
117
 
130
- from transformers.generation.configuration_utils import CompileConfig
118
+ from transformers.generation.configuration_utils import \
119
+ CompileConfig
131
120
 
132
121
  compile_config = CompileConfig()
133
122
 
134
123
  compile_config.fullgraph = False
135
124
  compile_config.dynamic = True
136
125
 
137
- generation_config.compile_config = compile_config
126
+ setattr(generation_config, "compile_config", compile_config)
138
127
 
139
128
  def export_edits(self, name:Optional[str] = None, export_dir: Optional[str] = None, variant: str = '__default__'):
140
129
  """TODO
@@ -213,6 +202,7 @@ class LanguageModel(RemoteableMixin):
213
202
  def _load_meta(
214
203
  self,
215
204
  repo_id: str,
205
+ revision:Optional[str] = "main",
216
206
  tokenizer_kwargs: Optional[Dict[str, Any]] = {},
217
207
  patch_llama_scan: bool = True,
218
208
  **kwargs,
@@ -220,9 +210,11 @@ class LanguageModel(RemoteableMixin):
220
210
 
221
211
  self.repo_id = repo_id
222
212
 
223
- self._load_config(repo_id, **kwargs)
213
+ self.revision = revision
224
214
 
225
- self._load_tokenizer(repo_id, **tokenizer_kwargs)
215
+ self._load_config(repo_id, revision=revision, **kwargs)
216
+
217
+ self._load_tokenizer(repo_id, revision=revision, **tokenizer_kwargs)
226
218
 
227
219
  if (
228
220
  patch_llama_scan
@@ -243,14 +235,15 @@ class LanguageModel(RemoteableMixin):
243
235
  def _load(
244
236
  self,
245
237
  repo_id: str,
238
+ revision:Optional[str] = "main",
246
239
  tokenizer_kwargs: Optional[Dict[str, Any]] = {},
247
240
  patch_llama_scan: bool = True,
248
241
  **kwargs,
249
242
  ) -> PreTrainedModel:
250
243
 
251
- self._load_config(repo_id, **kwargs)
244
+ self._load_config(repo_id, revision=revision, **kwargs)
252
245
 
253
- self._load_tokenizer(repo_id, **tokenizer_kwargs)
246
+ self._load_tokenizer(repo_id, revision=revision, **tokenizer_kwargs)
254
247
 
255
248
  if (
256
249
  patch_llama_scan
@@ -260,7 +253,7 @@ class LanguageModel(RemoteableMixin):
260
253
  ):
261
254
  self.config.rope_scaling["rope_type"] = "llama3"
262
255
 
263
- model = self.automodel.from_pretrained(repo_id, config=self.config, **kwargs)
256
+ model = self.automodel.from_pretrained(repo_id, config=self.config, revision=revision, **kwargs)
264
257
 
265
258
  self.config = model.config
266
259
 
@@ -399,7 +392,7 @@ class LanguageModel(RemoteableMixin):
399
392
 
400
393
  def _remoteable_model_key(self) -> str:
401
394
  return json.dumps(
402
- {"repo_id": self.repo_id} # , "torch_dtype": str(self._model.dtype)}
395
+ {"repo_id": self.repo_id, "revision": self.revision} # , "torch_dtype": str(self._model.dtype)}
403
396
  )
404
397
 
405
398
  @classmethod
@@ -409,7 +402,9 @@ class LanguageModel(RemoteableMixin):
409
402
 
410
403
  repo_id = kwargs.pop("repo_id")
411
404
 
412
- return LanguageModel(repo_id, **kwargs)
405
+ revision = kwargs.pop("revision", "main")
406
+
407
+ return LanguageModel(repo_id, revision=revision, **kwargs)
413
408
 
414
409
 
415
410
  if TYPE_CHECKING:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nnsight
3
- Version: 0.5.0.dev12
3
+ Version: 0.5.0.dev14
4
4
  Summary: Package for interpreting and manipulating the internals of deep learning models.
5
5
  Author-email: Jaden Fiotto-Kaufman <jadenfk@outlook.com>
6
6
  License: MIT License
@@ -872,6 +872,23 @@ def test_iter_and_skip(gpt2: nnsight.LanguageModel):
872
872
  assert not torch.all(arr_gen[2] == 0)
873
873
 
874
874
 
875
+ @torch.no_grad()
876
+ @pytest.mark.iter
877
+ def test_iter_with_batched_interventions(gpt2: nnsight.LanguageModel, ET_prompt: str, MSG_prompt: str):
878
+ with gpt2.generate(max_new_tokens=3) as tracer:
879
+ with tracer.invoke(ET_prompt):
880
+ logits_1 = list().save()
881
+ with tracer.iter[:]:
882
+ logits_1.append(gpt2.lm_head.output[0][-1].argmax(dim=-1))
883
+
884
+ with tracer.invoke(MSG_prompt):
885
+ logits_2 = list().save()
886
+ with tracer.iter[0:3]:
887
+ logits_2.append(gpt2.lm_head.output[0][-1].argmax(dim=-1))
888
+
889
+ assert all([not torch.equal(logit_1, logit_2) for logit_1, logit_2 in zip(logits_1, logits_2)])
890
+
891
+
875
892
  ######################### CACHE #################################
876
893
 
877
894
  @torch.no_grad()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes