interp-engine 1.2.5__tar.gz → 1.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interp_engine-1.2.5 → interp_engine-1.2.7}/PKG-INFO +1 -1
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/AGENT_INTEGRATION.md +1 -1
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/ARCHITECTURE_QUIRKS.md +9 -1
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/USAGE.md +31 -2
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/tokenize.py +56 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/freeze.py +39 -4
- {interp_engine-1.2.5 → interp_engine-1.2.7}/pyproject.toml +1 -1
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_chat_formatters.py +121 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_chat_templates.py +31 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_freeze_set.py +65 -5
- {interp_engine-1.2.5 → interp_engine-1.2.7}/.gitignore +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/LICENSE +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/README.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/README.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/__init__.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/bench_spec.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/cells.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/probe.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/publish.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/report_bench.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__eager.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__vllm-cudagraph.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__vllm-dspark-cudagraph.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__vllm-dspark.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__vllm-freeze.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__vllm.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/gemma-2-2b__eager.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/gemma-2-2b__vllm-cudagraph.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/gemma-2-2b__vllm-freeze.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/gemma-2-2b__vllm.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/llama-3.1-8b__eager.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/llama-3.1-8b__vllm-cudagraph.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/llama-3.1-8b__vllm-freeze.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/llama-3.1-8b__vllm.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3-4b__eager.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3-4b__vllm-cudagraph.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3-4b__vllm-freeze.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3-4b__vllm.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3.8-27b__eager.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3.8-27b__vllm-cudagraph.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3.8-27b__vllm-freeze.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3.8-27b__vllm.json +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results-latest.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/run_all.sh +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/run_bench.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/workloads.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/COMPATIBILITY.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/ENGINE_HOOK_MAPPINGS.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/GRADIENTS.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/INTERNALS.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/PERFORMANCE.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/PORTING.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/README.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/docs/SUPPORTED_POINTS.md +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/__init__.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/_loop.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/address.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/arch.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/attn_config.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/attn_scores.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/autograd_support.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/capture.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/chat_compose.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/chat_conventions.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/chat_formatters.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/cuda_preflight.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/dispatch.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/facts.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/hooks.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/lens.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/load.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/mappers.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/model.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/moe_routing.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/points.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/protocol.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/residual_basis.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/select.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/steer.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/steer_specs.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/sync.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_backend.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/__init__.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/_demux.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/_hooks.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/_payload.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/_tree.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/attn.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/capture.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/graphs.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/lens/__init__.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/lens/intervene.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/lens/readout.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/lens/unembed.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/mhc.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/native.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/requests.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_capture/steering.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/interp_engine/vllm_plugin.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/conftest.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/harness.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/model_expectations.yaml +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/synthetic_families.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_address.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_attn_config_tripwire.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_attn_probs_indexing.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_attn_scores.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_attn_z_gqa.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_autograd_support.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_bench_workloads.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_capability_refusals.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_capture_addressing.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_chat_compose.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_core.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_cuda_preflight.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_doc_code_fences.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_eager_autograd.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_facts.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_family_points.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_freeze_dsv4_gpu.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_freeze_parity_gpu.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_freeze_warmup.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_gated_attn_out.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_head_contributions.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_hook_call_conventions.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_layer_kinds.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_load.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_logit_transform.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_mappers.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_mlp_internals.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_model_expectations.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_moe.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_multimodal_arch.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_new_models_gpu.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_no_chat_template.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_normalized_hook.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_packaging.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_parity_gpt2.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_per_layer_attn_dims.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_points_registry.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_protocol.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_published_benchmarks.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_qk_norm.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_qkv_layout.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_reasoning_spans.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_release.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_resid_mid.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_residual_basis.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_sandwich_norms.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_select.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_sliding_window_attn.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_small_models_gpu.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_steer_context.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_steer_math_parity.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_sync_loop.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_sync_parity.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_unified_free_functions.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_unresolved_families.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_capture_gpu.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_capture_scales.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_graph_path.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_graphs_on_gpu.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_hook_availability.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_hyper_connections.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_kv_isolation.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_new_points.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_only_families.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_plugin.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vllm_wire_grammar.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_vocabulary_boundary.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_worker_lens_capture_readout.py +0 -0
- {interp_engine-1.2.5 → interp_engine-1.2.7}/tests/test_worker_lens_readout.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: interp-engine
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.7
|
|
4
4
|
Summary: A fast, standardized interpretability engine that supports most modern models and architectures. Powers Neuronpedia.
|
|
5
5
|
Project-URL: Homepage, https://github.com/decoderesearch/interp-engine
|
|
6
6
|
Project-URL: Repository, https://github.com/decoderesearch/interp-engine
|
|
@@ -236,7 +236,7 @@ mistake you made.
|
|
|
236
236
|
| `UnknownCoordinate` | an address carried a coordinate this version has no field for | version skew between processes — align the engine version on both |
|
|
237
237
|
| `{key} fired twice in one forward pass` | one address resolved to a module invoked more than once | a re-entrant trunk; capture per-invocation or name a narrower address |
|
|
238
238
|
| `Captured nothing at {missing}: the resolved module(s) did not run` | a quantized or fused kernel replaced the block's forward and computes the tensor inline | different point, or an unfused load |
|
|
239
|
-
| `NoChatTemplateError` | the
|
|
239
|
+
| `NoChatTemplateError` | the model has no chat format (no Jinja template and no code formatter) | `model.tok.has_chat_template()` first; do not hand-write one, and do not read `tokenizer.chat_template` — it is `None` for families that define their format in code |
|
|
240
240
|
| `CapabilityUnsupported` | you asked one backend for something only the other can do | the message names the call that works; `CAPABILITIES` is the table it came from (rule 13) |
|
|
241
241
|
| `... cannot run from inside a running event loop` | a sync free function or `sync_model` call from async code | `await` the method it wraps; the message names it |
|
|
242
242
|
| `... needs vLLM, but vLLM is not installed` | missing extra, or a non-Linux/CUDA box | `pip install 'interp-engine[vllm]'`, or `backend="eager"`; gate on `vllm_installed()` to branch instead of catching |
|
|
@@ -38,16 +38,24 @@ in `facts.py` instead.
|
|
|
38
38
|
| `points.py` | The canonical hook points as data: scope, width, vLLM support and the reason for each limit. Consumers (the vLLM served-point gate, the width guard, the reshape set, the docs footnotes) derive from it. | **Never for a new model** — a point is architecture-independent by construction. Only when adding a _point_, and then `tests/test_points_registry.py` tells you what else needs updating. |
|
|
39
39
|
| `chat_conventions.py` | How a model structures what it **generates**: harmony markers (gpt-oss), reasoning delimiters (`REASONING_TAGS`, e.g. `<think>`/`</think>`), and generic turn-end tokens. | Only when a model introduces a _new_ reasoning delimiter pair — append one `ReasoningTags` entry. Models reusing `<think>` need no change. |
|
|
40
40
|
| `chat_compose.py` | Consumes the table above to turn a generation back into messages (`compose_assistant_turns`), normalizing every family's reasoning to `<think>…</think>` in the message content. | Never for a new model — it reads `chat_conventions.py`. Only to change the _wire shape_ clients receive. |
|
|
41
|
+
| `chat_formatters.py` | `CODE_CHAT_FORMATS`: the few architectures that ship **no** `chat_template` and define their prompt format in Python instead (DeepSeek-V4), mapped to a loader that imports the checkpoint's own encoder. | Only when a model has no chat template at all. A model with a template needs no entry — that is the other 99%, and the default path below. |
|
|
41
42
|
|
|
42
43
|
Two deliberate non-entries:
|
|
43
44
|
|
|
44
|
-
- **Prompt-side chat structure
|
|
45
|
+
- **Prompt-side chat structure needs no per-family entry, in either direction.** Going _in_,
|
|
45
46
|
`Tokenize.message_spans` renders through the model's real chat template and derives per-token
|
|
46
47
|
role/section by diffing renders, so every family (ChatML, Gemma, Llama, harmony) works without
|
|
47
48
|
per-family code. Coming back _out_, `chat_compose.py` reads only the generated text and lets the
|
|
48
49
|
caller supply the prompt messages it already holds, so no code ever parses a rendered prompt back
|
|
49
50
|
into messages. Between them they replaced the frontend's per-family state machines and the
|
|
50
51
|
inference app's per-model response parsers.
|
|
52
|
+
|
|
53
|
+
`CODE_CHAT_FORMATS` is the one exception, and it is narrow on purpose: it holds only checkpoints
|
|
54
|
+
that publish **no** template, where there is no render to diff and the format exists solely as
|
|
55
|
+
code. Even then the entry is a _loader_, not a description — the format itself is read from the
|
|
56
|
+
encoder shipped beside the weights, so the table names which architectures are affected and
|
|
57
|
+
never what their prompts look like. Adding a model that _has_ a template here would be the
|
|
58
|
+
mistake the paragraph above is about.
|
|
51
59
|
- **Nothing keys on a model-name _substring_.** Both tables above key on architecture class or on
|
|
52
60
|
added-token-vocab membership, i.e. on **capability**. A substring match (`"llama-3" in
|
|
53
61
|
name_or_path`) is the anti-pattern this structure exists to avoid: a substring is a claim about a
|
|
@@ -197,8 +197,37 @@ prompt = model.tok.apply_chat_template([{"role": "user", "content": "Hi"}])
|
|
|
197
197
|
token_ids = model.tok.apply_chat_template([{"role": "user", "content": "Hi"}], tokenize=True)
|
|
198
198
|
```
|
|
199
199
|
|
|
200
|
-
`apply_chat_template` raises `NoChatTemplateError` when the
|
|
201
|
-
|
|
200
|
+
`apply_chat_template` raises `NoChatTemplateError` when the model has no chat format at all, rather
|
|
201
|
+
than inventing one it was never trained on; `model.tok.has_chat_template()` asks first.
|
|
202
|
+
|
|
203
|
+
A few checkpoints ship no Jinja `chat_template` because they define their format in Python instead —
|
|
204
|
+
DeepSeek-V4 carries an `encoding/encoding_dsv4.py` in the repo, beside the weights, and a Jinja
|
|
205
|
+
template could not express its other half (`parse_message_from_completion_text`). The engine
|
|
206
|
+
downloads and imports that file rather than vendoring a copy of it, so those models render chat
|
|
207
|
+
through the same three calls above, and `has_chat_template()` says yes. Reading
|
|
208
|
+
`tokenizer.chat_template` directly is what gets this wrong: it is `None` for a model that renders
|
|
209
|
+
chat perfectly well. Loading the file is remote code execution, so it needs `trust_remote_code=True`;
|
|
210
|
+
without it the model still loads and chat is simply unavailable.
|
|
211
|
+
|
|
212
|
+
`model.tok.accepted_template_kwargs([...])` reports which optional controls (`enable_thinking`,
|
|
213
|
+
`reasoning_effort`) this model actually reads, whichever of the two renders it. Adding a family means
|
|
214
|
+
one entry in `chat_formatters.CODE_CHAT_FORMATS`.
|
|
215
|
+
|
|
216
|
+
To attribute *tokens* to messages there are two methods, and the difference matters. `message_spans`
|
|
217
|
+
gives per-token role, channel and section (`header` / `content` / `footer`), leaving the trailing
|
|
218
|
+
generation scaffold owned by no message — use it to read or display structure. `message_partition`
|
|
219
|
+
gives one contiguous `[start, end)` span per message that together cover every token, which is what
|
|
220
|
+
mean-pooling activations per turn needs:
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
token_ids, spans = model.tok.message_partition([{"role": "user", "content": "Hi"}])
|
|
224
|
+
per_turn = [acts[start:end].mean(0) for start, end in spans]
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
Only `message_partition` is correct for a code-rendered model. Computing the same spans by rendering
|
|
228
|
+
growing message prefixes and taking length deltas assumes appending a message only appends tokens;
|
|
229
|
+
DeepSeek-V4 rewrites earlier turns once a later user turn exists (it drops their reasoning), so the
|
|
230
|
+
deltas land in the wrong places and still look like a partition.
|
|
202
231
|
|
|
203
232
|
## Capture while generating
|
|
204
233
|
|
|
@@ -297,6 +297,62 @@ class Tokenize:
|
|
|
297
297
|
)
|
|
298
298
|
)
|
|
299
299
|
|
|
300
|
+
def message_partition(
|
|
301
|
+
self,
|
|
302
|
+
messages: list[dict[str, str]],
|
|
303
|
+
**template_kwargs: Any,
|
|
304
|
+
) -> tuple[list[int], list[tuple[int, int]]]:
|
|
305
|
+
"""The closed conversation's ids, plus one contiguous ``[start, end)`` span per message.
|
|
306
|
+
|
|
307
|
+
A *partition*, which is a different contract from :meth:`message_spans`: the spans are
|
|
308
|
+
contiguous, cover every token, and align 1:1 with ``messages``. ``message_spans`` splits
|
|
309
|
+
each message into header/content/footer sections and leaves the trailing generation
|
|
310
|
+
scaffold owned by nobody, so a caller that mean-pools activations per turn wants this
|
|
311
|
+
one. Rendered closed, since a scaffold belongs to the turn nobody has written yet.
|
|
312
|
+
|
|
313
|
+
The two branches below are deliberately NOT unified. Where a code formatter renders, the
|
|
314
|
+
boundaries are exact, because the formatter reports its own message blocks. Where a Jinja
|
|
315
|
+
template does, they come from the length each render grows by as one more message is
|
|
316
|
+
appended -- an assumption that the template only ever appends, and also the arithmetic
|
|
317
|
+
every existing caller's numbers were computed with. Preserving it verbatim is the point:
|
|
318
|
+
a mean pooled over a span whose edge moved by one token is a different number, and these
|
|
319
|
+
feed persona projections that are compared across runs. So a template-rendered model is
|
|
320
|
+
unaffected by this method existing, and a code-rendered one gets boundaries the
|
|
321
|
+
prefix-delta could not have found.
|
|
322
|
+
"""
|
|
323
|
+
if not messages:
|
|
324
|
+
return [], []
|
|
325
|
+
|
|
326
|
+
if self.formatter is not None:
|
|
327
|
+
rendered = self.formatter.render(
|
|
328
|
+
messages,
|
|
329
|
+
add_generation_prompt=False,
|
|
330
|
+
continue_final_message=False,
|
|
331
|
+
**template_kwargs,
|
|
332
|
+
)
|
|
333
|
+
full_ids = self._encode_rendered(rendered.text)
|
|
334
|
+
bounds = [0]
|
|
335
|
+
for j in range(1, len(messages)):
|
|
336
|
+
cut = _common_prefix_len(self._encode_rendered(rendered.upto(j)), full_ids)
|
|
337
|
+
bounds.append(max(bounds[-1], min(cut, len(full_ids))))
|
|
338
|
+
# The last message closes the sequence by construction (`upto(n)` is the whole
|
|
339
|
+
# render once the generation scaffold is off), stated rather than measured so the
|
|
340
|
+
# spans cover every token even if a tokenizer merges across the final cut.
|
|
341
|
+
bounds.append(len(full_ids))
|
|
342
|
+
return full_ids, list(zip(bounds, bounds[1:]))
|
|
343
|
+
|
|
344
|
+
bounds = [0]
|
|
345
|
+
full_ids: list[int] = []
|
|
346
|
+
for j in range(1, len(messages) + 1):
|
|
347
|
+
full_ids = self._render_ids(
|
|
348
|
+
messages[:j],
|
|
349
|
+
add_generation_prompt=False,
|
|
350
|
+
continue_final_message=False,
|
|
351
|
+
**template_kwargs,
|
|
352
|
+
)
|
|
353
|
+
bounds.append(len(full_ids))
|
|
354
|
+
return full_ids, list(zip(bounds, bounds[1:]))
|
|
355
|
+
|
|
300
356
|
def message_spans(
|
|
301
357
|
self,
|
|
302
358
|
messages: list[dict[str, str]],
|
|
@@ -252,8 +252,24 @@ def resolve_freeze_points(
|
|
|
252
252
|
``graph_replay`` is False only when freeze is omitted and ``enforce_eager`` is not False
|
|
253
253
|
(today's hooked vLLM). ``freeze_points="auto"`` or ``enforce_eager=False`` with no list
|
|
254
254
|
freezes ``resid_post`` at every layer on a conventional trunk, and ``resid_streams`` at
|
|
255
|
-
every layer on a hyper-connection trunk.
|
|
255
|
+
every layer on a hyper-connection trunk -- to read AND to write.
|
|
256
|
+
|
|
257
|
+
Auto covers the write because the two are one decision for the caller who asks for it. ``auto``
|
|
258
|
+
says "serve the residual endpoints on this engine", and read taps alone serve half of them: a
|
|
259
|
+
lens read-out works, and every steer, ablation and swap derived from that read-out is refused
|
|
260
|
+
for want of a site at the very address already being tapped. Nothing in the caller's vocabulary
|
|
261
|
+
distinguished the two halves, either -- ``freeze_writes`` is a list of addresses, so asking for
|
|
262
|
+
the write meant restating every layer of a set ``auto`` had just built.
|
|
263
|
+
|
|
264
|
+
An explicit list is left alone, since a caller who named the points named what they wanted. So
|
|
265
|
+
is ``freeze_writes=[]``, which is how to ask for the reads WITHOUT the write buffers: those
|
|
266
|
+
buffers are per layer and per token, and they are what steps ``max_num_batched_tokens`` down
|
|
267
|
+
when the ladder in :func:`fit_max_num_batched_tokens` cannot fit them.
|
|
256
268
|
"""
|
|
269
|
+
# `None` and `[]` mean different things here and nowhere else in this signature: the first is
|
|
270
|
+
# "say nothing about writes", which auto fills in, and the second is "no writes", which it must
|
|
271
|
+
# not. Read before the comprehension below, which cannot tell them apart.
|
|
272
|
+
writes_declared = freeze_writes is not None
|
|
257
273
|
writes = [to_address(a) for a in (freeze_writes or ())]
|
|
258
274
|
graph = freeze_points is not None or bool(writes) or enforce_eager is False
|
|
259
275
|
if not graph:
|
|
@@ -264,14 +280,21 @@ def resolve_freeze_points(
|
|
|
264
280
|
"Do not pass enforce_eager=True with a freeze set; omit freeze_points for hooked vLLM."
|
|
265
281
|
)
|
|
266
282
|
|
|
283
|
+
auto = False
|
|
267
284
|
if freeze_points is None:
|
|
268
|
-
|
|
285
|
+
# Writes without a read list stay writes-only: naming a write site is already explicit
|
|
286
|
+
# about what this engine is for, and auto-reading every layer beside it is not implied.
|
|
287
|
+
auto = not writes
|
|
288
|
+
reads: list[Address] = _auto_reads(n_layers, n_streams) if auto else []
|
|
269
289
|
elif isinstance(freeze_points, str):
|
|
270
290
|
if freeze_points != "auto":
|
|
271
291
|
raise ValueError(f"freeze_points must be 'auto', a list of addresses, or []; got {freeze_points!r}")
|
|
272
292
|
reads = _auto_reads(n_layers, n_streams)
|
|
293
|
+
auto = True
|
|
273
294
|
else:
|
|
274
295
|
reads = [to_address(a) for a in freeze_points]
|
|
296
|
+
if auto and not writes_declared:
|
|
297
|
+
writes = list(reads)
|
|
275
298
|
|
|
276
299
|
for address in (*reads, *writes):
|
|
277
300
|
reason = freeze_unsupported_reason(address.name) or multi_stream_refusal_reason(address.name, n_streams)
|
|
@@ -990,7 +1013,16 @@ def worker_install_freeze(worker: object) -> None:
|
|
|
990
1013
|
|
|
991
1014
|
|
|
992
1015
|
def _install_mhc_freeze(worker: object, mhc_sites: Sequence[_Site]) -> None:
|
|
993
|
-
|
|
1016
|
+
"""Wrap the mHC kernels for every stacked freeze site, refusing one this model cannot serve.
|
|
1017
|
+
|
|
1018
|
+
A site that carries a ``delta`` is asked the STEER question rather than the capture one, because
|
|
1019
|
+
they differ by exactly the thing a write depends on: ``resid_streams`` is written by running the
|
|
1020
|
+
fused kernel's second half again on the edited stack, so that half has to be forwardable
|
|
1021
|
+
(:func:`~interp_engine.vllm_capture.mhc.pre_rerun_gap`). Asked here because freeze installs in
|
|
1022
|
+
``Worker.load_model`` -- a gap found later surfaces mid-forward, on an engine that has already
|
|
1023
|
+
started and reported itself healthy.
|
|
1024
|
+
"""
|
|
1025
|
+
from interp_engine.vllm_capture.mhc import mhc_taps, require_available, require_steerable
|
|
994
1026
|
|
|
995
1027
|
model = _worker_model(worker)
|
|
996
1028
|
taps = mhc_taps(worker)
|
|
@@ -999,7 +1031,10 @@ def _install_mhc_freeze(worker: object, mhc_sites: Sequence[_Site]) -> None:
|
|
|
999
1031
|
if id(site) in seen:
|
|
1000
1032
|
continue
|
|
1001
1033
|
seen.add(id(site))
|
|
1002
|
-
|
|
1034
|
+
if site.delta is not None:
|
|
1035
|
+
require_steerable(model, site.address.name, site.address.layer)
|
|
1036
|
+
else:
|
|
1037
|
+
require_available(model, site.address.name, site.address.layer)
|
|
1003
1038
|
taps.add(site.address, _freeze_mhc_recorder(worker, site))
|
|
1004
1039
|
|
|
1005
1040
|
|
|
@@ -393,6 +393,127 @@ def test_prefill_spans_end_on_content(tok: Tokenize):
|
|
|
393
393
|
assert spans[-1].role == "assistant"
|
|
394
394
|
|
|
395
395
|
|
|
396
|
+
# --------------------------------------------------------------------------- #
|
|
397
|
+
# Per-message partition (what activation pooling indexes by)
|
|
398
|
+
# --------------------------------------------------------------------------- #
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _legacy_partition(tok: Tokenize, messages: list[dict[str, str]]) -> tuple[list[int], list[tuple[int, int]]]:
|
|
402
|
+
"""The prefix-delta arithmetic that callers ran inline before ``message_partition``.
|
|
403
|
+
|
|
404
|
+
Kept here verbatim as the reference implementation. A template-rendered model has to
|
|
405
|
+
partition identically to this forever: the spans index activation mean-pooling, so an edge
|
|
406
|
+
that moves by one token silently changes a published number.
|
|
407
|
+
"""
|
|
408
|
+
spans: list[tuple[int, int]] = []
|
|
409
|
+
previous = 0
|
|
410
|
+
full_ids: list[int] = []
|
|
411
|
+
for index in range(len(messages)):
|
|
412
|
+
current = tok.apply_chat_template(messages[: index + 1], tokenize=True, add_generation_prompt=False)
|
|
413
|
+
assert isinstance(current, list)
|
|
414
|
+
spans.append((previous, len(current)))
|
|
415
|
+
previous = len(current)
|
|
416
|
+
full_ids = current
|
|
417
|
+
return full_ids, spans
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
class _RecordingTokenizer(_CharTokenizer):
|
|
421
|
+
"""A templated tokenizer that records how it was asked to render."""
|
|
422
|
+
|
|
423
|
+
chat_template = "{# present; contents never evaluated here #}"
|
|
424
|
+
|
|
425
|
+
def __init__(self):
|
|
426
|
+
super().__init__()
|
|
427
|
+
self.calls: list[tuple] = []
|
|
428
|
+
|
|
429
|
+
def apply_chat_template(
|
|
430
|
+
self,
|
|
431
|
+
messages: list[dict[str, str]],
|
|
432
|
+
tokenize: bool = True,
|
|
433
|
+
add_generation_prompt: bool = False,
|
|
434
|
+
continue_final_message: bool = False,
|
|
435
|
+
**_: Any,
|
|
436
|
+
):
|
|
437
|
+
self.calls.append(
|
|
438
|
+
(tuple(m["content"] for m in messages), add_generation_prompt, continue_final_message, tokenize)
|
|
439
|
+
)
|
|
440
|
+
return list(range(1, 2 * len(messages) + 1))
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def test_partition_asks_a_template_exactly_what_the_old_inline_code_asked():
|
|
444
|
+
"""The guarantee that no template-rendered model's pooled activations move.
|
|
445
|
+
|
|
446
|
+
Asserted on the *calls* rather than on one template's output, because that covers every
|
|
447
|
+
template rather than whichever ones happen to be cached: same message slices, same flags,
|
|
448
|
+
same tokenizing path means the same ids come back, whatever the template does with them.
|
|
449
|
+
"""
|
|
450
|
+
tokenizer = _RecordingTokenizer()
|
|
451
|
+
messages = [{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}]
|
|
452
|
+
|
|
453
|
+
full_ids, spans = Tokenize(tokenizer).message_partition(messages)
|
|
454
|
+
|
|
455
|
+
assert tokenizer.calls == [
|
|
456
|
+
(("a",), False, False, True),
|
|
457
|
+
(("a", "b"), False, False, True),
|
|
458
|
+
]
|
|
459
|
+
assert full_ids == [1, 2, 3, 4]
|
|
460
|
+
assert spans == [(0, 2), (2, 4)]
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def test_partition_equals_the_legacy_arithmetic_on_a_template():
|
|
464
|
+
tokenizer = _RecordingTokenizer()
|
|
465
|
+
messages = [{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}]
|
|
466
|
+
tok = Tokenize(tokenizer)
|
|
467
|
+
|
|
468
|
+
assert tok.message_partition(messages) == _legacy_partition(tok, messages)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def test_partition_of_no_messages_is_empty():
|
|
472
|
+
assert Tokenize(_RecordingTokenizer()).message_partition([]) == ([], [])
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def test_partition_covers_every_token_exactly_once(tok: Tokenize):
|
|
476
|
+
"""Contiguous, gapless, and ending on the last token — a partition, not a tagging.
|
|
477
|
+
|
|
478
|
+
Pooling reads `acts[start:end]` per message, so a gap drops activations from every mean and
|
|
479
|
+
an overlap double-counts them into two turns.
|
|
480
|
+
"""
|
|
481
|
+
full_ids, spans = tok.message_partition(MULTI_TURN)
|
|
482
|
+
|
|
483
|
+
assert len(spans) == len(MULTI_TURN)
|
|
484
|
+
assert spans[0][0] == 0
|
|
485
|
+
assert spans[-1][1] == len(full_ids)
|
|
486
|
+
assert all(before[1] == after[0] for before, after in zip(spans, spans[1:]))
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def test_partition_blocks_match_the_formatters_own_boundaries(tok: Tokenize, formatter: DeepseekV4Formatter):
|
|
490
|
+
"""Where a formatter renders, the boundaries are read off it rather than inferred.
|
|
491
|
+
|
|
492
|
+
Message 0 also carries the leading scaffold (the BOS token, in ``RenderedChat.prefix``),
|
|
493
|
+
which is where the prefix-delta arithmetic puts it too — there is no earlier message to
|
|
494
|
+
attribute it to, and leaving it unowned would break the partition.
|
|
495
|
+
"""
|
|
496
|
+
rendered = formatter.render(MULTI_TURN, add_generation_prompt=False)
|
|
497
|
+
_, spans = tok.message_partition(MULTI_TURN)
|
|
498
|
+
|
|
499
|
+
for index, (start, end) in enumerate(spans):
|
|
500
|
+
expected = rendered.blocks[index]
|
|
501
|
+
if index == 0:
|
|
502
|
+
expected = rendered.prefix + expected
|
|
503
|
+
assert end - start == len(tok.tokenizer.encode(expected)), f"message {index} span is not its own block"
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def test_the_legacy_arithmetic_would_misplace_a_code_rendered_conversation(tok: Tokenize):
|
|
507
|
+
"""Why this method exists rather than the caller keeping its inline loop.
|
|
508
|
+
|
|
509
|
+
The prefix delta assumes appending a message only appends tokens. DeepSeek-V4 rewrites
|
|
510
|
+
earlier turns once a later user turn exists (a turn stops being the *last* user turn), so
|
|
511
|
+
the deltas land in the wrong places — and the failure is silent, since they still look like
|
|
512
|
+
a partition.
|
|
513
|
+
"""
|
|
514
|
+
assert tok.message_partition(MULTI_TURN) != _legacy_partition(tok, MULTI_TURN)
|
|
515
|
+
|
|
516
|
+
|
|
396
517
|
# --------------------------------------------------------------------------- #
|
|
397
518
|
# The real encoder, from the checkpoint
|
|
398
519
|
# --------------------------------------------------------------------------- #
|
|
@@ -186,3 +186,34 @@ def test_reasoning_markers_present_in_tokenizer_vocab():
|
|
|
186
186
|
"""
|
|
187
187
|
model = _load(QWEN_THINKING)
|
|
188
188
|
assert detect_reasoning_tags(model.tokenizer.get_added_vocab().keys()) is not None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@pytest.mark.parametrize("spec", CHAT_PARAMS)
|
|
192
|
+
def test_message_partition_matches_the_prefix_delta_it_replaced(spec: ModelSpec):
|
|
193
|
+
"""``message_partition`` must not move a template-rendered model's message boundaries.
|
|
194
|
+
|
|
195
|
+
Callers that mean-pool activations per turn (the persona / assistant-axis path) used to run
|
|
196
|
+
this arithmetic inline against the tokenizer. The spans index the pooling, so an edge that
|
|
197
|
+
shifts by one token quietly changes a published projection — on every deployed instruct
|
|
198
|
+
model, none of which needed the change. The generalisation to every template is asserted on
|
|
199
|
+
the render calls in ``test_chat_formatters.py``; this is the same claim against a real
|
|
200
|
+
template and a real BPE vocabulary.
|
|
201
|
+
"""
|
|
202
|
+
model = _load(spec)
|
|
203
|
+
messages = [
|
|
204
|
+
{"role": "user", "content": "What is 2+2?"},
|
|
205
|
+
{"role": "assistant", "content": "Four."},
|
|
206
|
+
{"role": "user", "content": "And 3+3?"},
|
|
207
|
+
]
|
|
208
|
+
|
|
209
|
+
spans: list[tuple[int, int]] = []
|
|
210
|
+
previous = 0
|
|
211
|
+
reference_ids: list[int] = []
|
|
212
|
+
for count in range(1, len(messages) + 1):
|
|
213
|
+
reference_ids = _coerce_ids(
|
|
214
|
+
model.tokenizer.apply_chat_template(messages[:count], tokenize=True, add_generation_prompt=False)
|
|
215
|
+
)
|
|
216
|
+
spans.append((previous, len(reference_ids)))
|
|
217
|
+
previous = len(reference_ids)
|
|
218
|
+
|
|
219
|
+
assert model.tok.message_partition(messages) == (reference_ids, spans)
|
|
@@ -8,6 +8,7 @@ import pytest
|
|
|
8
8
|
import torch
|
|
9
9
|
|
|
10
10
|
from interp_engine.address import Address
|
|
11
|
+
from interp_engine.points import steer_refusal_reason
|
|
11
12
|
from interp_engine.vllm_capture.freeze import (
|
|
12
13
|
BREAKABLE_ENV,
|
|
13
14
|
FREEZE_SKIP_ABSENT_ENV,
|
|
@@ -59,21 +60,22 @@ def test_empty_list_is_graphs_with_no_taps():
|
|
|
59
60
|
def test_auto_is_resid_post_at_every_layer():
|
|
60
61
|
reads, writes, graph = resolve_freeze_points("auto", n_layers=3, n_streams=1)
|
|
61
62
|
assert graph is True
|
|
62
|
-
assert writes == []
|
|
63
63
|
assert reads == [Address("resid_post", 0), Address("resid_post", 1), Address("resid_post", 2)]
|
|
64
|
+
assert writes == reads
|
|
64
65
|
|
|
65
66
|
|
|
66
67
|
def test_enforce_eager_false_with_no_list_is_auto():
|
|
67
|
-
reads,
|
|
68
|
+
reads, writes, graph = resolve_freeze_points(None, n_layers=2, n_streams=1, enforce_eager=False)
|
|
68
69
|
assert graph is True
|
|
69
70
|
assert reads == [Address("resid_post", 0), Address("resid_post", 1)]
|
|
71
|
+
assert writes == reads, "the same set either way in; auto is auto however it was reached"
|
|
70
72
|
|
|
71
73
|
|
|
72
74
|
def test_auto_is_resid_streams_on_a_hyper_connection_trunk():
|
|
73
75
|
reads, writes, graph = resolve_freeze_points("auto", n_layers=3, n_streams=4)
|
|
74
76
|
assert graph is True
|
|
75
|
-
assert writes == []
|
|
76
77
|
assert reads == [Address("resid_streams", 0), Address("resid_streams", 1), Address("resid_streams", 2)]
|
|
78
|
+
assert writes == reads
|
|
77
79
|
|
|
78
80
|
|
|
79
81
|
def test_resid_streams_can_be_frozen():
|
|
@@ -105,11 +107,69 @@ def test_explicit_mlp_out_is_allowed_on_a_hyper_connection_trunk():
|
|
|
105
107
|
assert reads == [Address("mlp_out", 21)]
|
|
106
108
|
|
|
107
109
|
|
|
108
|
-
def
|
|
109
|
-
|
|
110
|
+
def test_auto_writes_where_it_reads_so_a_read_out_can_be_intervened_on():
|
|
111
|
+
"""The point of the change: an auto engine can steer at the addresses it captures.
|
|
112
|
+
|
|
113
|
+
Reads alone made the two halves of the residual endpoints disagree -- a lens read at layer 7
|
|
114
|
+
came back, and the steer, ablation or swap that read implies was refused for want of a site at
|
|
115
|
+
``resid_post.7``, which was already tapped a few bytes away.
|
|
116
|
+
"""
|
|
117
|
+
reads, writes, _ = resolve_freeze_points("auto", n_layers=2, n_streams=1)
|
|
118
|
+
assert writes == reads
|
|
119
|
+
assert writes == [Address("resid_post", 0), Address("resid_post", 1)]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_an_explicit_read_list_still_implies_no_writes():
|
|
123
|
+
"""Auto is a default, not a rewrite rule. A caller who named the points named all of them."""
|
|
124
|
+
reads, writes, _ = resolve_freeze_points([Address("resid_post", 1)], n_layers=4, n_streams=1)
|
|
125
|
+
assert reads == [Address("resid_post", 1)]
|
|
110
126
|
assert writes == []
|
|
111
127
|
|
|
112
128
|
|
|
129
|
+
def test_an_empty_write_list_asks_auto_for_the_reads_without_the_write_buffers():
|
|
130
|
+
"""``[]`` and None have to part company here, and nowhere else in this signature.
|
|
131
|
+
|
|
132
|
+
This is the opt-out for the memory: write buffers are per layer and per token, so on a pod
|
|
133
|
+
where they would step ``max_num_batched_tokens`` down, a caller who only ever reads can say so.
|
|
134
|
+
"""
|
|
135
|
+
reads, writes, graph = resolve_freeze_points("auto", n_layers=2, n_streams=1, freeze_writes=[])
|
|
136
|
+
assert graph is True
|
|
137
|
+
assert reads == [Address("resid_post", 0), Address("resid_post", 1)]
|
|
138
|
+
assert writes == []
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_an_empty_write_list_alone_is_still_not_a_reason_to_capture_graphs():
|
|
142
|
+
"""Distinguishing ``[]`` from None must not turn ``freeze_writes=[]`` into a freeze request."""
|
|
143
|
+
reads, writes, graph = resolve_freeze_points(None, n_layers=2, n_streams=1, freeze_writes=[])
|
|
144
|
+
assert (reads, writes, graph) == ([], [], False)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_a_named_write_is_not_joined_by_reads_it_did_not_ask_for():
|
|
148
|
+
reads, writes, graph = resolve_freeze_points(
|
|
149
|
+
None, n_layers=4, n_streams=1, freeze_writes=[Address("resid_post", 1)], enforce_eager=False
|
|
150
|
+
)
|
|
151
|
+
assert graph is True
|
|
152
|
+
assert reads == []
|
|
153
|
+
assert writes == [Address("resid_post", 1)]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_a_generation_only_engine_gains_neither_half():
|
|
157
|
+
"""``[]`` is graphs with no taps at all, and the auto write must not creep into it."""
|
|
158
|
+
reads, writes, graph = resolve_freeze_points([], n_layers=12, n_streams=1)
|
|
159
|
+
assert (reads, writes, graph) == ([], [], True)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_the_auto_writes_on_a_stacked_trunk_are_ones_the_engine_agrees_are_writable():
|
|
163
|
+
"""Auto now hands ``resid_streams`` to the write validation below, which used to see only reads.
|
|
164
|
+
|
|
165
|
+
A write set built by default has to clear the same bar as one a caller typed, or DeepSeek-V4
|
|
166
|
+
would refuse to load with `cannot freeze-write` from a set nobody asked for.
|
|
167
|
+
"""
|
|
168
|
+
_, writes, _ = resolve_freeze_points("auto", n_layers=3, n_streams=4)
|
|
169
|
+
assert {a.name for a in writes} == {"resid_streams"}
|
|
170
|
+
assert all(steer_refusal_reason(a.name) is None for a in writes)
|
|
171
|
+
|
|
172
|
+
|
|
113
173
|
def test_freeze_writes_without_reads_is_still_graph_mode():
|
|
114
174
|
reads, writes, graph = resolve_freeze_points(
|
|
115
175
|
None, n_layers=4, n_streams=1, freeze_writes=[Address("resid_post", 1)]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__eager.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/deepseek-v4-flash-0731__vllm.json
RENAMED
|
File without changes
|
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/gemma-2-2b__vllm-cudagraph.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/llama-3.1-8b__vllm-cudagraph.json
RENAMED
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/llama-3.1-8b__vllm-freeze.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3-4b__vllm-cudagraph.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3.8-27b__vllm-cudagraph.json
RENAMED
|
File without changes
|
{interp_engine-1.2.5 → interp_engine-1.2.7}/benchmarks/results/qwen3.8-27b__vllm-freeze.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|