hulder 0.1.0__tar.gz

hulder-0.1.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

hulder-0.1.0/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: hulder
+Version: 0.1.0
+Summary: Release gate for thresholded scores, such as activation probes, across LLM serving configs
+Keywords: activation probes,llm,vllm,quantization,release gate,mlops,monitoring
+Author: Utku Özbek
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Dist: numpy>=2.3
+Requires-Dist: scipy>=1.16
+Requires-Dist: scikit-learn>=1.7 ; extra == 'train'
+Requires-Dist: vllm==0.30.0 ; extra == 'vllm'
+Requires-Dist: pyyaml>=6.0.3 ; extra == 'vllm'
+Requires-Dist: huggingface-hub>=1.33.0 ; extra == 'vllm'
+Requires-Python: >=3.12
+Project-URL: Source, https://github.com/revtheundead/hulder
+Project-URL: Issues, https://github.com/revtheundead/hulder/issues
+Provides-Extra: train
+Provides-Extra: vllm
+Description-Content-Type: text/markdown
+
+# Hulder
+
+[![CI](https://github.com/revtheundead/hulder/actions/workflows/ci.yml/badge.svg)](https://github.com/revtheundead/hulder/actions/workflows/ci.yml)
+
+A release gate for activation probes, and any other thresholded scorer, across LLM serving configs.
+
+Teams tune a probe's threshold on one serving config, then change it: FP8 or AWQ weights, an FP8 KV cache, a vLLM upgrade. The hidden states move, and the threshold silently stops giving its target false-positive rate. `hulder check` compares a reference and a candidate on the same labelled examples and returns PASS, THRESHOLD STALE, SCORER DEGRADED or INCONCLUSIVE, with confidence intervals and a CI exit code.
+
+## Install
+
+```bash
+pip install hulder
+```
+
+The base install is numpy and scipy only and runs on CPU. Capturing activations from vLLM needs the `vllm` extra (`pip install "hulder[vllm]"`), which adds `hulder.capture`, experimental in v0.1.
+
+## Quickstart
+
+```bash
+hulder quickstart
+```
+
+It runs two checks on bundled data with a demo probe, under a minute on CPU. An fp16 rerun as candidate gives PASS. The official AWQ int4 checkpoint of Qwen2.5-1.5B-Instruct gives INCONCLUSIVE, with the shift flagged against the rerun's noise floor. `hulder check --help` lists the options.
+
+v0.1 is an early release. It has run only on a Turing GPU in fp16 and AWQ, and its tolerance bands are fixed.
+
+## More
+
+Design, statistical method, results and the cases where it loses: https://github.com/revtheundead/hulder
+
+Apache-2.0. The bundled demo data is derived from a CC-BY-4.0 dataset, attributed in `hulder/data/NOTICE.md`.

hulder-0.1.0/docs/pypi.md

@@ -0,0 +1,31 @@
+# Hulder
+
+[![CI](https://github.com/revtheundead/hulder/actions/workflows/ci.yml/badge.svg)](https://github.com/revtheundead/hulder/actions/workflows/ci.yml)
+
+A release gate for activation probes, and any other thresholded scorer, across LLM serving configs.
+
+Teams tune a probe's threshold on one serving config, then change it: FP8 or AWQ weights, an FP8 KV cache, a vLLM upgrade. The hidden states move, and the threshold silently stops giving its target false-positive rate. `hulder check` compares a reference and a candidate on the same labelled examples and returns PASS, THRESHOLD STALE, SCORER DEGRADED or INCONCLUSIVE, with confidence intervals and a CI exit code.
+
+## Install
+
+```bash
+pip install hulder
+```
+
+The base install is numpy and scipy only and runs on CPU. Capturing activations from vLLM needs the `vllm` extra (`pip install "hulder[vllm]"`), which adds `hulder.capture`, experimental in v0.1.
+
+## Quickstart
+
+```bash
+hulder quickstart
+```
+
+It runs two checks on bundled data with a demo probe, under a minute on CPU. An fp16 rerun as candidate gives PASS. The official AWQ int4 checkpoint of Qwen2.5-1.5B-Instruct gives INCONCLUSIVE, with the shift flagged against the rerun's noise floor. `hulder check --help` lists the options.
+
+v0.1 is an early release. It has run only on a Turing GPU in fp16 and AWQ, and its tolerance bands are fixed.
+
+## More
+
+Design, statistical method, results and the cases where it loses: https://github.com/revtheundead/hulder
+
+Apache-2.0. The bundled demo data is derived from a CC-BY-4.0 dataset, attributed in `hulder/data/NOTICE.md`.

hulder-0.1.0/pyproject.toml

@@ -0,0 +1,74 @@
+[project]
+name = "hulder"
+version = "0.1.0"
+description = "Release gate for thresholded scores, such as activation probes, across LLM serving configs"
+readme = "docs/pypi.md"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+keywords = [
+    "activation probes",
+    "llm",
+    "vllm",
+    "quantization",
+    "release gate",
+    "mlops",
+    "monitoring",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = [
+    "numpy>=2.3",
+    "scipy>=1.16",
+]
+
+[[project.authors]]
+name = "Utku Özbek"
+
+[project.urls]
+Source = "https://github.com/revtheundead/hulder"
+Issues = "https://github.com/revtheundead/hulder/issues"
+
+[project.optional-dependencies]
+vllm = [
+    "vllm==0.30.0",
+    "pyyaml>=6.0.3",
+    "huggingface-hub>=1.33.0",
+]
+train = ["scikit-learn>=1.7"]
+
+[project.scripts]
+hulder = "hulder.cli:main"
+
+[dependency-groups]
+dev = [
+    { include-group = "test" },
+    "hulder[vllm,train]",
+    "pandas>=2.3",
+    "pyarrow>=21",
+    "huggingface-hub",
+    "torch",
+]
+test = [
+    "pytest>=8",
+    "ruff==0.16.10",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.21,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.uv]
+python-preference = "only-managed"
+python-downloads = "manual"
+
+[tool.ruff]
+line-length = 120

hulder-0.1.0/pyproject.toml.orig

@@ -0,0 +1,69 @@
+[project]
+name = "hulder"
+version = "0.1.0"
+description = "Release gate for thresholded scores, such as activation probes, across LLM serving configs"
+readme = "docs/pypi.md"
+authors = [
+    { name = "Utku Özbek" }
+]
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+keywords = ["activation probes", "llm", "vllm", "quantization", "release gate", "mlops", "monitoring"]
+# No License :: classifier: PEP 639 forbids one alongside the license expression above.
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Quality Assurance",
+]
+# Base install: the gate library and CLI, numpy and scipy only (docs/design.md §4).
+dependencies = [
+    "numpy>=2.3",
+    "scipy>=1.16",
+]
+
+[project.urls]
+Source = "https://github.com/revtheundead/hulder"
+Issues = "https://github.com/revtheundead/hulder/issues"
+
+[project.optional-dependencies]
+# hulder.capture imports pyyaml and huggingface_hub itself; the floors are the versions
+# uv.lock resolves under the vllm pin.
+vllm = ["vllm==0.30.0", "pyyaml>=6.0.3", "huggingface-hub>=1.33.0"]
+train = ["scikit-learn>=1.7"]
+
+[dependency-groups]
+# Default for `uv run`/`uv sync`, so local work keeps both extras installed.
+dev = [
+    {include-group = "test"},
+    "hulder[vllm,train]",
+    "pandas>=2.3",
+    "pyarrow>=21",
+    # Imported directly by experiments/; versions come from the vllm pin via uv.lock.
+    "huggingface-hub",
+    "torch",
+]
+# The fast suite and lint on the base install, as CI runs them.
+test = ["pytest>=8", "ruff==0.16.10"]
+
+[project.scripts]
+hulder = "hulder.cli:main"
+
+[build-system]
+requires = ["uv_build>=0.12.21,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.uv]
+# Managed interpreter only (it ships Python.h, which Triton's JIT needs), never
+# downloaded implicitly. scripts/dev-env.sh points UV_PYTHON_INSTALL_DIR into the repo.
+python-preference = "only-managed"
+python-downloads = "manual"
+
+[tool.ruff]
+# Matches the existing code; most lines stay within 120.
+line-length = 120

hulder-0.1.0/src/hulder/__init__.py

@@ -0,0 +1,21 @@
+from importlib.metadata import version
+
+from hulder.gate.check import Estimate, Report, Verdict, check, check_snapshots
+from hulder.gate.pairing import PairingError, prompt_ids_sha256
+from hulder.gate.probe import Probe, apply_probe
+from hulder.gate.snapshot import Snapshot
+
+__all__ = [
+    "Estimate",
+    "PairingError",
+    "Probe",
+    "Report",
+    "Snapshot",
+    "Verdict",
+    "apply_probe",
+    "check",
+    "check_snapshots",
+    "prompt_ids_sha256",
+]
+
+__version__ = version("hulder")

hulder-0.1.0/src/hulder/capture/__init__.py

@@ -0,0 +1,21 @@
+"""Activation capture from vLLM, the hulder[vllm] extra (docs/design.md §5). Public but experimental
+in v0.1: these names may change in any 0.x release (docs/design.md §4).
+
+capture, CaptureConfig and Capture import vLLM and torch, so they load on first use. The
+other names, and importing this package, need neither.
+"""
+
+import importlib
+
+from hulder.capture.base_model import base_model_for
+from hulder.capture.metadata import snapshot_metadata
+
+__all__ = ["Capture", "CaptureConfig", "base_model_for", "capture", "snapshot_metadata"]
+
+_NEEDS_VLLM = ("Capture", "CaptureConfig", "capture")
+
+
+def __getattr__(name: str):
+    if name in _NEEDS_VLLM:
+        return getattr(importlib.import_module("hulder.capture.hidden_states"), name)
+    raise AttributeError(f"module 'hulder.capture' has no attribute {name!r}")

hulder-0.1.0/src/hulder/capture/base_model.py

@@ -0,0 +1,43 @@
+"""The base_model a capture records, which check requires to match across runs.
+
+Kept apart from hidden_states.py so it imports without vLLM.
+"""
+
+import re
+from collections.abc import Callable
+from pathlib import Path
+
+
+def card_base_model(model: str, revision: str) -> str | None:
+    """The base_model field of a model card's YAML front matter at a pinned revision."""
+    # Imported here so base_model_for, and its tests, run on the base install.
+    import yaml
+    from huggingface_hub import hf_hub_download
+
+    text = Path(hf_hub_download(model, "README.md", revision=revision)).read_text()
+    front = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
+    value = yaml.safe_load(front.group(1)).get("base_model") if front else None
+    return value if isinstance(value, str) else None
+
+
+def base_model_for(
+    model: str,
+    revision: str,
+    quantization: str | None,
+    override: str | None = None,
+    card: Callable[[str, str], str | None] = card_base_model,
+) -> tuple[str | None, str | None]:
+    """(base_model, warning). Unquantized, the model is its own base: a card's base_model
+    names the parent it was fine-tuned from, not what it re-encodes. Quantized, the override
+    or else the card's field; with neither, None and a warning rather than a guess."""
+    if override is not None:
+        return override, None
+    if quantization is None:
+        return model, None
+    base_model = card(model, revision)
+    if base_model is None:
+        return None, (
+            f"{model}@{revision} is quantized but its model card names no single base_model; "
+            "pass the base model explicitly so check can confirm both runs serve the same model."
+        )
+    return base_model, None

hulder-0.1.0/src/hulder/capture/hidden_states.py

@@ -0,0 +1,106 @@
+"""Activation capture from vLLM, the hulder[vllm] extra (docs/design.md §4 and §5).
+
+Offline extraction through vLLM's hidden-state connector, prefix caching off, chunked
+prefill off, fixed prompt order and batch size. Never imported by the base package.
+Any script calling `capture` needs an `if __name__ == "__main__":` guard (spawn on WSL).
+"""
+
+import os
+import tempfile
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+from vllm import LLM, SamplingParams
+from vllm.config.kv_transfer import KVTransferConfig
+from vllm.distributed.kv_transfer.kv_connector.v1 import example_hidden_states_connector
+
+
+@dataclass(frozen=True)
+class CaptureConfig:
+    """Everything that fixes the captured values. Recorded verbatim in the run manifest."""
+
+    model: str
+    revision: str
+    layers: tuple[int, ...]  # id k = residual stream after k decoder layers, 0 = embeddings
+    positions: tuple[int, ...]  # negative indices into the templated prompt
+    system_prompt: str
+    quantization: str | None = None
+    dtype: str = "float16"
+    batch_size: int = 8
+    # Fits Qwen2.5-1.5B on an 8 GB RTX 2070 with extraction on (measured 2026-10-02).
+    gpu_memory_utilization: float = 0.7
+    max_model_len: int = 1024
+    max_num_batched_tokens: int = 1024
+    seed: int = 0
+    # Passed to the chat template when set. Pinned for hybrid-thinking models such as Qwen3,
+    # whose templated prompt, and so its last token, depends on it. None leaves the template's default.
+    enable_thinking: bool | None = None
+    # Required off for reproducible snapshots. Fields so the manifest records them.
+    enable_prefix_caching: bool = False
+    enable_chunked_prefill: bool = False
+
+    def __post_init__(self) -> None:
+        if self.enable_prefix_caching or self.enable_chunked_prefill:
+            raise ValueError("capture needs prefix caching and chunked prefill off (docs/design.md §5)")
+
+
+@dataclass(frozen=True)
+class Capture:
+    activations: np.ndarray  # (n, layers, positions, d), float16, prompt order preserved
+    position_tokens: list[list[str]]  # token string at each requested position, per prompt
+    # torch.cuda.get_device_name() of the current device, the first visible one. Under tensor
+    # parallelism the other GPUs aren't recorded, a known limit. Another GPU is another serving
+    # config (docs/design.md §6, metadata rules).
+    gpu: str
+    enable_thinking: bool | None  # as passed to the chat template, None for its default
+
+
+def capture(prompts: list[str], cfg: CaptureConfig) -> Capture:
+    convs = [[{"role": "system", "content": cfg.system_prompt}, {"role": "user", "content": p}] for p in prompts]
+    params = SamplingParams(max_tokens=1, temperature=0)
+    template = {} if cfg.enable_thinking is None else {"chat_template_kwargs": {"enable_thinking": cfg.enable_thinking}}
+    acts, tokens = [], []
+    with tempfile.TemporaryDirectory() as tmp:
+        llm = LLM(
+            model=cfg.model,
+            revision=cfg.revision,
+            quantization=cfg.quantization,
+            dtype=cfg.dtype,
+            gpu_memory_utilization=cfg.gpu_memory_utilization,
+            max_model_len=cfg.max_model_len,
+            max_num_batched_tokens=cfg.max_num_batched_tokens,
+            max_num_seqs=cfg.batch_size,
+            seed=cfg.seed,
+            enable_chunked_prefill=cfg.enable_chunked_prefill,
+            enable_prefix_caching=cfg.enable_prefix_caching,
+            speculative_config={
+                "method": "extract_hidden_states",
+                "num_speculative_tokens": 1,
+                "draft_model_config": {"hf_config": {"eagle_aux_hidden_state_layer_ids": list(cfg.layers)}},
+            },
+            kv_transfer_config=KVTransferConfig(
+                kv_connector="ExampleHiddenStatesConnector",
+                kv_role="kv_producer",
+                kv_connector_extra_config={"shared_storage_path": tmp},
+            ),
+        )
+        tok = llm.get_tokenizer()
+        pos = list(cfg.positions)
+        for start in range(0, len(convs), cfg.batch_size):
+            for out in llm.chat(convs[start : start + cfg.batch_size], params, use_tqdm=False, **template):
+                path = out.kv_transfer_params["hidden_states_path"]
+                obj = example_hidden_states_connector.load_hidden_states(path)
+                assert obj["token_ids"].tolist() == list(out.prompt_token_ids)
+                # hidden_states is [tokens, layers, d]; keep only the requested positions.
+                acts.append(obj["hidden_states"][pos].transpose(0, 1).to("cpu").numpy().astype(np.float16))
+                tokens.append(tok.convert_ids_to_tokens(obj["token_ids"][pos].tolist()))
+                os.remove(path)
+            if start // cfg.batch_size % 100 == 0:
+                print(f"captured {min(start + cfg.batch_size, len(convs))}/{len(convs)}", flush=True)
+    return Capture(
+        activations=np.stack(acts),
+        position_tokens=tokens,
+        gpu=torch.cuda.get_device_name(),
+        enable_thinking=cfg.enable_thinking,
+    )