embedxray 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
embedxray/__init__.py ADDED
@@ -0,0 +1,46 @@
1
+ """embedxray: Diagnostic and evaluation library for sentence-transformers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from embedxray.evaluator import EmbeddingXRayEvaluator
6
+ from embedxray.modules import (
7
+ compute_anisotropy,
8
+ compute_dimension_health,
9
+ compute_hubness,
10
+ compute_pair_collisions,
11
+ compute_token_attribution,
12
+ evaluate_fracture,
13
+ )
14
+ from embedxray.report import XRayReport, compute_composite_health_score
15
+ from embedxray.utils import (
16
+ binarize_embeddings,
17
+ chunked_topk_binary,
18
+ chunked_topk_cosine,
19
+ safe_l2_normalize,
20
+ safe_svdvals,
21
+ simulate_fp16,
22
+ simulate_int4,
23
+ simulate_int8,
24
+ )
25
+
26
+ __version__ = "0.1.1"
27
+
28
+ __all__ = [
29
+ "EmbeddingXRayEvaluator",
30
+ "XRayReport",
31
+ "compute_composite_health_score",
32
+ "compute_token_attribution",
33
+ "compute_pair_collisions",
34
+ "compute_anisotropy",
35
+ "compute_dimension_health",
36
+ "compute_hubness",
37
+ "evaluate_fracture",
38
+ "safe_l2_normalize",
39
+ "safe_svdvals",
40
+ "chunked_topk_cosine",
41
+ "simulate_fp16",
42
+ "simulate_int8",
43
+ "simulate_int4",
44
+ "binarize_embeddings",
45
+ "chunked_topk_binary",
46
+ ]
embedxray/evaluator.py ADDED
@@ -0,0 +1,250 @@
1
+ """SentenceEvaluator integration for embedxray."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ import torch
9
+
10
+ try:
11
+ from sentence_transformers.sentence_transformer.evaluation import SentenceEvaluator
12
+ except ImportError:
13
+ try:
14
+ from sentence_transformers.evaluation import SentenceEvaluator
15
+ except ImportError:
16
+ from sentence_transformers.base.evaluation import BaseEvaluator as SentenceEvaluator
17
+
18
+ from embedxray.modules.attribution import (
19
+ DEFAULT_BOILERPLATE_PATTERN,
20
+ compute_pair_collisions,
21
+ compute_token_attribution,
22
+ )
23
+ from embedxray.modules.fracture import evaluate_fracture
24
+ from embedxray.modules.geometry import (
25
+ compute_anisotropy,
26
+ compute_dimension_health,
27
+ compute_hubness,
28
+ )
29
+ from embedxray.report import XRayReport, compute_composite_health_score
30
+ from embedxray.utils.model import encode_sentences, preserve_training_mode
31
+
32
+
33
+ class EmbeddingXRayEvaluator(SentenceEvaluator):
34
+ """Read-only diagnostic evaluator for sentence-transformers models."""
35
+
36
+ def __init__(
37
+ self,
38
+ corpus_texts: list[str] | None = None,
39
+ query_texts: list[str] | None = None,
40
+ contrast_pairs: list[tuple[str, str]] | None = None,
41
+ attribution_texts: list[str] | None = None,
42
+ matryoshka_dims: list[int] | None = None,
43
+ quantization_modes: tuple[str, ...] = ("fp16", "int8", "int4", "binary"),
44
+ k: int = 10,
45
+ batch_size: int = 32,
46
+ chunk_size: int = 2048,
47
+ max_anisotropy_pairs: int = 50000,
48
+ cliff_threshold: float = 0.95,
49
+ collision_threshold: float = 0.90,
50
+ boilerplate_pattern: str = DEFAULT_BOILERPLATE_PATTERN,
51
+ name: str = "",
52
+ write_csv: bool = True,
53
+ write_report: bool = True,
54
+ seed: int = 42,
55
+ *,
56
+ corpus: list[str] | None = None,
57
+ queries: list[str] | None = None,
58
+ ) -> None:
59
+ super().__init__()
60
+ resolved_corpus = corpus_texts if corpus_texts is not None else corpus
61
+ resolved_queries = query_texts if query_texts is not None else queries
62
+
63
+ if not resolved_corpus:
64
+ raise ValueError("corpus_texts (or corpus) must contain at least one text sample.")
65
+ if int(batch_size) <= 0:
66
+ raise ValueError("batch_size must be a positive integer.")
67
+
68
+ self.corpus_texts = list(resolved_corpus)
69
+ self.corpus = self.corpus_texts
70
+ self.query_texts = list(resolved_queries) if resolved_queries is not None else None
71
+ self.queries = self.query_texts
72
+ self.contrast_pairs = list(contrast_pairs) if contrast_pairs is not None else None
73
+ self.attribution_texts = (
74
+ list(attribution_texts)
75
+ if attribution_texts is not None
76
+ else self.corpus_texts[:64]
77
+ )
78
+ self.matryoshka_dims = list(matryoshka_dims) if matryoshka_dims is not None else None
79
+ self.quantization_modes = tuple(quantization_modes)
80
+ self.k = int(k)
81
+ self.batch_size = int(batch_size)
82
+ self.chunk_size = int(chunk_size)
83
+ self.max_anisotropy_pairs = int(max_anisotropy_pairs)
84
+ self.cliff_threshold = float(cliff_threshold)
85
+ self.collision_threshold = float(collision_threshold)
86
+ self.boilerplate_pattern = boilerplate_pattern
87
+ self.name = name
88
+ self.write_csv = bool(write_csv)
89
+ self.write_report = bool(write_report)
90
+ self.seed = int(seed)
91
+
92
+ self.greater_is_better = True
93
+ self.primary_metric = (
94
+ f"{self.name}_xray_health_score" if self.name else "xray_health_score"
95
+ )
96
+ self.csv_file = (
97
+ f"xray_evaluation_{self.name}_results.csv"
98
+ if self.name
99
+ else "xray_evaluation_results.csv"
100
+ )
101
+ self.last_report: XRayReport | None = None
102
+
103
+ def _prefix_metrics(self, base_metrics: dict[str, float]) -> dict[str, float]:
104
+ """Applies evaluator name prefix using SentenceEvaluator conventions."""
105
+ if hasattr(self, "prefix_name_to_metrics") and callable(self.prefix_name_to_metrics):
106
+ return self.prefix_name_to_metrics(base_metrics, self.name)
107
+ if not self.name:
108
+ return dict(base_metrics)
109
+ return {f"{self.name}_{key}": val for key, val in base_metrics.items()}
110
+
111
+ def evaluate_report(
112
+ self,
113
+ model: Any,
114
+ output_path: str | None = None,
115
+ epoch: int = -1,
116
+ steps: int = -1,
117
+ ) -> XRayReport:
118
+ """Runs the full read-only X-Ray diagnostic suite and returns a structured XRayReport."""
119
+ with preserve_training_mode(model), torch.no_grad():
120
+ corpus_emb = encode_sentences(model, self.corpus_texts, batch_size=self.batch_size)
121
+ if self.query_texts is None:
122
+ query_emb = None
123
+ elif self.query_texts:
124
+ query_emb = encode_sentences(model, self.query_texts, batch_size=self.batch_size)
125
+ else:
126
+ query_emb = corpus_emb.new_empty((0, corpus_emb.size(1)))
127
+
128
+ anisotropy = compute_anisotropy(
129
+ corpus_emb,
130
+ max_pairs=self.max_anisotropy_pairs,
131
+ seed=self.seed,
132
+ )
133
+ dim_health = compute_dimension_health(corpus_emb)
134
+ hubness = compute_hubness(
135
+ corpus_embeddings=corpus_emb,
136
+ query_embeddings=query_emb,
137
+ k=self.k,
138
+ chunk_size=self.chunk_size,
139
+ )
140
+ fracture = evaluate_fracture(
141
+ corpus_embeddings=corpus_emb,
142
+ query_embeddings=query_emb,
143
+ matryoshka_dims=self.matryoshka_dims,
144
+ quantization_modes=self.quantization_modes,
145
+ k=self.k,
146
+ chunk_size=self.chunk_size,
147
+ cliff_threshold=self.cliff_threshold,
148
+ )
149
+ attribution = compute_token_attribution(
150
+ model=model,
151
+ texts=self.attribution_texts,
152
+ batch_size=self.batch_size,
153
+ boilerplate_pattern=self.boilerplate_pattern,
154
+ )
155
+ collisions = compute_pair_collisions(
156
+ model=model,
157
+ contrast_pairs=self.contrast_pairs or [],
158
+ batch_size=self.batch_size,
159
+ collision_threshold=self.collision_threshold,
160
+ )
161
+
162
+ health_score = compute_composite_health_score(
163
+ geometry=anisotropy,
164
+ dimension_health=dim_health,
165
+ hubness=hubness,
166
+ fracture=fracture,
167
+ attribution=attribution,
168
+ collisions=collisions if self.contrast_pairs else None,
169
+ )
170
+
171
+ raw_metrics: dict[str, float] = {
172
+ "xray_health_score": float(health_score),
173
+ "mean_pairwise_cosine": float(anisotropy["mean_pairwise_cosine"]),
174
+ "spectral_entropy": float(anisotropy["spectral_entropy"]),
175
+ "dead_dimension_count": float(dim_health["dead_dimension_count"]),
176
+ "dead_dimension_ratio": float(dim_health["dead_dimension_ratio"]),
177
+ "hubness_skewness": float(hubness["hubness_skewness"]),
178
+ "robin_hood_index": float(hubness["robin_hood_index"]),
179
+ "mean_contrast_similarity": float(collisions["mean_contrast_similarity"]),
180
+ "collision_rate": float(collisions["collision_rate"]),
181
+ "fracture_cliff_bytes": float(fracture["fracture_cliff_bytes"]),
182
+ "compression_ratio": float(fracture["compression_ratio"]),
183
+ }
184
+
185
+ for key in ("boilerplate_hijack_score", "max_token_dominance"):
186
+ if attribution[key] is not None:
187
+ raw_metrics[key] = float(attribution[key])
188
+
189
+ for entry in fracture["grid"]:
190
+ dim = entry["dim"]
191
+ mode = entry["mode"]
192
+ if entry["overlap_at_k"] is not None:
193
+ raw_metrics[f"overlap_at_{self.k}_d{dim}_{mode}"] = float(entry["overlap_at_k"])
194
+ if entry["self_similarity_drift"] is not None:
195
+ raw_metrics[f"drift_d{dim}_{mode}"] = float(entry["self_similarity_drift"])
196
+
197
+ prefixed_metrics = self._prefix_metrics(raw_metrics)
198
+
199
+ report = XRayReport(
200
+ health_score=float(health_score),
201
+ anisotropy=anisotropy,
202
+ dimension_health=dim_health,
203
+ hubness=hubness,
204
+ fracture=fracture,
205
+ attribution=attribution,
206
+ collisions=collisions,
207
+ metrics=prefixed_metrics,
208
+ metadata={
209
+ "name": self.name,
210
+ "epoch": epoch,
211
+ "steps": steps,
212
+ "num_corpus": int(corpus_emb.size(0)),
213
+ "embedding_dim": int(corpus_emb.size(1)),
214
+ "k": self.k,
215
+ "actual_k": fracture["actual_k"],
216
+ "num_queries": fracture["num_queries"],
217
+ },
218
+ )
219
+ self.last_report = report
220
+
221
+ if output_path is not None:
222
+ out_dir = Path(output_path)
223
+ out_dir.mkdir(parents=True, exist_ok=True)
224
+ if self.write_csv:
225
+ report.append_csv(out_dir / self.csv_file, epoch=epoch, steps=steps)
226
+ if self.write_report:
227
+ json_name = f"xray_report_{self.name}.json" if self.name else "xray_report.json"
228
+ md_name = f"xray_report_{self.name}.md" if self.name else "xray_report.md"
229
+ report.save_json(out_dir / json_name)
230
+ report.save_markdown(out_dir / md_name)
231
+
232
+ return report
233
+
234
+ def __call__(
235
+ self,
236
+ model: Any,
237
+ output_path: str | None = None,
238
+ epoch: int = -1,
239
+ steps: int = -1,
240
+ *args: Any,
241
+ **kwargs: Any,
242
+ ) -> dict[str, float]:
243
+ """Executes the X-Ray evaluation and returns a flat dictionary of metrics."""
244
+ report = self.evaluate_report(
245
+ model=model,
246
+ output_path=output_path,
247
+ epoch=epoch,
248
+ steps=steps,
249
+ )
250
+ return report.metrics
@@ -0,0 +1,25 @@
1
+ """Diagnostic modules for attribution, geometry, and fracture analysis."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from embedxray.modules.attribution import (
6
+ DEFAULT_BOILERPLATE_PATTERN,
7
+ compute_pair_collisions,
8
+ compute_token_attribution,
9
+ )
10
+ from embedxray.modules.fracture import evaluate_fracture
11
+ from embedxray.modules.geometry import (
12
+ compute_anisotropy,
13
+ compute_dimension_health,
14
+ compute_hubness,
15
+ )
16
+
17
+ __all__ = [
18
+ "DEFAULT_BOILERPLATE_PATTERN",
19
+ "compute_token_attribution",
20
+ "compute_pair_collisions",
21
+ "compute_anisotropy",
22
+ "compute_dimension_health",
23
+ "compute_hubness",
24
+ "evaluate_fracture",
25
+ ]
@@ -0,0 +1,333 @@
1
+ """Token and pooling directional attribution and contrast-pair collision diagnostics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Mapping
7
+ from typing import Any
8
+
9
+ import torch
10
+ from torch import nn
11
+
12
+ try:
13
+ from sentence_transformers.sentence_transformer.modules import Normalize, Pooling
14
+ except ImportError:
15
+ from sentence_transformers.models import Normalize, Pooling
16
+
17
+ from embedxray.utils.model import (
18
+ encode_sentences,
19
+ model_module,
20
+ preserve_training_mode,
21
+ to_tensor,
22
+ )
23
+ from embedxray.utils.tensor_ops import safe_l2_normalize
24
+
25
+ DEFAULT_BOILERPLATE_PATTERN = (
26
+ r"^(\[.*\]|\{.*\}|<.*>|#+|\*+|-+|_+|[0-9]{2,4}[-/:][0-9]{2}.*|[\W_]+)$"
27
+ )
28
+
29
+
30
+ def _model_tokenizer(model: Any) -> Any:
31
+ """Resolves a tokenizer exposed directly or through a model wrapper."""
32
+ tokenizer = getattr(model, "tokenizer", None)
33
+ if tokenizer is not None:
34
+ return tokenizer
35
+ return getattr(model_module(model), "tokenizer", None)
36
+
37
+
38
+ def _extract_tokens(model: Any, text: str, record: Mapping[str, Any]) -> list[str]:
39
+ """Resolves token labels from the encoded record or the model tokenizer."""
40
+ tokens = record.get("tokens")
41
+ if isinstance(tokens, (tuple, list)):
42
+ return [str(token) for token in tokens]
43
+
44
+ tokenizer = _model_tokenizer(model)
45
+ ids = record.get("input_ids")
46
+ if ids is not None and hasattr(tokenizer, "convert_ids_to_tokens"):
47
+ return tokenizer.convert_ids_to_tokens(to_tensor(ids).reshape(-1).tolist())
48
+
49
+ tokenize = getattr(tokenizer, "tokenize", None)
50
+ if callable(tokenize):
51
+ return list(tokenize(text))
52
+ if callable(tokenizer) and hasattr(tokenizer, "convert_ids_to_tokens"):
53
+ encoded = tokenizer(text, add_special_tokens=True)
54
+ if isinstance(encoded, Mapping) and "input_ids" in encoded:
55
+ ids = to_tensor(encoded["input_ids"]).reshape(-1).tolist()
56
+ return tokenizer.convert_ids_to_tokens(ids)
57
+ return text.split()
58
+
59
+
60
+ def _feature_records(model: Any, texts: list[str], batch_size: int) -> list[dict[str, Any]]:
61
+ """Reads per-sample features from supported encode output layouts."""
62
+ try:
63
+ raw = model.encode(
64
+ texts,
65
+ batch_size=batch_size,
66
+ output_value=None,
67
+ convert_to_tensor=True,
68
+ show_progress_bar=False,
69
+ )
70
+ except (TypeError, ValueError, NotImplementedError):
71
+ raw = None
72
+
73
+ if isinstance(raw, list) and len(raw) == len(texts):
74
+ return [dict(item) if isinstance(item, Mapping) else {} for item in raw]
75
+ if isinstance(raw, Mapping) and raw.get("token_embeddings") is not None:
76
+ tokens = to_tensor(raw["token_embeddings"])
77
+ if len(texts) == 1 and tokens.ndim == 2:
78
+ return [dict(raw)]
79
+ if tokens.ndim == 3 and tokens.size(0) == len(texts):
80
+ batched = {}
81
+ for key in ("sentence_embedding", "attention_mask", "input_ids", "tokens"):
82
+ value = raw.get(key)
83
+ if isinstance(value, (list, tuple)) or getattr(value, "ndim", 0) > 0:
84
+ if len(value) == len(texts):
85
+ batched[key] = value
86
+ return [
87
+ {
88
+ **{key: value[index] for key, value in batched.items()},
89
+ "token_embeddings": tokens[index],
90
+ }
91
+ for index in range(len(texts))
92
+ ]
93
+ return [{} for _ in texts]
94
+
95
+
96
+ def _hook_record(model: Any, text: str) -> dict[str, Any]:
97
+ """Captures real token features for one input, independent of encode sorting."""
98
+ root = model_module(model)
99
+ if root is None:
100
+ return {}
101
+
102
+ named: dict[str, Any] = {}
103
+ unnamed: dict[str, Any] = {}
104
+ handles: list[torch.utils.hooks.RemovableHandle] = []
105
+
106
+ def capture(_module: nn.Module, inputs: tuple[Any, ...], output: Any) -> None:
107
+ if isinstance(output, Mapping):
108
+ tensor = output.get("token_embeddings", output.get("last_hidden_state"))
109
+ elif isinstance(output, (list, tuple)) and output:
110
+ tensor = output[0]
111
+ else:
112
+ tensor = output
113
+ if not isinstance(tensor, torch.Tensor) or tensor.ndim != 3 or tensor.size(0) != 1:
114
+ return
115
+
116
+ record: dict[str, Any] = {"token_embeddings": tensor[0].detach()}
117
+ input_features = inputs[0] if inputs and isinstance(inputs[0], Mapping) else {}
118
+ output_features = output if isinstance(output, Mapping) else {}
119
+ for key in ("attention_mask", "input_ids"):
120
+ value = output_features.get(key, input_features.get(key))
121
+ if isinstance(value, torch.Tensor):
122
+ record[key] = value.reshape(-1).detach()
123
+ captured = named if isinstance(output, Mapping) and "token_embeddings" in output else unnamed
124
+ captured.clear()
125
+ captured.update(record)
126
+
127
+ try:
128
+ for module in root.modules():
129
+ handles.append(module.register_forward_hook(capture))
130
+ pooled = encode_sentences(model, [text], batch_size=1)[0]
131
+ finally:
132
+ for handle in handles:
133
+ handle.remove()
134
+
135
+ captured = named or unnamed
136
+ if captured:
137
+ captured["sentence_embedding"] = pooled
138
+ return captured
139
+
140
+
141
+ def _projection_direction(
142
+ model: Any, sentence: torch.Tensor, token_dim: int, eps: float
143
+ ) -> torch.Tensor | None:
144
+ """Maps a pooled direction into the token space for prefix and concatenated pooling."""
145
+ full_dim = token_dim
146
+ root = model_module(model)
147
+ if root is not None:
148
+ layers = list(root.children())
149
+ pool = next((layer for layer in reversed(layers) if isinstance(layer, Pooling)), None)
150
+ if pool is not None:
151
+ pool_dim = getattr(
152
+ pool, "embedding_dimension", getattr(pool, "word_embedding_dimension", None)
153
+ )
154
+ if pool_dim != token_dim or any(
155
+ not isinstance(layer, Normalize) for layer in layers[layers.index(pool) + 1 :]
156
+ ):
157
+ return None
158
+ full_dim = pool.pooling_output_dimension
159
+
160
+ sent_dim = sentence.numel()
161
+ truncate_dim = getattr(model, "truncate_dim", getattr(root, "truncate_dim", None))
162
+ if sent_dim != full_dim and not (0 < sent_dim < full_dim and sent_dim == truncate_dim):
163
+ return None
164
+
165
+ unit = sentence / (torch.linalg.vector_norm(sentence) + eps)
166
+ if full_dim == token_dim:
167
+ return unit
168
+ padded = torch.nn.functional.pad(unit, (0, full_dim - sent_dim))
169
+ return padded.reshape(-1, token_dim).sum(dim=0)
170
+
171
+
172
+ def _score_tokens(
173
+ model: Any,
174
+ text: str,
175
+ record: Mapping[str, Any],
176
+ boilerplate: re.Pattern[str],
177
+ special_tokens: set[str],
178
+ eps: float,
179
+ ) -> dict[str, Any]:
180
+ """Scores valid token features and records why a sample cannot be measured."""
181
+ sample: dict[str, Any] = {
182
+ "text": text,
183
+ "boilerplate_hijack_score": None,
184
+ "max_token_weight": None,
185
+ "tokens": [],
186
+ }
187
+ if record.get("token_embeddings") is None or record.get("sentence_embedding") is None:
188
+ return {**sample, "unavailable_reason": "Token or sentence embeddings are unavailable."}
189
+
190
+ embeddings = to_tensor(record["token_embeddings"]).to(torch.float32)
191
+ if embeddings.ndim == 3 and embeddings.size(0) == 1:
192
+ embeddings = embeddings.squeeze(0)
193
+ if embeddings.ndim != 2 or embeddings.size(1) == 0:
194
+ return {**sample, "unavailable_reason": "Token embedding shape is unsupported."}
195
+
196
+ raw_mask = record.get("attention_mask")
197
+ mask = (
198
+ to_tensor(raw_mask).reshape(-1).to(device=embeddings.device) == 1
199
+ if raw_mask is not None
200
+ else torch.ones(embeddings.size(0), dtype=torch.bool, device=embeddings.device)
201
+ )
202
+ if mask.numel() != embeddings.size(0):
203
+ reason = "Attention mask does not align with token embeddings."
204
+ return {**sample, "unavailable_reason": reason}
205
+ positions = mask.nonzero(as_tuple=True)[0]
206
+ if positions.numel() == 0:
207
+ return {**sample, "unavailable_reason": "No tokens are valid under the attention mask."}
208
+
209
+ tokens = _extract_tokens(model, text, record)
210
+ if len(tokens) == embeddings.size(0):
211
+ tokens = [tokens[index] for index in positions.tolist()]
212
+ elif len(tokens) != positions.numel():
213
+ return {**sample, "unavailable_reason": "Token labels do not align with token embeddings."}
214
+
215
+ sentence = to_tensor(record["sentence_embedding"]).reshape(-1).to(embeddings)
216
+ direction = _projection_direction(model, sentence, embeddings.size(1), eps)
217
+ if direction is None:
218
+ reason = "Token and sentence embedding spaces cannot be aligned."
219
+ return {**sample, "unavailable_reason": reason}
220
+
221
+ projections = torch.mv(embeddings[positions, : direction.numel()], direction)
222
+ positive = projections.clamp(min=0.0)
223
+ weights = positive / (positive.sum() + eps)
224
+
225
+ token_entries: list[dict[str, Any]] = []
226
+ hijack_score = 0.0
227
+ for token, projection, weight in zip(tokens, projections.tolist(), weights.tolist()):
228
+ cleaned_token = token.lstrip("Ġ▁")
229
+ is_boilerplate = bool(
230
+ token not in special_tokens
231
+ and (boilerplate.match(token) or boilerplate.match(cleaned_token))
232
+ )
233
+ if is_boilerplate:
234
+ hijack_score += weight
235
+ token_entries.append(
236
+ {
237
+ "token": token,
238
+ "directional_projection": projection,
239
+ "weight": weight,
240
+ "is_boilerplate": is_boilerplate,
241
+ }
242
+ )
243
+ return {
244
+ **sample,
245
+ "boilerplate_hijack_score": min(hijack_score, 1.0),
246
+ "max_token_weight": weights.max().item(),
247
+ "tokens": token_entries,
248
+ }
249
+
250
+
251
+ def compute_token_attribution(
252
+ model: Any,
253
+ texts: list[str],
254
+ batch_size: int = 32,
255
+ boilerplate_pattern: str = DEFAULT_BOILERPLATE_PATTERN,
256
+ eps: float = 1e-12,
257
+ ) -> dict[str, Any]:
258
+ """Computes directional token attribution on samples whose features can be aligned."""
259
+ if batch_size <= 0:
260
+ raise ValueError("batch_size must be a positive integer.")
261
+ boilerplate = re.compile(boilerplate_pattern)
262
+ tokenizer = _model_tokenizer(model)
263
+ special_tokens = set(getattr(tokenizer, "all_special_tokens", []) or [])
264
+ samples: list[dict[str, Any]] = []
265
+ with preserve_training_mode(model), torch.no_grad():
266
+ for start in range(0, len(texts), batch_size):
267
+ batch = texts[start : start + batch_size]
268
+ for text, record in zip(batch, _feature_records(model, batch, batch_size)):
269
+ if (
270
+ record.get("token_embeddings") is None
271
+ or record.get("sentence_embedding") is None
272
+ ):
273
+ record = _hook_record(model, text)
274
+ sample = _score_tokens(model, text, record, boilerplate, special_tokens, eps)
275
+ samples.append(sample)
276
+
277
+ evaluated = [sample for sample in samples if sample["boilerplate_hijack_score"] is not None]
278
+ count = len(evaluated)
279
+ return {
280
+ "boilerplate_hijack_score": (
281
+ sum(sample["boilerplate_hijack_score"] for sample in evaluated) / count
282
+ if count else None
283
+ ),
284
+ "max_token_dominance": (
285
+ sum(sample["max_token_weight"] for sample in evaluated) / count if count else None
286
+ ),
287
+ "evaluated_samples": count,
288
+ "samples": samples,
289
+ }
290
+
291
+
292
+ def compute_pair_collisions(
293
+ model: Any,
294
+ contrast_pairs: list[tuple[str, str]],
295
+ batch_size: int = 32,
296
+ collision_threshold: float = 0.90,
297
+ ) -> dict[str, Any]:
298
+ """Computes cosine similarity and collision rate across contrasting sentence pairs."""
299
+ if not contrast_pairs:
300
+ return {
301
+ "mean_contrast_similarity": 0.0,
302
+ "collision_rate": 0.0,
303
+ "offending_pair_indices": [],
304
+ "pair_similarities": [],
305
+ "offending_pairs": [],
306
+ }
307
+
308
+ left_texts = [pair[0] for pair in contrast_pairs]
309
+ right_texts = [pair[1] for pair in contrast_pairs]
310
+ with preserve_training_mode(model), torch.no_grad():
311
+ left_emb = safe_l2_normalize(encode_sentences(model, left_texts, batch_size))
312
+ right_emb = safe_l2_normalize(encode_sentences(model, right_texts, batch_size))
313
+ if left_emb.shape != right_emb.shape:
314
+ raise ValueError("Contrast pairs must have matching embedding dimensions.")
315
+ sim_list = (left_emb * right_emb).sum(dim=-1).clamp(-1.0, 1.0).tolist()
316
+ offending_indices = [index for index, sim in enumerate(sim_list) if sim > collision_threshold]
317
+ offending_pairs = [
318
+ {
319
+ "index": index,
320
+ "left": contrast_pairs[index][0],
321
+ "right": contrast_pairs[index][1],
322
+ "similarity": sim_list[index],
323
+ }
324
+ for index in offending_indices
325
+ ]
326
+
327
+ return {
328
+ "mean_contrast_similarity": sum(sim_list) / len(sim_list),
329
+ "collision_rate": len(offending_indices) / len(sim_list),
330
+ "offending_pair_indices": offending_indices,
331
+ "pair_similarities": sim_list,
332
+ "offending_pairs": offending_pairs,
333
+ }