entroptics-context 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,78 @@
1
+ """Reading a context that spans several sources as a subspace rather than a centre.
2
+
3
+ A context assembled from several documents — a retrieval-augmented prompt, an agent
4
+ scratchpad, a long conversation — has no single direction. Its centroid sits between its
5
+ sources, in a direction none of them occupies, and every guard built on that centroid gets
6
+ worse as the context spreads. The subspace the context resolves above its own noise floor
7
+ spans the sources instead, so legitimate material lies inside it however many topics it
8
+ covers and foreign material does not. Nothing is trained, no threshold is set, and no
9
+ constant is fitted per corpus.
10
+
11
+ Quick start
12
+ -----------
13
+ ::
14
+
15
+ from entroptics_context import coherent_basis, most_anomalous, coherence
16
+
17
+ V, K = coherent_basis(context_embeddings) # the K directions the context resolves
18
+ suspect = most_anomalous(context_embeddings) # the item furthest outside them
19
+ score = coherence(answer_embedding, context_embeddings)
20
+
21
+ When it beats a centroid
22
+ ------------------------
23
+ Exactly when the context spans **more than one source**, and the margin grows with the
24
+ number of sources. On real prompt injections the edge over a centroid is +0.069, +0.131 and
25
+ +0.174 AUROC at 2, 3 and 5 sources; on the on-topic decision, +0.038, +0.084 and +0.155. The
26
+ largest margin is on selection: where relevance spreads across several facets the derived cut
27
+ holds F1 at 1.000 while a cosine cut falls to 0.000. research/PAPER.md has the construction
28
+ and what every number was measured on.
29
+
30
+ ``coherence`` reads topic coherence rather than factual correctness, and the anomaly reads
31
+ rank items within one set rather than returning a verdict about it.
32
+
33
+ Layout
34
+ ------
35
+ ``engine`` the one seam onto `entroptics`. Nothing else imports it.
36
+ ``subspace`` coherent basis, in-subspace fraction, anomaly, coherence.
37
+ ``lock`` ``gap_split`` / ``top_break`` — where a salience spectrum stops.
38
+ ``cut`` the retrieval budget: a query-relative screen, and the lock on cosine.
39
+ A separate and weaker result; see the scope note in README.md.
40
+ """
41
+ from __future__ import annotations
42
+
43
+ from . import cut, engine, lock, subspace
44
+ from .cut import (cosine_salience, derive_heads, head_screen, select,
45
+ select_by_cosine, select_with_info, signal_power)
46
+ from .engine import CutUnavailable, focus, occupancy, resolved_rank
47
+ from .lock import gap_split, top_break
48
+ from .subspace import (
49
+ anomaly,
50
+ anomaly_rank,
51
+ anomaly_z,
52
+ chance_overlap,
53
+ coherence,
54
+ coherent_basis,
55
+ grounding,
56
+ in_subspace_fraction,
57
+ most_anomalous,
58
+ novelty_score,
59
+ subspace_overlap,
60
+ )
61
+
62
+ __version__ = "0.2.0"
63
+
64
+ __all__ = [
65
+ # modules
66
+ "engine", "lock", "cut", "subspace",
67
+ # engine
68
+ "CutUnavailable", "resolved_rank", "occupancy", "focus",
69
+ # the lock
70
+ "gap_split", "top_break",
71
+ # the retrieval cut
72
+ "select", "select_by_cosine", "cosine_salience",
73
+ "select_with_info", "head_screen", "signal_power", "derive_heads",
74
+ # set geometry
75
+ "coherent_basis", "in_subspace_fraction", "subspace_overlap", "chance_overlap",
76
+ "coherence", "grounding", "anomaly", "anomaly_rank", "most_anomalous", "anomaly_z",
77
+ "novelty_score",
78
+ ]
@@ -0,0 +1,370 @@
1
+ """The retrieval cut: which of a candidate pool to keep for a query.
2
+
3
+ The difficulty this solves
4
+ --------------------------
5
+ The coherent modes of a raw candidate set denote *intrinsic* structure — the dominant
6
+ shared directions, equivalently the outliers. Retrieval relevance is *extrinsic*: it
7
+ is defined by the query, and a relevant item sits near the query rather than away from
8
+ the crowd. Reading the candidate embeddings directly therefore selects the atypical
9
+ items and never the gold; measured F1 **0.000**. Two other obvious routes fail too: an
10
+ entropy-scale cut on the score distribution finds it too uniform to fold and correctly
11
+ refuses, and word- or sub-query facets are cruder than the full-query cosine.
12
+
13
+ What works is to make the screen query-relative before reading it. Subdivide each
14
+ embedding into ``H`` contiguous heads, build ``W[item, head]`` = the per-head cosine of
15
+ that item's block to the query's block, and read *that*. Now the screen's coherent
16
+ structure means "matches the query across the heads" — a gold item — rather than "is
17
+ an outlier", and the intrinsic read becomes an extrinsic one without changing the
18
+ engine.
19
+
20
+ What it is measured to be worth
21
+ -------------------------------
22
+ On four real BEIR corpora, against the identical ``top_break`` lock run on plain cosine
23
+ salience so the screen is the only difference, this ties on all four, every interval
24
+ containing zero. That is the whole of the real-corpus evidence, and it does not include a
25
+ win or a loss.
26
+
27
+ The reason they tie is the law this package is built on: BEIR relevance is
28
+ unimodal — one claim wants one supporting abstract, one argument wants one
29
+ counterargument — and at one coherent mode this read **is** cosine. A tie there is the
30
+ correct result rather than a shortfall. The screen's stated win condition is
31
+ multi-modal relevance, and it has not been demonstrated on a real corpus.
32
+
33
+ What the module *is* measured to buy is the derived count rather than an edge over
34
+ cosine: the best single constant *k* gives up as much as 0.086 F1 against a corpus's
35
+ own tuned *k*, where reading the count per query gives up at most 0.019. That is a
36
+ bound on the cost of having no constant to sweep, not a claim of superiority.
37
+
38
+ It is precision-oriented and does not do bulk recall: the lock isolates one clean cluster
39
+ above a gap, so it under-selects when recall is what is wanted. That has a measured price,
40
+ and the price is the COUNT rather than the ranking. End to end on HotpotQA, 150 questions:
41
+ handed the same number of paragraphs it derived, cosine scores the same answers (+0.000 at
42
+ 1.5B, -0.001 at 7B), so the selection is not the problem. Against a fixed top-5 it spends
43
+ 36% of the context tokens and costs 0.133 F1 with Qwen2.5-7B-Instruct, all of it because it
44
+ keeps 2.1 paragraphs where a multi-hop answer needs two. Use a generous top-k, put this read
45
+ downstream of it, and do not use it where the evidence spans passages.
46
+
47
+ This is a separate and weaker result than the claim in ``research/PAPER.md``, and it is
48
+ separate from it. research/PAPER.md section 7 carries every row.
49
+ """
50
+ from __future__ import annotations
51
+
52
+ from typing import Any
53
+
54
+ import numpy as np
55
+
56
+ from . import engine
57
+ from .engine import CutUnavailable
58
+ from .lock import top_break
59
+
60
+ __all__ = [
61
+ "cosine_salience", "select_by_cosine",
62
+ "head_screen",
63
+ "signal_power",
64
+ "derive_heads",
65
+ "select",
66
+ "select_with_info",
67
+ "MIN_HEADS",
68
+ ]
69
+
70
+ #: Added to every norm before dividing by it. Stated, not derived — a guard against
71
+ #: dividing by a zero-length vector. The honest replacement is the forward error of the
72
+ #: summation actually performed, read off the frame's own dtype; it is not applied
73
+ #: because ``|v| + 1e-9`` and ``|v| + tiny`` are different denominators and every
74
+ #: downstream number would move.
75
+ _UNIT_NORM_FLOOR = 1e-9
76
+
77
+
78
+ def _smallest_readable_head_count() -> int:
79
+ """The fewest heads at which :func:`signal_power`'s own read can be taken.
80
+
81
+ Recovered from the engine's gate rather than typed. ``head_screen`` hands
82
+ ``signal_power`` an ``(n, heads)`` matrix, which asks the engine for its resolved
83
+ rank; the engine gives no reading below ``min(n, heads) >= 2``. A one-head screen
84
+ is therefore a screen whose read cannot be taken, while ``signal_power`` would
85
+ still return a number — a single column's energy dressed as a spectral read.
86
+
87
+ The probe holds the row axis one wider than the head axis so the heads are always
88
+ the binding side, and uses distinct column values so nothing but the shape can make
89
+ it unreadable. ``min(h + 1, h) = h`` is strictly increasing, so this terminates by
90
+ the algebra and needs no iteration bound. It lands on 2.
91
+ """
92
+ heads = 0
93
+ while True:
94
+ heads += 1
95
+ probe = np.arange(1.0, float((heads + 1) * heads) + 1.0).reshape(heads + 1, heads)
96
+ try:
97
+ engine.resolved_rank(probe)
98
+ except CutUnavailable:
99
+ continue
100
+ return heads
101
+
102
+
103
+ #: The floor on the derived head count — the engine's readability gate seen from the
104
+ #: head axis. Computed, and equal to 2.
105
+ MIN_HEADS = _smallest_readable_head_count()
106
+
107
+ #: The ceiling on the derived head count. Two bounds are taken together and the tighter
108
+ #: wins. ``n_features // 2`` **is** derived — it is the geometry, since a head block
109
+ #: narrower than two coordinates has a "cosine" carrying only a sign. ``64`` is
110
+ #: **stated, not derived**, and it is the one that binds.
111
+ #:
112
+ #: It binds sooner than one might hope, and on structured input too: measured on
113
+ #: retrieval-shaped pools the raw count is 18 at n=20, 37 at n=40 and **70 at n=80**, so
114
+ #: 64 binds from a horizon of about 80 upward — not only on isotropic sets.
115
+ #:
116
+ #: It does not decide the answer. Swept across a 12x range of ceilings on the same pools,
117
+ #: F1 is flat across hi = 16 / 32 / 64 / 128 / 192. So it is a guard, not a tuning knob
118
+ #: wearing a guard's name.
119
+ MAX_HEADS = 64
120
+
121
+
122
+ def head_screen(item_embs, query_emb, heads: int) -> np.ndarray:
123
+ """``(n, heads)`` — each item's per-head cosine to the query, block by block.
124
+
125
+ The feature axis is cut into ``heads`` contiguous blocks and each block compared on
126
+ its own, so an item matching the query strongly on part of the coordinate and not
127
+ at all elsewhere is visible as such rather than averaged into one scalar.
128
+
129
+ Negative cosines are clipped to zero. A block pointing away from the query is not
130
+ negative evidence, it is *no* evidence, and letting it subtract would let two
131
+ irrelevant blocks cancel into an apparent match.
132
+
133
+ Consequence of the clip, measured and bounded. For unrelated blocks the cosine is
134
+ symmetric about zero, so the clip makes a large share of every column exactly zero
135
+ and the per-column MAD can be exactly 0 — and that is the frame :func:`signal_power`
136
+ reads its mode count off. Its reach is bounded:
137
+
138
+ * On **retrieval-shaped** pools only a minority of each column is zeroed, and breaking
139
+ the degeneracy with a jitter far below the smallest nonzero entry leaves the rank
140
+ unchanged in almost every draw and the F1 essentially unchanged.
141
+ * On **pure-random** pools — no planted relevance, nothing to find — about half of
142
+ each column is zeroed, the rank moves in half the draws, and |dk| reaches 2.
143
+
144
+ So the degeneracy bites where there is no signal to read and not where there is.
145
+
146
+ The obvious repair is measurably wrong. ``entroptics.occupied_modes`` — the rank
147
+ edge read off the profile's own step, which needs no noise bulk — returns **k=1** on
148
+ a head screen, because the largest step in that spectrum sits right after the
149
+ dominant "everything matches the query a bit" mode rather than at the signal/noise
150
+ boundary. At k=1 the cut *is* cosine, which is the one thing this screen exists to
151
+ stop being. The whitened count stays.
152
+ """
153
+ E = np.asarray(item_embs, dtype=np.float64)
154
+ q = np.asarray(query_emb, dtype=np.float64).ravel()
155
+ if E.ndim != 2:
156
+ raise CutUnavailable(f"item embeddings must be (n, d); got shape {E.shape}")
157
+ if q.shape[0] != E.shape[1]:
158
+ raise CutUnavailable(
159
+ f"query dim {q.shape[0]} != item dim {E.shape[1]} — the blocks would not correspond"
160
+ )
161
+ if not np.isfinite(E).all() or not np.isfinite(q).all():
162
+ raise CutUnavailable("the pool or the query carries NaN or infinite coordinates")
163
+ if np.linalg.norm(q) <= 0.0:
164
+ raise CutUnavailable(
165
+ "the query has zero norm, so it points nowhere and every per-head cosine is 0. "
166
+ "The screen would be all zeros and the lock would keep everything — which is a "
167
+ "read that never happened wearing the shape of one that found no break."
168
+ )
169
+ d = E.shape[1] // heads
170
+ if d < 1:
171
+ raise CutUnavailable(
172
+ f"{heads} heads over {E.shape[1]} features gives an empty block. The screen would "
173
+ f"be all zeros and the lock would keep one arbitrary item while reporting a cut."
174
+ )
175
+ Eb = E[:, : heads * d].reshape(len(E), heads, d)
176
+ qb = q[: heads * d].reshape(heads, d)
177
+ Eb = Eb / (np.linalg.norm(Eb, axis=2, keepdims=True) + _UNIT_NORM_FLOOR)
178
+ qb = qb / (np.linalg.norm(qb, axis=1, keepdims=True) + _UNIT_NORM_FLOOR)
179
+ return np.clip(np.einsum("bhd,hd->bh", Eb, qb), 0.0, None)
180
+
181
+
182
+ def signal_power(W) -> np.ndarray:
183
+ """``(n,)`` — each item's energy in the screen's signal modes.
184
+
185
+ Delegates to :func:`entroptics_context.engine.raw_signal_power`; see that docstring for
186
+ why this substrate keeps its raw magnitudes while the KV substrate does not.
187
+ """
188
+ return engine.raw_signal_power(W)
189
+
190
+
191
+ def derive_heads(item_embs, lo: int | None = None, hi: int | None = None) -> int:
192
+ """The head count, from the candidate set's own effective rank: ``H = round(occupancy * n)``.
193
+
194
+ ``occupancy`` is ``2^{H_sv}/n``, the fraction of modes carrying the energy, so
195
+ ``occupancy * n = 2^{H_sv}`` is the effective number of modes the set actually
196
+ occupies. The screen is cut into that many blocks: as many heads as the data has
197
+ independent things to say. Nothing is picked.
198
+
199
+ On LongMemEval this self-selects to about 16 per query, which is why a fixed 16 had
200
+ looked like the best choice before it was derived.
201
+
202
+ ``lo`` and ``hi`` are not the same kind of number and are documented separately at
203
+ :data:`MIN_HEADS` and :data:`MAX_HEADS`. Passing either overrides the guard, not
204
+ the derivation.
205
+
206
+ Rows are normalised to unit length first. Without it the count is read off a
207
+ spectrum dominated by whichever items happen to have the largest norm, and the head
208
+ count then tracks embedding magnitude rather than the set's structure.
209
+ """
210
+ lo = MIN_HEADS if lo is None else int(lo)
211
+ E = np.asarray(item_embs, dtype=np.float64)
212
+ if E.ndim != 2 or min(E.shape) < 2:
213
+ return lo
214
+ ceiling = MAX_HEADS if hi is None else int(hi)
215
+ ceiling = max(lo, min(ceiling, int(E.shape[1]) // 2))
216
+ En = E / (np.linalg.norm(E, axis=1, keepdims=True) + _UNIT_NORM_FLOOR)
217
+ occ = engine.occupancy(En)
218
+ if not np.isfinite(occ): # a set carrying no power has no active modes
219
+ return lo
220
+ return int(max(lo, min(ceiling, round(occ * E.shape[0]))))
221
+
222
+
223
+ def select(item_embs, query_emb, heads: int | None = None) -> np.ndarray:
224
+ """The adaptive cut: which of ``item_embs`` to keep for ``query_emb``, as indices.
225
+
226
+ ``heads`` is derived from the data when ``None``; pass an int only to override.
227
+ The lock is :func:`~entroptics_context.lock.top_break` on the per-item signal power —
228
+ the top cluster above the largest relative gap — so there is no fixed k, no MAD
229
+ multiple and no threshold. The screen decides how many to keep, which is the entire
230
+ difference between an adaptive cut and a top-k with extra steps.
231
+
232
+ ``item_embs`` is expected to be a **bounded candidate pool** — a top-B cosine
233
+ horizon, not a corpus. The read is over the structure of the set it is given, so
234
+ what is in the set is part of the question; handing it everything asks a different
235
+ question and gets a different answer, and the SVD then sees an unbounded matrix.
236
+
237
+ ``n <= 1`` returns every index. One candidate has no spectrum, so there is nothing
238
+ to cut, and discarding the only evidence there is on the strength of a read that
239
+ never happened would be worse than keeping it.
240
+ """
241
+ E = np.asarray(item_embs, dtype=np.float64)
242
+ if len(E) <= 1:
243
+ return np.arange(len(E))
244
+ H = derive_heads(E) if heads is None else int(heads)
245
+ power = signal_power(head_screen(E, query_emb, H))
246
+ keep, _ = top_break(power)
247
+ return np.where(keep)[0]
248
+
249
+
250
+ def select_with_info(
251
+ candidate_embeddings,
252
+ query_embedding,
253
+ *,
254
+ max_keep: int | None = None,
255
+ min_horizon: int = 8,
256
+ ) -> tuple[list[int], dict[str, Any]]:
257
+ """:func:`select`, power-descending, with the evidence for what it did.
258
+
259
+ Args:
260
+ candidate_embeddings: ``(n, d)`` — a bounded cosine horizon.
261
+ query_embedding: ``(d,)``.
262
+ max_keep: a hard ceiling on the number kept, for a token budget. Reported as
263
+ ``capped``: a truncated selection is not the cut's answer and must not be
264
+ read as one.
265
+ min_horizon: below this many candidates, decline and hand back everything with
266
+ ``fell_back`` set. This is **caller policy, not a property of the read** —
267
+ the engine's own gate is ``n >= 2``. 8 is the value the live pilot ships;
268
+ it says how much structure a caller wants present before it trusts a cut,
269
+ and a caller with a different appetite should pass a different number.
270
+
271
+ Returns ``(kept_indices, info)``. ``info`` carries ``horizon``, ``heads_derived``,
272
+ ``kept``, ``rel_gap``, ``separability``, ``found_gap``, ``fell_back``, ``capped``.
273
+
274
+ ``separability`` is the evidence that there was anything to cut: eta^2 of the best
275
+ two-population split of the signal power, 0.0 for one population. It discriminates
276
+ better than ``rel_gap`` and not well enough to trust alone: the structured and
277
+ structureless populations touch at the edges. Both are reported and neither is gated on,
278
+ because a gate needs a threshold and a structureless pool scores too high to set one.
279
+
280
+ ``found_gap`` is False when the lock kept everything — the cut ran and found no
281
+ break — and that is a different fact from ``fell_back``, where it did not run.
282
+ """
283
+ E = np.asarray(candidate_embeddings, dtype=np.float64)
284
+ n = len(E)
285
+ info: dict[str, Any] = {
286
+ "horizon": n, "heads_derived": 0, "kept": 0, "rel_gap": None,
287
+ "separability": None, "found_gap": False, "fell_back": False, "capped": False,
288
+ }
289
+ if n == 0:
290
+ return [], info
291
+ if n < min_horizon:
292
+ info["fell_back"] = True
293
+ info["kept"] = n
294
+ return list(range(n)), info
295
+
296
+ heads = derive_heads(E)
297
+ info["heads_derived"] = heads
298
+ power = signal_power(head_screen(E, query_embedding, heads))
299
+ keep_mask, rel_gap = top_break(power)
300
+ info["rel_gap"] = float(rel_gap)
301
+ info["separability"] = round(engine.separability(power), 4)
302
+ order = np.argsort(power)[::-1]
303
+ kept = [int(i) for i in order if keep_mask[i]]
304
+ info["found_gap"] = len(kept) < n
305
+ if max_keep is not None and len(kept) > max_keep:
306
+ info["capped"] = True
307
+ kept = kept[:max_keep]
308
+ info["kept"] = len(kept)
309
+ return kept, info
310
+
311
+ # ──────────────────────────────────────────────────────────────────────────
312
+ # The lock on plain cosine salience: the same lock, without the screen.
313
+ # ──────────────────────────────────────────────────────────────────────────
314
+
315
+
316
+ def cosine_salience(item_embs, query_emb) -> np.ndarray:
317
+ """``(n,)`` — cosine of each candidate against the query, clipped at zero.
318
+
319
+ Clipped rather than shifted. A negative cosine says the candidate points away from the
320
+ query, and every such candidate is equally uninformative about where the spectrum
321
+ stops; mapping them all to zero says that, where shifting to the minimum would spread
322
+ them into a tail that the lock then has to look past.
323
+ """
324
+ E = np.asarray(item_embs, dtype=np.float64)
325
+ q = np.asarray(query_emb, dtype=np.float64).ravel()
326
+ if E.ndim != 2:
327
+ raise CutUnavailable(f"candidates must be (n, d); got shape {E.shape}")
328
+ if E.shape[1] != q.size:
329
+ raise CutUnavailable(
330
+ f"candidates are {E.shape[1]}-dimensional and the query is {q.size}"
331
+ )
332
+ en = np.linalg.norm(E, axis=1)
333
+ qn = float(np.linalg.norm(q))
334
+ if qn == 0.0 or not np.isfinite(qn) or not np.isfinite(en).all() or (en == 0).any():
335
+ raise CutUnavailable("a zero-length or non-finite vector has no direction to compare")
336
+ return np.clip((E @ q) / (en * qn), 0.0, None)
337
+
338
+
339
+ def select_by_cosine(item_embs, query_emb) -> np.ndarray:
340
+ """Which of a candidate horizon to keep for this query. Returns ascending indices.
341
+
342
+ The count is read from the candidates' own similarity spectrum: cut at the largest
343
+ relative gap at or above the median (:func:`~entroptics_context.lock.top_break`). No fixed
344
+ *k*, no similarity threshold, nothing fitted to a corpus, and no additional model call —
345
+ the similarities are ones the retriever already computed.
346
+
347
+ ``n <= 1`` returns every index. One candidate has no spectrum, and discarding the only
348
+ evidence there is on the strength of a read that never happened would be worse than
349
+ keeping it.
350
+
351
+ What it is for
352
+ --------------
353
+ The best fixed *k* is a different number on each corpus, so a service that answers
354
+ against more than one of them, or against a corpus whose content moves, has no single
355
+ right constant to ship. This reads the number per query instead, and the measured
356
+ consequence is a **bound on what that costs**: across five BEIR corpora the best single
357
+ constant gives up as much as 0.086 F1 against a corpus's own tuned *k*, where this gives
358
+ up at most 0.019. ``research/PAPER.md`` has the measurement.
359
+
360
+ Where a constant is better
361
+ --------------------------
362
+ One corpus, stable, with relevance labels to sweep against. Find the *k* and ship it —
363
+ it is cheaper than this and at least as good. What this is for is the case where that
364
+ sweep is not available.
365
+ """
366
+ E = np.asarray(item_embs, dtype=np.float64)
367
+ if E.ndim != 2 or len(E) <= 1:
368
+ return np.arange(len(E))
369
+ keep, _ = top_break(cosine_salience(E, query_emb))
370
+ return np.where(keep)[0]
@@ -0,0 +1,221 @@
1
+ """The one seam onto ``entroptics``.
2
+
3
+ Every spectral read in this package arrives here. No other module imports
4
+ ``entroptics``, and ``tests/test_no_shadow.py`` enforces that, because the way this
5
+ family of code goes wrong is not a wrong formula — it is two copies of the right one
6
+ drifting a percent apart and neither being wrong enough to notice.
7
+
8
+ What the engine is asked for
9
+ ----------------------------
10
+ ========================= ============================================================
11
+ ``resolved_rank(M)`` how many directions ``M`` really has: singular values above
12
+ the derived noise floor. Whitened, native resolution.
13
+ ``raw_signal_power(W)`` each row's energy in the top-``K`` left singular modes of
14
+ the **raw** screen, ``K`` from ``resolved_rank``. The
15
+ retrieval path — see "two substrates" below.
16
+ ``occupancy(M)`` ``2^{H_sv}/n``, the fraction of modes carrying the energy.
17
+ ``focus(E)`` ``sigma_1^2 / M``, the power fraction on the leading axis.
18
+ ========================= ============================================================
19
+
20
+ Two substrates, read two ways
21
+ ------------------------------------------
22
+ The library's resolved read whitens each feature channel to a common noise scale
23
+ before decomposing, which is right when the channels are *incommensurate* — a KV
24
+ head's ``head_dim`` coordinates carry arbitrary learned scales, and a channel that
25
+ happens to be large is not thereby more informative.
26
+
27
+ A retrieval head screen is the opposite case. ``W[item, head]`` is a per-head cosine:
28
+ every entry is already in the same units, on the same bounded interval, and the
29
+ per-head magnitude *is* the relevance. Whitening it would divide exactly the signal
30
+ out. So ``raw_signal_power`` takes its modes off the raw screen.
31
+
32
+ It still takes its *count* from the whitened frame, because the noise floor is a
33
+ statement about unit-noise channels and is not meaningful otherwise. That mix — rank
34
+ from the whitened frame, modes from the raw one — is what the shipped implementation
35
+ does and what the measured retrieval win was measured on. It is stated here rather
36
+ than left to be discovered in the arithmetic.
37
+
38
+ Where this sits relative to the other copies
39
+ --------------------------------------------
40
+ ``mantle.search.beacon.engine`` is a numpy-only reimplementation of the same reads,
41
+ carried in the Agience tree so a store can ship without this dependency. The two are
42
+ not assumed to agree; ``tests/test_conformance_mantle.py`` measures it, and the one
43
+ place they genuinely diverge is recorded there with the measured size: a channel of
44
+ measured zeros is a mode the record HAS, which ``occupancy`` counts in its denominator
45
+ and mantle's ``occupancy_fraction`` drops as a dead line.
46
+ """
47
+ from __future__ import annotations
48
+
49
+ import numpy as np
50
+
51
+ from entroptics import resolved_batch
52
+ from entroptics.reads import concentration as _concentration
53
+ from entroptics.reads import phi as _phi
54
+
55
+ __all__ = [
56
+ "CutUnavailable",
57
+ "resolved_rank",
58
+ "raw_signal_power",
59
+ "occupancy",
60
+ "focus",
61
+ "noise_floor",
62
+ "machine_eps",
63
+ "separability",
64
+ ]
65
+
66
+ #: Passed to every read as the noise floor's false-alarm level. It is the library's own
67
+ #: default and the only externally-set number in this package: not a tuned constant but
68
+ #: a stated tolerance — how often the floor is allowed to admit a mode that is noise.
69
+ FAR = 0.05
70
+
71
+
72
+ class CutUnavailable(RuntimeError):
73
+ """A read could not be taken. Callers keep a fixed-cut fallback path.
74
+
75
+ NOT a "no signal" result. A read that happened and found one coherent direction
76
+ returns 1; a read that did not happen raises. Conflating the two is how a fallback
77
+ becomes the product — every caller that catches this must do something
78
+ visibly different from what it does on a rank of 1.
79
+ """
80
+
81
+
82
+ def _frame(M) -> np.ndarray:
83
+ A = np.asarray(M, dtype=np.float64)
84
+ if A.ndim != 2:
85
+ raise CutUnavailable(f"a screen must be 2-D; got shape {A.shape}")
86
+ if A.size == 0 or min(A.shape) < 2:
87
+ raise CutUnavailable(f"a {A.shape} frame has no spectrum to read")
88
+ return A
89
+
90
+
91
+ def _read(M, *, energy: bool = False, basis: bool = False):
92
+ """One resolved read of a single frame, at native feature resolution.
93
+
94
+ ``fold=False`` throughout. The feature axis here is always an unordered learned
95
+ basis — embedding coordinates, a head's ``head_dim``, or the head index of a
96
+ query-relative screen — and the library's area-average fold is valid only for a
97
+ dense smooth continuum. ``fold="auto"`` would reach the same conclusion per frame;
98
+ pinning it says so once instead of relying on it every call.
99
+ """
100
+ A = _frame(M)
101
+ try:
102
+ return resolved_batch(A[None], fold=False, far=FAR, energy=energy, basis=basis)
103
+ except Exception as exc: # a genuinely unreadable frame
104
+ raise CutUnavailable(f"resolved read failed on a {A.shape} frame: {exc!r}") from exc
105
+
106
+
107
+ def resolved_rank(M) -> int:
108
+ """How many directions ``M`` has: singular values above the derived noise floor.
109
+
110
+ Floored at 1. A caller here builds a subspace out of the answer immediately and
111
+ cannot defer, so a genuine count of 0 becomes the coarsest usable basis rather
112
+ than an empty one. The floor is this package's policy, not the engine's — the
113
+ engine reports 0 when it means 0, which is why this wrapper is where the floor
114
+ lives and not somewhere it could be applied twice.
115
+ """
116
+ return max(1, int(_read(M).K_signal[0]))
117
+
118
+
119
+ def raw_signal_power(W) -> np.ndarray:
120
+ """``(n,)`` — each row's energy in the top-``K`` left singular modes of the raw ``W``.
121
+
122
+ ``K = resolved_rank(W)``, read on the whitened frame; the modes are taken off ``W``
123
+ as given. See the module docstring for why this substrate is not whitened.
124
+
125
+ A row aligned with the coherent structure the screen actually has carries the
126
+ power; a row that matches weakly and evenly does not. That is the whole difference
127
+ between this and a row-sum, and therefore between an adaptive cut and a re-ranking.
128
+ """
129
+ A = _frame(W)
130
+ k = min(resolved_rank(A), min(A.shape))
131
+ U, S, _ = np.linalg.svd(A, full_matrices=False)
132
+ return ((U[:, :k] * S[:k]) ** 2).sum(1)
133
+
134
+
135
+ def occupancy(M) -> float:
136
+ """``2^{H_sv}/n`` in (0, 1] — the fraction of ``M``'s modes that carry the energy.
137
+
138
+ Read on the block as recorded. A constant level is a mode and is counted as one.
139
+
140
+ This read and ``mantle.search.beacon.engine.occupancy_fraction`` agree exactly —
141
+ measured over 60 real retrieval horizons, a maximum absolute difference of 0.0.
142
+ They diverged until 2026-09-02, when the library still centred before decomposing:
143
+ on a set of unit direction vectors the shared mean direction is the topic the set
144
+ is about, so de-meaning deleted the dominant mode and roughly doubled the fill.
145
+ That reached this package as twice the head count, and
146
+ therefore as a different cut.
147
+
148
+ NaN for a frame carrying no power: there is no fraction of active modes to report
149
+ when no mode is active, and reporting ``1/n`` instead would be indistinguishable
150
+ from a perfect single mode.
151
+ """
152
+ return float(_phi(_frame(M)))
153
+
154
+
155
+ def focus(E) -> float:
156
+ """``sigma_1^2 / M`` in (0, 1] — the power fraction on the set's leading axis.
157
+
158
+ The bounded MAGNITUDE companion to the rank read: :func:`resolved_rank` answers
159
+ whether structure is present, this answers how concentrated it is. 1 = the set
160
+ points one way; toward 0 = isotropic. Axial, so an antipodal cloud still reads ~1.
161
+
162
+ A diagnostic rather than a decision. The concentration of a kept set tracks cut
163
+ reliability only weakly, and the head screen's own concentration runs weakly the
164
+ other way: a query matching everything is less discriminable. Use it to rank or
165
+ compare sets, or to report a confidence beside a selection, and read the cut
166
+ itself from the lock.
167
+ """
168
+ A = np.asarray(E, dtype=np.float64)
169
+ if A.ndim != 2 or len(A) < 1:
170
+ return 0.0
171
+ return float(_concentration(A).focus)
172
+
173
+
174
+ def separability(salience) -> float:
175
+ """``eta^2`` in [0, 1] — how cleanly a salience profile falls into two populations.
176
+
177
+ 0.0 means one population and nothing to cut at; toward 1.0 means two groups plainly
178
+ apart. Delegates to ``entroptics.level_edge``.
179
+
180
+ This is the **whether**, and the lock's ``rel_gap`` is the **where**. They are not
181
+ interchangeable and neither substitutes for the other. Measured on 50 structured
182
+ against 50 structureless candidate pools, as a detector of "is there anything here":
183
+ this separates structured from structureless pools better than ``rel_gap``, whose
184
+ ranges cross more broadly. Better, and not a solution — a structureless pool still
185
+ scores well above zero, so this
186
+ is evidence to report and not a gate to trust. It also measures *cleanness*, not
187
+ distance: an exact two-level profile reads 1.000 whatever the ratio across it, so a
188
+ caller who wants the size of the break still wants ``rel_gap``.
189
+
190
+ Report it beside a selection. It is not a gate here, because turning it into one means
191
+ choosing a threshold and that is the thing this package does not do.
192
+ """
193
+ from entroptics import level_edge
194
+ return float(level_edge(np.asarray(salience, dtype=np.float64)).separability)
195
+
196
+
197
+ def machine_eps(ref) -> float:
198
+ """The working dtype's machine epsilon — the smallest relative difference the arithmetic
199
+ can represent, read off the array rather than asserted.
200
+
201
+ Delegates to ``entroptics.entropy.macheps``, which is the library's own answer and
202
+ follows the backend and the compute precision. It is exposed through this seam because
203
+ :mod:`entroptics_context.lock` needs it — a ratio of two computed numbers is not a drop
204
+ unless it clears the round-off of the computation that produced them — and the lock
205
+ does not import the library.
206
+
207
+ This is the same construction as the library's ``resolution_floor`` (``typical * eps``,
208
+ derived on both sides: the scale from the data, the epsilon from the dtype). What the
209
+ lock adds is the term count, since its inputs are sums.
210
+ """
211
+ from entroptics.entropy import macheps
212
+ return float(macheps(np, np.asarray(ref)))
213
+
214
+
215
+ def noise_floor(M) -> float:
216
+ """The singular-value floor ``resolved_rank`` counted against, on the whitened frame.
217
+
218
+ Exposed for reporting — "the top mode stood this far above the floor" is the
219
+ evidence behind a rank, and a rank without it is a number with no error bar.
220
+ """
221
+ return float(_read(M).noise_floor[0])