entroptics-context 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- entroptics_context/__init__.py +78 -0
- entroptics_context/cut.py +370 -0
- entroptics_context/engine.py +221 -0
- entroptics_context/lock.py +138 -0
- entroptics_context/py.typed +0 -0
- entroptics_context/subspace.py +290 -0
- entroptics_context-0.2.0.dist-info/METADATA +209 -0
- entroptics_context-0.2.0.dist-info/RECORD +13 -0
- entroptics_context-0.2.0.dist-info/WHEEL +5 -0
- entroptics_context-0.2.0.dist-info/licenses/COMMERCIAL_LICENSE.md +91 -0
- entroptics_context-0.2.0.dist-info/licenses/LICENSE +661 -0
- entroptics_context-0.2.0.dist-info/licenses/NOTICE +5 -0
- entroptics_context-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Reading a context that spans several sources as a subspace rather than a centre.
|
|
2
|
+
|
|
3
|
+
A context assembled from several documents — a retrieval-augmented prompt, an agent
|
|
4
|
+
scratchpad, a long conversation — has no single direction. Its centroid sits between its
|
|
5
|
+
sources, in a direction none of them occupies, and every guard built on that centroid gets
|
|
6
|
+
worse as the context spreads. The subspace the context resolves above its own noise floor
|
|
7
|
+
spans the sources instead, so legitimate material lies inside it however many topics it
|
|
8
|
+
covers and foreign material does not. Nothing is trained, no threshold is set, and no
|
|
9
|
+
constant is fitted per corpus.
|
|
10
|
+
|
|
11
|
+
Quick start
|
|
12
|
+
-----------
|
|
13
|
+
::
|
|
14
|
+
|
|
15
|
+
from entroptics_context import coherent_basis, most_anomalous, coherence
|
|
16
|
+
|
|
17
|
+
V, K = coherent_basis(context_embeddings) # the K directions the context resolves
|
|
18
|
+
suspect = most_anomalous(context_embeddings) # the item furthest outside them
|
|
19
|
+
score = coherence(answer_embedding, context_embeddings)
|
|
20
|
+
|
|
21
|
+
When it beats a centroid
|
|
22
|
+
------------------------
|
|
23
|
+
Exactly when the context spans **more than one source**, and the margin grows with the
|
|
24
|
+
number of sources. On real prompt injections the edge over a centroid is +0.069, +0.131 and
|
|
25
|
+
+0.174 AUROC at 2, 3 and 5 sources; on the on-topic decision, +0.038, +0.084 and +0.155. The
|
|
26
|
+
largest margin is on selection: where relevance spreads across several facets the derived cut
|
|
27
|
+
holds F1 at 1.000 while a cosine cut falls to 0.000. research/PAPER.md has the construction
|
|
28
|
+
and what every number was measured on.
|
|
29
|
+
|
|
30
|
+
``coherence`` reads topic coherence rather than factual correctness, and the anomaly reads
|
|
31
|
+
rank items within one set rather than returning a verdict about it.
|
|
32
|
+
|
|
33
|
+
Layout
|
|
34
|
+
------
|
|
35
|
+
``engine`` the one seam onto `entroptics`. Nothing else imports it.
|
|
36
|
+
``subspace`` coherent basis, in-subspace fraction, anomaly, coherence.
|
|
37
|
+
``lock`` ``gap_split`` / ``top_break`` — where a salience spectrum stops.
|
|
38
|
+
``cut`` the retrieval budget: a query-relative screen, and the lock on cosine.
|
|
39
|
+
A separate and weaker result; see the scope note in README.md.
|
|
40
|
+
"""
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
from . import cut, engine, lock, subspace
|
|
44
|
+
from .cut import (cosine_salience, derive_heads, head_screen, select,
|
|
45
|
+
select_by_cosine, select_with_info, signal_power)
|
|
46
|
+
from .engine import CutUnavailable, focus, occupancy, resolved_rank
|
|
47
|
+
from .lock import gap_split, top_break
|
|
48
|
+
from .subspace import (
|
|
49
|
+
anomaly,
|
|
50
|
+
anomaly_rank,
|
|
51
|
+
anomaly_z,
|
|
52
|
+
chance_overlap,
|
|
53
|
+
coherence,
|
|
54
|
+
coherent_basis,
|
|
55
|
+
grounding,
|
|
56
|
+
in_subspace_fraction,
|
|
57
|
+
most_anomalous,
|
|
58
|
+
novelty_score,
|
|
59
|
+
subspace_overlap,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
__version__ = "0.2.0"
|
|
63
|
+
|
|
64
|
+
__all__ = [
|
|
65
|
+
# modules
|
|
66
|
+
"engine", "lock", "cut", "subspace",
|
|
67
|
+
# engine
|
|
68
|
+
"CutUnavailable", "resolved_rank", "occupancy", "focus",
|
|
69
|
+
# the lock
|
|
70
|
+
"gap_split", "top_break",
|
|
71
|
+
# the retrieval cut
|
|
72
|
+
"select", "select_by_cosine", "cosine_salience",
|
|
73
|
+
"select_with_info", "head_screen", "signal_power", "derive_heads",
|
|
74
|
+
# set geometry
|
|
75
|
+
"coherent_basis", "in_subspace_fraction", "subspace_overlap", "chance_overlap",
|
|
76
|
+
"coherence", "grounding", "anomaly", "anomaly_rank", "most_anomalous", "anomaly_z",
|
|
77
|
+
"novelty_score",
|
|
78
|
+
]
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""The retrieval cut: which of a candidate pool to keep for a query.
|
|
2
|
+
|
|
3
|
+
The difficulty this solves
|
|
4
|
+
--------------------------
|
|
5
|
+
The coherent modes of a raw candidate set denote *intrinsic* structure — the dominant
|
|
6
|
+
shared directions, equivalently the outliers. Retrieval relevance is *extrinsic*: it
|
|
7
|
+
is defined by the query, and a relevant item sits near the query rather than away from
|
|
8
|
+
the crowd. Reading the candidate embeddings directly therefore selects the atypical
|
|
9
|
+
items and never the gold; measured F1 **0.000**. Two other obvious routes fail too: an
|
|
10
|
+
entropy-scale cut on the score distribution finds it too uniform to fold and correctly
|
|
11
|
+
refuses, and word- or sub-query facets are cruder than the full-query cosine.
|
|
12
|
+
|
|
13
|
+
What works is to make the screen query-relative before reading it. Subdivide each
|
|
14
|
+
embedding into ``H`` contiguous heads, build ``W[item, head]`` = the per-head cosine of
|
|
15
|
+
that item's block to the query's block, and read *that*. Now the screen's coherent
|
|
16
|
+
structure means "matches the query across the heads" — a gold item — rather than "is
|
|
17
|
+
an outlier", and the intrinsic read becomes an extrinsic one without changing the
|
|
18
|
+
engine.
|
|
19
|
+
|
|
20
|
+
What it is measured to be worth
|
|
21
|
+
-------------------------------
|
|
22
|
+
On four real BEIR corpora, against the identical ``top_break`` lock run on plain cosine
|
|
23
|
+
salience so the screen is the only difference, this ties on all four, every interval
|
|
24
|
+
containing zero. That is the whole of the real-corpus evidence, and it does not include a
|
|
25
|
+
win or a loss.
|
|
26
|
+
|
|
27
|
+
The reason they tie is the law this package is built on: BEIR relevance is
|
|
28
|
+
unimodal — one claim wants one supporting abstract, one argument wants one
|
|
29
|
+
counterargument — and at one coherent mode this read **is** cosine. A tie there is the
|
|
30
|
+
correct result rather than a shortfall. The screen's stated win condition is
|
|
31
|
+
multi-modal relevance, and it has not been demonstrated on a real corpus.
|
|
32
|
+
|
|
33
|
+
What the module *is* measured to buy is the derived count rather than an edge over
|
|
34
|
+
cosine: the best single constant *k* gives up as much as 0.086 F1 against a corpus's
|
|
35
|
+
own tuned *k*, where reading the count per query gives up at most 0.019. That is a
|
|
36
|
+
bound on the cost of having no constant to sweep, not a claim of superiority.
|
|
37
|
+
|
|
38
|
+
It is precision-oriented and does not do bulk recall: the lock isolates one clean cluster
|
|
39
|
+
above a gap, so it under-selects when recall is what is wanted. That has a measured price,
|
|
40
|
+
and the price is the COUNT rather than the ranking. End to end on HotpotQA, 150 questions:
|
|
41
|
+
handed the same number of paragraphs it derived, cosine scores the same answers (+0.000 at
|
|
42
|
+
1.5B, -0.001 at 7B), so the selection is not the problem. Against a fixed top-5 it spends
|
|
43
|
+
36% of the context tokens and costs 0.133 F1 with Qwen2.5-7B-Instruct, all of it because it
|
|
44
|
+
keeps 2.1 paragraphs where a multi-hop answer needs two. Use a generous top-k, put this read
|
|
45
|
+
downstream of it, and do not use it where the evidence spans passages.
|
|
46
|
+
|
|
47
|
+
This is a separate and weaker result than the claim in ``research/PAPER.md``, and it is
|
|
48
|
+
separate from it. research/PAPER.md section 7 carries every row.
|
|
49
|
+
"""
|
|
50
|
+
from __future__ import annotations
|
|
51
|
+
|
|
52
|
+
from typing import Any
|
|
53
|
+
|
|
54
|
+
import numpy as np
|
|
55
|
+
|
|
56
|
+
from . import engine
|
|
57
|
+
from .engine import CutUnavailable
|
|
58
|
+
from .lock import top_break
|
|
59
|
+
|
|
60
|
+
__all__ = [
|
|
61
|
+
"cosine_salience", "select_by_cosine",
|
|
62
|
+
"head_screen",
|
|
63
|
+
"signal_power",
|
|
64
|
+
"derive_heads",
|
|
65
|
+
"select",
|
|
66
|
+
"select_with_info",
|
|
67
|
+
"MIN_HEADS",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
#: Added to every norm before dividing by it. Stated, not derived — a guard against
|
|
71
|
+
#: dividing by a zero-length vector. The honest replacement is the forward error of the
|
|
72
|
+
#: summation actually performed, read off the frame's own dtype; it is not applied
|
|
73
|
+
#: because ``|v| + 1e-9`` and ``|v| + tiny`` are different denominators and every
|
|
74
|
+
#: downstream number would move.
|
|
75
|
+
_UNIT_NORM_FLOOR = 1e-9
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _smallest_readable_head_count() -> int:
|
|
79
|
+
"""The fewest heads at which :func:`signal_power`'s own read can be taken.
|
|
80
|
+
|
|
81
|
+
Recovered from the engine's gate rather than typed. ``head_screen`` hands
|
|
82
|
+
``signal_power`` an ``(n, heads)`` matrix, which asks the engine for its resolved
|
|
83
|
+
rank; the engine gives no reading below ``min(n, heads) >= 2``. A one-head screen
|
|
84
|
+
is therefore a screen whose read cannot be taken, while ``signal_power`` would
|
|
85
|
+
still return a number — a single column's energy dressed as a spectral read.
|
|
86
|
+
|
|
87
|
+
The probe holds the row axis one wider than the head axis so the heads are always
|
|
88
|
+
the binding side, and uses distinct column values so nothing but the shape can make
|
|
89
|
+
it unreadable. ``min(h + 1, h) = h`` is strictly increasing, so this terminates by
|
|
90
|
+
the algebra and needs no iteration bound. It lands on 2.
|
|
91
|
+
"""
|
|
92
|
+
heads = 0
|
|
93
|
+
while True:
|
|
94
|
+
heads += 1
|
|
95
|
+
probe = np.arange(1.0, float((heads + 1) * heads) + 1.0).reshape(heads + 1, heads)
|
|
96
|
+
try:
|
|
97
|
+
engine.resolved_rank(probe)
|
|
98
|
+
except CutUnavailable:
|
|
99
|
+
continue
|
|
100
|
+
return heads
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
#: The floor on the derived head count — the engine's readability gate seen from the
|
|
104
|
+
#: head axis. Computed, and equal to 2.
|
|
105
|
+
MIN_HEADS = _smallest_readable_head_count()
|
|
106
|
+
|
|
107
|
+
#: The ceiling on the derived head count. Two bounds are taken together and the tighter
|
|
108
|
+
#: wins. ``n_features // 2`` **is** derived — it is the geometry, since a head block
|
|
109
|
+
#: narrower than two coordinates has a "cosine" carrying only a sign. ``64`` is
|
|
110
|
+
#: **stated, not derived**, and it is the one that binds.
|
|
111
|
+
#:
|
|
112
|
+
#: It binds sooner than one might hope, and on structured input too: measured on
|
|
113
|
+
#: retrieval-shaped pools the raw count is 18 at n=20, 37 at n=40 and **70 at n=80**, so
|
|
114
|
+
#: 64 binds from a horizon of about 80 upward — not only on isotropic sets.
|
|
115
|
+
#:
|
|
116
|
+
#: It does not decide the answer. Swept across a 12x range of ceilings on the same pools,
|
|
117
|
+
#: F1 is flat across hi = 16 / 32 / 64 / 128 / 192. So it is a guard, not a tuning knob
|
|
118
|
+
#: wearing a guard's name.
|
|
119
|
+
MAX_HEADS = 64
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def head_screen(item_embs, query_emb, heads: int) -> np.ndarray:
|
|
123
|
+
"""``(n, heads)`` — each item's per-head cosine to the query, block by block.
|
|
124
|
+
|
|
125
|
+
The feature axis is cut into ``heads`` contiguous blocks and each block compared on
|
|
126
|
+
its own, so an item matching the query strongly on part of the coordinate and not
|
|
127
|
+
at all elsewhere is visible as such rather than averaged into one scalar.
|
|
128
|
+
|
|
129
|
+
Negative cosines are clipped to zero. A block pointing away from the query is not
|
|
130
|
+
negative evidence, it is *no* evidence, and letting it subtract would let two
|
|
131
|
+
irrelevant blocks cancel into an apparent match.
|
|
132
|
+
|
|
133
|
+
Consequence of the clip, measured and bounded. For unrelated blocks the cosine is
|
|
134
|
+
symmetric about zero, so the clip makes a large share of every column exactly zero
|
|
135
|
+
and the per-column MAD can be exactly 0 — and that is the frame :func:`signal_power`
|
|
136
|
+
reads its mode count off. Its reach is bounded:
|
|
137
|
+
|
|
138
|
+
* On **retrieval-shaped** pools only a minority of each column is zeroed, and breaking
|
|
139
|
+
the degeneracy with a jitter far below the smallest nonzero entry leaves the rank
|
|
140
|
+
unchanged in almost every draw and the F1 essentially unchanged.
|
|
141
|
+
* On **pure-random** pools — no planted relevance, nothing to find — about half of
|
|
142
|
+
each column is zeroed, the rank moves in half the draws, and |dk| reaches 2.
|
|
143
|
+
|
|
144
|
+
So the degeneracy bites where there is no signal to read and not where there is.
|
|
145
|
+
|
|
146
|
+
The obvious repair is measurably wrong. ``entroptics.occupied_modes`` — the rank
|
|
147
|
+
edge read off the profile's own step, which needs no noise bulk — returns **k=1** on
|
|
148
|
+
a head screen, because the largest step in that spectrum sits right after the
|
|
149
|
+
dominant "everything matches the query a bit" mode rather than at the signal/noise
|
|
150
|
+
boundary. At k=1 the cut *is* cosine, which is the one thing this screen exists to
|
|
151
|
+
stop being. The whitened count stays.
|
|
152
|
+
"""
|
|
153
|
+
E = np.asarray(item_embs, dtype=np.float64)
|
|
154
|
+
q = np.asarray(query_emb, dtype=np.float64).ravel()
|
|
155
|
+
if E.ndim != 2:
|
|
156
|
+
raise CutUnavailable(f"item embeddings must be (n, d); got shape {E.shape}")
|
|
157
|
+
if q.shape[0] != E.shape[1]:
|
|
158
|
+
raise CutUnavailable(
|
|
159
|
+
f"query dim {q.shape[0]} != item dim {E.shape[1]} — the blocks would not correspond"
|
|
160
|
+
)
|
|
161
|
+
if not np.isfinite(E).all() or not np.isfinite(q).all():
|
|
162
|
+
raise CutUnavailable("the pool or the query carries NaN or infinite coordinates")
|
|
163
|
+
if np.linalg.norm(q) <= 0.0:
|
|
164
|
+
raise CutUnavailable(
|
|
165
|
+
"the query has zero norm, so it points nowhere and every per-head cosine is 0. "
|
|
166
|
+
"The screen would be all zeros and the lock would keep everything — which is a "
|
|
167
|
+
"read that never happened wearing the shape of one that found no break."
|
|
168
|
+
)
|
|
169
|
+
d = E.shape[1] // heads
|
|
170
|
+
if d < 1:
|
|
171
|
+
raise CutUnavailable(
|
|
172
|
+
f"{heads} heads over {E.shape[1]} features gives an empty block. The screen would "
|
|
173
|
+
f"be all zeros and the lock would keep one arbitrary item while reporting a cut."
|
|
174
|
+
)
|
|
175
|
+
Eb = E[:, : heads * d].reshape(len(E), heads, d)
|
|
176
|
+
qb = q[: heads * d].reshape(heads, d)
|
|
177
|
+
Eb = Eb / (np.linalg.norm(Eb, axis=2, keepdims=True) + _UNIT_NORM_FLOOR)
|
|
178
|
+
qb = qb / (np.linalg.norm(qb, axis=1, keepdims=True) + _UNIT_NORM_FLOOR)
|
|
179
|
+
return np.clip(np.einsum("bhd,hd->bh", Eb, qb), 0.0, None)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def signal_power(W) -> np.ndarray:
|
|
183
|
+
"""``(n,)`` — each item's energy in the screen's signal modes.
|
|
184
|
+
|
|
185
|
+
Delegates to :func:`entroptics_context.engine.raw_signal_power`; see that docstring for
|
|
186
|
+
why this substrate keeps its raw magnitudes while the KV substrate does not.
|
|
187
|
+
"""
|
|
188
|
+
return engine.raw_signal_power(W)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def derive_heads(item_embs, lo: int | None = None, hi: int | None = None) -> int:
|
|
192
|
+
"""The head count, from the candidate set's own effective rank: ``H = round(occupancy * n)``.
|
|
193
|
+
|
|
194
|
+
``occupancy`` is ``2^{H_sv}/n``, the fraction of modes carrying the energy, so
|
|
195
|
+
``occupancy * n = 2^{H_sv}`` is the effective number of modes the set actually
|
|
196
|
+
occupies. The screen is cut into that many blocks: as many heads as the data has
|
|
197
|
+
independent things to say. Nothing is picked.
|
|
198
|
+
|
|
199
|
+
On LongMemEval this self-selects to about 16 per query, which is why a fixed 16 had
|
|
200
|
+
looked like the best choice before it was derived.
|
|
201
|
+
|
|
202
|
+
``lo`` and ``hi`` are not the same kind of number and are documented separately at
|
|
203
|
+
:data:`MIN_HEADS` and :data:`MAX_HEADS`. Passing either overrides the guard, not
|
|
204
|
+
the derivation.
|
|
205
|
+
|
|
206
|
+
Rows are normalised to unit length first. Without it the count is read off a
|
|
207
|
+
spectrum dominated by whichever items happen to have the largest norm, and the head
|
|
208
|
+
count then tracks embedding magnitude rather than the set's structure.
|
|
209
|
+
"""
|
|
210
|
+
lo = MIN_HEADS if lo is None else int(lo)
|
|
211
|
+
E = np.asarray(item_embs, dtype=np.float64)
|
|
212
|
+
if E.ndim != 2 or min(E.shape) < 2:
|
|
213
|
+
return lo
|
|
214
|
+
ceiling = MAX_HEADS if hi is None else int(hi)
|
|
215
|
+
ceiling = max(lo, min(ceiling, int(E.shape[1]) // 2))
|
|
216
|
+
En = E / (np.linalg.norm(E, axis=1, keepdims=True) + _UNIT_NORM_FLOOR)
|
|
217
|
+
occ = engine.occupancy(En)
|
|
218
|
+
if not np.isfinite(occ): # a set carrying no power has no active modes
|
|
219
|
+
return lo
|
|
220
|
+
return int(max(lo, min(ceiling, round(occ * E.shape[0]))))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def select(item_embs, query_emb, heads: int | None = None) -> np.ndarray:
|
|
224
|
+
"""The adaptive cut: which of ``item_embs`` to keep for ``query_emb``, as indices.
|
|
225
|
+
|
|
226
|
+
``heads`` is derived from the data when ``None``; pass an int only to override.
|
|
227
|
+
The lock is :func:`~entroptics_context.lock.top_break` on the per-item signal power —
|
|
228
|
+
the top cluster above the largest relative gap — so there is no fixed k, no MAD
|
|
229
|
+
multiple and no threshold. The screen decides how many to keep, which is the entire
|
|
230
|
+
difference between an adaptive cut and a top-k with extra steps.
|
|
231
|
+
|
|
232
|
+
``item_embs`` is expected to be a **bounded candidate pool** — a top-B cosine
|
|
233
|
+
horizon, not a corpus. The read is over the structure of the set it is given, so
|
|
234
|
+
what is in the set is part of the question; handing it everything asks a different
|
|
235
|
+
question and gets a different answer, and the SVD then sees an unbounded matrix.
|
|
236
|
+
|
|
237
|
+
``n <= 1`` returns every index. One candidate has no spectrum, so there is nothing
|
|
238
|
+
to cut, and discarding the only evidence there is on the strength of a read that
|
|
239
|
+
never happened would be worse than keeping it.
|
|
240
|
+
"""
|
|
241
|
+
E = np.asarray(item_embs, dtype=np.float64)
|
|
242
|
+
if len(E) <= 1:
|
|
243
|
+
return np.arange(len(E))
|
|
244
|
+
H = derive_heads(E) if heads is None else int(heads)
|
|
245
|
+
power = signal_power(head_screen(E, query_emb, H))
|
|
246
|
+
keep, _ = top_break(power)
|
|
247
|
+
return np.where(keep)[0]
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def select_with_info(
|
|
251
|
+
candidate_embeddings,
|
|
252
|
+
query_embedding,
|
|
253
|
+
*,
|
|
254
|
+
max_keep: int | None = None,
|
|
255
|
+
min_horizon: int = 8,
|
|
256
|
+
) -> tuple[list[int], dict[str, Any]]:
|
|
257
|
+
""":func:`select`, power-descending, with the evidence for what it did.
|
|
258
|
+
|
|
259
|
+
Args:
|
|
260
|
+
candidate_embeddings: ``(n, d)`` — a bounded cosine horizon.
|
|
261
|
+
query_embedding: ``(d,)``.
|
|
262
|
+
max_keep: a hard ceiling on the number kept, for a token budget. Reported as
|
|
263
|
+
``capped``: a truncated selection is not the cut's answer and must not be
|
|
264
|
+
read as one.
|
|
265
|
+
min_horizon: below this many candidates, decline and hand back everything with
|
|
266
|
+
``fell_back`` set. This is **caller policy, not a property of the read** —
|
|
267
|
+
the engine's own gate is ``n >= 2``. 8 is the value the live pilot ships;
|
|
268
|
+
it says how much structure a caller wants present before it trusts a cut,
|
|
269
|
+
and a caller with a different appetite should pass a different number.
|
|
270
|
+
|
|
271
|
+
Returns ``(kept_indices, info)``. ``info`` carries ``horizon``, ``heads_derived``,
|
|
272
|
+
``kept``, ``rel_gap``, ``separability``, ``found_gap``, ``fell_back``, ``capped``.
|
|
273
|
+
|
|
274
|
+
``separability`` is the evidence that there was anything to cut: eta^2 of the best
|
|
275
|
+
two-population split of the signal power, 0.0 for one population. It discriminates
|
|
276
|
+
better than ``rel_gap`` and not well enough to trust alone: the structured and
|
|
277
|
+
structureless populations touch at the edges. Both are reported and neither is gated on,
|
|
278
|
+
because a gate needs a threshold and a structureless pool scores too high to set one.
|
|
279
|
+
|
|
280
|
+
``found_gap`` is False when the lock kept everything — the cut ran and found no
|
|
281
|
+
break — and that is a different fact from ``fell_back``, where it did not run.
|
|
282
|
+
"""
|
|
283
|
+
E = np.asarray(candidate_embeddings, dtype=np.float64)
|
|
284
|
+
n = len(E)
|
|
285
|
+
info: dict[str, Any] = {
|
|
286
|
+
"horizon": n, "heads_derived": 0, "kept": 0, "rel_gap": None,
|
|
287
|
+
"separability": None, "found_gap": False, "fell_back": False, "capped": False,
|
|
288
|
+
}
|
|
289
|
+
if n == 0:
|
|
290
|
+
return [], info
|
|
291
|
+
if n < min_horizon:
|
|
292
|
+
info["fell_back"] = True
|
|
293
|
+
info["kept"] = n
|
|
294
|
+
return list(range(n)), info
|
|
295
|
+
|
|
296
|
+
heads = derive_heads(E)
|
|
297
|
+
info["heads_derived"] = heads
|
|
298
|
+
power = signal_power(head_screen(E, query_embedding, heads))
|
|
299
|
+
keep_mask, rel_gap = top_break(power)
|
|
300
|
+
info["rel_gap"] = float(rel_gap)
|
|
301
|
+
info["separability"] = round(engine.separability(power), 4)
|
|
302
|
+
order = np.argsort(power)[::-1]
|
|
303
|
+
kept = [int(i) for i in order if keep_mask[i]]
|
|
304
|
+
info["found_gap"] = len(kept) < n
|
|
305
|
+
if max_keep is not None and len(kept) > max_keep:
|
|
306
|
+
info["capped"] = True
|
|
307
|
+
kept = kept[:max_keep]
|
|
308
|
+
info["kept"] = len(kept)
|
|
309
|
+
return kept, info
|
|
310
|
+
|
|
311
|
+
# ──────────────────────────────────────────────────────────────────────────
|
|
312
|
+
# The lock on plain cosine salience: the same lock, without the screen.
|
|
313
|
+
# ──────────────────────────────────────────────────────────────────────────
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def cosine_salience(item_embs, query_emb) -> np.ndarray:
|
|
317
|
+
"""``(n,)`` — cosine of each candidate against the query, clipped at zero.
|
|
318
|
+
|
|
319
|
+
Clipped rather than shifted. A negative cosine says the candidate points away from the
|
|
320
|
+
query, and every such candidate is equally uninformative about where the spectrum
|
|
321
|
+
stops; mapping them all to zero says that, where shifting to the minimum would spread
|
|
322
|
+
them into a tail that the lock then has to look past.
|
|
323
|
+
"""
|
|
324
|
+
E = np.asarray(item_embs, dtype=np.float64)
|
|
325
|
+
q = np.asarray(query_emb, dtype=np.float64).ravel()
|
|
326
|
+
if E.ndim != 2:
|
|
327
|
+
raise CutUnavailable(f"candidates must be (n, d); got shape {E.shape}")
|
|
328
|
+
if E.shape[1] != q.size:
|
|
329
|
+
raise CutUnavailable(
|
|
330
|
+
f"candidates are {E.shape[1]}-dimensional and the query is {q.size}"
|
|
331
|
+
)
|
|
332
|
+
en = np.linalg.norm(E, axis=1)
|
|
333
|
+
qn = float(np.linalg.norm(q))
|
|
334
|
+
if qn == 0.0 or not np.isfinite(qn) or not np.isfinite(en).all() or (en == 0).any():
|
|
335
|
+
raise CutUnavailable("a zero-length or non-finite vector has no direction to compare")
|
|
336
|
+
return np.clip((E @ q) / (en * qn), 0.0, None)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def select_by_cosine(item_embs, query_emb) -> np.ndarray:
|
|
340
|
+
"""Which of a candidate horizon to keep for this query. Returns ascending indices.
|
|
341
|
+
|
|
342
|
+
The count is read from the candidates' own similarity spectrum: cut at the largest
|
|
343
|
+
relative gap at or above the median (:func:`~entroptics_context.lock.top_break`). No fixed
|
|
344
|
+
*k*, no similarity threshold, nothing fitted to a corpus, and no additional model call —
|
|
345
|
+
the similarities are ones the retriever already computed.
|
|
346
|
+
|
|
347
|
+
``n <= 1`` returns every index. One candidate has no spectrum, and discarding the only
|
|
348
|
+
evidence there is on the strength of a read that never happened would be worse than
|
|
349
|
+
keeping it.
|
|
350
|
+
|
|
351
|
+
What it is for
|
|
352
|
+
--------------
|
|
353
|
+
The best fixed *k* is a different number on each corpus, so a service that answers
|
|
354
|
+
against more than one of them, or against a corpus whose content moves, has no single
|
|
355
|
+
right constant to ship. This reads the number per query instead, and the measured
|
|
356
|
+
consequence is a **bound on what that costs**: across five BEIR corpora the best single
|
|
357
|
+
constant gives up as much as 0.086 F1 against a corpus's own tuned *k*, where this gives
|
|
358
|
+
up at most 0.019. ``research/PAPER.md`` has the measurement.
|
|
359
|
+
|
|
360
|
+
Where a constant is better
|
|
361
|
+
--------------------------
|
|
362
|
+
One corpus, stable, with relevance labels to sweep against. Find the *k* and ship it —
|
|
363
|
+
it is cheaper than this and at least as good. What this is for is the case where that
|
|
364
|
+
sweep is not available.
|
|
365
|
+
"""
|
|
366
|
+
E = np.asarray(item_embs, dtype=np.float64)
|
|
367
|
+
if E.ndim != 2 or len(E) <= 1:
|
|
368
|
+
return np.arange(len(E))
|
|
369
|
+
keep, _ = top_break(cosine_salience(E, query_emb))
|
|
370
|
+
return np.where(keep)[0]
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""The one seam onto ``entroptics``.
|
|
2
|
+
|
|
3
|
+
Every spectral read in this package arrives here. No other module imports
|
|
4
|
+
``entroptics``, and ``tests/test_no_shadow.py`` enforces that, because the way this
|
|
5
|
+
family of code goes wrong is not a wrong formula — it is two copies of the right one
|
|
6
|
+
drifting a percent apart and neither being wrong enough to notice.
|
|
7
|
+
|
|
8
|
+
What the engine is asked for
|
|
9
|
+
----------------------------
|
|
10
|
+
========================= ============================================================
|
|
11
|
+
``resolved_rank(M)`` how many directions ``M`` really has: singular values above
|
|
12
|
+
the derived noise floor. Whitened, native resolution.
|
|
13
|
+
``raw_signal_power(W)`` each row's energy in the top-``K`` left singular modes of
|
|
14
|
+
the **raw** screen, ``K`` from ``resolved_rank``. The
|
|
15
|
+
retrieval path — see "two substrates" below.
|
|
16
|
+
``occupancy(M)`` ``2^{H_sv}/n``, the fraction of modes carrying the energy.
|
|
17
|
+
``focus(E)`` ``sigma_1^2 / M``, the power fraction on the leading axis.
|
|
18
|
+
========================= ============================================================
|
|
19
|
+
|
|
20
|
+
Two substrates, read two ways
|
|
21
|
+
------------------------------------------
|
|
22
|
+
The library's resolved read whitens each feature channel to a common noise scale
|
|
23
|
+
before decomposing, which is right when the channels are *incommensurate* — a KV
|
|
24
|
+
head's ``head_dim`` coordinates carry arbitrary learned scales, and a channel that
|
|
25
|
+
happens to be large is not thereby more informative.
|
|
26
|
+
|
|
27
|
+
A retrieval head screen is the opposite case. ``W[item, head]`` is a per-head cosine:
|
|
28
|
+
every entry is already in the same units, on the same bounded interval, and the
|
|
29
|
+
per-head magnitude *is* the relevance. Whitening it would divide exactly the signal
|
|
30
|
+
out. So ``raw_signal_power`` takes its modes off the raw screen.
|
|
31
|
+
|
|
32
|
+
It still takes its *count* from the whitened frame, because the noise floor is a
|
|
33
|
+
statement about unit-noise channels and is not meaningful otherwise. That mix — rank
|
|
34
|
+
from the whitened frame, modes from the raw one — is what the shipped implementation
|
|
35
|
+
does and what the measured retrieval win was measured on. It is stated here rather
|
|
36
|
+
than left to be discovered in the arithmetic.
|
|
37
|
+
|
|
38
|
+
Where this sits relative to the other copies
|
|
39
|
+
--------------------------------------------
|
|
40
|
+
``mantle.search.beacon.engine`` is a numpy-only reimplementation of the same reads,
|
|
41
|
+
carried in the Agience tree so a store can ship without this dependency. The two are
|
|
42
|
+
not assumed to agree; ``tests/test_conformance_mantle.py`` measures it, and the one
|
|
43
|
+
place they genuinely diverge is recorded there with the measured size: a channel of
|
|
44
|
+
measured zeros is a mode the record HAS, which ``occupancy`` counts in its denominator
|
|
45
|
+
and mantle's ``occupancy_fraction`` drops as a dead line.
|
|
46
|
+
"""
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import numpy as np
|
|
50
|
+
|
|
51
|
+
from entroptics import resolved_batch
|
|
52
|
+
from entroptics.reads import concentration as _concentration
|
|
53
|
+
from entroptics.reads import phi as _phi
|
|
54
|
+
|
|
55
|
+
__all__ = [
|
|
56
|
+
"CutUnavailable",
|
|
57
|
+
"resolved_rank",
|
|
58
|
+
"raw_signal_power",
|
|
59
|
+
"occupancy",
|
|
60
|
+
"focus",
|
|
61
|
+
"noise_floor",
|
|
62
|
+
"machine_eps",
|
|
63
|
+
"separability",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
#: Passed to every read as the noise floor's false-alarm level. It is the library's own
|
|
67
|
+
#: default and the only externally-set number in this package: not a tuned constant but
|
|
68
|
+
#: a stated tolerance — how often the floor is allowed to admit a mode that is noise.
|
|
69
|
+
FAR = 0.05
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class CutUnavailable(RuntimeError):
|
|
73
|
+
"""A read could not be taken. Callers keep a fixed-cut fallback path.
|
|
74
|
+
|
|
75
|
+
NOT a "no signal" result. A read that happened and found one coherent direction
|
|
76
|
+
returns 1; a read that did not happen raises. Conflating the two is how a fallback
|
|
77
|
+
becomes the product — every caller that catches this must do something
|
|
78
|
+
visibly different from what it does on a rank of 1.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _frame(M) -> np.ndarray:
|
|
83
|
+
A = np.asarray(M, dtype=np.float64)
|
|
84
|
+
if A.ndim != 2:
|
|
85
|
+
raise CutUnavailable(f"a screen must be 2-D; got shape {A.shape}")
|
|
86
|
+
if A.size == 0 or min(A.shape) < 2:
|
|
87
|
+
raise CutUnavailable(f"a {A.shape} frame has no spectrum to read")
|
|
88
|
+
return A
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _read(M, *, energy: bool = False, basis: bool = False):
|
|
92
|
+
"""One resolved read of a single frame, at native feature resolution.
|
|
93
|
+
|
|
94
|
+
``fold=False`` throughout. The feature axis here is always an unordered learned
|
|
95
|
+
basis — embedding coordinates, a head's ``head_dim``, or the head index of a
|
|
96
|
+
query-relative screen — and the library's area-average fold is valid only for a
|
|
97
|
+
dense smooth continuum. ``fold="auto"`` would reach the same conclusion per frame;
|
|
98
|
+
pinning it says so once instead of relying on it every call.
|
|
99
|
+
"""
|
|
100
|
+
A = _frame(M)
|
|
101
|
+
try:
|
|
102
|
+
return resolved_batch(A[None], fold=False, far=FAR, energy=energy, basis=basis)
|
|
103
|
+
except Exception as exc: # a genuinely unreadable frame
|
|
104
|
+
raise CutUnavailable(f"resolved read failed on a {A.shape} frame: {exc!r}") from exc
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def resolved_rank(M) -> int:
|
|
108
|
+
"""How many directions ``M`` has: singular values above the derived noise floor.
|
|
109
|
+
|
|
110
|
+
Floored at 1. A caller here builds a subspace out of the answer immediately and
|
|
111
|
+
cannot defer, so a genuine count of 0 becomes the coarsest usable basis rather
|
|
112
|
+
than an empty one. The floor is this package's policy, not the engine's — the
|
|
113
|
+
engine reports 0 when it means 0, which is why this wrapper is where the floor
|
|
114
|
+
lives and not somewhere it could be applied twice.
|
|
115
|
+
"""
|
|
116
|
+
return max(1, int(_read(M).K_signal[0]))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def raw_signal_power(W) -> np.ndarray:
|
|
120
|
+
"""``(n,)`` — each row's energy in the top-``K`` left singular modes of the raw ``W``.
|
|
121
|
+
|
|
122
|
+
``K = resolved_rank(W)``, read on the whitened frame; the modes are taken off ``W``
|
|
123
|
+
as given. See the module docstring for why this substrate is not whitened.
|
|
124
|
+
|
|
125
|
+
A row aligned with the coherent structure the screen actually has carries the
|
|
126
|
+
power; a row that matches weakly and evenly does not. That is the whole difference
|
|
127
|
+
between this and a row-sum, and therefore between an adaptive cut and a re-ranking.
|
|
128
|
+
"""
|
|
129
|
+
A = _frame(W)
|
|
130
|
+
k = min(resolved_rank(A), min(A.shape))
|
|
131
|
+
U, S, _ = np.linalg.svd(A, full_matrices=False)
|
|
132
|
+
return ((U[:, :k] * S[:k]) ** 2).sum(1)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def occupancy(M) -> float:
|
|
136
|
+
"""``2^{H_sv}/n`` in (0, 1] — the fraction of ``M``'s modes that carry the energy.
|
|
137
|
+
|
|
138
|
+
Read on the block as recorded. A constant level is a mode and is counted as one.
|
|
139
|
+
|
|
140
|
+
This read and ``mantle.search.beacon.engine.occupancy_fraction`` agree exactly —
|
|
141
|
+
measured over 60 real retrieval horizons, a maximum absolute difference of 0.0.
|
|
142
|
+
They diverged until 2026-09-02, when the library still centred before decomposing:
|
|
143
|
+
on a set of unit direction vectors the shared mean direction is the topic the set
|
|
144
|
+
is about, so de-meaning deleted the dominant mode and roughly doubled the fill.
|
|
145
|
+
That reached this package as twice the head count, and
|
|
146
|
+
therefore as a different cut.
|
|
147
|
+
|
|
148
|
+
NaN for a frame carrying no power: there is no fraction of active modes to report
|
|
149
|
+
when no mode is active, and reporting ``1/n`` instead would be indistinguishable
|
|
150
|
+
from a perfect single mode.
|
|
151
|
+
"""
|
|
152
|
+
return float(_phi(_frame(M)))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def focus(E) -> float:
|
|
156
|
+
"""``sigma_1^2 / M`` in (0, 1] — the power fraction on the set's leading axis.
|
|
157
|
+
|
|
158
|
+
The bounded MAGNITUDE companion to the rank read: :func:`resolved_rank` answers
|
|
159
|
+
whether structure is present, this answers how concentrated it is. 1 = the set
|
|
160
|
+
points one way; toward 0 = isotropic. Axial, so an antipodal cloud still reads ~1.
|
|
161
|
+
|
|
162
|
+
A diagnostic rather than a decision. The concentration of a kept set tracks cut
|
|
163
|
+
reliability only weakly, and the head screen's own concentration runs weakly the
|
|
164
|
+
other way: a query matching everything is less discriminable. Use it to rank or
|
|
165
|
+
compare sets, or to report a confidence beside a selection, and read the cut
|
|
166
|
+
itself from the lock.
|
|
167
|
+
"""
|
|
168
|
+
A = np.asarray(E, dtype=np.float64)
|
|
169
|
+
if A.ndim != 2 or len(A) < 1:
|
|
170
|
+
return 0.0
|
|
171
|
+
return float(_concentration(A).focus)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def separability(salience) -> float:
|
|
175
|
+
"""``eta^2`` in [0, 1] — how cleanly a salience profile falls into two populations.
|
|
176
|
+
|
|
177
|
+
0.0 means one population and nothing to cut at; toward 1.0 means two groups plainly
|
|
178
|
+
apart. Delegates to ``entroptics.level_edge``.
|
|
179
|
+
|
|
180
|
+
This is the **whether**, and the lock's ``rel_gap`` is the **where**. They are not
|
|
181
|
+
interchangeable and neither substitutes for the other. Measured on 50 structured
|
|
182
|
+
against 50 structureless candidate pools, as a detector of "is there anything here":
|
|
183
|
+
this separates structured from structureless pools better than ``rel_gap``, whose
|
|
184
|
+
ranges cross more broadly. Better, and not a solution — a structureless pool still
|
|
185
|
+
scores well above zero, so this
|
|
186
|
+
is evidence to report and not a gate to trust. It also measures *cleanness*, not
|
|
187
|
+
distance: an exact two-level profile reads 1.000 whatever the ratio across it, so a
|
|
188
|
+
caller who wants the size of the break still wants ``rel_gap``.
|
|
189
|
+
|
|
190
|
+
Report it beside a selection. It is not a gate here, because turning it into one means
|
|
191
|
+
choosing a threshold and that is the thing this package does not do.
|
|
192
|
+
"""
|
|
193
|
+
from entroptics import level_edge
|
|
194
|
+
return float(level_edge(np.asarray(salience, dtype=np.float64)).separability)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def machine_eps(ref) -> float:
|
|
198
|
+
"""The working dtype's machine epsilon — the smallest relative difference the arithmetic
|
|
199
|
+
can represent, read off the array rather than asserted.
|
|
200
|
+
|
|
201
|
+
Delegates to ``entroptics.entropy.macheps``, which is the library's own answer and
|
|
202
|
+
follows the backend and the compute precision. It is exposed through this seam because
|
|
203
|
+
:mod:`entroptics_context.lock` needs it — a ratio of two computed numbers is not a drop
|
|
204
|
+
unless it clears the round-off of the computation that produced them — and the lock
|
|
205
|
+
does not import the library.
|
|
206
|
+
|
|
207
|
+
This is the same construction as the library's ``resolution_floor`` (``typical * eps``,
|
|
208
|
+
derived on both sides: the scale from the data, the epsilon from the dtype). What the
|
|
209
|
+
lock adds is the term count, since its inputs are sums.
|
|
210
|
+
"""
|
|
211
|
+
from entroptics.entropy import macheps
|
|
212
|
+
return float(macheps(np, np.asarray(ref)))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def noise_floor(M) -> float:
|
|
216
|
+
"""The singular-value floor ``resolved_rank`` counted against, on the whitened frame.
|
|
217
|
+
|
|
218
|
+
Exposed for reporting — "the top mode stood this far above the floor" is the
|
|
219
|
+
evidence behind a rank, and a rank without it is a number with no error bar.
|
|
220
|
+
"""
|
|
221
|
+
return float(_read(M).noise_floor[0])
|