graphify-mesh 0.0.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphify_mesh/__init__.py +20 -0
- graphify_mesh/_version.py +24 -0
- graphify_mesh/server/__init__.py +61 -0
- graphify_mesh/server/anchors.py +223 -0
- graphify_mesh/server/citation.py +17 -0
- graphify_mesh/server/config.py +195 -0
- graphify_mesh/server/context_pack.py +212 -0
- graphify_mesh/server/embed_query.py +206 -0
- graphify_mesh/server/frames.py +93 -0
- graphify_mesh/server/http_app.py +367 -0
- graphify_mesh/server/lexical_read.py +208 -0
- graphify_mesh/server/project_map.py +92 -0
- graphify_mesh/server/ranking.py +209 -0
- graphify_mesh/server/retrieval.py +343 -0
- graphify_mesh/server/rwlock.py +61 -0
- graphify_mesh/server/scope.py +207 -0
- graphify_mesh/server/sdk_app.py +108 -0
- graphify_mesh/server/server.py +813 -0
- graphify_mesh/server/similar.py +284 -0
- graphify_mesh/server/stdio_guard.py +287 -0
- graphify_mesh/server/store.py +733 -0
- graphify_mesh/server/traverse.py +305 -0
- graphify_mesh/sync/__init__.py +26 -0
- graphify_mesh/sync/backend.py +80 -0
- graphify_mesh/sync/cli.py +249 -0
- graphify_mesh/sync/clustering.py +182 -0
- graphify_mesh/sync/config.py +815 -0
- graphify_mesh/sync/discovery.py +668 -0
- graphify_mesh/sync/embed_similarity.py +516 -0
- graphify_mesh/sync/embedding.py +1372 -0
- graphify_mesh/sync/graphify_cli.py +941 -0
- graphify_mesh/sync/lexical_index.py +293 -0
- graphify_mesh/sync/locking.py +30 -0
- graphify_mesh/sync/naming.py +563 -0
- graphify_mesh/sync/naming_state.py +143 -0
- graphify_mesh/sync/overlay.py +146 -0
- graphify_mesh/sync/overlay_api.py +312 -0
- graphify_mesh/sync/overlay_depends.py +324 -0
- graphify_mesh/sync/overlay_refs.py +143 -0
- graphify_mesh/sync/overlay_similar.py +357 -0
- graphify_mesh/sync/perms.py +116 -0
- graphify_mesh/sync/pipeline.py +1891 -0
- graphify_mesh/sync/progress.py +41 -0
- graphify_mesh/sync/publish.py +374 -0
- graphify_mesh/sync/reaper.py +201 -0
- graphify_mesh/sync/reaper_cli.py +62 -0
- graphify_mesh/sync/registry.py +184 -0
- graphify_mesh/sync/repo_tags.py +299 -0
- graphify_mesh/sync/rss.py +68 -0
- graphify_mesh/sync/source_cache.py +149 -0
- graphify_mesh/sync/state.py +268 -0
- graphify_mesh/sync/sync_project.py +793 -0
- graphify_mesh/sync/tls.py +119 -0
- graphify_mesh/sync/validate.py +277 -0
- graphify_mesh/sync/vectors.py +234 -0
- graphify_mesh-0.0.22.dist-info/METADATA +219 -0
- graphify_mesh-0.0.22.dist-info/RECORD +60 -0
- graphify_mesh-0.0.22.dist-info/WHEEL +4 -0
- graphify_mesh-0.0.22.dist-info/entry_points.txt +4 -0
- graphify_mesh-0.0.22.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""WS5 fusion + ranking contract: pinned RRF constants, hub/DEPRECATED
|
|
2
|
+
penalties, and MMR diversification. Every constant here is named and
|
|
3
|
+
documented, per project style rule (no magic numbers).
|
|
4
|
+
|
|
5
|
+
Fusion contract (plan WS5 bullet 3):
|
|
6
|
+
* Reciprocal Rank Fusion (RRF) combines however many retrievers actually
|
|
7
|
+
returned candidates this call (lexical always; vector only if the
|
|
8
|
+
generation has embeddings; structural/community always).
|
|
9
|
+
* Exact FQCN/label hits (resolved via the lexical index's `alias_exact`
|
|
10
|
+
table, see `retrieval.exact_alias_hits`) bypass fusion, the hub-degree
|
|
11
|
+
penalty, and MMR entirely — they are ranked first, deterministically.
|
|
12
|
+
* If a retriever has nothing to contribute this generation (most notably:
|
|
13
|
+
no embeddings at all, cold start / Ollama down during the sync run),
|
|
14
|
+
RRF simply sums over whichever retrievers DID return something — there
|
|
15
|
+
is no per-retriever normalization constant that assumes a fixed retriever
|
|
16
|
+
count, so "renormalizing over available retrievers" falls out for free;
|
|
17
|
+
the caller is responsible for surfacing the `degraded` flag (e.g.
|
|
18
|
+
`"embeddings_unavailable"`) so this is visible to the client, never
|
|
19
|
+
silently absorbed.
|
|
20
|
+
* Deterministic tie-breaking: candidates with an identical fused score are
|
|
21
|
+
ordered by their logical-ref string (`repo\x1fsource_file\x1flabel`),
|
|
22
|
+
so repeated identical queries return identical order regardless of dict
|
|
23
|
+
iteration order.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
# --- RRF (Reciprocal Rank Fusion) ------------------------------------------
|
|
29
|
+
|
|
30
|
+
# Standard RRF smoothing constant (Cormack et al. 2009's k=60 is the
|
|
31
|
+
# widely-cited default; kept as a named, overridable constant rather than a
|
|
32
|
+
# bare literal at each call site).
|
|
33
|
+
RRF_K = 60
|
|
34
|
+
|
|
35
|
+
# Per-retriever candidate depth: how many top-ranked candidates each of the
|
|
36
|
+
# lexical/vector/structural retrievers contributes into the fusion pool
|
|
37
|
+
# before RRF combines them. Kept well above the largest supported `k` so
|
|
38
|
+
# fusion has enough material to diversify from.
|
|
39
|
+
CANDIDATE_DEPTH_LEXICAL = 50
|
|
40
|
+
CANDIDATE_DEPTH_VECTOR = 50
|
|
41
|
+
CANDIDATE_DEPTH_STRUCTURAL = 50
|
|
42
|
+
|
|
43
|
+
# --- Ranking penalties -------------------------------------------------
|
|
44
|
+
|
|
45
|
+
# Hub/degree penalty: nodes above this total-degree threshold are generic
|
|
46
|
+
# hubs (CacheKey/User/Where-style god-nodes that flooded every baseline
|
|
47
|
+
# probe — see graphify/baseline-2026-07-20.md probes 2/11) and get
|
|
48
|
+
# multiplicatively down-weighted rather than excluded outright (a real hit
|
|
49
|
+
# that happens to be a hub should still be findable, just not favored).
|
|
50
|
+
HUB_DEGREE_THRESHOLD = 50
|
|
51
|
+
HUB_PENALTY_FACTOR = 0.5
|
|
52
|
+
|
|
53
|
+
# DEPRECATED-path down-weight (baseline systemic failure #8: DEPRECATED
|
|
54
|
+
# code dominated gamestream API results).
|
|
55
|
+
DEPRECATED_PATH_MARKER = "/DEPRECATED/"
|
|
56
|
+
DEPRECATED_PENALTY_FACTOR = 0.3
|
|
57
|
+
|
|
58
|
+
# Confidence handling is an allowlist: only EXTRACTED edges are followed by
|
|
59
|
+
# default in traversal (structural candidates, similar, neighbors). graphify
|
|
60
|
+
# also emits AMBIGUOUS, scored below INFERRED, so a denylist of INFERRED alone
|
|
61
|
+
# lets the weaker class through. Callers opt in to every non-EXTRACTED edge
|
|
62
|
+
# via `include_inferred=True`. EXTRACTED is the graphify default confidence
|
|
63
|
+
# class for edges that don't carry an explicit `confidence` attribute (see
|
|
64
|
+
# graphify/export.py's `_CONFIDENCE_SCORE_DEFAULTS`).
|
|
65
|
+
CONFIDENCE_EXTRACTED = "EXTRACTED"
|
|
66
|
+
CONFIDENCE_INFERRED = "INFERRED"
|
|
67
|
+
CONFIDENCE_AMBIGUOUS = "AMBIGUOUS"
|
|
68
|
+
|
|
69
|
+
# --- MMR (maximal marginal relevance) diversification -----------------
|
|
70
|
+
|
|
71
|
+
# Trade-off between relevance (fused score) and novelty vs already-selected
|
|
72
|
+
# results. 1.0 = pure relevance (no diversification), 0.0 = pure novelty.
|
|
73
|
+
MMR_LAMBDA = 0.7
|
|
74
|
+
|
|
75
|
+
# --- Tool contracts -----------------------------------------------------
|
|
76
|
+
|
|
77
|
+
MAX_K = 100
|
|
78
|
+
DEFAULT_K = 10
|
|
79
|
+
PAGE_SIZE = 20
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def edge_followable(edge: dict, include_inferred: bool) -> bool:
|
|
83
|
+
"""True when traversal may follow `edge`: EXTRACTED (the default when the
|
|
84
|
+
edge carries no `confidence`), or any confidence once the caller opted in."""
|
|
85
|
+
return include_inferred or edge.get("confidence", CONFIDENCE_EXTRACTED) == CONFIDENCE_EXTRACTED
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def rrf_contribution(rank: int) -> float:
|
|
89
|
+
"""`rank` is 0-indexed position within one retriever's ranked list."""
|
|
90
|
+
return 1.0 / (RRF_K + rank + 1)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def fuse_rankings(rankings: dict[str, list[str]]) -> dict[str, float]:
|
|
94
|
+
"""`rankings`: retriever name -> its ranked list of doc keys (best
|
|
95
|
+
first). Returns doc key -> summed RRF score. Retrievers that returned
|
|
96
|
+
nothing simply don't contribute — no fixed-N normalization assumption."""
|
|
97
|
+
scores: dict[str, float] = {}
|
|
98
|
+
for ranked in rankings.values():
|
|
99
|
+
for idx, key in enumerate(ranked):
|
|
100
|
+
scores[key] = scores.get(key, 0.0) + rrf_contribution(idx)
|
|
101
|
+
return scores
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def apply_penalties(
|
|
105
|
+
doc_key: str,
|
|
106
|
+
base_score: float,
|
|
107
|
+
degree_by_key: dict[str, int],
|
|
108
|
+
path_by_key: dict[str, str],
|
|
109
|
+
) -> float:
|
|
110
|
+
"""Multiplicative hub-degree + DEPRECATED-path penalties. Never applied
|
|
111
|
+
to exact-alias bypass hits (those skip this function entirely — see
|
|
112
|
+
`retrieval.rank_candidates`)."""
|
|
113
|
+
score = base_score
|
|
114
|
+
if degree_by_key.get(doc_key, 0) > HUB_DEGREE_THRESHOLD:
|
|
115
|
+
score *= HUB_PENALTY_FACTOR
|
|
116
|
+
path = path_by_key.get(doc_key, "") or ""
|
|
117
|
+
if DEPRECATED_PATH_MARKER in path:
|
|
118
|
+
score *= DEPRECATED_PENALTY_FACTOR
|
|
119
|
+
return score
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _structural_similarity_proxy(key_a: str, key_b: str, path_by_key: dict[str, str]) -> float:
|
|
123
|
+
"""Cheap, dependency-free doc-doc similarity proxy for MMR, used
|
|
124
|
+
instead of vector cosine so diversification still functions when
|
|
125
|
+
embeddings are unavailable (degraded mode, C9 cold start). Same source
|
|
126
|
+
file => high similarity (near-duplicate results from the same file);
|
|
127
|
+
otherwise 0. Deterministic and requires nothing beyond data already
|
|
128
|
+
loaded for ranking."""
|
|
129
|
+
path_a = path_by_key.get(key_a)
|
|
130
|
+
path_b = path_by_key.get(key_b)
|
|
131
|
+
if path_a and path_a == path_b:
|
|
132
|
+
return 0.9
|
|
133
|
+
return 0.0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def mmr_select(
|
|
137
|
+
scored_candidates: list[tuple[str, float]],
|
|
138
|
+
path_by_key: dict[str, str],
|
|
139
|
+
k: int,
|
|
140
|
+
lam: float = MMR_LAMBDA,
|
|
141
|
+
) -> list[str]:
|
|
142
|
+
"""Greedy MMR selection over an already-scored candidate pool.
|
|
143
|
+
Deterministic: ties broken by the candidate's own key (sorted lexical-
|
|
144
|
+
ref string) at every selection step.
|
|
145
|
+
|
|
146
|
+
Incremental implementation: each remaining candidate carries a running
|
|
147
|
+
`max_sim` against the already-selected set, updated only against the
|
|
148
|
+
item selected in the previous step (max over a growing set == running
|
|
149
|
+
max), so each selection step is one O(n) pass instead of a full re-sort
|
|
150
|
+
that recomputes similarity against ALL selected items. Selection output
|
|
151
|
+
is identical to the previous sort-based implementation: the first pick
|
|
152
|
+
uses the pure-relevance `(-score, key)` order, every later pick
|
|
153
|
+
maximizes `mmr_value` with ties broken by ascending key — exactly the
|
|
154
|
+
ordering `sorted(pool, key=lambda c: (-mmr_value(c), c[0]))[0]`
|
|
155
|
+
produced."""
|
|
156
|
+
if k <= 0:
|
|
157
|
+
return []
|
|
158
|
+
if not scored_candidates:
|
|
159
|
+
return []
|
|
160
|
+
|
|
161
|
+
# Mutable pool entries: [key, score, running max_sim vs selected].
|
|
162
|
+
pool: list[list] = [[key, score, 0.0] for key, score in scored_candidates]
|
|
163
|
+
selected: list[str] = []
|
|
164
|
+
|
|
165
|
+
def _pop(index: int) -> list:
|
|
166
|
+
# O(1) removal: swap with the last entry. Pool order is irrelevant —
|
|
167
|
+
# every pick below scans the whole pool with an explicit
|
|
168
|
+
# deterministic tie-break, never relies on pool order.
|
|
169
|
+
entry = pool[index]
|
|
170
|
+
pool[index] = pool[-1]
|
|
171
|
+
pool.pop()
|
|
172
|
+
return entry
|
|
173
|
+
|
|
174
|
+
def _pick_first() -> None:
|
|
175
|
+
best_index = 0
|
|
176
|
+
best_rank = (-pool[0][1], pool[0][0])
|
|
177
|
+
for index in range(1, len(pool)):
|
|
178
|
+
rank = (-pool[index][1], pool[index][0])
|
|
179
|
+
if rank < best_rank:
|
|
180
|
+
best_rank = rank
|
|
181
|
+
best_index = index
|
|
182
|
+
selected.append(_pop(best_index)[0])
|
|
183
|
+
|
|
184
|
+
def _pick_next() -> None:
|
|
185
|
+
last_selected = selected[-1]
|
|
186
|
+
best_index = 0
|
|
187
|
+
best_rank: tuple[float, str] | None = None
|
|
188
|
+
for index, entry in enumerate(pool):
|
|
189
|
+
sim = _structural_similarity_proxy(entry[0], last_selected, path_by_key)
|
|
190
|
+
if sim > entry[2]:
|
|
191
|
+
entry[2] = sim
|
|
192
|
+
value = lam * entry[1] - (1 - lam) * entry[2]
|
|
193
|
+
rank = (-value, entry[0])
|
|
194
|
+
if best_rank is None or rank < best_rank:
|
|
195
|
+
best_rank = rank
|
|
196
|
+
best_index = index
|
|
197
|
+
selected.append(_pop(best_index)[0])
|
|
198
|
+
|
|
199
|
+
_pick_first()
|
|
200
|
+
while pool and len(selected) < k:
|
|
201
|
+
_pick_next()
|
|
202
|
+
return selected
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def deterministic_sort_key(logical_ref_key: str) -> str:
|
|
206
|
+
"""Canonical tie-break key: the logical ref string itself
|
|
207
|
+
(`repo\x1fsource_file\x1fqualified_label`, same shape as
|
|
208
|
+
`overlay_refs.LogicalRef.to_key()`)."""
|
|
209
|
+
return logical_ref_key
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""WS5 candidate generation + fusion orchestration.
|
|
2
|
+
|
|
3
|
+
Scope filtering happens INSIDE each `*_candidates` function, before any
|
|
4
|
+
ranking/scoring is computed — never as a post-filter of an already-ranked
|
|
5
|
+
list (see `scope.py` module docstring for why that ordering matters).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
from graphify_mesh.server import anchors, lexical_read, ranking
|
|
17
|
+
from graphify_mesh.server.store import Generation
|
|
18
|
+
from graphify_mesh.sync.lexical_index import normalize_alias_query, tokenize_text
|
|
19
|
+
from graphify_mesh.sync.vectors import RepoVectors
|
|
20
|
+
|
|
21
|
+
EmbedQueryFn = Callable[[str], list[float] | None]
|
|
22
|
+
|
|
23
|
+
# Score carried by exact-alias bypass hits. It must dominate every fused
|
|
24
|
+
# score (RRF sums stay below 1.0 even with all three retrievers agreeing)
|
|
25
|
+
# while remaining a finite float: the score is serialized with `json.dumps`,
|
|
26
|
+
# and `float("inf")` would emit the bare token `Infinity`, which strict JSON
|
|
27
|
+
# parsers reject.
|
|
28
|
+
EXACT_MATCH_SCORE = 1_000_000.0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Hit:
|
|
33
|
+
key: str
|
|
34
|
+
repo: str
|
|
35
|
+
source_file: str
|
|
36
|
+
label: str
|
|
37
|
+
node_id: str
|
|
38
|
+
degree: int
|
|
39
|
+
score: float
|
|
40
|
+
match_type: str # "exact" | "anchor" | "fused"
|
|
41
|
+
deprecated: bool
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class RankedResult:
|
|
46
|
+
hits: list[Hit] = field(default_factory=list)
|
|
47
|
+
degraded: list[str] = field(default_factory=list)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def exact_alias_hits(query: str, lexical: dict, repo_filter: frozenset[str] | None) -> list[str]:
|
|
51
|
+
"""Exact FQCN/label bypass candidates (WS5 fusion contract): looked up
|
|
52
|
+
through the version-aware lexical reader, normalized with the SAME
|
|
53
|
+
normalization the index was built with (`normalize_alias_query`)."""
|
|
54
|
+
if not query or not query.strip():
|
|
55
|
+
return []
|
|
56
|
+
norm = normalize_alias_query(query.strip())
|
|
57
|
+
refs = lexical_read.alias_refs(lexical, norm)
|
|
58
|
+
keys = {key for repo, key in refs if repo_filter is None or repo in repo_filter}
|
|
59
|
+
return sorted(keys)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def lexical_candidates(
|
|
63
|
+
query: str,
|
|
64
|
+
lexical: dict,
|
|
65
|
+
repo_filter: frozenset[str] | None,
|
|
66
|
+
depth: int = ranking.CANDIDATE_DEPTH_LEXICAL,
|
|
67
|
+
) -> list[str]:
|
|
68
|
+
"""Rank lexical candidates using version-aware postings and document metadata."""
|
|
69
|
+
tokens = tokenize_text(query)
|
|
70
|
+
if not tokens:
|
|
71
|
+
return []
|
|
72
|
+
total_docs = max(lexical_read.document_count(lexical), 1)
|
|
73
|
+
field_boosts = lexical.get("field_boosts", {})
|
|
74
|
+
|
|
75
|
+
scores: dict[str, float] = {}
|
|
76
|
+
for term in tokens:
|
|
77
|
+
triples = lexical_read.term_postings(lexical, term)
|
|
78
|
+
df = lexical_read.term_doc_freq(lexical, term) or 1
|
|
79
|
+
idf = math.log(1 + (total_docs / df))
|
|
80
|
+
for repo, key, field_name in triples:
|
|
81
|
+
if repo_filter is not None and repo not in repo_filter:
|
|
82
|
+
continue
|
|
83
|
+
if key is None:
|
|
84
|
+
continue
|
|
85
|
+
weight = field_boosts.get(field_name, 1.0)
|
|
86
|
+
scores[key] = scores.get(key, 0.0) + weight * idf
|
|
87
|
+
|
|
88
|
+
ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
89
|
+
return [k for k, _ in ranked[:depth]]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _repo_scores(repo_vectors: RepoVectors, normalized_query: np.ndarray) -> np.ndarray:
|
|
93
|
+
"""Per-repo cosine scores against an already-normalized query vector.
|
|
94
|
+
Mixed-dimension shard vs query carries no comparable signal (same rule
|
|
95
|
+
`cosine_similarity` applied per key) — every key in that repo scores 0
|
|
96
|
+
rather than being dropped from the pool, so it still tie-breaks by key
|
|
97
|
+
like the old per-key comparison did."""
|
|
98
|
+
if repo_vectors.dim != normalized_query.shape[0]:
|
|
99
|
+
return np.zeros(len(repo_vectors), dtype=np.float32)
|
|
100
|
+
return repo_vectors.normalized() @ normalized_query
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _embeddings_degraded_marker(
|
|
104
|
+
embeddings: dict[str, RepoVectors], embed_query_fn: EmbedQueryFn
|
|
105
|
+
) -> str:
|
|
106
|
+
"""The `degraded` entry for a call where the vector retriever contributed
|
|
107
|
+
nothing.
|
|
108
|
+
|
|
109
|
+
Two very different situations used to collapse onto the same bare
|
|
110
|
+
`"embeddings_unavailable"` string: a generation that published no vectors
|
|
111
|
+
at all (a build/config problem) and a query-time embed call that failed
|
|
112
|
+
(a backend problem). A client could not tell them apart, which is the
|
|
113
|
+
whole reason a wedged backend looked like a misconfiguration for a day.
|
|
114
|
+
|
|
115
|
+
The bare marker is kept for the not-published case — `store` already
|
|
116
|
+
reports it that way for the same condition, and it stays the stable
|
|
117
|
+
string those checks match. A query-time failure instead reports
|
|
118
|
+
`"embeddings_unavailable:<reason>"`, taking the reason from the
|
|
119
|
+
embedder when it exposes one; a plain callable (the `EmbedQueryFn`
|
|
120
|
+
contract is only `str -> list[float] | None`) has no reason to give, so
|
|
121
|
+
it falls back to the bare marker rather than inventing one.
|
|
122
|
+
"""
|
|
123
|
+
if not embeddings:
|
|
124
|
+
return "embeddings_unavailable"
|
|
125
|
+
reason_fn = getattr(embed_query_fn, "failure_reason", None)
|
|
126
|
+
if not callable(reason_fn):
|
|
127
|
+
return "embeddings_unavailable"
|
|
128
|
+
return f"embeddings_unavailable:{reason_fn()}"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def vector_candidates(
|
|
132
|
+
query: str,
|
|
133
|
+
embeddings: dict[str, RepoVectors],
|
|
134
|
+
repo_filter: frozenset[str] | None,
|
|
135
|
+
embed_query_fn: EmbedQueryFn,
|
|
136
|
+
depth: int = ranking.CANDIDATE_DEPTH_VECTOR,
|
|
137
|
+
) -> tuple[list[str], bool]:
|
|
138
|
+
"""Returns (ranked keys, degraded). `degraded=True` means the vector
|
|
139
|
+
retriever contributed nothing this call (no embeddings published this
|
|
140
|
+
generation, or the query-embed call failed) — the caller renormalizes
|
|
141
|
+
fusion over whichever OTHER retrievers succeeded and must surface the
|
|
142
|
+
marker `_embeddings_degraded_marker` builds in the response's `degraded`
|
|
143
|
+
field, which distinguishes those two causes.
|
|
144
|
+
|
|
145
|
+
Scoring is matrix cosine similarity per repo: each `RepoVectors`
|
|
146
|
+
matrix's L2-normalized copy is cached on the instance
|
|
147
|
+
(`RepoVectors.normalized()`, computed once per process no matter how
|
|
148
|
+
many queries hit it this session) and multiplied against the
|
|
149
|
+
L2-normalized query vector in one `@` call — equivalent to calling
|
|
150
|
+
`cosine_similarity` per key, just batched over the whole repo instead
|
|
151
|
+
of a Python-level loop.
|
|
152
|
+
"""
|
|
153
|
+
if not embeddings:
|
|
154
|
+
return [], True
|
|
155
|
+
vector = embed_query_fn(query)
|
|
156
|
+
if not vector:
|
|
157
|
+
return [], True
|
|
158
|
+
|
|
159
|
+
query_vec = np.asarray(vector, dtype=np.float32)
|
|
160
|
+
query_norm = float(np.linalg.norm(query_vec))
|
|
161
|
+
# Mirror `cosine_similarity`'s own zero-vector rule (score 0 rather
|
|
162
|
+
# than raising/NaN) via a safe divisor instead of skipping the whole
|
|
163
|
+
# query — a literal all-zero query vector should still tie-break by
|
|
164
|
+
# key, exactly like the old per-key comparison did.
|
|
165
|
+
safe_query_norm = query_norm if query_norm != 0.0 else 1.0
|
|
166
|
+
normalized_query = query_vec / safe_query_norm
|
|
167
|
+
|
|
168
|
+
# Per repo, pull only the top-`depth` scores via np.argpartition instead
|
|
169
|
+
# of materializing (key, score) for EVERY embedded node across all repos
|
|
170
|
+
# and full-sorting that one giant Python list per query. Boundary ties
|
|
171
|
+
# are kept (everything scoring >= the depth-th largest value), so the
|
|
172
|
+
# merged shortlist provably contains every key the old global sort could
|
|
173
|
+
# have selected — the final sort below then applies the exact same
|
|
174
|
+
# tie-break semantics (score desc, then key) on <= depth x n_repos
|
|
175
|
+
# candidates (plus ties).
|
|
176
|
+
scored: list[tuple[str, float]] = []
|
|
177
|
+
for repo_id, repo_vectors in embeddings.items():
|
|
178
|
+
if repo_filter is not None and repo_id not in repo_filter:
|
|
179
|
+
continue
|
|
180
|
+
if len(repo_vectors) == 0:
|
|
181
|
+
continue
|
|
182
|
+
scores = _repo_scores(repo_vectors, normalized_query)
|
|
183
|
+
n_rows = scores.shape[0]
|
|
184
|
+
if depth >= n_rows:
|
|
185
|
+
top_idx = np.arange(n_rows)
|
|
186
|
+
else:
|
|
187
|
+
partition = np.argpartition(scores, -depth)
|
|
188
|
+
threshold = scores[partition[-depth]]
|
|
189
|
+
top_idx = np.nonzero(scores >= threshold)[0]
|
|
190
|
+
for idx in top_idx:
|
|
191
|
+
scored.append((repo_vectors.keys[int(idx)], float(scores[int(idx)])))
|
|
192
|
+
scored.sort(key=lambda kv: (-kv[1], kv[0]))
|
|
193
|
+
return [k for k, _ in scored[:depth]], False
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def structural_candidates(
|
|
197
|
+
seed_keys: list[str],
|
|
198
|
+
generation: Generation,
|
|
199
|
+
repo_filter: frozenset[str] | None,
|
|
200
|
+
include_inferred: bool = False,
|
|
201
|
+
depth: int = ranking.CANDIDATE_DEPTH_STRUCTURAL,
|
|
202
|
+
) -> list[str]:
|
|
203
|
+
"""1-hop neighbors of `seed_keys` in the structural graph. Non-EXTRACTED
|
|
204
|
+
(INFERRED/AMBIGUOUS) edges are excluded by default (only EXTRACTED-confidence
|
|
205
|
+
edges count as traversal candidates unless the caller opts in) — baseline
|
|
206
|
+
systemic failure #4 (INFERRED edge pollution)."""
|
|
207
|
+
proximity: dict[str, int] = {}
|
|
208
|
+
for seed_key in seed_keys:
|
|
209
|
+
seed_id = generation.node_id_by_key.get(seed_key)
|
|
210
|
+
if seed_id is None:
|
|
211
|
+
continue
|
|
212
|
+
for neighbor_id, edge in generation.adjacency.get(seed_id, []):
|
|
213
|
+
if not ranking.edge_followable(edge, include_inferred):
|
|
214
|
+
continue
|
|
215
|
+
neighbor_key = generation.key_by_node_id.get(neighbor_id)
|
|
216
|
+
if neighbor_key is None:
|
|
217
|
+
continue
|
|
218
|
+
neighbor_node = generation.node_by_id.get(neighbor_id, {})
|
|
219
|
+
if repo_filter is not None and neighbor_node.get("repo") not in repo_filter:
|
|
220
|
+
continue
|
|
221
|
+
proximity[neighbor_key] = proximity.get(neighbor_key, 0) + 1
|
|
222
|
+
|
|
223
|
+
ranked = sorted(proximity.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
224
|
+
return [k for k, _ in ranked[:depth]]
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _hit_from_key(key: str, generation: Generation, score: float, match_type: str) -> Hit | None:
|
|
228
|
+
node_id = generation.node_id_by_key.get(key)
|
|
229
|
+
if node_id is None:
|
|
230
|
+
return None
|
|
231
|
+
node = generation.node_by_id.get(node_id, {})
|
|
232
|
+
source_file = node.get("source_file") or ""
|
|
233
|
+
return Hit(
|
|
234
|
+
key=key,
|
|
235
|
+
repo=node.get("repo", ""),
|
|
236
|
+
source_file=source_file,
|
|
237
|
+
label=node.get("label", ""),
|
|
238
|
+
node_id=node_id,
|
|
239
|
+
degree=generation.degree(node_id),
|
|
240
|
+
score=score,
|
|
241
|
+
match_type=match_type,
|
|
242
|
+
deprecated=ranking.DEPRECATED_PATH_MARKER in source_file,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def rank(
|
|
247
|
+
query: str,
|
|
248
|
+
generation: Generation,
|
|
249
|
+
repo_filter: frozenset[str] | None,
|
|
250
|
+
k: int,
|
|
251
|
+
embed_query_fn: EmbedQueryFn,
|
|
252
|
+
include_inferred: bool = False,
|
|
253
|
+
) -> RankedResult:
|
|
254
|
+
"""Top-level WS5 search orchestration: exact-alias bypass, symbol
|
|
255
|
+
anchors (`anchors.py`), then lexical+vector+structural candidate
|
|
256
|
+
generation, RRF fusion, hub/DEPRECATED penalties, MMR diversification,
|
|
257
|
+
deterministic tie-break."""
|
|
258
|
+
k = max(1, min(k, ranking.MAX_K))
|
|
259
|
+
# Store-level degraded markers (reload rejections, embeddings stamp
|
|
260
|
+
# problems, ...) are merged into every tool response by the server layer
|
|
261
|
+
# (`GraphifyMeshServer.call_tool`), not read out of the manifest here.
|
|
262
|
+
degraded: list[str] = []
|
|
263
|
+
|
|
264
|
+
exact_keys = exact_alias_hits(query, generation.lexical, repo_filter)
|
|
265
|
+
exact_hits = [
|
|
266
|
+
h
|
|
267
|
+
for h in (_hit_from_key(k_, generation, EXACT_MATCH_SCORE, "exact") for k_ in exact_keys)
|
|
268
|
+
if h
|
|
269
|
+
]
|
|
270
|
+
exact_hits.sort(key=lambda h: h.key)
|
|
271
|
+
# An alias shared by many nodes must not blow past the caller's `k`;
|
|
272
|
+
# truncating after the sort keeps which hits survive deterministic.
|
|
273
|
+
del exact_hits[k:]
|
|
274
|
+
selected_keys = {h.key for h in exact_hits}
|
|
275
|
+
remaining_slots = max(0, k - len(exact_hits))
|
|
276
|
+
|
|
277
|
+
anchor_hits: list[Hit] = []
|
|
278
|
+
anchor_key_set: set[str] = set()
|
|
279
|
+
if remaining_slots > 0:
|
|
280
|
+
for key in anchors.anchor_keys(query, generation, repo_filter, exclude=selected_keys):
|
|
281
|
+
hit = _hit_from_key(key, generation, anchors.ANCHOR_MATCH_SCORE, "anchor")
|
|
282
|
+
if hit:
|
|
283
|
+
anchor_hits.append(hit)
|
|
284
|
+
del anchor_hits[remaining_slots:]
|
|
285
|
+
# Anchors stay in the fused pool so MMR diversifies against them exactly
|
|
286
|
+
# as when they rank there on their own; they are dropped after selection.
|
|
287
|
+
anchor_key_set = {h.key for h in anchor_hits}
|
|
288
|
+
remaining_slots -= len(anchor_hits)
|
|
289
|
+
|
|
290
|
+
fused_hits: list[Hit] = []
|
|
291
|
+
if remaining_slots > 0:
|
|
292
|
+
lexical_ranked = lexical_candidates(query, generation.lexical, repo_filter)
|
|
293
|
+
vector_ranked, vec_degraded = vector_candidates(
|
|
294
|
+
query, generation.embeddings, repo_filter, embed_query_fn
|
|
295
|
+
)
|
|
296
|
+
if vec_degraded:
|
|
297
|
+
vec_marker = _embeddings_degraded_marker(generation.embeddings, embed_query_fn)
|
|
298
|
+
if vec_marker not in degraded:
|
|
299
|
+
degraded.append(vec_marker)
|
|
300
|
+
|
|
301
|
+
seed_for_structural = (exact_keys + lexical_ranked)[:10]
|
|
302
|
+
structural_ranked = structural_candidates(
|
|
303
|
+
seed_for_structural, generation, repo_filter, include_inferred=include_inferred
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
rankings = {
|
|
307
|
+
"lexical": lexical_ranked,
|
|
308
|
+
"vector": vector_ranked,
|
|
309
|
+
"structural": structural_ranked,
|
|
310
|
+
}
|
|
311
|
+
fused_scores = ranking.fuse_rankings(rankings)
|
|
312
|
+
for excluded in selected_keys:
|
|
313
|
+
fused_scores.pop(excluded, None)
|
|
314
|
+
|
|
315
|
+
degree_by_key = {
|
|
316
|
+
key: generation.degree(generation.node_id_by_key[key])
|
|
317
|
+
for key in fused_scores
|
|
318
|
+
if key in generation.node_id_by_key
|
|
319
|
+
}
|
|
320
|
+
path_by_key = {
|
|
321
|
+
key: generation.node_by_id.get(generation.node_id_by_key.get(key, ""), {}).get(
|
|
322
|
+
"source_file", ""
|
|
323
|
+
)
|
|
324
|
+
for key in fused_scores
|
|
325
|
+
if key in generation.node_id_by_key
|
|
326
|
+
}
|
|
327
|
+
penalized = [
|
|
328
|
+
(key, ranking.apply_penalties(key, score, degree_by_key, path_by_key))
|
|
329
|
+
for key, score in fused_scores.items()
|
|
330
|
+
]
|
|
331
|
+
diversified_keys = ranking.mmr_select(
|
|
332
|
+
penalized, path_by_key, remaining_slots + len(anchor_key_set)
|
|
333
|
+
)
|
|
334
|
+
diversified_keys = [key for key in diversified_keys if key not in anchor_key_set][
|
|
335
|
+
:remaining_slots
|
|
336
|
+
]
|
|
337
|
+
final_scores = dict(penalized)
|
|
338
|
+
for key in diversified_keys:
|
|
339
|
+
hit = _hit_from_key(key, generation, final_scores.get(key, 0.0), "fused")
|
|
340
|
+
if hit:
|
|
341
|
+
fused_hits.append(hit)
|
|
342
|
+
|
|
343
|
+
return RankedResult(hits=exact_hits + anchor_hits + fused_hits, degraded=sorted(set(degraded)))
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Writer-preferring read/write lock.
|
|
2
|
+
|
|
3
|
+
The shared daemon answers many reads against one loaded generation while a
|
|
4
|
+
sync publish occasionally replaces it. Reads must not serialize behind each
|
|
5
|
+
other, and the swap must not run while a read is in flight. Writer-preferring
|
|
6
|
+
matters because reads are frequent: a reader-preferring lock would let a
|
|
7
|
+
steady stream of queries postpone the reload indefinitely, and the daemon
|
|
8
|
+
would keep serving a generation the pipeline already replaced.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import threading
|
|
14
|
+
from collections.abc import Iterator
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ReadWriteLock:
|
|
19
|
+
def __init__(self) -> None:
|
|
20
|
+
self._cond = threading.Condition()
|
|
21
|
+
self._readers = 0
|
|
22
|
+
self._writer = False
|
|
23
|
+
self._waiting_writers = 0
|
|
24
|
+
|
|
25
|
+
@contextmanager
|
|
26
|
+
def read(self) -> Iterator[None]:
|
|
27
|
+
with self._cond:
|
|
28
|
+
while self._writer or self._waiting_writers:
|
|
29
|
+
self._cond.wait()
|
|
30
|
+
self._readers += 1
|
|
31
|
+
try:
|
|
32
|
+
yield
|
|
33
|
+
finally:
|
|
34
|
+
with self._cond:
|
|
35
|
+
self._readers -= 1
|
|
36
|
+
if self._readers == 0:
|
|
37
|
+
self._cond.notify_all()
|
|
38
|
+
|
|
39
|
+
@contextmanager
|
|
40
|
+
def write(self) -> Iterator[None]:
|
|
41
|
+
with self._cond:
|
|
42
|
+
self._waiting_writers += 1
|
|
43
|
+
try:
|
|
44
|
+
while self._writer or self._readers:
|
|
45
|
+
self._cond.wait()
|
|
46
|
+
self._writer = True
|
|
47
|
+
finally:
|
|
48
|
+
self._waiting_writers -= 1
|
|
49
|
+
# If `wait()` itself raised before granting the writer (e.g. an
|
|
50
|
+
# injected exception, or interpreter shutdown), threads already
|
|
51
|
+
# blocked in `wait()` would otherwise never be woken by this
|
|
52
|
+
# writer's exit — nothing else releases them. Notifying here
|
|
53
|
+
# too is a no-op cost on the normal path (readers/writers just
|
|
54
|
+
# re-check their own condition and wait again if not runnable).
|
|
55
|
+
self._cond.notify_all()
|
|
56
|
+
try:
|
|
57
|
+
yield
|
|
58
|
+
finally:
|
|
59
|
+
with self._cond:
|
|
60
|
+
self._writer = False
|
|
61
|
+
self._cond.notify_all()
|