graphify-mesh 0.0.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphify_mesh/__init__.py +20 -0
- graphify_mesh/_version.py +24 -0
- graphify_mesh/server/__init__.py +61 -0
- graphify_mesh/server/anchors.py +223 -0
- graphify_mesh/server/citation.py +17 -0
- graphify_mesh/server/config.py +195 -0
- graphify_mesh/server/context_pack.py +212 -0
- graphify_mesh/server/embed_query.py +206 -0
- graphify_mesh/server/frames.py +93 -0
- graphify_mesh/server/http_app.py +367 -0
- graphify_mesh/server/lexical_read.py +208 -0
- graphify_mesh/server/project_map.py +92 -0
- graphify_mesh/server/ranking.py +209 -0
- graphify_mesh/server/retrieval.py +343 -0
- graphify_mesh/server/rwlock.py +61 -0
- graphify_mesh/server/scope.py +207 -0
- graphify_mesh/server/sdk_app.py +108 -0
- graphify_mesh/server/server.py +813 -0
- graphify_mesh/server/similar.py +284 -0
- graphify_mesh/server/stdio_guard.py +287 -0
- graphify_mesh/server/store.py +733 -0
- graphify_mesh/server/traverse.py +305 -0
- graphify_mesh/sync/__init__.py +26 -0
- graphify_mesh/sync/backend.py +80 -0
- graphify_mesh/sync/cli.py +249 -0
- graphify_mesh/sync/clustering.py +182 -0
- graphify_mesh/sync/config.py +815 -0
- graphify_mesh/sync/discovery.py +668 -0
- graphify_mesh/sync/embed_similarity.py +516 -0
- graphify_mesh/sync/embedding.py +1372 -0
- graphify_mesh/sync/graphify_cli.py +941 -0
- graphify_mesh/sync/lexical_index.py +293 -0
- graphify_mesh/sync/locking.py +30 -0
- graphify_mesh/sync/naming.py +563 -0
- graphify_mesh/sync/naming_state.py +143 -0
- graphify_mesh/sync/overlay.py +146 -0
- graphify_mesh/sync/overlay_api.py +312 -0
- graphify_mesh/sync/overlay_depends.py +324 -0
- graphify_mesh/sync/overlay_refs.py +143 -0
- graphify_mesh/sync/overlay_similar.py +357 -0
- graphify_mesh/sync/perms.py +116 -0
- graphify_mesh/sync/pipeline.py +1891 -0
- graphify_mesh/sync/progress.py +41 -0
- graphify_mesh/sync/publish.py +374 -0
- graphify_mesh/sync/reaper.py +201 -0
- graphify_mesh/sync/reaper_cli.py +62 -0
- graphify_mesh/sync/registry.py +184 -0
- graphify_mesh/sync/repo_tags.py +299 -0
- graphify_mesh/sync/rss.py +68 -0
- graphify_mesh/sync/source_cache.py +149 -0
- graphify_mesh/sync/state.py +268 -0
- graphify_mesh/sync/sync_project.py +793 -0
- graphify_mesh/sync/tls.py +119 -0
- graphify_mesh/sync/validate.py +277 -0
- graphify_mesh/sync/vectors.py +234 -0
- graphify_mesh-0.0.22.dist-info/METADATA +219 -0
- graphify_mesh-0.0.22.dist-info/RECORD +60 -0
- graphify_mesh-0.0.22.dist-info/WHEEL +4 -0
- graphify_mesh-0.0.22.dist-info/entry_points.txt +4 -0
- graphify_mesh-0.0.22.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""Streamable-HTTP transport: one shared daemon for every local agent.
|
|
2
|
+
|
|
3
|
+
Stateless by design — `cwd` travels in each tool call, so there is no
|
|
4
|
+
per-session state worth keeping, and a client that disappears leaves nothing
|
|
5
|
+
behind. The bearer token is mandatory: unlike the unix socket this replaces,
|
|
6
|
+
a loopback TCP port is reachable by every local user and process, and what it
|
|
7
|
+
serves is the merged graph of every repository in registry.json.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import contextlib
|
|
13
|
+
import hmac
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from collections import deque
|
|
17
|
+
from collections.abc import AsyncIterator, Callable
|
|
18
|
+
|
|
19
|
+
import anyio
|
|
20
|
+
from anyio import to_thread
|
|
21
|
+
from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
|
|
22
|
+
from mcp.server.transport_security import TransportSecuritySettings
|
|
23
|
+
from starlette.applications import Starlette
|
|
24
|
+
from starlette.middleware import Middleware
|
|
25
|
+
from starlette.routing import Route
|
|
26
|
+
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
|
27
|
+
|
|
28
|
+
from graphify_mesh.server.config import ServerConfig, is_loopback_bind
|
|
29
|
+
from graphify_mesh.server.frames import (
|
|
30
|
+
MAX_MESSAGE_BYTES,
|
|
31
|
+
PARSE_ERROR,
|
|
32
|
+
envelope_error,
|
|
33
|
+
error_frame_text,
|
|
34
|
+
frame_id,
|
|
35
|
+
)
|
|
36
|
+
from graphify_mesh.server.sdk_app import build_sdk_server
|
|
37
|
+
from graphify_mesh.server.server import GraphifyMeshServer
|
|
38
|
+
|
|
39
|
+
log = logging.getLogger("graphify_mesh.server.http_app")
|
|
40
|
+
|
|
41
|
+
# A bound exists because a request being read holds up to `MAX_MESSAGE_BYTES`
|
|
42
|
+
# of body in this one shared process, so unbounded concurrent reads make
|
|
43
|
+
# inbound memory unbounded too. It bounds body buffering specifically, and it
|
|
44
|
+
# QUEUES rather than rejecting: making a local agent's request wait a few
|
|
45
|
+
# milliseconds is better than failing it. Not uvicorn's `limit_concurrency`,
|
|
46
|
+
# which answers 503 once the number of open CONNECTIONS reaches the limit —
|
|
47
|
+
# on a daemon serving many long-lived keep-alive clients that rejects the
|
|
48
|
+
# 33rd client with nothing in flight at all.
|
|
49
|
+
BODY_BUFFER_SLOTS = 8
|
|
50
|
+
|
|
51
|
+
# Bodies above this size are parsed in a worker thread: a multi-megabyte
|
|
52
|
+
# `json.loads` on the event loop stalls every other client of this shared
|
|
53
|
+
# daemon for its whole duration. Below it the thread hop costs more than the
|
|
54
|
+
# parse, so small frames stay inline.
|
|
55
|
+
INLINE_PARSE_MAX_BYTES = 64 * 1024
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _security_settings(config: ServerConfig) -> TransportSecuritySettings:
|
|
59
|
+
"""DNS-rebinding protection, decided by the same rule
|
|
60
|
+
`config.is_loopback_bind` applies to the bind permission. A non-loopback
|
|
61
|
+
bind, a wildcard or a concrete interface address alike, both of which
|
|
62
|
+
require `--allow-public-bind`, is reachable under many names, so it
|
|
63
|
+
accepts any Host and leaves the bearer token as the gate, like the upstream
|
|
64
|
+
`graphify` server (`serve.py:_build_http_app`). A loopback bind restricts
|
|
65
|
+
Host to that address plus the localhost aliases, each with and without
|
|
66
|
+
the port."""
|
|
67
|
+
if not is_loopback_bind(config.http_host):
|
|
68
|
+
return TransportSecuritySettings(enable_dns_rebinding_protection=False)
|
|
69
|
+
allowed = {config.http_host, "localhost", "127.0.0.1"}
|
|
70
|
+
allowed |= {f"{host}:{config.http_port}" for host in list(allowed)}
|
|
71
|
+
return TransportSecuritySettings(allowed_hosts=sorted(allowed))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class _MCPASGIApp:
|
|
75
|
+
"""Raw-ASGI wrapper around the streamable-HTTP session manager, so
|
|
76
|
+
Starlette treats this as the ASGI app for the mount path directly with
|
|
77
|
+
no request/response wrapping that would break the streaming path."""
|
|
78
|
+
|
|
79
|
+
def __init__(self, manager: StreamableHTTPSessionManager) -> None:
|
|
80
|
+
self._manager = manager
|
|
81
|
+
|
|
82
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
83
|
+
await self._manager.handle_request(scope, receive, send)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class _TokenAuthMiddleware:
|
|
87
|
+
"""Mandatory bearer-token gate, ahead of the session manager.
|
|
88
|
+
|
|
89
|
+
Raw ASGI, not Starlette's `BaseHTTPMiddleware`: that middleware buffers
|
|
90
|
+
the response and breaks the Streamable HTTP stream. Never logs the
|
|
91
|
+
token, the Authorization header, or the request body — a rejection logs
|
|
92
|
+
only the remote address and the fact of rejection.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
def __init__(self, app: ASGIApp, token: str) -> None:
|
|
96
|
+
self.app = app
|
|
97
|
+
self._expected = token.encode("utf-8")
|
|
98
|
+
|
|
99
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
100
|
+
if scope["type"] != "http":
|
|
101
|
+
await self.app(scope, receive, send)
|
|
102
|
+
return
|
|
103
|
+
|
|
104
|
+
# The raw list, not a dict: dict() keeps only the last value of a
|
|
105
|
+
# repeated header, so a request carrying both a wrong and a right
|
|
106
|
+
# `authorization` would be accepted or rejected on header order
|
|
107
|
+
# alone. More than one of them is ambiguous, so it is rejected
|
|
108
|
+
# before any comparison runs.
|
|
109
|
+
supplied = [
|
|
110
|
+
value for name, value in (scope.get("headers") or []) if name == b"authorization"
|
|
111
|
+
]
|
|
112
|
+
provided: bytes | None = None
|
|
113
|
+
if len(supplied) == 1:
|
|
114
|
+
scheme, _, candidate = supplied[0].partition(b" ")
|
|
115
|
+
if scheme.lower() == b"bearer" and candidate:
|
|
116
|
+
provided = candidate.strip()
|
|
117
|
+
|
|
118
|
+
if provided is None or not hmac.compare_digest(provided, self._expected):
|
|
119
|
+
client = scope.get("client")
|
|
120
|
+
remote = client[0] if client else "unknown"
|
|
121
|
+
log.warning("rejected unauthorized request from %s", remote)
|
|
122
|
+
body = b'{"error": "unauthorized"}'
|
|
123
|
+
await send(
|
|
124
|
+
{
|
|
125
|
+
"type": "http.response.start",
|
|
126
|
+
"status": 401,
|
|
127
|
+
"headers": [
|
|
128
|
+
(b"content-type", b"application/json"),
|
|
129
|
+
(b"content-length", str(len(body)).encode("ascii")),
|
|
130
|
+
],
|
|
131
|
+
}
|
|
132
|
+
)
|
|
133
|
+
await send({"type": "http.response.body", "body": body})
|
|
134
|
+
return
|
|
135
|
+
|
|
136
|
+
await self.app(scope, receive, send)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def _send_json(send: Send, status: int, body: bytes) -> None:
|
|
140
|
+
await send(
|
|
141
|
+
{
|
|
142
|
+
"type": "http.response.start",
|
|
143
|
+
"status": status,
|
|
144
|
+
"headers": [
|
|
145
|
+
(b"content-type", b"application/json"),
|
|
146
|
+
(b"content-length", str(len(body)).encode("ascii")),
|
|
147
|
+
],
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
await send({"type": "http.response.body", "body": body})
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class _FrameGuardMiddleware:
|
|
154
|
+
"""Size cap and envelope check for POST bodies, between the token gate
|
|
155
|
+
and the session manager.
|
|
156
|
+
|
|
157
|
+
Two things the SDK does differently from `stdio_guard`, both visible to a
|
|
158
|
+
client that got past the token gate:
|
|
159
|
+
|
|
160
|
+
* its body limit is its own default rather than this package's number,
|
|
161
|
+
so the two transports stated different caps;
|
|
162
|
+
* its parse and envelope errors carry `str(JSONDecodeError)` or
|
|
163
|
+
`str(ValidationError)` — validation details and excerpts of the input —
|
|
164
|
+
straight to the client, where stdio answers generically.
|
|
165
|
+
|
|
166
|
+
So the body is read here, under `MAX_MESSAGE_BYTES` counted on bytes
|
|
167
|
+
actually received (a chunked client has no `Content-Length` to check),
|
|
168
|
+
checked with the same `frames.envelope_error` stdio uses, and replayed
|
|
169
|
+
downstream untouched when it is well-formed. The detail goes to the log;
|
|
170
|
+
the client gets a fixed message.
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
def __init__(self, app: ASGIApp) -> None:
|
|
174
|
+
self.app = app
|
|
175
|
+
# One semaphore per app, so a test app and the daemon never share one.
|
|
176
|
+
self._slots = anyio.Semaphore(BODY_BUFFER_SLOTS)
|
|
177
|
+
|
|
178
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
179
|
+
if scope["type"] != "http" or scope.get("method") != "POST":
|
|
180
|
+
await self.app(scope, receive, send)
|
|
181
|
+
return
|
|
182
|
+
|
|
183
|
+
headers = dict(scope.get("headers") or [])
|
|
184
|
+
declared = headers.get(b"content-length")
|
|
185
|
+
if declared is not None and declared.isdigit() and int(declared) > MAX_MESSAGE_BYTES:
|
|
186
|
+
# Nothing is buffered on this path, so it takes no slot.
|
|
187
|
+
await self._too_large(scope, send)
|
|
188
|
+
return
|
|
189
|
+
|
|
190
|
+
await self._slots.acquire()
|
|
191
|
+
released = False
|
|
192
|
+
|
|
193
|
+
def release_slot() -> None:
|
|
194
|
+
# The slot covers the span where THIS middleware holds body
|
|
195
|
+
# bytes, and nothing more: it is handed back as soon as the body
|
|
196
|
+
# reaches the app below, never held across the downstream call,
|
|
197
|
+
# which would cap real request concurrency at `BODY_BUFFER_SLOTS`
|
|
198
|
+
# instead of bounding memory.
|
|
199
|
+
nonlocal released
|
|
200
|
+
if not released:
|
|
201
|
+
released = True
|
|
202
|
+
self._slots.release()
|
|
203
|
+
|
|
204
|
+
try:
|
|
205
|
+
await self._guarded(scope, receive, send, release_slot)
|
|
206
|
+
finally:
|
|
207
|
+
release_slot()
|
|
208
|
+
|
|
209
|
+
async def _guarded(
|
|
210
|
+
self, scope: Scope, receive: Receive, send: Send, release_slot: Callable[[], None]
|
|
211
|
+
) -> None:
|
|
212
|
+
body = bytearray()
|
|
213
|
+
body_complete = False
|
|
214
|
+
trailing: Message | None = None
|
|
215
|
+
while True:
|
|
216
|
+
message = await receive()
|
|
217
|
+
if message["type"] != "http.request":
|
|
218
|
+
trailing = dict(message)
|
|
219
|
+
break
|
|
220
|
+
chunk = message.get("body", b"")
|
|
221
|
+
if len(body) + len(chunk) > MAX_MESSAGE_BYTES:
|
|
222
|
+
# Answer now; the rest of the body is never read, so an
|
|
223
|
+
# over-limit client cannot make this process buffer it.
|
|
224
|
+
await self._too_large(scope, send)
|
|
225
|
+
return
|
|
226
|
+
body.extend(chunk)
|
|
227
|
+
if not message.get("more_body", False):
|
|
228
|
+
body_complete = True
|
|
229
|
+
break
|
|
230
|
+
|
|
231
|
+
try:
|
|
232
|
+
if len(body) > INLINE_PARSE_MAX_BYTES:
|
|
233
|
+
# `json.loads` accepts the bytearray directly, so the thread
|
|
234
|
+
# gets the same single buffer, not a copy of it.
|
|
235
|
+
parsed = await to_thread.run_sync(json.loads, body)
|
|
236
|
+
else:
|
|
237
|
+
parsed = json.loads(body)
|
|
238
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
239
|
+
log.warning("rejected unparseable request body from %s: %s", _remote(scope), exc)
|
|
240
|
+
await _send_json(send, 400, error_frame_text(*PARSE_ERROR).encode("utf-8"))
|
|
241
|
+
return
|
|
242
|
+
|
|
243
|
+
coded = envelope_error(parsed)
|
|
244
|
+
if coded is not None:
|
|
245
|
+
log.warning(
|
|
246
|
+
"rejected malformed JSON-RPC frame from %s (code %d)", _remote(scope), coded[0]
|
|
247
|
+
)
|
|
248
|
+
frame = error_frame_text(*coded, frame_id(parsed))
|
|
249
|
+
await _send_json(send, 400, frame.encode("utf-8"))
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
# Both references are dropped before dispatching downstream: the
|
|
253
|
+
# parsed object would otherwise stay live for the whole call (several
|
|
254
|
+
# times the byte size of the body, as Python objects), and `body` is
|
|
255
|
+
# handed to `_replayed_receive` as-is rather than copied, so exactly
|
|
256
|
+
# one buffer of these bytes exists here at any moment.
|
|
257
|
+
del parsed
|
|
258
|
+
replay = _replayed_receive(body, body_complete, trailing, receive, release_slot)
|
|
259
|
+
del body
|
|
260
|
+
await self.app(scope, replay, send)
|
|
261
|
+
|
|
262
|
+
async def _too_large(self, scope: Scope, send: Send) -> None:
|
|
263
|
+
log.warning(
|
|
264
|
+
"rejected oversized request body from %s (> %d bytes)",
|
|
265
|
+
_remote(scope),
|
|
266
|
+
MAX_MESSAGE_BYTES,
|
|
267
|
+
)
|
|
268
|
+
await _send_json(send, 413, b'{"error": "request body too large"}')
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _remote(scope: Scope) -> str:
|
|
272
|
+
client = scope.get("client")
|
|
273
|
+
return client[0] if client else "unknown"
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _replayed_receive(
|
|
277
|
+
body: bytearray,
|
|
278
|
+
body_complete: bool,
|
|
279
|
+
trailing: Message | None,
|
|
280
|
+
receive: Receive,
|
|
281
|
+
on_handoff: Callable[[], None],
|
|
282
|
+
) -> Receive:
|
|
283
|
+
"""Hands the already-read body to the app below exactly once, then falls
|
|
284
|
+
back to the real `receive` (for the disconnect message).
|
|
285
|
+
|
|
286
|
+
`on_handoff` runs when the body message has been delivered: at that point
|
|
287
|
+
the app below owns these bytes and this layer holds none, which is what
|
|
288
|
+
the body-buffer slot was reserving. The buffer is passed through rather
|
|
289
|
+
than copied to `bytes`; the only consumer is the SDK's
|
|
290
|
+
`RequestBodyLimitMiddleware`, which extends its own bytearray from it and
|
|
291
|
+
never requires an immutable `bytes`.
|
|
292
|
+
"""
|
|
293
|
+
queued: deque[Message] = deque(
|
|
294
|
+
[{"type": "http.request", "body": body, "more_body": not body_complete}]
|
|
295
|
+
)
|
|
296
|
+
if trailing is not None:
|
|
297
|
+
queued.append(trailing)
|
|
298
|
+
|
|
299
|
+
async def replay() -> Message:
|
|
300
|
+
if queued:
|
|
301
|
+
message = queued.popleft()
|
|
302
|
+
if message["type"] == "http.request":
|
|
303
|
+
on_handoff()
|
|
304
|
+
return message
|
|
305
|
+
return await receive()
|
|
306
|
+
|
|
307
|
+
return replay
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def build_http_app(mesh: GraphifyMeshServer, config: ServerConfig) -> Starlette:
|
|
311
|
+
"""The ASGI app for the shared streamable-HTTP daemon. No server
|
|
312
|
+
started — for in-process tests and for `serve_http` to run under
|
|
313
|
+
uvicorn."""
|
|
314
|
+
if not config.http_token:
|
|
315
|
+
raise ValueError("http transport requires config.http_token")
|
|
316
|
+
|
|
317
|
+
sdk = build_sdk_server(mesh)
|
|
318
|
+
security = _security_settings(config)
|
|
319
|
+
|
|
320
|
+
# Stateless: no sessions, no SSE, no session reaper. `cwd` travels in
|
|
321
|
+
# each tool call instead.
|
|
322
|
+
manager = StreamableHTTPSessionManager(
|
|
323
|
+
app=sdk,
|
|
324
|
+
json_response=True,
|
|
325
|
+
stateless=True,
|
|
326
|
+
security_settings=security,
|
|
327
|
+
# The same number stdio caps a line at. Left to its default, the
|
|
328
|
+
# HTTP bound would be the SDK's, which is both a different number
|
|
329
|
+
# and one that can change between SDK releases. `session_idle_timeout`
|
|
330
|
+
# is deliberately NOT passed: the SDK documents it as unused in
|
|
331
|
+
# stateless mode, and a parameter this adapter does not need is one
|
|
332
|
+
# more thing an SDK release can move.
|
|
333
|
+
max_request_body_size=MAX_MESSAGE_BYTES,
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
@contextlib.asynccontextmanager
|
|
337
|
+
async def lifespan(_app: Starlette) -> AsyncIterator[None]:
|
|
338
|
+
# The session manager owns an anyio task group that must wrap the
|
|
339
|
+
# whole server lifetime, so enter it here rather than per-request.
|
|
340
|
+
async with manager.run():
|
|
341
|
+
yield
|
|
342
|
+
|
|
343
|
+
return Starlette(
|
|
344
|
+
routes=[Route(config.http_path, endpoint=_MCPASGIApp(manager))],
|
|
345
|
+
# Order matters: the token gate stays outermost, so an
|
|
346
|
+
# unauthenticated request is answered before this process reads,
|
|
347
|
+
# buffers or parses a single byte of its body.
|
|
348
|
+
middleware=[
|
|
349
|
+
Middleware(_TokenAuthMiddleware, token=config.http_token),
|
|
350
|
+
Middleware(_FrameGuardMiddleware),
|
|
351
|
+
],
|
|
352
|
+
lifespan=lifespan,
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def serve_http(mesh: GraphifyMeshServer, config: ServerConfig) -> None:
|
|
357
|
+
"""Blocking streamable-HTTP transport on the configured host and port."""
|
|
358
|
+
import uvicorn
|
|
359
|
+
|
|
360
|
+
app = build_http_app(mesh, config)
|
|
361
|
+
log.info(
|
|
362
|
+
"graphify-mesh http daemon: host=%s port=%s path=%s token required",
|
|
363
|
+
config.http_host,
|
|
364
|
+
config.http_port,
|
|
365
|
+
config.http_path,
|
|
366
|
+
)
|
|
367
|
+
uvicorn.run(app, host=config.http_host, port=config.http_port)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Version-aware read-path for the `lexical-index.json` bundle.
|
|
2
|
+
|
|
3
|
+
The sync pipeline writes exactly one schema version (the current
|
|
4
|
+
`LEXICAL_SCHEMA_VERSION` in sync/lexical_index.py), but the server must keep
|
|
5
|
+
serving whatever the CURRENT published generation contains — publish can lag
|
|
6
|
+
behind a deploy (e.g. shrink-guard blocking), so the reader understands the
|
|
7
|
+
previous on-disk shape too. All schema branching lives HERE; retrieval.py and
|
|
8
|
+
similar.py consume shape-agnostic tuples.
|
|
9
|
+
|
|
10
|
+
v2 shape: postings `term -> [[repo, key, field], ...]`,
|
|
11
|
+
alias_exact `alias -> [[repo, key], ...]`,
|
|
12
|
+
doc_freq stored under `doc_freq.global`.
|
|
13
|
+
v3 shape: `documents` table (list index = doc id), `fields` table
|
|
14
|
+
(list index = field id), postings `term -> [packed_int, ...]`
|
|
15
|
+
with `packed = (doc_id << FIELD_PACK_BITS) | field_index`,
|
|
16
|
+
alias_exact `alias -> [doc_id, ...]`, doc_freq NOT stored
|
|
17
|
+
(derived: distinct doc ids in the term's postings).
|
|
18
|
+
|
|
19
|
+
Malformed entries are skipped, never raised: a partially-corrupt index
|
|
20
|
+
degrades retrieval quality, it must not take the server down.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import bisect
|
|
26
|
+
from collections.abc import Iterable
|
|
27
|
+
|
|
28
|
+
SCHEMA_V2 = 2
|
|
29
|
+
SCHEMA_V3 = 3
|
|
30
|
+
# Low bits reserved for the field index inside a packed v3 posting entry.
|
|
31
|
+
FIELD_PACK_BITS = 2
|
|
32
|
+
_FIELD_MASK = (1 << FIELD_PACK_BITS) - 1
|
|
33
|
+
|
|
34
|
+
_V2_POSTING_LEN = 3
|
|
35
|
+
_V2_ALIAS_LEN = 2
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _get_dict(container: object, key: str) -> dict:
|
|
39
|
+
"""Fetch `container[key]` and guarantee a dict comes back, no matter how
|
|
40
|
+
malformed `container` or the value at `key` is. Container-level
|
|
41
|
+
hardening: `container` itself may be a non-dict scalar (e.g. the whole
|
|
42
|
+
`lexical` artifact got corrupted into a string), or the value stored at
|
|
43
|
+
`key` may be the wrong shape (a list, an int, ...) — either way this
|
|
44
|
+
degrades to an empty dict rather than raising."""
|
|
45
|
+
if not isinstance(container, dict):
|
|
46
|
+
return {}
|
|
47
|
+
value = container.get(key, {})
|
|
48
|
+
if not isinstance(value, dict):
|
|
49
|
+
return {}
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _get_list(container: object, key: str) -> list:
|
|
54
|
+
"""Same contract as `_get_dict`, but for list-shaped members
|
|
55
|
+
(`documents`, `fields`)."""
|
|
56
|
+
if not isinstance(container, dict):
|
|
57
|
+
return []
|
|
58
|
+
value = container.get(key, [])
|
|
59
|
+
if not isinstance(value, list):
|
|
60
|
+
return []
|
|
61
|
+
return value
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _get_entries(container: dict, key: str) -> list:
|
|
65
|
+
"""Fetch `container[key]` and guarantee a list comes back. Used for a
|
|
66
|
+
single term's/alias's posting list, which must be iterable — a
|
|
67
|
+
corrupted artifact may store a scalar (e.g. an int) there instead."""
|
|
68
|
+
value = container.get(key, [])
|
|
69
|
+
if not isinstance(value, list):
|
|
70
|
+
return []
|
|
71
|
+
return value
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _schema(lexical: object) -> int:
|
|
75
|
+
if not isinstance(lexical, dict):
|
|
76
|
+
return SCHEMA_V2
|
|
77
|
+
version = lexical.get("schema_version")
|
|
78
|
+
if isinstance(version, int) and not isinstance(version, bool):
|
|
79
|
+
return version
|
|
80
|
+
return SCHEMA_V2
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _doc_ref(lexical: object, doc_id: int) -> tuple[str, str] | None:
|
|
84
|
+
documents = _get_list(lexical, "documents")
|
|
85
|
+
if not isinstance(doc_id, int) or isinstance(doc_id, bool):
|
|
86
|
+
return None
|
|
87
|
+
if doc_id < 0 or doc_id >= len(documents):
|
|
88
|
+
return None
|
|
89
|
+
entry = documents[doc_id]
|
|
90
|
+
if not isinstance(entry, list) or len(entry) != 2:
|
|
91
|
+
return None
|
|
92
|
+
return (entry[0], entry[1])
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def alias_refs(lexical: object, norm_alias: str) -> list[tuple[str, str]]:
|
|
96
|
+
alias_exact = _get_dict(lexical, "alias_exact")
|
|
97
|
+
entries = _get_entries(alias_exact, norm_alias)
|
|
98
|
+
if _schema(lexical) == SCHEMA_V2:
|
|
99
|
+
return [(e[0], e[1]) for e in entries if isinstance(e, list) and len(e) == _V2_ALIAS_LEN]
|
|
100
|
+
refs: list[tuple[str, str]] = []
|
|
101
|
+
for doc_id in entries:
|
|
102
|
+
ref = _doc_ref(lexical, doc_id)
|
|
103
|
+
if ref is None:
|
|
104
|
+
continue
|
|
105
|
+
refs.append(ref)
|
|
106
|
+
return refs
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def term_postings(lexical: object, term: str) -> list[tuple[str, str, str]]:
|
|
110
|
+
postings = _get_dict(lexical, "postings")
|
|
111
|
+
entries = _get_entries(postings, term)
|
|
112
|
+
if _schema(lexical) == SCHEMA_V2:
|
|
113
|
+
return [
|
|
114
|
+
(e[0], e[1], e[2]) for e in entries if isinstance(e, list) and len(e) == _V2_POSTING_LEN
|
|
115
|
+
]
|
|
116
|
+
fields = _get_list(lexical, "fields")
|
|
117
|
+
triples: list[tuple[str, str, str]] = []
|
|
118
|
+
for packed in entries:
|
|
119
|
+
if not isinstance(packed, int) or isinstance(packed, bool):
|
|
120
|
+
continue
|
|
121
|
+
ref = _doc_ref(lexical, packed >> FIELD_PACK_BITS)
|
|
122
|
+
if ref is None:
|
|
123
|
+
continue
|
|
124
|
+
field_index = packed & _FIELD_MASK
|
|
125
|
+
if field_index >= len(fields):
|
|
126
|
+
continue
|
|
127
|
+
triples.append((ref[0], ref[1], fields[field_index]))
|
|
128
|
+
return triples
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def term_doc_freq(lexical: object, term: str) -> int:
|
|
132
|
+
postings = _get_dict(lexical, "postings")
|
|
133
|
+
if _schema(lexical) == SCHEMA_V2:
|
|
134
|
+
doc_freq = _get_dict(lexical, "doc_freq")
|
|
135
|
+
global_freq = _get_dict(doc_freq, "global")
|
|
136
|
+
stored = global_freq.get(term)
|
|
137
|
+
if isinstance(stored, int) and not isinstance(stored, bool):
|
|
138
|
+
return stored
|
|
139
|
+
return len(_get_entries(postings, term))
|
|
140
|
+
entries = _get_entries(postings, term)
|
|
141
|
+
distinct = {
|
|
142
|
+
packed >> FIELD_PACK_BITS
|
|
143
|
+
for packed in entries
|
|
144
|
+
if isinstance(packed, int) and not isinstance(packed, bool)
|
|
145
|
+
}
|
|
146
|
+
return len(distinct)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def doc_ids_by_key(lexical: object) -> dict[str, list[int]] | None:
|
|
150
|
+
"""v3 only (`None` for v2): key -> doc ids from the `documents` table.
|
|
151
|
+
A list, not one id: two nodes sharing repo, source file and label share a
|
|
152
|
+
key but are separate documents. Malformed entries are skipped exactly as
|
|
153
|
+
`_doc_ref` skips them."""
|
|
154
|
+
if _schema(lexical) == SCHEMA_V2:
|
|
155
|
+
return None
|
|
156
|
+
by_key: dict[str, list[int]] = {}
|
|
157
|
+
for doc_id, entry in enumerate(_get_list(lexical, "documents")):
|
|
158
|
+
if not isinstance(entry, list) or len(entry) != 2 or not isinstance(entry[1], str):
|
|
159
|
+
continue
|
|
160
|
+
by_key.setdefault(entry[1], []).append(doc_id)
|
|
161
|
+
return by_key
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _is_packed(entry: object) -> bool:
|
|
165
|
+
return isinstance(entry, int) and not isinstance(entry, bool)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def term_doc_hits(lexical: object, term: str, doc_ids: Iterable[int]) -> set[int]:
|
|
169
|
+
"""v3 only: the subset of `doc_ids` that `term` has a posting for, with
|
|
170
|
+
the same entry validity `term_postings` applies (int entry, known field).
|
|
171
|
+
Binary search per doc id over the sorted posting list; a list whose
|
|
172
|
+
entries do not compare (corrupt, mixed types) falls back to a scan."""
|
|
173
|
+
entries = _get_entries(_get_dict(lexical, "postings"), term)
|
|
174
|
+
field_count = len(_get_list(lexical, "fields"))
|
|
175
|
+
wanted = set(doc_ids)
|
|
176
|
+
try:
|
|
177
|
+
return {d for d in wanted if _bisect_hit(entries, d, field_count)}
|
|
178
|
+
except TypeError:
|
|
179
|
+
hits: set[int] = set()
|
|
180
|
+
for packed in entries:
|
|
181
|
+
if _is_packed(packed) and (packed & _FIELD_MASK) < field_count:
|
|
182
|
+
doc_id = packed >> FIELD_PACK_BITS
|
|
183
|
+
if doc_id in wanted:
|
|
184
|
+
hits.add(doc_id)
|
|
185
|
+
return hits
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _bisect_hit(entries: list, doc_id: int, field_count: int) -> bool:
|
|
189
|
+
index = bisect.bisect_left(entries, doc_id << FIELD_PACK_BITS)
|
|
190
|
+
while index < len(entries):
|
|
191
|
+
packed = entries[index]
|
|
192
|
+
index += 1
|
|
193
|
+
if not _is_packed(packed):
|
|
194
|
+
continue
|
|
195
|
+
if packed >> FIELD_PACK_BITS != doc_id:
|
|
196
|
+
return False
|
|
197
|
+
if (packed & _FIELD_MASK) < field_count:
|
|
198
|
+
return True
|
|
199
|
+
return False
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def document_count(lexical: object) -> int:
|
|
203
|
+
if not isinstance(lexical, dict):
|
|
204
|
+
return 0
|
|
205
|
+
count = lexical.get("document_count")
|
|
206
|
+
if isinstance(count, int) and not isinstance(count, bool):
|
|
207
|
+
return count
|
|
208
|
+
return len(_get_list(lexical, "documents"))
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""`project_map(repo)` tool (WS5 tool 4): a structural overview of one
|
|
2
|
+
registered repo in the CURRENT generation — node count, a trimmed
|
|
3
|
+
community_name breakdown, and the top hub nodes by structural degree. Purely
|
|
4
|
+
read-only over the already-loaded `Generation`; `Generation.nodes_by_repo` / `.adjacency`
|
|
5
|
+
already cover everything this needs, so no separate index is built here.
|
|
6
|
+
|
|
7
|
+
Fails closed like everything else in this package: an unresolvable repo
|
|
8
|
+
(unknown to the registry, or simply absent from the current generation —
|
|
9
|
+
e.g. registered but never synced yet) returns `resolved=False` with a
|
|
10
|
+
`degraded` reason, never a silently-empty-looking success.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
|
|
17
|
+
from graphify_mesh.server import ranking
|
|
18
|
+
from graphify_mesh.server.store import Generation
|
|
19
|
+
|
|
20
|
+
# How many top-degree nodes to surface per repo. Kept small and named
|
|
21
|
+
# (project style: no magic numbers) — this is an orientation summary, not a
|
|
22
|
+
# full node dump.
|
|
23
|
+
TOP_HUBS_LIMIT = 15
|
|
24
|
+
|
|
25
|
+
# `community_breakdown` keeps only communities of at least this size, and at
|
|
26
|
+
# most this many of them (largest first). Community labels are LLM-named
|
|
27
|
+
# clusters: an unreliable orientation hint, not a module map. Singletons were
|
|
28
|
+
# most of the list — about 4 KB of noise per call on cem.hub's 128
|
|
29
|
+
# communities. `communities_total` / `communities_omitted` report the cut.
|
|
30
|
+
COMMUNITY_MIN_SIZE = 2
|
|
31
|
+
COMMUNITY_BREAKDOWN_LIMIT = 25
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class ProjectMapResult:
|
|
36
|
+
resolved: bool
|
|
37
|
+
repo: str = ""
|
|
38
|
+
node_count: int = 0
|
|
39
|
+
community_breakdown: dict = field(default_factory=dict)
|
|
40
|
+
communities_total: int = 0 # every community in the repo, "unassigned" included
|
|
41
|
+
communities_omitted: int = 0 # communities_total minus len(community_breakdown)
|
|
42
|
+
top_hubs: list = field(default_factory=list) # [{key, label, degree, source_file, is_hub}]
|
|
43
|
+
degraded: list = field(default_factory=list)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def project_map(repo_id: str, generation: Generation) -> ProjectMapResult:
|
|
47
|
+
node_ids = generation.nodes_by_repo.get(repo_id)
|
|
48
|
+
if not node_ids:
|
|
49
|
+
return ProjectMapResult(
|
|
50
|
+
resolved=False, repo=repo_id, degraded=["repo_not_in_current_generation"]
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
community_breakdown: dict[str, int] = {}
|
|
54
|
+
scored: list[tuple[int, str, str]] = [] # (degree, key, node_id) — key used as tie-break
|
|
55
|
+
for node_id in node_ids:
|
|
56
|
+
node = generation.node_by_id.get(node_id, {})
|
|
57
|
+
community = node.get("community_name") or "unassigned"
|
|
58
|
+
community_breakdown[community] = community_breakdown.get(community, 0) + 1
|
|
59
|
+
key = generation.key_by_node_id.get(node_id)
|
|
60
|
+
if key is None:
|
|
61
|
+
continue
|
|
62
|
+
scored.append((generation.degree(node_id), key, node_id))
|
|
63
|
+
|
|
64
|
+
# Deterministic: degree desc, then key asc (never dict/set iteration order).
|
|
65
|
+
scored.sort(key=lambda t: (-t[0], t[1]))
|
|
66
|
+
top_hubs = []
|
|
67
|
+
for degree, key, node_id in scored[:TOP_HUBS_LIMIT]:
|
|
68
|
+
node = generation.node_by_id.get(node_id, {})
|
|
69
|
+
top_hubs.append(
|
|
70
|
+
{
|
|
71
|
+
"key": key,
|
|
72
|
+
"label": node.get("label", ""),
|
|
73
|
+
"degree": degree,
|
|
74
|
+
"source_file": node.get("source_file", ""),
|
|
75
|
+
"is_hub": degree > ranking.HUB_DEGREE_THRESHOLD,
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# Deterministic: size desc, then name asc; then the size floor and limit.
|
|
80
|
+
ranked_communities = sorted(community_breakdown.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
81
|
+
kept = [kv for kv in ranked_communities if kv[1] >= COMMUNITY_MIN_SIZE]
|
|
82
|
+
kept = kept[:COMMUNITY_BREAKDOWN_LIMIT]
|
|
83
|
+
|
|
84
|
+
return ProjectMapResult(
|
|
85
|
+
resolved=True,
|
|
86
|
+
repo=repo_id,
|
|
87
|
+
node_count=len(node_ids),
|
|
88
|
+
community_breakdown=dict(kept),
|
|
89
|
+
communities_total=len(ranked_communities),
|
|
90
|
+
communities_omitted=len(ranked_communities) - len(kept),
|
|
91
|
+
top_hubs=top_hubs,
|
|
92
|
+
)
|