graphify-mesh 0.0.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. graphify_mesh/__init__.py +20 -0
  2. graphify_mesh/_version.py +24 -0
  3. graphify_mesh/server/__init__.py +61 -0
  4. graphify_mesh/server/anchors.py +223 -0
  5. graphify_mesh/server/citation.py +17 -0
  6. graphify_mesh/server/config.py +195 -0
  7. graphify_mesh/server/context_pack.py +212 -0
  8. graphify_mesh/server/embed_query.py +206 -0
  9. graphify_mesh/server/frames.py +93 -0
  10. graphify_mesh/server/http_app.py +367 -0
  11. graphify_mesh/server/lexical_read.py +208 -0
  12. graphify_mesh/server/project_map.py +92 -0
  13. graphify_mesh/server/ranking.py +209 -0
  14. graphify_mesh/server/retrieval.py +343 -0
  15. graphify_mesh/server/rwlock.py +61 -0
  16. graphify_mesh/server/scope.py +207 -0
  17. graphify_mesh/server/sdk_app.py +108 -0
  18. graphify_mesh/server/server.py +813 -0
  19. graphify_mesh/server/similar.py +284 -0
  20. graphify_mesh/server/stdio_guard.py +287 -0
  21. graphify_mesh/server/store.py +733 -0
  22. graphify_mesh/server/traverse.py +305 -0
  23. graphify_mesh/sync/__init__.py +26 -0
  24. graphify_mesh/sync/backend.py +80 -0
  25. graphify_mesh/sync/cli.py +249 -0
  26. graphify_mesh/sync/clustering.py +182 -0
  27. graphify_mesh/sync/config.py +815 -0
  28. graphify_mesh/sync/discovery.py +668 -0
  29. graphify_mesh/sync/embed_similarity.py +516 -0
  30. graphify_mesh/sync/embedding.py +1372 -0
  31. graphify_mesh/sync/graphify_cli.py +941 -0
  32. graphify_mesh/sync/lexical_index.py +293 -0
  33. graphify_mesh/sync/locking.py +30 -0
  34. graphify_mesh/sync/naming.py +563 -0
  35. graphify_mesh/sync/naming_state.py +143 -0
  36. graphify_mesh/sync/overlay.py +146 -0
  37. graphify_mesh/sync/overlay_api.py +312 -0
  38. graphify_mesh/sync/overlay_depends.py +324 -0
  39. graphify_mesh/sync/overlay_refs.py +143 -0
  40. graphify_mesh/sync/overlay_similar.py +357 -0
  41. graphify_mesh/sync/perms.py +116 -0
  42. graphify_mesh/sync/pipeline.py +1891 -0
  43. graphify_mesh/sync/progress.py +41 -0
  44. graphify_mesh/sync/publish.py +374 -0
  45. graphify_mesh/sync/reaper.py +201 -0
  46. graphify_mesh/sync/reaper_cli.py +62 -0
  47. graphify_mesh/sync/registry.py +184 -0
  48. graphify_mesh/sync/repo_tags.py +299 -0
  49. graphify_mesh/sync/rss.py +68 -0
  50. graphify_mesh/sync/source_cache.py +149 -0
  51. graphify_mesh/sync/state.py +268 -0
  52. graphify_mesh/sync/sync_project.py +793 -0
  53. graphify_mesh/sync/tls.py +119 -0
  54. graphify_mesh/sync/validate.py +277 -0
  55. graphify_mesh/sync/vectors.py +234 -0
  56. graphify_mesh-0.0.22.dist-info/METADATA +219 -0
  57. graphify_mesh-0.0.22.dist-info/RECORD +60 -0
  58. graphify_mesh-0.0.22.dist-info/WHEEL +4 -0
  59. graphify_mesh-0.0.22.dist-info/entry_points.txt +4 -0
  60. graphify_mesh-0.0.22.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,367 @@
1
+ """Streamable-HTTP transport: one shared daemon for every local agent.
2
+
3
+ Stateless by design — `cwd` travels in each tool call, so there is no
4
+ per-session state worth keeping, and a client that disappears leaves nothing
5
+ behind. The bearer token is mandatory: unlike the unix socket this replaces,
6
+ a loopback TCP port is reachable by every local user and process, and what it
7
+ serves is the merged graph of every repository in registry.json.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import contextlib
13
+ import hmac
14
+ import json
15
+ import logging
16
+ from collections import deque
17
+ from collections.abc import AsyncIterator, Callable
18
+
19
+ import anyio
20
+ from anyio import to_thread
21
+ from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
22
+ from mcp.server.transport_security import TransportSecuritySettings
23
+ from starlette.applications import Starlette
24
+ from starlette.middleware import Middleware
25
+ from starlette.routing import Route
26
+ from starlette.types import ASGIApp, Message, Receive, Scope, Send
27
+
28
+ from graphify_mesh.server.config import ServerConfig, is_loopback_bind
29
+ from graphify_mesh.server.frames import (
30
+ MAX_MESSAGE_BYTES,
31
+ PARSE_ERROR,
32
+ envelope_error,
33
+ error_frame_text,
34
+ frame_id,
35
+ )
36
+ from graphify_mesh.server.sdk_app import build_sdk_server
37
+ from graphify_mesh.server.server import GraphifyMeshServer
38
+
39
+ log = logging.getLogger("graphify_mesh.server.http_app")
40
+
41
+ # A bound exists because a request being read holds up to `MAX_MESSAGE_BYTES`
42
+ # of body in this one shared process, so unbounded concurrent reads make
43
+ # inbound memory unbounded too. It bounds body buffering specifically, and it
44
+ # QUEUES rather than rejecting: making a local agent's request wait a few
45
+ # milliseconds is better than failing it. Not uvicorn's `limit_concurrency`,
46
+ # which answers 503 once the number of open CONNECTIONS reaches the limit —
47
+ # on a daemon serving many long-lived keep-alive clients that rejects the
48
+ # 33rd client with nothing in flight at all.
49
+ BODY_BUFFER_SLOTS = 8
50
+
51
+ # Bodies above this size are parsed in a worker thread: a multi-megabyte
52
+ # `json.loads` on the event loop stalls every other client of this shared
53
+ # daemon for its whole duration. Below it the thread hop costs more than the
54
+ # parse, so small frames stay inline.
55
+ INLINE_PARSE_MAX_BYTES = 64 * 1024
56
+
57
+
58
+ def _security_settings(config: ServerConfig) -> TransportSecuritySettings:
59
+ """DNS-rebinding protection, decided by the same rule
60
+ `config.is_loopback_bind` applies to the bind permission. A non-loopback
61
+ bind, a wildcard or a concrete interface address alike, both of which
62
+ require `--allow-public-bind`, is reachable under many names, so it
63
+ accepts any Host and leaves the bearer token as the gate, like the upstream
64
+ `graphify` server (`serve.py:_build_http_app`). A loopback bind restricts
65
+ Host to that address plus the localhost aliases, each with and without
66
+ the port."""
67
+ if not is_loopback_bind(config.http_host):
68
+ return TransportSecuritySettings(enable_dns_rebinding_protection=False)
69
+ allowed = {config.http_host, "localhost", "127.0.0.1"}
70
+ allowed |= {f"{host}:{config.http_port}" for host in list(allowed)}
71
+ return TransportSecuritySettings(allowed_hosts=sorted(allowed))
72
+
73
+
74
+ class _MCPASGIApp:
75
+ """Raw-ASGI wrapper around the streamable-HTTP session manager, so
76
+ Starlette treats this as the ASGI app for the mount path directly with
77
+ no request/response wrapping that would break the streaming path."""
78
+
79
+ def __init__(self, manager: StreamableHTTPSessionManager) -> None:
80
+ self._manager = manager
81
+
82
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
83
+ await self._manager.handle_request(scope, receive, send)
84
+
85
+
86
+ class _TokenAuthMiddleware:
87
+ """Mandatory bearer-token gate, ahead of the session manager.
88
+
89
+ Raw ASGI, not Starlette's `BaseHTTPMiddleware`: that middleware buffers
90
+ the response and breaks the Streamable HTTP stream. Never logs the
91
+ token, the Authorization header, or the request body — a rejection logs
92
+ only the remote address and the fact of rejection.
93
+ """
94
+
95
+ def __init__(self, app: ASGIApp, token: str) -> None:
96
+ self.app = app
97
+ self._expected = token.encode("utf-8")
98
+
99
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
100
+ if scope["type"] != "http":
101
+ await self.app(scope, receive, send)
102
+ return
103
+
104
+ # The raw list, not a dict: dict() keeps only the last value of a
105
+ # repeated header, so a request carrying both a wrong and a right
106
+ # `authorization` would be accepted or rejected on header order
107
+ # alone. More than one of them is ambiguous, so it is rejected
108
+ # before any comparison runs.
109
+ supplied = [
110
+ value for name, value in (scope.get("headers") or []) if name == b"authorization"
111
+ ]
112
+ provided: bytes | None = None
113
+ if len(supplied) == 1:
114
+ scheme, _, candidate = supplied[0].partition(b" ")
115
+ if scheme.lower() == b"bearer" and candidate:
116
+ provided = candidate.strip()
117
+
118
+ if provided is None or not hmac.compare_digest(provided, self._expected):
119
+ client = scope.get("client")
120
+ remote = client[0] if client else "unknown"
121
+ log.warning("rejected unauthorized request from %s", remote)
122
+ body = b'{"error": "unauthorized"}'
123
+ await send(
124
+ {
125
+ "type": "http.response.start",
126
+ "status": 401,
127
+ "headers": [
128
+ (b"content-type", b"application/json"),
129
+ (b"content-length", str(len(body)).encode("ascii")),
130
+ ],
131
+ }
132
+ )
133
+ await send({"type": "http.response.body", "body": body})
134
+ return
135
+
136
+ await self.app(scope, receive, send)
137
+
138
+
139
+ async def _send_json(send: Send, status: int, body: bytes) -> None:
140
+ await send(
141
+ {
142
+ "type": "http.response.start",
143
+ "status": status,
144
+ "headers": [
145
+ (b"content-type", b"application/json"),
146
+ (b"content-length", str(len(body)).encode("ascii")),
147
+ ],
148
+ }
149
+ )
150
+ await send({"type": "http.response.body", "body": body})
151
+
152
+
153
+ class _FrameGuardMiddleware:
154
+ """Size cap and envelope check for POST bodies, between the token gate
155
+ and the session manager.
156
+
157
+ Two things the SDK does differently from `stdio_guard`, both visible to a
158
+ client that got past the token gate:
159
+
160
+ * its body limit is its own default rather than this package's number,
161
+ so the two transports stated different caps;
162
+ * its parse and envelope errors carry `str(JSONDecodeError)` or
163
+ `str(ValidationError)` — validation details and excerpts of the input —
164
+ straight to the client, where stdio answers generically.
165
+
166
+ So the body is read here, under `MAX_MESSAGE_BYTES` counted on bytes
167
+ actually received (a chunked client has no `Content-Length` to check),
168
+ checked with the same `frames.envelope_error` stdio uses, and replayed
169
+ downstream untouched when it is well-formed. The detail goes to the log;
170
+ the client gets a fixed message.
171
+ """
172
+
173
+ def __init__(self, app: ASGIApp) -> None:
174
+ self.app = app
175
+ # One semaphore per app, so a test app and the daemon never share one.
176
+ self._slots = anyio.Semaphore(BODY_BUFFER_SLOTS)
177
+
178
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
179
+ if scope["type"] != "http" or scope.get("method") != "POST":
180
+ await self.app(scope, receive, send)
181
+ return
182
+
183
+ headers = dict(scope.get("headers") or [])
184
+ declared = headers.get(b"content-length")
185
+ if declared is not None and declared.isdigit() and int(declared) > MAX_MESSAGE_BYTES:
186
+ # Nothing is buffered on this path, so it takes no slot.
187
+ await self._too_large(scope, send)
188
+ return
189
+
190
+ await self._slots.acquire()
191
+ released = False
192
+
193
+ def release_slot() -> None:
194
+ # The slot covers the span where THIS middleware holds body
195
+ # bytes, and nothing more: it is handed back as soon as the body
196
+ # reaches the app below, never held across the downstream call,
197
+ # which would cap real request concurrency at `BODY_BUFFER_SLOTS`
198
+ # instead of bounding memory.
199
+ nonlocal released
200
+ if not released:
201
+ released = True
202
+ self._slots.release()
203
+
204
+ try:
205
+ await self._guarded(scope, receive, send, release_slot)
206
+ finally:
207
+ release_slot()
208
+
209
+ async def _guarded(
210
+ self, scope: Scope, receive: Receive, send: Send, release_slot: Callable[[], None]
211
+ ) -> None:
212
+ body = bytearray()
213
+ body_complete = False
214
+ trailing: Message | None = None
215
+ while True:
216
+ message = await receive()
217
+ if message["type"] != "http.request":
218
+ trailing = dict(message)
219
+ break
220
+ chunk = message.get("body", b"")
221
+ if len(body) + len(chunk) > MAX_MESSAGE_BYTES:
222
+ # Answer now; the rest of the body is never read, so an
223
+ # over-limit client cannot make this process buffer it.
224
+ await self._too_large(scope, send)
225
+ return
226
+ body.extend(chunk)
227
+ if not message.get("more_body", False):
228
+ body_complete = True
229
+ break
230
+
231
+ try:
232
+ if len(body) > INLINE_PARSE_MAX_BYTES:
233
+ # `json.loads` accepts the bytearray directly, so the thread
234
+ # gets the same single buffer, not a copy of it.
235
+ parsed = await to_thread.run_sync(json.loads, body)
236
+ else:
237
+ parsed = json.loads(body)
238
+ except (json.JSONDecodeError, UnicodeDecodeError) as exc:
239
+ log.warning("rejected unparseable request body from %s: %s", _remote(scope), exc)
240
+ await _send_json(send, 400, error_frame_text(*PARSE_ERROR).encode("utf-8"))
241
+ return
242
+
243
+ coded = envelope_error(parsed)
244
+ if coded is not None:
245
+ log.warning(
246
+ "rejected malformed JSON-RPC frame from %s (code %d)", _remote(scope), coded[0]
247
+ )
248
+ frame = error_frame_text(*coded, frame_id(parsed))
249
+ await _send_json(send, 400, frame.encode("utf-8"))
250
+ return
251
+
252
+ # Both references are dropped before dispatching downstream: the
253
+ # parsed object would otherwise stay live for the whole call (several
254
+ # times the byte size of the body, as Python objects), and `body` is
255
+ # handed to `_replayed_receive` as-is rather than copied, so exactly
256
+ # one buffer of these bytes exists here at any moment.
257
+ del parsed
258
+ replay = _replayed_receive(body, body_complete, trailing, receive, release_slot)
259
+ del body
260
+ await self.app(scope, replay, send)
261
+
262
+ async def _too_large(self, scope: Scope, send: Send) -> None:
263
+ log.warning(
264
+ "rejected oversized request body from %s (> %d bytes)",
265
+ _remote(scope),
266
+ MAX_MESSAGE_BYTES,
267
+ )
268
+ await _send_json(send, 413, b'{"error": "request body too large"}')
269
+
270
+
271
+ def _remote(scope: Scope) -> str:
272
+ client = scope.get("client")
273
+ return client[0] if client else "unknown"
274
+
275
+
276
+ def _replayed_receive(
277
+ body: bytearray,
278
+ body_complete: bool,
279
+ trailing: Message | None,
280
+ receive: Receive,
281
+ on_handoff: Callable[[], None],
282
+ ) -> Receive:
283
+ """Hands the already-read body to the app below exactly once, then falls
284
+ back to the real `receive` (for the disconnect message).
285
+
286
+ `on_handoff` runs when the body message has been delivered: at that point
287
+ the app below owns these bytes and this layer holds none, which is what
288
+ the body-buffer slot was reserving. The buffer is passed through rather
289
+ than copied to `bytes`; the only consumer is the SDK's
290
+ `RequestBodyLimitMiddleware`, which extends its own bytearray from it and
291
+ never requires an immutable `bytes`.
292
+ """
293
+ queued: deque[Message] = deque(
294
+ [{"type": "http.request", "body": body, "more_body": not body_complete}]
295
+ )
296
+ if trailing is not None:
297
+ queued.append(trailing)
298
+
299
+ async def replay() -> Message:
300
+ if queued:
301
+ message = queued.popleft()
302
+ if message["type"] == "http.request":
303
+ on_handoff()
304
+ return message
305
+ return await receive()
306
+
307
+ return replay
308
+
309
+
310
+ def build_http_app(mesh: GraphifyMeshServer, config: ServerConfig) -> Starlette:
311
+ """The ASGI app for the shared streamable-HTTP daemon. No server
312
+ started — for in-process tests and for `serve_http` to run under
313
+ uvicorn."""
314
+ if not config.http_token:
315
+ raise ValueError("http transport requires config.http_token")
316
+
317
+ sdk = build_sdk_server(mesh)
318
+ security = _security_settings(config)
319
+
320
+ # Stateless: no sessions, no SSE, no session reaper. `cwd` travels in
321
+ # each tool call instead.
322
+ manager = StreamableHTTPSessionManager(
323
+ app=sdk,
324
+ json_response=True,
325
+ stateless=True,
326
+ security_settings=security,
327
+ # The same number stdio caps a line at. Left to its default, the
328
+ # HTTP bound would be the SDK's, which is both a different number
329
+ # and one that can change between SDK releases. `session_idle_timeout`
330
+ # is deliberately NOT passed: the SDK documents it as unused in
331
+ # stateless mode, and a parameter this adapter does not need is one
332
+ # more thing an SDK release can move.
333
+ max_request_body_size=MAX_MESSAGE_BYTES,
334
+ )
335
+
336
+ @contextlib.asynccontextmanager
337
+ async def lifespan(_app: Starlette) -> AsyncIterator[None]:
338
+ # The session manager owns an anyio task group that must wrap the
339
+ # whole server lifetime, so enter it here rather than per-request.
340
+ async with manager.run():
341
+ yield
342
+
343
+ return Starlette(
344
+ routes=[Route(config.http_path, endpoint=_MCPASGIApp(manager))],
345
+ # Order matters: the token gate stays outermost, so an
346
+ # unauthenticated request is answered before this process reads,
347
+ # buffers or parses a single byte of its body.
348
+ middleware=[
349
+ Middleware(_TokenAuthMiddleware, token=config.http_token),
350
+ Middleware(_FrameGuardMiddleware),
351
+ ],
352
+ lifespan=lifespan,
353
+ )
354
+
355
+
356
+ def serve_http(mesh: GraphifyMeshServer, config: ServerConfig) -> None:
357
+ """Blocking streamable-HTTP transport on the configured host and port."""
358
+ import uvicorn
359
+
360
+ app = build_http_app(mesh, config)
361
+ log.info(
362
+ "graphify-mesh http daemon: host=%s port=%s path=%s token required",
363
+ config.http_host,
364
+ config.http_port,
365
+ config.http_path,
366
+ )
367
+ uvicorn.run(app, host=config.http_host, port=config.http_port)
@@ -0,0 +1,208 @@
1
+ """Version-aware read-path for the `lexical-index.json` bundle.
2
+
3
+ The sync pipeline writes exactly one schema version (the current
4
+ `LEXICAL_SCHEMA_VERSION` in sync/lexical_index.py), but the server must keep
5
+ serving whatever the CURRENT published generation contains — publish can lag
6
+ behind a deploy (e.g. shrink-guard blocking), so the reader understands the
7
+ previous on-disk shape too. All schema branching lives HERE; retrieval.py and
8
+ similar.py consume shape-agnostic tuples.
9
+
10
+ v2 shape: postings `term -> [[repo, key, field], ...]`,
11
+ alias_exact `alias -> [[repo, key], ...]`,
12
+ doc_freq stored under `doc_freq.global`.
13
+ v3 shape: `documents` table (list index = doc id), `fields` table
14
+ (list index = field id), postings `term -> [packed_int, ...]`
15
+ with `packed = (doc_id << FIELD_PACK_BITS) | field_index`,
16
+ alias_exact `alias -> [doc_id, ...]`, doc_freq NOT stored
17
+ (derived: distinct doc ids in the term's postings).
18
+
19
+ Malformed entries are skipped, never raised: a partially-corrupt index
20
+ degrades retrieval quality, it must not take the server down.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import bisect
26
+ from collections.abc import Iterable
27
+
28
+ SCHEMA_V2 = 2
29
+ SCHEMA_V3 = 3
30
+ # Low bits reserved for the field index inside a packed v3 posting entry.
31
+ FIELD_PACK_BITS = 2
32
+ _FIELD_MASK = (1 << FIELD_PACK_BITS) - 1
33
+
34
+ _V2_POSTING_LEN = 3
35
+ _V2_ALIAS_LEN = 2
36
+
37
+
38
+ def _get_dict(container: object, key: str) -> dict:
39
+ """Fetch `container[key]` and guarantee a dict comes back, no matter how
40
+ malformed `container` or the value at `key` is. Container-level
41
+ hardening: `container` itself may be a non-dict scalar (e.g. the whole
42
+ `lexical` artifact got corrupted into a string), or the value stored at
43
+ `key` may be the wrong shape (a list, an int, ...) — either way this
44
+ degrades to an empty dict rather than raising."""
45
+ if not isinstance(container, dict):
46
+ return {}
47
+ value = container.get(key, {})
48
+ if not isinstance(value, dict):
49
+ return {}
50
+ return value
51
+
52
+
53
+ def _get_list(container: object, key: str) -> list:
54
+ """Same contract as `_get_dict`, but for list-shaped members
55
+ (`documents`, `fields`)."""
56
+ if not isinstance(container, dict):
57
+ return []
58
+ value = container.get(key, [])
59
+ if not isinstance(value, list):
60
+ return []
61
+ return value
62
+
63
+
64
+ def _get_entries(container: dict, key: str) -> list:
65
+ """Fetch `container[key]` and guarantee a list comes back. Used for a
66
+ single term's/alias's posting list, which must be iterable — a
67
+ corrupted artifact may store a scalar (e.g. an int) there instead."""
68
+ value = container.get(key, [])
69
+ if not isinstance(value, list):
70
+ return []
71
+ return value
72
+
73
+
74
+ def _schema(lexical: object) -> int:
75
+ if not isinstance(lexical, dict):
76
+ return SCHEMA_V2
77
+ version = lexical.get("schema_version")
78
+ if isinstance(version, int) and not isinstance(version, bool):
79
+ return version
80
+ return SCHEMA_V2
81
+
82
+
83
+ def _doc_ref(lexical: object, doc_id: int) -> tuple[str, str] | None:
84
+ documents = _get_list(lexical, "documents")
85
+ if not isinstance(doc_id, int) or isinstance(doc_id, bool):
86
+ return None
87
+ if doc_id < 0 or doc_id >= len(documents):
88
+ return None
89
+ entry = documents[doc_id]
90
+ if not isinstance(entry, list) or len(entry) != 2:
91
+ return None
92
+ return (entry[0], entry[1])
93
+
94
+
95
+ def alias_refs(lexical: object, norm_alias: str) -> list[tuple[str, str]]:
96
+ alias_exact = _get_dict(lexical, "alias_exact")
97
+ entries = _get_entries(alias_exact, norm_alias)
98
+ if _schema(lexical) == SCHEMA_V2:
99
+ return [(e[0], e[1]) for e in entries if isinstance(e, list) and len(e) == _V2_ALIAS_LEN]
100
+ refs: list[tuple[str, str]] = []
101
+ for doc_id in entries:
102
+ ref = _doc_ref(lexical, doc_id)
103
+ if ref is None:
104
+ continue
105
+ refs.append(ref)
106
+ return refs
107
+
108
+
109
+ def term_postings(lexical: object, term: str) -> list[tuple[str, str, str]]:
110
+ postings = _get_dict(lexical, "postings")
111
+ entries = _get_entries(postings, term)
112
+ if _schema(lexical) == SCHEMA_V2:
113
+ return [
114
+ (e[0], e[1], e[2]) for e in entries if isinstance(e, list) and len(e) == _V2_POSTING_LEN
115
+ ]
116
+ fields = _get_list(lexical, "fields")
117
+ triples: list[tuple[str, str, str]] = []
118
+ for packed in entries:
119
+ if not isinstance(packed, int) or isinstance(packed, bool):
120
+ continue
121
+ ref = _doc_ref(lexical, packed >> FIELD_PACK_BITS)
122
+ if ref is None:
123
+ continue
124
+ field_index = packed & _FIELD_MASK
125
+ if field_index >= len(fields):
126
+ continue
127
+ triples.append((ref[0], ref[1], fields[field_index]))
128
+ return triples
129
+
130
+
131
+ def term_doc_freq(lexical: object, term: str) -> int:
132
+ postings = _get_dict(lexical, "postings")
133
+ if _schema(lexical) == SCHEMA_V2:
134
+ doc_freq = _get_dict(lexical, "doc_freq")
135
+ global_freq = _get_dict(doc_freq, "global")
136
+ stored = global_freq.get(term)
137
+ if isinstance(stored, int) and not isinstance(stored, bool):
138
+ return stored
139
+ return len(_get_entries(postings, term))
140
+ entries = _get_entries(postings, term)
141
+ distinct = {
142
+ packed >> FIELD_PACK_BITS
143
+ for packed in entries
144
+ if isinstance(packed, int) and not isinstance(packed, bool)
145
+ }
146
+ return len(distinct)
147
+
148
+
149
+ def doc_ids_by_key(lexical: object) -> dict[str, list[int]] | None:
150
+ """v3 only (`None` for v2): key -> doc ids from the `documents` table.
151
+ A list, not one id: two nodes sharing repo, source file and label share a
152
+ key but are separate documents. Malformed entries are skipped exactly as
153
+ `_doc_ref` skips them."""
154
+ if _schema(lexical) == SCHEMA_V2:
155
+ return None
156
+ by_key: dict[str, list[int]] = {}
157
+ for doc_id, entry in enumerate(_get_list(lexical, "documents")):
158
+ if not isinstance(entry, list) or len(entry) != 2 or not isinstance(entry[1], str):
159
+ continue
160
+ by_key.setdefault(entry[1], []).append(doc_id)
161
+ return by_key
162
+
163
+
164
+ def _is_packed(entry: object) -> bool:
165
+ return isinstance(entry, int) and not isinstance(entry, bool)
166
+
167
+
168
+ def term_doc_hits(lexical: object, term: str, doc_ids: Iterable[int]) -> set[int]:
169
+ """v3 only: the subset of `doc_ids` that `term` has a posting for, with
170
+ the same entry validity `term_postings` applies (int entry, known field).
171
+ Binary search per doc id over the sorted posting list; a list whose
172
+ entries do not compare (corrupt, mixed types) falls back to a scan."""
173
+ entries = _get_entries(_get_dict(lexical, "postings"), term)
174
+ field_count = len(_get_list(lexical, "fields"))
175
+ wanted = set(doc_ids)
176
+ try:
177
+ return {d for d in wanted if _bisect_hit(entries, d, field_count)}
178
+ except TypeError:
179
+ hits: set[int] = set()
180
+ for packed in entries:
181
+ if _is_packed(packed) and (packed & _FIELD_MASK) < field_count:
182
+ doc_id = packed >> FIELD_PACK_BITS
183
+ if doc_id in wanted:
184
+ hits.add(doc_id)
185
+ return hits
186
+
187
+
188
+ def _bisect_hit(entries: list, doc_id: int, field_count: int) -> bool:
189
+ index = bisect.bisect_left(entries, doc_id << FIELD_PACK_BITS)
190
+ while index < len(entries):
191
+ packed = entries[index]
192
+ index += 1
193
+ if not _is_packed(packed):
194
+ continue
195
+ if packed >> FIELD_PACK_BITS != doc_id:
196
+ return False
197
+ if (packed & _FIELD_MASK) < field_count:
198
+ return True
199
+ return False
200
+
201
+
202
+ def document_count(lexical: object) -> int:
203
+ if not isinstance(lexical, dict):
204
+ return 0
205
+ count = lexical.get("document_count")
206
+ if isinstance(count, int) and not isinstance(count, bool):
207
+ return count
208
+ return len(_get_list(lexical, "documents"))
@@ -0,0 +1,92 @@
1
+ """`project_map(repo)` tool (WS5 tool 4): a structural overview of one
2
+ registered repo in the CURRENT generation — node count, a trimmed
3
+ community_name breakdown, and the top hub nodes by structural degree. Purely
4
+ read-only over the already-loaded `Generation`; `Generation.nodes_by_repo` / `.adjacency`
5
+ already cover everything this needs, so no separate index is built here.
6
+
7
+ Fails closed like everything else in this package: an unresolvable repo
8
+ (unknown to the registry, or simply absent from the current generation —
9
+ e.g. registered but never synced yet) returns `resolved=False` with a
10
+ `degraded` reason, never a silently-empty-looking success.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import dataclass, field
16
+
17
+ from graphify_mesh.server import ranking
18
+ from graphify_mesh.server.store import Generation
19
+
20
+ # How many top-degree nodes to surface per repo. Kept small and named
21
+ # (project style: no magic numbers) — this is an orientation summary, not a
22
+ # full node dump.
23
+ TOP_HUBS_LIMIT = 15
24
+
25
+ # `community_breakdown` keeps only communities of at least this size, and at
26
+ # most this many of them (largest first). Community labels are LLM-named
27
+ # clusters: an unreliable orientation hint, not a module map. Singletons were
28
+ # most of the list — about 4 KB of noise per call on cem.hub's 128
29
+ # communities. `communities_total` / `communities_omitted` report the cut.
30
+ COMMUNITY_MIN_SIZE = 2
31
+ COMMUNITY_BREAKDOWN_LIMIT = 25
32
+
33
+
34
+ @dataclass
35
+ class ProjectMapResult:
36
+ resolved: bool
37
+ repo: str = ""
38
+ node_count: int = 0
39
+ community_breakdown: dict = field(default_factory=dict)
40
+ communities_total: int = 0 # every community in the repo, "unassigned" included
41
+ communities_omitted: int = 0 # communities_total minus len(community_breakdown)
42
+ top_hubs: list = field(default_factory=list) # [{key, label, degree, source_file, is_hub}]
43
+ degraded: list = field(default_factory=list)
44
+
45
+
46
+ def project_map(repo_id: str, generation: Generation) -> ProjectMapResult:
47
+ node_ids = generation.nodes_by_repo.get(repo_id)
48
+ if not node_ids:
49
+ return ProjectMapResult(
50
+ resolved=False, repo=repo_id, degraded=["repo_not_in_current_generation"]
51
+ )
52
+
53
+ community_breakdown: dict[str, int] = {}
54
+ scored: list[tuple[int, str, str]] = [] # (degree, key, node_id) — key used as tie-break
55
+ for node_id in node_ids:
56
+ node = generation.node_by_id.get(node_id, {})
57
+ community = node.get("community_name") or "unassigned"
58
+ community_breakdown[community] = community_breakdown.get(community, 0) + 1
59
+ key = generation.key_by_node_id.get(node_id)
60
+ if key is None:
61
+ continue
62
+ scored.append((generation.degree(node_id), key, node_id))
63
+
64
+ # Deterministic: degree desc, then key asc (never dict/set iteration order).
65
+ scored.sort(key=lambda t: (-t[0], t[1]))
66
+ top_hubs = []
67
+ for degree, key, node_id in scored[:TOP_HUBS_LIMIT]:
68
+ node = generation.node_by_id.get(node_id, {})
69
+ top_hubs.append(
70
+ {
71
+ "key": key,
72
+ "label": node.get("label", ""),
73
+ "degree": degree,
74
+ "source_file": node.get("source_file", ""),
75
+ "is_hub": degree > ranking.HUB_DEGREE_THRESHOLD,
76
+ }
77
+ )
78
+
79
+ # Deterministic: size desc, then name asc; then the size floor and limit.
80
+ ranked_communities = sorted(community_breakdown.items(), key=lambda kv: (-kv[1], kv[0]))
81
+ kept = [kv for kv in ranked_communities if kv[1] >= COMMUNITY_MIN_SIZE]
82
+ kept = kept[:COMMUNITY_BREAKDOWN_LIMIT]
83
+
84
+ return ProjectMapResult(
85
+ resolved=True,
86
+ repo=repo_id,
87
+ node_count=len(node_ids),
88
+ community_breakdown=dict(kept),
89
+ communities_total=len(ranked_communities),
90
+ communities_omitted=len(ranked_communities) - len(kept),
91
+ top_hubs=top_hubs,
92
+ )