pycode-kg 0.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pycode_kg/.DS_Store +0 -0
- pycode_kg/__init__.py +91 -0
- pycode_kg/__main__.py +11 -0
- pycode_kg/analysis/__init__.py +15 -0
- pycode_kg/analysis/bridge.py +108 -0
- pycode_kg/analysis/centrality.py +412 -0
- pycode_kg/analysis/framework_detector.py +103 -0
- pycode_kg/analysis/hybrid_rank.py +53 -0
- pycode_kg/app.py +1335 -0
- pycode_kg/architecture.py +624 -0
- pycode_kg/build_pycodekg_lancedb.py +15 -0
- pycode_kg/build_pycodekg_sqlite.py +15 -0
- pycode_kg/cli/__init__.py +29 -0
- pycode_kg/cli/cmd_analyze.py +96 -0
- pycode_kg/cli/cmd_architecture.py +150 -0
- pycode_kg/cli/cmd_bridges.py +26 -0
- pycode_kg/cli/cmd_build.py +154 -0
- pycode_kg/cli/cmd_build_full.py +242 -0
- pycode_kg/cli/cmd_centrality.py +131 -0
- pycode_kg/cli/cmd_explain.py +180 -0
- pycode_kg/cli/cmd_framework_nodes.py +18 -0
- pycode_kg/cli/cmd_hooks.py +137 -0
- pycode_kg/cli/cmd_init.py +312 -0
- pycode_kg/cli/cmd_mcp.py +71 -0
- pycode_kg/cli/cmd_model.py +53 -0
- pycode_kg/cli/cmd_query.py +211 -0
- pycode_kg/cli/cmd_snapshot.py +421 -0
- pycode_kg/cli/cmd_viz.py +180 -0
- pycode_kg/cli/main.py +23 -0
- pycode_kg/cli/options.py +63 -0
- pycode_kg/config.py +78 -0
- pycode_kg/graph.py +125 -0
- pycode_kg/index.py +542 -0
- pycode_kg/kg.py +220 -0
- pycode_kg/layout3d.py +470 -0
- pycode_kg/mcp/bridge_tools.py +19 -0
- pycode_kg/mcp/framework_tools.py +18 -0
- pycode_kg/mcp_server.py +1965 -0
- pycode_kg/module/__init__.py +83 -0
- pycode_kg/module/base.py +720 -0
- pycode_kg/module/extractor.py +276 -0
- pycode_kg/module/types.py +532 -0
- pycode_kg/pycodekg.py +543 -0
- pycode_kg/pycodekg_query.py +1 -0
- pycode_kg/pycodekg_snippet_packer.py +1 -0
- pycode_kg/pycodekg_thorough_analysis.py +2751 -0
- pycode_kg/pycodekg_viz.py +1 -0
- pycode_kg/pycodekg_viz3d.py +1 -0
- pycode_kg/ranking/__init__.py +1 -0
- pycode_kg/ranking/cli_rank.py +92 -0
- pycode_kg/ranking/coderank.py +555 -0
- pycode_kg/snapshots.py +612 -0
- pycode_kg/sql/004_add_centrality_table.sql +12 -0
- pycode_kg/store.py +766 -0
- pycode_kg/utils.py +39 -0
- pycode_kg/visitor.py +413 -0
- pycode_kg/viz3d.py +1353 -0
- pycode_kg/viz3d_timeline.py +364 -0
- pycode_kg-0.16.0.dist-info/METADATA +305 -0
- pycode_kg-0.16.0.dist-info/RECORD +63 -0
- pycode_kg-0.16.0.dist-info/WHEEL +4 -0
- pycode_kg-0.16.0.dist-info/entry_points.txt +19 -0
- pycode_kg-0.16.0.dist-info/licenses/LICENSE +94 -0
pycode_kg/module/base.py
ADDED
|
@@ -0,0 +1,720 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
module/base.py
|
|
4
|
+
|
|
5
|
+
KGModule — abstract base class for production-grade knowledge graph modules.
|
|
6
|
+
|
|
7
|
+
Provides the full PyCodeKG-equivalent infrastructure:
|
|
8
|
+
- build pipeline (extractor → GraphStore → SemanticIndex)
|
|
9
|
+
- hybrid query (semantic seeding + structural graph expansion)
|
|
10
|
+
- snippet packing (source-grounded context for LLMs)
|
|
11
|
+
- lazy-initialised layers (store, index, embedder)
|
|
12
|
+
- snapshot support (delegates to SnapshotManager)
|
|
13
|
+
|
|
14
|
+
Domain authors subclass KGModule and implement exactly two abstract methods:
|
|
15
|
+
- make_extractor() → KGExtractor
|
|
16
|
+
- kind() → str
|
|
17
|
+
|
|
18
|
+
Everything else is provided by this class.
|
|
19
|
+
|
|
20
|
+
Author: Eric G. Suchanek, PhD
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from abc import ABC, abstractmethod
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import TYPE_CHECKING
|
|
28
|
+
|
|
29
|
+
from pycode_kg.index import (
|
|
30
|
+
Embedder,
|
|
31
|
+
SemanticIndex,
|
|
32
|
+
SentenceTransformerEmbedder,
|
|
33
|
+
suppress_ingestion_logging,
|
|
34
|
+
)
|
|
35
|
+
from pycode_kg.module.extractor import EdgeSpec, KGExtractor, NodeSpec
|
|
36
|
+
from pycode_kg.module.types import (
|
|
37
|
+
BuildStats,
|
|
38
|
+
QueryResult,
|
|
39
|
+
SnippetPack,
|
|
40
|
+
compute_span,
|
|
41
|
+
docstring_signal,
|
|
42
|
+
lexical_overlap_score,
|
|
43
|
+
make_module_summary,
|
|
44
|
+
make_snippet,
|
|
45
|
+
normalize_query_text,
|
|
46
|
+
query_tokens,
|
|
47
|
+
read_lines,
|
|
48
|
+
safe_join,
|
|
49
|
+
semantic_score_from_distance,
|
|
50
|
+
spans_overlap,
|
|
51
|
+
)
|
|
52
|
+
from pycode_kg.pycodekg import DEFAULT_MODEL
|
|
53
|
+
from pycode_kg.store import DEFAULT_RELS, GraphStore
|
|
54
|
+
|
|
55
|
+
if TYPE_CHECKING:
|
|
56
|
+
from pycode_kg.pycodekg import Edge, Node
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ---------------------------------------------------------------------------
|
|
60
|
+
# Conversion helpers: NodeSpec/EdgeSpec → v0 Node/Edge
|
|
61
|
+
# ---------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _nodespec_to_node(spec: NodeSpec) -> Node:
|
|
65
|
+
"""Convert a NodeSpec to the v0-locked Node primitive.
|
|
66
|
+
|
|
67
|
+
:param spec: NodeSpec from a KGExtractor.
|
|
68
|
+
:return: Node dataclass instance.
|
|
69
|
+
"""
|
|
70
|
+
from pycode_kg.pycodekg import Node # pylint: disable=import-outside-toplevel
|
|
71
|
+
|
|
72
|
+
return Node(
|
|
73
|
+
id=spec.node_id,
|
|
74
|
+
kind=spec.kind,
|
|
75
|
+
name=spec.name,
|
|
76
|
+
qualname=spec.qualname,
|
|
77
|
+
module_path=spec.source_path,
|
|
78
|
+
lineno=spec.lineno,
|
|
79
|
+
end_lineno=spec.end_lineno,
|
|
80
|
+
docstring=spec.docstring,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _edgespec_to_edge(spec: EdgeSpec) -> Edge:
|
|
85
|
+
"""Convert an EdgeSpec to the v0-locked Edge primitive.
|
|
86
|
+
|
|
87
|
+
:param spec: EdgeSpec from a KGExtractor.
|
|
88
|
+
:return: Edge dataclass instance.
|
|
89
|
+
"""
|
|
90
|
+
from pycode_kg.pycodekg import Edge # pylint: disable=import-outside-toplevel
|
|
91
|
+
|
|
92
|
+
evidence = spec.metadata if spec.metadata else None
|
|
93
|
+
return Edge(src=spec.source_id, dst=spec.target_id, rel=spec.relation, evidence=evidence)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
# KGModule
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class KGModule(ABC):
|
|
102
|
+
"""Abstract base class for a production-grade knowledge graph module.
|
|
103
|
+
|
|
104
|
+
Provides the full build/query/pack infrastructure so domain authors
|
|
105
|
+
only need to implement :meth:`make_extractor` and :meth:`kind`.
|
|
106
|
+
|
|
107
|
+
Typical usage by a domain author::
|
|
108
|
+
|
|
109
|
+
class TypeScriptKG(KGModule):
|
|
110
|
+
def make_extractor(self):
|
|
111
|
+
return TypeScriptExtractor(self.repo_root)
|
|
112
|
+
def kind(self):
|
|
113
|
+
return "code"
|
|
114
|
+
def analyze(self):
|
|
115
|
+
return "# TypeScript KG Analysis\\n..."
|
|
116
|
+
|
|
117
|
+
kg = TypeScriptKG("/path/to/ts-repo")
|
|
118
|
+
kg.build(wipe=True)
|
|
119
|
+
result = kg.query("authentication middleware")
|
|
120
|
+
pack = kg.pack("error handling")
|
|
121
|
+
|
|
122
|
+
:param repo_root: Repository root directory.
|
|
123
|
+
:param db_path: SQLite database path (defaults to ``.<kind>kg/graph.sqlite``
|
|
124
|
+
under ``repo_root``; set ``_default_dir`` in subclass to override).
|
|
125
|
+
:param lancedb_dir: LanceDB directory (defaults to ``.<kind>kg/lancedb``).
|
|
126
|
+
:param model: Sentence-transformer model name for embedding.
|
|
127
|
+
:param table: LanceDB table name.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
#: Override in subclass to change the default artefact directory name.
|
|
131
|
+
#: e.g. ``_default_dir = ".tskg"`` for a TypeScript KG.
|
|
132
|
+
_default_dir: str = ".pycodekg"
|
|
133
|
+
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
repo_root: str | Path,
|
|
137
|
+
db_path: str | Path | None = None,
|
|
138
|
+
lancedb_dir: str | Path | None = None,
|
|
139
|
+
*,
|
|
140
|
+
model: str = DEFAULT_MODEL,
|
|
141
|
+
table: str = "kg_nodes",
|
|
142
|
+
) -> None:
|
|
143
|
+
"""Initialise KGModule and resolve all paths.
|
|
144
|
+
|
|
145
|
+
:param repo_root: Repository root directory; resolved to an absolute path.
|
|
146
|
+
:param db_path: Path to the SQLite database file.
|
|
147
|
+
:param lancedb_dir: Directory used by LanceDB for the vector index.
|
|
148
|
+
:param model: Sentence-transformer model name.
|
|
149
|
+
:param table: LanceDB table name.
|
|
150
|
+
"""
|
|
151
|
+
self.repo_root = Path(repo_root).resolve()
|
|
152
|
+
_dir = self.repo_root / self._default_dir
|
|
153
|
+
self.db_path = Path(db_path) if db_path is not None else _dir / "graph.sqlite"
|
|
154
|
+
self.lancedb_dir = Path(lancedb_dir) if lancedb_dir is not None else _dir / "lancedb"
|
|
155
|
+
self.model_name = model
|
|
156
|
+
self.table_name = table
|
|
157
|
+
|
|
158
|
+
# Lazy-initialised layers
|
|
159
|
+
self._store: GraphStore | None = None
|
|
160
|
+
self._index: SemanticIndex | None = None
|
|
161
|
+
self._embedder: Embedder | None = None
|
|
162
|
+
|
|
163
|
+
# ------------------------------------------------------------------
|
|
164
|
+
# Abstract interface — domain authors implement these
|
|
165
|
+
# ------------------------------------------------------------------
|
|
166
|
+
|
|
167
|
+
@abstractmethod
|
|
168
|
+
def make_extractor(self) -> KGExtractor:
|
|
169
|
+
"""Return the domain-specific extractor for this module.
|
|
170
|
+
|
|
171
|
+
Called once per :meth:`build_graph` invocation. The extractor
|
|
172
|
+
determines what nodes and edges are harvested from the source.
|
|
173
|
+
|
|
174
|
+
:return: A :class:`~pycode_kg.module.extractor.KGExtractor` instance.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
@abstractmethod
|
|
178
|
+
def kind(self) -> str:
|
|
179
|
+
"""Return the KG kind string for this module.
|
|
180
|
+
|
|
181
|
+
For code KGs return ``"code"``, for document KGs ``"doc"``, for
|
|
182
|
+
domain-specific KGs ``"meta"`` (or register a new value in
|
|
183
|
+
``kg_rag.primitives.KGKind``).
|
|
184
|
+
|
|
185
|
+
:return: Kind string used in registry entries and MCP tool names.
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
@abstractmethod
|
|
189
|
+
def analyze(self) -> str:
|
|
190
|
+
"""Run a full analysis of this KG and return a Markdown report.
|
|
191
|
+
|
|
192
|
+
The report format is domain-specific but must:
|
|
193
|
+
|
|
194
|
+
1. Begin with ``# <KG Type> Analysis Report``
|
|
195
|
+
2. Include the KG name/path for identification
|
|
196
|
+
3. Include structural metrics (node count, edge count)
|
|
197
|
+
4. Include domain-appropriate quality signals
|
|
198
|
+
|
|
199
|
+
Must not raise — return a Markdown error message on failure.
|
|
200
|
+
|
|
201
|
+
:return: Markdown-formatted analysis report string.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
# ------------------------------------------------------------------
|
|
205
|
+
# Layer accessors (lazy init)
|
|
206
|
+
# ------------------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
@property
|
|
209
|
+
def store(self) -> GraphStore:
|
|
210
|
+
"""SQLite persistence layer (lazy-initialised)."""
|
|
211
|
+
if self._store is None:
|
|
212
|
+
self._store = GraphStore(self.db_path)
|
|
213
|
+
return self._store
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def embedder(self) -> Embedder:
|
|
217
|
+
"""Embedding backend (lazy-initialised, shared between index and query)."""
|
|
218
|
+
if self._embedder is None:
|
|
219
|
+
suppress_ingestion_logging()
|
|
220
|
+
self._embedder = SentenceTransformerEmbedder(self.model_name)
|
|
221
|
+
return self._embedder
|
|
222
|
+
|
|
223
|
+
@property
|
|
224
|
+
def index(self) -> SemanticIndex:
|
|
225
|
+
"""LanceDB semantic index (lazy-initialised)."""
|
|
226
|
+
if self._index is None:
|
|
227
|
+
extractor = self.make_extractor()
|
|
228
|
+
self._index = SemanticIndex(
|
|
229
|
+
self.lancedb_dir,
|
|
230
|
+
embedder=self.embedder,
|
|
231
|
+
table=self.table_name,
|
|
232
|
+
index_kinds=extractor.meaningful_node_kinds(),
|
|
233
|
+
)
|
|
234
|
+
return self._index
|
|
235
|
+
|
|
236
|
+
# ------------------------------------------------------------------
|
|
237
|
+
# Build
|
|
238
|
+
# ------------------------------------------------------------------
|
|
239
|
+
|
|
240
|
+
def build(self, *, wipe: bool = False) -> BuildStats:
|
|
241
|
+
"""Full pipeline: extraction → SQLite → LanceDB.
|
|
242
|
+
|
|
243
|
+
:param wipe: Clear existing data before writing.
|
|
244
|
+
:return: :class:`~pycode_kg.module.types.BuildStats`.
|
|
245
|
+
"""
|
|
246
|
+
graph_stats = self.build_graph(wipe=wipe)
|
|
247
|
+
index_stats = self.build_index(wipe=wipe)
|
|
248
|
+
graph_stats.indexed_rows = index_stats.indexed_rows
|
|
249
|
+
graph_stats.index_dim = index_stats.index_dim
|
|
250
|
+
return graph_stats
|
|
251
|
+
|
|
252
|
+
def build_graph(self, *, wipe: bool = False) -> BuildStats:
|
|
253
|
+
"""Extraction → SQLite only (no vector indexing).
|
|
254
|
+
|
|
255
|
+
Calls :meth:`make_extractor`, drains the iterator into two lists
|
|
256
|
+
of :class:`~pycode_kg.module.extractor.NodeSpec` and
|
|
257
|
+
:class:`~pycode_kg.module.extractor.EdgeSpec`, converts them to the
|
|
258
|
+
v0 :class:`~pycode_kg.pycodekg.Node` / :class:`~pycode_kg.pycodekg.Edge`
|
|
259
|
+
primitives, writes to SQLite, then calls :meth:`_post_build_hook`
|
|
260
|
+
for any domain-specific post-processing (e.g. symbol resolution).
|
|
261
|
+
|
|
262
|
+
:param wipe: Clear existing graph before writing.
|
|
263
|
+
:return: :class:`~pycode_kg.module.types.BuildStats`
|
|
264
|
+
(``indexed_rows`` will be ``None``).
|
|
265
|
+
"""
|
|
266
|
+
extractor = self.make_extractor()
|
|
267
|
+
node_specs: list[NodeSpec] = []
|
|
268
|
+
edge_specs: list[EdgeSpec] = []
|
|
269
|
+
for item in extractor.extract():
|
|
270
|
+
if isinstance(item, NodeSpec):
|
|
271
|
+
node_specs.append(item)
|
|
272
|
+
else:
|
|
273
|
+
edge_specs.append(item)
|
|
274
|
+
|
|
275
|
+
nodes = [_nodespec_to_node(s) for s in node_specs]
|
|
276
|
+
edges = [_edgespec_to_edge(s) for s in edge_specs]
|
|
277
|
+
|
|
278
|
+
self.store.write(nodes, edges, wipe=wipe)
|
|
279
|
+
self._post_build_hook(self.store)
|
|
280
|
+
|
|
281
|
+
s = self.store.stats()
|
|
282
|
+
return BuildStats(
|
|
283
|
+
repo_root=str(self.repo_root),
|
|
284
|
+
db_path=str(self.db_path),
|
|
285
|
+
total_nodes=s["total_nodes"],
|
|
286
|
+
total_edges=s["total_edges"],
|
|
287
|
+
node_counts=s["node_counts"],
|
|
288
|
+
edge_counts=s["edge_counts"],
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
def build_index(self, *, wipe: bool = False) -> BuildStats:
|
|
292
|
+
"""SQLite → LanceDB only (graph must already exist).
|
|
293
|
+
|
|
294
|
+
:param wipe: Delete existing vectors before indexing.
|
|
295
|
+
:return: :class:`~pycode_kg.module.types.BuildStats` with
|
|
296
|
+
``indexed_rows`` and ``index_dim`` set.
|
|
297
|
+
"""
|
|
298
|
+
idx_stats = self.index.build(self.store, wipe=wipe)
|
|
299
|
+
s = self.store.stats()
|
|
300
|
+
return BuildStats(
|
|
301
|
+
repo_root=str(self.repo_root),
|
|
302
|
+
db_path=str(self.db_path),
|
|
303
|
+
total_nodes=s["total_nodes"],
|
|
304
|
+
total_edges=s["total_edges"],
|
|
305
|
+
node_counts=s["node_counts"],
|
|
306
|
+
edge_counts=s["edge_counts"],
|
|
307
|
+
indexed_rows=idx_stats["indexed_rows"],
|
|
308
|
+
index_dim=idx_stats["dim"],
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
def _post_build_hook(self, store: GraphStore) -> None:
|
|
312
|
+
"""Hook called after ``store.write()`` in :meth:`build_graph`.
|
|
313
|
+
|
|
314
|
+
Override in subclasses for domain-specific post-processing, e.g.
|
|
315
|
+
symbol resolution (``store.resolve_symbols()`` for Python KGs).
|
|
316
|
+
|
|
317
|
+
:param store: The :class:`~pycode_kg.store.GraphStore` just written to.
|
|
318
|
+
"""
|
|
319
|
+
|
|
320
|
+
# ------------------------------------------------------------------
|
|
321
|
+
# Query
|
|
322
|
+
# ------------------------------------------------------------------
|
|
323
|
+
|
|
324
|
+
def query(
|
|
325
|
+
self,
|
|
326
|
+
q: str,
|
|
327
|
+
*,
|
|
328
|
+
k: int = 8,
|
|
329
|
+
hop: int = 1,
|
|
330
|
+
rels: tuple[str, ...] = DEFAULT_RELS,
|
|
331
|
+
include_symbols: bool = False,
|
|
332
|
+
max_nodes: int = 25,
|
|
333
|
+
min_score: float = 0.0,
|
|
334
|
+
max_per_module: int | None = None,
|
|
335
|
+
rerank_mode: str = "legacy",
|
|
336
|
+
rerank_semantic_weight: float = 0.7,
|
|
337
|
+
rerank_lexical_weight: float = 0.3,
|
|
338
|
+
) -> QueryResult:
|
|
339
|
+
"""Hybrid query: semantic seeding + structural graph expansion.
|
|
340
|
+
|
|
341
|
+
Seeds with vector ANN search (LanceDB), expands through the graph
|
|
342
|
+
(SQLite) for ``hop`` hops along ``rels`` edge types, then reranks
|
|
343
|
+
results by the selected strategy.
|
|
344
|
+
|
|
345
|
+
:param q: Natural-language query.
|
|
346
|
+
:param k: Top-K semantic hits.
|
|
347
|
+
:param hop: Graph expansion hops (0 = pure semantic).
|
|
348
|
+
:param rels: Edge types to follow during expansion.
|
|
349
|
+
:param include_symbols: Include unresolved stub nodes in results.
|
|
350
|
+
:param max_nodes: Maximum nodes to return.
|
|
351
|
+
:param min_score: Minimum semantic score for seed inclusion.
|
|
352
|
+
:param max_per_module: Optional cap on returned nodes per module path.
|
|
353
|
+
:param rerank_mode: ``'legacy'`` | ``'semantic'`` | ``'hybrid'``.
|
|
354
|
+
:param rerank_semantic_weight: Semantic weight for ``'hybrid'`` mode.
|
|
355
|
+
:param rerank_lexical_weight: Lexical weight for ``'hybrid'`` mode.
|
|
356
|
+
:return: :class:`~pycode_kg.module.types.QueryResult`.
|
|
357
|
+
"""
|
|
358
|
+
q_norm = normalize_query_text(q)
|
|
359
|
+
hits = self.index.search(q_norm, k=k)
|
|
360
|
+
if min_score > 0.0:
|
|
361
|
+
hits = [h for h in hits if semantic_score_from_distance(h.distance) >= min_score]
|
|
362
|
+
seed_ids: set[str] = {h.id for h in hits}
|
|
363
|
+
seed_rank: dict[str, float] = {h.id: h.distance for h in hits}
|
|
364
|
+
q_toks = query_tokens(q_norm)
|
|
365
|
+
|
|
366
|
+
meta = self.store.expand(seed_ids, hop=hop, rels=rels)
|
|
367
|
+
all_ids = set(meta.keys())
|
|
368
|
+
|
|
369
|
+
nodes: list[dict] = []
|
|
370
|
+
kept_ids: set[str] = set()
|
|
371
|
+
module_counts: dict[str, int] = {}
|
|
372
|
+
|
|
373
|
+
def _rank_key(nid: str) -> tuple[float, float, int, int, str]:
|
|
374
|
+
prov = meta[nid]
|
|
375
|
+
dist = seed_rank.get(prov.via_seed, 1e9)
|
|
376
|
+
n = self.store.node(nid)
|
|
377
|
+
kind = n["kind"] if n else "symbol"
|
|
378
|
+
sem = semantic_score_from_distance(dist)
|
|
379
|
+
lex = lexical_overlap_score(q_toks, n or {})
|
|
380
|
+
hybrid = (
|
|
381
|
+
rerank_semantic_weight * sem + rerank_lexical_weight * lex
|
|
382
|
+
if (rerank_semantic_weight + rerank_lexical_weight) > 0
|
|
383
|
+
else sem
|
|
384
|
+
)
|
|
385
|
+
if rerank_mode == "hybrid":
|
|
386
|
+
return (-hybrid, float(prov.best_hop), self._kind_priority(kind), 0, nid)
|
|
387
|
+
if rerank_mode == "semantic":
|
|
388
|
+
return (-sem, float(prov.best_hop), self._kind_priority(kind), 0, nid)
|
|
389
|
+
return (float(prov.best_hop), dist, self._kind_priority(kind), 0, nid)
|
|
390
|
+
|
|
391
|
+
for nid in sorted(all_ids, key=_rank_key):
|
|
392
|
+
if len(nodes) >= max_nodes:
|
|
393
|
+
break
|
|
394
|
+
n = self.store.node(nid)
|
|
395
|
+
if not n:
|
|
396
|
+
continue
|
|
397
|
+
if not include_symbols and n["kind"] == "symbol":
|
|
398
|
+
continue
|
|
399
|
+
module_path = n.get("module_path") or ""
|
|
400
|
+
if max_per_module is not None and module_path:
|
|
401
|
+
if module_counts.get(module_path, 0) >= max_per_module:
|
|
402
|
+
continue
|
|
403
|
+
module_counts[module_path] = module_counts.get(module_path, 0) + 1
|
|
404
|
+
|
|
405
|
+
prov = meta[nid]
|
|
406
|
+
dist = seed_rank.get(prov.via_seed, 1e9)
|
|
407
|
+
sem = semantic_score_from_distance(dist)
|
|
408
|
+
lex = lexical_overlap_score(q_toks, n)
|
|
409
|
+
n["relevance"] = {
|
|
410
|
+
"score": (
|
|
411
|
+
rerank_semantic_weight * sem + rerank_lexical_weight * lex
|
|
412
|
+
if rerank_mode == "hybrid"
|
|
413
|
+
else sem
|
|
414
|
+
),
|
|
415
|
+
"semantic": sem,
|
|
416
|
+
"lexical": lex,
|
|
417
|
+
"docstring_signal": docstring_signal(n.get("docstring")),
|
|
418
|
+
"hop": prov.best_hop,
|
|
419
|
+
"via_seed": prov.via_seed,
|
|
420
|
+
"mode": rerank_mode,
|
|
421
|
+
}
|
|
422
|
+
kept_ids.add(nid)
|
|
423
|
+
nodes.append(n)
|
|
424
|
+
|
|
425
|
+
# Normalize composite scores so the top hit approaches 1.0, making
|
|
426
|
+
# min_score thresholds consistent across queries with different embedding distances.
|
|
427
|
+
if nodes:
|
|
428
|
+
max_score = max(n["relevance"]["score"] for n in nodes) or 1.0
|
|
429
|
+
if max_score > 0.0:
|
|
430
|
+
for n in nodes:
|
|
431
|
+
n["relevance"]["score"] = round(n["relevance"]["score"] / max_score, 6)
|
|
432
|
+
|
|
433
|
+
edges = self.store.edges_within(kept_ids)
|
|
434
|
+
return QueryResult(
|
|
435
|
+
query=q,
|
|
436
|
+
seeds=len(seed_ids),
|
|
437
|
+
expanded_nodes=len(all_ids),
|
|
438
|
+
returned_nodes=len(nodes),
|
|
439
|
+
hop=hop,
|
|
440
|
+
rels=list(rels),
|
|
441
|
+
nodes=nodes,
|
|
442
|
+
edges=edges,
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
# ------------------------------------------------------------------
|
|
446
|
+
# Snippet pack
|
|
447
|
+
# ------------------------------------------------------------------
|
|
448
|
+
|
|
449
|
+
def pack(
|
|
450
|
+
self,
|
|
451
|
+
q: str,
|
|
452
|
+
*,
|
|
453
|
+
k: int = 8,
|
|
454
|
+
hop: int = 1,
|
|
455
|
+
rels: tuple[str, ...] = DEFAULT_RELS,
|
|
456
|
+
include_symbols: bool = False,
|
|
457
|
+
context: int = 5,
|
|
458
|
+
max_lines: int = 60,
|
|
459
|
+
max_nodes: int | None = 15,
|
|
460
|
+
min_score: float = 0.0,
|
|
461
|
+
max_per_module: int | None = None,
|
|
462
|
+
rerank_mode: str = "legacy",
|
|
463
|
+
rerank_semantic_weight: float = 0.7,
|
|
464
|
+
rerank_lexical_weight: float = 0.3,
|
|
465
|
+
missing_lineno_policy: str = "cap_or_skip",
|
|
466
|
+
) -> SnippetPack:
|
|
467
|
+
"""Hybrid query + source-grounded snippet extraction.
|
|
468
|
+
|
|
469
|
+
:param q: Natural-language query.
|
|
470
|
+
:param k: Top-K semantic hits.
|
|
471
|
+
:param hop: Graph expansion hops.
|
|
472
|
+
:param rels: Edge types to follow during expansion.
|
|
473
|
+
:param include_symbols: Include unresolved stub nodes.
|
|
474
|
+
:param context: Extra context lines around each definition span.
|
|
475
|
+
:param max_lines: Maximum lines per snippet block.
|
|
476
|
+
:param max_nodes: Maximum nodes to return (``None`` = no limit).
|
|
477
|
+
:param min_score: Minimum semantic score for seed inclusion.
|
|
478
|
+
:param max_per_module: Optional cap on returned nodes per module.
|
|
479
|
+
:param rerank_mode: ``'legacy'`` | ``'semantic'`` | ``'hybrid'``.
|
|
480
|
+
:param rerank_semantic_weight: Semantic weight for ``'hybrid'`` mode.
|
|
481
|
+
:param rerank_lexical_weight: Lexical weight for ``'hybrid'`` mode.
|
|
482
|
+
:param missing_lineno_policy: ``'cap_or_skip'`` (default) or ``'legacy'``.
|
|
483
|
+
:return: :class:`~pycode_kg.module.types.SnippetPack`.
|
|
484
|
+
"""
|
|
485
|
+
q_norm = normalize_query_text(q)
|
|
486
|
+
hits = self.index.search(q_norm, k=k)
|
|
487
|
+
if min_score > 0.0:
|
|
488
|
+
hits = [h for h in hits if semantic_score_from_distance(h.distance) >= min_score]
|
|
489
|
+
seed_rank: dict[str, dict] = {h.id: {"rank": h.rank, "dist": h.distance} for h in hits}
|
|
490
|
+
seed_ids: set[str] = set(seed_rank.keys())
|
|
491
|
+
q_toks = query_tokens(q_norm)
|
|
492
|
+
warnings: list[str] = []
|
|
493
|
+
|
|
494
|
+
meta = self.store.expand(seed_ids, hop=hop, rels=rels)
|
|
495
|
+
all_ids = set(meta.keys())
|
|
496
|
+
|
|
497
|
+
raw_nodes: list[dict] = []
|
|
498
|
+
for nid in sorted(all_ids):
|
|
499
|
+
n = self.store.node(nid)
|
|
500
|
+
if not n:
|
|
501
|
+
continue
|
|
502
|
+
if not include_symbols and n["kind"] == "symbol":
|
|
503
|
+
continue
|
|
504
|
+
|
|
505
|
+
prov = meta[nid]
|
|
506
|
+
base_dist = seed_rank.get(prov.via_seed, {"dist": 1e9})["dist"]
|
|
507
|
+
kind_pri = self._kind_priority(n["kind"])
|
|
508
|
+
sem = semantic_score_from_distance(base_dist)
|
|
509
|
+
lex = lexical_overlap_score(q_toks, n)
|
|
510
|
+
hybrid = (
|
|
511
|
+
rerank_semantic_weight * sem + rerank_lexical_weight * lex
|
|
512
|
+
if (rerank_semantic_weight + rerank_lexical_weight) > 0
|
|
513
|
+
else sem
|
|
514
|
+
)
|
|
515
|
+
if rerank_mode == "hybrid":
|
|
516
|
+
n["_rank_key"] = (-hybrid, float(prov.best_hop), kind_pri, 0, n["id"])
|
|
517
|
+
elif rerank_mode == "semantic":
|
|
518
|
+
n["_rank_key"] = (-sem, float(prov.best_hop), kind_pri, 0, n["id"])
|
|
519
|
+
else:
|
|
520
|
+
n["_rank_key"] = (float(prov.best_hop), base_dist, kind_pri, 0, n["id"])
|
|
521
|
+
n["_best_hop"] = prov.best_hop
|
|
522
|
+
n["_via_seed"] = prov.via_seed
|
|
523
|
+
n["relevance"] = {
|
|
524
|
+
"score": hybrid if rerank_mode == "hybrid" else sem,
|
|
525
|
+
"semantic": sem,
|
|
526
|
+
"lexical": lex,
|
|
527
|
+
"docstring_signal": docstring_signal(n.get("docstring")),
|
|
528
|
+
"hop": prov.best_hop,
|
|
529
|
+
"via_seed": prov.via_seed,
|
|
530
|
+
"mode": rerank_mode,
|
|
531
|
+
}
|
|
532
|
+
raw_nodes.append(n)
|
|
533
|
+
|
|
534
|
+
# Attach spans (needed for dedup)
|
|
535
|
+
file_cache: dict[str, list[str]] = {}
|
|
536
|
+
spans_by_qualname: dict[tuple[str, str], tuple[int, int]] = {}
|
|
537
|
+
for n in raw_nodes:
|
|
538
|
+
mp = n.get("module_path")
|
|
539
|
+
if not mp:
|
|
540
|
+
n["_span"] = None
|
|
541
|
+
continue
|
|
542
|
+
if mp not in file_cache:
|
|
543
|
+
file_cache[mp] = read_lines(safe_join(self.repo_root, mp))
|
|
544
|
+
lines = file_cache[mp]
|
|
545
|
+
lineno = n.get("lineno")
|
|
546
|
+
if n.get("kind") != "module" and lineno is None and missing_lineno_policy != "legacy":
|
|
547
|
+
qualname = n.get("qualname") or ""
|
|
548
|
+
parent_span = None
|
|
549
|
+
if qualname and "." in qualname:
|
|
550
|
+
parent_key = (mp, qualname.rsplit(".", 1)[0])
|
|
551
|
+
parent_span = spans_by_qualname.get(parent_key)
|
|
552
|
+
if parent_span:
|
|
553
|
+
fallback_cap = min(max_lines, max(20, context * 4))
|
|
554
|
+
p_start, p_end = parent_span
|
|
555
|
+
capped_end = min(p_end, p_start + fallback_cap - 1)
|
|
556
|
+
n["_span"] = (p_start, capped_end)
|
|
557
|
+
warnings.append(
|
|
558
|
+
f"Missing line metadata for `{n['id']}`; "
|
|
559
|
+
f"using capped parent span {p_start}-{capped_end}."
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
n["_span"] = None
|
|
563
|
+
warnings.append(
|
|
564
|
+
f"Missing line metadata for `{n['id']}`; "
|
|
565
|
+
"snippet omitted (no parent span available)."
|
|
566
|
+
)
|
|
567
|
+
else:
|
|
568
|
+
n["_span"] = compute_span(
|
|
569
|
+
n["kind"],
|
|
570
|
+
lineno,
|
|
571
|
+
n.get("end_lineno"),
|
|
572
|
+
context=context,
|
|
573
|
+
max_lines=max_lines,
|
|
574
|
+
file_nlines=len(lines),
|
|
575
|
+
)
|
|
576
|
+
if n.get("qualname") and n.get("_span"):
|
|
577
|
+
spans_by_qualname[(mp, n["qualname"])] = n["_span"]
|
|
578
|
+
|
|
579
|
+
raw_nodes.sort(key=lambda x: x["_rank_key"])
|
|
580
|
+
|
|
581
|
+
# Deduplicate by file + overlapping span
|
|
582
|
+
kept: list[dict] = []
|
|
583
|
+
kept_by_file: dict[str, list[tuple[tuple[int, int], str]]] = {}
|
|
584
|
+
module_counts: dict[str, int] = {}
|
|
585
|
+
|
|
586
|
+
for n in raw_nodes:
|
|
587
|
+
if max_nodes is not None and len(kept) >= max_nodes:
|
|
588
|
+
break
|
|
589
|
+
mp = n.get("module_path") or ""
|
|
590
|
+
span = n.get("_span")
|
|
591
|
+
|
|
592
|
+
if not mp or not span or span[1] < span[0]:
|
|
593
|
+
kept.append(n)
|
|
594
|
+
continue
|
|
595
|
+
|
|
596
|
+
if any(spans_overlap(span, s2) for s2, _ in kept_by_file.get(mp, [])):
|
|
597
|
+
continue
|
|
598
|
+
|
|
599
|
+
if max_per_module is not None and mp:
|
|
600
|
+
if module_counts.get(mp, 0) >= max_per_module:
|
|
601
|
+
continue
|
|
602
|
+
module_counts[mp] = module_counts.get(mp, 0) + 1
|
|
603
|
+
|
|
604
|
+
kept.append(n)
|
|
605
|
+
kept_by_file.setdefault(mp, []).append((span, n["id"]))
|
|
606
|
+
|
|
607
|
+
kept_ids: set[str] = {n["id"] for n in kept}
|
|
608
|
+
edges = self.store.edges_within(kept_ids)
|
|
609
|
+
|
|
610
|
+
# Attach snippets
|
|
611
|
+
for n in kept:
|
|
612
|
+
mp = n.get("module_path")
|
|
613
|
+
span = n.get("_span")
|
|
614
|
+
if not mp or not span:
|
|
615
|
+
continue
|
|
616
|
+
if mp not in file_cache:
|
|
617
|
+
file_cache[mp] = read_lines(safe_join(self.repo_root, mp))
|
|
618
|
+
lines = file_cache[mp]
|
|
619
|
+
start, end = span
|
|
620
|
+
|
|
621
|
+
if n.get("kind") == "module" and len(lines) > max_lines:
|
|
622
|
+
contained = [
|
|
623
|
+
self.store.node(cn_id)
|
|
624
|
+
for cn_id in (
|
|
625
|
+
row[0]
|
|
626
|
+
for row in self.store.con.execute(
|
|
627
|
+
"SELECT dst FROM edges WHERE src = ? AND rel = 'CONTAINS'",
|
|
628
|
+
(n["id"],),
|
|
629
|
+
).fetchall()
|
|
630
|
+
)
|
|
631
|
+
if self.store.node(cn_id) is not None
|
|
632
|
+
]
|
|
633
|
+
contained_nodes: list[dict] = [c for c in contained if c is not None]
|
|
634
|
+
n["snippet"] = make_module_summary(
|
|
635
|
+
mp, lines, n.get("docstring"), contained_nodes, max_lines
|
|
636
|
+
)
|
|
637
|
+
elif end >= start and lines:
|
|
638
|
+
n["snippet"] = make_snippet(mp, lines, start, end)
|
|
639
|
+
|
|
640
|
+
# Strip internal keys
|
|
641
|
+
for n in kept:
|
|
642
|
+
for key in [k for k in n if k.startswith("_")]:
|
|
643
|
+
del n[key]
|
|
644
|
+
|
|
645
|
+
return SnippetPack(
|
|
646
|
+
query=q,
|
|
647
|
+
seeds=len(seed_ids),
|
|
648
|
+
expanded_nodes=len(all_ids),
|
|
649
|
+
returned_nodes=len(kept),
|
|
650
|
+
hop=hop,
|
|
651
|
+
rels=list(rels),
|
|
652
|
+
model=self.model_name,
|
|
653
|
+
nodes=kept,
|
|
654
|
+
edges=edges,
|
|
655
|
+
warnings=warnings,
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
# ------------------------------------------------------------------
|
|
659
|
+
# Convenience
|
|
660
|
+
# ------------------------------------------------------------------
|
|
661
|
+
|
|
662
|
+
def callers(self, node_id: str, *, rel: str = "CALLS") -> list[dict]:
|
|
663
|
+
"""Return all nodes that call *node_id*, resolving through stubs.
|
|
664
|
+
|
|
665
|
+
:param node_id: Target node identifier.
|
|
666
|
+
:param rel: Relation type to invert (default ``"CALLS"``).
|
|
667
|
+
:return: Deduplicated list of caller node dicts.
|
|
668
|
+
"""
|
|
669
|
+
return self.store.callers_of(node_id, rel=rel)
|
|
670
|
+
|
|
671
|
+
def stats(self) -> dict:
|
|
672
|
+
"""Return store statistics (node/edge counts by kind/relation).
|
|
673
|
+
|
|
674
|
+
:return: Dictionary from :meth:`~pycode_kg.store.GraphStore.stats`.
|
|
675
|
+
"""
|
|
676
|
+
return self.store.stats()
|
|
677
|
+
|
|
678
|
+
def node(self, node_id: str) -> dict | None:
|
|
679
|
+
"""Fetch a single node by ID from the store.
|
|
680
|
+
|
|
681
|
+
:param node_id: Stable node identifier.
|
|
682
|
+
:return: Node dict or ``None`` if not found.
|
|
683
|
+
"""
|
|
684
|
+
return self.store.node(node_id)
|
|
685
|
+
|
|
686
|
+
def close(self) -> None:
|
|
687
|
+
"""Close the underlying SQLite connection."""
|
|
688
|
+
if self._store is not None:
|
|
689
|
+
self._store.close()
|
|
690
|
+
|
|
691
|
+
def __enter__(self) -> KGModule:
|
|
692
|
+
"""Enter the runtime context.
|
|
693
|
+
|
|
694
|
+
:return: This :class:`KGModule` instance.
|
|
695
|
+
"""
|
|
696
|
+
return self
|
|
697
|
+
|
|
698
|
+
def __exit__(self, *_: object) -> None:
|
|
699
|
+
"""Exit the runtime context and close the SQLite connection."""
|
|
700
|
+
self.close()
|
|
701
|
+
|
|
702
|
+
# ------------------------------------------------------------------
|
|
703
|
+
# Domain hook — subclasses override for node kind ordering
|
|
704
|
+
# ------------------------------------------------------------------
|
|
705
|
+
|
|
706
|
+
def _kind_priority(self, kind: str) -> int:
|
|
707
|
+
"""Return the sort priority for a node kind (lower = higher priority).
|
|
708
|
+
|
|
709
|
+
Used as a tiebreaker in query/pack ranking when scores are equal.
|
|
710
|
+
Default: all unknown kinds sort last (priority 99).
|
|
711
|
+
|
|
712
|
+
Override in subclasses for domain-specific ordering::
|
|
713
|
+
|
|
714
|
+
def _kind_priority(self, kind):
|
|
715
|
+
return {"function": 0, "class": 1, "module": 2}.get(kind, 99)
|
|
716
|
+
|
|
717
|
+
:param kind: Node kind string.
|
|
718
|
+
:return: Integer priority (lower = ranked first).
|
|
719
|
+
"""
|
|
720
|
+
return 99
|