hubmesh 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hubmesh-0.1.0/LICENSE +21 -0
- hubmesh-0.1.0/PKG-INFO +253 -0
- hubmesh-0.1.0/README.md +204 -0
- hubmesh-0.1.0/pyproject.toml +53 -0
- hubmesh-0.1.0/setup.cfg +4 -0
- hubmesh-0.1.0/src/hubmesh/__init__.py +21 -0
- hubmesh-0.1.0/src/hubmesh/adapters/__init__.py +19 -0
- hubmesh-0.1.0/src/hubmesh/adapters/base.py +38 -0
- hubmesh-0.1.0/src/hubmesh/adapters/chroma.py +162 -0
- hubmesh-0.1.0/src/hubmesh/adapters/inmemory.py +111 -0
- hubmesh-0.1.0/src/hubmesh/adapters/qdrant.py +203 -0
- hubmesh-0.1.0/src/hubmesh/chunking.py +158 -0
- hubmesh-0.1.0/src/hubmesh/entity_linker.py +140 -0
- hubmesh-0.1.0/src/hubmesh/graph.py +84 -0
- hubmesh-0.1.0/src/hubmesh/kg.py +230 -0
- hubmesh-0.1.0/src/hubmesh/kg_llm.py +231 -0
- hubmesh-0.1.0/src/hubmesh/packing.py +72 -0
- hubmesh-0.1.0/src/hubmesh/paths.py +107 -0
- hubmesh-0.1.0/src/hubmesh/planner.py +341 -0
- hubmesh-0.1.0/src/hubmesh/ppr.py +110 -0
- hubmesh-0.1.0/src/hubmesh/scoring.py +143 -0
- hubmesh-0.1.0/src/hubmesh/types.py +42 -0
- hubmesh-0.1.0/src/hubmesh.egg-info/PKG-INFO +253 -0
- hubmesh-0.1.0/src/hubmesh.egg-info/SOURCES.txt +28 -0
- hubmesh-0.1.0/src/hubmesh.egg-info/dependency_links.txt +1 -0
- hubmesh-0.1.0/src/hubmesh.egg-info/requires.txt +36 -0
- hubmesh-0.1.0/src/hubmesh.egg-info/top_level.txt +1 -0
- hubmesh-0.1.0/tests/test_chroma_adapter.py +49 -0
- hubmesh-0.1.0/tests/test_planner.py +173 -0
- hubmesh-0.1.0/tests/test_qdrant_adapter.py +57 -0
hubmesh-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Datta Sai Krishna Naidu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
hubmesh-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hubmesh
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Centrality-aware GraphRAG retrieval planner — drop-in layer over any vector DB
|
|
5
|
+
Author-email: Datta Sai Krishna Naidu <dattasai293@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/dattasaikrishnanaidu/hubmesh
|
|
8
|
+
Project-URL: Issues, https://github.com/dattasaikrishnanaidu/hubmesh/issues
|
|
9
|
+
Keywords: rag,graphrag,vector-search,personalized-pagerank,context-selection,multi-hop-retrieval
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: numpy>=1.26
|
|
20
|
+
Requires-Dist: networkx>=3.2
|
|
21
|
+
Requires-Dist: scipy>=1.11
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
24
|
+
Requires-Dist: pytest-cov>=4.1; extra == "dev"
|
|
25
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
26
|
+
Requires-Dist: qdrant-client>=1.10; extra == "dev"
|
|
27
|
+
Requires-Dist: chromadb>=0.5; extra == "dev"
|
|
28
|
+
Provides-Extra: benchmarks
|
|
29
|
+
Requires-Dist: datasets>=2.18; extra == "benchmarks"
|
|
30
|
+
Requires-Dist: tqdm>=4.66; extra == "benchmarks"
|
|
31
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "benchmarks"
|
|
32
|
+
Requires-Dist: spacy>=3.7; extra == "benchmarks"
|
|
33
|
+
Provides-Extra: qdrant
|
|
34
|
+
Requires-Dist: qdrant-client>=1.10; extra == "qdrant"
|
|
35
|
+
Provides-Extra: chroma
|
|
36
|
+
Requires-Dist: chromadb>=0.5; extra == "chroma"
|
|
37
|
+
Provides-Extra: kg
|
|
38
|
+
Requires-Dist: spacy>=3.7; extra == "kg"
|
|
39
|
+
Provides-Extra: linker
|
|
40
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "linker"
|
|
41
|
+
Provides-Extra: all
|
|
42
|
+
Requires-Dist: qdrant-client>=1.10; extra == "all"
|
|
43
|
+
Requires-Dist: chromadb>=0.5; extra == "all"
|
|
44
|
+
Requires-Dist: spacy>=3.7; extra == "all"
|
|
45
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "all"
|
|
46
|
+
Requires-Dist: datasets>=2.18; extra == "all"
|
|
47
|
+
Requires-Dist: tqdm>=4.66; extra == "all"
|
|
48
|
+
Dynamic: license-file
|
|
49
|
+
|
|
50
|
+
# hubmesh
|
|
51
|
+
|
|
52
|
+
[](https://github.com/DemigodDSK/hubmesh/actions/workflows/test.yml)
|
|
53
|
+
[](https://github.com/DemigodDSK/hubmesh)
|
|
54
|
+
[](https://github.com/DemigodDSK/hubmesh/blob/main/LICENSE)
|
|
55
|
+
[](https://github.com/DemigodDSK/hubmesh/releases)
|
|
56
|
+
|
|
57
|
+
**Centrality-aware GraphRAG retrieval planner. Drop-in layer over any vector DB.**
|
|
58
|
+
|
|
59
|
+
`hubmesh` is a Python library that improves multi-hop RAG quality on top of an existing
|
|
60
|
+
vector database. You don't replace your infrastructure — you add a smart planner between
|
|
61
|
+
your vector DB and your LLM.
|
|
62
|
+
|
|
63
|
+
## What problem this solves
|
|
64
|
+
|
|
65
|
+
Naive vector retrieval ("embed query, get top-k by cosine similarity") fails on multi-hop
|
|
66
|
+
questions like *"Where was the founder of the company that acquired Slack born?"* The
|
|
67
|
+
correct answer requires retrieving entities along a reasoning path, not the single most
|
|
68
|
+
similar item.
|
|
69
|
+
|
|
70
|
+
GraphRAG and HippoRAG showed that running a small Personalized PageRank over a knowledge
|
|
71
|
+
graph at query time can substantially improve multi-hop retrieval. `hubmesh` extends
|
|
72
|
+
that line with two contributions:
|
|
73
|
+
|
|
74
|
+
1. **Multi-component seed selection.** Instead of picking PPR seeds by raw query
|
|
75
|
+
similarity (which picks wrong-community seeds at high feature overlap), seeds are
|
|
76
|
+
chosen by a multi-component score combining query relevance, structural fit, and
|
|
77
|
+
coverage diversity.
|
|
78
|
+
2. **Budget-aware context packing.** Once relevant entities are scored, pack them into
|
|
79
|
+
the LLM's context window with explicit coverage and redundancy control rather than
|
|
80
|
+
just truncating top-k.
|
|
81
|
+
|
|
82
|
+
The multi-component scoring pattern is adapted from the NNSI framework
|
|
83
|
+
([Naidu Dsk, iComp 2025](https://example.invalid)) for SDN topology
|
|
84
|
+
optimization, repurposed here for retrieval planning.
|
|
85
|
+
|
|
86
|
+
## Quickstart
|
|
87
|
+
|
|
88
|
+
### In-memory (testing, small corpora)
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from hubmesh import Planner
|
|
92
|
+
from hubmesh.adapters import InMemoryStore
|
|
93
|
+
|
|
94
|
+
embed = ... # callable: text -> np.ndarray
|
|
95
|
+
docs = [...] # list of Document or strings or dicts
|
|
96
|
+
|
|
97
|
+
store = InMemoryStore.from_documents(docs, embed=embed)
|
|
98
|
+
planner = Planner(store=store, embed=embed)
|
|
99
|
+
result = planner.retrieve(query="...", top_k=10, budget_tokens=4000)
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Qdrant adapter (production)
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from hubmesh import Planner
|
|
106
|
+
from hubmesh.adapters import QdrantStore
|
|
107
|
+
|
|
108
|
+
store = QdrantStore.from_documents(docs) # in-memory
|
|
109
|
+
store = QdrantStore.from_documents(docs, path="./qdrant_data") # on-disk
|
|
110
|
+
store = QdrantStore.from_documents(docs, url="http://localhost:6333") # remote
|
|
111
|
+
|
|
112
|
+
planner = Planner(store=store, embed=embed)
|
|
113
|
+
result = planner.retrieve(query="...", top_k=10)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Chroma adapter
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from hubmesh.adapters import ChromaStore
|
|
120
|
+
|
|
121
|
+
store = ChromaStore.from_documents(docs) # ephemeral
|
|
122
|
+
store = ChromaStore.from_documents(docs, persist_directory="./chroma_data")
|
|
123
|
+
store = ChromaStore.from_documents(docs, host="localhost", port=8000)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### Multi-hop / KG mode
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from hubmesh.kg import build_entity_kg
|
|
130
|
+
import spacy
|
|
131
|
+
|
|
132
|
+
nlp = spacy.load("en_core_web_sm")
|
|
133
|
+
kg = build_entity_kg(docs, nlp=nlp)
|
|
134
|
+
|
|
135
|
+
planner = Planner(store=store, kg=kg, nlp=nlp)
|
|
136
|
+
result = planner.retrieve(query="Where was the founder of the company that bought Slack born?",
|
|
137
|
+
top_k=10, budget_tokens=4000)
|
|
138
|
+
|
|
139
|
+
# RetrievalResult includes reasoning paths showing why each doc was returned
|
|
140
|
+
for path in result.reasoning:
|
|
141
|
+
print(f" score={path.score:.3f} {' → '.join(path.node_ids)}")
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### LLM-extracted KG (richer than spaCy)
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from hubmesh.kg_llm import build_entity_kg_llm
|
|
148
|
+
|
|
149
|
+
def llm(prompt): # provider-agnostic — bring your own
|
|
150
|
+
return your_llm_call(prompt)
|
|
151
|
+
|
|
152
|
+
kg = build_entity_kg_llm(docs, llm=llm, cache_path="kg_cache.json")
|
|
153
|
+
planner = Planner(store=store, kg=kg)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Better entity linking
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from hubmesh.kg import build_entity_kg
|
|
160
|
+
from hubmesh.entity_linker import EmbeddingLinker, make_st_embedder
|
|
161
|
+
|
|
162
|
+
# Cluster surface variations: "United States" / "U.S." / "USA" → one entity
|
|
163
|
+
linker = EmbeddingLinker(embed=make_st_embedder(), threshold=0.82)
|
|
164
|
+
kg = build_entity_kg(docs, linker=linker)
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
### Chunking long documents
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from hubmesh import chunk_by_sentences, chunk_documents
|
|
171
|
+
|
|
172
|
+
chunks = chunk_documents(
|
|
173
|
+
[{"id": "doc1", "text": long_text}, ...],
|
|
174
|
+
strategy="sentences", target_tokens=200,
|
|
175
|
+
)
|
|
176
|
+
# Then embed chunks and index normally
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Installation
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
pip install hubmesh # core
|
|
183
|
+
pip install "hubmesh[qdrant]" # Qdrant adapter
|
|
184
|
+
pip install "hubmesh[chroma]" # Chroma adapter
|
|
185
|
+
pip install "hubmesh[kg]" # entity-linked KG (spaCy)
|
|
186
|
+
pip install "hubmesh[linker]" # embedding-based entity linker
|
|
187
|
+
pip install "hubmesh[all]" # everything
|
|
188
|
+
python -m spacy download en_core_web_sm # required for KG mode
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## Design
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
query → first-pass ANN → induced subgraph → multi-component scoring
|
|
195
|
+
↓ ↓
|
|
196
|
+
community anchoring → Personalized PageRank
|
|
197
|
+
↓ ↓
|
|
198
|
+
└─────→ ranking → budget-aware packing → context
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Each layer is independently testable and replaceable. Adapters wrap your existing vector
|
|
202
|
+
DB so you don't have to migrate.
|
|
203
|
+
|
|
204
|
+
## Benchmarks
|
|
205
|
+
|
|
206
|
+
**Headline:** on multi-hop QA, hubmesh's KG mode beats both naive cosine
|
|
207
|
+
retrieval and a HippoRAG-style PPR-only ablation that uses the same KG.
|
|
208
|
+
The win grows with hop count — exactly the regime where graph-structural
|
|
209
|
+
retrieval should help most.
|
|
210
|
+
|
|
211
|
+
| Benchmark | Setting | recall@10 vs naive |
|
|
212
|
+
|---|---|---:|
|
|
213
|
+
| HotpotQA dev, N=500 | KG mode | **+3.7 pts** |
|
|
214
|
+
| MuSiQue dev, N=300, 2-hop | KG mode | **+1.7 pts** |
|
|
215
|
+
| MuSiQue dev, N=300, 3-hop | KG mode | **+1.9 pts** |
|
|
216
|
+
| MuSiQue dev, N=300, 4-hop | KG mode | **+2.8 pts** |
|
|
217
|
+
|
|
218
|
+
vs PPR-only ablation on the same KG: **+29.1 pts** on HotpotQA — the
|
|
219
|
+
multi-component scoring is doing the work, not just "having a graph."
|
|
220
|
+
|
|
221
|
+
Latency: **~22 ms** mean / 26 ms p95 per query on a 7K-node KG (after PPR
|
|
222
|
+
matrix caching).
|
|
223
|
+
|
|
224
|
+
See [BENCHMARKS.md](BENCHMARKS.md) for the full methodology, ablations,
|
|
225
|
+
per-hop breakdown, and notes on what this proves and doesn't.
|
|
226
|
+
|
|
227
|
+
Reproduce:
|
|
228
|
+
```bash
|
|
229
|
+
python benchmarks/run_hotpotqa.py --n 500 --kg
|
|
230
|
+
python benchmarks/run_musique.py --n 300 --kg
|
|
231
|
+
python benchmarks/profile_query.py # latency profile
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
## Status
|
|
235
|
+
|
|
236
|
+
Pre-alpha (v0.1.0). Core algorithms implemented and validated; adapters for
|
|
237
|
+
in-memory, Qdrant, and Chroma; entity-linked KG with both spaCy NER and
|
|
238
|
+
LLM-based extraction; document chunking; reasoning-path explanation;
|
|
239
|
+
PPR-cache latency optimisation. Pinecone / pgvector / Weaviate adapters
|
|
240
|
+
and additional multi-hop benchmarks are tracked as
|
|
241
|
+
[good first issues](https://github.com/DemigodDSK/hubmesh/issues).
|
|
242
|
+
|
|
243
|
+
## Acknowledgements
|
|
244
|
+
|
|
245
|
+
The multi-component scoring pattern is adapted from the **Network Node Significance
|
|
246
|
+
Index (NNSI)** framework introduced in
|
|
247
|
+
[Naidu Dsk, "A Framework for Improving Network Topology Based on Graph
|
|
248
|
+
Theory in Software-Defined Networking", iComp 2025](#) — repurposed here from
|
|
249
|
+
SDN topology optimization to retrieval planning.
|
|
250
|
+
|
|
251
|
+
## License
|
|
252
|
+
|
|
253
|
+
MIT
|
hubmesh-0.1.0/README.md
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# hubmesh
|
|
2
|
+
|
|
3
|
+
[](https://github.com/DemigodDSK/hubmesh/actions/workflows/test.yml)
|
|
4
|
+
[](https://github.com/DemigodDSK/hubmesh)
|
|
5
|
+
[](https://github.com/DemigodDSK/hubmesh/blob/main/LICENSE)
|
|
6
|
+
[](https://github.com/DemigodDSK/hubmesh/releases)
|
|
7
|
+
|
|
8
|
+
**Centrality-aware GraphRAG retrieval planner. Drop-in layer over any vector DB.**
|
|
9
|
+
|
|
10
|
+
`hubmesh` is a Python library that improves multi-hop RAG quality on top of an existing
|
|
11
|
+
vector database. You don't replace your infrastructure — you add a smart planner between
|
|
12
|
+
your vector DB and your LLM.
|
|
13
|
+
|
|
14
|
+
## What problem this solves
|
|
15
|
+
|
|
16
|
+
Naive vector retrieval ("embed query, get top-k by cosine similarity") fails on multi-hop
|
|
17
|
+
questions like *"Where was the founder of the company that acquired Slack born?"* The
|
|
18
|
+
correct answer requires retrieving entities along a reasoning path, not the single most
|
|
19
|
+
similar item.
|
|
20
|
+
|
|
21
|
+
GraphRAG and HippoRAG showed that running a small Personalized PageRank over a knowledge
|
|
22
|
+
graph at query time can substantially improve multi-hop retrieval. `hubmesh` extends
|
|
23
|
+
that line with two contributions:
|
|
24
|
+
|
|
25
|
+
1. **Multi-component seed selection.** Instead of picking PPR seeds by raw query
|
|
26
|
+
similarity (which picks wrong-community seeds at high feature overlap), seeds are
|
|
27
|
+
chosen by a multi-component score combining query relevance, structural fit, and
|
|
28
|
+
coverage diversity.
|
|
29
|
+
2. **Budget-aware context packing.** Once relevant entities are scored, pack them into
|
|
30
|
+
the LLM's context window with explicit coverage and redundancy control rather than
|
|
31
|
+
just truncating top-k.
|
|
32
|
+
|
|
33
|
+
The multi-component scoring pattern is adapted from the NNSI framework
|
|
34
|
+
([Naidu Dsk, iComp 2025](https://example.invalid)) for SDN topology
|
|
35
|
+
optimization, repurposed here for retrieval planning.
|
|
36
|
+
|
|
37
|
+
## Quickstart
|
|
38
|
+
|
|
39
|
+
### In-memory (testing, small corpora)
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from hubmesh import Planner
|
|
43
|
+
from hubmesh.adapters import InMemoryStore
|
|
44
|
+
|
|
45
|
+
embed = ... # callable: text -> np.ndarray
|
|
46
|
+
docs = [...] # list of Document or strings or dicts
|
|
47
|
+
|
|
48
|
+
store = InMemoryStore.from_documents(docs, embed=embed)
|
|
49
|
+
planner = Planner(store=store, embed=embed)
|
|
50
|
+
result = planner.retrieve(query="...", top_k=10, budget_tokens=4000)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Qdrant adapter (production)
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from hubmesh import Planner
|
|
57
|
+
from hubmesh.adapters import QdrantStore
|
|
58
|
+
|
|
59
|
+
store = QdrantStore.from_documents(docs) # in-memory
|
|
60
|
+
store = QdrantStore.from_documents(docs, path="./qdrant_data") # on-disk
|
|
61
|
+
store = QdrantStore.from_documents(docs, url="http://localhost:6333") # remote
|
|
62
|
+
|
|
63
|
+
planner = Planner(store=store, embed=embed)
|
|
64
|
+
result = planner.retrieve(query="...", top_k=10)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### Chroma adapter
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from hubmesh.adapters import ChromaStore
|
|
71
|
+
|
|
72
|
+
store = ChromaStore.from_documents(docs) # ephemeral
|
|
73
|
+
store = ChromaStore.from_documents(docs, persist_directory="./chroma_data")
|
|
74
|
+
store = ChromaStore.from_documents(docs, host="localhost", port=8000)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Multi-hop / KG mode
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from hubmesh.kg import build_entity_kg
|
|
81
|
+
import spacy
|
|
82
|
+
|
|
83
|
+
nlp = spacy.load("en_core_web_sm")
|
|
84
|
+
kg = build_entity_kg(docs, nlp=nlp)
|
|
85
|
+
|
|
86
|
+
planner = Planner(store=store, kg=kg, nlp=nlp)
|
|
87
|
+
result = planner.retrieve(query="Where was the founder of the company that bought Slack born?",
|
|
88
|
+
top_k=10, budget_tokens=4000)
|
|
89
|
+
|
|
90
|
+
# RetrievalResult includes reasoning paths showing why each doc was returned
|
|
91
|
+
for path in result.reasoning:
|
|
92
|
+
print(f" score={path.score:.3f} {' → '.join(path.node_ids)}")
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### LLM-extracted KG (richer than spaCy)
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from hubmesh.kg_llm import build_entity_kg_llm
|
|
99
|
+
|
|
100
|
+
def llm(prompt): # provider-agnostic — bring your own
|
|
101
|
+
return your_llm_call(prompt)
|
|
102
|
+
|
|
103
|
+
kg = build_entity_kg_llm(docs, llm=llm, cache_path="kg_cache.json")
|
|
104
|
+
planner = Planner(store=store, kg=kg)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Better entity linking
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from hubmesh.kg import build_entity_kg
|
|
111
|
+
from hubmesh.entity_linker import EmbeddingLinker, make_st_embedder
|
|
112
|
+
|
|
113
|
+
# Cluster surface variations: "United States" / "U.S." / "USA" → one entity
|
|
114
|
+
linker = EmbeddingLinker(embed=make_st_embedder(), threshold=0.82)
|
|
115
|
+
kg = build_entity_kg(docs, linker=linker)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Chunking long documents
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from hubmesh import chunk_by_sentences, chunk_documents
|
|
122
|
+
|
|
123
|
+
chunks = chunk_documents(
|
|
124
|
+
[{"id": "doc1", "text": long_text}, ...],
|
|
125
|
+
strategy="sentences", target_tokens=200,
|
|
126
|
+
)
|
|
127
|
+
# Then embed chunks and index normally
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Installation
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
pip install hubmesh # core
|
|
134
|
+
pip install "hubmesh[qdrant]" # Qdrant adapter
|
|
135
|
+
pip install "hubmesh[chroma]" # Chroma adapter
|
|
136
|
+
pip install "hubmesh[kg]" # entity-linked KG (spaCy)
|
|
137
|
+
pip install "hubmesh[linker]" # embedding-based entity linker
|
|
138
|
+
pip install "hubmesh[all]" # everything
|
|
139
|
+
python -m spacy download en_core_web_sm # required for KG mode
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Design
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
query → first-pass ANN → induced subgraph → multi-component scoring
|
|
146
|
+
↓ ↓
|
|
147
|
+
community anchoring → Personalized PageRank
|
|
148
|
+
↓ ↓
|
|
149
|
+
└─────→ ranking → budget-aware packing → context
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Each layer is independently testable and replaceable. Adapters wrap your existing vector
|
|
153
|
+
DB so you don't have to migrate.
|
|
154
|
+
|
|
155
|
+
## Benchmarks
|
|
156
|
+
|
|
157
|
+
**Headline:** on multi-hop QA, hubmesh's KG mode beats both naive cosine
|
|
158
|
+
retrieval and a HippoRAG-style PPR-only ablation that uses the same KG.
|
|
159
|
+
The win grows with hop count — exactly the regime where graph-structural
|
|
160
|
+
retrieval should help most.
|
|
161
|
+
|
|
162
|
+
| Benchmark | Setting | recall@10 vs naive |
|
|
163
|
+
|---|---|---:|
|
|
164
|
+
| HotpotQA dev, N=500 | KG mode | **+3.7 pts** |
|
|
165
|
+
| MuSiQue dev, N=300, 2-hop | KG mode | **+1.7 pts** |
|
|
166
|
+
| MuSiQue dev, N=300, 3-hop | KG mode | **+1.9 pts** |
|
|
167
|
+
| MuSiQue dev, N=300, 4-hop | KG mode | **+2.8 pts** |
|
|
168
|
+
|
|
169
|
+
vs PPR-only ablation on the same KG: **+29.1 pts** on HotpotQA — the
|
|
170
|
+
multi-component scoring is doing the work, not just "having a graph."
|
|
171
|
+
|
|
172
|
+
Latency: **~22 ms** mean / 26 ms p95 per query on a 7K-node KG (after PPR
|
|
173
|
+
matrix caching).
|
|
174
|
+
|
|
175
|
+
See [BENCHMARKS.md](BENCHMARKS.md) for the full methodology, ablations,
|
|
176
|
+
per-hop breakdown, and notes on what this proves and doesn't.
|
|
177
|
+
|
|
178
|
+
Reproduce:
|
|
179
|
+
```bash
|
|
180
|
+
python benchmarks/run_hotpotqa.py --n 500 --kg
|
|
181
|
+
python benchmarks/run_musique.py --n 300 --kg
|
|
182
|
+
python benchmarks/profile_query.py # latency profile
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Status
|
|
186
|
+
|
|
187
|
+
Pre-alpha (v0.1.0). Core algorithms implemented and validated; adapters for
|
|
188
|
+
in-memory, Qdrant, and Chroma; entity-linked KG with both spaCy NER and
|
|
189
|
+
LLM-based extraction; document chunking; reasoning-path explanation;
|
|
190
|
+
PPR-cache latency optimisation. Pinecone / pgvector / Weaviate adapters
|
|
191
|
+
and additional multi-hop benchmarks are tracked as
|
|
192
|
+
[good first issues](https://github.com/DemigodDSK/hubmesh/issues).
|
|
193
|
+
|
|
194
|
+
## Acknowledgements
|
|
195
|
+
|
|
196
|
+
The multi-component scoring pattern is adapted from the **Network Node Significance
|
|
197
|
+
Index (NNSI)** framework introduced in
|
|
198
|
+
[Naidu Dsk, "A Framework for Improving Network Topology Based on Graph
|
|
199
|
+
Theory in Software-Defined Networking", iComp 2025](#) — repurposed here from
|
|
200
|
+
SDN topology optimization to retrieval planning.
|
|
201
|
+
|
|
202
|
+
## License
|
|
203
|
+
|
|
204
|
+
MIT
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "hubmesh"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Centrality-aware GraphRAG retrieval planner — drop-in layer over any vector DB"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Datta Sai Krishna Naidu", email = "dattasai293@gmail.com" }]
|
|
13
|
+
keywords = ["rag", "graphrag", "vector-search", "personalized-pagerank",
|
|
14
|
+
"context-selection", "multi-hop-retrieval"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
dependencies = [
|
|
25
|
+
"numpy>=1.26",
|
|
26
|
+
"networkx>=3.2",
|
|
27
|
+
"scipy>=1.11",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
dev = ["pytest>=8.0", "pytest-cov>=4.1", "ruff>=0.4",
|
|
32
|
+
"qdrant-client>=1.10", "chromadb>=0.5"]
|
|
33
|
+
benchmarks = ["datasets>=2.18", "tqdm>=4.66",
|
|
34
|
+
"sentence-transformers>=3.0", "spacy>=3.7"]
|
|
35
|
+
qdrant = ["qdrant-client>=1.10"]
|
|
36
|
+
chroma = ["chromadb>=0.5"]
|
|
37
|
+
kg = ["spacy>=3.7"]
|
|
38
|
+
linker = ["sentence-transformers>=3.0"]
|
|
39
|
+
all = ["qdrant-client>=1.10", "chromadb>=0.5",
|
|
40
|
+
"spacy>=3.7", "sentence-transformers>=3.0",
|
|
41
|
+
"datasets>=2.18", "tqdm>=4.66"]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/dattasaikrishnanaidu/hubmesh"
|
|
45
|
+
Issues = "https://github.com/dattasaikrishnanaidu/hubmesh/issues"
|
|
46
|
+
|
|
47
|
+
[tool.setuptools.packages.find]
|
|
48
|
+
where = ["src"]
|
|
49
|
+
include = ["hubmesh*"]
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
line-length = 100
|
|
53
|
+
target-version = "py310"
|
hubmesh-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""hubmesh — centrality-aware GraphRAG retrieval planner.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
Planner — main entry point
|
|
5
|
+
Document — input record (id, text, vector, metadata)
|
|
6
|
+
RetrievalResult — what Planner.retrieve returns
|
|
7
|
+
ReasoningPath — multi-hop trace returned alongside results
|
|
8
|
+
chunk_by_sentences,
|
|
9
|
+
chunk_by_chars,
|
|
10
|
+
chunk_documents — helpers for splitting long source documents
|
|
11
|
+
"""
|
|
12
|
+
from .types import Document, RetrievalResult, ReasoningPath, ScoredDocument
|
|
13
|
+
from .planner import Planner, PlannerConfig
|
|
14
|
+
from .chunking import chunk_by_sentences, chunk_by_chars, chunk_documents
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Document", "RetrievalResult", "ReasoningPath", "ScoredDocument",
|
|
18
|
+
"Planner", "PlannerConfig",
|
|
19
|
+
"chunk_by_sentences", "chunk_by_chars", "chunk_documents",
|
|
20
|
+
]
|
|
21
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Vector store adapters."""
|
|
2
|
+
from .base import VectorStore
|
|
3
|
+
from .inmemory import InMemoryStore
|
|
4
|
+
|
|
5
|
+
__all__ = ["VectorStore", "InMemoryStore"]
|
|
6
|
+
|
|
7
|
+
# Qdrant is an optional dependency — only export if installed.
|
|
8
|
+
try:
|
|
9
|
+
from .qdrant import QdrantStore # noqa: F401
|
|
10
|
+
__all__.append("QdrantStore")
|
|
11
|
+
except ImportError:
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
# Chroma is also optional.
|
|
15
|
+
try:
|
|
16
|
+
from .chroma import ChromaStore # noqa: F401
|
|
17
|
+
__all__.append("ChromaStore")
|
|
18
|
+
except ImportError:
|
|
19
|
+
pass
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Vector store protocol — implement this to plug a new backend in."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from typing import Protocol, runtime_checkable
|
|
4
|
+
import numpy as np
|
|
5
|
+
from ..types import Document
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@runtime_checkable
|
|
9
|
+
class VectorStore(Protocol):
|
|
10
|
+
"""Minimal contract any backend (Pinecone, Qdrant, pgvector, ...) must satisfy.
|
|
11
|
+
|
|
12
|
+
A store owns:
|
|
13
|
+
• document storage — id → Document
|
|
14
|
+
• a vector index — fast top-k by cosine similarity
|
|
15
|
+
• optional pre-computed kNN graph (cheap to derive if absent)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def search(self, query_vec: np.ndarray, top_k: int) -> list[tuple[str, float]]:
|
|
19
|
+
"""Return [(doc_id, similarity)] of the top_k nearest documents."""
|
|
20
|
+
|
|
21
|
+
def get(self, doc_id: str) -> Document:
|
|
22
|
+
"""Fetch a Document by id."""
|
|
23
|
+
|
|
24
|
+
def get_many(self, doc_ids: list[str]) -> list[Document]:
|
|
25
|
+
"""Bulk fetch."""
|
|
26
|
+
|
|
27
|
+
def neighbors(self, doc_id: str, k: int) -> list[str]:
|
|
28
|
+
"""Return up-to-k neighbor doc_ids in the proximity graph.
|
|
29
|
+
Adapters that don't maintain a pre-built kNN graph should derive it
|
|
30
|
+
from the vector index on demand."""
|
|
31
|
+
|
|
32
|
+
def all_ids(self) -> list[str]:
|
|
33
|
+
"""Return every doc_id (used for full-graph operations during prototype;
|
|
34
|
+
avoid calling on giant indices)."""
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def dim(self) -> int:
|
|
38
|
+
"""Embedding dimension."""
|