dotvectordb 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dotvectordb-0.1.0/MANIFEST.in +8 -0
- dotvectordb-0.1.0/PKG-INFO +251 -0
- dotvectordb-0.1.0/README.MD +215 -0
- dotvectordb-0.1.0/dotvectordb/ConversationMemoryTree.py +331 -0
- dotvectordb-0.1.0/dotvectordb/DatabaseViewer.py +66 -0
- dotvectordb-0.1.0/dotvectordb/DotVDB.py +1318 -0
- dotvectordb-0.1.0/dotvectordb/GraphRetriever.py +713 -0
- dotvectordb-0.1.0/dotvectordb/NaiveVDBConstructor.py +48 -0
- dotvectordb-0.1.0/dotvectordb/StructuredGraphConstructor.py +1221 -0
- dotvectordb-0.1.0/dotvectordb/UnstructuredGraphConstructor.py +801 -0
- dotvectordb-0.1.0/dotvectordb/__init__.py +3 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/Makefile +10 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/build.py +28 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/headers/sqlite3.h +14474 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/headers/sqlite3ext.h +745 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c +13170 -0
- dotvectordb-0.1.0/dotvectordb/compiled_vec1/original_files/manifest.uuid +1 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/ConversationMemoryPrompts.py +35 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/chatbot_prompts.py +61 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/combined_ER_prompt.py +129 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/entity_prompts.py +303 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/global_desc_prompt.py +73 -0
- dotvectordb-0.1.0/dotvectordb/construction_prompts/relationship_prompts.py +293 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/BitMapRetrieval.py +1135 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/RoaringBitMapConstruction.py +856 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/RoaringBitMapRetriever.py +384 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/_bitmap_bfs.cpp +55947 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/_bitmap_bfs.pyx +2349 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/edit_oldBitmapConstruction.py +2111 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/newer_BitmapConstruction.py +2064 -0
- dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/oldBitmapConstruction.py +1897 -0
- dotvectordb-0.1.0/dotvectordb/graph/Combined_ER_indexer.py +364 -0
- dotvectordb-0.1.0/dotvectordb/graph/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/graph/entity_indexer.py +1256 -0
- dotvectordb-0.1.0/dotvectordb/graph/generate_global_desc.py +243 -0
- dotvectordb-0.1.0/dotvectordb/graph/indexer_helpers.py +63 -0
- dotvectordb-0.1.0/dotvectordb/graph/relationship_indexer.py +1370 -0
- dotvectordb-0.1.0/dotvectordb/interfaces/ChunkingFunc.py +53 -0
- dotvectordb-0.1.0/dotvectordb/interfaces/EmbeddingModel.py +45 -0
- dotvectordb-0.1.0/dotvectordb/interfaces/HashingFunc.py +42 -0
- dotvectordb-0.1.0/dotvectordb/interfaces/LanguageModel.py +42 -0
- dotvectordb-0.1.0/dotvectordb/interfaces/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/plugins/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/plugins/chunk_functions/FixedSizeChunker.py +42 -0
- dotvectordb-0.1.0/dotvectordb/plugins/chunk_functions/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/plugins/embedding_providers/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/plugins/embedding_providers/openai_embeddings.py +43 -0
- dotvectordb-0.1.0/dotvectordb/plugins/embedding_providers/openrouter_embeddings.py +52 -0
- dotvectordb-0.1.0/dotvectordb/plugins/hash_functions/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/plugins/hash_functions/hashfuncs.py +15 -0
- dotvectordb-0.1.0/dotvectordb/plugins/llm_providers/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/plugins/llm_providers/openai_llms.py +84 -0
- dotvectordb-0.1.0/dotvectordb/plugins/llm_providers/openrouter_llms.py +98 -0
- dotvectordb-0.1.0/dotvectordb/plugins/loaders/TextConverter.py +144 -0
- dotvectordb-0.1.0/dotvectordb/plugins/loaders/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/schema/__init__.py +0 -0
- dotvectordb-0.1.0/dotvectordb/schema/chunk.sql +242 -0
- dotvectordb-0.1.0/dotvectordb/schema/conversation.sql +171 -0
- dotvectordb-0.1.0/dotvectordb/schema/entity.sql +416 -0
- dotvectordb-0.1.0/dotvectordb/schema/failure_log.sql +15 -0
- dotvectordb-0.1.0/dotvectordb/schema/query_embeddings.sql +17 -0
- dotvectordb-0.1.0/dotvectordb/schema/relationship.sql +292 -0
- dotvectordb-0.1.0/dotvectordb/schema/settings.sql +88 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/__init__.py +3 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/chat.html +84 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/home.html +66 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/index.html +143 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/server.py +530 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/university-of-michigan-vector-logo-white.svg +74 -0
- dotvectordb-0.1.0/dotvectordb/sql_browser/university-of-michigan-vector-logo.svg +73 -0
- dotvectordb-0.1.0/dotvectordb/web_interface/__init__.py +4 -0
- dotvectordb-0.1.0/dotvectordb/web_interface/index.html +151 -0
- dotvectordb-0.1.0/dotvectordb/web_interface/server.py +448 -0
- dotvectordb-0.1.0/dotvectordb.egg-info/PKG-INFO +251 -0
- dotvectordb-0.1.0/dotvectordb.egg-info/SOURCES.txt +93 -0
- dotvectordb-0.1.0/dotvectordb.egg-info/dependency_links.txt +1 -0
- dotvectordb-0.1.0/dotvectordb.egg-info/requires.txt +17 -0
- dotvectordb-0.1.0/dotvectordb.egg-info/top_level.txt +1 -0
- dotvectordb-0.1.0/pyproject.toml +8 -0
- dotvectordb-0.1.0/setup.cfg +4 -0
- dotvectordb-0.1.0/setup.py +107 -0
- dotvectordb-0.1.0/tests/test_chunk_batch.py +83 -0
- dotvectordb-0.1.0/tests/test_chunk_delete_cascade.py +232 -0
- dotvectordb-0.1.0/tests/test_entity_custom_prompt.py +51 -0
- dotvectordb-0.1.0/tests/test_lmdb_storage.py +86 -0
- dotvectordb-0.1.0/tests/test_relationship_custom_prompt.py +67 -0
- dotvectordb-0.1.0/tests/test_relationship_supporting_lines.py +190 -0
- dotvectordb-0.1.0/tests/test_roaring_later_hops.py +548 -0
- dotvectordb-0.1.0/tests/test_roaring_retrieval.py +260 -0
- dotvectordb-0.1.0/tests/test_skip_embeddings.py +87 -0
- dotvectordb-0.1.0/tests/test_unstructured_roaring_phases.py +201 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
include README.MD
|
|
2
|
+
include dotvectordb/compiled_vec1/Makefile
|
|
3
|
+
include dotvectordb/compiled_vec1/build.py
|
|
4
|
+
include dotvectordb/compiled_vec1/headers/*.h
|
|
5
|
+
include dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c
|
|
6
|
+
include dotvectordb/compiled_vec1/original_files/manifest.uuid
|
|
7
|
+
include dotvectordb/data_structures/bitmap/_bitmap_bfs.pyx
|
|
8
|
+
global-exclude *.so
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dotvectordb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: DotVectorDB is a simple, transparent vector and graph database that makes building GraphRAG pipelines easier without hiding the underlying process.
|
|
5
|
+
Author: Larnell Moore
|
|
6
|
+
Author-email: larnell@umich.edu
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
9
|
+
Requires-Python: >=3.9
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: torch==2.10.0
|
|
12
|
+
Requires-Dist: aiosqlite==0.22.1
|
|
13
|
+
Requires-Dist: sentence-transformers==5.5.1
|
|
14
|
+
Requires-Dist: python-dotenv==1.2.2
|
|
15
|
+
Requires-Dist: openai==2.44.0
|
|
16
|
+
Requires-Dist: google-genai==2.12.0
|
|
17
|
+
Requires-Dist: aiohttp==3.13.3
|
|
18
|
+
Requires-Dist: tiktoken==0.13.0
|
|
19
|
+
Requires-Dist: tqdm<5.0,>=4.66
|
|
20
|
+
Requires-Dist: igraph==1.0.0
|
|
21
|
+
Requires-Dist: pyroaring<2,>=1.0
|
|
22
|
+
Requires-Dist: lmdb<4,>=3.0
|
|
23
|
+
Requires-Dist: pdfplumber==0.11.6
|
|
24
|
+
Requires-Dist: python-docx==1.2.0
|
|
25
|
+
Requires-Dist: python-pptx==1.0.2
|
|
26
|
+
Requires-Dist: openpyxl==3.1.5
|
|
27
|
+
Requires-Dist: pydantic<3.0,>=2.0
|
|
28
|
+
Dynamic: author
|
|
29
|
+
Dynamic: author-email
|
|
30
|
+
Dynamic: classifier
|
|
31
|
+
Dynamic: description
|
|
32
|
+
Dynamic: description-content-type
|
|
33
|
+
Dynamic: requires-dist
|
|
34
|
+
Dynamic: requires-python
|
|
35
|
+
Dynamic: summary
|
|
36
|
+
|
|
37
|
+
# DotVectorDB
|
|
38
|
+
|
|
39
|
+
**A SQLite-based vector and graph database for building GraphRAG pipelines.**
|
|
40
|
+
|
|
41
|
+
DotVectorDB turns documents into a graph of entities and relationships, keeps relationships linked to their supporting text, and provides vector, BM25, and hybrid retrieval. Compressed Roaring bitmap indexes narrow searches to an anchor's neighborhood and entity types before ranking the candidates.
|
|
42
|
+
|
|
43
|
+
The database, graph construction, and retrieval components are separate Python classes. You can use the search primitives directly or build a retrieval algorithm on top of them.
|
|
44
|
+
|
|
45
|
+
## What it does
|
|
46
|
+
|
|
47
|
+
- **Document ingestion:** extract text from TXT, PDF, DOCX, Excel, and PowerPoint files, then chunk it with a configurable chunker.
|
|
48
|
+
- **Graph construction:** extract entities and relationships, assign entity types, and embed graph content. Extraction prompts and examples can be customized.
|
|
49
|
+
- **Evidence tracking:** relationships retain supporting line references for each source chunk, so retrieved facts can be traced back to stored text.
|
|
50
|
+
- **Search:** vector search through SQLite Vec1, BM25 through FTS5, and hybrid search using reciprocal rank fusion.
|
|
51
|
+
- **Constrained retrieval:** retrieve entity or relationship IDs by anchor, hop, and entity type, then rank only those IDs.
|
|
52
|
+
- **Incremental construction:** track new and updated graph records, extend bitmap indexes, and resume completed insertion and bitmap batches.
|
|
53
|
+
- **Reusable embeddings:** supply an existing embedding directly to vector or hybrid search, or retrieve stored embeddings as NumPy arrays.
|
|
54
|
+
|
|
55
|
+
SQLite stores documents, graph records, metadata, and embeddings. Roaring bitmap indexes are stored separately in LMDB.
|
|
56
|
+
|
|
57
|
+
## Install from source
|
|
58
|
+
|
|
59
|
+
The native build currently targets **Linux**. Use a C/C++ compiler with OpenMP support and `make`. Python 3.10 and 3.11 have been used during development.
|
|
60
|
+
|
|
61
|
+
On Debian or Ubuntu, install the build tools:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
sudo apt install build-essential
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Then install the library:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
git clone https://github.com/LuffyAI/DotVectorDB.git
|
|
71
|
+
cd DotVectorDB
|
|
72
|
+
python -m pip install .
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The package build compiles Vec1 from the bundled source. LMDB and PyRoaring are installed as Python dependencies. See [BUILD_LOCAL.md](BUILD_LOCAL.md) for local wheel-build commands.
|
|
76
|
+
|
|
77
|
+
## Build a graph
|
|
78
|
+
|
|
79
|
+
The default providers use OpenAI. Create a `.env` file with your credentials and model settings, and load it **before importing the library**:
|
|
80
|
+
|
|
81
|
+
```dotenv
|
|
82
|
+
OPENAI_API_KEY=your-api-key
|
|
83
|
+
OPENAI_EMBEDDING_MODEL=text-embedding-3-large
|
|
84
|
+
OPENAI_LLM_MODEL=gpt-4o-mini
|
|
85
|
+
TOKENIZER=gpt-4o
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
import asyncio
|
|
90
|
+
from dotenv import load_dotenv
|
|
91
|
+
|
|
92
|
+
load_dotenv()
|
|
93
|
+
|
|
94
|
+
from dotvectordb.UnstructuredGraphConstructor import UnstructuredGraphConstructor
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
async def main():
|
|
98
|
+
async with UnstructuredGraphConstructor(
|
|
99
|
+
path="./article_graph",
|
|
100
|
+
tokenizer_name="gpt-4o",
|
|
101
|
+
) as graph:
|
|
102
|
+
await graph.construct_GraphRAG(
|
|
103
|
+
input_document_list=["./article.txt"],
|
|
104
|
+
global_desc=False,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
asyncio.run(main())
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
This ingests and embeds chunks, builds the entity and relationship graph, creates reachability indexes, and builds graph BM25 indexes. `global_desc=False` skips LLM-generated global descriptions. Graph construction still calls the configured language and embedding models.
|
|
112
|
+
|
|
113
|
+
The path above creates a database directory containing `article_graph.db` and its graph index artifacts. Reuse the same path when opening a retriever.
|
|
114
|
+
|
|
115
|
+
## Search the graph
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
from dotvectordb.GraphRetriever import GraphRetriever
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
async def search():
|
|
122
|
+
async with GraphRetriever(
|
|
123
|
+
path="./article_graph",
|
|
124
|
+
tokenizer_name="gpt-4o",
|
|
125
|
+
bitmap_mode="exact_hop",
|
|
126
|
+
) as retriever:
|
|
127
|
+
results, query_tokens = await retriever.hybrid_vector_BM25_search(
|
|
128
|
+
query="What makes aviation difficult to decarbonize?",
|
|
129
|
+
table="relationship",
|
|
130
|
+
top_k=5,
|
|
131
|
+
)
|
|
132
|
+
return results
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Search tables are `chunk`, `entity`, and `relationship`. The return value is `(results, query_tokens)`.
|
|
136
|
+
|
|
137
|
+
To reuse embeddings rather than generate a new query embedding:
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
embeddings = await retriever.batch_get_embeddings(
|
|
141
|
+
ids={entity_id},
|
|
142
|
+
table="entity",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
results, query_tokens = await retriever.vector_search(
|
|
146
|
+
query=None,
|
|
147
|
+
table="entity",
|
|
148
|
+
embedding=embeddings[entity_id],
|
|
149
|
+
top_k=5,
|
|
150
|
+
)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Supplied embeddings must use the same model and dimensions as the target index. Hybrid search also accepts `embedding=...`; its BM25 component still requires query text.
|
|
154
|
+
|
|
155
|
+
## Search a neighborhood
|
|
156
|
+
|
|
157
|
+
Precomputed exact-hop buckets provide direct lookups for the indexed hop and entity type. They return database IDs that can be passed to `search_id_batch`:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
relationship_ids = await retriever.bitmap_retriever.constrained_relationship_search(
|
|
161
|
+
anchor_entity_id=anchor_id,
|
|
162
|
+
exact_hop=2,
|
|
163
|
+
entity_types=["person"],
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
results, query_tokens = await retriever.search_id_batch(
|
|
167
|
+
query="Who worked on these projects?",
|
|
168
|
+
table="relationship",
|
|
169
|
+
rowids=relationship_ids,
|
|
170
|
+
top_k=10,
|
|
171
|
+
search_type="hybrid",
|
|
172
|
+
)
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Entities can belong to multiple types. Type filters match any requested type by default; `match_all_types=True` requires all requested types.
|
|
176
|
+
|
|
177
|
+
Precomputed hops do not expand the graph during lookup. Requests beyond the precomputed range use one-hop traversal for the remaining distance. Choose `bitmap_mode="one_hop"` to use traversal directly, or `load_all_into_memory=True` to cache the bitmap stores in memory. A memory-loaded bitmap retriever can pick up index changes with `await retriever.bitmap_retriever.refresh()`.
|
|
178
|
+
|
|
179
|
+
## Document-only retrieval
|
|
180
|
+
|
|
181
|
+
Use `NaiveVDBConstructor` when you want document retrieval without entity or relationship extraction:
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
from dotvectordb.NaiveVDBConstructor import NaiveVDBConstructor
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
async def build_document_index():
|
|
188
|
+
async with NaiveVDBConstructor(
|
|
189
|
+
path="./document_index",
|
|
190
|
+
tokenizer_name="gpt-4o",
|
|
191
|
+
) as database:
|
|
192
|
+
await database.construct_document_bm25_index(
|
|
193
|
+
input_document_list=["./article.txt"],
|
|
194
|
+
)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
`construct_document_bm25_index` skips chunk embeddings. Database initialization currently still configures the default models and may generate an embedding to establish dimensions for a fresh database.
|
|
198
|
+
|
|
199
|
+
The same constructor also provides `construct_document_vectordatabase` and `construct_bm25_and_vdb`.
|
|
200
|
+
|
|
201
|
+
## Code layout
|
|
202
|
+
|
|
203
|
+
| Component | Responsibility |
|
|
204
|
+
|---|---|
|
|
205
|
+
| `DotVectorDB` | SQLite connection, schema, metadata, and embedding storage |
|
|
206
|
+
| `UnstructuredGraphConstructor` | Coordinate document ingestion and graph construction |
|
|
207
|
+
| `NaiveVDBConstructor` | Document-only BM25 and vector index construction |
|
|
208
|
+
| `GraphRetriever` | BM25, vector, hybrid, and ID-constrained search |
|
|
209
|
+
| `RetrievalAlgos/` | Retrieval algorithms built on the search primitives; several are still in development |
|
|
210
|
+
| `plugins/` | Document extraction, chunking, hashing, and model providers |
|
|
211
|
+
| `data_structures/bitmap/` | Roaring bitmap construction and retrieval backed by LMDB |
|
|
212
|
+
| `schema/` | SQL tables, indexes, and triggers |
|
|
213
|
+
|
|
214
|
+
Custom chunkers, hashing functions, embedding models, and language models can be supplied through the corresponding constructor arguments.
|
|
215
|
+
|
|
216
|
+
## Native extension details
|
|
217
|
+
|
|
218
|
+
The bundled source is pinned to **Vec1 0.8**, independently of the SQLite engine version:
|
|
219
|
+
|
|
220
|
+
- Source: `dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c`
|
|
221
|
+
- SQLite source revision: `213d76c009bcd860b5a624be6ffe3b89aab063bc6f2c30bdb40384c5d7f8b35f`
|
|
222
|
+
- `vec1.c` SHA-256: `b341c15fe10e41a9b96981ff3a9c217ea0774b4e04932b5acf3dac1320aeceb1`
|
|
223
|
+
|
|
224
|
+
Matching SQLite headers are bundled. Compilation requires no Vec1 download. After loading the extension, `SELECT vec1_info();` reports its version and enabled features.
|
|
225
|
+
|
|
226
|
+
The default Vec1 compiler flags include `-march=native`: build on the target machine rather than distributing that wheel across different CPUs. Compiler flags can be overridden through `CC`, `CPPFLAGS`, `CFLAGS`, and `LDFLAGS`.
|
|
227
|
+
|
|
228
|
+
## Status
|
|
229
|
+
|
|
230
|
+
DotVectorDB is a research project under active development. APIs and schemas can change between revisions. Performance depends on graph structure, storage, models, and the index configuration; this README does not claim a universal benchmark result.
|
|
231
|
+
|
|
232
|
+
A PyPI release for installation with `pip install dotvectordb` is planned once development is complete.
|
|
233
|
+
|
|
234
|
+
## Citation
|
|
235
|
+
|
|
236
|
+
If you use the DotRAG retrieval algorithm or build on DotVectorDB in your research, please cite our paper:
|
|
237
|
+
|
|
238
|
+
**[DotRAG: Retrieval-Time Reasoning Along Relational Paths](https://arxiv.org/abs/2605.18760)**
|
|
239
|
+
Larnell Moore, Naihao Deng, Rada Mihalcea, and Farnaz Jahanbakhsh. arXiv, 2026.
|
|
240
|
+
|
|
241
|
+
```bibtex
|
|
242
|
+
@misc{moore2026dotrag,
|
|
243
|
+
title = {{DotRAG}: Retrieval-Time Reasoning Along Relational Paths},
|
|
244
|
+
author = {Moore, Larnell and Deng, Naihao and Mihalcea, Rada and Jahanbakhsh, Farnaz},
|
|
245
|
+
year = {2026},
|
|
246
|
+
eprint = {2605.18760},
|
|
247
|
+
archivePrefix = {arXiv},
|
|
248
|
+
primaryClass = {cs.IR},
|
|
249
|
+
url = {https://arxiv.org/abs/2605.18760}
|
|
250
|
+
}
|
|
251
|
+
```
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# DotVectorDB
|
|
2
|
+
|
|
3
|
+
**A SQLite-based vector and graph database for building GraphRAG pipelines.**
|
|
4
|
+
|
|
5
|
+
DotVectorDB turns documents into a graph of entities and relationships, keeps relationships linked to their supporting text, and provides vector, BM25, and hybrid retrieval. Compressed Roaring bitmap indexes narrow searches to an anchor's neighborhood and entity types before ranking the candidates.
|
|
6
|
+
|
|
7
|
+
The database, graph construction, and retrieval components are separate Python classes. You can use the search primitives directly or build a retrieval algorithm on top of them.
|
|
8
|
+
|
|
9
|
+
## What it does
|
|
10
|
+
|
|
11
|
+
- **Document ingestion:** extract text from TXT, PDF, DOCX, Excel, and PowerPoint files, then chunk it with a configurable chunker.
|
|
12
|
+
- **Graph construction:** extract entities and relationships, assign entity types, and embed graph content. Extraction prompts and examples can be customized.
|
|
13
|
+
- **Evidence tracking:** relationships retain supporting line references for each source chunk, so retrieved facts can be traced back to stored text.
|
|
14
|
+
- **Search:** vector search through SQLite Vec1, BM25 through FTS5, and hybrid search using reciprocal rank fusion.
|
|
15
|
+
- **Constrained retrieval:** retrieve entity or relationship IDs by anchor, hop, and entity type, then rank only those IDs.
|
|
16
|
+
- **Incremental construction:** track new and updated graph records, extend bitmap indexes, and resume completed insertion and bitmap batches.
|
|
17
|
+
- **Reusable embeddings:** supply an existing embedding directly to vector or hybrid search, or retrieve stored embeddings as NumPy arrays.
|
|
18
|
+
|
|
19
|
+
SQLite stores documents, graph records, metadata, and embeddings. Roaring bitmap indexes are stored separately in LMDB.
|
|
20
|
+
|
|
21
|
+
## Install from source
|
|
22
|
+
|
|
23
|
+
The native build currently targets **Linux**. Use a C/C++ compiler with OpenMP support and `make`. Python 3.10 and 3.11 have been used during development.
|
|
24
|
+
|
|
25
|
+
On Debian or Ubuntu, install the build tools:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
sudo apt install build-essential
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Then install the library:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
git clone https://github.com/LuffyAI/DotVectorDB.git
|
|
35
|
+
cd DotVectorDB
|
|
36
|
+
python -m pip install .
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The package build compiles Vec1 from the bundled source. LMDB and PyRoaring are installed as Python dependencies. See [BUILD_LOCAL.md](BUILD_LOCAL.md) for local wheel-build commands.
|
|
40
|
+
|
|
41
|
+
## Build a graph
|
|
42
|
+
|
|
43
|
+
The default providers use OpenAI. Create a `.env` file with your credentials and model settings, and load it **before importing the library**:
|
|
44
|
+
|
|
45
|
+
```dotenv
|
|
46
|
+
OPENAI_API_KEY=your-api-key
|
|
47
|
+
OPENAI_EMBEDDING_MODEL=text-embedding-3-large
|
|
48
|
+
OPENAI_LLM_MODEL=gpt-4o-mini
|
|
49
|
+
TOKENIZER=gpt-4o
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import asyncio
|
|
54
|
+
from dotenv import load_dotenv
|
|
55
|
+
|
|
56
|
+
load_dotenv()
|
|
57
|
+
|
|
58
|
+
from dotvectordb.UnstructuredGraphConstructor import UnstructuredGraphConstructor
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def main():
|
|
62
|
+
async with UnstructuredGraphConstructor(
|
|
63
|
+
path="./article_graph",
|
|
64
|
+
tokenizer_name="gpt-4o",
|
|
65
|
+
) as graph:
|
|
66
|
+
await graph.construct_GraphRAG(
|
|
67
|
+
input_document_list=["./article.txt"],
|
|
68
|
+
global_desc=False,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
asyncio.run(main())
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
This ingests and embeds chunks, builds the entity and relationship graph, creates reachability indexes, and builds graph BM25 indexes. `global_desc=False` skips LLM-generated global descriptions. Graph construction still calls the configured language and embedding models.
|
|
76
|
+
|
|
77
|
+
The path above creates a database directory containing `article_graph.db` and its graph index artifacts. Reuse the same path when opening a retriever.
|
|
78
|
+
|
|
79
|
+
## Search the graph
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from dotvectordb.GraphRetriever import GraphRetriever
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
async def search():
|
|
86
|
+
async with GraphRetriever(
|
|
87
|
+
path="./article_graph",
|
|
88
|
+
tokenizer_name="gpt-4o",
|
|
89
|
+
bitmap_mode="exact_hop",
|
|
90
|
+
) as retriever:
|
|
91
|
+
results, query_tokens = await retriever.hybrid_vector_BM25_search(
|
|
92
|
+
query="What makes aviation difficult to decarbonize?",
|
|
93
|
+
table="relationship",
|
|
94
|
+
top_k=5,
|
|
95
|
+
)
|
|
96
|
+
return results
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Search tables are `chunk`, `entity`, and `relationship`. The return value is `(results, query_tokens)`.
|
|
100
|
+
|
|
101
|
+
To reuse embeddings rather than generate a new query embedding:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
embeddings = await retriever.batch_get_embeddings(
|
|
105
|
+
ids={entity_id},
|
|
106
|
+
table="entity",
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
results, query_tokens = await retriever.vector_search(
|
|
110
|
+
query=None,
|
|
111
|
+
table="entity",
|
|
112
|
+
embedding=embeddings[entity_id],
|
|
113
|
+
top_k=5,
|
|
114
|
+
)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Supplied embeddings must use the same model and dimensions as the target index. Hybrid search also accepts `embedding=...`; its BM25 component still requires query text.
|
|
118
|
+
|
|
119
|
+
## Search a neighborhood
|
|
120
|
+
|
|
121
|
+
Precomputed exact-hop buckets provide direct lookups for the indexed hop and entity type. They return database IDs that can be passed to `search_id_batch`:
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
relationship_ids = await retriever.bitmap_retriever.constrained_relationship_search(
|
|
125
|
+
anchor_entity_id=anchor_id,
|
|
126
|
+
exact_hop=2,
|
|
127
|
+
entity_types=["person"],
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
results, query_tokens = await retriever.search_id_batch(
|
|
131
|
+
query="Who worked on these projects?",
|
|
132
|
+
table="relationship",
|
|
133
|
+
rowids=relationship_ids,
|
|
134
|
+
top_k=10,
|
|
135
|
+
search_type="hybrid",
|
|
136
|
+
)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Entities can belong to multiple types. Type filters match any requested type by default; `match_all_types=True` requires all requested types.
|
|
140
|
+
|
|
141
|
+
Precomputed hops do not expand the graph during lookup. Requests beyond the precomputed range use one-hop traversal for the remaining distance. Choose `bitmap_mode="one_hop"` to use traversal directly, or `load_all_into_memory=True` to cache the bitmap stores in memory. A memory-loaded bitmap retriever can pick up index changes with `await retriever.bitmap_retriever.refresh()`.
|
|
142
|
+
|
|
143
|
+
## Document-only retrieval
|
|
144
|
+
|
|
145
|
+
Use `NaiveVDBConstructor` when you want document retrieval without entity or relationship extraction:
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from dotvectordb.NaiveVDBConstructor import NaiveVDBConstructor
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
async def build_document_index():
|
|
152
|
+
async with NaiveVDBConstructor(
|
|
153
|
+
path="./document_index",
|
|
154
|
+
tokenizer_name="gpt-4o",
|
|
155
|
+
) as database:
|
|
156
|
+
await database.construct_document_bm25_index(
|
|
157
|
+
input_document_list=["./article.txt"],
|
|
158
|
+
)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
`construct_document_bm25_index` skips chunk embeddings. Database initialization currently still configures the default models and may generate an embedding to establish dimensions for a fresh database.
|
|
162
|
+
|
|
163
|
+
The same constructor also provides `construct_document_vectordatabase` and `construct_bm25_and_vdb`.
|
|
164
|
+
|
|
165
|
+
## Code layout
|
|
166
|
+
|
|
167
|
+
| Component | Responsibility |
|
|
168
|
+
|---|---|
|
|
169
|
+
| `DotVectorDB` | SQLite connection, schema, metadata, and embedding storage |
|
|
170
|
+
| `UnstructuredGraphConstructor` | Coordinate document ingestion and graph construction |
|
|
171
|
+
| `NaiveVDBConstructor` | Document-only BM25 and vector index construction |
|
|
172
|
+
| `GraphRetriever` | BM25, vector, hybrid, and ID-constrained search |
|
|
173
|
+
| `RetrievalAlgos/` | Retrieval algorithms built on the search primitives; several are still in development |
|
|
174
|
+
| `plugins/` | Document extraction, chunking, hashing, and model providers |
|
|
175
|
+
| `data_structures/bitmap/` | Roaring bitmap construction and retrieval backed by LMDB |
|
|
176
|
+
| `schema/` | SQL tables, indexes, and triggers |
|
|
177
|
+
|
|
178
|
+
Custom chunkers, hashing functions, embedding models, and language models can be supplied through the corresponding constructor arguments.
|
|
179
|
+
|
|
180
|
+
## Native extension details
|
|
181
|
+
|
|
182
|
+
The bundled source is pinned to **Vec1 0.8**, independently of the SQLite engine version:
|
|
183
|
+
|
|
184
|
+
- Source: `dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c`
|
|
185
|
+
- SQLite source revision: `213d76c009bcd860b5a624be6ffe3b89aab063bc6f2c30bdb40384c5d7f8b35f`
|
|
186
|
+
- `vec1.c` SHA-256: `b341c15fe10e41a9b96981ff3a9c217ea0774b4e04932b5acf3dac1320aeceb1`
|
|
187
|
+
|
|
188
|
+
Matching SQLite headers are bundled. Compilation requires no Vec1 download. After loading the extension, `SELECT vec1_info();` reports its version and enabled features.
|
|
189
|
+
|
|
190
|
+
The default Vec1 compiler flags include `-march=native`: build on the target machine rather than distributing that wheel across different CPUs. Compiler flags can be overridden through `CC`, `CPPFLAGS`, `CFLAGS`, and `LDFLAGS`.
|
|
191
|
+
|
|
192
|
+
## Status
|
|
193
|
+
|
|
194
|
+
DotVectorDB is a research project under active development. APIs and schemas can change between revisions. Performance depends on graph structure, storage, models, and the index configuration; this README does not claim a universal benchmark result.
|
|
195
|
+
|
|
196
|
+
A PyPI release for installation with `pip install dotvectordb` is planned once development is complete.
|
|
197
|
+
|
|
198
|
+
## Citation
|
|
199
|
+
|
|
200
|
+
If you use the DotRAG retrieval algorithm or build on DotVectorDB in your research, please cite our paper:
|
|
201
|
+
|
|
202
|
+
**[DotRAG: Retrieval-Time Reasoning Along Relational Paths](https://arxiv.org/abs/2605.18760)**
|
|
203
|
+
Larnell Moore, Naihao Deng, Rada Mihalcea, and Farnaz Jahanbakhsh. arXiv, 2026.
|
|
204
|
+
|
|
205
|
+
```bibtex
|
|
206
|
+
@misc{moore2026dotrag,
|
|
207
|
+
title = {{DotRAG}: Retrieval-Time Reasoning Along Relational Paths},
|
|
208
|
+
author = {Moore, Larnell and Deng, Naihao and Mihalcea, Rada and Jahanbakhsh, Farnaz},
|
|
209
|
+
year = {2026},
|
|
210
|
+
eprint = {2605.18760},
|
|
211
|
+
archivePrefix = {arXiv},
|
|
212
|
+
primaryClass = {cs.IR},
|
|
213
|
+
url = {https://arxiv.org/abs/2605.18760}
|
|
214
|
+
}
|
|
215
|
+
```
|