dotvectordb 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. dotvectordb-0.1.0/MANIFEST.in +8 -0
  2. dotvectordb-0.1.0/PKG-INFO +251 -0
  3. dotvectordb-0.1.0/README.MD +215 -0
  4. dotvectordb-0.1.0/dotvectordb/ConversationMemoryTree.py +331 -0
  5. dotvectordb-0.1.0/dotvectordb/DatabaseViewer.py +66 -0
  6. dotvectordb-0.1.0/dotvectordb/DotVDB.py +1318 -0
  7. dotvectordb-0.1.0/dotvectordb/GraphRetriever.py +713 -0
  8. dotvectordb-0.1.0/dotvectordb/NaiveVDBConstructor.py +48 -0
  9. dotvectordb-0.1.0/dotvectordb/StructuredGraphConstructor.py +1221 -0
  10. dotvectordb-0.1.0/dotvectordb/UnstructuredGraphConstructor.py +801 -0
  11. dotvectordb-0.1.0/dotvectordb/__init__.py +3 -0
  12. dotvectordb-0.1.0/dotvectordb/compiled_vec1/Makefile +10 -0
  13. dotvectordb-0.1.0/dotvectordb/compiled_vec1/__init__.py +0 -0
  14. dotvectordb-0.1.0/dotvectordb/compiled_vec1/build.py +28 -0
  15. dotvectordb-0.1.0/dotvectordb/compiled_vec1/headers/sqlite3.h +14474 -0
  16. dotvectordb-0.1.0/dotvectordb/compiled_vec1/headers/sqlite3ext.h +745 -0
  17. dotvectordb-0.1.0/dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c +13170 -0
  18. dotvectordb-0.1.0/dotvectordb/compiled_vec1/original_files/manifest.uuid +1 -0
  19. dotvectordb-0.1.0/dotvectordb/construction_prompts/ConversationMemoryPrompts.py +35 -0
  20. dotvectordb-0.1.0/dotvectordb/construction_prompts/__init__.py +0 -0
  21. dotvectordb-0.1.0/dotvectordb/construction_prompts/chatbot_prompts.py +61 -0
  22. dotvectordb-0.1.0/dotvectordb/construction_prompts/combined_ER_prompt.py +129 -0
  23. dotvectordb-0.1.0/dotvectordb/construction_prompts/entity_prompts.py +303 -0
  24. dotvectordb-0.1.0/dotvectordb/construction_prompts/global_desc_prompt.py +73 -0
  25. dotvectordb-0.1.0/dotvectordb/construction_prompts/relationship_prompts.py +293 -0
  26. dotvectordb-0.1.0/dotvectordb/data_structures/__init__.py +0 -0
  27. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/BitMapRetrieval.py +1135 -0
  28. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/RoaringBitMapConstruction.py +856 -0
  29. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/RoaringBitMapRetriever.py +384 -0
  30. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/__init__.py +0 -0
  31. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/_bitmap_bfs.cpp +55947 -0
  32. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/_bitmap_bfs.pyx +2349 -0
  33. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/edit_oldBitmapConstruction.py +2111 -0
  34. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/newer_BitmapConstruction.py +2064 -0
  35. dotvectordb-0.1.0/dotvectordb/data_structures/bitmap/oldBitmapConstruction.py +1897 -0
  36. dotvectordb-0.1.0/dotvectordb/graph/Combined_ER_indexer.py +364 -0
  37. dotvectordb-0.1.0/dotvectordb/graph/__init__.py +0 -0
  38. dotvectordb-0.1.0/dotvectordb/graph/entity_indexer.py +1256 -0
  39. dotvectordb-0.1.0/dotvectordb/graph/generate_global_desc.py +243 -0
  40. dotvectordb-0.1.0/dotvectordb/graph/indexer_helpers.py +63 -0
  41. dotvectordb-0.1.0/dotvectordb/graph/relationship_indexer.py +1370 -0
  42. dotvectordb-0.1.0/dotvectordb/interfaces/ChunkingFunc.py +53 -0
  43. dotvectordb-0.1.0/dotvectordb/interfaces/EmbeddingModel.py +45 -0
  44. dotvectordb-0.1.0/dotvectordb/interfaces/HashingFunc.py +42 -0
  45. dotvectordb-0.1.0/dotvectordb/interfaces/LanguageModel.py +42 -0
  46. dotvectordb-0.1.0/dotvectordb/interfaces/__init__.py +0 -0
  47. dotvectordb-0.1.0/dotvectordb/plugins/__init__.py +0 -0
  48. dotvectordb-0.1.0/dotvectordb/plugins/chunk_functions/FixedSizeChunker.py +42 -0
  49. dotvectordb-0.1.0/dotvectordb/plugins/chunk_functions/__init__.py +0 -0
  50. dotvectordb-0.1.0/dotvectordb/plugins/embedding_providers/__init__.py +0 -0
  51. dotvectordb-0.1.0/dotvectordb/plugins/embedding_providers/openai_embeddings.py +43 -0
  52. dotvectordb-0.1.0/dotvectordb/plugins/embedding_providers/openrouter_embeddings.py +52 -0
  53. dotvectordb-0.1.0/dotvectordb/plugins/hash_functions/__init__.py +0 -0
  54. dotvectordb-0.1.0/dotvectordb/plugins/hash_functions/hashfuncs.py +15 -0
  55. dotvectordb-0.1.0/dotvectordb/plugins/llm_providers/__init__.py +0 -0
  56. dotvectordb-0.1.0/dotvectordb/plugins/llm_providers/openai_llms.py +84 -0
  57. dotvectordb-0.1.0/dotvectordb/plugins/llm_providers/openrouter_llms.py +98 -0
  58. dotvectordb-0.1.0/dotvectordb/plugins/loaders/TextConverter.py +144 -0
  59. dotvectordb-0.1.0/dotvectordb/plugins/loaders/__init__.py +0 -0
  60. dotvectordb-0.1.0/dotvectordb/schema/__init__.py +0 -0
  61. dotvectordb-0.1.0/dotvectordb/schema/chunk.sql +242 -0
  62. dotvectordb-0.1.0/dotvectordb/schema/conversation.sql +171 -0
  63. dotvectordb-0.1.0/dotvectordb/schema/entity.sql +416 -0
  64. dotvectordb-0.1.0/dotvectordb/schema/failure_log.sql +15 -0
  65. dotvectordb-0.1.0/dotvectordb/schema/query_embeddings.sql +17 -0
  66. dotvectordb-0.1.0/dotvectordb/schema/relationship.sql +292 -0
  67. dotvectordb-0.1.0/dotvectordb/schema/settings.sql +88 -0
  68. dotvectordb-0.1.0/dotvectordb/sql_browser/__init__.py +3 -0
  69. dotvectordb-0.1.0/dotvectordb/sql_browser/chat.html +84 -0
  70. dotvectordb-0.1.0/dotvectordb/sql_browser/home.html +66 -0
  71. dotvectordb-0.1.0/dotvectordb/sql_browser/index.html +143 -0
  72. dotvectordb-0.1.0/dotvectordb/sql_browser/server.py +530 -0
  73. dotvectordb-0.1.0/dotvectordb/sql_browser/university-of-michigan-vector-logo-white.svg +74 -0
  74. dotvectordb-0.1.0/dotvectordb/sql_browser/university-of-michigan-vector-logo.svg +73 -0
  75. dotvectordb-0.1.0/dotvectordb/web_interface/__init__.py +4 -0
  76. dotvectordb-0.1.0/dotvectordb/web_interface/index.html +151 -0
  77. dotvectordb-0.1.0/dotvectordb/web_interface/server.py +448 -0
  78. dotvectordb-0.1.0/dotvectordb.egg-info/PKG-INFO +251 -0
  79. dotvectordb-0.1.0/dotvectordb.egg-info/SOURCES.txt +93 -0
  80. dotvectordb-0.1.0/dotvectordb.egg-info/dependency_links.txt +1 -0
  81. dotvectordb-0.1.0/dotvectordb.egg-info/requires.txt +17 -0
  82. dotvectordb-0.1.0/dotvectordb.egg-info/top_level.txt +1 -0
  83. dotvectordb-0.1.0/pyproject.toml +8 -0
  84. dotvectordb-0.1.0/setup.cfg +4 -0
  85. dotvectordb-0.1.0/setup.py +107 -0
  86. dotvectordb-0.1.0/tests/test_chunk_batch.py +83 -0
  87. dotvectordb-0.1.0/tests/test_chunk_delete_cascade.py +232 -0
  88. dotvectordb-0.1.0/tests/test_entity_custom_prompt.py +51 -0
  89. dotvectordb-0.1.0/tests/test_lmdb_storage.py +86 -0
  90. dotvectordb-0.1.0/tests/test_relationship_custom_prompt.py +67 -0
  91. dotvectordb-0.1.0/tests/test_relationship_supporting_lines.py +190 -0
  92. dotvectordb-0.1.0/tests/test_roaring_later_hops.py +548 -0
  93. dotvectordb-0.1.0/tests/test_roaring_retrieval.py +260 -0
  94. dotvectordb-0.1.0/tests/test_skip_embeddings.py +87 -0
  95. dotvectordb-0.1.0/tests/test_unstructured_roaring_phases.py +201 -0
@@ -0,0 +1,8 @@
1
+ include README.MD
2
+ include dotvectordb/compiled_vec1/Makefile
3
+ include dotvectordb/compiled_vec1/build.py
4
+ include dotvectordb/compiled_vec1/headers/*.h
5
+ include dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c
6
+ include dotvectordb/compiled_vec1/original_files/manifest.uuid
7
+ include dotvectordb/data_structures/bitmap/_bitmap_bfs.pyx
8
+ global-exclude *.so
@@ -0,0 +1,251 @@
1
+ Metadata-Version: 2.4
2
+ Name: dotvectordb
3
+ Version: 0.1.0
4
+ Summary: DotVectorDB is a simple, transparent vector and graph database that makes building GraphRAG pipelines easier without hiding the underlying process.
5
+ Author: Larnell Moore
6
+ Author-email: larnell@umich.edu
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Operating System :: POSIX :: Linux
9
+ Requires-Python: >=3.9
10
+ Description-Content-Type: text/markdown
11
+ Requires-Dist: torch==2.10.0
12
+ Requires-Dist: aiosqlite==0.22.1
13
+ Requires-Dist: sentence-transformers==5.5.1
14
+ Requires-Dist: python-dotenv==1.2.2
15
+ Requires-Dist: openai==2.44.0
16
+ Requires-Dist: google-genai==2.12.0
17
+ Requires-Dist: aiohttp==3.13.3
18
+ Requires-Dist: tiktoken==0.13.0
19
+ Requires-Dist: tqdm<5.0,>=4.66
20
+ Requires-Dist: igraph==1.0.0
21
+ Requires-Dist: pyroaring<2,>=1.0
22
+ Requires-Dist: lmdb<4,>=3.0
23
+ Requires-Dist: pdfplumber==0.11.6
24
+ Requires-Dist: python-docx==1.2.0
25
+ Requires-Dist: python-pptx==1.0.2
26
+ Requires-Dist: openpyxl==3.1.5
27
+ Requires-Dist: pydantic<3.0,>=2.0
28
+ Dynamic: author
29
+ Dynamic: author-email
30
+ Dynamic: classifier
31
+ Dynamic: description
32
+ Dynamic: description-content-type
33
+ Dynamic: requires-dist
34
+ Dynamic: requires-python
35
+ Dynamic: summary
36
+
37
+ # DotVectorDB
38
+
39
+ **A SQLite-based vector and graph database for building GraphRAG pipelines.**
40
+
41
+ DotVectorDB turns documents into a graph of entities and relationships, keeps relationships linked to their supporting text, and provides vector, BM25, and hybrid retrieval. Compressed Roaring bitmap indexes narrow searches to an anchor's neighborhood and entity types before ranking the candidates.
42
+
43
+ The database, graph construction, and retrieval components are separate Python classes. You can use the search primitives directly or build a retrieval algorithm on top of them.
44
+
45
+ ## What it does
46
+
47
+ - **Document ingestion:** extract text from TXT, PDF, DOCX, Excel, and PowerPoint files, then chunk it with a configurable chunker.
48
+ - **Graph construction:** extract entities and relationships, assign entity types, and embed graph content. Extraction prompts and examples can be customized.
49
+ - **Evidence tracking:** relationships retain supporting line references for each source chunk, so retrieved facts can be traced back to stored text.
50
+ - **Search:** vector search through SQLite Vec1, BM25 through FTS5, and hybrid search using reciprocal rank fusion.
51
+ - **Constrained retrieval:** retrieve entity or relationship IDs by anchor, hop, and entity type, then rank only those IDs.
52
+ - **Incremental construction:** track new and updated graph records, extend bitmap indexes, and resume completed insertion and bitmap batches.
53
+ - **Reusable embeddings:** supply an existing embedding directly to vector or hybrid search, or retrieve stored embeddings as NumPy arrays.
54
+
55
+ SQLite stores documents, graph records, metadata, and embeddings. Roaring bitmap indexes are stored separately in LMDB.
56
+
57
+ ## Install from source
58
+
59
+ The native build currently targets **Linux**. Use a C/C++ compiler with OpenMP support and `make`. Python 3.10 and 3.11 have been used during development.
60
+
61
+ On Debian or Ubuntu, install the build tools:
62
+
63
+ ```bash
64
+ sudo apt install build-essential
65
+ ```
66
+
67
+ Then install the library:
68
+
69
+ ```bash
70
+ git clone https://github.com/LuffyAI/DotVectorDB.git
71
+ cd DotVectorDB
72
+ python -m pip install .
73
+ ```
74
+
75
+ The package build compiles Vec1 from the bundled source. LMDB and PyRoaring are installed as Python dependencies. See [BUILD_LOCAL.md](BUILD_LOCAL.md) for local wheel-build commands.
76
+
77
+ ## Build a graph
78
+
79
+ The default providers use OpenAI. Create a `.env` file with your credentials and model settings, and load it **before importing the library**:
80
+
81
+ ```dotenv
82
+ OPENAI_API_KEY=your-api-key
83
+ OPENAI_EMBEDDING_MODEL=text-embedding-3-large
84
+ OPENAI_LLM_MODEL=gpt-4o-mini
85
+ TOKENIZER=gpt-4o
86
+ ```
87
+
88
+ ```python
89
+ import asyncio
90
+ from dotenv import load_dotenv
91
+
92
+ load_dotenv()
93
+
94
+ from dotvectordb.UnstructuredGraphConstructor import UnstructuredGraphConstructor
95
+
96
+
97
+ async def main():
98
+ async with UnstructuredGraphConstructor(
99
+ path="./article_graph",
100
+ tokenizer_name="gpt-4o",
101
+ ) as graph:
102
+ await graph.construct_GraphRAG(
103
+ input_document_list=["./article.txt"],
104
+ global_desc=False,
105
+ )
106
+
107
+
108
+ asyncio.run(main())
109
+ ```
110
+
111
+ This ingests and embeds chunks, builds the entity and relationship graph, creates reachability indexes, and builds graph BM25 indexes. `global_desc=False` skips LLM-generated global descriptions. Graph construction still calls the configured language and embedding models.
112
+
113
+ The path above creates a database directory containing `article_graph.db` and its graph index artifacts. Reuse the same path when opening a retriever.
114
+
115
+ ## Search the graph
116
+
117
+ ```python
118
+ from dotvectordb.GraphRetriever import GraphRetriever
119
+
120
+
121
+ async def search():
122
+ async with GraphRetriever(
123
+ path="./article_graph",
124
+ tokenizer_name="gpt-4o",
125
+ bitmap_mode="exact_hop",
126
+ ) as retriever:
127
+ results, query_tokens = await retriever.hybrid_vector_BM25_search(
128
+ query="What makes aviation difficult to decarbonize?",
129
+ table="relationship",
130
+ top_k=5,
131
+ )
132
+ return results
133
+ ```
134
+
135
+ Search tables are `chunk`, `entity`, and `relationship`. The return value is `(results, query_tokens)`.
136
+
137
+ To reuse embeddings rather than generate a new query embedding:
138
+
139
+ ```python
140
+ embeddings = await retriever.batch_get_embeddings(
141
+ ids={entity_id},
142
+ table="entity",
143
+ )
144
+
145
+ results, query_tokens = await retriever.vector_search(
146
+ query=None,
147
+ table="entity",
148
+ embedding=embeddings[entity_id],
149
+ top_k=5,
150
+ )
151
+ ```
152
+
153
+ Supplied embeddings must use the same model and dimensions as the target index. Hybrid search also accepts `embedding=...`; its BM25 component still requires query text.
154
+
155
+ ## Search a neighborhood
156
+
157
+ Precomputed exact-hop buckets provide direct lookups for the indexed hop and entity type. They return database IDs that can be passed to `search_id_batch`:
158
+
159
+ ```python
160
+ relationship_ids = await retriever.bitmap_retriever.constrained_relationship_search(
161
+ anchor_entity_id=anchor_id,
162
+ exact_hop=2,
163
+ entity_types=["person"],
164
+ )
165
+
166
+ results, query_tokens = await retriever.search_id_batch(
167
+ query="Who worked on these projects?",
168
+ table="relationship",
169
+ rowids=relationship_ids,
170
+ top_k=10,
171
+ search_type="hybrid",
172
+ )
173
+ ```
174
+
175
+ Entities can belong to multiple types. Type filters match any requested type by default; `match_all_types=True` requires all requested types.
176
+
177
+ Precomputed hops do not expand the graph during lookup. Requests beyond the precomputed range use one-hop traversal for the remaining distance. Choose `bitmap_mode="one_hop"` to use traversal directly, or `load_all_into_memory=True` to cache the bitmap stores in memory. A memory-loaded bitmap retriever can pick up index changes with `await retriever.bitmap_retriever.refresh()`.
178
+
179
+ ## Document-only retrieval
180
+
181
+ Use `NaiveVDBConstructor` when you want document retrieval without entity or relationship extraction:
182
+
183
+ ```python
184
+ from dotvectordb.NaiveVDBConstructor import NaiveVDBConstructor
185
+
186
+
187
+ async def build_document_index():
188
+ async with NaiveVDBConstructor(
189
+ path="./document_index",
190
+ tokenizer_name="gpt-4o",
191
+ ) as database:
192
+ await database.construct_document_bm25_index(
193
+ input_document_list=["./article.txt"],
194
+ )
195
+ ```
196
+
197
+ `construct_document_bm25_index` skips chunk embeddings. Database initialization currently still configures the default models and may generate an embedding to establish dimensions for a fresh database.
198
+
199
+ The same constructor also provides `construct_document_vectordatabase` and `construct_bm25_and_vdb`.
200
+
201
+ ## Code layout
202
+
203
+ | Component | Responsibility |
204
+ |---|---|
205
+ | `DotVectorDB` | SQLite connection, schema, metadata, and embedding storage |
206
+ | `UnstructuredGraphConstructor` | Coordinate document ingestion and graph construction |
207
+ | `NaiveVDBConstructor` | Document-only BM25 and vector index construction |
208
+ | `GraphRetriever` | BM25, vector, hybrid, and ID-constrained search |
209
+ | `RetrievalAlgos/` | Retrieval algorithms built on the search primitives; several are still in development |
210
+ | `plugins/` | Document extraction, chunking, hashing, and model providers |
211
+ | `data_structures/bitmap/` | Roaring bitmap construction and retrieval backed by LMDB |
212
+ | `schema/` | SQL tables, indexes, and triggers |
213
+
214
+ Custom chunkers, hashing functions, embedding models, and language models can be supplied through the corresponding constructor arguments.
215
+
216
+ ## Native extension details
217
+
218
+ The bundled source is pinned to **Vec1 0.8**, independently of the SQLite engine version:
219
+
220
+ - Source: `dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c`
221
+ - SQLite source revision: `213d76c009bcd860b5a624be6ffe3b89aab063bc6f2c30bdb40384c5d7f8b35f`
222
+ - `vec1.c` SHA-256: `b341c15fe10e41a9b96981ff3a9c217ea0774b4e04932b5acf3dac1320aeceb1`
223
+
224
+ Matching SQLite headers are bundled. Compilation requires no Vec1 download. After loading the extension, `SELECT vec1_info();` reports its version and enabled features.
225
+
226
+ The default Vec1 compiler flags include `-march=native`: build on the target machine rather than distributing that wheel across different CPUs. Compiler flags can be overridden through `CC`, `CPPFLAGS`, `CFLAGS`, and `LDFLAGS`.
227
+
228
+ ## Status
229
+
230
+ DotVectorDB is a research project under active development. APIs and schemas can change between revisions. Performance depends on graph structure, storage, models, and the index configuration; this README does not claim a universal benchmark result.
231
+
232
+ A PyPI release for installation with `pip install dotvectordb` is planned once development is complete.
233
+
234
+ ## Citation
235
+
236
+ If you use the DotRAG retrieval algorithm or build on DotVectorDB in your research, please cite our paper:
237
+
238
+ **[DotRAG: Retrieval-Time Reasoning Along Relational Paths](https://arxiv.org/abs/2605.18760)**
239
+ Larnell Moore, Naihao Deng, Rada Mihalcea, and Farnaz Jahanbakhsh. arXiv, 2026.
240
+
241
+ ```bibtex
242
+ @misc{moore2026dotrag,
243
+ title = {{DotRAG}: Retrieval-Time Reasoning Along Relational Paths},
244
+ author = {Moore, Larnell and Deng, Naihao and Mihalcea, Rada and Jahanbakhsh, Farnaz},
245
+ year = {2026},
246
+ eprint = {2605.18760},
247
+ archivePrefix = {arXiv},
248
+ primaryClass = {cs.IR},
249
+ url = {https://arxiv.org/abs/2605.18760}
250
+ }
251
+ ```
@@ -0,0 +1,215 @@
1
+ # DotVectorDB
2
+
3
+ **A SQLite-based vector and graph database for building GraphRAG pipelines.**
4
+
5
+ DotVectorDB turns documents into a graph of entities and relationships, keeps relationships linked to their supporting text, and provides vector, BM25, and hybrid retrieval. Compressed Roaring bitmap indexes narrow searches to an anchor's neighborhood and entity types before ranking the candidates.
6
+
7
+ The database, graph construction, and retrieval components are separate Python classes. You can use the search primitives directly or build a retrieval algorithm on top of them.
8
+
9
+ ## What it does
10
+
11
+ - **Document ingestion:** extract text from TXT, PDF, DOCX, Excel, and PowerPoint files, then chunk it with a configurable chunker.
12
+ - **Graph construction:** extract entities and relationships, assign entity types, and embed graph content. Extraction prompts and examples can be customized.
13
+ - **Evidence tracking:** relationships retain supporting line references for each source chunk, so retrieved facts can be traced back to stored text.
14
+ - **Search:** vector search through SQLite Vec1, BM25 through FTS5, and hybrid search using reciprocal rank fusion.
15
+ - **Constrained retrieval:** retrieve entity or relationship IDs by anchor, hop, and entity type, then rank only those IDs.
16
+ - **Incremental construction:** track new and updated graph records, extend bitmap indexes, and resume completed insertion and bitmap batches.
17
+ - **Reusable embeddings:** supply an existing embedding directly to vector or hybrid search, or retrieve stored embeddings as NumPy arrays.
18
+
19
+ SQLite stores documents, graph records, metadata, and embeddings. Roaring bitmap indexes are stored separately in LMDB.
20
+
21
+ ## Install from source
22
+
23
+ The native build currently targets **Linux**. Use a C/C++ compiler with OpenMP support and `make`. Python 3.10 and 3.11 have been used during development.
24
+
25
+ On Debian or Ubuntu, install the build tools:
26
+
27
+ ```bash
28
+ sudo apt install build-essential
29
+ ```
30
+
31
+ Then install the library:
32
+
33
+ ```bash
34
+ git clone https://github.com/LuffyAI/DotVectorDB.git
35
+ cd DotVectorDB
36
+ python -m pip install .
37
+ ```
38
+
39
+ The package build compiles Vec1 from the bundled source. LMDB and PyRoaring are installed as Python dependencies. See [BUILD_LOCAL.md](BUILD_LOCAL.md) for local wheel-build commands.
40
+
41
+ ## Build a graph
42
+
43
+ The default providers use OpenAI. Create a `.env` file with your credentials and model settings, and load it **before importing the library**:
44
+
45
+ ```dotenv
46
+ OPENAI_API_KEY=your-api-key
47
+ OPENAI_EMBEDDING_MODEL=text-embedding-3-large
48
+ OPENAI_LLM_MODEL=gpt-4o-mini
49
+ TOKENIZER=gpt-4o
50
+ ```
51
+
52
+ ```python
53
+ import asyncio
54
+ from dotenv import load_dotenv
55
+
56
+ load_dotenv()
57
+
58
+ from dotvectordb.UnstructuredGraphConstructor import UnstructuredGraphConstructor
59
+
60
+
61
+ async def main():
62
+ async with UnstructuredGraphConstructor(
63
+ path="./article_graph",
64
+ tokenizer_name="gpt-4o",
65
+ ) as graph:
66
+ await graph.construct_GraphRAG(
67
+ input_document_list=["./article.txt"],
68
+ global_desc=False,
69
+ )
70
+
71
+
72
+ asyncio.run(main())
73
+ ```
74
+
75
+ This ingests and embeds chunks, builds the entity and relationship graph, creates reachability indexes, and builds graph BM25 indexes. `global_desc=False` skips LLM-generated global descriptions. Graph construction still calls the configured language and embedding models.
76
+
77
+ The path above creates a database directory containing `article_graph.db` and its graph index artifacts. Reuse the same path when opening a retriever.
78
+
79
+ ## Search the graph
80
+
81
+ ```python
82
+ from dotvectordb.GraphRetriever import GraphRetriever
83
+
84
+
85
+ async def search():
86
+ async with GraphRetriever(
87
+ path="./article_graph",
88
+ tokenizer_name="gpt-4o",
89
+ bitmap_mode="exact_hop",
90
+ ) as retriever:
91
+ results, query_tokens = await retriever.hybrid_vector_BM25_search(
92
+ query="What makes aviation difficult to decarbonize?",
93
+ table="relationship",
94
+ top_k=5,
95
+ )
96
+ return results
97
+ ```
98
+
99
+ Search tables are `chunk`, `entity`, and `relationship`. The return value is `(results, query_tokens)`.
100
+
101
+ To reuse embeddings rather than generate a new query embedding:
102
+
103
+ ```python
104
+ embeddings = await retriever.batch_get_embeddings(
105
+ ids={entity_id},
106
+ table="entity",
107
+ )
108
+
109
+ results, query_tokens = await retriever.vector_search(
110
+ query=None,
111
+ table="entity",
112
+ embedding=embeddings[entity_id],
113
+ top_k=5,
114
+ )
115
+ ```
116
+
117
+ Supplied embeddings must use the same model and dimensions as the target index. Hybrid search also accepts `embedding=...`; its BM25 component still requires query text.
118
+
119
+ ## Search a neighborhood
120
+
121
+ Precomputed exact-hop buckets provide direct lookups for the indexed hop and entity type. They return database IDs that can be passed to `search_id_batch`:
122
+
123
+ ```python
124
+ relationship_ids = await retriever.bitmap_retriever.constrained_relationship_search(
125
+ anchor_entity_id=anchor_id,
126
+ exact_hop=2,
127
+ entity_types=["person"],
128
+ )
129
+
130
+ results, query_tokens = await retriever.search_id_batch(
131
+ query="Who worked on these projects?",
132
+ table="relationship",
133
+ rowids=relationship_ids,
134
+ top_k=10,
135
+ search_type="hybrid",
136
+ )
137
+ ```
138
+
139
+ Entities can belong to multiple types. Type filters match any requested type by default; `match_all_types=True` requires all requested types.
140
+
141
+ Precomputed hops do not expand the graph during lookup. Requests beyond the precomputed range use one-hop traversal for the remaining distance. Choose `bitmap_mode="one_hop"` to use traversal directly, or `load_all_into_memory=True` to cache the bitmap stores in memory. A memory-loaded bitmap retriever can pick up index changes with `await retriever.bitmap_retriever.refresh()`.
142
+
143
+ ## Document-only retrieval
144
+
145
+ Use `NaiveVDBConstructor` when you want document retrieval without entity or relationship extraction:
146
+
147
+ ```python
148
+ from dotvectordb.NaiveVDBConstructor import NaiveVDBConstructor
149
+
150
+
151
+ async def build_document_index():
152
+ async with NaiveVDBConstructor(
153
+ path="./document_index",
154
+ tokenizer_name="gpt-4o",
155
+ ) as database:
156
+ await database.construct_document_bm25_index(
157
+ input_document_list=["./article.txt"],
158
+ )
159
+ ```
160
+
161
+ `construct_document_bm25_index` skips chunk embeddings. Database initialization currently still configures the default models and may generate an embedding to establish dimensions for a fresh database.
162
+
163
+ The same constructor also provides `construct_document_vectordatabase` and `construct_bm25_and_vdb`.
164
+
165
+ ## Code layout
166
+
167
+ | Component | Responsibility |
168
+ |---|---|
169
+ | `DotVectorDB` | SQLite connection, schema, metadata, and embedding storage |
170
+ | `UnstructuredGraphConstructor` | Coordinate document ingestion and graph construction |
171
+ | `NaiveVDBConstructor` | Document-only BM25 and vector index construction |
172
+ | `GraphRetriever` | BM25, vector, hybrid, and ID-constrained search |
173
+ | `RetrievalAlgos/` | Retrieval algorithms built on the search primitives; several are still in development |
174
+ | `plugins/` | Document extraction, chunking, hashing, and model providers |
175
+ | `data_structures/bitmap/` | Roaring bitmap construction and retrieval backed by LMDB |
176
+ | `schema/` | SQL tables, indexes, and triggers |
177
+
178
+ Custom chunkers, hashing functions, embedding models, and language models can be supplied through the corresponding constructor arguments.
179
+
180
+ ## Native extension details
181
+
182
+ The bundled source is pinned to **Vec1 0.8**, independently of the SQLite engine version:
183
+
184
+ - Source: `dotvectordb/compiled_vec1/original_files/ext/vec1/vec1.c`
185
+ - SQLite source revision: `213d76c009bcd860b5a624be6ffe3b89aab063bc6f2c30bdb40384c5d7f8b35f`
186
+ - `vec1.c` SHA-256: `b341c15fe10e41a9b96981ff3a9c217ea0774b4e04932b5acf3dac1320aeceb1`
187
+
188
+ Matching SQLite headers are bundled. Compilation requires no Vec1 download. After loading the extension, `SELECT vec1_info();` reports its version and enabled features.
189
+
190
+ The default Vec1 compiler flags include `-march=native`: build on the target machine rather than distributing that wheel across different CPUs. Compiler flags can be overridden through `CC`, `CPPFLAGS`, `CFLAGS`, and `LDFLAGS`.
191
+
192
+ ## Status
193
+
194
+ DotVectorDB is a research project under active development. APIs and schemas can change between revisions. Performance depends on graph structure, storage, models, and the index configuration; this README does not claim a universal benchmark result.
195
+
196
+ A PyPI release for installation with `pip install dotvectordb` is planned once development is complete.
197
+
198
+ ## Citation
199
+
200
+ If you use the DotRAG retrieval algorithm or build on DotVectorDB in your research, please cite our paper:
201
+
202
+ **[DotRAG: Retrieval-Time Reasoning Along Relational Paths](https://arxiv.org/abs/2605.18760)**
203
+ Larnell Moore, Naihao Deng, Rada Mihalcea, and Farnaz Jahanbakhsh. arXiv, 2026.
204
+
205
+ ```bibtex
206
+ @misc{moore2026dotrag,
207
+ title = {{DotRAG}: Retrieval-Time Reasoning Along Relational Paths},
208
+ author = {Moore, Larnell and Deng, Naihao and Mihalcea, Rada and Jahanbakhsh, Farnaz},
209
+ year = {2026},
210
+ eprint = {2605.18760},
211
+ archivePrefix = {arXiv},
212
+ primaryClass = {cs.IR},
213
+ url = {https://arxiv.org/abs/2605.18760}
214
+ }
215
+ ```