garuddb 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- garuddb-0.1.0/PKG-INFO +307 -0
- garuddb-0.1.0/README.md +258 -0
- garuddb-0.1.0/garuddb/__init__.py +72 -0
- garuddb-0.1.0/garuddb/cli.py +165 -0
- garuddb-0.1.0/garuddb/client.py +200 -0
- garuddb-0.1.0/garuddb/collection.py +573 -0
- garuddb-0.1.0/garuddb/embeddings.py +283 -0
- garuddb-0.1.0/garuddb/engine/__init__.py +10 -0
- garuddb-0.1.0/garuddb/engine/concurrency.py +89 -0
- garuddb-0.1.0/garuddb/engine/index.py +534 -0
- garuddb-0.1.0/garuddb/engine/weights.py +130 -0
- garuddb-0.1.0/garuddb/extractor.py +180 -0
- garuddb-0.1.0/garuddb/ingest/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/base_connector.py +34 -0
- garuddb-0.1.0/garuddb/ingest/connectors/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/connectors/local_fs.py +69 -0
- garuddb-0.1.0/garuddb/ingest/connectors/rest_api.py +67 -0
- garuddb-0.1.0/garuddb/ingest/connectors/sqlite.py +60 -0
- garuddb-0.1.0/garuddb/ingest/connectors/streaming_csv.py +46 -0
- garuddb-0.1.0/garuddb/ingest/embedding/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/embedding/base_provider.py +47 -0
- garuddb-0.1.0/garuddb/ingest/embedding/caching.py +85 -0
- garuddb-0.1.0/garuddb/ingest/embedding/embedding_store.py +135 -0
- garuddb-0.1.0/garuddb/ingest/embedding/providers/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/embedding/providers/garud_embed.py +86 -0
- garuddb-0.1.0/garuddb/ingest/embedding/providers/sentence_transformer.py +62 -0
- garuddb-0.1.0/garuddb/ingest/embedding/registry.py +23 -0
- garuddb-0.1.0/garuddb/ingest/fusion/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/fusion/engine.py +37 -0
- garuddb-0.1.0/garuddb/ingest/fusion/radius.py +39 -0
- garuddb-0.1.0/garuddb/ingest/fusion/strategies.py +133 -0
- garuddb-0.1.0/garuddb/ingest/fusion/validator.py +44 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/base_preprocessor.py +25 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/processors/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/processors/csv_proc.py +75 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/processors/json_proc.py +147 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/processors/markdown_proc.py +97 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/processors/pdf_proc.py +87 -0
- garuddb-0.1.0/garuddb/ingest/preprocess/registry.py +25 -0
- garuddb-0.1.0/garuddb/ingest/representation/engine.py +34 -0
- garuddb-0.1.0/garuddb/ingest/representation/native.py +42 -0
- garuddb-0.1.0/garuddb/ingest/schema.py +120 -0
- garuddb-0.1.0/garuddb/ingest/topology/__init__.py +1 -0
- garuddb-0.1.0/garuddb/ingest/topology/engine.py +114 -0
- garuddb-0.1.0/garuddb/ingest/topology/graph.py +168 -0
- garuddb-0.1.0/garuddb/ingest/topology/optimizer.py +15 -0
- garuddb-0.1.0/garuddb/models/__init__.py +1 -0
- garuddb-0.1.0/garuddb/models/garud_embed_v4.py +161 -0
- garuddb-0.1.0/garuddb/models/garud_embed_v5.py +101 -0
- garuddb-0.1.0/garuddb/models/hierarchy_miner.py +165 -0
- garuddb-0.1.0/garuddb/server/__init__.py +1 -0
- garuddb-0.1.0/garuddb/server/api.py +190 -0
- garuddb-0.1.0/garuddb.egg-info/PKG-INFO +307 -0
- garuddb-0.1.0/garuddb.egg-info/SOURCES.txt +61 -0
- garuddb-0.1.0/garuddb.egg-info/dependency_links.txt +1 -0
- garuddb-0.1.0/garuddb.egg-info/entry_points.txt +2 -0
- garuddb-0.1.0/garuddb.egg-info/requires.txt +28 -0
- garuddb-0.1.0/garuddb.egg-info/top_level.txt +5 -0
- garuddb-0.1.0/hyperbolic_index.cpp +937 -0
- garuddb-0.1.0/pyproject.toml +87 -0
- garuddb-0.1.0/setup.cfg +4 -0
- garuddb-0.1.0/setup.py +40 -0
garuddb-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: garuddb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Hyperbolic Vector Database for Hierarchical Data — Poincaré geometry meets HNSW
|
|
5
|
+
Home-page: https://github.com/AryanRajendraDalvi/GarudDB
|
|
6
|
+
Author: GarudDB Contributors
|
|
7
|
+
License: Proprietary
|
|
8
|
+
Project-URL: Homepage, https://github.com/AryanRajendraDalvi/GarudDB
|
|
9
|
+
Keywords: vector-database,hyperbolic,poincare,hnsw,knowledge-graph
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Database
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: C++
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
Requires-Dist: numpy>=1.24.0
|
|
24
|
+
Requires-Dist: pandas>=2.0.0
|
|
25
|
+
Requires-Dist: huggingface-hub>=0.20.0
|
|
26
|
+
Provides-Extra: embed
|
|
27
|
+
Requires-Dist: torch>=2.0; extra == "embed"
|
|
28
|
+
Requires-Dist: geoopt>=0.5; extra == "embed"
|
|
29
|
+
Requires-Dist: sentence-transformers>=2.0; extra == "embed"
|
|
30
|
+
Provides-Extra: adapter
|
|
31
|
+
Requires-Dist: torch>=2.0; extra == "adapter"
|
|
32
|
+
Requires-Dist: transformers>=4.30; extra == "adapter"
|
|
33
|
+
Requires-Dist: geoopt>=0.5; extra == "adapter"
|
|
34
|
+
Requires-Dist: sentence-transformers>=2.0; extra == "adapter"
|
|
35
|
+
Provides-Extra: server
|
|
36
|
+
Requires-Dist: fastapi>=0.100; extra == "server"
|
|
37
|
+
Requires-Dist: uvicorn[standard]>=0.20; extra == "server"
|
|
38
|
+
Requires-Dist: boto3>=1.28; extra == "server"
|
|
39
|
+
Requires-Dist: python-multipart>=0.0.9; extra == "server"
|
|
40
|
+
Provides-Extra: all
|
|
41
|
+
Requires-Dist: garuddb[embed]; extra == "all"
|
|
42
|
+
Requires-Dist: garuddb[adapter]; extra == "all"
|
|
43
|
+
Requires-Dist: garuddb[server]; extra == "all"
|
|
44
|
+
Requires-Dist: onnxruntime>=1.15; extra == "all"
|
|
45
|
+
Requires-Dist: requests>=2.28; extra == "all"
|
|
46
|
+
Requires-Dist: tqdm>=4.60; extra == "all"
|
|
47
|
+
Dynamic: home-page
|
|
48
|
+
Dynamic: requires-python
|
|
49
|
+
|
|
50
|
+
# GarudDB: Hyperbolic Vector Database 🦅
|
|
51
|
+
|
|
52
|
+
**A topology-aware vector database built on Poincaré geometry.** Free and open-source, like ChromaDB — but for hierarchical data.
|
|
53
|
+
|
|
54
|
+
GarudDB natively encodes parent-child relationships, corporate ownership trees, ontologies, and knowledge graphs using **Riemannian Hyperbolic Geometry (Poincaré Ball Model)**. It answers complex tree queries that flat Euclidean databases like ChromaDB, Pinecone, and FAISS fundamentally cannot.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install garuddb
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## ⚡ Quick Start
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import garuddb
|
|
66
|
+
|
|
67
|
+
# Create a persistent database
|
|
68
|
+
client = garuddb.PersistentClient(path="./my_database")
|
|
69
|
+
|
|
70
|
+
# Create a collection with your choice of embedding model
|
|
71
|
+
collection = client.create_collection(
|
|
72
|
+
"companies",
|
|
73
|
+
embedding_function=garuddb.DefaultEmbeddingFunction() # or GarudEmbedFunction()
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# Add documents — auto-embedded
|
|
77
|
+
collection.add(
|
|
78
|
+
documents=["Apple Inc.", "Braeburn Capital Inc.", "Microsoft Corp."],
|
|
79
|
+
metadatas=[{"country": "US"}, {"country": "US"}, {"country": "US"}],
|
|
80
|
+
ids=["apple", "braeburn", "microsoft"]
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# Standard query (ChromaDB-compatible)
|
|
84
|
+
results = collection.query(query_texts=["Apple subsidiary"], n_results=5)
|
|
85
|
+
|
|
86
|
+
# GarudDB-exclusive: Topology-aware search
|
|
87
|
+
subtree = collection.find_subtree("Apple Inc.", n_results=10)
|
|
88
|
+
ancestor = collection.find_common_ancestor("Apple Inc.", "Braeburn Capital Inc.")
|
|
89
|
+
neighbors = collection.search_neighbors("Apple Inc.", translation_mode="inward")
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## 🏗️ What Makes GarudDB Different
|
|
95
|
+
|
|
96
|
+
### The Problem with Flat Vector Databases
|
|
97
|
+
ChromaDB, Pinecone, FAISS, and Weaviate all store vectors in **Euclidean or cosine space**. This works fine for semantic similarity, but completely breaks down for **hierarchical data**:
|
|
98
|
+
|
|
99
|
+
- "Find all subsidiaries of Apple" → **0% recall** in ChromaDB
|
|
100
|
+
- "What is the common parent company of these two entities?" → **Impossible** in flat databases
|
|
101
|
+
- "Traverse the ownership tree from leaf to root" → **Not a concept** that exists
|
|
102
|
+
|
|
103
|
+
### GarudDB's Solution: Poincaré Geometry
|
|
104
|
+
The Poincaré ball model has a crucial mathematical property: **exponentially more space near the boundary**. This perfectly mirrors how hierarchies work — one root, exponentially more leaves.
|
|
105
|
+
|
|
106
|
+
In GarudDB:
|
|
107
|
+
- **Root nodes** (parent companies, chapter headers) → positioned near the **origin** of the disk
|
|
108
|
+
- **Leaf nodes** (subsidiaries, paragraph text) → positioned near the **boundary** of the disk
|
|
109
|
+
- **Tree distance** = **geodesic distance** along the curved Poincaré manifold
|
|
110
|
+
|
|
111
|
+
This means tree traversal is just geometry. No graph database needed.
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## 📊 Benchmark Summary
|
|
116
|
+
|
|
117
|
+
Built on the global GLEIF dataset (3.3M+ corporate entities):
|
|
118
|
+
|
|
119
|
+
| Metric | ChromaDB | GarudDB | Advantage |
|
|
120
|
+
|:---|:---|:---|:---|
|
|
121
|
+
| RAG Search Latency | 2.49 ms | **0.06 ms** | **~40x faster** |
|
|
122
|
+
| Max Scale (8GB RAM) | ~414k vectors | **2M+ vectors** | **5x more** |
|
|
123
|
+
| Recall@10 (Flat Text) | 10.2% | 11.2% | +1pp |
|
|
124
|
+
| Ownership Discovery R@10 | 0.0% | **26.1%** | **∞** |
|
|
125
|
+
| Deep Traversal R@10 | 0.0% | **14.0%** | **∞** |
|
|
126
|
+
| Metadata Filter Latency | 35–80 ms | **6–15 ms** | **~4x faster** |
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## 🔌 Pluggable Embedding Functions
|
|
131
|
+
|
|
132
|
+
GarudDB is embedding-model agnostic. Use whatever model you want:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
# Option 1: Default — SentenceTransformer (384D, Euclidean, cosine metric)
|
|
136
|
+
collection = client.create_collection(
|
|
137
|
+
"docs",
|
|
138
|
+
embedding_function=garuddb.DefaultEmbeddingFunction()
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# Option 2: Native — GarudEmbed V3 (64D, real Poincaré vectors)
|
|
142
|
+
# Trained for 60-75 epochs on hierarchical data — the gold standard
|
|
143
|
+
collection = client.create_collection(
|
|
144
|
+
"companies",
|
|
145
|
+
embedding_function=garuddb.GarudEmbedFunction()
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
# Option 3: Custom — Any model that returns vectors
|
|
149
|
+
class MyEmbedder:
|
|
150
|
+
dimension = 768
|
|
151
|
+
is_hyperbolic = False
|
|
152
|
+
def __call__(self, texts):
|
|
153
|
+
return my_model.encode(texts) # Your model here
|
|
154
|
+
|
|
155
|
+
collection = client.create_collection("custom", embedding_function=MyEmbedder())
|
|
156
|
+
|
|
157
|
+
# Option 4: Pre-computed vectors — skip embedding entirely
|
|
158
|
+
collection.add(embeddings=[[0.1, 0.2, ...]], ids=["1"])
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## 🧭 Topology Search API (The Moat)
|
|
164
|
+
|
|
165
|
+
These methods only exist in GarudDB. No other vector database can do this:
|
|
166
|
+
|
|
167
|
+
### `search_neighbors()` — Geodesic Translation
|
|
168
|
+
Navigate the Poincaré disk directionally:
|
|
169
|
+
```python
|
|
170
|
+
# Find parents/grandparents (move INWARD toward disk origin)
|
|
171
|
+
parents = collection.search_neighbors("Braeburn Capital", translation_mode="inward")
|
|
172
|
+
|
|
173
|
+
# Find children/subsidiaries (move OUTWARD toward disk boundary)
|
|
174
|
+
children = collection.search_neighbors("Apple Inc.", translation_mode="outward")
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### `find_subtree()` — Deep Hierarchy Traversal
|
|
178
|
+
Find all entities within a topological branch:
|
|
179
|
+
```python
|
|
180
|
+
subtree = collection.find_subtree("Apple Inc.", n_results=20)
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### `find_common_ancestor()` — Möbius Midpoint
|
|
184
|
+
Find the nearest common parent of two entities:
|
|
185
|
+
```python
|
|
186
|
+
ancestor = collection.find_common_ancestor("Entity A", "Entity B")
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### `query_filtered()` — Native C++ Bitmask Filtering
|
|
190
|
+
Hardware-accelerated metadata filtering inside the HNSW graph:
|
|
191
|
+
```python
|
|
192
|
+
results = collection.query_filtered(
|
|
193
|
+
"Apple Inc.",
|
|
194
|
+
country="US",
|
|
195
|
+
status="ACTIVE",
|
|
196
|
+
n_results=10
|
|
197
|
+
)
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
---
|
|
201
|
+
|
|
202
|
+
## 🛠️ Installation & Setup
|
|
203
|
+
|
|
204
|
+
### 1. Install dependencies
|
|
205
|
+
```bash
|
|
206
|
+
pip install -r requirements.txt
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### 2. Build the C++ engine
|
|
210
|
+
```bash
|
|
211
|
+
pip install -e .
|
|
212
|
+
```
|
|
213
|
+
This compiles `hyperbolic_index.cpp` into the `adk_core` Python extension with AVX2 SIMD acceleration.
|
|
214
|
+
|
|
215
|
+
### 3. Optional extras
|
|
216
|
+
```bash
|
|
217
|
+
pip install garuddb[embed] # GarudEmbed V3 (PyTorch + Geoopt)
|
|
218
|
+
pip install garuddb[adapter] # GarudAdapter migration (SentenceTransformers)
|
|
219
|
+
pip install garuddb[server] # FastAPI server
|
|
220
|
+
pip install garuddb[all] # Everything
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## 🏛️ Architecture
|
|
226
|
+
|
|
227
|
+
### C++ Engine (`adk_core`)
|
|
228
|
+
The heart of GarudDB — a custom HNSW implementation written in C++ with:
|
|
229
|
+
- **Three distance kernels:** Poincaré geodesic, cosine, Euclidean — all SIMD-accelerated (AVX2)
|
|
230
|
+
- **Multi-layer HNSW:** Proper O(log N) ingest and search with geometric random level assignment
|
|
231
|
+
- **64-bit in-graph bitmask filtering:** Metadata filters evaluated at each HNSW node during traversal
|
|
232
|
+
- **Dual Index Architecture:** Dense HNSW (norm < 0.995) + flat scan for boundary singletons (norm ≥ 0.995)
|
|
233
|
+
- **GIL release:** Python threads run concurrently during C++ execution
|
|
234
|
+
|
|
235
|
+
### Python SDK (`garuddb/`)
|
|
236
|
+
```
|
|
237
|
+
garuddb/
|
|
238
|
+
├── __init__.py # Public API exports
|
|
239
|
+
├── client.py # EphemeralClient, PersistentClient
|
|
240
|
+
├── collection.py # Collection API (add, query, search_neighbors, etc.)
|
|
241
|
+
├── embeddings.py # Pluggable embedding functions
|
|
242
|
+
├── cli.py # Command-line interface
|
|
243
|
+
├── engine/
|
|
244
|
+
│ ├── index.py # TriIndex wrapper (dense + singleton + semantic)
|
|
245
|
+
│ ├── concurrency.py # Read-Write lock for thread safety
|
|
246
|
+
│ └── weights.py # GarudEmbed checkpoint downloader
|
|
247
|
+
└── server/ # FastAPI server (optional)
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
### Tri-Index Architecture
|
|
251
|
+
Every collection maintains three C++ indices:
|
|
252
|
+
|
|
253
|
+
| Index | Metric | Purpose |
|
|
254
|
+
|:---|:---|:---|
|
|
255
|
+
| `dense_index` | Poincaré | HNSW graph for multi-hop topology traversal |
|
|
256
|
+
| `singleton_index` | Poincaré | Flat scan for boundary nodes (no HNSW edges) |
|
|
257
|
+
| `semantic_index` | Cosine | Text-to-entity anchor resolution |
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
## 🧠 Mathematical Features
|
|
262
|
+
|
|
263
|
+
### 1. Geodesic Translation
|
|
264
|
+
Before searching, the query vector is shifted along the Poincaré geodesic:
|
|
265
|
+
- **Inward (× 0.92):** Traverse toward parent nodes near the disk origin
|
|
266
|
+
- **Outward (→ 0.995):** Traverse toward leaf subsidiaries near the disk boundary
|
|
267
|
+
|
|
268
|
+
### 2. Geometric Density Filter
|
|
269
|
+
Ported from the battle-tested `garud_search_engine.py`. Prevents results from bleeding across tree branches:
|
|
270
|
+
- **Gap threshold (0.6):** If consecutive results have a distance jump > 0.6, we've crossed a branch boundary — stop
|
|
271
|
+
- **Absolute cutoff (2.0):** Never return results beyond structural distance 2.0
|
|
272
|
+
|
|
273
|
+
### 3. 64-Bit In-Graph Bitmask Filtering
|
|
274
|
+
Metadata is encoded as `uint64_t` bitmasks at ingest time. Filters are evaluated with hardware `AND` inside the HNSW traversal — no post-filtering. Filter selectivity has zero impact on latency.
|
|
275
|
+
|
|
276
|
+
### 4. Multi-Hop Gradient Traversal
|
|
277
|
+
After initial HNSW search, the engine re-searches from the nearest non-trivial neighbor to expand the retrieval neighborhood. This walks along HNSW graph edges, which in Poincaré space correspond to actual tree topology links.
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## 📁 Project Structure
|
|
282
|
+
|
|
283
|
+
| Directory | Purpose |
|
|
284
|
+
|:---|:---|
|
|
285
|
+
| `garuddb/` | **Core SDK** — the pip-installable package |
|
|
286
|
+
| `hyperbolic_index.cpp` | C++ HNSW engine source |
|
|
287
|
+
| `garud_embed_model.py` | GarudEmbed V3 model architecture |
|
|
288
|
+
| `train_garud_embed.py` | Training script for GarudEmbed |
|
|
289
|
+
| `garud_adapter.py` | Euclidean → GarudDB migration tool |
|
|
290
|
+
| `garud_search_engine.py` | Reference implementation (corporate graph) |
|
|
291
|
+
| `examples/` | Example applications |
|
|
292
|
+
| `garud_ingest/` | 6-layer ingestion pipeline |
|
|
293
|
+
|
|
294
|
+
---
|
|
295
|
+
|
|
296
|
+
## 🚀 Roadmap
|
|
297
|
+
|
|
298
|
+
- **v0.1.0:** Core SDK release (`pip install garuddb`) with pluggable embeddings
|
|
299
|
+
- **v0.2.0:** Built-in data connectors (CSV, JSON, PDF parsing into hierarchy)
|
|
300
|
+
- **v0.3.0:** LLM Intent Routing for natural language queries
|
|
301
|
+
- **v1.0.0:** GPU-accelerated CUDA kernel for billion-scale traversal
|
|
302
|
+
|
|
303
|
+
---
|
|
304
|
+
|
|
305
|
+
## 📜 License
|
|
306
|
+
|
|
307
|
+
Proprietary — see LICENSE file for details.
|
garuddb-0.1.0/README.md
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
# GarudDB: Hyperbolic Vector Database 🦅
|
|
2
|
+
|
|
3
|
+
**A topology-aware vector database built on Poincaré geometry.** Free and open-source, like ChromaDB — but for hierarchical data.
|
|
4
|
+
|
|
5
|
+
GarudDB natively encodes parent-child relationships, corporate ownership trees, ontologies, and knowledge graphs using **Riemannian Hyperbolic Geometry (Poincaré Ball Model)**. It answers complex tree queries that flat Euclidean databases like ChromaDB, Pinecone, and FAISS fundamentally cannot.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install garuddb
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## ⚡ Quick Start
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import garuddb
|
|
17
|
+
|
|
18
|
+
# Create a persistent database
|
|
19
|
+
client = garuddb.PersistentClient(path="./my_database")
|
|
20
|
+
|
|
21
|
+
# Create a collection with your choice of embedding model
|
|
22
|
+
collection = client.create_collection(
|
|
23
|
+
"companies",
|
|
24
|
+
embedding_function=garuddb.DefaultEmbeddingFunction() # or GarudEmbedFunction()
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# Add documents — auto-embedded
|
|
28
|
+
collection.add(
|
|
29
|
+
documents=["Apple Inc.", "Braeburn Capital Inc.", "Microsoft Corp."],
|
|
30
|
+
metadatas=[{"country": "US"}, {"country": "US"}, {"country": "US"}],
|
|
31
|
+
ids=["apple", "braeburn", "microsoft"]
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# Standard query (ChromaDB-compatible)
|
|
35
|
+
results = collection.query(query_texts=["Apple subsidiary"], n_results=5)
|
|
36
|
+
|
|
37
|
+
# GarudDB-exclusive: Topology-aware search
|
|
38
|
+
subtree = collection.find_subtree("Apple Inc.", n_results=10)
|
|
39
|
+
ancestor = collection.find_common_ancestor("Apple Inc.", "Braeburn Capital Inc.")
|
|
40
|
+
neighbors = collection.search_neighbors("Apple Inc.", translation_mode="inward")
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 🏗️ What Makes GarudDB Different
|
|
46
|
+
|
|
47
|
+
### The Problem with Flat Vector Databases
|
|
48
|
+
ChromaDB, Pinecone, FAISS, and Weaviate all store vectors in **Euclidean or cosine space**. This works fine for semantic similarity, but completely breaks down for **hierarchical data**:
|
|
49
|
+
|
|
50
|
+
- "Find all subsidiaries of Apple" → **0% recall** in ChromaDB
|
|
51
|
+
- "What is the common parent company of these two entities?" → **Impossible** in flat databases
|
|
52
|
+
- "Traverse the ownership tree from leaf to root" → **Not a concept** that exists
|
|
53
|
+
|
|
54
|
+
### GarudDB's Solution: Poincaré Geometry
|
|
55
|
+
The Poincaré ball model has a crucial mathematical property: **exponentially more space near the boundary**. This perfectly mirrors how hierarchies work — one root, exponentially more leaves.
|
|
56
|
+
|
|
57
|
+
In GarudDB:
|
|
58
|
+
- **Root nodes** (parent companies, chapter headers) → positioned near the **origin** of the disk
|
|
59
|
+
- **Leaf nodes** (subsidiaries, paragraph text) → positioned near the **boundary** of the disk
|
|
60
|
+
- **Tree distance** = **geodesic distance** along the curved Poincaré manifold
|
|
61
|
+
|
|
62
|
+
This means tree traversal is just geometry. No graph database needed.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## 📊 Benchmark Summary
|
|
67
|
+
|
|
68
|
+
Built on the global GLEIF dataset (3.3M+ corporate entities):
|
|
69
|
+
|
|
70
|
+
| Metric | ChromaDB | GarudDB | Advantage |
|
|
71
|
+
|:---|:---|:---|:---|
|
|
72
|
+
| RAG Search Latency | 2.49 ms | **0.06 ms** | **~40x faster** |
|
|
73
|
+
| Max Scale (8GB RAM) | ~414k vectors | **2M+ vectors** | **5x more** |
|
|
74
|
+
| Recall@10 (Flat Text) | 10.2% | 11.2% | +1pp |
|
|
75
|
+
| Ownership Discovery R@10 | 0.0% | **26.1%** | **∞** |
|
|
76
|
+
| Deep Traversal R@10 | 0.0% | **14.0%** | **∞** |
|
|
77
|
+
| Metadata Filter Latency | 35–80 ms | **6–15 ms** | **~4x faster** |
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## 🔌 Pluggable Embedding Functions
|
|
82
|
+
|
|
83
|
+
GarudDB is embedding-model agnostic. Use whatever model you want:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
# Option 1: Default — SentenceTransformer (384D, Euclidean, cosine metric)
|
|
87
|
+
collection = client.create_collection(
|
|
88
|
+
"docs",
|
|
89
|
+
embedding_function=garuddb.DefaultEmbeddingFunction()
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Option 2: Native — GarudEmbed V3 (64D, real Poincaré vectors)
|
|
93
|
+
# Trained for 60-75 epochs on hierarchical data — the gold standard
|
|
94
|
+
collection = client.create_collection(
|
|
95
|
+
"companies",
|
|
96
|
+
embedding_function=garuddb.GarudEmbedFunction()
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Option 3: Custom — Any model that returns vectors
|
|
100
|
+
class MyEmbedder:
|
|
101
|
+
dimension = 768
|
|
102
|
+
is_hyperbolic = False
|
|
103
|
+
def __call__(self, texts):
|
|
104
|
+
return my_model.encode(texts) # Your model here
|
|
105
|
+
|
|
106
|
+
collection = client.create_collection("custom", embedding_function=MyEmbedder())
|
|
107
|
+
|
|
108
|
+
# Option 4: Pre-computed vectors — skip embedding entirely
|
|
109
|
+
collection.add(embeddings=[[0.1, 0.2, ...]], ids=["1"])
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## 🧭 Topology Search API (The Moat)
|
|
115
|
+
|
|
116
|
+
These methods only exist in GarudDB. No other vector database can do this:
|
|
117
|
+
|
|
118
|
+
### `search_neighbors()` — Geodesic Translation
|
|
119
|
+
Navigate the Poincaré disk directionally:
|
|
120
|
+
```python
|
|
121
|
+
# Find parents/grandparents (move INWARD toward disk origin)
|
|
122
|
+
parents = collection.search_neighbors("Braeburn Capital", translation_mode="inward")
|
|
123
|
+
|
|
124
|
+
# Find children/subsidiaries (move OUTWARD toward disk boundary)
|
|
125
|
+
children = collection.search_neighbors("Apple Inc.", translation_mode="outward")
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### `find_subtree()` — Deep Hierarchy Traversal
|
|
129
|
+
Find all entities within a topological branch:
|
|
130
|
+
```python
|
|
131
|
+
subtree = collection.find_subtree("Apple Inc.", n_results=20)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### `find_common_ancestor()` — Möbius Midpoint
|
|
135
|
+
Find the nearest common parent of two entities:
|
|
136
|
+
```python
|
|
137
|
+
ancestor = collection.find_common_ancestor("Entity A", "Entity B")
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### `query_filtered()` — Native C++ Bitmask Filtering
|
|
141
|
+
Hardware-accelerated metadata filtering inside the HNSW graph:
|
|
142
|
+
```python
|
|
143
|
+
results = collection.query_filtered(
|
|
144
|
+
"Apple Inc.",
|
|
145
|
+
country="US",
|
|
146
|
+
status="ACTIVE",
|
|
147
|
+
n_results=10
|
|
148
|
+
)
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## 🛠️ Installation & Setup
|
|
154
|
+
|
|
155
|
+
### 1. Install dependencies
|
|
156
|
+
```bash
|
|
157
|
+
pip install -r requirements.txt
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### 2. Build the C++ engine
|
|
161
|
+
```bash
|
|
162
|
+
pip install -e .
|
|
163
|
+
```
|
|
164
|
+
This compiles `hyperbolic_index.cpp` into the `adk_core` Python extension with AVX2 SIMD acceleration.
|
|
165
|
+
|
|
166
|
+
### 3. Optional extras
|
|
167
|
+
```bash
|
|
168
|
+
pip install garuddb[embed] # GarudEmbed V3 (PyTorch + Geoopt)
|
|
169
|
+
pip install garuddb[adapter] # GarudAdapter migration (SentenceTransformers)
|
|
170
|
+
pip install garuddb[server] # FastAPI server
|
|
171
|
+
pip install garuddb[all] # Everything
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## 🏛️ Architecture
|
|
177
|
+
|
|
178
|
+
### C++ Engine (`adk_core`)
|
|
179
|
+
The heart of GarudDB — a custom HNSW implementation written in C++ with:
|
|
180
|
+
- **Three distance kernels:** Poincaré geodesic, cosine, Euclidean — all SIMD-accelerated (AVX2)
|
|
181
|
+
- **Multi-layer HNSW:** Proper O(log N) ingest and search with geometric random level assignment
|
|
182
|
+
- **64-bit in-graph bitmask filtering:** Metadata filters evaluated at each HNSW node during traversal
|
|
183
|
+
- **Dual Index Architecture:** Dense HNSW (norm < 0.995) + flat scan for boundary singletons (norm ≥ 0.995)
|
|
184
|
+
- **GIL release:** Python threads run concurrently during C++ execution
|
|
185
|
+
|
|
186
|
+
### Python SDK (`garuddb/`)
|
|
187
|
+
```
|
|
188
|
+
garuddb/
|
|
189
|
+
├── __init__.py # Public API exports
|
|
190
|
+
├── client.py # EphemeralClient, PersistentClient
|
|
191
|
+
├── collection.py # Collection API (add, query, search_neighbors, etc.)
|
|
192
|
+
├── embeddings.py # Pluggable embedding functions
|
|
193
|
+
├── cli.py # Command-line interface
|
|
194
|
+
├── engine/
|
|
195
|
+
│ ├── index.py # TriIndex wrapper (dense + singleton + semantic)
|
|
196
|
+
│ ├── concurrency.py # Read-Write lock for thread safety
|
|
197
|
+
│ └── weights.py # GarudEmbed checkpoint downloader
|
|
198
|
+
└── server/ # FastAPI server (optional)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### Tri-Index Architecture
|
|
202
|
+
Every collection maintains three C++ indices:
|
|
203
|
+
|
|
204
|
+
| Index | Metric | Purpose |
|
|
205
|
+
|:---|:---|:---|
|
|
206
|
+
| `dense_index` | Poincaré | HNSW graph for multi-hop topology traversal |
|
|
207
|
+
| `singleton_index` | Poincaré | Flat scan for boundary nodes (no HNSW edges) |
|
|
208
|
+
| `semantic_index` | Cosine | Text-to-entity anchor resolution |
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## 🧠 Mathematical Features
|
|
213
|
+
|
|
214
|
+
### 1. Geodesic Translation
|
|
215
|
+
Before searching, the query vector is shifted along the Poincaré geodesic:
|
|
216
|
+
- **Inward (× 0.92):** Traverse toward parent nodes near the disk origin
|
|
217
|
+
- **Outward (→ 0.995):** Traverse toward leaf subsidiaries near the disk boundary
|
|
218
|
+
|
|
219
|
+
### 2. Geometric Density Filter
|
|
220
|
+
Ported from the battle-tested `garud_search_engine.py`. Prevents results from bleeding across tree branches:
|
|
221
|
+
- **Gap threshold (0.6):** If consecutive results have a distance jump > 0.6, we've crossed a branch boundary — stop
|
|
222
|
+
- **Absolute cutoff (2.0):** Never return results beyond structural distance 2.0
|
|
223
|
+
|
|
224
|
+
### 3. 64-Bit In-Graph Bitmask Filtering
|
|
225
|
+
Metadata is encoded as `uint64_t` bitmasks at ingest time. Filters are evaluated with hardware `AND` inside the HNSW traversal — no post-filtering. Filter selectivity has zero impact on latency.
|
|
226
|
+
|
|
227
|
+
### 4. Multi-Hop Gradient Traversal
|
|
228
|
+
After initial HNSW search, the engine re-searches from the nearest non-trivial neighbor to expand the retrieval neighborhood. This walks along HNSW graph edges, which in Poincaré space correspond to actual tree topology links.
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
|
|
232
|
+
## 📁 Project Structure
|
|
233
|
+
|
|
234
|
+
| Directory | Purpose |
|
|
235
|
+
|:---|:---|
|
|
236
|
+
| `garuddb/` | **Core SDK** — the pip-installable package |
|
|
237
|
+
| `hyperbolic_index.cpp` | C++ HNSW engine source |
|
|
238
|
+
| `garud_embed_model.py` | GarudEmbed V3 model architecture |
|
|
239
|
+
| `train_garud_embed.py` | Training script for GarudEmbed |
|
|
240
|
+
| `garud_adapter.py` | Euclidean → GarudDB migration tool |
|
|
241
|
+
| `garud_search_engine.py` | Reference implementation (corporate graph) |
|
|
242
|
+
| `examples/` | Example applications |
|
|
243
|
+
| `garud_ingest/` | 6-layer ingestion pipeline |
|
|
244
|
+
|
|
245
|
+
---
|
|
246
|
+
|
|
247
|
+
## 🚀 Roadmap
|
|
248
|
+
|
|
249
|
+
- **v0.1.0:** Core SDK release (`pip install garuddb`) with pluggable embeddings
|
|
250
|
+
- **v0.2.0:** Built-in data connectors (CSV, JSON, PDF parsing into hierarchy)
|
|
251
|
+
- **v0.3.0:** LLM Intent Routing for natural language queries
|
|
252
|
+
- **v1.0.0:** GPU-accelerated CUDA kernel for billion-scale traversal
|
|
253
|
+
|
|
254
|
+
---
|
|
255
|
+
|
|
256
|
+
## 📜 License
|
|
257
|
+
|
|
258
|
+
Proprietary — see LICENSE file for details.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
GarudDB — Hyperbolic Vector Database for Hierarchical Data
|
|
3
|
+
==========================================================
|
|
4
|
+
|
|
5
|
+
A high-performance vector database built on Poincaré geometry,
|
|
6
|
+
optimized for corporate ownership trees, knowledge graphs,
|
|
7
|
+
and any data with natural parent-child hierarchies.
|
|
8
|
+
|
|
9
|
+
Usage:
|
|
10
|
+
import garuddb
|
|
11
|
+
|
|
12
|
+
# In-memory (ephemeral) — great for testing
|
|
13
|
+
client = garuddb.EphemeralClient()
|
|
14
|
+
|
|
15
|
+
# Persistent — syncs to local disk
|
|
16
|
+
client = garuddb.PersistentClient(path="./my_data")
|
|
17
|
+
|
|
18
|
+
# Create a collection with auto-embedding
|
|
19
|
+
from garuddb.embeddings import DefaultEmbeddingFunction
|
|
20
|
+
collection = client.get_or_create_collection(
|
|
21
|
+
"companies",
|
|
22
|
+
embedding_function=DefaultEmbeddingFunction()
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
# Add documents (auto-embedded)
|
|
26
|
+
collection.add(
|
|
27
|
+
documents=["Apple Inc.", "Braeburn Capital Inc."],
|
|
28
|
+
metadatas=[{"country": "US"}, {"country": "US"}],
|
|
29
|
+
ids=["lei_001", "lei_002"]
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# Query (auto-embedded)
|
|
33
|
+
results = collection.query(query_texts=["Apple"], n_results=5)
|
|
34
|
+
|
|
35
|
+
# Or bring your own Poincaré vectors
|
|
36
|
+
collection.add(
|
|
37
|
+
embeddings=[[0.1, 0.2, ...], [0.3, 0.4, ...]],
|
|
38
|
+
ids=["custom_001", "custom_002"]
|
|
39
|
+
)
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
__version__ = "0.1.0"
|
|
43
|
+
__author__ = "GarudDB Contributors"
|
|
44
|
+
|
|
45
|
+
from garuddb.client import EphemeralClient, PersistentClient
|
|
46
|
+
from garuddb.embeddings import (
|
|
47
|
+
EmbeddingFunction,
|
|
48
|
+
DefaultEmbeddingFunction,
|
|
49
|
+
GarudEmbedFunction,
|
|
50
|
+
GarudEmbedV4Function,
|
|
51
|
+
GarudEmbedV5Function,
|
|
52
|
+
)
|
|
53
|
+
from garuddb.extractor import (
|
|
54
|
+
CSVRelationExtractor,
|
|
55
|
+
OllamaRelationExtractor,
|
|
56
|
+
REBELRelationExtractor,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
__all__ = [
|
|
60
|
+
"EphemeralClient",
|
|
61
|
+
"PersistentClient",
|
|
62
|
+
"EmbeddingFunction",
|
|
63
|
+
"DefaultEmbeddingFunction",
|
|
64
|
+
"GarudEmbedFunction",
|
|
65
|
+
"GarudEmbedV4Function",
|
|
66
|
+
"GarudEmbedV5Function",
|
|
67
|
+
"CSVRelationExtractor",
|
|
68
|
+
"OllamaRelationExtractor",
|
|
69
|
+
"REBELRelationExtractor",
|
|
70
|
+
"__version__",
|
|
71
|
+
]
|
|
72
|
+
|