kodachi 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kodachi-0.1.0/.gitignore +32 -0
- kodachi-0.1.0/LICENSE +21 -0
- kodachi-0.1.0/PKG-INFO +201 -0
- kodachi-0.1.0/README.ja.md +178 -0
- kodachi-0.1.0/README.md +165 -0
- kodachi-0.1.0/kodachi/__init__.py +6 -0
- kodachi-0.1.0/kodachi/main.py +288 -0
- kodachi-0.1.0/kodachi/py.typed +0 -0
- kodachi-0.1.0/pyproject.toml +53 -0
- kodachi-0.1.0/tests/test_index.py +144 -0
kodachi-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
|
|
7
|
+
# Build
|
|
8
|
+
build/
|
|
9
|
+
dist/
|
|
10
|
+
|
|
11
|
+
# Virtual environments
|
|
12
|
+
.venv/
|
|
13
|
+
venv/
|
|
14
|
+
env/
|
|
15
|
+
|
|
16
|
+
# Docs
|
|
17
|
+
docs/_build/
|
|
18
|
+
|
|
19
|
+
# Testing / tooling
|
|
20
|
+
.pytest_cache/
|
|
21
|
+
.coverage
|
|
22
|
+
htmlcov/
|
|
23
|
+
.mypy_cache/
|
|
24
|
+
.ruff_cache/
|
|
25
|
+
|
|
26
|
+
# Editor
|
|
27
|
+
.vscode/
|
|
28
|
+
.idea/
|
|
29
|
+
|
|
30
|
+
# OS
|
|
31
|
+
.DS_Store
|
|
32
|
+
Thumbs.db
|
kodachi-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 yuzujelly2222
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
kodachi-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: kodachi
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Approximate nearest neighbor search with random projection trees.
|
|
5
|
+
Project-URL: Homepage, https://github.com/yuzujelly2222/kodachi
|
|
6
|
+
Project-URL: Documentation, https://yuzujelly2222.github.io/kodachi/
|
|
7
|
+
Project-URL: Repository, https://github.com/yuzujelly2222/kodachi
|
|
8
|
+
Project-URL: Issues, https://github.com/yuzujelly2222/kodachi/issues
|
|
9
|
+
Author: yuzujelly2222
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ann,cosine-similarity,nearest-neighbor,random-projection,vector-search
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: numpy>=1.23
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: build; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
30
|
+
Requires-Dist: twine; extra == 'dev'
|
|
31
|
+
Provides-Extra: docs
|
|
32
|
+
Requires-Dist: furo; extra == 'docs'
|
|
33
|
+
Requires-Dist: myst-parser; extra == 'docs'
|
|
34
|
+
Requires-Dist: sphinx>=7; extra == 'docs'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
[English](https://github.com/yuzujelly2222/kodachi/blob/main/README.md) | [日本語](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md)
|
|
38
|
+
|
|
39
|
+
# kodachi
|
|
40
|
+
|
|
41
|
+
A modern, pure-Python ANN library built on a forest of random projection trees.
|
|
42
|
+
|
|
43
|
+
[](https://pypi.org/project/kodachi/)
|
|
44
|
+
[](https://github.com/yuzujelly2222/kodachi/blob/main/LICENSE)
|
|
45
|
+
[](https://www.python.org/)
|
|
46
|
+
[](https://github.com/yuzujelly2222/kodachi/actions/workflows/tests.yml)
|
|
47
|
+
|
|
48
|
+
> 詳しい日本語の解説は [README.ja.md](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md) にあります。
|
|
49
|
+
|
|
50
|
+
## What is this
|
|
51
|
+
|
|
52
|
+
kodachi is an approximate nearest neighbor (ANN) search library built on a forest of
|
|
53
|
+
random projection trees, written in pure Python with NumPy as its only dependency.
|
|
54
|
+
|
|
55
|
+
Graph-based methods such as HNSW (Hierarchical Navigable Small World) dominate modern
|
|
56
|
+
ANN and are faster. Random projection forests, on the other hand, generally use less
|
|
57
|
+
memory and are simpler to understand. kodachi favors ease of installation (no C
|
|
58
|
+
extensions, works anywhere) and readable code over raw speed.
|
|
59
|
+
|
|
60
|
+
Input is a plain numeric array of shape `(n_points, dim)`, so the vectors can come from
|
|
61
|
+
anywhere: text or image embeddings, feature vectors, user/item vectors for
|
|
62
|
+
recommendation — anything you want to search by cosine similarity.
|
|
63
|
+
|
|
64
|
+
The whole implementation fits in a single file, `kodachi/main.py` (about 300 lines),
|
|
65
|
+
so it also works as a readable reference for the algorithm.
|
|
66
|
+
|
|
67
|
+
## Install
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install kodachi
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Depends only on `numpy`. Requires Python 3.10+.
|
|
74
|
+
|
|
75
|
+
## Usage
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import numpy as np
|
|
79
|
+
from kodachi import Index
|
|
80
|
+
|
|
81
|
+
# Vectors to index (e.g. 10,000 vectors with 64 dimensions)
|
|
82
|
+
data = np.random.default_rng(0).normal(size=(10000, 64))
|
|
83
|
+
|
|
84
|
+
index = Index(dim=64, n_trees=20, seed=42)
|
|
85
|
+
index.build_index(data)
|
|
86
|
+
|
|
87
|
+
# Indices of the top 10 neighbors, sorted by cosine similarity
|
|
88
|
+
neighbors = index.search(data[0], k=10)
|
|
89
|
+
print(neighbors) # -> [0 ...] (the query itself ranks first)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Similarity search over embeddings
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
texts = ["I like apples", "I like oranges", "I commute by train", ...]
|
|
96
|
+
|
|
97
|
+
embeddings = embed(texts) # any embedding model, shape (len(texts), dim)
|
|
98
|
+
|
|
99
|
+
index = Index(dim=embeddings.shape[1], n_trees=30, seed=0)
|
|
100
|
+
index.build_index(embeddings)
|
|
101
|
+
|
|
102
|
+
ids = index.search(embed(["I like fruit"])[0], k=3)
|
|
103
|
+
print([texts[i] for i in ids])
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Trading accuracy for speed
|
|
107
|
+
|
|
108
|
+
Raising `search_k` (the number of candidates collected before ranking) improves recall
|
|
109
|
+
at the cost of speed.
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
index.search(query, k=10) # default: search_k = k * n_trees * 10
|
|
113
|
+
index.search(query, k=10, search_k=5000) # more candidates, higher recall
|
|
114
|
+
index.search(query, k=10, search_k=len(data)) # >= n_points: exact brute-force search
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## How it works
|
|
118
|
+
|
|
119
|
+
**Building** (`build_index` -> `_make_tree`)
|
|
120
|
+
|
|
121
|
+
1. **Normalize** all data to unit length, so splitting by Euclidean distance gives the
|
|
122
|
+
same ranking as cosine similarity.
|
|
123
|
+
2. **Split** by picking two random points and cutting the space with the hyperplane
|
|
124
|
+
equidistant from them. Points exactly on the hyperplane are split evenly.
|
|
125
|
+
3. **Recurse** on each side until it holds at most `max_leaf_size` (default 32) points.
|
|
126
|
+
4. Repeat `n_trees` times to get a forest of trees that each split the space differently.
|
|
127
|
+
|
|
128
|
+
**Searching** (`search` -> `_collect_candidates` -> `down_node`)
|
|
129
|
+
|
|
130
|
+
1. In each tree, descend straight to the leaf on the query's side. Branches not taken
|
|
131
|
+
are pushed onto a priority queue.
|
|
132
|
+
2. Until `search_k` candidates are collected, explore deferred branches in order of how
|
|
133
|
+
close the query was to their hyperplane.
|
|
134
|
+
3. Rank all candidates by exact cosine similarity and return the top `k`.
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
data -> normalize -> recursively split by random bisecting hyperplanes -> n_trees trees
|
|
138
|
+
|
|
139
|
+
query -> normalize -> descend each tree to a leaf (stage 1)
|
|
140
|
+
-> explore branches closest to their hyperplane first (stage 2, up to search_k)
|
|
141
|
+
-> return the top k candidates by cosine similarity
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Let `n` be the number of points, `d` the dimensionality, and `T` the number of trees.
|
|
145
|
+
Building takes roughly `O(T · n · d · log n)`; searching takes `O(T · d · log n)` plus
|
|
146
|
+
`O(search_k · d)` to rank candidates.
|
|
147
|
+
|
|
148
|
+
On 20,000 random 32-dimensional points, `n_trees=20, search_k=4000` reaches recall@10 of
|
|
149
|
+
0.95 at 3.5 ms/query — but NumPy brute force takes only 0.6 ms at this scale. Trees pay
|
|
150
|
+
off with hundreds of thousands of points or more and with structured data such as real
|
|
151
|
+
embeddings. Start with `n_trees` around 20–50 and tune `search_k` on your own data. See
|
|
152
|
+
[README.ja.md](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md) for the
|
|
153
|
+
full benchmark table.
|
|
154
|
+
|
|
155
|
+
## API
|
|
156
|
+
|
|
157
|
+
Only the members below are part of the public API; `_Node`, `_Helper`, `_make_tree`,
|
|
158
|
+
and `_collect_candidates` are internal.
|
|
159
|
+
|
|
160
|
+
| Member | Description |
|
|
161
|
+
|---|---|
|
|
162
|
+
| `Index(dim, n_trees=10, max_leaf_size=32, seed=None)` | Create an empty index. Fix `seed` for reproducible trees |
|
|
163
|
+
| `build_index(data)` | Build the forest from an array of shape `(n_points, dim)`. Calling it again rebuilds from scratch |
|
|
164
|
+
| `search(query, k=10, search_k=None)` | Return up to `k` data indices for a `(dim,)` query, sorted by descending cosine similarity |
|
|
165
|
+
| `down_node(heap, counter, node, query, space)` | Low-level helper that descends one branch to a leaf (rarely needed directly) |
|
|
166
|
+
|
|
167
|
+
Invalid input raises:
|
|
168
|
+
|
|
169
|
+
- `ValueError` if `build_index` gets an array that is not 2-D or whose width is not `dim`
|
|
170
|
+
- `ValueError` if `search` gets a query whose shape is not `(dim,)`
|
|
171
|
+
- `RuntimeError` if `search` is called before `build_index`
|
|
172
|
+
|
|
173
|
+
If `k` exceeds the number of points, all points are returned in similarity order.
|
|
174
|
+
|
|
175
|
+
## Documentation & tests
|
|
176
|
+
|
|
177
|
+
The API reference is published at <https://yuzujelly2222.github.io/kodachi/>.
|
|
178
|
+
Its Sphinx source lives in `docs/` and is deployed to GitHub Pages on every push to
|
|
179
|
+
`main`. To build it locally:
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
pip install -e ".[docs]"
|
|
183
|
+
sphinx-build docs docs/_build/html
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
A pytest suite lives in `tests/` and runs on GitHub Actions across Windows / macOS /
|
|
187
|
+
Linux and Python 3.10–3.13:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
pip install -e ".[dev]"
|
|
191
|
+
pytest
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## License & credits
|
|
195
|
+
|
|
196
|
+
Distributed under the **MIT License** — see
|
|
197
|
+
[LICENSE](https://github.com/yuzujelly2222/kodachi/blob/main/LICENSE).
|
|
198
|
+
|
|
199
|
+
The algorithm is an independent Python implementation inspired by Spotify's
|
|
200
|
+
[Annoy](https://github.com/spotify/annoy) by Erik Bernhardsson: a forest of trees split
|
|
201
|
+
by random bisecting hyperplanes, searched with a priority queue. No Annoy code is included.
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
[English](https://github.com/yuzujelly2222/kodachi/blob/main/README.md) | [日本語](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md)
|
|
2
|
+
|
|
3
|
+
# kodachi
|
|
4
|
+
|
|
5
|
+
ランダム射影の森を使ったモダンなANNライブラリのPythonネイティブ実装
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/kodachi/)
|
|
8
|
+
[](https://github.com/yuzujelly2222/kodachi/blob/main/LICENSE)
|
|
9
|
+
[](https://www.python.org/)
|
|
10
|
+
[](https://github.com/yuzujelly2222/kodachi/actions/workflows/tests.yml)
|
|
11
|
+
|
|
12
|
+
## これは何か
|
|
13
|
+
|
|
14
|
+
kodachi は、ランダム射影木の森による近似最近傍探索(ANN)を、NumPy だけに依存する純粋な Python で実装したライブラリです。
|
|
15
|
+
最近の ANN では HNSW(階層的航行可能スモールワールド)のようなグラフ型の手法が主流で、速度ではそちらが優れています。一方で、ランダム射影の森は一般にメモリ使用量が少なく、仕組みもシンプルです。
|
|
16
|
+
kodachi は速度よりも、C 拡張なしでどこでもすぐ入る手軽さと、コードの読みやすさを重視しています。
|
|
17
|
+
|
|
18
|
+
`build_index()` / `search()` に渡すのは `(点の数, 次元数)` の数値配列だけなので、ベクトルの出所は問いません。
|
|
19
|
+
文章や画像の埋め込みベクトル、特徴量、推薦用のユーザー・アイテムベクトルなど、コサイン類似度で
|
|
20
|
+
「似ているもの」を探したい場面であればそのまま使えます。
|
|
21
|
+
|
|
22
|
+
本体は `kodachi/main.py` の1ファイル(約300行)にまとまっており、アルゴリズムを読んで学ぶ教材としても使えます。
|
|
23
|
+
|
|
24
|
+
## インストール
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install kodachi
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
依存パッケージは `numpy` のみです。Python 3.10 以上が必要です。
|
|
31
|
+
|
|
32
|
+
## 使い方
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
import numpy as np
|
|
36
|
+
from kodachi import Index
|
|
37
|
+
|
|
38
|
+
# インデックスに登録するベクトル(例: 10,000 個の 64 次元ベクトル)
|
|
39
|
+
data = np.random.default_rng(0).normal(size=(10000, 64))
|
|
40
|
+
|
|
41
|
+
index = Index(dim=64, n_trees=20, seed=42)
|
|
42
|
+
index.build_index(data)
|
|
43
|
+
|
|
44
|
+
# クエリに近い上位 10 件のデータ番号(コサイン類似度の高い順)
|
|
45
|
+
neighbors = index.search(data[0], k=10)
|
|
46
|
+
print(neighbors) # -> [0 ...](自分自身が 1 位)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### 埋め込みベクトルで類似検索する例
|
|
50
|
+
|
|
51
|
+
ベクトルの出所は問わないため、任意のモデルで作った埋め込みをそのまま渡すだけで使えます。
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
texts = ["りんごが好き", "みかんが好き", "電車で通勤する", ...]
|
|
55
|
+
|
|
56
|
+
# 埋め込みの作り方は用途に応じて用意する(sentence-transformers など)
|
|
57
|
+
embeddings = embed(texts) # 形は (len(texts), dim)
|
|
58
|
+
|
|
59
|
+
index = Index(dim=embeddings.shape[1], n_trees=30, seed=0)
|
|
60
|
+
index.build_index(embeddings)
|
|
61
|
+
|
|
62
|
+
ids = index.search(embed(["果物が好き"])[0], k=3)
|
|
63
|
+
print([texts[i] for i in ids])
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### 精度と速度の調整
|
|
67
|
+
|
|
68
|
+
`search_k`(順位付けの前に集める候補の数)を増やすと recall が上がり、その分遅くなります。
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
index.search(query, k=10) # 既定: search_k = k * n_trees * 10
|
|
72
|
+
index.search(query, k=10, search_k=5000) # 候補を増やして精度を上げる
|
|
73
|
+
index.search(query, k=10, search_k=len(data)) # データ数以上なら総当たりで正確に計算
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## アルゴリズム解説
|
|
77
|
+
|
|
78
|
+
kodachi はデータをランダムな超平面で何度も 2 つに分けた木を複数本作り、検索時は
|
|
79
|
+
クエリが入る葉と、その近くの葉だけを調べることで計算を減らします。
|
|
80
|
+
|
|
81
|
+
### 木を作る(`build_index` → `_make_tree`)
|
|
82
|
+
|
|
83
|
+
1. **正規化** — 全データを長さ 1 に正規化する。正規化済みならユークリッド距離で分けても、
|
|
84
|
+
順位がコサイン類似度と一致する。
|
|
85
|
+
2. **分割** — 点の集合からランダムに 2 点を選び、その 2 点からちょうど等距離にある超平面
|
|
86
|
+
(`_Helper.make_offset_and_normal`)で空間を 2 つに分ける。面のちょうど上にある点は左右に半分ずつ振り分ける。
|
|
87
|
+
3. **再帰** — 点の数が `max_leaf_size`(既定 32)以下になるまで、左右それぞれで 2 を繰り返す。
|
|
88
|
+
4. これを `n_trees` 回行い、少しずつ違う分け方をした木の森を作る。
|
|
89
|
+
|
|
90
|
+
### 検索する(`search` → `_collect_candidates` → `down_node`)
|
|
91
|
+
|
|
92
|
+
1. **第1段階** — 各木でクエリがある側にまっすぐ降りて、葉を 1 枚ずつ集める。
|
|
93
|
+
通らなかった側の枝は優先度付きキューに後回しとして入れておく。
|
|
94
|
+
2. **第2段階** — 候補が `search_k` 個たまるまで、後回しにした枝を「クエリが超平面にどれだけ近かったか」の順に探す。
|
|
95
|
+
超平面に近いほど、反対側にも近い点がある可能性が高いため。
|
|
96
|
+
3. **順位付け** — 集めた候補とクエリのコサイン類似度をまとめて計算し、上位 `k` 個を返す。
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
データ → 正規化 → ランダムな2点の垂直二等分面で再帰的に分割 → n_trees 本の木
|
|
100
|
+
|
|
101
|
+
クエリ → 正規化 → 各木で葉まで降りる(第1段階)
|
|
102
|
+
→ 超平面に近かった枝から優先的に探索(第2段階、search_k 個まで)
|
|
103
|
+
→ 候補とのコサイン類似度で上位 k 件を返す
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### 計算量と目安
|
|
107
|
+
|
|
108
|
+
`n` を点の数、`d` を次元数、`T` を木の本数とします。
|
|
109
|
+
|
|
110
|
+
- **構築:** およそ `O(T · n · d · log n)`
|
|
111
|
+
- **検索:** 木を降りる部分が `O(T · d · log n)`、候補の順位付けが `O(search_k · d)`
|
|
112
|
+
- **メモリ:** データ本体の `n · d` に加えて、木 1 本あたり内部ノード(およそ `n / max_leaf_size` 個)× `d` の法線ベクトル
|
|
113
|
+
|
|
114
|
+
20,000 点 × 32 次元のランダムなデータでの測定結果:
|
|
115
|
+
|
|
116
|
+
| n_trees | search_k | recall@10 | ms/クエリ |
|
|
117
|
+
|---|---|---|---|
|
|
118
|
+
| 10 | 2,000 | 0.76 | 1.7 |
|
|
119
|
+
| 20 | 4,000 | 0.95 | 3.5 |
|
|
120
|
+
| 50 | 5,000 | 0.98 | 5.0 |
|
|
121
|
+
| 総当たり | — | 1.00 | 0.6 |
|
|
122
|
+
|
|
123
|
+
この規模では NumPy による総当たりのほうが速いことに注意してください。ランダムなデータは近傍探索にとって
|
|
124
|
+
最も難しいケースで、さらに点の数も少ないためです。木を使う効果が出るのは、点の数が数十万以上で、
|
|
125
|
+
実際の埋め込みベクトルのように偏りのあるデータです。
|
|
126
|
+
|
|
127
|
+
`n_trees` は 20〜50 くらいから始めて、実際に使うデータで「目標の recall を満たす最小の `search_k`」を測って決めるのがおすすめです。
|
|
128
|
+
|
|
129
|
+
## API
|
|
130
|
+
|
|
131
|
+
### `Index`
|
|
132
|
+
|
|
133
|
+
外部から使うのは以下の公開メンバのみです。`_Node`(木のノード)、`_Helper`(幾何計算)、
|
|
134
|
+
`_make_tree`、`_collect_candidates` は内部実装用のため `_` を付けて非公開にしています。
|
|
135
|
+
|
|
136
|
+
| メンバ | 説明 |
|
|
137
|
+
|---|---|
|
|
138
|
+
| `Index(dim, n_trees=10, max_leaf_size=32, seed=None)` | 空のインデックスを作る。`seed` を固定すると毎回同じ木ができる |
|
|
139
|
+
| `build_index(data)` | `(点の数, dim)` の配列から木を作る。もう一度呼ぶと前の木は捨てて作り直す |
|
|
140
|
+
| `search(query, k=10, search_k=None)` | `(dim,)` のクエリに対し、コサイン類似度の高い順に最大 `k` 個のデータ番号を返す |
|
|
141
|
+
| `down_node(heap, counter, node, query, space)` | 1 本の枝を葉まで降りる低レベル関数(通常は直接使わない) |
|
|
142
|
+
|
|
143
|
+
### バリデーション
|
|
144
|
+
|
|
145
|
+
不正な入力は次の例外で弾きます。
|
|
146
|
+
|
|
147
|
+
- `build_index` に 2 次元でない配列、または列数が `dim` と違う配列を渡した → `ValueError`
|
|
148
|
+
- `search` に形が `(dim,)` でないクエリを渡した → `ValueError`
|
|
149
|
+
- `build_index` を呼ぶ前に `search` を呼んだ → `RuntimeError`
|
|
150
|
+
|
|
151
|
+
`k` がデータ数より大きい場合はエラーにせず、全データを類似度順に返します。
|
|
152
|
+
|
|
153
|
+
## ドキュメント・テスト
|
|
154
|
+
|
|
155
|
+
API リファレンスは <https://yuzujelly2222.github.io/kodachi/> で公開しています。
|
|
156
|
+
Sphinx のソースが `docs/` にあり、`main` に push するたびに GitHub Actions で GitHub Pages に自動公開されます。
|
|
157
|
+
ローカルでビルドする場合:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
pip install -e ".[docs]"
|
|
161
|
+
sphinx-build docs docs/_build/html
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
`tests/` には pytest ベースのテストスイートがあり、GitHub Actions で Windows / macOS / Linux ×
|
|
165
|
+
Python 3.10〜3.13 の組み合わせで実行しています。
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
pip install -e ".[dev]"
|
|
169
|
+
pytest
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## ライセンスと出典
|
|
173
|
+
|
|
174
|
+
本リポジトリのコードは MIT License([LICENSE](https://github.com/yuzujelly2222/kodachi/blob/main/LICENSE))で公開しています。
|
|
175
|
+
|
|
176
|
+
アルゴリズムは Spotify の [Annoy](https://github.com/spotify/annoy)(Erik Bernhardsson 作)で使われている、
|
|
177
|
+
ランダムな 2 点の垂直二等分面で分割する木の森と、優先度付きキューによる探索の考え方を参考に、
|
|
178
|
+
独自に Python で実装したものです。Annoy のコードは含んでいません。
|
kodachi-0.1.0/README.md
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
[English](https://github.com/yuzujelly2222/kodachi/blob/main/README.md) | [日本語](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md)
|
|
2
|
+
|
|
3
|
+
# kodachi
|
|
4
|
+
|
|
5
|
+
A modern, pure-Python ANN library built on a forest of random projection trees.
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/kodachi/)
|
|
8
|
+
[](https://github.com/yuzujelly2222/kodachi/blob/main/LICENSE)
|
|
9
|
+
[](https://www.python.org/)
|
|
10
|
+
[](https://github.com/yuzujelly2222/kodachi/actions/workflows/tests.yml)
|
|
11
|
+
|
|
12
|
+
> 詳しい日本語の解説は [README.ja.md](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md) にあります。
|
|
13
|
+
|
|
14
|
+
## What is this
|
|
15
|
+
|
|
16
|
+
kodachi is an approximate nearest neighbor (ANN) search library built on a forest of
|
|
17
|
+
random projection trees, written in pure Python with NumPy as its only dependency.
|
|
18
|
+
|
|
19
|
+
Graph-based methods such as HNSW (Hierarchical Navigable Small World) dominate modern
|
|
20
|
+
ANN and are faster. Random projection forests, on the other hand, generally use less
|
|
21
|
+
memory and are simpler to understand. kodachi favors ease of installation (no C
|
|
22
|
+
extensions, works anywhere) and readable code over raw speed.
|
|
23
|
+
|
|
24
|
+
Input is a plain numeric array of shape `(n_points, dim)`, so the vectors can come from
|
|
25
|
+
anywhere: text or image embeddings, feature vectors, user/item vectors for
|
|
26
|
+
recommendation — anything you want to search by cosine similarity.
|
|
27
|
+
|
|
28
|
+
The whole implementation fits in a single file, `kodachi/main.py` (about 300 lines),
|
|
29
|
+
so it also works as a readable reference for the algorithm.
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install kodachi
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Depends only on `numpy`. Requires Python 3.10+.
|
|
38
|
+
|
|
39
|
+
## Usage
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import numpy as np
|
|
43
|
+
from kodachi import Index
|
|
44
|
+
|
|
45
|
+
# Vectors to index (e.g. 10,000 vectors with 64 dimensions)
|
|
46
|
+
data = np.random.default_rng(0).normal(size=(10000, 64))
|
|
47
|
+
|
|
48
|
+
index = Index(dim=64, n_trees=20, seed=42)
|
|
49
|
+
index.build_index(data)
|
|
50
|
+
|
|
51
|
+
# Indices of the top 10 neighbors, sorted by cosine similarity
|
|
52
|
+
neighbors = index.search(data[0], k=10)
|
|
53
|
+
print(neighbors) # -> [0 ...] (the query itself ranks first)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Similarity search over embeddings
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
texts = ["I like apples", "I like oranges", "I commute by train", ...]
|
|
60
|
+
|
|
61
|
+
embeddings = embed(texts) # any embedding model, shape (len(texts), dim)
|
|
62
|
+
|
|
63
|
+
index = Index(dim=embeddings.shape[1], n_trees=30, seed=0)
|
|
64
|
+
index.build_index(embeddings)
|
|
65
|
+
|
|
66
|
+
ids = index.search(embed(["I like fruit"])[0], k=3)
|
|
67
|
+
print([texts[i] for i in ids])
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Trading accuracy for speed
|
|
71
|
+
|
|
72
|
+
Raising `search_k` (the number of candidates collected before ranking) improves recall
|
|
73
|
+
at the cost of speed.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
index.search(query, k=10) # default: search_k = k * n_trees * 10
|
|
77
|
+
index.search(query, k=10, search_k=5000) # more candidates, higher recall
|
|
78
|
+
index.search(query, k=10, search_k=len(data)) # >= n_points: exact brute-force search
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## How it works
|
|
82
|
+
|
|
83
|
+
**Building** (`build_index` -> `_make_tree`)
|
|
84
|
+
|
|
85
|
+
1. **Normalize** all data to unit length, so splitting by Euclidean distance gives the
|
|
86
|
+
same ranking as cosine similarity.
|
|
87
|
+
2. **Split** by picking two random points and cutting the space with the hyperplane
|
|
88
|
+
equidistant from them. Points exactly on the hyperplane are split evenly.
|
|
89
|
+
3. **Recurse** on each side until it holds at most `max_leaf_size` (default 32) points.
|
|
90
|
+
4. Repeat `n_trees` times to get a forest of trees that each split the space differently.
|
|
91
|
+
|
|
92
|
+
**Searching** (`search` -> `_collect_candidates` -> `down_node`)
|
|
93
|
+
|
|
94
|
+
1. In each tree, descend straight to the leaf on the query's side. Branches not taken
|
|
95
|
+
are pushed onto a priority queue.
|
|
96
|
+
2. Until `search_k` candidates are collected, explore deferred branches in order of how
|
|
97
|
+
close the query was to their hyperplane.
|
|
98
|
+
3. Rank all candidates by exact cosine similarity and return the top `k`.
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
data -> normalize -> recursively split by random bisecting hyperplanes -> n_trees trees
|
|
102
|
+
|
|
103
|
+
query -> normalize -> descend each tree to a leaf (stage 1)
|
|
104
|
+
-> explore branches closest to their hyperplane first (stage 2, up to search_k)
|
|
105
|
+
-> return the top k candidates by cosine similarity
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Let `n` be the number of points, `d` the dimensionality, and `T` the number of trees.
|
|
109
|
+
Building takes roughly `O(T · n · d · log n)`; searching takes `O(T · d · log n)` plus
|
|
110
|
+
`O(search_k · d)` to rank candidates.
|
|
111
|
+
|
|
112
|
+
On 20,000 random 32-dimensional points, `n_trees=20, search_k=4000` reaches recall@10 of
|
|
113
|
+
0.95 at 3.5 ms/query — but NumPy brute force takes only 0.6 ms at this scale. Trees pay
|
|
114
|
+
off with hundreds of thousands of points or more and with structured data such as real
|
|
115
|
+
embeddings. Start with `n_trees` around 20–50 and tune `search_k` on your own data. See
|
|
116
|
+
[README.ja.md](https://github.com/yuzujelly2222/kodachi/blob/main/README.ja.md) for the
|
|
117
|
+
full benchmark table.
|
|
118
|
+
|
|
119
|
+
## API
|
|
120
|
+
|
|
121
|
+
Only the members below are part of the public API; `_Node`, `_Helper`, `_make_tree`,
|
|
122
|
+
and `_collect_candidates` are internal.
|
|
123
|
+
|
|
124
|
+
| Member | Description |
|
|
125
|
+
|---|---|
|
|
126
|
+
| `Index(dim, n_trees=10, max_leaf_size=32, seed=None)` | Create an empty index. Fix `seed` for reproducible trees |
|
|
127
|
+
| `build_index(data)` | Build the forest from an array of shape `(n_points, dim)`. Calling it again rebuilds from scratch |
|
|
128
|
+
| `search(query, k=10, search_k=None)` | Return up to `k` data indices for a `(dim,)` query, sorted by descending cosine similarity |
|
|
129
|
+
| `down_node(heap, counter, node, query, space)` | Low-level helper that descends one branch to a leaf (rarely needed directly) |
|
|
130
|
+
|
|
131
|
+
Invalid input raises:
|
|
132
|
+
|
|
133
|
+
- `ValueError` if `build_index` gets an array that is not 2-D or whose width is not `dim`
|
|
134
|
+
- `ValueError` if `search` gets a query whose shape is not `(dim,)`
|
|
135
|
+
- `RuntimeError` if `search` is called before `build_index`
|
|
136
|
+
|
|
137
|
+
If `k` exceeds the number of points, all points are returned in similarity order.
|
|
138
|
+
|
|
139
|
+
## Documentation & tests
|
|
140
|
+
|
|
141
|
+
The API reference is published at <https://yuzujelly2222.github.io/kodachi/>.
|
|
142
|
+
Its Sphinx source lives in `docs/` and is deployed to GitHub Pages on every push to
|
|
143
|
+
`main`. To build it locally:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
pip install -e ".[docs]"
|
|
147
|
+
sphinx-build docs docs/_build/html
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
A pytest suite lives in `tests/` and runs on GitHub Actions across Windows / macOS /
|
|
151
|
+
Linux and Python 3.10–3.13:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
pip install -e ".[dev]"
|
|
155
|
+
pytest
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## License & credits
|
|
159
|
+
|
|
160
|
+
Distributed under the **MIT License** — see
|
|
161
|
+
[LICENSE](https://github.com/yuzujelly2222/kodachi/blob/main/LICENSE).
|
|
162
|
+
|
|
163
|
+
The algorithm is an independent Python implementation inspired by Spotify's
|
|
164
|
+
[Annoy](https://github.com/spotify/annoy) by Erik Bernhardsson: a forest of trees split
|
|
165
|
+
by random bisecting hyperplanes, searched with a priority queue. No Annoy code is included.
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Approximate nearest neighbor search with random projection trees.
|
|
2
|
+
|
|
3
|
+
This module provides an Annoy-style index that builds a forest of random
|
|
4
|
+
hyperplane trees and searches it for the vectors with the highest cosine
|
|
5
|
+
similarity to a query.
|
|
6
|
+
|
|
7
|
+
Example:
|
|
8
|
+
>>> import numpy as np
|
|
9
|
+
>>> data = np.random.default_rng(0).normal(size=(1000, 16))
|
|
10
|
+
>>> index = Index(dim=16, n_trees=10, seed=42)
|
|
11
|
+
>>> index.build_index(data)
|
|
12
|
+
>>> index.search(data[0], k=5)
|
|
13
|
+
array([...])
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
import numpy as np
|
|
19
|
+
import heapq
|
|
20
|
+
import itertools
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(slots=True)
|
|
24
|
+
class _Node:
|
|
25
|
+
"""A node of a random projection tree.
|
|
26
|
+
|
|
27
|
+
An internal node splits space by the hyperplane ``normal · x + offset = 0``.
|
|
28
|
+
A leaf node stores the indices of the data points that fall into it.
|
|
29
|
+
|
|
30
|
+
Attributes:
|
|
31
|
+
normal: Unit normal vector of the splitting hyperplane. ``None`` for leaves.
|
|
32
|
+
offset: Bias term of the splitting hyperplane.
|
|
33
|
+
left: Child containing points with ``normal · x + offset < 0``.
|
|
34
|
+
right: Child containing points with ``normal · x + offset > 0``.
|
|
35
|
+
indices: Indices of the data points in this leaf. ``None`` for internal nodes.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
normal: np.ndarray | None = None
|
|
39
|
+
offset: float = 0.0
|
|
40
|
+
left: _Node | None = None
|
|
41
|
+
right: _Node | None = None
|
|
42
|
+
indices: np.ndarray | None = None
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def is_leaf(self) -> bool:
|
|
46
|
+
"""bool: Whether this node is a leaf."""
|
|
47
|
+
return self.indices is not None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class _Helper:
|
|
51
|
+
"""Stateless numerical helpers used by :class:`Index`."""
|
|
52
|
+
|
|
53
|
+
@staticmethod
|
|
54
|
+
def make_offset_and_normal(point_1: np.ndarray, point_2: np.ndarray) -> tuple[np.ndarray, float]:
|
|
55
|
+
"""Computes the hyperplane that equidistantly separates two points.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
point_1: First point, shape ``(dim,)``.
|
|
59
|
+
point_2: Second point, shape ``(dim,)``.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
A tuple ``(normal, offset)`` where ``normal`` is the unit vector
|
|
63
|
+
pointing from ``point_2`` to ``point_1`` and ``offset`` places the
|
|
64
|
+
hyperplane at their midpoint. If the points are identical,
|
|
65
|
+
``normal`` is the zero vector.
|
|
66
|
+
"""
|
|
67
|
+
normal = point_1 - point_2
|
|
68
|
+
norm = np.linalg.norm(normal)
|
|
69
|
+
if norm > 0:
|
|
70
|
+
normal = normal / norm
|
|
71
|
+
offset = float(np.dot(-normal, (point_1 + point_2) / 2))
|
|
72
|
+
return normal, offset
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def normalize_rows(a: np.ndarray) -> np.ndarray:
|
|
76
|
+
"""L2-normalizes each row of a 2-D array.
|
|
77
|
+
|
|
78
|
+
Rows with zero norm are left unchanged.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
a: Array of shape ``(n, dim)``.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
A new array of shape ``(n, dim)`` whose non-zero rows have unit length.
|
|
85
|
+
"""
|
|
86
|
+
norms = np.linalg.norm(a, axis=1, keepdims=True)
|
|
87
|
+
norms[norms == 0] = 1
|
|
88
|
+
return a / norms
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class Index:
|
|
92
|
+
"""Approximate nearest neighbor index based on cosine similarity.
|
|
93
|
+
|
|
94
|
+
The index builds ``n_trees`` random projection trees over L2-normalized
|
|
95
|
+
data. Searching descends every tree toward the query and then explores
|
|
96
|
+
the most promising unvisited branches until enough candidates are
|
|
97
|
+
collected, finally ranking the candidates by exact cosine similarity.
|
|
98
|
+
|
|
99
|
+
Attributes:
|
|
100
|
+
dim: Dimensionality of the vectors.
|
|
101
|
+
n_trees: Number of trees in the forest.
|
|
102
|
+
max_leaf_size: Maximum number of points stored in a single leaf.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
def __init__(self, dim: int, n_trees: int = 10, max_leaf_size: int = 32, seed: int | None = None):
|
|
106
|
+
"""Initializes an empty index.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
dim: Dimensionality of the vectors to be indexed.
|
|
110
|
+
n_trees: Number of trees to build. More trees improve recall at
|
|
111
|
+
the cost of memory and build time.
|
|
112
|
+
max_leaf_size: Maximum number of points in a leaf node.
|
|
113
|
+
seed: Seed for the random number generator. Use a fixed value
|
|
114
|
+
for reproducible trees.
|
|
115
|
+
"""
|
|
116
|
+
self.dim = dim
|
|
117
|
+
self.n_trees = n_trees
|
|
118
|
+
self.max_leaf_size = max_leaf_size
|
|
119
|
+
self._rng = np.random.default_rng(seed)
|
|
120
|
+
self._roots: list[_Node] = []
|
|
121
|
+
self._data: np.ndarray | None = None
|
|
122
|
+
|
|
123
|
+
def build_index(self, data: np.ndarray) -> None:
|
|
124
|
+
"""Builds the forest from the given data.
|
|
125
|
+
|
|
126
|
+
Any previously built trees are discarded. The data is copied, cast to
|
|
127
|
+
``float32`` and L2-normalized before indexing.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
data: Array-like of shape ``(n_points, dim)``.
|
|
131
|
+
|
|
132
|
+
Raises:
|
|
133
|
+
ValueError: If ``data`` is not 2-D or its second dimension does
|
|
134
|
+
not equal ``dim``.
|
|
135
|
+
"""
|
|
136
|
+
data = np.asarray(data, dtype=np.float32)
|
|
137
|
+
if data.ndim != 2 or data.shape[1] != self.dim:
|
|
138
|
+
raise ValueError(f"Expected data of shape (n_points, {self.dim}), but got {data.shape}.")
|
|
139
|
+
self._data = _Helper.normalize_rows(data)
|
|
140
|
+
self._roots = []
|
|
141
|
+
all_idx = np.arange(data.shape[0])
|
|
142
|
+
for _ in range(self.n_trees):
|
|
143
|
+
self._roots.append(self._make_tree(all_idx))
|
|
144
|
+
|
|
145
|
+
def _make_tree(self, idx: np.ndarray) -> _Node:
|
|
146
|
+
"""Recursively builds a tree over a subset of the data.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
idx: Indices of the data points to place in this subtree.
|
|
150
|
+
|
|
151
|
+
Returns:
|
|
152
|
+
The root node of the built subtree.
|
|
153
|
+
"""
|
|
154
|
+
node = _Node()
|
|
155
|
+
if len(idx) <= self.max_leaf_size:
|
|
156
|
+
node.indices = idx
|
|
157
|
+
return node
|
|
158
|
+
|
|
159
|
+
point_1, point_2 = self._rng.choice(idx, size=2, replace=False)
|
|
160
|
+
normal, offset = _Helper.make_offset_and_normal(self._data[point_1], self._data[point_2])
|
|
161
|
+
node.normal = normal
|
|
162
|
+
node.offset = offset
|
|
163
|
+
s = self._data[idx] @ normal + offset
|
|
164
|
+
left_mask = 0 > s
|
|
165
|
+
on_mask = 0 == s
|
|
166
|
+
right_mask = 0 < s
|
|
167
|
+
|
|
168
|
+
on = self._rng.permutation(idx[on_mask])
|
|
169
|
+
half = len(on) // 2
|
|
170
|
+
left = np.concatenate((idx[left_mask], on[half:]))
|
|
171
|
+
right = np.concatenate((idx[right_mask], on[:half]))
|
|
172
|
+
|
|
173
|
+
node.left = self._make_tree(left)
|
|
174
|
+
node.right = self._make_tree(right)
|
|
175
|
+
return node
|
|
176
|
+
|
|
177
|
+
def search(self, query: np.ndarray, k: int = 10, search_k: int | None = None) -> np.ndarray:
|
|
178
|
+
"""Finds the approximate ``k`` nearest neighbors of a query.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
query: Query vector of shape ``(dim,)``.
|
|
182
|
+
k: Number of neighbors to return.
|
|
183
|
+
search_k: Number of candidate points to collect before ranking.
|
|
184
|
+
Larger values improve recall at the cost of speed. Defaults
|
|
185
|
+
to ``k * n_trees * 10``. If it is at least the number of
|
|
186
|
+
indexed points, an exact brute-force search is performed
|
|
187
|
+
instead of traversing the trees.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
Array of up to ``k`` data indices, sorted by descending cosine
|
|
191
|
+
similarity to ``query``.
|
|
192
|
+
|
|
193
|
+
Raises:
|
|
194
|
+
RuntimeError: If :meth:`build_index` has not been called.
|
|
195
|
+
ValueError: If ``query`` does not have shape ``(dim,)``.
|
|
196
|
+
"""
|
|
197
|
+
if self._data is None:
|
|
198
|
+
raise RuntimeError("Index has not been built. Call build_index() first.")
|
|
199
|
+
query = np.asarray(query, dtype=np.float32)
|
|
200
|
+
if query.shape != (self.dim,):
|
|
201
|
+
raise ValueError(f"Expected query of shape ({self.dim},), but got {query.shape}.")
|
|
202
|
+
query = _Helper.normalize_rows(query[None, :])[0]
|
|
203
|
+
if search_k is None:
|
|
204
|
+
search_k = k * self.n_trees * 10
|
|
205
|
+
|
|
206
|
+
if search_k >= len(self._data):
|
|
207
|
+
cand = np.arange(len(self._data))
|
|
208
|
+
else:
|
|
209
|
+
cand = self._collect_candidates(query, search_k)
|
|
210
|
+
|
|
211
|
+
scores = self._data[cand] @ query
|
|
212
|
+
order = np.argsort(-scores)[:k]
|
|
213
|
+
return cand[order]
|
|
214
|
+
|
|
215
|
+
def _collect_candidates(self, query: np.ndarray, search_k: int) -> np.ndarray:
|
|
216
|
+
"""Collects candidate points by best-first traversal of the forest.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
query: Normalized query vector of shape ``(dim,)``.
|
|
220
|
+
search_k: Number of candidates to collect before stopping.
|
|
221
|
+
|
|
222
|
+
Returns:
|
|
223
|
+
Array of unique data indices of the collected candidates.
|
|
224
|
+
"""
|
|
225
|
+
heap: list[tuple[float, int, _Node]] = []
|
|
226
|
+
counter = itertools.count()
|
|
227
|
+
candidates: set[int] = set()
|
|
228
|
+
|
|
229
|
+
for node in self._roots:
|
|
230
|
+
leaf = self.down_node(heap, counter, node, query, np.inf)
|
|
231
|
+
candidates.update(leaf.indices.tolist())
|
|
232
|
+
|
|
233
|
+
while heap and len(candidates) < search_k:
|
|
234
|
+
neg_priority, _, node = heapq.heappop(heap)
|
|
235
|
+
leaf = self.down_node(heap, counter, node, query, -neg_priority)
|
|
236
|
+
candidates.update(leaf.indices.tolist())
|
|
237
|
+
|
|
238
|
+
return np.fromiter(candidates, dtype=np.int64)
|
|
239
|
+
|
|
240
|
+
def down_node(self, heap: list[tuple[float, int, _Node]], counter: itertools.count,
|
|
241
|
+
node: _Node, query: np.ndarray, space: float) -> _Node:
|
|
242
|
+
"""Descends from a node to a leaf, deferring the other branches.
|
|
243
|
+
|
|
244
|
+
At each internal node the query follows the side of the hyperplane it
|
|
245
|
+
lies on, and the opposite branch is pushed onto ``heap`` with a
|
|
246
|
+
priority based on the query's margin to the hyperplane.
|
|
247
|
+
|
|
248
|
+
Args:
|
|
249
|
+
heap: Priority queue of deferred branches, stored as
|
|
250
|
+
``(negated_priority, tie_breaker, node)`` tuples. Modified in place.
|
|
251
|
+
counter: Monotonic counter used as a tie-breaker in ``heap``.
|
|
252
|
+
node: Node to start descending from.
|
|
253
|
+
query: Normalized query vector of shape ``(dim,)``.
|
|
254
|
+
space: Smallest margin accumulated along the path to ``node``.
|
|
255
|
+
|
|
256
|
+
Returns:
|
|
257
|
+
The leaf node reached by the descent.
|
|
258
|
+
"""
|
|
259
|
+
while not node.is_leaf:
|
|
260
|
+
s = float(query @ node.normal + node.offset)
|
|
261
|
+
if s <= 0:
|
|
262
|
+
near, far = node.left, node.right
|
|
263
|
+
else:
|
|
264
|
+
near, far = node.right, node.left
|
|
265
|
+
heapq.heappush(heap, (-min(space, -abs(s)), next(counter), far))
|
|
266
|
+
space = min(space, abs(s))
|
|
267
|
+
node = near
|
|
268
|
+
return node
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
if __name__ == "__main__":
|
|
272
|
+
rng = np.random.default_rng(0)
|
|
273
|
+
n, dim, k = 20000, 32, 10
|
|
274
|
+
data = rng.normal(size=(n, dim))
|
|
275
|
+
queries = rng.normal(size=(100, dim))
|
|
276
|
+
|
|
277
|
+
index = Index(dim=dim, n_trees=10, seed=42)
|
|
278
|
+
index.build_index(data)
|
|
279
|
+
|
|
280
|
+
data_n = _Helper.normalize_rows(data)
|
|
281
|
+
queries_n = _Helper.normalize_rows(queries)
|
|
282
|
+
for search_k in (None, 1000, 5000):
|
|
283
|
+
hits = 0
|
|
284
|
+
for q, qn in zip(queries, queries_n):
|
|
285
|
+
truth = set(np.argsort(-(data_n @ qn))[:k].tolist())
|
|
286
|
+
got = set(index.search(q, k=k, search_k=search_k).tolist())
|
|
287
|
+
hits += len(truth & got)
|
|
288
|
+
print(f"search_k={search_k}: recall@{k} = {hits / (len(queries) * k):.3f}")
|
|
File without changes
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "kodachi"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Approximate nearest neighbor search with random projection trees."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "yuzujelly2222" }]
|
|
14
|
+
keywords = ["ann", "nearest-neighbor", "vector-search", "cosine-similarity", "random-projection"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
dependencies = ["numpy>=1.23"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/yuzujelly2222/kodachi"
|
|
33
|
+
Documentation = "https://yuzujelly2222.github.io/kodachi/"
|
|
34
|
+
Repository = "https://github.com/yuzujelly2222/kodachi"
|
|
35
|
+
Issues = "https://github.com/yuzujelly2222/kodachi/issues"
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
dev = ["pytest>=7", "build", "twine"]
|
|
39
|
+
docs = ["sphinx>=7", "furo", "myst-parser"]
|
|
40
|
+
|
|
41
|
+
[tool.hatch.version]
|
|
42
|
+
path = "kodachi/__init__.py"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.wheel]
|
|
45
|
+
packages = ["kodachi"]
|
|
46
|
+
|
|
47
|
+
[tool.hatch.build.targets.sdist]
|
|
48
|
+
include = ["kodachi", "tests", "README.md", "README.ja.md", "LICENSE", "pyproject.toml"]
|
|
49
|
+
|
|
50
|
+
[tool.pytest.ini_options]
|
|
51
|
+
testpaths = ["tests"]
|
|
52
|
+
pythonpath = ["."]
|
|
53
|
+
addopts = "-ra"
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from kodachi import Index
|
|
5
|
+
from kodachi.main import _Helper
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
DIM = 16
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def brute_force(data: np.ndarray, query: np.ndarray, k: int) -> np.ndarray:
|
|
12
|
+
data_n = _Helper.normalize_rows(np.asarray(data, dtype=np.float32))
|
|
13
|
+
query_n = _Helper.normalize_rows(np.asarray(query, dtype=np.float32)[None, :])[0]
|
|
14
|
+
return np.argsort(-(data_n @ query_n))[:k]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def data() -> np.ndarray:
|
|
19
|
+
return np.random.default_rng(0).normal(size=(2000, DIM))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.fixture
|
|
23
|
+
def index(data: np.ndarray) -> Index:
|
|
24
|
+
idx = Index(dim=DIM, n_trees=10, max_leaf_size=16, seed=42)
|
|
25
|
+
idx.build_index(data)
|
|
26
|
+
return idx
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def collect_leaves(node, leaves):
|
|
30
|
+
if node.is_leaf:
|
|
31
|
+
leaves.append(node)
|
|
32
|
+
else:
|
|
33
|
+
collect_leaves(node.left, leaves)
|
|
34
|
+
collect_leaves(node.right, leaves)
|
|
35
|
+
return leaves
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class TestValidation:
|
|
39
|
+
def test_search_before_build_raises(self):
|
|
40
|
+
with pytest.raises(RuntimeError, match="build_index"):
|
|
41
|
+
Index(dim=DIM).search(np.zeros(DIM))
|
|
42
|
+
|
|
43
|
+
@pytest.mark.parametrize("shape", [(10,), (10, DIM + 1), (2, 5, DIM)])
|
|
44
|
+
def test_build_with_wrong_shape_raises(self, shape):
|
|
45
|
+
with pytest.raises(ValueError, match="Expected data of shape"):
|
|
46
|
+
Index(dim=DIM).build_index(np.zeros(shape))
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize("shape", [(DIM + 1,), (1, DIM)])
|
|
49
|
+
def test_search_with_wrong_shape_raises(self, index, shape):
|
|
50
|
+
with pytest.raises(ValueError, match="Expected query of shape"):
|
|
51
|
+
index.search(np.zeros(shape))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class TestTreeStructure:
|
|
55
|
+
def test_each_tree_partitions_all_points(self, index, data):
|
|
56
|
+
for root in index._roots:
|
|
57
|
+
leaves = collect_leaves(root, [])
|
|
58
|
+
all_idx = np.concatenate([leaf.indices for leaf in leaves])
|
|
59
|
+
assert len(all_idx) == len(data)
|
|
60
|
+
assert set(all_idx.tolist()) == set(range(len(data)))
|
|
61
|
+
|
|
62
|
+
def test_leaf_size_is_bounded(self, index):
|
|
63
|
+
for root in index._roots:
|
|
64
|
+
for leaf in collect_leaves(root, []):
|
|
65
|
+
assert len(leaf.indices) <= index.max_leaf_size
|
|
66
|
+
|
|
67
|
+
def test_number_of_trees(self, index):
|
|
68
|
+
assert len(index._roots) == index.n_trees
|
|
69
|
+
|
|
70
|
+
def test_rebuild_replaces_trees(self, index, data):
|
|
71
|
+
index.build_index(data[:100])
|
|
72
|
+
assert len(index._roots) == index.n_trees
|
|
73
|
+
assert len(index._data) == 100
|
|
74
|
+
|
|
75
|
+
def test_duplicate_points_terminate(self):
|
|
76
|
+
data = np.ones((200, DIM))
|
|
77
|
+
idx = Index(dim=DIM, n_trees=3, max_leaf_size=8, seed=0)
|
|
78
|
+
idx.build_index(data)
|
|
79
|
+
assert len(idx.search(np.ones(DIM), k=5)) == 5
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TestSearch:
|
|
83
|
+
def test_returns_k_unique_indices(self, index, data):
|
|
84
|
+
result = index.search(data[0], k=10)
|
|
85
|
+
assert result.shape == (10,)
|
|
86
|
+
assert len(set(result.tolist())) == 10
|
|
87
|
+
assert np.all((0 <= result) & (result < len(data)))
|
|
88
|
+
|
|
89
|
+
def test_results_sorted_by_similarity(self, index, data):
|
|
90
|
+
query = np.random.default_rng(1).normal(size=DIM)
|
|
91
|
+
result = index.search(query, k=20)
|
|
92
|
+
q = query / np.linalg.norm(query)
|
|
93
|
+
scores = index._data[result] @ q
|
|
94
|
+
assert np.all(np.diff(scores) <= 1e-6)
|
|
95
|
+
|
|
96
|
+
def test_finds_itself(self, index, data):
|
|
97
|
+
for i in range(0, len(data), 200):
|
|
98
|
+
assert index.search(data[i], k=1)[0] == i
|
|
99
|
+
|
|
100
|
+
def test_large_search_k_is_exact(self, index, data):
|
|
101
|
+
rng = np.random.default_rng(2)
|
|
102
|
+
for _ in range(10):
|
|
103
|
+
query = rng.normal(size=DIM)
|
|
104
|
+
got = index.search(query, k=10, search_k=len(data))
|
|
105
|
+
np.testing.assert_array_equal(got, brute_force(data, query, 10))
|
|
106
|
+
|
|
107
|
+
def test_k_larger_than_data(self):
|
|
108
|
+
data = np.random.default_rng(0).normal(size=(5, DIM))
|
|
109
|
+
idx = Index(dim=DIM, seed=0)
|
|
110
|
+
idx.build_index(data)
|
|
111
|
+
result = idx.search(data[0], k=10)
|
|
112
|
+
assert sorted(result.tolist()) == list(range(5))
|
|
113
|
+
|
|
114
|
+
def test_scale_invariant(self, index, data):
|
|
115
|
+
np.testing.assert_array_equal(
|
|
116
|
+
index.search(data[3], k=10), index.search(data[3] * 100, k=10)
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def test_zero_query_does_not_crash(self, index):
|
|
120
|
+
assert len(index.search(np.zeros(DIM), k=5)) == 5
|
|
121
|
+
|
|
122
|
+
def test_seed_is_reproducible(self, data):
|
|
123
|
+
query = np.random.default_rng(3).normal(size=DIM)
|
|
124
|
+
results = []
|
|
125
|
+
for _ in range(2):
|
|
126
|
+
idx = Index(dim=DIM, n_trees=5, seed=123)
|
|
127
|
+
idx.build_index(data)
|
|
128
|
+
results.append(idx.search(query, k=10, search_k=200))
|
|
129
|
+
np.testing.assert_array_equal(results[0], results[1])
|
|
130
|
+
|
|
131
|
+
def test_recall_improves_with_search_k(self, index, data):
|
|
132
|
+
queries = np.random.default_rng(4).normal(size=(50, DIM))
|
|
133
|
+
k = 10
|
|
134
|
+
|
|
135
|
+
def recall(search_k):
|
|
136
|
+
hits = 0
|
|
137
|
+
for q in queries:
|
|
138
|
+
truth = set(brute_force(data, q, k).tolist())
|
|
139
|
+
hits += len(truth & set(index.search(q, k=k, search_k=search_k).tolist()))
|
|
140
|
+
return hits / (len(queries) * k)
|
|
141
|
+
|
|
142
|
+
low, high = recall(100), recall(1000)
|
|
143
|
+
assert high >= low
|
|
144
|
+
assert high >= 0.8
|