fastlinkage 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastlinkage-0.1.1/.gitignore +9 -0
- fastlinkage-0.1.1/CHANGELOG.md +21 -0
- fastlinkage-0.1.1/CITATION.cff +16 -0
- fastlinkage-0.1.1/LICENSE +21 -0
- fastlinkage-0.1.1/PKG-INFO +309 -0
- fastlinkage-0.1.1/README.md +270 -0
- fastlinkage-0.1.1/experiments/README.md +26 -0
- fastlinkage-0.1.1/pyproject.toml +63 -0
- fastlinkage-0.1.1/src/fastlinkage/__init__.py +6 -0
- fastlinkage-0.1.1/src/fastlinkage/_clusters.py +274 -0
- fastlinkage-0.1.1/src/fastlinkage/_distance.py +40 -0
- fastlinkage-0.1.1/src/fastlinkage/_search.py +179 -0
- fastlinkage-0.1.1/src/fastlinkage/_tree.py +39 -0
- fastlinkage-0.1.1/src/fastlinkage/cluster.py +77 -0
- fastlinkage-0.1.1/src/fastlinkage/core.py +144 -0
- fastlinkage-0.1.1/tests/conftest.py +20 -0
- fastlinkage-0.1.1/tests/test_linkage.py +101 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.1 (2026-11-16)
|
|
4
|
+
|
|
5
|
+
- Verified on an NVIDIA GPU (Tesla T4): same trees as SciPy and fastcluster.
|
|
6
|
+
- On CUDA the float64 checks run on the GPU: 100,000 x 384 points went from
|
|
7
|
+
31-49 s to 11-15 s on a T4 with 2 CPU cores.
|
|
8
|
+
- Tests run on every available device (CPU, CUDA, MPS).
|
|
9
|
+
- CI covers Linux, macOS and Windows.
|
|
10
|
+
- Colab notebook that runs the CUDA check.
|
|
11
|
+
- First release on PyPI: `pip install fastlinkage`.
|
|
12
|
+
|
|
13
|
+
## 0.1.0 (2026-11-15)
|
|
14
|
+
|
|
15
|
+
First release.
|
|
16
|
+
|
|
17
|
+
- `linkage`: exact Ward, average-cosine and average-squared-Euclidean trees in
|
|
18
|
+
SciPy's format, without the n x n distance matrix.
|
|
19
|
+
- `AgglomerativeClustering`: scikit-learn style estimator on top of it.
|
|
20
|
+
- `cut`: flat clusters from a tree by cluster count or distance threshold.
|
|
21
|
+
- Runs on Apple GPUs (PyTorch MPS) and on the CPU. CUDA is untested.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
title: "fastlinkage: exact hierarchical clustering on the GPU without the distance matrix"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Li
|
|
7
|
+
given-names: Michael
|
|
8
|
+
version: 0.1.1
|
|
9
|
+
date-released: 2026-11-16
|
|
10
|
+
license: MIT
|
|
11
|
+
repository-code: "https://github.com/michael-L-i/fastlinkage"
|
|
12
|
+
keywords:
|
|
13
|
+
- hierarchical clustering
|
|
14
|
+
- agglomerative clustering
|
|
15
|
+
- GPU
|
|
16
|
+
- embeddings
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Michael Li
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fastlinkage
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Exact hierarchical clustering (Ward, average linkage) on the GPU, without the n x n distance matrix
|
|
5
|
+
Project-URL: Homepage, https://github.com/michael-L-i/fastlinkage
|
|
6
|
+
Project-URL: Issues, https://github.com/michael-L-i/fastlinkage/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/michael-L-i/fastlinkage/blob/main/CHANGELOG.md
|
|
8
|
+
Author: Michael Li
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agglomerative,embeddings,gpu,hierarchical clustering,linkage
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: numpy>=1.24
|
|
24
|
+
Requires-Dist: scipy>=1.10
|
|
25
|
+
Requires-Dist: torch>=2.1
|
|
26
|
+
Provides-Extra: bench
|
|
27
|
+
Requires-Dist: datasets; extra == 'bench'
|
|
28
|
+
Requires-Dist: fastcluster; extra == 'bench'
|
|
29
|
+
Requires-Dist: scikit-learn; extra == 'bench'
|
|
30
|
+
Requires-Dist: sentence-transformers; extra == 'bench'
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
34
|
+
Requires-Dist: scikit-learn; extra == 'dev'
|
|
35
|
+
Provides-Extra: test
|
|
36
|
+
Requires-Dist: pytest; extra == 'test'
|
|
37
|
+
Requires-Dist: scikit-learn; extra == 'test'
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
|
|
40
|
+
# fastlinkage
|
|
41
|
+
|
|
42
|
+
[](https://github.com/michael-L-i/fastlinkage/actions/workflows/test.yml)
|
|
43
|
+
[](LICENSE)
|
|
44
|
+

|
|
45
|
+
|
|
46
|
+
Exact hierarchical clustering on the GPU, without the n x n distance matrix.
|
|
47
|
+
|
|
48
|
+
It returns the same tree as `scipy.cluster.hierarchy.linkage` for Ward and
|
|
49
|
+
average linkage, runs in seconds at 100,000+ points, and uses memory that
|
|
50
|
+
grows with the number of points, not its square.
|
|
51
|
+
|
|
52
|
+
<picture>
|
|
53
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/speed-dark.svg">
|
|
54
|
+
<img src="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/speed-light.svg" alt="Time to build the exact tree. 20,000 text embeddings, average linkage with cosine distance: 0.8 s against 47 s for SciPy. 100,000 points with 32 features, Ward: 2.2 s against 2.3 minutes for fastcluster. 120,000 text embeddings, Ward: 8.7 s against 59 minutes for fastcluster." width="800">
|
|
55
|
+
</picture>
|
|
56
|
+
|
|
57
|
+
> **Status: early release.** Tested on NVIDIA GPUs (CUDA), Apple GPUs (PyTorch
|
|
58
|
+
> MPS) and the CPU on Linux, macOS and Windows.
|
|
59
|
+
|
|
60
|
+
## Why
|
|
61
|
+
|
|
62
|
+
SciPy and scikit-learn build the full distance matrix. At 120,000 points that
|
|
63
|
+
is 58 GB, so most people give up on hierarchical clustering long before that
|
|
64
|
+
and switch to k-means or HDBSCAN. This package keeps the exact tree and drops
|
|
65
|
+
the matrix:
|
|
66
|
+
|
|
67
|
+
| | SciPy / scikit-learn | fastlinkage |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| Memory | n x n distances | one mean vector per cluster |
|
|
70
|
+
| 20,000 text embeddings (384 features), average cosine | 47 s | 0.8 s |
|
|
71
|
+
| 120,000 text embeddings, average cosine | needs a 58 GB matrix | 6.9 s |
|
|
72
|
+
| Result | exact | exact, same tree |
|
|
73
|
+
|
|
74
|
+
## Install
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
pip install fastlinkage
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Needs Python 3.10+, NumPy, SciPy and PyTorch. It uses CUDA or Apple MPS when
|
|
81
|
+
PyTorch sees one, and the CPU otherwise.
|
|
82
|
+
|
|
83
|
+
| Hardware | Status |
|
|
84
|
+
|---|---|
|
|
85
|
+
| Apple GPU (MPS) | tested; main benchmarks below |
|
|
86
|
+
| CPU on Linux, macOS, Windows | tested on every commit |
|
|
87
|
+
| NVIDIA GPU (CUDA) | tested on a Tesla T4 ([notebook](https://colab.research.google.com/github/michael-L-i/fastlinkage/blob/main/notebooks/cuda_check.ipynb)) |
|
|
88
|
+
|
|
89
|
+
The package is pure Python on top of PyTorch, so there is nothing to compile.
|
|
90
|
+
|
|
91
|
+
## Use
|
|
92
|
+
|
|
93
|
+
Drop-in for SciPy's `linkage`:
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from fastlinkage import linkage
|
|
97
|
+
|
|
98
|
+
Z = linkage(X, method="ward") # scipy: linkage(X, "ward")
|
|
99
|
+
Z = linkage(X, method="average_cosine") # scipy: linkage(X, "average", metric="cosine")
|
|
100
|
+
Z = linkage(X, method="average_sqeuclidean") # scipy: linkage(X, "average", metric="sqeuclidean")
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`Z` is SciPy's linkage matrix, so everything downstream keeps working:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from scipy.cluster.hierarchy import dendrogram, fcluster
|
|
107
|
+
|
|
108
|
+
labels = fcluster(Z, t=50, criterion="maxclust")
|
|
109
|
+
dendrogram(Z, truncate_mode="lastp", p=40)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Cluster embeddings without choosing the number of clusters:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from fastlinkage import AgglomerativeClustering
|
|
116
|
+
|
|
117
|
+
model = AgglomerativeClustering(
|
|
118
|
+
n_clusters=None, distance_threshold=0.35, linkage="average", metric="cosine"
|
|
119
|
+
).fit(embeddings)
|
|
120
|
+
|
|
121
|
+
model.labels_ # cluster of every row
|
|
122
|
+
model.n_clusters_ # how many clusters the threshold gave
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The class follows scikit-learn's `AgglomerativeClustering` (`labels_`,
|
|
126
|
+
`children_`, `distances_`, `fit_predict`). Cutting one tree at several levels
|
|
127
|
+
costs almost nothing:
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from fastlinkage import cut, linkage
|
|
131
|
+
|
|
132
|
+
Z = linkage(embeddings, method="average_cosine")
|
|
133
|
+
coarse = cut(Z, n_clusters=20)
|
|
134
|
+
fine = cut(Z, distance_threshold=0.25)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
A runnable version is in [`examples/quickstart.py`](examples/quickstart.py).
|
|
138
|
+
|
|
139
|
+
### Reference
|
|
140
|
+
|
|
141
|
+
| | |
|
|
142
|
+
|---|---|
|
|
143
|
+
| `linkage(X, method="ward", device=None)` | SciPy-format linkage matrix. `method` is `"ward"`, `"average_cosine"` or `"average_sqeuclidean"`. `device` is `"cuda"`, `"mps"` or `"cpu"`; by default the GPU if there is one. |
|
|
144
|
+
| `cut(Z, n_clusters=None, distance_threshold=None)` | Labels from a tree. Give exactly one of the two. |
|
|
145
|
+
| `AgglomerativeClustering(n_clusters=2, *, metric="euclidean", linkage="ward", distance_threshold=None, device=None)` | scikit-learn style estimator. Supported pairs: `ward` + `euclidean`, `average` + `cosine`, `average` + `sqeuclidean`. |
|
|
146
|
+
|
|
147
|
+
## Benchmarks
|
|
148
|
+
|
|
149
|
+
Measured on an Apple laptop GPU (PyTorch MPS, 18 CPU cores for the baselines).
|
|
150
|
+
Every timing builds the full tree.
|
|
151
|
+
|
|
152
|
+
| Data | Points | Linkage | fastlinkage | Baseline |
|
|
153
|
+
|---|---|---|---|---|
|
|
154
|
+
| Text embeddings, 384 features | 20,000 | Average, cosine | 0.8 s | SciPy 47 s |
|
|
155
|
+
| Text embeddings, 384 features | 20,000 | Ward | 1.0 s | SciPy 16 s |
|
|
156
|
+
| Text embeddings, 384 features | 120,000 | Average, cosine | 6.9 s | SciPy needs 58 GB |
|
|
157
|
+
| Text embeddings, 384 features | 120,000 | Ward | 8.7 s | fastcluster 59 min |
|
|
158
|
+
| Synthetic, 32 features | 100,000 | Ward | 2.2 s | fastcluster 135 s |
|
|
159
|
+
| Synthetic, 32 features | 200,000 | all three | 5.9 to 6.4 s | not run |
|
|
160
|
+
| Synthetic, 32 features | 500,000 | Average, cosine | 33 s | not run |
|
|
161
|
+
| Synthetic, 32 features | 500,000 | Ward | 36 s | not run |
|
|
162
|
+
|
|
163
|
+
- The text embeddings are AG News titles and descriptions, embedded with
|
|
164
|
+
all-MiniLM-L6-v2.
|
|
165
|
+
- fastcluster's `linkage_vector` is the standard exact Ward that avoids the
|
|
166
|
+
matrix. It is single-threaded.
|
|
167
|
+
- The same code on the CPU is 2x to 4x slower than on this GPU, so most of
|
|
168
|
+
the gap to SciPy comes from the method, not the hardware.
|
|
169
|
+
|
|
170
|
+
<picture>
|
|
171
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/scaling-dark.svg">
|
|
172
|
+
<img src="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/scaling-light.svg" alt="Exact Ward tree on 32-feature data. fastlinkage: 0.4 s at 20,000 points, 2.2 s at 100,000, 6.4 s at 200,000, 36 s at 500,000. fastcluster: 5.2 s at 20,000, 32 s at 50,000, 2.3 minutes at 100,000." width="800">
|
|
173
|
+
</picture>
|
|
174
|
+
|
|
175
|
+
Time grows with the square of the number of points. About a million points is
|
|
176
|
+
the practical limit.
|
|
177
|
+
|
|
178
|
+
### NVIDIA GPU
|
|
179
|
+
|
|
180
|
+
A free Google Colab machine: Tesla T4 with 2 virtual CPUs. Synthetic data.
|
|
181
|
+
|
|
182
|
+
| Points | Features | Ward | Average, cosine | Average, sq. Euclidean | Baseline on the same machine |
|
|
183
|
+
|---|---|---|---|---|---|
|
|
184
|
+
| 20,000 | 32 | 1.0 s | 0.8 s | 0.9 s | SciPy 23 to 24 s |
|
|
185
|
+
| 100,000 | 32 | 3.9 s | 4.0 s | 5.3 s | fastcluster Ward 490 s |
|
|
186
|
+
| 100,000 | 384 | 11.4 s | 12.7 s | 14.8 s | not run |
|
|
187
|
+
| 200,000 | 32 | 10.8 s | 10.0 s | 11.1 s | not run |
|
|
188
|
+
|
|
189
|
+
The trees match SciPy at 20,000 points and fastcluster at 100,000 to 1e-12.
|
|
190
|
+
[`notebooks/cuda_check.ipynb`](notebooks/cuda_check.ipynb) reproduces this.
|
|
191
|
+
|
|
192
|
+
## Exactness
|
|
193
|
+
|
|
194
|
+
The tree is checked against independent implementations:
|
|
195
|
+
|
|
196
|
+
| Check | Result |
|
|
197
|
+
|---|---|
|
|
198
|
+
| SciPy, 5,000 and 20,000 points, all three linkages, synthetic and real embeddings | merge heights equal to 1e-12; flat clusterings identical |
|
|
199
|
+
| fastcluster Ward, 100,000 synthetic points | heights equal to 5e-13 |
|
|
200
|
+
| fastcluster Ward, 120,000 text embeddings | heights equal to 1.4e-14 |
|
|
201
|
+
| Duplicated rows, near-duplicates (1e-7 noise), data offset by 1e4 | equal to SciPy to 1e-13 |
|
|
202
|
+
|
|
203
|
+
This takes some care. The GPU search runs in float32, and a plain float32
|
|
204
|
+
search picks a slightly wrong neighbour often enough to change the tree: an
|
|
205
|
+
earlier version of this code was off by up to 1.6 in merge height (tallest
|
|
206
|
+
merge 82) on the 120,000 embeddings. The current version accepts a float32
|
|
207
|
+
result only when it wins by more than the possible rounding error, and
|
|
208
|
+
otherwise compares the close contenders in float64.
|
|
209
|
+
|
|
210
|
+
With exact ties between different pairs (an integer grid, for example) the
|
|
211
|
+
tree is not unique, and this package and SciPy can return different valid
|
|
212
|
+
trees.
|
|
213
|
+
|
|
214
|
+
## How it works
|
|
215
|
+
|
|
216
|
+
Three known ideas and two additions.
|
|
217
|
+
|
|
218
|
+
1. **Cluster summaries instead of a matrix.** For these linkages the distance
|
|
219
|
+
between two clusters follows from each cluster's size, mean vector and one
|
|
220
|
+
scalar:
|
|
221
|
+
|
|
222
|
+
| Linkage | Distance between clusters A and B |
|
|
223
|
+
|---|---|
|
|
224
|
+
| Ward | sqrt(2 \|A\|\|B\| / (\|A\|+\|B\|)) x \|\|mean(A) - mean(B)\|\| |
|
|
225
|
+
| Average, cosine | 1 - mean(A) . mean(B), rows scaled to unit length |
|
|
226
|
+
| Average, squared Euclidean | \|\|mean(A) - mean(B)\|\|^2 + var(A) + var(B) |
|
|
227
|
+
|
|
228
|
+
2. **Merge in rounds.** Every pair of clusters that are each other's nearest
|
|
229
|
+
neighbour merges in the same round. For these linkages that gives exactly
|
|
230
|
+
the tree of the one-merge-at-a-time algorithm. A full run takes 50 to 180
|
|
231
|
+
rounds.
|
|
232
|
+
|
|
233
|
+
3. **Nearest-neighbour search as one matrix product** on the GPU.
|
|
234
|
+
|
|
235
|
+
4. **Neighbour lists that survive merges.** A search returns 16 candidates and
|
|
236
|
+
a radius beyond which nothing else can be. When a cluster's neighbour is
|
|
237
|
+
merged away, the next one usually comes from that list with no new search,
|
|
238
|
+
and a merged cluster inherits its parents' lists. This halves the search
|
|
239
|
+
work (from about 3 n^2 to 1.5 n^2 pair evaluations).
|
|
240
|
+
|
|
241
|
+
5. **A float32 rounding check** with a float64 fallback, described above.
|
|
242
|
+
|
|
243
|
+
[`docs/DESIGN.md`](docs/DESIGN.md) has the details: the neighbour-list rule,
|
|
244
|
+
the rounding check, more measurements, and what was tried and dropped.
|
|
245
|
+
|
|
246
|
+
## Related work
|
|
247
|
+
|
|
248
|
+
None of the underlying algorithm is new, and this package does not claim it.
|
|
249
|
+
|
|
250
|
+
| Idea | Where it comes from |
|
|
251
|
+
|---|---|
|
|
252
|
+
| Merging reciprocal nearest neighbours in rounds | Murtagh (1980s); [ParChain](https://arxiv.org/abs/2106.04727) (2021, CPU); [RAC](https://arxiv.org/abs/2105.11653) (2021) |
|
|
253
|
+
| Average linkage from means and variances in linear memory | ParChain (CPU, kd-trees, low dimensions); [arXiv 1404.3439](https://arxiv.org/abs/1404.3439) |
|
|
254
|
+
| Exact Ward on the GPU | [cuhc](https://pypi.org/project/cuhc/) (2026, CUDA) |
|
|
255
|
+
|
|
256
|
+
What I could not find elsewhere is a GPU implementation of exact average
|
|
257
|
+
linkage (cosine or squared Euclidean) that avoids the distance matrix and
|
|
258
|
+
works on high-dimensional embeddings. cuhc keeps the matrix for average
|
|
259
|
+
linkage; ParChain's version is CPU-only and built for low dimensions. The
|
|
260
|
+
search was not exhaustive, so treat "first" as unconfirmed. I have also not
|
|
261
|
+
checked whether the neighbour-list rule (item 4) is published.
|
|
262
|
+
|
|
263
|
+
## Limits
|
|
264
|
+
|
|
265
|
+
- Three linkages only: Ward, average with cosine, average with squared
|
|
266
|
+
Euclidean. Complete linkage and average linkage with plain Euclidean
|
|
267
|
+
distance have no per-cluster summary and are not supported.
|
|
268
|
+
- Time is quadratic in the number of points.
|
|
269
|
+
- A float32 copy of every cluster mean lives on the GPU: 8 n d bytes (3 GB
|
|
270
|
+
for a million points with 384 features). On CUDA a float64 copy is kept
|
|
271
|
+
there too when it fits in a quarter of the free memory.
|
|
272
|
+
- When distances are below float32 resolution (cosine distance between
|
|
273
|
+
almost parallel vectors), the float64 fallback does most of the work and is
|
|
274
|
+
slow. A warning is raised.
|
|
275
|
+
- Tested on one NVIDIA GPU (Tesla T4) so far, and not compared with cuhc on
|
|
276
|
+
the same hardware.
|
|
277
|
+
|
|
278
|
+
## Reproduce
|
|
279
|
+
|
|
280
|
+
```
|
|
281
|
+
git clone https://github.com/michael-L-i/fastlinkage && cd fastlinkage
|
|
282
|
+
pip install -e ".[test,bench]"
|
|
283
|
+
pytest # exactness tests, CPU only
|
|
284
|
+
python experiments/correctness.py # against SciPy
|
|
285
|
+
python experiments/make_embeddings.py # AG News embeddings (176 MB)
|
|
286
|
+
python experiments/neighbour_lists.py # 20k to 120k points
|
|
287
|
+
python experiments/large_runs.py # fastcluster check, 200k, 500k
|
|
288
|
+
python experiments/make_charts.py # the charts above
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
[`experiments/`](experiments) lists every script, and the raw numbers are in
|
|
292
|
+
[`results/`](results).
|
|
293
|
+
|
|
294
|
+
## Repository layout
|
|
295
|
+
|
|
296
|
+
```
|
|
297
|
+
src/fastlinkage/ the package
|
|
298
|
+
tests/ exactness tests against SciPy and scikit-learn
|
|
299
|
+
notebooks/ CUDA check for Google Colab
|
|
300
|
+
examples/ quickstart
|
|
301
|
+
experiments/ benchmark and correctness scripts
|
|
302
|
+
results/ their output (JSON)
|
|
303
|
+
docs/DESIGN.md how it works, measurements, prior work
|
|
304
|
+
assets/ README charts
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
## License
|
|
308
|
+
|
|
309
|
+
MIT
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# fastlinkage
|
|
2
|
+
|
|
3
|
+
[](https://github.com/michael-L-i/fastlinkage/actions/workflows/test.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
Exact hierarchical clustering on the GPU, without the n x n distance matrix.
|
|
8
|
+
|
|
9
|
+
It returns the same tree as `scipy.cluster.hierarchy.linkage` for Ward and
|
|
10
|
+
average linkage, runs in seconds at 100,000+ points, and uses memory that
|
|
11
|
+
grows with the number of points, not its square.
|
|
12
|
+
|
|
13
|
+
<picture>
|
|
14
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/speed-dark.svg">
|
|
15
|
+
<img src="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/speed-light.svg" alt="Time to build the exact tree. 20,000 text embeddings, average linkage with cosine distance: 0.8 s against 47 s for SciPy. 100,000 points with 32 features, Ward: 2.2 s against 2.3 minutes for fastcluster. 120,000 text embeddings, Ward: 8.7 s against 59 minutes for fastcluster." width="800">
|
|
16
|
+
</picture>
|
|
17
|
+
|
|
18
|
+
> **Status: early release.** Tested on NVIDIA GPUs (CUDA), Apple GPUs (PyTorch
|
|
19
|
+
> MPS) and the CPU on Linux, macOS and Windows.
|
|
20
|
+
|
|
21
|
+
## Why
|
|
22
|
+
|
|
23
|
+
SciPy and scikit-learn build the full distance matrix. At 120,000 points that
|
|
24
|
+
is 58 GB, so most people give up on hierarchical clustering long before that
|
|
25
|
+
and switch to k-means or HDBSCAN. This package keeps the exact tree and drops
|
|
26
|
+
the matrix:
|
|
27
|
+
|
|
28
|
+
| | SciPy / scikit-learn | fastlinkage |
|
|
29
|
+
|---|---|---|
|
|
30
|
+
| Memory | n x n distances | one mean vector per cluster |
|
|
31
|
+
| 20,000 text embeddings (384 features), average cosine | 47 s | 0.8 s |
|
|
32
|
+
| 120,000 text embeddings, average cosine | needs a 58 GB matrix | 6.9 s |
|
|
33
|
+
| Result | exact | exact, same tree |
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
pip install fastlinkage
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Needs Python 3.10+, NumPy, SciPy and PyTorch. It uses CUDA or Apple MPS when
|
|
42
|
+
PyTorch sees one, and the CPU otherwise.
|
|
43
|
+
|
|
44
|
+
| Hardware | Status |
|
|
45
|
+
|---|---|
|
|
46
|
+
| Apple GPU (MPS) | tested; main benchmarks below |
|
|
47
|
+
| CPU on Linux, macOS, Windows | tested on every commit |
|
|
48
|
+
| NVIDIA GPU (CUDA) | tested on a Tesla T4 ([notebook](https://colab.research.google.com/github/michael-L-i/fastlinkage/blob/main/notebooks/cuda_check.ipynb)) |
|
|
49
|
+
|
|
50
|
+
The package is pure Python on top of PyTorch, so there is nothing to compile.
|
|
51
|
+
|
|
52
|
+
## Use
|
|
53
|
+
|
|
54
|
+
Drop-in for SciPy's `linkage`:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from fastlinkage import linkage
|
|
58
|
+
|
|
59
|
+
Z = linkage(X, method="ward") # scipy: linkage(X, "ward")
|
|
60
|
+
Z = linkage(X, method="average_cosine") # scipy: linkage(X, "average", metric="cosine")
|
|
61
|
+
Z = linkage(X, method="average_sqeuclidean") # scipy: linkage(X, "average", metric="sqeuclidean")
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`Z` is SciPy's linkage matrix, so everything downstream keeps working:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from scipy.cluster.hierarchy import dendrogram, fcluster
|
|
68
|
+
|
|
69
|
+
labels = fcluster(Z, t=50, criterion="maxclust")
|
|
70
|
+
dendrogram(Z, truncate_mode="lastp", p=40)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Cluster embeddings without choosing the number of clusters:
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from fastlinkage import AgglomerativeClustering
|
|
77
|
+
|
|
78
|
+
model = AgglomerativeClustering(
|
|
79
|
+
n_clusters=None, distance_threshold=0.35, linkage="average", metric="cosine"
|
|
80
|
+
).fit(embeddings)
|
|
81
|
+
|
|
82
|
+
model.labels_ # cluster of every row
|
|
83
|
+
model.n_clusters_ # how many clusters the threshold gave
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The class follows scikit-learn's `AgglomerativeClustering` (`labels_`,
|
|
87
|
+
`children_`, `distances_`, `fit_predict`). Cutting one tree at several levels
|
|
88
|
+
costs almost nothing:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from fastlinkage import cut, linkage
|
|
92
|
+
|
|
93
|
+
Z = linkage(embeddings, method="average_cosine")
|
|
94
|
+
coarse = cut(Z, n_clusters=20)
|
|
95
|
+
fine = cut(Z, distance_threshold=0.25)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
A runnable version is in [`examples/quickstart.py`](examples/quickstart.py).
|
|
99
|
+
|
|
100
|
+
### Reference
|
|
101
|
+
|
|
102
|
+
| | |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `linkage(X, method="ward", device=None)` | SciPy-format linkage matrix. `method` is `"ward"`, `"average_cosine"` or `"average_sqeuclidean"`. `device` is `"cuda"`, `"mps"` or `"cpu"`; by default the GPU if there is one. |
|
|
105
|
+
| `cut(Z, n_clusters=None, distance_threshold=None)` | Labels from a tree. Give exactly one of the two. |
|
|
106
|
+
| `AgglomerativeClustering(n_clusters=2, *, metric="euclidean", linkage="ward", distance_threshold=None, device=None)` | scikit-learn style estimator. Supported pairs: `ward` + `euclidean`, `average` + `cosine`, `average` + `sqeuclidean`. |
|
|
107
|
+
|
|
108
|
+
## Benchmarks
|
|
109
|
+
|
|
110
|
+
Measured on an Apple laptop GPU (PyTorch MPS, 18 CPU cores for the baselines).
|
|
111
|
+
Every timing builds the full tree.
|
|
112
|
+
|
|
113
|
+
| Data | Points | Linkage | fastlinkage | Baseline |
|
|
114
|
+
|---|---|---|---|---|
|
|
115
|
+
| Text embeddings, 384 features | 20,000 | Average, cosine | 0.8 s | SciPy 47 s |
|
|
116
|
+
| Text embeddings, 384 features | 20,000 | Ward | 1.0 s | SciPy 16 s |
|
|
117
|
+
| Text embeddings, 384 features | 120,000 | Average, cosine | 6.9 s | SciPy needs 58 GB |
|
|
118
|
+
| Text embeddings, 384 features | 120,000 | Ward | 8.7 s | fastcluster 59 min |
|
|
119
|
+
| Synthetic, 32 features | 100,000 | Ward | 2.2 s | fastcluster 135 s |
|
|
120
|
+
| Synthetic, 32 features | 200,000 | all three | 5.9 to 6.4 s | not run |
|
|
121
|
+
| Synthetic, 32 features | 500,000 | Average, cosine | 33 s | not run |
|
|
122
|
+
| Synthetic, 32 features | 500,000 | Ward | 36 s | not run |
|
|
123
|
+
|
|
124
|
+
- The text embeddings are AG News titles and descriptions, embedded with
|
|
125
|
+
all-MiniLM-L6-v2.
|
|
126
|
+
- fastcluster's `linkage_vector` is the standard exact Ward that avoids the
|
|
127
|
+
matrix. It is single-threaded.
|
|
128
|
+
- The same code on the CPU is 2x to 4x slower than on this GPU, so most of
|
|
129
|
+
the gap to SciPy comes from the method, not the hardware.
|
|
130
|
+
|
|
131
|
+
<picture>
|
|
132
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/scaling-dark.svg">
|
|
133
|
+
<img src="https://raw.githubusercontent.com/michael-L-i/fastlinkage/main/assets/scaling-light.svg" alt="Exact Ward tree on 32-feature data. fastlinkage: 0.4 s at 20,000 points, 2.2 s at 100,000, 6.4 s at 200,000, 36 s at 500,000. fastcluster: 5.2 s at 20,000, 32 s at 50,000, 2.3 minutes at 100,000." width="800">
|
|
134
|
+
</picture>
|
|
135
|
+
|
|
136
|
+
Time grows with the square of the number of points. About a million points is
|
|
137
|
+
the practical limit.
|
|
138
|
+
|
|
139
|
+
### NVIDIA GPU
|
|
140
|
+
|
|
141
|
+
A free Google Colab machine: Tesla T4 with 2 virtual CPUs. Synthetic data.
|
|
142
|
+
|
|
143
|
+
| Points | Features | Ward | Average, cosine | Average, sq. Euclidean | Baseline on the same machine |
|
|
144
|
+
|---|---|---|---|---|---|
|
|
145
|
+
| 20,000 | 32 | 1.0 s | 0.8 s | 0.9 s | SciPy 23 to 24 s |
|
|
146
|
+
| 100,000 | 32 | 3.9 s | 4.0 s | 5.3 s | fastcluster Ward 490 s |
|
|
147
|
+
| 100,000 | 384 | 11.4 s | 12.7 s | 14.8 s | not run |
|
|
148
|
+
| 200,000 | 32 | 10.8 s | 10.0 s | 11.1 s | not run |
|
|
149
|
+
|
|
150
|
+
The trees match SciPy at 20,000 points and fastcluster at 100,000 to 1e-12.
|
|
151
|
+
[`notebooks/cuda_check.ipynb`](notebooks/cuda_check.ipynb) reproduces this.
|
|
152
|
+
|
|
153
|
+
## Exactness
|
|
154
|
+
|
|
155
|
+
The tree is checked against independent implementations:
|
|
156
|
+
|
|
157
|
+
| Check | Result |
|
|
158
|
+
|---|---|
|
|
159
|
+
| SciPy, 5,000 and 20,000 points, all three linkages, synthetic and real embeddings | merge heights equal to 1e-12; flat clusterings identical |
|
|
160
|
+
| fastcluster Ward, 100,000 synthetic points | heights equal to 5e-13 |
|
|
161
|
+
| fastcluster Ward, 120,000 text embeddings | heights equal to 1.4e-14 |
|
|
162
|
+
| Duplicated rows, near-duplicates (1e-7 noise), data offset by 1e4 | equal to SciPy to 1e-13 |
|
|
163
|
+
|
|
164
|
+
This takes some care. The GPU search runs in float32, and a plain float32
|
|
165
|
+
search picks a slightly wrong neighbour often enough to change the tree: an
|
|
166
|
+
earlier version of this code was off by up to 1.6 in merge height (tallest
|
|
167
|
+
merge 82) on the 120,000 embeddings. The current version accepts a float32
|
|
168
|
+
result only when it wins by more than the possible rounding error, and
|
|
169
|
+
otherwise compares the close contenders in float64.
|
|
170
|
+
|
|
171
|
+
With exact ties between different pairs (an integer grid, for example) the
|
|
172
|
+
tree is not unique, and this package and SciPy can return different valid
|
|
173
|
+
trees.
|
|
174
|
+
|
|
175
|
+
## How it works
|
|
176
|
+
|
|
177
|
+
Three known ideas and two additions.
|
|
178
|
+
|
|
179
|
+
1. **Cluster summaries instead of a matrix.** For these linkages the distance
|
|
180
|
+
between two clusters follows from each cluster's size, mean vector and one
|
|
181
|
+
scalar:
|
|
182
|
+
|
|
183
|
+
| Linkage | Distance between clusters A and B |
|
|
184
|
+
|---|---|
|
|
185
|
+
| Ward | sqrt(2 \|A\|\|B\| / (\|A\|+\|B\|)) x \|\|mean(A) - mean(B)\|\| |
|
|
186
|
+
| Average, cosine | 1 - mean(A) . mean(B), rows scaled to unit length |
|
|
187
|
+
| Average, squared Euclidean | \|\|mean(A) - mean(B)\|\|^2 + var(A) + var(B) |
|
|
188
|
+
|
|
189
|
+
2. **Merge in rounds.** Every pair of clusters that are each other's nearest
|
|
190
|
+
neighbour merges in the same round. For these linkages that gives exactly
|
|
191
|
+
the tree of the one-merge-at-a-time algorithm. A full run takes 50 to 180
|
|
192
|
+
rounds.
|
|
193
|
+
|
|
194
|
+
3. **Nearest-neighbour search as one matrix product** on the GPU.
|
|
195
|
+
|
|
196
|
+
4. **Neighbour lists that survive merges.** A search returns 16 candidates and
|
|
197
|
+
a radius beyond which nothing else can be. When a cluster's neighbour is
|
|
198
|
+
merged away, the next one usually comes from that list with no new search,
|
|
199
|
+
and a merged cluster inherits its parents' lists. This halves the search
|
|
200
|
+
work (from about 3 n^2 to 1.5 n^2 pair evaluations).
|
|
201
|
+
|
|
202
|
+
5. **A float32 rounding check** with a float64 fallback, described above.
|
|
203
|
+
|
|
204
|
+
[`docs/DESIGN.md`](docs/DESIGN.md) has the details: the neighbour-list rule,
|
|
205
|
+
the rounding check, more measurements, and what was tried and dropped.
|
|
206
|
+
|
|
207
|
+
## Related work
|
|
208
|
+
|
|
209
|
+
None of the underlying algorithm is new, and this package does not claim it.
|
|
210
|
+
|
|
211
|
+
| Idea | Where it comes from |
|
|
212
|
+
|---|---|
|
|
213
|
+
| Merging reciprocal nearest neighbours in rounds | Murtagh (1980s); [ParChain](https://arxiv.org/abs/2106.04727) (2021, CPU); [RAC](https://arxiv.org/abs/2105.11653) (2021) |
|
|
214
|
+
| Average linkage from means and variances in linear memory | ParChain (CPU, kd-trees, low dimensions); [arXiv 1404.3439](https://arxiv.org/abs/1404.3439) |
|
|
215
|
+
| Exact Ward on the GPU | [cuhc](https://pypi.org/project/cuhc/) (2026, CUDA) |
|
|
216
|
+
|
|
217
|
+
What I could not find elsewhere is a GPU implementation of exact average
|
|
218
|
+
linkage (cosine or squared Euclidean) that avoids the distance matrix and
|
|
219
|
+
works on high-dimensional embeddings. cuhc keeps the matrix for average
|
|
220
|
+
linkage; ParChain's version is CPU-only and built for low dimensions. The
|
|
221
|
+
search was not exhaustive, so treat "first" as unconfirmed. I have also not
|
|
222
|
+
checked whether the neighbour-list rule (item 4) is published.
|
|
223
|
+
|
|
224
|
+
## Limits
|
|
225
|
+
|
|
226
|
+
- Three linkages only: Ward, average with cosine, average with squared
|
|
227
|
+
Euclidean. Complete linkage and average linkage with plain Euclidean
|
|
228
|
+
distance have no per-cluster summary and are not supported.
|
|
229
|
+
- Time is quadratic in the number of points.
|
|
230
|
+
- A float32 copy of every cluster mean lives on the GPU: 8 n d bytes (3 GB
|
|
231
|
+
for a million points with 384 features). On CUDA a float64 copy is kept
|
|
232
|
+
there too when it fits in a quarter of the free memory.
|
|
233
|
+
- When distances are below float32 resolution (cosine distance between
|
|
234
|
+
almost parallel vectors), the float64 fallback does most of the work and is
|
|
235
|
+
slow. A warning is raised.
|
|
236
|
+
- Tested on one NVIDIA GPU (Tesla T4) so far, and not compared with cuhc on
|
|
237
|
+
the same hardware.
|
|
238
|
+
|
|
239
|
+
## Reproduce
|
|
240
|
+
|
|
241
|
+
```
|
|
242
|
+
git clone https://github.com/michael-L-i/fastlinkage && cd fastlinkage
|
|
243
|
+
pip install -e ".[test,bench]"
|
|
244
|
+
pytest # exactness tests, CPU only
|
|
245
|
+
python experiments/correctness.py # against SciPy
|
|
246
|
+
python experiments/make_embeddings.py # AG News embeddings (176 MB)
|
|
247
|
+
python experiments/neighbour_lists.py # 20k to 120k points
|
|
248
|
+
python experiments/large_runs.py # fastcluster check, 200k, 500k
|
|
249
|
+
python experiments/make_charts.py # the charts above
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
[`experiments/`](experiments) lists every script, and the raw numbers are in
|
|
253
|
+
[`results/`](results).
|
|
254
|
+
|
|
255
|
+
## Repository layout
|
|
256
|
+
|
|
257
|
+
```
|
|
258
|
+
src/fastlinkage/ the package
|
|
259
|
+
tests/ exactness tests against SciPy and scikit-learn
|
|
260
|
+
notebooks/ CUDA check for Google Colab
|
|
261
|
+
examples/ quickstart
|
|
262
|
+
experiments/ benchmark and correctness scripts
|
|
263
|
+
results/ their output (JSON)
|
|
264
|
+
docs/DESIGN.md how it works, measurements, prior work
|
|
265
|
+
assets/ README charts
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
## License
|
|
269
|
+
|
|
270
|
+
MIT
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Experiments
|
|
2
|
+
|
|
3
|
+
Scripts behind the numbers in the main README. Each one writes a JSON file of
|
|
4
|
+
the same name to [`../results`](../results). Run them from the repository root
|
|
5
|
+
after `pip install -e ".[test,bench]"`.
|
|
6
|
+
|
|
7
|
+
| Script | What it measures | Needs |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| `correctness.py` | Same tree as SciPy, three linkages, 5,000 and 20,000 synthetic points | |
|
|
10
|
+
| `scaling.py` | Time against SciPy and fastcluster as the number of points grows | |
|
|
11
|
+
| `make_embeddings.py` | Builds `data/agnews_minilm.npy`: 120,000 AG News items embedded with all-MiniLM-L6-v2 (176 MB, not in the repository) | |
|
|
12
|
+
| `embeddings.py` | Same tree as SciPy on 20,000 embeddings; timing on all 120,000 | embeddings |
|
|
13
|
+
| `neighbour_lists.py` | The package against the brute-force baseline: time, search work, list hit rates | embeddings |
|
|
14
|
+
| `large_runs.py` | fastcluster Ward check at 100,000 points; timings at 200,000 and 500,000 | |
|
|
15
|
+
| `ward_embeddings_check.py` | fastcluster Ward on all 120,000 embeddings (about an hour) | embeddings |
|
|
16
|
+
| `pruning_bound.py` | A search-pruning idea that was measured and dropped | embeddings |
|
|
17
|
+
| `make_charts.py` | Draws the charts in `assets/` from the result files | |
|
|
18
|
+
|
|
19
|
+
`brute_force_baseline.py` is the first version of the algorithm: a full search
|
|
20
|
+
for every cluster that lost its neighbour, every round, with no neighbour
|
|
21
|
+
lists and no float32 rounding check. It is kept only as the comparison point
|
|
22
|
+
for `neighbour_lists.py` and `ward_embeddings_check.py`, and it is not exact
|
|
23
|
+
on large inputs.
|
|
24
|
+
|
|
25
|
+
The timings were taken on an Apple laptop GPU. The largest run (500,000
|
|
26
|
+
points) takes about a minute and a few GB of memory.
|