everalgo-clustering 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- everalgo_clustering-0.1.0/.gitignore +58 -0
- everalgo_clustering-0.1.0/CHANGELOG.md +27 -0
- everalgo_clustering-0.1.0/LICENSE +21 -0
- everalgo_clustering-0.1.0/PKG-INFO +177 -0
- everalgo_clustering-0.1.0/README.md +157 -0
- everalgo_clustering-0.1.0/pyproject.toml +31 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/__init__.py +26 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/algorithm.py +210 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/prompts/__init__.py +5 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/prompts/en/__init__.py +1 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/prompts/en/cluster.py +38 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/prompts/zh/__init__.py +1 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/prompts/zh/cluster.py +10 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/py.typed +0 -0
- everalgo_clustering-0.1.0/src/everalgo/clustering/state.py +31 -0
- everalgo_clustering-0.1.0/tests/clustering/__init__.py +0 -0
- everalgo_clustering-0.1.0/tests/clustering/test_clustering_geometry.py +154 -0
- everalgo_clustering-0.1.0/tests/clustering/test_clustering_llm.py +235 -0
- everalgo_clustering-0.1.0/tests/clustering/test_clustering_state.py +46 -0
- everalgo_clustering-0.1.0/tests/clustering/test_public_api.py +34 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
wheels/
|
|
12
|
+
|
|
13
|
+
# Environments
|
|
14
|
+
.env
|
|
15
|
+
.env.local
|
|
16
|
+
.venv
|
|
17
|
+
env/
|
|
18
|
+
venv/
|
|
19
|
+
|
|
20
|
+
# uv
|
|
21
|
+
.uv/
|
|
22
|
+
|
|
23
|
+
# Test / coverage / cache
|
|
24
|
+
.pytest_cache/
|
|
25
|
+
.cache
|
|
26
|
+
.coverage
|
|
27
|
+
.coverage.*
|
|
28
|
+
coverage.xml
|
|
29
|
+
report.xml
|
|
30
|
+
htmlcov/
|
|
31
|
+
.tox/
|
|
32
|
+
.nox/
|
|
33
|
+
.hypothesis/
|
|
34
|
+
|
|
35
|
+
# Type checkers / linters
|
|
36
|
+
.mypy_cache/
|
|
37
|
+
.dmypy.json
|
|
38
|
+
.pyre/
|
|
39
|
+
.pytype/
|
|
40
|
+
.ruff_cache/
|
|
41
|
+
|
|
42
|
+
# Jupyter
|
|
43
|
+
.ipynb_checkpoints/
|
|
44
|
+
|
|
45
|
+
# IDE / OS
|
|
46
|
+
.vscode/
|
|
47
|
+
.idea/
|
|
48
|
+
*.swp
|
|
49
|
+
.DS_Store
|
|
50
|
+
|
|
51
|
+
# Logs
|
|
52
|
+
*.log
|
|
53
|
+
|
|
54
|
+
# Project local artifacts (placed under local/ per project convention)
|
|
55
|
+
local/
|
|
56
|
+
|
|
57
|
+
# Claude Code session-local state (scheduled tasks, agent caches, locks)
|
|
58
|
+
.claude/
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this package are documented here. Format follows
|
|
4
|
+
[Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/). Versioning
|
|
5
|
+
follows [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `cluster_by_geometry(embedding, state, *, threshold, time_window_days) -> tuple[int, ClusterState]`: cosine similarity + time-window incremental assignment; no LLM; pure-sync function.
|
|
12
|
+
- `cluster_by_llm(embedding, state, cluster_previews, *, llm, threshold, time_window_days, k_candidates, llm_skip_threshold) -> tuple[int, ClusterState]`: embedding top-K recall + cosine fast-path skip + LLM ranking; async; raises on LLM failure or malformed JSON.
|
|
13
|
+
- `ClusterState` frozen pydantic model with four fields: `centroids: dict[int, list[float]]`, `counts: dict[int, int]`, `last_ts: dict[int, int]` (ms-epoch int), `next_idx: int`. Provides `empty()`, `to_dict()`, `from_dict()`, and a private `_assign()` mutation method.
|
|
14
|
+
- English clustering prompt `CLUSTER_LLM_ASSIGN_PROMPT` in `prompts/en/cluster.py`; Chinese variant re-exports the English prompt (the template is language-neutral; responses adapt to the corpus language).
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- `ClusterState.last_ts` stores millisecond epoch integers instead of float seconds, aligning with `MemCell.timestamp` and `Episode.timestamp` conventions across EverAlgo.
|
|
19
|
+
- `ClusterState` tracks `next_idx` as an explicit field instead of deriving cluster numbering from `max(centroids.keys()) + 1`, which was a footgun when cluster IDs were deleted.
|
|
20
|
+
|
|
21
|
+
### Removed
|
|
22
|
+
|
|
23
|
+
- `ClusterConfig` dataclass: threshold, time-window, and candidate-count parameters are now keyword arguments on each function, allowing callers to pass only what they use.
|
|
24
|
+
- Geometric fallback path inside `cluster_by_llm`: the function raises on LLM failure rather than silently falling back to geometry.
|
|
25
|
+
- 3-retry loop in `cluster_by_llm`: the function raises `ValueError` immediately on bad LLM JSON.
|
|
26
|
+
|
|
27
|
+
[Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/main...HEAD
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EverMind
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: everalgo-clustering
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: EverAlgo clustering: cluster_by_geometry / cluster_by_llm functional operators with ClusterState value object.
|
|
5
|
+
Project-URL: Homepage, https://github.com/EverMind-AI/EverAlgo
|
|
6
|
+
Project-URL: Repository, https://github.com/EverMind-AI/EverAlgo
|
|
7
|
+
Project-URL: Issues, https://github.com/EverMind-AI/EverAlgo/issues
|
|
8
|
+
Project-URL: Documentation, https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-clustering
|
|
9
|
+
Project-URL: Changelog, https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-clustering/CHANGELOG.md
|
|
10
|
+
Author: EverMind
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Requires-Dist: everalgo-core<2.0.0,>=0.1.0
|
|
18
|
+
Requires-Dist: numpy>=1.26
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# everalgo-clustering
|
|
22
|
+
|
|
23
|
+
Online incremental clustering for EverAlgo — two async functions (`cluster_by_geometry` / `cluster_by_llm`) operating on caller-owned `list[Cluster]` state.
|
|
24
|
+
|
|
25
|
+
Stateless: the package never embeds, never queries storage, never holds a lock. The caller owns embedding computation, persistence (`list[Cluster]` serialisation), and read-modify-write coordination across concurrent writers.
|
|
26
|
+
|
|
27
|
+
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install everalgo-clustering
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## What this distribution provides
|
|
36
|
+
|
|
37
|
+
| Symbol | Role |
|
|
38
|
+
|---|---|
|
|
39
|
+
| `Cluster` | Frozen Pydantic value object — one cluster snapshot. Caller-supplied `id` / `members`; algorithm supplies merged `centroid`, `count`, `last_ts`, `preview`. |
|
|
40
|
+
| `cluster_by_geometry` | Cosine similarity + time-window filter + threshold; no LLM; async |
|
|
41
|
+
| `cluster_by_llm` | Top-K geometric recall → fast-path skip → LLM semantic ranking; raises on LLM failure; async |
|
|
42
|
+
|
|
43
|
+
## Cluster type
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
class Cluster(BaseModel):
|
|
47
|
+
id: str | None = None # caller-supplied business id; algo only passes through, never mints
|
|
48
|
+
centroid: np.ndarray
|
|
49
|
+
count: int = 1
|
|
50
|
+
last_ts: int # Unix epoch milliseconds
|
|
51
|
+
preview: list[str] = []
|
|
52
|
+
members: list[str] = [] # caller-supplied entity ids; algo appends on merge, never inspects
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The caller wraps each incoming item as a size-1 `Cluster` (count=1) before passing to either function.
|
|
56
|
+
Both functions return `Cluster | None`: a merged snapshot when the item is assigned to an existing cluster, or `None` when no match — the caller then appends the original size-1 `Cluster` as a brand-new entry and mints its own `id`.
|
|
57
|
+
|
|
58
|
+
## Quick start
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import asyncio
|
|
62
|
+
import numpy as np
|
|
63
|
+
from everalgo.clustering import Cluster, cluster_by_geometry
|
|
64
|
+
|
|
65
|
+
async def main() -> None:
|
|
66
|
+
existing: list[Cluster] = [] # caller loads from storage; empty on first run
|
|
67
|
+
vector = np.random.rand(2560).astype(np.float32)
|
|
68
|
+
timestamp_ms = 1_700_000_000_000
|
|
69
|
+
|
|
70
|
+
new_cluster = Cluster(centroid=vector, last_ts=timestamp_ms)
|
|
71
|
+
merged = await cluster_by_geometry(
|
|
72
|
+
new_cluster,
|
|
73
|
+
existing,
|
|
74
|
+
threshold=0.65, # cosine similarity floor
|
|
75
|
+
time_window_days=7.0, # ignore clusters older than this window
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
if merged is not None:
|
|
79
|
+
# item assigned to an existing cluster; caller updates the matching entry
|
|
80
|
+
print(f"merged into cluster id={merged.id!r}, new count={merged.count}")
|
|
81
|
+
else:
|
|
82
|
+
# no match — caller appends new_cluster and stamps its own id
|
|
83
|
+
new_cluster_with_id = new_cluster.model_copy(update={"id": "cid_001"})
|
|
84
|
+
existing.append(new_cluster_with_id)
|
|
85
|
+
print("created new cluster")
|
|
86
|
+
|
|
87
|
+
asyncio.run(main())
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## LLM-refined clustering
|
|
91
|
+
|
|
92
|
+
`cluster_by_llm` adds a semantic ranking step over the top-K geometrically-nearest candidates. It **raises** on LLM failure — there is no internal fallback; the caller decides whether to retry or fall back to `cluster_by_geometry`.
|
|
93
|
+
|
|
94
|
+
Populate `new_cluster.preview` with the item's representative text so the LLM has something to rank against.
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
import asyncio
|
|
98
|
+
import numpy as np
|
|
99
|
+
from everalgo.clustering import Cluster, cluster_by_llm
|
|
100
|
+
from everalgo.llm.types import ChatResponse
|
|
101
|
+
from everalgo.testing.fake_llm import FakeLLMClient
|
|
102
|
+
|
|
103
|
+
_LLM_JSON = '{"idx": 0}'
|
|
104
|
+
|
|
105
|
+
async def main() -> None:
|
|
106
|
+
fake = FakeLLMClient(responses=[ChatResponse(content=_LLM_JSON, model="fake")])
|
|
107
|
+
existing: list[Cluster] = []
|
|
108
|
+
vector = np.random.rand(2560).astype(np.float32)
|
|
109
|
+
|
|
110
|
+
new_cluster = Cluster(
|
|
111
|
+
centroid=vector,
|
|
112
|
+
last_ts=1_700_000_000_000,
|
|
113
|
+
preview=["Python async retry patterns"], # shown to the LLM
|
|
114
|
+
)
|
|
115
|
+
merged = await cluster_by_llm(
|
|
116
|
+
new_cluster,
|
|
117
|
+
existing,
|
|
118
|
+
llm=fake,
|
|
119
|
+
k_candidates=30,
|
|
120
|
+
llm_skip_threshold=0.85,
|
|
121
|
+
)
|
|
122
|
+
print(f"merged: {merged}")
|
|
123
|
+
|
|
124
|
+
asyncio.run(main())
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Persistence pattern
|
|
128
|
+
|
|
129
|
+
`Cluster` is a frozen Pydantic model — serialise with `model_dump()` and reconstruct with `Cluster.model_validate()`. The caller owns the list and the lock:
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
raw_list = await store.load(user_id) or []
|
|
133
|
+
clusters = [Cluster.model_validate(r) for r in raw_list]
|
|
134
|
+
|
|
135
|
+
async with caller.lock(f"cluster:{user_id}"):
|
|
136
|
+
merged = await cluster_by_geometry(new_cluster, clusters)
|
|
137
|
+
if merged is not None:
|
|
138
|
+
idx = next(i for i, c in enumerate(clusters) if c.id == merged.id)
|
|
139
|
+
clusters[idx] = merged
|
|
140
|
+
else:
|
|
141
|
+
new_cluster_stamped = new_cluster.model_copy(update={"id": new_id})
|
|
142
|
+
clusters.append(new_cluster_stamped)
|
|
143
|
+
await store.save(user_id, [c.model_dump() for c in clusters])
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## API reference
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
async def cluster_by_geometry(
|
|
150
|
+
new_cluster: Cluster,
|
|
151
|
+
existing_clusters: list[Cluster],
|
|
152
|
+
*,
|
|
153
|
+
threshold: float = 0.65,
|
|
154
|
+
time_window_days: float = 7.0,
|
|
155
|
+
preview_cap: int = 5,
|
|
156
|
+
) -> Cluster | None: ...
|
|
157
|
+
|
|
158
|
+
async def cluster_by_llm(
|
|
159
|
+
new_cluster: Cluster,
|
|
160
|
+
existing_clusters: list[Cluster],
|
|
161
|
+
*,
|
|
162
|
+
llm: LLMClient,
|
|
163
|
+
k_candidates: int = 30,
|
|
164
|
+
llm_skip_threshold: float = 0.85,
|
|
165
|
+
prompt: str | None = None,
|
|
166
|
+
preview_cap: int = 5,
|
|
167
|
+
) -> Cluster | None: ...
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Both functions return the **merged** `Cluster` (existing cluster's `id` preserved, centroid/count/members updated) or `None` (no match — caller creates a new cluster entry and mints its own `id`).
|
|
171
|
+
|
|
172
|
+
Tested embedding model: `Qwen3-Embedding-4B` (2560-dim float32). Any consistent-dimension embedding works; EverAlgo does not import or manage embedding SDKs.
|
|
173
|
+
|
|
174
|
+
## Related distributions
|
|
175
|
+
|
|
176
|
+
- [`everalgo-user-memory`](../everalgo-user-memory/) — uses `cluster_by_geometry` in the full user-memory pipeline
|
|
177
|
+
- [`everalgo-agent-memory`](../everalgo-agent-memory/) — uses `cluster_by_llm` for skill clustering
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# everalgo-clustering
|
|
2
|
+
|
|
3
|
+
Online incremental clustering for EverAlgo — two async functions (`cluster_by_geometry` / `cluster_by_llm`) operating on caller-owned `list[Cluster]` state.
|
|
4
|
+
|
|
5
|
+
Stateless: the package never embeds, never queries storage, never holds a lock. The caller owns embedding computation, persistence (`list[Cluster]` serialisation), and read-modify-write coordination across concurrent writers.
|
|
6
|
+
|
|
7
|
+
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install everalgo-clustering
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## What this distribution provides
|
|
16
|
+
|
|
17
|
+
| Symbol | Role |
|
|
18
|
+
|---|---|
|
|
19
|
+
| `Cluster` | Frozen Pydantic value object — one cluster snapshot. Caller-supplied `id` / `members`; algorithm supplies merged `centroid`, `count`, `last_ts`, `preview`. |
|
|
20
|
+
| `cluster_by_geometry` | Cosine similarity + time-window filter + threshold; no LLM; async |
|
|
21
|
+
| `cluster_by_llm` | Top-K geometric recall → fast-path skip → LLM semantic ranking; raises on LLM failure; async |
|
|
22
|
+
|
|
23
|
+
## Cluster type
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
class Cluster(BaseModel):
|
|
27
|
+
id: str | None = None # caller-supplied business id; algo only passes through, never mints
|
|
28
|
+
centroid: np.ndarray
|
|
29
|
+
count: int = 1
|
|
30
|
+
last_ts: int # Unix epoch milliseconds
|
|
31
|
+
preview: list[str] = []
|
|
32
|
+
members: list[str] = [] # caller-supplied entity ids; algo appends on merge, never inspects
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The caller wraps each incoming item as a size-1 `Cluster` (count=1) before passing to either function.
|
|
36
|
+
Both functions return `Cluster | None`: a merged snapshot when the item is assigned to an existing cluster, or `None` when no match — the caller then appends the original size-1 `Cluster` as a brand-new entry and mints its own `id`.
|
|
37
|
+
|
|
38
|
+
## Quick start
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import asyncio
|
|
42
|
+
import numpy as np
|
|
43
|
+
from everalgo.clustering import Cluster, cluster_by_geometry
|
|
44
|
+
|
|
45
|
+
async def main() -> None:
|
|
46
|
+
existing: list[Cluster] = [] # caller loads from storage; empty on first run
|
|
47
|
+
vector = np.random.rand(2560).astype(np.float32)
|
|
48
|
+
timestamp_ms = 1_700_000_000_000
|
|
49
|
+
|
|
50
|
+
new_cluster = Cluster(centroid=vector, last_ts=timestamp_ms)
|
|
51
|
+
merged = await cluster_by_geometry(
|
|
52
|
+
new_cluster,
|
|
53
|
+
existing,
|
|
54
|
+
threshold=0.65, # cosine similarity floor
|
|
55
|
+
time_window_days=7.0, # ignore clusters older than this window
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if merged is not None:
|
|
59
|
+
# item assigned to an existing cluster; caller updates the matching entry
|
|
60
|
+
print(f"merged into cluster id={merged.id!r}, new count={merged.count}")
|
|
61
|
+
else:
|
|
62
|
+
# no match — caller appends new_cluster and stamps its own id
|
|
63
|
+
new_cluster_with_id = new_cluster.model_copy(update={"id": "cid_001"})
|
|
64
|
+
existing.append(new_cluster_with_id)
|
|
65
|
+
print("created new cluster")
|
|
66
|
+
|
|
67
|
+
asyncio.run(main())
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## LLM-refined clustering
|
|
71
|
+
|
|
72
|
+
`cluster_by_llm` adds a semantic ranking step over the top-K geometrically-nearest candidates. It **raises** on LLM failure — there is no internal fallback; the caller decides whether to retry or fall back to `cluster_by_geometry`.
|
|
73
|
+
|
|
74
|
+
Populate `new_cluster.preview` with the item's representative text so the LLM has something to rank against.
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import asyncio
|
|
78
|
+
import numpy as np
|
|
79
|
+
from everalgo.clustering import Cluster, cluster_by_llm
|
|
80
|
+
from everalgo.llm.types import ChatResponse
|
|
81
|
+
from everalgo.testing.fake_llm import FakeLLMClient
|
|
82
|
+
|
|
83
|
+
_LLM_JSON = '{"idx": 0}'
|
|
84
|
+
|
|
85
|
+
async def main() -> None:
|
|
86
|
+
fake = FakeLLMClient(responses=[ChatResponse(content=_LLM_JSON, model="fake")])
|
|
87
|
+
existing: list[Cluster] = []
|
|
88
|
+
vector = np.random.rand(2560).astype(np.float32)
|
|
89
|
+
|
|
90
|
+
new_cluster = Cluster(
|
|
91
|
+
centroid=vector,
|
|
92
|
+
last_ts=1_700_000_000_000,
|
|
93
|
+
preview=["Python async retry patterns"], # shown to the LLM
|
|
94
|
+
)
|
|
95
|
+
merged = await cluster_by_llm(
|
|
96
|
+
new_cluster,
|
|
97
|
+
existing,
|
|
98
|
+
llm=fake,
|
|
99
|
+
k_candidates=30,
|
|
100
|
+
llm_skip_threshold=0.85,
|
|
101
|
+
)
|
|
102
|
+
print(f"merged: {merged}")
|
|
103
|
+
|
|
104
|
+
asyncio.run(main())
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Persistence pattern
|
|
108
|
+
|
|
109
|
+
`Cluster` is a frozen Pydantic model — serialise with `model_dump()` and reconstruct with `Cluster.model_validate()`. The caller owns the list and the lock:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
raw_list = await store.load(user_id) or []
|
|
113
|
+
clusters = [Cluster.model_validate(r) for r in raw_list]
|
|
114
|
+
|
|
115
|
+
async with caller.lock(f"cluster:{user_id}"):
|
|
116
|
+
merged = await cluster_by_geometry(new_cluster, clusters)
|
|
117
|
+
if merged is not None:
|
|
118
|
+
idx = next(i for i, c in enumerate(clusters) if c.id == merged.id)
|
|
119
|
+
clusters[idx] = merged
|
|
120
|
+
else:
|
|
121
|
+
new_cluster_stamped = new_cluster.model_copy(update={"id": new_id})
|
|
122
|
+
clusters.append(new_cluster_stamped)
|
|
123
|
+
await store.save(user_id, [c.model_dump() for c in clusters])
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## API reference
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
async def cluster_by_geometry(
|
|
130
|
+
new_cluster: Cluster,
|
|
131
|
+
existing_clusters: list[Cluster],
|
|
132
|
+
*,
|
|
133
|
+
threshold: float = 0.65,
|
|
134
|
+
time_window_days: float = 7.0,
|
|
135
|
+
preview_cap: int = 5,
|
|
136
|
+
) -> Cluster | None: ...
|
|
137
|
+
|
|
138
|
+
async def cluster_by_llm(
|
|
139
|
+
new_cluster: Cluster,
|
|
140
|
+
existing_clusters: list[Cluster],
|
|
141
|
+
*,
|
|
142
|
+
llm: LLMClient,
|
|
143
|
+
k_candidates: int = 30,
|
|
144
|
+
llm_skip_threshold: float = 0.85,
|
|
145
|
+
prompt: str | None = None,
|
|
146
|
+
preview_cap: int = 5,
|
|
147
|
+
) -> Cluster | None: ...
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Both functions return the **merged** `Cluster` (existing cluster's `id` preserved, centroid/count/members updated) or `None` (no match — caller creates a new cluster entry and mints its own `id`).
|
|
151
|
+
|
|
152
|
+
Tested embedding model: `Qwen3-Embedding-4B` (2560-dim float32). Any consistent-dimension embedding works; EverAlgo does not import or manage embedding SDKs.
|
|
153
|
+
|
|
154
|
+
## Related distributions
|
|
155
|
+
|
|
156
|
+
- [`everalgo-user-memory`](../everalgo-user-memory/) — uses `cluster_by_geometry` in the full user-memory pipeline
|
|
157
|
+
- [`everalgo-agent-memory`](../everalgo-agent-memory/) — uses `cluster_by_llm` for skill clustering
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "everalgo-clustering"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "EverAlgo clustering: cluster_by_geometry / cluster_by_llm functional operators with ClusterState value object."
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
authors = [{ name = "EverMind" }]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
]
|
|
18
|
+
dependencies = [
|
|
19
|
+
"everalgo-core>=0.1.0,<2.0.0",
|
|
20
|
+
"numpy>=1.26",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/EverMind-AI/EverAlgo"
|
|
25
|
+
Repository = "https://github.com/EverMind-AI/EverAlgo"
|
|
26
|
+
Issues = "https://github.com/EverMind-AI/EverAlgo/issues"
|
|
27
|
+
Documentation = "https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-clustering"
|
|
28
|
+
Changelog = "https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-clustering/CHANGELOG.md"
|
|
29
|
+
|
|
30
|
+
[tool.hatch.build.targets.wheel]
|
|
31
|
+
packages = ["src/everalgo"]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Online incremental clustering operators — pure-function state-in / state-out.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
|
|
5
|
+
* :func:`cluster_by_geometry` — cosine similarity + time-window filter + threshold; no LLM.
|
|
6
|
+
* :func:`cluster_by_llm` — embedding top-K recall + fast-path skip + LLM ranking.
|
|
7
|
+
* :class:`Cluster` — frozen value object the caller threads through and persists as list[Cluster].
|
|
8
|
+
|
|
9
|
+
Caller responsibilities: embedding (EverAlgo accepts any np.ndarray dimension as long as it is
|
|
10
|
+
consistent across calls), persistence (Cluster.model_dump() / model_validate()), and
|
|
11
|
+
read-modify-write serialisation if multiple writers share a list.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
|
|
16
|
+
from everalgo.clustering.algorithm import cluster_by_geometry, cluster_by_llm
|
|
17
|
+
from everalgo.clustering.state import Cluster
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"Cluster",
|
|
21
|
+
"cluster_by_geometry",
|
|
22
|
+
"cluster_by_llm",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
# ADR-013: per-subpackage logger gets a NullHandler so library use never emits to stderr by default.
|
|
26
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Cluster operators — geometry and LLM-refined paths over a list[Cluster] state.
|
|
2
|
+
|
|
3
|
+
Both operators are async and stateless: accept a new_cluster + existing_clusters list, return
|
|
4
|
+
a merged Cluster or None without mutating any input. The caller owns persistence and concurrency control.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from everalgo.clustering.prompts.en.cluster import CLUSTER_LLM_ASSIGN_PROMPT
|
|
16
|
+
from everalgo.clustering.state import Cluster
|
|
17
|
+
from everalgo.llm.parse import parse_llm_json_object
|
|
18
|
+
from everalgo.llm.types import ChatMessage as LLMChatMessage
|
|
19
|
+
from everalgo.prompts import render_prompt
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from everalgo.llm.protocols import LLMClient
|
|
23
|
+
|
|
24
|
+
__all__ = ["cluster_by_geometry", "cluster_by_llm"]
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
_MS_PER_DAY = 86_400_000
|
|
29
|
+
_NORM_EPSILON = 1e-9
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
async def cluster_by_geometry(
|
|
33
|
+
new_cluster: Cluster,
|
|
34
|
+
existing_clusters: list[Cluster],
|
|
35
|
+
*,
|
|
36
|
+
threshold: float = 0.65,
|
|
37
|
+
time_window_days: float = 7.0,
|
|
38
|
+
preview_cap: int = 5,
|
|
39
|
+
) -> Cluster | None:
|
|
40
|
+
"""Cosine + time-window assignment.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
new_cluster: Size-1 Cluster representing the incoming item.
|
|
44
|
+
existing_clusters: Caller-owned list; never mutated.
|
|
45
|
+
threshold: Cosine similarity >= this assigns to top-1. Default 0.65.
|
|
46
|
+
time_window_days: Clusters whose last_ts is older than this gap are excluded. Default 7.0.
|
|
47
|
+
preview_cap: Maximum preview length on the merged snapshot. Default 5.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
Merged Cluster when assigned to an existing one (id is passed through from
|
|
51
|
+
existing_clusters[i].id), or None when no match — caller appends new_cluster as a brand-new
|
|
52
|
+
cluster and mints its own id.
|
|
53
|
+
"""
|
|
54
|
+
if not existing_clusters:
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
idx = _find_best_within_window(
|
|
58
|
+
existing_clusters, new_cluster.centroid, new_cluster.last_ts, threshold, time_window_days
|
|
59
|
+
)
|
|
60
|
+
if idx is None:
|
|
61
|
+
return None
|
|
62
|
+
return _merge(existing_clusters[idx], new_cluster, preview_cap=preview_cap)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def cluster_by_llm(
|
|
66
|
+
new_cluster: Cluster,
|
|
67
|
+
existing_clusters: list[Cluster],
|
|
68
|
+
*,
|
|
69
|
+
llm: LLMClient,
|
|
70
|
+
k_candidates: int = 30,
|
|
71
|
+
llm_skip_threshold: float = 0.85,
|
|
72
|
+
prompt: str | None = None,
|
|
73
|
+
preview_cap: int = 5,
|
|
74
|
+
) -> Cluster | None:
|
|
75
|
+
"""LLM-refined assignment: top-K recall → fast path → LLM ranking by idx.
|
|
76
|
+
|
|
77
|
+
LLM ranking query text comes from new_cluster.preview[0] if preview else "".
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
new_cluster: Size-1 Cluster representing the incoming item.
|
|
81
|
+
existing_clusters: Caller-owned list; never mutated.
|
|
82
|
+
llm: LLM client for the Stage-2 ranking call.
|
|
83
|
+
k_candidates: Number of nearest clusters fetched before LLM ranking. Default 30.
|
|
84
|
+
llm_skip_threshold: Skip LLM and assign to top-1 when its similarity meets this. Default 0.85.
|
|
85
|
+
prompt: Optional override of the built-in LLM ranking prompt template.
|
|
86
|
+
preview_cap: Maximum preview length on the merged snapshot. Default 5.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
Merged Cluster when LLM ranks a match (id passed through), or None when LLM picks
|
|
90
|
+
"new" — caller appends new_cluster as a brand-new cluster and mints its own id.
|
|
91
|
+
LLM internally uses idx; idx -> existing_clusters[idx].id transparent to caller.
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
LLMError: Propagated from the underlying LLM client call.
|
|
95
|
+
ValueError: If the LLM response cannot be parsed or is missing the 'idx' field.
|
|
96
|
+
"""
|
|
97
|
+
if not existing_clusters:
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
candidates = _find_top_k_clusters(existing_clusters, new_cluster.centroid, k_candidates)
|
|
101
|
+
if not candidates:
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
top_idx, top_sim = candidates[0]
|
|
105
|
+
|
|
106
|
+
if top_sim >= llm_skip_threshold:
|
|
107
|
+
return _merge(existing_clusters[top_idx], new_cluster, preview_cap=preview_cap)
|
|
108
|
+
|
|
109
|
+
query_text = new_cluster.preview[0] if new_cluster.preview else ""
|
|
110
|
+
candidate_indices = [idx for idx, _ in candidates]
|
|
111
|
+
clusters_json = _build_clusters_json(existing_clusters, candidate_indices)
|
|
112
|
+
rendered = render_prompt(
|
|
113
|
+
CLUSTER_LLM_ASSIGN_PROMPT,
|
|
114
|
+
prompt,
|
|
115
|
+
memcell_text=query_text,
|
|
116
|
+
clusters_json=clusters_json,
|
|
117
|
+
)
|
|
118
|
+
response = await llm.chat(
|
|
119
|
+
messages=[LLMChatMessage(role="user", content=rendered)],
|
|
120
|
+
response_format={"type": "json_object"},
|
|
121
|
+
)
|
|
122
|
+
llm_result = parse_llm_json_object(response.content)
|
|
123
|
+
if "idx" not in llm_result:
|
|
124
|
+
raise ValueError("cluster_by_llm: LLM response missing 'idx' field")
|
|
125
|
+
|
|
126
|
+
chosen_idx = llm_result.get("idx")
|
|
127
|
+
if isinstance(chosen_idx, int) and 0 <= chosen_idx < len(existing_clusters):
|
|
128
|
+
return _merge(existing_clusters[chosen_idx], new_cluster, preview_cap=preview_cap)
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# --- private helpers --------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _merge(existing: Cluster, new: Cluster, *, preview_cap: int) -> Cluster:
|
|
136
|
+
"""Weighted centroid merge + preview concat + members append."""
|
|
137
|
+
total = existing.count + new.count
|
|
138
|
+
return Cluster(
|
|
139
|
+
id=existing.id, # transparent passthrough — algo never generates id
|
|
140
|
+
centroid=(existing.centroid * existing.count + new.centroid * new.count) / total,
|
|
141
|
+
count=total,
|
|
142
|
+
last_ts=max(existing.last_ts, new.last_ts),
|
|
143
|
+
preview=(existing.preview + new.preview)[-preview_cap:],
|
|
144
|
+
members=existing.members + new.members,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _cosine(v: np.ndarray, c: np.ndarray) -> float:
|
|
149
|
+
"""Cosine similarity with _NORM_EPSILON divide-by-zero guard."""
|
|
150
|
+
v_norm = float(np.linalg.norm(v)) + _NORM_EPSILON
|
|
151
|
+
c_norm = float(np.linalg.norm(c)) + _NORM_EPSILON
|
|
152
|
+
return float(np.dot(v, c) / (v_norm * c_norm))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _find_best_within_window(
|
|
156
|
+
clusters: list[Cluster],
|
|
157
|
+
vector: np.ndarray,
|
|
158
|
+
timestamp_ms: int,
|
|
159
|
+
threshold: float,
|
|
160
|
+
time_window_days: float,
|
|
161
|
+
) -> int | None:
|
|
162
|
+
"""Top-1 cluster index within the time window clearing threshold; None otherwise."""
|
|
163
|
+
max_gap_ms = int(time_window_days * _MS_PER_DAY)
|
|
164
|
+
best_idx: int | None = None
|
|
165
|
+
best_sim = -1.0
|
|
166
|
+
|
|
167
|
+
for i, cluster in enumerate(clusters):
|
|
168
|
+
if cluster.centroid.size == 0:
|
|
169
|
+
continue
|
|
170
|
+
if abs(timestamp_ms - cluster.last_ts) > max_gap_ms:
|
|
171
|
+
continue
|
|
172
|
+
sim = _cosine(vector, cluster.centroid)
|
|
173
|
+
if sim > best_sim:
|
|
174
|
+
best_sim = sim
|
|
175
|
+
best_idx = i
|
|
176
|
+
|
|
177
|
+
if best_idx is not None and best_sim >= threshold:
|
|
178
|
+
return best_idx
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _find_top_k_clusters(
|
|
183
|
+
clusters: list[Cluster],
|
|
184
|
+
vector: np.ndarray,
|
|
185
|
+
k: int,
|
|
186
|
+
) -> list[tuple[int, float]]:
|
|
187
|
+
"""Top-K clusters by cosine similarity (no time-window filter); empty-centroid clusters skipped."""
|
|
188
|
+
scored: list[tuple[int, float]] = []
|
|
189
|
+
for i, cluster in enumerate(clusters):
|
|
190
|
+
if cluster.centroid.size == 0:
|
|
191
|
+
continue
|
|
192
|
+
scored.append((i, _cosine(vector, cluster.centroid)))
|
|
193
|
+
scored.sort(key=lambda x: x[1], reverse=True)
|
|
194
|
+
return scored[:k]
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _build_clusters_json(clusters: list[Cluster], candidate_indices: list[int]) -> str:
|
|
198
|
+
"""Serialise candidate clusters to JSON for the {clusters_json} prompt placeholder."""
|
|
199
|
+
if not candidate_indices:
|
|
200
|
+
return "(No existing clusters)"
|
|
201
|
+
|
|
202
|
+
items = [
|
|
203
|
+
{
|
|
204
|
+
"idx": idx,
|
|
205
|
+
"count": clusters[idx].count,
|
|
206
|
+
"preview": clusters[idx].preview,
|
|
207
|
+
}
|
|
208
|
+
for idx in candidate_indices
|
|
209
|
+
]
|
|
210
|
+
return json.dumps(items, ensure_ascii=False, indent=2)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""English clustering prompts."""
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""English clustering prompts for LLM-based cluster assignment.
|
|
2
|
+
|
|
3
|
+
Placeholders (rendered via :py:meth:`str.format`):
|
|
4
|
+
- ``{memcell_text}`` — text representation of the new event (caller-supplied; typically the
|
|
5
|
+
preview[0] of the incoming Cluster).
|
|
6
|
+
- ``{clusters_json}`` — JSON list of candidate clusters as
|
|
7
|
+
``[{"idx": int, "count": int, "preview": [...]}]``.
|
|
8
|
+
|
|
9
|
+
Output schema (the LLM must return strictly): ``{"idx": int, "reason": str}``.
|
|
10
|
+
Use ``{"idx": -1, ...}`` to signal a new cluster (sentinel; avoids null ambiguity).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
CLUSTER_LLM_ASSIGN_PROMPT = """You are a clustering expert. Your goal is to group similar and related tasks together so that patterns and reusable strategies can be extracted from each cluster. Assign the new task intent to an existing cluster, or signal a new cluster if no existing cluster fits.
|
|
14
|
+
|
|
15
|
+
[How to decide]
|
|
16
|
+
The goal of clustering is to group cases that would produce a **specific, actionable skill** — not generic advice. Use this test: "Would an agent who solved one task in this cluster have a **concrete advantage** (reusable tools, domain knowledge, verified strategies) when facing the other tasks?"
|
|
17
|
+
|
|
18
|
+
1. **Identify two dimensions**: the task's **subject domain** (e.g., medical research, urban planning, e-commerce) and its **problem-solving pattern** (e.g., root cause analysis, constraint satisfaction, data pipeline design).
|
|
19
|
+
2. **Cluster by the more specific dimension**. If the domain is already narrow (e.g., "clinical trial data extraction"), domain alone is enough. If the domain is broad (e.g., "software engineering"), use the problem-solving pattern to differentiate (e.g., "performance profiling" vs. "schema migration").
|
|
20
|
+
3. **Do NOT merge across unrelated domains just because the strategy is similar.** "Diagnose a patient's symptoms via differential diagnosis" and "diagnose a supply chain bottleneck via constraint analysis" both use diagnostic reasoning, but involve completely different domain knowledge and belong in separate clusters.
|
|
21
|
+
4. Scan candidate clusters. Prefer the cluster whose existing items would **benefit most from sharing a skill** with the new task.
|
|
22
|
+
5. Signal a new cluster (idx: -1) only when no candidate cluster is a good fit.
|
|
23
|
+
|
|
24
|
+
[Candidate Clusters]
|
|
25
|
+
Each cluster is represented by its list index (idx), item count, and recent preview texts.
|
|
26
|
+
{clusters_json}
|
|
27
|
+
|
|
28
|
+
[New Task Intent]
|
|
29
|
+
{memcell_text}
|
|
30
|
+
|
|
31
|
+
[Rules]
|
|
32
|
+
- Output decision as JSON. Keep "reason" under 50 tokens.
|
|
33
|
+
- To assign to an existing cluster: use its idx (non-negative integer).
|
|
34
|
+
- To create a new cluster: use idx -1.
|
|
35
|
+
|
|
36
|
+
Return ONLY valid JSON (no markdown fences, no explanation):
|
|
37
|
+
{{"idx": <existing_idx or -1>, "reason": "short reason"}}
|
|
38
|
+
"""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Chinese clustering prompts."""
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Chinese clustering prompts — re-export of the English template.
|
|
2
|
+
|
|
3
|
+
The clustering LLM template is language-neutral: responses match the candidate corpus' language while
|
|
4
|
+
the template itself works either way. This module re-exports the English constant so callers that
|
|
5
|
+
import from ``prompts/zh/`` get the same prompt without duplication.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from everalgo.clustering.prompts.en.cluster import CLUSTER_LLM_ASSIGN_PROMPT
|
|
9
|
+
|
|
10
|
+
__all__ = ["CLUSTER_LLM_ASSIGN_PROMPT"]
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Cluster value object — frozen Pydantic model; caller persists list[Cluster] across calls.
|
|
2
|
+
|
|
3
|
+
Each Cluster carries its own centroid/count/last_ts/preview. The clustering operators treat an
|
|
4
|
+
incoming item as a size-1 Cluster (BIRCH CF-tree N=1 equivalence), so input and existing
|
|
5
|
+
elements share one type.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np # noqa: TC002 — np.ndarray is a Pydantic field type; Pydantic resolves it at runtime
|
|
11
|
+
from pydantic import BaseModel, ConfigDict
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Cluster(BaseModel):
|
|
15
|
+
"""Frozen value object representing one cluster snapshot.
|
|
16
|
+
|
|
17
|
+
``members`` carries caller-supplied entity ids; the algorithm appends them on merge but never
|
|
18
|
+
inspects their semantics.
|
|
19
|
+
|
|
20
|
+
For an incoming item, build a size-1 Cluster (count=1 default; preview=[item_text] when
|
|
21
|
+
cluster_by_llm will be called).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
id: str | None = None # caller-supplied business id; algo only passes it through, never mints
|
|
25
|
+
centroid: np.ndarray
|
|
26
|
+
count: int = 1
|
|
27
|
+
last_ts: int # ms epoch
|
|
28
|
+
preview: list[str] = []
|
|
29
|
+
members: list[str] = [] # caller-supplied entity ids (e.g. MemCell.id); algo appends on merge, no cap
|
|
30
|
+
|
|
31
|
+
model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
|
|
File without changes
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Behaviour tests for :func:`everalgo.clustering.cluster_by_geometry`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from everalgo.clustering import Cluster, cluster_by_geometry
|
|
8
|
+
|
|
9
|
+
_BASE_TS_MS = 1_700_000_000_000 # 2023-11-14, far enough from epoch to be realistic.
|
|
10
|
+
_DAY_MS = 86_400_000
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _c(vec: list[float], ts: int = _BASE_TS_MS, count: int = 1, cid: str | None = None) -> Cluster:
|
|
14
|
+
return Cluster(centroid=np.array(vec, dtype=np.float32), last_ts=ts, count=count, id=cid)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
async def test_empty_existing_returns_none() -> None:
|
|
18
|
+
new_c = _c([1.0, 0.0])
|
|
19
|
+
|
|
20
|
+
result = await cluster_by_geometry(new_c, [])
|
|
21
|
+
|
|
22
|
+
assert result is None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
async def test_above_threshold_returns_merged_cluster() -> None:
|
|
26
|
+
existing = [_c([1.0, 0.0], cid="cid_0")]
|
|
27
|
+
# cosine([0.99, 0.01], [1,0]) ≈ 0.9999 > 0.65 threshold.
|
|
28
|
+
new_c = _c([0.99, 0.01], ts=_BASE_TS_MS + 1000)
|
|
29
|
+
|
|
30
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
31
|
+
|
|
32
|
+
assert result is not None
|
|
33
|
+
assert result.id == "cid_0"
|
|
34
|
+
assert result.count == 2
|
|
35
|
+
assert result.last_ts == _BASE_TS_MS + 1000
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def test_below_threshold_returns_none() -> None:
|
|
39
|
+
existing = [_c([1.0, 0.0])]
|
|
40
|
+
# cosine([1,0], [0,1]) = 0 < 0.65 threshold → new cluster.
|
|
41
|
+
new_c = _c([0.0, 1.0], ts=_BASE_TS_MS + 1000)
|
|
42
|
+
|
|
43
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
44
|
+
|
|
45
|
+
assert result is None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
async def test_time_window_excludes_old_clusters() -> None:
|
|
49
|
+
old_ts = _BASE_TS_MS - 10 * _DAY_MS # 10 days ago > 7-day window.
|
|
50
|
+
existing = [_c([1.0, 0.0], ts=old_ts)]
|
|
51
|
+
# Cosine match would be perfect, but time window kicks the candidate out → new cluster.
|
|
52
|
+
new_c = _c([1.0, 0.0])
|
|
53
|
+
|
|
54
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
55
|
+
|
|
56
|
+
assert result is None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def test_centroid_merges_via_weighted_average() -> None:
|
|
60
|
+
# count=2, centroid=[1.0, 0.0]. New vector [0.95, 0.05] → weighted avg.
|
|
61
|
+
existing = [_c([1.0, 0.0], count=2)]
|
|
62
|
+
new_vec = np.array([0.95, 0.05], dtype=np.float32)
|
|
63
|
+
new_c = Cluster(centroid=new_vec, last_ts=_BASE_TS_MS, count=1)
|
|
64
|
+
|
|
65
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
66
|
+
|
|
67
|
+
assert result is not None
|
|
68
|
+
expected = (np.array([1.0, 0.0], dtype=np.float32) * 2 + new_vec) / 3
|
|
69
|
+
np.testing.assert_allclose(result.centroid, expected, rtol=1e-5)
|
|
70
|
+
assert result.count == 3
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def test_existing_unchanged_after_call() -> None:
|
|
74
|
+
existing = [_c([1.0, 0.0])]
|
|
75
|
+
original_centroid = existing[0].centroid.copy()
|
|
76
|
+
new_c = _c([0.99, 0.01], ts=_BASE_TS_MS + 1)
|
|
77
|
+
|
|
78
|
+
await cluster_by_geometry(new_c, existing)
|
|
79
|
+
|
|
80
|
+
# existing list itself and Cluster inside must be unchanged (frozen).
|
|
81
|
+
np.testing.assert_array_equal(existing[0].centroid, original_centroid)
|
|
82
|
+
assert existing[0].count == 1
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
async def test_last_ts_uses_max() -> None:
|
|
86
|
+
# Existing ts is newer; merge should keep the larger of the two.
|
|
87
|
+
later_ts = _BASE_TS_MS + 5_000
|
|
88
|
+
existing = [_c([1.0, 0.0], ts=later_ts)]
|
|
89
|
+
new_c = _c([0.99, 0.01], ts=_BASE_TS_MS) # older timestamp
|
|
90
|
+
|
|
91
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
92
|
+
|
|
93
|
+
assert result is not None
|
|
94
|
+
assert result.last_ts == later_ts
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
async def test_preview_is_merged_and_capped() -> None:
|
|
98
|
+
existing = [Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=_BASE_TS_MS, count=1, preview=["old"])]
|
|
99
|
+
new_c = Cluster(
|
|
100
|
+
centroid=np.array([0.99, 0.01], dtype=np.float32), last_ts=_BASE_TS_MS + 1, count=1, preview=["new"]
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
result = await cluster_by_geometry(new_c, existing, preview_cap=5)
|
|
104
|
+
|
|
105
|
+
assert result is not None
|
|
106
|
+
assert result.preview == ["old", "new"]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
async def test_pick_best_among_multiple_existing() -> None:
|
|
110
|
+
existing = [
|
|
111
|
+
_c([1.0, 0.0], cid="cid_0"), # cosine([0.9, 0.1], ...) ≈ 0.994
|
|
112
|
+
_c([0.0, 1.0], cid="cid_1"), # cosine([0.9, 0.1], ...) ≈ 0.099
|
|
113
|
+
]
|
|
114
|
+
new_c = _c([0.9, 0.1], ts=_BASE_TS_MS + 1)
|
|
115
|
+
|
|
116
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
117
|
+
|
|
118
|
+
assert result is not None
|
|
119
|
+
assert result.id == "cid_0" # best match
|
|
120
|
+
assert result.count == 2
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
async def test_merge_appends_members() -> None:
|
|
124
|
+
existing = [
|
|
125
|
+
Cluster(
|
|
126
|
+
centroid=np.array([1.0, 0.0], dtype=np.float32),
|
|
127
|
+
last_ts=_BASE_TS_MS,
|
|
128
|
+
members=["a", "b"],
|
|
129
|
+
)
|
|
130
|
+
]
|
|
131
|
+
new_c = Cluster(
|
|
132
|
+
centroid=np.array([0.99, 0.01], dtype=np.float32),
|
|
133
|
+
last_ts=_BASE_TS_MS + 1,
|
|
134
|
+
members=["c"],
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
result = await cluster_by_geometry(new_c, existing)
|
|
138
|
+
|
|
139
|
+
assert result is not None
|
|
140
|
+
assert result.members == ["a", "b", "c"]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async def test_none_path_preserves_new_members() -> None:
|
|
144
|
+
# No existing clusters → None returned; caller uses new_c as-is with its members intact.
|
|
145
|
+
new_c = Cluster(
|
|
146
|
+
centroid=np.array([1.0, 0.0], dtype=np.float32),
|
|
147
|
+
last_ts=_BASE_TS_MS,
|
|
148
|
+
members=["entity_x"],
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
result = await cluster_by_geometry(new_c, [])
|
|
152
|
+
|
|
153
|
+
assert result is None
|
|
154
|
+
assert new_c.members == ["entity_x"]
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Behaviour tests for :func:`everalgo.clustering.cluster_by_llm`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from everalgo.clustering import Cluster, cluster_by_llm
|
|
11
|
+
from everalgo.llm.types import ChatMessage, ChatResponse
|
|
12
|
+
from everalgo.testing.fake_llm import FakeLLMClient
|
|
13
|
+
|
|
14
|
+
_BASE_TS_MS = 1_700_000_000_000
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _c(vec: list[float], ts: int = _BASE_TS_MS, count: int = 1, preview: list[str] | None = None) -> Cluster:
|
|
18
|
+
return Cluster(centroid=np.array(vec, dtype=np.float32), last_ts=ts, count=count, preview=preview or [])
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _two_clusters() -> list[Cluster]:
|
|
22
|
+
return [
|
|
23
|
+
_c([1.0, 0.0], count=3),
|
|
24
|
+
_c([0.0, 1.0], count=2),
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
async def test_empty_existing_skips_llm_and_returns_none() -> None:
|
|
29
|
+
new_c = _c([1.0, 0.0])
|
|
30
|
+
llm = FakeLLMClient(responses=[])
|
|
31
|
+
|
|
32
|
+
result = await cluster_by_llm(new_c, [], llm=llm)
|
|
33
|
+
|
|
34
|
+
assert result is None
|
|
35
|
+
assert llm.call_count == 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def test_fast_path_top1_above_skip_threshold_skips_llm() -> None:
|
|
39
|
+
existing = _two_clusters()
|
|
40
|
+
# cosine([0.999, 0.001], [1,0]) ≈ 0.9999 > 0.85 skip threshold.
|
|
41
|
+
new_c = _c([0.999, 0.001])
|
|
42
|
+
llm = FakeLLMClient(responses=[])
|
|
43
|
+
|
|
44
|
+
result = await cluster_by_llm(new_c, existing, llm=llm)
|
|
45
|
+
|
|
46
|
+
assert result is not None
|
|
47
|
+
assert result.count == 4
|
|
48
|
+
assert llm.call_count == 0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
async def test_llm_called_when_top1_below_skip_threshold() -> None:
|
|
52
|
+
existing = _two_clusters()
|
|
53
|
+
# cosine([0.7, 0.7], [1,0]) ≈ 0.707 — below 0.85 skip.
|
|
54
|
+
new_c = _c([0.7, 0.7], preview=["ambiguous task"])
|
|
55
|
+
llm = FakeLLMClient(responses=['{"idx": 1, "reason": "matches case bucket"}'])
|
|
56
|
+
|
|
57
|
+
result = await cluster_by_llm(new_c, existing, llm=llm)
|
|
58
|
+
|
|
59
|
+
assert llm.call_count == 1
|
|
60
|
+
assert result is not None
|
|
61
|
+
assert result.count == 3 # LLM's choice (existing[1], count=2) + new
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
async def test_llm_returns_minus_one_creates_new() -> None:
|
|
65
|
+
existing = _two_clusters()
|
|
66
|
+
new_c = _c([0.7, 0.7], preview=["novel task"])
|
|
67
|
+
llm = FakeLLMClient(responses=['{"idx": -1, "reason": "new domain"}'])
|
|
68
|
+
|
|
69
|
+
result = await cluster_by_llm(new_c, existing, llm=llm)
|
|
70
|
+
|
|
71
|
+
assert result is None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
async def test_llm_returns_out_of_range_idx_creates_new() -> None:
|
|
75
|
+
existing = _two_clusters()
|
|
76
|
+
new_c = _c([0.7, 0.7])
|
|
77
|
+
llm = FakeLLMClient(responses=['{"idx": 999, "reason": "new domain"}'])
|
|
78
|
+
|
|
79
|
+
result = await cluster_by_llm(new_c, existing, llm=llm)
|
|
80
|
+
|
|
81
|
+
assert result is None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
async def test_bad_llm_response_raises_value_error() -> None:
|
|
85
|
+
"""Non-JSON response from LLM → ValueError propagates (no retry, no fallback)."""
|
|
86
|
+
existing = _two_clusters()
|
|
87
|
+
new_c = _c([0.7, 0.7])
|
|
88
|
+
llm = FakeLLMClient(responses=["not json"])
|
|
89
|
+
|
|
90
|
+
with pytest.raises(ValueError):
|
|
91
|
+
await cluster_by_llm(new_c, existing, llm=llm)
|
|
92
|
+
|
|
93
|
+
assert llm.call_count == 1
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def test_llm_missing_idx_field_raises_value_error() -> None:
|
|
97
|
+
"""Valid JSON but missing 'idx' field → ValueError propagates (no fallback)."""
|
|
98
|
+
existing = _two_clusters()
|
|
99
|
+
new_c = _c([0.7, 0.7])
|
|
100
|
+
llm = FakeLLMClient(responses=['{"reason": "x"}'])
|
|
101
|
+
|
|
102
|
+
with pytest.raises(ValueError, match="missing 'idx' field"):
|
|
103
|
+
await cluster_by_llm(new_c, existing, llm=llm)
|
|
104
|
+
|
|
105
|
+
assert llm.call_count == 1
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
async def test_llm_clusters_json_contains_idx_count_preview() -> None:
|
|
109
|
+
existing = [
|
|
110
|
+
_c([1.0, 0.0], count=3, preview=["fix login bug"]),
|
|
111
|
+
_c([0.0, 1.0], count=2, preview=["refactor auth"]),
|
|
112
|
+
]
|
|
113
|
+
new_c = _c([0.7, 0.7], preview=["investigate token leak"])
|
|
114
|
+
captured: dict[str, str] = {}
|
|
115
|
+
|
|
116
|
+
def handler(messages: list[ChatMessage], **_kwargs: Any) -> ChatResponse:
|
|
117
|
+
assert isinstance(messages[0].content, str) # narrow for test
|
|
118
|
+
captured["prompt"] = messages[0].content
|
|
119
|
+
return ChatResponse(
|
|
120
|
+
content='{"idx": 0, "reason": "test"}',
|
|
121
|
+
model="fake",
|
|
122
|
+
usage=None,
|
|
123
|
+
finish_reason="stop",
|
|
124
|
+
raw=None,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
llm = FakeLLMClient(handler=handler)
|
|
128
|
+
|
|
129
|
+
await cluster_by_llm(new_c, existing, llm=llm)
|
|
130
|
+
|
|
131
|
+
rendered = captured["prompt"]
|
|
132
|
+
assert "investigate token leak" in rendered
|
|
133
|
+
assert "fix login bug" in rendered
|
|
134
|
+
assert "refactor auth" in rendered
|
|
135
|
+
assert '"count": 3' in rendered
|
|
136
|
+
assert '"idx": 0' in rendered
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def test_caller_supplied_prompt_overrides_default() -> None:
|
|
140
|
+
existing = _two_clusters()
|
|
141
|
+
new_c = _c([0.7, 0.7], preview=["hello"])
|
|
142
|
+
custom = "OVERRIDE memcell={memcell_text} clusters={clusters_json}"
|
|
143
|
+
captured: dict[str, str] = {}
|
|
144
|
+
|
|
145
|
+
def handler(messages: list[ChatMessage], **_kwargs: Any) -> ChatResponse:
|
|
146
|
+
assert isinstance(messages[0].content, str) # narrow for test
|
|
147
|
+
captured["prompt"] = messages[0].content
|
|
148
|
+
return ChatResponse(
|
|
149
|
+
content='{"idx": 0, "reason": "ok"}',
|
|
150
|
+
model="fake",
|
|
151
|
+
usage=None,
|
|
152
|
+
finish_reason="stop",
|
|
153
|
+
raw=None,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
llm = FakeLLMClient(handler=handler)
|
|
157
|
+
|
|
158
|
+
await cluster_by_llm(new_c, existing, llm=llm, prompt=custom)
|
|
159
|
+
|
|
160
|
+
assert captured["prompt"].startswith("OVERRIDE memcell=hello")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@pytest.mark.parametrize(
|
|
164
|
+
"raw",
|
|
165
|
+
[
|
|
166
|
+
'```json\n{"idx": 0, "reason": "fence-wrapped"}\n```',
|
|
167
|
+
],
|
|
168
|
+
)
|
|
169
|
+
async def test_llm_response_with_json_fence_is_parsed(raw: str) -> None:
|
|
170
|
+
existing = _two_clusters()
|
|
171
|
+
new_c = _c([0.7, 0.7])
|
|
172
|
+
llm = FakeLLMClient(responses=[raw])
|
|
173
|
+
|
|
174
|
+
result = await cluster_by_llm(new_c, existing, llm=llm)
|
|
175
|
+
|
|
176
|
+
assert result is not None
|
|
177
|
+
assert llm.call_count == 1
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
async def test_preview_empty_when_new_cluster_has_no_preview() -> None:
|
|
181
|
+
"""When new_cluster.preview is empty, query_text is '' and prompt still renders."""
|
|
182
|
+
existing = _two_clusters()
|
|
183
|
+
new_c = _c([0.7, 0.7]) # no preview
|
|
184
|
+
captured: dict[str, str] = {}
|
|
185
|
+
|
|
186
|
+
def handler(messages: list[ChatMessage], **_kwargs: Any) -> ChatResponse:
|
|
187
|
+
assert isinstance(messages[0].content, str) # narrow for test
|
|
188
|
+
captured["prompt"] = messages[0].content
|
|
189
|
+
return ChatResponse(
|
|
190
|
+
content='{"idx": 0, "reason": "ok"}', model="fake", usage=None, finish_reason="stop", raw=None
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
llm = FakeLLMClient(handler=handler)
|
|
194
|
+
await cluster_by_llm(new_c, existing, llm=llm)
|
|
195
|
+
|
|
196
|
+
assert "idx" in captured["prompt"] # prompt rendered without error
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
async def test_merge_appends_members_via_llm_path() -> None:
|
|
200
|
+
existing = [
|
|
201
|
+
Cluster(
|
|
202
|
+
centroid=np.array([1.0, 0.0], dtype=np.float32),
|
|
203
|
+
last_ts=_BASE_TS_MS,
|
|
204
|
+
count=2,
|
|
205
|
+
members=["a", "b"],
|
|
206
|
+
)
|
|
207
|
+
]
|
|
208
|
+
new_c = Cluster(
|
|
209
|
+
centroid=np.array([0.999, 0.001], dtype=np.float32),
|
|
210
|
+
last_ts=_BASE_TS_MS + 1,
|
|
211
|
+
members=["c"],
|
|
212
|
+
)
|
|
213
|
+
# cosine ~0.9999 >= 0.85 skip threshold → fast-path merge, no LLM call.
|
|
214
|
+
llm = FakeLLMClient(responses=[])
|
|
215
|
+
|
|
216
|
+
result = await cluster_by_llm(new_c, existing, llm=llm)
|
|
217
|
+
|
|
218
|
+
assert result is not None
|
|
219
|
+
assert result.members == ["a", "b", "c"]
|
|
220
|
+
assert llm.call_count == 0
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
async def test_none_path_preserves_new_members_llm() -> None:
|
|
224
|
+
# Empty existing → None; caller keeps new_c with its members.
|
|
225
|
+
new_c = Cluster(
|
|
226
|
+
centroid=np.array([1.0, 0.0], dtype=np.float32),
|
|
227
|
+
last_ts=_BASE_TS_MS,
|
|
228
|
+
members=["entity_y"],
|
|
229
|
+
)
|
|
230
|
+
llm = FakeLLMClient(responses=[])
|
|
231
|
+
|
|
232
|
+
result = await cluster_by_llm(new_c, [], llm=llm)
|
|
233
|
+
|
|
234
|
+
assert result is None
|
|
235
|
+
assert new_c.members == ["entity_y"]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Cluster value-object tests — frozen invariants + basic field access."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pytest
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
|
|
9
|
+
from everalgo.clustering import Cluster
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_cluster_default_count_is_one() -> None:
|
|
13
|
+
c = Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=1_700_000_000_000)
|
|
14
|
+
assert c.count == 1
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_cluster_preview_defaults_to_empty() -> None:
|
|
18
|
+
c = Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=1_700_000_000_000)
|
|
19
|
+
assert c.preview == []
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_cluster_is_frozen_raises_on_assignment() -> None:
|
|
23
|
+
c = Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=1_700_000_000_000)
|
|
24
|
+
with pytest.raises(ValidationError):
|
|
25
|
+
c.count = 99 # type: ignore[misc]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_cluster_arbitrary_ndarray_allowed() -> None:
|
|
29
|
+
vec = np.array([0.1, 0.2, 0.3], dtype=np.float32)
|
|
30
|
+
c = Cluster(centroid=vec, last_ts=0)
|
|
31
|
+
np.testing.assert_array_equal(c.centroid, vec)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_cluster_with_preview() -> None:
|
|
35
|
+
c = Cluster(centroid=np.array([1.0], dtype=np.float32), last_ts=0, preview=["a", "b"])
|
|
36
|
+
assert c.preview == ["a", "b"]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_cluster_members_defaults_to_empty() -> None:
|
|
40
|
+
c = Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=1_700_000_000_000)
|
|
41
|
+
assert c.members == []
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_cluster_members_roundtrip() -> None:
|
|
45
|
+
c = Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=0, members=["id_a", "id_b"])
|
|
46
|
+
assert c.members == ["id_a", "id_b"]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Surface tests for ``everalgo.clustering`` public API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_top_level_exports() -> None:
|
|
9
|
+
from everalgo.clustering import __all__
|
|
10
|
+
|
|
11
|
+
assert sorted(__all__) == sorted(
|
|
12
|
+
[
|
|
13
|
+
"Cluster",
|
|
14
|
+
"cluster_by_geometry",
|
|
15
|
+
"cluster_by_llm",
|
|
16
|
+
]
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_cluster_instantiates_with_required_fields() -> None:
|
|
21
|
+
import numpy as np
|
|
22
|
+
|
|
23
|
+
from everalgo.clustering import Cluster
|
|
24
|
+
|
|
25
|
+
c = Cluster(centroid=np.array([1.0, 0.0], dtype=np.float32), last_ts=0)
|
|
26
|
+
assert c.count == 1
|
|
27
|
+
assert c.preview == []
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_cluster_functions_are_async() -> None:
|
|
31
|
+
from everalgo.clustering import cluster_by_geometry, cluster_by_llm
|
|
32
|
+
|
|
33
|
+
assert inspect.iscoroutinefunction(cluster_by_geometry)
|
|
34
|
+
assert inspect.iscoroutinefunction(cluster_by_llm)
|