wavemind 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wavemind/__init__.py +22 -0
- wavemind/__main__.py +5 -0
- wavemind/api.py +177 -0
- wavemind/benchmark.py +67 -0
- wavemind/cli.py +245 -0
- wavemind/core.py +428 -0
- wavemind/encoders.py +127 -0
- wavemind/importers.py +136 -0
- wavemind/indexes.py +214 -0
- wavemind/storage.py +222 -0
- wavemind-2.0.0.dist-info/METADATA +134 -0
- wavemind-2.0.0.dist-info/RECORD +16 -0
- wavemind-2.0.0.dist-info/WHEEL +5 -0
- wavemind-2.0.0.dist-info/entry_points.txt +2 -0
- wavemind-2.0.0.dist-info/licenses/LICENSE +21 -0
- wavemind-2.0.0.dist-info/top_level.txt +1 -0
wavemind/indexes.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Iterable
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class IndexResult:
|
|
11
|
+
id: int
|
|
12
|
+
score: float
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _normalize(vector: np.ndarray) -> np.ndarray:
|
|
16
|
+
vector = np.asarray(vector, dtype=np.float32)
|
|
17
|
+
norm = float(np.linalg.norm(vector))
|
|
18
|
+
if norm <= 1e-12:
|
|
19
|
+
return vector
|
|
20
|
+
return (vector / norm).astype(np.float32)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NumpyVectorIndex:
|
|
24
|
+
name = "numpy-exact"
|
|
25
|
+
|
|
26
|
+
def __init__(self, vector_dim: int):
|
|
27
|
+
self.vector_dim = int(vector_dim)
|
|
28
|
+
self._vectors: dict[int, np.ndarray] = {}
|
|
29
|
+
self._ids = np.array([], dtype=np.int64)
|
|
30
|
+
self._matrix = np.zeros((0, self.vector_dim), dtype=np.float32)
|
|
31
|
+
self._dirty = True
|
|
32
|
+
|
|
33
|
+
def add(self, id: int, vector: np.ndarray) -> None:
|
|
34
|
+
self._vectors[int(id)] = _normalize(vector)
|
|
35
|
+
self._dirty = True
|
|
36
|
+
|
|
37
|
+
def remove(self, id: int) -> None:
|
|
38
|
+
self._vectors.pop(int(id), None)
|
|
39
|
+
self._dirty = True
|
|
40
|
+
|
|
41
|
+
def build(self, records: Iterable) -> None:
|
|
42
|
+
self._vectors.clear()
|
|
43
|
+
for record in records:
|
|
44
|
+
self.add(record.id, record.vector)
|
|
45
|
+
self._dirty = True
|
|
46
|
+
|
|
47
|
+
def _ensure_matrix(self) -> None:
|
|
48
|
+
if not self._dirty:
|
|
49
|
+
return
|
|
50
|
+
if not self._vectors:
|
|
51
|
+
self._ids = np.array([], dtype=np.int64)
|
|
52
|
+
self._matrix = np.zeros((0, self.vector_dim), dtype=np.float32)
|
|
53
|
+
self._dirty = False
|
|
54
|
+
return
|
|
55
|
+
items = sorted(self._vectors.items())
|
|
56
|
+
self._ids = np.array([id for id, _ in items], dtype=np.int64)
|
|
57
|
+
self._matrix = np.stack([vector for _, vector in items]).astype(np.float32)
|
|
58
|
+
self._dirty = False
|
|
59
|
+
|
|
60
|
+
def search(
|
|
61
|
+
self,
|
|
62
|
+
vector: np.ndarray,
|
|
63
|
+
top_k: int = 3,
|
|
64
|
+
allowed_ids: set[int] | None = None,
|
|
65
|
+
) -> list[IndexResult]:
|
|
66
|
+
if top_k <= 0 or not self._vectors:
|
|
67
|
+
return []
|
|
68
|
+
|
|
69
|
+
self._ensure_matrix()
|
|
70
|
+
query = _normalize(vector)
|
|
71
|
+
if allowed_ids is None:
|
|
72
|
+
ids = self._ids
|
|
73
|
+
matrix = self._matrix
|
|
74
|
+
else:
|
|
75
|
+
mask = np.fromiter((int(id) in allowed_ids for id in self._ids), dtype=bool)
|
|
76
|
+
if not np.any(mask):
|
|
77
|
+
return []
|
|
78
|
+
ids = self._ids[mask]
|
|
79
|
+
matrix = self._matrix[mask]
|
|
80
|
+
if ids.size == 0:
|
|
81
|
+
return []
|
|
82
|
+
|
|
83
|
+
scores = matrix @ query
|
|
84
|
+
order = np.argsort(scores)[::-1][:top_k]
|
|
85
|
+
return [IndexResult(int(ids[int(i)]), float(scores[int(i)])) for i in order]
|
|
86
|
+
|
|
87
|
+
def __len__(self) -> int:
|
|
88
|
+
return len(self._vectors)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class FaissVectorIndex(NumpyVectorIndex):
|
|
92
|
+
name = "faiss-flat-ip"
|
|
93
|
+
|
|
94
|
+
def __init__(self, vector_dim: int):
|
|
95
|
+
try:
|
|
96
|
+
import faiss
|
|
97
|
+
except ImportError as exc:
|
|
98
|
+
raise ImportError("Install faiss-cpu to use FaissVectorIndex") from exc
|
|
99
|
+
super().__init__(vector_dim)
|
|
100
|
+
self._faiss = faiss
|
|
101
|
+
self._index = faiss.IndexFlatIP(self.vector_dim)
|
|
102
|
+
self._id_order: list[int] = []
|
|
103
|
+
|
|
104
|
+
def build(self, records: Iterable) -> None:
|
|
105
|
+
self._vectors.clear()
|
|
106
|
+
self._id_order.clear()
|
|
107
|
+
vectors = []
|
|
108
|
+
for record in records:
|
|
109
|
+
vector = _normalize(record.vector)
|
|
110
|
+
self._vectors[int(record.id)] = vector
|
|
111
|
+
self._id_order.append(int(record.id))
|
|
112
|
+
vectors.append(vector)
|
|
113
|
+
self._index = self._faiss.IndexFlatIP(self.vector_dim)
|
|
114
|
+
if vectors:
|
|
115
|
+
self._index.add(np.stack(vectors).astype(np.float32))
|
|
116
|
+
self._dirty = True
|
|
117
|
+
|
|
118
|
+
def add(self, id: int, vector: np.ndarray) -> None:
|
|
119
|
+
self._vectors[int(id)] = _normalize(vector)
|
|
120
|
+
self.build([type("Record", (), {"id": k, "vector": v}) for k, v in self._vectors.items()])
|
|
121
|
+
|
|
122
|
+
def remove(self, id: int) -> None:
|
|
123
|
+
self._vectors.pop(int(id), None)
|
|
124
|
+
self.build([type("Record", (), {"id": k, "vector": v}) for k, v in self._vectors.items()])
|
|
125
|
+
|
|
126
|
+
def search(
|
|
127
|
+
self,
|
|
128
|
+
vector: np.ndarray,
|
|
129
|
+
top_k: int = 3,
|
|
130
|
+
allowed_ids: set[int] | None = None,
|
|
131
|
+
) -> list[IndexResult]:
|
|
132
|
+
if allowed_ids is not None:
|
|
133
|
+
return super().search(vector, top_k=top_k, allowed_ids=allowed_ids)
|
|
134
|
+
if top_k <= 0 or not self._id_order:
|
|
135
|
+
return []
|
|
136
|
+
query = _normalize(vector).reshape(1, -1).astype(np.float32)
|
|
137
|
+
scores, positions = self._index.search(query, min(top_k, len(self._id_order)))
|
|
138
|
+
results = []
|
|
139
|
+
for score, pos in zip(scores[0], positions[0]):
|
|
140
|
+
if pos < 0:
|
|
141
|
+
continue
|
|
142
|
+
results.append(IndexResult(self._id_order[int(pos)], float(score)))
|
|
143
|
+
return results
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class AnnoyVectorIndex(NumpyVectorIndex):
|
|
147
|
+
name = "annoy-angular"
|
|
148
|
+
|
|
149
|
+
def __init__(self, vector_dim: int, n_trees: int = 16):
|
|
150
|
+
try:
|
|
151
|
+
from annoy import AnnoyIndex
|
|
152
|
+
except ImportError as exc:
|
|
153
|
+
raise ImportError("Install annoy to use AnnoyVectorIndex") from exc
|
|
154
|
+
super().__init__(vector_dim)
|
|
155
|
+
self._AnnoyIndex = AnnoyIndex
|
|
156
|
+
self.n_trees = int(n_trees)
|
|
157
|
+
self._index = AnnoyIndex(self.vector_dim, "angular")
|
|
158
|
+
self._id_order: list[int] = []
|
|
159
|
+
self._built = False
|
|
160
|
+
|
|
161
|
+
def build(self, records: Iterable) -> None:
|
|
162
|
+
self._vectors.clear()
|
|
163
|
+
self._id_order.clear()
|
|
164
|
+
self._index = self._AnnoyIndex(self.vector_dim, "angular")
|
|
165
|
+
for pos, record in enumerate(records):
|
|
166
|
+
vector = _normalize(record.vector)
|
|
167
|
+
self._vectors[int(record.id)] = vector
|
|
168
|
+
self._id_order.append(int(record.id))
|
|
169
|
+
self._index.add_item(pos, vector.tolist())
|
|
170
|
+
if self._id_order:
|
|
171
|
+
self._index.build(self.n_trees)
|
|
172
|
+
self._built = True
|
|
173
|
+
self._dirty = True
|
|
174
|
+
|
|
175
|
+
def add(self, id: int, vector: np.ndarray) -> None:
|
|
176
|
+
self._vectors[int(id)] = _normalize(vector)
|
|
177
|
+
self.build([type("Record", (), {"id": k, "vector": v}) for k, v in self._vectors.items()])
|
|
178
|
+
|
|
179
|
+
def remove(self, id: int) -> None:
|
|
180
|
+
self._vectors.pop(int(id), None)
|
|
181
|
+
self.build([type("Record", (), {"id": k, "vector": v}) for k, v in self._vectors.items()])
|
|
182
|
+
|
|
183
|
+
def search(
|
|
184
|
+
self,
|
|
185
|
+
vector: np.ndarray,
|
|
186
|
+
top_k: int = 3,
|
|
187
|
+
allowed_ids: set[int] | None = None,
|
|
188
|
+
) -> list[IndexResult]:
|
|
189
|
+
if allowed_ids is not None:
|
|
190
|
+
return super().search(vector, top_k=top_k, allowed_ids=allowed_ids)
|
|
191
|
+
if top_k <= 0 or not self._id_order or not self._built:
|
|
192
|
+
return []
|
|
193
|
+
positions, distances = self._index.get_nns_by_vector(
|
|
194
|
+
_normalize(vector).tolist(),
|
|
195
|
+
min(top_k, len(self._id_order)),
|
|
196
|
+
include_distances=True,
|
|
197
|
+
)
|
|
198
|
+
return [
|
|
199
|
+
IndexResult(self._id_order[int(pos)], float(1.0 - distance / 2.0))
|
|
200
|
+
for pos, distance in zip(positions, distances)
|
|
201
|
+
]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def create_vector_index(kind: str, vector_dim: int):
|
|
205
|
+
kind = (kind or "numpy").lower()
|
|
206
|
+
if kind in {"numpy", "exact"}:
|
|
207
|
+
return NumpyVectorIndex(vector_dim)
|
|
208
|
+
if kind == "faiss":
|
|
209
|
+
return FaissVectorIndex(vector_dim)
|
|
210
|
+
if kind == "annoy":
|
|
211
|
+
return AnnoyVectorIndex(vector_dim)
|
|
212
|
+
raise ValueError(
|
|
213
|
+
f"Unknown vector index kind: {kind}. Choose an explicit index: numpy, faiss, or annoy."
|
|
214
|
+
)
|
wavemind/storage.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
import time
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Iterable
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class MemoryRecord:
|
|
15
|
+
text: str
|
|
16
|
+
vector: np.ndarray
|
|
17
|
+
pattern: np.ndarray
|
|
18
|
+
namespace: str = "default"
|
|
19
|
+
tags: tuple[str, ...] = ()
|
|
20
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
21
|
+
created_at: float = field(default_factory=time.time)
|
|
22
|
+
updated_at: float = field(default_factory=time.time)
|
|
23
|
+
expires_at: float | None = None
|
|
24
|
+
priority: float = 1.0
|
|
25
|
+
access_count: int = 0
|
|
26
|
+
id: int | None = None
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def is_expired(self) -> bool:
|
|
30
|
+
return self.expires_at is not None and self.expires_at <= time.time()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _array_to_blob(array: np.ndarray) -> bytes:
|
|
34
|
+
return np.asarray(array, dtype=np.float32).tobytes()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _array_from_blob(blob: bytes, shape: Iterable[int]) -> np.ndarray:
|
|
38
|
+
return np.frombuffer(blob, dtype=np.float32).copy().reshape(tuple(shape))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SQLiteMemoryStore:
|
|
42
|
+
def __init__(self, path: str | Path | None = None):
|
|
43
|
+
self.path = str(path or ":memory:")
|
|
44
|
+
if self.path != ":memory:":
|
|
45
|
+
Path(self.path).parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
self.conn = sqlite3.connect(self.path, check_same_thread=False)
|
|
47
|
+
self.conn.row_factory = sqlite3.Row
|
|
48
|
+
self.ensure_schema()
|
|
49
|
+
|
|
50
|
+
def ensure_schema(self) -> None:
|
|
51
|
+
self.conn.execute(
|
|
52
|
+
"""
|
|
53
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
54
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
55
|
+
namespace TEXT NOT NULL,
|
|
56
|
+
text TEXT NOT NULL,
|
|
57
|
+
vector BLOB NOT NULL,
|
|
58
|
+
vector_dim INTEGER NOT NULL,
|
|
59
|
+
pattern BLOB NOT NULL,
|
|
60
|
+
pattern_shape TEXT NOT NULL,
|
|
61
|
+
tags TEXT NOT NULL,
|
|
62
|
+
metadata TEXT NOT NULL,
|
|
63
|
+
created_at REAL NOT NULL,
|
|
64
|
+
updated_at REAL NOT NULL,
|
|
65
|
+
expires_at REAL,
|
|
66
|
+
priority REAL NOT NULL,
|
|
67
|
+
access_count INTEGER NOT NULL DEFAULT 0
|
|
68
|
+
)
|
|
69
|
+
"""
|
|
70
|
+
)
|
|
71
|
+
self.conn.execute("CREATE INDEX IF NOT EXISTS idx_memories_namespace ON memories(namespace)")
|
|
72
|
+
self.conn.execute("CREATE INDEX IF NOT EXISTS idx_memories_expires_at ON memories(expires_at)")
|
|
73
|
+
self.conn.commit()
|
|
74
|
+
|
|
75
|
+
def insert(self, record: MemoryRecord) -> int:
|
|
76
|
+
now = time.time()
|
|
77
|
+
record.created_at = record.created_at or now
|
|
78
|
+
record.updated_at = now
|
|
79
|
+
cur = self.conn.execute(
|
|
80
|
+
"""
|
|
81
|
+
INSERT INTO memories (
|
|
82
|
+
namespace, text, vector, vector_dim, pattern, pattern_shape,
|
|
83
|
+
tags, metadata, created_at, updated_at, expires_at, priority, access_count
|
|
84
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
85
|
+
""",
|
|
86
|
+
(
|
|
87
|
+
record.namespace,
|
|
88
|
+
record.text,
|
|
89
|
+
_array_to_blob(record.vector),
|
|
90
|
+
int(record.vector.shape[0]),
|
|
91
|
+
_array_to_blob(record.pattern),
|
|
92
|
+
json.dumps(list(record.pattern.shape)),
|
|
93
|
+
json.dumps(list(record.tags), ensure_ascii=False),
|
|
94
|
+
json.dumps(record.metadata, ensure_ascii=False),
|
|
95
|
+
record.created_at,
|
|
96
|
+
record.updated_at,
|
|
97
|
+
record.expires_at,
|
|
98
|
+
float(record.priority),
|
|
99
|
+
int(record.access_count),
|
|
100
|
+
),
|
|
101
|
+
)
|
|
102
|
+
self.conn.commit()
|
|
103
|
+
record.id = int(cur.lastrowid)
|
|
104
|
+
return record.id
|
|
105
|
+
|
|
106
|
+
def get(self, id: int) -> MemoryRecord | None:
|
|
107
|
+
row = self.conn.execute("SELECT * FROM memories WHERE id = ?", (int(id),)).fetchone()
|
|
108
|
+
return self._row_to_record(row) if row else None
|
|
109
|
+
|
|
110
|
+
def list(
|
|
111
|
+
self,
|
|
112
|
+
namespace: str | None = None,
|
|
113
|
+
include_expired: bool = False,
|
|
114
|
+
tags: Iterable[str] | None = None,
|
|
115
|
+
) -> list[MemoryRecord]:
|
|
116
|
+
params: list[Any] = []
|
|
117
|
+
where = []
|
|
118
|
+
if namespace is not None:
|
|
119
|
+
where.append("namespace = ?")
|
|
120
|
+
params.append(namespace)
|
|
121
|
+
if not include_expired:
|
|
122
|
+
where.append("(expires_at IS NULL OR expires_at > ?)")
|
|
123
|
+
params.append(time.time())
|
|
124
|
+
sql = "SELECT * FROM memories"
|
|
125
|
+
if where:
|
|
126
|
+
sql += " WHERE " + " AND ".join(where)
|
|
127
|
+
rows = self.conn.execute(sql, params).fetchall()
|
|
128
|
+
records = [self._row_to_record(row) for row in rows]
|
|
129
|
+
required_tags = set(tags or [])
|
|
130
|
+
if required_tags:
|
|
131
|
+
records = [
|
|
132
|
+
record
|
|
133
|
+
for record in records
|
|
134
|
+
if required_tags.issubset(set(record.tags))
|
|
135
|
+
]
|
|
136
|
+
return records
|
|
137
|
+
|
|
138
|
+
def delete(
|
|
139
|
+
self,
|
|
140
|
+
id: int | None = None,
|
|
141
|
+
text: str | None = None,
|
|
142
|
+
namespace: str | None = None,
|
|
143
|
+
) -> list[MemoryRecord]:
|
|
144
|
+
params: list[Any] = []
|
|
145
|
+
where = []
|
|
146
|
+
if id is not None:
|
|
147
|
+
where.append("id = ?")
|
|
148
|
+
params.append(int(id))
|
|
149
|
+
if text is not None:
|
|
150
|
+
where.append("text = ?")
|
|
151
|
+
params.append(text)
|
|
152
|
+
if namespace is not None:
|
|
153
|
+
where.append("namespace = ?")
|
|
154
|
+
params.append(namespace)
|
|
155
|
+
if not where:
|
|
156
|
+
raise ValueError("delete requires id or text")
|
|
157
|
+
|
|
158
|
+
sql_where = " AND ".join(where)
|
|
159
|
+
rows = self.conn.execute(f"SELECT * FROM memories WHERE {sql_where}", params).fetchall()
|
|
160
|
+
records = [self._row_to_record(row) for row in rows]
|
|
161
|
+
self.conn.execute(f"DELETE FROM memories WHERE {sql_where}", params)
|
|
162
|
+
self.conn.commit()
|
|
163
|
+
return records
|
|
164
|
+
|
|
165
|
+
def purge_expired(self) -> int:
|
|
166
|
+
rows = self.conn.execute(
|
|
167
|
+
"SELECT * FROM memories WHERE expires_at IS NOT NULL AND expires_at <= ?",
|
|
168
|
+
(time.time(),),
|
|
169
|
+
).fetchall()
|
|
170
|
+
ids = [int(row["id"]) for row in rows]
|
|
171
|
+
if ids:
|
|
172
|
+
placeholders = ",".join("?" for _ in ids)
|
|
173
|
+
self.conn.execute(f"DELETE FROM memories WHERE id IN ({placeholders})", ids)
|
|
174
|
+
self.conn.commit()
|
|
175
|
+
return len(ids)
|
|
176
|
+
|
|
177
|
+
def touch(self, id: int, priority_delta: float = 0.05) -> None:
|
|
178
|
+
self.conn.execute(
|
|
179
|
+
"""
|
|
180
|
+
UPDATE memories
|
|
181
|
+
SET access_count = access_count + 1,
|
|
182
|
+
priority = priority + ?,
|
|
183
|
+
updated_at = ?
|
|
184
|
+
WHERE id = ?
|
|
185
|
+
""",
|
|
186
|
+
(float(priority_delta), time.time(), int(id)),
|
|
187
|
+
)
|
|
188
|
+
self.conn.commit()
|
|
189
|
+
|
|
190
|
+
def count(self, namespace: str | None = None, include_expired: bool = False) -> int:
|
|
191
|
+
return len(self.list(namespace=namespace, include_expired=include_expired))
|
|
192
|
+
|
|
193
|
+
def backup(self, destination: str | Path) -> Path:
|
|
194
|
+
destination = Path(destination)
|
|
195
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
196
|
+
target = sqlite3.connect(str(destination))
|
|
197
|
+
with target:
|
|
198
|
+
self.conn.backup(target)
|
|
199
|
+
target.close()
|
|
200
|
+
return destination
|
|
201
|
+
|
|
202
|
+
def close(self) -> None:
|
|
203
|
+
self.conn.close()
|
|
204
|
+
|
|
205
|
+
def _row_to_record(self, row: sqlite3.Row) -> MemoryRecord:
|
|
206
|
+
pattern_shape = json.loads(row["pattern_shape"])
|
|
207
|
+
vector_dim = int(row["vector_dim"])
|
|
208
|
+
return MemoryRecord(
|
|
209
|
+
id=int(row["id"]),
|
|
210
|
+
namespace=row["namespace"],
|
|
211
|
+
text=row["text"],
|
|
212
|
+
vector=_array_from_blob(row["vector"], (vector_dim,)),
|
|
213
|
+
pattern=_array_from_blob(row["pattern"], pattern_shape),
|
|
214
|
+
tags=tuple(json.loads(row["tags"])),
|
|
215
|
+
metadata=json.loads(row["metadata"]),
|
|
216
|
+
created_at=float(row["created_at"]),
|
|
217
|
+
updated_at=float(row["updated_at"]),
|
|
218
|
+
expires_at=row["expires_at"],
|
|
219
|
+
priority=float(row["priority"]),
|
|
220
|
+
access_count=int(row["access_count"]),
|
|
221
|
+
)
|
|
222
|
+
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wavemind
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Persistent dynamic memory engine with vector search and wave-field re-ranking
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/CaspianG/wavemind
|
|
7
|
+
Project-URL: Repository, https://github.com/CaspianG/wavemind
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: numpy>=1.24
|
|
12
|
+
Requires-Dist: fastapi>=0.110
|
|
13
|
+
Requires-Dist: uvicorn[standard]>=0.27
|
|
14
|
+
Requires-Dist: pydantic>=2
|
|
15
|
+
Requires-Dist: pypdf>=4
|
|
16
|
+
Provides-Extra: sentence
|
|
17
|
+
Requires-Dist: sentence-transformers>=3; extra == "sentence"
|
|
18
|
+
Provides-Extra: ml
|
|
19
|
+
Requires-Dist: sentence-transformers>=3; extra == "ml"
|
|
20
|
+
Provides-Extra: indexes
|
|
21
|
+
Requires-Dist: annoy>=1.17; extra == "indexes"
|
|
22
|
+
Requires-Dist: faiss-cpu>=1.8; platform_system != "Windows" and extra == "indexes"
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
25
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# WaveMind is persistent dynamic memory for AI agents: vector search first, wave-field priority second, SQLite as the source of truth.
|
|
29
|
+
|
|
30
|
+

|
|
31
|
+
[](https://github.com/CaspianG/wavemind/actions/workflows/tests.yml)
|
|
32
|
+

|
|
33
|
+
|
|
34
|
+
## Terminal Demo
|
|
35
|
+
|
|
36
|
+
```text
|
|
37
|
+
$ python examples/demo.py
|
|
38
|
+
✓ Remembered: "Andrey is a trader who tracks market breakouts."
|
|
39
|
+
✓ Remembered: "Andrey prefers short practical answers about AI agents."
|
|
40
|
+
|
|
41
|
+
Query: "Andrey trader agent"
|
|
42
|
+
→ Result 1 (0.54): "Andrey is a trader who tracks market breakouts."
|
|
43
|
+
→ Result 2 (0.30): "Andrey prefers short practical answers about AI agents."
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The demo is offline, keyless, and uses the built-in hash encoder.
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
python -m pip install -e .
|
|
52
|
+
wavemind remember "Andrey is a trader" --namespace demo
|
|
53
|
+
wavemind query "trader" --namespace demo
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
This creates `wavemind.sqlite3` in your current working directory.
|
|
57
|
+
|
|
58
|
+
For sentence-transformer embeddings:
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
python -m pip install -e ".[sentence]"
|
|
62
|
+
wavemind --encoder sentence remember "Andrey is a trader" --namespace demo
|
|
63
|
+
wavemind --encoder sentence query "What does Andrey do?" --namespace demo
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
One-file setup scripts are also included:
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
sh install.sh
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
```bat
|
|
73
|
+
install.bat
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Benchmark
|
|
77
|
+
|
|
78
|
+
Real Russian sentences from Tatoeba, 50 one-word queries, NumPy exact index.
|
|
79
|
+
|
|
80
|
+
| metric | hash | sentence-transformers |
|
|
81
|
+
|---|---:|---:|
|
|
82
|
+
| precision@1 | 1.00 | 1.00 |
|
|
83
|
+
| precision@3 | 1.00 | 1.00 |
|
|
84
|
+
| avg query | 0.49 ms | 52.84 ms |
|
|
85
|
+
|
|
86
|
+
Capacity check with the hash encoder:
|
|
87
|
+
|
|
88
|
+
| memories | precision@1 | precision@3 | avg query |
|
|
89
|
+
|---:|---:|---:|---:|
|
|
90
|
+
| 200 | 1.00 | 1.00 | 0.49 ms |
|
|
91
|
+
| 1000 | 0.88 | 1.00 | 1.50 ms |
|
|
92
|
+
| 5000 | 0.72 | 0.88 | 5.68 ms |
|
|
93
|
+
|
|
94
|
+
Run locally:
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
python benchmarks/ru_sentences_benchmark.py --sentences 200 --queries 50 --encoder hash --index numpy
|
|
98
|
+
python benchmarks/ru_sentences_benchmark.py --sentences 200 --queries 50 --encoder sentence --index numpy
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Comparison
|
|
102
|
+
|
|
103
|
+
| feature | WaveMind | Chroma | Qdrant |
|
|
104
|
+
|---|---|---|---|
|
|
105
|
+
| Primary role | Agent memory engine | Embedding database | Production vector database |
|
|
106
|
+
| Local SQLite persistence | Yes | Yes | No, separate service/storage |
|
|
107
|
+
| HTTP API | FastAPI included | Included | Included |
|
|
108
|
+
| Dynamic memory priority | Wave-field hotness, TTL, priority | Metadata/filter driven | Payload/filter driven |
|
|
109
|
+
| Built-in forgetting | TTL and explicit forget | Manual delete/filtering | Manual delete/filtering |
|
|
110
|
+
| Best fit | Small to medium agent memory with dynamic recall | Local RAG apps and prototypes | Large-scale vector search |
|
|
111
|
+
| Scale target today | Up to 1000 optimal on NumPy, FAISS recommended beyond 5000 | Larger than WaveMind local mode | Production scale |
|
|
112
|
+
|
|
113
|
+
WaveMind is not trying to replace dedicated vector databases at scale. Its difference is dynamic priority: frequently used memories can become hotter while old or low-priority memories fade.
|
|
114
|
+
|
|
115
|
+
## Known Limitations
|
|
116
|
+
|
|
117
|
+
- Optimal capacity on the current NumPy exact index is up to 1000 records.
|
|
118
|
+
- At 5000 records, one-word `precision@1` is currently 0.72 with the hash encoder; many misses are ambiguous queries where another sentence containing the same word ranks first.
|
|
119
|
+
- For `N > 5000`, use the FAISS backend with `--index faiss` or another production vector index.
|
|
120
|
+
- `sentence-transformers/paraphrase-multilingual-mpnet-base-v2` requires about 420 MB of model files and measured about 53 ms per query on the benchmark machine.
|
|
121
|
+
- The bundled benchmark is a retrieval sanity check, not a full agent-memory benchmark against Chroma or Qdrant yet.
|
|
122
|
+
|
|
123
|
+
## Roadmap
|
|
124
|
+
|
|
125
|
+
- FAISS-first production index path with persisted index rebuilds.
|
|
126
|
+
- Larger public benchmark against Chroma and Qdrant on agent-memory tasks.
|
|
127
|
+
- Better semantic query expansion for short and ambiguous queries.
|
|
128
|
+
- Namespace quotas, backups, and daemon hardening for SaaS use.
|
|
129
|
+
- Webhook on recall for agent runtimes.
|
|
130
|
+
- OHLCV pattern-memory experiments for market research and backtests.
|
|
131
|
+
|
|
132
|
+
## License
|
|
133
|
+
|
|
134
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
wavemind/__init__.py,sha256=6FmY1qcTZByc-cDOEUvb6uShlKUghXWkRB6lp-Kjv4A,496
|
|
2
|
+
wavemind/__main__.py,sha256=4BTAfHHF2u3j_oUg_Wluqu0HyxnXfAHBQUcnPQ-e1xs,50
|
|
3
|
+
wavemind/api.py,sha256=vthe2fUxv4dTeXhIU8Z4boqULSk4gAFJPyO5OCFRMQw,5146
|
|
4
|
+
wavemind/benchmark.py,sha256=oL3zAiyRwzj5Puh1Ex6c9U-0_mAY9bS1pSA-xUxEpUg,2081
|
|
5
|
+
wavemind/cli.py,sha256=c40q_34_FJA1ojuoPoiIG1jELmNeW5jO7z-2_H8PBto,8341
|
|
6
|
+
wavemind/core.py,sha256=nsgb4Yuw6BND2SE8dm6s7YK_jHU-jCMl2zdI-0yeYz8,16198
|
|
7
|
+
wavemind/encoders.py,sha256=c9BfW3SBGxinpJVtoYBN7l1wqxi3aGoA9h2lX6LuKUA,4338
|
|
8
|
+
wavemind/importers.py,sha256=cwuBIwEXQqHM35iWg23BGfHJTSCVLNt4MoFW-AEN7Hw,4023
|
|
9
|
+
wavemind/indexes.py,sha256=gUX7uvfD5KQjqr0Hgz18q5gAuq8ipAlcWvbVHyo0330,7300
|
|
10
|
+
wavemind/storage.py,sha256=Yx3f7xfDEJrBRS2oS1U8zrPh7zFP10gGTZ-hBhJeEkA,7867
|
|
11
|
+
wavemind-2.0.0.dist-info/licenses/LICENSE,sha256=3774fYet-ActxR0N3xkWV1n-qGV8usVH8lO95fa0TSw,1078
|
|
12
|
+
wavemind-2.0.0.dist-info/METADATA,sha256=MVNXOyt3-VFLZBnoS4YK9kaeI2WQuqdw7zXoAi5tD_4,5099
|
|
13
|
+
wavemind-2.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
14
|
+
wavemind-2.0.0.dist-info/entry_points.txt,sha256=2t66WjNTzGr9_af4T8eQO9OyMhJsA5RqzUBUCi1x1fU,47
|
|
15
|
+
wavemind-2.0.0.dist-info/top_level.txt,sha256=n3jjV6Gcmic3b4D5TgTObdq2sQOCJwb7hZgdIv305QI,9
|
|
16
|
+
wavemind-2.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 WaveMind contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
wavemind
|