genesis-l0 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- genesis_l0-0.1.0/LICENSE +49 -0
- genesis_l0-0.1.0/MANIFEST.in +4 -0
- genesis_l0-0.1.0/NOTICE +8 -0
- genesis_l0-0.1.0/PKG-INFO +172 -0
- genesis_l0-0.1.0/README.md +145 -0
- genesis_l0-0.1.0/benchmarks/bench_e2e.py +98 -0
- genesis_l0-0.1.0/csrc/l0_api.c +193 -0
- genesis_l0-0.1.0/csrc/l0_api.h +79 -0
- genesis_l0-0.1.0/csrc/srp_lsh.c +104 -0
- genesis_l0-0.1.0/csrc/srp_lsh.h +70 -0
- genesis_l0-0.1.0/pyproject.toml +47 -0
- genesis_l0-0.1.0/setup.cfg +4 -0
- genesis_l0-0.1.0/setup.py +43 -0
- genesis_l0-0.1.0/src/genesis_l0/__init__.py +11 -0
- genesis_l0-0.1.0/src/genesis_l0/_c_core.py +194 -0
- genesis_l0-0.1.0/src/genesis_l0/adapters/__init__.py +2 -0
- genesis_l0-0.1.0/src/genesis_l0/adapters/langgraph.py +37 -0
- genesis_l0-0.1.0/src/genesis_l0/memory.py +154 -0
- genesis_l0-0.1.0/src/genesis_l0/storage.py +133 -0
- genesis_l0-0.1.0/src/genesis_l0.egg-info/PKG-INFO +172 -0
- genesis_l0-0.1.0/src/genesis_l0.egg-info/SOURCES.txt +28 -0
- genesis_l0-0.1.0/src/genesis_l0.egg-info/dependency_links.txt +1 -0
- genesis_l0-0.1.0/src/genesis_l0.egg-info/requires.txt +9 -0
- genesis_l0-0.1.0/src/genesis_l0.egg-info/top_level.txt +1 -0
- genesis_l0-0.1.0/tests/__init__.py +0 -0
- genesis_l0-0.1.0/tests/conftest.py +16 -0
- genesis_l0-0.1.0/tests/test_c_core.py +109 -0
- genesis_l0-0.1.0/tests/test_end_to_end.py +87 -0
- genesis_l0-0.1.0/tests/test_langgraph.py +40 -0
- genesis_l0-0.1.0/tests/test_mmap_storage.py +72 -0
genesis_l0-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Business Source License 1.1
|
|
2
|
+
|
|
3
|
+
Licensor: Maksym Babych / Synapse Core Infrastructure
|
|
4
|
+
Licensed Work: genesis-l0 0.1.0
|
|
5
|
+
(c) 2026 Maksym Babych / Synapse Core Infrastructure
|
|
6
|
+
(csrc/srp_lsh.c and csrc/srp_lsh.h are licensed
|
|
7
|
+
separately under Apache 2.0; see NOTICE.)
|
|
8
|
+
Additional Use Grant: You may use the Licensed Work for non-commercial,
|
|
9
|
+
research, educational, and personal purposes free of
|
|
10
|
+
charge. Commercial use — including SaaS deployment,
|
|
11
|
+
API monetization, embedding in commercial products,
|
|
12
|
+
or providing managed services based on this work —
|
|
13
|
+
requires a separate written commercial license from
|
|
14
|
+
the Licensor. Contact: research@syn.ai
|
|
15
|
+
Change Date: 2030-01-01
|
|
16
|
+
Change License: Apache License, Version 2.0
|
|
17
|
+
|
|
18
|
+
-----------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
Terms
|
|
21
|
+
|
|
22
|
+
The Licensor hereby grants you the right to copy, modify, create derivative
|
|
23
|
+
works, redistribute, and make non-production use of the Licensed Work. The
|
|
24
|
+
Licensor may make an Additional Use Grant, above, permitting limited
|
|
25
|
+
production use.
|
|
26
|
+
|
|
27
|
+
Effective on the Change Date, or the fourth anniversary of the first publicly
|
|
28
|
+
available distribution of a specific version of the Licensed Work under this
|
|
29
|
+
License, whichever comes first, the Licensor hereby grants you rights under
|
|
30
|
+
the terms of the Change License, and the rights granted in the paragraph
|
|
31
|
+
above terminate.
|
|
32
|
+
|
|
33
|
+
If your use of the Licensed Work does not comply with the requirements
|
|
34
|
+
currently in effect as described in this License, you must purchase a
|
|
35
|
+
commercial license from the Licensor, its affiliated entities, or authorized
|
|
36
|
+
resellers, or you must refrain from using the Licensed Work.
|
|
37
|
+
|
|
38
|
+
All copies of the original and modified Licensed Work, and derivative works
|
|
39
|
+
of the Licensed Work, are subject to this License.
|
|
40
|
+
|
|
41
|
+
Any use of the Licensed Work in violation of this License will automatically
|
|
42
|
+
terminate your rights under this License for the current and all other
|
|
43
|
+
versions of the Licensed Work.
|
|
44
|
+
|
|
45
|
+
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
|
|
46
|
+
AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
|
|
47
|
+
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
|
|
48
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
|
|
49
|
+
TITLE.
|
genesis_l0-0.1.0/NOTICE
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
genesis-l0
|
|
2
|
+
Copyright 2026 Maksym Babych / Synapse Core Infrastructure
|
|
3
|
+
|
|
4
|
+
csrc/srp_lsh.c and csrc/srp_lsh.h are taken unchanged from genesis-live-core
|
|
5
|
+
(https://github.com/mister3ai-cmyk/genesis-live-core, DOI 10.5281/zenodo.23246581)
|
|
6
|
+
and are licensed under the Apache License, Version 2.0
|
|
7
|
+
(http://www.apache.org/licenses/LICENSE-2.0). All other files are licensed
|
|
8
|
+
under the Business Source License 1.1 (see LICENSE).
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: genesis-l0
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: In-process C99 L0 vector index (SRP-LSH + exact re-rank) with a SQLite mmap system of record for AI agent loops
|
|
5
|
+
Author-email: Maksym Babych / Synapse Core Infrastructure <research@syn.ai>
|
|
6
|
+
License: BUSL-1.1
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: C
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
License-File: NOTICE
|
|
19
|
+
Requires-Dist: numpy>=1.20.0
|
|
20
|
+
Provides-Extra: langgraph
|
|
21
|
+
Requires-Dist: langgraph>=0.2; extra == "langgraph"
|
|
22
|
+
Requires-Dist: langgraph-checkpoint-sqlite>=2.0; extra == "langgraph"
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
25
|
+
Requires-Dist: build; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# genesis-l0
|
|
29
|
+
|
|
30
|
+
In-process vector memory for AI-agent loops (LangGraph, AutoGen, CrewAI): a C99
|
|
31
|
+
L0 index in front of a local SQLite system of record. No sockets, no server.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
query vector (float32)
|
|
35
|
+
│
|
|
36
|
+
▼ C99, ctypes (GIL released)
|
|
37
|
+
256-bit SRP-LSH code ─► POPCNT Hamming scan over all codes (1.01 MB for 33k vectors)
|
|
38
|
+
─► top-`pool` candidates ─► exact inner-product re-rank ─► top-k glyph ids
|
|
39
|
+
│
|
|
40
|
+
▼ SQLite (WAL, mmap_size = 256 MB)
|
|
41
|
+
typed payload of the requested face ("code", "research", ...)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The cascade is the measured one from
|
|
45
|
+
[genesis-live-core](https://github.com/mister3ai-cmyk/genesis-live-core)
|
|
46
|
+
(DOI [10.5281/zenodo.23246581](https://doi.org/10.5281/zenodo.23246581)).
|
|
47
|
+
|
|
48
|
+
## Measured
|
|
49
|
+
|
|
50
|
+
OpenAI `text-embedding-3-large` (DBpedia, truncated to 512-dim), 33,234 vectors,
|
|
51
|
+
100 held-out queries × 10 passes, mean-centered, top-10.
|
|
52
|
+
Contabo VPS, AMD EPYC 2.0 GHz (AVX2, L3 8 MiB), shared host, x86-64-v3 build.
|
|
53
|
+
Reproduce with `python benchmarks/bench_e2e.py --data /tmp/genesis_eval`.
|
|
54
|
+
|
|
55
|
+
| pool | Recall@10 | L0 index p50 | face fetch p50 | end-to-end p50 | end-to-end p99 |
|
|
56
|
+
|-----:|----------:|-------------:|---------------:|---------------:|---------------:|
|
|
57
|
+
| 512 | **90.6 %** | 306–336 µs | 59–70 µs | 359–400 µs | 465–1190 µs |
|
|
58
|
+
| 256 | 82.1 % | 226 µs | 61 µs | 285 µs | 389 µs |
|
|
59
|
+
| 128 | 73.9 % | 176 µs | 55 µs | 228 µs | 377 µs |
|
|
60
|
+
|
|
61
|
+
- Recall@10 is against exact brute force. Synthetic clustered 512-dim data: 100 %.
|
|
62
|
+
- Face fetch = one `SELECT … IN (10 ids)` right after the index query (caches
|
|
63
|
+
already warmed by the re-rank). On its own it measures ~18–20 µs p50.
|
|
64
|
+
- p99 varies between runs on the shared VPS; the range is three runs.
|
|
65
|
+
- Memory: 32 bytes of code per vector stay hot in L3; the float32 copy used by
|
|
66
|
+
the re-rank (2 KB per 512-dim vector, ~65 MB for 33k) is read from RAM.
|
|
67
|
+
|
|
68
|
+
## Modes
|
|
69
|
+
|
|
70
|
+
`pool` is the only knob between accuracy and latency:
|
|
71
|
+
|
|
72
|
+
| mode | `pool` | Recall@10 | end-to-end p50 |
|
|
73
|
+
|------|-------:|----------:|---------------:|
|
|
74
|
+
| **Precision** (default) | 512 | 90.6 % | ~380 µs |
|
|
75
|
+
| **Fast** | 128 | 73.9 % | ~228 µs (fits a hard < 250 µs budget) |
|
|
76
|
+
|
|
77
|
+
Precision is the default: inside an LLM step the ~150 µs difference is noise,
|
|
78
|
+
while a missed context hit is not.
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
mem = GenesisL0Memory("agent_memory.db", pool=128) # fast mode
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Install
|
|
85
|
+
|
|
86
|
+
Linux or macOS, Python ≥ 3.9, a C compiler.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pip install . # or: pip install -e ".[dev,langgraph]"
|
|
90
|
+
GENESIS_L0_NATIVE=1 pip install . # -march=native instead of portable x86-64-v3
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
x86_64 builds target x86-64-v3 (AVX2 + POPCNT: Intel Haswell / AMD Zen and newer).
|
|
94
|
+
|
|
95
|
+
## Quickstart
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
import numpy as np
|
|
99
|
+
from genesis_l0 import GenesisL0Memory
|
|
100
|
+
|
|
101
|
+
corpus = np.random.randn(1000, 512).astype(np.float32)
|
|
102
|
+
|
|
103
|
+
with GenesisL0Memory(db_path="agent_memory.db", capacity=50_000, dim=512,
|
|
104
|
+
pool=512, mean=corpus.mean(axis=0)) as mem:
|
|
105
|
+
mem.add_batch(
|
|
106
|
+
np.arange(1000), corpus,
|
|
107
|
+
[{"slug": f"step_{i}", "faces": {"code": f"def tool_{i}(): ..."}} for i in range(1000)],
|
|
108
|
+
)
|
|
109
|
+
mem.add_glyph(5000, corpus[0], slug="auth_agent", faces={"code": "def verify(): pass"})
|
|
110
|
+
|
|
111
|
+
for hit in mem.query(corpus[0], top_k=5, face="code"):
|
|
112
|
+
print(hit["glyph_id"], round(hit["score"], 3), hit["payload"])
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
- Vectors are stored in SQLite; reopening the database rebuilds the index.
|
|
116
|
+
- `mean` (optional) enables mean-centering. It is fixed once vectors exist and
|
|
117
|
+
is stored in the database. For anisotropic embeddings it adds a few recall points.
|
|
118
|
+
- `dim` must be a multiple of 16; `top_k` is capped at 64; glyph ids are uint32.
|
|
119
|
+
- Writes are serialized; `query()` is safe from any number of threads.
|
|
120
|
+
Do not call `close()` while queries are running.
|
|
121
|
+
|
|
122
|
+
## LangGraph checkpointer
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install "genesis-l0[langgraph]"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from genesis_l0.adapters.langgraph import GenesisL0CheckpointSaver
|
|
130
|
+
|
|
131
|
+
saver = GenesisL0CheckpointSaver.from_path("checkpoints.db")
|
|
132
|
+
graph = builder.compile(checkpointer=saver)
|
|
133
|
+
graph.invoke(inputs, {"configurable": {"thread_id": "agent-1"}})
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
It is the official `langgraph-checkpoint-sqlite` `SqliteSaver` on a connection
|
|
137
|
+
with the genesis-l0 pragmas, so `put_writes`, `list`, history and serialization
|
|
138
|
+
behave exactly as upstream.
|
|
139
|
+
|
|
140
|
+
## Docker / Kubernetes: locked pages
|
|
141
|
+
|
|
142
|
+
The code array is `mlock()`ed. If `RLIMIT_MEMLOCK` refuses it, the index falls
|
|
143
|
+
back to `madvise(MADV_WILLNEED)` and emits `CMemoryLockWarning`; it still works,
|
|
144
|
+
but the codes can be paged out under memory pressure. To allow locking:
|
|
145
|
+
|
|
146
|
+
```yaml
|
|
147
|
+
services:
|
|
148
|
+
agent-worker:
|
|
149
|
+
image: my-agent:latest
|
|
150
|
+
ulimits:
|
|
151
|
+
memlock:
|
|
152
|
+
soft: -1
|
|
153
|
+
hard: -1
|
|
154
|
+
cap_add:
|
|
155
|
+
- IPC_LOCK
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Tests
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
pytest # functional tests
|
|
162
|
+
pytest -m benchmark -s # latency checks (prints p50/p99)
|
|
163
|
+
GENESIS_L0_E2E_BUDGET_US=300 pytest -m benchmark
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The end-to-end benchmark (precision mode) asserts p50 < 450 µs by default;
|
|
167
|
+
override `GENESIS_L0_E2E_BUDGET_US` for your hardware.
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
Business Source License 1.1 (see `LICENSE`), changing to Apache 2.0 on
|
|
172
|
+
2030-01-01. `csrc/srp_lsh.c` and `csrc/srp_lsh.h` are Apache 2.0 (see `NOTICE`).
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# genesis-l0
|
|
2
|
+
|
|
3
|
+
In-process vector memory for AI-agent loops (LangGraph, AutoGen, CrewAI): a C99
|
|
4
|
+
L0 index in front of a local SQLite system of record. No sockets, no server.
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
query vector (float32)
|
|
8
|
+
│
|
|
9
|
+
▼ C99, ctypes (GIL released)
|
|
10
|
+
256-bit SRP-LSH code ─► POPCNT Hamming scan over all codes (1.01 MB for 33k vectors)
|
|
11
|
+
─► top-`pool` candidates ─► exact inner-product re-rank ─► top-k glyph ids
|
|
12
|
+
│
|
|
13
|
+
▼ SQLite (WAL, mmap_size = 256 MB)
|
|
14
|
+
typed payload of the requested face ("code", "research", ...)
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
The cascade is the measured one from
|
|
18
|
+
[genesis-live-core](https://github.com/mister3ai-cmyk/genesis-live-core)
|
|
19
|
+
(DOI [10.5281/zenodo.23246581](https://doi.org/10.5281/zenodo.23246581)).
|
|
20
|
+
|
|
21
|
+
## Measured
|
|
22
|
+
|
|
23
|
+
OpenAI `text-embedding-3-large` (DBpedia, truncated to 512-dim), 33,234 vectors,
|
|
24
|
+
100 held-out queries × 10 passes, mean-centered, top-10.
|
|
25
|
+
Contabo VPS, AMD EPYC 2.0 GHz (AVX2, L3 8 MiB), shared host, x86-64-v3 build.
|
|
26
|
+
Reproduce with `python benchmarks/bench_e2e.py --data /tmp/genesis_eval`.
|
|
27
|
+
|
|
28
|
+
| pool | Recall@10 | L0 index p50 | face fetch p50 | end-to-end p50 | end-to-end p99 |
|
|
29
|
+
|-----:|----------:|-------------:|---------------:|---------------:|---------------:|
|
|
30
|
+
| 512 | **90.6 %** | 306–336 µs | 59–70 µs | 359–400 µs | 465–1190 µs |
|
|
31
|
+
| 256 | 82.1 % | 226 µs | 61 µs | 285 µs | 389 µs |
|
|
32
|
+
| 128 | 73.9 % | 176 µs | 55 µs | 228 µs | 377 µs |
|
|
33
|
+
|
|
34
|
+
- Recall@10 is against exact brute force. Synthetic clustered 512-dim data: 100 %.
|
|
35
|
+
- Face fetch = one `SELECT … IN (10 ids)` right after the index query (caches
|
|
36
|
+
already warmed by the re-rank). On its own it measures ~18–20 µs p50.
|
|
37
|
+
- p99 varies between runs on the shared VPS; the range is three runs.
|
|
38
|
+
- Memory: 32 bytes of code per vector stay hot in L3; the float32 copy used by
|
|
39
|
+
the re-rank (2 KB per 512-dim vector, ~65 MB for 33k) is read from RAM.
|
|
40
|
+
|
|
41
|
+
## Modes
|
|
42
|
+
|
|
43
|
+
`pool` is the only knob between accuracy and latency:
|
|
44
|
+
|
|
45
|
+
| mode | `pool` | Recall@10 | end-to-end p50 |
|
|
46
|
+
|------|-------:|----------:|---------------:|
|
|
47
|
+
| **Precision** (default) | 512 | 90.6 % | ~380 µs |
|
|
48
|
+
| **Fast** | 128 | 73.9 % | ~228 µs (fits a hard < 250 µs budget) |
|
|
49
|
+
|
|
50
|
+
Precision is the default: inside an LLM step the ~150 µs difference is noise,
|
|
51
|
+
while a missed context hit is not.
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
mem = GenesisL0Memory("agent_memory.db", pool=128) # fast mode
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
Linux or macOS, Python ≥ 3.9, a C compiler.
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install . # or: pip install -e ".[dev,langgraph]"
|
|
63
|
+
GENESIS_L0_NATIVE=1 pip install . # -march=native instead of portable x86-64-v3
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
x86_64 builds target x86-64-v3 (AVX2 + POPCNT: Intel Haswell / AMD Zen and newer).
|
|
67
|
+
|
|
68
|
+
## Quickstart
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
import numpy as np
|
|
72
|
+
from genesis_l0 import GenesisL0Memory
|
|
73
|
+
|
|
74
|
+
corpus = np.random.randn(1000, 512).astype(np.float32)
|
|
75
|
+
|
|
76
|
+
with GenesisL0Memory(db_path="agent_memory.db", capacity=50_000, dim=512,
|
|
77
|
+
pool=512, mean=corpus.mean(axis=0)) as mem:
|
|
78
|
+
mem.add_batch(
|
|
79
|
+
np.arange(1000), corpus,
|
|
80
|
+
[{"slug": f"step_{i}", "faces": {"code": f"def tool_{i}(): ..."}} for i in range(1000)],
|
|
81
|
+
)
|
|
82
|
+
mem.add_glyph(5000, corpus[0], slug="auth_agent", faces={"code": "def verify(): pass"})
|
|
83
|
+
|
|
84
|
+
for hit in mem.query(corpus[0], top_k=5, face="code"):
|
|
85
|
+
print(hit["glyph_id"], round(hit["score"], 3), hit["payload"])
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
- Vectors are stored in SQLite; reopening the database rebuilds the index.
|
|
89
|
+
- `mean` (optional) enables mean-centering. It is fixed once vectors exist and
|
|
90
|
+
is stored in the database. For anisotropic embeddings it adds a few recall points.
|
|
91
|
+
- `dim` must be a multiple of 16; `top_k` is capped at 64; glyph ids are uint32.
|
|
92
|
+
- Writes are serialized; `query()` is safe from any number of threads.
|
|
93
|
+
Do not call `close()` while queries are running.
|
|
94
|
+
|
|
95
|
+
## LangGraph checkpointer
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
pip install "genesis-l0[langgraph]"
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from genesis_l0.adapters.langgraph import GenesisL0CheckpointSaver
|
|
103
|
+
|
|
104
|
+
saver = GenesisL0CheckpointSaver.from_path("checkpoints.db")
|
|
105
|
+
graph = builder.compile(checkpointer=saver)
|
|
106
|
+
graph.invoke(inputs, {"configurable": {"thread_id": "agent-1"}})
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
It is the official `langgraph-checkpoint-sqlite` `SqliteSaver` on a connection
|
|
110
|
+
with the genesis-l0 pragmas, so `put_writes`, `list`, history and serialization
|
|
111
|
+
behave exactly as upstream.
|
|
112
|
+
|
|
113
|
+
## Docker / Kubernetes: locked pages
|
|
114
|
+
|
|
115
|
+
The code array is `mlock()`ed. If `RLIMIT_MEMLOCK` refuses it, the index falls
|
|
116
|
+
back to `madvise(MADV_WILLNEED)` and emits `CMemoryLockWarning`; it still works,
|
|
117
|
+
but the codes can be paged out under memory pressure. To allow locking:
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
services:
|
|
121
|
+
agent-worker:
|
|
122
|
+
image: my-agent:latest
|
|
123
|
+
ulimits:
|
|
124
|
+
memlock:
|
|
125
|
+
soft: -1
|
|
126
|
+
hard: -1
|
|
127
|
+
cap_add:
|
|
128
|
+
- IPC_LOCK
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Tests
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
pytest # functional tests
|
|
135
|
+
pytest -m benchmark -s # latency checks (prints p50/p99)
|
|
136
|
+
GENESIS_L0_E2E_BUDGET_US=300 pytest -m benchmark
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The end-to-end benchmark (precision mode) asserts p50 < 450 µs by default;
|
|
140
|
+
override `GENESIS_L0_E2E_BUDGET_US` for your hardware.
|
|
141
|
+
|
|
142
|
+
## License
|
|
143
|
+
|
|
144
|
+
Business Source License 1.1 (see `LICENSE`), changing to Apache 2.0 on
|
|
145
|
+
2030-01-01. `csrc/srp_lsh.c` and `csrc/srp_lsh.h` are Apache 2.0 (see `NOTICE`).
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
bench_e2e.py - Recall@10 and latency of the installed genesis-l0 package.
|
|
4
|
+
|
|
5
|
+
python benchmarks/bench_e2e.py # synthetic clustered 512-dim
|
|
6
|
+
python benchmarks/bench_e2e.py --data /tmp/genesis_eval # corpus_512.fvecs + queries_512.fvecs
|
|
7
|
+
# from genesis-live-core eval_real_embeddings.py
|
|
8
|
+
|
|
9
|
+
Ground truth = exact inner-product top-10 (numpy brute force). Every query is
|
|
10
|
+
distinct, so the re-rank rows come from RAM rather than a replayed cache.
|
|
11
|
+
"""
|
|
12
|
+
import argparse
|
|
13
|
+
import os
|
|
14
|
+
import tempfile
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
from genesis_l0 import GenesisL0Memory
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def read_fvecs(path):
|
|
23
|
+
raw = np.fromfile(path, dtype=np.int32)
|
|
24
|
+
dim = raw[0]
|
|
25
|
+
return raw.reshape(-1, dim + 1)[:, 1:].view(np.float32).copy()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def synthetic(n, n_query, dim=512, clusters=1000, noise=1.0, seed=0):
|
|
29
|
+
rng = np.random.default_rng(seed)
|
|
30
|
+
centers = rng.standard_normal((clusters, dim)).astype(np.float32)
|
|
31
|
+
|
|
32
|
+
def draw(m):
|
|
33
|
+
x = centers[rng.integers(0, clusters, m)] + noise * rng.standard_normal((m, dim)).astype(np.float32)
|
|
34
|
+
return x / np.linalg.norm(x, axis=1, keepdims=True)
|
|
35
|
+
|
|
36
|
+
return draw(n), draw(n_query)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def pct(v, p):
|
|
40
|
+
return float(np.percentile(v, p))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main():
|
|
44
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
45
|
+
ap.add_argument("--data", help="directory with corpus_512.fvecs and queries_512.fvecs")
|
|
46
|
+
ap.add_argument("--n", type=int, default=33234, help="synthetic corpus size")
|
|
47
|
+
ap.add_argument("--pool", type=int, default=512)
|
|
48
|
+
ap.add_argument("--no-center", action="store_true")
|
|
49
|
+
ap.add_argument("--passes", type=int, default=10)
|
|
50
|
+
args = ap.parse_args()
|
|
51
|
+
|
|
52
|
+
if args.data:
|
|
53
|
+
corpus = read_fvecs(os.path.join(args.data, "corpus_512.fvecs"))
|
|
54
|
+
queries = read_fvecs(os.path.join(args.data, "queries_512.fvecs"))
|
|
55
|
+
source = f"fvecs {args.data}"
|
|
56
|
+
else:
|
|
57
|
+
corpus, queries = synthetic(args.n, 200)
|
|
58
|
+
source = "synthetic clustered"
|
|
59
|
+
n, dim = corpus.shape
|
|
60
|
+
mean = None if args.no_center else corpus.mean(axis=0)
|
|
61
|
+
truth = np.argsort(-(queries @ corpus.T), axis=1)[:, :10]
|
|
62
|
+
|
|
63
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
64
|
+
db = os.path.join(tmp, "bench.db")
|
|
65
|
+
with GenesisL0Memory(db, capacity=n, dim=dim, pool=args.pool, mean=mean) as mem:
|
|
66
|
+
t = time.perf_counter()
|
|
67
|
+
mem.add_batch(np.arange(n), corpus,
|
|
68
|
+
[{"slug": f"g{i}", "faces": {"code": f"payload {i}"}} for i in range(n)])
|
|
69
|
+
build_s = time.perf_counter() - t
|
|
70
|
+
|
|
71
|
+
hits = sum(len(set(mem.index.query(q, 10)[0]) & set(tr.tolist())) for q, tr in zip(queries, truth))
|
|
72
|
+
|
|
73
|
+
for q in queries: # warm-up pass
|
|
74
|
+
mem.query(q, top_k=10, face="code")
|
|
75
|
+
l0, e2e, face = [], [], []
|
|
76
|
+
for _ in range(args.passes):
|
|
77
|
+
for q in queries:
|
|
78
|
+
t0 = time.perf_counter_ns()
|
|
79
|
+
ids, _ = mem.index.query(q, 10)
|
|
80
|
+
t1 = time.perf_counter_ns()
|
|
81
|
+
mem.storage.fetch_payloads(ids, "code")
|
|
82
|
+
t2 = time.perf_counter_ns()
|
|
83
|
+
l0.append((t1 - t0) / 1e3)
|
|
84
|
+
face.append((t2 - t1) / 1e3)
|
|
85
|
+
e2e.append((t2 - t0) / 1e3)
|
|
86
|
+
code_mb = mem.index.code_bytes / 2**20
|
|
87
|
+
|
|
88
|
+
print(f"data: {source}, {n} x {dim}, {len(queries)} queries x {args.passes} passes, "
|
|
89
|
+
f"{mem.index.bits}-bit, pool {args.pool}, centered={mean is not None}")
|
|
90
|
+
print(f"codes {code_mb:.2f} MB, build+store {build_s:.1f} s")
|
|
91
|
+
print(f"Recall@10 vs exact brute force: {100.0 * hits / (10 * len(queries)):.1f} %")
|
|
92
|
+
print(f"{'':28}{'p50':>8}{'p99':>9} (us)")
|
|
93
|
+
for name, v in (("L0 index (ctypes call)", l0), ("SQLite face fetch (10 ids)", face), ("end-to-end", e2e)):
|
|
94
|
+
print(f"{name:28}{pct(v, 50):8.1f}{pct(v, 99):9.1f}")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
main()
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* csrc/l0_api.c — handle-based facade over srp_lsh.c (see l0_api.h).
|
|
3
|
+
* Same cascade as genesis-live-core benchmarks/run_benchmark.c:
|
|
4
|
+
* planes seed 7, optional mean-centering, Hamming pool -> exact re-rank.
|
|
5
|
+
*
|
|
6
|
+
* Licensed under the Business Source License 1.1 (see LICENSE).
|
|
7
|
+
*/
|
|
8
|
+
#define _GNU_SOURCE
|
|
9
|
+
#include "l0_api.h"
|
|
10
|
+
#include "srp_lsh.h"
|
|
11
|
+
|
|
12
|
+
#include <pthread.h>
|
|
13
|
+
#include <stdlib.h>
|
|
14
|
+
#include <string.h>
|
|
15
|
+
#include <sys/mman.h>
|
|
16
|
+
|
|
17
|
+
#define L0_PLANE_SEED 7ULL
|
|
18
|
+
#define L0_ALIGN 64
|
|
19
|
+
|
|
20
|
+
struct l0_index {
|
|
21
|
+
int capacity;
|
|
22
|
+
int dim;
|
|
23
|
+
int pool;
|
|
24
|
+
int num_vectors;
|
|
25
|
+
int memlock_status;
|
|
26
|
+
int centered;
|
|
27
|
+
|
|
28
|
+
float *planes; /* SRP_BITS * dim */
|
|
29
|
+
float offsets[SRP_BITS]; /* <plane_b, mean> when centered */
|
|
30
|
+
srp_code *codes; /* capacity, mlock()ed */
|
|
31
|
+
float *vectors; /* capacity * dim, for the re-rank */
|
|
32
|
+
uint32_t *ids; /* capacity: row -> glyph id */
|
|
33
|
+
size_t code_bytes;
|
|
34
|
+
|
|
35
|
+
pthread_rwlock_t lock;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
static void *aligned_alloc_zero(size_t bytes) {
|
|
39
|
+
void *p = NULL;
|
|
40
|
+
if (posix_memalign(&p, L0_ALIGN, bytes ? bytes : L0_ALIGN) != 0) return NULL;
|
|
41
|
+
memset(p, 0, bytes);
|
|
42
|
+
return p;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
int l0_bits(void) { return SRP_BITS; }
|
|
46
|
+
|
|
47
|
+
l0_index_t *l0_create_index(int capacity, int dim, int pool) {
|
|
48
|
+
if (capacity <= 0 || dim <= 0 || dim % 16 != 0 || pool <= 0) return NULL;
|
|
49
|
+
|
|
50
|
+
l0_index_t *index = calloc(1, sizeof(*index));
|
|
51
|
+
if (!index) return NULL;
|
|
52
|
+
|
|
53
|
+
index->capacity = capacity;
|
|
54
|
+
index->dim = dim;
|
|
55
|
+
index->pool = pool;
|
|
56
|
+
index->code_bytes = (size_t)capacity * sizeof(srp_code);
|
|
57
|
+
|
|
58
|
+
index->planes = aligned_alloc_zero((size_t)SRP_BITS * dim * sizeof(float));
|
|
59
|
+
index->codes = aligned_alloc_zero(index->code_bytes);
|
|
60
|
+
index->vectors = aligned_alloc_zero((size_t)capacity * dim * sizeof(float));
|
|
61
|
+
index->ids = malloc((size_t)capacity * sizeof(uint32_t));
|
|
62
|
+
if (!index->planes || !index->codes || !index->vectors || !index->ids ||
|
|
63
|
+
pthread_rwlock_init(&index->lock, NULL) != 0) {
|
|
64
|
+
free(index->planes);
|
|
65
|
+
free(index->codes);
|
|
66
|
+
free(index->vectors);
|
|
67
|
+
free(index->ids);
|
|
68
|
+
free(index);
|
|
69
|
+
return NULL;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
srp_make_planes(index->planes, dim, L0_PLANE_SEED);
|
|
73
|
+
|
|
74
|
+
if (mlock(index->codes, index->code_bytes) == 0) {
|
|
75
|
+
index->memlock_status = L0_STATUS_OK;
|
|
76
|
+
} else {
|
|
77
|
+
#ifdef MADV_WILLNEED
|
|
78
|
+
madvise(index->codes, index->code_bytes, MADV_WILLNEED);
|
|
79
|
+
#endif
|
|
80
|
+
index->memlock_status = L0_STATUS_WARN_MEMLOCK_FALLBACK;
|
|
81
|
+
}
|
|
82
|
+
return index;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
int l0_get_memlock_status(const l0_index_t *index) {
|
|
86
|
+
return index ? index->memlock_status : L0_STATUS_ERR_INVALID_PARAM;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
int l0_count(const l0_index_t *index) {
|
|
90
|
+
if (!index) return L0_STATUS_ERR_INVALID_PARAM;
|
|
91
|
+
pthread_rwlock_rdlock((pthread_rwlock_t *)&index->lock);
|
|
92
|
+
int n = index->num_vectors;
|
|
93
|
+
pthread_rwlock_unlock((pthread_rwlock_t *)&index->lock);
|
|
94
|
+
return n;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
size_t l0_code_bytes(const l0_index_t *index) {
|
|
98
|
+
return index ? (size_t)l0_count(index) * sizeof(srp_code) : 0;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
int l0_set_mean(l0_index_t *index, const float *mean) {
|
|
102
|
+
if (!index || !mean) return L0_STATUS_ERR_INVALID_PARAM;
|
|
103
|
+
int rc = L0_STATUS_OK;
|
|
104
|
+
pthread_rwlock_wrlock(&index->lock);
|
|
105
|
+
if (index->num_vectors != 0) {
|
|
106
|
+
rc = L0_STATUS_ERR_NOT_EMPTY;
|
|
107
|
+
} else {
|
|
108
|
+
srp_make_offsets(index->planes, index->dim, mean, index->offsets);
|
|
109
|
+
index->centered = 1;
|
|
110
|
+
}
|
|
111
|
+
pthread_rwlock_unlock(&index->lock);
|
|
112
|
+
return rc;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/* Caller holds the write lock and has checked capacity. */
|
|
116
|
+
static void add_locked(l0_index_t *index, uint32_t id, const float *vec) {
|
|
117
|
+
int row = index->num_vectors++;
|
|
118
|
+
float *dst = index->vectors + (size_t)row * index->dim;
|
|
119
|
+
memcpy(dst, vec, (size_t)index->dim * sizeof(float));
|
|
120
|
+
index->ids[row] = id;
|
|
121
|
+
index->codes[row] = srp_encode(index->planes, index->dim, dst,
|
|
122
|
+
index->centered ? index->offsets : NULL);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
int l0_add_vector(l0_index_t *index, uint32_t id, const float *vec) {
|
|
126
|
+
return l0_add_vectors_batch(index, &id, vec, 1);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
int l0_add_vectors_batch(l0_index_t *index, const uint32_t *ids,
|
|
130
|
+
const float *vectors, int count) {
|
|
131
|
+
if (!index || count < 0 || (count > 0 && (!ids || !vectors)))
|
|
132
|
+
return L0_STATUS_ERR_INVALID_PARAM;
|
|
133
|
+
int rc = L0_STATUS_OK;
|
|
134
|
+
pthread_rwlock_wrlock(&index->lock);
|
|
135
|
+
if (count > index->capacity - index->num_vectors) {
|
|
136
|
+
rc = L0_STATUS_ERR_CAPACITY;
|
|
137
|
+
} else {
|
|
138
|
+
for (int i = 0; i < count; i++)
|
|
139
|
+
add_locked(index, ids[i], vectors + (size_t)i * index->dim);
|
|
140
|
+
}
|
|
141
|
+
pthread_rwlock_unlock(&index->lock);
|
|
142
|
+
return rc;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
int l0_query(l0_index_t *index, const float *query_vec, int top_k,
|
|
146
|
+
uint32_t *out_ids, float *out_scores) {
|
|
147
|
+
if (!index || !query_vec || !out_ids || !out_scores || top_k <= 0)
|
|
148
|
+
return L0_STATUS_ERR_INVALID_PARAM;
|
|
149
|
+
if (top_k > SRP_RERANK_MAX) top_k = SRP_RERANK_MAX;
|
|
150
|
+
|
|
151
|
+
pthread_rwlock_rdlock(&index->lock);
|
|
152
|
+
size_t n = (size_t)index->num_vectors;
|
|
153
|
+
if (n == 0) {
|
|
154
|
+
pthread_rwlock_unlock(&index->lock);
|
|
155
|
+
return 0;
|
|
156
|
+
}
|
|
157
|
+
size_t pool = (size_t)index->pool < n ? (size_t)index->pool : n;
|
|
158
|
+
if (pool < (size_t)top_k) pool = (size_t)top_k < n ? (size_t)top_k : n;
|
|
159
|
+
|
|
160
|
+
uint32_t *cand = malloc(pool * sizeof(uint32_t));
|
|
161
|
+
if (!cand) {
|
|
162
|
+
pthread_rwlock_unlock(&index->lock);
|
|
163
|
+
return L0_STATUS_ERR_ALLOC;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
srp_code qc = srp_encode(index->planes, index->dim, query_vec,
|
|
167
|
+
index->centered ? index->offsets : NULL);
|
|
168
|
+
size_t m = srp_scan_topk(&qc, index->codes, n, pool, cand);
|
|
169
|
+
|
|
170
|
+
uint32_t rows[SRP_RERANK_MAX];
|
|
171
|
+
size_t r = srp_rerank_topk(index->vectors, index->dim, query_vec,
|
|
172
|
+
cand, m, (size_t)top_k, rows);
|
|
173
|
+
for (size_t i = 0; i < r; i++) {
|
|
174
|
+
out_ids[i] = index->ids[rows[i]];
|
|
175
|
+
out_scores[i] = srp_dot(index->vectors + (size_t)rows[i] * index->dim,
|
|
176
|
+
query_vec, index->dim);
|
|
177
|
+
}
|
|
178
|
+
pthread_rwlock_unlock(&index->lock);
|
|
179
|
+
free(cand);
|
|
180
|
+
return (int)r;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
void l0_free_index(l0_index_t *index) {
|
|
184
|
+
if (!index) return;
|
|
185
|
+
if (index->memlock_status == L0_STATUS_OK)
|
|
186
|
+
munlock(index->codes, index->code_bytes);
|
|
187
|
+
pthread_rwlock_destroy(&index->lock);
|
|
188
|
+
free(index->planes);
|
|
189
|
+
free(index->codes);
|
|
190
|
+
free(index->vectors);
|
|
191
|
+
free(index->ids);
|
|
192
|
+
free(index);
|
|
193
|
+
}
|