vector-engine 1.0.2__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vector_engine-1.0.2 → vector_engine-1.1.0}/PKG-INFO +10 -5
- {vector_engine-1.0.2 → vector_engine-1.1.0}/README.md +9 -4
- {vector_engine-1.0.2 → vector_engine-1.1.0}/pyproject.toml +1 -1
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_core.py +14 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_dataset_benchmark_tooling.py +95 -2
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_ingest_pipeline.py +16 -0
- vector_engine-1.1.0/tests/test_matrix_profile_advisor.py +24 -0
- vector_engine-1.1.0/tests/test_release_performance_workflow.py +125 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/__init__.py +1 -1
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/backends/base.py +3 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/backends/bruteforce.py +14 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/backends/faiss_backend.py +13 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/index.py +31 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine.egg-info/PKG-INFO +10 -5
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine.egg-info/SOURCES.txt +2 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/LICENSE +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/setup.cfg +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_api_stability.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_artifact_contracts.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_credibility_audit.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_env_diagnostics.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_eval_surface_v1.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_faiss_optional.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_hardening.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_install_smoke.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_ml_eval.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_perf_smoke.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_persistence_compat.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_profile_local.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_rag_reliability.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_real_corpus_eval.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_release_bundle.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/tests/test_v02_features.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/array.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/backends/__init__.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/backends/registry.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/eval/__init__.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/eval/retrieval.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/io/__init__.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/io/manifest.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/metric.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/ml/__init__.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/ml/clustering.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/ml/knn.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/results.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/training/__init__.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine/training/hard_negative.py +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine.egg-info/dependency_links.txt +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine.egg-info/requires.txt +0 -0
- {vector_engine-1.0.2 → vector_engine-1.1.0}/vector_engine.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vector-engine
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: ML-first vector computation and retrieval engine.
|
|
5
5
|
Author: Neel Panchal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,7 +16,7 @@ Provides-Extra: dev
|
|
|
16
16
|
Requires-Dist: pytest<9,>=8.2; extra == "dev"
|
|
17
17
|
Dynamic: license-file
|
|
18
18
|
|
|
19
|
-
# Vector Engine v1.
|
|
19
|
+
# Vector Engine v1.1.0
|
|
20
20
|
|
|
21
21
|
Reproducibility-first vector retrieval toolkit for local ML and IR workflows.
|
|
22
22
|
|
|
@@ -95,7 +95,12 @@ results = index.search(xq, k=5)
|
|
|
95
95
|
print(results.ids[0], results.scores[0])
|
|
96
96
|
```
|
|
97
97
|
|
|
98
|
-
##
|
|
98
|
+
## Colab Demos
|
|
99
|
+
|
|
100
|
+
- Demo A (basic): [Google Colab](https://colab.research.google.com/drive/1VUkf8mc1lBEQQ9iBWr_WitJJgbcbIPZ6?usp=sharing)
|
|
101
|
+
- Scaling expansion: [Google Colab](https://colab.research.google.com/drive/1HCd1_R56eBLuTzobYTeiCeAIwT-K7XZb?usp=sharing)
|
|
102
|
+
|
|
103
|
+
## v1.1.0 Surface
|
|
99
104
|
|
|
100
105
|
- Core: `VectorArray`, `VectorIndex`, `Metric`, `SearchResult`
|
|
101
106
|
- ML: `knn_classify`, `knn_regress`, `kmeans`, `KMeansResult`
|
|
@@ -192,8 +197,8 @@ python scripts/credibility_audit.py --matrix-summary artifacts/benchmark_matrix/
|
|
|
192
197
|
|
|
193
198
|
## Project Links
|
|
194
199
|
|
|
195
|
-
- `docs/releases/v1.
|
|
196
|
-
- `docs/releases/v1.
|
|
200
|
+
- `docs/releases/v1.1.0.md`
|
|
201
|
+
- `docs/releases/v1.1.0-checklist.md`
|
|
197
202
|
- `docs/reproducibility.md`
|
|
198
203
|
- `docs/use_cases.md`
|
|
199
204
|
- `docs/api_stability.md`
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Vector Engine v1.
|
|
1
|
+
# Vector Engine v1.1.0
|
|
2
2
|
|
|
3
3
|
Reproducibility-first vector retrieval toolkit for local ML and IR workflows.
|
|
4
4
|
|
|
@@ -77,7 +77,12 @@ results = index.search(xq, k=5)
|
|
|
77
77
|
print(results.ids[0], results.scores[0])
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
-
##
|
|
80
|
+
## Colab Demos
|
|
81
|
+
|
|
82
|
+
- Demo A (basic): [Google Colab](https://colab.research.google.com/drive/1VUkf8mc1lBEQQ9iBWr_WitJJgbcbIPZ6?usp=sharing)
|
|
83
|
+
- Scaling expansion: [Google Colab](https://colab.research.google.com/drive/1HCd1_R56eBLuTzobYTeiCeAIwT-K7XZb?usp=sharing)
|
|
84
|
+
|
|
85
|
+
## v1.1.0 Surface
|
|
81
86
|
|
|
82
87
|
- Core: `VectorArray`, `VectorIndex`, `Metric`, `SearchResult`
|
|
83
88
|
- ML: `knn_classify`, `knn_regress`, `kmeans`, `KMeansResult`
|
|
@@ -174,8 +179,8 @@ python scripts/credibility_audit.py --matrix-summary artifacts/benchmark_matrix/
|
|
|
174
179
|
|
|
175
180
|
## Project Links
|
|
176
181
|
|
|
177
|
-
- `docs/releases/v1.
|
|
178
|
-
- `docs/releases/v1.
|
|
182
|
+
- `docs/releases/v1.1.0.md`
|
|
183
|
+
- `docs/releases/v1.1.0-checklist.md`
|
|
179
184
|
- `docs/reproducibility.md`
|
|
180
185
|
- `docs/use_cases.md`
|
|
181
186
|
- `docs/api_stability.md`
|
|
@@ -37,3 +37,17 @@ def test_index_save_and_load_roundtrip(tmp_path):
|
|
|
37
37
|
loaded = VectorIndex.load(str(path))
|
|
38
38
|
second = loaded.search(xq, k=4)
|
|
39
39
|
assert np.array_equal(first.ids, second.ids)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_index_backend_capabilities_and_runtime_stats():
|
|
43
|
+
xb = VectorArray.from_numpy(np.random.randn(16, 4).astype(np.float32), ids=np.arange(16))
|
|
44
|
+
xq = VectorArray.from_numpy(np.random.randn(2, 4).astype(np.float32))
|
|
45
|
+
index = VectorIndex.create(xb, metric="cosine", backend="bruteforce")
|
|
46
|
+
caps = index.backend_capabilities()
|
|
47
|
+
assert caps["supports_add"] is True
|
|
48
|
+
assert caps["supports_ann_tuning"] is False
|
|
49
|
+
index.search(xq, k=3)
|
|
50
|
+
stats = index.runtime_stats()
|
|
51
|
+
assert stats["search_calls"] == 1
|
|
52
|
+
assert stats["last_search_k"] == 3
|
|
53
|
+
assert stats["backend_stats"]["backend"] == "bruteforce"
|
|
@@ -4,8 +4,8 @@ import subprocess
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
|
-
from scripts.benchmark_matrix import _apply_memory_limit, _parse_matrix, run_matrix
|
|
8
|
-
from scripts.datasets import load_jsonl_bundle, load_numpy_bundle, with_deterministic_splits
|
|
7
|
+
from scripts.benchmark_matrix import _apply_memory_limit, _build_run_variants, _parse_matrix, run_matrix
|
|
8
|
+
from scripts.datasets import estimate_numpy_bundle_memory_mb, load_jsonl_bundle, load_numpy_bundle, with_deterministic_splits
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def test_parse_matrix_profile_defaults_to_dev():
|
|
@@ -14,6 +14,29 @@ def test_parse_matrix_profile_defaults_to_dev():
|
|
|
14
14
|
assert {"name", "n", "d", "nq", "k"}.issubset(set(matrix[0].keys()))
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
def test_parse_matrix_profile_scale():
|
|
18
|
+
matrix = _parse_matrix(None, profile="scale")
|
|
19
|
+
assert len(matrix) >= 1
|
|
20
|
+
assert "ann_index_factory_options" in matrix[0]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_build_run_variants_ann_expands():
|
|
24
|
+
variants = _build_run_variants(
|
|
25
|
+
{
|
|
26
|
+
"name": "cfg",
|
|
27
|
+
"n": 1000,
|
|
28
|
+
"d": 64,
|
|
29
|
+
"nq": 32,
|
|
30
|
+
"k": 5,
|
|
31
|
+
"ann_index_factory_options": ["IVF64,Flat", "IVF128,Flat"],
|
|
32
|
+
"ann_nprobe_options": [4, 8],
|
|
33
|
+
},
|
|
34
|
+
mode="ann",
|
|
35
|
+
)
|
|
36
|
+
assert len(variants) == 5
|
|
37
|
+
assert any(v["faiss_ivf_nprobe"] == 8 for v in variants)
|
|
38
|
+
|
|
39
|
+
|
|
17
40
|
def test_apply_memory_limit_filters_large_configs():
|
|
18
41
|
matrix = [
|
|
19
42
|
{"name": "a", "n": 1000, "d": 32, "nq": 64, "k": 5},
|
|
@@ -40,6 +63,24 @@ def test_load_numpy_bundle(tmp_path):
|
|
|
40
63
|
assert bundle.metadata is not None
|
|
41
64
|
|
|
42
65
|
|
|
66
|
+
def test_load_numpy_bundle_memmap_mode(tmp_path):
|
|
67
|
+
xb = np.random.randn(6, 12).astype(np.float32)
|
|
68
|
+
emb = tmp_path / "emb.npy"
|
|
69
|
+
ids = tmp_path / "ids.json"
|
|
70
|
+
np.save(emb, xb)
|
|
71
|
+
ids.write_text(json.dumps([f"doc-{i}" for i in range(6)]), encoding="utf-8")
|
|
72
|
+
bundle = load_numpy_bundle(str(emb), str(ids), mmap_mode="r")
|
|
73
|
+
assert bundle.embeddings.shape == (6, 12)
|
|
74
|
+
assert bundle.ids[0] == "doc-0"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_estimate_numpy_bundle_memory_mb():
|
|
78
|
+
estimate = estimate_numpy_bundle_memory_mb(1000, 64)
|
|
79
|
+
assert estimate > 0.0
|
|
80
|
+
with pytest.raises(ValueError, match="dataset_error"):
|
|
81
|
+
estimate_numpy_bundle_memory_mb(0, 64)
|
|
82
|
+
|
|
83
|
+
|
|
43
84
|
def test_load_numpy_bundle_with_extended_fields(tmp_path):
|
|
44
85
|
xb = np.random.randn(4, 8).astype(np.float32)
|
|
45
86
|
emb = tmp_path / "emb.npy"
|
|
@@ -189,3 +230,55 @@ def test_run_matrix_fails_when_overlap_gate_enabled_without_faiss(tmp_path, monk
|
|
|
189
230
|
profile="dev",
|
|
190
231
|
max_memory_mb=1024.0,
|
|
191
232
|
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def test_run_matrix_ann_sweep_invokes_multiple_configs(tmp_path, monkeypatch):
|
|
236
|
+
calls: list[list[str]] = []
|
|
237
|
+
|
|
238
|
+
def fake_check_call(cmd):
|
|
239
|
+
calls.append(cmd)
|
|
240
|
+
out_path = cmd[cmd.index("--output") + 1]
|
|
241
|
+
payload = {
|
|
242
|
+
"timestamp_utc": "2026-01-01T00:00:00+00:00",
|
|
243
|
+
"config": {"mode": "ann", "k": 5, "min_flat_overlap": None},
|
|
244
|
+
"environment": {"platform": "darwin", "python_version": "3.12", "machine": "arm64", "processor": "arm"},
|
|
245
|
+
"results": [
|
|
246
|
+
{
|
|
247
|
+
"backend": "bruteforce",
|
|
248
|
+
"qps": 900.0,
|
|
249
|
+
"latency_p50_ms": 1.1,
|
|
250
|
+
"latency_p95_ms": 2.2,
|
|
251
|
+
"overlap_vs_bruteforce": 1.0,
|
|
252
|
+
"memory_mb_estimate": 10.0,
|
|
253
|
+
}
|
|
254
|
+
],
|
|
255
|
+
"artifact_contract_version": "1.0",
|
|
256
|
+
}
|
|
257
|
+
with open(out_path, "w", encoding="utf-8") as f:
|
|
258
|
+
json.dump(payload, f)
|
|
259
|
+
|
|
260
|
+
monkeypatch.setattr(subprocess, "check_call", fake_check_call)
|
|
261
|
+
summary = run_matrix(
|
|
262
|
+
matrix=[
|
|
263
|
+
{
|
|
264
|
+
"name": "tiny",
|
|
265
|
+
"n": 2000,
|
|
266
|
+
"d": 32,
|
|
267
|
+
"nq": 64,
|
|
268
|
+
"k": 5,
|
|
269
|
+
"ann_index_factory_options": ["IVF64,Flat"],
|
|
270
|
+
"ann_nprobe_options": [4, 8],
|
|
271
|
+
}
|
|
272
|
+
],
|
|
273
|
+
mode="ann",
|
|
274
|
+
warmup=1,
|
|
275
|
+
loops=1,
|
|
276
|
+
seed=7,
|
|
277
|
+
min_flat_overlap=None,
|
|
278
|
+
out_dir=str(tmp_path / "out"),
|
|
279
|
+
profile="dev",
|
|
280
|
+
max_memory_mb=1024.0,
|
|
281
|
+
)
|
|
282
|
+
assert len(calls) == 3
|
|
283
|
+
assert summary["protocol"]["expanded_run_count"] == 3
|
|
284
|
+
assert "pareto_frontier" in summary
|
|
@@ -81,3 +81,19 @@ def test_run_ingest_rejects_duplicate_ids(tmp_path):
|
|
|
81
81
|
embedding_dim=8,
|
|
82
82
|
seed=7,
|
|
83
83
|
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_run_ingest_honors_memory_cap(tmp_path):
|
|
87
|
+
source = tmp_path / "source.jsonl"
|
|
88
|
+
rows = [{"id": f"d{i}", "text": f"text-{i}"} for i in range(10)]
|
|
89
|
+
_write_jsonl(source, rows)
|
|
90
|
+
with pytest.raises(ValueError, match="max_memory_mb"):
|
|
91
|
+
run_ingest(
|
|
92
|
+
input_jsonl=str(source),
|
|
93
|
+
output_dir=str(tmp_path / "out"),
|
|
94
|
+
id_field="id",
|
|
95
|
+
text_field="text",
|
|
96
|
+
embedding_dim=64,
|
|
97
|
+
seed=7,
|
|
98
|
+
max_memory_mb=0.0001,
|
|
99
|
+
)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from scripts.matrix_profile_advisor import recommend_configs
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_recommend_configs_returns_sorted_recommendations(tmp_path):
|
|
7
|
+
run_dir = tmp_path / "runs"
|
|
8
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
9
|
+
matrix_summary = {
|
|
10
|
+
"expanded_runs": ["cfg_a", "cfg_b"],
|
|
11
|
+
"runs_dir": str(run_dir),
|
|
12
|
+
}
|
|
13
|
+
(tmp_path / "matrix_summary.json").write_text(json.dumps(matrix_summary), encoding="utf-8")
|
|
14
|
+
for name, qps, p95, overlap in [("cfg_a", 1200.0, 10.0, 0.90), ("cfg_b", 1000.0, 8.0, 0.98)]:
|
|
15
|
+
payload = {
|
|
16
|
+
"results": [
|
|
17
|
+
{"backend": "bruteforce", "qps": 500.0, "latency_p95_ms": 12.0, "overlap_vs_bruteforce": 1.0},
|
|
18
|
+
{"backend": "faiss_ivf", "qps": qps, "latency_p95_ms": p95, "overlap_vs_bruteforce": overlap},
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
(run_dir / f"{name}.json").write_text(json.dumps(payload), encoding="utf-8")
|
|
22
|
+
out = recommend_configs(str(tmp_path / "matrix_summary.json"), top_n=1)
|
|
23
|
+
assert len(out["recommendations"]) == 1
|
|
24
|
+
assert out["recommendations"][0]["run"] in {"cfg_a", "cfg_b"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from scripts.performance_gates import run_performance_gates
|
|
4
|
+
from scripts.publishable_results import build_publishable_summary
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _write(path, payload):
|
|
8
|
+
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_performance_gates_passes_with_reasonable_metrics(tmp_path):
|
|
12
|
+
matrix = {
|
|
13
|
+
"timestamp_utc": "2026-01-01T00:00:00+00:00",
|
|
14
|
+
"protocol": {"mode": "exact", "warmup": 2, "loops": 5, "seed": 7, "matrix_size": 1, "min_flat_overlap": 0.99},
|
|
15
|
+
"environment": {},
|
|
16
|
+
"matrix": [{"name": "cfg", "n": 1000, "d": 32, "nq": 64, "k": 5}],
|
|
17
|
+
"backend_summary": {
|
|
18
|
+
"bruteforce": {
|
|
19
|
+
"latency_p50_ms": {"mean": 1.0, "median": 1.0, "min": 1.0, "max": 1.0},
|
|
20
|
+
"latency_p95_ms": {"mean": 2.0, "median": 2.0, "min": 2.0, "max": 2.0},
|
|
21
|
+
"qps": {"mean": 100.0, "median": 100.0, "min": 90.0, "max": 110.0},
|
|
22
|
+
"overlap_vs_bruteforce": {"mean": 1.0, "median": 1.0, "min": 1.0, "max": 1.0},
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"runs_dir": "artifacts/benchmark_matrix",
|
|
26
|
+
"artifact_contract_version": "1.0",
|
|
27
|
+
}
|
|
28
|
+
stability = {
|
|
29
|
+
"timestamp_utc": "2026-01-01T00:00:00+00:00",
|
|
30
|
+
"run_count": 3,
|
|
31
|
+
"backend": "bruteforce",
|
|
32
|
+
"config": {},
|
|
33
|
+
"environment": {},
|
|
34
|
+
"performance_summary": {
|
|
35
|
+
"latency_p50_ms": {"mean": 1.0},
|
|
36
|
+
"latency_p95_ms": {"mean": 2.0},
|
|
37
|
+
"qps": {"mean": 150.0},
|
|
38
|
+
},
|
|
39
|
+
"metric_summary": {"recall@10": {"mean": 0.95}, "ndcg@10": {"mean": 0.92}},
|
|
40
|
+
"check_pass_rate": {},
|
|
41
|
+
"input_files": {},
|
|
42
|
+
"runs_path": "runs.jsonl",
|
|
43
|
+
"artifact_contract_version": "1.0",
|
|
44
|
+
}
|
|
45
|
+
matrix_path = tmp_path / "matrix.json"
|
|
46
|
+
stability_path = tmp_path / "stability.json"
|
|
47
|
+
out_path = tmp_path / "gates.json"
|
|
48
|
+
_write(matrix_path, matrix)
|
|
49
|
+
_write(stability_path, stability)
|
|
50
|
+
report = run_performance_gates(
|
|
51
|
+
matrix_summary_path=str(matrix_path),
|
|
52
|
+
stability_summary_path=str(stability_path),
|
|
53
|
+
output_path=str(out_path),
|
|
54
|
+
min_recall=0.8,
|
|
55
|
+
min_ndcg=0.8,
|
|
56
|
+
max_latency_p95_ms=50.0,
|
|
57
|
+
min_qps=50.0,
|
|
58
|
+
)
|
|
59
|
+
assert report["status"] == "pass"
|
|
60
|
+
assert out_path.exists()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_build_publishable_summary_includes_release_delta(tmp_path):
|
|
64
|
+
matrix = {
|
|
65
|
+
"timestamp_utc": "2026-01-01T00:00:00+00:00",
|
|
66
|
+
"protocol": {"mode": "exact", "warmup": 2, "loops": 8, "seed": 7, "matrix_size": 1},
|
|
67
|
+
"environment": {},
|
|
68
|
+
"matrix": [{"name": "cfg", "n": 1000, "d": 32, "nq": 64, "k": 5}],
|
|
69
|
+
"backend_summary": {
|
|
70
|
+
"bruteforce": {
|
|
71
|
+
"latency_p50_ms": {"mean": 1.0, "median": 1.0, "min": 1.0, "max": 1.0},
|
|
72
|
+
"latency_p95_ms": {"mean": 2.0, "median": 2.0, "min": 2.0, "max": 2.0},
|
|
73
|
+
"qps": {"mean": 100.0, "median": 100.0, "min": 95.0, "max": 105.0},
|
|
74
|
+
"overlap_vs_bruteforce": {"mean": 1.0, "median": 1.0, "min": 1.0, "max": 1.0},
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
"runs_dir": "artifacts/benchmark_matrix",
|
|
78
|
+
"artifact_contract_version": "1.0",
|
|
79
|
+
}
|
|
80
|
+
stability = {
|
|
81
|
+
"timestamp_utc": "2026-01-01T00:00:00+00:00",
|
|
82
|
+
"run_count": 2,
|
|
83
|
+
"backend": "bruteforce",
|
|
84
|
+
"config": {"run_count": 2},
|
|
85
|
+
"environment": {},
|
|
86
|
+
"performance_summary": {
|
|
87
|
+
"latency_p50_ms": {"mean": 1.0, "median": 1.0, "std": 0.0, "cv": 0.0, "p02_5": 1.0, "p97_5": 1.0, "min": 1.0, "max": 1.0},
|
|
88
|
+
"latency_p95_ms": {"mean": 2.0, "median": 2.0, "std": 0.0, "cv": 0.0, "p02_5": 2.0, "p97_5": 2.0, "min": 2.0, "max": 2.0},
|
|
89
|
+
"qps": {"mean": 120.0, "median": 120.0, "std": 0.0, "cv": 0.0, "p02_5": 120.0, "p97_5": 120.0, "min": 120.0, "max": 120.0},
|
|
90
|
+
},
|
|
91
|
+
"metric_summary": {"recall@1": {"mean": 0.9}},
|
|
92
|
+
"check_pass_rate": {},
|
|
93
|
+
"input_files": {},
|
|
94
|
+
"runs_path": "runs.jsonl",
|
|
95
|
+
"artifact_contract_version": "1.0",
|
|
96
|
+
}
|
|
97
|
+
previous = {
|
|
98
|
+
"matrix_backend_summary": {
|
|
99
|
+
"bruteforce": {
|
|
100
|
+
"latency_p95_ms": {"mean": 3.0},
|
|
101
|
+
"qps": {"mean": 100.0},
|
|
102
|
+
"overlap_vs_bruteforce": {"mean": 1.0},
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
"stability_performance_summary": {
|
|
106
|
+
"latency_p95_ms": {"mean": 2.5},
|
|
107
|
+
"qps": {"mean": 100.0},
|
|
108
|
+
},
|
|
109
|
+
}
|
|
110
|
+
matrix_path = tmp_path / "matrix.json"
|
|
111
|
+
stability_path = tmp_path / "stability.json"
|
|
112
|
+
previous_path = tmp_path / "previous.json"
|
|
113
|
+
out_path = tmp_path / "publishable.json"
|
|
114
|
+
_write(matrix_path, matrix)
|
|
115
|
+
_write(stability_path, stability)
|
|
116
|
+
_write(previous_path, previous)
|
|
117
|
+
payload = build_publishable_summary(
|
|
118
|
+
matrix_summary_path=str(matrix_path),
|
|
119
|
+
stability_summary_path=str(stability_path),
|
|
120
|
+
out_path=str(out_path),
|
|
121
|
+
previous_summary_path=str(previous_path),
|
|
122
|
+
)
|
|
123
|
+
assert "release_over_release_delta" in payload
|
|
124
|
+
delta = payload["release_over_release_delta"]["stability_performance_delta"]
|
|
125
|
+
assert delta["latency_p95_ms_mean_delta"] < 0.0
|
|
@@ -23,6 +23,8 @@ class BruteForceBackend:
|
|
|
23
23
|
"supports_delete": False,
|
|
24
24
|
"supports_custom_metric": True,
|
|
25
25
|
"supports_persistence": True,
|
|
26
|
+
"supports_add": True,
|
|
27
|
+
"supports_ann_tuning": False,
|
|
26
28
|
}
|
|
27
29
|
)
|
|
28
30
|
xb: np.ndarray | None = None
|
|
@@ -94,6 +96,18 @@ class BruteForceBackend:
|
|
|
94
96
|
f,
|
|
95
97
|
)
|
|
96
98
|
|
|
99
|
+
def get_runtime_stats(self) -> dict[str, float | int | str]:
|
|
100
|
+
count = int(self.xb.shape[0]) if self.xb is not None else 0
|
|
101
|
+
dim = int(self.xb.shape[1]) if self.xb is not None else 0
|
|
102
|
+
metric_name = self.metric.name if self.metric is not None else "unknown"
|
|
103
|
+
return {
|
|
104
|
+
"backend": self.name,
|
|
105
|
+
"count": count,
|
|
106
|
+
"dim": dim,
|
|
107
|
+
"metric": metric_name,
|
|
108
|
+
"vector_bytes": int(self.xb.nbytes) if self.xb is not None else 0,
|
|
109
|
+
}
|
|
110
|
+
|
|
97
111
|
@classmethod
|
|
98
112
|
def load(cls, path: str) -> "BruteForceBackend":
|
|
99
113
|
with open(os.path.join(path, "backend_meta.json"), "r", encoding="utf-8") as f:
|
|
@@ -23,6 +23,8 @@ class FaissBackend:
|
|
|
23
23
|
"supports_delete": False,
|
|
24
24
|
"supports_custom_metric": False,
|
|
25
25
|
"supports_persistence": True,
|
|
26
|
+
"supports_add": True,
|
|
27
|
+
"supports_ann_tuning": True,
|
|
26
28
|
}
|
|
27
29
|
)
|
|
28
30
|
_index: object | None = None
|
|
@@ -110,6 +112,17 @@ class FaissBackend:
|
|
|
110
112
|
f,
|
|
111
113
|
)
|
|
112
114
|
|
|
115
|
+
def get_runtime_stats(self) -> dict[str, float | int | str]:
|
|
116
|
+
metric_name = self._metric.name if self._metric is not None else "unknown"
|
|
117
|
+
config = self._config or {}
|
|
118
|
+
return {
|
|
119
|
+
"backend": self.name,
|
|
120
|
+
"count": int(self._count),
|
|
121
|
+
"metric": metric_name,
|
|
122
|
+
"index_factory": str(config.get("index_factory", "Flat")),
|
|
123
|
+
"nprobe": int(config.get("nprobe", 0)) if config.get("nprobe") is not None else 0,
|
|
124
|
+
}
|
|
125
|
+
|
|
113
126
|
@classmethod
|
|
114
127
|
def load(cls, path: str) -> "FaissBackend":
|
|
115
128
|
backend = cls()
|
|
@@ -55,6 +55,9 @@ class VectorIndex:
|
|
|
55
55
|
_row_to_external: list[int | str] = field(default_factory=list)
|
|
56
56
|
_metadata: list[dict[str, Any]] | None = None
|
|
57
57
|
_dim: int = 0
|
|
58
|
+
_search_calls: int = 0
|
|
59
|
+
_last_search_k: int = 0
|
|
60
|
+
_last_search_nq: int = 0
|
|
58
61
|
|
|
59
62
|
@classmethod
|
|
60
63
|
def create(
|
|
@@ -121,6 +124,9 @@ class VectorIndex:
|
|
|
121
124
|
if k <= 0:
|
|
122
125
|
raise ValueError("index_error: k must be > 0")
|
|
123
126
|
scores, internal_ids = self._backend.search(queries.values, k)
|
|
127
|
+
self._search_calls += 1
|
|
128
|
+
self._last_search_k = int(k)
|
|
129
|
+
self._last_search_nq = int(queries.shape[0])
|
|
124
130
|
external_ids = _to_external_ids(internal_ids, self._row_to_external)
|
|
125
131
|
md_out = None
|
|
126
132
|
if return_metadata and self._metadata is not None:
|
|
@@ -136,6 +142,31 @@ class VectorIndex:
|
|
|
136
142
|
md_out.append(items)
|
|
137
143
|
return SearchResult(ids=external_ids, scores=scores, metadata=md_out)
|
|
138
144
|
|
|
145
|
+
def backend_capabilities(self) -> dict[str, bool]:
|
|
146
|
+
if self._backend is None:
|
|
147
|
+
raise RuntimeError("index_error: index is not initialized")
|
|
148
|
+
caps = getattr(self._backend, "capabilities", None)
|
|
149
|
+
if not isinstance(caps, dict):
|
|
150
|
+
return {}
|
|
151
|
+
return {str(k): bool(v) for k, v in caps.items()}
|
|
152
|
+
|
|
153
|
+
def runtime_stats(self) -> dict[str, Any]:
|
|
154
|
+
if self._backend is None:
|
|
155
|
+
raise RuntimeError("index_error: index is not initialized")
|
|
156
|
+
backend_stats: dict[str, Any] = {}
|
|
157
|
+
if hasattr(self._backend, "get_runtime_stats"):
|
|
158
|
+
backend_stats = dict(getattr(self._backend, "get_runtime_stats")())
|
|
159
|
+
return {
|
|
160
|
+
"backend": self.backend_name,
|
|
161
|
+
"dim": int(self._dim),
|
|
162
|
+
"count": int(len(self._row_to_external)),
|
|
163
|
+
"search_calls": int(self._search_calls),
|
|
164
|
+
"last_search_k": int(self._last_search_k),
|
|
165
|
+
"last_search_nq": int(self._last_search_nq),
|
|
166
|
+
"capabilities": self.backend_capabilities(),
|
|
167
|
+
"backend_stats": backend_stats,
|
|
168
|
+
}
|
|
169
|
+
|
|
139
170
|
def save(self, path: str) -> None:
|
|
140
171
|
if self._backend is None:
|
|
141
172
|
raise RuntimeError("index_error: index is not initialized")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vector-engine
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: ML-first vector computation and retrieval engine.
|
|
5
5
|
Author: Neel Panchal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,7 +16,7 @@ Provides-Extra: dev
|
|
|
16
16
|
Requires-Dist: pytest<9,>=8.2; extra == "dev"
|
|
17
17
|
Dynamic: license-file
|
|
18
18
|
|
|
19
|
-
# Vector Engine v1.
|
|
19
|
+
# Vector Engine v1.1.0
|
|
20
20
|
|
|
21
21
|
Reproducibility-first vector retrieval toolkit for local ML and IR workflows.
|
|
22
22
|
|
|
@@ -95,7 +95,12 @@ results = index.search(xq, k=5)
|
|
|
95
95
|
print(results.ids[0], results.scores[0])
|
|
96
96
|
```
|
|
97
97
|
|
|
98
|
-
##
|
|
98
|
+
## Colab Demos
|
|
99
|
+
|
|
100
|
+
- Demo A (basic): [Google Colab](https://colab.research.google.com/drive/1VUkf8mc1lBEQQ9iBWr_WitJJgbcbIPZ6?usp=sharing)
|
|
101
|
+
- Scaling expansion: [Google Colab](https://colab.research.google.com/drive/1HCd1_R56eBLuTzobYTeiCeAIwT-K7XZb?usp=sharing)
|
|
102
|
+
|
|
103
|
+
## v1.1.0 Surface
|
|
99
104
|
|
|
100
105
|
- Core: `VectorArray`, `VectorIndex`, `Metric`, `SearchResult`
|
|
101
106
|
- ML: `knn_classify`, `knn_regress`, `kmeans`, `KMeansResult`
|
|
@@ -192,8 +197,8 @@ python scripts/credibility_audit.py --matrix-summary artifacts/benchmark_matrix/
|
|
|
192
197
|
|
|
193
198
|
## Project Links
|
|
194
199
|
|
|
195
|
-
- `docs/releases/v1.
|
|
196
|
-
- `docs/releases/v1.
|
|
200
|
+
- `docs/releases/v1.1.0.md`
|
|
201
|
+
- `docs/releases/v1.1.0-checklist.md`
|
|
197
202
|
- `docs/reproducibility.md`
|
|
198
203
|
- `docs/use_cases.md`
|
|
199
204
|
- `docs/api_stability.md`
|
|
@@ -12,6 +12,7 @@ tests/test_faiss_optional.py
|
|
|
12
12
|
tests/test_hardening.py
|
|
13
13
|
tests/test_ingest_pipeline.py
|
|
14
14
|
tests/test_install_smoke.py
|
|
15
|
+
tests/test_matrix_profile_advisor.py
|
|
15
16
|
tests/test_ml_eval.py
|
|
16
17
|
tests/test_perf_smoke.py
|
|
17
18
|
tests/test_persistence_compat.py
|
|
@@ -19,6 +20,7 @@ tests/test_profile_local.py
|
|
|
19
20
|
tests/test_rag_reliability.py
|
|
20
21
|
tests/test_real_corpus_eval.py
|
|
21
22
|
tests/test_release_bundle.py
|
|
23
|
+
tests/test_release_performance_workflow.py
|
|
22
24
|
tests/test_v02_features.py
|
|
23
25
|
vector_engine/__init__.py
|
|
24
26
|
vector_engine/array.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|