isotrieve 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- isotrieve-0.2.1/.gitignore +68 -0
- isotrieve-0.2.1/CLAIMS.md +41 -0
- isotrieve-0.2.1/DECISIONS.md +285 -0
- isotrieve-0.2.1/LICENSE +17 -0
- isotrieve-0.2.1/PKG-INFO +283 -0
- isotrieve-0.2.1/README.md +200 -0
- isotrieve-0.2.1/integrations/langchain/README.md +54 -0
- isotrieve-0.2.1/pyproject.toml +164 -0
- isotrieve-0.2.1/src/isotrieve/__init__.py +22 -0
- isotrieve-0.2.1/src/isotrieve/adapters/__init__.py +6 -0
- isotrieve-0.2.1/src/isotrieve/adapters/base.py +127 -0
- isotrieve-0.2.1/src/isotrieve/adapters/chroma.py +281 -0
- isotrieve-0.2.1/src/isotrieve/adapters/langchain.py +117 -0
- isotrieve-0.2.1/src/isotrieve/adapters/llamaindex_store.py +208 -0
- isotrieve-0.2.1/src/isotrieve/adapters/pinecone.py +180 -0
- isotrieve-0.2.1/src/isotrieve/adapters/qdrant.py +173 -0
- isotrieve-0.2.1/src/isotrieve/calibration/__init__.py +17 -0
- isotrieve-0.2.1/src/isotrieve/calibration/calib_v1.py +150 -0
- isotrieve-0.2.1/src/isotrieve/calibration/corpus.py +81 -0
- isotrieve-0.2.1/src/isotrieve/calibration/planner.py +87 -0
- isotrieve-0.2.1/src/isotrieve/cli.py +323 -0
- isotrieve-0.2.1/src/isotrieve/cli_doctor.py +183 -0
- isotrieve-0.2.1/src/isotrieve/cli_gate.py +205 -0
- isotrieve-0.2.1/src/isotrieve/cli_report.py +52 -0
- isotrieve-0.2.1/src/isotrieve/mapping/__init__.py +29 -0
- isotrieve-0.2.1/src/isotrieve/mapping/base.py +331 -0
- isotrieve-0.2.1/src/isotrieve/mapping/linear.py +547 -0
- isotrieve-0.2.1/src/isotrieve/mapping/mlp.py +316 -0
- isotrieve-0.2.1/src/isotrieve/mapping/registry.py +165 -0
- isotrieve-0.2.1/src/isotrieve/migrate.py +185 -0
- isotrieve-0.2.1/src/isotrieve/providers/__init__.py +17 -0
- isotrieve-0.2.1/src/isotrieve/providers/base.py +33 -0
- isotrieve-0.2.1/src/isotrieve/providers/cached.py +121 -0
- isotrieve-0.2.1/src/isotrieve/providers/cohere.py +68 -0
- isotrieve-0.2.1/src/isotrieve/providers/factory.py +92 -0
- isotrieve-0.2.1/src/isotrieve/providers/gemini.py +64 -0
- isotrieve-0.2.1/src/isotrieve/providers/openai.py +84 -0
- isotrieve-0.2.1/src/isotrieve/providers/sentence_transformers.py +52 -0
- isotrieve-0.2.1/src/isotrieve/providers/voyage.py +61 -0
- isotrieve-0.2.1/src/isotrieve/py.typed +0 -0
- isotrieve-0.2.1/src/isotrieve/quality/__init__.py +19 -0
- isotrieve-0.2.1/src/isotrieve/quality/gate.py +283 -0
- isotrieve-0.2.1/src/isotrieve/quality/gate_model_v1.json +52 -0
- isotrieve-0.2.1/src/isotrieve/quality/metrics.py +198 -0
- isotrieve-0.2.1/src/isotrieve/quality/thresholds.json +11 -0
- isotrieve-0.2.1/src/isotrieve/quality/thresholds.yaml +16 -0
- isotrieve-0.2.1/src/isotrieve/recalibration.py +238 -0
- isotrieve-0.2.1/src/isotrieve/reporting/__init__.py +1 -0
- isotrieve-0.2.1/src/isotrieve/reporting/html_report.py +62 -0
- isotrieve-0.2.1/src/isotrieve/reranking.py +217 -0
- isotrieve-0.2.1/src/isotrieve/serve.py +208 -0
- isotrieve-0.2.1/src/isotrieve/stores/__init__.py +6 -0
- isotrieve-0.2.1/src/isotrieve/stores/base.py +41 -0
- isotrieve-0.2.1/src/isotrieve/stores/numpy_files.py +168 -0
- isotrieve-0.2.1/src/isotrieve/stores/qdrant_store.py +181 -0
- isotrieve-0.2.1/src/isotrieve/wrappers/__init__.py +31 -0
- isotrieve-0.2.1/src/isotrieve/wrappers/llamaindex.py +161 -0
- isotrieve-0.2.1/src/isotrieve/wrappers/openai_shim.py +118 -0
- isotrieve-0.2.1/src/isotrieve/wrappers/telemetry.py +79 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
*.egg-info/
|
|
7
|
+
*.egg
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
.eggs/
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
env/
|
|
16
|
+
|
|
17
|
+
# Testing
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.coverage
|
|
20
|
+
htmlcov/
|
|
21
|
+
|
|
22
|
+
# Type checking
|
|
23
|
+
.mypy_cache/
|
|
24
|
+
|
|
25
|
+
# IDE
|
|
26
|
+
.vscode/
|
|
27
|
+
.idea/
|
|
28
|
+
.cursor/
|
|
29
|
+
*.swp
|
|
30
|
+
*.swo
|
|
31
|
+
|
|
32
|
+
# macOS
|
|
33
|
+
.DS_Store
|
|
34
|
+
|
|
35
|
+
# Environment
|
|
36
|
+
.env
|
|
37
|
+
|
|
38
|
+
# Jupyter
|
|
39
|
+
.ipynb_checkpoints/
|
|
40
|
+
|
|
41
|
+
# Node
|
|
42
|
+
node_modules/
|
|
43
|
+
.npm/
|
|
44
|
+
|
|
45
|
+
# Model/artifact files (too large for git)
|
|
46
|
+
*.pkl
|
|
47
|
+
*.npy
|
|
48
|
+
*.pt
|
|
49
|
+
*.pth
|
|
50
|
+
*.onnx
|
|
51
|
+
*.bin
|
|
52
|
+
*.safetensors
|
|
53
|
+
|
|
54
|
+
# Embedding cache
|
|
55
|
+
benchmarks/.embed_cache/
|
|
56
|
+
|
|
57
|
+
# Benchmark results (keep committed, ignore regenerated)
|
|
58
|
+
# benchmarks/results/ is committed via explicit add
|
|
59
|
+
|
|
60
|
+
# Build artifacts
|
|
61
|
+
*.log
|
|
62
|
+
*.tsbuildinfo
|
|
63
|
+
*.map
|
|
64
|
+
|
|
65
|
+
# Scratch files at repo root
|
|
66
|
+
*.ipynb_checkpoints/
|
|
67
|
+
aecp-python/aecp_training_loss.png
|
|
68
|
+
cleanup/
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# CLAIMS.md — Public quantitative claims → benchmark artifacts
|
|
2
|
+
|
|
3
|
+
Every number that appears in README, docs, or marketing must have a row here
|
|
4
|
+
pointing at a committed file under `benchmarks/results/`.
|
|
5
|
+
|
|
6
|
+
## Active claims
|
|
7
|
+
|
|
8
|
+
| Claim text (exact or paraphrase) | Where used | Artifact path | Verified | Notes |
|
|
9
|
+
|----------------------------------|------------|---------------|----------|-------|
|
|
10
|
+
| SciFact MiniLM→bge-large nDCG@10 retention ≈ 0.871 ± 0.006 (K=4000, Ridge adapter, 3 seeds) | `isotrieve-python/README.md` | `benchmarks/results/beir_scifact_*__ridge__k4000__seed{0,1,2}__*.json` | 2026-07-21 | Floor=0.0, ceiling=0.735, Isotrieve=0.634–0.646. Actual: 0.872, 0.863, 0.879. |
|
|
11
|
+
| LowRank adapter nDCG@10 retention ≈ 0.857 ± 0.009 (K=4000, 3 seeds) | same | `benchmarks/results/beir_scifact_*__lowrank__k4000__seed{0,1,2}__*.json` | 2026-07-21 | Phase 2 adapter sweep. Actual: 0.867, 0.846, 0.858. |
|
|
12
|
+
| MLP adapter nDCG@10 retention ≈ 0.727 ± 0.007 (K=4000, 3 seeds) | same | `benchmarks/results/beir_scifact_*__mlp__k4000__seed{0,1,2}__*.json` | 2026-07-21 | No hyperparameter tuning. Actual: 0.736, 0.724, 0.721. |
|
|
13
|
+
| K-sweep (ridge only): K=500 0.704±0.004, K=1000 0.781±0.008, K=2000 0.818±0.007, K=4000 0.871±0.006 | same | `benchmarks/results/beir_scifact_*__ridge__k{500,1000,2000,4000}__*.json` | 2026-07-21 | Ridge-only K-sweep. Monotonic improvement with K. |
|
|
14
|
+
| K-sweep (all adapters avg): K=500 0.671±0.041, K=1000 0.735±0.058, K=2000 0.785±0.052, K=4000 0.832±0.061 | same | `benchmarks/results/beir_scifact_*__k{500,1000,2000,4000}__*.json` | 2026-07-21 | All adapters averaged across 3 seeds. |
|
|
15
|
+
| Floor nDCG@10 = 0.0 when dims differ (384≠1024) | same | same | 2026-07-21 | Raw cross-space vectors cannot be queried. |
|
|
16
|
+
| Ceiling (full re-embed) nDCG@10 ≈ 0.735 | same | same | 2026-07-21 | Quality upper bound for this model pair. |
|
|
17
|
+
| Same-dim pair (bge-large→e5-large, 1024→1024): floor=0.0, Isotrieve≈0.667, ceiling=0.722, retention=0.923±0.010 | `isotrieve-python/README.md` | `benchmarks/results/beir_scifact_*__BAAI_bge-large-en-v1.5__to__intfloat_e5-large-v2__ridge__k2000__seed{0,1,2}__*.json` | 2026-07-21 | Actual: 0.908, 0.930, 0.931. With e5 prefixes. Without prefixes, ceiling=0.355 and retention=0.95 (broken). |
|
|
18
|
+
| Gate model trained on local pairs only (no API model pairs) | `isotrieve-python/src/isotrieve/quality/gate_model_v1.json` | `benchmarks/results/gate_lopo.json` | — | Gate model valid for local model pairs. API pair performance may differ. |
|
|
19
|
+
| WS-A: bge→e5 raw scores agree at 100% for τ≤0.75; recalibration helps at τ=0.80 (+4.7% agreement, +2.36pt recall) | `README.md` | `benchmarks/results/ws_a_bge_to_e5_recall_tables.json` | — | MAE=0.095, margin compression=0.83x. |
|
|
20
|
+
| WS-A: MiniLM→bge rectangular pair — raw scores severely compressed (mean 0.157 vs ceiling 0.521, MAE=0.364). Recalibration essential. | `README.md` | `benchmarks/results/ws_a_minilm_to_bge_recall_tables.json` | — | τ=0.60 goes 78%→100% (+22%), τ=0.70 goes 27%→67% (+40%). |
|
|
21
|
+
| WS-B: Confidence flags (adaptive P33/P67 margins) are predictive across both pairs. bge→e5 high=0.955/low=0.637; MiniLM→bge high=0.875/low=0.651 | `README.md` | `benchmarks/results/ws_b_confidence_flags.json` | — | 50-56ms/query latency. |
|
|
22
|
+
| WS-B: Cross-encoder reranking NULL RESULT (-10.7pts bge→e5, -9.8pts MiniLM→bge — MS MARCO domain-mismatched for sci-text) | `DECISIONS.md` | `benchmarks/results/ws_b_cross_encoder.json` | — | Not shipped. |
|
|
23
|
+
| WS-C: Independent inverse-α: +2.17pts (bge→e5), +2.23pts (MiniLM→bge). Consistent across pairs. | `README.md` | `benchmarks/results/ws_c_independent_inv_alpha.json` | — | Optimal inv alpha differs from forward (0.178 vs 0.316 on bge→e5). |
|
|
24
|
+
| WS-C: TSVD shrinkage NULL RESULT (rank=512 only -0.33pt) | `DECISIONS.md` | `benchmarks/results/ws_c_tsvd_shrinkage.json` | — | Not worth complexity. |
|
|
25
|
+
| WS-E: Rectangular pair re-validation — MiniLM→bge: 86% retention, margin compression 0.85x, independent inverse-α +2.23pts | `README.md` | `benchmarks/results/ws_e_minilm_to_bge_revalidation.json` | — | |
|
|
26
|
+
| WS-D: Gate v3 — margin compression <0.85 widens prediction interval | `DECISIONS.md` | `src/isotrieve/quality/gate_model_v1.json` | — | `_predict_retention()` accepts `margin_compression` parameter. |
|
|
27
|
+
|
|
28
|
+
## Retired / deleted claims (must not reappear)
|
|
29
|
+
|
|
30
|
+
These appeared in prior docs without reproducible artifacts and were removed:
|
|
31
|
+
|
|
32
|
+
- "97% semantic fidelity" / "97.2%" / "97.35%"
|
|
33
|
+
- "<10ms" / "<1ms" transfer latency as a product claim
|
|
34
|
+
- "85% Top-1" abort threshold as a validated number
|
|
35
|
+
- "86% corpus fidelity" / "43% text baseline"
|
|
36
|
+
- "150x faster" / "200x"
|
|
37
|
+
- "Validated on 300k vocab, zero overfitting"
|
|
38
|
+
- Ridge ">90%" / future MLP ">99%" as stated facts
|
|
39
|
+
|
|
40
|
+
Replacement rule: only cite numbers produced by `benchmarks/run_benchmark.py`
|
|
41
|
+
and stored under `benchmarks/results/`.
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
# Isotrieve Design Decisions
|
|
2
|
+
|
|
3
|
+
Running log of design choices and open questions.
|
|
4
|
+
Bias toward the boring, verifiable choice.
|
|
5
|
+
|
|
6
|
+
## Postmortems
|
|
7
|
+
|
|
8
|
+
### 2026-07-19 — Number Discrepancy Postmortem (WS-0)
|
|
9
|
+
|
|
10
|
+
**Symptom:** Adapter sweep reported Ridge/K=4000/SciFact = 0.866 ± 0.008. K-sweep reported the same nominal config = 0.814 ± 0.068. The ±0.068 (8× larger std) flagged a discrepancy.
|
|
11
|
+
|
|
12
|
+
**Root cause:** The K-sweep summary (`run_benchmark.py` line 538) filtered by `k_cal` but **not by adapter**. When running `--adapter ridge lowrank mlp --k 500 1000 2000 4000`, the K-sweep summary averaged across ALL adapters for each K value:
|
|
13
|
+
- ridge K=4000: 0.866 ± 0.008
|
|
14
|
+
- lowrank K=4000: 0.857 ± 0.009
|
|
15
|
+
- mlp K=4000: 0.719 ± 0.008
|
|
16
|
+
- **All-adapter average at K=4000: 0.814 ± 0.068** ← this was the K-sweep number
|
|
17
|
+
|
|
18
|
+
The adapter sweep was adapter-specific (ridge only). Two valid but different experiments, presented in a way that implied they should match.
|
|
19
|
+
|
|
20
|
+
**Resolution:**
|
|
21
|
+
1. K-sweep summary now broken down by adapter (`run_benchmark.py` updated).
|
|
22
|
+
2. `protocol` field added to result JSON (`fixed_calibration_per_seed`).
|
|
23
|
+
3. Audit script `benchmarks/audit_configs.py` committed for field-by-field comparison.
|
|
24
|
+
4. CLAIMS.md and DECISIONS.md updated to distinguish adapter-specific vs cross-adapter numbers.
|
|
25
|
+
|
|
26
|
+
**Lesson:** Always declare which dimension (adapter, K, seed) a summary averages over. The `protocol` field in result JSONs makes this explicit.
|
|
27
|
+
|
|
28
|
+
## Locked decisions
|
|
29
|
+
|
|
30
|
+
### 2026-07-19 — Package strategy (Phase 1)
|
|
31
|
+
|
|
32
|
+
**Choice:** Rewrite `isotrieve-python` onto a `src/isotrieve/` layout as the migration toolkit.
|
|
33
|
+
The old agent-protocol API under the flat `isotrieve/` package is excluded from the
|
|
34
|
+
build and deferred to a later "advanced usage" docs page. TypeScript (`@isotrieve/core`)
|
|
35
|
+
is out of scope for v1.
|
|
36
|
+
|
|
37
|
+
**Why:** The product contract repositions Isotrieve as embedding-space migration, not
|
|
38
|
+
agent handoff. One excellent Python package beats two mediocre packages. The PyPI
|
|
39
|
+
name `isotrieve` must not collide with a second package.
|
|
40
|
+
|
|
41
|
+
### 2026-07-19 — Version reset
|
|
42
|
+
|
|
43
|
+
**Choice:** Ship as `0.1.0` (alpha), not continue from marketed `1.0.0`.
|
|
44
|
+
|
|
45
|
+
**Why:** Public claims from the old package are unverified. Starting at 0.1.0
|
|
46
|
+
signals that numbers are earned via the benchmark harness.
|
|
47
|
+
|
|
48
|
+
### 2026-07-19 — License
|
|
49
|
+
|
|
50
|
+
**Choice:** Apache-2.0 (per product contract). Previous MIT license in this
|
|
51
|
+
directory is superseded for the new package.
|
|
52
|
+
|
|
53
|
+
### 2026-07-19 — Mapping defaults
|
|
54
|
+
|
|
55
|
+
**Choice:** `RidgeMapping` is primary; `OrthogonalProcrustesMapping` only when
|
|
56
|
+
`d_src == d_tgt`. All `transform` outputs are L2-normalized. Ridge uses an
|
|
57
|
+
augmented bias column and `alpha="auto"` via leave-one-out GCV over a log grid
|
|
58
|
+
(`sklearn.linear_model.RidgeCV`).
|
|
59
|
+
|
|
60
|
+
**Why:** Rectangular maps (1536→3072) are the normal case; Procrustes cannot
|
|
61
|
+
handle them. Normalization matches retrieval practice and makes cosine metrics
|
|
62
|
+
stable.
|
|
63
|
+
|
|
64
|
+
### 2026-07-19 — No wall-clock expiry on `.isotrieve` files
|
|
65
|
+
|
|
66
|
+
**Choice:** Header includes `expires_hint` for operator convenience but the
|
|
67
|
+
library never auto-expires. Drift detection is `isotrieve verify` (fresh probe embeds).
|
|
68
|
+
|
|
69
|
+
**Why:** Model weights do not drift on a calendar; silent provider updates do.
|
|
70
|
+
|
|
71
|
+
### 2026-07-19 — Phase 1 scope
|
|
72
|
+
|
|
73
|
+
**Choice:** Implement core math, quality metrics, numpy store, minimal file CLI,
|
|
74
|
+
offline tests, and a local-model SciFact harness before any API providers or
|
|
75
|
+
vector-DB adapters.
|
|
76
|
+
|
|
77
|
+
### 2026-07-19 — Phase 1 gate result (first honest number)
|
|
78
|
+
|
|
79
|
+
**Measured** on SciFact test (300 queries with qrels), in-domain K=4000,
|
|
80
|
+
`all-MiniLM-L6-v2` → `BAAI/bge-large-en-v1.5`, 3 seeds:
|
|
81
|
+
|
|
82
|
+
| | nDCG@10 |
|
|
83
|
+
|--|---------|
|
|
84
|
+
| Floor (raw cross-space; dims 384≠1024) | 0.000 |
|
|
85
|
+
| Isotrieve (mapped) | ≈ 0.640 |
|
|
86
|
+
| Ceiling (full re-embed) | ≈ 0.735 |
|
|
87
|
+
| **Retention (Isotrieve÷ceiling)** | **0.871 ± 0.006** |
|
|
88
|
+
|
|
89
|
+
Artifacts under `benchmarks/results/`. Rank warning: MiniLM source matrix was
|
|
90
|
+
slightly rank-deficient (382/384) at K=4000 — logged, not hidden.
|
|
91
|
+
|
|
92
|
+
**Gate status:** PASSED — Isotrieve retention meaningfully above floor.
|
|
93
|
+
|
|
94
|
+
### 2026-07-19 — Phase 1 prerequisite fixes (before Phase 2)
|
|
95
|
+
|
|
96
|
+
Inferred from Phase 1 debt + contaminated artifacts (PM referenced "four fixes
|
|
97
|
+
above" without the prior message in-thread). Logged here for audit:
|
|
98
|
+
|
|
99
|
+
| Fix | Action |
|
|
100
|
+
|-----|--------|
|
|
101
|
+
| F1 | Purge `self_retrieval_fallback` result JSONs; harness **requires** real BEIR qrels for claimable runs — no silent fallback |
|
|
102
|
+
| F2 | Record **holdout-vs-corpus optimism gap** (fit holdout top-1 − retrieval retention) in every result + QualityGate |
|
|
103
|
+
| F3 | **CachedEmbedder mandatory** on all provider/CLI/benchmark embed paths — never embed the same text twice |
|
|
104
|
+
| F4 | Gate/corpus sample must be **disjoint from calibration**; rank-deficiency stays a warning (ridge handles it) |
|
|
105
|
+
|
|
106
|
+
### 2026-07-19 — Phase 2 adapter zoo
|
|
107
|
+
|
|
108
|
+
**Choice:** Add 4 adapters to the existing Ridge: Procrustes (square only), Procrustes+Diagonal, LowRankAffine, ResidualMLP. Ridge remains the default for rectangular mappings.
|
|
109
|
+
|
|
110
|
+
**Why:** Drift-Adapter (EMNLP 2025) validates Procrustes and MLP approaches. LowRankAffine provides compressed matrices. Each adapter has different compute/memory/quality tradeoffs.
|
|
111
|
+
|
|
112
|
+
**Measured (SciFact, all-MiniLM-L6-v2 → bge-large-en-v1.5, K=4000, 3 seeds):**
|
|
113
|
+
|
|
114
|
+
| Adapter | nDCG@10 retention | Notes |
|
|
115
|
+
|---------|------------------|-------|
|
|
116
|
+
| Ridge | 0.866 ± 0.008 | Default. Fast, stable. |
|
|
117
|
+
| LowRank | 0.857 ± 0.009 | Compressed matrix. ~1% worse than Ridge. |
|
|
118
|
+
| MLP | 0.719 ± 0.008 | No hyperparameter tuning. Significantly worse than linear. |
|
|
119
|
+
|
|
120
|
+
### 2026-07-19 — Phase 2 K-sweep (data-driven gate thresholds)
|
|
121
|
+
|
|
122
|
+
**Choice:** Benchmark across K=500,1000,2000,4000 to derive QualityGate thresholds from data.
|
|
123
|
+
|
|
124
|
+
**Measured (SciFact, all adapters averaged, 3 seeds):**
|
|
125
|
+
|
|
126
|
+
| K | nDCG@10 retention | Notes |
|
|
127
|
+
|---|------------------|-------|
|
|
128
|
+
| 500 | 0.667 ± 0.039 | Below recommended minimum. Rank-deficient. |
|
|
129
|
+
| 1000 | 0.732 ± 0.056 | Still below minimum. Better but high variance. |
|
|
130
|
+
| 2000 | 0.788 ± 0.054 | Approaching acceptable quality. |
|
|
131
|
+
| 4000 | 0.814 ± 0.068 | Full quality. Recommended minimum. |
|
|
132
|
+
|
|
133
|
+
**Gate thresholds derived:**
|
|
134
|
+
- PASS: retention ≥ 0.75 (K≥2000 typically achieves this)
|
|
135
|
+
- WARN: 0.55 ≤ retention < 0.75 (K=500-1000 range, or poor mapping)
|
|
136
|
+
- FAIL: retention < 0.55 (mapping fundamentally broken)
|
|
137
|
+
|
|
138
|
+
### 2026-07-19 — K minimum constraint downgraded to warning
|
|
139
|
+
|
|
140
|
+
**Choice:** Change adapter `fit()` from raising `ValueError` on small K to issuing `UserWarning`. The 10×min_dim rule is a guideline, not a hard error.
|
|
141
|
+
|
|
142
|
+
**Why:** Users need to see what happens with small K. The benchmark harness needs to sweep across K values for threshold calibration. Warning gives visibility without blocking experimentation.
|
|
143
|
+
|
|
144
|
+
### 2026-07-19 — MLP rectangular dims
|
|
145
|
+
|
|
146
|
+
**Choice:** ResidualMLP uses plain MLP (no residual skip) when d_src ≠ d_tgt. Residual skip only when dimensions match.
|
|
147
|
+
|
|
148
|
+
**Why:** The residual connection `x + net(x)` requires matching input/output dims. Rectangular projections need a feedforward architecture.
|
|
149
|
+
|
|
150
|
+
### 2026-07-19 — QualityGate v2: proxy-based prediction (not raw holdout)
|
|
151
|
+
|
|
152
|
+
**Choice:** Redefine gate thresholds from "requires ceiling measurement" to "predicts retention from holdout proxy using isotonic regression." Gate model ships with the package (`gate_model_v1.json`). Gate verdict uses prediction intervals.
|
|
153
|
+
|
|
154
|
+
**Why:** The old gate required ceiling quality (full re-embed) to measure retention — defeating the purpose of a migration tool. The new gate predicts retention from holdout proxies that are always available: `top1_retention`, `holdout_rank_corr`, `cosine_mean`. Trained on 48 benchmark results (4 adapter pairs), LOPO CV: MAE=0.0443, RMSE=0.0592, 80% interval half-width=0.0706.
|
|
155
|
+
|
|
156
|
+
**Artifacts:** `gate_model_v1.json`, `benchmarks/fit_gate_model.py`, `benchmarks/results/gate_lopo.json`
|
|
157
|
+
|
|
158
|
+
### 2026-07-19 — Same-dimension floor is 0.0
|
|
159
|
+
|
|
160
|
+
**Choice:** Measure bge-large→e5-large (1024→1024) raw cross-space retrieval. Floor = 0.0.
|
|
161
|
+
|
|
162
|
+
**Why:** "Same dimension" is not "same space." Different models produce geometrically incompatible embeddings even at the same dimension. Raw cross-space nDCG@10 = 0.0 — search runs fine, results are garbage. This is the core problem Isotrieve solves. Retention = 0.908 with mapping (with e5 prefixes).
|
|
163
|
+
|
|
164
|
+
### 2026-07-19 — e5 prefix bug (postmortem)
|
|
165
|
+
|
|
166
|
+
**Symptom:** Same-dim pair (bge-large→e5-large) showed ceiling nDCG@10 = 0.355, retention = 95%. e5-large alone scores ~0.74 on SciFact — ceiling was clearly broken.
|
|
167
|
+
|
|
168
|
+
**Root cause:** e5 models require `"query: "` / `"passage: "` text prefixes for correct embeddings. The benchmark harness passed raw texts with no prefix handling. Both documents and queries were embedded without prefixes, degrading e5's retrieval quality by ~50%.
|
|
169
|
+
|
|
170
|
+
**Resolution:** Added `MODEL_PREFIXES` dict to `run_benchmark.py` with model-specific prefix mappings. Modified `embed_texts()` and `load_or_embed()` to apply correct prefixes. Cache invalidated via `prefix_version` field.
|
|
171
|
+
|
|
172
|
+
**Corrected results:**
|
|
173
|
+
- Before fix: ceiling=0.355, Isotrieve=0.337, retention=0.950 (broken)
|
|
174
|
+
- After fix: ceiling=0.722, Isotrieve=0.656, retention=0.908 (honest)
|
|
175
|
+
|
|
176
|
+
**Lesson:** Model-specific preprocessing (prefixes, tokenization quirks) must be part of the embedding pipeline, not assumed to be "just encode text." When a number looks too good (95% retention), verify the ceiling is realistic.
|
|
177
|
+
|
|
178
|
+
### 2026-07-19 — Bidirectional evaluation
|
|
179
|
+
|
|
180
|
+
**Choice:** Evaluate both directions: source→target (transform corpus) AND target→source (map queries). Report "query→legacy retention" = 86.2% of ceiling.
|
|
181
|
+
|
|
182
|
+
**Why:** Serve-mode quality is measured by mapping queries into legacy space (query→legacy), not by transforming corpus (legacy→target). Both directions matter for the "before/after" narrative.
|
|
183
|
+
|
|
184
|
+
### 2026-07-20 — WS-A: Score recalibration (isotonic regression)
|
|
185
|
+
|
|
186
|
+
**Choice:** Fit isotonic regression (sklearn `IsotonicRegression`) on held-out query scores to map cross-space cosine scores to ceiling-equivalent scores. Recalibrator saved alongside mapping in `.isotrieve` format under `score_recal_v1` section.
|
|
187
|
+
|
|
188
|
+
**Why:** Raw cross-space scores are compressed (mean 0.157 vs ceiling 0.521 for MiniLM→bge rectangular pair). This means τ-thresholding based on ceiling scores fails on migrated data. Recalibration makes τ-thresholding reliable.
|
|
189
|
+
|
|
190
|
+
**Measured:**
|
|
191
|
+
- bge→e5 (same-dim): Raw scores agree at 100% for τ≤0.75. Recalibration helps only at τ=0.80 (+4.7% agreement, +2.36pt recall). MAE=0.095, margin compression=0.83x.
|
|
192
|
+
- MiniLM→bge (rectangular): Raw scores severely compressed (MAE=0.364). Recalibration essential: τ=0.60 goes 78%→100% (+22%), τ=0.70 goes 27%→67% (+40%). Margin compression=0.85x.
|
|
193
|
+
|
|
194
|
+
**Artifacts:** `benchmarks/results/ws_a_*_recall_tables.json`
|
|
195
|
+
|
|
196
|
+
### 2026-07-20 — WS-B: Cross-encoder reranking NULL RESULT
|
|
197
|
+
|
|
198
|
+
**Choice:** Do NOT ship cross-encoder reranking. MS MARCO cross-encoder (`cross-encoder/ms-marco-MiniLM-L-6-v2`) is domain-mismatched for scientific text.
|
|
199
|
+
|
|
200
|
+
**Measured:** -10.7pts on bge→e5, -9.8pts on MiniLM→bge. Consistent degradation.
|
|
201
|
+
|
|
202
|
+
**Why not:** Would need domain-matched cross-encoder (e.g., SciFact-trained). Not worth the complexity until proven on in-domain data.
|
|
203
|
+
|
|
204
|
+
**Artifacts:** `benchmarks/results/ws_b_cross_encoder.json`
|
|
205
|
+
|
|
206
|
+
### 2026-07-20 — WS-B: Confidence flags (adaptive margins)
|
|
207
|
+
|
|
208
|
+
**Choice:** Confidence flags using adaptive P33/P67 margin thresholds from calibration data. Predictive across both pairs.
|
|
209
|
+
|
|
210
|
+
**Measured:**
|
|
211
|
+
- bge→e5: high-conf R@10=0.955, low-conf R@10=0.637 (gap=0.318)
|
|
212
|
+
- MiniLM→bge: high-conf R@10=0.875, low-conf R@10=0.651 (gap=0.224)
|
|
213
|
+
|
|
214
|
+
**Why:** Users need per-query confidence to know when to trust migrated results. Adaptive thresholds (percentile-based) work across different score distributions without manual tuning.
|
|
215
|
+
|
|
216
|
+
**Artifacts:** `benchmarks/results/ws_b_confidence_flags.json`
|
|
217
|
+
|
|
218
|
+
### 2026-07-20 — WS-C: Independent inverse-α
|
|
219
|
+
|
|
220
|
+
**Choice:** RidgeMapping now has separate `_chosen_inv_alpha` for inverse direction, fitted independently from forward α. Stored in `.isotrieve` header.
|
|
221
|
+
|
|
222
|
+
**Why:** Optimal regularization differs between forward (Y = XW) and inverse (X = YW_inv) directions. Fitting α independently for each direction improves inverse quality without affecting forward.
|
|
223
|
+
|
|
224
|
+
**Measured:** +2.17pts (bge→e5), +2.23pts (MiniLM→bge). Consistent across pairs. Optimal inv alpha differs from forward (0.178 vs 0.316 on bge→e5).
|
|
225
|
+
|
|
226
|
+
**Artifacts:** `benchmarks/results/ws_c_independent_inv_alpha.json`
|
|
227
|
+
|
|
228
|
+
### 2026-07-20 — WS-C: TSVD shrinkage NULL RESULT
|
|
229
|
+
|
|
230
|
+
**Choice:** RidgeMapping now accepts `rank=r` parameter for TSVD shrinkage, but NOT recommended. Default is `rank=None` (no shrinkage).
|
|
231
|
+
|
|
232
|
+
**Measured:** rank=512 only -0.33pt, rank=256 -1.69pt, rank=128 -2.33pt. Quality degrades monotonically with rank reduction.
|
|
233
|
+
|
|
234
|
+
**Why keep it:** Useful for storage-constrained deployments (large mappings). But for most users, the quality loss isn't worth it.
|
|
235
|
+
|
|
236
|
+
**Artifacts:** `benchmarks/results/ws_c_tsvd_shrinkage.json`
|
|
237
|
+
|
|
238
|
+
### 2026-07-20 — WS-C: Procrustes centering REVERTED
|
|
239
|
+
|
|
240
|
+
**Choice:** Do NOT auto-center embeddings before Procrustes mapping. Production `OrthogonalProcrustesMapping` uses `V @ R^T` for inverse (no centering).
|
|
241
|
+
|
|
242
|
+
**Why reverted:** Q3 measured held-out cosine alignment (X_ho mapped to Y space vs Y_ho), NOT serve-mode retrieval. Centering breaks serve-mode because centered queries don't match uncentered source doc distribution. The -55pt swing was a measurement artifact, not a production bug.
|
|
243
|
+
|
|
244
|
+
**Lesson:** Always validate against actual serve-mode retrieval (R@10), not just alignment metrics.
|
|
245
|
+
|
|
246
|
+
### 2026-07-20 — WS-D: Gate v3 (margin compression)
|
|
247
|
+
|
|
248
|
+
**Choice:** `_predict_retention()` now accepts `margin_compression` parameter. When margin_compression < 0.85, widen prediction interval.
|
|
249
|
+
|
|
250
|
+
**Why:** Score compression indicates the mapping quality is uncertain. Wider intervals make the gate more conservative, reducing false PASS verdicts.
|
|
251
|
+
|
|
252
|
+
### 2026-07-20 — Vector DB adapters (v0.2)
|
|
253
|
+
|
|
254
|
+
**Choice:** Add ChromaDB and LangChain adapters. ChromaDB gets serve-mode (`IsotrieveChromaFunction`) and offline migration (`migrate_collection`). LangChain gets `IsotrieveEmbeddings` shim.
|
|
255
|
+
|
|
256
|
+
**Why:** Most users interact with vector databases through ORMs/clients. Thin adapters that apply Isotrieve mappings at the DB layer eliminate the need for users to manage mapping files manually.
|
|
257
|
+
|
|
258
|
+
**Adapters implemented:**
|
|
259
|
+
- `isotrieve.adapters.chroma.IsotrieveChromaFunction` — Chroma `EmbeddingFunction`
|
|
260
|
+
- `isotrieve.adapters.chroma.migrate_collection()` — offline migration
|
|
261
|
+
- `isotrieve.adapters.langchain.IsotrieveEmbeddings` — LangChain `Embeddings` shim
|
|
262
|
+
- `isotrieve.adapters.base.EmbeddingAdapter` — core ABC
|
|
263
|
+
- `isotrieve.adapters.base.VectorStoreAdapter` — core ABC
|
|
264
|
+
- `isotrieve.adapters.base.MigrationReport` — migration metadata
|
|
265
|
+
|
|
266
|
+
**Tests:** 21 tests for adapters (13 ChromaDB, 8 LangChain), all passing.
|
|
267
|
+
|
|
268
|
+
## Open questions
|
|
269
|
+
|
|
270
|
+
None for Phase 1 gates. Questions that affect later phases:
|
|
271
|
+
|
|
272
|
+
| ID | Question | Affects | Status |
|
|
273
|
+
|----|----------|---------|--------|
|
|
274
|
+
| Q1 | Exact QualityGate PASS/WARN/FAIL thresholds | Phase 2 | **Resolved** — data-driven from K-sweep. See thresholds.json. |
|
|
275
|
+
| Q2 | Frozen calibration corpus composition (`isotrieve-calib-v1`) | Phase 2 | Open — need permissive sources + checksum |
|
|
276
|
+
| Q3 | Fate of legacy `isotrieve/` protocol modules and `isotrieve-npm` | Phase 4 | Open — archive vs delete |
|
|
277
|
+
| Q4 | PyPI publish account / GitHub org URLs | Phase 4 | Open — placeholders remain |
|
|
278
|
+
| Q5 | Procrustes adapters for square pairs | Phase 2 | Open — only works when d_src == d_tgt |
|
|
279
|
+
| Q6 | MLP hyperparameter tuning (epochs, lr, hidden_dim) | Phase 3 | Open — current defaults unoptimized |
|
|
280
|
+
|
|
281
|
+
## PM sign-off
|
|
282
|
+
|
|
283
|
+
- [ ] Results table reviewed
|
|
284
|
+
- [ ] "When NOT to use Isotrieve" section reviewed
|
|
285
|
+
- [ ] Sign-off date: _
|
isotrieve-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Apache License
|
|
2
|
+
# Version 2.0, January 2004
|
|
3
|
+
# http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
Copyright 2026 AECP Contributors
|
|
6
|
+
|
|
7
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
you may not use this file except in compliance with the License.
|
|
9
|
+
You may obtain a copy of the License at
|
|
10
|
+
|
|
11
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
|
|
13
|
+
Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
See the License for the specific language governing permissions and
|
|
17
|
+
limitations under the License.
|