isotrieve 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. isotrieve-0.2.1/.gitignore +68 -0
  2. isotrieve-0.2.1/CLAIMS.md +41 -0
  3. isotrieve-0.2.1/DECISIONS.md +285 -0
  4. isotrieve-0.2.1/LICENSE +17 -0
  5. isotrieve-0.2.1/PKG-INFO +283 -0
  6. isotrieve-0.2.1/README.md +200 -0
  7. isotrieve-0.2.1/integrations/langchain/README.md +54 -0
  8. isotrieve-0.2.1/pyproject.toml +164 -0
  9. isotrieve-0.2.1/src/isotrieve/__init__.py +22 -0
  10. isotrieve-0.2.1/src/isotrieve/adapters/__init__.py +6 -0
  11. isotrieve-0.2.1/src/isotrieve/adapters/base.py +127 -0
  12. isotrieve-0.2.1/src/isotrieve/adapters/chroma.py +281 -0
  13. isotrieve-0.2.1/src/isotrieve/adapters/langchain.py +117 -0
  14. isotrieve-0.2.1/src/isotrieve/adapters/llamaindex_store.py +208 -0
  15. isotrieve-0.2.1/src/isotrieve/adapters/pinecone.py +180 -0
  16. isotrieve-0.2.1/src/isotrieve/adapters/qdrant.py +173 -0
  17. isotrieve-0.2.1/src/isotrieve/calibration/__init__.py +17 -0
  18. isotrieve-0.2.1/src/isotrieve/calibration/calib_v1.py +150 -0
  19. isotrieve-0.2.1/src/isotrieve/calibration/corpus.py +81 -0
  20. isotrieve-0.2.1/src/isotrieve/calibration/planner.py +87 -0
  21. isotrieve-0.2.1/src/isotrieve/cli.py +323 -0
  22. isotrieve-0.2.1/src/isotrieve/cli_doctor.py +183 -0
  23. isotrieve-0.2.1/src/isotrieve/cli_gate.py +205 -0
  24. isotrieve-0.2.1/src/isotrieve/cli_report.py +52 -0
  25. isotrieve-0.2.1/src/isotrieve/mapping/__init__.py +29 -0
  26. isotrieve-0.2.1/src/isotrieve/mapping/base.py +331 -0
  27. isotrieve-0.2.1/src/isotrieve/mapping/linear.py +547 -0
  28. isotrieve-0.2.1/src/isotrieve/mapping/mlp.py +316 -0
  29. isotrieve-0.2.1/src/isotrieve/mapping/registry.py +165 -0
  30. isotrieve-0.2.1/src/isotrieve/migrate.py +185 -0
  31. isotrieve-0.2.1/src/isotrieve/providers/__init__.py +17 -0
  32. isotrieve-0.2.1/src/isotrieve/providers/base.py +33 -0
  33. isotrieve-0.2.1/src/isotrieve/providers/cached.py +121 -0
  34. isotrieve-0.2.1/src/isotrieve/providers/cohere.py +68 -0
  35. isotrieve-0.2.1/src/isotrieve/providers/factory.py +92 -0
  36. isotrieve-0.2.1/src/isotrieve/providers/gemini.py +64 -0
  37. isotrieve-0.2.1/src/isotrieve/providers/openai.py +84 -0
  38. isotrieve-0.2.1/src/isotrieve/providers/sentence_transformers.py +52 -0
  39. isotrieve-0.2.1/src/isotrieve/providers/voyage.py +61 -0
  40. isotrieve-0.2.1/src/isotrieve/py.typed +0 -0
  41. isotrieve-0.2.1/src/isotrieve/quality/__init__.py +19 -0
  42. isotrieve-0.2.1/src/isotrieve/quality/gate.py +283 -0
  43. isotrieve-0.2.1/src/isotrieve/quality/gate_model_v1.json +52 -0
  44. isotrieve-0.2.1/src/isotrieve/quality/metrics.py +198 -0
  45. isotrieve-0.2.1/src/isotrieve/quality/thresholds.json +11 -0
  46. isotrieve-0.2.1/src/isotrieve/quality/thresholds.yaml +16 -0
  47. isotrieve-0.2.1/src/isotrieve/recalibration.py +238 -0
  48. isotrieve-0.2.1/src/isotrieve/reporting/__init__.py +1 -0
  49. isotrieve-0.2.1/src/isotrieve/reporting/html_report.py +62 -0
  50. isotrieve-0.2.1/src/isotrieve/reranking.py +217 -0
  51. isotrieve-0.2.1/src/isotrieve/serve.py +208 -0
  52. isotrieve-0.2.1/src/isotrieve/stores/__init__.py +6 -0
  53. isotrieve-0.2.1/src/isotrieve/stores/base.py +41 -0
  54. isotrieve-0.2.1/src/isotrieve/stores/numpy_files.py +168 -0
  55. isotrieve-0.2.1/src/isotrieve/stores/qdrant_store.py +181 -0
  56. isotrieve-0.2.1/src/isotrieve/wrappers/__init__.py +31 -0
  57. isotrieve-0.2.1/src/isotrieve/wrappers/llamaindex.py +161 -0
  58. isotrieve-0.2.1/src/isotrieve/wrappers/openai_shim.py +118 -0
  59. isotrieve-0.2.1/src/isotrieve/wrappers/telemetry.py +79 -0
@@ -0,0 +1,68 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ *.egg-info/
7
+ *.egg
8
+ dist/
9
+ build/
10
+ .eggs/
11
+
12
+ # Virtual environments
13
+ .venv/
14
+ venv/
15
+ env/
16
+
17
+ # Testing
18
+ .pytest_cache/
19
+ .coverage
20
+ htmlcov/
21
+
22
+ # Type checking
23
+ .mypy_cache/
24
+
25
+ # IDE
26
+ .vscode/
27
+ .idea/
28
+ .cursor/
29
+ *.swp
30
+ *.swo
31
+
32
+ # macOS
33
+ .DS_Store
34
+
35
+ # Environment
36
+ .env
37
+
38
+ # Jupyter
39
+ .ipynb_checkpoints/
40
+
41
+ # Node
42
+ node_modules/
43
+ .npm/
44
+
45
+ # Model/artifact files (too large for git)
46
+ *.pkl
47
+ *.npy
48
+ *.pt
49
+ *.pth
50
+ *.onnx
51
+ *.bin
52
+ *.safetensors
53
+
54
+ # Embedding cache
55
+ benchmarks/.embed_cache/
56
+
57
+ # Benchmark results (keep committed, ignore regenerated)
58
+ # benchmarks/results/ is committed via explicit add
59
+
60
+ # Build artifacts
61
+ *.log
62
+ *.tsbuildinfo
63
+ *.map
64
+
65
+ # Scratch files at repo root
66
+ *.ipynb_checkpoints/
67
+ aecp-python/aecp_training_loss.png
68
+ cleanup/
@@ -0,0 +1,41 @@
1
+ # CLAIMS.md — Public quantitative claims → benchmark artifacts
2
+
3
+ Every number that appears in README, docs, or marketing must have a row here
4
+ pointing at a committed file under `benchmarks/results/`.
5
+
6
+ ## Active claims
7
+
8
+ | Claim text (exact or paraphrase) | Where used | Artifact path | Verified | Notes |
9
+ |----------------------------------|------------|---------------|----------|-------|
10
+ | SciFact MiniLM→bge-large nDCG@10 retention ≈ 0.871 ± 0.006 (K=4000, Ridge adapter, 3 seeds) | `isotrieve-python/README.md` | `benchmarks/results/beir_scifact_*__ridge__k4000__seed{0,1,2}__*.json` | 2026-07-21 | Floor=0.0, ceiling=0.735, Isotrieve=0.634–0.646. Actual: 0.872, 0.863, 0.879. |
11
+ | LowRank adapter nDCG@10 retention ≈ 0.857 ± 0.009 (K=4000, 3 seeds) | same | `benchmarks/results/beir_scifact_*__lowrank__k4000__seed{0,1,2}__*.json` | 2026-07-21 | Phase 2 adapter sweep. Actual: 0.867, 0.846, 0.858. |
12
+ | MLP adapter nDCG@10 retention ≈ 0.727 ± 0.007 (K=4000, 3 seeds) | same | `benchmarks/results/beir_scifact_*__mlp__k4000__seed{0,1,2}__*.json` | 2026-07-21 | No hyperparameter tuning. Actual: 0.736, 0.724, 0.721. |
13
+ | K-sweep (ridge only): K=500 0.704±0.004, K=1000 0.781±0.008, K=2000 0.818±0.007, K=4000 0.871±0.006 | same | `benchmarks/results/beir_scifact_*__ridge__k{500,1000,2000,4000}__*.json` | 2026-07-21 | Ridge-only K-sweep. Monotonic improvement with K. |
14
+ | K-sweep (all adapters avg): K=500 0.671±0.041, K=1000 0.735±0.058, K=2000 0.785±0.052, K=4000 0.832±0.061 | same | `benchmarks/results/beir_scifact_*__k{500,1000,2000,4000}__*.json` | 2026-07-21 | All adapters averaged across 3 seeds. |
15
+ | Floor nDCG@10 = 0.0 when dims differ (384≠1024) | same | same | 2026-07-21 | Raw cross-space vectors cannot be queried. |
16
+ | Ceiling (full re-embed) nDCG@10 ≈ 0.735 | same | same | 2026-07-21 | Quality upper bound for this model pair. |
17
+ | Same-dim pair (bge-large→e5-large, 1024→1024): floor=0.0, Isotrieve≈0.667, ceiling=0.722, retention=0.923±0.010 | `isotrieve-python/README.md` | `benchmarks/results/beir_scifact_*__BAAI_bge-large-en-v1.5__to__intfloat_e5-large-v2__ridge__k2000__seed{0,1,2}__*.json` | 2026-07-21 | Actual: 0.908, 0.930, 0.931. With e5 prefixes. Without prefixes, ceiling=0.355 and retention=0.95 (broken). |
18
+ | Gate model trained on local pairs only (no API model pairs) | `isotrieve-python/src/isotrieve/quality/gate_model_v1.json` | `benchmarks/results/gate_lopo.json` | — | Gate model valid for local model pairs. API pair performance may differ. |
19
+ | WS-A: bge→e5 raw scores agree at 100% for τ≤0.75; recalibration helps at τ=0.80 (+4.7% agreement, +2.36pt recall) | `README.md` | `benchmarks/results/ws_a_bge_to_e5_recall_tables.json` | — | MAE=0.095, margin compression=0.83x. |
20
+ | WS-A: MiniLM→bge rectangular pair — raw scores severely compressed (mean 0.157 vs ceiling 0.521, MAE=0.364). Recalibration essential. | `README.md` | `benchmarks/results/ws_a_minilm_to_bge_recall_tables.json` | — | τ=0.60 goes 78%→100% (+22%), τ=0.70 goes 27%→67% (+40%). |
21
+ | WS-B: Confidence flags (adaptive P33/P67 margins) are predictive across both pairs. bge→e5 high=0.955/low=0.637; MiniLM→bge high=0.875/low=0.651 | `README.md` | `benchmarks/results/ws_b_confidence_flags.json` | — | 50-56ms/query latency. |
22
+ | WS-B: Cross-encoder reranking NULL RESULT (-10.7pts bge→e5, -9.8pts MiniLM→bge — MS MARCO domain-mismatched for sci-text) | `DECISIONS.md` | `benchmarks/results/ws_b_cross_encoder.json` | — | Not shipped. |
23
+ | WS-C: Independent inverse-α: +2.17pts (bge→e5), +2.23pts (MiniLM→bge). Consistent across pairs. | `README.md` | `benchmarks/results/ws_c_independent_inv_alpha.json` | — | Optimal inv alpha differs from forward (0.178 vs 0.316 on bge→e5). |
24
+ | WS-C: TSVD shrinkage NULL RESULT (rank=512 only -0.33pt) | `DECISIONS.md` | `benchmarks/results/ws_c_tsvd_shrinkage.json` | — | Not worth complexity. |
25
+ | WS-E: Rectangular pair re-validation — MiniLM→bge: 86% retention, margin compression 0.85x, independent inverse-α +2.23pts | `README.md` | `benchmarks/results/ws_e_minilm_to_bge_revalidation.json` | — | |
26
+ | WS-D: Gate v3 — margin compression <0.85 widens prediction interval | `DECISIONS.md` | `src/isotrieve/quality/gate_model_v1.json` | — | `_predict_retention()` accepts `margin_compression` parameter. |
27
+
28
+ ## Retired / deleted claims (must not reappear)
29
+
30
+ These appeared in prior docs without reproducible artifacts and were removed:
31
+
32
+ - "97% semantic fidelity" / "97.2%" / "97.35%"
33
+ - "<10ms" / "<1ms" transfer latency as a product claim
34
+ - "85% Top-1" abort threshold as a validated number
35
+ - "86% corpus fidelity" / "43% text baseline"
36
+ - "150x faster" / "200x"
37
+ - "Validated on 300k vocab, zero overfitting"
38
+ - Ridge ">90%" / future MLP ">99%" as stated facts
39
+
40
+ Replacement rule: only cite numbers produced by `benchmarks/run_benchmark.py`
41
+ and stored under `benchmarks/results/`.
@@ -0,0 +1,285 @@
1
+ # Isotrieve Design Decisions
2
+
3
+ Running log of design choices and open questions.
4
+ Bias toward the boring, verifiable choice.
5
+
6
+ ## Postmortems
7
+
8
+ ### 2026-07-19 — Number Discrepancy Postmortem (WS-0)
9
+
10
+ **Symptom:** Adapter sweep reported Ridge/K=4000/SciFact = 0.866 ± 0.008. K-sweep reported the same nominal config = 0.814 ± 0.068. The ±0.068 (8× larger std) flagged a discrepancy.
11
+
12
+ **Root cause:** The K-sweep summary (`run_benchmark.py` line 538) filtered by `k_cal` but **not by adapter**. When running `--adapter ridge lowrank mlp --k 500 1000 2000 4000`, the K-sweep summary averaged across ALL adapters for each K value:
13
+ - ridge K=4000: 0.866 ± 0.008
14
+ - lowrank K=4000: 0.857 ± 0.009
15
+ - mlp K=4000: 0.719 ± 0.008
16
+ - **All-adapter average at K=4000: 0.814 ± 0.068** ← this was the K-sweep number
17
+
18
+ The adapter sweep was adapter-specific (ridge only). Two valid but different experiments, presented in a way that implied they should match.
19
+
20
+ **Resolution:**
21
+ 1. K-sweep summary now broken down by adapter (`run_benchmark.py` updated).
22
+ 2. `protocol` field added to result JSON (`fixed_calibration_per_seed`).
23
+ 3. Audit script `benchmarks/audit_configs.py` committed for field-by-field comparison.
24
+ 4. CLAIMS.md and DECISIONS.md updated to distinguish adapter-specific vs cross-adapter numbers.
25
+
26
+ **Lesson:** Always declare which dimension (adapter, K, seed) a summary averages over. The `protocol` field in result JSONs makes this explicit.
27
+
28
+ ## Locked decisions
29
+
30
+ ### 2026-07-19 — Package strategy (Phase 1)
31
+
32
+ **Choice:** Rewrite `isotrieve-python` onto a `src/isotrieve/` layout as the migration toolkit.
33
+ The old agent-protocol API under the flat `isotrieve/` package is excluded from the
34
+ build and deferred to a later "advanced usage" docs page. TypeScript (`@isotrieve/core`)
35
+ is out of scope for v1.
36
+
37
+ **Why:** The product contract repositions Isotrieve as embedding-space migration, not
38
+ agent handoff. One excellent Python package beats two mediocre packages. The PyPI
39
+ name `isotrieve` must not collide with a second package.
40
+
41
+ ### 2026-07-19 — Version reset
42
+
43
+ **Choice:** Ship as `0.1.0` (alpha), not continue from marketed `1.0.0`.
44
+
45
+ **Why:** Public claims from the old package are unverified. Starting at 0.1.0
46
+ signals that numbers are earned via the benchmark harness.
47
+
48
+ ### 2026-07-19 — License
49
+
50
+ **Choice:** Apache-2.0 (per product contract). Previous MIT license in this
51
+ directory is superseded for the new package.
52
+
53
+ ### 2026-07-19 — Mapping defaults
54
+
55
+ **Choice:** `RidgeMapping` is primary; `OrthogonalProcrustesMapping` only when
56
+ `d_src == d_tgt`. All `transform` outputs are L2-normalized. Ridge uses an
57
+ augmented bias column and `alpha="auto"` via leave-one-out GCV over a log grid
58
+ (`sklearn.linear_model.RidgeCV`).
59
+
60
+ **Why:** Rectangular maps (1536→3072) are the normal case; Procrustes cannot
61
+ handle them. Normalization matches retrieval practice and makes cosine metrics
62
+ stable.
63
+
64
+ ### 2026-07-19 — No wall-clock expiry on `.isotrieve` files
65
+
66
+ **Choice:** Header includes `expires_hint` for operator convenience but the
67
+ library never auto-expires. Drift detection is `isotrieve verify` (fresh probe embeds).
68
+
69
+ **Why:** Model weights do not drift on a calendar; silent provider updates do.
70
+
71
+ ### 2026-07-19 — Phase 1 scope
72
+
73
+ **Choice:** Implement core math, quality metrics, numpy store, minimal file CLI,
74
+ offline tests, and a local-model SciFact harness before any API providers or
75
+ vector-DB adapters.
76
+
77
+ ### 2026-07-19 — Phase 1 gate result (first honest number)
78
+
79
+ **Measured** on SciFact test (300 queries with qrels), in-domain K=4000,
80
+ `all-MiniLM-L6-v2` → `BAAI/bge-large-en-v1.5`, 3 seeds:
81
+
82
+ | | nDCG@10 |
83
+ |--|---------|
84
+ | Floor (raw cross-space; dims 384≠1024) | 0.000 |
85
+ | Isotrieve (mapped) | ≈ 0.640 |
86
+ | Ceiling (full re-embed) | ≈ 0.735 |
87
+ | **Retention (Isotrieve÷ceiling)** | **0.871 ± 0.006** |
88
+
89
+ Artifacts under `benchmarks/results/`. Rank warning: MiniLM source matrix was
90
+ slightly rank-deficient (382/384) at K=4000 — logged, not hidden.
91
+
92
+ **Gate status:** PASSED — Isotrieve retention meaningfully above floor.
93
+
94
+ ### 2026-07-19 — Phase 1 prerequisite fixes (before Phase 2)
95
+
96
+ Inferred from Phase 1 debt + contaminated artifacts (PM referenced "four fixes
97
+ above" without the prior message in-thread). Logged here for audit:
98
+
99
+ | Fix | Action |
100
+ |-----|--------|
101
+ | F1 | Purge `self_retrieval_fallback` result JSONs; harness **requires** real BEIR qrels for claimable runs — no silent fallback |
102
+ | F2 | Record **holdout-vs-corpus optimism gap** (fit holdout top-1 − retrieval retention) in every result + QualityGate |
103
+ | F3 | **CachedEmbedder mandatory** on all provider/CLI/benchmark embed paths — never embed the same text twice |
104
+ | F4 | Gate/corpus sample must be **disjoint from calibration**; rank-deficiency stays a warning (ridge handles it) |
105
+
106
+ ### 2026-07-19 — Phase 2 adapter zoo
107
+
108
+ **Choice:** Add 4 adapters to the existing Ridge: Procrustes (square only), Procrustes+Diagonal, LowRankAffine, ResidualMLP. Ridge remains the default for rectangular mappings.
109
+
110
+ **Why:** Drift-Adapter (EMNLP 2025) validates Procrustes and MLP approaches. LowRankAffine provides compressed matrices. Each adapter has different compute/memory/quality tradeoffs.
111
+
112
+ **Measured (SciFact, all-MiniLM-L6-v2 → bge-large-en-v1.5, K=4000, 3 seeds):**
113
+
114
+ | Adapter | nDCG@10 retention | Notes |
115
+ |---------|------------------|-------|
116
+ | Ridge | 0.866 ± 0.008 | Default. Fast, stable. |
117
+ | LowRank | 0.857 ± 0.009 | Compressed matrix. ~1% worse than Ridge. |
118
+ | MLP | 0.719 ± 0.008 | No hyperparameter tuning. Significantly worse than linear. |
119
+
120
+ ### 2026-07-19 — Phase 2 K-sweep (data-driven gate thresholds)
121
+
122
+ **Choice:** Benchmark across K=500,1000,2000,4000 to derive QualityGate thresholds from data.
123
+
124
+ **Measured (SciFact, all adapters averaged, 3 seeds):**
125
+
126
+ | K | nDCG@10 retention | Notes |
127
+ |---|------------------|-------|
128
+ | 500 | 0.667 ± 0.039 | Below recommended minimum. Rank-deficient. |
129
+ | 1000 | 0.732 ± 0.056 | Still below minimum. Better but high variance. |
130
+ | 2000 | 0.788 ± 0.054 | Approaching acceptable quality. |
131
+ | 4000 | 0.814 ± 0.068 | Full quality. Recommended minimum. |
132
+
133
+ **Gate thresholds derived:**
134
+ - PASS: retention ≥ 0.75 (K≥2000 typically achieves this)
135
+ - WARN: 0.55 ≤ retention < 0.75 (K=500-1000 range, or poor mapping)
136
+ - FAIL: retention < 0.55 (mapping fundamentally broken)
137
+
138
+ ### 2026-07-19 — K minimum constraint downgraded to warning
139
+
140
+ **Choice:** Change adapter `fit()` from raising `ValueError` on small K to issuing `UserWarning`. The 10×min_dim rule is a guideline, not a hard error.
141
+
142
+ **Why:** Users need to see what happens with small K. The benchmark harness needs to sweep across K values for threshold calibration. Warning gives visibility without blocking experimentation.
143
+
144
+ ### 2026-07-19 — MLP rectangular dims
145
+
146
+ **Choice:** ResidualMLP uses plain MLP (no residual skip) when d_src ≠ d_tgt. Residual skip only when dimensions match.
147
+
148
+ **Why:** The residual connection `x + net(x)` requires matching input/output dims. Rectangular projections need a feedforward architecture.
149
+
150
+ ### 2026-07-19 — QualityGate v2: proxy-based prediction (not raw holdout)
151
+
152
+ **Choice:** Redefine gate thresholds from "requires ceiling measurement" to "predicts retention from holdout proxy using isotonic regression." Gate model ships with the package (`gate_model_v1.json`). Gate verdict uses prediction intervals.
153
+
154
+ **Why:** The old gate required ceiling quality (full re-embed) to measure retention — defeating the purpose of a migration tool. The new gate predicts retention from holdout proxies that are always available: `top1_retention`, `holdout_rank_corr`, `cosine_mean`. Trained on 48 benchmark results (4 adapter pairs), LOPO CV: MAE=0.0443, RMSE=0.0592, 80% interval half-width=0.0706.
155
+
156
+ **Artifacts:** `gate_model_v1.json`, `benchmarks/fit_gate_model.py`, `benchmarks/results/gate_lopo.json`
157
+
158
+ ### 2026-07-19 — Same-dimension floor is 0.0
159
+
160
+ **Choice:** Measure bge-large→e5-large (1024→1024) raw cross-space retrieval. Floor = 0.0.
161
+
162
+ **Why:** "Same dimension" is not "same space." Different models produce geometrically incompatible embeddings even at the same dimension. Raw cross-space nDCG@10 = 0.0 — search runs fine, results are garbage. This is the core problem Isotrieve solves. Retention = 0.908 with mapping (with e5 prefixes).
163
+
164
+ ### 2026-07-19 — e5 prefix bug (postmortem)
165
+
166
+ **Symptom:** Same-dim pair (bge-large→e5-large) showed ceiling nDCG@10 = 0.355, retention = 95%. e5-large alone scores ~0.74 on SciFact — ceiling was clearly broken.
167
+
168
+ **Root cause:** e5 models require `"query: "` / `"passage: "` text prefixes for correct embeddings. The benchmark harness passed raw texts with no prefix handling. Both documents and queries were embedded without prefixes, degrading e5's retrieval quality by ~50%.
169
+
170
+ **Resolution:** Added `MODEL_PREFIXES` dict to `run_benchmark.py` with model-specific prefix mappings. Modified `embed_texts()` and `load_or_embed()` to apply correct prefixes. Cache invalidated via `prefix_version` field.
171
+
172
+ **Corrected results:**
173
+ - Before fix: ceiling=0.355, Isotrieve=0.337, retention=0.950 (broken)
174
+ - After fix: ceiling=0.722, Isotrieve=0.656, retention=0.908 (honest)
175
+
176
+ **Lesson:** Model-specific preprocessing (prefixes, tokenization quirks) must be part of the embedding pipeline, not assumed to be "just encode text." When a number looks too good (95% retention), verify the ceiling is realistic.
177
+
178
+ ### 2026-07-19 — Bidirectional evaluation
179
+
180
+ **Choice:** Evaluate both directions: source→target (transform corpus) AND target→source (map queries). Report "query→legacy retention" = 86.2% of ceiling.
181
+
182
+ **Why:** Serve-mode quality is measured by mapping queries into legacy space (query→legacy), not by transforming corpus (legacy→target). Both directions matter for the "before/after" narrative.
183
+
184
+ ### 2026-07-20 — WS-A: Score recalibration (isotonic regression)
185
+
186
+ **Choice:** Fit isotonic regression (sklearn `IsotonicRegression`) on held-out query scores to map cross-space cosine scores to ceiling-equivalent scores. Recalibrator saved alongside mapping in `.isotrieve` format under `score_recal_v1` section.
187
+
188
+ **Why:** Raw cross-space scores are compressed (mean 0.157 vs ceiling 0.521 for MiniLM→bge rectangular pair). This means τ-thresholding based on ceiling scores fails on migrated data. Recalibration makes τ-thresholding reliable.
189
+
190
+ **Measured:**
191
+ - bge→e5 (same-dim): Raw scores agree at 100% for τ≤0.75. Recalibration helps only at τ=0.80 (+4.7% agreement, +2.36pt recall). MAE=0.095, margin compression=0.83x.
192
+ - MiniLM→bge (rectangular): Raw scores severely compressed (MAE=0.364). Recalibration essential: τ=0.60 goes 78%→100% (+22%), τ=0.70 goes 27%→67% (+40%). Margin compression=0.85x.
193
+
194
+ **Artifacts:** `benchmarks/results/ws_a_*_recall_tables.json`
195
+
196
+ ### 2026-07-20 — WS-B: Cross-encoder reranking NULL RESULT
197
+
198
+ **Choice:** Do NOT ship cross-encoder reranking. MS MARCO cross-encoder (`cross-encoder/ms-marco-MiniLM-L-6-v2`) is domain-mismatched for scientific text.
199
+
200
+ **Measured:** -10.7pts on bge→e5, -9.8pts on MiniLM→bge. Consistent degradation.
201
+
202
+ **Why not:** Would need domain-matched cross-encoder (e.g., SciFact-trained). Not worth the complexity until proven on in-domain data.
203
+
204
+ **Artifacts:** `benchmarks/results/ws_b_cross_encoder.json`
205
+
206
+ ### 2026-07-20 — WS-B: Confidence flags (adaptive margins)
207
+
208
+ **Choice:** Confidence flags using adaptive P33/P67 margin thresholds from calibration data. Predictive across both pairs.
209
+
210
+ **Measured:**
211
+ - bge→e5: high-conf R@10=0.955, low-conf R@10=0.637 (gap=0.318)
212
+ - MiniLM→bge: high-conf R@10=0.875, low-conf R@10=0.651 (gap=0.224)
213
+
214
+ **Why:** Users need per-query confidence to know when to trust migrated results. Adaptive thresholds (percentile-based) work across different score distributions without manual tuning.
215
+
216
+ **Artifacts:** `benchmarks/results/ws_b_confidence_flags.json`
217
+
218
+ ### 2026-07-20 — WS-C: Independent inverse-α
219
+
220
+ **Choice:** RidgeMapping now has separate `_chosen_inv_alpha` for inverse direction, fitted independently from forward α. Stored in `.isotrieve` header.
221
+
222
+ **Why:** Optimal regularization differs between forward (Y = XW) and inverse (X = YW_inv) directions. Fitting α independently for each direction improves inverse quality without affecting forward.
223
+
224
+ **Measured:** +2.17pts (bge→e5), +2.23pts (MiniLM→bge). Consistent across pairs. Optimal inv alpha differs from forward (0.178 vs 0.316 on bge→e5).
225
+
226
+ **Artifacts:** `benchmarks/results/ws_c_independent_inv_alpha.json`
227
+
228
+ ### 2026-07-20 — WS-C: TSVD shrinkage NULL RESULT
229
+
230
+ **Choice:** RidgeMapping now accepts `rank=r` parameter for TSVD shrinkage, but NOT recommended. Default is `rank=None` (no shrinkage).
231
+
232
+ **Measured:** rank=512 only -0.33pt, rank=256 -1.69pt, rank=128 -2.33pt. Quality degrades monotonically with rank reduction.
233
+
234
+ **Why keep it:** Useful for storage-constrained deployments (large mappings). But for most users, the quality loss isn't worth it.
235
+
236
+ **Artifacts:** `benchmarks/results/ws_c_tsvd_shrinkage.json`
237
+
238
+ ### 2026-07-20 — WS-C: Procrustes centering REVERTED
239
+
240
+ **Choice:** Do NOT auto-center embeddings before Procrustes mapping. Production `OrthogonalProcrustesMapping` uses `V @ R^T` for inverse (no centering).
241
+
242
+ **Why reverted:** Q3 measured held-out cosine alignment (X_ho mapped to Y space vs Y_ho), NOT serve-mode retrieval. Centering breaks serve-mode because centered queries don't match uncentered source doc distribution. The -55pt swing was a measurement artifact, not a production bug.
243
+
244
+ **Lesson:** Always validate against actual serve-mode retrieval (R@10), not just alignment metrics.
245
+
246
+ ### 2026-07-20 — WS-D: Gate v3 (margin compression)
247
+
248
+ **Choice:** `_predict_retention()` now accepts `margin_compression` parameter. When margin_compression < 0.85, widen prediction interval.
249
+
250
+ **Why:** Score compression indicates the mapping quality is uncertain. Wider intervals make the gate more conservative, reducing false PASS verdicts.
251
+
252
+ ### 2026-07-20 — Vector DB adapters (v0.2)
253
+
254
+ **Choice:** Add ChromaDB and LangChain adapters. ChromaDB gets serve-mode (`IsotrieveChromaFunction`) and offline migration (`migrate_collection`). LangChain gets `IsotrieveEmbeddings` shim.
255
+
256
+ **Why:** Most users interact with vector databases through ORMs/clients. Thin adapters that apply Isotrieve mappings at the DB layer eliminate the need for users to manage mapping files manually.
257
+
258
+ **Adapters implemented:**
259
+ - `isotrieve.adapters.chroma.IsotrieveChromaFunction` — Chroma `EmbeddingFunction`
260
+ - `isotrieve.adapters.chroma.migrate_collection()` — offline migration
261
+ - `isotrieve.adapters.langchain.IsotrieveEmbeddings` — LangChain `Embeddings` shim
262
+ - `isotrieve.adapters.base.EmbeddingAdapter` — core ABC
263
+ - `isotrieve.adapters.base.VectorStoreAdapter` — core ABC
264
+ - `isotrieve.adapters.base.MigrationReport` — migration metadata
265
+
266
+ **Tests:** 21 tests for adapters (13 ChromaDB, 8 LangChain), all passing.
267
+
268
+ ## Open questions
269
+
270
+ None for Phase 1 gates. Questions that affect later phases:
271
+
272
+ | ID | Question | Affects | Status |
273
+ |----|----------|---------|--------|
274
+ | Q1 | Exact QualityGate PASS/WARN/FAIL thresholds | Phase 2 | **Resolved** — data-driven from K-sweep. See thresholds.json. |
275
+ | Q2 | Frozen calibration corpus composition (`isotrieve-calib-v1`) | Phase 2 | Open — need permissive sources + checksum |
276
+ | Q3 | Fate of legacy `isotrieve/` protocol modules and `isotrieve-npm` | Phase 4 | Open — archive vs delete |
277
+ | Q4 | PyPI publish account / GitHub org URLs | Phase 4 | Open — placeholders remain |
278
+ | Q5 | Procrustes adapters for square pairs | Phase 2 | Open — only works when d_src == d_tgt |
279
+ | Q6 | MLP hyperparameter tuning (epochs, lr, hidden_dim) | Phase 3 | Open — current defaults unoptimized |
280
+
281
+ ## PM sign-off
282
+
283
+ - [ ] Results table reviewed
284
+ - [ ] "When NOT to use Isotrieve" section reviewed
285
+ - [ ] Sign-off date: _
@@ -0,0 +1,17 @@
1
+ # Apache License
2
+ # Version 2.0, January 2004
3
+ # http://www.apache.org/licenses/
4
+
5
+ Copyright 2026 AECP Contributors
6
+
7
+ Licensed under the Apache License, Version 2.0 (the "License");
8
+ you may not use this file except in compliance with the License.
9
+ You may obtain a copy of the License at
10
+
11
+ http://www.apache.org/licenses/LICENSE-2.0
12
+
13
+ Unless required by applicable law or agreed to in writing, software
14
+ distributed under the License is distributed on an "AS IS" BASIS,
15
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ See the License for the specific language governing permissions and
17
+ limitations under the License.