semanticmemo 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. semanticmemo-1.2.0/.github/workflows/ci.yml +42 -0
  2. semanticmemo-1.2.0/.github/workflows/publish-pypi.yml +84 -0
  3. semanticmemo-1.2.0/.gitignore +21 -0
  4. semanticmemo-1.2.0/AGENTS.md +232 -0
  5. semanticmemo-1.2.0/CHANGELOG.md +183 -0
  6. semanticmemo-1.2.0/CONTRIBUTING.md +73 -0
  7. semanticmemo-1.2.0/LICENSE +21 -0
  8. semanticmemo-1.2.0/PKG-INFO +403 -0
  9. semanticmemo-1.2.0/README.md +362 -0
  10. semanticmemo-1.2.0/RELEASE_NOTES.md +126 -0
  11. semanticmemo-1.2.0/benchmarks/classifier_vs_cosine.py +168 -0
  12. semanticmemo-1.2.0/benchmarks/cosine_baseline_customer_support.py +133 -0
  13. semanticmemo-1.2.0/benchmarks/data/customer_support.jsonl +20 -0
  14. semanticmemo-1.2.0/benchmarks/data/finance.jsonl +20 -0
  15. semanticmemo-1.2.0/benchmarks/data/hard_negatives.jsonl +12 -0
  16. semanticmemo-1.2.0/benchmarks/data/high_stakes_pairs.jsonl +28 -0
  17. semanticmemo-1.2.0/benchmarks/data/medical.jsonl +20 -0
  18. semanticmemo-1.2.0/benchmarks/data/security.jsonl +20 -0
  19. semanticmemo-1.2.0/benchmarks/false_positive_eval.py +183 -0
  20. semanticmemo-1.2.0/benchmarks/prompt_mutation_benchmark.py +300 -0
  21. semanticmemo-1.2.0/benchmarks/results/classifier_vs_cosine.json +53 -0
  22. semanticmemo-1.2.0/benchmarks/results/comparison_matrix.json +489 -0
  23. semanticmemo-1.2.0/benchmarks/results/false_positive_eval.json +323 -0
  24. semanticmemo-1.2.0/benchmarks/results/mutation_benchmark.json +69 -0
  25. semanticmemo-1.2.0/benchmarks/results/semantic_drift_benchmark.json +505 -0
  26. semanticmemo-1.2.0/benchmarks/results/threshold_report.md +127 -0
  27. semanticmemo-1.2.0/benchmarks/run_benchmarks.py +413 -0
  28. semanticmemo-1.2.0/benchmarks/semantic_drift_benchmark.py +499 -0
  29. semanticmemo-1.2.0/benchmarks/sweep_thresholds.py +478 -0
  30. semanticmemo-1.2.0/data/README.md +20 -0
  31. semanticmemo-1.2.0/data/corpus/action_templates.jsonl +109 -0
  32. semanticmemo-1.2.0/data/corpus/base_prompts.jsonl +96 -0
  33. semanticmemo-1.2.0/data/fixtures/customer_support_pairs.jsonl +12 -0
  34. semanticmemo-1.2.0/data/gold/equivalence_gold.jsonl +84 -0
  35. semanticmemo-1.2.0/data/training/manifest.json +32 -0
  36. semanticmemo-1.2.0/data/training/pairs_v2.train.jsonl +13576 -0
  37. semanticmemo-1.2.0/data/training/pairs_v2.validation.jsonl +3000 -0
  38. semanticmemo-1.2.0/docs/API.md +158 -0
  39. semanticmemo-1.2.0/docs/adr/adr-001-why-mlp-classifier.md +25 -0
  40. semanticmemo-1.2.0/docs/adr/adr-002-why-cross-encoder-verification.md +19 -0
  41. semanticmemo-1.2.0/docs/adr/adr-003-why-domain-conditioned-risk-policies.md +21 -0
  42. semanticmemo-1.2.0/docs/architecture.md +235 -0
  43. semanticmemo-1.2.0/docs/benchmark_methodology.md +65 -0
  44. semanticmemo-1.2.0/docs/decision_engine.md +72 -0
  45. semanticmemo-1.2.0/docs/deployment.md +67 -0
  46. semanticmemo-1.2.0/docs/feedback.md +99 -0
  47. semanticmemo-1.2.0/docs/images/active_learning.png +0 -0
  48. semanticmemo-1.2.0/docs/images/architecture.png +0 -0
  49. semanticmemo-1.2.0/docs/images/benchmark_fpr.png +0 -0
  50. semanticmemo-1.2.0/docs/images/benchmark_latency.png +0 -0
  51. semanticmemo-1.2.0/docs/images/drift_detection.png +0 -0
  52. semanticmemo-1.2.0/docs/images/precision_recall_curves.png +0 -0
  53. semanticmemo-1.2.0/docs/limitations.md +88 -0
  54. semanticmemo-1.2.0/docs/ml/evaluation.md +47 -0
  55. semanticmemo-1.2.0/docs/ml/how-the-classifier-works.md +44 -0
  56. semanticmemo-1.2.0/docs/ml/training-your-own.md +97 -0
  57. semanticmemo-1.2.0/docs/quickstart.md +116 -0
  58. semanticmemo-1.2.0/docs/results.md +291 -0
  59. semanticmemo-1.2.0/docs/why-semanticmemo.md +66 -0
  60. semanticmemo-1.2.0/examples/basic_usage.py +58 -0
  61. semanticmemo-1.2.0/examples/bundled_classifier.py +77 -0
  62. semanticmemo-1.2.0/examples/train_custom_classifier.py +42 -0
  63. semanticmemo-1.2.0/examples/with_agenteval.py +66 -0
  64. semanticmemo-1.2.0/examples/with_guardloop.py +83 -0
  65. semanticmemo-1.2.0/examples/with_orchflow.py +78 -0
  66. semanticmemo-1.2.0/models/.gitkeep +1 -0
  67. semanticmemo-1.2.0/pyproject.toml +99 -0
  68. semanticmemo-1.2.0/scripts/generate_training_data.py +467 -0
  69. semanticmemo-1.2.0/scripts/train_classifier.py +208 -0
  70. semanticmemo-1.2.0/src/semanticmemo/__init__.py +48 -0
  71. semanticmemo-1.2.0/src/semanticmemo/_logging.py +29 -0
  72. semanticmemo-1.2.0/src/semanticmemo/_models/equivalence-net-v1.pt +0 -0
  73. semanticmemo-1.2.0/src/semanticmemo/_models/equivalence-net-v1.report.json +248 -0
  74. semanticmemo-1.2.0/src/semanticmemo/cache.py +149 -0
  75. semanticmemo-1.2.0/src/semanticmemo/classifier/__init__.py +28 -0
  76. semanticmemo-1.2.0/src/semanticmemo/classifier/cross_encoder_service.py +84 -0
  77. semanticmemo-1.2.0/src/semanticmemo/classifier/data.py +135 -0
  78. semanticmemo-1.2.0/src/semanticmemo/classifier/evaluate.py +160 -0
  79. semanticmemo-1.2.0/src/semanticmemo/classifier/model.py +53 -0
  80. semanticmemo-1.2.0/src/semanticmemo/classifier/service.py +108 -0
  81. semanticmemo-1.2.0/src/semanticmemo/classifier/train.py +151 -0
  82. semanticmemo-1.2.0/src/semanticmemo/cli.py +251 -0
  83. semanticmemo-1.2.0/src/semanticmemo/domain_detector.py +103 -0
  84. semanticmemo-1.2.0/src/semanticmemo/embedding/__init__.py +19 -0
  85. semanticmemo-1.2.0/src/semanticmemo/embedding/service.py +194 -0
  86. semanticmemo-1.2.0/src/semanticmemo/entity_change_detection.py +619 -0
  87. semanticmemo-1.2.0/src/semanticmemo/exceptions.py +22 -0
  88. semanticmemo-1.2.0/src/semanticmemo/feedback/__init__.py +9 -0
  89. semanticmemo-1.2.0/src/semanticmemo/feedback/retrain.py +241 -0
  90. semanticmemo-1.2.0/src/semanticmemo/metrics.py +61 -0
  91. semanticmemo-1.2.0/src/semanticmemo/models.py +277 -0
  92. semanticmemo-1.2.0/src/semanticmemo/orchestrator.py +778 -0
  93. semanticmemo-1.2.0/src/semanticmemo/resources.py +37 -0
  94. semanticmemo-1.2.0/src/semanticmemo/store/__init__.py +5 -0
  95. semanticmemo-1.2.0/src/semanticmemo/store/schema.sql +73 -0
  96. semanticmemo-1.2.0/src/semanticmemo/store/sqlite_store.py +551 -0
  97. semanticmemo-1.2.0/src/semanticmemo/types.py +34 -0
  98. semanticmemo-1.2.0/tests/conftest.py +59 -0
  99. semanticmemo-1.2.0/tests/test_bundled_classifier.py +78 -0
  100. semanticmemo-1.2.0/tests/test_cache.py +277 -0
  101. semanticmemo-1.2.0/tests/test_classifier_data.py +60 -0
  102. semanticmemo-1.2.0/tests/test_classifier_evaluate.py +32 -0
  103. semanticmemo-1.2.0/tests/test_classifier_model.py +30 -0
  104. semanticmemo-1.2.0/tests/test_classifier_service.py +68 -0
  105. semanticmemo-1.2.0/tests/test_classifier_train.py +59 -0
  106. semanticmemo-1.2.0/tests/test_cli.py +48 -0
  107. semanticmemo-1.2.0/tests/test_context_manager.py +49 -0
  108. semanticmemo-1.2.0/tests/test_decision_engine.py +342 -0
  109. semanticmemo-1.2.0/tests/test_embedding.py +90 -0
  110. semanticmemo-1.2.0/tests/test_entity_change_detection.py +347 -0
  111. semanticmemo-1.2.0/tests/test_feedback_retrain.py +261 -0
  112. semanticmemo-1.2.0/tests/test_implicit_feedback.py +309 -0
  113. semanticmemo-1.2.0/tests/test_logging.py +82 -0
  114. semanticmemo-1.2.0/tests/test_models.py +72 -0
  115. semanticmemo-1.2.0/tests/test_public_api.py +20 -0
  116. semanticmemo-1.2.0/tests/test_retry.py +148 -0
  117. semanticmemo-1.2.0/tests/test_store.py +132 -0
  118. semanticmemo-1.2.0/tests/test_store_concurrency.py +68 -0
  119. semanticmemo-1.2.0/uv.lock +1598 -0
@@ -0,0 +1,42 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: ["main"]
6
+ pull_request:
7
+ branches: ["main"]
8
+
9
+ jobs:
10
+ test:
11
+ name: Test (py${{ matrix.python-version }})
12
+ runs-on: ubuntu-latest
13
+ permissions:
14
+ contents: read
15
+ strategy:
16
+ fail-fast: false
17
+ matrix:
18
+ python-version: ["3.11", "3.12", "3.13", "3.14"]
19
+
20
+ steps:
21
+ - name: Check out repository
22
+ uses: actions/checkout@v6
23
+
24
+ - name: Install uv
25
+ uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
26
+ with:
27
+ python-version: "${{ matrix.python-version }}"
28
+
29
+ - name: Install dependencies
30
+ run: uv sync --all-extras --frozen
31
+
32
+ - name: Check formatting
33
+ run: uv run ruff format --check
34
+
35
+ - name: Lint
36
+ run: uv run ruff check
37
+
38
+ - name: Type check
39
+ run: uv run pyright
40
+
41
+ - name: Test
42
+ run: uv run pytest
@@ -0,0 +1,84 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*.*.*"
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ test:
11
+ name: Test before publish
12
+ runs-on: ubuntu-latest
13
+ permissions:
14
+ contents: read
15
+
16
+ steps:
17
+ - name: Check out repository
18
+ uses: actions/checkout@v6
19
+
20
+ - name: Install uv
21
+ uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
22
+ with:
23
+ python-version: "3.12"
24
+
25
+ - name: Install dependencies
26
+ run: uv sync --all-extras --frozen
27
+
28
+ - name: Check formatting
29
+ run: uv run ruff format --check
30
+
31
+ - name: Lint
32
+ run: uv run ruff check
33
+
34
+ - name: Type check
35
+ run: uv run pyright
36
+
37
+ - name: Test
38
+ run: uv run pytest
39
+
40
+ build:
41
+ name: Build distributions
42
+ needs: test
43
+ runs-on: ubuntu-latest
44
+ permissions:
45
+ contents: read
46
+
47
+ steps:
48
+ - name: Check out repository
49
+ uses: actions/checkout@v6
50
+
51
+ - name: Install uv
52
+ uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
53
+ with:
54
+ python-version: "3.12"
55
+
56
+ - name: Build package
57
+ run: uv build
58
+
59
+ - name: Upload distributions
60
+ uses: actions/upload-artifact@v7
61
+ with:
62
+ name: python-package-distributions
63
+ path: dist/
64
+
65
+ publish:
66
+ name: Publish distributions to PyPI
67
+ needs: build
68
+ runs-on: ubuntu-latest
69
+ environment:
70
+ name: pypi
71
+ url: https://pypi.org/p/semanticmemo
72
+ permissions:
73
+ actions: read
74
+ id-token: write
75
+
76
+ steps:
77
+ - name: Download distributions
78
+ uses: actions/download-artifact@v8
79
+ with:
80
+ name: python-package-distributions
81
+ path: dist/
82
+
83
+ - name: Publish package distributions to PyPI
84
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,21 @@
1
+ .DS_Store
2
+ .venv/
3
+ .ruff_cache/
4
+ .pytest_cache/
5
+ .pyright/
6
+ __pycache__/
7
+ *.py[cod]
8
+ *.egg-info/
9
+ build/
10
+ dist/
11
+ .coverage
12
+ htmlcov/
13
+ *.db
14
+ *.sqlite
15
+ *.sqlite3
16
+ models/*.pt
17
+ !models/.gitkeep
18
+ data/generated/
19
+ data/training/.paraphrase_cache.json
20
+ scratch/
21
+
@@ -0,0 +1,232 @@
1
+ # AGENTS.md — SemanticMemo Project Guide
2
+
3
+ > **Author & Lead Engineer:** Rajveer Singh Saggu
4
+ > **Repository:** [rajveer100704/semanticmemo](https://github.com/rajveer100704/semanticmemo)
5
+ > This guide is the single source of truth for the SemanticMemo project. It documents the ML rationale, systems architecture, and engineering principles behind this learned semantic caching layer.
6
+
7
+ ---
8
+
9
+ ## The Philosophy & Vision
10
+
11
+ ### Why Cosine Similarity Fails in Production
12
+ Semantic caching is essential for scaling LLM applications. However, existing industry solutions (like GPTCache) rely on a naive cosine similarity threshold in embedding space. In my work, I realized a foundational truth:
13
+
14
+ **Cosine similarity is not equivalence.**
15
+
16
+ In embedding space, prompts asking for opposite actions can sit extremely close. For example, *"Should I accept this meeting request?"* and *"Should I reject this meeting request?"* have a cosine similarity of ~0.97 using standard sentence encoders. A naive cache treats them as equivalent and serves the cached answer of one to the other.
17
+
18
+ In critical domains like finance, healthcare, or security, this is not just a cheap cache hit — it is a silent, catastrophic bug. It is a false-positive that returns wrong data to an agent, prompting incorrect actions.
19
+
20
+ ### The SemanticMemo Solution
21
+ SemanticMemo replaces the arbitrary, static cosine threshold with a **learned equivalence classifier** fine-tuned on domain-specific prompt traces. By modeling semantic equivalence as a supervised matching problem, it distinguishes intent from surface similarity.
22
+
23
+ **The Vision:**
24
+ > Providing a highly reliable, domain-adapted semantic cache that teams can confidently run in production. Cache hit rates of 60-80% with classifier-validated correctness, preventing silent bugs and keeping LLM operating costs predictable.
25
+
26
+ ---
27
+
28
+ ## The Problem in Concrete Terms
29
+
30
+ When building real-world AI agents, developers are forced into two compromises:
31
+ - **No Caching:** Every query hits the LLM API. High latency, high API costs, and linear scaling with traffic.
32
+ - **Naive Cosine Caching:** Fast and cheap, but high false-positive rates (FPR). The agent receives cached responses to different prompts, leading to incorrect actions that are notoriously hard to debug.
33
+
34
+ SemanticMemo introduces a third option: **Double-Verification Caching**. It combines fast vector retrieval with a high-precision, low-latency classifier to ensure only functionally equivalent prompts trigger cache hits.
35
+
36
+ ---
37
+
38
+ ## The Mental Model: A Multi-Stage Verification Pipeline
39
+
40
+ SemanticMemo works like a layered security checkpoint:
41
+ 1. **Metal Detector (High Recall, Low Latency):** Uses fast vector similarity (FAISS / Qdrant) to quickly retrieve the top $K$ candidates ($O(\log N)$).
42
+ 2. **First Gate (MLP Classifier):** Evaluates candidate pairs via a fast (~1ms) classifier ($O(K)$). If the score is extremely high (e.g. $\ge 0.995$), it bypasses the next gate.
43
+ 3. **Second Gate (Cross-Encoder):** Runs a deep joint-attention transformer pass on candidate pairs where the MLP is uncertain, providing high-precision validation.
44
+ 4. **Final Gate (Entity Change Detector):** Runs a zero-latency (<1ms) battery of 11 regex detectors to veto any matches that contain critical entity drift (e.g. quarter, drug, year, numeric, or proper noun changes).
45
+
46
+ This structure ensures both sub-millisecond retrieval scaling and maximum correctness in production.
47
+
48
+ ---
49
+
50
+ ## Core Architecture
51
+
52
+ The codebase is structured around five core concepts:
53
+
54
+ ### 1. The Embedding Layer (Fast Filter)
55
+ Embeds incoming queries and queries the vector index (FAISS or Qdrant) for the top $K$ candidate vectors.
56
+ - **Default Encoder:** `sentence-transformers/all-MiniLM-L6-v2` (384-dimensional).
57
+ - **Index:** `faiss.IndexFlatIP` (normalized inner product for cosine similarity).
58
+
59
+ ### 2. The Equivalence Classifier (Stage 1 MLP)
60
+ A lightweight neural network computing equivalence probabilities on concatenated features ($[u, v, |u - v|, u * v]$).
61
+ - **Architecture:** MLP head with 2 hidden layers (128 → 64 → 1) and ReLU activation.
62
+ - **Inference Latency:** < 1.5ms on CPU.
63
+
64
+ ### 3. The Cross-Encoder (Stage 2 Re-ranker)
65
+ A transformer-based cross-encoder (`cross-encoder/ms-marco-MiniLM-L-6-v2`) that parses joint-attention representations.
66
+ - **Domain Tuning:** Thresholds are dynamically conditioned per domain (Medical, Finance, Security, Customer Support).
67
+ - **Latency Optimization:** Bypassed when the MLP score $\ge$ skip threshold (triggering on ~94.7% of hits).
68
+
69
+ ### 4. The Entity Change Detector (Stage 3 Veto)
70
+ A battery of 11 lightweight, regex-based check rules validating key entities (e.g. Q3 vs Q4, Tesla vs Apple).
71
+ - **Latency:** < 1ms on CPU, no ML inference.
72
+ - **Configurability:** Fully toggleable per detector or system-wide.
73
+
74
+ ### 5. The Cache Store & Active Learning Ledger (Persistence)
75
+ SQLite database managing cache storage (LRU/TTL eviction) and active learning logging.
76
+ - **Active Learning:** Logs MLP/CE disagreements and entity-vetoed misses to `active_learning_pairs` as training data for future classifier retraining.
77
+
78
+ ---
79
+
80
+ ## Complete Request Lifecycle
81
+
82
+ The diagram below outlines the cache lookup workflow:
83
+
84
+ ```
85
+ agent calls semanticmemo.get_or_call(prompt, llm_function)
86
+ │
87
+ ▼
88
+ 1. embed(prompt) → query_embedding
89
+ │
90
+ ▼
91
+ 2. FAISS/Qdrant search → candidate_ids (K=5)
92
+ │
93
+ ▼
94
+ 3. Run MLP Classifier on candidates
95
+ ├── Score >= Skip Threshold (0.995) ──┐
96
+ └── Score < Skip Threshold │
97
+ │ │
98
+ ▼ ▼
99
+ Run Cross-Encoder [Bypass CE]
100
+ │ │
101
+ ▼ │
102
+ Verify thresholds │
103
+ (Passed?) │
104
+ ├── Yes ──────────────────────┤
105
+ └── No ──► Cache Miss │
106
+ │ │
107
+ ▼ ▼
108
+ Store LLM response ◄─── Run Entity Detector
109
+ (Entity Changed?)
110
+ ├── Yes (Drift) ──► Cache Miss & Log AL
111
+ └── No ───────────► Cache Hit
112
+ ```
113
+
114
+ ---
115
+
116
+ ## Public API Surface
117
+
118
+ ### Minimal Usage
119
+ ```python
120
+ from semanticmemo import SemanticMemo
121
+ from anthropic import AsyncAnthropic
122
+
123
+ client = AsyncAnthropic()
124
+ cache = SemanticMemo(domain="customer-support")
125
+
126
+ async def call_llm(prompt: str) -> str:
127
+ response = await client.messages.create(
128
+ model="claude-3-5-sonnet-latest",
129
+ max_tokens=1024,
130
+ messages=[{"role": "user", "content": prompt}],
131
+ )
132
+ return response.content[0].text
133
+
134
+ # Wrapper handles cache check, lookup, decision, and LLM call fallback
135
+ result = await cache.get_or_call(
136
+ prompt="List the active tickets for user 102",
137
+ llm_function=call_llm,
138
+ )
139
+ print(result.response)
140
+ print(result.was_cache_hit)
141
+ print(result.cost_saved_usd)
142
+ ```
143
+
144
+ ### Complete Configuration
145
+ ```python
146
+ from semanticmemo import SemanticMemo, CacheConfig, ClassifierConfig, EvictionPolicy
147
+
148
+ cache = SemanticMemo(
149
+ domain="finance",
150
+ config=CacheConfig(
151
+ embedding_dim=384,
152
+ candidate_k=5,
153
+ max_entries=5000,
154
+ eviction_policy=EvictionPolicy.LRU,
155
+ ttl_seconds=86400 * 14, # 14 days
156
+ ),
157
+ classifier=ClassifierConfig(
158
+ model_path="./models/equivalence-net-v1.pt",
159
+ device="cpu"
160
+ )
161
+ )
162
+ ```
163
+
164
+ ---
165
+
166
+ ## Internal Code Structure
167
+
168
+ ```
169
+ semanticmemo/
170
+ ├── src/
171
+ │ └── semanticmemo/
172
+ │ ├── __init__.py # Public API exports
173
+ │ ├── cache.py # Main SemanticMemo orchestrator
174
+ │ ├── orchestrator.py # CacheOrchestrator pipeline
175
+ │ ├── embedding/
176
+ │ │ ├── __init__.py
177
+ │ │ ├── service.py # EmbeddingService (Model + FAISS)
178
+ │ │ └── models.py # Embedding providers
179
+ │ ├── classifier/
180
+ │ │ ├── __init__.py
181
+ │ │ ├── service.py # ClassifierService (Inference wrapper)
182
+ │ │ ├── model.py # PairClassifier (PyTorch Module)
183
+ │ │ ├── train.py # Classifier training pipeline
184
+ │ │ ├── data.py # Dataset loaders and transformers
185
+ │ │ └── evaluate.py # Evaluation helper utilities
186
+ │ ├── store/
187
+ │ │ ├── __init__.py
188
+ │ │ ├── sqlite_store.py # SQLite CacheStore implementation
189
+ │ │ ├── eviction.py # Eviction policy modules
190
+ │ │ └── schema.sql # SQL Schema DDL
191
+ │ ├── feedback/
192
+ │ │ ├── __init__.py
193
+ │ │ ├── ledger.py # FeedbackLedger database
194
+ │ │ └── retrain.py # Retraining orchestration
195
+ │ ├── stats.py # Cache stats collectors
196
+ │ ├── models.py # Pydantic models (CacheResult, CacheDecision)
197
+ │ ├── exceptions.py # Custom exceptions
198
+ │ └── cli.py # CLI entry points
199
+ ├── benchmarks/
200
+ │ ├── run_benchmarks.py # Complete suite runner
201
+ │ ├── sweep_thresholds.py # Threshold optimization grid-search
202
+ │ ├── false_positive_eval.py # Hard negative / opposite action tests
203
+ │ ├── prompt_mutation_benchmark.py # Mutation robustness suite
204
+ │ └── data/ # Benchmark datasets (.jsonl)
205
+ ├── examples/ # Reference integration files
206
+ ├── tests/ # Pytest suite
207
+ └── pyproject.toml # Package dependencies and settings
208
+ ```
209
+
210
+ ---
211
+
212
+ ## Development History & Roadmap
213
+
214
+ ### Version 1.0.0 (Production Stable)
215
+ - **Double Verification Caching:** FAISS + MLP Classifier + Cross-Encoder verification.
216
+ - **Domain Auto-Detection:** Automatically routes queries using domain centroid distance.
217
+ - **Risk-Aware Policies:** Allows stricter classification thresholds on sensitive domains.
218
+ - **Performance Optimizations:** Global model caching and CPU-based warmup profiles.
219
+ - **Full Benchmarks:** Hard negative FPR reduced from 33.3% to 0% with minimal latency overhead.
220
+
221
+ ### Future Enhancements
222
+ - **pgvector & Qdrant Backends:** Support for external vector databases in distributed setups.
223
+ - **Online Continual Learning:** Background training worker updating the classifier weights asynchronously.
224
+
225
+ ---
226
+
227
+ ## Engineering Standards
228
+
229
+ 1. **Typing & Validation:** Strict type annotations across all code. Run `uv run pyright` to verify types.
230
+ 2. **Silent by Default:** Library logging propagates to a root `semanticmemo` namespace using `logging.NullHandler`.
231
+ 3. **No Unmanaged Latency:** CPU inference must be optimized. Models are initialized lazily and cached globally.
232
+ 4. **Test-Driven Rigor:** Maintain >90% test coverage. Every new component must have corresponding unit tests in `tests/`.
@@ -0,0 +1,183 @@
1
+ # Changelog
2
+
3
+ ## 1.2.0
4
+
5
+ Upgrade release featuring zero-latency entity drift protection, domain-conditioned thresholds, active learning builder, and safety-framed benchmarking.
6
+
7
+ **New features & upgrades:**
8
+
9
+ - **Entity Drift Protection** — The `EntityChangeDetector` runs 11 fast, regex-based validation checks in `< 1ms` to intercept critical entity changes (e.g. Q3 vs Q4, Tesla vs Apple, 100mg vs 200mg) that neural models and cross-encoders often miss.
10
+ - **Domain-Conditioned Thresholds** — Replaces generic risk tiers with domain-specific MLP and Cross-Encoder thresholds (Medical: 0.995/0.97, Finance: 0.990/0.95, Security: 0.997/0.98, Customer Support: 0.90/0.85) to optimize precision/recall trade-offs.
11
+ - **Active Learning Dataset Builder** — Automatically logs MLP vs Cross-Encoder disagreements to the `active_learning_pairs` database table to build hard-negative datasets for continuous retraining.
12
+ - **Safety-Framed Benchmarking** — Rewrote the semantic drift benchmark to compare three systems (Cosine, SM v1, SM v2) on Safety Accuracy (correct blocks), Dangerous False Positive Rate (DFP), and Reuse Accuracy (valid hits).
13
+ - **Comprehensive Tests** — Added 33 unit tests under `tests/test_entity_change_detection.py` validating each regex detector and disabled configurations.
14
+
15
+ ## 1.0.0
16
+
17
+ First stable release. Published to PyPI as `semanticmemo` (internal import name unchanged: `import semanticmemo`).
18
+
19
+ This release promotes SemanticMemo from a cosine-threshold semantic cache to a four-stage
20
+ verification pipeline with measured, reproducible false-positive reduction across four
21
+ high-stakes production domains.
22
+
23
+ **Architecture additions:**
24
+
25
+ - **Double Verification pipeline** — MLP classifier (stage 3) + Cross-Encoder (stage 4) replace the
26
+ single cosine threshold. The MLP is fast (~1ms); the Cross-Encoder is high-precision (~3–8ms) and
27
+ is only invoked when the MLP is uncertain.
28
+ - **Opposite-action veto** — rule-based pre-filter that blocks antonym pairs (approve/deny,
29
+ buy/sell, increase/decrease, enable/disable, allow/block, grant/revoke, etc.) before any ML
30
+ stage runs. Catches the canonical hard-negative failure case with zero latency overhead.
31
+ - **Latency-aware MLP bypass** — when the MLP score exceeds `high_precision_skip_threshold`
32
+ (default 0.995), the Cross-Encoder is skipped entirely. Measured bypass rate: **94.7%** of
33
+ cache hits; average latency with bypass: **~27ms** vs **~39ms** without.
34
+ - **Domain auto-detection** — `DomainDetector` routes each query to its risk tier using embedding
35
+ centroids, enabling per-domain threshold policies without explicit domain tagging by the caller.
36
+ - **Risk-aware policy engine** — `RiskPolicy` maps domains to `RiskTier.HIGH` or `RiskTier.LOW`
37
+ and applies the corresponding MLP/CE thresholds. Ships with sensible defaults:
38
+ medical/finance/security/legal → HIGH (MLP=0.99, CE=0.95); general/customer-support → LOW (MLP=0.90, CE=0.85).
39
+ - **Explainable `CacheDecision`** — every `CacheResult.decision` records the decision reason,
40
+ risk tier, per-stage scores, and per-stage latency. Available on both hits and misses.
41
+
42
+ **Performance and profiling:**
43
+
44
+ - **Global `_MODEL_CACHE`** in `CrossEncoderService` — the Cross-Encoder is loaded once and
45
+ shared across all `SemanticMemo` instances in the same process. Eliminates the 1–5 second
46
+ cold-start penalty on the first inference.
47
+ - **Five new latency profiling fields** on `CacheResult` and `CacheDecision`:
48
+ `embedding_latency_ms`, `retrieval_latency_ms`, `mlp_latency_ms`, `cross_encoder_latency_ms`,
49
+ `total_latency_ms`. Enables per-stage bottleneck diagnosis in production.
50
+
51
+ **Benchmark suite expansion:**
52
+
53
+ - `benchmarks/run_benchmarks.py` — full 4-method × 4-domain comparison matrix (Cosine Baseline,
54
+ MLP Classifier, Double Verification, SemanticMemo). Includes per-stage latency breakout table,
55
+ hard-negative stress test, and Cross-Encoder bypass statistics. Output: `comparison_matrix.json`.
56
+ - `benchmarks/sweep_thresholds.py` — 40 MLP thresholds × 30 CE thresholds = 1,200 configurations
57
+ per domain, with per-domain FPR constraints (Security < 5%, Finance < 10%). Output:
58
+ `threshold_report.md` with optimal per-domain config and production recommendation.
59
+ - `benchmarks/data/hard_negatives.jsonl` — 12 curated opposite-action pairs across finance,
60
+ security, medical, and software domains.
61
+ - All four domain datasets expanded to 20 pairs each (10 positive + 10 hard-negative),
62
+ replacing the previous 5–10 pair sets.
63
+ - Model warmup in both benchmark scripts eliminates download/load time from measured latency.
64
+
65
+ **Key measured results:**
66
+
67
+ | Metric | Cosine Baseline | SemanticMemo |
68
+ | :--- | :---: | :---: |
69
+ | Hard-Negative FPR | 33.3% | **0.0%** |
70
+ | Finance FPR | 20% | **0%** |
71
+ | Security FPR | 20% | **0%** |
72
+ | Medical FPR | 30% | **10%** |
73
+ | Classifier Precision (gold set, equal recall) | 0.527 | **0.829** |
74
+ | CE Bypass Rate | — | **94.7%** |
75
+ | Avg Cache-Hit Latency | — | **~27ms** |
76
+
77
+ **Documentation:**
78
+
79
+ - `docs/results.md` — comprehensive 9-section benchmark results document: architecture diagram,
80
+ full comparison matrix, hard-negative stress test results, latency breakdown, bypass rate
81
+ analysis, threshold sweep results, cost savings model, and classifier gold-set evaluation.
82
+ - `README.md` rewritten for v1.0.0: problem statement, architecture mermaid diagram, all
83
+ benchmark tables, full quickstart with production config, project structure, and roadmap.
84
+
85
+ **Packaging:**
86
+
87
+ - PyPI package renamed to `semanticmemo`. Internal import name unchanged: `import semanticmemo`.
88
+ - `Development Status` classifier upgraded from Beta to `5 - Production/Stable`.
89
+ - Added `Topic :: Scientific/Engineering :: Artificial Intelligence` classifier.
90
+ - `__version__ = "1.0.0"` exported from `semanticmemo.__init__`.
91
+
92
+ ## 0.3.0
93
+
94
+ A hardening release: no new product features, but the existing ones become safer to run
95
+ in production. Every change is backward compatible; new configuration is opt-in and off
96
+ by default.
97
+
98
+ - Add structured logging under the `semanticmemo` logger namespace. The library is silent by
99
+ default (it attaches only a `NullHandler`); applications opt in by configuring the
100
+ `semanticmemo` logger.
101
+ - Open the SQLite store in WAL mode with a 5-second busy timeout, and serialize writes
102
+ with an internal re-entrant lock so one store instance is safe to use from multiple
103
+ threads of a process. The concurrency guarantees are documented honestly — it is not a
104
+ distributed cache.
105
+ - Bug fix: enable `PRAGMA foreign_keys`, so the schema's declared `ON DELETE CASCADE`
106
+ foreign keys now actually fire. Deleting or evicting a cache entry previously left
107
+ orphan `lookup_records` and `feedback_events` rows behind; it now cleans them up.
108
+ - `SemanticMemo` supports `async with` for guaranteed cleanup; `close()` is unchanged.
109
+ - Add opt-in LLM-call retries: `CacheConfig(retry=RetryConfig(...))` retries transient
110
+ failures of `llm_function` with bounded exponential backoff. Off by default; only the
111
+ cache-miss path is retried. Adds the `RetryConfig` model and the `LLMCallError`
112
+ exception (raised only when retries are enabled and exhausted).
113
+ - CI now tests Python 3.11 through 3.14; the PyPI publish workflow runs the test suite
114
+ before building and releasing.
115
+ - Add `CONTRIBUTING.md`.
116
+
117
+ ## 0.2.0
118
+
119
+ - Retrain the bundled classifier as `equivalence-net-v1`: trained on 16,576 labeled pairs
120
+ across nine domains (adds medical, legal, and finance, plus negation-by-insertion hard
121
+ negatives). It beats the cosine baseline by +30 precision points at equal recall on the
122
+ gold set and replaces `classifier-v1` as the shipped checkpoint.
123
+ - Add opt-in implicit-feedback detection: with `CacheConfig(implicit_feedback=...)`,
124
+ re-issuing the same prompt within a window after a cache hit auto-records a bad-hit
125
+ event. Off by default; explicit feedback always takes precedence.
126
+ - Add the `ImplicitFeedbackConfig` model and `CacheResult.implicit_bad_hit_recorded`.
127
+ - `get_or_call` now trims surrounding whitespace from the prompt, so the cache key is
128
+ whitespace-insensitive — prompts differing only by leading or trailing whitespace no
129
+ longer create duplicate cache entries.
130
+ - Add `benchmarks/false_positive_eval.py` and `benchmarks/data/high_stakes_pairs.jsonl`:
131
+ a high-stakes medical/legal/finance opposite-action evaluation.
132
+ - Add optional integration examples (`examples/with_orchflow.py`, `with_guardloop.py`,
133
+ `with_agenteval.py`) composing SemanticMemo with companion libraries; none are
134
+ dependencies of semanticmemo.
135
+ - Add `docs/feedback.md` covering explicit and implicit feedback and retraining.
136
+
137
+ ## 0.1.0
138
+
139
+ - Ship `classifier-v1`, a pretrained generic equivalence classifier, inside the package.
140
+ - Add `ClassifierConfig.bundled()` for opt-in, zero-training classifier-gated caching.
141
+ - Add `scripts/generate_training_data.py`: a local-LLM (Ollama) paraphrase and templated
142
+ hard-negative training-data pipeline, with a committed 10,800-pair dataset.
143
+ - Add a hand-curated 84-pair equivalence gold test set under `data/gold/`.
144
+ - Add `scripts/train_classifier_v1.py` and `benchmarks/classifier_vs_cosine.py`:
145
+ `classifier-v1` beats the cosine baseline by +32 precision points at equal recall on
146
+ the gold set.
147
+ - Document that the optional `[ml]` dependencies (PyTorch, FAISS, SentenceTransformers)
148
+ are required to import semanticmemo.
149
+
150
+ ## 0.0.4
151
+
152
+ - Add `semanticmemo retrain` for manual feedback-to-checkpoint retraining.
153
+ - Add `semanticmemo.feedback.retrain_from_feedback(...)` with auditable retrain reports.
154
+ - Support optional seed data, validation gates, and checkpoint promotion when gates pass.
155
+ - Document the manual feedback loop from collection through deliberate classifier promotion.
156
+
157
+ ## 0.0.3
158
+
159
+ - Add durable lookup records for cache hits so feedback can be exported later.
160
+ - Add durable good/bad feedback events tied to cache-hit query IDs.
161
+ - Add `SemanticMemo.export_feedback_pairs(...)` and `semanticmemo export-feedback`.
162
+ - Export feedback-derived JSONL compatible with the classifier training pipeline.
163
+
164
+ ## 0.0.2
165
+
166
+ - Add optional classifier-gated cache decisions through `SemanticMemo(..., classifier=...)`.
167
+ - Populate `CacheResult.classifier_score` when the classifier evaluates cache candidates.
168
+ - Preserve cosine-threshold behavior when no classifier checkpoint is configured.
169
+ - Document classifier-enabled usage and the current cold-start boundary.
170
+
171
+ ## 0.0.1
172
+
173
+ Initial SemanticMemo release.
174
+
175
+ - Async semantic-cache facade with SQLite persistence.
176
+ - Dependency-light hash embeddings for tests and smoke demos.
177
+ - Optional SentenceTransformers and FAISS support through the `ml` extra.
178
+ - Customer-support cosine-baseline benchmark fixture.
179
+ - Pair-classifier model, dataset, training, evaluation, and checkpoint inference utilities.
180
+ - CLI commands for cache stats, classifier training, and classifier evaluation.
181
+ - GitHub Actions CI and PyPI trusted publishing workflow.
182
+
183
+
@@ -0,0 +1,73 @@
1
+ # Contributing to SemanticMemo
2
+
3
+ Thanks for your interest in SemanticMemo. This guide covers local development, the quality
4
+ gates, and how releases are made.
5
+
6
+ ## Development setup
7
+
8
+ SemanticMemo uses [uv](https://docs.astral.sh/uv/) for environment and dependency
9
+ management. Python 3.11–3.14 are supported.
10
+
11
+ ```bash
12
+ uv sync --all-extras
13
+ ```
14
+
15
+ The `[ml]` extra (PyTorch, FAISS, SentenceTransformers) is required to import `semanticmemo`
16
+ itself, so `--all-extras` is the normal development sync.
17
+
18
+ ## Quality gates
19
+
20
+ These four commands are exactly what CI runs. All must pass before a change merges:
21
+
22
+ ```bash
23
+ uv run ruff format --check # formatting
24
+ uv run ruff check # lint
25
+ uv run pyright # type checking
26
+ uv run pytest # tests
27
+ ```
28
+
29
+ CI runs them on Python 3.11, 3.12, 3.13, and 3.14.
30
+
31
+ ## Concurrency model
32
+
33
+ The SQLite store opens the database in WAL mode with a 5-second busy timeout. A single
34
+ store instance is safe to use from multiple threads of one process — writes are
35
+ serialized by an internal re-entrant lock, and the connection is created with
36
+ `check_same_thread=False`. SemanticMemo is not a distributed cache: multiple processes
37
+ writing the same database file rely solely on SQLite's file locking, and heavy
38
+ multi-process write contention can still raise `sqlite3.OperationalError`.
39
+
40
+ ## The bundled classifier
41
+
42
+ The package ships a pretrained equivalence classifier at
43
+ `src/semanticmemo/_models/equivalence-net-v1.pt`, force-included into the wheel by the
44
+ `[tool.hatch.build.targets.wheel.force-include]` table in `pyproject.toml`.
45
+
46
+ It is reproducible from committed data:
47
+
48
+ ```bash
49
+ # Optional: regenerate the dataset from the prompt corpus (requires a local Ollama model)
50
+ python scripts/generate_training_data.py
51
+
52
+ # Train and package the classifier from the committed dataset
53
+ uv run python scripts/train_classifier.py
54
+ ```
55
+
56
+ `train_classifier.py` writes the checkpoint plus an auditable model card
57
+ (`equivalence-net-v1.report.json`) next to it. SemanticMemo is precision-first: the acceptance
58
+ gate requires the classifier to beat the cosine baseline by at least 10 precision points
59
+ at equal recall on the held-out gold set. See `docs/ml/` for details.
60
+
61
+ ## Release process
62
+
63
+ Releases are tag-triggered:
64
+
65
+ 1. Bump `version` in `pyproject.toml`.
66
+ 2. Add a new section to `CHANGELOG.md` describing what shipped.
67
+ 3. Commit the changes to `main`.
68
+ 4. Tag the commit `vX.Y.Z` and push the tag.
69
+
70
+ Pushing the tag triggers `.github/workflows/publish-pypi.yml`, which runs the full test
71
+ suite before building and publishing to PyPI — a broken tag cannot publish.
72
+
73
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Rajveer Singh Saggu
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.