context-engineering-core 0.12.1__tar.gz → 0.14.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/CHANGELOG.md +36 -1
  2. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/PKG-INFO +33 -7
  3. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/README.md +27 -6
  4. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/pyproject.toml +4 -2
  5. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/__init__.py +1 -1
  6. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/cli.py +26 -1
  7. context_engineering_core-0.14.0/src/context_engine/embeddings.py +189 -0
  8. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/data/freeze.json +2 -2
  9. context_engineering_core-0.14.0/src/context_engine/evaluation/data/paraphrases.json +38 -0
  10. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/strategies.py +129 -19
  11. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/retrieve.py +28 -6
  12. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/mcp_server.py +19 -4
  13. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/pipeline.py +11 -1
  14. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/.gitignore +0 -0
  15. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/LICENSE +0 -0
  16. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/evidence_answer_policy.py +0 -0
  17. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/incident-demo.json +0 -0
  18. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/inspection-input.json +0 -0
  19. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/rolling_summary.py +0 -0
  20. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/sdk_quickstart.py +0 -0
  21. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/service-config.example.json +0 -0
  22. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/structured_answers.py +0 -0
  23. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/examples/validated_answer.py +0 -0
  24. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/__main__.py +0 -0
  25. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/_sqlite.py +0 -0
  26. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/_windows_security.py +0 -0
  27. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/_windows_storage.py +0 -0
  28. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/adapters.py +0 -0
  29. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/answers.py +0 -0
  30. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/budget.py +0 -0
  31. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/config.py +0 -0
  32. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/context.py +0 -0
  33. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/demo.py +0 -0
  34. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/errors.py +0 -0
  35. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/__init__.py +0 -0
  36. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/commands.py +0 -0
  37. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/corpus.py +0 -0
  38. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/data/ground_truth.json +0 -0
  39. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/data/pin_retention.json +0 -0
  40. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/data/protocol.json +0 -0
  41. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/data/scenario.json +0 -0
  42. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/data/summary_fixture.json +0 -0
  43. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/execution.py +0 -0
  44. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/fixtures.py +0 -0
  45. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/grading.py +0 -0
  46. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/harness.py +0 -0
  47. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/__init__.py +0 -0
  48. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/__main__.py +0 -0
  49. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/data/ground_truth.json +0 -0
  50. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/data/manifest.json +0 -0
  51. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/data/protocol.json +0 -0
  52. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/data/scenarios.json +0 -0
  53. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/data/scoring_manifest.json +0 -0
  54. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/data/scoring_protocol.json +0 -0
  55. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/heldout/scoring.py +0 -0
  56. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/protocol.py +0 -0
  57. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/reporting.py +0 -0
  58. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/results.py +0 -0
  59. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/evaluation/run_commands.py +0 -0
  60. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/inspection.py +0 -0
  61. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/__init__.py +0 -0
  62. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/cap.py +0 -0
  63. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/common.py +0 -0
  64. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/pin.py +0 -0
  65. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/summarize.py +0 -0
  66. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/layers/window.py +0 -0
  67. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/memory/__init__.py +0 -0
  68. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/memory/codec.py +0 -0
  69. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/memory/demo.py +0 -0
  70. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/memory/store.py +0 -0
  71. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/models.py +0 -0
  72. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/__init__.py +0 -0
  73. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/client.py +0 -0
  74. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/commands.py +0 -0
  75. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/contracts.py +0 -0
  76. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/credentials.py +0 -0
  77. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/memory.py +0 -0
  78. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/store.py +0 -0
  79. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/transport.py +0 -0
  80. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/providers/windows_credentials.py +0 -0
  81. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/py.typed +0 -0
  82. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/__init__.py +0 -0
  83. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/__main__.py +0 -0
  84. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/app.py +0 -0
  85. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/auth.py +0 -0
  86. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/client.py +0 -0
  87. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/control.py +0 -0
  88. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/demo.py +0 -0
  89. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/integrations.py +0 -0
  90. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/schemas.py +0 -0
  91. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/worker.py +0 -0
  92. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/service/workers.py +0 -0
  93. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/summaries.py +0 -0
  94. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/tokens.py +0 -0
  95. {context_engineering_core-0.12.1 → context_engineering_core-0.14.0}/src/context_engine/work.py +0 -0
@@ -3,7 +3,42 @@
3
3
  Earlier versions (0.1.0–0.9.6) are documented in `docs/C01_REPORT.md` through
4
4
  `docs/C10_STORAGE_ACL_REPORT.md` and preserved as wheels under `archives/`.
5
5
 
6
- ## 0.12.1 — 2026-10-31 (unreleased)
6
+ ## 0.14.0 — 2026-10-31 (unreleased)
7
+
8
+ ### Added
9
+ - Optional hybrid retrieval: `assemble_context(..., embedder=, semantic_weight=0.7)`
10
+ (also via `MemoryStore.assemble`) ranks chunks by 0.7 x meaning + 0.3 x keyword
11
+ score. `context_engine.embeddings`: `Embedder` protocol, `Model2VecEmbedder`
12
+ (default model pinned: `minishlab/potion-base-8M@bf8b0566`, extra `[semantic]`),
13
+ `FastEmbedEmbedder` (extra `[fastembed]`), `FunctionEmbedder`. Bounded vector cache.
14
+ - `strategy-eval --semantic` (adds `retrieval_semantic`, `combined_semantic`) and
15
+ `context-engine-mcp --semantic`.
16
+ - Reworded questions: default retention 3/31 → 21/31 with the embedder; original
17
+ questions stay 31/31. Bake-off evidence in `output/embedding-bakeoff-001/`.
18
+
19
+ ### Changed
20
+ - `numpy` is now a declared dependency (it was already installed via `rank-bm25`).
21
+ - Missing optional extras give an install hint instead of a traceback.
22
+
23
+ ### Unchanged
24
+ - Without an embedder, all 310 strategy-benchmark assemblies are byte-identical to
25
+ 0.13.0 (archived in `archives/c14-semantic-0140/`).
26
+
27
+ ## 0.13.0 — 2026-10-31 (unreleased)
28
+
29
+ ### Added
30
+ - Reworded-question stress test: `strategy-eval --questions paraphrased` and
31
+ `compare_strategies(..., questions="paraphrased")`. 31 questions share no content
32
+ word (or 4-letter start of a 5+ letter word) with their evidence, enforced by
33
+ `paraphrase_overlaps`. Result: default retention falls from 31/31 to 3/31 at 900
34
+ and 3000 tokens (retrieval alone: 6/31 and 7/31). README updated.
35
+ - `assemble_strategy(..., question=...)` to assemble a strategy for any question.
36
+
37
+ ### Unchanged
38
+ - Engine behavior and defaults. 0.12.1 wheel/freeze archived in
39
+ `archives/c13-paraphrase-0130/`.
40
+
41
+ ## 0.12.1 — 2026-10-31 (PyPI)
7
42
 
8
43
  ### Changed (performance only; output identical)
9
44
  - Budget fit checks compute only the request's estimated token count
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: context-engineering-core
3
- Version: 0.12.1
3
+ Version: 0.14.0
4
4
  Summary: Provider-independent context layers, source contracts, and token budget accounting
5
5
  Project-URL: Homepage, https://github.com/Nandu1729/Context-Engine
6
6
  Project-URL: Repository, https://github.com/Nandu1729/Context-Engine
@@ -18,15 +18,20 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
18
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
19
19
  Classifier: Typing :: Typed
20
20
  Requires-Python: >=3.12
21
+ Requires-Dist: numpy>=1.26
21
22
  Requires-Dist: pywin32==311; sys_platform == 'win32'
22
23
  Requires-Dist: rank-bm25<0.3,>=0.2.2
23
24
  Requires-Dist: tiktoken<0.15,>=0.14
25
+ Provides-Extra: fastembed
26
+ Requires-Dist: fastembed<0.10,>=0.9; extra == 'fastembed'
24
27
  Provides-Extra: langchain
25
28
  Requires-Dist: langchain-core<2,>=1.0; extra == 'langchain'
26
29
  Provides-Extra: provider
27
30
  Requires-Dist: httpx<0.29,>=0.28; extra == 'provider'
28
31
  Provides-Extra: reports
29
32
  Requires-Dist: matplotlib<3.12,>=3.11; extra == 'reports'
33
+ Provides-Extra: semantic
34
+ Requires-Dist: model2vec<0.11,>=0.10; extra == 'semantic'
30
35
  Provides-Extra: service
31
36
  Requires-Dist: fastapi<0.136,>=0.135; extra == 'service'
32
37
  Requires-Dist: httpx<0.29,>=0.28; extra == 'service'
@@ -45,7 +50,7 @@ that fits the budget. Those messages keep the recent turns, the facts you pinned
45
50
  and the older evidence that matches the question. It never calls a model by
46
51
  itself, so you send the messages to any provider you like.
47
52
 
48
- > **Status: alpha (0.12.0).** Good for developers to try locally. It is not yet
53
+ > **Status: alpha (0.14.0).** Good for developers to try locally. It is not yet
49
54
  > proven on real-world data, and there is no hosted service. See
50
55
  > [What is not proven yet](#what-is-not-proven-yet).
51
56
 
@@ -106,6 +111,23 @@ chat_model.invoke(langchain_messages(result)) # pip install 'context-engineerin
106
111
  history = turns_from_openai(existing_openai_messages, scope) # reuse a chat history you already have
107
112
  ```
108
113
 
114
+ ### Better recall for reworded questions (optional)
115
+
116
+ Keyword search needs the question to reuse the fact's words. Add a small local
117
+ embedding model so meaning matches too ("retailer" finds "merchant"):
118
+
119
+ ```python
120
+ # pip install 'context-engineering-core[semantic]' (~30 MB model, downloaded once)
121
+ from context_engine.embeddings import Model2VecEmbedder
122
+
123
+ embedder = Model2VecEmbedder() # pinned minishlab/potion-base-8M; loads only when you create it
124
+ result = assemble_context(history, question, pins, budget, system=system, embedder=embedder)
125
+ ```
126
+
127
+ Ranking becomes 0.7 x meaning score + 0.3 x keyword score. Prefer accuracy over
128
+ speed? `FastEmbedEmbedder("BAAI/bge-small-en-v1.5")` (extra `[fastembed]`). Or wrap
129
+ your own embedding function or API with `FunctionEmbedder`.
130
+
109
131
  ### Use it from an MCP client
110
132
 
111
133
  `context-engine-mcp --data-dir ./context-memory` runs a local MCP server (stdio) with
@@ -154,11 +176,15 @@ These are enforced in code and covered by about 1,400 tests:
154
176
  - **Answer quality.** Tests measure whether the needed facts *reach* the model's
155
177
  context, not whether the model then answers correctly. On the built-in
156
178
  benchmark, the default settings keep 31 of 31 planted facts at 900 and 3000
157
- tokens. But that benchmark is one synthetic conversation written alongside
158
- the engine.
159
- - **Real-world data.** There are no results yet on real user conversations or
160
- on questions phrased differently from the source text. Retrieval is
161
- keyword-based (BM25), so heavily reworded questions may miss evidence.
179
+ tokens, and a live model run answered 31 of 31 correctly at 900 tokens. But
180
+ that benchmark is one synthetic conversation written alongside the engine.
181
+ - **Reworded questions.** By default, retrieval is keyword-based (BM25). When we
182
+ reword the 31 benchmark questions so they share no words with the stored facts,
183
+ the default settings keep only 3 of 31 facts. With the optional embedder (see
184
+ above) they keep 21 of 31 (retrieval alone: 24–25 of 31), with no loss on the
185
+ original questions. Run it yourself:
186
+ `context-engine strategy-eval --questions paraphrased --semantic`.
187
+ - **Real-world data.** There are no results yet on real user conversations.
162
188
  - **Exact provider token counts.** Budgets use a local tokenizer estimate.
163
189
  Providers may count differently. On small requests, provider counts were up
164
190
  to 1.78× the local estimate in past tests, so leave headroom or calibrate.
@@ -9,7 +9,7 @@ that fits the budget. Those messages keep the recent turns, the facts you pinned
9
9
  and the older evidence that matches the question. It never calls a model by
10
10
  itself, so you send the messages to any provider you like.
11
11
 
12
- > **Status: alpha (0.12.0).** Good for developers to try locally. It is not yet
12
+ > **Status: alpha (0.14.0).** Good for developers to try locally. It is not yet
13
13
  > proven on real-world data, and there is no hosted service. See
14
14
  > [What is not proven yet](#what-is-not-proven-yet).
15
15
 
@@ -70,6 +70,23 @@ chat_model.invoke(langchain_messages(result)) # pip install 'context-engineerin
70
70
  history = turns_from_openai(existing_openai_messages, scope) # reuse a chat history you already have
71
71
  ```
72
72
 
73
+ ### Better recall for reworded questions (optional)
74
+
75
+ Keyword search needs the question to reuse the fact's words. Add a small local
76
+ embedding model so meaning matches too ("retailer" finds "merchant"):
77
+
78
+ ```python
79
+ # pip install 'context-engineering-core[semantic]' (~30 MB model, downloaded once)
80
+ from context_engine.embeddings import Model2VecEmbedder
81
+
82
+ embedder = Model2VecEmbedder() # pinned minishlab/potion-base-8M; loads only when you create it
83
+ result = assemble_context(history, question, pins, budget, system=system, embedder=embedder)
84
+ ```
85
+
86
+ Ranking becomes 0.7 x meaning score + 0.3 x keyword score. Prefer accuracy over
87
+ speed? `FastEmbedEmbedder("BAAI/bge-small-en-v1.5")` (extra `[fastembed]`). Or wrap
88
+ your own embedding function or API with `FunctionEmbedder`.
89
+
73
90
  ### Use it from an MCP client
74
91
 
75
92
  `context-engine-mcp --data-dir ./context-memory` runs a local MCP server (stdio) with
@@ -118,11 +135,15 @@ These are enforced in code and covered by about 1,400 tests:
118
135
  - **Answer quality.** Tests measure whether the needed facts *reach* the model's
119
136
  context, not whether the model then answers correctly. On the built-in
120
137
  benchmark, the default settings keep 31 of 31 planted facts at 900 and 3000
121
- tokens. But that benchmark is one synthetic conversation written alongside
122
- the engine.
123
- - **Real-world data.** There are no results yet on real user conversations or
124
- on questions phrased differently from the source text. Retrieval is
125
- keyword-based (BM25), so heavily reworded questions may miss evidence.
138
+ tokens, and a live model run answered 31 of 31 correctly at 900 tokens. But
139
+ that benchmark is one synthetic conversation written alongside the engine.
140
+ - **Reworded questions.** By default, retrieval is keyword-based (BM25). When we
141
+ reword the 31 benchmark questions so they share no words with the stored facts,
142
+ the default settings keep only 3 of 31 facts. With the optional embedder (see
143
+ above) they keep 21 of 31 (retrieval alone: 24–25 of 31), with no loss on the
144
+ original questions. Run it yourself:
145
+ `context-engine strategy-eval --questions paraphrased --semantic`.
146
+ - **Real-world data.** There are no results yet on real user conversations.
126
147
  - **Exact provider token counts.** Budgets use a local tokenizer estimate.
127
148
  Providers may count differently. On small requests, provider counts were up
128
149
  to 1.78× the local estimate in past tests, so leave headroom or calibrate.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "context-engineering-core"
7
- version = "0.12.1"
7
+ version = "0.14.0"
8
8
  description = "Provider-independent context layers, source contracts, and token budget accounting"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -21,11 +21,13 @@ classifiers = [
21
21
  "Topic :: Software Development :: Libraries :: Python Modules",
22
22
  "Typing :: Typed",
23
23
  ]
24
- dependencies = ["tiktoken>=0.14,<0.15", "rank-bm25>=0.2.2,<0.3", "pywin32==311; sys_platform == 'win32'"]
24
+ dependencies = ["tiktoken>=0.14,<0.15", "rank-bm25>=0.2.2,<0.3", "numpy>=1.26", "pywin32==311; sys_platform == 'win32'"]
25
25
 
26
26
  [project.optional-dependencies]
27
27
  provider = ["httpx>=0.28,<0.29"]
28
28
  langchain = ["langchain-core>=1.0,<2"]
29
+ semantic = ["model2vec>=0.10,<0.11"]
30
+ fastembed = ["fastembed>=0.9,<0.10"]
29
31
  reports = ["matplotlib>=3.11,<3.12"]
30
32
  service = ["fastapi>=0.135,<0.136", "uvicorn>=0.41,<0.42", "PyJWT[crypto]>=2.12,<3", "httpx>=0.28,<0.29"]
31
33
 
@@ -1,6 +1,6 @@
1
1
  """Core contracts. Importing this package performs no configuration or network I/O."""
2
2
 
3
- __version__ = "0.12.1"
3
+ __version__ = "0.14.0"
4
4
 
5
5
 
6
6
  def __getattr__(name):
@@ -81,6 +81,17 @@ def main(argv: list[str] | None = None) -> int:
81
81
  "--budget", type=int, action="append", help="Input allowance; repeatable"
82
82
  )
83
83
  strategies.add_argument("--probes", action="store_true", help="Include per-fact results")
84
+ strategies.add_argument(
85
+ "--questions",
86
+ choices=("original", "paraphrased"),
87
+ default="original",
88
+ help="paraphrased: reworded questions with no shared words (lexical stress test)",
89
+ )
90
+ strategies.add_argument(
91
+ "--semantic",
92
+ action="store_true",
93
+ help="Also run hybrid strategies with the default embedder (needs the [semantic] extra)",
94
+ )
84
95
  strategies.add_argument("--data-dir", type=Path)
85
96
  strategies.add_argument("--output", type=Path, help="Save JSON to a new file; never overwrite")
86
97
  planned = commands.add_parser(
@@ -136,7 +147,18 @@ def main(argv: list[str] | None = None) -> int:
136
147
  budgets = None if args.budget is None else tuple(args.budget)
137
148
  if budgets is not None and any(b <= 0 for b in budgets):
138
149
  raise ContextEngineError("Budgets must be positive integers")
139
- result = compare_strategies(bundle, budgets, include_probes=args.probes)
150
+ embedder = None
151
+ if args.semantic:
152
+ from .embeddings import Model2VecEmbedder
153
+
154
+ embedder = Model2VecEmbedder()
155
+ result = compare_strategies(
156
+ bundle,
157
+ budgets,
158
+ include_probes=args.probes,
159
+ questions=args.questions,
160
+ embedder=embedder,
161
+ )
140
162
  if args.output is not None:
141
163
  save_inspection(args.output, result)
142
164
  print(json.dumps(result, indent=2, ensure_ascii=False))
@@ -216,6 +238,9 @@ def main(argv: list[str] | None = None) -> int:
216
238
  except ContextEngineError as error:
217
239
  print(json.dumps(error.to_dict()), file=sys.stderr)
218
240
  return 1
241
+ except ImportError as error: # An optional extra (e.g. [semantic]) is missing.
242
+ print(json.dumps({"code": "missing_extra", "message": str(error)}), file=sys.stderr)
243
+ return 2
219
244
 
220
245
 
221
246
  if __name__ == "__main__":
@@ -0,0 +1,189 @@
1
+ """Optional semantic retrieval: a small embedder interface and ready-made adapters.
2
+
3
+ Keyword (BM25) retrieval needs the question to reuse the evidence's words. An
4
+ embedder maps text to meaning vectors so "retailer" can find "merchant". Pass one to
5
+ `assemble_context(..., embedder=...)` (or `MemoryStore.assemble`) to rank chunks by
6
+ `semantic_weight` x meaning score + (1 - weight) x keyword score.
7
+
8
+ Models load only when you construct an embedder; assembly never downloads anything.
9
+
10
+ from context_engine.embeddings import Model2VecEmbedder
11
+ embedder = Model2VecEmbedder() # pip install 'context-engineering-core[semantic]'
12
+ """
13
+
14
+ from collections import OrderedDict
15
+ from collections.abc import Callable, Sequence
16
+ from pathlib import Path
17
+ from typing import Protocol, runtime_checkable
18
+
19
+ import numpy as np
20
+
21
+ from .errors import ContractError
22
+
23
+ # minishlab/potion-base-8M (MIT): chosen by measurement on the reworded-question
24
+ # benchmark (docs/BENCHMARK_REPORT.md). Pinned for reproducible vectors.
25
+ DEFAULT_MODEL = "minishlab/potion-base-8M"
26
+ DEFAULT_REVISION = "bf8b056651a2c21b8d2565580b8569da283cab23"
27
+ DEFAULT_SEMANTIC_WEIGHT = 0.7
28
+ _CACHE_SIZE = 16384
29
+ _MAX_TEXT = 100_000
30
+
31
+
32
+ @runtime_checkable
33
+ class Embedder(Protocol):
34
+ """`name` must change whenever vectors would change (model, revision, settings)."""
35
+
36
+ name: str
37
+
38
+ def embed_documents(self, texts: Sequence[str]) -> Sequence[Sequence[float]]: ...
39
+
40
+ def embed_query(self, text: str) -> Sequence[float]: ...
41
+
42
+
43
+ class FunctionEmbedder:
44
+ """Wrap your own functions, e.g. a hosted embedding API you call and pay for."""
45
+
46
+ def __init__(
47
+ self,
48
+ name: str,
49
+ documents: Callable[[list[str]], Sequence[Sequence[float]]],
50
+ query: Callable[[str], Sequence[float]] | None = None,
51
+ ):
52
+ if not isinstance(name, str) or not name.strip():
53
+ raise ContractError("Embedder name is required")
54
+ self.name = name
55
+ self._documents = documents
56
+ self._query = query or (lambda text: documents([text])[0])
57
+
58
+ def embed_documents(self, texts):
59
+ return self._documents(list(texts))
60
+
61
+ def embed_query(self, text):
62
+ return self._query(text)
63
+
64
+
65
+ class Model2VecEmbedder:
66
+ """Static embeddings (Model2Vec); ~30 MB model, fast on CPU, no ONNX runtime."""
67
+
68
+ def __init__(self, model: str = DEFAULT_MODEL, *, revision: str | None = DEFAULT_REVISION):
69
+ """Downloads (once, then cached) the pinned model revision; a local path also works."""
70
+ try:
71
+ from huggingface_hub import snapshot_download
72
+ from model2vec import StaticModel
73
+ except ImportError:
74
+ raise ImportError(
75
+ "Model2VecEmbedder needs model2vec: "
76
+ "pip install 'context-engineering-core[semantic]'"
77
+ ) from None
78
+ local = Path(model).is_dir()
79
+ self.name = f"model2vec:{model}@{'local' if local else revision or 'latest'}"
80
+ path = (
81
+ model
82
+ if local
83
+ else snapshot_download(
84
+ model, revision=revision, allow_patterns=["*.json", "*.safetensors"]
85
+ )
86
+ )
87
+ self._model = StaticModel.from_pretrained(path, force_download=False)
88
+
89
+ def embed_documents(self, texts):
90
+ return self._model.encode(list(texts))
91
+
92
+ def embed_query(self, text):
93
+ return self._model.encode([text])[0]
94
+
95
+
96
+ class FastEmbedEmbedder:
97
+ """ONNX transformer embeddings via fastembed (e.g. BAAI/bge-small-en-v1.5).
98
+
99
+ More accurate on the reworded benchmark but much slower to index than the default.
100
+ """
101
+
102
+ def __init__(self, model: str = "BAAI/bge-small-en-v1.5", **options):
103
+ try:
104
+ from fastembed import TextEmbedding
105
+ except ImportError:
106
+ raise ImportError(
107
+ "FastEmbedEmbedder needs fastembed: "
108
+ "pip install 'context-engineering-core[fastembed]'"
109
+ ) from None
110
+ self.name = f"fastembed:{model}"
111
+ self._model = TextEmbedding(model, **options)
112
+
113
+ def embed_documents(self, texts):
114
+ return list(self._model.passage_embed(list(texts)))
115
+
116
+ def embed_query(self, text):
117
+ return next(iter(self._model.query_embed([text])))
118
+
119
+
120
+ def _unit(vectors, dimension: int | None = None) -> np.ndarray:
121
+ array = np.asarray(vectors, dtype=np.float32)
122
+ if array.ndim != 2 or array.shape[1] == 0 or not np.all(np.isfinite(array)):
123
+ raise ContractError("Embedder returned invalid vectors")
124
+ if dimension is not None and array.shape[1] != dimension:
125
+ raise ContractError("Embedder vector dimensions differ")
126
+ norms = np.linalg.norm(array, axis=1, keepdims=True)
127
+ return np.divide(array, norms, out=np.zeros_like(array), where=norms > 0)
128
+
129
+
130
+ # Process-wide cache of unit document vectors keyed by (embedder name, text): a chat
131
+ # loop re-embeds the same history on every call otherwise.
132
+ _VECTORS: OrderedDict[tuple[str, str], np.ndarray] = OrderedDict()
133
+
134
+
135
+ def document_vectors(embedder: Embedder, texts: Sequence[str]) -> np.ndarray:
136
+ keys = [(embedder.name, text) for text in texts]
137
+ missing = list(dict.fromkeys(t for k, t in zip(keys, texts, strict=True) if k not in _VECTORS))
138
+ if missing:
139
+ if any(len(t) > _MAX_TEXT for t in missing):
140
+ raise ContractError("Text too long to embed")
141
+ fresh = _unit(embedder.embed_documents(missing))
142
+ if len(fresh) != len(missing):
143
+ raise ContractError("Embedder returned the wrong number of vectors")
144
+ for text, vector in zip(missing, fresh, strict=True):
145
+ _VECTORS[(embedder.name, text)] = vector
146
+ _VECTORS.move_to_end((embedder.name, text))
147
+ result = np.stack([_VECTORS[k] for k in keys]) if keys else np.zeros((0, 1), np.float32)
148
+ for key in keys:
149
+ _VECTORS.move_to_end(key)
150
+ while len(_VECTORS) > _CACHE_SIZE:
151
+ _VECTORS.popitem(last=False)
152
+ return result
153
+
154
+
155
+ def query_vector(embedder: Embedder, text: str, dimension: int) -> np.ndarray:
156
+ return _unit([embedder.embed_query(text)], dimension)[0]
157
+
158
+
159
+ def _scaled(values: np.ndarray) -> np.ndarray:
160
+ low, high = float(values.min()), float(values.max())
161
+ return np.zeros_like(values) if high <= low else (values - low) / (high - low)
162
+
163
+
164
+ def hybrid_scores(
165
+ semantic: np.ndarray, lexical: np.ndarray, weight: float = DEFAULT_SEMANTIC_WEIGHT
166
+ ) -> np.ndarray:
167
+ """Per-query min-max scaled fusion: weight x meaning + (1 - weight) x keyword."""
168
+ return weight * _scaled(np.asarray(semantic, np.float64)) + (1 - weight) * _scaled(
169
+ np.asarray(lexical, np.float64)
170
+ )
171
+
172
+
173
+ def validate_semantic(embedder, weight) -> None:
174
+ if not isinstance(embedder, Embedder) or not isinstance(getattr(embedder, "name", None), str):
175
+ raise ContractError("embedder must provide name, embed_documents and embed_query")
176
+ if type(weight) not in (int, float) or not 0 <= weight <= 1:
177
+ raise ContractError("semantic_weight must be between 0 and 1")
178
+
179
+
180
+ __all__ = [
181
+ "DEFAULT_MODEL",
182
+ "DEFAULT_REVISION",
183
+ "DEFAULT_SEMANTIC_WEIGHT",
184
+ "Embedder",
185
+ "FastEmbedEmbedder",
186
+ "FunctionEmbedder",
187
+ "Model2VecEmbedder",
188
+ "hybrid_scores",
189
+ ]
@@ -7,7 +7,7 @@
7
7
  "summary_fixture": "b3c1b76bb330b52493e5150647747bad77d5de3a55d05d032bb5703be1c70196"
8
8
  },
9
9
  "runtime": {
10
- "code_hash": "c3d4bfbdad58327eb09a763ae398fd97277e33339ce00cfb5395b2615f2d269f",
10
+ "code_hash": "9a6b1e2584db1f98a58011dc680ea7dc00f7a06d648831d1d39c46de32735976",
11
11
  "dependencies": {
12
12
  "certifi": "2026.10.23",
13
13
  "charset-normalizer": "3.5.1",
@@ -19,7 +19,7 @@
19
19
  "tiktoken": "0.14.0",
20
20
  "urllib3": "2.8.0"
21
21
  },
22
- "package_version": "0.12.1",
22
+ "package_version": "0.14.0",
23
23
  "python": "3.12.13"
24
24
  },
25
25
  "schema_version": 1
@@ -0,0 +1,38 @@
1
+ {
2
+ "schema_version": 1,
3
+ "scenario_id": "incident-100-v1",
4
+ "rule": "Each question asks for the same fact as the original but shares no content word (and, for words of 5+ letters, no 4-letter word start) with the fact's evidence sentence or answer value. Written by the engine author; not independently reviewed.",
5
+ "questions": {
6
+ "merchant": "Which retailer or seller took part?",
7
+ "ticket": "What case number was opened for this outage?",
8
+ "service": "Which component directed card purchases to processors?",
9
+ "owner": "Which group was responsible for fixing the problem?",
10
+ "database": "Where were purchase records persisted?",
11
+ "queue": "Which channel transported the clearing notifications?",
12
+ "rollback": "Which version did they revert to?",
13
+ "runbook": "Which written procedure did the on-call staff follow?",
14
+ "cluster": "On which group of machines did the job run?",
15
+ "bucket": "Where were the compliance downloads kept in object storage?",
16
+ "region": "In which geographic location did the impacted rollout run?",
17
+ "canary": "Which early-warning probe spotted the performance drop?",
18
+ "dashboard": "Which monitoring screen showed response delay?",
19
+ "certificate": "Which TLS credential needed to be swapped out?",
20
+ "feature": "Which toggle got switched off?",
21
+ "cache": "Which in-memory store got cleared?",
22
+ "trace": "What correlation ID tied the broken calls together?",
23
+ "replica": "Which copy of the data returned outdated results?",
24
+ "schema": "Which data-model revision caused the incompatibility?",
25
+ "retries": "How many repeat attempts were set up?",
26
+ "timeout": "What wait limit was used before giving up?",
27
+ "region_backup": "Which location was kept on standby for disaster scenarios?",
28
+ "policy": "Which permission rule got changed?",
29
+ "namespace": "Which isolation scope held the background jobs?",
30
+ "endpoint": "Where was the rerouted load sent?",
31
+ "checksum": "Which hash confirmed the restored backup was intact?",
32
+ "shard": "Which data segment was found to be at fault?",
33
+ "batch": "Which group of jobs was run again once things were restored?",
34
+ "approval": "Which sign-off permitted rebooting the system?",
35
+ "handoff": "Which transfer note wrapped up the inquiry?",
36
+ "archive": "Where was the last proof kept long-term?"
37
+ }
38
+ }