somtum 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/LICENSE +663 -0
  2. package/README.md +459 -0
  3. package/dist/cli/config_cmd.d.ts +10 -0
  4. package/dist/cli/config_cmd.d.ts.map +1 -0
  5. package/dist/cli/config_cmd.js +96 -0
  6. package/dist/cli/config_cmd.js.map +1 -0
  7. package/dist/cli/doctor.d.ts +17 -0
  8. package/dist/cli/doctor.d.ts.map +1 -0
  9. package/dist/cli/doctor.js +169 -0
  10. package/dist/cli/doctor.js.map +1 -0
  11. package/dist/cli/edit.d.ts +4 -0
  12. package/dist/cli/edit.d.ts.map +1 -0
  13. package/dist/cli/edit.js +63 -0
  14. package/dist/cli/edit.js.map +1 -0
  15. package/dist/cli/export.d.ts +19 -0
  16. package/dist/cli/export.d.ts.map +1 -0
  17. package/dist/cli/export.js +84 -0
  18. package/dist/cli/export.js.map +1 -0
  19. package/dist/cli/forget.d.ts +11 -0
  20. package/dist/cli/forget.d.ts.map +1 -0
  21. package/dist/cli/forget.js +32 -0
  22. package/dist/cli/forget.js.map +1 -0
  23. package/dist/cli/hook.d.ts +2 -0
  24. package/dist/cli/hook.d.ts.map +1 -0
  25. package/dist/cli/hook.js +66 -0
  26. package/dist/cli/hook.js.map +1 -0
  27. package/dist/cli/import.d.ts +18 -0
  28. package/dist/cli/import.d.ts.map +1 -0
  29. package/dist/cli/import.js +90 -0
  30. package/dist/cli/import.js.map +1 -0
  31. package/dist/cli/index.d.ts +3 -0
  32. package/dist/cli/index.d.ts.map +1 -0
  33. package/dist/cli/index.js +223 -0
  34. package/dist/cli/index.js.map +1 -0
  35. package/dist/cli/init.d.ts +24 -0
  36. package/dist/cli/init.d.ts.map +1 -0
  37. package/dist/cli/init.js +144 -0
  38. package/dist/cli/init.js.map +1 -0
  39. package/dist/cli/purge.d.ts +16 -0
  40. package/dist/cli/purge.d.ts.map +1 -0
  41. package/dist/cli/purge.js +69 -0
  42. package/dist/cli/purge.js.map +1 -0
  43. package/dist/cli/rebuild.d.ts +15 -0
  44. package/dist/cli/rebuild.d.ts.map +1 -0
  45. package/dist/cli/rebuild.js +41 -0
  46. package/dist/cli/rebuild.js.map +1 -0
  47. package/dist/cli/reindex.d.ts +6 -0
  48. package/dist/cli/reindex.d.ts.map +1 -0
  49. package/dist/cli/reindex.js +47 -0
  50. package/dist/cli/reindex.js.map +1 -0
  51. package/dist/cli/search.d.ts +16 -0
  52. package/dist/cli/search.d.ts.map +1 -0
  53. package/dist/cli/search.js +62 -0
  54. package/dist/cli/search.js.map +1 -0
  55. package/dist/cli/show.d.ts +12 -0
  56. package/dist/cli/show.d.ts.map +1 -0
  57. package/dist/cli/show.js +49 -0
  58. package/dist/cli/show.js.map +1 -0
  59. package/dist/cli/stats.d.ts +27 -0
  60. package/dist/cli/stats.d.ts.map +1 -0
  61. package/dist/cli/stats.js +80 -0
  62. package/dist/cli/stats.js.map +1 -0
  63. package/dist/cli/sync.d.ts +18 -0
  64. package/dist/cli/sync.d.ts.map +1 -0
  65. package/dist/cli/sync.js +138 -0
  66. package/dist/cli/sync.js.map +1 -0
  67. package/dist/config.d.ts +12 -0
  68. package/dist/config.d.ts.map +1 -0
  69. package/dist/config.js +52 -0
  70. package/dist/config.js.map +1 -0
  71. package/dist/core/cache.d.ts +29 -0
  72. package/dist/core/cache.d.ts.map +1 -0
  73. package/dist/core/cache.js +137 -0
  74. package/dist/core/cache.js.map +1 -0
  75. package/dist/core/db.d.ts +12 -0
  76. package/dist/core/db.d.ts.map +1 -0
  77. package/dist/core/db.js +80 -0
  78. package/dist/core/db.js.map +1 -0
  79. package/dist/core/embeddings.d.ts +15 -0
  80. package/dist/core/embeddings.d.ts.map +1 -0
  81. package/dist/core/embeddings.js +74 -0
  82. package/dist/core/embeddings.js.map +1 -0
  83. package/dist/core/embeddings_bootstrap.d.ts +3 -0
  84. package/dist/core/embeddings_bootstrap.d.ts.map +1 -0
  85. package/dist/core/embeddings_bootstrap.js +16 -0
  86. package/dist/core/embeddings_bootstrap.js.map +1 -0
  87. package/dist/core/embeddings_xenova.d.ts +3 -0
  88. package/dist/core/embeddings_xenova.d.ts.map +1 -0
  89. package/dist/core/embeddings_xenova.js +28 -0
  90. package/dist/core/embeddings_xenova.js.map +1 -0
  91. package/dist/core/extractor.d.ts +27 -0
  92. package/dist/core/extractor.d.ts.map +1 -0
  93. package/dist/core/extractor.js +133 -0
  94. package/dist/core/extractor.js.map +1 -0
  95. package/dist/core/file_summary.d.ts +33 -0
  96. package/dist/core/file_summary.d.ts.map +1 -0
  97. package/dist/core/file_summary.js +127 -0
  98. package/dist/core/file_summary.js.map +1 -0
  99. package/dist/core/fingerprint.d.ts +13 -0
  100. package/dist/core/fingerprint.d.ts.map +1 -0
  101. package/dist/core/fingerprint.js +36 -0
  102. package/dist/core/fingerprint.js.map +1 -0
  103. package/dist/core/index_gen.d.ts +16 -0
  104. package/dist/core/index_gen.d.ts.map +1 -0
  105. package/dist/core/index_gen.js +173 -0
  106. package/dist/core/index_gen.js.map +1 -0
  107. package/dist/core/memory_files.d.ts +4 -0
  108. package/dist/core/memory_files.d.ts.map +1 -0
  109. package/dist/core/memory_files.js +36 -0
  110. package/dist/core/memory_files.js.map +1 -0
  111. package/dist/core/privacy.d.ts +9 -0
  112. package/dist/core/privacy.d.ts.map +1 -0
  113. package/dist/core/privacy.js +32 -0
  114. package/dist/core/privacy.js.map +1 -0
  115. package/dist/core/project_id.d.ts +3 -0
  116. package/dist/core/project_id.d.ts.map +1 -0
  117. package/dist/core/project_id.js +31 -0
  118. package/dist/core/project_id.js.map +1 -0
  119. package/dist/core/reindex.d.ts +10 -0
  120. package/dist/core/reindex.d.ts.map +1 -0
  121. package/dist/core/reindex.js +40 -0
  122. package/dist/core/reindex.js.map +1 -0
  123. package/dist/core/retrieval_stats.d.ts +22 -0
  124. package/dist/core/retrieval_stats.d.ts.map +1 -0
  125. package/dist/core/retrieval_stats.js +56 -0
  126. package/dist/core/retrieval_stats.js.map +1 -0
  127. package/dist/core/retriever/bm25.d.ts +15 -0
  128. package/dist/core/retriever/bm25.d.ts.map +1 -0
  129. package/dist/core/retriever/bm25.js +51 -0
  130. package/dist/core/retriever/bm25.js.map +1 -0
  131. package/dist/core/retriever/embeddings.d.ts +9 -0
  132. package/dist/core/retriever/embeddings.d.ts.map +1 -0
  133. package/dist/core/retriever/embeddings.js +34 -0
  134. package/dist/core/retriever/embeddings.js.map +1 -0
  135. package/dist/core/retriever/factory.d.ts +6 -0
  136. package/dist/core/retriever/factory.d.ts.map +1 -0
  137. package/dist/core/retriever/factory.js +53 -0
  138. package/dist/core/retriever/factory.js.map +1 -0
  139. package/dist/core/retriever/hybrid.d.ts +14 -0
  140. package/dist/core/retriever/hybrid.d.ts.map +1 -0
  141. package/dist/core/retriever/hybrid.js +52 -0
  142. package/dist/core/retriever/hybrid.js.map +1 -0
  143. package/dist/core/retriever/llm_index.d.ts +18 -0
  144. package/dist/core/retriever/llm_index.d.ts.map +1 -0
  145. package/dist/core/retriever/llm_index.js +90 -0
  146. package/dist/core/retriever/llm_index.js.map +1 -0
  147. package/dist/core/retriever/types.d.ts +16 -0
  148. package/dist/core/retriever/types.d.ts.map +1 -0
  149. package/dist/core/retriever/types.js +2 -0
  150. package/dist/core/retriever/types.js.map +1 -0
  151. package/dist/core/schema.d.ts +470 -0
  152. package/dist/core/schema.d.ts.map +1 -0
  153. package/dist/core/schema.js +153 -0
  154. package/dist/core/schema.js.map +1 -0
  155. package/dist/core/store.d.ts +35 -0
  156. package/dist/core/store.d.ts.map +1 -0
  157. package/dist/core/store.js +172 -0
  158. package/dist/core/store.js.map +1 -0
  159. package/dist/core/tokens.d.ts +3 -0
  160. package/dist/core/tokens.d.ts.map +1 -0
  161. package/dist/core/tokens.js +12 -0
  162. package/dist/core/tokens.js.map +1 -0
  163. package/dist/core/transcript.d.ts +16 -0
  164. package/dist/core/transcript.d.ts.map +1 -0
  165. package/dist/core/transcript.js +128 -0
  166. package/dist/core/transcript.js.map +1 -0
  167. package/dist/db/migrations/001_init.sql +90 -0
  168. package/dist/db/migrations/002_cache_files.sql +6 -0
  169. package/dist/db/migrations/003_retrieval_stats.sql +17 -0
  170. package/dist/hooks/post_session.d.ts +65 -0
  171. package/dist/hooks/post_session.d.ts.map +1 -0
  172. package/dist/hooks/post_session.js +249 -0
  173. package/dist/hooks/post_session.js.map +1 -0
  174. package/dist/hooks/pre_prompt.d.ts +55 -0
  175. package/dist/hooks/pre_prompt.d.ts.map +1 -0
  176. package/dist/hooks/pre_prompt.js +98 -0
  177. package/dist/hooks/pre_prompt.js.map +1 -0
  178. package/dist/hooks/pre_read.d.ts +56 -0
  179. package/dist/hooks/pre_read.d.ts.map +1 -0
  180. package/dist/hooks/pre_read.js +77 -0
  181. package/dist/hooks/pre_read.js.map +1 -0
  182. package/dist/index.d.ts +15 -0
  183. package/dist/index.d.ts.map +1 -0
  184. package/dist/index.js +15 -0
  185. package/dist/index.js.map +1 -0
  186. package/dist/mcp/server.d.ts +20 -0
  187. package/dist/mcp/server.d.ts.map +1 -0
  188. package/dist/mcp/server.js +128 -0
  189. package/dist/mcp/server.js.map +1 -0
  190. package/dist/mcp/tools.d.ts +72 -0
  191. package/dist/mcp/tools.d.ts.map +1 -0
  192. package/dist/mcp/tools.js +153 -0
  193. package/dist/mcp/tools.js.map +1 -0
  194. package/package.json +79 -0
package/README.md ADDED
@@ -0,0 +1,459 @@
1
+ <img src="./assets/logo.png" width="500" alt="Somtum Logo">
2
+
3
+ **Local-first memory and prompt-cache layer for Claude Code.**
4
+
5
+ [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
6
+ [![npm version](https://badge.fury.io/js/somtum.svg)](https://www.npmjs.com/package/somtum)
7
+
8
+ Somtum captures durable observations from your Claude Code sessions — decisions, bugfixes, learnings, file summaries — stores them in a local SQLite database, and injects the relevant ones back into context the next time you need them. It also caches repeated prompt→response pairs so identical (or near-identical) prompts never hit the model twice.
9
+
10
+ Zero-config: one `somtum init` in an existing Claude Code project and every session end is captured automatically. No server, no cloud account, no mandatory tuning.
11
+
12
+ ---
13
+
14
+ ## Table of Contents
15
+ - [Why Somtum?](#why-somtum)
16
+ - [How it works](#how-it-works)
17
+ - [Requirements](#requirements)
18
+ - [Install](#install)
19
+ - [Quickstart](#quickstart)
20
+ - [CLI Reference](#cli-reference)
21
+ - [MCP Server](#mcp-server)
22
+ - [Storage Layout](#storage-layout)
23
+ - [Configuration](#configuration)
24
+ - [Privacy](#privacy)
25
+ - [Token Accounting](#token-accounting)
26
+ - [Performance](#performance)
27
+ - [Development](#development)
28
+ - [Troubleshooting](#troubleshooting)
29
+ - [License](#license)
30
+
31
+ ---
32
+
33
+ ## Why Somtum?
34
+
35
+ LLM agents like Claude Code typically start every session with a "blank slate." This leads to:
36
+ * **Repetitive Explanations:** Having to re-explain architectural choices or local conventions.
37
+ * **Regressions:** Claude might suggest a fix you've already tried and discarded.
38
+ * **Context Waste:** Large codebases eat up tokens just to "set the scene."
39
+
40
+ **Somtum gives Claude a long-term memory.** It ensures that once a decision is made or a bug is fixed, it stays "remembered" across all future sessions without bloating your context window.
41
+
42
+ ---
43
+
44
+ ## How it works
45
+
46
+ ```
47
+ ┌─────────────────────────────────────────────────────────────┐
48
+ │ Claude Code / Agent │
49
+ └──────────┬──────────────────────────────┬───────────────────┘
50
+ │ hooks │ MCP tools
51
+ ▼ ▼
52
+ ┌─────────────────────┐ ┌──────────────────────┐
53
+ │ Capture Pipeline │ │ Query Pipeline │
54
+ │ │ │ │
55
+ │ UserPromptSubmit ──┼─────────┼▶ cache_lookup │
56
+ │ SessionEnd ────────┼─────────┼▶ recall / get │
57
+ │ PreCompact ────────┼─────────┼▶ remember / forget │
58
+ │ PreToolUse (Read) ─┼─────────┼▶ stats │
59
+ └──────────┬──────────┘ └──────────┬───────────┘
60
+ │ │
61
+ ▼ ▼
62
+ ┌─────────────────────────────────────────────────────────────┐
63
+ │ Core (TypeScript) │
64
+ │ │
65
+ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │
66
+ │ │ PromptCache │ │ MemoryStore │ │ Retriever │ │
67
+ │ │ │ │ │ │ │ │
68
+ │ │ exact hash │ │ observations │ │ bm25 (FTS5) │ │
69
+ │ │ fuzzy embed │ │ + embeddings │ │ embeddings │ │
70
+ │ │ fingerprint │ │ + redaction │ │ index (Haiku) │ │
71
+ │ │ invalidation │ │ │ │ hybrid (default)│ │
72
+ │ └──────────────┘ └──────────────┘ └──────────────────┘ │
73
+ └─────────────────────────────────┬───────────────────────────┘
74
+ │
75
+ ▼
76
+ ┌─────────────────────────┐
77
+ │ SQLite WAL + ~/.somtum/ │
78
+ │ /projects/<hash>/ │
79
+ │ db.sqlite │
80
+ │ index.md │
81
+ │ memories/YYYY-MM/ │
82
+ │ <ulid>.md │
83
+ └─────────────────────────┘
84
+ ```
85
+
86
+ ### Three retrieval strategies — all first-class
87
+
88
+ | Strategy | How it works | Best for | Cost |
89
+ | ------------------------ | ---------------------------------------------------------------- | ---------------------------------------- | ----------------------------------- |
90
+ | **`bm25`** | SQLite FTS5 lexical search over title + body + tags | Keyword queries, offline-forever setups | Near-zero |
91
+ | **`embeddings`** | Cosine similarity over 384-dim vectors (bge-small-en-v1.5, ONNX) | Semantic queries, large corpora | ~30 MB model, ~5 ms at 10k memories |
92
+ | **`index`** | Compact catalog sent to Haiku; model picks relevant IDs | Paraphrased queries, zero-embedding mode | 1 Haiku call per query |
93
+ | **`hybrid`** _(default)_ | BM25 top-50 ∪ embeddings top-50 → RRF blend → Haiku rerank | General case | BM25 + embeddings + 1 Haiku call |
94
+
95
+ ---
96
+
97
+ ## Requirements
98
+
99
+ - **Node 20+**
100
+ - **pnpm** (`npm i -g pnpm`)
101
+ - **`ANTHROPIC_API_KEY`** — used by the capture hook to summarise session transcripts via Claude Haiku, and optionally by the `index` and `hybrid` retrieval strategies
102
+
103
+ ---
104
+
105
+ ## Install
106
+
107
+ ### From npm / yarn / pnpm
108
+
109
+ ```bash
110
+ # npm
111
+ npm install -g somtum
112
+
113
+ # yarn
114
+ yarn global add somtum
115
+
116
+ # pnpm
117
+ pnpm add -g somtum
118
+ ```
119
+
120
+ Or as a project dependency:
121
+
122
+ ```bash
123
+ npm install somtum
124
+ yarn add somtum
125
+ pnpm add somtum
126
+ ```
127
+
128
+ ### From source
129
+
130
+ ```bash
131
+ git clone https://github.com/riz007/somtum
132
+ cd somtum
133
+ pnpm install
134
+ pnpm build
135
+ # Optional: link globally
136
+ pnpm link --global
137
+ ```
138
+
139
+ ### Native module note
140
+
141
+ Somtum uses [`better-sqlite3`](https://github.com/WiseLibs/better-sqlite3), which contains a native C++ addon. On most platforms (macOS, Linux x64/arm64, Windows x64) a prebuilt binary is downloaded automatically. On Alpine Linux / musl or unusual architectures the addon compiles from source, which requires `python`, `make`, and `gcc` to be available. If the install fails with a node-gyp error, install those build tools and retry.
142
+
143
+ ---
144
+
145
+ ## Quickstart
146
+
147
+ ```bash
148
+ # 1. Inside an existing Claude Code project:
149
+ somtum init
150
+
151
+ # Installs a SessionEnd hook in .claude/settings.json.
152
+ # Use Claude Code as normal — observations are captured at session end.
153
+
154
+ # 2. Search your memory
155
+ somtum search "auth jwt rotation"
156
+ somtum search "why we use pnpm" --strategy hybrid
157
+
158
+ # 3. Read a full observation
159
+ somtum show 01JBZ...
160
+
161
+ # 4. Check your token savings
162
+ somtum stats
163
+
164
+ # 5. Diagnose any issues
165
+ somtum doctor
166
+ ```
167
+
168
+ ### Enable all features at once
169
+
170
+ ```bash
171
+ somtum init --all
172
+ # Installs:
173
+ # - SessionEnd capture hook
174
+ # - UserPromptSubmit prompt-cache hook
175
+ # - PreToolUse file-gating hook
176
+ # - MCP server registration in .mcp.json
177
+ ```
178
+
179
+ ---
180
+
181
+ ## CLI reference
182
+
183
+ ### Setup
184
+
185
+ | Command | Description |
186
+ | --------------------------- | --------------------------------------------------- |
187
+ | `somtum init` | Install the SessionEnd capture hook |
188
+ | `somtum init --cache` | Also install the UserPromptSubmit cache hook |
189
+ | `somtum init --file-gating` | Also install the PreToolUse file-gating hook |
190
+ | `somtum init --all` | Install all hooks + MCP server |
191
+ | `somtum init --force` | Reinstall even if hooks already present |
192
+ | `somtum doctor` | Check DB health, migrations, hooks, breakeven ratio |
193
+
194
+ ### Memory
195
+
196
+ | Command | Description |
197
+ | --------------------------------------- | ------------------------------------------------ |
198
+ | `somtum search <query>` | Search observations (default: `hybrid` strategy) |
199
+ | `somtum search <query> --strategy bm25` | Force a specific retrieval strategy |
200
+ | `somtum search <query> -k 16` | Return more results |
201
+ | `somtum show <id>` | Print the full body of an observation |
202
+ | `somtum forget <id>` | Soft-delete an observation |
203
+ | `somtum edit <id>` | Open an observation body in `$EDITOR` |
204
+ | `somtum rebuild` | Regenerate `index.md` from all observations |
205
+
206
+ ### Data management
207
+
208
+ | Command | Description |
209
+ | ------------------------------------------------- | --------------------------------------------------- |
210
+ | `somtum export` | Export observations to stdout as JSON |
211
+ | `somtum export --format jsonl --output obs.jsonl` | Export as JSONL file |
212
+ | `somtum export --format markdown` | Export as readable Markdown |
213
+ | `somtum export --include-deleted` | Include soft-deleted entries |
214
+ | `somtum import <file>` | Import observations from JSON or JSONL |
215
+ | `somtum purge --older-than 30d` | Hard-delete soft-deleted entries older than 30 days |
216
+ | `somtum purge --older-than 30d --dry-run` | Preview without deleting |
217
+
218
+ ### Embeddings
219
+
220
+ | Command | Description |
221
+ | ---------------- | ------------------------------------------- |
222
+ | `somtum reindex` | Embed observations that are missing vectors |
223
+
224
+ ### Stats
225
+
226
+ | Command | Description |
227
+ | --------------------- | ------------------------------------------------- |
228
+ | `somtum stats` | Tokens saved, cache hit rate, retrieval breakdown |
229
+ | `somtum stats --json` | Machine-readable JSON output |
230
+
231
+ ### Configuration
232
+
233
+ | Command | Description |
234
+ | -------------------------------------------------------------- | --------------------------------- |
235
+ | `somtum config get` | Print the full resolved config |
236
+ | `somtum config get retrieval.strategy` | Read a single key (dot-separated) |
237
+ | `somtum config set retrieval.strategy hybrid` | Write to `.somtum/config.json` |
238
+ | `somtum config set retrieval.embeddings.enabled true --global` | Write to `~/.somtum/config.json` |
239
+
240
+ ### Sync (M6)
241
+
242
+ | Command | Description |
243
+ | -------------------- | ----------------------------------------- |
244
+ | `somtum sync status` | Compare local vs remote observation count |
245
+ | `somtum sync push` | Export and scp observations to remote |
246
+ | `somtum sync pull` | scp from remote and merge into local DB |
247
+
248
+ Remote configured in config: `somtum config set sync.remote "user@host:/path/.somtum/projects/<id>"`
249
+
250
+ ---
251
+
252
+ ## MCP Server
253
+
254
+ Somtum includes a [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) server. When you run `somtum init --all`, it registers tools that Claude can use autonomously:
255
+
256
+ - `recall`: Claude can search your memories when it's unsure about a project detail.
257
+ - `get`: Retrieve the full body of specific observations.
258
+ - `remember`: Claude can manually store a specific observation.
259
+ - `cache_lookup`: Used internally to check the prompt cache.
260
+ - `forget`: Soft-delete an observation.
261
+ - `stats`: Claude can report on how much it has "learned" from you.
262
+
263
+ ---
264
+
265
+ ## Storage layout
266
+
267
+ ```
268
+ ~/.somtum/
269
+ ├── config.json ← global config (overridden by project config)
270
+ └── projects/
271
+ └── <project_id>/
272
+ ├── db.sqlite ← source of truth (SQLite WAL)
273
+ ├── index.md ← regenerated human-readable mirror
274
+ └── memories/
275
+ └── YYYY-MM/
276
+ └── <ulid>.md ← per-observation markdown files
277
+ ```
278
+
279
+ SQLite is the source of truth. `index.md` and `memories/*.md` are derived mirrors regenerated on each capture. Edit observations with `somtum edit <id>`, not by hand.
280
+
281
+ ---
282
+
283
+ ## Configuration
284
+
285
+ Global config lives at `~/.somtum/config.json`. Per-project config at `.somtum/config.json` — project values override global ones (deep merge).
286
+
287
+ ```jsonc
288
+ {
289
+ "extraction": {
290
+ "model": "claude-haiku-4-5-20251001",
291
+ "trigger": ["SessionEnd", "PreCompact"],
292
+ "max_observations_per_session": 10,
293
+ },
294
+ "cache": {
295
+ "enabled": true,
296
+ "fuzzy_match": true,
297
+ "fuzzy_threshold": 0.92, // conservative — raise to 0.95 once you have signal
298
+ "max_entries": 10000,
299
+ "ttl_days": 90,
300
+ },
301
+ "retrieval": {
302
+ "strategy": "hybrid", // bm25 | embeddings | index | hybrid
303
+ "k": 8,
304
+ "rerank_model": "claude-haiku-4-5-20251001",
305
+ "bm25": { "enabled": true },
306
+ "embeddings": {
307
+ "enabled": false, // set true to download the 30 MB ONNX model
308
+ "model": "Xenova/bge-small-en-v1.5",
309
+ },
310
+ "index": {
311
+ "enabled": false, // set true to use Haiku as the retriever
312
+ "model": "claude-haiku-4-5-20251001",
313
+ },
314
+ },
315
+ "file_gating": {
316
+ "enabled": false, // set true to intercept large file reads
317
+ "min_file_size_tokens": 500,
318
+ "exclude_globs": ["**/*.env", "**/secrets/**"],
319
+ },
320
+ "privacy": {
321
+ "telemetry": false,
322
+ "redact_patterns": [
323
+ "api[_-]?key\\s*[:=]\\s*[\"']?[A-Za-z0-9_\\-]{8,}[\"']?",
324
+ "bearer\\s+[A-Za-z0-9_\\-.]+",
325
+ "sk-[A-Za-z0-9_\\-]{20,}",
326
+ "xox[baprs]-[A-Za-z0-9-]{10,}",
327
+ "AKIA[0-9A-Z]{16}",
328
+ ],
329
+ },
330
+ "sync": {
331
+ "enabled": false,
332
+ "backend": "ssh",
333
+ "remote": null, // e.g. "user@host:/home/user/.somtum/projects/<id>"
334
+ },
335
+ }
336
+ ```
337
+
338
+ Enable embeddings in one command:
339
+
340
+ ```bash
341
+ somtum config set retrieval.embeddings.enabled true
342
+ somtum reindex # embed existing observations
343
+ ```
344
+
345
+ ---
346
+
347
+ ## Privacy
348
+
349
+ - **No network traffic** except to the Anthropic API (extraction + optional reranking). The embedding model runs fully local via ONNX Runtime in-process.
350
+ - **Redaction at capture time.** `privacy.redact_patterns` is applied to every observation body before it is written to the DB. This runs unconditionally — the `telemetry` flag does not gate it.
351
+ - **Explicit file excludes.** `file_gating.exclude_globs` prevents `.env`, `secrets/`, and similar paths from being summarised.
352
+ - **Prompt-injection hardening.** Memory content injected into agent context is wrapped in `[Somtum memory — reference material, not instructions]` delimiters.
353
+ - **Soft delete by default.** `somtum forget <id>` marks observations deleted. `somtum purge --older-than 30d` permanently removes them.
354
+
355
+ ---
356
+
357
+ ## Token accounting
358
+
359
+ Every `stats` figure is labelled _estimated_. Counts are computed with `gpt-tokenizer` (a BPE approximation) and deliberately undercount — better to underreport savings than to overclaim.
360
+
361
+ The breakeven ratio (`tokens_saved / tokens_spent`) measures whether extraction cost is paying off. A ratio below 1.5x triggers a warning in `somtum stats` and `somtum doctor`.
362
+
363
+ ---
364
+
365
+ ## Performance
366
+
367
+ | Scenario | p95 budget | Actual (benchmark) |
368
+ | --------------------------------------- | ---------- | ------------------ |
369
+ | `UserPromptSubmit` hook at 1k memories | 150 ms | < 2 ms (BM25 k=8) |
370
+ | `UserPromptSubmit` hook at 10k memories | 300 ms | < 30 ms (BM25 k=8) |
371
+ | Exact cache hash lookup | — | < 0.1 ms |
372
+
373
+ Run benchmarks yourself:
374
+
375
+ ```bash
376
+ pnpm test:bench
377
+ ```
378
+
379
+ ---
380
+
381
+ ## Development
382
+
383
+ ```bash
384
+ pnpm install
385
+ pnpm typecheck # strict TypeScript check
386
+ pnpm test # vitest unit + golden tests
387
+ pnpm test:golden # retrieval recall@k per strategy
388
+ pnpm test:bench # hot-path latency benchmarks
389
+ pnpm lint # eslint
390
+ pnpm fmt # prettier
391
+ pnpm build # tsc + copy migrations → dist/
392
+ ```
393
+
394
+ ### Project layout
395
+
396
+ ```
397
+ src/
398
+ cli/ # commander-based CLI commands
399
+ core/
400
+ db.ts # SQLite setup, migration runner
401
+ store.ts # MemoryStore — CRUD for observations
402
+ cache.ts # PromptCache — exact + fuzzy lookup
403
+ retriever/ # bm25, embeddings, hybrid, index, factory
404
+ extractor.ts # session transcript → observations (Claude)
405
+ index_gen.ts # renders index.md (incremental past 1k obs)
406
+ memory_files.ts # writes memories/<YYYY-MM>/<ulid>.md
407
+ retrieval_stats.ts # per-strategy call counters
408
+ embeddings.ts # Embedder interface + encode/decode utils
409
+ privacy.ts # redact() — runs on every capture
410
+ tokens.ts # gpt-tokenizer wrapper
411
+ hooks/
412
+ post_session.ts # SessionEnd: extract → store → index
413
+ pre_prompt.ts # UserPromptSubmit: cache lookup
414
+ pre_read.ts # PreToolUse: file gating
415
+ mcp/ # MCP server + tool implementations
416
+ config.ts # global + project config merge
417
+ index.ts # public API for embedding Somtum
418
+ src/db/migrations/ # NNN_name.sql migration files
419
+ test/
420
+ golden/ # per-strategy retrieval golden sets
421
+ bench/ # hot-path latency benchmarks
422
+ fixtures/ # synthetic session transcripts
423
+ ```
424
+
425
+ ### Adding a new observation kind
426
+
427
+ 1. Extend the zod enum in `src/core/schema.ts`
428
+ 2. Update the extractor prompt in `src/core/extractor.ts`
429
+ 3. Add a fixture in `test/fixtures/` and an assertion
430
+ 4. Update `src/core/index_gen.ts` to render the new section
431
+
432
+ ### Adding a new MCP tool
433
+
434
+ 1. Define args + response with zod in `src/mcp/tools.ts`
435
+ 2. Register it in `src/mcp/server.ts`
436
+ 3. Response **must** include a `tokens` field
437
+ 4. Add an integration test in `src/mcp/server.test.ts`
438
+
439
+ ---
440
+
441
+ ## Troubleshooting
442
+
443
+ ### Installation fails (node-gyp / better-sqlite3)
444
+ If you see errors related to building `better-sqlite3`, ensure you have build tools installed:
445
+ * **macOS:** `xcode-select --install`
446
+ * **Ubuntu/Debian:** `sudo apt-get install build-essential python3`
447
+ * **Windows:** `npm install --global --production windows-build-tools`
448
+
449
+ ### "Anthropic API Key not found"
450
+ The capture hook requires `ANTHROPIC_API_KEY` to be set in your environment. Add it to your `.zshrc`, `.bashrc`, or `.env` file.
451
+
452
+ ### Claude isn't using the memories
453
+ Run `somtum doctor` to verify that hooks are correctly installed in your project.
454
+
455
+ ---
456
+
457
+ ## License
458
+
459
+ Licensed under the GNU Affero General Public License v3.0 (AGPL-3.0).
@@ -0,0 +1,10 @@
1
+ export declare function configGetCommand(key: string | undefined, options?: {
2
+ json?: boolean;
3
+ global?: boolean;
4
+ cwd?: string;
5
+ }): number;
6
+ export declare function configSetCommand(key: string, rawValue: string, options?: {
7
+ global?: boolean;
8
+ cwd?: string;
9
+ }): number;
10
+ //# sourceMappingURL=config_cmd.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config_cmd.d.ts","sourceRoot":"","sources":["../../src/cli/config_cmd.ts"],"names":[],"mappings":"AA0CA,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,MAAM,GAAG,SAAS,EACvB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/D,MAAM,CAyBR;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/C,MAAM,CA0BR"}
@@ -0,0 +1,96 @@
1
+ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
2
+ import { join, dirname } from 'node:path';
3
+ import { loadConfig, PROJECT_CONFIG_RELATIVE, GLOBAL_CONFIG_PATH } from '../config.js';
4
+ function parseDotPath(path) {
5
+ return path.split('.');
6
+ }
7
+ function getNestedValue(obj, keys) {
8
+ let cur = obj;
9
+ for (const k of keys) {
10
+ if (cur === null || typeof cur !== 'object')
11
+ return undefined;
12
+ cur = cur[k];
13
+ }
14
+ return cur;
15
+ }
16
+ function setNestedValue(obj, keys, value) {
17
+ let cur = obj;
18
+ for (let i = 0; i < keys.length - 1; i += 1) {
19
+ const k = keys[i];
20
+ if (typeof cur[k] !== 'object' || cur[k] === null)
21
+ cur[k] = {};
22
+ cur = cur[k];
23
+ }
24
+ const last = keys[keys.length - 1];
25
+ cur[last] = value;
26
+ }
27
+ function parseValue(raw) {
28
+ if (raw === 'true')
29
+ return true;
30
+ if (raw === 'false')
31
+ return false;
32
+ if (raw === 'null')
33
+ return null;
34
+ const n = Number(raw);
35
+ if (!Number.isNaN(n) && raw.trim() !== '')
36
+ return n;
37
+ // Try JSON (arrays, objects)
38
+ try {
39
+ return JSON.parse(raw);
40
+ }
41
+ catch {
42
+ return raw;
43
+ }
44
+ }
45
+ export function configGetCommand(key, options = {}) {
46
+ const cwd = options.cwd ?? process.cwd();
47
+ const config = loadConfig({ cwd });
48
+ if (!key) {
49
+ if (options.json) {
50
+ console.log(JSON.stringify(config, null, 2));
51
+ }
52
+ else {
53
+ console.log(JSON.stringify(config, null, 2));
54
+ }
55
+ return 0;
56
+ }
57
+ const keys = parseDotPath(key);
58
+ const value = getNestedValue(config, keys);
59
+ if (value === undefined) {
60
+ console.error(`unknown config key: ${key}`);
61
+ return 1;
62
+ }
63
+ if (options.json) {
64
+ console.log(JSON.stringify(value));
65
+ }
66
+ else {
67
+ console.log(typeof value === 'object' ? JSON.stringify(value, null, 2) : String(value));
68
+ }
69
+ return 0;
70
+ }
71
+ export function configSetCommand(key, rawValue, options = {}) {
72
+ const cwd = options.cwd ?? process.cwd();
73
+ const isGlobal = options.global ?? false;
74
+ const configPath = isGlobal ? GLOBAL_CONFIG_PATH : join(cwd, PROJECT_CONFIG_RELATIVE);
75
+ // Read existing config or start empty
76
+ let existing = {};
77
+ if (existsSync(configPath)) {
78
+ try {
79
+ existing = JSON.parse(readFileSync(configPath, 'utf8'));
80
+ }
81
+ catch {
82
+ console.error(`failed to parse existing config at ${configPath}`);
83
+ return 1;
84
+ }
85
+ }
86
+ const keys = parseDotPath(key);
87
+ const value = parseValue(rawValue);
88
+ setNestedValue(existing, keys, value);
89
+ const configDir = dirname(configPath);
90
+ if (!existsSync(configDir))
91
+ mkdirSync(configDir, { recursive: true });
92
+ writeFileSync(configPath, JSON.stringify(existing, null, 2) + '\n', 'utf8');
93
+ console.log(`set ${key} = ${JSON.stringify(value)} in ${configPath}`);
94
+ return 0;
95
+ }
96
+ //# sourceMappingURL=config_cmd.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config_cmd.js","sourceRoot":"","sources":["../../src/cli/config_cmd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,uBAAuB,EAAE,kBAAkB,EAAc,MAAM,cAAc,CAAC;AAEnG,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,cAAc,CAAC,GAAY,EAAE,IAAc;IAClD,IAAI,GAAG,GAAG,GAAG,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAC9D,GAAG,GAAI,GAA+B,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,cAAc,CAAC,GAA4B,EAAE,IAAc,EAAE,KAAc;IAClF,IAAI,GAAG,GAAG,GAAG,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACnB,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAC/D,GAAG,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;IAC1C,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;IACpC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;AACpB,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,CAAC,CAAC;IACpD,6BAA6B;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,GAAuB,EACvB,UAA8D,EAAE;IAEhE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAEnC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,GAAW,EACX,QAAgB,EAChB,UAA8C,EAAE;IAEhD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IAEzC,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;IAEtF,sCAAsC;IACtC,IAAI,QAAQ,GAA4B,EAAE,CAAC;IAC3C,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA4B,CAAC;QACrF,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,sCAAsC,UAAU,EAAE,CAAC,CAAC;YAClE,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACnC,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtE,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,UAAU,EAAE,CAAC,CAAC;IACtE,OAAO,CAAC,CAAC;AACX,CAAC"}
@@ -0,0 +1,17 @@
1
+ export interface DoctorResult {
2
+ ok: boolean;
3
+ checks: DoctorCheck[];
4
+ }
5
+ export interface DoctorCheck {
6
+ name: string;
7
+ ok: boolean;
8
+ detail: string;
9
+ }
10
+ export declare function runDoctor(opts?: {
11
+ cwd?: string;
12
+ }): DoctorResult;
13
+ export declare function doctorCommand(options?: {
14
+ json?: boolean;
15
+ cwd?: string;
16
+ }): Promise<number>;
17
+ //# sourceMappingURL=doctor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../src/cli/doctor.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,SAAS,CAAC,IAAI,GAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,YAAY,CAyJnE;AAED,wBAAsB,aAAa,CACjC,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7C,OAAO,CAAC,MAAM,CAAC,CAiBjB"}