sessionmem 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/sessionmem-logo.svg +57 -0
- package/CHANGELOG.md +84 -0
- package/CONTRIBUTING.md +130 -0
- package/README.md +153 -82
- package/SECURITY.md +33 -0
- package/dist/cli/commands/dedupe.js +89 -22
- package/dist/cli/commands/import.js +15 -44
- package/dist/cli/commands/ingestHook.js +107 -18
- package/dist/cli/commands/install.js +3 -3
- package/dist/cli/commands/list.js +3 -0
- package/dist/cli/commands/run.js +10 -12
- package/dist/cli/commands/savings.js +3 -1
- package/dist/cli/commands/search.js +8 -4
- package/dist/cli/commands/stats.js +7 -0
- package/dist/cli/commands/uninstall.js +2 -1
- package/dist/cli/context.js +8 -6
- package/dist/cli/index.js +8 -17
- package/dist/cli/projectId.js +48 -2
- package/dist/core/api/contracts.js +6 -1
- package/dist/core/api/memoryCoreService.js +17 -3
- package/dist/core/config/policyConfig.js +5 -1
- package/dist/core/injection/tokenBudget.js +10 -2
- package/dist/core/retrieve/retrieveMemories.js +1 -1
- package/dist/core/schema/runMigrations.js +9 -5
- package/dist/core/storage/db.js +31 -9
- package/dist/core/storage/memoryRepo.js +1 -1
- package/dist/core/storage/memorySearchRepo.js +36 -14
- package/dist/core/storage/sessionEventsRepo.js +21 -0
- package/dist/core/summarize/cloudSummarizer.js +3 -8
- package/dist/core/summarize/redaction.js +70 -10
- package/dist/core/summarize/summaryShape.js +50 -20
- package/docs/architecture.md +53 -0
- package/docs/benchmark.md +50 -0
- package/docs/cloud-summarization.md +57 -0
- package/docs/migration.md +42 -0
- package/docs/privacy-and-retention.md +139 -0
- package/docs/team-mode.md +117 -0
- package/docs/troubleshooting.md +78 -0
- package/docs/typescript-upgrade.md +48 -0
- package/package.json +26 -13
- package/dist/core/session.js +0 -4
package/dist/cli/index.js
CHANGED
|
@@ -30,7 +30,7 @@ import { browseCommand } from "./commands/browse.js";
|
|
|
30
30
|
// "../../package.json" specifier resolves to the package root.
|
|
31
31
|
const pkg = createRequire(import.meta.url)("../../package.json");
|
|
32
32
|
const program = new Command();
|
|
33
|
-
program.name("sessionmem").version(pkg.version);
|
|
33
|
+
program.name("sessionmem").description("Local-first memory for MCP coding agents").version(pkg.version);
|
|
34
34
|
program
|
|
35
35
|
.command("run")
|
|
36
36
|
.description("Start the sessionmem MCP server")
|
|
@@ -74,15 +74,15 @@ program
|
|
|
74
74
|
program
|
|
75
75
|
.command("search <query>")
|
|
76
76
|
.description("Search memories by semantic query")
|
|
77
|
-
.option("--limit <n>", "Maximum number of results", "10")
|
|
77
|
+
.option("--limit <n>", "Maximum number of results (1-100)", "10")
|
|
78
78
|
.action((query, options) => searchCommand(query, options));
|
|
79
79
|
program
|
|
80
80
|
.command("list")
|
|
81
|
-
.description("List
|
|
81
|
+
.description("List up to 200 recent memories for the current project")
|
|
82
82
|
.action(() => listCommand());
|
|
83
83
|
program
|
|
84
84
|
.command("show <id>")
|
|
85
|
-
.description("Show
|
|
85
|
+
.description("Show memory details by ID (content capped at 2000 characters)")
|
|
86
86
|
.action((id) => showCommand(id));
|
|
87
87
|
program
|
|
88
88
|
.command("forget <id>")
|
|
@@ -96,7 +96,7 @@ program
|
|
|
96
96
|
program
|
|
97
97
|
.command("import <path>")
|
|
98
98
|
.description("Import memories from a JSON file")
|
|
99
|
-
.option("--merge", "
|
|
99
|
+
.option("--merge", "Merge new records; existing IDs are preserved")
|
|
100
100
|
.action((path, options) => importCommand(path, options));
|
|
101
101
|
program
|
|
102
102
|
.command("stats")
|
|
@@ -126,8 +126,6 @@ retention
|
|
|
126
126
|
.option("--days <n>", "Override the retention window in days")
|
|
127
127
|
.action((options) => retentionPruneCommand(options));
|
|
128
128
|
// config command group — generic get/set over ~/.sessionmem/config.json.
|
|
129
|
-
// config get/set are synchronous and take no CliContext, so the arrow-wrap here
|
|
130
|
-
// only drops commander's trailing Command argument.
|
|
131
129
|
const config = program
|
|
132
130
|
.command("config")
|
|
133
131
|
.description("Read and write sessionmem policy config");
|
|
@@ -140,10 +138,6 @@ config
|
|
|
140
138
|
.description("Persist a config key to ~/.sessionmem/config.json")
|
|
141
139
|
.action((key, value) => configSetCommand(key, value));
|
|
142
140
|
// team command group — turn shared-memory mode on/off and inspect it.
|
|
143
|
-
// enable/status take no CliContext (config-only), so their arrow-wraps just drop
|
|
144
|
-
// commander's trailing Command argument. disable accepts an optional CliContext
|
|
145
|
-
// seam for the --remove-team-memories DB delete; the arrow-wrap forwards only
|
|
146
|
-
// options so production falls through to createCliContext() (NOTE above).
|
|
147
141
|
const team = program
|
|
148
142
|
.command("team")
|
|
149
143
|
.description("Manage shared-path team memory mode");
|
|
@@ -161,15 +155,12 @@ team
|
|
|
161
155
|
.description("Show team mode state and shared-path availability")
|
|
162
156
|
.action(() => teamStatusCommand());
|
|
163
157
|
// re-embed — bulk-update stale embeddings to the current version.
|
|
164
|
-
// reEmbedCommand declares a trailing `ctx?` test seam, so arrow-wrap to drop
|
|
165
|
-
// commander's trailing Command argument (NOTE above).
|
|
166
158
|
program
|
|
167
159
|
.command("re-embed")
|
|
168
160
|
.description("Re-embed all memories with stale embedding versions")
|
|
169
161
|
.action(() => reEmbedCommand());
|
|
170
|
-
// sync — push a local snapshot to the shared path and pull every
|
|
171
|
-
//
|
|
172
|
-
// arrow-wrap to drop commander's trailing Command argument (NOTE above).
|
|
162
|
+
// sync — push a local snapshot to the shared path and pull every teammate
|
|
163
|
+
// snapshot back.
|
|
173
164
|
program
|
|
174
165
|
.command("sync")
|
|
175
166
|
.description("Push local memories and pull teammate memories via the shared path")
|
|
@@ -180,7 +171,7 @@ program
|
|
|
180
171
|
.command("dedupe")
|
|
181
172
|
.description("Find and optionally remove near-duplicate memories")
|
|
182
173
|
.option("--apply", "Delete the lower-importance duplicate in each pair (default: dry-run)")
|
|
183
|
-
.option("--threshold <n>", "Cosine similarity threshold 0
|
|
174
|
+
.option("--threshold <n>", "Cosine similarity threshold >0 to 1 (default: 0.85)")
|
|
184
175
|
.action((options) => dedupeCommand({ dry: !options.apply, threshold: options.threshold }));
|
|
185
176
|
// browse — cross-project memory viewer.
|
|
186
177
|
program
|
package/dist/cli/projectId.js
CHANGED
|
@@ -1,4 +1,43 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
/** Safety bound on the upward walk in {@link findProjectRoot}. */
|
|
5
|
+
const MAX_PARENT_WALK = 64;
|
|
6
|
+
/**
|
|
7
|
+
* Walk up from `startDir` to the nearest directory containing `.git`, and
|
|
8
|
+
* return it. Returns null when there is no repository above `startDir`.
|
|
9
|
+
*
|
|
10
|
+
* Memory is keyed by project, and a project is the repository — not whichever
|
|
11
|
+
* directory a command happened to run in. Hashing the raw cwd meant every
|
|
12
|
+
* subdirectory got its OWN bucket: the Claude Code hooks fire at the repo root
|
|
13
|
+
* and write there, then `sessionmem stats` run from `src/` reported an empty
|
|
14
|
+
* store, because it was reading a different project entirely.
|
|
15
|
+
*
|
|
16
|
+
* `.git` is matched as a path entry rather than a directory so linked worktrees
|
|
17
|
+
* and submodules (where `.git` is a FILE pointing at the real gitdir) anchor
|
|
18
|
+
* correctly too.
|
|
19
|
+
*/
|
|
20
|
+
export function findProjectRoot(startDir) {
|
|
21
|
+
let current = resolve(startDir);
|
|
22
|
+
for (let depth = 0; depth < MAX_PARENT_WALK; depth += 1) {
|
|
23
|
+
try {
|
|
24
|
+
// nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal
|
|
25
|
+
if (existsSync(join(current, ".git"))) {
|
|
26
|
+
return current;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// Unreadable directory (permissions, race): stop rather than throw — the
|
|
31
|
+
// caller falls back to the cwd-derived id.
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
const parent = dirname(current);
|
|
35
|
+
if (parent === current)
|
|
36
|
+
break; // filesystem root
|
|
37
|
+
current = parent;
|
|
38
|
+
}
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
2
41
|
/**
|
|
3
42
|
* Sanitize a raw token to the filename-safe character set used across
|
|
4
43
|
* sessionmem (mirrors localUsername). Any character outside [A-Za-z0-9._-]
|
|
@@ -45,7 +84,13 @@ export function projectIdFromCwd(cwd) {
|
|
|
45
84
|
/**
|
|
46
85
|
* Resolve the effective project id: the `SESSIONMEM_PROJECT_ID` env override
|
|
47
86
|
* (operator-controlled test/migration seam; also lets a user pin a legacy
|
|
48
|
-
* basename-only id) wins, otherwise derive from
|
|
87
|
+
* basename-only id) wins, otherwise derive from the enclosing repository root,
|
|
88
|
+
* falling back to `process.cwd()` when the cwd is not inside a repository.
|
|
89
|
+
*
|
|
90
|
+
* Anchoring to the repository keeps every command — hooks firing at the root,
|
|
91
|
+
* a `sessionmem stats` typed three directories down — on the same memory
|
|
92
|
+
* bucket. The id is unchanged for anyone already running at the repo root,
|
|
93
|
+
* since the root IS the cwd in that case.
|
|
49
94
|
*/
|
|
50
95
|
export function deriveProjectId() {
|
|
51
96
|
const envProjectId = process.env.SESSIONMEM_PROJECT_ID;
|
|
@@ -65,5 +110,6 @@ export function deriveProjectId() {
|
|
|
65
110
|
return sanitized;
|
|
66
111
|
}
|
|
67
112
|
}
|
|
68
|
-
|
|
113
|
+
const cwd = process.cwd();
|
|
114
|
+
return projectIdFromCwd(findProjectRoot(cwd) ?? cwd);
|
|
69
115
|
}
|
|
@@ -65,7 +65,12 @@ export const memorySchema = z.object({
|
|
|
65
65
|
});
|
|
66
66
|
export const ingestSessionEventSchema = z.object({
|
|
67
67
|
id: z.string().min(1).max(200),
|
|
68
|
-
|
|
68
|
+
// Optional: callers that already track an ordering key (batch ingestion of a
|
|
69
|
+
// transcript) supply it; callers that fire one event at a time from separate
|
|
70
|
+
// processes (the PostToolUse hook) omit it and the service assigns the next
|
|
71
|
+
// free index per session. See nextSessionEventIndex for why inventing a
|
|
72
|
+
// wall-clock index silently loses events.
|
|
73
|
+
eventIndex: z.number().int().nonnegative().optional(),
|
|
69
74
|
eventType: z.string().min(1).max(100),
|
|
70
75
|
payloadJson: z
|
|
71
76
|
.string()
|
|
@@ -8,7 +8,7 @@ import { applyRedaction } from "../summarize/redaction.js";
|
|
|
8
8
|
import { countAllMemoriesByProject, countMemoriesBySession, countMemoriesOlderThan, deleteMemoriesOlderThan, deleteMemoryById, getMemoryOwnerProjectId, getMemoryRecordById, incrementAccessCounts, insertMemory, listMemoriesByProject, resetAccessCounts, updateMemoryContent, upsertImportedMemory, upsertPulledMemory, upsertSessionSummaryMemory, } from "../storage/memoryRepo.js";
|
|
9
9
|
import { SESSION_WRITE_SOFT_LIMIT, configFilePath, DEEP_MODE_RETRIEVAL_CAP, readPolicyConfig, resolvePolicySettings, } from "../config/policyConfig.js";
|
|
10
10
|
import { insertMemoryFeedbackEvent } from "../storage/memoryFeedbackRepo.js";
|
|
11
|
-
import { countAllSessionEvents, insertSessionEvent } from "../storage/sessionEventsRepo.js";
|
|
11
|
+
import { countAllSessionEvents, insertSessionEvent, nextSessionEventIndex, } from "../storage/sessionEventsRepo.js";
|
|
12
12
|
import { batchStoreMemoryItemSchema, batchStoreMemoryRequestSchema, exportMemoriesRequestSchema, forgetMemoryRequestSchema, getMemoryRequestSchema, handleSessionEndRequestSchema, importMemoriesRequestSchema, pullMemoriesRequestSchema, ingestSessionEventsRequestSchema, LIST_MEMORIES_DEFAULT_LIMIT, listMemoriesRequestSchema, pruneMemoriesRequestSchema, redactExistingRequestSchema, resetAccessCountsRequestSchema, retrieveMemoriesRequestSchema, statsRequestSchema, storeMemoryRequestSchema, summarizeSessionToMemoryRequestSchema, } from "./contracts.js";
|
|
13
13
|
import { DomainError, toErrorEnvelope } from "./errors.js";
|
|
14
14
|
import { assertLocalOnlyPolicy, } from "./localOnlyPolicy.js";
|
|
@@ -186,15 +186,25 @@ export function createMemoryCoreService(deps) {
|
|
|
186
186
|
const redactionEnabled = resolveRedactionEnabled(undefined);
|
|
187
187
|
const ingest = db.transaction(() => {
|
|
188
188
|
let written = 0;
|
|
189
|
+
// Events that omit eventIndex are appended after the session's current
|
|
190
|
+
// maximum. Read it once inside the transaction and increment locally so
|
|
191
|
+
// a multi-event batch stays contiguous.
|
|
192
|
+
let nextIndex;
|
|
189
193
|
for (const event of parsed.events) {
|
|
190
194
|
const redactedPayload = applyRedaction(event.payloadJson, {
|
|
191
195
|
redactionEnabled,
|
|
192
196
|
}).text;
|
|
197
|
+
let eventIndex = event.eventIndex;
|
|
198
|
+
if (eventIndex === undefined) {
|
|
199
|
+
nextIndex ??= nextSessionEventIndex(db, parsed.projectId, parsed.sessionId);
|
|
200
|
+
eventIndex = nextIndex;
|
|
201
|
+
nextIndex += 1;
|
|
202
|
+
}
|
|
193
203
|
written += insertSessionEvent(db, {
|
|
194
204
|
id: event.id,
|
|
195
205
|
project_id: parsed.projectId,
|
|
196
206
|
session_id: parsed.sessionId,
|
|
197
|
-
event_index:
|
|
207
|
+
event_index: eventIndex,
|
|
198
208
|
event_type: event.eventType,
|
|
199
209
|
payload_json: redactedPayload,
|
|
200
210
|
created_at: event.createdAt,
|
|
@@ -202,7 +212,11 @@ export function createMemoryCoreService(deps) {
|
|
|
202
212
|
}
|
|
203
213
|
return written;
|
|
204
214
|
});
|
|
205
|
-
|
|
215
|
+
// `immediate` takes the write lock up front so two concurrent hook
|
|
216
|
+
// processes cannot both read the same next index and have one insert
|
|
217
|
+
// silently ignored. busy_timeout (db.ts) makes the loser wait rather
|
|
218
|
+
// than fail.
|
|
219
|
+
const ingested = ingest.immediate();
|
|
206
220
|
return {
|
|
207
221
|
ok: true,
|
|
208
222
|
ingested,
|
|
@@ -88,7 +88,11 @@ export function readPolicyConfig(filePath) {
|
|
|
88
88
|
let obj;
|
|
89
89
|
try {
|
|
90
90
|
raw = readFileSync(filePath, "utf8");
|
|
91
|
-
|
|
91
|
+
const parsed = JSON.parse(raw);
|
|
92
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
93
|
+
return { ...DEFAULT_POLICY_CONFIG };
|
|
94
|
+
}
|
|
95
|
+
obj = parsed;
|
|
92
96
|
}
|
|
93
97
|
catch {
|
|
94
98
|
return { ...DEFAULT_POLICY_CONFIG };
|
|
@@ -10,8 +10,16 @@ export function capTokens(text, cap) {
|
|
|
10
10
|
if (tokens.length <= cap) {
|
|
11
11
|
return text;
|
|
12
12
|
}
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
// Reserve room for the truncation marker, then verify the re-encoded text:
|
|
14
|
+
// token boundaries can change when a decoded prefix and suffix are joined.
|
|
15
|
+
const suffix = " ...";
|
|
16
|
+
for (let keep = Math.max(0, Math.floor(cap) - countTokens(suffix)); keep >= 0; keep--) {
|
|
17
|
+
const trimmed = encoding.decode(tokens.slice(0, keep)).trimEnd();
|
|
18
|
+
const result = trimmed ? `${trimmed}${suffix}` : suffix.trim();
|
|
19
|
+
if (countTokens(result) <= cap)
|
|
20
|
+
return result;
|
|
21
|
+
}
|
|
22
|
+
return "";
|
|
15
23
|
}
|
|
16
24
|
export function trimLowestPriorityContent(included, options = {}) {
|
|
17
25
|
const minContentTokens = options.minContentTokens ?? DEFAULT_MIN_CONTENT_TOKENS;
|
|
@@ -43,7 +43,7 @@ export function retrieveMemories(input) {
|
|
|
43
43
|
const now = input.now ?? new Date();
|
|
44
44
|
// Use FTS5 pre-filtering to limit cosine similarity computation to ~50 candidates.
|
|
45
45
|
// queryText is guaranteed non-empty by the Zod schema (z.string().min(1)) upstream.
|
|
46
|
-
const candidates = searchMemoryCandidatesFTS(input.db, input.projectId, queryText);
|
|
46
|
+
const candidates = searchMemoryCandidatesFTS(input.db, input.projectId, queryText, now);
|
|
47
47
|
const decayedCandidates = decayOldBoosts(candidates, now);
|
|
48
48
|
const dimension = resolveEmbeddingDimension(candidates);
|
|
49
49
|
const queryVector = deterministicEmbed(queryText, dimension).vector;
|
|
@@ -57,18 +57,19 @@ export function runMigrations(db, migrationsDir = DEFAULT_MIGRATIONS_DIR) {
|
|
|
57
57
|
const hasMigrationStmt = db.prepare("SELECT name FROM _migrations WHERE name = ? LIMIT 1");
|
|
58
58
|
const insertMigrationStmt = db.prepare("INSERT INTO _migrations(name) VALUES (?)");
|
|
59
59
|
const runMigration = db.transaction((fileName) => {
|
|
60
|
+
if (hasMigrationStmt.get(fileName))
|
|
61
|
+
return;
|
|
60
62
|
const filePath = path.join(migrationsDir, fileName);
|
|
61
63
|
const sql = fs.readFileSync(filePath, "utf8");
|
|
62
64
|
db.exec(sql);
|
|
63
65
|
insertMigrationStmt.run(fileName);
|
|
64
66
|
});
|
|
65
67
|
for (const fileName of files) {
|
|
66
|
-
|
|
67
|
-
if (existing) {
|
|
68
|
+
if (hasMigrationStmt.get(fileName))
|
|
68
69
|
continue;
|
|
69
|
-
}
|
|
70
70
|
try {
|
|
71
|
-
|
|
71
|
+
// Serialize the check and apply across processes opening the same DB.
|
|
72
|
+
runMigration.immediate(fileName);
|
|
72
73
|
}
|
|
73
74
|
catch (err) {
|
|
74
75
|
// Idempotency guard for ALTER TABLE ADD COLUMN migrations (005/006).
|
|
@@ -90,7 +91,10 @@ export function runMigrations(db, migrationsDir = DEFAULT_MIGRATIONS_DIR) {
|
|
|
90
91
|
const filePath = path.join(migrationsDir, fileName);
|
|
91
92
|
const sql = fs.readFileSync(filePath, "utf8");
|
|
92
93
|
if (allAddedColumnsExist(db, sql)) {
|
|
93
|
-
|
|
94
|
+
db.transaction(() => {
|
|
95
|
+
if (!hasMigrationStmt.get(fileName))
|
|
96
|
+
insertMigrationStmt.run(fileName);
|
|
97
|
+
}).immediate();
|
|
94
98
|
continue;
|
|
95
99
|
}
|
|
96
100
|
}
|
package/dist/core/storage/db.js
CHANGED
|
@@ -2,13 +2,35 @@ import BetterSqlite3 from "better-sqlite3";
|
|
|
2
2
|
import { runMigrations } from "../schema/runMigrations.js";
|
|
3
3
|
export function openDb(options = {}) {
|
|
4
4
|
const db = new BetterSqlite3(options.dbPath ?? ":memory:");
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
5
|
+
try {
|
|
6
|
+
// Performance pragmas — WAL enables concurrent reads during writes;
|
|
7
|
+
// busy_timeout prevents SQLITE_BUSY under concurrent MCP tool calls.
|
|
8
|
+
// Concurrent first opens can race while switching to WAL. SQLite may
|
|
9
|
+
// bypass the busy handler for this lock upgrade, so retry within the same
|
|
10
|
+
// five-second budget used for normal writes.
|
|
11
|
+
const deadline = Date.now() + 5000;
|
|
12
|
+
const pause = new Int32Array(new SharedArrayBuffer(4));
|
|
13
|
+
for (;;) {
|
|
14
|
+
try {
|
|
15
|
+
db.pragma("journal_mode = WAL");
|
|
16
|
+
break;
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
if (!(err instanceof Error) || !("code" in err) || err.code !== "SQLITE_BUSY" || Date.now() >= deadline) {
|
|
20
|
+
throw err;
|
|
21
|
+
}
|
|
22
|
+
Atomics.wait(pause, 0, 0, 10);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
db.pragma("synchronous = NORMAL");
|
|
26
|
+
db.pragma("foreign_keys = ON");
|
|
27
|
+
db.pragma("busy_timeout = 5000");
|
|
28
|
+
db.pragma("cache_size = -32000");
|
|
29
|
+
runMigrations(db, options.migrationsDir);
|
|
30
|
+
return db;
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
db.close();
|
|
34
|
+
throw err;
|
|
35
|
+
}
|
|
14
36
|
}
|
|
@@ -107,7 +107,7 @@ function getStatements(db) {
|
|
|
107
107
|
FROM memories
|
|
108
108
|
WHERE project_id = ?
|
|
109
109
|
AND (expires_at IS NULL OR expires_at > strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
110
|
-
ORDER BY updated_at DESC
|
|
110
|
+
ORDER BY updated_at DESC, id ASC
|
|
111
111
|
`),
|
|
112
112
|
// Lightweight projection for token-savings accounting: only `content` is
|
|
113
113
|
// needed to count tokens, so we deliberately avoid pulling the (potentially
|
|
@@ -1,23 +1,36 @@
|
|
|
1
1
|
import { EMBEDDING_VERSION } from "../embed/embeddingVersion.js";
|
|
2
2
|
const FTS_CANDIDATE_LIMIT = 50;
|
|
3
3
|
const FTS_FALLBACK_THRESHOLD = 5;
|
|
4
|
+
/** Recency window for the fallback candidate scan. */
|
|
5
|
+
const FALLBACK_RECENCY_DAYS = 90;
|
|
6
|
+
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
7
|
+
/** Storage timestamp format: ISO-8601 UTC with millisecond precision. */
|
|
8
|
+
function toStorageTimestamp(date) {
|
|
9
|
+
return date.toISOString();
|
|
10
|
+
}
|
|
4
11
|
const searchStmtCache = new WeakMap();
|
|
5
12
|
function getSearchStatements(db) {
|
|
6
13
|
let stmts = searchStmtCache.get(db);
|
|
7
14
|
if (stmts)
|
|
8
15
|
return stmts;
|
|
9
16
|
stmts = {
|
|
17
|
+
// `now` and the recency cutoff are bound parameters rather than
|
|
18
|
+
// strftime('now') so the caller's clock drives both filters. With SQL's own
|
|
19
|
+
// clock, a caller that passes an explicit `now` (retrieveMemories, and every
|
|
20
|
+
// test that pins a date) silently got a different window than it asked for,
|
|
21
|
+
// and fixed-date fixtures aged out of the 90-day band over real calendar
|
|
22
|
+
// time — turning deterministic tests into time bombs.
|
|
10
23
|
searchCandidates: db.prepare(`
|
|
11
24
|
SELECT
|
|
12
25
|
id, project_id, session_id, source_adapter, kind, content, normalized_content,
|
|
13
26
|
importance, author, origin_project_id, access_count, created_at, updated_at,
|
|
14
27
|
embedding, embedding_dim, embedding_version
|
|
15
28
|
FROM memories
|
|
16
|
-
WHERE project_id =
|
|
17
|
-
AND (expires_at IS NULL OR expires_at >
|
|
29
|
+
WHERE project_id = @projectId
|
|
30
|
+
AND (expires_at IS NULL OR expires_at > @now)
|
|
18
31
|
AND (
|
|
19
32
|
importance >= 8
|
|
20
|
-
OR updated_at >
|
|
33
|
+
OR updated_at > @recencyCutoff
|
|
21
34
|
)
|
|
22
35
|
`),
|
|
23
36
|
searchCandidatesFTS: db.prepare(`
|
|
@@ -28,11 +41,11 @@ function getSearchStatements(db) {
|
|
|
28
41
|
m.embedding, m.embedding_dim, m.embedding_version
|
|
29
42
|
FROM memories_fts
|
|
30
43
|
JOIN memories m ON m.rowid = memories_fts.rowid
|
|
31
|
-
WHERE memories_fts MATCH
|
|
32
|
-
AND m.project_id =
|
|
33
|
-
AND (m.expires_at IS NULL OR m.expires_at >
|
|
44
|
+
WHERE memories_fts MATCH @match
|
|
45
|
+
AND m.project_id = @projectId
|
|
46
|
+
AND (m.expires_at IS NULL OR m.expires_at > @now)
|
|
34
47
|
ORDER BY rank
|
|
35
|
-
LIMIT
|
|
48
|
+
LIMIT @limit
|
|
36
49
|
`),
|
|
37
50
|
};
|
|
38
51
|
searchStmtCache.set(db, stmts);
|
|
@@ -74,8 +87,12 @@ function mapRows(rows) {
|
|
|
74
87
|
};
|
|
75
88
|
});
|
|
76
89
|
}
|
|
77
|
-
export function searchMemoryCandidates(db, projectId) {
|
|
78
|
-
const rows = getSearchStatements(db).searchCandidates.all(
|
|
90
|
+
export function searchMemoryCandidates(db, projectId, now = new Date()) {
|
|
91
|
+
const rows = getSearchStatements(db).searchCandidates.all({
|
|
92
|
+
projectId,
|
|
93
|
+
now: toStorageTimestamp(now),
|
|
94
|
+
recencyCutoff: toStorageTimestamp(new Date(now.getTime() - FALLBACK_RECENCY_DAYS * MS_PER_DAY)),
|
|
95
|
+
});
|
|
79
96
|
return mapRows(rows);
|
|
80
97
|
}
|
|
81
98
|
/**
|
|
@@ -107,23 +124,28 @@ function sanitizeFtsQuery(queryText) {
|
|
|
107
124
|
* (<8) — exactly the rows the fallback's filter excludes — returning zero
|
|
108
125
|
* candidates for a query that matched only such rows.
|
|
109
126
|
*/
|
|
110
|
-
export function searchMemoryCandidatesFTS(db, projectId, queryText) {
|
|
127
|
+
export function searchMemoryCandidatesFTS(db, projectId, queryText, now = new Date()) {
|
|
111
128
|
const sanitized = sanitizeFtsQuery(queryText);
|
|
112
129
|
if (!sanitized) {
|
|
113
|
-
return searchMemoryCandidates(db, projectId);
|
|
130
|
+
return searchMemoryCandidates(db, projectId, now);
|
|
114
131
|
}
|
|
115
132
|
let rows;
|
|
116
133
|
try {
|
|
117
|
-
rows = getSearchStatements(db).searchCandidatesFTS.all(
|
|
134
|
+
rows = getSearchStatements(db).searchCandidatesFTS.all({
|
|
135
|
+
match: sanitized,
|
|
136
|
+
projectId,
|
|
137
|
+
now: toStorageTimestamp(now),
|
|
138
|
+
limit: FTS_CANDIDATE_LIMIT,
|
|
139
|
+
});
|
|
118
140
|
}
|
|
119
141
|
catch {
|
|
120
142
|
// FTS5 MATCH can throw on malformed queries — fall back to full scan
|
|
121
|
-
return searchMemoryCandidates(db, projectId);
|
|
143
|
+
return searchMemoryCandidates(db, projectId, now);
|
|
122
144
|
}
|
|
123
145
|
if (rows.length >= FTS_FALLBACK_THRESHOLD) {
|
|
124
146
|
return dedupById(mapRows(rows));
|
|
125
147
|
}
|
|
126
|
-
const fallback = searchMemoryCandidates(db, projectId);
|
|
148
|
+
const fallback = searchMemoryCandidates(db, projectId, now);
|
|
127
149
|
if (rows.length === 0) {
|
|
128
150
|
return fallback;
|
|
129
151
|
}
|
|
@@ -22,6 +22,11 @@ function getSessionEventsStatements(db) {
|
|
|
22
22
|
ORDER BY event_index ASC
|
|
23
23
|
`),
|
|
24
24
|
countAll: db.prepare("SELECT COUNT(*) AS count FROM session_events WHERE project_id = ?"),
|
|
25
|
+
nextEventIndex: db.prepare(`
|
|
26
|
+
SELECT COALESCE(MAX(event_index), -1) + 1 AS next
|
|
27
|
+
FROM session_events
|
|
28
|
+
WHERE project_id = ? AND session_id = ?
|
|
29
|
+
`),
|
|
25
30
|
};
|
|
26
31
|
sessionEventsStmtCache.set(db, stmts);
|
|
27
32
|
return stmts;
|
|
@@ -34,6 +39,22 @@ function getSessionEventsStatements(db) {
|
|
|
34
39
|
export function insertSessionEvent(db, input) {
|
|
35
40
|
return getSessionEventsStatements(db).insertEvent.run(input).changes;
|
|
36
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Next free `event_index` for a session (max + 1, or 0 when the session has no
|
|
44
|
+
* events yet).
|
|
45
|
+
*
|
|
46
|
+
* Callers that have no natural ordering key of their own (the PostToolUse hook
|
|
47
|
+
* fires once per tool use, in separate processes) use this instead of inventing
|
|
48
|
+
* one. A wall-clock index collides whenever two tool uses land in the same
|
|
49
|
+
* millisecond — which parallel tool calls routinely do — and the UNIQUE index
|
|
50
|
+
* on (project_id, session_id, event_index) then makes `INSERT OR IGNORE` drop
|
|
51
|
+
* the loser silently. Must be called inside the same immediate transaction as
|
|
52
|
+
* the insert so concurrent writers cannot read the same value.
|
|
53
|
+
*/
|
|
54
|
+
export function nextSessionEventIndex(db, projectId, sessionId) {
|
|
55
|
+
const row = getSessionEventsStatements(db).nextEventIndex.get(projectId, sessionId);
|
|
56
|
+
return row.next;
|
|
57
|
+
}
|
|
37
58
|
export function countAllSessionEvents(db, projectId) {
|
|
38
59
|
const row = getSessionEventsStatements(db).countAll.get(projectId);
|
|
39
60
|
return row.count;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import Anthropic from "@anthropic-ai/sdk";
|
|
2
|
+
import { capTokens } from "../injection/tokenBudget.js";
|
|
2
3
|
import { summarizeLocalSessionEvents } from "./localSummarizer.js";
|
|
3
4
|
import { DEFAULT_SUMMARIZER_MODEL } from "../config/policyConfig.js";
|
|
4
5
|
const CLOUD_SYSTEM_PROMPT = "You are a memory compressor for AI coding sessions. Given this session transcript summary, produce a compact, high-signal list of facts, decisions, and context that should be remembered. Be extremely concise.";
|
|
@@ -26,7 +27,7 @@ export async function summarizeWithCloud(input) {
|
|
|
26
27
|
messages: [{ role: "user", content: localResult.summary }],
|
|
27
28
|
});
|
|
28
29
|
// Extract text from response content blocks
|
|
29
|
-
|
|
30
|
+
const text = response.content
|
|
30
31
|
.filter((block) => block.type === "text")
|
|
31
32
|
.map((block) => block.text)
|
|
32
33
|
.join("\n");
|
|
@@ -35,14 +36,8 @@ export async function summarizeWithCloud(input) {
|
|
|
35
36
|
if (!text || text.trim().length === 0) {
|
|
36
37
|
throw new Error("Cloud summarizer returned empty response");
|
|
37
38
|
}
|
|
38
|
-
// Cap cloud output to summaryTokenCap (rough token→char ratio) for parity
|
|
39
|
-
// with the local summarizer, so a verbose cloud response cannot blow past the
|
|
40
|
-
// configured budget.
|
|
41
|
-
if (input.summaryTokenCap && text.length > input.summaryTokenCap * 4) {
|
|
42
|
-
text = text.slice(0, input.summaryTokenCap * 4);
|
|
43
|
-
}
|
|
44
39
|
return {
|
|
45
|
-
summary: text,
|
|
40
|
+
summary: capTokens(text, input.summaryTokenCap),
|
|
46
41
|
warningCodes: localResult.warningCodes,
|
|
47
42
|
};
|
|
48
43
|
}
|
|
@@ -9,6 +9,43 @@
|
|
|
9
9
|
// applyRedaction invocation — a measurable saving in batch/import/pull loops.
|
|
10
10
|
// The regex literals carry no `lastIndex` state because every pattern is used
|
|
11
11
|
// with String.prototype.replace (not stateful .test()/.exec() on a shared regex).
|
|
12
|
+
/**
|
|
13
|
+
* Key names that mark an assignment's value as a secret. Matched
|
|
14
|
+
* case-insensitively and allowed to carry bounded affixes on either side, so
|
|
15
|
+
* `AWS_SECRET_ACCESS_KEY` and `SECRET_TOKEN_VALUE` match as readily as `secret`.
|
|
16
|
+
*/
|
|
17
|
+
const SECRET_KEY_NAMES = [
|
|
18
|
+
"password",
|
|
19
|
+
"passwd",
|
|
20
|
+
"pwd",
|
|
21
|
+
"secret",
|
|
22
|
+
"api[_-]?key",
|
|
23
|
+
"apikey",
|
|
24
|
+
"access[_-]?token",
|
|
25
|
+
"refresh[_-]?token",
|
|
26
|
+
"auth[_-]?token",
|
|
27
|
+
"client[_-]?secret",
|
|
28
|
+
"private[_-]?key",
|
|
29
|
+
"credentials?",
|
|
30
|
+
"token",
|
|
31
|
+
].join("|");
|
|
32
|
+
/**
|
|
33
|
+
* `<key><separator><value>` where the key names a secret. Capture groups are
|
|
34
|
+
* (key, separator, value) so the replacement can keep the key and separator
|
|
35
|
+
* verbatim and swap only the value.
|
|
36
|
+
*
|
|
37
|
+
* Both `:` and `=` are accepted (YAML, JSON, `.env`, query strings, log lines),
|
|
38
|
+
* and the value may be backslash-escaped-quoted, double-quoted, single-quoted,
|
|
39
|
+
* or bare. Every quantifier is bounded, so the pattern is ReDoS-safe.
|
|
40
|
+
*
|
|
41
|
+
* The escaped-quote branch comes first and the bare branch excludes `\`, which
|
|
42
|
+
* together keep a JSON payload parseable after redaction. Session events store
|
|
43
|
+
* JSON, so a secret inside a stringified field arrives as `password=\"a b\"`;
|
|
44
|
+
* letting the bare branch match the lone backslash replaced it with
|
|
45
|
+
* `[REDACTED]` and left the rest of the quoted run stranded, producing
|
|
46
|
+
* unparseable JSON.
|
|
47
|
+
*/
|
|
48
|
+
const SECRET_ASSIGNMENT_PATTERN = new RegExp(String.raw `\b([A-Za-z0-9_.-]{0,40}(?:${SECRET_KEY_NAMES})[A-Za-z0-9_.-]{0,40}\\?"?)(\s*[:=]\s*)(\\"(?:\\.|[^"\\]){0,4096}\\"|"[^"\n]{1,4096}"|'[^'\n]{1,4096}'|[^\s"'\\,;&}]{1,4096})`, "gi");
|
|
12
49
|
const DEFAULT_RULES = createDefaultRules();
|
|
13
50
|
function createDefaultRules() {
|
|
14
51
|
return [
|
|
@@ -49,16 +86,39 @@ function createDefaultRules() {
|
|
|
49
86
|
(input) => input.replace(/\bsk-(?:[a-zA-Z0-9]{1,32}-){0,4}[a-zA-Z0-9]{12,200}\b/g, "[REDACTED_API_KEY]"),
|
|
50
87
|
// Bearer token header value: "Bearer <token>" (case-insensitive), token redacted.
|
|
51
88
|
(input) => input.replace(/\bBearer\s+[A-Za-z0-9._~+/-]{8,}=*/gi, "Bearer [REDACTED_BEARER_TOKEN]"),
|
|
52
|
-
//
|
|
53
|
-
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
(input) => input.replace(
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
//
|
|
61
|
-
|
|
89
|
+
// HTTP Basic auth header: the base64 blob encodes user:password verbatim.
|
|
90
|
+
(input) => input.replace(/\bBasic\s+[A-Za-z0-9+/]{12,}={0,2}/g, "Basic [REDACTED_BASIC_AUTH]"),
|
|
91
|
+
// Slack incoming-webhook URLs — the path IS the credential, so the whole
|
|
92
|
+
// path is collapsed rather than a token-shaped substring of it.
|
|
93
|
+
(input) => input.replace(/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]{10,200}/g, "https://hooks.slack.com/services/[REDACTED_SLACK_WEBHOOK]"),
|
|
94
|
+
// Secret-bearing assignments: `<key><sep><value>` for a known secret-ish key
|
|
95
|
+
// name, covering config files, .env exports, YAML, JSON bodies, log lines
|
|
96
|
+
// and URL query strings in one rule.
|
|
97
|
+
//
|
|
98
|
+
// Three things the narrower predecessors missed, each a real leak:
|
|
99
|
+
// - `:` as a separator (YAML, JSON, `Header: value` log lines). Only `=`
|
|
100
|
+
// was handled, so `aws_secret_access_key: wJalr…` passed through.
|
|
101
|
+
// - Affixed key names. The keyword had to sit on a \b boundary, and `_` is
|
|
102
|
+
// a word character, so `AWS_SECRET_ACCESS_KEY` and `SECRET_TOKEN_VALUE`
|
|
103
|
+
// never matched. Allow bounded affixes on both sides.
|
|
104
|
+
// - Quoted values. The value class excluded quotes, so `password="a b c"`
|
|
105
|
+
// matched nothing at all.
|
|
106
|
+
//
|
|
107
|
+
// Quoted values keep their delimiters so a redacted JSON payload stays
|
|
108
|
+
// parseable. All quantifiers are bounded and the alternation is
|
|
109
|
+
// non-backtracking on failure, so the rule stays ReDoS-safe.
|
|
110
|
+
(input) => input.replace(SECRET_ASSIGNMENT_PATTERN, (_match, key, separator, value) => {
|
|
111
|
+
// Preserve the value's quote style so a redacted JSON payload stays
|
|
112
|
+
// parseable and a redacted YAML line stays well-formed.
|
|
113
|
+
const quote = value.startsWith('\\"')
|
|
114
|
+
? '\\"'
|
|
115
|
+
: value.startsWith('"')
|
|
116
|
+
? '"'
|
|
117
|
+
: value.startsWith("'")
|
|
118
|
+
? "'"
|
|
119
|
+
: "";
|
|
120
|
+
return `${key}${separator}${quote}[REDACTED]${quote}`;
|
|
121
|
+
}),
|
|
62
122
|
];
|
|
63
123
|
}
|
|
64
124
|
export function applyRedaction(input, options) {
|