sessionmem 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +39 -0
- package/dist/adapters/claudeMdInjector.js +45 -67
- package/dist/adapters/diagnostics.js +29 -4
- package/dist/adapters/generic.js +31 -5
- package/dist/adapters/global/claudeCode.js +6 -3
- package/dist/adapters/ide/cline.js +2 -1
- package/dist/adapters/ide/cursor.js +3 -3
- package/dist/adapters/ide/installer.js +1 -0
- package/dist/adapters/installedHosts.js +4 -1
- package/dist/cli/commands/config.js +7 -0
- package/dist/cli/commands/doctor.js +26 -2
- package/dist/cli/commands/install.js +4 -3
- package/dist/cli/commands/promptSubmit.js +58 -0
- package/dist/cli/commands/sessionStart.js +14 -2
- package/dist/cli/index.js +2 -0
- package/dist/cli/projectId.js +17 -2
- package/dist/core/api/contracts.js +19 -0
- package/dist/core/api/memoryCoreService.js +58 -31
- package/dist/core/config/policyConfig.js +10 -0
- package/dist/core/retrieve/memoryIndex.js +6 -0
- package/dist/core/retrieve/retrieveMemories.js +40 -8
- package/dist/core/retrieve/signalFusion.js +21 -0
- package/dist/core/retrieve/temporalIntent.js +59 -0
- package/dist/core/storage/entityLinkRepo.js +47 -0
- package/dist/core/storage/memorySearchRepo.js +5 -3
- package/dist/core/storage/temporalRepo.js +13 -0
- package/dist/core/storage/timelineRepo.js +27 -0
- package/docs/automatic-memory.md +86 -0
- package/docs/benchmark.md +7 -0
- package/docs/local-semantic.md +3 -1
- package/docs/retrieval-tools.md +144 -0
- package/package.json +1 -1
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { createCliContext } from "../context.js";
|
|
2
|
+
import { deriveProjectId } from "../projectId.js";
|
|
3
|
+
/** Bounded hook input. Invalid input produces no context and never blocks a prompt. */
|
|
4
|
+
export async function promptSubmitCommand(ctx, supplied) {
|
|
5
|
+
let context;
|
|
6
|
+
try {
|
|
7
|
+
let payload = supplied;
|
|
8
|
+
if (payload === undefined) {
|
|
9
|
+
payload = await new Promise(resolve => {
|
|
10
|
+
let text = "", finished = false;
|
|
11
|
+
const done = () => {
|
|
12
|
+
if (finished)
|
|
13
|
+
return;
|
|
14
|
+
finished = true;
|
|
15
|
+
clearTimeout(timer);
|
|
16
|
+
process.stdin.removeListener("data", data);
|
|
17
|
+
process.stdin.removeListener("end", done);
|
|
18
|
+
process.stdin.removeListener("error", done);
|
|
19
|
+
process.stdin.pause();
|
|
20
|
+
try {
|
|
21
|
+
resolve(JSON.parse(text));
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
resolve({});
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
const data = (chunk) => { text += chunk; if (text.length > 65536) {
|
|
28
|
+
text = "";
|
|
29
|
+
done();
|
|
30
|
+
} };
|
|
31
|
+
const timer = setTimeout(done, 500);
|
|
32
|
+
timer.unref();
|
|
33
|
+
process.stdin.setEncoding("utf8");
|
|
34
|
+
process.stdin.on("data", data);
|
|
35
|
+
process.stdin.on("end", done);
|
|
36
|
+
process.stdin.on("error", done);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
if (!payload || typeof payload !== "object")
|
|
40
|
+
return;
|
|
41
|
+
const { prompt, cwd } = payload;
|
|
42
|
+
if (typeof prompt !== "string" || !prompt.trim())
|
|
43
|
+
return;
|
|
44
|
+
context = ctx ?? createCliContext(typeof cwd === "string" ? { projectId: deriveProjectId(cwd) } : {});
|
|
45
|
+
const result = await context.service.call("retrieveMemories", {
|
|
46
|
+
projectId: context.projectId, query: prompt.slice(0, 1000), limit: 5, mode: "on-demand", depth: "default",
|
|
47
|
+
});
|
|
48
|
+
if (result.ok && result.startupInjection.trim())
|
|
49
|
+
process.stdout.write(JSON.stringify({
|
|
50
|
+
hookSpecificOutput: { hookEventName: "UserPromptSubmit", additionalContext: result.startupInjection },
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
catch { /* Hook failures do not interrupt the host. Doctor diagnoses the setup. */ }
|
|
54
|
+
finally {
|
|
55
|
+
if (!ctx)
|
|
56
|
+
context?.db.close();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sweepTranscripts } from "../transcriptSweep.js";
|
|
1
2
|
import { createCliContext } from "../context.js";
|
|
2
3
|
/**
|
|
3
4
|
* Generic startup query. When few memories contain these literal tokens the
|
|
@@ -25,8 +26,15 @@ const STARTUP_QUERY = "session startup context recent decisions architecture war
|
|
|
25
26
|
* starts clean.
|
|
26
27
|
*/
|
|
27
28
|
export async function sessionStartCommand(ctx) {
|
|
29
|
+
let context;
|
|
28
30
|
try {
|
|
29
|
-
|
|
31
|
+
context = ctx ?? createCliContext();
|
|
32
|
+
if (!ctx) {
|
|
33
|
+
try {
|
|
34
|
+
await sweepTranscripts(context, {});
|
|
35
|
+
}
|
|
36
|
+
catch { /* Existing recall still works when transcript discovery fails. */ }
|
|
37
|
+
}
|
|
30
38
|
const result = await context.service.call("retrieveMemories", {
|
|
31
39
|
projectId: context.projectId,
|
|
32
40
|
query: STARTUP_QUERY,
|
|
@@ -47,6 +55,10 @@ export async function sessionStartCommand(ctx) {
|
|
|
47
55
|
}
|
|
48
56
|
}
|
|
49
57
|
catch {
|
|
50
|
-
// Never block session start
|
|
58
|
+
// Never block session start.
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
if (!ctx)
|
|
62
|
+
context?.db.close();
|
|
51
63
|
}
|
|
52
64
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { autostartCommand } from "./commands/autostart.js";
|
|
|
3
3
|
import { Command } from "commander";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
5
|
import { runMcpServer } from "./commands/run.js";
|
|
6
|
+
import { promptSubmitCommand } from "./commands/promptSubmit.js";
|
|
6
7
|
import { sessionStartCommand } from "./commands/sessionStart.js";
|
|
7
8
|
import { sessionEndCommand } from "./commands/sessionEnd.js";
|
|
8
9
|
import { installCommand } from "./commands/install.js";
|
|
@@ -51,6 +52,7 @@ program
|
|
|
51
52
|
.command("session-start")
|
|
52
53
|
.description("Print prior memory context for the current project as Claude Code SessionStart hook output (invoked automatically by the hook installed during `sessionmem install`)")
|
|
53
54
|
.action(() => sessionStartCommand());
|
|
55
|
+
program.command("prompt-submit").description("Inject relevant context for a UserPromptSubmit hook").action(() => promptSubmitCommand());
|
|
54
56
|
program
|
|
55
57
|
.command("session-end")
|
|
56
58
|
.description("Run the session-end pipeline (auto-summarize ingested session events + light retention prune) for the current project (invoked automatically by the SessionEnd hook installed during `sessionmem install`)")
|
package/dist/cli/projectId.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { existsSync } from "node:fs";
|
|
2
|
+
import { existsSync, realpathSync } from "node:fs";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
/** Safety bound on the upward walk in {@link findProjectRoot}. */
|
|
5
5
|
const MAX_PARENT_WALK = 64;
|
|
@@ -110,5 +110,20 @@ export function deriveProjectId(cwd = process.cwd()) {
|
|
|
110
110
|
return sanitized;
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
|
-
|
|
113
|
+
const canonical = canonicalDir(cwd);
|
|
114
|
+
return projectIdFromCwd(findProjectRoot(canonical) ?? canonical);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Resolve symlinks so the same directory hashes to one id however it is
|
|
118
|
+
* spelled. process.cwd() is already symlink-resolved, but a hook payload cwd is
|
|
119
|
+
* not (on macOS /var/... vs /private/var/...). Falls back to the input when the
|
|
120
|
+
* path does not exist.
|
|
121
|
+
*/
|
|
122
|
+
function canonicalDir(dir) {
|
|
123
|
+
try {
|
|
124
|
+
return realpathSync(dir);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return dir;
|
|
128
|
+
}
|
|
114
129
|
}
|
|
@@ -179,6 +179,25 @@ export const getMemoryRequestSchema = z.object({
|
|
|
179
179
|
projectId: z.string().min(1),
|
|
180
180
|
memoryId: z.string().min(1).max(200),
|
|
181
181
|
});
|
|
182
|
+
export const searchMemoryIndexRequestSchema = retrieveMemoriesRequestSchema
|
|
183
|
+
.pick({ projectId: true, query: true, includeHistory: true, tags: true })
|
|
184
|
+
.extend({ limit: z.number().int().min(1).max(50).default(10) });
|
|
185
|
+
export const memoryTimelineRequestSchema = z.object({
|
|
186
|
+
projectId: z.string().min(1),
|
|
187
|
+
memoryId: z.string().min(1).max(200).optional(),
|
|
188
|
+
fromDate: z.iso.datetime().optional(),
|
|
189
|
+
toDate: z.iso.datetime().optional(),
|
|
190
|
+
includeHistory: z.boolean().default(true),
|
|
191
|
+
limit: z.number().int().min(1).max(50).default(10),
|
|
192
|
+
}).refine(value => value.memoryId !== undefined
|
|
193
|
+
? value.fromDate === undefined && value.toDate === undefined
|
|
194
|
+
: value.fromDate !== undefined && value.toDate !== undefined && Date.parse(value.fromDate) <= Date.parse(value.toDate), { message: "Provide memoryId alone, or fromDate and toDate in chronological order" });
|
|
195
|
+
export const memoryIndexItemSchema = z.object({
|
|
196
|
+
id: z.string(), summary: z.string().max(160), kind: z.string(), date: z.string(), score: z.number().nullable(),
|
|
197
|
+
});
|
|
198
|
+
export const memoryIndexResponseSchema = z.object({
|
|
199
|
+
ok: z.literal(true), memories: z.array(memoryIndexItemSchema), hasMore: z.boolean(),
|
|
200
|
+
});
|
|
182
201
|
export const forgetMemoryRequestSchema = z.object({
|
|
183
202
|
projectId: z.string().min(1),
|
|
184
203
|
memoryId: z.string().min(1).max(200),
|
|
@@ -3,6 +3,9 @@ import { ZodError } from "zod";
|
|
|
3
3
|
import { deterministicEmbed } from "../embed/deterministicEmbed.js";
|
|
4
4
|
import { loadLocalSemanticEmbedder } from "../embed/localSemanticEmbedder.js";
|
|
5
5
|
import { indexSemanticMemories, searchSemanticMemories } from "../storage/semanticSearchRepo.js";
|
|
6
|
+
import { detectTemporalIntent } from "../retrieve/temporalIntent.js";
|
|
7
|
+
import { memoryIndexItem } from "../retrieve/memoryIndex.js";
|
|
8
|
+
import { memoryTimeline } from "../storage/timelineRepo.js";
|
|
6
9
|
import { retrieveMemories } from "../retrieve/retrieveMemories.js";
|
|
7
10
|
import { computeEffectiveImportance } from "../retrieve/score.js";
|
|
8
11
|
import { formatStartupInjection } from "../injection/formatStartupInjection.js";
|
|
@@ -11,7 +14,7 @@ import { countAllMemoriesByProject, countMemoriesBySession, countMemoriesOlderTh
|
|
|
11
14
|
import { SESSION_WRITE_SOFT_LIMIT, configFilePath, DEEP_MODE_RETRIEVAL_CAP, readPolicyConfig, resolvePolicySettings, } from "../config/policyConfig.js";
|
|
12
15
|
import { insertMemoryFeedbackEvent } from "../storage/memoryFeedbackRepo.js";
|
|
13
16
|
import { countAllSessionEvents, insertSessionEvent, nextSessionEventIndex, } from "../storage/sessionEventsRepo.js";
|
|
14
|
-
import { batchStoreMemoryItemSchema, batchStoreMemoryRequestSchema, exportMemoriesRequestSchema, forgetMemoryRequestSchema, getMemoryRequestSchema, handleSessionEndRequestSchema, importMemoriesRequestSchema, pullMemoriesRequestSchema, ingestSessionEventsRequestSchema, LIST_MEMORIES_DEFAULT_LIMIT, listMemoriesRequestSchema, pruneMemoriesRequestSchema, redactExistingRequestSchema, resetAccessCountsRequestSchema, retrieveMemoriesRequestSchema, statsRequestSchema, storeMemoryRequestSchema, summarizeSessionToMemoryRequestSchema, } from "./contracts.js";
|
|
17
|
+
import { batchStoreMemoryItemSchema, batchStoreMemoryRequestSchema, exportMemoriesRequestSchema, forgetMemoryRequestSchema, getMemoryRequestSchema, handleSessionEndRequestSchema, importMemoriesRequestSchema, pullMemoriesRequestSchema, ingestSessionEventsRequestSchema, LIST_MEMORIES_DEFAULT_LIMIT, listMemoriesRequestSchema, pruneMemoriesRequestSchema, redactExistingRequestSchema, resetAccessCountsRequestSchema, retrieveMemoriesRequestSchema, searchMemoryIndexRequestSchema, memoryTimelineRequestSchema, statsRequestSchema, storeMemoryRequestSchema, summarizeSessionToMemoryRequestSchema, } from "./contracts.js";
|
|
15
18
|
import { DomainError, toErrorEnvelope } from "./errors.js";
|
|
16
19
|
import { assertLocalOnlyPolicy, } from "./localOnlyPolicy.js";
|
|
17
20
|
import { createSessionLifecycleService } from "./sessionLifecycleService.js";
|
|
@@ -189,6 +192,48 @@ export function createMemoryCoreService(deps) {
|
|
|
189
192
|
now: deps.now,
|
|
190
193
|
deleteOldMemories: deps.deleteOldMemories,
|
|
191
194
|
});
|
|
195
|
+
async function rankMemories(parsed, limit) {
|
|
196
|
+
const now = deps.now?.() ?? new Date();
|
|
197
|
+
const config = readPolicyConfig(policyConfigPath);
|
|
198
|
+
const intent = config.temporalRetrieval ? detectTemporalIntent(parsed.query, now) : { kind: "none" };
|
|
199
|
+
const includeHistory = parsed.includeHistory ?? intent.kind === "past";
|
|
200
|
+
let semanticCandidates;
|
|
201
|
+
try {
|
|
202
|
+
const embedder = await semanticEmbedder();
|
|
203
|
+
if (embedder)
|
|
204
|
+
semanticCandidates = await searchSemanticMemories(db, parsed.projectId, parsed.query, now, includeHistory, embedder);
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
semanticWarning();
|
|
208
|
+
}
|
|
209
|
+
const rankedAll = retrieveMemories({
|
|
210
|
+
db,
|
|
211
|
+
projectId: parsed.projectId,
|
|
212
|
+
queryText: parsed.query,
|
|
213
|
+
includeHistory,
|
|
214
|
+
temporalRetrieval: config.temporalRetrieval,
|
|
215
|
+
// Measured mode-specific default: coverage helps BM25, but regresses
|
|
216
|
+
// MiniLM recall. An explicit override always wins, including false.
|
|
217
|
+
entityLinkBoost: config.entityLinkBoost ?? (!semanticCandidates?.length && (!config.retrievalRanking || config.retrievalRanking === "legacy")),
|
|
218
|
+
ranking: config.retrievalRanking,
|
|
219
|
+
limit,
|
|
220
|
+
now,
|
|
221
|
+
semanticCandidates,
|
|
222
|
+
});
|
|
223
|
+
// Optional tag filter: keep a memory only when its stored tags contain ALL
|
|
224
|
+
// requested tags. The semantic candidate projection omits tags, so re-read
|
|
225
|
+
// each row's tags by id (this is an explicit, infrequent filter path).
|
|
226
|
+
const ranked = parsed.tags && parsed.tags.length > 0
|
|
227
|
+
? rankedAll.filter((m) => {
|
|
228
|
+
const rec = getMemoryRecordById(db, parsed.projectId, m.id);
|
|
229
|
+
const recTags = rec?.tags
|
|
230
|
+
? JSON.parse(rec.tags)
|
|
231
|
+
: [];
|
|
232
|
+
return parsed.tags.every((t) => recTags.includes(t));
|
|
233
|
+
})
|
|
234
|
+
: rankedAll;
|
|
235
|
+
return ranked.slice(0, limit);
|
|
236
|
+
}
|
|
192
237
|
const methods = {
|
|
193
238
|
async ingestSessionEvents(request) {
|
|
194
239
|
const parsed = parseRequest(ingestSessionEventsRequestSchema, request);
|
|
@@ -340,36 +385,7 @@ export function createMemoryCoreService(deps) {
|
|
|
340
385
|
async retrieveMemories(request) {
|
|
341
386
|
const parsed = parseRequest(retrieveMemoriesRequestSchema, request);
|
|
342
387
|
const limit = parsed.depth === "deep" ? Math.min(parsed.limit * 2, DEEP_MODE_RETRIEVAL_CAP) : parsed.limit;
|
|
343
|
-
|
|
344
|
-
try {
|
|
345
|
-
const embedder = await semanticEmbedder();
|
|
346
|
-
if (embedder)
|
|
347
|
-
semanticCandidates = await searchSemanticMemories(db, parsed.projectId, parsed.query, deps.now?.() ?? new Date(), parsed.includeHistory ?? false, embedder);
|
|
348
|
-
}
|
|
349
|
-
catch {
|
|
350
|
-
semanticWarning();
|
|
351
|
-
}
|
|
352
|
-
const rankedAll = retrieveMemories({
|
|
353
|
-
db,
|
|
354
|
-
projectId: parsed.projectId,
|
|
355
|
-
queryText: parsed.query,
|
|
356
|
-
includeHistory: parsed.includeHistory,
|
|
357
|
-
limit,
|
|
358
|
-
now: deps.now?.(),
|
|
359
|
-
semanticCandidates,
|
|
360
|
-
});
|
|
361
|
-
// Optional tag filter: keep a memory only when its stored tags contain ALL
|
|
362
|
-
// requested tags. The semantic candidate projection omits tags, so re-read
|
|
363
|
-
// each row's tags by id (this is an explicit, infrequent filter path).
|
|
364
|
-
const ranked = parsed.tags && parsed.tags.length > 0
|
|
365
|
-
? rankedAll.filter((m) => {
|
|
366
|
-
const rec = getMemoryRecordById(db, parsed.projectId, m.id);
|
|
367
|
-
const recTags = rec?.tags
|
|
368
|
-
? JSON.parse(rec.tags)
|
|
369
|
-
: [];
|
|
370
|
-
return parsed.tags.every((t) => recTags.includes(t));
|
|
371
|
-
})
|
|
372
|
-
: rankedAll;
|
|
388
|
+
const ranked = await rankMemories(parsed, limit);
|
|
373
389
|
if (ranked.length > 0 && parsed.mode !== "on-demand") {
|
|
374
390
|
// Only boost access counts for startup injection (mode='auto'), not for
|
|
375
391
|
// explicit on-demand retrieval, so a mid-session lookup does not inflate
|
|
@@ -389,6 +405,17 @@ export function createMemoryCoreService(deps) {
|
|
|
389
405
|
}),
|
|
390
406
|
};
|
|
391
407
|
},
|
|
408
|
+
async searchMemoryIndex(request) {
|
|
409
|
+
const parsed = parseRequest(searchMemoryIndexRequestSchema, request);
|
|
410
|
+
const ranked = await rankMemories(parsed, parsed.limit + 1);
|
|
411
|
+
return { ok: true, memories: ranked.slice(0, parsed.limit).map(row => memoryIndexItem(row, row.score.total)),
|
|
412
|
+
hasMore: ranked.length > parsed.limit };
|
|
413
|
+
},
|
|
414
|
+
async memoryTimeline(request) {
|
|
415
|
+
const parsed = parseRequest(memoryTimelineRequestSchema, request);
|
|
416
|
+
const { rows, hasMore } = memoryTimeline(db, parsed, deps.now?.() ?? new Date());
|
|
417
|
+
return { ok: true, memories: rows.map(row => memoryIndexItem(row)), hasMore };
|
|
418
|
+
},
|
|
392
419
|
async listMemories(request) {
|
|
393
420
|
const parsed = parseRequest(listMemoriesRequestSchema, request);
|
|
394
421
|
const all = listMemoriesByProject(db, parsed.projectId);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { RETRIEVAL_RANKINGS } from "../retrieve/signalFusion.js";
|
|
2
3
|
import { homedir } from "os";
|
|
3
4
|
import { join, dirname } from "path";
|
|
4
5
|
import { mkdirSync, readFileSync, writeFileSync, renameSync } from "fs";
|
|
@@ -54,6 +55,9 @@ const policyConfigShape = {
|
|
|
54
55
|
// `savings` analytics. Omitted by default so existing configs keep the default.
|
|
55
56
|
injectionCap: z.number().int().min(100).max(10000).optional(),
|
|
56
57
|
semanticSearch: z.boolean().optional(),
|
|
58
|
+
temporalRetrieval: z.boolean().optional(),
|
|
59
|
+
entityLinkBoost: z.boolean().optional(),
|
|
60
|
+
retrievalRanking: z.enum(RETRIEVAL_RANKINGS).optional(),
|
|
57
61
|
team: teamConfigShape.default({ enabled: false }),
|
|
58
62
|
};
|
|
59
63
|
/**
|
|
@@ -109,12 +113,18 @@ export function readPolicyConfig(filePath) {
|
|
|
109
113
|
const re = s.redactionEnabled.safeParse(obj.redactionEnabled);
|
|
110
114
|
const ic = s.injectionCap.safeParse(obj.injectionCap);
|
|
111
115
|
const ss = s.semanticSearch.safeParse(obj.semanticSearch);
|
|
116
|
+
const tr = s.temporalRetrieval.safeParse(obj.temporalRetrieval);
|
|
117
|
+
const el = s.entityLinkBoost.safeParse(obj.entityLinkBoost);
|
|
118
|
+
const rr = s.retrievalRanking.safeParse(obj.retrievalRanking);
|
|
112
119
|
const tm = s.team.safeParse(obj.team);
|
|
113
120
|
return {
|
|
114
121
|
retentionDays: rd.success ? rd.data : DEFAULT_POLICY_CONFIG.retentionDays,
|
|
115
122
|
redactionEnabled: re.success ? re.data : DEFAULT_POLICY_CONFIG.redactionEnabled,
|
|
116
123
|
injectionCap: ic.success ? ic.data : undefined,
|
|
117
124
|
semanticSearch: ss.success ? ss.data : undefined,
|
|
125
|
+
temporalRetrieval: tr.success ? tr.data : undefined,
|
|
126
|
+
entityLinkBoost: el.success ? el.data : undefined,
|
|
127
|
+
retrievalRanking: rr.success ? rr.data : undefined,
|
|
118
128
|
team: tm.success ? tm.data : { ...DEFAULT_POLICY_CONFIG.team },
|
|
119
129
|
};
|
|
120
130
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Extractive preview only. No generated summary or extra inference. */
|
|
2
|
+
export function memoryIndexItem(memory, score = null) {
|
|
3
|
+
const text = memory.content.replace(/\s+/g, " ").trim();
|
|
4
|
+
return { id: memory.id, summary: text.length > 160 ? `${text.slice(0, 157)}...` : text,
|
|
5
|
+
kind: memory.kind, date: memory.created_at, score: score === null ? null : Math.round(score * 10000) / 10000 };
|
|
6
|
+
}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { entityOverlap } from "../storage/knowledgeRepo.js";
|
|
2
|
+
import { linkedEntityScores } from "../storage/entityLinkRepo.js";
|
|
3
|
+
import { memoryTemporalMetadata } from "../storage/temporalRepo.js";
|
|
4
|
+
import { adaptiveLexicalScore, additiveRelevance, SEMANTIC_GATE } from "./signalFusion.js";
|
|
5
|
+
import { detectTemporalIntent, temporalRelevance } from "./temporalIntent.js";
|
|
2
6
|
import { deterministicEmbed } from "../embed/deterministicEmbed.js";
|
|
3
7
|
import { decayOldBoosts } from "./decay.js";
|
|
4
8
|
import { scoreMemoryCandidate, } from "./score.js";
|
|
5
|
-
import { searchMemoryCandidatesFTS, } from "../storage/memorySearchRepo.js";
|
|
9
|
+
import { searchMemoryCandidatesFTS, searchMemoryCandidatesByIds, } from "../storage/memorySearchRepo.js";
|
|
6
10
|
const DEFAULT_EMBEDDING_DIMENSION = 32;
|
|
7
11
|
// Small, fixed contribution inside the existing relevance component. Keep the
|
|
8
12
|
// public score shape/outer recency and importance weights unchanged.
|
|
@@ -45,11 +49,25 @@ export function retrieveMemories(input) {
|
|
|
45
49
|
}
|
|
46
50
|
const topK = input.topK ?? input.limit ?? 20;
|
|
47
51
|
const now = input.now ?? new Date();
|
|
52
|
+
const ranking = input.ranking ?? "legacy";
|
|
53
|
+
const additive = ranking === "hub-additive" || ranking === "adaptive-hub-additive";
|
|
54
|
+
const adaptive = ranking === "adaptive-bm25" || ranking === "adaptive-hub-additive";
|
|
55
|
+
const intent = input.temporalRetrieval ? detectTemporalIntent(queryText, now) : { kind: "none" };
|
|
56
|
+
const includeHistory = input.includeHistory ?? intent.kind === "past";
|
|
48
57
|
// FTS5 retrieves up to 50 BM25 matches; sparse matches retain the existing
|
|
49
58
|
// recency/importance fallback. The public request schema validates queryText.
|
|
50
|
-
const lexicalCandidates = searchMemoryCandidatesFTS(input.db, input.projectId, queryText, now,
|
|
59
|
+
const lexicalCandidates = searchMemoryCandidatesFTS(input.db, input.projectId, queryText, now, includeHistory);
|
|
51
60
|
const candidates = [...lexicalCandidates];
|
|
52
61
|
const ids = new Set(candidates.map(c => c.id));
|
|
62
|
+
const linked = input.entityLinkBoost || additive
|
|
63
|
+
? linkedEntityScores(input.db, input.projectId, queryText, now, includeHistory, additive)
|
|
64
|
+
: new Map();
|
|
65
|
+
for (const candidate of searchMemoryCandidatesByIds(input.db, input.projectId, [...linked.keys()].slice(0, 50), now, includeHistory)) {
|
|
66
|
+
if (!ids.has(candidate.id)) {
|
|
67
|
+
candidates.push(candidate);
|
|
68
|
+
ids.add(candidate.id);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
53
71
|
for (const { candidate } of input.semanticCandidates ?? []) {
|
|
54
72
|
if (!ids.has(candidate.id)) {
|
|
55
73
|
candidates.push(candidate);
|
|
@@ -61,10 +79,12 @@ export function retrieveMemories(input) {
|
|
|
61
79
|
const dimension = resolveEmbeddingDimension(candidates);
|
|
62
80
|
const queryVector = deterministicEmbed(queryText, dimension).vector;
|
|
63
81
|
const overlap = entityOverlap(input.db, input.projectId, queryText, now);
|
|
82
|
+
const temporal = intent.kind !== "none" ? memoryTemporalMetadata(input.db, input.projectId, candidates.map(c => c.id)) : new Map();
|
|
64
83
|
// FTS5 returns negative BM25 ranks; storage converts them to positive scores.
|
|
65
84
|
// Normalize within this query so BM25 fits the existing relevance weight.
|
|
66
85
|
const maxLexicalScore = candidates.reduce((max, candidate) => Math.max(max, candidate.lexical_score), 0);
|
|
67
86
|
const ranked = decayedCandidates
|
|
87
|
+
.filter(candidate => !additive || !input.semanticCandidates?.length || (dense.get(candidate.id) ?? 0) >= SEMANTIC_GATE)
|
|
68
88
|
.map((candidate) => {
|
|
69
89
|
// When embedding is null (version mismatch or missing), use a neutral
|
|
70
90
|
// score of 0.5 so the memory is neither penalized nor boosted. A non-null
|
|
@@ -78,7 +98,7 @@ export function retrieveMemories(input) {
|
|
|
78
98
|
// Hash cosine is useful for identical text, but adds noise to lexical
|
|
79
99
|
// matches. Keep it as a diagnostic and as the no-match fallback only.
|
|
80
100
|
const lexicalRelevance = maxLexicalScore > 0
|
|
81
|
-
? candidate.lexical_score / maxLexicalScore
|
|
101
|
+
? adaptive ? adaptiveLexicalScore(candidate.lexical_score, queryText) : candidate.lexical_score / maxLexicalScore
|
|
82
102
|
: semantic;
|
|
83
103
|
// The local dense variant was weaker alone. Preserve lexical dominance
|
|
84
104
|
// while admitting candidates absent from FTS, using a fixed 2:1 blend.
|
|
@@ -87,15 +107,27 @@ export function retrieveMemories(input) {
|
|
|
87
107
|
? (2 * lexicalRelevance + Math.max(0, dense.get(candidate.id) ?? 0)) / 3
|
|
88
108
|
: Math.max(0, dense.get(candidate.id) ?? 0)
|
|
89
109
|
: lexicalRelevance;
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
:
|
|
110
|
+
const fused = additive ? additiveRelevance({
|
|
111
|
+
lexical: maxLexicalScore > 0 ? lexicalRelevance : undefined,
|
|
112
|
+
dense: input.semanticCandidates?.length ? Math.max(0, dense.get(candidate.id) ?? 0) : undefined,
|
|
113
|
+
entity: linked.size > 0 ? linked.get(candidate.id) ?? 0 : undefined,
|
|
114
|
+
}) : null;
|
|
115
|
+
const entityRelevance = additive && fused !== null ? fused : linked.size > 0
|
|
116
|
+
? 0.8 * baseRelevance + 0.2 * (linked.get(candidate.id) ?? 0)
|
|
117
|
+
: overlap.size > 0
|
|
118
|
+
? (1 - ENTITY_RELEVANCE_WEIGHT) * baseRelevance + ENTITY_RELEVANCE_WEIGHT * (overlap.get(candidate.id) ?? 0)
|
|
119
|
+
: baseRelevance;
|
|
120
|
+
const metadata = temporal.get(candidate.id);
|
|
121
|
+
const relevance = metadata
|
|
122
|
+
? 0.9 * entityRelevance + 0.1 * temporalRelevance(intent, metadata.timestamp, metadata.supersededFraction, now)
|
|
123
|
+
: entityRelevance;
|
|
93
124
|
const score = scoreMemoryCandidate({
|
|
94
125
|
semantic: relevance,
|
|
95
|
-
|
|
126
|
+
// Historical lookup should not lose evidence merely because it is old.
|
|
127
|
+
updated_at: intent.kind === "past" ? now.toISOString() : candidate.updated_at,
|
|
96
128
|
importance: candidate.importance,
|
|
97
129
|
access_count: candidate.access_count,
|
|
98
|
-
decayedImportance: candidate.decayedImportance,
|
|
130
|
+
decayedImportance: intent.kind === "past" ? candidate.importance : candidate.decayedImportance,
|
|
99
131
|
}, now);
|
|
100
132
|
return {
|
|
101
133
|
id: candidate.id,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { lexicalQueryTerms } from "../storage/memorySearchRepo.js";
|
|
2
|
+
export const RETRIEVAL_RANKINGS = ["legacy", "adaptive-bm25", "hub-additive", "adaptive-hub-additive"];
|
|
3
|
+
/** Fixed hypotheses from the requested public-source audit, independently implemented. */
|
|
4
|
+
export function adaptiveLexicalScore(raw, query) {
|
|
5
|
+
if (raw <= 0 || !Number.isFinite(raw))
|
|
6
|
+
return 0;
|
|
7
|
+
const count = lexicalQueryTerms(query).length;
|
|
8
|
+
const bands = [[3, 5, 0.7], [6, 7, 0.6], [9, 9, 0.5], [15, 10, 0.5], [Infinity, 12, 0.5]];
|
|
9
|
+
const [, midpoint, steepness] = bands.find(([ceiling]) => count <= ceiling);
|
|
10
|
+
return 1 / (1 + Math.exp(steepness * (midpoint - raw)));
|
|
11
|
+
}
|
|
12
|
+
export const SEMANTIC_GATE = 0.2;
|
|
13
|
+
export function additiveRelevance(signals) {
|
|
14
|
+
if (signals.dense !== undefined && signals.dense < SEMANTIC_GATE)
|
|
15
|
+
return null;
|
|
16
|
+
const capacity = Number(signals.lexical !== undefined) + Number(signals.dense !== undefined) + 0.5 * Number(signals.entity !== undefined);
|
|
17
|
+
if (!capacity)
|
|
18
|
+
return 0;
|
|
19
|
+
const sum = (signals.lexical ?? 0) + (signals.dense ?? 0) + (signals.entity ?? 0);
|
|
20
|
+
return Math.max(0, Math.min(1, sum / capacity));
|
|
21
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
const DAY = 86_400_000;
|
|
2
|
+
function range(start, end) {
|
|
3
|
+
return { kind: "past", start, end };
|
|
4
|
+
}
|
|
5
|
+
export function detectTemporalIntent(query, now) {
|
|
6
|
+
const text = query.normalize("NFKC").toLowerCase();
|
|
7
|
+
// A dated or historical clause takes precedence over a current-state cue.
|
|
8
|
+
const iso = /\b(\d{4})-(\d{2})(?:-(\d{2}))?\b/.exec(text);
|
|
9
|
+
if (iso) {
|
|
10
|
+
const year = Number(iso[1]), month = Number(iso[2]) - 1, day = Number(iso[3] ?? 1);
|
|
11
|
+
const start = Date.UTC(year, month, day);
|
|
12
|
+
const date = new Date(start);
|
|
13
|
+
if (date.getUTCFullYear() === year && date.getUTCMonth() === month && date.getUTCDate() === day) {
|
|
14
|
+
return range(start, iso[3] ? start + DAY : Date.UTC(year, month + 1, 1));
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
|
|
18
|
+
if (/\byesterday\b/.test(text))
|
|
19
|
+
return range(today - DAY, today);
|
|
20
|
+
const relative = /\b(?:last|previous) (week|month|year)\b/.exec(text);
|
|
21
|
+
if (relative) {
|
|
22
|
+
const year = now.getUTCFullYear(), month = now.getUTCMonth();
|
|
23
|
+
if (relative[1] === "month")
|
|
24
|
+
return range(Date.UTC(year, month - 1, 1), Date.UTC(year, month, 1));
|
|
25
|
+
if (relative[1] === "year")
|
|
26
|
+
return range(Date.UTC(year - 1, 0, 1), Date.UTC(year, 0, 1));
|
|
27
|
+
// Calendar weeks start on Monday, in UTC.
|
|
28
|
+
const monday = today - ((now.getUTCDay() + 6) % 7) * DAY;
|
|
29
|
+
return range(monday - 7 * DAY, monday);
|
|
30
|
+
}
|
|
31
|
+
const ago = /\b(\d{1,3}) (day|week)s? ago\b/.exec(text);
|
|
32
|
+
if (ago) {
|
|
33
|
+
const start = today - Number(ago[1]) * (ago[2] === "week" ? 7 : 1) * DAY;
|
|
34
|
+
return range(start, start + (ago[2] === "week" ? 7 : 1) * DAY);
|
|
35
|
+
}
|
|
36
|
+
const year = /\b(?:in|during) ((?:19|20)\d{2})\b/.exec(text);
|
|
37
|
+
if (year)
|
|
38
|
+
return range(Date.UTC(Number(year[1]), 0, 1), Date.UTC(Number(year[1]) + 1, 0, 1));
|
|
39
|
+
if (/\b(before|previously|formerly|prior|used to|no longer)\b/.test(text))
|
|
40
|
+
return { kind: "past", previous: true };
|
|
41
|
+
if (/\b(when (?:did|was|were)|in the past|historical|history)\b/.test(text))
|
|
42
|
+
return { kind: "past" };
|
|
43
|
+
if (/\b(current|currently|now|latest|today|at present)\b/.test(text))
|
|
44
|
+
return { kind: "current" };
|
|
45
|
+
return { kind: "none" };
|
|
46
|
+
}
|
|
47
|
+
/** A bounded signal, separate from lexical relevance. Missing dates stay neutral. */
|
|
48
|
+
export function temporalRelevance(intent, timestamp, supersededFraction, now) {
|
|
49
|
+
const time = Date.parse(timestamp);
|
|
50
|
+
if (intent.kind === "current") {
|
|
51
|
+
const freshness = Number.isFinite(time) ? 1 / (1 + Math.max(0, now.getTime() - time) / (30 * DAY)) : 0.5;
|
|
52
|
+
return (1 - supersededFraction + freshness) / 2;
|
|
53
|
+
}
|
|
54
|
+
if (intent.start !== undefined && intent.end !== undefined && Number.isFinite(time)) {
|
|
55
|
+
const distance = time < intent.start ? intent.start - time : time >= intent.end ? time - intent.end + 1 : 0;
|
|
56
|
+
return 1 / (1 + distance / (30 * DAY));
|
|
57
|
+
}
|
|
58
|
+
return intent.previous ? supersededFraction : 0.5;
|
|
59
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { queryEntityNames } from "../extract/knowledge.js";
|
|
2
|
+
// Explicit spelling aliases only. No fuzzy matching or inferred coreference.
|
|
3
|
+
const ALIASES = [["postgres", "postgresql"], ["node", "node.js"], ["k8s", "kubernetes"]];
|
|
4
|
+
/** Link literal query names to existing entities, then weight rare names more. */
|
|
5
|
+
export function linkedEntityScores(db, projectId, query, now, includeHistory, hubDamping = false) {
|
|
6
|
+
const names = new Set(queryEntityNames(query));
|
|
7
|
+
for (const group of ALIASES)
|
|
8
|
+
if (group.some(name => names.has(name)))
|
|
9
|
+
for (const name of group)
|
|
10
|
+
names.add(name);
|
|
11
|
+
const rows = db.prepare(`SELECT DISTINCT e.memory_id, e.kind, e.name FROM memory_entities e
|
|
12
|
+
JOIN memories m ON m.id = e.memory_id
|
|
13
|
+
WHERE m.project_id = ? AND (m.expires_at IS NULL OR m.expires_at > ?)
|
|
14
|
+
AND (? = 1 OR m.id NOT IN (SELECT memory_id FROM superseded_memories))
|
|
15
|
+
AND e.name IN (SELECT value FROM json_each(?)) ORDER BY e.kind, e.name, e.memory_id`)
|
|
16
|
+
.all(projectId, now.toISOString(), Number(includeHistory), JSON.stringify([...names]));
|
|
17
|
+
const frequencies = new Map();
|
|
18
|
+
const key = (row) => `${row.kind}:${row.name}`;
|
|
19
|
+
for (const row of rows)
|
|
20
|
+
frequencies.set(key(row), (frequencies.get(key(row)) ?? 0) + 1);
|
|
21
|
+
if (hubDamping) {
|
|
22
|
+
// Match at most eight stored names, preferring specific spans. Exact and
|
|
23
|
+
// explicit alias matches have similarity 1; no entity model is implied.
|
|
24
|
+
const selected = new Set([...new Set(rows.map(row => row.name))]
|
|
25
|
+
.sort((a, b) => b.length - a.length || a.localeCompare(b)).slice(0, 8));
|
|
26
|
+
const boosts = new Map();
|
|
27
|
+
for (const row of rows) {
|
|
28
|
+
if (!selected.has(row.name))
|
|
29
|
+
continue;
|
|
30
|
+
const degree = frequencies.get(key(row));
|
|
31
|
+
const boost = 0.5 / (1 + 0.001 * (degree - 1) ** 2);
|
|
32
|
+
boosts.set(row.memory_id, Math.max(boosts.get(row.memory_id) ?? 0, boost));
|
|
33
|
+
}
|
|
34
|
+
return new Map([...boosts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
|
|
35
|
+
}
|
|
36
|
+
const total = db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE project_id = ?
|
|
37
|
+
AND (expires_at IS NULL OR expires_at > ?)
|
|
38
|
+
AND (? = 1 OR id NOT IN (SELECT memory_id FROM superseded_memories))`)
|
|
39
|
+
.get(projectId, now.toISOString(), Number(includeHistory)).n;
|
|
40
|
+
const weights = new Map([...frequencies].map(([name, count]) => [name, 1 + Math.log((total + 1) / (count + 1))]));
|
|
41
|
+
const denominator = [...weights.values()].reduce((a, b) => a + b, 0);
|
|
42
|
+
const scores = new Map();
|
|
43
|
+
for (const row of rows)
|
|
44
|
+
scores.set(row.memory_id, (scores.get(row.memory_id) ?? 0) + weights.get(key(row)) / denominator);
|
|
45
|
+
// Keep scores for all existing lexical/dense candidates. Only admission is capped.
|
|
46
|
+
return new Map([...scores].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
|
|
47
|
+
}
|
|
@@ -118,13 +118,15 @@ const QUERY_STOP_WORDS = new Set(("a an and are as at be been being but by can c
|
|
|
118
118
|
"where which who whom why will with would you your yours yourself yourselves " +
|
|
119
119
|
"s t don doesn didn isn wasn weren won wouldn couldn shouldn").split(" "));
|
|
120
120
|
/** Match any meaningful query term; question grammar must not exclude evidence. */
|
|
121
|
-
function
|
|
121
|
+
export function lexicalQueryTerms(queryText) {
|
|
122
122
|
const tokens = [...new Set(queryText.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])];
|
|
123
123
|
const terms = tokens.filter((token) => !QUERY_STOP_WORDS.has(token));
|
|
124
124
|
// Preserve literal searches consisting entirely of stop words. Tokens contain
|
|
125
125
|
// only letters/numbers, so FTS operators and punctuation cannot become syntax.
|
|
126
|
-
return (terms.length ? terms : tokens)
|
|
127
|
-
|
|
126
|
+
return (terms.length ? terms : tokens).slice(0, 64);
|
|
127
|
+
}
|
|
128
|
+
function sanitizeFtsQuery(queryText) {
|
|
129
|
+
return lexicalQueryTerms(queryText)
|
|
128
130
|
.map((token) => `"${token}"`)
|
|
129
131
|
.join(" OR ");
|
|
130
132
|
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Use claim observation times when available, otherwise the original capture time. */
|
|
2
|
+
export function memoryTemporalMetadata(db, projectId, ids) {
|
|
3
|
+
const rows = db.prepare(`SELECT m.id, m.created_at, MAX(c.observed_at) AS observed_at,
|
|
4
|
+
COUNT(c.claim_key) AS claims, COUNT(s.claim_key) AS superseded
|
|
5
|
+
FROM memories m LEFT JOIN memory_claims c ON c.memory_id = m.id
|
|
6
|
+
LEFT JOIN memory_supersessions s ON s.supersedes = c.memory_id AND s.claim_key = c.claim_key
|
|
7
|
+
WHERE m.project_id = ? AND m.id IN (SELECT value FROM json_each(?)) GROUP BY m.id`)
|
|
8
|
+
.all(projectId, JSON.stringify(ids));
|
|
9
|
+
return new Map(rows.map(row => [row.id, {
|
|
10
|
+
timestamp: row.observed_at ?? row.created_at,
|
|
11
|
+
supersededFraction: row.claims ? row.superseded / row.claims : 0,
|
|
12
|
+
}]));
|
|
13
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { DomainError } from "../api/errors.js";
|
|
2
|
+
/** Small SQL projections, stable ordering, and no access-count writes. */
|
|
3
|
+
export function memoryTimeline(db, input, now) {
|
|
4
|
+
const visible = `project_id = @projectId AND (expires_at IS NULL OR expires_at > @now)
|
|
5
|
+
AND (@history = 1 OR id NOT IN (SELECT memory_id FROM superseded_memories))`;
|
|
6
|
+
const bindings = { projectId: input.projectId, now: now.toISOString(), history: Number(input.includeHistory) };
|
|
7
|
+
// Project only enough text for the one-line summary, never embeddings or bodies.
|
|
8
|
+
const select = `SELECT id, substr(content, 1, 1000) AS content, kind, created_at FROM memories WHERE ${visible}`;
|
|
9
|
+
if (!input.memoryId) {
|
|
10
|
+
const rows = db.prepare(`${select} AND created_at >= @fromDate AND created_at <= @toDate
|
|
11
|
+
ORDER BY created_at, id LIMIT @limit`).all({ ...bindings,
|
|
12
|
+
fromDate: new Date(input.fromDate).toISOString(), toDate: new Date(input.toDate).toISOString(), limit: input.limit + 1, });
|
|
13
|
+
return { rows: rows.slice(0, input.limit), hasMore: rows.length > input.limit };
|
|
14
|
+
}
|
|
15
|
+
const anchor = db.prepare(`${select} AND id = @memoryId`).get({ ...bindings, memoryId: input.memoryId });
|
|
16
|
+
if (!anchor)
|
|
17
|
+
throw new DomainError("NOT_FOUND", `Memory not found: ${input.memoryId}`);
|
|
18
|
+
const params = { ...bindings, date: anchor.created_at, id: anchor.id, limit: input.limit + 1 };
|
|
19
|
+
const before = db.prepare(`${select} AND (created_at < @date OR (created_at = @date AND id < @id))
|
|
20
|
+
ORDER BY created_at DESC, id DESC LIMIT @limit`).all(params);
|
|
21
|
+
const after = db.prepare(`${select} AND (created_at > @date OR (created_at = @date AND id > @id))
|
|
22
|
+
ORDER BY created_at, id LIMIT @limit`).all(params);
|
|
23
|
+
const left = Math.min(before.length, Math.max(Math.floor((input.limit - 1) / 2), input.limit - 1 - after.length));
|
|
24
|
+
const right = Math.min(after.length, input.limit - 1 - left);
|
|
25
|
+
return { rows: [...before.slice(0, left).reverse(), anchor, ...after.slice(0, right)],
|
|
26
|
+
hasMore: before.length + after.length + 1 > input.limit };
|
|
27
|
+
}
|