sessionmem 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +122 -61
- package/dist/adapters/claudeMdInjector.js +45 -67
- package/dist/adapters/diagnostics.js +29 -4
- package/dist/adapters/generic.js +31 -5
- package/dist/adapters/global/claudeCode.js +6 -3
- package/dist/adapters/ide/cline.js +2 -1
- package/dist/adapters/ide/cursor.js +3 -3
- package/dist/adapters/ide/installer.js +1 -0
- package/dist/adapters/installedHosts.js +4 -1
- package/dist/cli/autostart/index.js +246 -0
- package/dist/cli/commands/autostart.js +24 -0
- package/dist/cli/commands/config.js +8 -0
- package/dist/cli/commands/doctor.js +45 -2
- package/dist/cli/commands/install.js +14 -3
- package/dist/cli/commands/promptSubmit.js +58 -0
- package/dist/cli/commands/sessionStart.js +14 -2
- package/dist/cli/commands/uninstall.js +4 -0
- package/dist/cli/commands/watch.js +14 -2
- package/dist/cli/index.js +12 -3
- package/dist/cli/projectId.js +17 -2
- package/dist/cli/watchCheckpoint.js +35 -0
- package/dist/cli/watchLog.js +23 -0
- package/dist/core/api/contracts.js +19 -0
- package/dist/core/api/memoryCoreService.js +84 -21
- package/dist/core/config/policyConfig.js +13 -0
- package/dist/core/embed/localSemanticEmbedder.js +62 -0
- package/dist/core/retrieve/memoryIndex.js +6 -0
- package/dist/core/retrieve/retrieveMemories.js +57 -9
- package/dist/core/retrieve/signalFusion.js +21 -0
- package/dist/core/retrieve/temporalIntent.js +59 -0
- package/dist/core/schema/migrations/016_semantic_embeddings.sql +12 -0
- package/dist/core/storage/entityLinkRepo.js +47 -0
- package/dist/core/storage/memorySearchRepo.js +16 -3
- package/dist/core/storage/semanticSearchRepo.js +56 -0
- package/dist/core/storage/temporalRepo.js +13 -0
- package/dist/core/storage/timelineRepo.js +27 -0
- package/docs/architecture.md +13 -1
- package/docs/automatic-memory.md +86 -0
- package/docs/benchmark.md +16 -0
- package/docs/local-semantic.md +58 -0
- package/docs/research/automatic-capture.md +62 -0
- package/docs/retrieval-tools.md +144 -0
- package/docs/troubleshooting.md +5 -0
- package/package.json +21 -2
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { openWatchLog } from "../watchLog.js";
|
|
2
|
+
import { saveWatchCheckpoint } from "../watchCheckpoint.js";
|
|
1
3
|
import { setTimeout } from "node:timers/promises";
|
|
2
4
|
import { createCliContext } from "../context.js";
|
|
3
5
|
import { printSweepSummary, sweepTranscripts } from "../transcriptSweep.js";
|
|
@@ -6,16 +8,20 @@ export async function watchCommand(options = {}, ctx) {
|
|
|
6
8
|
if (!Number.isFinite(interval) || interval <= 0 || interval > 86400) {
|
|
7
9
|
throw new Error("--interval must be a number greater than 0 and at most 86400 seconds");
|
|
8
10
|
}
|
|
9
|
-
const
|
|
11
|
+
const closeLog = openWatchLog(options.logFile);
|
|
12
|
+
let context;
|
|
10
13
|
const controller = new AbortController();
|
|
11
14
|
const stop = () => controller.abort();
|
|
12
15
|
process.once("SIGINT", stop);
|
|
13
16
|
process.once("SIGTERM", stop);
|
|
14
17
|
let firstSweep = true;
|
|
15
18
|
try {
|
|
19
|
+
console.log(`sessionmem watch starting (pid ${process.pid}, interval ${interval}s)`);
|
|
20
|
+
context = ctx ?? createCliContext({ dbPath: options.dbPath });
|
|
16
21
|
do {
|
|
17
22
|
try {
|
|
18
23
|
const summary = await sweepTranscripts(context, options);
|
|
24
|
+
saveWatchCheckpoint(context, summary, interval);
|
|
19
25
|
if (firstSweep || options.once || summary.sessionsImported || summary.errors) {
|
|
20
26
|
printSweepSummary("watch", summary);
|
|
21
27
|
}
|
|
@@ -39,10 +45,16 @@ export async function watchCommand(options = {}, ctx) {
|
|
|
39
45
|
}
|
|
40
46
|
} while (!controller.signal.aborted);
|
|
41
47
|
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
console.error(`[sessionmem] watch failed: ${error instanceof Error ? error.stack ?? error.message : String(error)}`);
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
42
52
|
finally {
|
|
43
53
|
process.removeListener("SIGINT", stop);
|
|
44
54
|
process.removeListener("SIGTERM", stop);
|
|
45
55
|
if (!ctx)
|
|
46
|
-
context
|
|
56
|
+
context?.db.close();
|
|
57
|
+
console.log("sessionmem watch stopped");
|
|
58
|
+
closeLog();
|
|
47
59
|
}
|
|
48
60
|
}
|
package/dist/cli/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { autostartCommand } from "./commands/autostart.js";
|
|
2
3
|
import { Command } from "commander";
|
|
3
4
|
import { createRequire } from "node:module";
|
|
4
5
|
import { runMcpServer } from "./commands/run.js";
|
|
6
|
+
import { promptSubmitCommand } from "./commands/promptSubmit.js";
|
|
5
7
|
import { sessionStartCommand } from "./commands/sessionStart.js";
|
|
6
8
|
import { sessionEndCommand } from "./commands/sessionEnd.js";
|
|
7
9
|
import { installCommand } from "./commands/install.js";
|
|
@@ -50,6 +52,7 @@ program
|
|
|
50
52
|
.command("session-start")
|
|
51
53
|
.description("Print prior memory context for the current project as Claude Code SessionStart hook output (invoked automatically by the hook installed during `sessionmem install`)")
|
|
52
54
|
.action(() => sessionStartCommand());
|
|
55
|
+
program.command("prompt-submit").description("Inject relevant context for a UserPromptSubmit hook").action(() => promptSubmitCommand());
|
|
53
56
|
program
|
|
54
57
|
.command("session-end")
|
|
55
58
|
.description("Run the session-end pipeline (auto-summarize ingested session events + light retention prune) for the current project (invoked automatically by the SessionEnd hook installed during `sessionmem install`)")
|
|
@@ -69,22 +72,28 @@ program
|
|
|
69
72
|
.description("Poll Claude Code, Codex, and Cursor transcripts and capture sessions offline")
|
|
70
73
|
.option("--transcripts-dir <path>", "Override transcript roots (Cursor: User directory or state.vscdb)")
|
|
71
74
|
.option("--host <host>", "Select a transcript host: claude-code, codex, or cursor")
|
|
75
|
+
.option("--db-path <path>", "Database file for headless capture")
|
|
76
|
+
.option("--log-file <path>", "Append timestamped stdout, stderr and startup failures to a file")
|
|
72
77
|
.option("--once", "Run one sweep and exit")
|
|
73
78
|
.option("--interval <seconds>", "Seconds between sweeps", "5")
|
|
74
79
|
.action((options) => watchCommand(options));
|
|
75
80
|
program
|
|
76
81
|
.command("install")
|
|
77
|
-
.description("Install
|
|
82
|
+
.description("Install integrations and enable login transcript capture")
|
|
78
83
|
.option("--adapter <name>", "Force a host adapter instead of auto-detecting (claude-code, cursor, windsurf, cline, codex, antigravity, qcoder, generic)")
|
|
79
84
|
.action((options) => installCommand(options));
|
|
85
|
+
program
|
|
86
|
+
.command("autostart <action>")
|
|
87
|
+
.description("Control login capture: enable, disable, or status")
|
|
88
|
+
.action((action) => autostartCommand(action));
|
|
80
89
|
program
|
|
81
90
|
.command("doctor")
|
|
82
|
-
.description("Check storage,
|
|
91
|
+
.description("Check login capture, storage, transcripts, hooks, MCP wiring, and recall offline")
|
|
83
92
|
.option("--adapter <name>", "Check a specific host instead of installed-host detection")
|
|
84
93
|
.action(async (options) => { await doctorCommand(options); });
|
|
85
94
|
program
|
|
86
95
|
.command("uninstall")
|
|
87
|
-
.description("
|
|
96
|
+
.description("Stop login capture and remove sessionmem from the current MCP host")
|
|
88
97
|
.option("--purge", "Also delete the local memories database")
|
|
89
98
|
.option("--adapter <name>", "Force a host adapter instead of auto-detecting (claude-code, cursor, windsurf, cline, codex, antigravity, qcoder, generic)")
|
|
90
99
|
.action((options) => uninstallCommand(options));
|
package/dist/cli/projectId.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { existsSync } from "node:fs";
|
|
2
|
+
import { existsSync, realpathSync } from "node:fs";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
/** Safety bound on the upward walk in {@link findProjectRoot}. */
|
|
5
5
|
const MAX_PARENT_WALK = 64;
|
|
@@ -110,5 +110,20 @@ export function deriveProjectId(cwd = process.cwd()) {
|
|
|
110
110
|
return sanitized;
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
|
-
|
|
113
|
+
const canonical = canonicalDir(cwd);
|
|
114
|
+
return projectIdFromCwd(findProjectRoot(canonical) ?? canonical);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Resolve symlinks so the same directory hashes to one id however it is
|
|
118
|
+
* spelled. process.cwd() is already symlink-resolved, but a hook payload cwd is
|
|
119
|
+
* not (on macOS /var/... vs /private/var/...). Falls back to the input when the
|
|
120
|
+
* path does not exist.
|
|
121
|
+
*/
|
|
122
|
+
function canonicalDir(dir) {
|
|
123
|
+
try {
|
|
124
|
+
return realpathSync(dir);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return dir;
|
|
128
|
+
}
|
|
114
129
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
3
|
+
export const watchCheckpointPath = (dbPath) => join(dirname(resolve(dbPath)), `${basename(dbPath)}.watch-checkpoint.json`);
|
|
4
|
+
export function saveWatchCheckpoint(context, summary, intervalSeconds) {
|
|
5
|
+
if (context.dbPath === ":memory:")
|
|
6
|
+
return;
|
|
7
|
+
const path = watchCheckpointPath(context.dbPath);
|
|
8
|
+
const checkpoint = {
|
|
9
|
+
dbPath: resolve(context.dbPath), completedAt: new Date().toISOString(), intervalSeconds,
|
|
10
|
+
checkpointCount: context.db.prepare("SELECT count(*) AS n FROM transcript_checkpoints").get().n,
|
|
11
|
+
summary,
|
|
12
|
+
};
|
|
13
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
14
|
+
writeFileSync(temp, JSON.stringify(checkpoint), { mode: 0o600 });
|
|
15
|
+
renameSync(temp, path);
|
|
16
|
+
}
|
|
17
|
+
export function inspectWatchCheckpoint(dbPath, now = Date.now()) {
|
|
18
|
+
const checkpoint = JSON.parse(readFileSync(watchCheckpointPath(dbPath), "utf8"));
|
|
19
|
+
const age = now - Date.parse(checkpoint.completedAt);
|
|
20
|
+
if (checkpoint.dbPath !== resolve(dbPath))
|
|
21
|
+
throw new Error("Watcher checkpoint belongs to a different database");
|
|
22
|
+
if (!Number.isFinite(checkpoint.intervalSeconds) || checkpoint.intervalSeconds <= 0 || checkpoint.intervalSeconds > 86400)
|
|
23
|
+
throw new Error("Invalid watcher checkpoint interval");
|
|
24
|
+
if (!Number.isFinite(age) || age < -60000 || age > Math.max(120000, checkpoint.intervalSeconds * 3000))
|
|
25
|
+
throw new Error(`Watcher checkpoint is stale or invalid (${checkpoint.completedAt})`);
|
|
26
|
+
if (!checkpoint.summary || Object.values(checkpoint.summary).length !== 5 || Object.values(checkpoint.summary).some(value => !Number.isInteger(value) || value < 0))
|
|
27
|
+
throw new Error("Invalid watcher sweep counters");
|
|
28
|
+
if (checkpoint.summary.errors !== 0)
|
|
29
|
+
throw new Error(`Last watcher sweep had ${checkpoint.summary.errors} capture error(s)`);
|
|
30
|
+
if (!Number.isInteger(checkpoint.checkpointCount) || checkpoint.checkpointCount < 0)
|
|
31
|
+
throw new Error("Invalid transcript checkpoint count");
|
|
32
|
+
if (checkpoint.summary.filesFound > 0 && checkpoint.checkpointCount === 0)
|
|
33
|
+
throw new Error("Transcripts found but none checkpointed yet");
|
|
34
|
+
return `Sweep completed ${Math.max(0, Math.round(age / 1000))}s ago; ${checkpoint.checkpointCount} transcript checkpoint(s); ${checkpoint.summary.filesFound} source file(s)`;
|
|
35
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { mkdirSync, openSync, closeSync, writeSync } from "node:fs";
|
|
2
|
+
import { dirname, resolve } from "node:path";
|
|
3
|
+
/** Redirect both streams before opening storage so headless startup failures land
|
|
4
|
+
* in the same log as per-file capture failures. Never reads stdin or uses a TTY. */
|
|
5
|
+
export function openWatchLog(path) {
|
|
6
|
+
if (!path)
|
|
7
|
+
return () => { };
|
|
8
|
+
const absolute = resolve(path);
|
|
9
|
+
mkdirSync(dirname(absolute), { recursive: true });
|
|
10
|
+
const fd = openSync(absolute, "a", 0o600);
|
|
11
|
+
const stdout = process.stdout.write;
|
|
12
|
+
const stderr = process.stderr.write;
|
|
13
|
+
const write = ((chunk, encodingOrCallback, callback) => {
|
|
14
|
+
const value = typeof chunk === "string" ? chunk : Buffer.from(chunk).toString();
|
|
15
|
+
writeSync(fd, `${new Date().toISOString()} ${value}`);
|
|
16
|
+
const done = typeof encodingOrCallback === "function" ? encodingOrCallback : callback;
|
|
17
|
+
done?.();
|
|
18
|
+
return true;
|
|
19
|
+
});
|
|
20
|
+
process.stdout.write = write;
|
|
21
|
+
process.stderr.write = write;
|
|
22
|
+
return () => { process.stdout.write = stdout; process.stderr.write = stderr; closeSync(fd); };
|
|
23
|
+
}
|
|
@@ -179,6 +179,25 @@ export const getMemoryRequestSchema = z.object({
|
|
|
179
179
|
projectId: z.string().min(1),
|
|
180
180
|
memoryId: z.string().min(1).max(200),
|
|
181
181
|
});
|
|
182
|
+
export const searchMemoryIndexRequestSchema = retrieveMemoriesRequestSchema
|
|
183
|
+
.pick({ projectId: true, query: true, includeHistory: true, tags: true })
|
|
184
|
+
.extend({ limit: z.number().int().min(1).max(50).default(10) });
|
|
185
|
+
export const memoryTimelineRequestSchema = z.object({
|
|
186
|
+
projectId: z.string().min(1),
|
|
187
|
+
memoryId: z.string().min(1).max(200).optional(),
|
|
188
|
+
fromDate: z.iso.datetime().optional(),
|
|
189
|
+
toDate: z.iso.datetime().optional(),
|
|
190
|
+
includeHistory: z.boolean().default(true),
|
|
191
|
+
limit: z.number().int().min(1).max(50).default(10),
|
|
192
|
+
}).refine(value => value.memoryId !== undefined
|
|
193
|
+
? value.fromDate === undefined && value.toDate === undefined
|
|
194
|
+
: value.fromDate !== undefined && value.toDate !== undefined && Date.parse(value.fromDate) <= Date.parse(value.toDate), { message: "Provide memoryId alone, or fromDate and toDate in chronological order" });
|
|
195
|
+
export const memoryIndexItemSchema = z.object({
|
|
196
|
+
id: z.string(), summary: z.string().max(160), kind: z.string(), date: z.string(), score: z.number().nullable(),
|
|
197
|
+
});
|
|
198
|
+
export const memoryIndexResponseSchema = z.object({
|
|
199
|
+
ok: z.literal(true), memories: z.array(memoryIndexItemSchema), hasMore: z.boolean(),
|
|
200
|
+
});
|
|
182
201
|
export const forgetMemoryRequestSchema = z.object({
|
|
183
202
|
projectId: z.string().min(1),
|
|
184
203
|
memoryId: z.string().min(1).max(200),
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { userInfo } from "node:os";
|
|
2
2
|
import { ZodError } from "zod";
|
|
3
3
|
import { deterministicEmbed } from "../embed/deterministicEmbed.js";
|
|
4
|
+
import { loadLocalSemanticEmbedder } from "../embed/localSemanticEmbedder.js";
|
|
5
|
+
import { indexSemanticMemories, searchSemanticMemories } from "../storage/semanticSearchRepo.js";
|
|
6
|
+
import { detectTemporalIntent } from "../retrieve/temporalIntent.js";
|
|
7
|
+
import { memoryIndexItem } from "../retrieve/memoryIndex.js";
|
|
8
|
+
import { memoryTimeline } from "../storage/timelineRepo.js";
|
|
4
9
|
import { retrieveMemories } from "../retrieve/retrieveMemories.js";
|
|
5
10
|
import { computeEffectiveImportance } from "../retrieve/score.js";
|
|
6
11
|
import { formatStartupInjection } from "../injection/formatStartupInjection.js";
|
|
@@ -9,7 +14,7 @@ import { countAllMemoriesByProject, countMemoriesBySession, countMemoriesOlderTh
|
|
|
9
14
|
import { SESSION_WRITE_SOFT_LIMIT, configFilePath, DEEP_MODE_RETRIEVAL_CAP, readPolicyConfig, resolvePolicySettings, } from "../config/policyConfig.js";
|
|
10
15
|
import { insertMemoryFeedbackEvent } from "../storage/memoryFeedbackRepo.js";
|
|
11
16
|
import { countAllSessionEvents, insertSessionEvent, nextSessionEventIndex, } from "../storage/sessionEventsRepo.js";
|
|
12
|
-
import { batchStoreMemoryItemSchema, batchStoreMemoryRequestSchema, exportMemoriesRequestSchema, forgetMemoryRequestSchema, getMemoryRequestSchema, handleSessionEndRequestSchema, importMemoriesRequestSchema, pullMemoriesRequestSchema, ingestSessionEventsRequestSchema, LIST_MEMORIES_DEFAULT_LIMIT, listMemoriesRequestSchema, pruneMemoriesRequestSchema, redactExistingRequestSchema, resetAccessCountsRequestSchema, retrieveMemoriesRequestSchema, statsRequestSchema, storeMemoryRequestSchema, summarizeSessionToMemoryRequestSchema, } from "./contracts.js";
|
|
17
|
+
import { batchStoreMemoryItemSchema, batchStoreMemoryRequestSchema, exportMemoriesRequestSchema, forgetMemoryRequestSchema, getMemoryRequestSchema, handleSessionEndRequestSchema, importMemoriesRequestSchema, pullMemoriesRequestSchema, ingestSessionEventsRequestSchema, LIST_MEMORIES_DEFAULT_LIMIT, listMemoriesRequestSchema, pruneMemoriesRequestSchema, redactExistingRequestSchema, resetAccessCountsRequestSchema, retrieveMemoriesRequestSchema, searchMemoryIndexRequestSchema, memoryTimelineRequestSchema, statsRequestSchema, storeMemoryRequestSchema, summarizeSessionToMemoryRequestSchema, } from "./contracts.js";
|
|
13
18
|
import { DomainError, toErrorEnvelope } from "./errors.js";
|
|
14
19
|
import { assertLocalOnlyPolicy, } from "./localOnlyPolicy.js";
|
|
15
20
|
import { createSessionLifecycleService } from "./sessionLifecycleService.js";
|
|
@@ -150,6 +155,21 @@ export function createMemoryCoreService(deps) {
|
|
|
150
155
|
// Resolve the local author identity once per service instance.
|
|
151
156
|
const localAuthor = resolveServiceUsername(deps.username);
|
|
152
157
|
const policyConfigPath = deps.policyConfigPath ?? configFilePath();
|
|
158
|
+
let semanticWarningShown = false;
|
|
159
|
+
async function semanticEmbedder() {
|
|
160
|
+
if (!readPolicyConfig(policyConfigPath).semanticSearch)
|
|
161
|
+
return null;
|
|
162
|
+
const embedder = deps.semanticEmbedder === undefined ? await loadLocalSemanticEmbedder() : deps.semanticEmbedder;
|
|
163
|
+
if (!embedder)
|
|
164
|
+
semanticWarning();
|
|
165
|
+
return embedder;
|
|
166
|
+
}
|
|
167
|
+
function semanticWarning() {
|
|
168
|
+
if (!semanticWarningShown) {
|
|
169
|
+
console.error("Local semantic retrieval unavailable; using BM25.");
|
|
170
|
+
semanticWarningShown = true;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
153
173
|
/**
|
|
154
174
|
* Resolve the effective redactionEnabled flag using precedence
|
|
155
175
|
* override (explicit per-request value) > config.json > default,
|
|
@@ -172,6 +192,48 @@ export function createMemoryCoreService(deps) {
|
|
|
172
192
|
now: deps.now,
|
|
173
193
|
deleteOldMemories: deps.deleteOldMemories,
|
|
174
194
|
});
|
|
195
|
+
async function rankMemories(parsed, limit) {
|
|
196
|
+
const now = deps.now?.() ?? new Date();
|
|
197
|
+
const config = readPolicyConfig(policyConfigPath);
|
|
198
|
+
const intent = config.temporalRetrieval ? detectTemporalIntent(parsed.query, now) : { kind: "none" };
|
|
199
|
+
const includeHistory = parsed.includeHistory ?? intent.kind === "past";
|
|
200
|
+
let semanticCandidates;
|
|
201
|
+
try {
|
|
202
|
+
const embedder = await semanticEmbedder();
|
|
203
|
+
if (embedder)
|
|
204
|
+
semanticCandidates = await searchSemanticMemories(db, parsed.projectId, parsed.query, now, includeHistory, embedder);
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
semanticWarning();
|
|
208
|
+
}
|
|
209
|
+
const rankedAll = retrieveMemories({
|
|
210
|
+
db,
|
|
211
|
+
projectId: parsed.projectId,
|
|
212
|
+
queryText: parsed.query,
|
|
213
|
+
includeHistory,
|
|
214
|
+
temporalRetrieval: config.temporalRetrieval,
|
|
215
|
+
// Measured mode-specific default: coverage helps BM25, but regresses
|
|
216
|
+
// MiniLM recall. An explicit override always wins, including false.
|
|
217
|
+
entityLinkBoost: config.entityLinkBoost ?? (!semanticCandidates?.length && (!config.retrievalRanking || config.retrievalRanking === "legacy")),
|
|
218
|
+
ranking: config.retrievalRanking,
|
|
219
|
+
limit,
|
|
220
|
+
now,
|
|
221
|
+
semanticCandidates,
|
|
222
|
+
});
|
|
223
|
+
// Optional tag filter: keep a memory only when its stored tags contain ALL
|
|
224
|
+
// requested tags. The semantic candidate projection omits tags, so re-read
|
|
225
|
+
// each row's tags by id (this is an explicit, infrequent filter path).
|
|
226
|
+
const ranked = parsed.tags && parsed.tags.length > 0
|
|
227
|
+
? rankedAll.filter((m) => {
|
|
228
|
+
const rec = getMemoryRecordById(db, parsed.projectId, m.id);
|
|
229
|
+
const recTags = rec?.tags
|
|
230
|
+
? JSON.parse(rec.tags)
|
|
231
|
+
: [];
|
|
232
|
+
return parsed.tags.every((t) => recTags.includes(t));
|
|
233
|
+
})
|
|
234
|
+
: rankedAll;
|
|
235
|
+
return ranked.slice(0, limit);
|
|
236
|
+
}
|
|
175
237
|
const methods = {
|
|
176
238
|
async ingestSessionEvents(request) {
|
|
177
239
|
const parsed = parseRequest(ingestSessionEventsRequestSchema, request);
|
|
@@ -299,6 +361,15 @@ export function createMemoryCoreService(deps) {
|
|
|
299
361
|
if (!inserted) {
|
|
300
362
|
throw new DomainError("INTERNAL", "Memory insert did not persist");
|
|
301
363
|
}
|
|
364
|
+
// Optional indexing never turns a successful durable capture into a failure.
|
|
365
|
+
try {
|
|
366
|
+
const embedder = await semanticEmbedder();
|
|
367
|
+
if (embedder)
|
|
368
|
+
await indexSemanticMemories(db, parsed.projectId, embedder, parsed.memoryId);
|
|
369
|
+
}
|
|
370
|
+
catch {
|
|
371
|
+
semanticWarning();
|
|
372
|
+
}
|
|
302
373
|
return {
|
|
303
374
|
ok: true,
|
|
304
375
|
// Single-record write echo-back: return the FULL stored body (uncapped),
|
|
@@ -314,26 +385,7 @@ export function createMemoryCoreService(deps) {
|
|
|
314
385
|
async retrieveMemories(request) {
|
|
315
386
|
const parsed = parseRequest(retrieveMemoriesRequestSchema, request);
|
|
316
387
|
const limit = parsed.depth === "deep" ? Math.min(parsed.limit * 2, DEEP_MODE_RETRIEVAL_CAP) : parsed.limit;
|
|
317
|
-
const
|
|
318
|
-
db,
|
|
319
|
-
projectId: parsed.projectId,
|
|
320
|
-
queryText: parsed.query,
|
|
321
|
-
includeHistory: parsed.includeHistory,
|
|
322
|
-
limit,
|
|
323
|
-
now: deps.now?.(),
|
|
324
|
-
});
|
|
325
|
-
// Optional tag filter: keep a memory only when its stored tags contain ALL
|
|
326
|
-
// requested tags. The semantic candidate projection omits tags, so re-read
|
|
327
|
-
// each row's tags by id (this is an explicit, infrequent filter path).
|
|
328
|
-
const ranked = parsed.tags && parsed.tags.length > 0
|
|
329
|
-
? rankedAll.filter((m) => {
|
|
330
|
-
const rec = getMemoryRecordById(db, parsed.projectId, m.id);
|
|
331
|
-
const recTags = rec?.tags
|
|
332
|
-
? JSON.parse(rec.tags)
|
|
333
|
-
: [];
|
|
334
|
-
return parsed.tags.every((t) => recTags.includes(t));
|
|
335
|
-
})
|
|
336
|
-
: rankedAll;
|
|
388
|
+
const ranked = await rankMemories(parsed, limit);
|
|
337
389
|
if (ranked.length > 0 && parsed.mode !== "on-demand") {
|
|
338
390
|
// Only boost access counts for startup injection (mode='auto'), not for
|
|
339
391
|
// explicit on-demand retrieval, so a mid-session lookup does not inflate
|
|
@@ -353,6 +405,17 @@ export function createMemoryCoreService(deps) {
|
|
|
353
405
|
}),
|
|
354
406
|
};
|
|
355
407
|
},
|
|
408
|
+
async searchMemoryIndex(request) {
|
|
409
|
+
const parsed = parseRequest(searchMemoryIndexRequestSchema, request);
|
|
410
|
+
const ranked = await rankMemories(parsed, parsed.limit + 1);
|
|
411
|
+
return { ok: true, memories: ranked.slice(0, parsed.limit).map(row => memoryIndexItem(row, row.score.total)),
|
|
412
|
+
hasMore: ranked.length > parsed.limit };
|
|
413
|
+
},
|
|
414
|
+
async memoryTimeline(request) {
|
|
415
|
+
const parsed = parseRequest(memoryTimelineRequestSchema, request);
|
|
416
|
+
const { rows, hasMore } = memoryTimeline(db, parsed, deps.now?.() ?? new Date());
|
|
417
|
+
return { ok: true, memories: rows.map(row => memoryIndexItem(row)), hasMore };
|
|
418
|
+
},
|
|
356
419
|
async listMemories(request) {
|
|
357
420
|
const parsed = parseRequest(listMemoriesRequestSchema, request);
|
|
358
421
|
const all = listMemoriesByProject(db, parsed.projectId);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { RETRIEVAL_RANKINGS } from "../retrieve/signalFusion.js";
|
|
2
3
|
import { homedir } from "os";
|
|
3
4
|
import { join, dirname } from "path";
|
|
4
5
|
import { mkdirSync, readFileSync, writeFileSync, renameSync } from "fs";
|
|
@@ -53,6 +54,10 @@ const policyConfigShape = {
|
|
|
53
54
|
// built-in DEFAULT_INJECTION_CAP used by formatStartupInjection and the
|
|
54
55
|
// `savings` analytics. Omitted by default so existing configs keep the default.
|
|
55
56
|
injectionCap: z.number().int().min(100).max(10000).optional(),
|
|
57
|
+
semanticSearch: z.boolean().optional(),
|
|
58
|
+
temporalRetrieval: z.boolean().optional(),
|
|
59
|
+
entityLinkBoost: z.boolean().optional(),
|
|
60
|
+
retrievalRanking: z.enum(RETRIEVAL_RANKINGS).optional(),
|
|
56
61
|
team: teamConfigShape.default({ enabled: false }),
|
|
57
62
|
};
|
|
58
63
|
/**
|
|
@@ -107,11 +112,19 @@ export function readPolicyConfig(filePath) {
|
|
|
107
112
|
const rd = s.retentionDays.safeParse(obj.retentionDays);
|
|
108
113
|
const re = s.redactionEnabled.safeParse(obj.redactionEnabled);
|
|
109
114
|
const ic = s.injectionCap.safeParse(obj.injectionCap);
|
|
115
|
+
const ss = s.semanticSearch.safeParse(obj.semanticSearch);
|
|
116
|
+
const tr = s.temporalRetrieval.safeParse(obj.temporalRetrieval);
|
|
117
|
+
const el = s.entityLinkBoost.safeParse(obj.entityLinkBoost);
|
|
118
|
+
const rr = s.retrievalRanking.safeParse(obj.retrievalRanking);
|
|
110
119
|
const tm = s.team.safeParse(obj.team);
|
|
111
120
|
return {
|
|
112
121
|
retentionDays: rd.success ? rd.data : DEFAULT_POLICY_CONFIG.retentionDays,
|
|
113
122
|
redactionEnabled: re.success ? re.data : DEFAULT_POLICY_CONFIG.redactionEnabled,
|
|
114
123
|
injectionCap: ic.success ? ic.data : undefined,
|
|
124
|
+
semanticSearch: ss.success ? ss.data : undefined,
|
|
125
|
+
temporalRetrieval: tr.success ? tr.data : undefined,
|
|
126
|
+
entityLinkBoost: el.success ? el.data : undefined,
|
|
127
|
+
retrievalRanking: rr.success ? rr.data : undefined,
|
|
115
128
|
team: tm.success ? tm.data : { ...DEFAULT_POLICY_CONFIG.team },
|
|
116
129
|
};
|
|
117
130
|
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
const MODEL_SHA = "6fd5d72fe4589f189f8ebc006442dbb529bb7ce38f8082112682524616046452";
|
|
5
|
+
const TOKENIZER_SHA = "be50c3628f2bf5bb5e3a7f17b1f74611b2561a3a27eeab05e5aa30f411572037";
|
|
6
|
+
export const LOCAL_SEMANTIC_MODEL = `minilm-l6-v2:${MODEL_SHA}:${TOKENIZER_SHA}:mean256-v1`;
|
|
7
|
+
let loaded;
|
|
8
|
+
/** Only local package files are opened. No download or remote provider path. */
|
|
9
|
+
export function loadLocalSemanticEmbedder() {
|
|
10
|
+
return loaded ??= load().catch(() => null);
|
|
11
|
+
}
|
|
12
|
+
async function load() {
|
|
13
|
+
const require = createRequire(import.meta.url);
|
|
14
|
+
const model = require.resolve("@xcidos/genesis-memory-model/onnx/model.onnx");
|
|
15
|
+
const tokenizerPath = require.resolve("@xcidos/genesis-memory-model/tokenizer.json");
|
|
16
|
+
for (const [file, expected] of [[model, MODEL_SHA], [tokenizerPath, TOKENIZER_SHA]]) {
|
|
17
|
+
if (createHash("sha256").update(readFileSync(file)).digest("hex") !== expected) {
|
|
18
|
+
throw new Error("Local semantic model checksum mismatch");
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
// ORT 1.30 can initialize a native HTTP telemetry uploader on POSIX.
|
|
22
|
+
// Its full opt-out must be set before the native environment is created;
|
|
23
|
+
// JavaScript network guards cannot intercept native library sockets.
|
|
24
|
+
process.env.ORT_DISABLE_TELEMETRY = "1";
|
|
25
|
+
const ort = await import("onnxruntime-node");
|
|
26
|
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
|
27
|
+
const tokenizer = new Tokenizer(JSON.parse(readFileSync(tokenizerPath, "utf8")), {});
|
|
28
|
+
const session = await ort.InferenceSession.create(model, {
|
|
29
|
+
executionProviders: ["cpu"], intraOpNumThreads: 1, interOpNumThreads: 1,
|
|
30
|
+
logSeverityLevel: 3,
|
|
31
|
+
});
|
|
32
|
+
return {
|
|
33
|
+
id: LOCAL_SEMANTIC_MODEL, dimension: 384,
|
|
34
|
+
async embed(text) {
|
|
35
|
+
let ids = tokenizer.encode(text).ids;
|
|
36
|
+
if (ids.length > 256)
|
|
37
|
+
ids = [...ids.slice(0, 255), 102]; // retain BERT's SEP
|
|
38
|
+
const input = new ort.Tensor("int64", BigInt64Array.from(ids, BigInt), [1, ids.length]);
|
|
39
|
+
const ones = new ort.Tensor("int64", new BigInt64Array(ids.length).fill(1n), [1, ids.length]);
|
|
40
|
+
const zeros = new ort.Tensor("int64", new BigInt64Array(ids.length), [1, ids.length]);
|
|
41
|
+
let output;
|
|
42
|
+
try {
|
|
43
|
+
output = await session.run({ input_ids: input, attention_mask: ones, token_type_ids: zeros });
|
|
44
|
+
const hidden = output.last_hidden_state.data;
|
|
45
|
+
const vector = new Array(384).fill(0);
|
|
46
|
+
for (let token = 0; token < ids.length; token++) {
|
|
47
|
+
for (let dim = 0; dim < 384; dim++)
|
|
48
|
+
vector[dim] += hidden[token * 384 + dim] / ids.length;
|
|
49
|
+
}
|
|
50
|
+
const norm = Math.hypot(...vector);
|
|
51
|
+
return vector.map(v => v / (norm || 1));
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
input.dispose();
|
|
55
|
+
ones.dispose();
|
|
56
|
+
zeros.dispose();
|
|
57
|
+
for (const tensor of Object.values(output ?? {}))
|
|
58
|
+
tensor.dispose();
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Extractive preview only. No generated summary or extra inference. */
|
|
2
|
+
export function memoryIndexItem(memory, score = null) {
|
|
3
|
+
const text = memory.content.replace(/\s+/g, " ").trim();
|
|
4
|
+
return { id: memory.id, summary: text.length > 160 ? `${text.slice(0, 157)}...` : text,
|
|
5
|
+
kind: memory.kind, date: memory.created_at, score: score === null ? null : Math.round(score * 10000) / 10000 };
|
|
6
|
+
}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { entityOverlap } from "../storage/knowledgeRepo.js";
|
|
2
|
+
import { linkedEntityScores } from "../storage/entityLinkRepo.js";
|
|
3
|
+
import { memoryTemporalMetadata } from "../storage/temporalRepo.js";
|
|
4
|
+
import { adaptiveLexicalScore, additiveRelevance, SEMANTIC_GATE } from "./signalFusion.js";
|
|
5
|
+
import { detectTemporalIntent, temporalRelevance } from "./temporalIntent.js";
|
|
2
6
|
import { deterministicEmbed } from "../embed/deterministicEmbed.js";
|
|
3
7
|
import { decayOldBoosts } from "./decay.js";
|
|
4
8
|
import { scoreMemoryCandidate, } from "./score.js";
|
|
5
|
-
import { searchMemoryCandidatesFTS, } from "../storage/memorySearchRepo.js";
|
|
9
|
+
import { searchMemoryCandidatesFTS, searchMemoryCandidatesByIds, } from "../storage/memorySearchRepo.js";
|
|
6
10
|
const DEFAULT_EMBEDDING_DIMENSION = 32;
|
|
7
11
|
// Small, fixed contribution inside the existing relevance component. Keep the
|
|
8
12
|
// public score shape/outer recency and importance weights unchanged.
|
|
@@ -45,17 +49,42 @@ export function retrieveMemories(input) {
|
|
|
45
49
|
}
|
|
46
50
|
const topK = input.topK ?? input.limit ?? 20;
|
|
47
51
|
const now = input.now ?? new Date();
|
|
52
|
+
const ranking = input.ranking ?? "legacy";
|
|
53
|
+
const additive = ranking === "hub-additive" || ranking === "adaptive-hub-additive";
|
|
54
|
+
const adaptive = ranking === "adaptive-bm25" || ranking === "adaptive-hub-additive";
|
|
55
|
+
const intent = input.temporalRetrieval ? detectTemporalIntent(queryText, now) : { kind: "none" };
|
|
56
|
+
const includeHistory = input.includeHistory ?? intent.kind === "past";
|
|
48
57
|
// FTS5 retrieves up to 50 BM25 matches; sparse matches retain the existing
|
|
49
58
|
// recency/importance fallback. The public request schema validates queryText.
|
|
50
|
-
const
|
|
59
|
+
const lexicalCandidates = searchMemoryCandidatesFTS(input.db, input.projectId, queryText, now, includeHistory);
|
|
60
|
+
const candidates = [...lexicalCandidates];
|
|
61
|
+
const ids = new Set(candidates.map(c => c.id));
|
|
62
|
+
const linked = input.entityLinkBoost || additive
|
|
63
|
+
? linkedEntityScores(input.db, input.projectId, queryText, now, includeHistory, additive)
|
|
64
|
+
: new Map();
|
|
65
|
+
for (const candidate of searchMemoryCandidatesByIds(input.db, input.projectId, [...linked.keys()].slice(0, 50), now, includeHistory)) {
|
|
66
|
+
if (!ids.has(candidate.id)) {
|
|
67
|
+
candidates.push(candidate);
|
|
68
|
+
ids.add(candidate.id);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
for (const { candidate } of input.semanticCandidates ?? []) {
|
|
72
|
+
if (!ids.has(candidate.id)) {
|
|
73
|
+
candidates.push(candidate);
|
|
74
|
+
ids.add(candidate.id);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const dense = new Map(input.semanticCandidates?.map(c => [c.candidate.id, c.similarity]));
|
|
51
78
|
const decayedCandidates = decayOldBoosts(candidates, now);
|
|
52
79
|
const dimension = resolveEmbeddingDimension(candidates);
|
|
53
80
|
const queryVector = deterministicEmbed(queryText, dimension).vector;
|
|
54
81
|
const overlap = entityOverlap(input.db, input.projectId, queryText, now);
|
|
82
|
+
const temporal = intent.kind !== "none" ? memoryTemporalMetadata(input.db, input.projectId, candidates.map(c => c.id)) : new Map();
|
|
55
83
|
// FTS5 returns negative BM25 ranks; storage converts them to positive scores.
|
|
56
84
|
// Normalize within this query so BM25 fits the existing relevance weight.
|
|
57
85
|
const maxLexicalScore = candidates.reduce((max, candidate) => Math.max(max, candidate.lexical_score), 0);
|
|
58
86
|
const ranked = decayedCandidates
|
|
87
|
+
.filter(candidate => !additive || !input.semanticCandidates?.length || (dense.get(candidate.id) ?? 0) >= SEMANTIC_GATE)
|
|
59
88
|
.map((candidate) => {
|
|
60
89
|
// When embedding is null (version mismatch or missing), use a neutral
|
|
61
90
|
// score of 0.5 so the memory is neither penalized nor boosted. A non-null
|
|
@@ -68,18 +97,37 @@ export function retrieveMemories(input) {
|
|
|
68
97
|
: cosineSimilarity(queryVector, candidate.embedding);
|
|
69
98
|
// Hash cosine is useful for identical text, but adds noise to lexical
|
|
70
99
|
// matches. Keep it as a diagnostic and as the no-match fallback only.
|
|
71
|
-
const
|
|
72
|
-
? candidate.lexical_score / maxLexicalScore
|
|
100
|
+
const lexicalRelevance = maxLexicalScore > 0
|
|
101
|
+
? adaptive ? adaptiveLexicalScore(candidate.lexical_score, queryText) : candidate.lexical_score / maxLexicalScore
|
|
73
102
|
: semantic;
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
103
|
+
// The local dense variant was weaker alone. Preserve lexical dominance
|
|
104
|
+
// while admitting candidates absent from FTS, using a fixed 2:1 blend.
|
|
105
|
+
const baseRelevance = input.semanticCandidates?.length
|
|
106
|
+
? maxLexicalScore > 0
|
|
107
|
+
? (2 * lexicalRelevance + Math.max(0, dense.get(candidate.id) ?? 0)) / 3
|
|
108
|
+
: Math.max(0, dense.get(candidate.id) ?? 0)
|
|
109
|
+
: lexicalRelevance;
|
|
110
|
+
const fused = additive ? additiveRelevance({
|
|
111
|
+
lexical: maxLexicalScore > 0 ? lexicalRelevance : undefined,
|
|
112
|
+
dense: input.semanticCandidates?.length ? Math.max(0, dense.get(candidate.id) ?? 0) : undefined,
|
|
113
|
+
entity: linked.size > 0 ? linked.get(candidate.id) ?? 0 : undefined,
|
|
114
|
+
}) : null;
|
|
115
|
+
const entityRelevance = additive && fused !== null ? fused : linked.size > 0
|
|
116
|
+
? 0.8 * baseRelevance + 0.2 * (linked.get(candidate.id) ?? 0)
|
|
117
|
+
: overlap.size > 0
|
|
118
|
+
? (1 - ENTITY_RELEVANCE_WEIGHT) * baseRelevance + ENTITY_RELEVANCE_WEIGHT * (overlap.get(candidate.id) ?? 0)
|
|
119
|
+
: baseRelevance;
|
|
120
|
+
const metadata = temporal.get(candidate.id);
|
|
121
|
+
const relevance = metadata
|
|
122
|
+
? 0.9 * entityRelevance + 0.1 * temporalRelevance(intent, metadata.timestamp, metadata.supersededFraction, now)
|
|
123
|
+
: entityRelevance;
|
|
77
124
|
const score = scoreMemoryCandidate({
|
|
78
125
|
semantic: relevance,
|
|
79
|
-
|
|
126
|
+
// Historical lookup should not lose evidence merely because it is old.
|
|
127
|
+
updated_at: intent.kind === "past" ? now.toISOString() : candidate.updated_at,
|
|
80
128
|
importance: candidate.importance,
|
|
81
129
|
access_count: candidate.access_count,
|
|
82
|
-
decayedImportance: candidate.decayedImportance,
|
|
130
|
+
decayedImportance: intent.kind === "past" ? candidate.importance : candidate.decayedImportance,
|
|
83
131
|
}, now);
|
|
84
132
|
return {
|
|
85
133
|
id: candidate.id,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { lexicalQueryTerms } from "../storage/memorySearchRepo.js";
|
|
2
|
+
export const RETRIEVAL_RANKINGS = ["legacy", "adaptive-bm25", "hub-additive", "adaptive-hub-additive"];
|
|
3
|
+
/** Fixed hypotheses from the requested public-source audit, independently implemented. */
|
|
4
|
+
export function adaptiveLexicalScore(raw, query) {
|
|
5
|
+
if (raw <= 0 || !Number.isFinite(raw))
|
|
6
|
+
return 0;
|
|
7
|
+
const count = lexicalQueryTerms(query).length;
|
|
8
|
+
const bands = [[3, 5, 0.7], [6, 7, 0.6], [9, 9, 0.5], [15, 10, 0.5], [Infinity, 12, 0.5]];
|
|
9
|
+
const [, midpoint, steepness] = bands.find(([ceiling]) => count <= ceiling);
|
|
10
|
+
return 1 / (1 + Math.exp(steepness * (midpoint - raw)));
|
|
11
|
+
}
|
|
12
|
+
export const SEMANTIC_GATE = 0.2;
|
|
13
|
+
export function additiveRelevance(signals) {
|
|
14
|
+
if (signals.dense !== undefined && signals.dense < SEMANTIC_GATE)
|
|
15
|
+
return null;
|
|
16
|
+
const capacity = Number(signals.lexical !== undefined) + Number(signals.dense !== undefined) + 0.5 * Number(signals.entity !== undefined);
|
|
17
|
+
if (!capacity)
|
|
18
|
+
return 0;
|
|
19
|
+
const sum = (signals.lexical ?? 0) + (signals.dense ?? 0) + (signals.entity ?? 0);
|
|
20
|
+
return Math.max(0, Math.min(1, sum / capacity));
|
|
21
|
+
}
|