sessionmem 1.0.5 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +372 -365
- package/dist/adapters/capabilities/fallbackTools.js +33 -18
- package/dist/adapters/claudeMdInjector.js +164 -0
- package/dist/adapters/factory.js +68 -9
- package/dist/adapters/generic.js +221 -15
- package/dist/adapters/global/antigravity.js +14 -7
- package/dist/adapters/global/claudeCode.js +46 -10
- package/dist/adapters/global/codex.js +73 -13
- package/dist/adapters/global/qcoder.js +18 -5
- package/dist/adapters/ide/cline.js +54 -9
- package/dist/adapters/ide/cursor.js +15 -13
- package/dist/adapters/ide/installer.js +201 -8
- package/dist/adapters/ide/windsurf.js +14 -13
- package/dist/adapters/tools/ping.js +4 -1
- package/dist/cli/commands/config.js +10 -1
- package/dist/cli/commands/import.js +6 -1
- package/dist/cli/commands/install.js +63 -5
- package/dist/cli/commands/ping.js +42 -8
- package/dist/cli/commands/reEmbed.js +48 -0
- package/dist/cli/commands/run.js +18 -2
- package/dist/cli/commands/savings.js +91 -0
- package/dist/cli/commands/sessionEnd.js +124 -0
- package/dist/cli/commands/sessionStart.js +52 -0
- package/dist/cli/commands/sync.js +39 -9
- package/dist/cli/commands/uninstall.js +37 -1
- package/dist/cli/context.js +14 -18
- package/dist/cli/index.js +30 -4
- package/dist/cli/output.js +11 -3
- package/dist/cli/projectId.js +69 -0
- package/dist/core/api/contracts.js +182 -45
- package/dist/core/api/errors.js +4 -7
- package/dist/core/api/memoryCoreService.js +409 -240
- package/dist/core/api/sessionLifecycleService.js +20 -2
- package/dist/core/config/policyConfig.js +53 -6
- package/dist/core/injection/formatStartupInjection.js +55 -10
- package/dist/core/injection/tokenBudget.js +8 -0
- package/dist/core/retrieve/importance.js +4 -3
- package/dist/core/retrieve/recencyBands.js +6 -10
- package/dist/core/retrieve/retrieveMemories.js +19 -4
- package/dist/core/retrieve/score.js +11 -1
- package/dist/core/schema/migrations/005_team_provenance.sql +14 -9
- package/dist/core/schema/migrations/006_access_pattern_boosting.sql +10 -0
- package/dist/core/schema/migrations/007_feedback_manual_delete.sql +23 -0
- package/dist/core/schema/migrations/008_fts5_search.sql +37 -0
- package/dist/core/schema/migrations/009_session_events_unique.sql +24 -0
- package/dist/core/schema/runMigrations.js +64 -2
- package/dist/core/storage/db.js +6 -0
- package/dist/core/storage/memoryFeedbackRepo.js +14 -4
- package/dist/core/storage/memoryRepo.js +292 -121
- package/dist/core/storage/memorySearchRepo.js +125 -13
- package/dist/core/storage/sessionEventsRepo.js +33 -10
- package/dist/core/storage/summarizationFailuresRepo.js +36 -26
- package/dist/core/storage/tokenSavingsRepo.js +20 -0
- package/dist/core/summarize/cloudSummarizer.js +34 -5
- package/dist/core/summarize/localSummarizer.js +1 -10
- package/dist/core/summarize/redaction.js +45 -8
- package/package.json +50 -48
|
@@ -111,6 +111,14 @@ export function createSessionLifecycleService(deps) {
|
|
|
111
111
|
* Hard-deletes memories older than the effective retentionDays for this
|
|
112
112
|
* project. retentionDays<=0 disables pruning. Any failure is swallowed
|
|
113
113
|
* so it can never block or fail summarization.
|
|
114
|
+
*
|
|
115
|
+
* IMPORTANT: retention pruning fires INDEPENDENTLY of summarization. It runs
|
|
116
|
+
* on every handleSessionEnd path — including the `skipped_disabled`
|
|
117
|
+
* (autoSummarize=false) and `skipped_threshold` (too few events) early
|
|
118
|
+
* returns — because retention is a time-based policy that must apply whether
|
|
119
|
+
* or not a summary was produced this session. It is gated only by the
|
|
120
|
+
* retentionDays config (retentionDays<=0 disables it), not by whether the
|
|
121
|
+
* lifecycle proceeded past the summarization threshold.
|
|
114
122
|
*/
|
|
115
123
|
function runLightPrune(projectId) {
|
|
116
124
|
try {
|
|
@@ -185,8 +193,18 @@ export function createSessionLifecycleService(deps) {
|
|
|
185
193
|
memoryId,
|
|
186
194
|
};
|
|
187
195
|
}
|
|
188
|
-
catch {
|
|
189
|
-
|
|
196
|
+
catch (cloudError) {
|
|
197
|
+
const failureRecordId = createFailureId();
|
|
198
|
+
insertSummarizationFailure(deps.db, {
|
|
199
|
+
id: failureRecordId,
|
|
200
|
+
project_id: request.projectId,
|
|
201
|
+
session_id: request.sessionId,
|
|
202
|
+
source_adapter: request.sourceAdapter,
|
|
203
|
+
reason: "cloud_failed",
|
|
204
|
+
attempt_count: CLOUD_RETRY_CONFIG.retries + 1,
|
|
205
|
+
last_error_json: toErrorJson(cloudError),
|
|
206
|
+
});
|
|
207
|
+
// fall through to local summarizer
|
|
190
208
|
}
|
|
191
209
|
try {
|
|
192
210
|
const fallbackResult = await summarizeLocal(baseInput);
|
|
@@ -1,7 +1,20 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { homedir } from "os";
|
|
3
3
|
import { join, dirname } from "path";
|
|
4
|
-
import { mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
4
|
+
import { mkdirSync, readFileSync, writeFileSync, renameSync } from "fs";
|
|
5
|
+
/** Lower bound of the 1-10 importance scale. */
|
|
6
|
+
export const MIN_IMPORTANCE = 1;
|
|
7
|
+
/** Upper bound of the 1-10 importance scale. */
|
|
8
|
+
export const MAX_IMPORTANCE = 10;
|
|
9
|
+
/** Importance threshold at or above which a warning is considered critical. */
|
|
10
|
+
export const CRITICAL_WARNING_IMPORTANCE_THRESHOLD = 9;
|
|
11
|
+
/** Maximum number of memories returned by a "deep" retrieval. */
|
|
12
|
+
export const DEEP_MODE_RETRIEVAL_CAP = 100;
|
|
13
|
+
/**
|
|
14
|
+
* Default model for cloud summarization via the Anthropic API.
|
|
15
|
+
* Consumed by {@link import("../summarize/cloudSummarizer.js").summarizeWithCloud}.
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_SUMMARIZER_MODEL = "claude-sonnet-4-6";
|
|
5
18
|
/**
|
|
6
19
|
* Built-in policy defaults. Used whenever the config file is missing, malformed,
|
|
7
20
|
* or fails validation, and as the lowest-precedence source in
|
|
@@ -34,8 +47,12 @@ const teamConfigShape = z
|
|
|
34
47
|
* built-in defaults.
|
|
35
48
|
*/
|
|
36
49
|
const policyConfigShape = {
|
|
37
|
-
retentionDays: z.number().int().default(DEFAULT_POLICY_CONFIG.retentionDays),
|
|
50
|
+
retentionDays: z.number().int().min(0).default(DEFAULT_POLICY_CONFIG.retentionDays),
|
|
38
51
|
redactionEnabled: z.boolean().default(DEFAULT_POLICY_CONFIG.redactionEnabled),
|
|
52
|
+
// Optional per-session startup-injection token cap. When set, it overrides the
|
|
53
|
+
// built-in DEFAULT_INJECTION_CAP used by formatStartupInjection and the
|
|
54
|
+
// `savings` analytics. Omitted by default so existing configs keep the default.
|
|
55
|
+
injectionCap: z.number().int().min(100).max(10000).optional(),
|
|
39
56
|
team: teamConfigShape.default({ enabled: false }),
|
|
40
57
|
};
|
|
41
58
|
/**
|
|
@@ -67,14 +84,32 @@ export function configFilePath() {
|
|
|
67
84
|
* are merged over defaults via the schema's per-field `.default()`.
|
|
68
85
|
*/
|
|
69
86
|
export function readPolicyConfig(filePath) {
|
|
87
|
+
let raw;
|
|
88
|
+
let obj;
|
|
70
89
|
try {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
return policyConfigReadSchema.parse(parsed);
|
|
90
|
+
raw = readFileSync(filePath, "utf8");
|
|
91
|
+
obj = JSON.parse(raw);
|
|
74
92
|
}
|
|
75
93
|
catch {
|
|
76
94
|
return { ...DEFAULT_POLICY_CONFIG };
|
|
77
95
|
}
|
|
96
|
+
// Fast path: strict parse succeeds (the common case).
|
|
97
|
+
const strict = policyConfigReadSchema.safeParse(obj);
|
|
98
|
+
if (strict.success)
|
|
99
|
+
return strict.data;
|
|
100
|
+
// Lenient fallback: salvage valid individual fields so a single bad value
|
|
101
|
+
// (e.g. injectionCap: 50, below min 100) doesn't wipe out retentionDays etc.
|
|
102
|
+
const s = policyConfigShape;
|
|
103
|
+
const rd = s.retentionDays.safeParse(obj.retentionDays);
|
|
104
|
+
const re = s.redactionEnabled.safeParse(obj.redactionEnabled);
|
|
105
|
+
const ic = s.injectionCap.safeParse(obj.injectionCap);
|
|
106
|
+
const tm = s.team.safeParse(obj.team);
|
|
107
|
+
return {
|
|
108
|
+
retentionDays: rd.success ? rd.data : DEFAULT_POLICY_CONFIG.retentionDays,
|
|
109
|
+
redactionEnabled: re.success ? re.data : DEFAULT_POLICY_CONFIG.redactionEnabled,
|
|
110
|
+
injectionCap: ic.success ? ic.data : undefined,
|
|
111
|
+
team: tm.success ? tm.data : { ...DEFAULT_POLICY_CONFIG.team },
|
|
112
|
+
};
|
|
78
113
|
}
|
|
79
114
|
/**
|
|
80
115
|
* Persist a partial policy config, merged over the current on-disk values (or
|
|
@@ -91,7 +126,12 @@ export function writePolicyConfig(filePath, partial) {
|
|
|
91
126
|
const current = readPolicyConfig(filePath);
|
|
92
127
|
const merged = policyConfigSchema.parse({ ...current, ...validatedPartial });
|
|
93
128
|
mkdirSync(dirname(filePath), { recursive: true });
|
|
94
|
-
|
|
129
|
+
// Atomic write: write to a temp file then rename over the target so a crash
|
|
130
|
+
// mid-write can't leave a truncated/corrupt config. On Windows, renameSync
|
|
131
|
+
// over an existing file works (Node wraps it via MoveFileExW).
|
|
132
|
+
const tmpPath = `${filePath}.tmp`;
|
|
133
|
+
writeFileSync(tmpPath, `${JSON.stringify(merged, null, 2)}\n`, "utf8");
|
|
134
|
+
renameSync(tmpPath, filePath);
|
|
95
135
|
return merged;
|
|
96
136
|
}
|
|
97
137
|
/**
|
|
@@ -116,6 +156,13 @@ export function resolvePolicySettings(input) {
|
|
|
116
156
|
redactionEnabled: resolve("redactionEnabled"),
|
|
117
157
|
};
|
|
118
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Per-session write soft limit. When a session has stored at least this many
|
|
161
|
+
* memories, subsequent storeMemory calls still succeed but the response
|
|
162
|
+
* includes a "session_write_limit_warning" warningCode, giving the agent
|
|
163
|
+
* feedback to stop storing excessive memories in a single session.
|
|
164
|
+
*/
|
|
165
|
+
export const SESSION_WRITE_SOFT_LIMIT = 50;
|
|
119
166
|
/**
|
|
120
167
|
* Resolve the `team` config as a single object unit using precedence
|
|
121
168
|
* override > config.json > default (RESEARCH Pitfall 5). Unlike
|
|
@@ -1,13 +1,37 @@
|
|
|
1
|
+
import { CRITICAL_WARNING_IMPORTANCE_THRESHOLD } from "../config/policyConfig.js";
|
|
1
2
|
import { countTokens, trimLowestPriorityContent } from "./tokenBudget.js";
|
|
2
3
|
const DEFAULT_TOKEN_CAP = 450;
|
|
3
4
|
const HEADER = "Relevant prior context";
|
|
5
|
+
// At most this many preserved (critical-warning) entries may bypass trimming.
|
|
6
|
+
// Critical warnings rarely need more, and an unbounded count would let
|
|
7
|
+
// preserved entries dominate the injection block past the token cap.
|
|
8
|
+
const MAX_PRESERVED = 5;
|
|
9
|
+
// Strip control characters and hard-cap per-entry content before it is rendered
|
|
10
|
+
// into the injection block. Prevents a single memory from breaking out of the
|
|
11
|
+
// block (newlines/control chars) or dominating it (length). The full content
|
|
12
|
+
// remains retrievable via retrieveMemories — this only affects startup display.
|
|
13
|
+
const safeContent = (content) =>
|
|
14
|
+
// eslint-disable-next-line no-control-regex -- intentional control-char strip
|
|
15
|
+
content.replace(/[\n\r\x00-\x08\x0e-\x1f\x7f]/g, " ").slice(0, 500);
|
|
16
|
+
// Sanitize source_adapter before it is rendered verbatim onto the metadata
|
|
17
|
+
// line. Like `author`, a malformed row could otherwise smuggle newlines/control
|
|
18
|
+
// chars (and thus a prompt-injection payload) into the injection block.
|
|
19
|
+
const safeSourceAdapter = (s) =>
|
|
20
|
+
// eslint-disable-next-line no-control-regex -- intentional control-char strip
|
|
21
|
+
(s ?? "").replace(/[\n\r\x00-\x08\x0e-\x1f\x7f]/g, "").slice(0, 100);
|
|
22
|
+
// Allow-list of known kinds. Any unrecognized kind renders as "memory" so a
|
|
23
|
+
// malformed row cannot inject arbitrary text into the rendered `[kind]` label.
|
|
24
|
+
const KNOWN_KINDS = new Set(["fact", "decision", "preference", "warning", "summary", "memory", "context"]);
|
|
25
|
+
function safeKind(kind) {
|
|
26
|
+
return KNOWN_KINDS.has(kind) ? kind : "memory";
|
|
27
|
+
}
|
|
4
28
|
const KIND_ORDER = ["warning", "decision", "fact", "summary", "preference"];
|
|
5
29
|
const KIND_RANK = new Map(KIND_ORDER.map((kind, index) => [kind, index]));
|
|
6
30
|
function kindRank(kind) {
|
|
7
31
|
return KIND_RANK.get(kind) ?? KIND_ORDER.length;
|
|
8
32
|
}
|
|
9
33
|
function isCriticalWarning(memory) {
|
|
10
|
-
return memory.kind === "warning" && memory.importance >=
|
|
34
|
+
return memory.kind === "warning" && memory.importance >= CRITICAL_WARNING_IMPORTANCE_THRESHOLD;
|
|
11
35
|
}
|
|
12
36
|
function sortMemories(memories) {
|
|
13
37
|
return [...memories].sort((left, right) => {
|
|
@@ -35,7 +59,12 @@ function authorPrefix(memory, localUsername) {
|
|
|
35
59
|
if (memory.author &&
|
|
36
60
|
localUsername &&
|
|
37
61
|
memory.author !== localUsername) {
|
|
38
|
-
|
|
62
|
+
// Defense in depth: even though `author` is constrained at the contract
|
|
63
|
+
// boundary, strip any control characters before rendering so a malformed
|
|
64
|
+
// row cannot break out of the injection block.
|
|
65
|
+
// eslint-disable-next-line no-control-regex -- intentional control-char strip
|
|
66
|
+
const safeAuthor = memory.author.replace(/[\n\r\x00-\x1f\x7f]/g, "");
|
|
67
|
+
return `${safeAuthor}: `;
|
|
39
68
|
}
|
|
40
69
|
return "";
|
|
41
70
|
}
|
|
@@ -44,8 +73,8 @@ function formatLine(entry, localUsername) {
|
|
|
44
73
|
const score = memory.score;
|
|
45
74
|
const prefix = authorPrefix(memory, localUsername);
|
|
46
75
|
return [
|
|
47
|
-
`- [${memory.kind}] ${prefix}${entry.content}`,
|
|
48
|
-
`(score total=${formatScore(score.total)}, semantic=${formatScore(score.raw.semantic)}, recency=${formatScore(score.raw.recency)}, importance=${formatScore(score.raw.importance)}; source=${memory.source_adapter}; date=${memory.updated_at})`,
|
|
76
|
+
`- [${safeKind(memory.kind)}] ${prefix}${safeContent(entry.content)}`,
|
|
77
|
+
`(score total=${formatScore(score.total)}, semantic=${formatScore(score.raw.semantic)}, recency=${formatScore(score.raw.recency)}, importance=${formatScore(score.raw.importance)}; source=${safeSourceAdapter(memory.source_adapter)}; date=${memory.updated_at})`,
|
|
49
78
|
].join(" ");
|
|
50
79
|
}
|
|
51
80
|
function render(entries, localUsername) {
|
|
@@ -72,12 +101,28 @@ function lowestDroppableIndex(entries) {
|
|
|
72
101
|
export function formatStartupInjection(rankedMemories, options = {}) {
|
|
73
102
|
const tokenCap = options.tokenCap ?? DEFAULT_TOKEN_CAP;
|
|
74
103
|
const localUsername = options.localUsername;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
104
|
+
// Cap how many entries may be marked `preserve`. Excess critical warnings
|
|
105
|
+
// beyond MAX_PRESERVED become droppable so they cannot bypass the trim/drop
|
|
106
|
+
// loop and blow past the token cap. Sorting first keeps the highest-ranked
|
|
107
|
+
// warnings preserved.
|
|
108
|
+
let preservedCount = 0;
|
|
109
|
+
let included = sortMemories(rankedMemories).map((memory) => {
|
|
110
|
+
let preserve = isCriticalWarning(memory);
|
|
111
|
+
if (preserve) {
|
|
112
|
+
if (preservedCount >= MAX_PRESERVED) {
|
|
113
|
+
preserve = false;
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
preservedCount += 1;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
memory,
|
|
121
|
+
content: memory.content,
|
|
122
|
+
priority: KIND_ORDER.length - kindRank(memory.kind),
|
|
123
|
+
preserve,
|
|
124
|
+
};
|
|
125
|
+
});
|
|
81
126
|
let output = render(included, localUsername);
|
|
82
127
|
while (included.length > 0 && countTokens(output) > tokenCap) {
|
|
83
128
|
const trimmed = trimLowestPriorityContent(included);
|
|
@@ -5,6 +5,14 @@ const DEFAULT_TRIM_RATIO = 0.75;
|
|
|
5
5
|
export function countTokens(text) {
|
|
6
6
|
return encoding.encode(text).length;
|
|
7
7
|
}
|
|
8
|
+
export function capTokens(text, cap) {
|
|
9
|
+
const tokens = encoding.encode(text);
|
|
10
|
+
if (tokens.length <= cap) {
|
|
11
|
+
return text;
|
|
12
|
+
}
|
|
13
|
+
const trimmed = encoding.decode(tokens.slice(0, cap)).trimEnd();
|
|
14
|
+
return `${trimmed} ...`;
|
|
15
|
+
}
|
|
8
16
|
export function trimLowestPriorityContent(included, options = {}) {
|
|
9
17
|
const minContentTokens = options.minContentTokens ?? DEFAULT_MIN_CONTENT_TOKENS;
|
|
10
18
|
const trimRatio = options.trimRatio ?? DEFAULT_TRIM_RATIO;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { MIN_IMPORTANCE, MAX_IMPORTANCE } from "../config/policyConfig.js";
|
|
1
2
|
export function normalizeImportance(score1to10) {
|
|
2
|
-
if (!Number.isFinite(score1to10) || score1to10 <
|
|
3
|
-
throw new Error(
|
|
3
|
+
if (!Number.isFinite(score1to10) || score1to10 < MIN_IMPORTANCE || score1to10 > MAX_IMPORTANCE) {
|
|
4
|
+
throw new Error(`importance must be between ${MIN_IMPORTANCE} and ${MAX_IMPORTANCE}`);
|
|
4
5
|
}
|
|
5
|
-
return (score1to10 -
|
|
6
|
+
return (score1to10 - MIN_IMPORTANCE) / (MAX_IMPORTANCE - MIN_IMPORTANCE);
|
|
6
7
|
}
|
|
@@ -4,15 +4,11 @@ function toDate(value) {
|
|
|
4
4
|
}
|
|
5
5
|
export function getRecencyBandScore(updatedAt, now = new Date()) {
|
|
6
6
|
const updatedDate = toDate(updatedAt);
|
|
7
|
-
const ageDays =
|
|
8
|
-
if (ageDays
|
|
9
|
-
return
|
|
7
|
+
const ageDays = (now.getTime() - updatedDate.getTime()) / DAY_IN_MS;
|
|
8
|
+
if (!Number.isFinite(ageDays) || ageDays < 0) {
|
|
9
|
+
return 0.05; // invalid/future date -> treat as max age (lowest recency score)
|
|
10
10
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if (ageDays <= 30) {
|
|
15
|
-
return 0.5;
|
|
16
|
-
}
|
|
17
|
-
return 0.25;
|
|
11
|
+
const HALF_LIFE_DAYS = 14;
|
|
12
|
+
const lambda = Math.LN2 / HALF_LIFE_DAYS;
|
|
13
|
+
return Math.max(0.05, Math.exp(-lambda * ageDays));
|
|
18
14
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { deterministicEmbed } from "../embed/deterministicEmbed.js";
|
|
2
|
+
import { decayOldBoosts } from "./decay.js";
|
|
2
3
|
import { scoreMemoryCandidate, } from "./score.js";
|
|
3
|
-
import {
|
|
4
|
+
import { searchMemoryCandidatesFTS, } from "../storage/memorySearchRepo.js";
|
|
4
5
|
const DEFAULT_EMBEDDING_DIMENSION = 32;
|
|
5
6
|
function resolveEmbeddingDimension(candidates) {
|
|
6
7
|
for (const candidate of candidates) {
|
|
@@ -40,16 +41,29 @@ export function retrieveMemories(input) {
|
|
|
40
41
|
}
|
|
41
42
|
const topK = input.topK ?? input.limit ?? 20;
|
|
42
43
|
const now = input.now ?? new Date();
|
|
43
|
-
|
|
44
|
+
// Use FTS5 pre-filtering to limit cosine similarity computation to ~50 candidates.
|
|
45
|
+
// queryText is guaranteed non-empty by the Zod schema (z.string().min(1)) upstream.
|
|
46
|
+
const candidates = searchMemoryCandidatesFTS(input.db, input.projectId, queryText);
|
|
47
|
+
const decayedCandidates = decayOldBoosts(candidates, now);
|
|
44
48
|
const dimension = resolveEmbeddingDimension(candidates);
|
|
45
49
|
const queryVector = deterministicEmbed(queryText, dimension).vector;
|
|
46
|
-
const ranked =
|
|
50
|
+
const ranked = decayedCandidates
|
|
47
51
|
.map((candidate) => {
|
|
48
|
-
|
|
52
|
+
// When embedding is null (version mismatch or missing), use a neutral
|
|
53
|
+
// score of 0.5 so the memory is neither penalized nor boosted. A non-null
|
|
54
|
+
// embedding whose length differs from the query vector (a stored
|
|
55
|
+
// embedding from a different dimension) is treated the same way: passing
|
|
56
|
+
// it to cosineSimilarity would return 0 and actively penalize the row.
|
|
57
|
+
const semantic = candidate.embedding === null ||
|
|
58
|
+
candidate.embedding.length !== queryVector.length
|
|
59
|
+
? 0.5
|
|
60
|
+
: cosineSimilarity(queryVector, candidate.embedding);
|
|
49
61
|
const score = scoreMemoryCandidate({
|
|
50
62
|
semantic,
|
|
51
63
|
updated_at: candidate.updated_at,
|
|
52
64
|
importance: candidate.importance,
|
|
65
|
+
access_count: candidate.access_count,
|
|
66
|
+
decayedImportance: candidate.decayedImportance,
|
|
53
67
|
}, now);
|
|
54
68
|
return {
|
|
55
69
|
id: candidate.id,
|
|
@@ -62,6 +76,7 @@ export function retrieveMemories(input) {
|
|
|
62
76
|
importance: candidate.importance,
|
|
63
77
|
author: candidate.author,
|
|
64
78
|
origin_project_id: candidate.origin_project_id,
|
|
79
|
+
access_count: candidate.access_count,
|
|
65
80
|
created_at: candidate.created_at,
|
|
66
81
|
updated_at: candidate.updated_at,
|
|
67
82
|
embedding_dim: candidate.embedding_dim,
|
|
@@ -5,9 +5,19 @@ export const SCORING_WEIGHTS = {
|
|
|
5
5
|
recency: 0.25,
|
|
6
6
|
importance: 0.15,
|
|
7
7
|
};
|
|
8
|
+
export const ACCESS_BOOST_THRESHOLD = 3;
|
|
9
|
+
export const ACCESS_BOOST_AMOUNT = 2;
|
|
10
|
+
export function computeEffectiveImportance(storedImportance, accessCount) {
|
|
11
|
+
if (accessCount >= ACCESS_BOOST_THRESHOLD) {
|
|
12
|
+
return Math.min(storedImportance + ACCESS_BOOST_AMOUNT, 10);
|
|
13
|
+
}
|
|
14
|
+
return storedImportance;
|
|
15
|
+
}
|
|
8
16
|
export function scoreMemoryCandidate(candidate, now = new Date()) {
|
|
9
17
|
const recency = getRecencyBandScore(candidate.updated_at, now);
|
|
10
|
-
const
|
|
18
|
+
const baseImportance = candidate.decayedImportance ?? candidate.importance;
|
|
19
|
+
const effectiveImportance = computeEffectiveImportance(baseImportance, candidate.access_count ?? 0);
|
|
20
|
+
const importance = normalizeImportance(effectiveImportance);
|
|
11
21
|
const weighted = {
|
|
12
22
|
semantic: candidate.semantic * SCORING_WEIGHTS.semantic,
|
|
13
23
|
recency: recency * SCORING_WEIGHTS.recency,
|
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
-- Team-mode provenance: record who authored each memory and, for
|
|
2
|
-
-- synced/shared rows, which project they originated in. Both columns are added
|
|
3
|
-
-- to the existing `memories` table without rewriting it so pre-existing rows
|
|
4
|
-
-- survive. `author` is NOT NULL with a DEFAULT '' sentinel because a
|
|
5
|
-
-- NOT NULL ADD COLUMN requires a default and the local OS username is not
|
|
6
|
-
-- available inside static SQL. `origin_project_id` is nullable and
|
|
7
|
-
-- only set on rows pulled in from another project's store.
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
-- Team-mode provenance: record who authored each memory and, for
|
|
2
|
+
-- synced/shared rows, which project they originated in. Both columns are added
|
|
3
|
+
-- to the existing `memories` table without rewriting it so pre-existing rows
|
|
4
|
+
-- survive. `author` is NOT NULL with a DEFAULT '' sentinel because a
|
|
5
|
+
-- NOT NULL ADD COLUMN requires a default and the local OS username is not
|
|
6
|
+
-- available inside static SQL. `origin_project_id` is nullable and
|
|
7
|
+
-- only set on rows pulled in from another project's store.
|
|
8
|
+
--
|
|
9
|
+
-- Idempotency: SQLite has no `ADD COLUMN IF NOT EXISTS`. Re-running this
|
|
10
|
+
-- migration (only possible if the _migrations record was lost) throws
|
|
11
|
+
-- "duplicate column name", which runMigrations catches and treats as
|
|
12
|
+
-- already-applied. See src/core/schema/runMigrations.ts.
|
|
13
|
+
ALTER TABLE memories ADD COLUMN author TEXT NOT NULL DEFAULT '';
|
|
14
|
+
ALTER TABLE memories ADD COLUMN origin_project_id TEXT;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
-- Access-pattern boosting: track how often each memory is included in
|
|
2
|
+
-- retrieval output. access_count drives a read-time effective_importance
|
|
3
|
+
-- boost without mutating the stored importance score.
|
|
4
|
+
--
|
|
5
|
+
-- Idempotency: SQLite has no `ADD COLUMN IF NOT EXISTS`. Re-running this
|
|
6
|
+
-- migration (only possible if the _migrations record was lost) throws
|
|
7
|
+
-- "duplicate column name", which runMigrations catches and treats as
|
|
8
|
+
-- already-applied. See src/core/schema/runMigrations.ts.
|
|
9
|
+
ALTER TABLE memories ADD COLUMN access_count INTEGER NOT NULL DEFAULT 0;
|
|
10
|
+
ALTER TABLE memories ADD COLUMN last_accessed TEXT;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
-- Allow 'manual_delete' feedback_type and new_importance = 0 for deletion records.
|
|
2
|
+
-- Remove FOREIGN KEY CASCADE so feedback rows survive when a memory is deleted.
|
|
3
|
+
-- SQLite requires table recreation to alter CHECK constraints and FK behavior.
|
|
4
|
+
|
|
5
|
+
CREATE TABLE IF NOT EXISTS memory_feedback_new (
|
|
6
|
+
id TEXT PRIMARY KEY,
|
|
7
|
+
memory_id TEXT NOT NULL,
|
|
8
|
+
feedback_type TEXT NOT NULL CHECK (feedback_type IN ('auto_use', 'manual', 'manual_delete')),
|
|
9
|
+
previous_importance INTEGER NOT NULL CHECK (previous_importance >= 0 AND previous_importance <= 10),
|
|
10
|
+
new_importance INTEGER NOT NULL CHECK (new_importance >= 0 AND new_importance <= 10),
|
|
11
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
12
|
+
);
|
|
13
|
+
|
|
14
|
+
INSERT OR IGNORE INTO memory_feedback_new (id, memory_id, feedback_type, previous_importance, new_importance, created_at)
|
|
15
|
+
SELECT id, memory_id, feedback_type, previous_importance, new_importance, created_at
|
|
16
|
+
FROM memory_feedback;
|
|
17
|
+
|
|
18
|
+
DROP TABLE memory_feedback;
|
|
19
|
+
|
|
20
|
+
ALTER TABLE memory_feedback_new RENAME TO memory_feedback;
|
|
21
|
+
|
|
22
|
+
CREATE INDEX IF NOT EXISTS idx_memory_feedback_memory_created
|
|
23
|
+
ON memory_feedback(memory_id, created_at DESC);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
-- FTS5 full-text search index on memory content for candidate pre-filtering.
|
|
2
|
+
-- Using content-sync (external content) mode: the FTS index mirrors the
|
|
3
|
+
-- memories table without duplicating storage. Triggers keep it in sync.
|
|
4
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
5
|
+
content,
|
|
6
|
+
normalized_content,
|
|
7
|
+
content='memories',
|
|
8
|
+
content_rowid='rowid'
|
|
9
|
+
);
|
|
10
|
+
|
|
11
|
+
-- Populate FTS index from existing rows. Guard against double-backfill: if this
|
|
12
|
+
-- migration is re-run on a DB that already has FTS data (e.g. after the
|
|
13
|
+
-- _migrations record was lost), re-inserting every row would duplicate the
|
|
14
|
+
-- index entries. Only backfill when the FTS table is empty.
|
|
15
|
+
INSERT INTO memories_fts(rowid, content, normalized_content)
|
|
16
|
+
SELECT rowid, content, normalized_content FROM memories
|
|
17
|
+
WHERE NOT EXISTS (SELECT 1 FROM memories_fts LIMIT 1);
|
|
18
|
+
|
|
19
|
+
-- Keep FTS index in sync: INSERT trigger
|
|
20
|
+
CREATE TRIGGER IF NOT EXISTS memories_fts_insert AFTER INSERT ON memories BEGIN
|
|
21
|
+
INSERT INTO memories_fts(rowid, content, normalized_content)
|
|
22
|
+
VALUES (new.rowid, new.content, new.normalized_content);
|
|
23
|
+
END;
|
|
24
|
+
|
|
25
|
+
-- Keep FTS index in sync: DELETE trigger
|
|
26
|
+
CREATE TRIGGER IF NOT EXISTS memories_fts_delete AFTER DELETE ON memories BEGIN
|
|
27
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, normalized_content)
|
|
28
|
+
VALUES ('delete', old.rowid, old.content, old.normalized_content);
|
|
29
|
+
END;
|
|
30
|
+
|
|
31
|
+
-- Keep FTS index in sync: UPDATE trigger (content or normalized_content changed)
|
|
32
|
+
CREATE TRIGGER IF NOT EXISTS memories_fts_update AFTER UPDATE OF content, normalized_content ON memories BEGIN
|
|
33
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, normalized_content)
|
|
34
|
+
VALUES ('delete', old.rowid, old.content, old.normalized_content);
|
|
35
|
+
INSERT INTO memories_fts(rowid, content, normalized_content)
|
|
36
|
+
VALUES (new.rowid, new.content, new.normalized_content);
|
|
37
|
+
END;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
-- Deduplicate session events by their logical key.
|
|
2
|
+
--
|
|
3
|
+
-- `ingestSessionEvents` is now reachable from agents (MCP tool) and CLI, so it
|
|
4
|
+
-- can be called more than once for the same session. The original index on
|
|
5
|
+
-- (project_id, session_id, event_index) was NOT unique, so re-ingesting the
|
|
6
|
+
-- same logical event with a fresh `id` silently created duplicate rows — which
|
|
7
|
+
-- then inflated session-event counts and the local-summarizer input.
|
|
8
|
+
--
|
|
9
|
+
-- Replace it with a UNIQUE index so `INSERT OR IGNORE` makes re-ingestion a
|
|
10
|
+
-- no-op. session_events has no production writer before this migration, so there
|
|
11
|
+
-- are no pre-existing duplicates to reconcile.
|
|
12
|
+
DROP INDEX IF EXISTS idx_session_events_project_session_event_index;
|
|
13
|
+
|
|
14
|
+
-- Remove duplicate rows before adding unique constraint.
|
|
15
|
+
-- Keep the row with the smallest rowid for each logical key.
|
|
16
|
+
DELETE FROM session_events
|
|
17
|
+
WHERE rowid NOT IN (
|
|
18
|
+
SELECT MIN(rowid)
|
|
19
|
+
FROM session_events
|
|
20
|
+
GROUP BY project_id, session_id, event_index
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_session_events_project_session_event_index
|
|
24
|
+
ON session_events(project_id, session_id, event_index);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
const DEFAULT_MIGRATIONS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "migrations");
|
|
4
5
|
function ensureMigrationsTable(db) {
|
|
5
6
|
db.exec(`
|
|
6
7
|
CREATE TABLE IF NOT EXISTS _migrations (
|
|
@@ -18,6 +19,38 @@ function listMigrationFiles(migrationsDir) {
|
|
|
18
19
|
.filter((fileName) => fileName.endsWith(".sql"))
|
|
19
20
|
.sort((left, right) => left.localeCompare(right));
|
|
20
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Verify that every column an ADD COLUMN migration declares already exists in
|
|
24
|
+
* its target table. Parses `ALTER TABLE <table> ADD COLUMN <name>` statements
|
|
25
|
+
* from the migration SQL and checks each against `PRAGMA table_info`. Returns
|
|
26
|
+
* false the moment a declared column is missing (or its table doesn't exist),
|
|
27
|
+
* so a partially-applied migration is never marked complete.
|
|
28
|
+
*/
|
|
29
|
+
function allAddedColumnsExist(db, sql) {
|
|
30
|
+
const addColumnRe = /ALTER\s+TABLE\s+(\w+)\s+ADD\s+COLUMN\s+(\w+)/gi;
|
|
31
|
+
const byTable = new Map();
|
|
32
|
+
let match;
|
|
33
|
+
while ((match = addColumnRe.exec(sql)) !== null) {
|
|
34
|
+
const [, table, column] = match;
|
|
35
|
+
const cols = byTable.get(table) ?? [];
|
|
36
|
+
cols.push(column);
|
|
37
|
+
byTable.set(table, cols);
|
|
38
|
+
}
|
|
39
|
+
// No ADD COLUMN statements parsed → we cannot prove the schema is consistent,
|
|
40
|
+
// so treat it as unsafe and let the caller re-throw.
|
|
41
|
+
if (byTable.size === 0) {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
for (const [table, columns] of byTable) {
|
|
45
|
+
const existing = new Set(db.prepare(`PRAGMA table_info(${table})`).all().map((row) => row.name));
|
|
46
|
+
for (const column of columns) {
|
|
47
|
+
if (!existing.has(column)) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
21
54
|
export function runMigrations(db, migrationsDir = DEFAULT_MIGRATIONS_DIR) {
|
|
22
55
|
ensureMigrationsTable(db);
|
|
23
56
|
const files = listMigrationFiles(migrationsDir);
|
|
@@ -31,8 +64,37 @@ export function runMigrations(db, migrationsDir = DEFAULT_MIGRATIONS_DIR) {
|
|
|
31
64
|
});
|
|
32
65
|
for (const fileName of files) {
|
|
33
66
|
const existing = hasMigrationStmt.get(fileName);
|
|
34
|
-
if (
|
|
67
|
+
if (existing) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
35
71
|
runMigration(fileName);
|
|
36
72
|
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
// Idempotency guard for ALTER TABLE ADD COLUMN migrations (005/006).
|
|
75
|
+
// SQLite has no `ADD COLUMN IF NOT EXISTS`, so re-running a column-adding
|
|
76
|
+
// migration on a DB that already has the column throws "duplicate column
|
|
77
|
+
// name". This only happens when the _migrations record was lost (e.g. the
|
|
78
|
+
// table was dropped) while the schema change survived.
|
|
79
|
+
//
|
|
80
|
+
// Each migration runs in a transaction (all-or-nothing), so the duplicate
|
|
81
|
+
// error rolls the whole body back. Migrations 005/006 add TWO columns
|
|
82
|
+
// each: if only the FIRST already exists, the throw fires on the first
|
|
83
|
+
// ALTER and the second column is never added. Blindly marking the
|
|
84
|
+
// migration applied would leave that second column permanently missing.
|
|
85
|
+
//
|
|
86
|
+
// So instead of trusting the error, verify that EVERY column this
|
|
87
|
+
// migration was supposed to add actually exists. Only then is it safe to
|
|
88
|
+
// record as applied; otherwise re-throw so the failure surfaces.
|
|
89
|
+
if (err instanceof Error && /duplicate column name/i.test(err.message)) {
|
|
90
|
+
const filePath = path.join(migrationsDir, fileName);
|
|
91
|
+
const sql = fs.readFileSync(filePath, "utf8");
|
|
92
|
+
if (allAddedColumnsExist(db, sql)) {
|
|
93
|
+
insertMigrationStmt.run(fileName);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
throw err;
|
|
98
|
+
}
|
|
37
99
|
}
|
|
38
100
|
}
|
package/dist/core/storage/db.js
CHANGED
|
@@ -2,7 +2,13 @@ import BetterSqlite3 from "better-sqlite3";
|
|
|
2
2
|
import { runMigrations } from "../schema/runMigrations.js";
|
|
3
3
|
export function openDb(options = {}) {
|
|
4
4
|
const db = new BetterSqlite3(options.dbPath ?? ":memory:");
|
|
5
|
+
// Performance pragmas — WAL enables concurrent reads during writes;
|
|
6
|
+
// busy_timeout prevents SQLITE_BUSY under concurrent MCP tool calls.
|
|
7
|
+
db.pragma("journal_mode = WAL");
|
|
8
|
+
db.pragma("synchronous = NORMAL");
|
|
5
9
|
db.pragma("foreign_keys = ON");
|
|
10
|
+
db.pragma("busy_timeout = 5000");
|
|
11
|
+
db.pragma("cache_size = -32000");
|
|
6
12
|
runMigrations(db, options.migrationsDir);
|
|
7
13
|
return db;
|
|
8
14
|
}
|
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
const feedbackStmtCache = new WeakMap();
|
|
3
|
+
function getFeedbackStatements(db) {
|
|
4
|
+
let stmts = feedbackStmtCache.get(db);
|
|
5
|
+
if (stmts)
|
|
6
|
+
return stmts;
|
|
7
|
+
stmts = {
|
|
8
|
+
insertFeedback: db.prepare(`
|
|
4
9
|
INSERT INTO memory_feedback (
|
|
5
10
|
id, memory_id, feedback_type, previous_importance, new_importance, created_at
|
|
6
11
|
) VALUES (
|
|
7
12
|
@id, @memory_id, @feedback_type, @previous_importance, @new_importance,
|
|
8
13
|
COALESCE(@created_at, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
9
14
|
)
|
|
10
|
-
`)
|
|
11
|
-
|
|
15
|
+
`),
|
|
16
|
+
};
|
|
17
|
+
feedbackStmtCache.set(db, stmts);
|
|
18
|
+
return stmts;
|
|
19
|
+
}
|
|
20
|
+
export function insertMemoryFeedbackEvent(db, event) {
|
|
21
|
+
getFeedbackStatements(db).insertFeedback.run({
|
|
12
22
|
...event,
|
|
13
23
|
id: event.id ?? randomUUID(),
|
|
14
24
|
created_at: event.created_at ?? null,
|