stackmem 1.0.15 → 1.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark.js +20 -4
- package/dist/cli.js +94 -20
- package/dist/index.js +5 -1
- package/dist/scoring.js +80 -16
- package/package.json +1 -1
- package/src/benchmark.ts +22 -4
- package/src/cli.ts +103 -19
- package/src/index.ts +6 -1
- package/src/scoring.ts +85 -18
package/dist/benchmark.js
CHANGED
|
@@ -8,7 +8,7 @@ import { execSync } from "node:child_process";
|
|
|
8
8
|
import nodeFs from "node:fs";
|
|
9
9
|
import nodePath from "node:path";
|
|
10
10
|
import { fileURLToPath } from "node:url";
|
|
11
|
-
import { applyContextBudget, computeDecayScore,
|
|
11
|
+
import { applyContextBudget, computeDecayScore, decideSaveAction, estimateTokens, extractEntities, hybridScore, scoreMemoryRelevance, } from "./scoring.js";
|
|
12
12
|
import { supabase } from "./storage.js";
|
|
13
13
|
const CONTEXT_BUDGET_TOKENS = 2000;
|
|
14
14
|
const REPO_ROOT = nodePath.join(nodePath.dirname(fileURLToPath(import.meta.url)), "..");
|
|
@@ -256,7 +256,16 @@ function benchmarkDuplicateDetection(allMemories, unrelatedPairs) {
|
|
|
256
256
|
// a harder, unverified case this benchmark doesn't cover.
|
|
257
257
|
const CONTRADICTION_PREFIXES = ["switched from ", "no longer using "];
|
|
258
258
|
function benchmarkContradictionDetection(allMemories, unrelatedPairs) {
|
|
259
|
-
|
|
259
|
+
// Hook-generated templates (dependency changes, config/schema/TODO
|
|
260
|
+
// discoveries) are structural event records, not decisions — decideSaveAction
|
|
261
|
+
// deliberately exempts dependency templates from ever routing to
|
|
262
|
+
// "superseded" (see DEPENDENCY_TEMPLATE_RE in scoring.ts), and prepending
|
|
263
|
+
// "switched from X" to any of these templates isn't a phrase anyone would
|
|
264
|
+
// actually write. Sampling them here would just measure whether that
|
|
265
|
+
// exemption holds, not whether contradiction detection works — so they're
|
|
266
|
+
// excluded, keeping this test on genuine decision/discovery-style content.
|
|
267
|
+
const contradictableMemories = allMemories.filter((m) => !HOOK_TEMPLATE_PREFIXES.some((p) => m.content.startsWith(p.prefix)));
|
|
268
|
+
const baseSample = seededShuffle(contradictableMemories, BENCHMARK_SEED + 2).slice(0, SAMPLE_SIZE / 2);
|
|
260
269
|
let tp = 0;
|
|
261
270
|
let fn = 0;
|
|
262
271
|
for (const memory of baseSample) {
|
|
@@ -294,6 +303,14 @@ function benchmarkContradictionDetection(allMemories, unrelatedPairs) {
|
|
|
294
303
|
// exhaustively rather than randomly sampled — at this data size (a few
|
|
295
304
|
// thousand possible pairs at most) exhaustive is cheap and gives a more
|
|
296
305
|
// stable number than a small random draw would.
|
|
306
|
+
//
|
|
307
|
+
// CAVEAT: avgLinkScore below reads memory_links.score as stored — i.e.
|
|
308
|
+
// whatever computeMemoryLinks computed at save time. Rows saved before
|
|
309
|
+
// computeMemoryLinks switched from containmentScore to hybridScore still
|
|
310
|
+
// hold old-formula scores; only the baseline here is guaranteed to reflect
|
|
311
|
+
// the current formula (it's computed live). A mixed-vintage avgLinkScore
|
|
312
|
+
// vs. a current-formula baseline is still the most honest comparison
|
|
313
|
+
// available without backfilling every historical link.
|
|
297
314
|
function benchmarkSemanticLinking(allMemories, allLinks) {
|
|
298
315
|
const avgLinkScore = allLinks.length === 0 ? 0 : allLinks.reduce((sum, l) => sum + l.score, 0) / allLinks.length;
|
|
299
316
|
const linkedPairKeys = new Set(allLinks.map((l) => pairKey(l.source_id, l.target_id)));
|
|
@@ -306,13 +323,12 @@ function benchmarkSemanticLinking(allMemories, allLinks) {
|
|
|
306
323
|
let baselineSum = 0;
|
|
307
324
|
let baselineCount = 0;
|
|
308
325
|
for (const rows of byProject.values()) {
|
|
309
|
-
const entitiesById = new Map(rows.map((m) => [m.id, extractEntities(m.content)]));
|
|
310
326
|
for (let i = 0; i < rows.length; i++) {
|
|
311
327
|
for (let j = i + 1; j < rows.length; j++) {
|
|
312
328
|
const key = pairKey(rows[i].id, rows[j].id);
|
|
313
329
|
if (linkedPairKeys.has(key))
|
|
314
330
|
continue;
|
|
315
|
-
baselineSum +=
|
|
331
|
+
baselineSum += hybridScore(rows[i].content, rows[j].content);
|
|
316
332
|
baselineCount++;
|
|
317
333
|
}
|
|
318
334
|
}
|
package/dist/cli.js
CHANGED
|
@@ -234,6 +234,8 @@ async function cmdStart(project) {
|
|
|
234
234
|
const memories = data ?? [];
|
|
235
235
|
if (memories.length === 0) {
|
|
236
236
|
console.log(`No memories found for "${project}".`);
|
|
237
|
+
console.log("");
|
|
238
|
+
console.log("Tip: run 'cm init' to seed tech stack memories from your project files.");
|
|
237
239
|
return;
|
|
238
240
|
}
|
|
239
241
|
const scoredMemories = memories
|
|
@@ -310,6 +312,11 @@ async function cmdStart(project) {
|
|
|
310
312
|
if (omittedCount > 0) {
|
|
311
313
|
console.log(`${omittedCount} memories omitted (budget exceeded)`);
|
|
312
314
|
}
|
|
315
|
+
const hasTechStackMemory = memories.some((m) => /^tech stack:/i.test(m.content));
|
|
316
|
+
if (!hasTechStackMemory) {
|
|
317
|
+
console.log("");
|
|
318
|
+
console.log("Tip: run 'cm init' to seed tech stack memories from your project files.");
|
|
319
|
+
}
|
|
313
320
|
const incrementResults = await Promise.all(budgetedMemories.map((memory) => supabase
|
|
314
321
|
.from("memories")
|
|
315
322
|
.update({ access_count: (memory.access_count ?? 0) + 1 })
|
|
@@ -923,6 +930,24 @@ function registerMcpServer(distIndexPath) {
|
|
|
923
930
|
nodeFs.writeFileSync(claudeConfigPath, JSON.stringify(config, null, 2), "utf-8");
|
|
924
931
|
console.log("stackmem MCP server registered with Claude Code.");
|
|
925
932
|
}
|
|
933
|
+
const DEFAULT_GITIGNORE_CONTENT = `node_modules/
|
|
934
|
+
dist/
|
|
935
|
+
.env
|
|
936
|
+
.env.local
|
|
937
|
+
.DS_Store
|
|
938
|
+
*.log
|
|
939
|
+
coverage/
|
|
940
|
+
`;
|
|
941
|
+
// Returns true if the file was created, false if it already existed
|
|
942
|
+
// (and was left untouched).
|
|
943
|
+
function writeGitignore(resolvedPath) {
|
|
944
|
+
const gitignorePath = nodePath.join(resolvedPath, ".gitignore");
|
|
945
|
+
if (nodeFs.existsSync(gitignorePath)) {
|
|
946
|
+
return false;
|
|
947
|
+
}
|
|
948
|
+
nodeFs.writeFileSync(gitignorePath, DEFAULT_GITIGNORE_CONTENT, "utf-8");
|
|
949
|
+
return true;
|
|
950
|
+
}
|
|
926
951
|
function writeClaudeMd(project, resolvedPath) {
|
|
927
952
|
const claudeMdPath = nodePath.join(resolvedPath, "CLAUDE.md");
|
|
928
953
|
if (nodeFs.existsSync(claudeMdPath)) {
|
|
@@ -1040,6 +1065,21 @@ function writeProjectFile(project, resolvedPath) {
|
|
|
1040
1065
|
const stackmemPath = nodePath.join(resolvedPath, STACKMEM_FILE);
|
|
1041
1066
|
nodeFs.writeFileSync(stackmemPath, `${project}\n`, "utf-8");
|
|
1042
1067
|
}
|
|
1068
|
+
// Shared by every seedMemories check below: skip the insert if this exact
|
|
1069
|
+
// content already exists (decideSaveAction's duplicate check) so re-running
|
|
1070
|
+
// `cm init` on an already-seeded project doesn't pile up repeats.
|
|
1071
|
+
async function trySeedMemory(project, existingMemories, type, content) {
|
|
1072
|
+
const decision = decideSaveAction(content, existingMemories);
|
|
1073
|
+
if (decision.action === "duplicate")
|
|
1074
|
+
return false;
|
|
1075
|
+
const { error } = await supabase.from("memories").insert({
|
|
1076
|
+
project,
|
|
1077
|
+
type,
|
|
1078
|
+
content,
|
|
1079
|
+
device_id: getDeviceId(),
|
|
1080
|
+
});
|
|
1081
|
+
return !error;
|
|
1082
|
+
}
|
|
1043
1083
|
// Scans the freshly-initialized project for a few cheap, high-signal facts
|
|
1044
1084
|
// and saves them as memories directly (bypassing cmdSave's duplicate/conflict
|
|
1045
1085
|
// prompt — there's nothing to conflict with yet, and init must stay
|
|
@@ -1064,17 +1104,8 @@ async function seedMemories(project, resolvedPath) {
|
|
|
1064
1104
|
];
|
|
1065
1105
|
if (depNames.length > 0) {
|
|
1066
1106
|
const content = `tech stack: ${depNames.join(", ")}`;
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
const { error } = await supabase.from("memories").insert({
|
|
1070
|
-
project,
|
|
1071
|
-
type: "discovery",
|
|
1072
|
-
content,
|
|
1073
|
-
device_id: getDeviceId(),
|
|
1074
|
-
});
|
|
1075
|
-
if (!error)
|
|
1076
|
-
seededCount++;
|
|
1077
|
-
}
|
|
1107
|
+
if (await trySeedMemory(project, existingMemories, "discovery", content))
|
|
1108
|
+
seededCount++;
|
|
1078
1109
|
}
|
|
1079
1110
|
}
|
|
1080
1111
|
catch {
|
|
@@ -1090,18 +1121,57 @@ async function seedMemories(project, resolvedPath) {
|
|
|
1090
1121
|
.map((line) => line.split("=")[0]);
|
|
1091
1122
|
if (varNames.length > 0) {
|
|
1092
1123
|
const content = `required env vars: ${varNames.join(", ")}`;
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1124
|
+
if (await trySeedMemory(project, existingMemories, "constraint", content))
|
|
1125
|
+
seededCount++;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
const tsconfigPath = nodePath.join(resolvedPath, "tsconfig.json");
|
|
1129
|
+
if (nodeFs.existsSync(tsconfigPath)) {
|
|
1130
|
+
try {
|
|
1131
|
+
const tsconfig = JSON.parse(nodeFs.readFileSync(tsconfigPath, "utf-8"));
|
|
1132
|
+
const target = tsconfig.compilerOptions?.target;
|
|
1133
|
+
const moduleKind = tsconfig.compilerOptions?.module;
|
|
1134
|
+
if (target || moduleKind) {
|
|
1135
|
+
const content = `TypeScript config: target=${target ?? "unspecified"}, module=${moduleKind ?? "unspecified"}`;
|
|
1136
|
+
if (await trySeedMemory(project, existingMemories, "constraint", content))
|
|
1102
1137
|
seededCount++;
|
|
1103
1138
|
}
|
|
1104
1139
|
}
|
|
1140
|
+
catch {
|
|
1141
|
+
// Malformed tsconfig.json (or JSONC with comments) — skip.
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
const readmePath = nodePath.join(resolvedPath, "README.md");
|
|
1145
|
+
if (nodeFs.existsSync(readmePath)) {
|
|
1146
|
+
const words = nodeFs
|
|
1147
|
+
.readFileSync(readmePath, "utf-8")
|
|
1148
|
+
.split(/\s+/)
|
|
1149
|
+
.filter(Boolean)
|
|
1150
|
+
.slice(0, 100);
|
|
1151
|
+
if (words.length > 0) {
|
|
1152
|
+
const content = `project description: ${words.join(" ")}`;
|
|
1153
|
+
if (await trySeedMemory(project, existingMemories, "discovery", content))
|
|
1154
|
+
seededCount++;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
const FRAMEWORK_FILE_INDICATORS = [
|
|
1158
|
+
{ file: "next.config.ts", type: "constraint", content: "framework: Next.js" },
|
|
1159
|
+
{ file: "next.config.js", type: "constraint", content: "framework: Next.js" },
|
|
1160
|
+
{ file: "vite.config.ts", type: "constraint", content: "framework: Vite" },
|
|
1161
|
+
{ file: "vite.config.js", type: "constraint", content: "framework: Vite" },
|
|
1162
|
+
{ file: "Dockerfile", type: "constraint", content: "containerized: Docker" },
|
|
1163
|
+
];
|
|
1164
|
+
for (const indicator of FRAMEWORK_FILE_INDICATORS) {
|
|
1165
|
+
if (!nodeFs.existsSync(nodePath.join(resolvedPath, indicator.file)))
|
|
1166
|
+
continue;
|
|
1167
|
+
if (await trySeedMemory(project, existingMemories, indicator.type, indicator.content))
|
|
1168
|
+
seededCount++;
|
|
1169
|
+
}
|
|
1170
|
+
const githubWorkflowsPath = nodePath.join(resolvedPath, ".github", "workflows");
|
|
1171
|
+
if (nodeFs.existsSync(githubWorkflowsPath)) {
|
|
1172
|
+
const content = "CI/CD: GitHub Actions";
|
|
1173
|
+
if (await trySeedMemory(project, existingMemories, "discovery", content))
|
|
1174
|
+
seededCount++;
|
|
1105
1175
|
}
|
|
1106
1176
|
analyzeCodebase(resolvedPath);
|
|
1107
1177
|
return seededCount;
|
|
@@ -1122,6 +1192,7 @@ async function cmdInit(project, targetPath) {
|
|
|
1122
1192
|
console.error(`No git repo found at ${targetPath}`);
|
|
1123
1193
|
process.exit(1);
|
|
1124
1194
|
}
|
|
1195
|
+
const gitignoreCreated = writeGitignore(resolvedPath);
|
|
1125
1196
|
const codingMemoryDir = nodePath.join(resolvedPath, ".coding-memory");
|
|
1126
1197
|
nodeFs.mkdirSync(codingMemoryDir, { recursive: true });
|
|
1127
1198
|
const hookLogicPath = nodePath.join(codingMemoryDir, "post-commit-hook.mjs");
|
|
@@ -1141,6 +1212,9 @@ async function cmdInit(project, targetPath) {
|
|
|
1141
1212
|
const windsurfRulesWritten = writeWindsurfRules(resolvedPath, rulesContent);
|
|
1142
1213
|
const backendOk = await verifyBackendConnection(project);
|
|
1143
1214
|
console.log("");
|
|
1215
|
+
if (gitignoreCreated) {
|
|
1216
|
+
console.log("✓ .gitignore created");
|
|
1217
|
+
}
|
|
1144
1218
|
console.log("✓ Git hook installed");
|
|
1145
1219
|
console.log("✓ MCP server registered");
|
|
1146
1220
|
console.log("✓ CLAUDE.md written");
|
package/dist/index.js
CHANGED
|
@@ -89,8 +89,12 @@ server.registerTool("session_start", {
|
|
|
89
89
|
const budgetText = omittedCount > 0
|
|
90
90
|
? `Loaded ${memories.length} memory row(s) for "${project}" (${tokensUsed}/${CONTEXT_BUDGET_TOKENS} tokens used, ${omittedCount} omitted for budget).`
|
|
91
91
|
: `Loaded ${memories.length} memory row(s) for "${project}" (${tokensUsed}/${CONTEXT_BUDGET_TOKENS} tokens used).`;
|
|
92
|
+
const hasTechStackMemory = memories.some((m) => /^tech stack:/i.test(m.content));
|
|
93
|
+
const techStackTip = hasTechStackMemory
|
|
94
|
+
? ""
|
|
95
|
+
: "\n\nTip: run 'cm init' to seed tech stack memories from your project files.";
|
|
92
96
|
return {
|
|
93
|
-
content: [{ type: "text", text: budgetText }],
|
|
97
|
+
content: [{ type: "text", text: budgetText + techStackTip }],
|
|
94
98
|
structuredContent: {
|
|
95
99
|
memories: memoriesWithLinks,
|
|
96
100
|
fixes,
|
package/dist/scoring.js
CHANGED
|
@@ -3,10 +3,31 @@ const STOP_WORDS = new Set([
|
|
|
3
3
|
"we", "use", "for", "all", "a", "the", "is", "has", "have", "are", "to",
|
|
4
4
|
"of", "in", "it", "this", "that", "with", "and", "or", "but",
|
|
5
5
|
]);
|
|
6
|
+
// Bumping this from 5 to 6 was considered (see the benchmark's duplicate-
|
|
7
|
+
// detection findings) but doesn't actually fix anything: "removed" is 7
|
|
8
|
+
// chars and "added"/"dependency" aren't the problem either. The real issue
|
|
9
|
+
// is entity-level, not length-based — see DEPENDENCY_TEMPLATE_RE below.
|
|
6
10
|
const MIN_ENTITY_LENGTH = 5;
|
|
11
|
+
// Hook-generated dependency-change templates ("removed dependency: X" /
|
|
12
|
+
// "added dependency: X") reduce to just {"removed"/"added", "dependency"}
|
|
13
|
+
// under generic word-based extraction below, since package names are often
|
|
14
|
+
// short (node:path, zod, ./device.js — all filtered by MIN_ENTITY_LENGTH).
|
|
15
|
+
// That made two DIFFERENT packages removed in different commits look like
|
|
16
|
+
// duplicates of each other. Special-cased here instead: for these
|
|
17
|
+
// templates, the package name itself is the only entity, so containment
|
|
18
|
+
// (and hybridScore) only agrees when the package name actually matches.
|
|
19
|
+
const DEPENDENCY_TEMPLATE_RE = /^(removed|added) dependency:/i;
|
|
20
|
+
const DEPENDENCY_PACKAGE_RE = /^(?:removed|added) dependency:\s*(.+)$/i;
|
|
21
|
+
function extractDependencyPackageName(content) {
|
|
22
|
+
const match = DEPENDENCY_PACKAGE_RE.exec(content.trim());
|
|
23
|
+
return match ? match[1].trim().toLowerCase() : null;
|
|
24
|
+
}
|
|
7
25
|
// Key entities: words longer than 4 characters that aren't stop words.
|
|
8
26
|
// Short/common words carry little identifying signal for a memory.
|
|
9
27
|
export function extractEntities(text) {
|
|
28
|
+
const packageName = extractDependencyPackageName(text);
|
|
29
|
+
if (packageName !== null)
|
|
30
|
+
return new Set([packageName]);
|
|
10
31
|
const entities = new Set();
|
|
11
32
|
for (const word of text.toLowerCase().split(/\W+/)) {
|
|
12
33
|
if (word.length < MIN_ENTITY_LENGTH || STOP_WORDS.has(word))
|
|
@@ -29,13 +50,58 @@ export function containmentScore(entitiesA, entitiesB) {
|
|
|
29
50
|
}
|
|
30
51
|
return shared / smallerSize;
|
|
31
52
|
}
|
|
53
|
+
function trigramSet(text) {
|
|
54
|
+
const normalized = text.toLowerCase();
|
|
55
|
+
if (normalized.length < 3) {
|
|
56
|
+
return normalized.length > 0 ? new Set([normalized]) : new Set();
|
|
57
|
+
}
|
|
58
|
+
const trigrams = new Set();
|
|
59
|
+
for (let i = 0; i <= normalized.length - 3; i++) {
|
|
60
|
+
trigrams.add(normalized.slice(i, i + 3));
|
|
61
|
+
}
|
|
62
|
+
return trigrams;
|
|
63
|
+
}
|
|
64
|
+
// Character-trigram Jaccard overlap: catches surface-level lexical
|
|
65
|
+
// similarity (typos, partial substrings, shared phrasing) that word-based
|
|
66
|
+
// containment misses entirely, since containment only ever compares whole
|
|
67
|
+
// extracted entities.
|
|
68
|
+
function ngramOverlap(a, b) {
|
|
69
|
+
const trigramsA = trigramSet(a);
|
|
70
|
+
const trigramsB = trigramSet(b);
|
|
71
|
+
const union = new Set([...trigramsA, ...trigramsB]);
|
|
72
|
+
if (union.size === 0)
|
|
73
|
+
return 0;
|
|
74
|
+
let shared = 0;
|
|
75
|
+
for (const trigram of trigramsA) {
|
|
76
|
+
if (trigramsB.has(trigram))
|
|
77
|
+
shared++;
|
|
78
|
+
}
|
|
79
|
+
return shared / union.size;
|
|
80
|
+
}
|
|
81
|
+
// Blends entity containment (structural: do the two texts share key
|
|
82
|
+
// extracted terms) with character-trigram overlap (lexical: do the two
|
|
83
|
+
// texts look alike as raw strings) so near-duplicates with slightly
|
|
84
|
+
// different wording still score high even when their extracted entities
|
|
85
|
+
// don't line up neatly, and vice versa.
|
|
86
|
+
//
|
|
87
|
+
// NOTE ON THRESHOLDS: DUPLICATE_THRESHOLD (0.85), CONTRADICTION_MIN (0.4),
|
|
88
|
+
// computeMemoryLinks' default (0.15), and CLUSTER_THRESHOLD (0.2) were all
|
|
89
|
+
// tuned against pure containmentScore. The 0.3 trigram component shifts
|
|
90
|
+
// scores upward for almost any pair sharing common substrings (whitespace,
|
|
91
|
+
// "ing", "the", etc.), so these thresholds may need recalibration —
|
|
92
|
+
// re-validate against src/benchmark.ts rather than assuming they still
|
|
93
|
+
// hold as-is.
|
|
94
|
+
export function hybridScore(a, b) {
|
|
95
|
+
const containment = containmentScore(extractEntities(a), extractEntities(b));
|
|
96
|
+
const overlap = ngramOverlap(a, b);
|
|
97
|
+
return containment * 0.7 + overlap * 0.3;
|
|
98
|
+
}
|
|
32
99
|
export function computeMemoryLinks(newMemoryId, newContent, existingMemories, threshold = 0.15) {
|
|
33
|
-
const newEntities = extractEntities(newContent);
|
|
34
100
|
return existingMemories
|
|
35
101
|
.map((existing) => ({
|
|
36
102
|
source_id: newMemoryId,
|
|
37
103
|
target_id: existing.id,
|
|
38
|
-
score:
|
|
104
|
+
score: hybridScore(newContent, existing.content),
|
|
39
105
|
}))
|
|
40
106
|
.filter((link) => link.score > threshold);
|
|
41
107
|
}
|
|
@@ -52,11 +118,6 @@ function containsNegationWord(text) {
|
|
|
52
118
|
const lower = text.toLowerCase();
|
|
53
119
|
return NEGATION_WORDS.some((word) => lower.includes(word));
|
|
54
120
|
}
|
|
55
|
-
// Hook-generated dependency-change templates (e.g. "removed dependency: X")
|
|
56
|
-
// are structural facts, not decisions — "removed" trips the negation-word
|
|
57
|
-
// check below, but there's nothing for them to supersede/contradict, so
|
|
58
|
-
// they're exempted up front rather than ever entering the negation branch.
|
|
59
|
-
const DEPENDENCY_TEMPLATE_RE = /^(removed|added) dependency:/i;
|
|
60
121
|
// Decides whether a new memory is a near-duplicate of an existing one
|
|
61
122
|
// (skip it), supersedes an existing one (resolve the old, insert the new),
|
|
62
123
|
// or is genuinely new. A negation word (e.g. "switched", "no longer")
|
|
@@ -66,12 +127,15 @@ const DEPENDENCY_TEMPLATE_RE = /^(removed|added) dependency:/i;
|
|
|
66
127
|
// Scanning finds the best-scoring match in each band rather than stopping
|
|
67
128
|
// at the first hit, so the result doesn't depend on row order.
|
|
68
129
|
export function decideSaveAction(newContent, existingMemories) {
|
|
69
|
-
|
|
130
|
+
// Dependency-change templates are structural facts, not decisions —
|
|
131
|
+
// "removed"/"added" would otherwise trip containsNegationWord below, but
|
|
132
|
+
// there's nothing for them to supersede/contradict, so they're exempted
|
|
133
|
+
// up front and only ever fall through to the plain duplicate check.
|
|
70
134
|
const hasNegation = DEPENDENCY_TEMPLATE_RE.test(newContent) ? false : containsNegationWord(newContent);
|
|
71
135
|
let bestDuplicate = null;
|
|
72
136
|
let bestSupersede = null;
|
|
73
137
|
for (const existing of existingMemories) {
|
|
74
|
-
const score =
|
|
138
|
+
const score = hybridScore(newContent, existing.content);
|
|
75
139
|
if (score > DUPLICATE_THRESHOLD) {
|
|
76
140
|
if (hasNegation) {
|
|
77
141
|
if (!bestSupersede || score > bestSupersede.score) {
|
|
@@ -144,10 +208,9 @@ export function clusterMemories(memories) {
|
|
|
144
208
|
if (rootA !== rootB)
|
|
145
209
|
parent[rootA] = rootB;
|
|
146
210
|
}
|
|
147
|
-
const entities = memories.map((memory) => extractEntities(memory.content));
|
|
148
211
|
for (let i = 0; i < memories.length; i++) {
|
|
149
212
|
for (let j = i + 1; j < memories.length; j++) {
|
|
150
|
-
if (
|
|
213
|
+
if (hybridScore(memories[i].content, memories[j].content) > CLUSTER_THRESHOLD) {
|
|
151
214
|
union(i, j);
|
|
152
215
|
}
|
|
153
216
|
}
|
|
@@ -248,11 +311,12 @@ function splitWords(text) {
|
|
|
248
311
|
.split(/\s+/)
|
|
249
312
|
.filter((word) => word.length > 0 && !STOP_WORDS.has(word));
|
|
250
313
|
}
|
|
251
|
-
// Blends
|
|
252
|
-
// (surface-level phrasing match) so a memory that shares the task's
|
|
253
|
-
// wording scores well even if its extracted entities don't line up
|
|
314
|
+
// Blends hybrid structural/lexical overlap (see hybridScore) with raw word
|
|
315
|
+
// overlap (surface-level phrasing match) so a memory that shares the task's
|
|
316
|
+
// exact wording scores well even if its extracted entities don't line up
|
|
317
|
+
// neatly.
|
|
254
318
|
export function scoreMemoryRelevance(memoryContent, task) {
|
|
255
|
-
const
|
|
319
|
+
const structuralScore = hybridScore(memoryContent, task);
|
|
256
320
|
const memoryWords = new Set(splitWords(memoryContent));
|
|
257
321
|
const taskWords = splitWords(task);
|
|
258
322
|
let shared = 0;
|
|
@@ -261,6 +325,6 @@ export function scoreMemoryRelevance(memoryContent, task) {
|
|
|
261
325
|
shared++;
|
|
262
326
|
}
|
|
263
327
|
const wordOverlapRatio = taskWords.length === 0 ? 0 : shared / taskWords.length;
|
|
264
|
-
return
|
|
328
|
+
return structuralScore * 0.7 + wordOverlapRatio * 0.3;
|
|
265
329
|
}
|
|
266
330
|
// TODO: optimize the clustering algorithm
|
package/package.json
CHANGED
package/src/benchmark.ts
CHANGED
|
@@ -11,10 +11,10 @@ import { fileURLToPath } from "node:url";
|
|
|
11
11
|
import {
|
|
12
12
|
applyContextBudget,
|
|
13
13
|
computeDecayScore,
|
|
14
|
-
containmentScore,
|
|
15
14
|
decideSaveAction,
|
|
16
15
|
estimateTokens,
|
|
17
16
|
extractEntities,
|
|
17
|
+
hybridScore,
|
|
18
18
|
scoreMemoryRelevance,
|
|
19
19
|
type ExistingMemory,
|
|
20
20
|
} from "./scoring.js";
|
|
@@ -407,7 +407,18 @@ function benchmarkContradictionDetection(
|
|
|
407
407
|
allMemories: MemoryRow[],
|
|
408
408
|
unrelatedPairs: [MemoryRow, MemoryRow][]
|
|
409
409
|
): ContradictionDetectionResult {
|
|
410
|
-
|
|
410
|
+
// Hook-generated templates (dependency changes, config/schema/TODO
|
|
411
|
+
// discoveries) are structural event records, not decisions — decideSaveAction
|
|
412
|
+
// deliberately exempts dependency templates from ever routing to
|
|
413
|
+
// "superseded" (see DEPENDENCY_TEMPLATE_RE in scoring.ts), and prepending
|
|
414
|
+
// "switched from X" to any of these templates isn't a phrase anyone would
|
|
415
|
+
// actually write. Sampling them here would just measure whether that
|
|
416
|
+
// exemption holds, not whether contradiction detection works — so they're
|
|
417
|
+
// excluded, keeping this test on genuine decision/discovery-style content.
|
|
418
|
+
const contradictableMemories = allMemories.filter(
|
|
419
|
+
(m) => !HOOK_TEMPLATE_PREFIXES.some((p) => m.content.startsWith(p.prefix))
|
|
420
|
+
);
|
|
421
|
+
const baseSample = seededShuffle(contradictableMemories, BENCHMARK_SEED + 2).slice(0, SAMPLE_SIZE / 2);
|
|
411
422
|
|
|
412
423
|
let tp = 0;
|
|
413
424
|
let fn = 0;
|
|
@@ -457,6 +468,14 @@ interface SemanticLinkingResult {
|
|
|
457
468
|
// exhaustively rather than randomly sampled — at this data size (a few
|
|
458
469
|
// thousand possible pairs at most) exhaustive is cheap and gives a more
|
|
459
470
|
// stable number than a small random draw would.
|
|
471
|
+
//
|
|
472
|
+
// CAVEAT: avgLinkScore below reads memory_links.score as stored — i.e.
|
|
473
|
+
// whatever computeMemoryLinks computed at save time. Rows saved before
|
|
474
|
+
// computeMemoryLinks switched from containmentScore to hybridScore still
|
|
475
|
+
// hold old-formula scores; only the baseline here is guaranteed to reflect
|
|
476
|
+
// the current formula (it's computed live). A mixed-vintage avgLinkScore
|
|
477
|
+
// vs. a current-formula baseline is still the most honest comparison
|
|
478
|
+
// available without backfilling every historical link.
|
|
460
479
|
function benchmarkSemanticLinking(allMemories: MemoryRow[], allLinks: MemoryLinkRow[]): SemanticLinkingResult {
|
|
461
480
|
const avgLinkScore =
|
|
462
481
|
allLinks.length === 0 ? 0 : allLinks.reduce((sum, l) => sum + l.score, 0) / allLinks.length;
|
|
@@ -472,12 +491,11 @@ function benchmarkSemanticLinking(allMemories: MemoryRow[], allLinks: MemoryLink
|
|
|
472
491
|
let baselineSum = 0;
|
|
473
492
|
let baselineCount = 0;
|
|
474
493
|
for (const rows of byProject.values()) {
|
|
475
|
-
const entitiesById = new Map(rows.map((m) => [m.id, extractEntities(m.content)]));
|
|
476
494
|
for (let i = 0; i < rows.length; i++) {
|
|
477
495
|
for (let j = i + 1; j < rows.length; j++) {
|
|
478
496
|
const key = pairKey(rows[i].id, rows[j].id);
|
|
479
497
|
if (linkedPairKeys.has(key)) continue;
|
|
480
|
-
baselineSum +=
|
|
498
|
+
baselineSum += hybridScore(rows[i].content, rows[j].content);
|
|
481
499
|
baselineCount++;
|
|
482
500
|
}
|
|
483
501
|
}
|
package/src/cli.ts
CHANGED
|
@@ -270,6 +270,8 @@ async function cmdStart(project: string) {
|
|
|
270
270
|
const memories = data ?? [];
|
|
271
271
|
if (memories.length === 0) {
|
|
272
272
|
console.log(`No memories found for "${project}".`);
|
|
273
|
+
console.log("");
|
|
274
|
+
console.log("Tip: run 'cm init' to seed tech stack memories from your project files.");
|
|
273
275
|
return;
|
|
274
276
|
}
|
|
275
277
|
|
|
@@ -358,6 +360,12 @@ async function cmdStart(project: string) {
|
|
|
358
360
|
console.log(`${omittedCount} memories omitted (budget exceeded)`);
|
|
359
361
|
}
|
|
360
362
|
|
|
363
|
+
const hasTechStackMemory = memories.some((m) => /^tech stack:/i.test(m.content));
|
|
364
|
+
if (!hasTechStackMemory) {
|
|
365
|
+
console.log("");
|
|
366
|
+
console.log("Tip: run 'cm init' to seed tech stack memories from your project files.");
|
|
367
|
+
}
|
|
368
|
+
|
|
361
369
|
const incrementResults = await Promise.all(
|
|
362
370
|
budgetedMemories.map((memory) =>
|
|
363
371
|
supabase
|
|
@@ -1051,6 +1059,28 @@ function registerMcpServer(distIndexPath: string): void {
|
|
|
1051
1059
|
console.log("stackmem MCP server registered with Claude Code.");
|
|
1052
1060
|
}
|
|
1053
1061
|
|
|
1062
|
+
const DEFAULT_GITIGNORE_CONTENT = `node_modules/
|
|
1063
|
+
dist/
|
|
1064
|
+
.env
|
|
1065
|
+
.env.local
|
|
1066
|
+
.DS_Store
|
|
1067
|
+
*.log
|
|
1068
|
+
coverage/
|
|
1069
|
+
`;
|
|
1070
|
+
|
|
1071
|
+
// Returns true if the file was created, false if it already existed
|
|
1072
|
+
// (and was left untouched).
|
|
1073
|
+
function writeGitignore(resolvedPath: string): boolean {
|
|
1074
|
+
const gitignorePath = nodePath.join(resolvedPath, ".gitignore");
|
|
1075
|
+
|
|
1076
|
+
if (nodeFs.existsSync(gitignorePath)) {
|
|
1077
|
+
return false;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
nodeFs.writeFileSync(gitignorePath, DEFAULT_GITIGNORE_CONTENT, "utf-8");
|
|
1081
|
+
return true;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1054
1084
|
function writeClaudeMd(project: string, resolvedPath: string): void {
|
|
1055
1085
|
const claudeMdPath = nodePath.join(resolvedPath, "CLAUDE.md");
|
|
1056
1086
|
|
|
@@ -1189,6 +1219,27 @@ function writeProjectFile(project: string, resolvedPath: string): void {
|
|
|
1189
1219
|
nodeFs.writeFileSync(stackmemPath, `${project}\n`, "utf-8");
|
|
1190
1220
|
}
|
|
1191
1221
|
|
|
1222
|
+
// Shared by every seedMemories check below: skip the insert if this exact
|
|
1223
|
+
// content already exists (decideSaveAction's duplicate check) so re-running
|
|
1224
|
+
// `cm init` on an already-seeded project doesn't pile up repeats.
|
|
1225
|
+
async function trySeedMemory(
|
|
1226
|
+
project: string,
|
|
1227
|
+
existingMemories: { id: string; content: string }[],
|
|
1228
|
+
type: MemoryType,
|
|
1229
|
+
content: string
|
|
1230
|
+
): Promise<boolean> {
|
|
1231
|
+
const decision = decideSaveAction(content, existingMemories);
|
|
1232
|
+
if (decision.action === "duplicate") return false;
|
|
1233
|
+
|
|
1234
|
+
const { error } = await supabase.from("memories").insert({
|
|
1235
|
+
project,
|
|
1236
|
+
type,
|
|
1237
|
+
content,
|
|
1238
|
+
device_id: getDeviceId(),
|
|
1239
|
+
});
|
|
1240
|
+
return !error;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1192
1243
|
// Scans the freshly-initialized project for a few cheap, high-signal facts
|
|
1193
1244
|
// and saves them as memories directly (bypassing cmdSave's duplicate/conflict
|
|
1194
1245
|
// prompt — there's nothing to conflict with yet, and init must stay
|
|
@@ -1216,16 +1267,7 @@ async function seedMemories(project: string, resolvedPath: string): Promise<numb
|
|
|
1216
1267
|
];
|
|
1217
1268
|
if (depNames.length > 0) {
|
|
1218
1269
|
const content = `tech stack: ${depNames.join(", ")}`;
|
|
1219
|
-
|
|
1220
|
-
if (decision.action !== "duplicate") {
|
|
1221
|
-
const { error } = await supabase.from("memories").insert({
|
|
1222
|
-
project,
|
|
1223
|
-
type: "discovery",
|
|
1224
|
-
content,
|
|
1225
|
-
device_id: getDeviceId(),
|
|
1226
|
-
});
|
|
1227
|
-
if (!error) seededCount++;
|
|
1228
|
-
}
|
|
1270
|
+
if (await trySeedMemory(project, existingMemories, "discovery", content)) seededCount++;
|
|
1229
1271
|
}
|
|
1230
1272
|
} catch {
|
|
1231
1273
|
// Malformed package.json — skip the tech-stack memory.
|
|
@@ -1242,19 +1284,56 @@ async function seedMemories(project: string, resolvedPath: string): Promise<numb
|
|
|
1242
1284
|
|
|
1243
1285
|
if (varNames.length > 0) {
|
|
1244
1286
|
const content = `required env vars: ${varNames.join(", ")}`;
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1287
|
+
if (await trySeedMemory(project, existingMemories, "constraint", content)) seededCount++;
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
const tsconfigPath = nodePath.join(resolvedPath, "tsconfig.json");
|
|
1292
|
+
if (nodeFs.existsSync(tsconfigPath)) {
|
|
1293
|
+
try {
|
|
1294
|
+
const tsconfig = JSON.parse(nodeFs.readFileSync(tsconfigPath, "utf-8"));
|
|
1295
|
+
const target = tsconfig.compilerOptions?.target;
|
|
1296
|
+
const moduleKind = tsconfig.compilerOptions?.module;
|
|
1297
|
+
if (target || moduleKind) {
|
|
1298
|
+
const content = `TypeScript config: target=${target ?? "unspecified"}, module=${moduleKind ?? "unspecified"}`;
|
|
1299
|
+
if (await trySeedMemory(project, existingMemories, "constraint", content)) seededCount++;
|
|
1254
1300
|
}
|
|
1301
|
+
} catch {
|
|
1302
|
+
// Malformed tsconfig.json (or JSONC with comments) — skip.
|
|
1255
1303
|
}
|
|
1256
1304
|
}
|
|
1257
1305
|
|
|
1306
|
+
const readmePath = nodePath.join(resolvedPath, "README.md");
|
|
1307
|
+
if (nodeFs.existsSync(readmePath)) {
|
|
1308
|
+
const words = nodeFs
|
|
1309
|
+
.readFileSync(readmePath, "utf-8")
|
|
1310
|
+
.split(/\s+/)
|
|
1311
|
+
.filter(Boolean)
|
|
1312
|
+
.slice(0, 100);
|
|
1313
|
+
if (words.length > 0) {
|
|
1314
|
+
const content = `project description: ${words.join(" ")}`;
|
|
1315
|
+
if (await trySeedMemory(project, existingMemories, "discovery", content)) seededCount++;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
const FRAMEWORK_FILE_INDICATORS: { file: string; type: MemoryType; content: string }[] = [
|
|
1320
|
+
{ file: "next.config.ts", type: "constraint", content: "framework: Next.js" },
|
|
1321
|
+
{ file: "next.config.js", type: "constraint", content: "framework: Next.js" },
|
|
1322
|
+
{ file: "vite.config.ts", type: "constraint", content: "framework: Vite" },
|
|
1323
|
+
{ file: "vite.config.js", type: "constraint", content: "framework: Vite" },
|
|
1324
|
+
{ file: "Dockerfile", type: "constraint", content: "containerized: Docker" },
|
|
1325
|
+
];
|
|
1326
|
+
for (const indicator of FRAMEWORK_FILE_INDICATORS) {
|
|
1327
|
+
if (!nodeFs.existsSync(nodePath.join(resolvedPath, indicator.file))) continue;
|
|
1328
|
+
if (await trySeedMemory(project, existingMemories, indicator.type, indicator.content)) seededCount++;
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
const githubWorkflowsPath = nodePath.join(resolvedPath, ".github", "workflows");
|
|
1332
|
+
if (nodeFs.existsSync(githubWorkflowsPath)) {
|
|
1333
|
+
const content = "CI/CD: GitHub Actions";
|
|
1334
|
+
if (await trySeedMemory(project, existingMemories, "discovery", content)) seededCount++;
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1258
1337
|
analyzeCodebase(resolvedPath);
|
|
1259
1338
|
|
|
1260
1339
|
return seededCount;
|
|
@@ -1280,6 +1359,8 @@ async function cmdInit(project: string, targetPath: string) {
|
|
|
1280
1359
|
process.exit(1);
|
|
1281
1360
|
}
|
|
1282
1361
|
|
|
1362
|
+
const gitignoreCreated = writeGitignore(resolvedPath);
|
|
1363
|
+
|
|
1283
1364
|
const codingMemoryDir = nodePath.join(resolvedPath, ".coding-memory");
|
|
1284
1365
|
nodeFs.mkdirSync(codingMemoryDir, { recursive: true });
|
|
1285
1366
|
|
|
@@ -1308,6 +1389,9 @@ async function cmdInit(project: string, targetPath: string) {
|
|
|
1308
1389
|
const backendOk = await verifyBackendConnection(project);
|
|
1309
1390
|
|
|
1310
1391
|
console.log("");
|
|
1392
|
+
if (gitignoreCreated) {
|
|
1393
|
+
console.log("✓ .gitignore created");
|
|
1394
|
+
}
|
|
1311
1395
|
console.log("✓ Git hook installed");
|
|
1312
1396
|
console.log("✓ MCP server registered");
|
|
1313
1397
|
console.log("✓ CLAUDE.md written");
|
package/src/index.ts
CHANGED
|
@@ -129,8 +129,13 @@ server.registerTool(
|
|
|
129
129
|
? `Loaded ${memories.length} memory row(s) for "${project}" (${tokensUsed}/${CONTEXT_BUDGET_TOKENS} tokens used, ${omittedCount} omitted for budget).`
|
|
130
130
|
: `Loaded ${memories.length} memory row(s) for "${project}" (${tokensUsed}/${CONTEXT_BUDGET_TOKENS} tokens used).`;
|
|
131
131
|
|
|
132
|
+
const hasTechStackMemory = memories.some((m) => /^tech stack:/i.test(m.content));
|
|
133
|
+
const techStackTip = hasTechStackMemory
|
|
134
|
+
? ""
|
|
135
|
+
: "\n\nTip: run 'cm init' to seed tech stack memories from your project files.";
|
|
136
|
+
|
|
132
137
|
return {
|
|
133
|
-
content: [{ type: "text", text: budgetText }],
|
|
138
|
+
content: [{ type: "text", text: budgetText + techStackTip }],
|
|
134
139
|
structuredContent: {
|
|
135
140
|
memories: memoriesWithLinks,
|
|
136
141
|
fixes,
|
package/src/scoring.ts
CHANGED
|
@@ -5,11 +5,34 @@ const STOP_WORDS = new Set([
|
|
|
5
5
|
"of", "in", "it", "this", "that", "with", "and", "or", "but",
|
|
6
6
|
]);
|
|
7
7
|
|
|
8
|
+
// Bumping this from 5 to 6 was considered (see the benchmark's duplicate-
|
|
9
|
+
// detection findings) but doesn't actually fix anything: "removed" is 7
|
|
10
|
+
// chars and "added"/"dependency" aren't the problem either. The real issue
|
|
11
|
+
// is entity-level, not length-based — see DEPENDENCY_TEMPLATE_RE below.
|
|
8
12
|
const MIN_ENTITY_LENGTH = 5;
|
|
9
13
|
|
|
14
|
+
// Hook-generated dependency-change templates ("removed dependency: X" /
|
|
15
|
+
// "added dependency: X") reduce to just {"removed"/"added", "dependency"}
|
|
16
|
+
// under generic word-based extraction below, since package names are often
|
|
17
|
+
// short (node:path, zod, ./device.js — all filtered by MIN_ENTITY_LENGTH).
|
|
18
|
+
// That made two DIFFERENT packages removed in different commits look like
|
|
19
|
+
// duplicates of each other. Special-cased here instead: for these
|
|
20
|
+
// templates, the package name itself is the only entity, so containment
|
|
21
|
+
// (and hybridScore) only agrees when the package name actually matches.
|
|
22
|
+
const DEPENDENCY_TEMPLATE_RE = /^(removed|added) dependency:/i;
|
|
23
|
+
const DEPENDENCY_PACKAGE_RE = /^(?:removed|added) dependency:\s*(.+)$/i;
|
|
24
|
+
|
|
25
|
+
function extractDependencyPackageName(content: string): string | null {
|
|
26
|
+
const match = DEPENDENCY_PACKAGE_RE.exec(content.trim());
|
|
27
|
+
return match ? match[1].trim().toLowerCase() : null;
|
|
28
|
+
}
|
|
29
|
+
|
|
10
30
|
// Key entities: words longer than 4 characters that aren't stop words.
|
|
11
31
|
// Short/common words carry little identifying signal for a memory.
|
|
12
32
|
export function extractEntities(text: string): Set<string> {
|
|
33
|
+
const packageName = extractDependencyPackageName(text);
|
|
34
|
+
if (packageName !== null) return new Set([packageName]);
|
|
35
|
+
|
|
13
36
|
const entities = new Set<string>();
|
|
14
37
|
for (const word of text.toLowerCase().split(/\W+/)) {
|
|
15
38
|
if (word.length < MIN_ENTITY_LENGTH || STOP_WORDS.has(word)) continue;
|
|
@@ -33,6 +56,55 @@ export function containmentScore(entitiesA: Set<string>, entitiesB: Set<string>)
|
|
|
33
56
|
return shared / smallerSize;
|
|
34
57
|
}
|
|
35
58
|
|
|
59
|
+
function trigramSet(text: string): Set<string> {
|
|
60
|
+
const normalized = text.toLowerCase();
|
|
61
|
+
if (normalized.length < 3) {
|
|
62
|
+
return normalized.length > 0 ? new Set([normalized]) : new Set();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const trigrams = new Set<string>();
|
|
66
|
+
for (let i = 0; i <= normalized.length - 3; i++) {
|
|
67
|
+
trigrams.add(normalized.slice(i, i + 3));
|
|
68
|
+
}
|
|
69
|
+
return trigrams;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Character-trigram Jaccard overlap: catches surface-level lexical
|
|
73
|
+
// similarity (typos, partial substrings, shared phrasing) that word-based
|
|
74
|
+
// containment misses entirely, since containment only ever compares whole
|
|
75
|
+
// extracted entities.
|
|
76
|
+
function ngramOverlap(a: string, b: string): number {
|
|
77
|
+
const trigramsA = trigramSet(a);
|
|
78
|
+
const trigramsB = trigramSet(b);
|
|
79
|
+
const union = new Set([...trigramsA, ...trigramsB]);
|
|
80
|
+
if (union.size === 0) return 0;
|
|
81
|
+
|
|
82
|
+
let shared = 0;
|
|
83
|
+
for (const trigram of trigramsA) {
|
|
84
|
+
if (trigramsB.has(trigram)) shared++;
|
|
85
|
+
}
|
|
86
|
+
return shared / union.size;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Blends entity containment (structural: do the two texts share key
|
|
90
|
+
// extracted terms) with character-trigram overlap (lexical: do the two
|
|
91
|
+
// texts look alike as raw strings) so near-duplicates with slightly
|
|
92
|
+
// different wording still score high even when their extracted entities
|
|
93
|
+
// don't line up neatly, and vice versa.
|
|
94
|
+
//
|
|
95
|
+
// NOTE ON THRESHOLDS: DUPLICATE_THRESHOLD (0.85), CONTRADICTION_MIN (0.4),
|
|
96
|
+
// computeMemoryLinks' default (0.15), and CLUSTER_THRESHOLD (0.2) were all
|
|
97
|
+
// tuned against pure containmentScore. The 0.3 trigram component shifts
|
|
98
|
+
// scores upward for almost any pair sharing common substrings (whitespace,
|
|
99
|
+
// "ing", "the", etc.), so these thresholds may need recalibration —
|
|
100
|
+
// re-validate against src/benchmark.ts rather than assuming they still
|
|
101
|
+
// hold as-is.
|
|
102
|
+
export function hybridScore(a: string, b: string): number {
|
|
103
|
+
const containment = containmentScore(extractEntities(a), extractEntities(b));
|
|
104
|
+
const overlap = ngramOverlap(a, b);
|
|
105
|
+
return containment * 0.7 + overlap * 0.3;
|
|
106
|
+
}
|
|
107
|
+
|
|
36
108
|
// --- memory links ------------------------------------------------------
|
|
37
109
|
|
|
38
110
|
export interface ExistingMemory {
|
|
@@ -52,12 +124,11 @@ export function computeMemoryLinks(
|
|
|
52
124
|
existingMemories: ExistingMemory[],
|
|
53
125
|
threshold = 0.15
|
|
54
126
|
): MemoryLink[] {
|
|
55
|
-
const newEntities = extractEntities(newContent);
|
|
56
127
|
return existingMemories
|
|
57
128
|
.map((existing) => ({
|
|
58
129
|
source_id: newMemoryId,
|
|
59
130
|
target_id: existing.id,
|
|
60
|
-
score:
|
|
131
|
+
score: hybridScore(newContent, existing.content),
|
|
61
132
|
}))
|
|
62
133
|
.filter((link) => link.score > threshold);
|
|
63
134
|
}
|
|
@@ -79,12 +150,6 @@ function containsNegationWord(text: string): boolean {
|
|
|
79
150
|
return NEGATION_WORDS.some((word) => lower.includes(word));
|
|
80
151
|
}
|
|
81
152
|
|
|
82
|
-
// Hook-generated dependency-change templates (e.g. "removed dependency: X")
|
|
83
|
-
// are structural facts, not decisions — "removed" trips the negation-word
|
|
84
|
-
// check below, but there's nothing for them to supersede/contradict, so
|
|
85
|
-
// they're exempted up front rather than ever entering the negation branch.
|
|
86
|
-
const DEPENDENCY_TEMPLATE_RE = /^(removed|added) dependency:/i;
|
|
87
|
-
|
|
88
153
|
export type SaveAction =
|
|
89
154
|
| { action: "duplicate"; matchedId: string }
|
|
90
155
|
| { action: "superseded"; oldId: string }
|
|
@@ -102,14 +167,17 @@ export function decideSaveAction(
|
|
|
102
167
|
newContent: string,
|
|
103
168
|
existingMemories: ExistingMemory[]
|
|
104
169
|
): SaveAction {
|
|
105
|
-
|
|
170
|
+
// Dependency-change templates are structural facts, not decisions —
|
|
171
|
+
// "removed"/"added" would otherwise trip containsNegationWord below, but
|
|
172
|
+
// there's nothing for them to supersede/contradict, so they're exempted
|
|
173
|
+
// up front and only ever fall through to the plain duplicate check.
|
|
106
174
|
const hasNegation = DEPENDENCY_TEMPLATE_RE.test(newContent) ? false : containsNegationWord(newContent);
|
|
107
175
|
|
|
108
176
|
let bestDuplicate: { id: string; score: number } | null = null;
|
|
109
177
|
let bestSupersede: { id: string; score: number } | null = null;
|
|
110
178
|
|
|
111
179
|
for (const existing of existingMemories) {
|
|
112
|
-
const score =
|
|
180
|
+
const score = hybridScore(newContent, existing.content);
|
|
113
181
|
|
|
114
182
|
if (score > DUPLICATE_THRESHOLD) {
|
|
115
183
|
if (hasNegation) {
|
|
@@ -192,11 +260,9 @@ export function clusterMemories(memories: any[]): any[][] {
|
|
|
192
260
|
if (rootA !== rootB) parent[rootA] = rootB;
|
|
193
261
|
}
|
|
194
262
|
|
|
195
|
-
const entities = memories.map((memory) => extractEntities(memory.content));
|
|
196
|
-
|
|
197
263
|
for (let i = 0; i < memories.length; i++) {
|
|
198
264
|
for (let j = i + 1; j < memories.length; j++) {
|
|
199
|
-
if (
|
|
265
|
+
if (hybridScore(memories[i].content, memories[j].content) > CLUSTER_THRESHOLD) {
|
|
200
266
|
union(i, j);
|
|
201
267
|
}
|
|
202
268
|
}
|
|
@@ -313,11 +379,12 @@ function splitWords(text: string): string[] {
|
|
|
313
379
|
.filter((word) => word.length > 0 && !STOP_WORDS.has(word));
|
|
314
380
|
}
|
|
315
381
|
|
|
316
|
-
// Blends
|
|
317
|
-
// (surface-level phrasing match) so a memory that shares the task's
|
|
318
|
-
// wording scores well even if its extracted entities don't line up
|
|
382
|
+
// Blends hybrid structural/lexical overlap (see hybridScore) with raw word
|
|
383
|
+
// overlap (surface-level phrasing match) so a memory that shares the task's
|
|
384
|
+
// exact wording scores well even if its extracted entities don't line up
|
|
385
|
+
// neatly.
|
|
319
386
|
export function scoreMemoryRelevance(memoryContent: string, task: string): number {
|
|
320
|
-
const
|
|
387
|
+
const structuralScore = hybridScore(memoryContent, task);
|
|
321
388
|
|
|
322
389
|
const memoryWords = new Set(splitWords(memoryContent));
|
|
323
390
|
const taskWords = splitWords(task);
|
|
@@ -328,6 +395,6 @@ export function scoreMemoryRelevance(memoryContent: string, task: string): numbe
|
|
|
328
395
|
}
|
|
329
396
|
const wordOverlapRatio = taskWords.length === 0 ? 0 : shared / taskWords.length;
|
|
330
397
|
|
|
331
|
-
return
|
|
398
|
+
return structuralScore * 0.7 + wordOverlapRatio * 0.3;
|
|
332
399
|
}
|
|
333
400
|
// TODO: optimize the clustering algorithm
|