r2mcp 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/LICENSE +21 -0
- package/README.md +532 -0
- package/dist/breadcrumbs.d.ts +123 -0
- package/dist/breadcrumbs.js +135 -0
- package/dist/cli/classify-edges.d.ts +2 -0
- package/dist/cli/classify-edges.js +130 -0
- package/dist/cli/compile-wiki.d.ts +2 -0
- package/dist/cli/compile-wiki.js +173 -0
- package/dist/cli/dump-edges-json.d.ts +2 -0
- package/dist/cli/dump-edges-json.js +21 -0
- package/dist/cli/extract-entities.d.ts +17 -0
- package/dist/cli/extract-entities.js +166 -0
- package/dist/cli/lint-memory.d.ts +16 -0
- package/dist/cli/lint-memory.js +94 -0
- package/dist/cli/migrate.d.ts +17 -0
- package/dist/cli/migrate.js +146 -0
- package/dist/cli/setup-helpers.d.ts +7 -0
- package/dist/cli/setup-helpers.js +72 -0
- package/dist/cli/setup.d.ts +15 -0
- package/dist/cli/setup.js +95 -0
- package/dist/compiler/clustering.d.ts +29 -0
- package/dist/compiler/clustering.js +66 -0
- package/dist/compiler/frontmatter.d.ts +35 -0
- package/dist/compiler/frontmatter.js +168 -0
- package/dist/compiler/manifest.d.ts +32 -0
- package/dist/compiler/manifest.js +82 -0
- package/dist/compiler/prompts.d.ts +17 -0
- package/dist/compiler/prompts.js +82 -0
- package/dist/compiler/run.d.ts +52 -0
- package/dist/compiler/run.js +186 -0
- package/dist/compiler/tier.d.ts +10 -0
- package/dist/compiler/tier.js +85 -0
- package/dist/compiler/topic.d.ts +16 -0
- package/dist/compiler/topic.js +105 -0
- package/dist/compiler/types.d.ts +101 -0
- package/dist/compiler/types.js +4 -0
- package/dist/db.d.ts +10 -0
- package/dist/db.js +46 -0
- package/dist/edges/candidate-pairs.d.ts +24 -0
- package/dist/edges/candidate-pairs.js +35 -0
- package/dist/edges/classifier.d.ts +45 -0
- package/dist/edges/classifier.js +172 -0
- package/dist/edges/signals.d.ts +13 -0
- package/dist/edges/signals.js +45 -0
- package/dist/edges/stage1-haiku.d.ts +21 -0
- package/dist/edges/stage1-haiku.js +33 -0
- package/dist/edges/stage2-opus.d.ts +41 -0
- package/dist/edges/stage2-opus.js +101 -0
- package/dist/edges/state.d.ts +44 -0
- package/dist/edges/state.js +79 -0
- package/dist/edges/types.d.ts +20 -0
- package/dist/edges/types.js +1 -0
- package/dist/embeddings.d.ts +13 -0
- package/dist/embeddings.js +54 -0
- package/dist/entities/db.d.ts +49 -0
- package/dist/entities/db.js +109 -0
- package/dist/entities/extractor.d.ts +14 -0
- package/dist/entities/extractor.js +154 -0
- package/dist/entities/normalize.d.ts +5 -0
- package/dist/entities/normalize.js +7 -0
- package/dist/entities/prompt.d.ts +19 -0
- package/dist/entities/prompt.js +100 -0
- package/dist/entities/state.d.ts +44 -0
- package/dist/entities/state.js +99 -0
- package/dist/entities/types.d.ts +62 -0
- package/dist/entities/types.js +6 -0
- package/dist/env.d.ts +13 -0
- package/dist/env.js +32 -0
- package/dist/fingerprint.d.ts +2 -0
- package/dist/fingerprint.js +12 -0
- package/dist/graph-rebuild.d.ts +6 -0
- package/dist/graph-rebuild.js +20 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +403 -0
- package/dist/instrumentation.d.ts +10 -0
- package/dist/instrumentation.js +37 -0
- package/dist/lint/checks/contradictions.d.ts +30 -0
- package/dist/lint/checks/contradictions.js +52 -0
- package/dist/lint/checks/drift.d.ts +5 -0
- package/dist/lint/checks/drift.js +34 -0
- package/dist/lint/checks/orphans.d.ts +5 -0
- package/dist/lint/checks/orphans.js +25 -0
- package/dist/lint/checks/stale.d.ts +6 -0
- package/dist/lint/checks/stale.js +29 -0
- package/dist/lint/checks/superseded-unflagged.d.ts +5 -0
- package/dist/lint/checks/superseded-unflagged.js +47 -0
- package/dist/lint/run.d.ts +11 -0
- package/dist/lint/run.js +95 -0
- package/dist/lint/types.d.ts +60 -0
- package/dist/lint/types.js +13 -0
- package/dist/mcp-response.d.ts +7 -0
- package/dist/mcp-response.js +13 -0
- package/dist/providers/anthropic.d.ts +13 -0
- package/dist/providers/anthropic.js +56 -0
- package/dist/providers/claude-code.d.ts +35 -0
- package/dist/providers/claude-code.js +175 -0
- package/dist/providers/errors.d.ts +12 -0
- package/dist/providers/errors.js +19 -0
- package/dist/providers/index.d.ts +30 -0
- package/dist/providers/index.js +71 -0
- package/dist/providers/openrouter.d.ts +19 -0
- package/dist/providers/openrouter.js +76 -0
- package/dist/providers/semaphore.d.ts +19 -0
- package/dist/providers/semaphore.js +51 -0
- package/dist/providers/types.d.ts +27 -0
- package/dist/providers/types.js +7 -0
- package/dist/schema.sql +116 -0
- package/dist/server-instructions.d.ts +9 -0
- package/dist/server-instructions.js +20 -0
- package/dist/telemetry.d.ts +39 -0
- package/dist/telemetry.js +130 -0
- package/dist/tools/classify.d.ts +44 -0
- package/dist/tools/classify.js +121 -0
- package/dist/tools/compile.d.ts +31 -0
- package/dist/tools/compile.js +132 -0
- package/dist/tools/dump-edges-sidecar.d.ts +37 -0
- package/dist/tools/dump-edges-sidecar.js +80 -0
- package/dist/tools/extract-entities.d.ts +53 -0
- package/dist/tools/extract-entities.js +169 -0
- package/dist/tools/lint.d.ts +10 -0
- package/dist/tools/lint.js +13 -0
- package/dist/tools/meditate.d.ts +25 -0
- package/dist/tools/meditate.js +128 -0
- package/dist/tools/recall.d.ts +66 -0
- package/dist/tools/recall.js +409 -0
- package/dist/tools/reject.d.ts +10 -0
- package/dist/tools/reject.js +24 -0
- package/dist/tools/remember.d.ts +26 -0
- package/dist/tools/remember.js +140 -0
- package/dist/tools/search.d.ts +30 -0
- package/dist/tools/search.js +69 -0
- package/dist/tools/spawn-cli.d.ts +14 -0
- package/dist/tools/spawn-cli.js +41 -0
- package/dist/tools/stats.d.ts +31 -0
- package/dist/tools/stats.js +88 -0
- package/package.json +86 -0
- package/skills/remember/SKILL.md +357 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MCP tool handler for `lint()`.
|
|
3
|
+
*
|
|
4
|
+
* Lint is SQL-only — no LLM calls (C.R5) — so unlike `compile()` it runs
|
|
5
|
+
* directly in the MCP server process against the existing pgvector pool.
|
|
6
|
+
* No subprocess delegation needed.
|
|
7
|
+
*/
|
|
8
|
+
import { getPool } from '../db.js';
|
|
9
|
+
import { runLint } from '../lint/run.js';
|
|
10
|
+
export async function lint(input) {
|
|
11
|
+
const pool = getPool();
|
|
12
|
+
return runLint(input, pool);
|
|
13
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { LintFinding } from '../lint/types.js';
|
|
2
|
+
export interface MeditateInput {
|
|
3
|
+
mode: 'full';
|
|
4
|
+
dry_run: boolean;
|
|
5
|
+
/**
|
|
6
|
+
* SPEC-044 C.R4 — opt-in lint integration. Default false for backward
|
|
7
|
+
* compatibility with existing direct callers (Slack bot, programmatic).
|
|
8
|
+
* When true, lint runs and findings are surfaced in `lint_findings`.
|
|
9
|
+
*/
|
|
10
|
+
include_lint?: boolean;
|
|
11
|
+
}
|
|
12
|
+
export interface MeditateResult {
|
|
13
|
+
archived: number;
|
|
14
|
+
deduplicated: number;
|
|
15
|
+
cross_referenced: number;
|
|
16
|
+
clustered: number;
|
|
17
|
+
gaps_found: number;
|
|
18
|
+
total_changes: number;
|
|
19
|
+
/**
|
|
20
|
+
* Populated only when input.include_lint is true. Absent (undefined) for
|
|
21
|
+
* default callers — preserves the byte-identical default response shape.
|
|
22
|
+
*/
|
|
23
|
+
lint_findings?: LintFinding[];
|
|
24
|
+
}
|
|
25
|
+
export declare function meditate(input: MeditateInput, projectRoot?: string): Promise<MeditateResult>;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { getPool } from '../db.js';
|
|
2
|
+
import { triggerGraphRebuild } from '../graph-rebuild.js';
|
|
3
|
+
import { runLint } from '../lint/run.js';
|
|
4
|
+
export async function meditate(input, projectRoot) {
|
|
5
|
+
const pool = getPool();
|
|
6
|
+
// 1. Archive stale entries
|
|
7
|
+
const archived = await archiveStale(pool, input.dry_run);
|
|
8
|
+
// 2. Deduplicate (sanity check — fingerprints should be unique)
|
|
9
|
+
const deduplicated = await countDuplicateFingerprints(pool);
|
|
10
|
+
// 3. Cross-reference (find entries with 2+ shared topics)
|
|
11
|
+
const cross_referenced = await countCrossReferencePairs(pool);
|
|
12
|
+
// 4. Cluster by theme (count distinct topic clusters)
|
|
13
|
+
const clustered = await countTopicClusters(pool);
|
|
14
|
+
// 5. Surface gaps (topics in preferences but not project-context)
|
|
15
|
+
const gaps_found = await surfaceGaps(pool);
|
|
16
|
+
// Trigger graph rebuild after consolidation (only if not dry_run)
|
|
17
|
+
if (!input.dry_run && projectRoot) {
|
|
18
|
+
triggerGraphRebuild(projectRoot);
|
|
19
|
+
}
|
|
20
|
+
const result = {
|
|
21
|
+
archived,
|
|
22
|
+
deduplicated,
|
|
23
|
+
cross_referenced,
|
|
24
|
+
clustered,
|
|
25
|
+
gaps_found,
|
|
26
|
+
total_changes: archived + deduplicated,
|
|
27
|
+
};
|
|
28
|
+
if (input.include_lint) {
|
|
29
|
+
const lintResult = await runLint({}, pool);
|
|
30
|
+
result.lint_findings = lintResult.findings;
|
|
31
|
+
}
|
|
32
|
+
return result;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Archive stale entries:
|
|
36
|
+
* - conversations tier: older than 90 days
|
|
37
|
+
* - project-context tier: older than 180 days
|
|
38
|
+
* - preferences tier: never auto-archived
|
|
39
|
+
*/
|
|
40
|
+
async function archiveStale(pool, dryRun) {
|
|
41
|
+
// Count how many would be archived
|
|
42
|
+
const countResult = await pool.query(`
|
|
43
|
+
SELECT COUNT(*)::int AS count FROM memories
|
|
44
|
+
WHERE type != 'archived' AND (
|
|
45
|
+
(tier = 'conversations' AND created_at < NOW() - INTERVAL '90 days')
|
|
46
|
+
OR
|
|
47
|
+
(tier = 'project-context' AND created_at < NOW() - INTERVAL '180 days')
|
|
48
|
+
)
|
|
49
|
+
`);
|
|
50
|
+
const count = countResult.rows[0].count;
|
|
51
|
+
if (!dryRun && count > 0) {
|
|
52
|
+
await pool.query(`
|
|
53
|
+
UPDATE memories SET type = 'archived', updated_at = NOW()
|
|
54
|
+
WHERE type != 'archived' AND (
|
|
55
|
+
(tier = 'conversations' AND created_at < NOW() - INTERVAL '90 days')
|
|
56
|
+
OR
|
|
57
|
+
(tier = 'project-context' AND created_at < NOW() - INTERVAL '180 days')
|
|
58
|
+
)
|
|
59
|
+
`);
|
|
60
|
+
}
|
|
61
|
+
return count;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Count entries with duplicate fingerprints (sanity check).
|
|
65
|
+
* Shouldn't happen due to UNIQUE constraint, but counts them if they exist.
|
|
66
|
+
*/
|
|
67
|
+
async function countDuplicateFingerprints(pool) {
|
|
68
|
+
const result = await pool.query(`
|
|
69
|
+
SELECT COALESCE(SUM(dup_count - 1), 0)::int AS duplicates
|
|
70
|
+
FROM (
|
|
71
|
+
SELECT fingerprint, COUNT(*)::int AS dup_count
|
|
72
|
+
FROM memories
|
|
73
|
+
GROUP BY fingerprint
|
|
74
|
+
HAVING COUNT(*) > 1
|
|
75
|
+
) sub
|
|
76
|
+
`);
|
|
77
|
+
return result.rows[0].duplicates;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Find entries with overlapping topics (2+ shared topics) that could be cross-referenced.
|
|
81
|
+
* Returns count of such pairs. Informational only for Phase 2.
|
|
82
|
+
*/
|
|
83
|
+
async function countCrossReferencePairs(pool) {
|
|
84
|
+
const result = await pool.query(`
|
|
85
|
+
SELECT COUNT(*)::int AS pair_count
|
|
86
|
+
FROM (
|
|
87
|
+
SELECT m1.id AS id1, m2.id AS id2
|
|
88
|
+
FROM memories m1
|
|
89
|
+
JOIN memories m2 ON m1.id < m2.id
|
|
90
|
+
WHERE (
|
|
91
|
+
SELECT COUNT(*)
|
|
92
|
+
FROM unnest(m1.topics) t1
|
|
93
|
+
WHERE t1 = ANY(m2.topics)
|
|
94
|
+
) >= 2
|
|
95
|
+
) sub
|
|
96
|
+
`);
|
|
97
|
+
return result.rows[0].pair_count;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Group entries by most common topic. Returns count of distinct topic clusters.
|
|
101
|
+
* Informational only for Phase 2.
|
|
102
|
+
*/
|
|
103
|
+
async function countTopicClusters(pool) {
|
|
104
|
+
const result = await pool.query(`
|
|
105
|
+
SELECT COUNT(DISTINCT topic)::int AS cluster_count
|
|
106
|
+
FROM memories, unnest(topics) AS topic
|
|
107
|
+
`);
|
|
108
|
+
return result.rows[0].cluster_count;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Surface gaps: topics that appear in preferences but not in project-context.
|
|
112
|
+
* These might indicate missing architectural documentation for decided preferences.
|
|
113
|
+
*/
|
|
114
|
+
async function surfaceGaps(pool) {
|
|
115
|
+
const result = await pool.query(`
|
|
116
|
+
SELECT COUNT(*)::int AS gap_count
|
|
117
|
+
FROM (
|
|
118
|
+
SELECT DISTINCT topic
|
|
119
|
+
FROM memories, unnest(topics) AS topic
|
|
120
|
+
WHERE tier = 'preferences'
|
|
121
|
+
EXCEPT
|
|
122
|
+
SELECT DISTINCT topic
|
|
123
|
+
FROM memories, unnest(topics) AS topic
|
|
124
|
+
WHERE tier = 'project-context'
|
|
125
|
+
) sub
|
|
126
|
+
`);
|
|
127
|
+
return result.rows[0].gap_count;
|
|
128
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { RecallSignal } from '../edges/types.js';
|
|
2
|
+
import type { EntityType } from '../entities/types.js';
|
|
3
|
+
export type Tier = 'preferences' | 'project-context' | 'conversations';
|
|
4
|
+
export type MatchType = 'semantic' | 'fulltext' | 'hybrid';
|
|
5
|
+
export type SearchMode = 'semantic' | 'fulltext_only';
|
|
6
|
+
export interface EntityLink {
|
|
7
|
+
type: EntityType;
|
|
8
|
+
canonical_name: string;
|
|
9
|
+
confidence: number;
|
|
10
|
+
}
|
|
11
|
+
export interface RecallResult {
|
|
12
|
+
id: string;
|
|
13
|
+
tier: string;
|
|
14
|
+
content: string;
|
|
15
|
+
metadata: {
|
|
16
|
+
type: string;
|
|
17
|
+
topics: string[];
|
|
18
|
+
persons: string[];
|
|
19
|
+
created: string;
|
|
20
|
+
updated: string;
|
|
21
|
+
};
|
|
22
|
+
score: number;
|
|
23
|
+
match_type: MatchType;
|
|
24
|
+
/** SPEC-046: present only when recall() is called with an `entity` filter. */
|
|
25
|
+
entity_links?: EntityLink[];
|
|
26
|
+
}
|
|
27
|
+
export interface RecallResponse {
|
|
28
|
+
results: RecallResult[];
|
|
29
|
+
query: string;
|
|
30
|
+
total_results: number;
|
|
31
|
+
search_mode: SearchMode;
|
|
32
|
+
tiers_searched: string[];
|
|
33
|
+
tokens_used?: number;
|
|
34
|
+
early_stopped?: boolean;
|
|
35
|
+
/** Always present in v1.1+; optional for backward-compat with pre-edges client types. */
|
|
36
|
+
signals?: RecallSignal[];
|
|
37
|
+
/** SPEC-046: present only when recall() is called with an `entity` filter. */
|
|
38
|
+
entity_resolved?: boolean;
|
|
39
|
+
/** SPEC-046: present only when `entity_resolved` is true. */
|
|
40
|
+
entity_id?: string;
|
|
41
|
+
/** Present only when the search ran degraded, e.g. embeddings unavailable (claw-8cjf.2). */
|
|
42
|
+
warnings?: string[];
|
|
43
|
+
}
|
|
44
|
+
export interface RecallInput {
|
|
45
|
+
/** Free-text query. Optional only when `entity` is provided (SPEC-046 entity-only fast path). */
|
|
46
|
+
query?: string;
|
|
47
|
+
top_k?: number;
|
|
48
|
+
tier?: Tier;
|
|
49
|
+
max_tokens?: number;
|
|
50
|
+
min_score?: number;
|
|
51
|
+
diversity?: number;
|
|
52
|
+
progressive?: boolean;
|
|
53
|
+
confidence_threshold?: number;
|
|
54
|
+
/** SPEC-046: optional entity filter. Resolves via canonical_name or alias. */
|
|
55
|
+
entity?: string;
|
|
56
|
+
}
|
|
57
|
+
interface InternalResult extends RecallResult {
|
|
58
|
+
rawScore: number;
|
|
59
|
+
rawEmbedding?: number[];
|
|
60
|
+
}
|
|
61
|
+
export declare function cosineSimilarity(a: number[], b: number[]): number;
|
|
62
|
+
export declare function jaccardSimilarity(a: string, b: string): number;
|
|
63
|
+
export declare function estimateTokens(text: string): number;
|
|
64
|
+
export declare function applyMMR(candidates: InternalResult[], lambda: number, topK: number): InternalResult[];
|
|
65
|
+
export declare function recall(input: RecallInput): Promise<RecallResponse>;
|
|
66
|
+
export {};
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
import { getPool } from '../db.js';
|
|
2
|
+
import { embedText, embeddingWarning } from '../embeddings.js';
|
|
3
|
+
import pgvector from 'pgvector';
|
|
4
|
+
import { getSignalsForMemoryIds } from '../edges/signals.js';
|
|
5
|
+
import { findEntityByInput, getEntityLinksForMemories } from '../entities/db.js';
|
|
6
|
+
const { toSql } = pgvector;
|
|
7
|
+
const TIER_WEIGHTS = {
|
|
8
|
+
preferences: 1.3,
|
|
9
|
+
'project-context': 1.0,
|
|
10
|
+
conversations: 0.8,
|
|
11
|
+
};
|
|
12
|
+
const TIER_ORDER = ['preferences', 'project-context', 'conversations'];
|
|
13
|
+
// Default to no floor to maintain backward compatibility with v1 callers.
|
|
14
|
+
// Set min_score explicitly (e.g. 0.3 for hybrid, 0.1 for fulltext) to filter low-quality results.
|
|
15
|
+
const DEFAULT_MIN_SCORE_HYBRID = 0.0;
|
|
16
|
+
const DEFAULT_MIN_SCORE_FULLTEXT = 0.0;
|
|
17
|
+
const DEFAULT_DIVERSITY = 0.7;
|
|
18
|
+
const DEFAULT_CONFIDENCE_THRESHOLD = 0.82;
|
|
19
|
+
// --- Utility functions ---
|
|
20
|
+
export function cosineSimilarity(a, b) {
|
|
21
|
+
let dot = 0, normA = 0, normB = 0;
|
|
22
|
+
for (let i = 0; i < a.length; i++) {
|
|
23
|
+
dot += a[i] * b[i];
|
|
24
|
+
normA += a[i] * a[i];
|
|
25
|
+
normB += b[i] * b[i];
|
|
26
|
+
}
|
|
27
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
28
|
+
return denom === 0 ? 0 : dot / denom;
|
|
29
|
+
}
|
|
30
|
+
export function jaccardSimilarity(a, b) {
|
|
31
|
+
const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(Boolean));
|
|
32
|
+
const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(Boolean));
|
|
33
|
+
let intersection = 0;
|
|
34
|
+
for (const w of wordsA) {
|
|
35
|
+
if (wordsB.has(w))
|
|
36
|
+
intersection++;
|
|
37
|
+
}
|
|
38
|
+
const union = wordsA.size + wordsB.size - intersection;
|
|
39
|
+
return union === 0 ? 0 : intersection / union;
|
|
40
|
+
}
|
|
41
|
+
export function estimateTokens(text) {
|
|
42
|
+
const wordCount = text.split(/\s+/).filter(Boolean).length;
|
|
43
|
+
return Math.ceil(wordCount * 1.3);
|
|
44
|
+
}
|
|
45
|
+
function parseEmbedding(raw) {
|
|
46
|
+
if (!raw || typeof raw !== 'string')
|
|
47
|
+
return undefined;
|
|
48
|
+
try {
|
|
49
|
+
return raw.slice(1, -1).split(',').map(Number);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function docSimilarity(a, b) {
|
|
56
|
+
if (a.rawEmbedding && b.rawEmbedding) {
|
|
57
|
+
return cosineSimilarity(a.rawEmbedding, b.rawEmbedding);
|
|
58
|
+
}
|
|
59
|
+
return jaccardSimilarity(a.content, b.content);
|
|
60
|
+
}
|
|
61
|
+
function stripInternal(r) {
|
|
62
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
63
|
+
const { rawScore, rawEmbedding, ...rest } = r;
|
|
64
|
+
return rest;
|
|
65
|
+
}
|
|
66
|
+
function applyTierWeight(score, tier) {
|
|
67
|
+
return score * (TIER_WEIGHTS[tier] ?? 1.0);
|
|
68
|
+
}
|
|
69
|
+
// MMR: Maximum Marginal Relevance — balances relevance vs. diversity to eliminate redundant results.
|
|
70
|
+
// lambda=1.0 = pure relevance (same as top-K), lambda=0.0 = pure diversity.
|
|
71
|
+
export function applyMMR(candidates, lambda, topK) {
|
|
72
|
+
if (candidates.length === 0)
|
|
73
|
+
return [];
|
|
74
|
+
if (candidates.length <= 1)
|
|
75
|
+
return candidates.slice(0, topK);
|
|
76
|
+
const sorted = [...candidates].sort((a, b) => b.score - a.score);
|
|
77
|
+
const selected = [sorted[0]];
|
|
78
|
+
const remaining = sorted.slice(1);
|
|
79
|
+
while (selected.length < topK && remaining.length > 0) {
|
|
80
|
+
let bestIdx = -1;
|
|
81
|
+
let bestMMR = -Infinity;
|
|
82
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
83
|
+
const relevance = remaining[i].score;
|
|
84
|
+
let maxSim = 0;
|
|
85
|
+
for (const sel of selected) {
|
|
86
|
+
const sim = docSimilarity(remaining[i], sel);
|
|
87
|
+
if (sim > maxSim)
|
|
88
|
+
maxSim = sim;
|
|
89
|
+
}
|
|
90
|
+
const mmrScore = lambda * relevance - (1 - lambda) * maxSim;
|
|
91
|
+
if (mmrScore > bestMMR) {
|
|
92
|
+
bestMMR = mmrScore;
|
|
93
|
+
bestIdx = i;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
selected.push(remaining.splice(bestIdx, 1)[0]);
|
|
97
|
+
}
|
|
98
|
+
return selected;
|
|
99
|
+
}
|
|
100
|
+
// --- DB search functions ---
|
|
101
|
+
async function hybridSearchTier(pool, query, queryEmbedding, topK, tier, fetchEmbeddings = false, entityId) {
|
|
102
|
+
const embeddingSql = toSql(queryEmbedding);
|
|
103
|
+
const params = [embeddingSql, query];
|
|
104
|
+
let tierFilter = '';
|
|
105
|
+
if (tier) {
|
|
106
|
+
tierFilter = ` AND tier = $${params.length + 1}`;
|
|
107
|
+
params.push(tier);
|
|
108
|
+
}
|
|
109
|
+
// SPEC-046: narrow the candidate pool to memories linked to the resolved
|
|
110
|
+
// entity. Cheapest expression is a subquery in the WHERE clause — applied
|
|
111
|
+
// inside the CTE so ranking only ever runs over the entity-scoped pool.
|
|
112
|
+
let entityFilter = '';
|
|
113
|
+
if (entityId) {
|
|
114
|
+
entityFilter = ` AND id IN (SELECT memory_id FROM memory_entities WHERE entity_id = $${params.length + 1})`;
|
|
115
|
+
params.push(entityId);
|
|
116
|
+
}
|
|
117
|
+
const embeddingCol = fetchEmbeddings ? 'embedding::text AS raw_embedding,' : '';
|
|
118
|
+
const sql = `
|
|
119
|
+
WITH scored AS (
|
|
120
|
+
SELECT
|
|
121
|
+
id, content, tier, type, topics, people, created_at, updated_at,
|
|
122
|
+
${embeddingCol}
|
|
123
|
+
(1 - (embedding <=> $1::vector)) AS semantic_score,
|
|
124
|
+
ts_rank(tsv, plainto_tsquery('english', $2)) AS fulltext_score,
|
|
125
|
+
CASE
|
|
126
|
+
WHEN embedding IS NOT NULL AND ts_rank(tsv, plainto_tsquery('english', $2)) > 0 THEN 'hybrid'
|
|
127
|
+
WHEN embedding IS NOT NULL THEN 'semantic'
|
|
128
|
+
ELSE 'fulltext'
|
|
129
|
+
END AS match_type
|
|
130
|
+
FROM memories
|
|
131
|
+
WHERE type NOT IN ('rejection', 'archived')
|
|
132
|
+
AND (
|
|
133
|
+
embedding IS NOT NULL
|
|
134
|
+
OR tsv @@ plainto_tsquery('english', $2)
|
|
135
|
+
)${tierFilter}${entityFilter}
|
|
136
|
+
)
|
|
137
|
+
SELECT *,
|
|
138
|
+
CASE match_type
|
|
139
|
+
WHEN 'hybrid' THEN (0.7 * semantic_score + 0.3 * fulltext_score)
|
|
140
|
+
WHEN 'semantic' THEN semantic_score
|
|
141
|
+
ELSE fulltext_score
|
|
142
|
+
END AS combined_score
|
|
143
|
+
FROM scored
|
|
144
|
+
ORDER BY combined_score DESC
|
|
145
|
+
LIMIT ${topK}
|
|
146
|
+
`;
|
|
147
|
+
const { rows } = await pool.query(sql, params);
|
|
148
|
+
return rows
|
|
149
|
+
.map((row) => {
|
|
150
|
+
const rawScore = row.combined_score;
|
|
151
|
+
return {
|
|
152
|
+
id: row.id,
|
|
153
|
+
tier: row.tier,
|
|
154
|
+
content: row.content,
|
|
155
|
+
metadata: {
|
|
156
|
+
type: row.type,
|
|
157
|
+
topics: row.topics || [],
|
|
158
|
+
persons: row.people || [],
|
|
159
|
+
created: row.created_at.toISOString(),
|
|
160
|
+
updated: row.updated_at.toISOString(),
|
|
161
|
+
},
|
|
162
|
+
score: applyTierWeight(rawScore, row.tier),
|
|
163
|
+
match_type: row.match_type,
|
|
164
|
+
rawScore,
|
|
165
|
+
rawEmbedding: fetchEmbeddings ? parseEmbedding(row.raw_embedding) : undefined,
|
|
166
|
+
};
|
|
167
|
+
})
|
|
168
|
+
.sort((a, b) => b.score - a.score);
|
|
169
|
+
}
|
|
170
|
+
async function fulltextSearchTier(pool, query, topK, tier, entityId) {
|
|
171
|
+
const params = [query];
|
|
172
|
+
let tierFilter = '';
|
|
173
|
+
if (tier) {
|
|
174
|
+
tierFilter = ` AND tier = $${params.length + 1}`;
|
|
175
|
+
params.push(tier);
|
|
176
|
+
}
|
|
177
|
+
let entityFilter = '';
|
|
178
|
+
if (entityId) {
|
|
179
|
+
entityFilter = ` AND id IN (SELECT memory_id FROM memory_entities WHERE entity_id = $${params.length + 1})`;
|
|
180
|
+
params.push(entityId);
|
|
181
|
+
}
|
|
182
|
+
const sql = `
|
|
183
|
+
SELECT
|
|
184
|
+
id, content, tier, type, topics, people, created_at, updated_at,
|
|
185
|
+
ts_rank(tsv, plainto_tsquery('english', $1)) AS fulltext_score
|
|
186
|
+
FROM memories
|
|
187
|
+
WHERE type NOT IN ('rejection', 'archived')
|
|
188
|
+
AND tsv @@ plainto_tsquery('english', $1)${tierFilter}${entityFilter}
|
|
189
|
+
ORDER BY fulltext_score DESC
|
|
190
|
+
LIMIT ${topK}
|
|
191
|
+
`;
|
|
192
|
+
const { rows } = await pool.query(sql, params);
|
|
193
|
+
return rows
|
|
194
|
+
.map((row) => {
|
|
195
|
+
const rawScore = row.fulltext_score;
|
|
196
|
+
return {
|
|
197
|
+
id: row.id,
|
|
198
|
+
tier: row.tier,
|
|
199
|
+
content: row.content,
|
|
200
|
+
metadata: {
|
|
201
|
+
type: row.type,
|
|
202
|
+
topics: row.topics || [],
|
|
203
|
+
persons: row.people || [],
|
|
204
|
+
created: row.created_at.toISOString(),
|
|
205
|
+
updated: row.updated_at.toISOString(),
|
|
206
|
+
},
|
|
207
|
+
score: applyTierWeight(rawScore, row.tier),
|
|
208
|
+
match_type: 'fulltext',
|
|
209
|
+
rawScore,
|
|
210
|
+
};
|
|
211
|
+
})
|
|
212
|
+
.sort((a, b) => b.score - a.score);
|
|
213
|
+
}
|
|
214
|
+
// Progressive tier search: searches tier by tier, top-down, stopping early when a high-confidence
|
|
215
|
+
// result is found. This avoids searching lower-signal tiers when the top tier already answers the query.
|
|
216
|
+
async function progressiveHybridSearch(pool, query, queryEmbedding, topK, confidenceThreshold, entityId) {
|
|
217
|
+
const tiersSearched = [];
|
|
218
|
+
const allResults = [];
|
|
219
|
+
let earlyStopped = false;
|
|
220
|
+
for (const tier of TIER_ORDER) {
|
|
221
|
+
tiersSearched.push(tier);
|
|
222
|
+
const tierResults = await hybridSearchTier(pool, query, queryEmbedding, topK, tier, true, entityId);
|
|
223
|
+
// Merge without duplicates
|
|
224
|
+
for (const r of tierResults) {
|
|
225
|
+
if (!allResults.some((existing) => existing.id === r.id)) {
|
|
226
|
+
allResults.push(r);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
// Early stop: if best raw score exceeds confidence threshold, no need to dig deeper
|
|
230
|
+
if (allResults.length > 0) {
|
|
231
|
+
const topRawScore = Math.max(...allResults.map((r) => r.rawScore));
|
|
232
|
+
if (topRawScore >= confidenceThreshold) {
|
|
233
|
+
earlyStopped = true;
|
|
234
|
+
break;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return { results: allResults, tiersSearched, earlyStopped };
|
|
239
|
+
}
|
|
240
|
+
// --- Entity-only fast path ---
|
|
241
|
+
// When an entity filter is set but the caller provides no query (or an empty
|
|
242
|
+
// one), we skip semantic/fulltext ranking entirely — there's nothing to rank
|
|
243
|
+
// against. Return all memories linked to the entity, ordered by recency, so
|
|
244
|
+
// the entity_links attachment downstream still has a stable result set.
|
|
245
|
+
async function entityOnlySearch(pool, entityId, topK, tier) {
|
|
246
|
+
const params = [entityId];
|
|
247
|
+
let tierFilter = '';
|
|
248
|
+
if (tier) {
|
|
249
|
+
tierFilter = ` AND m.tier = $${params.length + 1}`;
|
|
250
|
+
params.push(tier);
|
|
251
|
+
}
|
|
252
|
+
const sql = `
|
|
253
|
+
SELECT m.id, m.content, m.tier, m.type, m.topics, m.people, m.created_at, m.updated_at
|
|
254
|
+
FROM memories m
|
|
255
|
+
WHERE m.type NOT IN ('rejection', 'archived')
|
|
256
|
+
AND m.id IN (SELECT memory_id FROM memory_entities WHERE entity_id = $1)${tierFilter}
|
|
257
|
+
ORDER BY m.updated_at DESC
|
|
258
|
+
LIMIT ${topK}
|
|
259
|
+
`;
|
|
260
|
+
const { rows } = await pool.query(sql, params);
|
|
261
|
+
return rows.map((row) => ({
|
|
262
|
+
id: row.id,
|
|
263
|
+
tier: row.tier,
|
|
264
|
+
content: row.content,
|
|
265
|
+
metadata: {
|
|
266
|
+
type: row.type,
|
|
267
|
+
topics: row.topics || [],
|
|
268
|
+
persons: row.people || [],
|
|
269
|
+
created: row.created_at.toISOString(),
|
|
270
|
+
updated: row.updated_at.toISOString(),
|
|
271
|
+
},
|
|
272
|
+
score: 1.0,
|
|
273
|
+
match_type: 'fulltext',
|
|
274
|
+
rawScore: 1.0,
|
|
275
|
+
}));
|
|
276
|
+
}
|
|
277
|
+
// --- Main recall function ---
|
|
278
|
+
export async function recall(input) {
|
|
279
|
+
const {
|
|
280
|
+
// SPEC-046: query is optional when `entity` is set (entity-only recall).
|
|
281
|
+
// Normalize undefined → '' so downstream code keeps the same string contract.
|
|
282
|
+
query = '', top_k = 10, tier, max_tokens, min_score, diversity = DEFAULT_DIVERSITY, progressive = true, confidence_threshold = DEFAULT_CONFIDENCE_THRESHOLD, entity, } = input;
|
|
283
|
+
const pool = getPool();
|
|
284
|
+
// SPEC-046: resolve entity BEFORE retrieval. The resolution is the cheapest
|
|
285
|
+
// possible signal — a single indexed lookup on entities.normalized_name.
|
|
286
|
+
// When it fails, short-circuit with an empty result + entity_resolved=false
|
|
287
|
+
// (no error, per AC3b).
|
|
288
|
+
const entityFilterActive = entity !== undefined && entity !== '';
|
|
289
|
+
let resolvedEntity = null;
|
|
290
|
+
if (entityFilterActive) {
|
|
291
|
+
resolvedEntity = await findEntityByInput(pool, entity);
|
|
292
|
+
if (!resolvedEntity) {
|
|
293
|
+
return {
|
|
294
|
+
results: [],
|
|
295
|
+
query: query ?? '',
|
|
296
|
+
total_results: 0,
|
|
297
|
+
search_mode: 'semantic',
|
|
298
|
+
tiers_searched: [],
|
|
299
|
+
entity_resolved: false,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
// Skip embedText entirely on the entity-only fast path: an empty query
|
|
304
|
+
// would either burn an embedding round-trip for nothing or break ranking.
|
|
305
|
+
const skipRanking = entityFilterActive && (!query || query === '');
|
|
306
|
+
const queryEmbedding = skipRanking ? null : await embedText(query);
|
|
307
|
+
// Only warn when an embedding was actually attempted (claw-8cjf.2).
|
|
308
|
+
const degradedWarning = skipRanking ? null : embeddingWarning(queryEmbedding);
|
|
309
|
+
let hasDbEmbeddings = false;
|
|
310
|
+
if (queryEmbedding) {
|
|
311
|
+
const embCheck = await pool.query('SELECT EXISTS(SELECT 1 FROM memories WHERE embedding IS NOT NULL) AS has_embeddings');
|
|
312
|
+
hasDbEmbeddings = embCheck.rows[0].has_embeddings;
|
|
313
|
+
}
|
|
314
|
+
const useHybrid = queryEmbedding !== null && hasDbEmbeddings;
|
|
315
|
+
const searchMode = useHybrid ? 'semantic' : 'fulltext_only';
|
|
316
|
+
const effectiveMinScore = min_score ?? (useHybrid ? DEFAULT_MIN_SCORE_HYBRID : DEFAULT_MIN_SCORE_FULLTEXT);
|
|
317
|
+
// Fetch a larger candidate pool so MMR has room to diversify
|
|
318
|
+
const candidateLimit = Math.max(top_k * 3, 30);
|
|
319
|
+
let rawResults;
|
|
320
|
+
let tiersSearched;
|
|
321
|
+
let earlyStopped = false;
|
|
322
|
+
const entityId = resolvedEntity?.id;
|
|
323
|
+
if (skipRanking && entityId) {
|
|
324
|
+
// SPEC-046: entity-only fast path. No query → no ranking signal; just
|
|
325
|
+
// return entity-linked memories ordered by recency.
|
|
326
|
+
rawResults = await entityOnlySearch(pool, entityId, candidateLimit, tier);
|
|
327
|
+
tiersSearched = tier ? [tier] : TIER_ORDER;
|
|
328
|
+
}
|
|
329
|
+
else if (useHybrid && progressive && !tier) {
|
|
330
|
+
// Phase 3: progressive tier search — most valuable in semantic mode
|
|
331
|
+
const r = await progressiveHybridSearch(pool, query, queryEmbedding, candidateLimit, confidence_threshold, entityId);
|
|
332
|
+
rawResults = r.results;
|
|
333
|
+
tiersSearched = r.tiersSearched;
|
|
334
|
+
earlyStopped = r.earlyStopped;
|
|
335
|
+
}
|
|
336
|
+
else if (useHybrid) {
|
|
337
|
+
// Flat hybrid search: tier explicitly set or progressive disabled
|
|
338
|
+
rawResults = await hybridSearchTier(pool, query, queryEmbedding, candidateLimit, tier, true, entityId);
|
|
339
|
+
tiersSearched = tier ? [tier] : TIER_ORDER;
|
|
340
|
+
}
|
|
341
|
+
else {
|
|
342
|
+
// Fulltext-only fallback: no embeddings available
|
|
343
|
+
rawResults = await fulltextSearchTier(pool, query, candidateLimit, tier, entityId);
|
|
344
|
+
tiersSearched = tier ? [tier] : TIER_ORDER;
|
|
345
|
+
}
|
|
346
|
+
// Phase 1a: relevance floor — drop results below minimum quality threshold
|
|
347
|
+
const floorPassed = rawResults.filter((r) => r.rawScore >= effectiveMinScore);
|
|
348
|
+
// Phase 1b: MMR diversity — re-rank to eliminate near-duplicate results
|
|
349
|
+
const diverse = applyMMR(floorPassed, diversity, max_tokens ? candidateLimit : top_k);
|
|
350
|
+
// Phase 2: context budgeting — fit within token budget, or apply top_k
|
|
351
|
+
let finalResults;
|
|
352
|
+
let tokensUsed;
|
|
353
|
+
if (max_tokens !== undefined) {
|
|
354
|
+
finalResults = [];
|
|
355
|
+
let tokenCount = 0;
|
|
356
|
+
for (const r of diverse) {
|
|
357
|
+
if (finalResults.length >= top_k)
|
|
358
|
+
break; // top_k is always an upper bound
|
|
359
|
+
const tokens = estimateTokens(r.content);
|
|
360
|
+
if (tokenCount + tokens > max_tokens)
|
|
361
|
+
break;
|
|
362
|
+
finalResults.push(r);
|
|
363
|
+
tokenCount += tokens;
|
|
364
|
+
}
|
|
365
|
+
tokensUsed = finalResults.reduce((sum, r) => sum + estimateTokens(r.content), 0);
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
finalResults = diverse.slice(0, top_k);
|
|
369
|
+
tokensUsed = finalResults.reduce((sum, r) => sum + estimateTokens(r.content), 0);
|
|
370
|
+
}
|
|
371
|
+
const ids = finalResults.map((r) => r.id);
|
|
372
|
+
const signals = await getSignalsForMemoryIds(pool, ids);
|
|
373
|
+
// SPEC-046: when the entity filter is active, attach per-memory entity_links
|
|
374
|
+
// (the FULL link set for each memory, not just the filter entity — callers
|
|
375
|
+
// can see the wider entity graph for these results) and set top-level
|
|
376
|
+
// entity_resolved + entity_id. When the filter is OFF, response shape must
|
|
377
|
+
// be byte-identical to SPEC-037 — no new keys, not even undefined ones.
|
|
378
|
+
const strippedResults = finalResults.map(stripInternal);
|
|
379
|
+
if (entityFilterActive && resolvedEntity) {
|
|
380
|
+
const linkMap = await getEntityLinksForMemories(pool, ids);
|
|
381
|
+
for (const r of strippedResults) {
|
|
382
|
+
r.entity_links = linkMap.get(r.id) ?? [];
|
|
383
|
+
}
|
|
384
|
+
return {
|
|
385
|
+
results: strippedResults,
|
|
386
|
+
query,
|
|
387
|
+
total_results: finalResults.length,
|
|
388
|
+
search_mode: searchMode,
|
|
389
|
+
tiers_searched: tiersSearched,
|
|
390
|
+
tokens_used: tokensUsed,
|
|
391
|
+
early_stopped: earlyStopped,
|
|
392
|
+
signals,
|
|
393
|
+
entity_resolved: true,
|
|
394
|
+
entity_id: resolvedEntity.id,
|
|
395
|
+
...(degradedWarning ? { warnings: [degradedWarning] } : {}),
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
return {
|
|
399
|
+
results: strippedResults,
|
|
400
|
+
query,
|
|
401
|
+
total_results: finalResults.length,
|
|
402
|
+
search_mode: searchMode,
|
|
403
|
+
tiers_searched: tiersSearched,
|
|
404
|
+
tokens_used: tokensUsed,
|
|
405
|
+
early_stopped: earlyStopped,
|
|
406
|
+
signals,
|
|
407
|
+
...(degradedWarning ? { warnings: [degradedWarning] } : {}),
|
|
408
|
+
};
|
|
409
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { getPool } from '../db.js';
|
|
2
|
+
import { fingerprint } from '../fingerprint.js';
|
|
3
|
+
export async function reject(input) {
|
|
4
|
+
const pool = getPool();
|
|
5
|
+
const { id, reason } = input;
|
|
6
|
+
// Mark the original memory's type as 'rejection' (matching schema CHECK constraint)
|
|
7
|
+
const updateResult = await pool.query(`UPDATE memories SET type = 'rejection', updated_at = NOW()
|
|
8
|
+
WHERE id = $1
|
|
9
|
+
RETURNING id, tier, topics, people`, [id]);
|
|
10
|
+
if (updateResult.rows.length === 0) {
|
|
11
|
+
throw new Error(`No memory found with id ${id}`);
|
|
12
|
+
}
|
|
13
|
+
const original = updateResult.rows[0];
|
|
14
|
+
// Store the rejection reason as a new memory entry
|
|
15
|
+
const fp = fingerprint(reason);
|
|
16
|
+
const reasonResult = await pool.query(`INSERT INTO memories (content, tier, type, section, topics, people, fingerprint)
|
|
17
|
+
VALUES ($1, $2, 'rejection', $3, $4, $5, $6)
|
|
18
|
+
RETURNING id`, [reason, original.tier, `rejection-of:${id}`, original.topics || [], original.people || [], fp]);
|
|
19
|
+
return {
|
|
20
|
+
rejected_id: id,
|
|
21
|
+
reason_id: reasonResult.rows[0].id,
|
|
22
|
+
message: `Memory ${id} marked as rejected. Reason stored as ${reasonResult.rows[0].id}.`,
|
|
23
|
+
};
|
|
24
|
+
}
|