scream-code 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.mjs
CHANGED
|
@@ -6,7 +6,7 @@ const __dirname = __cjsShimDirname(__filename);
|
|
|
6
6
|
import "./suppress-sqlite-warning-C2VB0doZ.mjs";
|
|
7
7
|
//#region src/main.ts
|
|
8
8
|
try {
|
|
9
|
-
(await import("./app-
|
|
9
|
+
(await import("./app-Dpd7ZDc_.mjs")).main();
|
|
10
10
|
} catch (error) {
|
|
11
11
|
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
|
|
12
12
|
process.exit(1);
|
|
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import {
|
|
7
|
-
export {
|
|
6
|
+
import { n as multiSearchWithTrace } from "./src-DWWVKqBK.mjs";
|
|
7
|
+
export { multiSearchWithTrace };
|
|
@@ -637,6 +637,42 @@ var KnowledgeStore = class {
|
|
|
637
637
|
JOIN knowledge_event_entities ee ON ee.entity_id = en.id
|
|
638
638
|
WHERE ee.event_id = ?`).all(eventId).map(rowToEntity);
|
|
639
639
|
}
|
|
640
|
+
async listEntities(sourceId) {
|
|
641
|
+
await this.init();
|
|
642
|
+
if (this.db === void 0) return [];
|
|
643
|
+
const sql = sourceId ? `SELECT e.*, COUNT(ee.id) AS event_count
|
|
644
|
+
FROM knowledge_entities e
|
|
645
|
+
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
646
|
+
WHERE e.source_id = ?
|
|
647
|
+
GROUP BY e.id
|
|
648
|
+
ORDER BY event_count DESC` : `SELECT e.*, COUNT(ee.id) AS event_count
|
|
649
|
+
FROM knowledge_entities e
|
|
650
|
+
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
651
|
+
GROUP BY e.id
|
|
652
|
+
ORDER BY event_count DESC`;
|
|
653
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
654
|
+
...rowToEntity(row),
|
|
655
|
+
eventCount: Number(row["event_count"] ?? 0)
|
|
656
|
+
}));
|
|
657
|
+
}
|
|
658
|
+
async listEvents(sourceId) {
|
|
659
|
+
await this.init();
|
|
660
|
+
if (this.db === void 0) return [];
|
|
661
|
+
const sql = sourceId ? "SELECT * FROM knowledge_events WHERE source_id = ? ORDER BY rank ASC" : "SELECT * FROM knowledge_events ORDER BY created_at DESC";
|
|
662
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map(rowToEvent);
|
|
663
|
+
}
|
|
664
|
+
async listEventEntities(sourceId) {
|
|
665
|
+
await this.init();
|
|
666
|
+
if (this.db === void 0) return [];
|
|
667
|
+
const sql = sourceId ? `SELECT ee.event_id, ee.entity_id
|
|
668
|
+
FROM knowledge_event_entities ee
|
|
669
|
+
JOIN knowledge_events e ON e.id = ee.event_id
|
|
670
|
+
WHERE e.source_id = ?` : "SELECT event_id, entity_id FROM knowledge_event_entities";
|
|
671
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
672
|
+
eventId: asString(row["event_id"]),
|
|
673
|
+
entityId: asString(row["entity_id"])
|
|
674
|
+
}));
|
|
675
|
+
}
|
|
640
676
|
async ftsSearchChunks(query, limit = 50) {
|
|
641
677
|
await this.init();
|
|
642
678
|
if (this.db === void 0) return [];
|
|
@@ -819,6 +855,13 @@ function buildFtsQuery(search) {
|
|
|
819
855
|
const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
820
856
|
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
|
821
857
|
/**
|
|
858
|
+
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the fastembed model used
|
|
859
|
+
* here) has a 512-token sequence limit; chunks longer than that get truncated
|
|
860
|
+
* at the embedding layer, hurting retrieval quality. We split at ~480 to leave
|
|
861
|
+
* headroom for the heading line that gets prepended at embed time.
|
|
862
|
+
*/
|
|
863
|
+
const MAX_CHUNK_TOKENS = 480;
|
|
864
|
+
/**
|
|
822
865
|
* Split a markdown document into sections using heading_strict strategy:
|
|
823
866
|
* each section begins at a heading and contains all body content until the
|
|
824
867
|
* next heading. Consecutive headings collapse to the lowest-level one as the
|
|
@@ -837,18 +880,25 @@ function chunkMarkdown(content) {
|
|
|
837
880
|
const flush = () => {
|
|
838
881
|
const raw = buffer.join("\n").trim();
|
|
839
882
|
const heading = currentHeading;
|
|
883
|
+
const level = currentLevel;
|
|
840
884
|
buffer = [];
|
|
841
885
|
if (raw.length === 0) return;
|
|
842
886
|
const body = stripMarkdown(raw);
|
|
843
887
|
if (body.length === 0 && heading === null) return;
|
|
844
|
-
|
|
888
|
+
const subSections = splitLargeSection({
|
|
845
889
|
heading,
|
|
846
|
-
headingLevel:
|
|
890
|
+
headingLevel: level,
|
|
847
891
|
content: body,
|
|
848
892
|
rawContent: raw,
|
|
849
893
|
rank
|
|
850
894
|
});
|
|
851
|
-
|
|
895
|
+
for (const section of subSections) {
|
|
896
|
+
sections.push({
|
|
897
|
+
...section,
|
|
898
|
+
rank
|
|
899
|
+
});
|
|
900
|
+
rank += 1;
|
|
901
|
+
}
|
|
852
902
|
};
|
|
853
903
|
for (const line of lines) {
|
|
854
904
|
if (CODE_FENCE_RE.exec(line)) {
|
|
@@ -882,10 +932,11 @@ function stripMarkdown(text) {
|
|
|
882
932
|
/**
|
|
883
933
|
* Split a plain text file into chunks by blank-line paragraphs.
|
|
884
934
|
* Each non-empty paragraph becomes one chunk with no heading.
|
|
885
|
-
* Long paragraphs are split at sentence boundaries to
|
|
935
|
+
* Long paragraphs are split at sentence boundaries to stay under the
|
|
936
|
+
* embedding model's token cap (same MAX_CHUNK_TOKENS as markdown).
|
|
886
937
|
*/
|
|
887
938
|
function chunkText(content) {
|
|
888
|
-
const MAX_PARA_TOKENS =
|
|
939
|
+
const MAX_PARA_TOKENS = MAX_CHUNK_TOKENS;
|
|
889
940
|
const MAX_PARA_CHARS = MAX_PARA_TOKENS * 4;
|
|
890
941
|
const paragraphs = content.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
891
942
|
const sections = [];
|
|
@@ -928,8 +979,144 @@ function chunkText(content) {
|
|
|
928
979
|
function estimateTokens(text) {
|
|
929
980
|
return Math.ceil(text.length / 4);
|
|
930
981
|
}
|
|
982
|
+
/**
|
|
983
|
+
* Split a section that exceeds MAX_CHUNK_TOKENS into smaller sub-sections.
|
|
984
|
+
* Strategy (mirrors SAG's `splitLargeSection`):
|
|
985
|
+
* 1. Split into paragraphs by blank lines.
|
|
986
|
+
* 2. Greedily pack paragraphs into chunks under the token cap.
|
|
987
|
+
* 3. A paragraph that alone exceeds the cap is split at sentence boundaries;
|
|
988
|
+
* a sentence that itself exceeds the cap is split by character limit.
|
|
989
|
+
*
|
|
990
|
+
* Sections under the cap are returned unchanged (single-element array).
|
|
991
|
+
* The heading is preserved across all sub-sections; rank is reassigned by the
|
|
992
|
+
* caller. rawContent is rebuilt from the sub-chunk's content paragraphs.
|
|
993
|
+
*/
|
|
994
|
+
function splitLargeSection(section) {
|
|
995
|
+
if (estimateTokens(section.content) <= MAX_CHUNK_TOKENS) return [section];
|
|
996
|
+
const paragraphs = section.rawContent.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
997
|
+
if (paragraphs.length <= 1) return splitBySentences(section, section.rawContent);
|
|
998
|
+
const result = [];
|
|
999
|
+
let buffer = [];
|
|
1000
|
+
let bufferTokens = 0;
|
|
1001
|
+
const flushBuffer = () => {
|
|
1002
|
+
if (buffer.length === 0) return;
|
|
1003
|
+
const raw = buffer.join("\n\n").trim();
|
|
1004
|
+
if (raw.length === 0) {
|
|
1005
|
+
buffer = [];
|
|
1006
|
+
bufferTokens = 0;
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
result.push({
|
|
1010
|
+
heading: section.heading,
|
|
1011
|
+
headingLevel: section.headingLevel,
|
|
1012
|
+
content: stripMarkdown(raw),
|
|
1013
|
+
rawContent: raw,
|
|
1014
|
+
rank: 0
|
|
1015
|
+
});
|
|
1016
|
+
buffer = [];
|
|
1017
|
+
bufferTokens = 0;
|
|
1018
|
+
};
|
|
1019
|
+
for (const para of paragraphs) {
|
|
1020
|
+
const paraTokens = estimateTokens(para);
|
|
1021
|
+
if (paraTokens > MAX_CHUNK_TOKENS) {
|
|
1022
|
+
flushBuffer();
|
|
1023
|
+
for (const frag of splitBySentences(section, para)) result.push(frag);
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
1026
|
+
if (buffer.length > 0 && bufferTokens + paraTokens > MAX_CHUNK_TOKENS) flushBuffer();
|
|
1027
|
+
buffer.push(para);
|
|
1028
|
+
bufferTokens += paraTokens;
|
|
1029
|
+
}
|
|
1030
|
+
flushBuffer();
|
|
1031
|
+
return result.length > 0 ? result : [section];
|
|
1032
|
+
}
|
|
1033
|
+
/**
|
|
1034
|
+
* Split a long text into sub-sections at sentence boundaries.
|
|
1035
|
+
* Falls back to a hard character cut when a single sentence exceeds the cap.
|
|
1036
|
+
*/
|
|
1037
|
+
function splitBySentences(section, text) {
|
|
1038
|
+
const sentences = splitSentences(text);
|
|
1039
|
+
const result = [];
|
|
1040
|
+
let buffer = "";
|
|
1041
|
+
let bufferTokens = 0;
|
|
1042
|
+
const flush = () => {
|
|
1043
|
+
const trimmed = buffer.trim();
|
|
1044
|
+
if (trimmed.length === 0) {
|
|
1045
|
+
buffer = "";
|
|
1046
|
+
bufferTokens = 0;
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
result.push({
|
|
1050
|
+
heading: section.heading,
|
|
1051
|
+
headingLevel: section.headingLevel,
|
|
1052
|
+
content: stripMarkdown(trimmed),
|
|
1053
|
+
rawContent: trimmed,
|
|
1054
|
+
rank: 0
|
|
1055
|
+
});
|
|
1056
|
+
buffer = "";
|
|
1057
|
+
bufferTokens = 0;
|
|
1058
|
+
};
|
|
1059
|
+
for (const sentence of sentences) {
|
|
1060
|
+
const sentTokens = estimateTokens(sentence);
|
|
1061
|
+
if (sentTokens > MAX_CHUNK_TOKENS) {
|
|
1062
|
+
flush();
|
|
1063
|
+
for (const frag of splitByCharLimit(sentence)) result.push({
|
|
1064
|
+
heading: section.heading,
|
|
1065
|
+
headingLevel: section.headingLevel,
|
|
1066
|
+
content: stripMarkdown(frag),
|
|
1067
|
+
rawContent: frag,
|
|
1068
|
+
rank: 0
|
|
1069
|
+
});
|
|
1070
|
+
continue;
|
|
1071
|
+
}
|
|
1072
|
+
if (buffer.length > 0 && bufferTokens + sentTokens > MAX_CHUNK_TOKENS) flush();
|
|
1073
|
+
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
1074
|
+
bufferTokens += sentTokens;
|
|
1075
|
+
}
|
|
1076
|
+
flush();
|
|
1077
|
+
return result.length > 0 ? result : [{
|
|
1078
|
+
heading: section.heading,
|
|
1079
|
+
headingLevel: section.headingLevel,
|
|
1080
|
+
content: section.content,
|
|
1081
|
+
rawContent: section.rawContent,
|
|
1082
|
+
rank: 0
|
|
1083
|
+
}];
|
|
1084
|
+
}
|
|
1085
|
+
/** Split text into sentences using CJK + ASCII punctuation. */
|
|
1086
|
+
function splitSentences(text) {
|
|
1087
|
+
return text.match(/[^。!?!?.\n]+[。!?!?.\n]+|[^。!?!?.\n]+$/g) ?? [text];
|
|
1088
|
+
}
|
|
1089
|
+
/** Hard character cut for pathological single sentences longer than the cap. */
|
|
1090
|
+
function splitByCharLimit(text) {
|
|
1091
|
+
const maxChars = MAX_CHUNK_TOKENS * 4;
|
|
1092
|
+
const chunks = [];
|
|
1093
|
+
let remaining = text.trim();
|
|
1094
|
+
while (remaining.length > maxChars) {
|
|
1095
|
+
chunks.push(remaining.slice(0, maxChars));
|
|
1096
|
+
remaining = remaining.slice(maxChars).trimStart();
|
|
1097
|
+
}
|
|
1098
|
+
if (remaining.length > 0) chunks.push(remaining);
|
|
1099
|
+
return chunks;
|
|
1100
|
+
}
|
|
931
1101
|
//#endregion
|
|
932
1102
|
//#region ../../packages/knowledge/src/extractor.ts
|
|
1103
|
+
/** Max LLM call attempts for extraction — fails fall back to a heading-derived event. */
|
|
1104
|
+
const EXTRACTION_MAX_ATTEMPTS = 3;
|
|
1105
|
+
/** Base delay (ms) for exponential backoff: 100, 200, 400. */
|
|
1106
|
+
const EXTRACTION_BACKOFF_BASE_MS = 100;
|
|
1107
|
+
function backoffDelay(attempt) {
|
|
1108
|
+
return EXTRACTION_BACKOFF_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1109
|
+
}
|
|
1110
|
+
async function callLlmWithBackoff(llm, systemPrompt, userPrompt) {
|
|
1111
|
+
let lastError;
|
|
1112
|
+
for (let attempt = 1; attempt <= EXTRACTION_MAX_ATTEMPTS; attempt += 1) try {
|
|
1113
|
+
return await llm.generate(systemPrompt, userPrompt);
|
|
1114
|
+
} catch (error) {
|
|
1115
|
+
lastError = error;
|
|
1116
|
+
if (attempt < EXTRACTION_MAX_ATTEMPTS) await new Promise((resolve) => setTimeout(resolve, backoffDelay(attempt)));
|
|
1117
|
+
}
|
|
1118
|
+
throw lastError;
|
|
1119
|
+
}
|
|
933
1120
|
const ENTITY_TYPES = [
|
|
934
1121
|
"person",
|
|
935
1122
|
"organization",
|
|
@@ -1104,11 +1291,12 @@ function fallbackEvent(chunk) {
|
|
|
1104
1291
|
}
|
|
1105
1292
|
/**
|
|
1106
1293
|
* Run extraction on a single chunk via the LLM caller.
|
|
1107
|
-
*
|
|
1294
|
+
* Retries with exponential backoff on transient failures; falls back to a
|
|
1295
|
+
* heading-derived event only after all attempts fail.
|
|
1108
1296
|
*/
|
|
1109
1297
|
async function extractEventFromChunk(llm, chunk) {
|
|
1110
1298
|
try {
|
|
1111
|
-
return parseExtractionResponse(await llm
|
|
1299
|
+
return parseExtractionResponse(await callLlmWithBackoff(llm, EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt(chunk)), chunk);
|
|
1112
1300
|
} catch {
|
|
1113
1301
|
return fallbackEvent(chunk);
|
|
1114
1302
|
}
|
|
@@ -1188,7 +1376,8 @@ Only include entities that explicitly appear in the query. If no clear entities,
|
|
|
1188
1376
|
}
|
|
1189
1377
|
//#endregion
|
|
1190
1378
|
//#region ../../packages/knowledge/src/ingest.ts
|
|
1191
|
-
|
|
1379
|
+
/** LLM concurrency for extraction — kept low to avoid rate-limit bursts. */
|
|
1380
|
+
const LLM_CONCURRENCY = 3;
|
|
1192
1381
|
const SUPPORTED_EXTENSIONS = new Set([
|
|
1193
1382
|
".md",
|
|
1194
1383
|
".markdown",
|
|
@@ -1534,6 +1723,28 @@ const EXPANDED_EVENT_LIMIT = 100;
|
|
|
1534
1723
|
const COARSE_RANK_LIMIT = 50;
|
|
1535
1724
|
const ENTITY_VECTOR_THRESHOLD = .7;
|
|
1536
1725
|
const TITLE_VECTOR_THRESHOLD = .4;
|
|
1726
|
+
async function timed(onStep, step, detail, fn, payload) {
|
|
1727
|
+
const start = performance.now();
|
|
1728
|
+
try {
|
|
1729
|
+
const result = await fn();
|
|
1730
|
+
const traceStep = {
|
|
1731
|
+
step,
|
|
1732
|
+
detail,
|
|
1733
|
+
durationMs: Math.round((performance.now() - start) * 100) / 100,
|
|
1734
|
+
...payload !== void 0 ? { payload: payload(result) } : {}
|
|
1735
|
+
};
|
|
1736
|
+
onStep?.(traceStep);
|
|
1737
|
+
return result;
|
|
1738
|
+
} catch (error) {
|
|
1739
|
+
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
|
1740
|
+
onStep?.({
|
|
1741
|
+
step,
|
|
1742
|
+
detail: `${detail} 失败:${error instanceof Error ? error.message : String(error)}`,
|
|
1743
|
+
durationMs
|
|
1744
|
+
});
|
|
1745
|
+
throw error;
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1537
1748
|
/**
|
|
1538
1749
|
* Multi-hop retrieval:
|
|
1539
1750
|
* 1. Vectorize query.
|
|
@@ -1545,62 +1756,155 @@ const TITLE_VECTOR_THRESHOLD = .4;
|
|
|
1545
1756
|
* 6. LLM rerank — pick top-K most relevant.
|
|
1546
1757
|
* 7. Return corresponding chunks (deduped by chunk_id) with scores and provenance.
|
|
1547
1758
|
* If rerank yields fewer than topK chunks, backfill by direct chunk vector search.
|
|
1759
|
+
*
|
|
1760
|
+
* Optional `onStep` callback receives a trace step at each phase — used by the
|
|
1761
|
+
* KnowledgeLookup tool to expose retrieval diagnostics to the agent.
|
|
1548
1762
|
*/
|
|
1549
1763
|
async function multiSearch(store, llm, query, options = {}) {
|
|
1764
|
+
return (await multiSearchWithTrace(store, llm, query, options)).results;
|
|
1765
|
+
}
|
|
1766
|
+
/** Like multiSearch but also returns the retrieval trace for diagnostics. */
|
|
1767
|
+
async function multiSearchWithTrace(store, llm, query, options = {}) {
|
|
1550
1768
|
const topK = Math.min(options.topK ?? DEFAULT_TOP_K, MAX_TOP_K);
|
|
1769
|
+
const steps = [];
|
|
1770
|
+
const onStep = (step) => steps.push(step);
|
|
1771
|
+
const rerankedEventTitles = [];
|
|
1772
|
+
let fallbackReason = null;
|
|
1551
1773
|
const engine = store.getEmbeddingEngine();
|
|
1552
|
-
if (engine === void 0 || !engine.available)
|
|
1553
|
-
|
|
1554
|
-
|
|
1774
|
+
if (engine === void 0 || !engine.available) {
|
|
1775
|
+
fallbackReason = "embedding engine unavailable; used FTS5 keyword fallback";
|
|
1776
|
+
return {
|
|
1777
|
+
results: await ftsFallback(store, query, topK),
|
|
1778
|
+
trace: {
|
|
1779
|
+
steps,
|
|
1780
|
+
rerankedEventTitles,
|
|
1781
|
+
fallbackReason
|
|
1782
|
+
}
|
|
1783
|
+
};
|
|
1784
|
+
}
|
|
1785
|
+
const queryEmbeddings = await timed(onStep, "queryEmbedding", "把用户问题转成向量,用于召回相关事件和切片。", async () => engine.embedBatch([query]));
|
|
1786
|
+
if (queryEmbeddings === null || queryEmbeddings.length === 0) {
|
|
1787
|
+
fallbackReason = "query embedding returned null; used FTS5 fallback";
|
|
1788
|
+
return {
|
|
1789
|
+
results: await ftsFallback(store, query, topK),
|
|
1790
|
+
trace: {
|
|
1791
|
+
steps,
|
|
1792
|
+
rerankedEventTitles,
|
|
1793
|
+
fallbackReason
|
|
1794
|
+
}
|
|
1795
|
+
};
|
|
1796
|
+
}
|
|
1555
1797
|
const queryVec = queryEmbeddings[0];
|
|
1556
|
-
const recalledEntities = await recallEntities(store, llm, query, queryVec);
|
|
1798
|
+
const recalledEntities = await timed(onStep, "entityRecall", "LLM 抽 query 实体 + 名字精确匹配 + 向量召回。", () => recallEntities(store, llm, query, queryVec), (entities) => ({ count: entities.length }));
|
|
1557
1799
|
const seedEventIds = /* @__PURE__ */ new Set();
|
|
1558
1800
|
for (const entity of recalledEntities) {
|
|
1559
1801
|
const events = await store.findEventsByEntity(entity.id);
|
|
1560
1802
|
for (const event of events) seedEventIds.add(event.id);
|
|
1561
1803
|
}
|
|
1562
|
-
const titleMatches = await store.findEventsByTitleVector(queryVec, {
|
|
1804
|
+
const titleMatches = await timed(onStep, "seedEventsByTitle", "按查询向量在事件标题向量上召回 seed events。", () => store.findEventsByTitleVector(queryVec, {
|
|
1563
1805
|
limit: SEED_EVENT_LIMIT,
|
|
1564
1806
|
threshold: TITLE_VECTOR_THRESHOLD
|
|
1565
|
-
});
|
|
1807
|
+
}), (matches) => ({ count: matches.length }));
|
|
1566
1808
|
for (const { event } of titleMatches) seedEventIds.add(event.id);
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1809
|
+
if (seedEventIds.size === 0) {
|
|
1810
|
+
fallbackReason = "no seed events; used direct chunk vector search";
|
|
1811
|
+
return {
|
|
1812
|
+
results: await chunkVectorFallback(store, queryVec, topK),
|
|
1813
|
+
trace: {
|
|
1814
|
+
steps,
|
|
1815
|
+
rerankedEventTitles,
|
|
1816
|
+
fallbackReason
|
|
1575
1817
|
}
|
|
1576
|
-
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
const expandedEventIds = await timed(onStep, "bfsExpand", "从 seed events 沿实体关系 1 跳扩展候选事件。", async () => {
|
|
1821
|
+
const ids = new Set(seedEventIds);
|
|
1822
|
+
let capped = false;
|
|
1823
|
+
for (const eventId of seedEventIds) {
|
|
1824
|
+
const entities = await store.findEntitiesByEvent(eventId);
|
|
1825
|
+
for (const entity of entities) {
|
|
1826
|
+
const neighborEvents = await store.findEventsByEntity(entity.id);
|
|
1827
|
+
for (const neighbor of neighborEvents) {
|
|
1828
|
+
ids.add(neighbor.id);
|
|
1829
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) {
|
|
1830
|
+
capped = true;
|
|
1831
|
+
break;
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
1835
|
+
}
|
|
1836
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
1577
1837
|
}
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1838
|
+
return {
|
|
1839
|
+
ids,
|
|
1840
|
+
capped
|
|
1841
|
+
};
|
|
1842
|
+
}, (result) => ({
|
|
1843
|
+
expandedCount: result.ids.size,
|
|
1844
|
+
capped: result.capped
|
|
1845
|
+
}));
|
|
1846
|
+
const { candidates, topCandidates } = await timed(onStep, "coarseRank", "按 content 向量相似度对候选事件排序,取前 COARSE_RANK_LIMIT 个。", async () => {
|
|
1847
|
+
const cands = [];
|
|
1848
|
+
for (const eventId of expandedEventIds.ids) {
|
|
1849
|
+
const event = await store.getEvent(eventId);
|
|
1850
|
+
if (event === void 0) continue;
|
|
1851
|
+
const vec = event.contentEmbedding;
|
|
1852
|
+
const score = vec === null ? 0 : engine.cosineSimilarity(queryVec, vec);
|
|
1853
|
+
cands.push({
|
|
1854
|
+
id: event.id,
|
|
1855
|
+
title: event.title,
|
|
1856
|
+
summary: event.summary ?? "",
|
|
1857
|
+
score,
|
|
1858
|
+
chunkId: event.chunkId
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1861
|
+
cands.sort((a, b) => b.score - a.score);
|
|
1862
|
+
return {
|
|
1863
|
+
candidates: cands,
|
|
1864
|
+
topCandidates: cands.slice(0, COARSE_RANK_LIMIT)
|
|
1865
|
+
};
|
|
1866
|
+
}, (result) => ({
|
|
1867
|
+
totalCandidates: result.candidates.length,
|
|
1868
|
+
kept: result.topCandidates.length,
|
|
1869
|
+
topScore: result.topCandidates[0]?.score ?? 0
|
|
1870
|
+
}));
|
|
1871
|
+
if (candidates.length === 0) {
|
|
1872
|
+
fallbackReason = "no graph-reachable events with content; used chunk vector fallback";
|
|
1873
|
+
return {
|
|
1874
|
+
results: await chunkVectorFallback(store, queryVec, topK),
|
|
1875
|
+
trace: {
|
|
1876
|
+
steps,
|
|
1877
|
+
rerankedEventTitles,
|
|
1878
|
+
fallbackReason
|
|
1879
|
+
}
|
|
1880
|
+
};
|
|
1593
1881
|
}
|
|
1594
|
-
if (candidates.length === 0) return chunkVectorFallback(store, queryVec, topK);
|
|
1595
|
-
candidates.sort((a, b) => b.score - a.score);
|
|
1596
|
-
const topCandidates = candidates.slice(0, COARSE_RANK_LIMIT);
|
|
1597
1882
|
let rankedIds;
|
|
1598
|
-
if (options.skipRerank === true
|
|
1599
|
-
|
|
1883
|
+
if (options.skipRerank === true) {
|
|
1884
|
+
rankedIds = topCandidates.map((c) => c.id);
|
|
1885
|
+
onStep({
|
|
1886
|
+
step: "rerank",
|
|
1887
|
+
detail: "跳过 LLM rerank(skipRerank=true),直接用 coarse rank 顺序。",
|
|
1888
|
+
durationMs: 0,
|
|
1889
|
+
payload: { count: rankedIds.length }
|
|
1890
|
+
});
|
|
1891
|
+
} else if (topCandidates.length <= topK) {
|
|
1892
|
+
rankedIds = topCandidates.map((c) => c.id);
|
|
1893
|
+
onStep({
|
|
1894
|
+
step: "rerank",
|
|
1895
|
+
detail: `候选数 ${topCandidates.length} ≤ topK ${topK},无需 LLM rerank。`,
|
|
1896
|
+
durationMs: 0,
|
|
1897
|
+
payload: { count: rankedIds.length }
|
|
1898
|
+
});
|
|
1899
|
+
} else rankedIds = await timed(onStep, "rerank", `LLM 从 ${topCandidates.length} 个候选中选最相关的 ${topK} 个。`, () => rerankEventsWithLlm(llm, query, topCandidates.map((c) => ({
|
|
1600
1900
|
id: c.id,
|
|
1601
1901
|
title: c.title,
|
|
1602
1902
|
summary: c.summary
|
|
1603
|
-
})), topK);
|
|
1903
|
+
})), topK), (ids) => ({ count: ids.length }));
|
|
1904
|
+
for (const id of rankedIds) {
|
|
1905
|
+
const c = topCandidates.find((x) => x.id === id);
|
|
1906
|
+
if (c !== void 0) rerankedEventTitles.push(c.title);
|
|
1907
|
+
}
|
|
1604
1908
|
const seen = /* @__PURE__ */ new Set();
|
|
1605
1909
|
const results = [];
|
|
1606
1910
|
for (const eventId of rankedIds) {
|
|
@@ -1612,17 +1916,29 @@ async function multiSearch(store, llm, query, options = {}) {
|
|
|
1612
1916
|
if (result !== void 0) results.push(result);
|
|
1613
1917
|
if (results.length >= topK) break;
|
|
1614
1918
|
}
|
|
1615
|
-
if (results.length < topK)
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1919
|
+
if (results.length < topK) {
|
|
1920
|
+
let supplemented = 0;
|
|
1921
|
+
for (const candidate of topCandidates) {
|
|
1922
|
+
if (results.length >= topK) break;
|
|
1923
|
+
if (rankedIds.includes(candidate.id)) continue;
|
|
1924
|
+
if (seen.has(candidate.chunkId)) continue;
|
|
1925
|
+
seen.add(candidate.chunkId);
|
|
1926
|
+
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, candidate.id);
|
|
1927
|
+
if (result !== void 0) {
|
|
1928
|
+
results.push(result);
|
|
1929
|
+
supplemented += 1;
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
if (supplemented > 0) onStep({
|
|
1933
|
+
step: "supplementFromCoarse",
|
|
1934
|
+
detail: `rerank 结果不足,从 coarse rank 残余补 ${supplemented} 个。`,
|
|
1935
|
+
durationMs: 0,
|
|
1936
|
+
payload: { supplemented }
|
|
1937
|
+
});
|
|
1622
1938
|
}
|
|
1623
1939
|
if (results.length < topK) {
|
|
1624
1940
|
const remaining = topK - results.length;
|
|
1625
|
-
const backfill = await store.searchChunksByVector(queryVec, { limit: remaining * 2 });
|
|
1941
|
+
const backfill = await timed(onStep, "backfill", `rerank 结果不足,回退到 chunk 向量搜索补 ${remaining} 个。`, () => store.searchChunksByVector(queryVec, { limit: remaining * 2 }), (matches) => ({ backfillCandidates: matches.length }));
|
|
1626
1942
|
for (const { chunk, score } of backfill) {
|
|
1627
1943
|
if (results.length >= topK) break;
|
|
1628
1944
|
if (seen.has(chunk.id)) continue;
|
|
@@ -1631,7 +1947,14 @@ async function multiSearch(store, llm, query, options = {}) {
|
|
|
1631
1947
|
if (result !== void 0) results.push(result);
|
|
1632
1948
|
}
|
|
1633
1949
|
}
|
|
1634
|
-
return
|
|
1950
|
+
return {
|
|
1951
|
+
results,
|
|
1952
|
+
trace: {
|
|
1953
|
+
steps,
|
|
1954
|
+
rerankedEventTitles,
|
|
1955
|
+
fallbackReason
|
|
1956
|
+
}
|
|
1957
|
+
};
|
|
1635
1958
|
}
|
|
1636
1959
|
/** Recall entities by name (LLM-extracted) and by vector similarity. */
|
|
1637
1960
|
async function recallEntities(store, llm, query, queryVec) {
|
|
@@ -1671,4 +1994,4 @@ async function chunkVectorFallback(store, queryVec, topK) {
|
|
|
1671
1994
|
return results;
|
|
1672
1995
|
}
|
|
1673
1996
|
//#endregion
|
|
1674
|
-
export {
|
|
1997
|
+
export { join as C, resolve as D, relative as E, isAbsolute as S, parse as T, splitLargeSection as _, isSupportedFile as a, basename as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, normalize as w, dirname as x, KnowledgeStore as y };
|