scream-code 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5,7 +5,7 @@ const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
6
|
import { a as __toESM, i as __require, r as __exportAll, t as __commonJSMin } from "./chunk-apG1qJts.mjs";
|
|
7
7
|
import "./suppress-sqlite-warning-C2VB0doZ.mjs";
|
|
8
|
-
import { C as join$1, D as resolve$1, E as relative$1, S as isAbsolute$1, T as parse$7, a as isSupportedFile, b as basename$1, i as ingestFile, r as ingestDirectory, t as multiSearch, w as normalize, x as dirname$2, y as KnowledgeStore } from "./src-
|
|
8
|
+
import { C as join$1, D as resolve$1, E as relative$1, S as isAbsolute$1, T as parse$7, a as isSupportedFile, b as basename$1, i as ingestFile, r as ingestDirectory, t as multiSearch, w as normalize, x as dirname$2, y as KnowledgeStore } from "./src-BHYcdFfg.mjs";
|
|
9
9
|
import { createRequire } from "node:module";
|
|
10
10
|
import { createHash, randomBytes, randomInt, randomUUID } from "node:crypto";
|
|
11
11
|
import Jn, { access, appendFile, chmod, copyFile, cp, lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
@@ -56624,10 +56624,31 @@ function buildEmbeddingText(memo) {
|
|
|
56624
56624
|
return `${memo.userNeed} ${memo.approach} ${memo.whatWorked}`;
|
|
56625
56625
|
}
|
|
56626
56626
|
/**
|
|
56627
|
+
* Cache of created engines keyed by cacheDir.
|
|
56628
|
+
* Guarantees that all callers sharing the same cacheDir reuse the same engine
|
|
56629
|
+
* instance and the same in-flight model download/load — avoiding duplicate
|
|
56630
|
+
* downloads and file-corruption races when /memory and /knowledge both start
|
|
56631
|
+
* before the model is cached.
|
|
56632
|
+
*/
|
|
56633
|
+
const engineCache = /* @__PURE__ */ new Map();
|
|
56634
|
+
/**
|
|
56627
56635
|
* Create an embedding engine backed by fastembed.
|
|
56628
56636
|
* Lazily loads the model on first use so startup is not blocked.
|
|
56629
|
-
|
|
56630
|
-
|
|
56637
|
+
* Engines are cached by cacheDir so repeated calls with the same cacheDir
|
|
56638
|
+
* return the same instance, sharing model state and download progress.
|
|
56639
|
+
* @param cacheDir Absolute path for model cache (e.g. ~/.scream-code/cache/fastembed).
|
|
56640
|
+
* Defaults to "local_cache" (CWD-relative) if not provided — prefer
|
|
56641
|
+
* passing an explicit path so the cache doesn't duplicate across CWDs.
|
|
56642
|
+
*/
|
|
56643
|
+
function createFastEmbedEngine(cacheDir) {
|
|
56644
|
+
const key = cacheDir ?? "";
|
|
56645
|
+
const cached = engineCache.get(key);
|
|
56646
|
+
if (cached !== void 0) return cached;
|
|
56647
|
+
const engine = createFastEmbedEngineImpl(cacheDir);
|
|
56648
|
+
engineCache.set(key, engine);
|
|
56649
|
+
return engine;
|
|
56650
|
+
}
|
|
56651
|
+
function createFastEmbedEngineImpl(cacheDir) {
|
|
56631
56652
|
let embedder = null;
|
|
56632
56653
|
let initPromise = null;
|
|
56633
56654
|
let loadFailed = false;
|
|
@@ -56640,7 +56661,7 @@ function createFastEmbedEngine() {
|
|
|
56640
56661
|
if (texts.length === 0) return [];
|
|
56641
56662
|
try {
|
|
56642
56663
|
if (embedder === null) {
|
|
56643
|
-
initPromise ??= loadEmbedder();
|
|
56664
|
+
initPromise ??= loadEmbedder(cacheDir);
|
|
56644
56665
|
embedder = await initPromise;
|
|
56645
56666
|
if (embedder === null) {
|
|
56646
56667
|
loadFailed = true;
|
|
@@ -56668,22 +56689,93 @@ function createFastEmbedEngine() {
|
|
|
56668
56689
|
}
|
|
56669
56690
|
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
56670
56691
|
return denom === 0 ? 0 : dot / denom;
|
|
56692
|
+
},
|
|
56693
|
+
async ensureReady() {
|
|
56694
|
+
if (embedder !== null) {
|
|
56695
|
+
loadFailed = false;
|
|
56696
|
+
return true;
|
|
56697
|
+
}
|
|
56698
|
+
try {
|
|
56699
|
+
if (loadFailed || initPromise === null) initPromise = loadEmbedder(cacheDir);
|
|
56700
|
+
embedder = await initPromise;
|
|
56701
|
+
if (embedder === null) {
|
|
56702
|
+
initPromise = null;
|
|
56703
|
+
return false;
|
|
56704
|
+
}
|
|
56705
|
+
loadFailed = false;
|
|
56706
|
+
return true;
|
|
56707
|
+
} catch {
|
|
56708
|
+
initPromise = null;
|
|
56709
|
+
return false;
|
|
56710
|
+
}
|
|
56671
56711
|
}
|
|
56672
56712
|
};
|
|
56673
56713
|
}
|
|
56674
|
-
async function loadEmbedder() {
|
|
56714
|
+
async function loadEmbedder(cacheDir) {
|
|
56675
56715
|
try {
|
|
56676
56716
|
const { FlagEmbedding, EmbeddingModel } = await import("./esm-Du2MwXei.mjs");
|
|
56677
|
-
|
|
56717
|
+
if (cacheDir !== void 0) mkdirSync(cacheDir, { recursive: true });
|
|
56718
|
+
const model = EmbeddingModel.BGESmallZH;
|
|
56719
|
+
const initOpts = cacheDir !== void 0 ? {
|
|
56720
|
+
model,
|
|
56721
|
+
cacheDir
|
|
56722
|
+
} : { model };
|
|
56723
|
+
try {
|
|
56724
|
+
return await FlagEmbedding.init(initOpts);
|
|
56725
|
+
} catch (initError) {
|
|
56726
|
+
const msg = initError instanceof Error ? initError.message : "";
|
|
56727
|
+
if (!/Config file not found|Tokenizer file not found|Tokens map file not found/iu.test(msg)) return null;
|
|
56728
|
+
try {
|
|
56729
|
+
await ensureFastembedModelSidecars(String(model), cacheDir);
|
|
56730
|
+
return await FlagEmbedding.init(initOpts);
|
|
56731
|
+
} catch {
|
|
56732
|
+
return null;
|
|
56733
|
+
}
|
|
56734
|
+
}
|
|
56678
56735
|
} catch {
|
|
56679
56736
|
return null;
|
|
56680
56737
|
}
|
|
56681
56738
|
}
|
|
56739
|
+
/**
|
|
56740
|
+
* Small config/tokenizer files that fastembed expects alongside model.onnx.
|
|
56741
|
+
* If these are missing (e.g. GCS download partially failed), fastembed throws.
|
|
56742
|
+
* We download them from HuggingFace so the model can load — this covers the
|
|
56743
|
+
* case where the GCS tarball was incomplete but the HF repo is reachable.
|
|
56744
|
+
*/
|
|
56745
|
+
const FASTEMBED_SIDECARS = [
|
|
56746
|
+
"config.json",
|
|
56747
|
+
"tokenizer.json",
|
|
56748
|
+
"tokenizer_config.json",
|
|
56749
|
+
"special_tokens_map.json"
|
|
56750
|
+
];
|
|
56751
|
+
const FASTEMBED_HF_REPOS = { "fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5" };
|
|
56752
|
+
async function ensureFastembedModelSidecars(model, cacheDir) {
|
|
56753
|
+
const repo = FASTEMBED_HF_REPOS[model];
|
|
56754
|
+
if (repo === void 0) return;
|
|
56755
|
+
const modelDir = join(cacheDir ?? "local_cache", model);
|
|
56756
|
+
mkdirSync(modelDir, { recursive: true });
|
|
56757
|
+
for (const fileName of FASTEMBED_SIDECARS) {
|
|
56758
|
+
const target = join(modelDir, fileName);
|
|
56759
|
+
try {
|
|
56760
|
+
const { access } = await import("node:fs/promises");
|
|
56761
|
+
await access(target);
|
|
56762
|
+
continue;
|
|
56763
|
+
} catch {}
|
|
56764
|
+
const hfUrl = `https://huggingface.co/${repo}/resolve/main/${fileName}`;
|
|
56765
|
+
try {
|
|
56766
|
+
const response = await fetch(hfUrl);
|
|
56767
|
+
if (!response.ok) continue;
|
|
56768
|
+
const { writeFile } = await import("node:fs/promises");
|
|
56769
|
+
await writeFile(target, Buffer.from(await response.arrayBuffer()));
|
|
56770
|
+
} catch {}
|
|
56771
|
+
}
|
|
56772
|
+
}
|
|
56682
56773
|
//#endregion
|
|
56683
56774
|
//#region ../../packages/memory/src/store.ts
|
|
56684
56775
|
const FILE_NAME = "entries.jsonl";
|
|
56685
56776
|
const MIGRATION_MARKER = ".migrated";
|
|
56686
56777
|
const SQLITE_MIGRATION_MARKER = ".migrated-to-sqlite";
|
|
56778
|
+
const EMBEDDING_LOAD_RETRY_MS = 300 * 1e3;
|
|
56687
56779
|
var MemoryMemoStore = class MemoryMemoStore {
|
|
56688
56780
|
projectDir;
|
|
56689
56781
|
jsonlPath;
|
|
@@ -56698,6 +56790,7 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
56698
56790
|
embeddingDegraded = false;
|
|
56699
56791
|
embeddingConsecutiveFailures = 0;
|
|
56700
56792
|
lastEmbeddingError;
|
|
56793
|
+
embeddingLoadTimer;
|
|
56701
56794
|
log;
|
|
56702
56795
|
constructor(projectDir, log) {
|
|
56703
56796
|
this.projectDir = projectDir;
|
|
@@ -56735,6 +56828,10 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
56735
56828
|
}
|
|
56736
56829
|
/** Close the database handle and checkpoint WAL. */
|
|
56737
56830
|
close() {
|
|
56831
|
+
if (this.embeddingLoadTimer !== void 0) {
|
|
56832
|
+
clearInterval(this.embeddingLoadTimer);
|
|
56833
|
+
this.embeddingLoadTimer = void 0;
|
|
56834
|
+
}
|
|
56738
56835
|
if (this.db !== void 0) {
|
|
56739
56836
|
this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
56740
56837
|
this.db.close();
|
|
@@ -57112,6 +57209,34 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
57112
57209
|
/** Set the embedding engine. Call once after construction, before any writes. */
|
|
57113
57210
|
setEmbeddingEngine(engine) {
|
|
57114
57211
|
this.embeddingEngine = engine;
|
|
57212
|
+
this.startEmbeddingLoadLoop();
|
|
57213
|
+
}
|
|
57214
|
+
/**
|
|
57215
|
+
* Background loop that retries model loading every 5 minutes until success.
|
|
57216
|
+
* Covers the case where the model download fails on first use (network issue,
|
|
57217
|
+
* GCS blocked, etc.) — without this, the engine stays loadFailed forever and
|
|
57218
|
+
* all future memos fall back to keyword search silently.
|
|
57219
|
+
* Stops itself once the model loads successfully.
|
|
57220
|
+
*/
|
|
57221
|
+
startEmbeddingLoadLoop() {
|
|
57222
|
+
if (this.embeddingEngine === void 0) return;
|
|
57223
|
+
if (this.embeddingLoadTimer !== void 0) return;
|
|
57224
|
+
const attempt = async () => {
|
|
57225
|
+
if (this.embeddingEngine === void 0) return;
|
|
57226
|
+
if (await this.embeddingEngine.ensureReady()) {
|
|
57227
|
+
if (this.embeddingLoadTimer !== void 0) {
|
|
57228
|
+
clearInterval(this.embeddingLoadTimer);
|
|
57229
|
+
this.embeddingLoadTimer = void 0;
|
|
57230
|
+
}
|
|
57231
|
+
this.embeddingDegraded = false;
|
|
57232
|
+
return;
|
|
57233
|
+
}
|
|
57234
|
+
};
|
|
57235
|
+
this.embeddingLoadTimer = setInterval(() => {
|
|
57236
|
+
attempt();
|
|
57237
|
+
}, EMBEDDING_LOAD_RETRY_MS);
|
|
57238
|
+
this.embeddingLoadTimer.unref?.();
|
|
57239
|
+
attempt();
|
|
57115
57240
|
}
|
|
57116
57241
|
/** Check whether the store has any vector embeddings. */
|
|
57117
57242
|
hasEmbeddings() {
|
|
@@ -58216,7 +58341,7 @@ var KnowledgeLookupTool = class {
|
|
|
58216
58341
|
const llm = { generate: async (systemPrompt, userPrompt) => {
|
|
58217
58342
|
return this.agent.generateText(systemPrompt, userPrompt);
|
|
58218
58343
|
} };
|
|
58219
|
-
const { multiSearchWithTrace } = await import("./src-
|
|
58344
|
+
const { multiSearchWithTrace } = await import("./src-CC8SSs-H.mjs");
|
|
58220
58345
|
const { results, trace } = await multiSearchWithTrace(store, llm, query, { topK });
|
|
58221
58346
|
if (results.length === 0) return {
|
|
58222
58347
|
isError: false,
|
|
@@ -98577,6 +98702,7 @@ var Agent = class {
|
|
|
98577
98702
|
dreamTracker;
|
|
98578
98703
|
replayBuilder;
|
|
98579
98704
|
lastLlmConfigLogSignature;
|
|
98705
|
+
sharedEmbeddingEngine;
|
|
98580
98706
|
constructor(options) {
|
|
98581
98707
|
this.type = options.type ?? "main";
|
|
98582
98708
|
this.jian = options.jian;
|
|
@@ -98591,6 +98717,8 @@ var Agent = class {
|
|
|
98591
98717
|
this.subagentHost = options.subagentHost;
|
|
98592
98718
|
this.mcp = options.mcp;
|
|
98593
98719
|
this.hooks = options.hookEngine;
|
|
98720
|
+
const embedCacheDir = options.screamHomeDir !== void 0 ? join$1(options.screamHomeDir, "cache", "fastembed") : void 0;
|
|
98721
|
+
this.sharedEmbeddingEngine = createFastEmbedEngine(embedCacheDir);
|
|
98594
98722
|
this.log = options.log ?? log;
|
|
98595
98723
|
this.blobStore = options.homedir ? new BlobStore({ blobsDir: join$1(options.homedir, "blobs") }) : void 0;
|
|
98596
98724
|
this.records = new AgentRecords(this, options.persistence ?? (options.homedir ? new FileSystemAgentRecordPersistence(join$1(options.homedir, "wire.jsonl"), {
|
|
@@ -98649,7 +98777,7 @@ var Agent = class {
|
|
|
98649
98777
|
this.log.error("memory legacy migration failed", error);
|
|
98650
98778
|
}
|
|
98651
98779
|
try {
|
|
98652
|
-
this.memoStore.setEmbeddingEngine(
|
|
98780
|
+
this.memoStore.setEmbeddingEngine(this.sharedEmbeddingEngine);
|
|
98653
98781
|
} catch (error) {
|
|
98654
98782
|
this.log.warn("embedding engine init failed; falling back to keyword search", error);
|
|
98655
98783
|
}
|
|
@@ -98664,7 +98792,7 @@ var Agent = class {
|
|
|
98664
98792
|
this.log.error("knowledge store init failed", error);
|
|
98665
98793
|
}
|
|
98666
98794
|
try {
|
|
98667
|
-
store.setEmbeddingEngine(
|
|
98795
|
+
store.setEmbeddingEngine(this.sharedEmbeddingEngine);
|
|
98668
98796
|
} catch (error) {
|
|
98669
98797
|
this.log.warn("knowledge embedding engine init failed", error);
|
|
98670
98798
|
}
|
|
@@ -122253,7 +122381,7 @@ function optionalBuildString(value) {
|
|
|
122253
122381
|
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
122254
122382
|
}
|
|
122255
122383
|
const SCREAM_BUILD_INFO = {
|
|
122256
|
-
version: optionalBuildString("0.8.
|
|
122384
|
+
version: optionalBuildString("0.8.5"),
|
|
122257
122385
|
channel: optionalBuildString(""),
|
|
122258
122386
|
commit: optionalBuildString(""),
|
|
122259
122387
|
buildTarget: optionalBuildString("darwin-arm64")
|
|
@@ -133093,11 +133221,16 @@ function openUrl(url) {
|
|
|
133093
133221
|
//#endregion
|
|
133094
133222
|
//#region src/tui/commands/knowledge-store.ts
|
|
133095
133223
|
let knowledgeStoreInstance;
|
|
133224
|
+
function getEmbeddingCacheDir() {
|
|
133225
|
+
const dir = join(getDataDir(), "cache", "fastembed");
|
|
133226
|
+
mkdirSync(dir, { recursive: true });
|
|
133227
|
+
return dir;
|
|
133228
|
+
}
|
|
133096
133229
|
async function getKnowledgeStore() {
|
|
133097
133230
|
if (knowledgeStoreInstance === void 0) {
|
|
133098
133231
|
knowledgeStoreInstance = new KnowledgeStore(getDataDir());
|
|
133099
133232
|
await knowledgeStoreInstance.init();
|
|
133100
|
-
knowledgeStoreInstance.setEmbeddingEngine(createFastEmbedEngine());
|
|
133233
|
+
knowledgeStoreInstance.setEmbeddingEngine(createFastEmbedEngine(getEmbeddingCacheDir()));
|
|
133101
133234
|
}
|
|
133102
133235
|
return knowledgeStoreInstance;
|
|
133103
133236
|
}
|
|
@@ -133567,7 +133700,7 @@ function serveHTML(res) {
|
|
|
133567
133700
|
});
|
|
133568
133701
|
res.end(HTML);
|
|
133569
133702
|
}
|
|
133570
|
-
async function handleWeb() {
|
|
133703
|
+
async function handleWeb(host) {
|
|
133571
133704
|
const store = await getKnowledgeStore();
|
|
133572
133705
|
const s = await store.stats();
|
|
133573
133706
|
if (s.entities === 0 && s.events === 0) throw new Error("知识库为空,请先用 /knowledge 摄入文档");
|
|
@@ -133588,7 +133721,7 @@ async function handleWeb() {
|
|
|
133588
133721
|
});
|
|
133589
133722
|
const url = `http://127.0.0.1:${server.address().port}`;
|
|
133590
133723
|
openUrl(url);
|
|
133591
|
-
|
|
133724
|
+
host.showStatus(`知识图谱已打开: ${url}`);
|
|
133592
133725
|
}
|
|
133593
133726
|
//#endregion
|
|
133594
133727
|
//#region src/tui/components/dialogs/knowledge-result-viewer.ts
|
|
@@ -133984,6 +134117,7 @@ function makeLlmCaller(host) {
|
|
|
133984
134117
|
}
|
|
133985
134118
|
function formatProgress(progress) {
|
|
133986
134119
|
switch (progress.stage) {
|
|
134120
|
+
case "embedding-check": return progress.message;
|
|
133987
134121
|
case "chunking": return `切分文件中...`;
|
|
133988
134122
|
case "embedding-chunks": return `嵌入 chunks: ${progress.chunkIndex}/${progress.totalChunks}`;
|
|
133989
134123
|
case "extracting": return `抽取事件: ${progress.chunkIndex}/${progress.totalChunks}`;
|
|
@@ -134116,11 +134250,17 @@ async function handleSearch(host) {
|
|
|
134116
134250
|
ok: true,
|
|
134117
134251
|
label: "搜索完成"
|
|
134118
134252
|
});
|
|
134253
|
+
const engine = store.getEmbeddingEngine();
|
|
134254
|
+
const degraded = engine === void 0 || !engine.available;
|
|
134119
134255
|
if (results.length === 0) {
|
|
134120
|
-
await showResultViewer(host, "搜索结果", `查询 "${query}" 未命中任何 chunk`);
|
|
134256
|
+
await showResultViewer(host, "搜索结果", `查询 "${query}" 未命中任何 chunk${degraded ? "\n\n⚠️ 向量模型未就绪,当前为关键词检索模式。如下载失败,建议开启科学上网后重启。" : ""}`);
|
|
134121
134257
|
return;
|
|
134122
134258
|
}
|
|
134123
134259
|
const lines = [`查询: ${query}`, ""];
|
|
134260
|
+
if (degraded) {
|
|
134261
|
+
lines.push("⚠️ 向量模型未就绪,当前为关键词检索模式。如下载失败,建议开启科学上网后重启。");
|
|
134262
|
+
lines.push("");
|
|
134263
|
+
}
|
|
134124
134264
|
for (const [i, r] of results.entries()) {
|
|
134125
134265
|
lines.push(`#${i + 1} [score=${r.score.toFixed(3)}] ${r.heading ?? "(无标题)"}`);
|
|
134126
134266
|
lines.push(` 来源: ${r.sourceName}`);
|
|
@@ -134271,7 +134411,7 @@ async function handleKnowledgeCommand(host, _args) {
|
|
|
134271
134411
|
else if (value === "search") await handleSearch(host);
|
|
134272
134412
|
else if (value === "delete") await handleDelete(host);
|
|
134273
134413
|
else if (value === "stats") await handleStats(host);
|
|
134274
|
-
else if (value === "web") await handleWeb();
|
|
134414
|
+
else if (value === "web") await handleWeb(host);
|
|
134275
134415
|
} catch (error) {
|
|
134276
134416
|
const msg = error instanceof Error ? error.message : String(error);
|
|
134277
134417
|
host.showError(`操作失败: ${msg}`);
|
package/dist/main.mjs
CHANGED
|
@@ -6,7 +6,7 @@ const __dirname = __cjsShimDirname(__filename);
|
|
|
6
6
|
import "./suppress-sqlite-warning-C2VB0doZ.mjs";
|
|
7
7
|
//#region src/main.ts
|
|
8
8
|
try {
|
|
9
|
-
(await import("./app-
|
|
9
|
+
(await import("./app-DFMT0YIn.mjs")).main();
|
|
10
10
|
} catch (error) {
|
|
11
11
|
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
|
|
12
12
|
process.exit(1);
|
|
@@ -243,6 +243,22 @@ var KnowledgeStore = class {
|
|
|
243
243
|
return this.embeddingEngine;
|
|
244
244
|
}
|
|
245
245
|
/**
|
|
246
|
+
* Proactively load the embedding model (downloads on first call).
|
|
247
|
+
* Call before ingestion so the user gets a clear download prompt instead
|
|
248
|
+
* of a mid-ingest failure.
|
|
249
|
+
*/
|
|
250
|
+
async ensureEmbeddingReady() {
|
|
251
|
+
if (this.embeddingEngine === void 0) return {
|
|
252
|
+
ok: false,
|
|
253
|
+
reason: "engine not set"
|
|
254
|
+
};
|
|
255
|
+
const ok = await this.embeddingEngine.ensureReady();
|
|
256
|
+
return {
|
|
257
|
+
ok,
|
|
258
|
+
reason: ok ? void 0 : "fastembed init failed"
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
246
262
|
* Begin a write transaction. Use commitTransaction / rollbackTransaction
|
|
247
263
|
* to close it. Used by ingest to keep chunks/events/entities/edges atomic —
|
|
248
264
|
* a mid-ingest failure leaves no partial rows.
|
|
@@ -1376,6 +1392,19 @@ Only include entities that explicitly appear in the query. If no clear entities,
|
|
|
1376
1392
|
}
|
|
1377
1393
|
//#endregion
|
|
1378
1394
|
//#region ../../packages/knowledge/src/ingest.ts
|
|
1395
|
+
/**
|
|
1396
|
+
* Embed texts, returning null-array if embeddings are skipped or on failure.
|
|
1397
|
+
* Caller should set skipEmbedding=true when the model didn't load in time,
|
|
1398
|
+
* so the entire ingestion is consistent (all or nothing for embeddings).
|
|
1399
|
+
* Search falls back to FTS5 keyword match for chunks without embeddings.
|
|
1400
|
+
*/
|
|
1401
|
+
async function embedOrNulls(engine, texts, skipEmbedding) {
|
|
1402
|
+
if (engine === void 0 || texts.length === 0 || skipEmbedding) return texts.map(() => null);
|
|
1403
|
+
if (!engine.available) return texts.map(() => null);
|
|
1404
|
+
const vectors = await engine.embedBatch(texts);
|
|
1405
|
+
if (vectors === null || vectors.length !== texts.length) return texts.map(() => null);
|
|
1406
|
+
return vectors;
|
|
1407
|
+
}
|
|
1379
1408
|
/** LLM concurrency for extraction — kept low to avoid rate-limit bursts. */
|
|
1380
1409
|
const LLM_CONCURRENCY = 3;
|
|
1381
1410
|
const SUPPORTED_EXTENSIONS = new Set([
|
|
@@ -1438,12 +1467,25 @@ async function mapWithConcurrency(items, limit, fn, onProgress) {
|
|
|
1438
1467
|
*/
|
|
1439
1468
|
async function ingestFile(store, llm, filePath, onProgress) {
|
|
1440
1469
|
const engine = store.getEmbeddingEngine();
|
|
1441
|
-
|
|
1470
|
+
let skipEmbed = engine === void 0;
|
|
1471
|
+
if (engine !== void 0) {
|
|
1442
1472
|
onProgress?.({
|
|
1443
|
-
stage: "
|
|
1444
|
-
message: "
|
|
1473
|
+
stage: "embedding-check",
|
|
1474
|
+
message: "检查向量模型..."
|
|
1445
1475
|
});
|
|
1446
|
-
|
|
1476
|
+
try {
|
|
1477
|
+
skipEmbed = !await Promise.race([engine.ensureReady(), new Promise((resolve) => setTimeout(() => resolve(false), 15e3))]);
|
|
1478
|
+
if (skipEmbed) onProgress?.({
|
|
1479
|
+
stage: "embedding-check",
|
|
1480
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1481
|
+
});
|
|
1482
|
+
} catch {
|
|
1483
|
+
skipEmbed = true;
|
|
1484
|
+
onProgress?.({
|
|
1485
|
+
stage: "embedding-check",
|
|
1486
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1447
1489
|
}
|
|
1448
1490
|
const existing = await store.findSourceByFilePath(filePath);
|
|
1449
1491
|
if (existing !== void 0) {
|
|
@@ -1451,7 +1493,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1451
1493
|
stage: "error",
|
|
1452
1494
|
message: `文件已摄入:${existing.name}(source id=${existing.id})`
|
|
1453
1495
|
});
|
|
1454
|
-
throw new Error(
|
|
1496
|
+
throw new Error(`文件已摄入过: ${filePath} (source ${existing.id})`);
|
|
1455
1497
|
}
|
|
1456
1498
|
onProgress?.({
|
|
1457
1499
|
stage: "chunking",
|
|
@@ -1464,7 +1506,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1464
1506
|
stage: "error",
|
|
1465
1507
|
message: "文件无有效内容"
|
|
1466
1508
|
});
|
|
1467
|
-
throw new Error("
|
|
1509
|
+
throw new Error("文件无有效内容可摄入");
|
|
1468
1510
|
}
|
|
1469
1511
|
store.beginTransaction();
|
|
1470
1512
|
try {
|
|
@@ -1485,14 +1527,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1485
1527
|
totalChunks: sections.length,
|
|
1486
1528
|
message: `嵌入 chunks: 0/${sections.length}`
|
|
1487
1529
|
});
|
|
1488
|
-
const chunkEmbeddings = await engine
|
|
1489
|
-
if (chunkEmbeddings === null) {
|
|
1490
|
-
onProgress?.({
|
|
1491
|
-
stage: "error",
|
|
1492
|
-
message: "chunk embedding failed"
|
|
1493
|
-
});
|
|
1494
|
-
throw new Error("chunk embedding failed");
|
|
1495
|
-
}
|
|
1530
|
+
const chunkEmbeddings = await embedOrNulls(engine, sections.map((s) => s.heading !== null ? `${s.heading}\n${s.content}` : s.content), skipEmbed);
|
|
1496
1531
|
const chunks = [];
|
|
1497
1532
|
for (let i = 0; i < sections.length; i++) {
|
|
1498
1533
|
const section = sections[i];
|
|
@@ -1536,14 +1571,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1536
1571
|
});
|
|
1537
1572
|
const titleTexts = extractedEvents.map((e) => e.title);
|
|
1538
1573
|
const contentTexts = extractedEvents.map((e) => `${e.title}\n\n${e.content}`);
|
|
1539
|
-
const [titleEmbeddings, contentEmbeddings] = await Promise.all([engine
|
|
1540
|
-
if (titleEmbeddings === null || contentEmbeddings === null) {
|
|
1541
|
-
onProgress?.({
|
|
1542
|
-
stage: "error",
|
|
1543
|
-
message: "event embedding failed"
|
|
1544
|
-
});
|
|
1545
|
-
throw new Error("event embedding failed");
|
|
1546
|
-
}
|
|
1574
|
+
const [titleEmbeddings, contentEmbeddings] = await Promise.all([embedOrNulls(engine, titleTexts, skipEmbed), embedOrNulls(engine, contentTexts, skipEmbed)]);
|
|
1547
1575
|
const events = [];
|
|
1548
1576
|
for (let i = 0; i < extractedEvents.length; i++) {
|
|
1549
1577
|
const extracted = extractedEvents[i];
|
|
@@ -1579,18 +1607,11 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1579
1607
|
if (!entityMap.has(key)) entityMap.set(key, entity);
|
|
1580
1608
|
}
|
|
1581
1609
|
const uniqueEntities = Array.from(entityMap.values());
|
|
1582
|
-
const entityEmbeddings = uniqueEntities.length > 0 ? await engine
|
|
1583
|
-
if (uniqueEntities.length > 0 && entityEmbeddings === null) {
|
|
1584
|
-
onProgress?.({
|
|
1585
|
-
stage: "error",
|
|
1586
|
-
message: "entity embedding failed"
|
|
1587
|
-
});
|
|
1588
|
-
throw new Error("entity embedding failed");
|
|
1589
|
-
}
|
|
1610
|
+
const entityEmbeddings = uniqueEntities.length > 0 ? await embedOrNulls(engine, uniqueEntities.map((e) => e.name), skipEmbed) : [];
|
|
1590
1611
|
const entityIdByEntityKey = /* @__PURE__ */ new Map();
|
|
1591
1612
|
for (let i = 0; i < uniqueEntities.length; i++) {
|
|
1592
1613
|
const entity = uniqueEntities[i];
|
|
1593
|
-
const embedding = entityEmbeddings
|
|
1614
|
+
const embedding = entityEmbeddings[i] ?? null;
|
|
1594
1615
|
const stored = await store.upsertEntity({
|
|
1595
1616
|
sourceId: source.id,
|
|
1596
1617
|
type: entity.type,
|
|
@@ -1616,14 +1637,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1616
1637
|
const event = events[eventIndex];
|
|
1617
1638
|
return entity.description.length > 0 ? entity.description : `${event.title} ${entity.name}`;
|
|
1618
1639
|
});
|
|
1619
|
-
const relationEmbeddings = relationPairs.length > 0 ? await engine
|
|
1620
|
-
if (relationPairs.length > 0 && relationEmbeddings === null) {
|
|
1621
|
-
onProgress?.({
|
|
1622
|
-
stage: "error",
|
|
1623
|
-
message: "relation embedding failed"
|
|
1624
|
-
});
|
|
1625
|
-
throw new Error("relation embedding failed");
|
|
1626
|
-
}
|
|
1640
|
+
const relationEmbeddings = relationPairs.length > 0 ? await embedOrNulls(engine, relationTexts, skipEmbed) : [];
|
|
1627
1641
|
for (let i = 0; i < relationPairs.length; i++) {
|
|
1628
1642
|
const pair = relationPairs[i];
|
|
1629
1643
|
const event = events[pair.eventIndex];
|
|
@@ -1668,7 +1682,7 @@ async function collectSupportedFiles(dirPath) {
|
|
|
1668
1682
|
}
|
|
1669
1683
|
async function ingestDirectory(store, llm, dirPath, onProgress) {
|
|
1670
1684
|
const files = await collectSupportedFiles(dirPath);
|
|
1671
|
-
if (files.length === 0) throw new Error(
|
|
1685
|
+
if (files.length === 0) throw new Error(`目录下没有支持的文件 (.md, .markdown, .txt): ${dirPath}`);
|
|
1672
1686
|
const errors = [];
|
|
1673
1687
|
let succeeded = 0;
|
|
1674
1688
|
let totalChunks = 0;
|
|
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { n as multiSearchWithTrace } from "./src-
|
|
6
|
+
import { n as multiSearchWithTrace } from "./src-BHYcdFfg.mjs";
|
|
7
7
|
export { multiSearchWithTrace };
|