scream-code 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.mjs
CHANGED
|
@@ -6,7 +6,7 @@ const __dirname = __cjsShimDirname(__filename);
|
|
|
6
6
|
import "./suppress-sqlite-warning-C2VB0doZ.mjs";
|
|
7
7
|
//#region src/main.ts
|
|
8
8
|
try {
|
|
9
|
-
(await import("./app-
|
|
9
|
+
(await import("./app-a4CtjPlv.mjs")).main();
|
|
10
10
|
} catch (error) {
|
|
11
11
|
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
|
|
12
12
|
process.exit(1);
|
|
@@ -1392,6 +1392,19 @@ Only include entities that explicitly appear in the query. If no clear entities,
|
|
|
1392
1392
|
}
|
|
1393
1393
|
//#endregion
|
|
1394
1394
|
//#region ../../packages/knowledge/src/ingest.ts
|
|
1395
|
+
/**
|
|
1396
|
+
* Embed texts, returning null-array if embeddings are skipped or on failure.
|
|
1397
|
+
* Caller should set skipEmbedding=true when the model didn't load in time,
|
|
1398
|
+
* so the entire ingestion is consistent (all or nothing for embeddings).
|
|
1399
|
+
* Search falls back to FTS5 keyword match for chunks without embeddings.
|
|
1400
|
+
*/
|
|
1401
|
+
async function embedOrNulls(engine, texts, skipEmbedding) {
|
|
1402
|
+
if (engine === void 0 || texts.length === 0 || skipEmbedding) return texts.map(() => null);
|
|
1403
|
+
if (!engine.available) return texts.map(() => null);
|
|
1404
|
+
const vectors = await engine.embedBatch(texts);
|
|
1405
|
+
if (vectors === null || vectors.length !== texts.length) return texts.map(() => null);
|
|
1406
|
+
return vectors;
|
|
1407
|
+
}
|
|
1395
1408
|
/** LLM concurrency for extraction — kept low to avoid rate-limit bursts. */
|
|
1396
1409
|
const LLM_CONCURRENCY = 3;
|
|
1397
1410
|
const SUPPORTED_EXTENSIONS = new Set([
|
|
@@ -1454,12 +1467,25 @@ async function mapWithConcurrency(items, limit, fn, onProgress) {
|
|
|
1454
1467
|
*/
|
|
1455
1468
|
async function ingestFile(store, llm, filePath, onProgress) {
|
|
1456
1469
|
const engine = store.getEmbeddingEngine();
|
|
1457
|
-
|
|
1470
|
+
let skipEmbed = engine === void 0;
|
|
1471
|
+
if (engine !== void 0) {
|
|
1458
1472
|
onProgress?.({
|
|
1459
|
-
stage: "
|
|
1460
|
-
message: "
|
|
1473
|
+
stage: "embedding-check",
|
|
1474
|
+
message: "检查向量模型..."
|
|
1461
1475
|
});
|
|
1462
|
-
|
|
1476
|
+
try {
|
|
1477
|
+
skipEmbed = !await Promise.race([engine.ensureReady(), new Promise((resolve) => setTimeout(() => resolve(false), 15e3))]);
|
|
1478
|
+
if (skipEmbed) onProgress?.({
|
|
1479
|
+
stage: "embedding-check",
|
|
1480
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1481
|
+
});
|
|
1482
|
+
} catch {
|
|
1483
|
+
skipEmbed = true;
|
|
1484
|
+
onProgress?.({
|
|
1485
|
+
stage: "embedding-check",
|
|
1486
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1463
1489
|
}
|
|
1464
1490
|
const existing = await store.findSourceByFilePath(filePath);
|
|
1465
1491
|
if (existing !== void 0) {
|
|
@@ -1501,14 +1527,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1501
1527
|
totalChunks: sections.length,
|
|
1502
1528
|
message: `嵌入 chunks: 0/${sections.length}`
|
|
1503
1529
|
});
|
|
1504
|
-
const chunkEmbeddings = await engine
|
|
1505
|
-
if (chunkEmbeddings === null) {
|
|
1506
|
-
onProgress?.({
|
|
1507
|
-
stage: "error",
|
|
1508
|
-
message: "chunk 向量嵌入失败"
|
|
1509
|
-
});
|
|
1510
|
-
throw new Error("chunk 向量嵌入失败");
|
|
1511
|
-
}
|
|
1530
|
+
const chunkEmbeddings = await embedOrNulls(engine, sections.map((s) => s.heading !== null ? `${s.heading}\n${s.content}` : s.content), skipEmbed);
|
|
1512
1531
|
const chunks = [];
|
|
1513
1532
|
for (let i = 0; i < sections.length; i++) {
|
|
1514
1533
|
const section = sections[i];
|
|
@@ -1552,14 +1571,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1552
1571
|
});
|
|
1553
1572
|
const titleTexts = extractedEvents.map((e) => e.title);
|
|
1554
1573
|
const contentTexts = extractedEvents.map((e) => `${e.title}\n\n${e.content}`);
|
|
1555
|
-
const [titleEmbeddings, contentEmbeddings] = await Promise.all([engine
|
|
1556
|
-
if (titleEmbeddings === null || contentEmbeddings === null) {
|
|
1557
|
-
onProgress?.({
|
|
1558
|
-
stage: "error",
|
|
1559
|
-
message: "event 向量嵌入失败"
|
|
1560
|
-
});
|
|
1561
|
-
throw new Error("event 向量嵌入失败");
|
|
1562
|
-
}
|
|
1574
|
+
const [titleEmbeddings, contentEmbeddings] = await Promise.all([embedOrNulls(engine, titleTexts, skipEmbed), embedOrNulls(engine, contentTexts, skipEmbed)]);
|
|
1563
1575
|
const events = [];
|
|
1564
1576
|
for (let i = 0; i < extractedEvents.length; i++) {
|
|
1565
1577
|
const extracted = extractedEvents[i];
|
|
@@ -1595,18 +1607,11 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1595
1607
|
if (!entityMap.has(key)) entityMap.set(key, entity);
|
|
1596
1608
|
}
|
|
1597
1609
|
const uniqueEntities = Array.from(entityMap.values());
|
|
1598
|
-
const entityEmbeddings = uniqueEntities.length > 0 ? await engine
|
|
1599
|
-
if (uniqueEntities.length > 0 && entityEmbeddings === null) {
|
|
1600
|
-
onProgress?.({
|
|
1601
|
-
stage: "error",
|
|
1602
|
-
message: "entity 向量嵌入失败"
|
|
1603
|
-
});
|
|
1604
|
-
throw new Error("entity 向量嵌入失败");
|
|
1605
|
-
}
|
|
1610
|
+
const entityEmbeddings = uniqueEntities.length > 0 ? await embedOrNulls(engine, uniqueEntities.map((e) => e.name), skipEmbed) : [];
|
|
1606
1611
|
const entityIdByEntityKey = /* @__PURE__ */ new Map();
|
|
1607
1612
|
for (let i = 0; i < uniqueEntities.length; i++) {
|
|
1608
1613
|
const entity = uniqueEntities[i];
|
|
1609
|
-
const embedding = entityEmbeddings
|
|
1614
|
+
const embedding = entityEmbeddings[i] ?? null;
|
|
1610
1615
|
const stored = await store.upsertEntity({
|
|
1611
1616
|
sourceId: source.id,
|
|
1612
1617
|
type: entity.type,
|
|
@@ -1632,14 +1637,7 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
1632
1637
|
const event = events[eventIndex];
|
|
1633
1638
|
return entity.description.length > 0 ? entity.description : `${event.title} ${entity.name}`;
|
|
1634
1639
|
});
|
|
1635
|
-
const relationEmbeddings = relationPairs.length > 0 ? await engine
|
|
1636
|
-
if (relationPairs.length > 0 && relationEmbeddings === null) {
|
|
1637
|
-
onProgress?.({
|
|
1638
|
-
stage: "error",
|
|
1639
|
-
message: "relation 向量嵌入失败"
|
|
1640
|
-
});
|
|
1641
|
-
throw new Error("relation 向量嵌入失败");
|
|
1642
|
-
}
|
|
1640
|
+
const relationEmbeddings = relationPairs.length > 0 ? await embedOrNulls(engine, relationTexts, skipEmbed) : [];
|
|
1643
1641
|
for (let i = 0; i < relationPairs.length; i++) {
|
|
1644
1642
|
const pair = relationPairs[i];
|
|
1645
1643
|
const event = events[pair.eventIndex];
|
|
@@ -3,5 +3,5 @@ import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
|
3
3
|
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
-
import { n as multiSearchWithTrace } from "./src-
|
|
6
|
+
import { n as multiSearchWithTrace } from "./src-BHYcdFfg.mjs";
|
|
7
7
|
export { multiSearchWithTrace };
|
package/icon.ico
CHANGED
|
Binary file
|