wiki-graph-core 0.3.1

package/dist/document/types.d.ts

@@ -0,0 +1,148 @@
+declare enum ChunkRetention {
+    Verbatim = "verbatim",
+    Detailed = "detailed",
+    Focused = "focused",
+    Relevant = "relevant"
+}
+declare const isChunkRetention: (value: string) => value is ChunkRetention;
+declare const expectChunkRetention: (value: string) => ChunkRetention;
+declare enum ChunkImportance {
+    Critical = "critical",
+    Important = "important",
+    Helpful = "helpful"
+}
+declare const isChunkImportance: (value: string) => value is ChunkImportance;
+declare const expectChunkImportance: (value: string) => ChunkImportance;
+type SentenceId = readonly [chapterId: number, sentenceIndex: number];
+interface SentenceRecord {
+    readonly text: string;
+    readonly wordsCount: number;
+}
+declare class Sentence implements SentenceRecord {
+    #private;
+    readonly rawText: string;
+    readonly wordsCount: number;
+    constructor(rawText: string, wordsCount: number);
+    get text(): string;
+    toJSON(): SentenceRecord;
+}
+interface FragmentRecord {
+    readonly serialId: number;
+    readonly fragmentId: number;
+    readonly summary: string;
+    readonly sentences: readonly SentenceRecord[];
+}
+interface SerialRecord {
+    readonly documentOrder: number;
+    readonly knowledgeGraphReady: boolean;
+    readonly knowledgeGraphParameterHash?: string;
+    readonly id: number;
+    readonly revision: number;
+    readonly topologyParameterHash?: string;
+    readonly topologyReady: boolean;
+}
+interface GraphBuildParameterRecord {
+    readonly createdAt: string;
+    readonly hash: string;
+    readonly language?: string;
+    readonly prompt: string;
+}
+interface ChunkRecord {
+    readonly id: number;
+    readonly generation: number;
+    readonly sentenceId: SentenceId;
+    readonly label: string;
+    readonly content: string;
+    readonly sentenceIds: readonly SentenceId[];
+    readonly retention?: ChunkRetention;
+    readonly importance?: ChunkImportance;
+    readonly wordsCount: number;
+    readonly weight: number;
+}
+type CreateChunkRecord = Omit<ChunkRecord, "id">;
+interface ReadingEdgeRecord {
+    readonly fromId: number;
+    readonly toId: number;
+    readonly strength?: string;
+    readonly weight: number;
+}
+interface MentionRecord {
+    readonly id: string;
+    readonly chapterId: number;
+    readonly fragmentId?: number;
+    readonly sentenceIndex?: number;
+    readonly rangeStart: number;
+    readonly rangeEnd: number;
+    readonly surface: string;
+    readonly qid: string;
+    readonly confidence?: number;
+    readonly note?: string;
+}
+interface MentionLinkRecord {
+    readonly id: string;
+    readonly sourceMentionId: string;
+    readonly targetMentionId: string;
+    readonly predicate: string;
+    readonly evidenceSentenceIds: readonly SentenceId[];
+    readonly confidence?: number;
+    readonly note?: string;
+}
+interface SnakeRecord {
+    readonly id: number;
+    readonly serialId: number;
+    readonly groupId: number;
+    readonly localSnakeId: number;
+    readonly size: number;
+    readonly firstLabel: string;
+    readonly lastLabel: string;
+    readonly wordsCount: number;
+    readonly weight: number;
+}
+interface CreateSnakeRecord {
+    readonly serialId: number;
+    readonly groupId: number;
+    readonly localSnakeId: number;
+    readonly size: number;
+    readonly firstLabel: string;
+    readonly lastLabel: string;
+    readonly wordsCount?: number;
+    readonly weight?: number;
+}
+interface SnakeChunkRecord {
+    readonly snakeId: number;
+    readonly chunkId: number;
+    readonly position: number;
+}
+interface SnakeEdgeRecord {
+    readonly fromSnakeId: number;
+    readonly toSnakeId: number;
+    readonly weight: number;
+}
+interface SentenceGroupRecord {
+    readonly serialId: number;
+    readonly groupId: number;
+    readonly startSentenceIndex: number;
+    readonly endSentenceIndex: number;
+    readonly fragmentId?: number;
+}
+type FragmentGroupRecord = SentenceGroupRecord;
+declare const enum ObjectMetadataKind {
+    Archive = 1,
+    Chapter = 2,
+    Chunk = 3,
+    Entity = 4,
+    Triple = 5,
+    Object = 6
+}
+interface ObjectMetadataTarget {
+    readonly kind: ObjectMetadataKind;
+    readonly objectPath: string;
+    readonly chapterId?: number;
+    readonly chunkId?: number;
+    readonly entityQid?: string;
+    readonly tripleSubjectQid?: string;
+    readonly triplePredicate?: string;
+    readonly tripleObjectQid?: string;
+}
+
+export { ChunkImportance, type ChunkRecord, ChunkRetention, type CreateChunkRecord, type CreateSnakeRecord, type FragmentGroupRecord, type FragmentRecord, type GraphBuildParameterRecord, type MentionLinkRecord, type MentionRecord, ObjectMetadataKind, type ObjectMetadataTarget, type ReadingEdgeRecord, Sentence, type SentenceGroupRecord, type SentenceId, type SentenceRecord, type SerialRecord, type SnakeChunkRecord, type SnakeEdgeRecord, type SnakeRecord, expectChunkImportance, expectChunkRetention, isChunkImportance, isChunkRetention };

package/dist/document/types.js

@@ -0,0 +1,69 @@
+import {
+  createEnumValueAsserter,
+  createEnumValueGuard
+} from "../utils/enum.js";
+var ChunkRetention = /* @__PURE__ */ ((ChunkRetention2) => {
+  ChunkRetention2["Verbatim"] = "verbatim";
+  ChunkRetention2["Detailed"] = "detailed";
+  ChunkRetention2["Focused"] = "focused";
+  ChunkRetention2["Relevant"] = "relevant";
+  return ChunkRetention2;
+})(ChunkRetention || {});
+const isChunkRetention = createEnumValueGuard(ChunkRetention);
+const expectChunkRetention = createEnumValueAsserter(
+  ChunkRetention,
+  "chunk retention"
+);
+var ChunkImportance = /* @__PURE__ */ ((ChunkImportance2) => {
+  ChunkImportance2["Critical"] = "critical";
+  ChunkImportance2["Important"] = "important";
+  ChunkImportance2["Helpful"] = "helpful";
+  return ChunkImportance2;
+})(ChunkImportance || {});
+const isChunkImportance = createEnumValueGuard(ChunkImportance);
+const expectChunkImportance = createEnumValueAsserter(
+  ChunkImportance,
+  "chunk importance"
+);
+class Sentence {
+  #text;
+  rawText;
+  wordsCount;
+  constructor(rawText, wordsCount) {
+    this.rawText = rawText;
+    this.wordsCount = wordsCount;
+  }
+  get text() {
+    this.#text ??= normalizeSentenceText(this.rawText);
+    return this.#text;
+  }
+  toJSON() {
+    return {
+      text: this.text,
+      wordsCount: this.wordsCount
+    };
+  }
+}
+function normalizeSentenceText(text) {
+  return text.replace(/\s+/gu, " ").trim();
+}
+var ObjectMetadataKind = /* @__PURE__ */ ((ObjectMetadataKind2) => {
+  ObjectMetadataKind2[ObjectMetadataKind2["Archive"] = 1] = "Archive";
+  ObjectMetadataKind2[ObjectMetadataKind2["Chapter"] = 2] = "Chapter";
+  ObjectMetadataKind2[ObjectMetadataKind2["Chunk"] = 3] = "Chunk";
+  ObjectMetadataKind2[ObjectMetadataKind2["Entity"] = 4] = "Entity";
+  ObjectMetadataKind2[ObjectMetadataKind2["Triple"] = 5] = "Triple";
+  ObjectMetadataKind2[ObjectMetadataKind2["Object"] = 6] = "Object";
+  return ObjectMetadataKind2;
+})(ObjectMetadataKind || {});
+export {
+  ChunkImportance,
+  ChunkRetention,
+  ObjectMetadataKind,
+  Sentence,
+  expectChunkImportance,
+  expectChunkRetention,
+  isChunkImportance,
+  isChunkRetention
+};
+//# sourceMappingURL=types.js.map

package/dist/document/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/document/types.ts"],"sourcesContent":["import {\n  createEnumValueAsserter,\n  createEnumValueGuard,\n} from \"../utils/enum.js\";\n\nexport enum ChunkRetention {\n  Verbatim = \"verbatim\",\n  Detailed = \"detailed\",\n  Focused = \"focused\",\n  Relevant = \"relevant\",\n}\n\nexport const isChunkRetention = createEnumValueGuard(ChunkRetention);\nexport const expectChunkRetention = createEnumValueAsserter(\n  ChunkRetention,\n  \"chunk retention\",\n);\n\nexport enum ChunkImportance {\n  Critical = \"critical\",\n  Important = \"important\",\n  Helpful = \"helpful\",\n}\n\nexport const isChunkImportance = createEnumValueGuard(ChunkImportance);\nexport const expectChunkImportance = createEnumValueAsserter(\n  ChunkImportance,\n  \"chunk importance\",\n);\n\nexport type SentenceId = readonly [chapterId: number, sentenceIndex: number];\n\nexport interface SentenceRecord {\n  readonly text: string;\n  readonly wordsCount: number;\n}\n\nexport class Sentence implements SentenceRecord {\n  #text: string | undefined;\n  public readonly rawText: string;\n  public readonly wordsCount: number;\n\n  public constructor(rawText: string, wordsCount: number) {\n    this.rawText = rawText;\n    this.wordsCount = wordsCount;\n  }\n\n  public get text(): string {\n    this.#text ??= normalizeSentenceText(this.rawText);\n\n    return this.#text;\n  }\n\n  public toJSON(): SentenceRecord {\n    return {\n      text: this.text,\n      wordsCount: this.wordsCount,\n    };\n  }\n}\n\nexport interface FragmentRecord {\n  readonly serialId: number;\n  readonly fragmentId: number;\n  readonly summary: string;\n  readonly sentences: readonly SentenceRecord[];\n}\n\nexport interface SerialRecord {\n  readonly documentOrder: number;\n  readonly knowledgeGraphReady: boolean;\n  readonly knowledgeGraphParameterHash?: string;\n  readonly id: number;\n  readonly revision: number;\n  readonly topologyParameterHash?: string;\n  readonly topologyReady: boolean;\n}\n\nexport interface GraphBuildParameterRecord {\n  readonly createdAt: string;\n  readonly hash: string;\n  readonly language?: string;\n  readonly prompt: string;\n}\n\nexport interface ChunkRecord {\n  readonly id: number;\n  readonly generation: number;\n  readonly sentenceId: SentenceId;\n  readonly label: string;\n  readonly content: string;\n  readonly sentenceIds: readonly SentenceId[];\n  readonly retention?: ChunkRetention;\n  readonly importance?: ChunkImportance;\n  readonly wordsCount: number;\n  readonly weight: number;\n}\n\nexport type CreateChunkRecord = Omit<ChunkRecord, \"id\">;\n\nexport interface ReadingEdgeRecord {\n  readonly fromId: number;\n  readonly toId: number;\n  readonly strength?: string;\n  readonly weight: number;\n}\n\nexport interface MentionRecord {\n  readonly id: string;\n  readonly chapterId: number;\n  readonly fragmentId?: number;\n  readonly sentenceIndex?: number;\n  readonly rangeStart: number;\n  readonly rangeEnd: number;\n  readonly surface: string;\n  readonly qid: string;\n  readonly confidence?: number;\n  readonly note?: string;\n}\n\nexport interface MentionLinkRecord {\n  readonly id: string;\n  readonly sourceMentionId: string;\n  readonly targetMentionId: string;\n  readonly predicate: string;\n  readonly evidenceSentenceIds: readonly SentenceId[];\n  readonly confidence?: number;\n  readonly note?: string;\n}\n\nexport interface SnakeRecord {\n  readonly id: number;\n  readonly serialId: number;\n  readonly groupId: number;\n  readonly localSnakeId: number;\n  readonly size: number;\n  readonly firstLabel: string;\n  readonly lastLabel: string;\n  readonly wordsCount: number;\n  readonly weight: number;\n}\n\nexport interface CreateSnakeRecord {\n  readonly serialId: number;\n  readonly groupId: number;\n  readonly localSnakeId: number;\n  readonly size: number;\n  readonly firstLabel: string;\n  readonly lastLabel: string;\n  readonly wordsCount?: number;\n  readonly weight?: number;\n}\n\nexport interface SnakeChunkRecord {\n  readonly snakeId: number;\n  readonly chunkId: number;\n  readonly position: number;\n}\n\nexport interface SnakeEdgeRecord {\n  readonly fromSnakeId: number;\n  readonly toSnakeId: number;\n  readonly weight: number;\n}\n\nfunction normalizeSentenceText(text: string): string {\n  return text.replace(/\\s+/gu, \" \").trim();\n}\n\nexport interface SentenceGroupRecord {\n  readonly serialId: number;\n  readonly groupId: number;\n  readonly startSentenceIndex: number;\n  readonly endSentenceIndex: number;\n  readonly fragmentId?: number;\n}\n\nexport type FragmentGroupRecord = SentenceGroupRecord;\n\nexport const enum ObjectMetadataKind {\n  Archive = 1,\n  Chapter = 2,\n  Chunk = 3,\n  Entity = 4,\n  Triple = 5,\n  Object = 6,\n}\n\nexport interface ObjectMetadataTarget {\n  readonly kind: ObjectMetadataKind;\n  readonly objectPath: string;\n  readonly chapterId?: number;\n  readonly chunkId?: number;\n  readonly entityQid?: string;\n  readonly tripleSubjectQid?: string;\n  readonly triplePredicate?: string;\n  readonly tripleObjectQid?: string;\n}\n"],"mappings":"AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;AAEA,IAAK,iBAAL,kBAAKA,oBAAL;AACL,EAAAA,gBAAA,cAAW;AACX,EAAAA,gBAAA,cAAW;AACX,EAAAA,gBAAA,aAAU;AACV,EAAAA,gBAAA,cAAW;AAJD,SAAAA;AAAA,GAAA;AAOL,MAAM,mBAAmB,qBAAqB,cAAc;AAC5D,MAAM,uBAAuB;AAAA,EAClC;AAAA,EACA;AACF;AAEO,IAAK,kBAAL,kBAAKC,qBAAL;AACL,EAAAA,iBAAA,cAAW;AACX,EAAAA,iBAAA,eAAY;AACZ,EAAAA,iBAAA,aAAU;AAHA,SAAAA;AAAA,GAAA;AAML,MAAM,oBAAoB,qBAAqB,eAAe;AAC9D,MAAM,wBAAwB;AAAA,EACnC;AAAA,EACA;AACF;AASO,MAAM,SAAmC;AAAA,EAC9C;AAAA,EACgB;AAAA,EACA;AAAA,EAET,YAAY,SAAiB,YAAoB;AACtD,SAAK,UAAU;AACf,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAW,OAAe;AACxB,SAAK,UAAU,sBAAsB,KAAK,OAAO;AAEjD,WAAO,KAAK;AAAA,EACd;AAAA,EAEO,SAAyB;AAC9B,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,YAAY,KAAK;AAAA,IACnB;AAAA,EACF;AACF;AA0GA,SAAS,sBAAsB,MAAsB;AACnD,SAAO,KAAK,QAAQ,SAAS,GAAG,EAAE,KAAK;AACzC;AAYO,IAAW,qBAAX,kBAAWC,wBAAX;AACL,EAAAA,wCAAA,aAAU,KAAV;AACA,EAAAA,wCAAA,aAAU,KAAV;AACA,EAAAA,wCAAA,WAAQ,KAAR;AACA,EAAAA,wCAAA,YAAS,KAAT;AACA,EAAAA,wCAAA,YAAS,KAAT;AACA,EAAAA,wCAAA,YAAS,KAAT;AANgB,SAAAA;AAAA,GAAA;","names":["ChunkRetention","ChunkImportance","ObjectMetadataKind"]}

package/dist/editor/clue.d.ts

@@ -0,0 +1,27 @@
+import { ChunkRecord } from '../document/types.js';
+import { ReadonlyDocument } from '../document/document.js';
+import '../source/meta.js';
+import 'zod';
+import '../source/toc.js';
+import '../source/types.js';
+import '../document/database.js';
+import 'sqlite3';
+import '../document/text-streams.js';
+import '../document/stores.js';
+
+interface Clue {
+    readonly clueId: number;
+    readonly chunks: readonly ChunkRecord[];
+    readonly isMerged: boolean;
+    readonly label: string;
+    readonly sourceSnakeIds: readonly number[];
+    readonly weight: number;
+}
+declare function extractCluesFromDocument(input: {
+    groupId: number;
+    maxClues: number;
+    serialId: number;
+    document: ReadonlyDocument;
+}): Promise<readonly Clue[]>;
+
+export { type Clue, extractCluesFromDocument };

package/dist/editor/clue.js

@@ -0,0 +1,158 @@
+async function extractCluesFromDocument(input) {
+  const snakeIds = await input.document.snakes.listIdsByGroup(
+    input.serialId,
+    input.groupId
+  );
+  const snakes = await Promise.all(
+    snakeIds.map(
+      async (snakeId) => await input.document.snakes.getById(snakeId)
+    )
+  );
+  const clues = await Promise.all(
+    snakes.filter((snake) => snake !== void 0).map(async (snake) => await convertSnakeToClue(snake, input.document))
+  );
+  if (clues.length <= input.maxClues) {
+    return normalizeClueWeights(clues);
+  }
+  let currentClues = [...clues];
+  let mergedClueId = -1;
+  while (currentClues.length > input.maxClues) {
+    currentClues.sort(compareClueByWeightDescending);
+    const cutoffRank = Math.floor(input.maxClues * 0.75);
+    const candidates = currentClues.slice(cutoffRank);
+    if (candidates.length < 2) {
+      break;
+    }
+    const pair = findBestMergePair(candidates);
+    const [leftClue, rightClue] = pair ?? [...currentClues].sort(compareClueByWeightAscending).slice(0, 2);
+    if (leftClue === void 0 || rightClue === void 0) {
+      break;
+    }
+    currentClues = currentClues.filter(
+      (clue) => clue !== leftClue && clue !== rightClue
+    );
+    currentClues.push(mergeClues(leftClue, rightClue, mergedClueId));
+    mergedClueId -= 1;
+  }
+  return normalizeClueWeights(currentClues);
+}
+function calculateSentenceReduction(leftClue, rightClue) {
+  const leftSentenceIndexes = collectSentenceIndexes(leftClue.chunks);
+  const rightSentenceIndexes = collectSentenceIndexes(rightClue.chunks);
+  const mergedSentenceIndexes = /* @__PURE__ */ Object.create(null);
+  let mergedCount = 0;
+  for (const sentenceIndex of leftSentenceIndexes) {
+    mergedSentenceIndexes[String(sentenceIndex)] = true;
+    mergedCount += 1;
+  }
+  for (const sentenceIndex of rightSentenceIndexes) {
+    const sentenceKey = String(sentenceIndex);
+    if (mergedSentenceIndexes[sentenceKey] === true) {
+      continue;
+    }
+    mergedSentenceIndexes[sentenceKey] = true;
+    mergedCount += 1;
+  }
+  return leftSentenceIndexes.length + rightSentenceIndexes.length - mergedCount;
+}
+function collectSentenceIndexes(chunks) {
+  const sentenceIndexRecord = /* @__PURE__ */ Object.create(null);
+  const sentenceIndexes = [];
+  for (const chunk of chunks) {
+    for (const sentenceId of chunk.sentenceIds) {
+      const sentenceIndex = sentenceId[1];
+      const sentenceKey = String(sentenceIndex);
+      if (sentenceIndexRecord[sentenceKey] === true) {
+        continue;
+      }
+      sentenceIndexRecord[sentenceKey] = true;
+      sentenceIndexes.push(sentenceIndex);
+    }
+  }
+  sentenceIndexes.sort(compareNumber);
+  return sentenceIndexes;
+}
+function compareClueByWeightAscending(left, right) {
+  return left.weight - right.weight;
+}
+function compareClueByWeightDescending(left, right) {
+  return right.weight - left.weight;
+}
+function compareChunkBySentenceId(left, right) {
+  const [leftSerialId, leftSentenceIndex] = left.sentenceId;
+  const [rightSerialId, rightSentenceIndex] = right.sentenceId;
+  if (leftSerialId !== rightSerialId) {
+    return leftSerialId - rightSerialId;
+  }
+  return leftSentenceIndex - rightSentenceIndex;
+}
+function compareNumber(left, right) {
+  return left - right;
+}
+async function convertSnakeToClue(snake, document) {
+  const chunkIds = await document.snakeChunks.listChunkIds(snake.id);
+  const chunks = (await Promise.all(
+    chunkIds.map(async (chunkId) => await document.chunks.getById(chunkId))
+  )).filter((chunk) => chunk !== void 0);
+  return {
+    clueId: snake.id,
+    chunks,
+    isMerged: false,
+    label: `${snake.firstLabel} -> ${snake.lastLabel}`,
+    sourceSnakeIds: [snake.id],
+    weight: snake.weight
+  };
+}
+function findBestMergePair(clues) {
+  let bestPair;
+  let bestReduction = -1;
+  for (let leftIndex = 0; leftIndex < clues.length; leftIndex += 1) {
+    const leftClue = clues[leftIndex];
+    if (leftClue === void 0) {
+      continue;
+    }
+    for (let rightIndex = leftIndex + 1; rightIndex < clues.length; rightIndex += 1) {
+      const rightClue = clues[rightIndex];
+      if (rightClue === void 0) {
+        continue;
+      }
+      const reduction = calculateSentenceReduction(leftClue, rightClue);
+      if (reduction <= bestReduction) {
+        continue;
+      }
+      bestReduction = reduction;
+      bestPair = [leftClue, rightClue];
+    }
+  }
+  return bestPair;
+}
+function mergeClues(leftClue, rightClue, clueId) {
+  const sourceSnakeIds = [
+    ...leftClue.sourceSnakeIds,
+    ...rightClue.sourceSnakeIds
+  ].sort(compareNumber);
+  const chunks = [...leftClue.chunks, ...rightClue.chunks].sort(
+    compareChunkBySentenceId
+  );
+  return {
+    clueId,
+    chunks,
+    isMerged: true,
+    label: `Merged minor clues (${sourceSnakeIds.length})`,
+    sourceSnakeIds,
+    weight: leftClue.weight + rightClue.weight
+  };
+}
+function normalizeClueWeights(clues) {
+  const totalWeight = clues.reduce((sum, clue) => sum + clue.weight, 0);
+  const normalizedClues = clues.map((clue) => ({
+    ...clue,
+    weight: totalWeight === 0 ? 0 : clue.weight / totalWeight
+  }));
+  normalizedClues.sort(compareClueByWeightDescending);
+  return normalizedClues;
+}
+export {
+  extractCluesFromDocument
+};
+//# sourceMappingURL=clue.js.map

package/dist/editor/clue.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/editor/clue.ts"],"sourcesContent":["import type {\n  ChunkRecord,\n  ReadonlyDocument,\n  SnakeRecord,\n} from \"../document/index.js\";\n\ninterface Clue {\n  readonly clueId: number;\n  readonly chunks: readonly ChunkRecord[];\n  readonly isMerged: boolean;\n  readonly label: string;\n  readonly sourceSnakeIds: readonly number[];\n  readonly weight: number;\n}\n\nexport async function extractCluesFromDocument(input: {\n  groupId: number;\n  maxClues: number;\n  serialId: number;\n  document: ReadonlyDocument;\n}): Promise<readonly Clue[]> {\n  const snakeIds = await input.document.snakes.listIdsByGroup(\n    input.serialId,\n    input.groupId,\n  );\n  const snakes = await Promise.all(\n    snakeIds.map(\n      async (snakeId) => await input.document.snakes.getById(snakeId),\n    ),\n  );\n  const clues = await Promise.all(\n    snakes\n      .filter((snake): snake is SnakeRecord => snake !== undefined)\n      .map(async (snake) => await convertSnakeToClue(snake, input.document)),\n  );\n\n  if (clues.length <= input.maxClues) {\n    return normalizeClueWeights(clues);\n  }\n\n  let currentClues = [...clues];\n  let mergedClueId = -1;\n\n  while (currentClues.length > input.maxClues) {\n    currentClues.sort(compareClueByWeightDescending);\n\n    const cutoffRank = Math.floor(input.maxClues * 0.75);\n    const candidates = currentClues.slice(cutoffRank);\n\n    if (candidates.length < 2) {\n      break;\n    }\n\n    const pair = findBestMergePair(candidates);\n    const [leftClue, rightClue] =\n      pair ?? [...currentClues].sort(compareClueByWeightAscending).slice(0, 2);\n\n    if (leftClue === undefined || rightClue === undefined) {\n      break;\n    }\n\n    currentClues = currentClues.filter(\n      (clue) => clue !== leftClue && clue !== rightClue,\n    );\n    currentClues.push(mergeClues(leftClue, rightClue, mergedClueId));\n    mergedClueId -= 1;\n  }\n\n  return normalizeClueWeights(currentClues);\n}\n\nfunction calculateSentenceReduction(leftClue: Clue, rightClue: Clue): number {\n  const leftSentenceIndexes = collectSentenceIndexes(leftClue.chunks);\n  const rightSentenceIndexes = collectSentenceIndexes(rightClue.chunks);\n  const mergedSentenceIndexes = Object.create(null) as Record<string, true>;\n  let mergedCount = 0;\n\n  for (const sentenceIndex of leftSentenceIndexes) {\n    mergedSentenceIndexes[String(sentenceIndex)] = true;\n    mergedCount += 1;\n  }\n\n  for (const sentenceIndex of rightSentenceIndexes) {\n    const sentenceKey = String(sentenceIndex);\n\n    if (mergedSentenceIndexes[sentenceKey] === true) {\n      continue;\n    }\n\n    mergedSentenceIndexes[sentenceKey] = true;\n    mergedCount += 1;\n  }\n\n  return leftSentenceIndexes.length + rightSentenceIndexes.length - mergedCount;\n}\n\nfunction collectSentenceIndexes(chunks: readonly ChunkRecord[]): number[] {\n  const sentenceIndexRecord = Object.create(null) as Record<string, true>;\n  const sentenceIndexes: number[] = [];\n\n  for (const chunk of chunks) {\n    for (const sentenceId of chunk.sentenceIds) {\n      const sentenceIndex = sentenceId[1];\n      const sentenceKey = String(sentenceIndex);\n\n      if (sentenceIndexRecord[sentenceKey] === true) {\n        continue;\n      }\n\n      sentenceIndexRecord[sentenceKey] = true;\n      sentenceIndexes.push(sentenceIndex);\n    }\n  }\n\n  sentenceIndexes.sort(compareNumber);\n\n  return sentenceIndexes;\n}\n\nfunction compareClueByWeightAscending(left: Clue, right: Clue): number {\n  return left.weight - right.weight;\n}\n\nfunction compareClueByWeightDescending(left: Clue, right: Clue): number {\n  return right.weight - left.weight;\n}\n\nfunction compareChunkBySentenceId(\n  left: ChunkRecord,\n  right: ChunkRecord,\n): number {\n  const [leftSerialId, leftSentenceIndex] = left.sentenceId;\n  const [rightSerialId, rightSentenceIndex] = right.sentenceId;\n\n  if (leftSerialId !== rightSerialId) {\n    return leftSerialId - rightSerialId;\n  }\n\n  return leftSentenceIndex - rightSentenceIndex;\n}\n\nfunction compareNumber(left: number, right: number): number {\n  return left - right;\n}\n\nasync function convertSnakeToClue(\n  snake: SnakeRecord,\n  document: ReadonlyDocument,\n): Promise<Clue> {\n  const chunkIds = await document.snakeChunks.listChunkIds(snake.id);\n  const chunks = (\n    await Promise.all(\n      chunkIds.map(async (chunkId) => await document.chunks.getById(chunkId)),\n    )\n  ).filter((chunk): chunk is ChunkRecord => chunk !== undefined);\n\n  return {\n    clueId: snake.id,\n    chunks,\n    isMerged: false,\n    label: `${snake.firstLabel} -> ${snake.lastLabel}`,\n    sourceSnakeIds: [snake.id],\n    weight: snake.weight,\n  };\n}\n\nfunction findBestMergePair(\n  clues: readonly Clue[],\n): readonly [Clue, Clue] | undefined {\n  let bestPair: [Clue, Clue] | undefined;\n  let bestReduction = -1;\n\n  for (let leftIndex = 0; leftIndex < clues.length; leftIndex += 1) {\n    const leftClue = clues[leftIndex];\n\n    if (leftClue === undefined) {\n      continue;\n    }\n\n    for (\n      let rightIndex = leftIndex + 1;\n      rightIndex < clues.length;\n      rightIndex += 1\n    ) {\n      const rightClue = clues[rightIndex];\n\n      if (rightClue === undefined) {\n        continue;\n      }\n\n      const reduction = calculateSentenceReduction(leftClue, rightClue);\n\n      if (reduction <= bestReduction) {\n        continue;\n      }\n\n      bestReduction = reduction;\n      bestPair = [leftClue, rightClue];\n    }\n  }\n\n  return bestPair;\n}\n\nfunction mergeClues(leftClue: Clue, rightClue: Clue, clueId: number): Clue {\n  const sourceSnakeIds = [\n    ...leftClue.sourceSnakeIds,\n    ...rightClue.sourceSnakeIds,\n  ].sort(compareNumber);\n  const chunks = [...leftClue.chunks, ...rightClue.chunks].sort(\n    compareChunkBySentenceId,\n  );\n\n  return {\n    clueId,\n    chunks,\n    isMerged: true,\n    label: `Merged minor clues (${sourceSnakeIds.length})`,\n    sourceSnakeIds,\n    weight: leftClue.weight + rightClue.weight,\n  };\n}\n\nfunction normalizeClueWeights(clues: readonly Clue[]): readonly Clue[] {\n  const totalWeight = clues.reduce((sum, clue) => sum + clue.weight, 0);\n  const normalizedClues = clues.map((clue) => ({\n    ...clue,\n    weight: totalWeight === 0 ? 0 : clue.weight / totalWeight,\n  }));\n\n  normalizedClues.sort(compareClueByWeightDescending);\n\n  return normalizedClues;\n}\n\nexport type { Clue };\n"],"mappings":"AAeA,eAAsB,yBAAyB,OAKlB;AAC3B,QAAM,WAAW,MAAM,MAAM,SAAS,OAAO;AAAA,IAC3C,MAAM;AAAA,IACN,MAAM;AAAA,EACR;AACA,QAAM,SAAS,MAAM,QAAQ;AAAA,IAC3B,SAAS;AAAA,MACP,OAAO,YAAY,MAAM,MAAM,SAAS,OAAO,QAAQ,OAAO;AAAA,IAChE;AAAA,EACF;AACA,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,OACG,OAAO,CAAC,UAAgC,UAAU,MAAS,EAC3D,IAAI,OAAO,UAAU,MAAM,mBAAmB,OAAO,MAAM,QAAQ,CAAC;AAAA,EACzE;AAEA,MAAI,MAAM,UAAU,MAAM,UAAU;AAClC,WAAO,qBAAqB,KAAK;AAAA,EACnC;AAEA,MAAI,eAAe,CAAC,GAAG,KAAK;AAC5B,MAAI,eAAe;AAEnB,SAAO,aAAa,SAAS,MAAM,UAAU;AAC3C,iBAAa,KAAK,6BAA6B;AAE/C,UAAM,aAAa,KAAK,MAAM,MAAM,WAAW,IAAI;AACnD,UAAM,aAAa,aAAa,MAAM,UAAU;AAEhD,QAAI,WAAW,SAAS,GAAG;AACzB;AAAA,IACF;AAEA,UAAM,OAAO,kBAAkB,UAAU;AACzC,UAAM,CAAC,UAAU,SAAS,IACxB,QAAQ,CAAC,GAAG,YAAY,EAAE,KAAK,4BAA4B,EAAE,MAAM,GAAG,CAAC;AAEzE,QAAI,aAAa,UAAa,cAAc,QAAW;AACrD;AAAA,IACF;AAEA,mBAAe,aAAa;AAAA,MAC1B,CAAC,SAAS,SAAS,YAAY,SAAS;AAAA,IAC1C;AACA,iBAAa,KAAK,WAAW,UAAU,WAAW,YAAY,CAAC;AAC/D,oBAAgB;AAAA,EAClB;AAEA,SAAO,qBAAqB,YAAY;AAC1C;AAEA,SAAS,2BAA2B,UAAgB,WAAyB;AAC3E,QAAM,sBAAsB,uBAAuB,SAAS,MAAM;AAClE,QAAM,uBAAuB,uBAAuB,UAAU,MAAM;AACpE,QAAM,wBAAwB,uBAAO,OAAO,IAAI;AAChD,MAAI,cAAc;AAElB,aAAW,iBAAiB,qBAAqB;AAC/C,0BAAsB,OAAO,aAAa,CAAC,IAAI;AAC/C,mBAAe;AAAA,EACjB;AAEA,aAAW,iBAAiB,sBAAsB;AAChD,UAAM,cAAc,OAAO,aAAa;AAExC,QAAI,sBAAsB,WAAW,MAAM,MAAM;AAC/C;AAAA,IACF;AAEA,0BAAsB,WAAW,IAAI;AACrC,mBAAe;AAAA,EACjB;AAEA,SAAO,oBAAoB,SAAS,qBAAqB,SAAS;AACpE;AAEA,SAAS,uBAAuB,QAA0C;AACxE,QAAM,sBAAsB,uBAAO,OAAO,IAAI;AAC9C,QAAM,kBAA4B,CAAC;AAEnC,aAAW,SAAS,QAAQ;AAC1B,eAAW,cAAc,MAAM,aAAa;AAC1C,YAAM,gBAAgB,WAAW,CAAC;AAClC,YAAM,cAAc,OAAO,aAAa;AAExC,UAAI,oBAAoB,WAAW,MAAM,MAAM;AAC7C;AAAA,MACF;AAEA,0BAAoB,WAAW,IAAI;AACnC,sBAAgB,KAAK,aAAa;AAAA,IACpC;AAAA,EACF;AAEA,kBAAgB,KAAK,aAAa;AAElC,SAAO;AACT;AAEA,SAAS,6BAA6B,MAAY,OAAqB;AACrE,SAAO,KAAK,SAAS,MAAM;AAC7B;AAEA,SAAS,8BAA8B,MAAY,OAAqB;AACtE,SAAO,MAAM,SAAS,KAAK;AAC7B;AAEA,SAAS,yBACP,MACA,OACQ;AACR,QAAM,CAAC,cAAc,iBAAiB,IAAI,KAAK;AAC/C,QAAM,CAAC,eAAe,kBAAkB,IAAI,MAAM;AAElD,MAAI,iBAAiB,eAAe;AAClC,WAAO,eAAe;AAAA,EACxB;AAEA,SAAO,oBAAoB;AAC7B;AAEA,SAAS,cAAc,MAAc,OAAuB;AAC1D,SAAO,OAAO;AAChB;AAEA,eAAe,mBACb,OACA,UACe;AACf,QAAM,WAAW,MAAM,SAAS,YAAY,aAAa,MAAM,EAAE;AACjE,QAAM,UACJ,MAAM,QAAQ;AAAA,IACZ,SAAS,IAAI,OAAO,YAAY,MAAM,SAAS,OAAO,QAAQ,OAAO,CAAC;AAAA,EACxE,GACA,OAAO,CAAC,UAAgC,UAAU,MAAS;AAE7D,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd;AAAA,IACA,UAAU;AAAA,IACV,OAAO,GAAG,MAAM,UAAU,OAAO,MAAM,SAAS;AAAA,IAChD,gBAAgB,CAAC,MAAM,EAAE;AAAA,IACzB,QAAQ,MAAM;AAAA,EAChB;AACF;AAEA,SAAS,kBACP,OACmC;AACnC,MAAI;AACJ,MAAI,gBAAgB;AAEpB,WAAS,YAAY,GAAG,YAAY,MAAM,QAAQ,aAAa,GAAG;AAChE,UAAM,WAAW,MAAM,SAAS;AAEhC,QAAI,aAAa,QAAW;AAC1B;AAAA,IACF;AAEA,aACM,aAAa,YAAY,GAC7B,aAAa,MAAM,QACnB,cAAc,GACd;AACA,YAAM,YAAY,MAAM,UAAU;AAElC,UAAI,cAAc,QAAW;AAC3B;AAAA,MACF;AAEA,YAAM,YAAY,2BAA2B,UAAU,SAAS;AAEhE,UAAI,aAAa,eAAe;AAC9B;AAAA,MACF;AAEA,sBAAgB;AAChB,iBAAW,CAAC,UAAU,SAAS;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,WAAW,UAAgB,WAAiB,QAAsB;AACzE,QAAM,iBAAiB;AAAA,IACrB,GAAG,SAAS;AAAA,IACZ,GAAG,UAAU;AAAA,EACf,EAAE,KAAK,aAAa;AACpB,QAAM,SAAS,CAAC,GAAG,SAAS,QAAQ,GAAG,UAAU,MAAM,EAAE;AAAA,IACvD;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,UAAU;AAAA,IACV,OAAO,uBAAuB,eAAe,MAAM;AAAA,IACnD;AAAA,IACA,QAAQ,SAAS,SAAS,UAAU;AAAA,EACtC;AACF;AAEA,SAAS,qBAAqB,OAAyC;AACrE,QAAM,cAAc,MAAM,OAAO,CAAC,KAAK,SAAS,MAAM,KAAK,QAAQ,CAAC;AACpE,QAAM,kBAAkB,MAAM,IAAI,CAAC,UAAU;AAAA,IAC3C,GAAG;AAAA,IACH,QAAQ,gBAAgB,IAAI,IAAI,KAAK,SAAS;AAAA,EAChD,EAAE;AAEF,kBAAgB,KAAK,6BAA6B;AAElD,SAAO;AACT;","names":[]}

package/dist/editor/compressor.d.ts

@@ -0,0 +1,18 @@
+import { LLM } from '../llm/client.js';
+import '../llm/context.js';
+import '../llm/cache.js';
+import '../llm/types.js';
+import 'ai';
+
+declare class CompressionRequester<S extends string> {
+    #private;
+    constructor(llm: LLM<S>, scope: S, compressionRatio: number, userLanguage?: string);
+    request(input: {
+        markedText: string;
+        previousCompressedText?: string;
+        revisionFeedback?: string;
+        targetLength: number;
+    }): Promise<string>;
+}
+
+export { CompressionRequester };

package/dist/editor/compressor.js

@@ -0,0 +1,68 @@
+import { TEXT_COMPRESSOR_PROMPT_TEMPLATE } from "./prompt-templates.js";
+class CompressionRequester {
+  #compressionRatio;
+  #llm;
+  #scope;
+  #userLanguage;
+  constructor(llm, scope, compressionRatio, userLanguage) {
+    this.#compressionRatio = compressionRatio;
+    this.#llm = llm;
+    this.#scope = scope;
+    this.#userLanguage = userLanguage;
+  }
+  async request(input) {
+    const acceptableMin = Math.floor(input.targetLength * 0.85);
+    const acceptableMax = Math.floor(input.targetLength * 1.15);
+    const systemPrompt = this.#llm.loadSystemPrompt(
+      TEXT_COMPRESSOR_PROMPT_TEMPLATE,
+      {
+        acceptable_max: acceptableMax,
+        acceptable_min: acceptableMin,
+        compression_ratio: Math.floor(this.#compressionRatio * 100),
+        original_length: input.markedText.length,
+        target_length: input.targetLength,
+        user_language: this.#userLanguage
+      }
+    );
+    const messages = buildCompressionMessages(
+      {
+        markedText: input.markedText,
+        systemPrompt
+      },
+      input.previousCompressedText,
+      input.revisionFeedback
+    );
+    return (await this.#llm.request(messages, {
+      scope: this.#scope
+    })).trim();
+  }
+}
+function buildCompressionMessages(input, previousCompressedText, revisionFeedback) {
+  const messages = [
+    {
+      content: input.systemPrompt,
+      role: "system"
+    },
+    {
+      content: input.markedText,
+      role: "user"
+    }
+  ];
+  if (previousCompressedText !== void 0 && revisionFeedback !== void 0) {
+    messages.push(
+      {
+        content: previousCompressedText,
+        role: "assistant"
+      },
+      {
+        content: revisionFeedback,
+        role: "user"
+      }
+    );
+  }
+  return messages;
+}
+export {
+  CompressionRequester
+};
+//# sourceMappingURL=compressor.js.map

package/dist/editor/compressor.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/editor/compressor.ts"],"sourcesContent":["import type { LLMessage, LLM } from \"../llm/index.js\";\nimport { TEXT_COMPRESSOR_PROMPT_TEMPLATE } from \"./prompt-templates.js\";\n\nexport class CompressionRequester<S extends string> {\n  readonly #compressionRatio: number;\n  readonly #llm: LLM<S>;\n  readonly #scope: S;\n  readonly #userLanguage: string | undefined;\n\n  public constructor(\n    llm: LLM<S>,\n    scope: S,\n    compressionRatio: number,\n    userLanguage?: string,\n  ) {\n    this.#compressionRatio = compressionRatio;\n    this.#llm = llm;\n    this.#scope = scope;\n    this.#userLanguage = userLanguage;\n  }\n\n  public async request(input: {\n    markedText: string;\n    previousCompressedText?: string;\n    revisionFeedback?: string;\n    targetLength: number;\n  }): Promise<string> {\n    const acceptableMin = Math.floor(input.targetLength * 0.85);\n    const acceptableMax = Math.floor(input.targetLength * 1.15);\n    const systemPrompt = this.#llm.loadSystemPrompt(\n      TEXT_COMPRESSOR_PROMPT_TEMPLATE,\n      {\n        acceptable_max: acceptableMax,\n        acceptable_min: acceptableMin,\n        compression_ratio: Math.floor(this.#compressionRatio * 100),\n        original_length: input.markedText.length,\n        target_length: input.targetLength,\n        user_language: this.#userLanguage,\n      },\n    );\n    const messages = buildCompressionMessages(\n      {\n        markedText: input.markedText,\n        systemPrompt,\n      },\n      input.previousCompressedText,\n      input.revisionFeedback,\n    );\n\n    return (\n      await this.#llm.request(messages, {\n        scope: this.#scope,\n      })\n    ).trim();\n  }\n}\n\nfunction buildCompressionMessages(\n  input: {\n    markedText: string;\n    systemPrompt: string;\n  },\n  previousCompressedText?: string,\n  revisionFeedback?: string,\n): LLMessage[] {\n  const messages: LLMessage[] = [\n    {\n      content: input.systemPrompt,\n      role: \"system\",\n    },\n    {\n      content: input.markedText,\n      role: \"user\",\n    },\n  ];\n\n  if (previousCompressedText !== undefined && revisionFeedback !== undefined) {\n    messages.push(\n      {\n        content: previousCompressedText,\n        role: \"assistant\",\n      },\n      {\n        content: revisionFeedback,\n        role: \"user\",\n      },\n    );\n  }\n\n  return messages;\n}\n"],"mappings":"AACA,SAAS,uCAAuC;AAEzC,MAAM,qBAAuC;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YACL,KACA,OACA,kBACA,cACA;AACA,SAAK,oBAAoB;AACzB,SAAK,OAAO;AACZ,SAAK,SAAS;AACd,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,MAAa,QAAQ,OAKD;AAClB,UAAM,gBAAgB,KAAK,MAAM,MAAM,eAAe,IAAI;AAC1D,UAAM,gBAAgB,KAAK,MAAM,MAAM,eAAe,IAAI;AAC1D,UAAM,eAAe,KAAK,KAAK;AAAA,MAC7B;AAAA,MACA;AAAA,QACE,gBAAgB;AAAA,QAChB,gBAAgB;AAAA,QAChB,mBAAmB,KAAK,MAAM,KAAK,oBAAoB,GAAG;AAAA,QAC1D,iBAAiB,MAAM,WAAW;AAAA,QAClC,eAAe,MAAM;AAAA,QACrB,eAAe,KAAK;AAAA,MACtB;AAAA,IACF;AACA,UAAM,WAAW;AAAA,MACf;AAAA,QACE,YAAY,MAAM;AAAA,QAClB;AAAA,MACF;AAAA,MACA,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAEA,YACE,MAAM,KAAK,KAAK,QAAQ,UAAU;AAAA,MAChC,OAAO,KAAK;AAAA,IACd,CAAC,GACD,KAAK;AAAA,EACT;AACF;AAEA,SAAS,yBACP,OAIA,wBACA,kBACa;AACb,QAAM,WAAwB;AAAA,IAC5B;AAAA,MACE,SAAS,MAAM;AAAA,MACf,MAAM;AAAA,IACR;AAAA,IACA;AAAA,MACE,SAAS,MAAM;AAAA,MACf,MAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,2BAA2B,UAAa,qBAAqB,QAAW;AAC1E,aAAS;AAAA,MACP;AAAA,QACE,SAAS;AAAA,QACT,MAAM;AAAA,MACR;AAAA,MACA;AAAA,QACE,SAAS;AAAA,QACT,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;","names":[]}

package/dist/editor/editor.d.ts

@@ -0,0 +1,39 @@
+import { Language } from '../common/language.js';
+import { LLM } from '../llm/client.js';
+import { ReadonlyDocument } from '../document/document.js';
+import '../llm/context.js';
+import '../llm/cache.js';
+import '../llm/types.js';
+import 'ai';
+import '../source/meta.js';
+import 'zod';
+import '../source/toc.js';
+import '../source/types.js';
+import '../document/database.js';
+import 'sqlite3';
+import '../document/text-streams.js';
+import '../document/types.js';
+import '../document/stores.js';
+
+interface EditorScopes<S extends string> {
+    readonly compress: S;
+    readonly review: S;
+    readonly reviewGuide: S;
+}
+interface EditorOptions<S extends string> {
+    readonly compressionRatio?: number;
+    readonly document?: ReadonlyDocument;
+    readonly groupId: number;
+    readonly llm: LLM<S>;
+    readonly logDirPath?: string;
+    readonly maxClues?: number;
+    readonly maxIterations?: number;
+    readonly scopes: EditorScopes<S>;
+    readonly serialId: number;
+    readonly userLanguage?: Language;
+    /** @deprecated Use `document` instead. */
+    readonly workspace?: ReadonlyDocument;
+}
+declare function compressText<S extends string>(options: EditorOptions<S>): Promise<string>;
+
+export { type EditorOptions, type EditorScopes, compressText };