wiki-graph-core 0.3.1 → 0.4.0

package/dist/facade/knowledge-graph-build.js

@@ -1,897 +0,0 @@
-import { createReadStream, createWriteStream } from "fs";
-import { mkdir, rm } from "fs/promises";
-import { createInterface } from "readline";
-import { join } from "path";
-import { z } from "zod";
-import { LanguageCode, normalizeLanguageCode } from "../common/language.js";
-import {
-  buildWikimatchSurfaceProtectionInput,
-  buildWikimatchWindows,
-  enrichWikimatchCandidates,
-  filterCandidateQidOptions,
-  judgeWikimatchPolicy,
-  judgeWikimatchSurfaceProtection,
-  listCandidateSelectableQids,
-  matchWikispineSentenceCandidates
-} from "../wikimatch/index.js";
-import {
-  buildWikilinkEvidenceWindows,
-  discoverWikilinkRelations
-} from "../wikilink/index.js";
-import { getChapterDetails } from "./chapter.js";
-const mentionRecordSchema = z.object({
-  id: z.string().min(1),
-  chapterId: z.number().int(),
-  sentenceIndex: z.number().int().nonnegative().optional(),
-  rangeStart: z.number().int().nonnegative(),
-  rangeEnd: z.number().int().nonnegative(),
-  surface: z.string().min(1),
-  qid: z.string().regex(/^Q[1-9][0-9]*$/),
-  confidence: z.number().min(0).max(1).optional(),
-  note: z.string().optional()
-});
-const sentenceIdSchema = z.tuple([z.number().int().nonnegative(), z.number().int().nonnegative()]).readonly();
-const mentionLinkRecordSchema = z.object({
-  id: z.string().min(1),
-  sourceMentionId: z.string().min(1),
-  targetMentionId: z.string().min(1),
-  predicate: z.string().min(1),
-  evidenceSentenceIds: z.array(sentenceIdSchema).min(1),
-  confidence: z.number().min(0).max(1).optional(),
-  note: z.string().optional()
-});
-const WIKIMATCH_GROUNDING_DEFAULT_OPTION_BUDGETS = [5, 10, 20, 35];
-const WIKIMATCH_GROUNDING_PRIOR_OPTION_BUDGETS = [3, 5, 10, 20, 35];
-const WIKIMATCH_GROUNDING_MAX_OPTION_BUDGET = 50;
-const WIKIMATCH_GROUNDING_SURFACE_PRIOR_THRESHOLD = 3;
-const WIKIMATCH_SURFACE_PROTECTION_PERCENTILE = 0.1;
-const WIKILINK_EVIDENCE_DISTANCE = 700;
-const WIKILINK_WINDOW_LENGTH = 1800;
-async function generateChapterKnowledgeGraphArtifact(document, chapterId, options) {
-  return await generateChapterKnowledgeGraphArtifactFromSnapshot(
-    chapterId,
-    await snapshotChapterKnowledgeGraphInput(document, chapterId),
-    options
-  );
-}
-async function snapshotChapterKnowledgeGraphInput(document, chapterId) {
-  const details = await getChapterDetails(document, chapterId);
-  if (details.stage === "planned") {
-    throw new Error(
-      `Chapter ${chapterId} is planned. Set source before generating Knowledge Graph.`
-    );
-  }
-  return {
-    details,
-    fragments: await readChapterFragments(document, chapterId)
-  };
-}
-async function generateChapterKnowledgeGraphArtifactFromSnapshot(chapterId, snapshot, options) {
-  if (snapshot.details.chapterId !== chapterId) {
-    throw new Error(
-      `Knowledge Graph snapshot belongs to chapter ${snapshot.details.chapterId}, not chapter ${chapterId}.`
-    );
-  }
-  if (snapshot.details.stage === "planned") {
-    throw new Error(
-      `Chapter ${chapterId} is planned. Set source before generating Knowledge Graph.`
-    );
-  }
-  const fragments = snapshot.fragments;
-  const text = joinFragmentText(fragments);
-  const sentences = createWikimatchSentences(fragments);
-  await options.progressTracker?.updatePhase({
-    done: 0,
-    phase: "matching",
-    phaseDetail: "text",
-    total: text.length,
-    unit: "char"
-  });
-  const rawCandidates = await matchWikispineSentenceCandidates({
-    includeDisambiguation: true,
-    onProgress: async (progress) => {
-      await options.progressTracker?.updatePhase({
-        done: progress.coveredRangeEnd,
-        force: false,
-        phase: "matching",
-        phaseDetail: "text",
-        total: text.length,
-        unit: "char"
-      });
-    },
-    ...options.wikispine ?? {},
-    sentences
-  });
-  await options.progressTracker?.throwIfStopped();
-  await options.progressTracker?.updatePhase({
-    done: text.length,
-    phase: "matching",
-    phaseDetail: "text",
-    total: text.length,
-    unit: "char"
-  });
-  const screenedCandidates = await screenCandidates({
-    candidates: rawCandidates,
-    policyPrompt: options.policyPrompt,
-    ...options.progressTracker === void 0 ? {} : { progressTracker: options.progressTracker },
-    request: options.request,
-    text
-  });
-  await options.progressTracker?.throwIfStopped();
-  const qidCount = countUniqueQids(screenedCandidates);
-  await options.progressTracker?.updatePhase({
-    done: 0,
-    phase: "enrichment",
-    total: qidCount,
-    unit: "qid"
-  });
-  const enrichedCandidates = await enrichWikimatchCandidates(
-    screenedCandidates,
-    {
-      ...options.progressTracker === void 0 ? {} : {
-        progress: createEnrichmentProgressReporter(options.progressTracker)
-      },
-      ...options.resolverOptions === void 0 ? {} : { resolverOptions: options.resolverOptions }
-    }
-  );
-  await options.progressTracker?.throwIfStopped();
-  await options.progressTracker?.updatePhase({
-    done: qidCount,
-    phase: "enrichment",
-    total: qidCount,
-    unit: "qid"
-  });
-  const mentions = await judgeCandidates({
-    candidates: enrichedCandidates,
-    chapterId,
-    fragments,
-    policyPrompt: options.policyPrompt,
-    ...options.progressTracker === void 0 ? {} : { progressTracker: options.progressTracker },
-    request: options.request,
-    text
-  });
-  await options.progressTracker?.throwIfStopped();
-  const mentionLinks = await discoverMentionLinks({
-    fragments,
-    mentions,
-    ...options.progressTracker === void 0 ? {} : { progressTracker: options.progressTracker },
-    request: options.request
-  });
-  await options.progressTracker?.throwIfStopped();
-  const artifact = await buildChapterKnowledgeGraphArtifact(chapterId, {
-    mentionLinks,
-    mentions,
-    parameter: createKnowledgeGraphParameterInput(options),
-    workspacePath: options.workspacePath
-  });
-  return artifact;
-}
-async function buildChapterKnowledgeGraphArtifact(chapterId, options) {
-  const workspacePath = join(
-    options.workspacePath,
-    "knowledge-graph",
-    `chapter-${chapterId}`
-  );
-  const mentionsPath = join(workspacePath, "mentions.jsonl");
-  const mentionLinksPath = join(workspacePath, "mention-links.jsonl");
-  await rm(workspacePath, { force: true, recursive: true });
-  await mkdir(workspacePath, { recursive: true });
-  await writeJsonl(mentionsPath, options.mentions, parseMentionRecord);
-  await writeJsonl(
-    mentionLinksPath,
-    options.mentionLinks,
-    parseMentionLinkRecord
-  );
-  return {
-    chapterId,
-    mentionLinksPath,
-    mentionsPath,
-    parameter: options.parameter ?? {
-      language: LanguageCode.Chinese,
-      prompt: ""
-    },
-    workspacePath
-  };
-}
-function createKnowledgeGraphParameterInput(options) {
-  return {
-    language: normalizeLanguageCode(options.resolverOptions?.language) ?? LanguageCode.Chinese,
-    prompt: options.policyPrompt
-  };
-}
-async function screenCandidates(input) {
-  if (input.candidates.length === 0) {
-    return [];
-  }
-  const protectionInput = buildWikimatchSurfaceProtectionInput({
-    candidates: input.candidates,
-    percentile: WIKIMATCH_SURFACE_PROTECTION_PERCENTILE,
-    text: input.text
-  });
-  await input.progressTracker?.updatePhase({
-    done: 0,
-    phase: "screening",
-    phaseDetail: `${protectionInput.suspiciousSurfaces.length} high-frequency surfaces`,
-    total: 1,
-    unit: "window"
-  });
-  await input.progressTracker?.throwIfStopped();
-  const protection = await judgeWikimatchSurfaceProtection({
-    policyPrompt: input.policyPrompt,
-    request: input.request,
-    suspiciousSurfaces: protectionInput.suspiciousSurfaces
-  });
-  await input.progressTracker?.throwIfStopped();
-  const protectedSurfaces = new Set(
-    protection.protectedSurfaces.map((surface) => surface.text)
-  );
-  const allowedCandidateKeys = new Set(
-    protectionInput.candidates.map(createCandidateRangeKey)
-  );
-  await input.progressTracker?.updatePhase({
-    done: 1,
-    phase: "screening",
-    phaseDetail: `${protectedSurfaces.size} protected surfaces`,
-    total: 1,
-    unit: "window"
-  });
-  return protectionInput.suppressedCandidates.filter(
-    (candidate) => allowedCandidateKeys.has(createCandidateRangeKey(candidate)) || protectedSurfaces.has(candidate.surface)
-  );
-}
-function createCandidateRangeKey(candidate) {
-  return `${candidate.range.start}\0${candidate.range.end}\0${candidate.surface}`;
-}
-async function judgeCandidates(input) {
-  const mentions = [];
-  const sentenceLocations = buildSentenceLocations(input.fragments);
-  const acceptedMentions = await groundWikimatchCandidates({
-    candidates: input.candidates,
-    policyPrompt: input.policyPrompt,
-    ...input.progressTracker === void 0 ? {} : { progressTracker: input.progressTracker },
-    request: input.request,
-    text: input.text
-  });
-  let mentionIndex = 1;
-  for (const mention of acceptedMentions) {
-    const location = locateMention(sentenceLocations, mention.range.start);
-    mentions.push(
-      toMentionRecord(input.chapterId, mention, location, mentionIndex)
-    );
-    mentionIndex += 1;
-  }
-  return mentions;
-}
-async function groundWikimatchCandidates(input) {
-  const mentions = [];
-  const candidatePages = createGroundingCandidatePages(input.candidates);
-  let completedWindows = 0;
-  let totalWindows = 0;
-  let activeCandidates = candidatePages.nextPage();
-  while (activeCandidates.length > 0) {
-    const windows = buildWikimatchWindows({
-      candidates: activeCandidates,
-      contextWords: 220,
-      optionBudget: WIKIMATCH_GROUNDING_MAX_OPTION_BUDGET,
-      text: input.text
-    });
-    totalWindows += windows.length;
-    if (windows.length === 0) {
-      break;
-    }
-    await input.progressTracker?.updatePhase({
-      done: completedWindows,
-      phase: "grounding",
-      total: totalWindows,
-      unit: "window"
-    });
-    const results = await mapLazyGuaranteedRequests(
-      input.request,
-      windows,
-      async (window, request) => {
-        try {
-          await input.progressTracker?.throwIfStopped();
-          return await judgeWikimatchPolicy({
-            candidates: window.candidates,
-            policyPrompt: input.policyPrompt,
-            request,
-            window
-          });
-        } finally {
-          completedWindows += 1;
-          await input.progressTracker?.updatePhase({
-            done: completedWindows,
-            phase: "grounding",
-            total: totalWindows,
-            unit: "window"
-          });
-        }
-      }
-    );
-    const continuedCandidateIds = /* @__PURE__ */ new Set();
-    for (const result of results) {
-      for (const mention of result.mentions) {
-        mentions.push(mention);
-        candidatePages.accept(mention);
-      }
-      for (const update of result.policyUpdates) {
-        candidatePages.close(update.candidateId, update.decision);
-      }
-      for (const continuation of result.continuations) {
-        for (const candidateId of continuation.candidateIds) {
-          continuedCandidateIds.add(candidateId);
-          candidatePages.continue(candidateId);
-        }
-      }
-    }
-    activeCandidates = candidatePages.nextPage(continuedCandidateIds);
-  }
-  await input.progressTracker?.updatePhase({
-    done: completedWindows,
-    phase: "grounding",
-    phaseDetail: formatGroundingEfficiency(
-      candidatePages.getStats(),
-      mentions.length
-    ),
-    total: totalWindows,
-    unit: "window"
-  });
-  return mentions;
-}
-function createGroundingCandidatePages(candidates) {
-  const candidatesById = new Map(
-    candidates.map((candidate) => [candidate.id, candidate])
-  );
-  const shownQidsByCandidateId = new Map(
-    candidates.map((candidate) => [candidate.id, /* @__PURE__ */ new Set()])
-  );
-  const pageIndexesByCandidateId = new Map(
-    candidates.map((candidate) => [candidate.id, 0])
-  );
-  const closedCandidateIds = /* @__PURE__ */ new Set();
-  const recallCounts = /* @__PURE__ */ new Map();
-  const surfaceStats = /* @__PURE__ */ new Map();
-  const stats = {
-    candidatePageCount: 0,
-    qidAppearanceCount: 0
-  };
-  return {
-    accept(mention) {
-      closedCandidateIds.add(mention.candidateId);
-      getSurfaceStats(surfaceStats, mention.surface).recallCount += 1;
-      recallCounts.set(
-        createSurfaceQidKey(mention.surface, mention.qid),
-        (recallCounts.get(createSurfaceQidKey(mention.surface, mention.qid)) ?? 0) + 1
-      );
-    },
-    close(candidateId, decision) {
-      closedCandidateIds.add(candidateId);
-      const candidate = candidatesById.get(candidateId);
-      if (candidate !== void 0 && (decision === "skip_this_time" || decision === "never_recall")) {
-        getSurfaceStats(surfaceStats, candidate.surface).rejectCount += 1;
-      }
-    },
-    continue(candidateId) {
-      const candidate = candidatesById.get(candidateId);
-      if (candidate !== void 0) {
-        getSurfaceStats(surfaceStats, candidate.surface).continueCount += 1;
-      }
-    },
-    getStats() {
-      return { ...stats };
-    },
-    nextPage(continuedCandidateIds) {
-      const pageCandidates = [];
-      const candidateIds = continuedCandidateIds === void 0 ? candidates.map((candidate) => candidate.id) : [...continuedCandidateIds];
-      for (const candidateId of candidateIds) {
-        if (closedCandidateIds.has(candidateId)) {
-          continue;
-        }
-        const candidate = candidatesById.get(candidateId);
-        const shownQids = shownQidsByCandidateId.get(candidateId);
-        if (candidate === void 0 || shownQids === void 0) {
-          continue;
-        }
-        const sortedCandidate = sortCandidateOptionsByRecall(
-          candidate,
-          recallCounts
-        );
-        const selectableQids = listCandidateSelectableQids(sortedCandidate);
-        const pageIndex = pageIndexesByCandidateId.get(candidateId) ?? 0;
-        const optionBudget = getGroundingCandidateOptionBudget(
-          getSurfaceStats(surfaceStats, candidate.surface),
-          pageIndex
-        );
-        const selectedQids = selectableQids.filter((qid) => !shownQids.has(qid)).slice(0, optionBudget);
-        if (selectedQids.length === 0) {
-          closedCandidateIds.add(candidateId);
-          continue;
-        }
-        for (const qid of selectedQids) {
-          shownQids.add(qid);
-        }
-        const hasMoreOptions = selectableQids.some(
-          (qid) => !shownQids.has(qid)
-        );
-        const pageCandidate = filterCandidateQidOptions(
-          sortedCandidate,
-          new Set(selectedQids)
-        );
-        if (!hasMoreOptions) {
-          closedCandidateIds.add(candidateId);
-        }
-        getSurfaceStats(surfaceStats, candidate.surface).seenCount += 1;
-        pageIndexesByCandidateId.set(candidateId, pageIndex + 1);
-        stats.candidatePageCount += 1;
-        stats.qidAppearanceCount += selectedQids.length;
-        pageCandidates.push({
-          ...pageCandidate,
-          ...hasMoreOptions ? { hasMoreOptions: true } : {}
-        });
-      }
-      return pageCandidates;
-    }
-  };
-}
-function getSurfaceStats(surfaceStats, surface) {
-  const existing = surfaceStats.get(surface);
-  if (existing !== void 0) {
-    return existing;
-  }
-  const created = {
-    continueCount: 0,
-    recallCount: 0,
-    rejectCount: 0,
-    seenCount: 0
-  };
-  surfaceStats.set(surface, created);
-  return created;
-}
-function getGroundingCandidateOptionBudget(stats, pageIndex) {
-  const budgets = hasStrongGroundingSurfacePrior(stats) ? WIKIMATCH_GROUNDING_PRIOR_OPTION_BUDGETS : WIKIMATCH_GROUNDING_DEFAULT_OPTION_BUDGETS;
-  return budgets[Math.min(pageIndex, budgets.length - 1)];
-}
-function hasStrongGroundingSurfacePrior(stats) {
-  if (stats.seenCount < WIKIMATCH_GROUNDING_SURFACE_PRIOR_THRESHOLD) {
-    return false;
-  }
-  return stats.recallCount >= 2 && stats.continueCount === 0 || stats.rejectCount >= 2 && stats.recallCount === 0 || stats.continueCount >= WIKIMATCH_GROUNDING_SURFACE_PRIOR_THRESHOLD;
-}
-function formatGroundingEfficiency(stats, mentionCount) {
-  const qidPerMention = mentionCount === 0 ? "n/a" : (stats.qidAppearanceCount / mentionCount).toFixed(1);
-  return `efficiency qid/mention=${qidPerMention} qids=${stats.qidAppearanceCount} mentions=${mentionCount} pages=${stats.candidatePageCount}`;
-}
-function sortCandidateOptionsByRecall(candidate, recallCounts) {
-  return {
-    ...candidate,
-    qidOptions: [...candidate.qidOptions].sort((left, right) => {
-      return getOptionRecallScore(candidate.surface, right, recallCounts) - getOptionRecallScore(candidate.surface, left, recallCounts);
-    })
-  };
-}
-function getOptionRecallScore(surface, option, recallCounts) {
-  const directScore = recallCounts.get(createSurfaceQidKey(surface, option.qid)) ?? 0;
-  const disambiguationScore = option.disambiguation?.linkedQids.reduce(
-    (total, item) => total + (recallCounts.get(createSurfaceQidKey(surface, item.qid)) ?? 0),
-    0
-  ) ?? 0;
-  const profileScore = option.disambiguation?.profile?.meanings.reduce(
-    (total, item) => total + (recallCounts.get(createSurfaceQidKey(surface, item.qid)) ?? 0),
-    0
-  ) ?? 0;
-  return directScore + disambiguationScore + profileScore;
-}
-function createSurfaceQidKey(surface, qid) {
-  return `${surface}\0${qid}`;
-}
-function createEnrichmentProgressReporter(progressTracker) {
-  return async (event) => {
-    await progressTracker.throwIfStopped();
-    const phase = formatEnrichmentProgressPhase(event);
-    await progressTracker.updatePhase({
-      done: event.done,
-      ...phase.detail === void 0 ? {} : { phaseDetail: phase.detail },
-      phase: "enrichment",
-      total: event.total,
-      unit: phase.unit
-    });
-  };
-}
-function formatEnrichmentProgressPhase(event) {
-  switch (event.detail) {
-    case "disambiguation-page":
-      return {
-        detail: "disambiguation",
-        unit: "page"
-      };
-    case "entity":
-      return {
-        detail: "entity",
-        unit: "record"
-      };
-    case "linked-page":
-      return {
-        detail: "linked",
-        unit: "page"
-      };
-    case "page":
-      return {
-        detail: "page",
-        unit: "page"
-      };
-    case "qid":
-      return {
-        unit: "qid"
-      };
-  }
-}
-async function mapLazyGuaranteedRequests(request, items, operation) {
-  const lazy = request.lazy;
-  if (lazy !== void 0) {
-    return await Promise.all(
-      items.map(
-        async (item) => await lazy(async (request2) => await operation(item, request2))
-      )
-    );
-  }
-  const results = [];
-  for (const item of items) {
-    results.push(await operation(item, request));
-  }
-  return results;
-}
-async function discoverMentionLinks(input) {
-  const fragmentWindows = buildMentionLinkWindows(
-    input.fragments,
-    input.mentions
-  );
-  let completedWindows = 0;
-  await input.progressTracker?.updatePhase({
-    done: 0,
-    phase: "relation-discovery",
-    total: fragmentWindows.length,
-    unit: "window"
-  });
-  const discoveredLinks = (await mapLazyGuaranteedRequests(
-    input.request,
-    fragmentWindows,
-    async (item, request) => {
-      try {
-        await input.progressTracker?.throwIfStopped();
-        return await discoverWikilinkRelations({
-          chapterId: item.fragment.serialId,
-          fragmentId: item.fragment.fragmentId,
-          request,
-          sentences: item.fragment.sentences,
-          window: item.window
-        });
-      } finally {
-        completedWindows += 1;
-        await input.progressTracker?.updatePhase({
-          done: completedWindows,
-          phase: "relation-discovery",
-          total: fragmentWindows.length,
-          unit: "window"
-        });
-      }
-    }
-  )).flat();
-  return discoveredLinks.map((link, index) => ({
-    ...link.confidence === void 0 ? {} : { confidence: link.confidence },
-    evidenceSentenceIds: link.evidenceSentenceIds,
-    id: `l${input.fragments[0]?.serialId ?? 0}-${index + 1}`,
-    predicate: link.predicate,
-    sourceMentionId: link.sourceMentionId,
-    targetMentionId: link.targetMentionId
-  }));
-}
-function buildMentionLinkWindows(fragments, mentions) {
-  return fragments.flatMap((fragment) => {
-    const startSentenceIndex = fragment.fragmentId;
-    const endSentenceIndex = startSentenceIndex + fragment.sentences.length - 1;
-    const fragmentMentions = toWikilinkMentions(
-      fragment.sentences,
-      mentions.filter(
-        (mention) => mention.sentenceIndex !== void 0 && mention.sentenceIndex >= startSentenceIndex && mention.sentenceIndex <= endSentenceIndex
-      ),
-      startSentenceIndex
-    );
-    const windows = buildWikilinkEvidenceWindows({
-      maxEvidenceDistance: WIKILINK_EVIDENCE_DISTANCE,
-      mentions: fragmentMentions,
-      text: joinSentences(fragment.sentences),
-      windowLength: WIKILINK_WINDOW_LENGTH
-    });
-    return windows.map((window) => ({
-      fragment,
-      window
-    }));
-  });
-}
-function toWikilinkMentions(sentences, mentions, startSentenceIndex) {
-  const sentenceOffsets = buildFragmentSentenceOffsets(sentences);
-  return mentions.flatMap((mention) => {
-    if (mention.sentenceIndex === void 0) {
-      return [];
-    }
-    const sentenceOffset = sentenceOffsets[mention.sentenceIndex - startSentenceIndex];
-    if (sentenceOffset === void 0) {
-      return [];
-    }
-    return [
-      {
-        id: mention.id,
-        qid: mention.qid,
-        range: {
-          end: sentenceOffset.start + mention.rangeEnd,
-          start: sentenceOffset.start + mention.rangeStart
-        },
-        surface: mention.surface
-      }
-    ];
-  });
-}
-function buildFragmentSentenceOffsets(sentences) {
-  const offsets = [];
-  let cursor = 0;
-  for (const sentence of sentences) {
-    const start = cursor;
-    const end = start + sentence.text.length;
-    offsets.push({ end, start });
-    cursor = end + 1;
-  }
-  return offsets;
-}
-function joinSentences(sentences) {
-  return sentences.map((sentence) => sentence.text).join(" ");
-}
-function countUniqueQids(candidates) {
-  return new Set(
-    candidates.flatMap(
-      (candidate) => candidate.qidOptions.map((option) => option.qid)
-    )
-  ).size;
-}
-function toMentionRecord(chapterId, mention, location, index) {
-  return {
-    chapterId,
-    ...mention.confidence === void 0 ? {} : { confidence: mention.confidence },
-    id: `m${chapterId}-${index}`,
-    ...mention.note === void 0 ? {} : { note: mention.note },
-    qid: mention.qid,
-    rangeEnd: location.rangeStart + mention.surface.length,
-    rangeStart: location.rangeStart,
-    sentenceIndex: location.sentenceIndex,
-    surface: mention.surface
-  };
-}
-function buildSentenceLocations(fragments) {
-  const locations = [];
-  let offset = 0;
-  for (const fragment of fragments) {
-    for (let sentenceIndex = 0; sentenceIndex < fragment.sentences.length; sentenceIndex += 1) {
-      const length = fragment.sentences[sentenceIndex].text.length;
-      locations.push({
-        absoluteStart: offset,
-        length,
-        sentenceIndex: fragment.fragmentId + sentenceIndex
-      });
-      offset += length + 1;
-    }
-  }
-  return locations;
-}
-function locateMention(locations, absoluteOffset) {
-  for (const location of locations) {
-    const rangeEnd = location.absoluteStart + location.length;
-    if (absoluteOffset >= location.absoluteStart && absoluteOffset < rangeEnd) {
-      return {
-        rangeStart: absoluteOffset - location.absoluteStart,
-        sentenceIndex: location.sentenceIndex
-      };
-    }
-  }
-  throw new Error(`Mention offset ${absoluteOffset} is outside chapter text.`);
-}
-async function readChapterFragments(document, chapterId) {
-  const serialFragments = document.getSerialFragments(chapterId);
-  return await Promise.all(
-    (await serialFragments.listFragmentIds()).map(
-      async (fragmentId) => await serialFragments.getFragment(fragmentId)
-    )
-  );
-}
-function joinFragmentText(fragments) {
-  return fragments.flatMap((fragment) => fragment.sentences.map((sentence) => sentence.text)).join(" ");
-}
-function createWikimatchSentences(fragments) {
-  const sentences = [];
-  let offset = 0;
-  for (const fragment of fragments) {
-    for (let index = 0; index < fragment.sentences.length; index += 1) {
-      const sentence = fragment.sentences[index];
-      sentences.push({
-        id: `${fragment.serialId}:${fragment.fragmentId + index}`,
-        range: {
-          end: offset + sentence.text.length,
-          start: offset
-        },
-        text: sentence.text
-      });
-      offset += sentence.text.length + 1;
-    }
-  }
-  return sentences;
-}
-async function commitChapterKnowledgeGraphArtifact(document, artifact) {
-  const mentions = await readJsonl(artifact.mentionsPath, parseMentionRecord);
-  const mentionLinks = await readJsonl(
-    artifact.mentionLinksPath,
-    parseMentionLinkRecord
-  );
-  validateChapterKnowledgeGraphArtifact(artifact.chapterId, {
-    mentionLinks,
-    mentions
-  });
-  await document.openSession(async (openedDocument) => {
-    await openedDocument.serials.ensure(artifact.chapterId);
-    const existingLinks = await openedDocument.mentionLinks.listByChapter(
-      artifact.chapterId
-    );
-    if (existingLinks.length > 0 && mentionLinks.length === 0) {
-      throw new Error(
-        `Refusing to replace chapter ${artifact.chapterId} knowledge graph with an artifact that contains no mention links.`
-      );
-    }
-    await openedDocument.mentionLinks.deleteByChapter(artifact.chapterId);
-    await openedDocument.mentions.deleteByChapter(artifact.chapterId);
-    await openedDocument.mentions.saveMany(mentions);
-    await openedDocument.mentionLinks.saveMany(mentionLinks);
-    const parameter = await openedDocument.graphBuildParameters.save(
-      artifact.parameter
-    );
-    await openedDocument.serials.setKnowledgeGraphReady(
-      artifact.chapterId,
-      true,
-      parameter.hash
-    );
-  });
-}
-async function clearChapterKnowledgeGraph(document, chapterId) {
-  await document.openSession(async (openedDocument) => {
-    await openedDocument.mentionLinks.deleteByChapter(chapterId);
-    await openedDocument.mentions.deleteByChapter(chapterId);
-    await openedDocument.serials.setKnowledgeGraphReady(chapterId, false);
-    await openedDocument.graphBuildParameters.deleteUnreferenced();
-  });
-}
-function validateChapterKnowledgeGraphArtifact(chapterId, records) {
-  const mentionIds = /* @__PURE__ */ new Set();
-  for (const mention of records.mentions) {
-    if (mention.chapterId !== chapterId) {
-      throw new Error(
-        `Mention ${mention.id} belongs to chapter ${mention.chapterId}, expected chapter ${chapterId}.`
-      );
-    }
-    if (mention.rangeEnd <= mention.rangeStart) {
-      throw new Error(
-        `Mention ${mention.id} has invalid range [${mention.rangeStart}, ${mention.rangeEnd}).`
-      );
-    }
-    if (mentionIds.has(mention.id)) {
-      throw new Error(`Duplicate mention id ${mention.id}.`);
-    }
-    mentionIds.add(mention.id);
-  }
-  const linkIds = /* @__PURE__ */ new Set();
-  for (const link of records.mentionLinks) {
-    if (linkIds.has(link.id)) {
-      throw new Error(`Duplicate mention link id ${link.id}.`);
-    }
-    if (!mentionIds.has(link.sourceMentionId)) {
-      throw new Error(
-        `Mention link ${link.id} references unknown source mention ${link.sourceMentionId}.`
-      );
-    }
-    if (!mentionIds.has(link.targetMentionId)) {
-      throw new Error(
-        `Mention link ${link.id} references unknown target mention ${link.targetMentionId}.`
-      );
-    }
-    if (link.evidenceSentenceIds.length === 0) {
-      throw new Error(`Mention link ${link.id} has no evidence sentences.`);
-    }
-    for (const sentenceId of link.evidenceSentenceIds) {
-      if (sentenceId[0] !== chapterId) {
-        throw new Error(
-          `Mention link ${link.id} evidence sentence ${formatSentenceId(sentenceId)} is outside chapter ${chapterId}.`
-        );
-      }
-    }
-    linkIds.add(link.id);
-  }
-}
-function formatSentenceId(sentenceId) {
-  return sentenceId.join(":");
-}
-async function writeJsonl(path, records, parseRecord) {
-  const stream = createWriteStream(path, { encoding: "utf8", flags: "wx" });
-  try {
-    for await (const record of records) {
-      stream.write(`${JSON.stringify(parseRecord(record))}
-`);
-    }
-  } finally {
-    await closeWritableStream(stream);
-  }
-}
-async function readJsonl(path, parseRecord) {
-  const records = [];
-  const lines = createInterface({
-    crlfDelay: Infinity,
-    input: createReadStream(path, { encoding: "utf8" })
-  });
-  let lineNumber = 0;
-  for await (const line of lines) {
-    lineNumber += 1;
-    if (line.trim() === "") {
-      continue;
-    }
-    try {
-      records.push(parseRecord(JSON.parse(line)));
-    } catch (error) {
-      throw new Error(`Invalid JSONL record at ${path}:${lineNumber}`, {
-        cause: error
-      });
-    }
-  }
-  return records;
-}
-function parseMentionRecord(record) {
-  const parsed = mentionRecordSchema.parse(record);
-  return {
-    chapterId: parsed.chapterId,
-    ...parsed.confidence === void 0 ? {} : { confidence: parsed.confidence },
-    id: parsed.id,
-    ...parsed.note === void 0 ? {} : { note: parsed.note },
-    qid: parsed.qid,
-    rangeEnd: parsed.rangeEnd,
-    rangeStart: parsed.rangeStart,
-    ...parsed.sentenceIndex === void 0 ? {} : { sentenceIndex: parsed.sentenceIndex },
-    surface: parsed.surface
-  };
-}
-function parseMentionLinkRecord(record) {
-  const parsed = mentionLinkRecordSchema.parse(record);
-  return {
-    ...parsed.confidence === void 0 ? {} : { confidence: parsed.confidence },
-    evidenceSentenceIds: parsed.evidenceSentenceIds,
-    id: parsed.id,
-    ...parsed.note === void 0 ? {} : { note: parsed.note },
-    predicate: parsed.predicate,
-    sourceMentionId: parsed.sourceMentionId,
-    targetMentionId: parsed.targetMentionId
-  };
-}
-async function closeWritableStream(stream) {
-  await new Promise((resolveClose, rejectClose) => {
-    stream.end((error) => {
-      if (error !== void 0 && error !== null) {
-        rejectClose(error);
-        return;
-      }
-      resolveClose();
-    });
-  });
-}
-export {
-  buildChapterKnowledgeGraphArtifact,
-  clearChapterKnowledgeGraph,
-  commitChapterKnowledgeGraphArtifact,
-  createEnrichmentProgressReporter,
-  generateChapterKnowledgeGraphArtifact,
-  generateChapterKnowledgeGraphArtifactFromSnapshot,
-  groundWikimatchCandidates,
-  snapshotChapterKnowledgeGraphInput
-};
-//# sourceMappingURL=knowledge-graph-build.js.map