wiki-graph-core 0.3.1 → 0.4.1

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/core.js

@@ -0,0 +1,77 @@
+import { rm } from "fs/promises";
+import { join } from "path";
+import { Database } from "../../../../../document/database.js";
+import {
+  createDuplicateHalfCanonicalizationPlan,
+  listLegacySourceSerials,
+  readLegacySourceFragments
+} from "./fragments.js";
+import { migrateLegacySentenceReferences } from "./references.js";
+import { migrateLegacySummariesToTextStreams } from "./summaries.js";
+import { writeLegacySourceTextStream } from "./source-text.js";
+async function migrateLegacyTextStorage(workspacePath) {
+  const sourceSerials = await listLegacySourceSerials(workspacePath);
+  const remaps = /* @__PURE__ */ new Map();
+  for (const serialId of sourceSerials) {
+    const fragments = await readLegacySourceFragments(workspacePath, serialId);
+    const plan = createDuplicateHalfCanonicalizationPlan(fragments);
+    const canonicalFragments = plan?.canonicalFragments ?? fragments;
+    const fragmentIdMap = plan?.fragmentIdMap ?? /* @__PURE__ */ new Map();
+    const canonicalById = new Map(
+      canonicalFragments.map((fragment) => [fragment.fragmentId, fragment])
+    );
+    const sentenceMap = /* @__PURE__ */ new Map();
+    const textParts = [];
+    let globalSentenceIndex = 0;
+    for (const fragment of canonicalFragments) {
+      for (let localSentenceIndex = 0; localSentenceIndex < fragment.content.sentences.length; localSentenceIndex += 1) {
+        const sentence = fragment.content.sentences[localSentenceIndex];
+        if (sentence === void 0) {
+          continue;
+        }
+        sentenceMap.set(
+          `${fragment.fragmentId}:${localSentenceIndex}`,
+          globalSentenceIndex
+        );
+        textParts.push(sentence.text);
+        globalSentenceIndex += 1;
+      }
+    }
+    for (const [oldFragmentId, canonicalFragmentId] of fragmentIdMap) {
+      const canonicalFragment = canonicalById.get(canonicalFragmentId);
+      if (canonicalFragment === void 0) {
+        continue;
+      }
+      for (let localSentenceIndex = 0; localSentenceIndex < canonicalFragment.content.sentences.length; localSentenceIndex += 1) {
+        const mapped = sentenceMap.get(
+          `${canonicalFragmentId}:${localSentenceIndex}`
+        );
+        if (mapped !== void 0) {
+          sentenceMap.set(`${oldFragmentId}:${localSentenceIndex}`, mapped);
+        }
+      }
+    }
+    remaps.set(serialId, {
+      get: (fragmentId, sentenceIndex) => sentenceMap.get(`${fragmentId}:${sentenceIndex}`),
+      serialId
+    });
+    const database = await Database.open(join(workspacePath, "database.db"));
+    try {
+      await writeLegacySourceTextStream(database, workspacePath, {
+        fragments: canonicalFragments,
+        serialId,
+        text: textParts.join("")
+      });
+    } finally {
+      await database.close();
+    }
+  }
+  await migrateLegacySentenceReferences(workspacePath, remaps);
+  await migrateLegacySummariesToTextStreams(workspacePath);
+  await rm(join(workspacePath, "fragments"), { force: true, recursive: true });
+  await rm(join(workspacePath, "summaries"), { force: true, recursive: true });
+}
+export {
+  migrateLegacyTextStorage
+};
+//# sourceMappingURL=core.js.map

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/core.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../src/storage/migration/legacy-sdpub/upgrade/text-storage/core.ts"],"sourcesContent":["import { rm } from \"fs/promises\";\nimport { join } from \"path\";\n\nimport { Database } from \"../../../../../document/database.js\";\nimport {\n  createDuplicateHalfCanonicalizationPlan,\n  listLegacySourceSerials,\n  readLegacySourceFragments,\n} from \"./fragments.js\";\nimport { migrateLegacySentenceReferences } from \"./references.js\";\nimport { migrateLegacySummariesToTextStreams } from \"./summaries.js\";\nimport { writeLegacySourceTextStream } from \"./source-text.js\";\nimport type { SentenceIndexRemap } from \"./types.js\";\n\nexport async function migrateLegacyTextStorage(\n  workspacePath: string,\n): Promise<void> {\n  const sourceSerials = await listLegacySourceSerials(workspacePath);\n  const remaps = new Map<number, SentenceIndexRemap>();\n\n  for (const serialId of sourceSerials) {\n    const fragments = await readLegacySourceFragments(workspacePath, serialId);\n    const plan = createDuplicateHalfCanonicalizationPlan(fragments);\n    const canonicalFragments = plan?.canonicalFragments ?? fragments;\n    const fragmentIdMap = plan?.fragmentIdMap ?? new Map<number, number>();\n    const canonicalById = new Map(\n      canonicalFragments.map((fragment) => [fragment.fragmentId, fragment]),\n    );\n    const sentenceMap = new Map<string, number>();\n    const textParts: string[] = [];\n    let globalSentenceIndex = 0;\n\n    for (const fragment of canonicalFragments) {\n      for (\n        let localSentenceIndex = 0;\n        localSentenceIndex < fragment.content.sentences.length;\n        localSentenceIndex += 1\n      ) {\n        const sentence = fragment.content.sentences[localSentenceIndex];\n\n        if (sentence === undefined) {\n          continue;\n        }\n\n        sentenceMap.set(\n          `${fragment.fragmentId}:${localSentenceIndex}`,\n          globalSentenceIndex,\n        );\n        textParts.push(sentence.text);\n        globalSentenceIndex += 1;\n      }\n    }\n\n    for (const [oldFragmentId, canonicalFragmentId] of fragmentIdMap) {\n      const canonicalFragment = canonicalById.get(canonicalFragmentId);\n\n      if (canonicalFragment === undefined) {\n        continue;\n      }\n\n      for (\n        let localSentenceIndex = 0;\n        localSentenceIndex < canonicalFragment.content.sentences.length;\n        localSentenceIndex += 1\n      ) {\n        const mapped = sentenceMap.get(\n          `${canonicalFragmentId}:${localSentenceIndex}`,\n        );\n\n        if (mapped !== undefined) {\n          sentenceMap.set(`${oldFragmentId}:${localSentenceIndex}`, mapped);\n        }\n      }\n    }\n\n    remaps.set(serialId, {\n      get: (fragmentId, sentenceIndex) =>\n        sentenceMap.get(`${fragmentId}:${sentenceIndex}`),\n      serialId,\n    });\n\n    const database = await Database.open(join(workspacePath, \"database.db\"));\n\n    try {\n      await writeLegacySourceTextStream(database, workspacePath, {\n        fragments: canonicalFragments,\n        serialId,\n        text: textParts.join(\"\"),\n      });\n    } finally {\n      await database.close();\n    }\n  }\n\n  await migrateLegacySentenceReferences(workspacePath, remaps);\n  await migrateLegacySummariesToTextStreams(workspacePath);\n  await rm(join(workspacePath, \"fragments\"), { force: true, recursive: true });\n  await rm(join(workspacePath, \"summaries\"), { force: true, recursive: true });\n}\n"],"mappings":"AAAA,SAAS,UAAU;AACnB,SAAS,YAAY;AAErB,SAAS,gBAAgB;AACzB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uCAAuC;AAChD,SAAS,2CAA2C;AACpD,SAAS,mCAAmC;AAG5C,eAAsB,yBACpB,eACe;AACf,QAAM,gBAAgB,MAAM,wBAAwB,aAAa;AACjE,QAAM,SAAS,oBAAI,IAAgC;AAEnD,aAAW,YAAY,eAAe;AACpC,UAAM,YAAY,MAAM,0BAA0B,eAAe,QAAQ;AACzE,UAAM,OAAO,wCAAwC,SAAS;AAC9D,UAAM,qBAAqB,MAAM,sBAAsB;AACvD,UAAM,gBAAgB,MAAM,iBAAiB,oBAAI,IAAoB;AACrE,UAAM,gBAAgB,IAAI;AAAA,MACxB,mBAAmB,IAAI,CAAC,aAAa,CAAC,SAAS,YAAY,QAAQ,CAAC;AAAA,IACtE;AACA,UAAM,cAAc,oBAAI,IAAoB;AAC5C,UAAM,YAAsB,CAAC;AAC7B,QAAI,sBAAsB;AAE1B,eAAW,YAAY,oBAAoB;AACzC,eACM,qBAAqB,GACzB,qBAAqB,SAAS,QAAQ,UAAU,QAChD,sBAAsB,GACtB;AACA,cAAM,WAAW,SAAS,QAAQ,UAAU,kBAAkB;AAE9D,YAAI,aAAa,QAAW;AAC1B;AAAA,QACF;AAEA,oBAAY;AAAA,UACV,GAAG,SAAS,UAAU,IAAI,kBAAkB;AAAA,UAC5C;AAAA,QACF;AACA,kBAAU,KAAK,SAAS,IAAI;AAC5B,+BAAuB;AAAA,MACzB;AAAA,IACF;AAEA,eAAW,CAAC,eAAe,mBAAmB,KAAK,eAAe;AAChE,YAAM,oBAAoB,cAAc,IAAI,mBAAmB;AAE/D,UAAI,sBAAsB,QAAW;AACnC;AAAA,MACF;AAEA,eACM,qBAAqB,GACzB,qBAAqB,kBAAkB,QAAQ,UAAU,QACzD,sBAAsB,GACtB;AACA,cAAM,SAAS,YAAY;AAAA,UACzB,GAAG,mBAAmB,IAAI,kBAAkB;AAAA,QAC9C;AAEA,YAAI,WAAW,QAAW;AACxB,sBAAY,IAAI,GAAG,aAAa,IAAI,kBAAkB,IAAI,MAAM;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,UAAU;AAAA,MACnB,KAAK,CAAC,YAAY,kBAChB,YAAY,IAAI,GAAG,UAAU,IAAI,aAAa,EAAE;AAAA,MAClD;AAAA,IACF,CAAC;AAED,UAAM,WAAW,MAAM,SAAS,KAAK,KAAK,eAAe,aAAa,CAAC;AAEvE,QAAI;AACF,YAAM,4BAA4B,UAAU,eAAe;AAAA,QACzD,WAAW;AAAA,QACX;AAAA,QACA,MAAM,UAAU,KAAK,EAAE;AAAA,MACzB,CAAC;AAAA,IACH,UAAE;AACA,YAAM,SAAS,MAAM;AAAA,IACvB;AAAA,EACF;AAEA,QAAM,gCAAgC,eAAe,MAAM;AAC3D,QAAM,oCAAoC,aAAa;AACvD,QAAM,GAAG,KAAK,eAAe,WAAW,GAAG,EAAE,OAAO,MAAM,WAAW,KAAK,CAAC;AAC3E,QAAM,GAAG,KAAK,eAAe,WAAW,GAAG,EAAE,OAAO,MAAM,WAAW,KAAK,CAAC;AAC7E;","names":[]}

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/fragments.d.ts

@@ -0,0 +1,10 @@
+import { LegacyFragmentRecord } from './types.js';
+
+declare function listLegacySourceSerials(workspacePath: string): Promise<readonly number[]>;
+declare function readLegacySourceFragments(workspacePath: string, serialId: number): Promise<readonly LegacyFragmentRecord[]>;
+declare function createDuplicateHalfCanonicalizationPlan(fragments: readonly LegacyFragmentRecord[]): {
+    readonly canonicalFragments: readonly LegacyFragmentRecord[];
+    readonly fragmentIdMap: ReadonlyMap<number, number>;
+} | undefined;
+
+export { createDuplicateHalfCanonicalizationPlan, listLegacySourceSerials, readLegacySourceFragments };

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/fragments.js

@@ -0,0 +1,123 @@
+import { readFile, readdir } from "fs/promises";
+import { join } from "path";
+import { isNodeError } from "../../../../../utils/node-error.js";
+async function listLegacySourceSerials(workspacePath) {
+  const fragmentsDirectory = join(workspacePath, "fragments");
+  try {
+    const entries = await readdir(fragmentsDirectory, { withFileTypes: true });
+    const serialIds = [];
+    for (const entry of entries) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      const match = /^serial-(\d+)$/u.exec(entry.name);
+      if (match !== null) {
+        serialIds.push(Number(match[1]));
+      }
+    }
+    return serialIds.sort((left, right) => left - right);
+  } catch (error) {
+    if (isNodeError(error) && error.code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+}
+async function readLegacySourceFragments(workspacePath, serialId) {
+  const serialDirectory = join(
+    workspacePath,
+    "fragments",
+    `serial-${serialId}`
+  );
+  const entries = await readdir(serialDirectory, { withFileTypes: true });
+  const records = [];
+  for (const entry of entries) {
+    if (!entry.isFile()) {
+      continue;
+    }
+    const match = /^fragment_(\d+)\.json$/u.exec(entry.name);
+    if (match === null) {
+      continue;
+    }
+    const path = join(serialDirectory, entry.name);
+    const content = parseLegacyFragmentFile(await readFile(path, "utf8"));
+    records.push({
+      content,
+      fragmentId: Number(match[1]),
+      path,
+      signature: createLegacyFragmentSignature(content)
+    });
+  }
+  return records.sort((left, right) => left.fragmentId - right.fragmentId);
+}
+function parseLegacyFragmentFile(content) {
+  const parsed = JSON.parse(content);
+  if (typeof parsed !== "object" || parsed === null) {
+    throw new TypeError("Legacy fragment file must contain sentences.");
+  }
+  const rawFragment = parsed;
+  if (!Array.isArray(rawFragment.sentences)) {
+    throw new TypeError("Legacy fragment file must contain sentences.");
+  }
+  const sentences = rawFragment.sentences.map((sentence) => {
+    if (typeof sentence !== "object" || sentence === null || !("text" in sentence) || typeof sentence.text !== "string") {
+      throw new TypeError("Legacy fragment sentence must contain text.");
+    }
+    const rawSentence = sentence;
+    const text = rawSentence.text;
+    const rawWordsCount = rawSentence.wordsCount;
+    const wordsCount = typeof rawWordsCount === "number" ? rawWordsCount : countWords(text);
+    return {
+      text,
+      wordsCount
+    };
+  });
+  const summary = typeof rawFragment.summary === "string" ? rawFragment.summary : "";
+  return { sentences, summary };
+}
+function countWords(text) {
+  const trimmed = text.trim();
+  return trimmed === "" ? 0 : trimmed.split(/\s+/u).length;
+}
+function createLegacyFragmentSignature(fragment) {
+  return JSON.stringify(fragment.sentences.map((sentence) => sentence.text));
+}
+function createDuplicateHalfCanonicalizationPlan(fragments) {
+  if (fragments.length < 2 || fragments.length % 2 !== 0) {
+    return void 0;
+  }
+  const halfLength = fragments.length / 2;
+  const leftHalf = fragments.slice(0, halfLength);
+  const rightHalf = fragments.slice(halfLength);
+  for (let index = 0; index < halfLength; index += 1) {
+    if (leftHalf[index]?.signature !== rightHalf[index]?.signature) {
+      return void 0;
+    }
+  }
+  const preferRightHalf = rightHalf.some(
+    (fragment) => fragment.content.summary.trim() !== ""
+  );
+  const sourceFragments = preferRightHalf ? rightHalf : leftHalf;
+  const fragmentIdMap = /* @__PURE__ */ new Map();
+  const canonicalFragments = sourceFragments.map((fragment, index) => {
+    const leftFragment = leftHalf[index];
+    const rightFragment = rightHalf[index];
+    if (leftFragment !== void 0) {
+      fragmentIdMap.set(leftFragment.fragmentId, index);
+    }
+    if (rightFragment !== void 0) {
+      fragmentIdMap.set(rightFragment.fragmentId, index);
+    }
+    return {
+      ...fragment,
+      fragmentId: index
+    };
+  });
+  return { canonicalFragments, fragmentIdMap };
+}
+export {
+  createDuplicateHalfCanonicalizationPlan,
+  listLegacySourceSerials,
+  readLegacySourceFragments
+};
+//# sourceMappingURL=fragments.js.map

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/fragments.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../src/storage/migration/legacy-sdpub/upgrade/text-storage/fragments.ts"],"sourcesContent":["import { readFile, readdir } from \"fs/promises\";\nimport { join } from \"path\";\n\nimport { isNodeError } from \"../../../../../utils/node-error.js\";\nimport type { LegacyFragmentFile, LegacyFragmentRecord } from \"./types.js\";\n\nexport async function listLegacySourceSerials(\n  workspacePath: string,\n): Promise<readonly number[]> {\n  const fragmentsDirectory = join(workspacePath, \"fragments\");\n\n  try {\n    const entries = await readdir(fragmentsDirectory, { withFileTypes: true });\n    const serialIds: number[] = [];\n\n    for (const entry of entries) {\n      if (!entry.isDirectory()) {\n        continue;\n      }\n\n      const match = /^serial-(\\d+)$/u.exec(entry.name);\n\n      if (match !== null) {\n        serialIds.push(Number(match[1]));\n      }\n    }\n\n    return serialIds.sort((left, right) => left - right);\n  } catch (error) {\n    if (isNodeError(error) && error.code === \"ENOENT\") {\n      return [];\n    }\n\n    throw error;\n  }\n}\n\nexport async function readLegacySourceFragments(\n  workspacePath: string,\n  serialId: number,\n): Promise<readonly LegacyFragmentRecord[]> {\n  const serialDirectory = join(\n    workspacePath,\n    \"fragments\",\n    `serial-${serialId}`,\n  );\n  const entries = await readdir(serialDirectory, { withFileTypes: true });\n  const records: LegacyFragmentRecord[] = [];\n\n  for (const entry of entries) {\n    if (!entry.isFile()) {\n      continue;\n    }\n\n    const match = /^fragment_(\\d+)\\.json$/u.exec(entry.name);\n\n    if (match === null) {\n      continue;\n    }\n\n    const path = join(serialDirectory, entry.name);\n    const content = parseLegacyFragmentFile(await readFile(path, \"utf8\"));\n\n    records.push({\n      content,\n      fragmentId: Number(match[1]),\n      path,\n      signature: createLegacyFragmentSignature(content),\n    });\n  }\n\n  return records.sort((left, right) => left.fragmentId - right.fragmentId);\n}\n\nfunction parseLegacyFragmentFile(content: string): LegacyFragmentFile {\n  const parsed = JSON.parse(content) as unknown;\n\n  if (typeof parsed !== \"object\" || parsed === null) {\n    throw new TypeError(\"Legacy fragment file must contain sentences.\");\n  }\n\n  const rawFragment = parsed as Record<string, unknown>;\n\n  if (!Array.isArray(rawFragment.sentences)) {\n    throw new TypeError(\"Legacy fragment file must contain sentences.\");\n  }\n\n  const sentences = rawFragment.sentences.map((sentence) => {\n    if (\n      typeof sentence !== \"object\" ||\n      sentence === null ||\n      !(\"text\" in sentence) ||\n      typeof (sentence as Record<string, unknown>).text !== \"string\"\n    ) {\n      throw new TypeError(\"Legacy fragment sentence must contain text.\");\n    }\n\n    const rawSentence = sentence as Record<string, unknown>;\n    const text = rawSentence.text as string;\n    const rawWordsCount = rawSentence.wordsCount;\n    const wordsCount =\n      typeof rawWordsCount === \"number\" ? rawWordsCount : countWords(text);\n\n    return {\n      text,\n      wordsCount,\n    };\n  });\n  const summary =\n    typeof rawFragment.summary === \"string\" ? rawFragment.summary : \"\";\n\n  return { sentences, summary };\n}\n\nfunction countWords(text: string): number {\n  const trimmed = text.trim();\n\n  return trimmed === \"\" ? 0 : trimmed.split(/\\s+/u).length;\n}\n\nfunction createLegacyFragmentSignature(fragment: LegacyFragmentFile): string {\n  return JSON.stringify(fragment.sentences.map((sentence) => sentence.text));\n}\n\nexport function createDuplicateHalfCanonicalizationPlan(\n  fragments: readonly LegacyFragmentRecord[],\n):\n  | {\n      readonly canonicalFragments: readonly LegacyFragmentRecord[];\n      readonly fragmentIdMap: ReadonlyMap<number, number>;\n    }\n  | undefined {\n  if (fragments.length < 2 || fragments.length % 2 !== 0) {\n    return undefined;\n  }\n\n  const halfLength = fragments.length / 2;\n  const leftHalf = fragments.slice(0, halfLength);\n  const rightHalf = fragments.slice(halfLength);\n\n  for (let index = 0; index < halfLength; index += 1) {\n    if (leftHalf[index]?.signature !== rightHalf[index]?.signature) {\n      return undefined;\n    }\n  }\n\n  const preferRightHalf = rightHalf.some(\n    (fragment) => fragment.content.summary.trim() !== \"\",\n  );\n  const sourceFragments = preferRightHalf ? rightHalf : leftHalf;\n  const fragmentIdMap = new Map<number, number>();\n  const canonicalFragments = sourceFragments.map((fragment, index) => {\n    const leftFragment = leftHalf[index];\n    const rightFragment = rightHalf[index];\n\n    if (leftFragment !== undefined) {\n      fragmentIdMap.set(leftFragment.fragmentId, index);\n    }\n    if (rightFragment !== undefined) {\n      fragmentIdMap.set(rightFragment.fragmentId, index);\n    }\n\n    return {\n      ...fragment,\n      fragmentId: index,\n    };\n  });\n\n  return { canonicalFragments, fragmentIdMap };\n}\n"],"mappings":"AAAA,SAAS,UAAU,eAAe;AAClC,SAAS,YAAY;AAErB,SAAS,mBAAmB;AAG5B,eAAsB,wBACpB,eAC4B;AAC5B,QAAM,qBAAqB,KAAK,eAAe,WAAW;AAE1D,MAAI;AACF,UAAM,UAAU,MAAM,QAAQ,oBAAoB,EAAE,eAAe,KAAK,CAAC;AACzE,UAAM,YAAsB,CAAC;AAE7B,eAAW,SAAS,SAAS;AAC3B,UAAI,CAAC,MAAM,YAAY,GAAG;AACxB;AAAA,MACF;AAEA,YAAM,QAAQ,kBAAkB,KAAK,MAAM,IAAI;AAE/C,UAAI,UAAU,MAAM;AAClB,kBAAU,KAAK,OAAO,MAAM,CAAC,CAAC,CAAC;AAAA,MACjC;AAAA,IACF;AAEA,WAAO,UAAU,KAAK,CAAC,MAAM,UAAU,OAAO,KAAK;AAAA,EACrD,SAAS,OAAO;AACd,QAAI,YAAY,KAAK,KAAK,MAAM,SAAS,UAAU;AACjD,aAAO,CAAC;AAAA,IACV;AAEA,UAAM;AAAA,EACR;AACF;AAEA,eAAsB,0BACpB,eACA,UAC0C;AAC1C,QAAM,kBAAkB;AAAA,IACtB;AAAA,IACA;AAAA,IACA,UAAU,QAAQ;AAAA,EACpB;AACA,QAAM,UAAU,MAAM,QAAQ,iBAAiB,EAAE,eAAe,KAAK,CAAC;AACtE,QAAM,UAAkC,CAAC;AAEzC,aAAW,SAAS,SAAS;AAC3B,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB;AAAA,IACF;AAEA,UAAM,QAAQ,0BAA0B,KAAK,MAAM,IAAI;AAEvD,QAAI,UAAU,MAAM;AAClB;AAAA,IACF;AAEA,UAAM,OAAO,KAAK,iBAAiB,MAAM,IAAI;AAC7C,UAAM,UAAU,wBAAwB,MAAM,SAAS,MAAM,MAAM,CAAC;AAEpE,YAAQ,KAAK;AAAA,MACX;AAAA,MACA,YAAY,OAAO,MAAM,CAAC,CAAC;AAAA,MAC3B;AAAA,MACA,WAAW,8BAA8B,OAAO;AAAA,IAClD,CAAC;AAAA,EACH;AAEA,SAAO,QAAQ,KAAK,CAAC,MAAM,UAAU,KAAK,aAAa,MAAM,UAAU;AACzE;AAEA,SAAS,wBAAwB,SAAqC;AACpE,QAAM,SAAS,KAAK,MAAM,OAAO;AAEjC,MAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AACjD,UAAM,IAAI,UAAU,8CAA8C;AAAA,EACpE;AAEA,QAAM,cAAc;AAEpB,MAAI,CAAC,MAAM,QAAQ,YAAY,SAAS,GAAG;AACzC,UAAM,IAAI,UAAU,8CAA8C;AAAA,EACpE;AAEA,QAAM,YAAY,YAAY,UAAU,IAAI,CAAC,aAAa;AACxD,QACE,OAAO,aAAa,YACpB,aAAa,QACb,EAAE,UAAU,aACZ,OAAQ,SAAqC,SAAS,UACtD;AACA,YAAM,IAAI,UAAU,6CAA6C;AAAA,IACnE;AAEA,UAAM,cAAc;AACpB,UAAM,OAAO,YAAY;AACzB,UAAM,gBAAgB,YAAY;AAClC,UAAM,aACJ,OAAO,kBAAkB,WAAW,gBAAgB,WAAW,IAAI;AAErE,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AACD,QAAM,UACJ,OAAO,YAAY,YAAY,WAAW,YAAY,UAAU;AAElE,SAAO,EAAE,WAAW,QAAQ;AAC9B;AAEA,SAAS,WAAW,MAAsB;AACxC,QAAM,UAAU,KAAK,KAAK;AAE1B,SAAO,YAAY,KAAK,IAAI,QAAQ,MAAM,MAAM,EAAE;AACpD;AAEA,SAAS,8BAA8B,UAAsC;AAC3E,SAAO,KAAK,UAAU,SAAS,UAAU,IAAI,CAAC,aAAa,SAAS,IAAI,CAAC;AAC3E;AAEO,SAAS,wCACd,WAMY;AACZ,MAAI,UAAU,SAAS,KAAK,UAAU,SAAS,MAAM,GAAG;AACtD,WAAO;AAAA,EACT;AAEA,QAAM,aAAa,UAAU,SAAS;AACtC,QAAM,WAAW,UAAU,MAAM,GAAG,UAAU;AAC9C,QAAM,YAAY,UAAU,MAAM,UAAU;AAE5C,WAAS,QAAQ,GAAG,QAAQ,YAAY,SAAS,GAAG;AAClD,QAAI,SAAS,KAAK,GAAG,cAAc,UAAU,KAAK,GAAG,WAAW;AAC9D,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,kBAAkB,UAAU;AAAA,IAChC,CAAC,aAAa,SAAS,QAAQ,QAAQ,KAAK,MAAM;AAAA,EACpD;AACA,QAAM,kBAAkB,kBAAkB,YAAY;AACtD,QAAM,gBAAgB,oBAAI,IAAoB;AAC9C,QAAM,qBAAqB,gBAAgB,IAAI,CAAC,UAAU,UAAU;AAClE,UAAM,eAAe,SAAS,KAAK;AACnC,UAAM,gBAAgB,UAAU,KAAK;AAErC,QAAI,iBAAiB,QAAW;AAC9B,oBAAc,IAAI,aAAa,YAAY,KAAK;AAAA,IAClD;AACA,QAAI,kBAAkB,QAAW;AAC/B,oBAAc,IAAI,cAAc,YAAY,KAAK;AAAA,IACnD;AAEA,WAAO;AAAA,MACL,GAAG;AAAA,MACH,YAAY;AAAA,IACd;AAAA,EACF,CAAC;AAED,SAAO,EAAE,oBAAoB,cAAc;AAC7C;","names":[]}

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/index.d.ts

@@ -0,0 +1 @@
+export { migrateLegacyTextStorage } from './core.js';

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/index.js

@@ -0,0 +1,2 @@
+export * from "./core.js";
+//# sourceMappingURL=index.js.map

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../src/storage/migration/legacy-sdpub/upgrade/text-storage/index.ts"],"sourcesContent":["export * from \"./core.js\";\n"],"mappings":"AAAA,cAAc;","names":[]}

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/references.d.ts

@@ -0,0 +1,5 @@
+import { SentenceIndexRemap } from './types.js';
+
+declare function migrateLegacySentenceReferences(workspacePath: string, remaps: ReadonlyMap<number, SentenceIndexRemap>): Promise<void>;
+
+export { migrateLegacySentenceReferences };

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/references.js

@@ -0,0 +1,305 @@
+import { join } from "path";
+import { Database } from "../../../../../document/database.js";
+import { listTableNames } from "../database.js";
+async function migrateLegacySentenceReferences(workspacePath, remaps) {
+  const database = await Database.open(join(workspacePath, "database.db"));
+  try {
+    const tableNames = await listTableNames(database);
+    await database.transaction(async () => {
+      if (tableNames.has("chunks")) {
+        await migrateLegacyChunks(database, remaps);
+      }
+      if (tableNames.has("chunk_sentences")) {
+        await migrateLegacyChunkSentences(database, remaps);
+      }
+      if (tableNames.has("mentions")) {
+        await migrateLegacyMentions(database, remaps);
+      }
+      if (tableNames.has("mention_link_evidence_sentences")) {
+        await migrateLegacyMentionLinkEvidenceSentences(database, remaps);
+      }
+      if (tableNames.has("fragment_groups")) {
+        await migrateLegacyFragmentGroups(database, remaps);
+      }
+    });
+  } finally {
+    await database.close();
+  }
+}
+async function migrateLegacyChunks(database, remaps) {
+  const rows = await database.queryAll(
+    `
+      SELECT *
+      FROM chunks
+      ORDER BY id
+    `,
+    void 0,
+    (row) => row
+  );
+  await database.run("ALTER TABLE chunks RENAME TO legacy_chunks");
+  await database.run(`
+    CREATE TABLE chunks (
+      id INTEGER PRIMARY KEY,
+      generation INTEGER NOT NULL,
+      serial_id INTEGER NOT NULL,
+      sentence_index INTEGER NOT NULL,
+      label TEXT NOT NULL,
+      content TEXT NOT NULL,
+      retention TEXT,
+      importance TEXT,
+      wordsCount INTEGER NOT NULL DEFAULT 0,
+      weight REAL NOT NULL DEFAULT 0.0
+    )
+  `);
+  for (const row of rows) {
+    const serialId = Number(row.serial_id);
+    const sentenceIndex = remapSentenceIndex(
+      remaps,
+      serialId,
+      Number(row.fragment_id),
+      Number(row.sentence_index)
+    );
+    await database.run(
+      `
+        INSERT INTO chunks (
+          id, generation, serial_id, sentence_index, label, content,
+          retention, importance, wordsCount, weight
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `,
+      [
+        row.id ?? null,
+        row.generation ?? 0,
+        serialId,
+        sentenceIndex,
+        row.label ?? "",
+        row.content ?? "",
+        row.retention ?? null,
+        row.importance ?? null,
+        row.wordsCount ?? 0,
+        row.weight ?? 0
+      ]
+    );
+  }
+  await database.run("DROP TABLE legacy_chunks");
+}
+async function migrateLegacyChunkSentences(database, remaps) {
+  const rows = await database.queryAll(
+    `
+      SELECT chunk_id, serial_id, fragment_id, sentence_index
+      FROM chunk_sentences
+    `,
+    void 0,
+    (row) => row
+  );
+  await database.run("DROP TABLE chunk_sentences");
+  await database.run(`
+    CREATE TABLE chunk_sentences (
+      chunk_id INTEGER NOT NULL,
+      serial_id INTEGER NOT NULL,
+      sentence_index INTEGER NOT NULL,
+      FOREIGN KEY (chunk_id) REFERENCES chunks(id),
+      PRIMARY KEY (chunk_id, serial_id, sentence_index)
+    )
+  `);
+  for (const row of rows) {
+    const serialId = Number(row.serial_id);
+    await database.run(
+      `
+        INSERT OR IGNORE INTO chunk_sentences (
+          chunk_id, serial_id, sentence_index
+        )
+        VALUES (?, ?, ?)
+      `,
+      [
+        getRequiredSqlBindValue(row.chunk_id),
+        serialId,
+        remapSentenceIndex(
+          remaps,
+          serialId,
+          Number(row.fragment_id),
+          Number(row.sentence_index)
+        )
+      ]
+    );
+  }
+}
+async function migrateLegacyMentions(database, remaps) {
+  const rows = await database.queryAll(
+    `
+      SELECT *
+      FROM mentions
+      ORDER BY id
+    `,
+    void 0,
+    (row) => row
+  );
+  await database.run("ALTER TABLE mentions RENAME TO legacy_mentions");
+  await database.run(`
+    CREATE TABLE mentions (
+      id TEXT PRIMARY KEY,
+      chapter_id INTEGER NOT NULL,
+      sentence_index INTEGER,
+      range_start INTEGER NOT NULL,
+      range_end INTEGER NOT NULL,
+      surface TEXT NOT NULL,
+      qid TEXT NOT NULL,
+      confidence REAL,
+      note TEXT,
+      FOREIGN KEY (chapter_id) REFERENCES serials(id)
+    )
+  `);
+  for (const row of rows) {
+    const chapterId = Number(row.chapter_id);
+    const sentenceIndex = row.sentence_index === null || row.sentence_index === void 0 ? null : remapSentenceIndex(
+      remaps,
+      chapterId,
+      Number(row.fragment_id),
+      Number(row.sentence_index)
+    );
+    await database.run(
+      `
+        INSERT OR REPLACE INTO mentions (
+          id, chapter_id, sentence_index, range_start, range_end, surface,
+          qid, confidence, note
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `,
+      [
+        getRequiredSqlBindValue(row.id),
+        chapterId,
+        sentenceIndex,
+        row.range_start ?? 0,
+        row.range_end ?? 0,
+        row.surface ?? "",
+        row.qid ?? "",
+        row.confidence ?? null,
+        row.note ?? null
+      ]
+    );
+  }
+  await database.run("DROP TABLE legacy_mentions");
+}
+async function migrateLegacyMentionLinkEvidenceSentences(database, remaps) {
+  const rows = await database.queryAll(
+    `
+      SELECT link_id, chapter_id, fragment_id, sentence_index
+      FROM mention_link_evidence_sentences
+    `,
+    void 0,
+    (row) => row
+  );
+  await database.run("DROP TABLE mention_link_evidence_sentences");
+  await database.run(`
+    CREATE TABLE mention_link_evidence_sentences (
+      link_id TEXT NOT NULL,
+      chapter_id INTEGER NOT NULL,
+      sentence_index INTEGER NOT NULL,
+      FOREIGN KEY (link_id) REFERENCES mention_links(id),
+      PRIMARY KEY (link_id, chapter_id, sentence_index)
+    )
+  `);
+  for (const row of rows) {
+    const chapterId = Number(row.chapter_id);
+    await database.run(
+      `
+        INSERT OR IGNORE INTO mention_link_evidence_sentences (
+          link_id, chapter_id, sentence_index
+        )
+        VALUES (?, ?, ?)
+      `,
+      [
+        getRequiredSqlBindValue(row.link_id),
+        chapterId,
+        remapSentenceIndex(
+          remaps,
+          chapterId,
+          Number(row.fragment_id),
+          Number(row.sentence_index)
+        )
+      ]
+    );
+  }
+}
+async function migrateLegacyFragmentGroups(database, remaps) {
+  const rows = await database.queryAll(
+    `
+      SELECT serial_id, group_id, fragment_id
+      FROM fragment_groups
+      ORDER BY serial_id, group_id, fragment_id
+    `,
+    void 0,
+    (row) => row
+  );
+  await database.run("DROP TABLE fragment_groups");
+  await database.run(`
+    CREATE TABLE sentence_groups (
+      serial_id INTEGER NOT NULL,
+      group_id INTEGER NOT NULL,
+      start_sentence_index INTEGER NOT NULL,
+      end_sentence_index INTEGER NOT NULL,
+      PRIMARY KEY (serial_id, group_id, start_sentence_index, end_sentence_index)
+    )
+  `);
+  for (const row of rows) {
+    const serialId = Number(row.serial_id);
+    const startSentenceIndex = remapSentenceIndex(
+      remaps,
+      serialId,
+      Number(row.fragment_id),
+      0
+    );
+    const endSentenceIndex = findFragmentEndSentenceIndex(
+      remaps,
+      serialId,
+      Number(row.fragment_id)
+    );
+    await database.run(
+      `
+        INSERT OR IGNORE INTO sentence_groups (
+          serial_id, group_id, start_sentence_index, end_sentence_index
+        )
+        VALUES (?, ?, ?, ?)
+      `,
+      [
+        serialId,
+        getRequiredSqlBindValue(row.group_id),
+        startSentenceIndex,
+        endSentenceIndex
+      ]
+    );
+  }
+}
+function remapSentenceIndex(remaps, serialId, fragmentId, sentenceIndex) {
+  const remap = remaps.get(serialId);
+  const mapped = remap?.get(fragmentId, sentenceIndex);
+  if (mapped === void 0) {
+    throw new Error(
+      `Cannot remap legacy sentence ${serialId}:${fragmentId}:${sentenceIndex}.`
+    );
+  }
+  return mapped;
+}
+function findFragmentEndSentenceIndex(remaps, serialId, fragmentId) {
+  let sentenceIndex = 0;
+  let last;
+  while (true) {
+    const mapped = remaps.get(serialId)?.get(fragmentId, sentenceIndex);
+    if (mapped === void 0) {
+      break;
+    }
+    last = mapped;
+    sentenceIndex += 1;
+  }
+  return last ?? remapSentenceIndex(remaps, serialId, fragmentId, 0);
+}
+function getRequiredSqlBindValue(value) {
+  if (value === void 0) {
+    throw new TypeError("Expected a SQLite bind value.");
+  }
+  return value;
+}
+export {
+  migrateLegacySentenceReferences
+};
+//# sourceMappingURL=references.js.map

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/references.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../../src/storage/migration/legacy-sdpub/upgrade/text-storage/references.ts"],"sourcesContent":["import { join } from \"path\";\n\nimport type { SqlBindValue } from \"../../../../../document/database.js\";\nimport { Database } from \"../../../../../document/database.js\";\nimport { listTableNames } from \"../database.js\";\nimport type { SentenceIndexRemap } from \"./types.js\";\n\nexport async function migrateLegacySentenceReferences(\n  workspacePath: string,\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n): Promise<void> {\n  const database = await Database.open(join(workspacePath, \"database.db\"));\n\n  try {\n    const tableNames = await listTableNames(database);\n\n    await database.transaction(async () => {\n      if (tableNames.has(\"chunks\")) {\n        await migrateLegacyChunks(database, remaps);\n      }\n      if (tableNames.has(\"chunk_sentences\")) {\n        await migrateLegacyChunkSentences(database, remaps);\n      }\n      if (tableNames.has(\"mentions\")) {\n        await migrateLegacyMentions(database, remaps);\n      }\n      if (tableNames.has(\"mention_link_evidence_sentences\")) {\n        await migrateLegacyMentionLinkEvidenceSentences(database, remaps);\n      }\n      if (tableNames.has(\"fragment_groups\")) {\n        await migrateLegacyFragmentGroups(database, remaps);\n      }\n    });\n  } finally {\n    await database.close();\n  }\n}\nasync function migrateLegacyChunks(\n  database: Database,\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n): Promise<void> {\n  const rows = await database.queryAll(\n    `\n      SELECT *\n      FROM chunks\n      ORDER BY id\n    `,\n    undefined,\n    (row) => row,\n  );\n\n  await database.run(\"ALTER TABLE chunks RENAME TO legacy_chunks\");\n  await database.run(`\n    CREATE TABLE chunks (\n      id INTEGER PRIMARY KEY,\n      generation INTEGER NOT NULL,\n      serial_id INTEGER NOT NULL,\n      sentence_index INTEGER NOT NULL,\n      label TEXT NOT NULL,\n      content TEXT NOT NULL,\n      retention TEXT,\n      importance TEXT,\n      wordsCount INTEGER NOT NULL DEFAULT 0,\n      weight REAL NOT NULL DEFAULT 0.0\n    )\n  `);\n\n  for (const row of rows) {\n    const serialId = Number(row.serial_id);\n    const sentenceIndex = remapSentenceIndex(\n      remaps,\n      serialId,\n      Number(row.fragment_id),\n      Number(row.sentence_index),\n    );\n\n    await database.run(\n      `\n        INSERT INTO chunks (\n          id, generation, serial_id, sentence_index, label, content,\n          retention, importance, wordsCount, weight\n        )\n        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n      `,\n      [\n        row.id ?? null,\n        row.generation ?? 0,\n        serialId,\n        sentenceIndex,\n        row.label ?? \"\",\n        row.content ?? \"\",\n        row.retention ?? null,\n        row.importance ?? null,\n        row.wordsCount ?? 0,\n        row.weight ?? 0,\n      ],\n    );\n  }\n\n  await database.run(\"DROP TABLE legacy_chunks\");\n}\n\nasync function migrateLegacyChunkSentences(\n  database: Database,\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n): Promise<void> {\n  const rows = await database.queryAll(\n    `\n      SELECT chunk_id, serial_id, fragment_id, sentence_index\n      FROM chunk_sentences\n    `,\n    undefined,\n    (row) => row,\n  );\n\n  await database.run(\"DROP TABLE chunk_sentences\");\n  await database.run(`\n    CREATE TABLE chunk_sentences (\n      chunk_id INTEGER NOT NULL,\n      serial_id INTEGER NOT NULL,\n      sentence_index INTEGER NOT NULL,\n      FOREIGN KEY (chunk_id) REFERENCES chunks(id),\n      PRIMARY KEY (chunk_id, serial_id, sentence_index)\n    )\n  `);\n\n  for (const row of rows) {\n    const serialId = Number(row.serial_id);\n\n    await database.run(\n      `\n        INSERT OR IGNORE INTO chunk_sentences (\n          chunk_id, serial_id, sentence_index\n        )\n        VALUES (?, ?, ?)\n      `,\n      [\n        getRequiredSqlBindValue(row.chunk_id),\n        serialId,\n        remapSentenceIndex(\n          remaps,\n          serialId,\n          Number(row.fragment_id),\n          Number(row.sentence_index),\n        ),\n      ],\n    );\n  }\n}\n\nasync function migrateLegacyMentions(\n  database: Database,\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n): Promise<void> {\n  const rows = await database.queryAll(\n    `\n      SELECT *\n      FROM mentions\n      ORDER BY id\n    `,\n    undefined,\n    (row) => row,\n  );\n\n  await database.run(\"ALTER TABLE mentions RENAME TO legacy_mentions\");\n  await database.run(`\n    CREATE TABLE mentions (\n      id TEXT PRIMARY KEY,\n      chapter_id INTEGER NOT NULL,\n      sentence_index INTEGER,\n      range_start INTEGER NOT NULL,\n      range_end INTEGER NOT NULL,\n      surface TEXT NOT NULL,\n      qid TEXT NOT NULL,\n      confidence REAL,\n      note TEXT,\n      FOREIGN KEY (chapter_id) REFERENCES serials(id)\n    )\n  `);\n\n  for (const row of rows) {\n    const chapterId = Number(row.chapter_id);\n    const sentenceIndex =\n      row.sentence_index === null || row.sentence_index === undefined\n        ? null\n        : remapSentenceIndex(\n            remaps,\n            chapterId,\n            Number(row.fragment_id),\n            Number(row.sentence_index),\n          );\n\n    await database.run(\n      `\n        INSERT OR REPLACE INTO mentions (\n          id, chapter_id, sentence_index, range_start, range_end, surface,\n          qid, confidence, note\n        )\n        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n      `,\n      [\n        getRequiredSqlBindValue(row.id),\n        chapterId,\n        sentenceIndex,\n        row.range_start ?? 0,\n        row.range_end ?? 0,\n        row.surface ?? \"\",\n        row.qid ?? \"\",\n        row.confidence ?? null,\n        row.note ?? null,\n      ],\n    );\n  }\n\n  await database.run(\"DROP TABLE legacy_mentions\");\n}\n\nasync function migrateLegacyMentionLinkEvidenceSentences(\n  database: Database,\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n): Promise<void> {\n  const rows = await database.queryAll(\n    `\n      SELECT link_id, chapter_id, fragment_id, sentence_index\n      FROM mention_link_evidence_sentences\n    `,\n    undefined,\n    (row) => row,\n  );\n\n  await database.run(\"DROP TABLE mention_link_evidence_sentences\");\n  await database.run(`\n    CREATE TABLE mention_link_evidence_sentences (\n      link_id TEXT NOT NULL,\n      chapter_id INTEGER NOT NULL,\n      sentence_index INTEGER NOT NULL,\n      FOREIGN KEY (link_id) REFERENCES mention_links(id),\n      PRIMARY KEY (link_id, chapter_id, sentence_index)\n    )\n  `);\n\n  for (const row of rows) {\n    const chapterId = Number(row.chapter_id);\n\n    await database.run(\n      `\n        INSERT OR IGNORE INTO mention_link_evidence_sentences (\n          link_id, chapter_id, sentence_index\n        )\n        VALUES (?, ?, ?)\n      `,\n      [\n        getRequiredSqlBindValue(row.link_id),\n        chapterId,\n        remapSentenceIndex(\n          remaps,\n          chapterId,\n          Number(row.fragment_id),\n          Number(row.sentence_index),\n        ),\n      ],\n    );\n  }\n}\n\nasync function migrateLegacyFragmentGroups(\n  database: Database,\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n): Promise<void> {\n  const rows = await database.queryAll(\n    `\n      SELECT serial_id, group_id, fragment_id\n      FROM fragment_groups\n      ORDER BY serial_id, group_id, fragment_id\n    `,\n    undefined,\n    (row) => row,\n  );\n\n  await database.run(\"DROP TABLE fragment_groups\");\n  await database.run(`\n    CREATE TABLE sentence_groups (\n      serial_id INTEGER NOT NULL,\n      group_id INTEGER NOT NULL,\n      start_sentence_index INTEGER NOT NULL,\n      end_sentence_index INTEGER NOT NULL,\n      PRIMARY KEY (serial_id, group_id, start_sentence_index, end_sentence_index)\n    )\n  `);\n\n  for (const row of rows) {\n    const serialId = Number(row.serial_id);\n    const startSentenceIndex = remapSentenceIndex(\n      remaps,\n      serialId,\n      Number(row.fragment_id),\n      0,\n    );\n    const endSentenceIndex = findFragmentEndSentenceIndex(\n      remaps,\n      serialId,\n      Number(row.fragment_id),\n    );\n\n    await database.run(\n      `\n        INSERT OR IGNORE INTO sentence_groups (\n          serial_id, group_id, start_sentence_index, end_sentence_index\n        )\n        VALUES (?, ?, ?, ?)\n      `,\n      [\n        serialId,\n        getRequiredSqlBindValue(row.group_id),\n        startSentenceIndex,\n        endSentenceIndex,\n      ],\n    );\n  }\n}\n\nfunction remapSentenceIndex(\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n  serialId: number,\n  fragmentId: number,\n  sentenceIndex: number,\n): number {\n  const remap = remaps.get(serialId);\n  const mapped = remap?.get(fragmentId, sentenceIndex);\n\n  if (mapped === undefined) {\n    throw new Error(\n      `Cannot remap legacy sentence ${serialId}:${fragmentId}:${sentenceIndex}.`,\n    );\n  }\n\n  return mapped;\n}\n\nfunction findFragmentEndSentenceIndex(\n  remaps: ReadonlyMap<number, SentenceIndexRemap>,\n  serialId: number,\n  fragmentId: number,\n): number {\n  let sentenceIndex = 0;\n  let last: number | undefined;\n\n  while (true) {\n    const mapped = remaps.get(serialId)?.get(fragmentId, sentenceIndex);\n\n    if (mapped === undefined) {\n      break;\n    }\n\n    last = mapped;\n    sentenceIndex += 1;\n  }\n\n  return last ?? remapSentenceIndex(remaps, serialId, fragmentId, 0);\n}\nfunction getRequiredSqlBindValue(\n  value: SqlBindValue | undefined,\n): SqlBindValue {\n  if (value === undefined) {\n    throw new TypeError(\"Expected a SQLite bind value.\");\n  }\n\n  return value;\n}\n"],"mappings":"AAAA,SAAS,YAAY;AAGrB,SAAS,gBAAgB;AACzB,SAAS,sBAAsB;AAG/B,eAAsB,gCACpB,eACA,QACe;AACf,QAAM,WAAW,MAAM,SAAS,KAAK,KAAK,eAAe,aAAa,CAAC;AAEvE,MAAI;AACF,UAAM,aAAa,MAAM,eAAe,QAAQ;AAEhD,UAAM,SAAS,YAAY,YAAY;AACrC,UAAI,WAAW,IAAI,QAAQ,GAAG;AAC5B,cAAM,oBAAoB,UAAU,MAAM;AAAA,MAC5C;AACA,UAAI,WAAW,IAAI,iBAAiB,GAAG;AACrC,cAAM,4BAA4B,UAAU,MAAM;AAAA,MACpD;AACA,UAAI,WAAW,IAAI,UAAU,GAAG;AAC9B,cAAM,sBAAsB,UAAU,MAAM;AAAA,MAC9C;AACA,UAAI,WAAW,IAAI,iCAAiC,GAAG;AACrD,cAAM,0CAA0C,UAAU,MAAM;AAAA,MAClE;AACA,UAAI,WAAW,IAAI,iBAAiB,GAAG;AACrC,cAAM,4BAA4B,UAAU,MAAM;AAAA,MACpD;AAAA,IACF,CAAC;AAAA,EACH,UAAE;AACA,UAAM,SAAS,MAAM;AAAA,EACvB;AACF;AACA,eAAe,oBACb,UACA,QACe;AACf,QAAM,OAAO,MAAM,SAAS;AAAA,IAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA;AAAA,IACA,CAAC,QAAQ;AAAA,EACX;AAEA,QAAM,SAAS,IAAI,4CAA4C;AAC/D,QAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAalB;AAED,aAAW,OAAO,MAAM;AACtB,UAAM,WAAW,OAAO,IAAI,SAAS;AACrC,UAAM,gBAAgB;AAAA,MACpB;AAAA,MACA;AAAA,MACA,OAAO,IAAI,WAAW;AAAA,MACtB,OAAO,IAAI,cAAc;AAAA,IAC3B;AAEA,UAAM,SAAS;AAAA,MACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOA;AAAA,QACE,IAAI,MAAM;AAAA,QACV,IAAI,cAAc;AAAA,QAClB;AAAA,QACA;AAAA,QACA,IAAI,SAAS;AAAA,QACb,IAAI,WAAW;AAAA,QACf,IAAI,aAAa;AAAA,QACjB,IAAI,cAAc;AAAA,QAClB,IAAI,cAAc;AAAA,QAClB,IAAI,UAAU;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,QAAM,SAAS,IAAI,0BAA0B;AAC/C;AAEA,eAAe,4BACb,UACA,QACe;AACf,QAAM,OAAO,MAAM,SAAS;AAAA,IAC1B;AAAA;AAAA;AAAA;AAAA,IAIA;AAAA,IACA,CAAC,QAAQ;AAAA,EACX;AAEA,QAAM,SAAS,IAAI,4BAA4B;AAC/C,QAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAQlB;AAED,aAAW,OAAO,MAAM;AACtB,UAAM,WAAW,OAAO,IAAI,SAAS;AAErC,UAAM,SAAS;AAAA,MACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAMA;AAAA,QACE,wBAAwB,IAAI,QAAQ;AAAA,QACpC;AAAA,QACA;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO,IAAI,WAAW;AAAA,UACtB,OAAO,IAAI,cAAc;AAAA,QAC3B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,sBACb,UACA,QACe;AACf,QAAM,OAAO,MAAM,SAAS;AAAA,IAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA;AAAA,IACA,CAAC,QAAQ;AAAA,EACX;AAEA,QAAM,SAAS,IAAI,gDAAgD;AACnE,QAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAalB;AAED,aAAW,OAAO,MAAM;AACtB,UAAM,YAAY,OAAO,IAAI,UAAU;AACvC,UAAM,gBACJ,IAAI,mBAAmB,QAAQ,IAAI,mBAAmB,SAClD,OACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,OAAO,IAAI,WAAW;AAAA,MACtB,OAAO,IAAI,cAAc;AAAA,IAC3B;AAEN,UAAM,SAAS;AAAA,MACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOA;AAAA,QACE,wBAAwB,IAAI,EAAE;AAAA,QAC9B;AAAA,QACA;AAAA,QACA,IAAI,eAAe;AAAA,QACnB,IAAI,aAAa;AAAA,QACjB,IAAI,WAAW;AAAA,QACf,IAAI,OAAO;AAAA,QACX,IAAI,cAAc;AAAA,QAClB,IAAI,QAAQ;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,QAAM,SAAS,IAAI,4BAA4B;AACjD;AAEA,eAAe,0CACb,UACA,QACe;AACf,QAAM,OAAO,MAAM,SAAS;AAAA,IAC1B;AAAA;AAAA;AAAA;AAAA,IAIA;AAAA,IACA,CAAC,QAAQ;AAAA,EACX;AAEA,QAAM,SAAS,IAAI,4CAA4C;AAC/D,QAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAQlB;AAED,aAAW,OAAO,MAAM;AACtB,UAAM,YAAY,OAAO,IAAI,UAAU;AAEvC,UAAM,SAAS;AAAA,MACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAMA;AAAA,QACE,wBAAwB,IAAI,OAAO;AAAA,QACnC;AAAA,QACA;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO,IAAI,WAAW;AAAA,UACtB,OAAO,IAAI,cAAc;AAAA,QAC3B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,4BACb,UACA,QACe;AACf,QAAM,OAAO,MAAM,SAAS;AAAA,IAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,IAKA;AAAA,IACA,CAAC,QAAQ;AAAA,EACX;AAEA,QAAM,SAAS,IAAI,4BAA4B;AAC/C,QAAM,SAAS,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAQlB;AAED,aAAW,OAAO,MAAM;AACtB,UAAM,WAAW,OAAO,IAAI,SAAS;AACrC,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA,OAAO,IAAI,WAAW;AAAA,MACtB;AAAA,IACF;AACA,UAAM,mBAAmB;AAAA,MACvB;AAAA,MACA;AAAA,MACA,OAAO,IAAI,WAAW;AAAA,IACxB;AAEA,UAAM,SAAS;AAAA,MACb;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAMA;AAAA,QACE;AAAA,QACA,wBAAwB,IAAI,QAAQ;AAAA,QACpC;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,mBACP,QACA,UACA,YACA,eACQ;AACR,QAAM,QAAQ,OAAO,IAAI,QAAQ;AACjC,QAAM,SAAS,OAAO,IAAI,YAAY,aAAa;AAEnD,MAAI,WAAW,QAAW;AACxB,UAAM,IAAI;AAAA,MACR,gCAAgC,QAAQ,IAAI,UAAU,IAAI,aAAa;AAAA,IACzE;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,6BACP,QACA,UACA,YACQ;AACR,MAAI,gBAAgB;AACpB,MAAI;AAEJ,SAAO,MAAM;AACX,UAAM,SAAS,OAAO,IAAI,QAAQ,GAAG,IAAI,YAAY,aAAa;AAElE,QAAI,WAAW,QAAW;AACxB;AAAA,IACF;AAEA,WAAO;AACP,qBAAiB;AAAA,EACnB;AAEA,SAAO,QAAQ,mBAAmB,QAAQ,UAAU,YAAY,CAAC;AACnE;AACA,SAAS,wBACP,OACc;AACd,MAAI,UAAU,QAAW;AACvB,UAAM,IAAI,UAAU,+BAA+B;AAAA,EACrD;AAEA,SAAO;AACT;","names":[]}

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/source-text.d.ts

@@ -0,0 +1,11 @@
+import { Database } from '../../../../../document/database.js';
+import { LegacyFragmentRecord } from './types.js';
+import 'sqlite3';
+
+declare function writeLegacySourceTextStream(database: Database, workspacePath: string, input: {
+    readonly fragments: readonly LegacyFragmentRecord[];
+    readonly serialId: number;
+    readonly text: string;
+}): Promise<void>;
+
+export { writeLegacySourceTextStream };

package/dist/storage/migration/legacy-sdpub/upgrade/text-storage/source-text.js

@@ -0,0 +1,62 @@
+import { mkdir, writeFile } from "fs/promises";
+import { join } from "path";
+async function writeLegacySourceTextStream(database, workspacePath, input) {
+  await database.run(`
+    CREATE TABLE IF NOT EXISTS text_sentence_records (
+      id INTEGER PRIMARY KEY,
+      kind INTEGER NOT NULL,
+      chapter_id INTEGER NOT NULL,
+      sentence_index INTEGER NOT NULL,
+      words_count INTEGER NOT NULL DEFAULT 0,
+      byte_offset INTEGER NOT NULL DEFAULT 0,
+      byte_length INTEGER NOT NULL DEFAULT 0,
+      UNIQUE(kind, chapter_id, sentence_index)
+    )
+  `);
+  await mkdir(join(workspacePath, "texts", "source"), { recursive: true });
+  await writeFile(
+    join(workspacePath, "texts", "source", `${input.serialId}.txt`),
+    input.text,
+    "utf8"
+  );
+  await database.run(
+    `
+      DELETE FROM text_sentence_records
+      WHERE kind = 1 AND chapter_id = ?
+    `,
+    [input.serialId]
+  );
+  let byteOffset = 0;
+  let sentenceIndex = 0;
+  for (const fragment of input.fragments) {
+    for (const sentence of fragment.content.sentences) {
+      const byteLength = Buffer.byteLength(sentence.text, "utf8");
+      await database.run(
+        `
+          INSERT OR REPLACE INTO text_sentence_records (
+            kind,
+            chapter_id,
+            sentence_index,
+            words_count,
+            byte_offset,
+            byte_length
+          )
+          VALUES (1, ?, ?, ?, ?, ?)
+        `,
+        [
+          input.serialId,
+          sentenceIndex,
+          sentence.wordsCount,
+          byteOffset,
+          byteLength
+        ]
+      );
+      byteOffset += byteLength;
+      sentenceIndex += 1;
+    }
+  }
+}
+export {
+  writeLegacySourceTextStream
+};
+//# sourceMappingURL=source-text.js.map