wiki-graph-core 0.3.1

package/dist/reader/chunk-batch/extractor.js

@@ -0,0 +1,337 @@
+import { z } from "zod";
+import {
+  GuaranteedParseValidationError,
+  GuaranteedRequestFailureError,
+  ParsedJsonError,
+  RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,
+  requestGuaranteedJson
+} from "../../guaranteed/index.js";
+import {
+  EVIDENCE_SELECTION_PROMPT_FRAGMENT,
+  normalizeEvidenceDisplayText
+} from "../../evidence-selection/index.js";
+import {
+  bookCoherenceResponseSchema,
+  ChunkBatchParser,
+  ChunkMetadataField,
+  userFocusedResponseSchema
+} from "./parser.js";
+import { FragmentProjection } from "./fragment-projection.js";
+import { projectFragmentText } from "./fragment-projection.js";
+import { needsTranslation } from "./language.js";
+import {
+  BOOK_COHERENCE_PROMPT_TEMPLATE,
+  EVIDENCE_CHOICE_PROMPT_TEMPLATE,
+  TRANSLATE_CHUNKS_PROMPT_TEMPLATE,
+  USER_FOCUSED_PROMPT_TEMPLATE
+} from "./prompt-templates.js";
+const MAX_CHUNK_REGENERATIONS = 7;
+const TRANSLATED_CHUNK_SCHEMA = z.object({
+  content: z.string(),
+  id: z.number(),
+  label: z.string()
+});
+const TRANSLATED_CHUNKS_SCHEMA = z.array(TRANSLATED_CHUNK_SCHEMA);
+const MAX_TRANSLATION_RETRIES = 3;
+class ChunkExtractor {
+  #extractionGuidance;
+  #llm;
+  #scopes;
+  #sentenceTextSource;
+  #userLanguage;
+  constructor(options) {
+    this.#extractionGuidance = options.extractionGuidance;
+    this.#llm = options.llm;
+    this.#scopes = options.scopes;
+    this.#sentenceTextSource = options.sentenceTextSource;
+    this.#userLanguage = options.userLanguage;
+  }
+  async extractUserFocused(input) {
+    const projection = new FragmentProjection(input.sentences);
+    const messages = this.#buildMessages({
+      promptTemplateName: USER_FOCUSED_PROMPT_TEMPLATE,
+      templateContext: {
+        extraction_guidance: this.#extractionGuidance,
+        evidence_selection_prompt: EVIDENCE_SELECTION_PROMPT_FRAGMENT,
+        user_language: this.#userLanguage,
+        working_memory: input.workingMemoryPrompt
+      },
+      text: formatEvidenceSelectionSourceText(projection)
+    });
+    const extraction = await this.#extractChunks({
+      emptyChunkBatch: {
+        chunks: [],
+        links: [],
+        orderCorrect: true,
+        tempIds: []
+      },
+      messages,
+      metadataField: ChunkMetadataField.Retention,
+      projection,
+      schema: userFocusedResponseSchema,
+      sentences: input.sentences,
+      visibleChunkIds: input.visibleChunkIds
+    });
+    return {
+      chunkBatch: await this.#ensureChunkBatchLanguage(
+        extraction.result.chunkBatch,
+        extraction.parser
+      ),
+      fragmentSummary: extraction.result.fragmentSummary ?? ""
+    };
+  }
+  async extractBookCoherence(input) {
+    const projection = new FragmentProjection(input.sentences);
+    const messages = this.#buildMessages({
+      promptTemplateName: BOOK_COHERENCE_PROMPT_TEMPLATE,
+      templateContext: {
+        user_focused_chunks: input.userFocusedChunks.map((chunk) => ({
+          content: chunk.content,
+          id: chunk.id,
+          label: chunk.label
+        })),
+        evidence_selection_prompt: EVIDENCE_SELECTION_PROMPT_FRAGMENT,
+        user_language: this.#userLanguage,
+        working_memory: input.workingMemoryPrompt
+      },
+      text: formatEvidenceSelectionSourceText(projection)
+    });
+    const extraction = await this.#extractChunks({
+      emptyChunkBatch: {
+        chunks: [],
+        importanceAnnotations: [],
+        links: [],
+        orderCorrect: true,
+        tempIds: []
+      },
+      messages,
+      metadataField: ChunkMetadataField.Importance,
+      projection,
+      schema: bookCoherenceResponseSchema,
+      sentences: input.sentences,
+      validImportanceChunkIds: input.userFocusedChunks.map((chunk) => chunk.id),
+      visibleChunkIds: input.visibleChunkIds
+    });
+    return await this.#ensureChunkBatchLanguage(
+      extraction.result.chunkBatch,
+      extraction.parser
+    );
+  }
+  #buildMessages(input) {
+    return [
+      {
+        content: this.#llm.loadSystemPrompt(
+          input.promptTemplateName,
+          input.templateContext
+        ),
+        role: "system"
+      },
+      {
+        content: input.text,
+        role: "user"
+      }
+    ];
+  }
+  async #extractChunks(input) {
+    return await this.#llm.withContext(
+      async (context) => {
+        const parser = new ChunkBatchParser({
+          metadataField: input.metadataField,
+          projection: input.projection,
+          responseIntentClassifierPrompt: this.#llm.loadSystemPrompt(
+            RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE
+          ),
+          sentenceTextSource: this.#sentenceTextSource,
+          sentences: input.sentences,
+          visibleChunkIds: input.visibleChunkIds,
+          choiceSystemPrompt: this.#llm.loadSystemPrompt(
+            EVIDENCE_CHOICE_PROMPT_TEMPLATE,
+            {
+              extraction_guidance: this.#extractionGuidance,
+              metadata_field: input.metadataField,
+              user_language: this.#userLanguage
+            }
+          ),
+          requestChoice: async (messages, index, maxRetries) => await context.request(messages, {
+            retryIndex: index,
+            retryMax: maxRetries,
+            scope: this.#scopes.choice,
+            useCache: false
+          }),
+          ...input.validImportanceChunkIds === void 0 ? {} : {
+            validImportanceChunkIds: input.validImportanceChunkIds
+          }
+        });
+        try {
+          const result = await requestGuaranteedJson({
+            messages: input.messages,
+            schema: input.schema,
+            maxRetries: MAX_CHUNK_REGENERATIONS,
+            parse: async (data, index, maxRetries) => await parser.parse(data, {
+              isLastGenerationAttempt: index >= maxRetries
+            }),
+            responseIntentClassifierPrompt: this.#llm.loadSystemPrompt(
+              RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE
+            ),
+            request: async (messages, index, maxRetries) => await context.request(messages, {
+              retryIndex: index,
+              retryMax: maxRetries,
+              scope: this.#scopes.extraction
+            })
+          });
+          return {
+            parser,
+            result
+          };
+        } catch (error) {
+          if (isRecoverableChunkExtractionFailure(error)) {
+            return {
+              parser,
+              result: {
+                chunkBatch: input.emptyChunkBatch
+              }
+            };
+          }
+          throw error;
+        }
+      }
+    );
+  }
+  /**
+   * 对于 AI 来说，原本应该直接生成正确语言的文字和摘要，但因为
+   * 强迫 AI 在不同语言之间切换非常困难，经常会出现语言不符合要求的情况。
+   *
+   * 所以我们的处理方案是：
+   * 1. 通过程序找到那些与要求语言不一致的句子
+   * 2. 直接再用 AI 把这些句子翻译过去
+   *
+   * 这是一个保底手段。
+   */
+  async #ensureChunkBatchLanguage(chunkBatch, parser) {
+    if (this.#userLanguage === void 0 || chunkBatch.chunks.length === 0) {
+      return chunkBatch;
+    }
+    const translationInput = [];
+    for (const [index, chunk] of chunkBatch.chunks.entries()) {
+      if (!needsTranslation({
+        content: chunk.content,
+        label: chunk.label,
+        targetLanguage: this.#userLanguage
+      })) {
+        continue;
+      }
+      translationInput.push({
+        content: chunk.content,
+        id: index,
+        label: chunk.label,
+        sourceSentences: await parser.getChunkSourceSentences(chunk)
+      });
+    }
+    if (translationInput.length === 0) {
+      return chunkBatch;
+    }
+    try {
+      const translatedChunks = await this.#translateChunks(
+        translationInput,
+        this.#userLanguage
+      );
+      const translatedById = createTranslatedChunkRecord(translatedChunks);
+      return {
+        ...chunkBatch,
+        chunks: chunkBatch.chunks.map((chunk, index) => {
+          const translated = translatedById[String(index)];
+          if (translated === void 0) {
+            return chunk;
+          }
+          return {
+            ...chunk,
+            content: translated.content,
+            label: translated.label
+          };
+        })
+      };
+    } catch {
+      return chunkBatch;
+    }
+  }
+  async #translateChunks(chunks, userLanguage) {
+    return await requestGuaranteedJson({
+      messages: [
+        {
+          content: this.#llm.loadSystemPrompt(
+            TRANSLATE_CHUNKS_PROMPT_TEMPLATE,
+            {
+              user_language: userLanguage
+            }
+          ),
+          role: "system"
+        },
+        {
+          content: JSON.stringify(chunks, void 0, 2),
+          role: "user"
+        }
+      ],
+      schema: TRANSLATED_CHUNKS_SCHEMA,
+      maxRetries: MAX_TRANSLATION_RETRIES,
+      parse: (data) => {
+        validateTranslatedChunks(chunks, data);
+        return data;
+      },
+      responseIntentClassifierPrompt: this.#llm.loadSystemPrompt(
+        RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE
+      ),
+      request: async (messages, index, maxRetries) => await this.#llm.request(messages, {
+        retryIndex: index,
+        retryMax: maxRetries,
+        scope: this.#scopes.extraction
+      })
+    });
+  }
+}
+function formatEvidenceSelectionSourceText(projection) {
+  return projection.sentences.map(
+    (sentence, index) => `S${index + 1}: ${projectFragmentText(
+      normalizeEvidenceDisplayText(sentence.rawText)
+    )}`
+  ).join("\n");
+}
+function isParsedJsonValidationFailure(error) {
+  return error instanceof GuaranteedParseValidationError && error.cause instanceof ParsedJsonError;
+}
+function isRecoverableChunkExtractionFailure(error) {
+  return isParsedJsonValidationFailure(error) || error instanceof GuaranteedRequestFailureError;
+}
+function createTranslatedChunkRecord(translatedChunks) {
+  const record = /* @__PURE__ */ Object.create(null);
+  for (const chunk of translatedChunks) {
+    record[String(chunk.id)] = chunk;
+  }
+  return record;
+}
+function validateTranslatedChunks(chunks, translatedChunks) {
+  const issues = [];
+  if (translatedChunks.length !== chunks.length) {
+    issues.push(
+      `Expected ${chunks.length} translated chunk(s), got ${translatedChunks.length}`
+    );
+  }
+  for (const [index, chunk] of chunks.entries()) {
+    const translatedChunk = translatedChunks[index];
+    if (translatedChunk === void 0) {
+      issues.push(`Missing translated chunk at position ${index}`);
+      continue;
+    }
+    if (translatedChunk.id !== chunk.id) {
+      issues.push(
+        `Translated chunk at position ${index} must keep id ${chunk.id}, got ${translatedChunk.id}`
+      );
+    }
+  }
+  if (issues.length > 0) {
+    throw new ParsedJsonError(issues);
+  }
+}
+export {
+  ChunkExtractor
+};
+//# sourceMappingURL=extractor.js.map

package/dist/reader/chunk-batch/extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/chunk-batch/extractor.ts"],"sourcesContent":["import { z, type ZodType } from \"zod\";\n\nimport {\n  GuaranteedParseValidationError,\n  GuaranteedRequestFailureError,\n  ParsedJsonError,\n  RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,\n  requestGuaranteedJson,\n} from \"../../guaranteed/index.js\";\nimport type { Language } from \"../../common/language.js\";\nimport {\n  EVIDENCE_SELECTION_PROMPT_FRAGMENT,\n  normalizeEvidenceDisplayText,\n} from \"../../evidence-selection/index.js\";\nimport type { LLMessage, LLM } from \"../../llm/index.js\";\nimport {\n  bookCoherenceResponseSchema,\n  ChunkBatchParser,\n  ChunkMetadataField,\n  type BookCoherenceResponseData,\n  type ExtractChunksResult,\n  type UserFocusedResponseData,\n  userFocusedResponseSchema,\n} from \"./parser.js\";\nimport { FragmentProjection } from \"./fragment-projection.js\";\nimport { projectFragmentText } from \"./fragment-projection.js\";\nimport { needsTranslation } from \"./language.js\";\nimport {\n  BOOK_COHERENCE_PROMPT_TEMPLATE,\n  EVIDENCE_CHOICE_PROMPT_TEMPLATE,\n  TRANSLATE_CHUNKS_PROMPT_TEMPLATE,\n  USER_FOCUSED_PROMPT_TEMPLATE,\n} from \"./prompt-templates.js\";\nimport type {\n  ChunkBatch,\n  ChunkBatchOptions,\n  ChunkExtractionSentence,\n  ChunkTranslationInput,\n  ChunkTranslationOutput,\n  ExtractBookCoherenceInput,\n  ExtractUserFocusedInput,\n  ExtractUserFocusedResult,\n} from \"./types.js\";\n\nconst MAX_CHUNK_REGENERATIONS = 7;\n\ninterface ExtractChunksInput<\n  TData extends UserFocusedResponseData | BookCoherenceResponseData,\n> {\n  readonly emptyChunkBatch: ChunkBatch;\n  readonly messages: readonly LLMessage[];\n  readonly metadataField: ChunkMetadataField;\n  readonly projection: FragmentProjection;\n  readonly schema: ZodType<TData>;\n  readonly sentences: readonly ChunkExtractionSentence[];\n  readonly validImportanceChunkIds?: readonly number[];\n  readonly visibleChunkIds: readonly number[];\n}\n\ninterface ExtractChunksOutput<\n  TData extends UserFocusedResponseData | BookCoherenceResponseData,\n> {\n  readonly parser: ChunkBatchParser<TData>;\n  readonly result: ExtractChunksResult;\n}\nconst TRANSLATED_CHUNK_SCHEMA = z.object({\n  content: z.string(),\n  id: z.number(),\n  label: z.string(),\n});\nconst TRANSLATED_CHUNKS_SCHEMA = z.array(TRANSLATED_CHUNK_SCHEMA);\nconst MAX_TRANSLATION_RETRIES = 3;\n\nexport class ChunkExtractor<S extends string> {\n  readonly #extractionGuidance: string;\n  readonly #llm: LLM<S>;\n  readonly #scopes: ChunkBatchOptions<S>[\"scopes\"];\n  readonly #sentenceTextSource: ChunkBatchOptions<S>[\"sentenceTextSource\"];\n  readonly #userLanguage: Language | undefined;\n\n  public constructor(options: ChunkBatchOptions<S>) {\n    this.#extractionGuidance = options.extractionGuidance;\n    this.#llm = options.llm;\n    this.#scopes = options.scopes;\n    this.#sentenceTextSource = options.sentenceTextSource;\n    this.#userLanguage = options.userLanguage;\n  }\n\n  public async extractUserFocused(\n    input: ExtractUserFocusedInput,\n  ): Promise<ExtractUserFocusedResult> {\n    const projection = new FragmentProjection(input.sentences);\n    const messages = this.#buildMessages({\n      promptTemplateName: USER_FOCUSED_PROMPT_TEMPLATE,\n      templateContext: {\n        extraction_guidance: this.#extractionGuidance,\n        evidence_selection_prompt: EVIDENCE_SELECTION_PROMPT_FRAGMENT,\n        user_language: this.#userLanguage,\n        working_memory: input.workingMemoryPrompt,\n      },\n      text: formatEvidenceSelectionSourceText(projection),\n    });\n    const extraction = await this.#extractChunks({\n      emptyChunkBatch: {\n        chunks: [],\n        links: [],\n        orderCorrect: true,\n        tempIds: [],\n      },\n      messages,\n      metadataField: ChunkMetadataField.Retention,\n      projection,\n      schema: userFocusedResponseSchema,\n      sentences: input.sentences,\n      visibleChunkIds: input.visibleChunkIds,\n    });\n\n    return {\n      chunkBatch: await this.#ensureChunkBatchLanguage(\n        extraction.result.chunkBatch,\n        extraction.parser,\n      ),\n      fragmentSummary: extraction.result.fragmentSummary ?? \"\",\n    };\n  }\n\n  public async extractBookCoherence(\n    input: ExtractBookCoherenceInput,\n  ): Promise<ChunkBatch> {\n    const projection = new FragmentProjection(input.sentences);\n    const messages = this.#buildMessages({\n      promptTemplateName: BOOK_COHERENCE_PROMPT_TEMPLATE,\n      templateContext: {\n        user_focused_chunks: input.userFocusedChunks.map((chunk) => ({\n          content: chunk.content,\n          id: chunk.id,\n          label: chunk.label,\n        })),\n        evidence_selection_prompt: EVIDENCE_SELECTION_PROMPT_FRAGMENT,\n        user_language: this.#userLanguage,\n        working_memory: input.workingMemoryPrompt,\n      },\n      text: formatEvidenceSelectionSourceText(projection),\n    });\n    const extraction = await this.#extractChunks({\n      emptyChunkBatch: {\n        chunks: [],\n        importanceAnnotations: [],\n        links: [],\n        orderCorrect: true,\n        tempIds: [],\n      },\n      messages,\n      metadataField: ChunkMetadataField.Importance,\n      projection,\n      schema: bookCoherenceResponseSchema,\n      sentences: input.sentences,\n      validImportanceChunkIds: input.userFocusedChunks.map((chunk) => chunk.id),\n      visibleChunkIds: input.visibleChunkIds,\n    });\n\n    return await this.#ensureChunkBatchLanguage(\n      extraction.result.chunkBatch,\n      extraction.parser,\n    );\n  }\n\n  #buildMessages(input: {\n    promptTemplateName: string;\n    templateContext: Record<string, unknown>;\n    text: string;\n  }): LLMessage[] {\n    return [\n      {\n        content: this.#llm.loadSystemPrompt(\n          input.promptTemplateName,\n          input.templateContext,\n        ),\n        role: \"system\",\n      },\n      {\n        content: input.text,\n        role: \"user\",\n      },\n    ];\n  }\n\n  async #extractChunks<\n    TData extends UserFocusedResponseData | BookCoherenceResponseData,\n  >(input: ExtractChunksInput<TData>): Promise<ExtractChunksOutput<TData>> {\n    return await this.#llm.withContext(\n      async (context): Promise<ExtractChunksOutput<TData>> => {\n        const parser = new ChunkBatchParser<TData>({\n          metadataField: input.metadataField,\n          projection: input.projection,\n          responseIntentClassifierPrompt: this.#llm.loadSystemPrompt(\n            RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,\n          ),\n          sentenceTextSource: this.#sentenceTextSource,\n          sentences: input.sentences,\n          visibleChunkIds: input.visibleChunkIds,\n          choiceSystemPrompt: this.#llm.loadSystemPrompt(\n            EVIDENCE_CHOICE_PROMPT_TEMPLATE,\n            {\n              extraction_guidance: this.#extractionGuidance,\n              metadata_field: input.metadataField,\n              user_language: this.#userLanguage,\n            },\n          ),\n          requestChoice: async (messages, index, maxRetries) =>\n            await context.request(messages, {\n              retryIndex: index,\n              retryMax: maxRetries,\n              scope: this.#scopes.choice,\n              useCache: false,\n            }),\n          ...(input.validImportanceChunkIds === undefined\n            ? {}\n            : {\n                validImportanceChunkIds: input.validImportanceChunkIds,\n              }),\n        });\n\n        try {\n          const result = await requestGuaranteedJson({\n            messages: input.messages,\n            schema: input.schema,\n            maxRetries: MAX_CHUNK_REGENERATIONS,\n            parse: async (data, index, maxRetries) =>\n              await parser.parse(data, {\n                isLastGenerationAttempt: index >= maxRetries,\n              }),\n            responseIntentClassifierPrompt: this.#llm.loadSystemPrompt(\n              RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,\n            ),\n            request: async (messages, index, maxRetries) =>\n              await context.request(messages, {\n                retryIndex: index,\n                retryMax: maxRetries,\n                scope: this.#scopes.extraction,\n              }),\n          });\n\n          return {\n            parser,\n            result,\n          };\n        } catch (error) {\n          if (isRecoverableChunkExtractionFailure(error)) {\n            return {\n              parser,\n              result: {\n                chunkBatch: input.emptyChunkBatch,\n              },\n            };\n          }\n          throw error;\n        }\n      },\n    );\n  }\n\n  /**\n   * 对于 AI 来说，原本应该直接生成正确语言的文字和摘要，但因为\n   * 强迫 AI 在不同语言之间切换非常困难，经常会出现语言不符合要求的情况。\n   *\n   * 所以我们的处理方案是：\n   * 1. 通过程序找到那些与要求语言不一致的句子\n   * 2. 直接再用 AI 把这些句子翻译过去\n   *\n   * 这是一个保底手段。\n   */\n  async #ensureChunkBatchLanguage(\n    chunkBatch: ChunkBatch,\n    parser: ChunkBatchParser<\n      UserFocusedResponseData | BookCoherenceResponseData\n    >,\n  ): Promise<ChunkBatch> {\n    if (this.#userLanguage === undefined || chunkBatch.chunks.length === 0) {\n      return chunkBatch;\n    }\n\n    const translationInput: ChunkTranslationInput[] = [];\n\n    for (const [index, chunk] of chunkBatch.chunks.entries()) {\n      if (\n        !needsTranslation({\n          content: chunk.content,\n          label: chunk.label,\n          targetLanguage: this.#userLanguage,\n        })\n      ) {\n        continue;\n      }\n\n      translationInput.push({\n        content: chunk.content,\n        id: index,\n        label: chunk.label,\n        sourceSentences: await parser.getChunkSourceSentences(chunk),\n      });\n    }\n\n    if (translationInput.length === 0) {\n      return chunkBatch;\n    }\n\n    try {\n      const translatedChunks = await this.#translateChunks(\n        translationInput,\n        this.#userLanguage,\n      );\n      const translatedById = createTranslatedChunkRecord(translatedChunks);\n\n      return {\n        ...chunkBatch,\n        chunks: chunkBatch.chunks.map((chunk, index) => {\n          const translated = translatedById[String(index)];\n\n          if (translated === undefined) {\n            return chunk;\n          }\n\n          return {\n            ...chunk,\n            content: translated.content,\n            label: translated.label,\n          };\n        }),\n      };\n    } catch {\n      return chunkBatch;\n    }\n  }\n\n  async #translateChunks(\n    chunks: readonly ChunkTranslationInput[],\n    userLanguage: Language,\n  ): Promise<readonly ChunkTranslationOutput[]> {\n    return await requestGuaranteedJson({\n      messages: [\n        {\n          content: this.#llm.loadSystemPrompt(\n            TRANSLATE_CHUNKS_PROMPT_TEMPLATE,\n            {\n              user_language: userLanguage,\n            },\n          ),\n          role: \"system\",\n        },\n        {\n          content: JSON.stringify(chunks, undefined, 2),\n          role: \"user\",\n        },\n      ],\n      schema: TRANSLATED_CHUNKS_SCHEMA,\n      maxRetries: MAX_TRANSLATION_RETRIES,\n      parse: (data) => {\n        validateTranslatedChunks(chunks, data);\n        return data;\n      },\n      responseIntentClassifierPrompt: this.#llm.loadSystemPrompt(\n        RESPONSE_INTENT_CLASSIFIER_PROMPT_TEMPLATE,\n      ),\n      request: async (messages, index, maxRetries) =>\n        await this.#llm.request(messages, {\n          retryIndex: index,\n          retryMax: maxRetries,\n          scope: this.#scopes.extraction,\n        }),\n    });\n  }\n}\n\nfunction formatEvidenceSelectionSourceText(\n  projection: FragmentProjection,\n): string {\n  return projection.sentences\n    .map(\n      (sentence, index) =>\n        `S${index + 1}: ${projectFragmentText(\n          normalizeEvidenceDisplayText(sentence.rawText),\n        )}`,\n    )\n    .join(\"\\n\");\n}\n\nfunction isParsedJsonValidationFailure(error: unknown): boolean {\n  return (\n    error instanceof GuaranteedParseValidationError &&\n    error.cause instanceof ParsedJsonError\n  );\n}\n\nfunction isRecoverableChunkExtractionFailure(error: unknown): boolean {\n  return (\n    isParsedJsonValidationFailure(error) ||\n    error instanceof GuaranteedRequestFailureError\n  );\n}\n\nfunction createTranslatedChunkRecord(\n  translatedChunks: readonly ChunkTranslationOutput[],\n): Readonly<Record<string, ChunkTranslationOutput>> {\n  const record = Object.create(null) as Record<string, ChunkTranslationOutput>;\n\n  for (const chunk of translatedChunks) {\n    record[String(chunk.id)] = chunk;\n  }\n\n  return record;\n}\n\nfunction validateTranslatedChunks(\n  chunks: readonly ChunkTranslationInput[],\n  translatedChunks: readonly ChunkTranslationOutput[],\n): void {\n  const issues: string[] = [];\n\n  if (translatedChunks.length !== chunks.length) {\n    issues.push(\n      `Expected ${chunks.length} translated chunk(s), got ${translatedChunks.length}`,\n    );\n  }\n\n  for (const [index, chunk] of chunks.entries()) {\n    const translatedChunk = translatedChunks[index];\n\n    if (translatedChunk === undefined) {\n      issues.push(`Missing translated chunk at position ${index}`);\n      continue;\n    }\n\n    if (translatedChunk.id !== chunk.id) {\n      issues.push(\n        `Translated chunk at position ${index} must keep id ${chunk.id}, got ${translatedChunk.id}`,\n      );\n    }\n  }\n\n  if (issues.length > 0) {\n    throw new ParsedJsonError(issues);\n  }\n}\n"],"mappings":"AAAA,SAAS,SAAuB;AAEhC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,OACK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EAIA;AAAA,OACK;AACP,SAAS,0BAA0B;AACnC,SAAS,2BAA2B;AACpC,SAAS,wBAAwB;AACjC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAYP,MAAM,0BAA0B;AAqBhC,MAAM,0BAA0B,EAAE,OAAO;AAAA,EACvC,SAAS,EAAE,OAAO;AAAA,EAClB,IAAI,EAAE,OAAO;AAAA,EACb,OAAO,EAAE,OAAO;AAClB,CAAC;AACD,MAAM,2BAA2B,EAAE,MAAM,uBAAuB;AAChE,MAAM,0BAA0B;AAEzB,MAAM,eAAiC;AAAA,EACnC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,SAA+B;AAChD,SAAK,sBAAsB,QAAQ;AACnC,SAAK,OAAO,QAAQ;AACpB,SAAK,UAAU,QAAQ;AACvB,SAAK,sBAAsB,QAAQ;AACnC,SAAK,gBAAgB,QAAQ;AAAA,EAC/B;AAAA,EAEA,MAAa,mBACX,OACmC;AACnC,UAAM,aAAa,IAAI,mBAAmB,MAAM,SAAS;AACzD,UAAM,WAAW,KAAK,eAAe;AAAA,MACnC,oBAAoB;AAAA,MACpB,iBAAiB;AAAA,QACf,qBAAqB,KAAK;AAAA,QAC1B,2BAA2B;AAAA,QAC3B,eAAe,KAAK;AAAA,QACpB,gBAAgB,MAAM;AAAA,MACxB;AAAA,MACA,MAAM,kCAAkC,UAAU;AAAA,IACpD,CAAC;AACD,UAAM,aAAa,MAAM,KAAK,eAAe;AAAA,MAC3C,iBAAiB;AAAA,QACf,QAAQ,CAAC;AAAA,QACT,OAAO,CAAC;AAAA,QACR,cAAc;AAAA,QACd,SAAS,CAAC;AAAA,MACZ;AAAA,MACA;AAAA,MACA,eAAe,mBAAmB;AAAA,MAClC;AAAA,MACA,QAAQ;AAAA,MACR,WAAW,MAAM;AAAA,MACjB,iBAAiB,MAAM;AAAA,IACzB,CAAC;AAED,WAAO;AAAA,MACL,YAAY,MAAM,KAAK;AAAA,QACrB,WAAW,OAAO;AAAA,QAClB,WAAW;AAAA,MACb;AAAA,MACA,iBAAiB,WAAW,OAAO,mBAAmB;AAAA,IACxD;AAAA,EACF;AAAA,EAEA,MAAa,qBACX,OACqB;AACrB,UAAM,aAAa,IAAI,mBAAmB,MAAM,SAAS;AACzD,UAAM,WAAW,KAAK,eAAe;AAAA,MACnC,oBAAoB;AAAA,MACpB,iBAAiB;AAAA,QACf,qBAAqB,MAAM,kBAAkB,IAAI,CAAC,WAAW;AAAA,UAC3D,SAAS,MAAM;AAAA,UACf,IAAI,MAAM;AAAA,UACV,OAAO,MAAM;AAAA,QACf,EAAE;AAAA,QACF,2BAA2B;AAAA,QAC3B,eAAe,KAAK;AAAA,QACpB,gBAAgB,MAAM;AAAA,MACxB;AAAA,MACA,MAAM,kCAAkC,UAAU;AAAA,IACpD,CAAC;AACD,UAAM,aAAa,MAAM,KAAK,eAAe;AAAA,MAC3C,iBAAiB;AAAA,QACf,QAAQ,CAAC;AAAA,QACT,uBAAuB,CAAC;AAAA,QACxB,OAAO,CAAC;AAAA,QACR,cAAc;AAAA,QACd,SAAS,CAAC;AAAA,MACZ;AAAA,MACA;AAAA,MACA,eAAe,mBAAmB;AAAA,MAClC;AAAA,MACA,QAAQ;AAAA,MACR,WAAW,MAAM;AAAA,MACjB,yBAAyB,MAAM,kBAAkB,IAAI,CAAC,UAAU,MAAM,EAAE;AAAA,MACxE,iBAAiB,MAAM;AAAA,IACzB,CAAC;AAED,WAAO,MAAM,KAAK;AAAA,MAChB,WAAW,OAAO;AAAA,MAClB,WAAW;AAAA,IACb;AAAA,EACF;AAAA,EAEA,eAAe,OAIC;AACd,WAAO;AAAA,MACL;AAAA,QACE,SAAS,KAAK,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,MAAM;AAAA,QACR;AAAA,QACA,MAAM;AAAA,MACR;AAAA,MACA;AAAA,QACE,SAAS,MAAM;AAAA,QACf,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,eAEJ,OAAuE;AACvE,WAAO,MAAM,KAAK,KAAK;AAAA,MACrB,OAAO,YAAiD;AACtD,cAAM,SAAS,IAAI,iBAAwB;AAAA,UACzC,eAAe,MAAM;AAAA,UACrB,YAAY,MAAM;AAAA,UAClB,gCAAgC,KAAK,KAAK;AAAA,YACxC;AAAA,UACF;AAAA,UACA,oBAAoB,KAAK;AAAA,UACzB,WAAW,MAAM;AAAA,UACjB,iBAAiB,MAAM;AAAA,UACvB,oBAAoB,KAAK,KAAK;AAAA,YAC5B;AAAA,YACA;AAAA,cACE,qBAAqB,KAAK;AAAA,cAC1B,gBAAgB,MAAM;AAAA,cACtB,eAAe,KAAK;AAAA,YACtB;AAAA,UACF;AAAA,UACA,eAAe,OAAO,UAAU,OAAO,eACrC,MAAM,QAAQ,QAAQ,UAAU;AAAA,YAC9B,YAAY;AAAA,YACZ,UAAU;AAAA,YACV,OAAO,KAAK,QAAQ;AAAA,YACpB,UAAU;AAAA,UACZ,CAAC;AAAA,UACH,GAAI,MAAM,4BAA4B,SAClC,CAAC,IACD;AAAA,YACE,yBAAyB,MAAM;AAAA,UACjC;AAAA,QACN,CAAC;AAED,YAAI;AACF,gBAAM,SAAS,MAAM,sBAAsB;AAAA,YACzC,UAAU,MAAM;AAAA,YAChB,QAAQ,MAAM;AAAA,YACd,YAAY;AAAA,YACZ,OAAO,OAAO,MAAM,OAAO,eACzB,MAAM,OAAO,MAAM,MAAM;AAAA,cACvB,yBAAyB,SAAS;AAAA,YACpC,CAAC;AAAA,YACH,gCAAgC,KAAK,KAAK;AAAA,cACxC;AAAA,YACF;AAAA,YACA,SAAS,OAAO,UAAU,OAAO,eAC/B,MAAM,QAAQ,QAAQ,UAAU;AAAA,cAC9B,YAAY;AAAA,cACZ,UAAU;AAAA,cACV,OAAO,KAAK,QAAQ;AAAA,YACtB,CAAC;AAAA,UACL,CAAC;AAED,iBAAO;AAAA,YACL;AAAA,YACA;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd,cAAI,oCAAoC,KAAK,GAAG;AAC9C,mBAAO;AAAA,cACL;AAAA,cACA,QAAQ;AAAA,gBACN,YAAY,MAAM;AAAA,cACpB;AAAA,YACF;AAAA,UACF;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,0BACJ,YACA,QAGqB;AACrB,QAAI,KAAK,kBAAkB,UAAa,WAAW,OAAO,WAAW,GAAG;AACtE,aAAO;AAAA,IACT;AAEA,UAAM,mBAA4C,CAAC;AAEnD,eAAW,CAAC,OAAO,KAAK,KAAK,WAAW,OAAO,QAAQ,GAAG;AACxD,UACE,CAAC,iBAAiB;AAAA,QAChB,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,QACb,gBAAgB,KAAK;AAAA,MACvB,CAAC,GACD;AACA;AAAA,MACF;AAEA,uBAAiB,KAAK;AAAA,QACpB,SAAS,MAAM;AAAA,QACf,IAAI;AAAA,QACJ,OAAO,MAAM;AAAA,QACb,iBAAiB,MAAM,OAAO,wBAAwB,KAAK;AAAA,MAC7D,CAAC;AAAA,IACH;AAEA,QAAI,iBAAiB,WAAW,GAAG;AACjC,aAAO;AAAA,IACT;AAEA,QAAI;AACF,YAAM,mBAAmB,MAAM,KAAK;AAAA,QAClC;AAAA,QACA,KAAK;AAAA,MACP;AACA,YAAM,iBAAiB,4BAA4B,gBAAgB;AAEnE,aAAO;AAAA,QACL,GAAG;AAAA,QACH,QAAQ,WAAW,OAAO,IAAI,CAAC,OAAO,UAAU;AAC9C,gBAAM,aAAa,eAAe,OAAO,KAAK,CAAC;AAE/C,cAAI,eAAe,QAAW;AAC5B,mBAAO;AAAA,UACT;AAEA,iBAAO;AAAA,YACL,GAAG;AAAA,YACH,SAAS,WAAW;AAAA,YACpB,OAAO,WAAW;AAAA,UACpB;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,iBACJ,QACA,cAC4C;AAC5C,WAAO,MAAM,sBAAsB;AAAA,MACjC,UAAU;AAAA,QACR;AAAA,UACE,SAAS,KAAK,KAAK;AAAA,YACjB;AAAA,YACA;AAAA,cACE,eAAe;AAAA,YACjB;AAAA,UACF;AAAA,UACA,MAAM;AAAA,QACR;AAAA,QACA;AAAA,UACE,SAAS,KAAK,UAAU,QAAQ,QAAW,CAAC;AAAA,UAC5C,MAAM;AAAA,QACR;AAAA,MACF;AAAA,MACA,QAAQ;AAAA,MACR,YAAY;AAAA,MACZ,OAAO,CAAC,SAAS;AACf,iCAAyB,QAAQ,IAAI;AACrC,eAAO;AAAA,MACT;AAAA,MACA,gCAAgC,KAAK,KAAK;AAAA,QACxC;AAAA,MACF;AAAA,MACA,SAAS,OAAO,UAAU,OAAO,eAC/B,MAAM,KAAK,KAAK,QAAQ,UAAU;AAAA,QAChC,YAAY;AAAA,QACZ,UAAU;AAAA,QACV,OAAO,KAAK,QAAQ;AAAA,MACtB,CAAC;AAAA,IACL,CAAC;AAAA,EACH;AACF;AAEA,SAAS,kCACP,YACQ;AACR,SAAO,WAAW,UACf;AAAA,IACC,CAAC,UAAU,UACT,IAAI,QAAQ,CAAC,KAAK;AAAA,MAChB,6BAA6B,SAAS,OAAO;AAAA,IAC/C,CAAC;AAAA,EACL,EACC,KAAK,IAAI;AACd;AAEA,SAAS,8BAA8B,OAAyB;AAC9D,SACE,iBAAiB,kCACjB,MAAM,iBAAiB;AAE3B;AAEA,SAAS,oCAAoC,OAAyB;AACpE,SACE,8BAA8B,KAAK,KACnC,iBAAiB;AAErB;AAEA,SAAS,4BACP,kBACkD;AAClD,QAAM,SAAS,uBAAO,OAAO,IAAI;AAEjC,aAAW,SAAS,kBAAkB;AACpC,WAAO,OAAO,MAAM,EAAE,CAAC,IAAI;AAAA,EAC7B;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,QACA,kBACM;AACN,QAAM,SAAmB,CAAC;AAE1B,MAAI,iBAAiB,WAAW,OAAO,QAAQ;AAC7C,WAAO;AAAA,MACL,YAAY,OAAO,MAAM,6BAA6B,iBAAiB,MAAM;AAAA,IAC/E;AAAA,EACF;AAEA,aAAW,CAAC,OAAO,KAAK,KAAK,OAAO,QAAQ,GAAG;AAC7C,UAAM,kBAAkB,iBAAiB,KAAK;AAE9C,QAAI,oBAAoB,QAAW;AACjC,aAAO,KAAK,wCAAwC,KAAK,EAAE;AAC3D;AAAA,IACF;AAEA,QAAI,gBAAgB,OAAO,MAAM,IAAI;AACnC,aAAO;AAAA,QACL,gCAAgC,KAAK,iBAAiB,MAAM,EAAE,SAAS,gBAAgB,EAAE;AAAA,MAC3F;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,IAAI,gBAAgB,MAAM;AAAA,EAClC;AACF;","names":[]}

package/dist/reader/chunk-batch/fragment-projection.d.ts

@@ -0,0 +1,34 @@
+import { SentenceId } from '../../document/types.js';
+import { ChunkExtractionSentence } from './types.js';
+import '../../common/language.js';
+import '../../llm/client.js';
+import '../../llm/context.js';
+import '../../llm/cache.js';
+import '../../llm/types.js';
+import 'ai';
+
+interface TextSpan {
+    readonly length: number;
+    readonly offset: number;
+}
+interface ProjectedFragmentSentence {
+    readonly endOffset: number;
+    readonly projectedText: string;
+    readonly rawText: string;
+    readonly sentenceId: SentenceId;
+    readonly startOffset: number;
+    readonly wordsCount: number;
+}
+declare class FragmentProjection {
+    #private;
+    constructor(sentences: readonly ChunkExtractionSentence[]);
+    get projectedText(): string;
+    get rawText(): string;
+    get sentences(): readonly ProjectedFragmentSentence[];
+    findExactMatches(text: string): readonly TextSpan[];
+    projectText(text: string): string;
+    resolveSentenceIds(span: TextSpan): SentenceId[];
+}
+declare function projectFragmentText(text: string): string;
+
+export { FragmentProjection, type ProjectedFragmentSentence, type TextSpan, projectFragmentText };

package/dist/reader/chunk-batch/fragment-projection.js

@@ -0,0 +1,94 @@
+const SENTENCE_SEPARATOR = " ";
+const STANDARDIZATION_MAP = {
+  '"': "\uFF02",
+  "\\": "\uFF3C"
+};
+class FragmentProjection {
+  #projectedText;
+  #rawText;
+  #sentences;
+  constructor(sentences) {
+    const projectedSentences = [];
+    let rawText = "";
+    let projectedText = "";
+    let offset = 0;
+    for (const [index, sentence] of sentences.entries()) {
+      if (index > 0) {
+        rawText += SENTENCE_SEPARATOR;
+        projectedText += SENTENCE_SEPARATOR;
+        offset += SENTENCE_SEPARATOR.length;
+      }
+      const projectedSentenceText = projectFragmentText(sentence.text);
+      const startOffset = offset;
+      const endOffset = startOffset + projectedSentenceText.length;
+      rawText += sentence.text;
+      projectedText += projectedSentenceText;
+      projectedSentences.push({
+        endOffset,
+        projectedText: projectedSentenceText,
+        rawText: sentence.text,
+        sentenceId: sentence.sentenceId,
+        startOffset,
+        wordsCount: sentence.wordsCount
+      });
+      offset = endOffset;
+    }
+    this.#rawText = rawText;
+    this.#projectedText = projectedText;
+    this.#sentences = projectedSentences;
+  }
+  get projectedText() {
+    return this.#projectedText;
+  }
+  get rawText() {
+    return this.#rawText;
+  }
+  get sentences() {
+    return this.#sentences;
+  }
+  findExactMatches(text) {
+    const query = projectFragmentText(text).trim();
+    if (query === "") {
+      return [];
+    }
+    const matches = [];
+    let startIndex = 0;
+    while (startIndex <= this.#projectedText.length - query.length) {
+      const offset = this.#projectedText.indexOf(query, startIndex);
+      if (offset === -1) {
+        break;
+      }
+      matches.push({
+        length: query.length,
+        offset
+      });
+      startIndex = offset + 1;
+    }
+    return matches;
+  }
+  projectText(text) {
+    return projectFragmentText(text);
+  }
+  resolveSentenceIds(span) {
+    if (span.length <= 0) {
+      return [];
+    }
+    const rangeStart = span.offset;
+    const rangeEnd = span.offset + span.length;
+    return this.#sentences.filter(
+      (sentence) => sentence.endOffset > rangeStart && sentence.startOffset < rangeEnd
+    ).map((sentence) => sentence.sentenceId);
+  }
+}
+function projectFragmentText(text) {
+  let result = "";
+  for (const char of text) {
+    result += STANDARDIZATION_MAP[char] ?? char;
+  }
+  return result;
+}
+export {
+  FragmentProjection,
+  projectFragmentText
+};
+//# sourceMappingURL=fragment-projection.js.map

package/dist/reader/chunk-batch/fragment-projection.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/chunk-batch/fragment-projection.ts"],"sourcesContent":["import type { SentenceId } from \"../../document/index.js\";\nimport type { ChunkExtractionSentence } from \"./types.js\";\n\nconst SENTENCE_SEPARATOR = \" \";\n\nconst STANDARDIZATION_MAP = {\n  '\"': \"＂\",\n  \"\\\\\": \"＼\",\n} as const;\n\nexport interface TextSpan {\n  readonly length: number;\n  readonly offset: number;\n}\n\nexport interface ProjectedFragmentSentence {\n  readonly endOffset: number;\n  readonly projectedText: string;\n  readonly rawText: string;\n  readonly sentenceId: SentenceId;\n  readonly startOffset: number;\n  readonly wordsCount: number;\n}\n\nexport class FragmentProjection {\n  readonly #projectedText: string;\n  readonly #rawText: string;\n  readonly #sentences: readonly ProjectedFragmentSentence[];\n\n  public constructor(sentences: readonly ChunkExtractionSentence[]) {\n    const projectedSentences: ProjectedFragmentSentence[] = [];\n    let rawText = \"\";\n    let projectedText = \"\";\n    let offset = 0;\n\n    for (const [index, sentence] of sentences.entries()) {\n      if (index > 0) {\n        rawText += SENTENCE_SEPARATOR;\n        projectedText += SENTENCE_SEPARATOR;\n        offset += SENTENCE_SEPARATOR.length;\n      }\n\n      const projectedSentenceText = projectFragmentText(sentence.text);\n      const startOffset = offset;\n      const endOffset = startOffset + projectedSentenceText.length;\n\n      rawText += sentence.text;\n      projectedText += projectedSentenceText;\n      projectedSentences.push({\n        endOffset,\n        projectedText: projectedSentenceText,\n        rawText: sentence.text,\n        sentenceId: sentence.sentenceId,\n        startOffset,\n        wordsCount: sentence.wordsCount,\n      });\n      offset = endOffset;\n    }\n\n    this.#rawText = rawText;\n    this.#projectedText = projectedText;\n    this.#sentences = projectedSentences;\n  }\n\n  public get projectedText(): string {\n    return this.#projectedText;\n  }\n\n  public get rawText(): string {\n    return this.#rawText;\n  }\n\n  public get sentences(): readonly ProjectedFragmentSentence[] {\n    return this.#sentences;\n  }\n\n  public findExactMatches(text: string): readonly TextSpan[] {\n    const query = projectFragmentText(text).trim();\n\n    if (query === \"\") {\n      return [];\n    }\n\n    const matches: TextSpan[] = [];\n    let startIndex = 0;\n\n    while (startIndex <= this.#projectedText.length - query.length) {\n      const offset = this.#projectedText.indexOf(query, startIndex);\n\n      if (offset === -1) {\n        break;\n      }\n\n      matches.push({\n        length: query.length,\n        offset,\n      });\n      startIndex = offset + 1;\n    }\n\n    return matches;\n  }\n\n  public projectText(text: string): string {\n    return projectFragmentText(text);\n  }\n\n  public resolveSentenceIds(span: TextSpan): SentenceId[] {\n    if (span.length <= 0) {\n      return [];\n    }\n\n    const rangeStart = span.offset;\n    const rangeEnd = span.offset + span.length;\n\n    return this.#sentences\n      .filter(\n        (sentence) =>\n          sentence.endOffset > rangeStart && sentence.startOffset < rangeEnd,\n      )\n      .map((sentence) => sentence.sentenceId);\n  }\n}\n\nexport function projectFragmentText(text: string): string {\n  let result = \"\";\n\n  for (const char of text) {\n    result +=\n      STANDARDIZATION_MAP[char as keyof typeof STANDARDIZATION_MAP] ?? char;\n  }\n\n  return result;\n}\n"],"mappings":"AAGA,MAAM,qBAAqB;AAE3B,MAAM,sBAAsB;AAAA,EAC1B,KAAK;AAAA,EACL,MAAM;AACR;AAgBO,MAAM,mBAAmB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,WAA+C;AAChE,UAAM,qBAAkD,CAAC;AACzD,QAAI,UAAU;AACd,QAAI,gBAAgB;AACpB,QAAI,SAAS;AAEb,eAAW,CAAC,OAAO,QAAQ,KAAK,UAAU,QAAQ,GAAG;AACnD,UAAI,QAAQ,GAAG;AACb,mBAAW;AACX,yBAAiB;AACjB,kBAAU,mBAAmB;AAAA,MAC/B;AAEA,YAAM,wBAAwB,oBAAoB,SAAS,IAAI;AAC/D,YAAM,cAAc;AACpB,YAAM,YAAY,cAAc,sBAAsB;AAEtD,iBAAW,SAAS;AACpB,uBAAiB;AACjB,yBAAmB,KAAK;AAAA,QACtB;AAAA,QACA,eAAe;AAAA,QACf,SAAS,SAAS;AAAA,QAClB,YAAY,SAAS;AAAA,QACrB;AAAA,QACA,YAAY,SAAS;AAAA,MACvB,CAAC;AACD,eAAS;AAAA,IACX;AAEA,SAAK,WAAW;AAChB,SAAK,iBAAiB;AACtB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,IAAW,gBAAwB;AACjC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAW,UAAkB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAW,YAAkD;AAC3D,WAAO,KAAK;AAAA,EACd;AAAA,EAEO,iBAAiB,MAAmC;AACzD,UAAM,QAAQ,oBAAoB,IAAI,EAAE,KAAK;AAE7C,QAAI,UAAU,IAAI;AAChB,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,UAAsB,CAAC;AAC7B,QAAI,aAAa;AAEjB,WAAO,cAAc,KAAK,eAAe,SAAS,MAAM,QAAQ;AAC9D,YAAM,SAAS,KAAK,eAAe,QAAQ,OAAO,UAAU;AAE5D,UAAI,WAAW,IAAI;AACjB;AAAA,MACF;AAEA,cAAQ,KAAK;AAAA,QACX,QAAQ,MAAM;AAAA,QACd;AAAA,MACF,CAAC;AACD,mBAAa,SAAS;AAAA,IACxB;AAEA,WAAO;AAAA,EACT;AAAA,EAEO,YAAY,MAAsB;AACvC,WAAO,oBAAoB,IAAI;AAAA,EACjC;AAAA,EAEO,mBAAmB,MAA8B;AACtD,QAAI,KAAK,UAAU,GAAG;AACpB,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,aAAa,KAAK;AACxB,UAAM,WAAW,KAAK,SAAS,KAAK;AAEpC,WAAO,KAAK,WACT;AAAA,MACC,CAAC,aACC,SAAS,YAAY,cAAc,SAAS,cAAc;AAAA,IAC9D,EACC,IAAI,CAAC,aAAa,SAAS,UAAU;AAAA,EAC1C;AACF;AAEO,SAAS,oBAAoB,MAAsB;AACxD,MAAI,SAAS;AAEb,aAAW,QAAQ,MAAM;AACvB,cACE,oBAAoB,IAAwC,KAAK;AAAA,EACrE;AAEA,SAAO;AACT;","names":[]}

package/dist/reader/chunk-batch/index.d.ts

@@ -0,0 +1,12 @@
+export { extractBookCoherenceChunkBatch, extractUserFocusedChunkBatch } from './extract.js';
+export { EvidenceResolver } from '../../evidence-selection/anchor-resolver.js';
+export { EvidenceResolutionFailure, EvidenceResolutionResult, RankedSentenceCandidate } from '../../evidence-selection/anchor-types.js';
+export { FragmentProjection, TextSpan } from './fragment-projection.js';
+export { ChunkBatch, ChunkBatchOptions, ChunkExtractionScopes, ChunkExtractionSentence, ChunkImportanceAnnotation, ChunkLink, ChunkTranslationInput, ChunkTranslationOutput, CognitiveChunk, ExtractBookCoherenceInput, ExtractUserFocusedInput, ExtractUserFocusedResult, SentenceTextSource } from './types.js';
+import '../../document/types.js';
+import '../../common/language.js';
+import '../../llm/client.js';
+import '../../llm/context.js';
+import '../../llm/cache.js';
+import '../../llm/types.js';
+import 'ai';

package/dist/reader/chunk-batch/index.js

@@ -0,0 +1,13 @@
+import {
+  extractBookCoherenceChunkBatch,
+  extractUserFocusedChunkBatch
+} from "./extract.js";
+import { EvidenceResolver } from "../../evidence-selection/index.js";
+import { FragmentProjection } from "./fragment-projection.js";
+export {
+  EvidenceResolver,
+  FragmentProjection,
+  extractBookCoherenceChunkBatch,
+  extractUserFocusedChunkBatch
+};
+//# sourceMappingURL=index.js.map

package/dist/reader/chunk-batch/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/chunk-batch/index.ts"],"sourcesContent":["export {\n  extractBookCoherenceChunkBatch,\n  extractUserFocusedChunkBatch,\n} from \"./extract.js\";\nexport { EvidenceResolver } from \"../../evidence-selection/index.js\";\nexport { FragmentProjection } from \"./fragment-projection.js\";\nexport type { TextSpan } from \"./fragment-projection.js\";\nexport type {\n  ChunkBatch,\n  ChunkBatchOptions,\n  ChunkExtractionScopes,\n  ChunkExtractionSentence,\n  ChunkImportanceAnnotation,\n  ChunkLink,\n  ChunkTranslationInput,\n  ChunkTranslationOutput,\n  CognitiveChunk,\n  ExtractBookCoherenceInput,\n  ExtractUserFocusedInput,\n  ExtractUserFocusedResult,\n  SentenceTextSource,\n} from \"./types.js\";\nexport type {\n  EvidenceResolutionFailure,\n  EvidenceResolutionResult,\n  RankedSentenceCandidate,\n} from \"../../evidence-selection/index.js\";\n"],"mappings":"AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;","names":[]}

package/dist/reader/chunk-batch/language.d.ts

@@ -0,0 +1,9 @@
+import { Language } from '../../common/language.js';
+
+declare function needsTranslation(input: {
+    content: string;
+    label: string;
+    targetLanguage: Language;
+}): boolean;
+
+export { needsTranslation };

package/dist/reader/chunk-batch/language.js

@@ -0,0 +1,23 @@
+import {
+  detectLanguageCode,
+  getLanguageDetectionCode
+} from "../../common/tinyld-language.js";
+import {} from "../../common/language.js";
+function needsTranslation(input) {
+  const targetLanguageCode = getLanguageDetectionCode(input.targetLanguage);
+  return fieldNeedsTranslation(input.label, targetLanguageCode) || fieldNeedsTranslation(input.content, targetLanguageCode);
+}
+function fieldNeedsTranslation(text, targetLanguageCode) {
+  if (text.trim() === "") {
+    return false;
+  }
+  const detectedLanguageCode = detectLanguageCode(text);
+  if (detectedLanguageCode === void 0) {
+    return true;
+  }
+  return detectedLanguageCode !== targetLanguageCode;
+}
+export {
+  needsTranslation
+};
+//# sourceMappingURL=language.js.map

package/dist/reader/chunk-batch/language.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/chunk-batch/language.ts"],"sourcesContent":["import {\n  detectLanguageCode,\n  getLanguageDetectionCode,\n} from \"../../common/tinyld-language.js\";\nimport { type Language } from \"../../common/language.js\";\n\nexport function needsTranslation(input: {\n  content: string;\n  label: string;\n  targetLanguage: Language;\n}): boolean {\n  const targetLanguageCode = getLanguageDetectionCode(input.targetLanguage);\n\n  return (\n    fieldNeedsTranslation(input.label, targetLanguageCode) ||\n    fieldNeedsTranslation(input.content, targetLanguageCode)\n  );\n}\n\nfunction fieldNeedsTranslation(\n  text: string,\n  targetLanguageCode: string,\n): boolean {\n  if (text.trim() === \"\") {\n    return false;\n  }\n\n  const detectedLanguageCode = detectLanguageCode(text);\n\n  if (detectedLanguageCode === undefined) {\n    return true;\n  }\n\n  return detectedLanguageCode !== targetLanguageCode;\n}\n"],"mappings":"AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,eAA8B;AAEvB,SAAS,iBAAiB,OAIrB;AACV,QAAM,qBAAqB,yBAAyB,MAAM,cAAc;AAExE,SACE,sBAAsB,MAAM,OAAO,kBAAkB,KACrD,sBAAsB,MAAM,SAAS,kBAAkB;AAE3D;AAEA,SAAS,sBACP,MACA,oBACS;AACT,MAAI,KAAK,KAAK,MAAM,IAAI;AACtB,WAAO;AAAA,EACT;AAEA,QAAM,uBAAuB,mBAAmB,IAAI;AAEpD,MAAI,yBAAyB,QAAW;AACtC,WAAO;AAAA,EACT;AAEA,SAAO,yBAAyB;AAClC;","names":[]}