wiki-graph-core 0.3.1

package/dist/reader/chunk-batch/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/chunk-batch/parser.ts"],"sourcesContent":["import { z } from \"zod\";\n\nimport {\n  GuaranteedParseValidationError,\n  ParsedJsonError,\n  requestGuaranteedJson,\n} from \"../../guaranteed/index.js\";\nimport type { LLMessage } from \"../../llm/index.js\";\nimport {\n  ChunkImportance,\n  ChunkRetention,\n  expectChunkImportance,\n  expectChunkRetention,\n  type SentenceId,\n} from \"../../document/index.js\";\nimport {\n  resolveEvidenceSelectionList,\n  type EvidenceSelectionCandidate,\n  type EvidenceSelectionList,\n  EvidenceResolver,\n  type EvidenceResolutionFailure,\n  type RankedSentenceCandidate,\n} from \"../../evidence-selection/index.js\";\nimport type {\n  ChunkBatch,\n  ChunkExtractionSentence,\n  ChunkImportanceAnnotation,\n  ChunkLink,\n  CognitiveChunk,\n  SentenceTextSource,\n} from \"./types.js\";\nimport type { FragmentProjection, TextSpan } from \"./fragment-projection.js\";\n\nconst MAX_CHOICE_RETRIES = 3;\n\nconst chunkLinkSchema = z.object({\n  from: z.union([z.number().int(), z.string()]),\n  strength: z.string().optional(),\n  to: z.union([z.number().int(), z.string()]),\n});\nconst evidenceSelectionItemSchema = z\n  .object({\n    quote: z.string().optional(),\n    sentence_id: z.string().optional(),\n  })\n  .passthrough();\nconst chunkEvidenceSchema = z.union([\n  z.record(z.string(), z.unknown()),\n  z.array(evidenceSelectionItemSchema),\n]);\n\nconst userFocusedChunkSchema = z\n  .object({\n    content: z.string(),\n    evidence: chunkEvidenceSchema.nullish(),\n    label: z.string(),\n    retention: z.enum([\n      ChunkRetention.Verbatim,\n      ChunkRetention.Detailed,\n      ChunkRetention.Focused,\n      ChunkRetention.Relevant,\n    ]),\n    temp_id: z.string(),\n  })\n  .passthrough();\n\nexport const userFocusedResponseSchema = z.object({\n  chunks: z.array(userFocusedChunkSchema),\n  fragment_summary: z.string(),\n  links: z.array(chunkLinkSchema),\n});\n\nconst bookCoherenceChunkSchema = z\n  .object({\n    content: z.string(),\n    evidence: chunkEvidenceSchema.nullish(),\n    importance: z.enum([\n      ChunkImportance.Critical,\n      ChunkImportance.Important,\n      ChunkImportance.Helpful,\n    ]),\n    label: z.string(),\n    temp_id: z.string(),\n  })\n  .passthrough();\n\nconst importanceAnnotationSchema = z.object({\n  chunk_id: z.number().int(),\n  importance: z.enum([\n    ChunkImportance.Critical,\n    ChunkImportance.Important,\n    ChunkImportance.Helpful,\n  ]),\n});\n\nexport const bookCoherenceResponseSchema = z.object({\n  chunks: z.array(bookCoherenceChunkSchema),\n  importance_annotations: z.array(importanceAnnotationSchema),\n  links: z.array(chunkLinkSchema),\n});\n\nconst choiceResponseSchema = z.object({\n  choice: z.string(),\n});\n\ntype UserFocusedChunkData = z.infer<typeof userFocusedChunkSchema>;\ntype BookCoherenceChunkData = z.infer<typeof bookCoherenceChunkSchema>;\nexport type UserFocusedResponseData = z.infer<typeof userFocusedResponseSchema>;\nexport type BookCoherenceResponseData = z.infer<\n  typeof bookCoherenceResponseSchema\n>;\ntype ExtractedChunkData = UserFocusedChunkData | BookCoherenceChunkData;\ntype RawChunkLink = z.infer<typeof chunkLinkSchema>;\ntype RawChunkEvidence = z.infer<typeof chunkEvidenceSchema>;\ntype ChoiceFieldName = \"evidence\" | \"start_anchor\" | \"end_anchor\";\n\nexport enum ChunkMetadataField {\n  Retention = \"retention\",\n  Importance = \"importance\",\n}\n\nexport interface ExtractChunksResult {\n  readonly chunkBatch: ChunkBatch;\n  readonly fragmentSummary?: string;\n}\n\ninterface ResolveChunkEvidenceInput {\n  readonly data: ExtractedChunkData;\n  readonly chunkIndex: number;\n  readonly chunkLabel: string;\n  readonly isLastGenerationAttempt: boolean;\n}\n\ninterface SelectAmbiguousCandidateInput {\n  readonly candidates: readonly RankedSentenceCandidate[];\n  readonly chunkData: ExtractedChunkData;\n  readonly chunkIndex: number;\n  readonly chunkLabel: string;\n  readonly fieldName: ChoiceFieldName;\n}\n\ntype GuaranteedChoiceRequest = (\n  messages: readonly LLMessage[],\n  index: number,\n  maxRetries: number,\n) => Promise<string>;\n\nexport class ChunkBatchParser<\n  TData extends UserFocusedResponseData | BookCoherenceResponseData,\n> {\n  readonly #choiceSystemPrompt: string;\n  readonly #evidenceResolver = new EvidenceResolver();\n  readonly #metadataField: ChunkMetadataField;\n  readonly #projection: FragmentProjection;\n  readonly #responseIntentClassifierPrompt: string;\n  readonly #requestChoice: GuaranteedChoiceRequest;\n  readonly #sentenceTextByKey: Readonly<Record<string, string>>;\n  readonly #sentenceTextSource: SentenceTextSource;\n  readonly #wordsCountByKey: Readonly<Record<string, number>>;\n  readonly #validImportanceChunkIds: Readonly<Record<string, true>> | undefined;\n  readonly #visibleChunkIds: readonly number[];\n\n  public constructor(input: {\n    choiceSystemPrompt: string;\n    metadataField: ChunkMetadataField;\n    projection: FragmentProjection;\n    responseIntentClassifierPrompt: string;\n    requestChoice: GuaranteedChoiceRequest;\n    sentenceTextSource: SentenceTextSource;\n    sentences: readonly ChunkExtractionSentence[];\n    validImportanceChunkIds?: readonly number[];\n    visibleChunkIds: readonly number[];\n  }) {\n    this.#choiceSystemPrompt = input.choiceSystemPrompt;\n    this.#metadataField = input.metadataField;\n    this.#projection = input.projection;\n    this.#responseIntentClassifierPrompt = input.responseIntentClassifierPrompt;\n    this.#requestChoice = input.requestChoice;\n    this.#sentenceTextSource = input.sentenceTextSource;\n    this.#visibleChunkIds = input.visibleChunkIds;\n\n    this.#wordsCountByKey = createWordsCountRecord(\n      input.projection.sentences.map((sentence) => ({\n        sentenceId: sentence.sentenceId,\n        wordsCount: sentence.wordsCount,\n      })),\n    );\n    this.#sentenceTextByKey = createSentenceTextRecord(input.projection);\n    this.#validImportanceChunkIds =\n      input.validImportanceChunkIds === undefined\n        ? undefined\n        : createMembershipRecord(input.validImportanceChunkIds);\n  }\n\n  public async parse(\n    parsedData: TData,\n    input: {\n      isLastGenerationAttempt: boolean;\n    },\n  ): Promise<ExtractChunksResult> {\n    const issues: string[] = [];\n    const chunks: ChunkBatch[\"chunks\"] = [];\n    const tempIds: string[] = [];\n    const importanceAnnotations =\n      \"importance_annotations\" in parsedData\n        ? this.#validateImportanceAnnotations(\n            parsedData.importance_annotations,\n            issues,\n          )\n        : undefined;\n    const fragmentSummary =\n      \"fragment_summary\" in parsedData\n        ? parsedData.fragment_summary\n        : undefined;\n    const links = normalizeChunkLinks(parsedData.links);\n\n    for (const [index, data] of parsedData.chunks.entries()) {\n      const chunkIndex = index + 1;\n      const chunkIssues: string[] = [];\n      const label = data.label.trim();\n      const content = data.content.trim();\n\n      if (label === \"\") {\n        chunkIssues.push(\n          `Chunk #${chunkIndex}: Missing or empty \"label\" field`,\n        );\n      }\n\n      if (content === \"\") {\n        chunkIssues.push(\n          `Chunk #${chunkIndex}: Missing or empty \"content\" field`,\n        );\n      }\n\n      const [matchedSentenceIds, evidenceFailure] =\n        await this.#resolveChunkEvidence({\n          chunkIndex,\n          chunkLabel: label,\n          data,\n          isLastGenerationAttempt: input.isLastGenerationAttempt,\n        });\n\n      if (matchedSentenceIds.length === 0) {\n        if (evidenceFailure !== undefined) {\n          chunkIssues.push(\n            `Chunk #${chunkIndex} (\"${label}\"): ${evidenceFailure.message}`,\n          );\n        } else {\n          chunkIssues.push(\n            `Chunk #${chunkIndex} (\"${label}\"): Missing evidence`,\n          );\n        }\n      }\n\n      if (chunkIssues.length > 0) {\n        issues.push(...chunkIssues);\n        continue;\n      }\n\n      const primarySentenceId = matchedSentenceIds[0];\n\n      if (primarySentenceId === undefined) {\n        issues.push(\n          `Chunk #${chunkIndex} (\"${label}\"): Unable to resolve any sentence IDs`,\n        );\n        continue;\n      }\n\n      const totalWordsCount = matchedSentenceIds.reduce((sum, sentenceId) => {\n        return (\n          sum + (this.#wordsCountByKey[createSentenceKey(sentenceId)] ?? 0)\n        );\n      }, 0);\n\n      if (this.#metadataField === ChunkMetadataField.Retention) {\n        const chunkData = data as UserFocusedChunkData;\n\n        chunks.push({\n          content,\n          generation: 0,\n          id: 0,\n          label,\n          links: [],\n          retention: expectChunkRetention(chunkData.retention),\n          sentenceId: primarySentenceId,\n          sentenceIds: [...matchedSentenceIds],\n          wordsCount: totalWordsCount,\n        });\n      } else {\n        const chunkData = data as BookCoherenceChunkData;\n\n        chunks.push({\n          content,\n          generation: 0,\n          id: 0,\n          importance: expectChunkImportance(chunkData.importance),\n          label,\n          links: [],\n          sentenceId: primarySentenceId,\n          sentenceIds: [...matchedSentenceIds],\n          wordsCount: totalWordsCount,\n        });\n      }\n\n      tempIds.push(data.temp_id);\n    }\n\n    const validLinks = this.#filterAndValidateLinks({\n      issues,\n      links,\n      tempIds,\n      visibleChunkIds: this.#visibleChunkIds,\n    });\n\n    if (issues.length > 0) {\n      throw new ParsedJsonError(issues);\n    }\n\n    return {\n      chunkBatch: {\n        chunks,\n        links: validLinks,\n        orderCorrect: true,\n        tempIds,\n        ...(importanceAnnotations === undefined\n          ? {}\n          : { importanceAnnotations }),\n      },\n      ...(fragmentSummary === undefined ? {} : { fragmentSummary }),\n    };\n  }\n\n  public async getChunkSourceSentences(\n    chunk: CognitiveChunk,\n  ): Promise<string[]> {\n    const sourceSentences: string[] = [];\n\n    for (const sentenceId of chunk.sentenceIds) {\n      const sentenceKey = createSentenceKey(sentenceId);\n      const sentenceText = this.#sentenceTextByKey[sentenceKey];\n\n      if (sentenceText !== undefined) {\n        sourceSentences.push(sentenceText);\n        continue;\n      }\n\n      sourceSentences.push(\n        await this.#sentenceTextSource.getSentence(sentenceId),\n      );\n    }\n\n    return sourceSentences;\n  }\n\n  async #resolveChunkEvidence(\n    input: ResolveChunkEvidenceInput,\n  ): Promise<\n    readonly [\n      sentenceIds: readonly SentenceId[],\n      failure: EvidenceResolutionFailure | undefined,\n    ]\n  > {\n    const evidence = input.data.evidence;\n\n    if (evidence === undefined || evidence === null) {\n      return [[], undefined];\n    }\n\n    if (!isRecord(evidence) && !Array.isArray(evidence)) {\n      return [\n        [],\n        {\n          candidates: [],\n          code: \"invalid_evidence\",\n          fieldName: \"evidence\",\n          message: `Chunk #${input.chunkIndex} (\"${input.chunkLabel}\"): evidence must be an object or array`,\n        },\n      ];\n    }\n\n    const projectedSentences = this.#projection.sentences;\n    const candidateSentenceIds = projectedSentences.map(\n      (sentence) => sentence.sentenceId,\n    );\n    const selectionSentences = projectedSentences.map((sentence, index) => ({\n      id: `S${index + 1}`,\n      sentenceId: sentence.sentenceId,\n      text: sentence.projectedText,\n    }));\n    const selectionEvidence = createEvidenceSelectionList(evidence);\n    const [selectionResolution, selectionFailure] =\n      selectionEvidence === undefined\n        ? [undefined, undefined]\n        : resolveEvidenceSelectionList({\n            evidence: selectionEvidence,\n            sentences: selectionSentences,\n          });\n\n    if (selectionResolution !== undefined) {\n      return [selectionResolution.sentenceIds, undefined];\n    }\n\n    const exactMatchSentenceIds = Array.isArray(evidence)\n      ? []\n      : this.#resolveExactProjectionEvidence(evidence);\n\n    if (exactMatchSentenceIds.length > 0) {\n      return [exactMatchSentenceIds, undefined];\n    }\n\n    const candidateTexts = projectedSentences.map(\n      (sentence) => sentence.projectedText,\n    );\n    const [resolution, failure] = Array.isArray(evidence)\n      ? [undefined, undefined]\n      : this.#evidenceResolver.resolve(\n          evidence,\n          candidateSentenceIds,\n          candidateTexts,\n        );\n\n    if (resolution !== undefined) {\n      return [resolution.sentenceIds, undefined];\n    }\n\n    const fallbackFailure =\n      selectionFailure === undefined\n        ? failure\n        : toEvidenceResolutionFailure(selectionFailure, \"evidence\");\n\n    if (fallbackFailure === undefined) {\n      return [[], undefined];\n    }\n\n    const shouldUseChoice =\n      fallbackFailure.code.startsWith(\"ambiguous\") ||\n      (fallbackFailure.code === \"low_confidence\" &&\n        input.isLastGenerationAttempt &&\n        fallbackFailure.candidates.length > 0);\n\n    if (!shouldUseChoice) {\n      return [[], fallbackFailure];\n    }\n\n    const choiceFieldName = toChoiceFieldName(fallbackFailure.fieldName);\n\n    if (choiceFieldName === undefined) {\n      return [[], fallbackFailure];\n    }\n\n    const [choiceCandidate, choiceFailure] =\n      await this.#chooseAmbiguousCandidate({\n        candidates: fallbackFailure.candidates,\n        chunkData: input.data,\n        chunkIndex: input.chunkIndex,\n        chunkLabel: input.chunkLabel,\n        fieldName: choiceFieldName,\n      });\n\n    if (choiceFailure !== undefined) {\n      if (choiceFailure.code === \"choice_parse_failed_full_fragment\") {\n        return [candidateSentenceIds, undefined];\n      }\n\n      return [[], choiceFailure];\n    }\n\n    if (choiceCandidate === undefined) {\n      return [\n        [],\n        {\n          candidates: fallbackFailure.candidates,\n          code: \"choice_failed\",\n          fieldName: fallbackFailure.fieldName,\n          message: `Second-stage choice failed for ${fallbackFailure.fieldName}: no candidate returned.`,\n        },\n      ];\n    }\n\n    if (choiceFieldName === \"evidence\") {\n      return [[choiceCandidate.sentenceId], undefined];\n    }\n\n    if (Array.isArray(evidence)) {\n      return [[], fallbackFailure];\n    }\n\n    if (input.isLastGenerationAttempt) {\n      const [resolved, resolveFailure] =\n        this.#evidenceResolver.resolveWithOverrides({\n          candidateSentenceIds,\n          candidateTexts,\n          evidence,\n          overrides: {\n            [choiceFieldName]: choiceCandidate,\n          },\n        });\n\n      if (resolved !== undefined) {\n        return [resolved.sentenceIds, undefined];\n      }\n\n      return [[], resolveFailure];\n    }\n\n    const repairedEvidence: Record<string, unknown> = {\n      ...evidence,\n      [choiceFieldName]: {\n        mode: \"full\",\n        text: choiceCandidate.text,\n      },\n    };\n    const [resolved, resolveFailure] = this.#evidenceResolver.resolve(\n      repairedEvidence,\n      candidateSentenceIds,\n      candidateTexts,\n    );\n\n    if (resolved !== undefined) {\n      return [resolved.sentenceIds, undefined];\n    }\n\n    return [[], resolveFailure];\n  }\n\n  #resolveExactProjectionEvidence(\n    evidence: Record<string, unknown>,\n  ): SentenceId[] {\n    const startValue = evidence.start_anchor ?? evidence.start;\n    const [startAnchor, startFailure] = this.#evidenceResolver.parseAnchor(\n      startValue,\n      \"start_anchor\",\n    );\n\n    if (startFailure !== undefined || startAnchor?.text === undefined) {\n      return [];\n    }\n\n    const startMatch = expectSingleSpan(\n      this.#projection.findExactMatches(startAnchor.text),\n    );\n\n    if (startMatch === undefined) {\n      return [];\n    }\n\n    const endValue = evidence.end_anchor ?? evidence.end;\n\n    if (endValue === undefined) {\n      return this.#projection.resolveSentenceIds(startMatch);\n    }\n\n    const [endAnchor, endFailure] = this.#evidenceResolver.parseAnchor(\n      endValue,\n      \"end_anchor\",\n    );\n\n    if (endFailure !== undefined || endAnchor?.text === undefined) {\n      return [];\n    }\n\n    const endMatch = expectSingleSpan(\n      this.#projection.findExactMatches(endAnchor.text),\n    );\n\n    if (endMatch === undefined || endMatch.offset < startMatch.offset) {\n      return [];\n    }\n\n    return this.#projection.resolveSentenceIds({\n      length: endMatch.offset + endMatch.length - startMatch.offset,\n      offset: startMatch.offset,\n    });\n  }\n\n  async #chooseAmbiguousCandidate(\n    input: SelectAmbiguousCandidateInput,\n  ): Promise<\n    readonly [\n      candidate: RankedSentenceCandidate | undefined,\n      failure: EvidenceResolutionFailure | undefined,\n    ]\n  > {\n    const messages = this.#buildChoiceMessages(input);\n    const candidateIds = input.candidates.map(\n      (candidate) => candidate.occurrenceId,\n    );\n\n    try {\n      const choice = await requestGuaranteedJson({\n        maxRetries: MAX_CHOICE_RETRIES,\n        messages,\n        parse: (data) => {\n          const candidate = input.candidates.find(\n            (item) => item.occurrenceId === data.choice,\n          );\n\n          if (candidate === undefined) {\n            throw new ParsedJsonError([\n              `Invalid choice \"${data.choice}\". Expected one of: ${candidateIds.join(\", \")}`,\n            ]);\n          }\n\n          return candidate;\n        },\n        responseIntentClassifierPrompt: this.#responseIntentClassifierPrompt,\n        request: this.#requestChoice,\n        schema: choiceResponseSchema,\n      });\n\n      return [choice, undefined];\n    } catch (error) {\n      if (isParsedJsonValidationFailure(error)) {\n        return [\n          undefined,\n          {\n            candidates: input.candidates,\n            code: \"choice_parse_failed_full_fragment\",\n            fieldName: input.fieldName,\n            message:\n              `Second-stage choice parse validation failed for ${input.fieldName}; ` +\n              \"falling back to the full fragment span.\",\n          },\n        ];\n      }\n\n      return [\n        undefined,\n        {\n          candidates: input.candidates,\n          code: \"choice_failed\",\n          fieldName: input.fieldName,\n          message: `Second-stage choice failed for ${input.fieldName}: ${formatError(error)}`,\n        },\n      ];\n    }\n  }\n\n  #buildChoiceMessages(input: SelectAmbiguousCandidateInput): LLMessage[] {\n    return [\n      {\n        content: this.#choiceSystemPrompt,\n        role: \"system\",\n      },\n      {\n        content:\n          `Previously generated chunk (do NOT rewrite it):\\n` +\n          `\\`\\`\\`json\\n${JSON.stringify(input.chunkData, null, 2)}\\n\\`\\`\\`\\n\\n` +\n          `Resolve only this field: \"${input.fieldName}\" for chunk #${input.chunkIndex} [${input.chunkLabel}].\\n` +\n          \"Choose exactly one candidate occurrence ID from the list below.\\n\\n\" +\n          input.candidates.map(formatChoiceCandidate).join(\"\\n\"),\n        role: \"user\",\n      },\n    ];\n  }\n\n  #validateImportanceAnnotations(\n    annotations: readonly {\n      readonly chunk_id: number;\n      readonly importance: ChunkImportance;\n    }[],\n    issues: string[],\n  ): ChunkImportanceAnnotation[] | undefined {\n    if (annotations.length === 0) {\n      return [];\n    }\n\n    if (this.#validImportanceChunkIds === undefined) {\n      return annotations.map((annotation) => ({\n        chunkId: annotation.chunk_id,\n        importance: expectChunkImportance(annotation.importance),\n      }));\n    }\n\n    const result: ChunkImportanceAnnotation[] = [];\n\n    for (const annotation of annotations) {\n      if (!hasMembership(this.#validImportanceChunkIds, annotation.chunk_id)) {\n        issues.push(\n          `importance_annotations references unknown chunk_id ${annotation.chunk_id}`,\n        );\n        continue;\n      }\n\n      result.push({\n        chunkId: annotation.chunk_id,\n        importance: expectChunkImportance(annotation.importance),\n      });\n    }\n\n    return result;\n  }\n\n  #filterAndValidateLinks(input: {\n    issues: string[];\n    links: readonly ChunkLink[];\n    tempIds: readonly string[];\n    visibleChunkIds: readonly number[];\n  }): ChunkLink[] {\n    const validTempIds = createMembershipRecord(input.tempIds);\n    const validChunkIds = createMembershipRecord(input.visibleChunkIds);\n    const retainedLinks: ChunkLink[] = [];\n\n    for (const [index, link] of input.links.entries()) {\n      const fromValid = this.#validateLinkReference({\n        fieldName: \"from\",\n        index: index + 1,\n        issues: input.issues,\n        reference: link.from,\n        validChunkIds,\n        validTempIds,\n      });\n      const toValid = this.#validateLinkReference({\n        fieldName: \"to\",\n        index: index + 1,\n        issues: input.issues,\n        reference: link.to,\n        validChunkIds,\n        validTempIds,\n      });\n\n      if (fromValid && toValid) {\n        retainedLinks.push(link);\n      }\n    }\n\n    return retainedLinks;\n  }\n\n  #validateLinkReference(input: {\n    fieldName: \"from\" | \"to\";\n    index: number;\n    issues: string[];\n    reference: number | string;\n    validChunkIds: Readonly<Record<string, true>>;\n    validTempIds: Readonly<Record<string, true>>;\n  }): boolean {\n    if (typeof input.reference === \"string\") {\n      return hasMembership(input.validTempIds, input.reference);\n    }\n\n    if (!hasMembership(input.validChunkIds, input.reference)) {\n      input.issues.push(\n        `Link #${input.index}: \"${input.fieldName}\" chunk_id ${input.reference} does not exist in visible chunks`,\n      );\n\n      return false;\n    }\n\n    return true;\n  }\n}\n\nfunction createWordsCountRecord(\n  sentences: readonly Pick<\n    ChunkExtractionSentence,\n    \"sentenceId\" | \"wordsCount\"\n  >[],\n): Readonly<Record<string, number>> {\n  const wordsCountByKey = createEmptyRecord<number>();\n\n  for (const sentence of sentences) {\n    const sentenceKey = createSentenceKey(sentence.sentenceId);\n    wordsCountByKey[sentenceKey] = sentence.wordsCount;\n  }\n\n  return wordsCountByKey;\n}\n\nfunction createSentenceTextRecord(\n  projection: FragmentProjection,\n): Readonly<Record<string, string>> {\n  const record = createEmptyRecord<string>();\n\n  for (const sentence of projection.sentences) {\n    record[createSentenceKey(sentence.sentenceId)] = sentence.rawText;\n  }\n\n  return record;\n}\n\nfunction normalizeChunkLinks(links: readonly RawChunkLink[]): ChunkLink[] {\n  return links.map((link) => {\n    if (link.strength === undefined) {\n      return {\n        from: link.from,\n        to: link.to,\n      };\n    }\n\n    return {\n      from: link.from,\n      strength: link.strength,\n      to: link.to,\n    };\n  });\n}\n\nfunction formatChoiceCandidate(candidate: RankedSentenceCandidate): string {\n  return [\n    candidate.occurrenceId,\n    `prev: ${formatChoiceText(candidate.prevText)}`,\n    `text: ${formatChoiceText(candidate.text)}`,\n    `next: ${formatChoiceText(candidate.nextText)}`,\n  ].join(\"\\n\");\n}\n\nfunction formatChoiceText(text: string): string {\n  const collapsed = text.replace(/\\s+/gu, \" \").trim();\n\n  return collapsed === \"\" ? \"(none)\" : collapsed;\n}\n\nfunction toChoiceFieldName(value: string): ChoiceFieldName | undefined {\n  return value === \"evidence\" ||\n    value === \"start_anchor\" ||\n    value === \"end_anchor\"\n    ? value\n    : undefined;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null;\n}\n\nfunction formatError(error: unknown): string {\n  if (error instanceof Error) {\n    return error.message;\n  }\n\n  return String(error);\n}\n\nfunction isParsedJsonValidationFailure(error: unknown): boolean {\n  return (\n    error instanceof GuaranteedParseValidationError &&\n    error.cause instanceof ParsedJsonError\n  );\n}\n\nfunction createSentenceKey(sentenceId: SentenceId): string {\n  return sentenceId.join(\":\");\n}\n\nfunction createMembershipRecord(\n  values: readonly (number | string)[],\n): Readonly<Record<string, true>> {\n  const record = createEmptyRecord<true>();\n\n  for (const value of values) {\n    record[String(value)] = true;\n  }\n\n  return record;\n}\n\nfunction hasMembership(\n  record: Readonly<Record<string, true>>,\n  value: number | string,\n): boolean {\n  return hasIndexedValue(record, String(value));\n}\n\nfunction hasIndexedValue<TValue>(\n  record: Readonly<Record<string, TValue>>,\n  key: string,\n): boolean {\n  return Object.hasOwn(record, key);\n}\n\nfunction createEmptyRecord<TValue>(): Record<string, TValue> {\n  return Object.create(null) as Record<string, TValue>;\n}\n\nfunction expectSingleSpan(spans: readonly TextSpan[]): TextSpan | undefined {\n  return spans.length === 1 ? spans[0] : undefined;\n}\n\nfunction createEvidenceSelectionList(\n  evidence: RawChunkEvidence,\n): EvidenceSelectionList | undefined {\n  if (Array.isArray(evidence)) {\n    return evidence.map(createEvidenceSelection);\n  }\n\n  const hasSelectionEvidence =\n    typeof evidence.quote === \"string\" ||\n    typeof evidence.sentence_id === \"string\";\n\n  return hasSelectionEvidence ? createEvidenceSelection(evidence) : undefined;\n}\n\nfunction createEvidenceSelection(evidence: {\n  readonly quote?: unknown;\n  readonly sentence_id?: unknown;\n}): {\n  readonly quote?: string;\n  readonly sentence_id?: string;\n} {\n  return {\n    ...(typeof evidence.quote === \"string\" ? { quote: evidence.quote } : {}),\n    ...(typeof evidence.sentence_id === \"string\"\n      ? { sentence_id: evidence.sentence_id }\n      : {}),\n  };\n}\n\nfunction toEvidenceResolutionFailure(\n  failure: {\n    readonly candidates: readonly EvidenceSelectionCandidate[];\n    readonly code: string;\n    readonly message: string;\n  },\n  fieldName: string,\n): EvidenceResolutionFailure {\n  return {\n    candidates: failure.candidates.map(toRankedSentenceCandidate),\n    code: failure.code,\n    fieldName,\n    message: failure.message,\n  };\n}\n\nfunction toRankedSentenceCandidate(\n  candidate: EvidenceSelectionCandidate,\n): RankedSentenceCandidate {\n  return {\n    exactNormalized: candidate.exactNormalized,\n    exactRaw: candidate.exactRaw,\n    exactSubstring: candidate.exactSubstring,\n    index: candidate.index,\n    nextText: candidate.nextText,\n    occurrenceId: candidate.occurrenceId,\n    prevText: candidate.prevText,\n    score: candidate.score,\n    sentenceId: candidate.sentence.sentenceId,\n    text: candidate.sentence.text,\n  };\n}\n"],"mappings":"AAAA,SAAS,SAAS;AAElB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP;AAAA,EACE;AAAA,EAGA;AAAA,OAGK;AAWP,MAAM,qBAAqB;AAE3B,MAAM,kBAAkB,EAAE,OAAO;AAAA,EAC/B,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;AAAA,EAC5C,UAAU,EAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,IAAI,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;AAC5C,CAAC;AACD,MAAM,8BAA8B,EACjC,OAAO;AAAA,EACN,OAAO,EAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,aAAa,EAAE,OAAO,EAAE,SAAS;AACnC,CAAC,EACA,YAAY;AACf,MAAM,sBAAsB,EAAE,MAAM;AAAA,EAClC,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC;AAAA,EAChC,EAAE,MAAM,2BAA2B;AACrC,CAAC;AAED,MAAM,yBAAyB,EAC5B,OAAO;AAAA,EACN,SAAS,EAAE,OAAO;AAAA,EAClB,UAAU,oBAAoB,QAAQ;AAAA,EACtC,OAAO,EAAE,OAAO;AAAA,EAChB,WAAW,EAAE,KAAK;AAAA,IAChB,eAAe;AAAA,IACf,eAAe;AAAA,IACf,eAAe;AAAA,IACf,eAAe;AAAA,EACjB,CAAC;AAAA,EACD,SAAS,EAAE,OAAO;AACpB,CAAC,EACA,YAAY;AAER,MAAM,4BAA4B,EAAE,OAAO;AAAA,EAChD,QAAQ,EAAE,MAAM,sBAAsB;AAAA,EACtC,kBAAkB,EAAE,OAAO;AAAA,EAC3B,OAAO,EAAE,MAAM,eAAe;AAChC,CAAC;AAED,MAAM,2BAA2B,EAC9B,OAAO;AAAA,EACN,SAAS,EAAE,OAAO;AAAA,EAClB,UAAU,oBAAoB,QAAQ;AAAA,EACtC,YAAY,EAAE,KAAK;AAAA,IACjB,gBAAgB;AAAA,IAChB,gBAAgB;AAAA,IAChB,gBAAgB;AAAA,EAClB,CAAC;AAAA,EACD,OAAO,EAAE,OAAO;AAAA,EAChB,SAAS,EAAE,OAAO;AACpB,CAAC,EACA,YAAY;AAEf,MAAM,6BAA6B,EAAE,OAAO;AAAA,EAC1C,UAAU,EAAE,OAAO,EAAE,IAAI;AAAA,EACzB,YAAY,EAAE,KAAK;AAAA,IACjB,gBAAgB;AAAA,IAChB,gBAAgB;AAAA,IAChB,gBAAgB;AAAA,EAClB,CAAC;AACH,CAAC;AAEM,MAAM,8BAA8B,EAAE,OAAO;AAAA,EAClD,QAAQ,EAAE,MAAM,wBAAwB;AAAA,EACxC,wBAAwB,EAAE,MAAM,0BAA0B;AAAA,EAC1D,OAAO,EAAE,MAAM,eAAe;AAChC,CAAC;AAED,MAAM,uBAAuB,EAAE,OAAO;AAAA,EACpC,QAAQ,EAAE,OAAO;AACnB,CAAC;AAaM,IAAK,qBAAL,kBAAKA,wBAAL;AACL,EAAAA,oBAAA,eAAY;AACZ,EAAAA,oBAAA,gBAAa;AAFH,SAAAA;AAAA,GAAA;AA+BL,MAAM,iBAEX;AAAA,EACS;AAAA,EACA,oBAAoB,IAAI,iBAAiB;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,OAUhB;AACD,SAAK,sBAAsB,MAAM;AACjC,SAAK,iBAAiB,MAAM;AAC5B,SAAK,cAAc,MAAM;AACzB,SAAK,kCAAkC,MAAM;AAC7C,SAAK,iBAAiB,MAAM;AAC5B,SAAK,sBAAsB,MAAM;AACjC,SAAK,mBAAmB,MAAM;AAE9B,SAAK,mBAAmB;AAAA,MACtB,MAAM,WAAW,UAAU,IAAI,CAAC,cAAc;AAAA,QAC5C,YAAY,SAAS;AAAA,QACrB,YAAY,SAAS;AAAA,MACvB,EAAE;AAAA,IACJ;AACA,SAAK,qBAAqB,yBAAyB,MAAM,UAAU;AACnE,SAAK,2BACH,MAAM,4BAA4B,SAC9B,SACA,uBAAuB,MAAM,uBAAuB;AAAA,EAC5D;AAAA,EAEA,MAAa,MACX,YACA,OAG8B;AAC9B,UAAM,SAAmB,CAAC;AAC1B,UAAM,SAA+B,CAAC;AACtC,UAAM,UAAoB,CAAC;AAC3B,UAAM,wBACJ,4BAA4B,aACxB,KAAK;AAAA,MACH,WAAW;AAAA,MACX;AAAA,IACF,IACA;AACN,UAAM,kBACJ,sBAAsB,aAClB,WAAW,mBACX;AACN,UAAM,QAAQ,oBAAoB,WAAW,KAAK;AAElD,eAAW,CAAC,OAAO,IAAI,KAAK,WAAW,OAAO,QAAQ,GAAG;AACvD,YAAM,aAAa,QAAQ;AAC3B,YAAM,cAAwB,CAAC;AAC/B,YAAM,QAAQ,KAAK,MAAM,KAAK;AAC9B,YAAM,UAAU,KAAK,QAAQ,KAAK;AAElC,UAAI,UAAU,IAAI;AAChB,oBAAY;AAAA,UACV,UAAU,UAAU;AAAA,QACtB;AAAA,MACF;AAEA,UAAI,YAAY,IAAI;AAClB,oBAAY;AAAA,UACV,UAAU,UAAU;AAAA,QACtB;AAAA,MACF;AAEA,YAAM,CAAC,oBAAoB,eAAe,IACxC,MAAM,KAAK,sBAAsB;AAAA,QAC/B;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,QACA,yBAAyB,MAAM;AAAA,MACjC,CAAC;AAEH,UAAI,mBAAmB,WAAW,GAAG;AACnC,YAAI,oBAAoB,QAAW;AACjC,sBAAY;AAAA,YACV,UAAU,UAAU,MAAM,KAAK,OAAO,gBAAgB,OAAO;AAAA,UAC/D;AAAA,QACF,OAAO;AACL,sBAAY;AAAA,YACV,UAAU,UAAU,MAAM,KAAK;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAEA,UAAI,YAAY,SAAS,GAAG;AAC1B,eAAO,KAAK,GAAG,WAAW;AAC1B;AAAA,MACF;AAEA,YAAM,oBAAoB,mBAAmB,CAAC;AAE9C,UAAI,sBAAsB,QAAW;AACnC,eAAO;AAAA,UACL,UAAU,UAAU,MAAM,KAAK;AAAA,QACjC;AACA;AAAA,MACF;AAEA,YAAM,kBAAkB,mBAAmB,OAAO,CAAC,KAAK,eAAe;AACrE,eACE,OAAO,KAAK,iBAAiB,kBAAkB,UAAU,CAAC,KAAK;AAAA,MAEnE,GAAG,CAAC;AAEJ,UAAI,KAAK,mBAAmB,6BAA8B;AACxD,cAAM,YAAY;AAElB,eAAO,KAAK;AAAA,UACV;AAAA,UACA,YAAY;AAAA,UACZ,IAAI;AAAA,UACJ;AAAA,UACA,OAAO,CAAC;AAAA,UACR,WAAW,qBAAqB,UAAU,SAAS;AAAA,UACnD,YAAY;AAAA,UACZ,aAAa,CAAC,GAAG,kBAAkB;AAAA,UACnC,YAAY;AAAA,QACd,CAAC;AAAA,MACH,OAAO;AACL,cAAM,YAAY;AAElB,eAAO,KAAK;AAAA,UACV;AAAA,UACA,YAAY;AAAA,UACZ,IAAI;AAAA,UACJ,YAAY,sBAAsB,UAAU,UAAU;AAAA,UACtD;AAAA,UACA,OAAO,CAAC;AAAA,UACR,YAAY;AAAA,UACZ,aAAa,CAAC,GAAG,kBAAkB;AAAA,UACnC,YAAY;AAAA,QACd,CAAC;AAAA,MACH;AAEA,cAAQ,KAAK,KAAK,OAAO;AAAA,IAC3B;AAEA,UAAM,aAAa,KAAK,wBAAwB;AAAA,MAC9C;AAAA,MACA;AAAA,MACA;AAAA,MACA,iBAAiB,KAAK;AAAA,IACxB,CAAC;AAED,QAAI,OAAO,SAAS,GAAG;AACrB,YAAM,IAAI,gBAAgB,MAAM;AAAA,IAClC;AAEA,WAAO;AAAA,MACL,YAAY;AAAA,QACV;AAAA,QACA,OAAO;AAAA,QACP,cAAc;AAAA,QACd;AAAA,QACA,GAAI,0BAA0B,SAC1B,CAAC,IACD,EAAE,sBAAsB;AAAA,MAC9B;AAAA,MACA,GAAI,oBAAoB,SAAY,CAAC,IAAI,EAAE,gBAAgB;AAAA,IAC7D;AAAA,EACF;AAAA,EAEA,MAAa,wBACX,OACmB;AACnB,UAAM,kBAA4B,CAAC;AAEnC,eAAW,cAAc,MAAM,aAAa;AAC1C,YAAM,cAAc,kBAAkB,UAAU;AAChD,YAAM,eAAe,KAAK,mBAAmB,WAAW;AAExD,UAAI,iBAAiB,QAAW;AAC9B,wBAAgB,KAAK,YAAY;AACjC;AAAA,MACF;AAEA,sBAAgB;AAAA,QACd,MAAM,KAAK,oBAAoB,YAAY,UAAU;AAAA,MACvD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,sBACJ,OAMA;AACA,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,UAAa,aAAa,MAAM;AAC/C,aAAO,CAAC,CAAC,GAAG,MAAS;AAAA,IACvB;AAEA,QAAI,CAAC,SAAS,QAAQ,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG;AACnD,aAAO;AAAA,QACL,CAAC;AAAA,QACD;AAAA,UACE,YAAY,CAAC;AAAA,UACb,MAAM;AAAA,UACN,WAAW;AAAA,UACX,SAAS,UAAU,MAAM,UAAU,MAAM,MAAM,UAAU;AAAA,QAC3D;AAAA,MACF;AAAA,IACF;AAEA,UAAM,qBAAqB,KAAK,YAAY;AAC5C,UAAM,uBAAuB,mBAAmB;AAAA,MAC9C,CAAC,aAAa,SAAS;AAAA,IACzB;AACA,UAAM,qBAAqB,mBAAmB,IAAI,CAAC,UAAU,WAAW;AAAA,MACtE,IAAI,IAAI,QAAQ,CAAC;AAAA,MACjB,YAAY,SAAS;AAAA,MACrB,MAAM,SAAS;AAAA,IACjB,EAAE;AACF,UAAM,oBAAoB,4BAA4B,QAAQ;AAC9D,UAAM,CAAC,qBAAqB,gBAAgB,IAC1C,sBAAsB,SAClB,CAAC,QAAW,MAAS,IACrB,6BAA6B;AAAA,MAC3B,UAAU;AAAA,MACV,WAAW;AAAA,IACb,CAAC;AAEP,QAAI,wBAAwB,QAAW;AACrC,aAAO,CAAC,oBAAoB,aAAa,MAAS;AAAA,IACpD;AAEA,UAAM,wBAAwB,MAAM,QAAQ,QAAQ,IAChD,CAAC,IACD,KAAK,gCAAgC,QAAQ;AAEjD,QAAI,sBAAsB,SAAS,GAAG;AACpC,aAAO,CAAC,uBAAuB,MAAS;AAAA,IAC1C;AAEA,UAAM,iBAAiB,mBAAmB;AAAA,MACxC,CAAC,aAAa,SAAS;AAAA,IACzB;AACA,UAAM,CAAC,YAAY,OAAO,IAAI,MAAM,QAAQ,QAAQ,IAChD,CAAC,QAAW,MAAS,IACrB,KAAK,kBAAkB;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEJ,QAAI,eAAe,QAAW;AAC5B,aAAO,CAAC,WAAW,aAAa,MAAS;AAAA,IAC3C;AAEA,UAAM,kBACJ,qBAAqB,SACjB,UACA,4BAA4B,kBAAkB,UAAU;AAE9D,QAAI,oBAAoB,QAAW;AACjC,aAAO,CAAC,CAAC,GAAG,MAAS;AAAA,IACvB;AAEA,UAAM,kBACJ,gBAAgB,KAAK,WAAW,WAAW,KAC1C,gBAAgB,SAAS,oBACxB,MAAM,2BACN,gBAAgB,WAAW,SAAS;AAExC,QAAI,CAAC,iBAAiB;AACpB,aAAO,CAAC,CAAC,GAAG,eAAe;AAAA,IAC7B;AAEA,UAAM,kBAAkB,kBAAkB,gBAAgB,SAAS;AAEnE,QAAI,oBAAoB,QAAW;AACjC,aAAO,CAAC,CAAC,GAAG,eAAe;AAAA,IAC7B;AAEA,UAAM,CAAC,iBAAiB,aAAa,IACnC,MAAM,KAAK,0BAA0B;AAAA,MACnC,YAAY,gBAAgB;AAAA,MAC5B,WAAW,MAAM;AAAA,MACjB,YAAY,MAAM;AAAA,MAClB,YAAY,MAAM;AAAA,MAClB,WAAW;AAAA,IACb,CAAC;AAEH,QAAI,kBAAkB,QAAW;AAC/B,UAAI,cAAc,SAAS,qCAAqC;AAC9D,eAAO,CAAC,sBAAsB,MAAS;AAAA,MACzC;AAEA,aAAO,CAAC,CAAC,GAAG,aAAa;AAAA,IAC3B;AAEA,QAAI,oBAAoB,QAAW;AACjC,aAAO;AAAA,QACL,CAAC;AAAA,QACD;AAAA,UACE,YAAY,gBAAgB;AAAA,UAC5B,MAAM;AAAA,UACN,WAAW,gBAAgB;AAAA,UAC3B,SAAS,kCAAkC,gBAAgB,SAAS;AAAA,QACtE;AAAA,MACF;AAAA,IACF;AAEA,QAAI,oBAAoB,YAAY;AAClC,aAAO,CAAC,CAAC,gBAAgB,UAAU,GAAG,MAAS;AAAA,IACjD;AAEA,QAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,aAAO,CAAC,CAAC,GAAG,eAAe;AAAA,IAC7B;AAEA,QAAI,MAAM,yBAAyB;AACjC,YAAM,CAACC,WAAUC,eAAc,IAC7B,KAAK,kBAAkB,qBAAqB;AAAA,QAC1C;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW;AAAA,UACT,CAAC,eAAe,GAAG;AAAA,QACrB;AAAA,MACF,CAAC;AAEH,UAAID,cAAa,QAAW;AAC1B,eAAO,CAACA,UAAS,aAAa,MAAS;AAAA,MACzC;AAEA,aAAO,CAAC,CAAC,GAAGC,eAAc;AAAA,IAC5B;AAEA,UAAM,mBAA4C;AAAA,MAChD,GAAG;AAAA,MACH,CAAC,eAAe,GAAG;AAAA,QACjB,MAAM;AAAA,QACN,MAAM,gBAAgB;AAAA,MACxB;AAAA,IACF;AACA,UAAM,CAAC,UAAU,cAAc,IAAI,KAAK,kBAAkB;AAAA,MACxD;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI,aAAa,QAAW;AAC1B,aAAO,CAAC,SAAS,aAAa,MAAS;AAAA,IACzC;AAEA,WAAO,CAAC,CAAC,GAAG,cAAc;AAAA,EAC5B;AAAA,EAEA,gCACE,UACc;AACd,UAAM,aAAa,SAAS,gBAAgB,SAAS;AACrD,UAAM,CAAC,aAAa,YAAY,IAAI,KAAK,kBAAkB;AAAA,MACzD;AAAA,MACA;AAAA,IACF;AAEA,QAAI,iBAAiB,UAAa,aAAa,SAAS,QAAW;AACjE,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,aAAa;AAAA,MACjB,KAAK,YAAY,iBAAiB,YAAY,IAAI;AAAA,IACpD;AAEA,QAAI,eAAe,QAAW;AAC5B,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,WAAW,SAAS,cAAc,SAAS;AAEjD,QAAI,aAAa,QAAW;AAC1B,aAAO,KAAK,YAAY,mBAAmB,UAAU;AAAA,IACvD;AAEA,UAAM,CAAC,WAAW,UAAU,IAAI,KAAK,kBAAkB;AAAA,MACrD;AAAA,MACA;AAAA,IACF;AAEA,QAAI,eAAe,UAAa,WAAW,SAAS,QAAW;AAC7D,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,WAAW;AAAA,MACf,KAAK,YAAY,iBAAiB,UAAU,IAAI;AAAA,IAClD;AAEA,QAAI,aAAa,UAAa,SAAS,SAAS,WAAW,QAAQ;AACjE,aAAO,CAAC;AAAA,IACV;AAEA,WAAO,KAAK,YAAY,mBAAmB;AAAA,MACzC,QAAQ,SAAS,SAAS,SAAS,SAAS,WAAW;AAAA,MACvD,QAAQ,WAAW;AAAA,IACrB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,0BACJ,OAMA;AACA,UAAM,WAAW,KAAK,qBAAqB,KAAK;AAChD,UAAM,eAAe,MAAM,WAAW;AAAA,MACpC,CAAC,cAAc,UAAU;AAAA,IAC3B;AAEA,QAAI;AACF,YAAM,SAAS,MAAM,sBAAsB;AAAA,QACzC,YAAY;AAAA,QACZ;AAAA,QACA,OAAO,CAAC,SAAS;AACf,gBAAM,YAAY,MAAM,WAAW;AAAA,YACjC,CAAC,SAAS,KAAK,iBAAiB,KAAK;AAAA,UACvC;AAEA,cAAI,cAAc,QAAW;AAC3B,kBAAM,IAAI,gBAAgB;AAAA,cACxB,mBAAmB,KAAK,MAAM,uBAAuB,aAAa,KAAK,IAAI,CAAC;AAAA,YAC9E,CAAC;AAAA,UACH;AAEA,iBAAO;AAAA,QACT;AAAA,QACA,gCAAgC,KAAK;AAAA,QACrC,SAAS,KAAK;AAAA,QACd,QAAQ;AAAA,MACV,CAAC;AAED,aAAO,CAAC,QAAQ,MAAS;AAAA,IAC3B,SAAS,OAAO;AACd,UAAI,8BAA8B,KAAK,GAAG;AACxC,eAAO;AAAA,UACL;AAAA,UACA;AAAA,YACE,YAAY,MAAM;AAAA,YAClB,MAAM;AAAA,YACN,WAAW,MAAM;AAAA,YACjB,SACE,mDAAmD,MAAM,SAAS;AAAA,UAEtE;AAAA,QACF;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,UACE,YAAY,MAAM;AAAA,UAClB,MAAM;AAAA,UACN,WAAW,MAAM;AAAA,UACjB,SAAS,kCAAkC,MAAM,SAAS,KAAK,YAAY,KAAK,CAAC;AAAA,QACnF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,qBAAqB,OAAmD;AACtE,WAAO;AAAA,MACL;AAAA,QACE,SAAS,KAAK;AAAA,QACd,MAAM;AAAA,MACR;AAAA,MACA;AAAA,QACE,SACE;AAAA;AAAA,EACe,KAAK,UAAU,MAAM,WAAW,MAAM,CAAC,CAAC;AAAA;AAAA;AAAA,4BAC1B,MAAM,SAAS,gBAAgB,MAAM,UAAU,KAAK,MAAM,UAAU;AAAA;AAAA;AAAA,IAEjG,MAAM,WAAW,IAAI,qBAAqB,EAAE,KAAK,IAAI;AAAA,QACvD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAAA,EAEA,+BACE,aAIA,QACyC;AACzC,QAAI,YAAY,WAAW,GAAG;AAC5B,aAAO,CAAC;AAAA,IACV;AAEA,QAAI,KAAK,6BAA6B,QAAW;AAC/C,aAAO,YAAY,IAAI,CAAC,gBAAgB;AAAA,QACtC,SAAS,WAAW;AAAA,QACpB,YAAY,sBAAsB,WAAW,UAAU;AAAA,MACzD,EAAE;AAAA,IACJ;AAEA,UAAM,SAAsC,CAAC;AAE7C,eAAW,cAAc,aAAa;AACpC,UAAI,CAAC,cAAc,KAAK,0BAA0B,WAAW,QAAQ,GAAG;AACtE,eAAO;AAAA,UACL,sDAAsD,WAAW,QAAQ;AAAA,QAC3E;AACA;AAAA,MACF;AAEA,aAAO,KAAK;AAAA,QACV,SAAS,WAAW;AAAA,QACpB,YAAY,sBAAsB,WAAW,UAAU;AAAA,MACzD,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,wBAAwB,OAKR;AACd,UAAM,eAAe,uBAAuB,MAAM,OAAO;AACzD,UAAM,gBAAgB,uBAAuB,MAAM,eAAe;AAClE,UAAM,gBAA6B,CAAC;AAEpC,eAAW,CAAC,OAAO,IAAI,KAAK,MAAM,MAAM,QAAQ,GAAG;AACjD,YAAM,YAAY,KAAK,uBAAuB;AAAA,QAC5C,WAAW;AAAA,QACX,OAAO,QAAQ;AAAA,QACf,QAAQ,MAAM;AAAA,QACd,WAAW,KAAK;AAAA,QAChB;AAAA,QACA;AAAA,MACF,CAAC;AACD,YAAM,UAAU,KAAK,uBAAuB;AAAA,QAC1C,WAAW;AAAA,QACX,OAAO,QAAQ;AAAA,QACf,QAAQ,MAAM;AAAA,QACd,WAAW,KAAK;AAAA,QAChB;AAAA,QACA;AAAA,MACF,CAAC;AAED,UAAI,aAAa,SAAS;AACxB,sBAAc,KAAK,IAAI;AAAA,MACzB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,OAOX;AACV,QAAI,OAAO,MAAM,cAAc,UAAU;AACvC,aAAO,cAAc,MAAM,cAAc,MAAM,SAAS;AAAA,IAC1D;AAEA,QAAI,CAAC,cAAc,MAAM,eAAe,MAAM,SAAS,GAAG;AACxD,YAAM,OAAO;AAAA,QACX,SAAS,MAAM,KAAK,MAAM,MAAM,SAAS,cAAc,MAAM,SAAS;AAAA,MACxE;AAEA,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AACF;AAEA,SAAS,uBACP,WAIkC;AAClC,QAAM,kBAAkB,kBAA0B;AAElD,aAAW,YAAY,WAAW;AAChC,UAAM,cAAc,kBAAkB,SAAS,UAAU;AACzD,oBAAgB,WAAW,IAAI,SAAS;AAAA,EAC1C;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,YACkC;AAClC,QAAM,SAAS,kBAA0B;AAEzC,aAAW,YAAY,WAAW,WAAW;AAC3C,WAAO,kBAAkB,SAAS,UAAU,CAAC,IAAI,SAAS;AAAA,EAC5D;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAA6C;AACxE,SAAO,MAAM,IAAI,CAAC,SAAS;AACzB,QAAI,KAAK,aAAa,QAAW;AAC/B,aAAO;AAAA,QACL,MAAM,KAAK;AAAA,QACX,IAAI,KAAK;AAAA,MACX;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM,KAAK;AAAA,MACX,UAAU,KAAK;AAAA,MACf,IAAI,KAAK;AAAA,IACX;AAAA,EACF,CAAC;AACH;AAEA,SAAS,sBAAsB,WAA4C;AACzE,SAAO;AAAA,IACL,UAAU;AAAA,IACV,SAAS,iBAAiB,UAAU,QAAQ,CAAC;AAAA,IAC7C,SAAS,iBAAiB,UAAU,IAAI,CAAC;AAAA,IACzC,SAAS,iBAAiB,UAAU,QAAQ,CAAC;AAAA,EAC/C,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,iBAAiB,MAAsB;AAC9C,QAAM,YAAY,KAAK,QAAQ,SAAS,GAAG,EAAE,KAAK;AAElD,SAAO,cAAc,KAAK,WAAW;AACvC;AAEA,SAAS,kBAAkB,OAA4C;AACrE,SAAO,UAAU,cACf,UAAU,kBACV,UAAU,eACR,QACA;AACN;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAEA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;AAAA,EACf;AAEA,SAAO,OAAO,KAAK;AACrB;AAEA,SAAS,8BAA8B,OAAyB;AAC9D,SACE,iBAAiB,kCACjB,MAAM,iBAAiB;AAE3B;AAEA,SAAS,kBAAkB,YAAgC;AACzD,SAAO,WAAW,KAAK,GAAG;AAC5B;AAEA,SAAS,uBACP,QACgC;AAChC,QAAM,SAAS,kBAAwB;AAEvC,aAAW,SAAS,QAAQ;AAC1B,WAAO,OAAO,KAAK,CAAC,IAAI;AAAA,EAC1B;AAEA,SAAO;AACT;AAEA,SAAS,cACP,QACA,OACS;AACT,SAAO,gBAAgB,QAAQ,OAAO,KAAK,CAAC;AAC9C;AAEA,SAAS,gBACP,QACA,KACS;AACT,SAAO,OAAO,OAAO,QAAQ,GAAG;AAClC;AAEA,SAAS,oBAAoD;AAC3D,SAAO,uBAAO,OAAO,IAAI;AAC3B;AAEA,SAAS,iBAAiB,OAAkD;AAC1E,SAAO,MAAM,WAAW,IAAI,MAAM,CAAC,IAAI;AACzC;AAEA,SAAS,4BACP,UACmC;AACnC,MAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,WAAO,SAAS,IAAI,uBAAuB;AAAA,EAC7C;AAEA,QAAM,uBACJ,OAAO,SAAS,UAAU,YAC1B,OAAO,SAAS,gBAAgB;AAElC,SAAO,uBAAuB,wBAAwB,QAAQ,IAAI;AACpE;AAEA,SAAS,wBAAwB,UAM/B;AACA,SAAO;AAAA,IACL,GAAI,OAAO,SAAS,UAAU,WAAW,EAAE,OAAO,SAAS,MAAM,IAAI,CAAC;AAAA,IACtE,GAAI,OAAO,SAAS,gBAAgB,WAChC,EAAE,aAAa,SAAS,YAAY,IACpC,CAAC;AAAA,EACP;AACF;AAEA,SAAS,4BACP,SAKA,WAC2B;AAC3B,SAAO;AAAA,IACL,YAAY,QAAQ,WAAW,IAAI,yBAAyB;AAAA,IAC5D,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,SAAS,QAAQ;AAAA,EACnB;AACF;AAEA,SAAS,0BACP,WACyB;AACzB,SAAO;AAAA,IACL,iBAAiB,UAAU;AAAA,IAC3B,UAAU,UAAU;AAAA,IACpB,gBAAgB,UAAU;AAAA,IAC1B,OAAO,UAAU;AAAA,IACjB,UAAU,UAAU;AAAA,IACpB,cAAc,UAAU;AAAA,IACxB,UAAU,UAAU;AAAA,IACpB,OAAO,UAAU;AAAA,IACjB,YAAY,UAAU,SAAS;AAAA,IAC/B,MAAM,UAAU,SAAS;AAAA,EAC3B;AACF;","names":["ChunkMetadataField","resolved","resolveFailure"]}

package/dist/reader/chunk-batch/prompt-templates.d.ts

@@ -0,0 +1,6 @@
+declare const USER_FOCUSED_PROMPT_TEMPLATE = "topologization/user_focused_extraction";
+declare const BOOK_COHERENCE_PROMPT_TEMPLATE = "topologization/book_coherence_extraction";
+declare const EVIDENCE_CHOICE_PROMPT_TEMPLATE = "topologization/evidence_choice";
+declare const TRANSLATE_CHUNKS_PROMPT_TEMPLATE = "topologization/translate_chunks";
+
+export { BOOK_COHERENCE_PROMPT_TEMPLATE, EVIDENCE_CHOICE_PROMPT_TEMPLATE, TRANSLATE_CHUNKS_PROMPT_TEMPLATE, USER_FOCUSED_PROMPT_TEMPLATE };

package/dist/reader/chunk-batch/prompt-templates.js

@@ -0,0 +1,11 @@
+const USER_FOCUSED_PROMPT_TEMPLATE = "topologization/user_focused_extraction";
+const BOOK_COHERENCE_PROMPT_TEMPLATE = "topologization/book_coherence_extraction";
+const EVIDENCE_CHOICE_PROMPT_TEMPLATE = "topologization/evidence_choice";
+const TRANSLATE_CHUNKS_PROMPT_TEMPLATE = "topologization/translate_chunks";
+export {
+  BOOK_COHERENCE_PROMPT_TEMPLATE,
+  EVIDENCE_CHOICE_PROMPT_TEMPLATE,
+  TRANSLATE_CHUNKS_PROMPT_TEMPLATE,
+  USER_FOCUSED_PROMPT_TEMPLATE
+};
+//# sourceMappingURL=prompt-templates.js.map

package/dist/reader/chunk-batch/prompt-templates.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/chunk-batch/prompt-templates.ts"],"sourcesContent":["export const USER_FOCUSED_PROMPT_TEMPLATE =\n  \"topologization/user_focused_extraction\";\nexport const BOOK_COHERENCE_PROMPT_TEMPLATE =\n  \"topologization/book_coherence_extraction\";\nexport const EVIDENCE_CHOICE_PROMPT_TEMPLATE = \"topologization/evidence_choice\";\nexport const TRANSLATE_CHUNKS_PROMPT_TEMPLATE =\n  \"topologization/translate_chunks\";\n"],"mappings":"AAAO,MAAM,+BACX;AACK,MAAM,iCACX;AACK,MAAM,kCAAkC;AACxC,MAAM,mCACX;","names":[]}

package/dist/reader/chunk-batch/types.d.ts

@@ -0,0 +1,85 @@
+import { Language } from '../../common/language.js';
+import { LLM } from '../../llm/client.js';
+import { SentenceId, ChunkRetention, ChunkImportance } from '../../document/types.js';
+import '../../llm/context.js';
+import '../../llm/cache.js';
+import '../../llm/types.js';
+import 'ai';
+
+interface ChunkLink {
+    readonly from: number | string;
+    readonly strength?: string;
+    readonly to: number | string;
+}
+interface ChunkImportanceAnnotation {
+    readonly chunkId: number;
+    readonly importance: ChunkImportance;
+}
+interface CognitiveChunk {
+    id: number;
+    generation: number;
+    sentenceId: SentenceId;
+    label: string;
+    content: string;
+    sentenceIds: SentenceId[];
+    links: number[];
+    retention?: ChunkRetention;
+    importance?: ChunkImportance;
+    wordsCount: number;
+}
+interface ChunkBatch {
+    readonly chunks: CognitiveChunk[];
+    readonly tempIds: string[];
+    readonly links: readonly ChunkLink[];
+    readonly orderCorrect: boolean;
+    readonly importanceAnnotations?: readonly ChunkImportanceAnnotation[];
+}
+interface ChunkExtractionScopes<S extends string> {
+    readonly choice: S;
+    readonly extraction: S;
+}
+interface SentenceTextSource {
+    getSentence(sentenceId: SentenceId): Promise<string>;
+}
+interface ChunkTranslationInput {
+    readonly content: string;
+    readonly id: number;
+    readonly label: string;
+    readonly sourceSentences: readonly string[];
+}
+interface ChunkTranslationOutput {
+    readonly content: string;
+    readonly id: number;
+    readonly label: string;
+}
+interface ChunkBatchOptions<S extends string> {
+    readonly extractionGuidance: string;
+    readonly llm: LLM<S>;
+    readonly scopes: ChunkExtractionScopes<S>;
+    readonly sentenceTextSource: SentenceTextSource;
+    readonly userLanguage?: Language;
+}
+interface ChunkExtractionSentence {
+    readonly sentenceId: SentenceId;
+    readonly text: string;
+    readonly wordsCount: number;
+}
+interface ExtractUserFocusedInput {
+    readonly text: string;
+    readonly workingMemoryPrompt: string;
+    readonly visibleChunkIds: readonly number[];
+    readonly sentences: readonly ChunkExtractionSentence[];
+}
+interface ExtractUserFocusedResult {
+    readonly chunkBatch: ChunkBatch;
+    readonly fragmentSummary: string;
+}
+interface ExtractBookCoherenceInput {
+    readonly text: string;
+    readonly workingMemoryPrompt: string;
+    readonly visibleChunkIds: readonly number[];
+    readonly sentences: readonly ChunkExtractionSentence[];
+    readonly userFocusedChunks: readonly CognitiveChunk[];
+}
+
+export type { ChunkBatch, ChunkBatchOptions, ChunkExtractionScopes, ChunkExtractionSentence, ChunkImportanceAnnotation, ChunkLink, ChunkTranslationInput, ChunkTranslationOutput, CognitiveChunk, ExtractBookCoherenceInput, ExtractUserFocusedInput, ExtractUserFocusedResult, SentenceTextSource };

package/dist/reader/chunk-batch/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/reader/chunk-batch/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/reader/index.d.ts

@@ -0,0 +1,13 @@
+export { Reader } from './reader.js';
+export { segmentTextStream } from './segment/segment.js';
+export { ReaderChunk, ReaderGraphDelta, ReaderGraphEdge, ReaderImportanceAnnotation, ReaderOptions, ReaderSegment, ReaderSegmenter, ReaderSentence, ReaderTextStream } from './types.js';
+import './segment/types.js';
+import './attention/attention.js';
+import './chunk-batch/types.js';
+import '../common/language.js';
+import '../llm/client.js';
+import '../llm/context.js';
+import '../llm/cache.js';
+import '../llm/types.js';
+import 'ai';
+import '../document/types.js';

package/dist/reader/index.js

@@ -0,0 +1,7 @@
+import { Reader } from "./reader.js";
+import { segmentTextStream } from "./segment/segment.js";
+export {
+  Reader,
+  segmentTextStream
+};
+//# sourceMappingURL=index.js.map

package/dist/reader/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/reader/index.ts"],"sourcesContent":["export { Reader } from \"./reader.js\";\nexport { segmentTextStream } from \"./segment/segment.js\";\nexport type {\n  ReaderChunk,\n  ReaderGraphDelta,\n  ReaderGraphEdge,\n  ReaderImportanceAnnotation,\n  ReaderOptions,\n  ReaderSegment,\n  ReaderSegmenter,\n  ReaderSentence,\n  ReaderTextStream,\n} from \"./types.js\";\n"],"mappings":"AAAA,SAAS,cAAc;AACvB,SAAS,yBAAyB;","names":[]}

package/dist/reader/reader.d.ts

@@ -0,0 +1,37 @@
+import { ReaderOptions, ReaderTextStream, ReaderSegment, ReaderSentence, ReaderGraphDelta, ReaderChunk } from './types.js';
+import './attention/attention.js';
+import './chunk-batch/types.js';
+import '../common/language.js';
+import '../llm/client.js';
+import '../llm/context.js';
+import '../llm/cache.js';
+import '../llm/types.js';
+import 'ai';
+import '../document/types.js';
+import './segment/types.js';
+
+declare class Reader<S extends string> {
+    #private;
+    constructor(options: ReaderOptions<S>);
+    get capacity(): number;
+    segment(stream: ReaderTextStream): AsyncIterable<ReaderSegment>;
+    extractUserFocused(input: {
+        readonly sentences: readonly ReaderSentence[];
+        readonly text: string;
+    }): Promise<{
+        readonly delta: ReaderGraphDelta;
+        readonly fragmentSummary: string;
+    }>;
+    extractBookCoherence(input: {
+        readonly sentences: readonly ReaderSentence[];
+        readonly text: string;
+        readonly userFocusedChunks: readonly ReaderChunk[];
+    }): Promise<ReaderGraphDelta>;
+    completeFragment(input: {
+        readonly allChunks: readonly ReaderChunk[];
+        readonly getSuccessorChunkIds: (chunkId: number) => readonly number[];
+    }): void;
+    clear(): void;
+}
+
+export { Reader };

package/dist/reader/reader.js

@@ -0,0 +1,82 @@
+import { Attention } from "./attention/attention.js";
+import {
+  extractBookCoherenceChunkBatch,
+  extractUserFocusedChunkBatch
+} from "./chunk-batch/extract.js";
+import { segmentTextStream } from "./segment/segment.js";
+class Reader {
+  #attention;
+  #chunkBatchOptions;
+  #segmenter;
+  constructor(options) {
+    this.#attention = new Attention(
+      options.attention.capacity,
+      options.attention.generationDecayFactor,
+      options.attention.idGenerator
+    );
+    this.#chunkBatchOptions = {
+      extractionGuidance: options.extractionGuidance,
+      llm: options.llm,
+      scopes: options.scopes,
+      sentenceTextSource: options.sentenceTextSource,
+      ...options.userLanguage === void 0 ? {} : {
+        userLanguage: options.userLanguage
+      }
+    };
+    this.#segmenter = options.segmenter;
+  }
+  get capacity() {
+    return this.#attention.capacity;
+  }
+  segment(stream) {
+    if (this.#segmenter === void 0) {
+      return segmentTextStream(stream);
+    }
+    return segmentTextStream(stream, {
+      adapter: this.#segmenter
+    });
+  }
+  async extractUserFocused(input) {
+    const context = this.#attention.createChunkBatchContext();
+    const result = await extractUserFocusedChunkBatch(this.#chunkBatchOptions, {
+      sentences: input.sentences,
+      text: input.text,
+      visibleChunkIds: context.visibleChunkIds,
+      workingMemoryPrompt: context.workingMemoryPrompt
+    });
+    return {
+      delta: await this.#attention.acceptChunkBatch(result.chunkBatch),
+      fragmentSummary: result.fragmentSummary
+    };
+  }
+  async extractBookCoherence(input) {
+    const context = this.#attention.createChunkBatchContext({
+      includeCurrentFragment: false
+    });
+    const linkableChunkIds = [
+      ...context.visibleChunkIds,
+      ...input.userFocusedChunks.map((chunk) => chunk.id)
+    ];
+    const chunkBatch = await extractBookCoherenceChunkBatch(
+      this.#chunkBatchOptions,
+      {
+        sentences: input.sentences,
+        text: input.text,
+        userFocusedChunks: input.userFocusedChunks,
+        visibleChunkIds: linkableChunkIds,
+        workingMemoryPrompt: context.workingMemoryPrompt
+      }
+    );
+    return await this.#attention.acceptChunkBatch(chunkBatch);
+  }
+  completeFragment(input) {
+    this.#attention.completeFragment(input);
+  }
+  clear() {
+    this.#attention.clear();
+  }
+}
+export {
+  Reader
+};
+//# sourceMappingURL=reader.js.map

package/dist/reader/reader.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/reader/reader.ts"],"sourcesContent":["import { Attention } from \"./attention/attention.js\";\nimport {\n  extractBookCoherenceChunkBatch,\n  extractUserFocusedChunkBatch,\n} from \"./chunk-batch/extract.js\";\nimport { segmentTextStream } from \"./segment/segment.js\";\nimport type { ChunkBatchOptions } from \"./chunk-batch/types.js\";\nimport type {\n  ReaderChunk,\n  ReaderGraphDelta,\n  ReaderOptions,\n  ReaderSegment,\n  ReaderSentence,\n  ReaderTextStream,\n} from \"./types.js\";\n\nexport class Reader<S extends string> {\n  readonly #attention: Attention;\n  readonly #chunkBatchOptions: ChunkBatchOptions<S>;\n  readonly #segmenter: ReaderOptions<S>[\"segmenter\"];\n\n  public constructor(options: ReaderOptions<S>) {\n    this.#attention = new Attention(\n      options.attention.capacity,\n      options.attention.generationDecayFactor,\n      options.attention.idGenerator,\n    );\n    this.#chunkBatchOptions = {\n      extractionGuidance: options.extractionGuidance,\n      llm: options.llm,\n      scopes: options.scopes,\n      sentenceTextSource: options.sentenceTextSource,\n      ...(options.userLanguage === undefined\n        ? {}\n        : {\n            userLanguage: options.userLanguage,\n          }),\n    };\n    this.#segmenter = options.segmenter;\n  }\n\n  public get capacity(): number {\n    return this.#attention.capacity;\n  }\n\n  public segment(stream: ReaderTextStream): AsyncIterable<ReaderSegment> {\n    if (this.#segmenter === undefined) {\n      return segmentTextStream(stream);\n    }\n\n    return segmentTextStream(stream, {\n      adapter: this.#segmenter,\n    });\n  }\n\n  public async extractUserFocused(input: {\n    readonly sentences: readonly ReaderSentence[];\n    readonly text: string;\n  }): Promise<{\n    readonly delta: ReaderGraphDelta;\n    readonly fragmentSummary: string;\n  }> {\n    const context = this.#attention.createChunkBatchContext();\n    const result = await extractUserFocusedChunkBatch(this.#chunkBatchOptions, {\n      sentences: input.sentences,\n      text: input.text,\n      visibleChunkIds: context.visibleChunkIds,\n      workingMemoryPrompt: context.workingMemoryPrompt,\n    });\n\n    return {\n      delta: await this.#attention.acceptChunkBatch(result.chunkBatch),\n      fragmentSummary: result.fragmentSummary,\n    };\n  }\n\n  public async extractBookCoherence(input: {\n    readonly sentences: readonly ReaderSentence[];\n    readonly text: string;\n    readonly userFocusedChunks: readonly ReaderChunk[];\n  }): Promise<ReaderGraphDelta> {\n    const context = this.#attention.createChunkBatchContext({\n      includeCurrentFragment: false,\n    });\n    const linkableChunkIds = [\n      ...context.visibleChunkIds,\n      ...input.userFocusedChunks.map((chunk) => chunk.id),\n    ];\n    const chunkBatch = await extractBookCoherenceChunkBatch(\n      this.#chunkBatchOptions,\n      {\n        sentences: input.sentences,\n        text: input.text,\n        userFocusedChunks: input.userFocusedChunks,\n        visibleChunkIds: linkableChunkIds,\n        workingMemoryPrompt: context.workingMemoryPrompt,\n      },\n    );\n\n    return await this.#attention.acceptChunkBatch(chunkBatch);\n  }\n\n  public completeFragment(input: {\n    readonly allChunks: readonly ReaderChunk[];\n    readonly getSuccessorChunkIds: (chunkId: number) => readonly number[];\n  }): void {\n    this.#attention.completeFragment(input);\n  }\n\n  public clear(): void {\n    this.#attention.clear();\n  }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAW3B,MAAM,OAAyB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,SAA2B;AAC5C,SAAK,aAAa,IAAI;AAAA,MACpB,QAAQ,UAAU;AAAA,MAClB,QAAQ,UAAU;AAAA,MAClB,QAAQ,UAAU;AAAA,IACpB;AACA,SAAK,qBAAqB;AAAA,MACxB,oBAAoB,QAAQ;AAAA,MAC5B,KAAK,QAAQ;AAAA,MACb,QAAQ,QAAQ;AAAA,MAChB,oBAAoB,QAAQ;AAAA,MAC5B,GAAI,QAAQ,iBAAiB,SACzB,CAAC,IACD;AAAA,QACE,cAAc,QAAQ;AAAA,MACxB;AAAA,IACN;AACA,SAAK,aAAa,QAAQ;AAAA,EAC5B;AAAA,EAEA,IAAW,WAAmB;AAC5B,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEO,QAAQ,QAAwD;AACrE,QAAI,KAAK,eAAe,QAAW;AACjC,aAAO,kBAAkB,MAAM;AAAA,IACjC;AAEA,WAAO,kBAAkB,QAAQ;AAAA,MAC/B,SAAS,KAAK;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,MAAa,mBAAmB,OAM7B;AACD,UAAM,UAAU,KAAK,WAAW,wBAAwB;AACxD,UAAM,SAAS,MAAM,6BAA6B,KAAK,oBAAoB;AAAA,MACzE,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,MACZ,iBAAiB,QAAQ;AAAA,MACzB,qBAAqB,QAAQ;AAAA,IAC/B,CAAC;AAED,WAAO;AAAA,MACL,OAAO,MAAM,KAAK,WAAW,iBAAiB,OAAO,UAAU;AAAA,MAC/D,iBAAiB,OAAO;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAa,qBAAqB,OAIJ;AAC5B,UAAM,UAAU,KAAK,WAAW,wBAAwB;AAAA,MACtD,wBAAwB;AAAA,IAC1B,CAAC;AACD,UAAM,mBAAmB;AAAA,MACvB,GAAG,QAAQ;AAAA,MACX,GAAG,MAAM,kBAAkB,IAAI,CAAC,UAAU,MAAM,EAAE;AAAA,IACpD;AACA,UAAM,aAAa,MAAM;AAAA,MACvB,KAAK;AAAA,MACL;AAAA,QACE,WAAW,MAAM;AAAA,QACjB,MAAM,MAAM;AAAA,QACZ,mBAAmB,MAAM;AAAA,QACzB,iBAAiB;AAAA,QACjB,qBAAqB,QAAQ;AAAA,MAC/B;AAAA,IACF;AAEA,WAAO,MAAM,KAAK,WAAW,iBAAiB,UAAU;AAAA,EAC1D;AAAA,EAEO,iBAAiB,OAGf;AACP,SAAK,WAAW,iBAAiB,KAAK;AAAA,EACxC;AAAA,EAEO,QAAc;AACnB,SAAK,WAAW,MAAM;AAAA,EACxB;AACF;","names":[]}

package/dist/reader/segment/index.d.ts

@@ -0,0 +1,2 @@
+export { segmentTextStream } from './segment.js';
+export { SegmentTextStreamOptions, SentenceStreamAdapter, SentenceStreamItem, TextStream } from './types.js';

package/dist/reader/segment/index.js

@@ -0,0 +1,5 @@
+import { segmentTextStream } from "./segment.js";
+export {
+  segmentTextStream
+};
+//# sourceMappingURL=index.js.map

package/dist/reader/segment/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/segment/index.ts"],"sourcesContent":["export { segmentTextStream } from \"./segment.js\";\nexport type {\n  SegmentTextStreamOptions,\n  SentenceStreamAdapter,\n  SentenceStreamItem,\n  TextStream,\n} from \"./types.js\";\n"],"mappings":"AAAA,SAAS,yBAAyB;","names":[]}

package/dist/reader/segment/intl-segmenter.d.ts

@@ -0,0 +1,5 @@
+import { SentenceStreamAdapter } from './types.js';
+
+declare function createDefaultSentenceStreamAdapter(): SentenceStreamAdapter;
+
+export { createDefaultSentenceStreamAdapter };

package/dist/reader/segment/intl-segmenter.js

@@ -0,0 +1,102 @@
+const SENTENCE_BOUNDARY_PATTERN = /(?:[.!?。！？…]+|(?:\r?\n)\s*(?:\r?\n)?|[.!?。！？…]+["'”’」』】）》〉〕〗]*)$/u;
+class IntlSegmenterSentenceStreamAdapter {
+  #graphemeSegmenter = new Intl.Segmenter(void 0, {
+    granularity: "grapheme"
+  });
+  #sentenceSegmenter = new Intl.Segmenter(void 0, {
+    granularity: "sentence"
+  });
+  #wordSegmenter = new Intl.Segmenter(void 0, {
+    granularity: "word"
+  });
+  async *pipe(stream) {
+    let buffer = "";
+    let offset = 0;
+    for await (const chunk of stream) {
+      if (chunk === "") {
+        continue;
+      }
+      buffer += chunk;
+      const { remainder, sentences: sentences2 } = this.#drainSentences(buffer, false);
+      buffer = remainder;
+      for (const sentenceText of sentences2) {
+        yield {
+          offset,
+          text: sentenceText,
+          wordsCount: this.#countWords(sentenceText)
+        };
+        offset += sentenceText.length;
+      }
+    }
+    const { sentences } = this.#drainSentences(buffer, true);
+    for (const sentenceText of sentences) {
+      yield {
+        offset,
+        text: sentenceText,
+        wordsCount: this.#countWords(sentenceText)
+      };
+      offset += sentenceText.length;
+    }
+  }
+  #drainSentences(buffer, flushAll) {
+    if (buffer.trim() === "") {
+      return {
+        remainder: "",
+        sentences: []
+      };
+    }
+    const segments = Array.from(this.#sentenceSegmenter.segment(buffer));
+    if (segments.length === 0) {
+      return {
+        remainder: buffer,
+        sentences: []
+      };
+    }
+    if (flushAll || this.#endsWithSentenceBoundary(buffer)) {
+      return {
+        remainder: "",
+        sentences: segments.map((segment) => segment.segment.trim()).filter((segment) => segment !== "")
+      };
+    }
+    const lastSegment = segments[segments.length - 1];
+    if (lastSegment === void 0) {
+      return {
+        remainder: buffer,
+        sentences: []
+      };
+    }
+    return {
+      remainder: buffer.slice(lastSegment.index),
+      sentences: segments.slice(0, -1).map((segment) => segment.segment.trim()).filter((segment) => segment !== "")
+    };
+  }
+  #countWords(text) {
+    let wordsCount = 0;
+    for (const segment of this.#wordSegmenter.segment(text)) {
+      if ("isWordLike" in segment && segment.isWordLike) {
+        wordsCount += 1;
+      }
+    }
+    if (wordsCount > 0) {
+      return wordsCount;
+    }
+    return this.#countGraphemes(text);
+  }
+  #countGraphemes(text) {
+    let unitCount = 0;
+    for (const _segment of this.#graphemeSegmenter.segment(text)) {
+      unitCount += 1;
+    }
+    return unitCount;
+  }
+  #endsWithSentenceBoundary(buffer) {
+    return SENTENCE_BOUNDARY_PATTERN.test(buffer.trimEnd());
+  }
+}
+function createDefaultSentenceStreamAdapter() {
+  return new IntlSegmenterSentenceStreamAdapter();
+}
+export {
+  createDefaultSentenceStreamAdapter
+};
+//# sourceMappingURL=intl-segmenter.js.map

package/dist/reader/segment/intl-segmenter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/segment/intl-segmenter.ts"],"sourcesContent":["import type {\n  SentenceStreamAdapter,\n  SentenceStreamItem,\n  TextStream,\n} from \"./types.js\";\n\nconst SENTENCE_BOUNDARY_PATTERN =\n  /(?:[.!?。！？…]+|(?:\\r?\\n)\\s*(?:\\r?\\n)?|[.!?。！？…]+[\"'”’」』】）》〉〕〗]*)$/u;\n\nclass IntlSegmenterSentenceStreamAdapter implements SentenceStreamAdapter {\n  readonly #graphemeSegmenter = new Intl.Segmenter(undefined, {\n    granularity: \"grapheme\",\n  });\n  readonly #sentenceSegmenter = new Intl.Segmenter(undefined, {\n    granularity: \"sentence\",\n  });\n  readonly #wordSegmenter = new Intl.Segmenter(undefined, {\n    granularity: \"word\",\n  });\n\n  public async *pipe(stream: TextStream): AsyncIterable<SentenceStreamItem> {\n    let buffer = \"\";\n    let offset = 0;\n\n    for await (const chunk of stream) {\n      if (chunk === \"\") {\n        continue;\n      }\n\n      buffer += chunk;\n\n      const { remainder, sentences } = this.#drainSentences(buffer, false);\n      buffer = remainder;\n\n      for (const sentenceText of sentences) {\n        yield {\n          offset,\n          text: sentenceText,\n          wordsCount: this.#countWords(sentenceText),\n        };\n        offset += sentenceText.length;\n      }\n    }\n\n    const { sentences } = this.#drainSentences(buffer, true);\n\n    for (const sentenceText of sentences) {\n      yield {\n        offset,\n        text: sentenceText,\n        wordsCount: this.#countWords(sentenceText),\n      };\n      offset += sentenceText.length;\n    }\n  }\n\n  #drainSentences(\n    buffer: string,\n    flushAll: boolean,\n  ): {\n    readonly remainder: string;\n    readonly sentences: readonly string[];\n  } {\n    if (buffer.trim() === \"\") {\n      return {\n        remainder: \"\",\n        sentences: [],\n      };\n    }\n\n    const segments = Array.from(this.#sentenceSegmenter.segment(buffer));\n\n    if (segments.length === 0) {\n      return {\n        remainder: buffer,\n        sentences: [],\n      };\n    }\n\n    if (flushAll || this.#endsWithSentenceBoundary(buffer)) {\n      return {\n        remainder: \"\",\n        sentences: segments\n          .map((segment) => segment.segment.trim())\n          .filter((segment) => segment !== \"\"),\n      };\n    }\n\n    const lastSegment = segments[segments.length - 1];\n\n    if (lastSegment === undefined) {\n      return {\n        remainder: buffer,\n        sentences: [],\n      };\n    }\n\n    return {\n      remainder: buffer.slice(lastSegment.index),\n      sentences: segments\n        .slice(0, -1)\n        .map((segment) => segment.segment.trim())\n        .filter((segment) => segment !== \"\"),\n    };\n  }\n\n  #countWords(text: string): number {\n    let wordsCount = 0;\n\n    for (const segment of this.#wordSegmenter.segment(text)) {\n      if (\"isWordLike\" in segment && segment.isWordLike) {\n        wordsCount += 1;\n      }\n    }\n\n    if (wordsCount > 0) {\n      return wordsCount;\n    }\n\n    return this.#countGraphemes(text);\n  }\n\n  #countGraphemes(text: string): number {\n    let unitCount = 0;\n\n    for (const _segment of this.#graphemeSegmenter.segment(text)) {\n      unitCount += 1;\n    }\n\n    return unitCount;\n  }\n\n  #endsWithSentenceBoundary(buffer: string): boolean {\n    return SENTENCE_BOUNDARY_PATTERN.test(buffer.trimEnd());\n  }\n}\n\nexport function createDefaultSentenceStreamAdapter(): SentenceStreamAdapter {\n  return new IntlSegmenterSentenceStreamAdapter();\n}\n"],"mappings":"AAMA,MAAM,4BACJ;AAEF,MAAM,mCAAoE;AAAA,EAC/D,qBAAqB,IAAI,KAAK,UAAU,QAAW;AAAA,IAC1D,aAAa;AAAA,EACf,CAAC;AAAA,EACQ,qBAAqB,IAAI,KAAK,UAAU,QAAW;AAAA,IAC1D,aAAa;AAAA,EACf,CAAC;AAAA,EACQ,iBAAiB,IAAI,KAAK,UAAU,QAAW;AAAA,IACtD,aAAa;AAAA,EACf,CAAC;AAAA,EAED,OAAc,KAAK,QAAuD;AACxE,QAAI,SAAS;AACb,QAAI,SAAS;AAEb,qBAAiB,SAAS,QAAQ;AAChC,UAAI,UAAU,IAAI;AAChB;AAAA,MACF;AAEA,gBAAU;AAEV,YAAM,EAAE,WAAW,WAAAA,WAAU,IAAI,KAAK,gBAAgB,QAAQ,KAAK;AACnE,eAAS;AAET,iBAAW,gBAAgBA,YAAW;AACpC,cAAM;AAAA,UACJ;AAAA,UACA,MAAM;AAAA,UACN,YAAY,KAAK,YAAY,YAAY;AAAA,QAC3C;AACA,kBAAU,aAAa;AAAA,MACzB;AAAA,IACF;AAEA,UAAM,EAAE,UAAU,IAAI,KAAK,gBAAgB,QAAQ,IAAI;AAEvD,eAAW,gBAAgB,WAAW;AACpC,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,QACN,YAAY,KAAK,YAAY,YAAY;AAAA,MAC3C;AACA,gBAAU,aAAa;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,gBACE,QACA,UAIA;AACA,QAAI,OAAO,KAAK,MAAM,IAAI;AACxB,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,CAAC;AAAA,MACd;AAAA,IACF;AAEA,UAAM,WAAW,MAAM,KAAK,KAAK,mBAAmB,QAAQ,MAAM,CAAC;AAEnE,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,CAAC;AAAA,MACd;AAAA,IACF;AAEA,QAAI,YAAY,KAAK,0BAA0B,MAAM,GAAG;AACtD,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,SACR,IAAI,CAAC,YAAY,QAAQ,QAAQ,KAAK,CAAC,EACvC,OAAO,CAAC,YAAY,YAAY,EAAE;AAAA,MACvC;AAAA,IACF;AAEA,UAAM,cAAc,SAAS,SAAS,SAAS,CAAC;AAEhD,QAAI,gBAAgB,QAAW;AAC7B,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,CAAC;AAAA,MACd;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW,OAAO,MAAM,YAAY,KAAK;AAAA,MACzC,WAAW,SACR,MAAM,GAAG,EAAE,EACX,IAAI,CAAC,YAAY,QAAQ,QAAQ,KAAK,CAAC,EACvC,OAAO,CAAC,YAAY,YAAY,EAAE;AAAA,IACvC;AAAA,EACF;AAAA,EAEA,YAAY,MAAsB;AAChC,QAAI,aAAa;AAEjB,eAAW,WAAW,KAAK,eAAe,QAAQ,IAAI,GAAG;AACvD,UAAI,gBAAgB,WAAW,QAAQ,YAAY;AACjD,sBAAc;AAAA,MAChB;AAAA,IACF;AAEA,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,gBAAgB,IAAI;AAAA,EAClC;AAAA,EAEA,gBAAgB,MAAsB;AACpC,QAAI,YAAY;AAEhB,eAAW,YAAY,KAAK,mBAAmB,QAAQ,IAAI,GAAG;AAC5D,mBAAa;AAAA,IACf;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,0BAA0B,QAAyB;AACjD,WAAO,0BAA0B,KAAK,OAAO,QAAQ,CAAC;AAAA,EACxD;AACF;AAEO,SAAS,qCAA4D;AAC1E,SAAO,IAAI,mCAAmC;AAChD;","names":["sentences"]}

package/dist/reader/segment/segment.d.ts

@@ -0,0 +1,5 @@
+import { TextStream, SegmentTextStreamOptions, SentenceStreamItem } from './types.js';
+
+declare function segmentTextStream(stream: TextStream, options?: SegmentTextStreamOptions): AsyncIterable<SentenceStreamItem>;
+
+export { segmentTextStream };

package/dist/reader/segment/segment.js

@@ -0,0 +1,9 @@
+import { createDefaultSentenceStreamAdapter } from "./intl-segmenter.js";
+function segmentTextStream(stream, options) {
+  const adapter = options?.adapter ?? createDefaultSentenceStreamAdapter();
+  return adapter.pipe(stream);
+}
+export {
+  segmentTextStream
+};
+//# sourceMappingURL=segment.js.map

package/dist/reader/segment/segment.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/reader/segment/segment.ts"],"sourcesContent":["import { createDefaultSentenceStreamAdapter } from \"./intl-segmenter.js\";\nimport type {\n  SegmentTextStreamOptions,\n  SentenceStreamItem,\n  TextStream,\n} from \"./types.js\";\n\nexport function segmentTextStream(\n  stream: TextStream,\n  options?: SegmentTextStreamOptions,\n): AsyncIterable<SentenceStreamItem> {\n  const adapter = options?.adapter ?? createDefaultSentenceStreamAdapter();\n\n  return adapter.pipe(stream);\n}\n"],"mappings":"AAAA,SAAS,0CAA0C;AAO5C,SAAS,kBACd,QACA,SACmC;AACnC,QAAM,UAAU,SAAS,WAAW,mCAAmC;AAEvE,SAAO,QAAQ,KAAK,MAAM;AAC5B;","names":[]}

package/dist/reader/segment/types.d.ts

@@ -0,0 +1,14 @@
+type TextStream = AsyncIterable<string> | Iterable<string>;
+interface SentenceStreamItem {
+    readonly offset: number;
+    readonly text: string;
+    readonly wordsCount: number;
+}
+interface SentenceStreamAdapter {
+    pipe(stream: TextStream): AsyncIterable<SentenceStreamItem>;
+}
+interface SegmentTextStreamOptions {
+    readonly adapter?: SentenceStreamAdapter;
+}
+
+export type { SegmentTextStreamOptions, SentenceStreamAdapter, SentenceStreamItem, TextStream };

package/dist/reader/segment/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/reader/segment/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/reader/types.d.ts

@@ -0,0 +1,29 @@
+import { ChunkGraphDelta, ChunkGraphEdge } from './attention/attention.js';
+import { ChunkBatchOptions, ChunkExtractionSentence, CognitiveChunk, ChunkImportanceAnnotation } from './chunk-batch/types.js';
+import { TextStream, SentenceStreamAdapter, SentenceStreamItem } from './segment/types.js';
+import '../common/language.js';
+import '../llm/client.js';
+import '../llm/context.js';
+import '../llm/cache.js';
+import '../llm/types.js';
+import 'ai';
+import '../document/types.js';
+
+type ReaderTextStream = TextStream;
+type ReaderSegment = SentenceStreamItem;
+type ReaderSegmenter = SentenceStreamAdapter;
+type ReaderSentence = ChunkExtractionSentence;
+type ReaderChunk = CognitiveChunk;
+type ReaderGraphEdge = ChunkGraphEdge;
+type ReaderImportanceAnnotation = ChunkImportanceAnnotation;
+type ReaderGraphDelta = ChunkGraphDelta;
+interface ReaderOptions<S extends string> extends ChunkBatchOptions<S> {
+    readonly attention: {
+        readonly capacity: number;
+        readonly generationDecayFactor: number;
+        readonly idGenerator: () => Promise<number>;
+    };
+    readonly segmenter?: ReaderSegmenter;
+}
+
+export type { ReaderChunk, ReaderGraphDelta, ReaderGraphEdge, ReaderImportanceAnnotation, ReaderOptions, ReaderSegment, ReaderSegmenter, ReaderSentence, ReaderTextStream };

package/dist/reader/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/reader/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}