wiki-graph-core 0.3.1 → 0.4.0

package/dist/text/reader/core.d.ts

@@ -0,0 +1,37 @@
+import { ReaderOptions, ReaderTextStream, ReaderSegment, ReaderSentence, ReaderGraphDelta, ReaderChunk } from './types.js';
+import './attention/core.js';
+import './chunk-batch/types.js';
+import '../../runtime/common/language.js';
+import '../../external/llm/client/core.js';
+import '../../external/llm/context.js';
+import '../../external/llm/cache.js';
+import '../../external/llm/types.js';
+import 'ai';
+import '../../document/types.js';
+import './segment/types.js';
+
+declare class Reader<S extends string> {
+    #private;
+    constructor(options: ReaderOptions<S>);
+    get capacity(): number;
+    segment(stream: ReaderTextStream): AsyncIterable<ReaderSegment>;
+    extractUserFocused(input: {
+        readonly sentences: readonly ReaderSentence[];
+        readonly text: string;
+    }): Promise<{
+        readonly delta: ReaderGraphDelta;
+        readonly fragmentSummary: string;
+    }>;
+    extractBookCoherence(input: {
+        readonly sentences: readonly ReaderSentence[];
+        readonly text: string;
+        readonly userFocusedChunks: readonly ReaderChunk[];
+    }): Promise<ReaderGraphDelta>;
+    completeFragment(input: {
+        readonly allChunks: readonly ReaderChunk[];
+        readonly getSuccessorChunkIds: (chunkId: number) => readonly number[];
+    }): void;
+    clear(): void;
+}
+
+export { Reader };

package/dist/text/reader/core.js

@@ -0,0 +1,82 @@
+import { Attention } from "./attention/core.js";
+import {
+  extractBookCoherenceChunkBatch,
+  extractUserFocusedChunkBatch
+} from "./chunk-batch/extract.js";
+import { segmentTextStream } from "./segment/core.js";
+class Reader {
+  #attention;
+  #chunkBatchOptions;
+  #segmenter;
+  constructor(options) {
+    this.#attention = new Attention(
+      options.attention.capacity,
+      options.attention.generationDecayFactor,
+      options.attention.idGenerator
+    );
+    this.#chunkBatchOptions = {
+      extractionGuidance: options.extractionGuidance,
+      llm: options.llm,
+      scopes: options.scopes,
+      sentenceTextSource: options.sentenceTextSource,
+      ...options.userLanguage === void 0 ? {} : {
+        userLanguage: options.userLanguage
+      }
+    };
+    this.#segmenter = options.segmenter;
+  }
+  get capacity() {
+    return this.#attention.capacity;
+  }
+  segment(stream) {
+    if (this.#segmenter === void 0) {
+      return segmentTextStream(stream);
+    }
+    return segmentTextStream(stream, {
+      adapter: this.#segmenter
+    });
+  }
+  async extractUserFocused(input) {
+    const context = this.#attention.createChunkBatchContext();
+    const result = await extractUserFocusedChunkBatch(this.#chunkBatchOptions, {
+      sentences: input.sentences,
+      text: input.text,
+      visibleChunkIds: context.visibleChunkIds,
+      workingMemoryPrompt: context.workingMemoryPrompt
+    });
+    return {
+      delta: await this.#attention.acceptChunkBatch(result.chunkBatch),
+      fragmentSummary: result.fragmentSummary
+    };
+  }
+  async extractBookCoherence(input) {
+    const context = this.#attention.createChunkBatchContext({
+      includeCurrentFragment: false
+    });
+    const linkableChunkIds = [
+      ...context.visibleChunkIds,
+      ...input.userFocusedChunks.map((chunk) => chunk.id)
+    ];
+    const chunkBatch = await extractBookCoherenceChunkBatch(
+      this.#chunkBatchOptions,
+      {
+        sentences: input.sentences,
+        text: input.text,
+        userFocusedChunks: input.userFocusedChunks,
+        visibleChunkIds: linkableChunkIds,
+        workingMemoryPrompt: context.workingMemoryPrompt
+      }
+    );
+    return await this.#attention.acceptChunkBatch(chunkBatch);
+  }
+  completeFragment(input) {
+    this.#attention.completeFragment(input);
+  }
+  clear() {
+    this.#attention.clear();
+  }
+}
+export {
+  Reader
+};
+//# sourceMappingURL=core.js.map

package/dist/text/reader/core.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/text/reader/core.ts"],"sourcesContent":["import { Attention } from \"./attention/core.js\";\nimport {\n  extractBookCoherenceChunkBatch,\n  extractUserFocusedChunkBatch,\n} from \"./chunk-batch/extract.js\";\nimport { segmentTextStream } from \"./segment/core.js\";\nimport type { ChunkBatchOptions } from \"./chunk-batch/types.js\";\nimport type {\n  ReaderChunk,\n  ReaderGraphDelta,\n  ReaderOptions,\n  ReaderSegment,\n  ReaderSentence,\n  ReaderTextStream,\n} from \"./types.js\";\n\nexport class Reader<S extends string> {\n  readonly #attention: Attention;\n  readonly #chunkBatchOptions: ChunkBatchOptions<S>;\n  readonly #segmenter: ReaderOptions<S>[\"segmenter\"];\n\n  public constructor(options: ReaderOptions<S>) {\n    this.#attention = new Attention(\n      options.attention.capacity,\n      options.attention.generationDecayFactor,\n      options.attention.idGenerator,\n    );\n    this.#chunkBatchOptions = {\n      extractionGuidance: options.extractionGuidance,\n      llm: options.llm,\n      scopes: options.scopes,\n      sentenceTextSource: options.sentenceTextSource,\n      ...(options.userLanguage === undefined\n        ? {}\n        : {\n            userLanguage: options.userLanguage,\n          }),\n    };\n    this.#segmenter = options.segmenter;\n  }\n\n  public get capacity(): number {\n    return this.#attention.capacity;\n  }\n\n  public segment(stream: ReaderTextStream): AsyncIterable<ReaderSegment> {\n    if (this.#segmenter === undefined) {\n      return segmentTextStream(stream);\n    }\n\n    return segmentTextStream(stream, {\n      adapter: this.#segmenter,\n    });\n  }\n\n  public async extractUserFocused(input: {\n    readonly sentences: readonly ReaderSentence[];\n    readonly text: string;\n  }): Promise<{\n    readonly delta: ReaderGraphDelta;\n    readonly fragmentSummary: string;\n  }> {\n    const context = this.#attention.createChunkBatchContext();\n    const result = await extractUserFocusedChunkBatch(this.#chunkBatchOptions, {\n      sentences: input.sentences,\n      text: input.text,\n      visibleChunkIds: context.visibleChunkIds,\n      workingMemoryPrompt: context.workingMemoryPrompt,\n    });\n\n    return {\n      delta: await this.#attention.acceptChunkBatch(result.chunkBatch),\n      fragmentSummary: result.fragmentSummary,\n    };\n  }\n\n  public async extractBookCoherence(input: {\n    readonly sentences: readonly ReaderSentence[];\n    readonly text: string;\n    readonly userFocusedChunks: readonly ReaderChunk[];\n  }): Promise<ReaderGraphDelta> {\n    const context = this.#attention.createChunkBatchContext({\n      includeCurrentFragment: false,\n    });\n    const linkableChunkIds = [\n      ...context.visibleChunkIds,\n      ...input.userFocusedChunks.map((chunk) => chunk.id),\n    ];\n    const chunkBatch = await extractBookCoherenceChunkBatch(\n      this.#chunkBatchOptions,\n      {\n        sentences: input.sentences,\n        text: input.text,\n        userFocusedChunks: input.userFocusedChunks,\n        visibleChunkIds: linkableChunkIds,\n        workingMemoryPrompt: context.workingMemoryPrompt,\n      },\n    );\n\n    return await this.#attention.acceptChunkBatch(chunkBatch);\n  }\n\n  public completeFragment(input: {\n    readonly allChunks: readonly ReaderChunk[];\n    readonly getSuccessorChunkIds: (chunkId: number) => readonly number[];\n  }): void {\n    this.#attention.completeFragment(input);\n  }\n\n  public clear(): void {\n    this.#attention.clear();\n  }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAW3B,MAAM,OAAyB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EAEF,YAAY,SAA2B;AAC5C,SAAK,aAAa,IAAI;AAAA,MACpB,QAAQ,UAAU;AAAA,MAClB,QAAQ,UAAU;AAAA,MAClB,QAAQ,UAAU;AAAA,IACpB;AACA,SAAK,qBAAqB;AAAA,MACxB,oBAAoB,QAAQ;AAAA,MAC5B,KAAK,QAAQ;AAAA,MACb,QAAQ,QAAQ;AAAA,MAChB,oBAAoB,QAAQ;AAAA,MAC5B,GAAI,QAAQ,iBAAiB,SACzB,CAAC,IACD;AAAA,QACE,cAAc,QAAQ;AAAA,MACxB;AAAA,IACN;AACA,SAAK,aAAa,QAAQ;AAAA,EAC5B;AAAA,EAEA,IAAW,WAAmB;AAC5B,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEO,QAAQ,QAAwD;AACrE,QAAI,KAAK,eAAe,QAAW;AACjC,aAAO,kBAAkB,MAAM;AAAA,IACjC;AAEA,WAAO,kBAAkB,QAAQ;AAAA,MAC/B,SAAS,KAAK;AAAA,IAChB,CAAC;AAAA,EACH;AAAA,EAEA,MAAa,mBAAmB,OAM7B;AACD,UAAM,UAAU,KAAK,WAAW,wBAAwB;AACxD,UAAM,SAAS,MAAM,6BAA6B,KAAK,oBAAoB;AAAA,MACzE,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,MACZ,iBAAiB,QAAQ;AAAA,MACzB,qBAAqB,QAAQ;AAAA,IAC/B,CAAC;AAED,WAAO;AAAA,MACL,OAAO,MAAM,KAAK,WAAW,iBAAiB,OAAO,UAAU;AAAA,MAC/D,iBAAiB,OAAO;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAa,qBAAqB,OAIJ;AAC5B,UAAM,UAAU,KAAK,WAAW,wBAAwB;AAAA,MACtD,wBAAwB;AAAA,IAC1B,CAAC;AACD,UAAM,mBAAmB;AAAA,MACvB,GAAG,QAAQ;AAAA,MACX,GAAG,MAAM,kBAAkB,IAAI,CAAC,UAAU,MAAM,EAAE;AAAA,IACpD;AACA,UAAM,aAAa,MAAM;AAAA,MACvB,KAAK;AAAA,MACL;AAAA,QACE,WAAW,MAAM;AAAA,QACjB,MAAM,MAAM;AAAA,QACZ,mBAAmB,MAAM;AAAA,QACzB,iBAAiB;AAAA,QACjB,qBAAqB,QAAQ;AAAA,MAC/B;AAAA,IACF;AAEA,WAAO,MAAM,KAAK,WAAW,iBAAiB,UAAU;AAAA,EAC1D;AAAA,EAEO,iBAAiB,OAGf;AACP,SAAK,WAAW,iBAAiB,KAAK;AAAA,EACxC;AAAA,EAEO,QAAc;AACnB,SAAK,WAAW,MAAM;AAAA,EACxB;AACF;","names":[]}

package/dist/text/reader/index.d.ts

@@ -0,0 +1,13 @@
+export { Reader } from './core.js';
+export { segmentTextStream } from './segment/core.js';
+export { ReaderChunk, ReaderGraphDelta, ReaderGraphEdge, ReaderImportanceAnnotation, ReaderOptions, ReaderSegment, ReaderSegmenter, ReaderSentence, ReaderTextStream } from './types.js';
+import './segment/types.js';
+import './attention/core.js';
+import './chunk-batch/types.js';
+import '../../runtime/common/language.js';
+import '../../external/llm/client/core.js';
+import '../../external/llm/context.js';
+import '../../external/llm/cache.js';
+import '../../external/llm/types.js';
+import 'ai';
+import '../../document/types.js';

package/dist/text/reader/index.js

@@ -0,0 +1,7 @@
+import { Reader } from "./core.js";
+import { segmentTextStream } from "./segment/core.js";
+export {
+  Reader,
+  segmentTextStream
+};
+//# sourceMappingURL=index.js.map

package/dist/text/reader/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/text/reader/index.ts"],"sourcesContent":["export { Reader } from \"./core.js\";\nexport { segmentTextStream } from \"./segment/core.js\";\nexport type {\n  ReaderChunk,\n  ReaderGraphDelta,\n  ReaderGraphEdge,\n  ReaderImportanceAnnotation,\n  ReaderOptions,\n  ReaderSegment,\n  ReaderSegmenter,\n  ReaderSentence,\n  ReaderTextStream,\n} from \"./types.js\";\n"],"mappings":"AAAA,SAAS,cAAc;AACvB,SAAS,yBAAyB;","names":[]}

package/dist/text/reader/segment/core.js

@@ -0,0 +1,9 @@
+import { createDefaultSentenceStreamAdapter } from "./intl-segmenter.js";
+function segmentTextStream(stream, options) {
+  const adapter = options?.adapter ?? createDefaultSentenceStreamAdapter();
+  return adapter.pipe(stream);
+}
+export {
+  segmentTextStream
+};
+//# sourceMappingURL=core.js.map

package/dist/text/reader/segment/core.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/text/reader/segment/core.ts"],"sourcesContent":["import { createDefaultSentenceStreamAdapter } from \"./intl-segmenter.js\";\nimport type {\n  SegmentTextStreamOptions,\n  SentenceStreamItem,\n  TextStream,\n} from \"./types.js\";\n\nexport function segmentTextStream(\n  stream: TextStream,\n  options?: SegmentTextStreamOptions,\n): AsyncIterable<SentenceStreamItem> {\n  const adapter = options?.adapter ?? createDefaultSentenceStreamAdapter();\n\n  return adapter.pipe(stream);\n}\n"],"mappings":"AAAA,SAAS,0CAA0C;AAO5C,SAAS,kBACd,QACA,SACmC;AACnC,QAAM,UAAU,SAAS,WAAW,mCAAmC;AAEvE,SAAO,QAAQ,KAAK,MAAM;AAC5B;","names":[]}

package/dist/text/reader/segment/index.d.ts

@@ -0,0 +1,2 @@
+export { segmentTextStream } from './core.js';
+export { SegmentTextStreamOptions, SentenceStreamAdapter, SentenceStreamItem, TextStream } from './types.js';

package/dist/text/reader/segment/index.js

@@ -0,0 +1,5 @@
+import { segmentTextStream } from "./core.js";
+export {
+  segmentTextStream
+};
+//# sourceMappingURL=index.js.map

package/dist/text/reader/segment/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/text/reader/segment/index.ts"],"sourcesContent":["export { segmentTextStream } from \"./core.js\";\nexport type {\n  SegmentTextStreamOptions,\n  SentenceStreamAdapter,\n  SentenceStreamItem,\n  TextStream,\n} from \"./types.js\";\n"],"mappings":"AAAA,SAAS,yBAAyB;","names":[]}

package/dist/text/reader/segment/intl-segmenter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/text/reader/segment/intl-segmenter.ts"],"sourcesContent":["import type {\n  SentenceStreamAdapter,\n  SentenceStreamItem,\n  TextStream,\n} from \"./types.js\";\n\nconst SENTENCE_BOUNDARY_PATTERN =\n  /(?:[.!?。！？…]+|(?:\\r?\\n)\\s*(?:\\r?\\n)?|[.!?。！？…]+[\"'”’」』】）》〉〕〗]*)$/u;\n\nclass IntlSegmenterSentenceStreamAdapter implements SentenceStreamAdapter {\n  readonly #graphemeSegmenter = new Intl.Segmenter(undefined, {\n    granularity: \"grapheme\",\n  });\n  readonly #sentenceSegmenter = new Intl.Segmenter(undefined, {\n    granularity: \"sentence\",\n  });\n  readonly #wordSegmenter = new Intl.Segmenter(undefined, {\n    granularity: \"word\",\n  });\n\n  public async *pipe(stream: TextStream): AsyncIterable<SentenceStreamItem> {\n    let buffer = \"\";\n    let offset = 0;\n\n    for await (const chunk of stream) {\n      if (chunk === \"\") {\n        continue;\n      }\n\n      buffer += chunk;\n\n      const { remainder, sentences } = this.#drainSentences(buffer, false);\n      buffer = remainder;\n\n      for (const sentenceText of sentences) {\n        yield {\n          offset,\n          text: sentenceText,\n          wordsCount: this.#countWords(sentenceText),\n        };\n        offset += sentenceText.length;\n      }\n    }\n\n    const { sentences } = this.#drainSentences(buffer, true);\n\n    for (const sentenceText of sentences) {\n      yield {\n        offset,\n        text: sentenceText,\n        wordsCount: this.#countWords(sentenceText),\n      };\n      offset += sentenceText.length;\n    }\n  }\n\n  #drainSentences(\n    buffer: string,\n    flushAll: boolean,\n  ): {\n    readonly remainder: string;\n    readonly sentences: readonly string[];\n  } {\n    if (buffer.trim() === \"\") {\n      return {\n        remainder: \"\",\n        sentences: [],\n      };\n    }\n\n    const segments = Array.from(this.#sentenceSegmenter.segment(buffer));\n\n    if (segments.length === 0) {\n      return {\n        remainder: buffer,\n        sentences: [],\n      };\n    }\n\n    if (flushAll || this.#endsWithSentenceBoundary(buffer)) {\n      return {\n        remainder: \"\",\n        sentences: segments\n          .map((segment) => segment.segment.trim())\n          .filter((segment) => segment !== \"\"),\n      };\n    }\n\n    const lastSegment = segments[segments.length - 1];\n\n    if (lastSegment === undefined) {\n      return {\n        remainder: buffer,\n        sentences: [],\n      };\n    }\n\n    return {\n      remainder: buffer.slice(lastSegment.index),\n      sentences: segments\n        .slice(0, -1)\n        .map((segment) => segment.segment.trim())\n        .filter((segment) => segment !== \"\"),\n    };\n  }\n\n  #countWords(text: string): number {\n    let wordsCount = 0;\n\n    for (const segment of this.#wordSegmenter.segment(text)) {\n      if (\"isWordLike\" in segment && segment.isWordLike) {\n        wordsCount += 1;\n      }\n    }\n\n    if (wordsCount > 0) {\n      return wordsCount;\n    }\n\n    return this.#countGraphemes(text);\n  }\n\n  #countGraphemes(text: string): number {\n    let unitCount = 0;\n\n    for (const _segment of this.#graphemeSegmenter.segment(text)) {\n      unitCount += 1;\n    }\n\n    return unitCount;\n  }\n\n  #endsWithSentenceBoundary(buffer: string): boolean {\n    return SENTENCE_BOUNDARY_PATTERN.test(buffer.trimEnd());\n  }\n}\n\nexport function createDefaultSentenceStreamAdapter(): SentenceStreamAdapter {\n  return new IntlSegmenterSentenceStreamAdapter();\n}\n"],"mappings":"AAMA,MAAM,4BACJ;AAEF,MAAM,mCAAoE;AAAA,EAC/D,qBAAqB,IAAI,KAAK,UAAU,QAAW;AAAA,IAC1D,aAAa;AAAA,EACf,CAAC;AAAA,EACQ,qBAAqB,IAAI,KAAK,UAAU,QAAW;AAAA,IAC1D,aAAa;AAAA,EACf,CAAC;AAAA,EACQ,iBAAiB,IAAI,KAAK,UAAU,QAAW;AAAA,IACtD,aAAa;AAAA,EACf,CAAC;AAAA,EAED,OAAc,KAAK,QAAuD;AACxE,QAAI,SAAS;AACb,QAAI,SAAS;AAEb,qBAAiB,SAAS,QAAQ;AAChC,UAAI,UAAU,IAAI;AAChB;AAAA,MACF;AAEA,gBAAU;AAEV,YAAM,EAAE,WAAW,WAAAA,WAAU,IAAI,KAAK,gBAAgB,QAAQ,KAAK;AACnE,eAAS;AAET,iBAAW,gBAAgBA,YAAW;AACpC,cAAM;AAAA,UACJ;AAAA,UACA,MAAM;AAAA,UACN,YAAY,KAAK,YAAY,YAAY;AAAA,QAC3C;AACA,kBAAU,aAAa;AAAA,MACzB;AAAA,IACF;AAEA,UAAM,EAAE,UAAU,IAAI,KAAK,gBAAgB,QAAQ,IAAI;AAEvD,eAAW,gBAAgB,WAAW;AACpC,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,QACN,YAAY,KAAK,YAAY,YAAY;AAAA,MAC3C;AACA,gBAAU,aAAa;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,gBACE,QACA,UAIA;AACA,QAAI,OAAO,KAAK,MAAM,IAAI;AACxB,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,CAAC;AAAA,MACd;AAAA,IACF;AAEA,UAAM,WAAW,MAAM,KAAK,KAAK,mBAAmB,QAAQ,MAAM,CAAC;AAEnE,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,CAAC;AAAA,MACd;AAAA,IACF;AAEA,QAAI,YAAY,KAAK,0BAA0B,MAAM,GAAG;AACtD,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,SACR,IAAI,CAAC,YAAY,QAAQ,QAAQ,KAAK,CAAC,EACvC,OAAO,CAAC,YAAY,YAAY,EAAE;AAAA,MACvC;AAAA,IACF;AAEA,UAAM,cAAc,SAAS,SAAS,SAAS,CAAC;AAEhD,QAAI,gBAAgB,QAAW;AAC7B,aAAO;AAAA,QACL,WAAW;AAAA,QACX,WAAW,CAAC;AAAA,MACd;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW,OAAO,MAAM,YAAY,KAAK;AAAA,MACzC,WAAW,SACR,MAAM,GAAG,EAAE,EACX,IAAI,CAAC,YAAY,QAAQ,QAAQ,KAAK,CAAC,EACvC,OAAO,CAAC,YAAY,YAAY,EAAE;AAAA,IACvC;AAAA,EACF;AAAA,EAEA,YAAY,MAAsB;AAChC,QAAI,aAAa;AAEjB,eAAW,WAAW,KAAK,eAAe,QAAQ,IAAI,GAAG;AACvD,UAAI,gBAAgB,WAAW,QAAQ,YAAY;AACjD,sBAAc;AAAA,MAChB;AAAA,IACF;AAEA,QAAI,aAAa,GAAG;AAClB,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,gBAAgB,IAAI;AAAA,EAClC;AAAA,EAEA,gBAAgB,MAAsB;AACpC,QAAI,YAAY;AAEhB,eAAW,YAAY,KAAK,mBAAmB,QAAQ,IAAI,GAAG;AAC5D,mBAAa;AAAA,IACf;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,0BAA0B,QAAyB;AACjD,WAAO,0BAA0B,KAAK,OAAO,QAAQ,CAAC;AAAA,EACxD;AACF;AAEO,SAAS,qCAA4D;AAC1E,SAAO,IAAI,mCAAmC;AAChD;","names":["sentences"]}

package/dist/text/reader/segment/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/text/reader/segment/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/text/reader/types.d.ts

@@ -0,0 +1,29 @@
+import { ChunkGraphDelta, ChunkGraphEdge } from './attention/core.js';
+import { ChunkBatchOptions, ChunkExtractionSentence, CognitiveChunk, ChunkImportanceAnnotation } from './chunk-batch/types.js';
+import { TextStream, SentenceStreamAdapter, SentenceStreamItem } from './segment/types.js';
+import '../../runtime/common/language.js';
+import '../../external/llm/client/core.js';
+import '../../external/llm/context.js';
+import '../../external/llm/cache.js';
+import '../../external/llm/types.js';
+import 'ai';
+import '../../document/types.js';
+
+type ReaderTextStream = TextStream;
+type ReaderSegment = SentenceStreamItem;
+type ReaderSegmenter = SentenceStreamAdapter;
+type ReaderSentence = ChunkExtractionSentence;
+type ReaderChunk = CognitiveChunk;
+type ReaderGraphEdge = ChunkGraphEdge;
+type ReaderImportanceAnnotation = ChunkImportanceAnnotation;
+type ReaderGraphDelta = ChunkGraphDelta;
+interface ReaderOptions<S extends string> extends ChunkBatchOptions<S> {
+    readonly attention: {
+        readonly capacity: number;
+        readonly generationDecayFactor: number;
+        readonly idGenerator: () => Promise<number>;
+    };
+    readonly segmenter?: ReaderSegmenter;
+}
+
+export type { ReaderChunk, ReaderGraphDelta, ReaderGraphEdge, ReaderImportanceAnnotation, ReaderOptions, ReaderSegment, ReaderSegmenter, ReaderSentence, ReaderTextStream };

package/dist/text/reader/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/text/reader/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/text/serial/discovery.d.ts

@@ -0,0 +1,36 @@
+export { discoverSerial } from './fragments.js';
+import '../../document/text-streams/types.js';
+import '../../document/types.js';
+import '../reader/types.js';
+import '../reader/attention/core.js';
+import '../reader/chunk-batch/types.js';
+import '../../runtime/common/language.js';
+import '../../external/llm/client/core.js';
+import '../../external/llm/context.js';
+import '../../external/llm/cache.js';
+import '../../external/llm/types.js';
+import 'ai';
+import '../reader/segment/types.js';
+import './options.js';
+import '../../runtime/common/llm-scope.js';
+import '../../document/directory/types.js';
+import '../source/meta.js';
+import 'zod';
+import '../source/toc.js';
+import '../source/types.js';
+import '../../document/database.js';
+import 'sqlite3';
+import '../../document/text-streams/serial.js';
+import '../../document/text-streams/draft.js';
+import '../../document/stores/chunk.js';
+import '../../document/stores/types.js';
+import '../../document/stores/fragment-group.js';
+import '../../document/stores/graph-build-parameter.js';
+import '../../document/stores/mention-link.js';
+import '../../document/stores/mention.js';
+import '../../document/stores/object-metadata.js';
+import '../../document/stores/reading-edge.js';
+import '../../document/stores/serial.js';
+import '../../document/stores/snake/chunk.js';
+import '../../document/stores/snake/core.js';
+import '../../document/stores/snake/edge.js';

package/dist/text/serial/discovery.js

@@ -0,0 +1,5 @@
+import { discoverSerial } from "./fragments.js";
+export {
+  discoverSerial
+};
+//# sourceMappingURL=discovery.js.map

package/dist/text/serial/discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/text/serial/discovery.ts"],"sourcesContent":["export { discoverSerial } from \"./fragments.js\";\n"],"mappings":"AAAA,SAAS,sBAAsB;","names":[]}

package/dist/text/serial/fragments.d.ts

@@ -0,0 +1,67 @@
+import { ReadonlySerialTextStream } from '../../document/text-streams/types.js';
+import { ReaderSegmenter, ReaderTextStream } from '../reader/types.js';
+import { SerialDiscovery } from './options.js';
+import '../../document/types.js';
+import '../reader/attention/core.js';
+import '../reader/chunk-batch/types.js';
+import '../../runtime/common/language.js';
+import '../../external/llm/client/core.js';
+import '../../external/llm/context.js';
+import '../../external/llm/cache.js';
+import '../../external/llm/types.js';
+import 'ai';
+import '../reader/segment/types.js';
+import '../../runtime/common/llm-scope.js';
+import '../../document/directory/types.js';
+import '../source/meta.js';
+import 'zod';
+import '../source/toc.js';
+import '../source/types.js';
+import '../../document/database.js';
+import 'sqlite3';
+import '../../document/text-streams/serial.js';
+import '../../document/text-streams/draft.js';
+import '../../document/stores/chunk.js';
+import '../../document/stores/types.js';
+import '../../document/stores/fragment-group.js';
+import '../../document/stores/graph-build-parameter.js';
+import '../../document/stores/mention-link.js';
+import '../../document/stores/mention.js';
+import '../../document/stores/object-metadata.js';
+import '../../document/stores/reading-edge.js';
+import '../../document/stores/serial.js';
+import '../../document/stores/snake/chunk.js';
+import '../../document/stores/snake/core.js';
+import '../../document/stores/snake/edge.js';
+
+declare function discoverSerial(input: {
+    readonly segmenter?: ReaderSegmenter;
+    readonly stream: ReaderTextStream;
+}): Promise<SerialDiscovery>;
+declare function streamFragments(input: {
+    maxWordsCount: number;
+    stream: AsyncIterable<{
+        readonly text: string;
+        readonly wordsCount: number;
+    }>;
+}): AsyncIterable<{
+    readonly sentences: ReadonlyArray<{
+        readonly text: string;
+        readonly wordsCount: number;
+    }>;
+}>;
+declare function listSerialProcessingFragments(fragments: ReadonlySerialTextStream, maxWordsCount: number): Promise<ReadonlyArray<{
+    readonly startSentenceIndex: number;
+    readonly sentences: ReadonlyArray<{
+        readonly text: string;
+        readonly wordsCount: number;
+    }>;
+}>>;
+declare function collectTextStream(stream: ReaderTextStream): Promise<string>;
+declare function countFragmentWords(sentences: ReadonlyArray<{
+    readonly text: string;
+    readonly wordsCount: number;
+}>): number;
+declare function readSerialPassthroughSummary(fragments: ReadonlySerialTextStream, fragmentIds: readonly number[]): Promise<string>;
+
+export { collectTextStream, countFragmentWords, discoverSerial, listSerialProcessingFragments, readSerialPassthroughSummary, streamFragments };

package/dist/text/serial/fragments.js

@@ -0,0 +1,121 @@
+import {
+  segmentTextStream
+} from "../reader/index.js";
+import { DEFAULT_FRAGMENT_WORDS_COUNT } from "./options.js";
+async function discoverSerial(input) {
+  let fragments = 0;
+  let words = 0;
+  for await (const fragment of streamFragments({
+    maxWordsCount: DEFAULT_FRAGMENT_WORDS_COUNT,
+    stream: segmentTextStream(input.stream, {
+      ...input.segmenter === void 0 ? {} : { adapter: input.segmenter }
+    })
+  })) {
+    fragments += 1;
+    words += countFragmentWords(fragment.sentences);
+  }
+  return {
+    fragments,
+    words
+  };
+}
+async function* streamFragments(input) {
+  let currentSentences = [];
+  let currentWordsCount = 0;
+  for await (const sentence of input.stream) {
+    const sentenceText = sentence.text.trim();
+    if (sentenceText === "") {
+      continue;
+    }
+    if (currentSentences.length > 0 && currentWordsCount + sentence.wordsCount > input.maxWordsCount) {
+      yield {
+        sentences: currentSentences
+      };
+      currentSentences = [];
+      currentWordsCount = 0;
+    }
+    currentSentences.push({
+      text: sentenceText,
+      wordsCount: sentence.wordsCount
+    });
+    currentWordsCount += sentence.wordsCount;
+  }
+  if (currentSentences.length > 0) {
+    yield {
+      sentences: currentSentences
+    };
+  }
+}
+async function listSerialProcessingFragments(fragments, maxWordsCount) {
+  const sentences = fragments.listSentences === void 0 ? await listFragmentSentences(fragments) : await fragments.listSentences();
+  const batches = [];
+  let currentSentences = [];
+  let currentStartSentenceIndex = 0;
+  let currentWordsCount = 0;
+  for (let index = 0; index < sentences.length; index += 1) {
+    const sentence = sentences[index];
+    if (sentence === void 0 || sentence.text.trim() === "") {
+      continue;
+    }
+    if (currentSentences.length > 0 && currentWordsCount + sentence.wordsCount > maxWordsCount) {
+      batches.push({
+        startSentenceIndex: currentStartSentenceIndex,
+        sentences: currentSentences
+      });
+      currentSentences = [];
+      currentWordsCount = 0;
+    }
+    if (currentSentences.length === 0) {
+      currentStartSentenceIndex = index;
+    }
+    currentSentences.push(sentence);
+    currentWordsCount += sentence.wordsCount;
+  }
+  if (currentSentences.length > 0) {
+    batches.push({
+      startSentenceIndex: currentStartSentenceIndex,
+      sentences: currentSentences
+    });
+  }
+  return batches;
+}
+async function collectTextStream(stream) {
+  const parts = [];
+  for await (const chunk of stream) {
+    parts.push(chunk);
+  }
+  return parts.join("");
+}
+function countFragmentWords(sentences) {
+  return sentences.reduce((sum, sentence) => sum + sentence.wordsCount, 0);
+}
+async function readSerialPassthroughSummary(fragments, fragmentIds) {
+  if (fragmentIds.length === 0) {
+    return "";
+  }
+  const records = await Promise.all(
+    fragmentIds.map(
+      async (fragmentId) => await fragments.getFragment(fragmentId)
+    )
+  );
+  return records.flatMap(
+    (fragment) => fragment.sentences.map((sentence) => sentence.text)
+  ).join(" ").trim();
+}
+async function listFragmentSentences(fragments) {
+  const records = await Promise.all(
+    (await fragments.listFragmentIds()).map(
+      async (fragmentId) => await fragments.getFragment(fragmentId)
+    )
+  );
+  return records.sort((left, right) => left.fragmentId - right.fragmentId).flatMap((fragment) => fragment.sentences);
+}
+export {
+  collectTextStream,
+  countFragmentWords,
+  discoverSerial,
+  listSerialProcessingFragments,
+  readSerialPassthroughSummary,
+  streamFragments
+};
+//# sourceMappingURL=fragments.js.map

package/dist/text/serial/fragments.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/text/serial/fragments.ts"],"sourcesContent":["import type { ReadonlySerialFragments } from \"../../document/index.js\";\nimport {\n  segmentTextStream,\n  type ReaderSegmenter,\n  type ReaderTextStream,\n} from \"../reader/index.js\";\nimport { DEFAULT_FRAGMENT_WORDS_COUNT } from \"./options.js\";\nimport type { SerialDiscovery } from \"./options.js\";\n\nexport async function discoverSerial(input: {\n  readonly segmenter?: ReaderSegmenter;\n  readonly stream: ReaderTextStream;\n}): Promise<SerialDiscovery> {\n  let fragments = 0;\n  let words = 0;\n\n  for await (const fragment of streamFragments({\n    maxWordsCount: DEFAULT_FRAGMENT_WORDS_COUNT,\n    stream: segmentTextStream(input.stream, {\n      ...(input.segmenter === undefined ? {} : { adapter: input.segmenter }),\n    }),\n  })) {\n    fragments += 1;\n    words += countFragmentWords(fragment.sentences);\n  }\n\n  return {\n    fragments,\n    words,\n  };\n}\n\nexport async function* streamFragments(input: {\n  maxWordsCount: number;\n  stream: AsyncIterable<{\n    readonly text: string;\n    readonly wordsCount: number;\n  }>;\n}): AsyncIterable<{\n  readonly sentences: ReadonlyArray<{\n    readonly text: string;\n    readonly wordsCount: number;\n  }>;\n}> {\n  let currentSentences: Array<{\n    readonly text: string;\n    readonly wordsCount: number;\n  }> = [];\n  let currentWordsCount = 0;\n\n  for await (const sentence of input.stream) {\n    const sentenceText = sentence.text.trim();\n\n    if (sentenceText === \"\") {\n      continue;\n    }\n    if (\n      currentSentences.length > 0 &&\n      currentWordsCount + sentence.wordsCount > input.maxWordsCount\n    ) {\n      yield {\n        sentences: currentSentences,\n      };\n      currentSentences = [];\n      currentWordsCount = 0;\n    }\n    currentSentences.push({\n      text: sentenceText,\n      wordsCount: sentence.wordsCount,\n    });\n    currentWordsCount += sentence.wordsCount;\n  }\n\n  if (currentSentences.length > 0) {\n    yield {\n      sentences: currentSentences,\n    };\n  }\n}\n\nexport async function listSerialProcessingFragments(\n  fragments: ReadonlySerialFragments,\n  maxWordsCount: number,\n): Promise<\n  ReadonlyArray<{\n    readonly startSentenceIndex: number;\n    readonly sentences: ReadonlyArray<{\n      readonly text: string;\n      readonly wordsCount: number;\n    }>;\n  }>\n> {\n  const sentences =\n    fragments.listSentences === undefined\n      ? await listFragmentSentences(fragments)\n      : await fragments.listSentences();\n  const batches: Array<{\n    startSentenceIndex: number;\n    sentences: Array<{\n      readonly text: string;\n      readonly wordsCount: number;\n    }>;\n  }> = [];\n  let currentSentences: Array<{\n    readonly text: string;\n    readonly wordsCount: number;\n  }> = [];\n  let currentStartSentenceIndex = 0;\n  let currentWordsCount = 0;\n\n  for (let index = 0; index < sentences.length; index += 1) {\n    const sentence = sentences[index];\n\n    if (sentence === undefined || sentence.text.trim() === \"\") {\n      continue;\n    }\n    if (\n      currentSentences.length > 0 &&\n      currentWordsCount + sentence.wordsCount > maxWordsCount\n    ) {\n      batches.push({\n        startSentenceIndex: currentStartSentenceIndex,\n        sentences: currentSentences,\n      });\n      currentSentences = [];\n      currentWordsCount = 0;\n    }\n    if (currentSentences.length === 0) {\n      currentStartSentenceIndex = index;\n    }\n    currentSentences.push(sentence);\n    currentWordsCount += sentence.wordsCount;\n  }\n\n  if (currentSentences.length > 0) {\n    batches.push({\n      startSentenceIndex: currentStartSentenceIndex,\n      sentences: currentSentences,\n    });\n  }\n\n  return batches;\n}\n\nexport async function collectTextStream(\n  stream: ReaderTextStream,\n): Promise<string> {\n  const parts: string[] = [];\n\n  for await (const chunk of stream) {\n    parts.push(chunk);\n  }\n\n  return parts.join(\"\");\n}\n\nexport function countFragmentWords(\n  sentences: ReadonlyArray<{\n    readonly text: string;\n    readonly wordsCount: number;\n  }>,\n): number {\n  return sentences.reduce((sum, sentence) => sum + sentence.wordsCount, 0);\n}\n\nexport async function readSerialPassthroughSummary(\n  fragments: ReadonlySerialFragments,\n  fragmentIds: readonly number[],\n): Promise<string> {\n  if (fragmentIds.length === 0) {\n    return \"\";\n  }\n\n  const records = await Promise.all(\n    fragmentIds.map(\n      async (fragmentId) => await fragments.getFragment(fragmentId),\n    ),\n  );\n\n  return records\n    .flatMap((fragment: Awaited<ReturnType<typeof fragments.getFragment>>) =>\n      fragment.sentences.map((sentence) => sentence.text),\n    )\n    .join(\" \")\n    .trim();\n}\n\nasync function listFragmentSentences(\n  fragments: ReadonlySerialFragments,\n): Promise<\n  ReadonlyArray<{\n    readonly text: string;\n    readonly wordsCount: number;\n  }>\n> {\n  const records = await Promise.all(\n    (await fragments.listFragmentIds()).map(\n      async (fragmentId) => await fragments.getFragment(fragmentId),\n    ),\n  );\n\n  return records\n    .sort((left, right) => left.fragmentId - right.fragmentId)\n    .flatMap((fragment) => fragment.sentences);\n}\n"],"mappings":"AACA;AAAA,EACE;AAAA,OAGK;AACP,SAAS,oCAAoC;AAG7C,eAAsB,eAAe,OAGR;AAC3B,MAAI,YAAY;AAChB,MAAI,QAAQ;AAEZ,mBAAiB,YAAY,gBAAgB;AAAA,IAC3C,eAAe;AAAA,IACf,QAAQ,kBAAkB,MAAM,QAAQ;AAAA,MACtC,GAAI,MAAM,cAAc,SAAY,CAAC,IAAI,EAAE,SAAS,MAAM,UAAU;AAAA,IACtE,CAAC;AAAA,EACH,CAAC,GAAG;AACF,iBAAa;AACb,aAAS,mBAAmB,SAAS,SAAS;AAAA,EAChD;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,OAWpC;AACD,MAAI,mBAGC,CAAC;AACN,MAAI,oBAAoB;AAExB,mBAAiB,YAAY,MAAM,QAAQ;AACzC,UAAM,eAAe,SAAS,KAAK,KAAK;AAExC,QAAI,iBAAiB,IAAI;AACvB;AAAA,IACF;AACA,QACE,iBAAiB,SAAS,KAC1B,oBAAoB,SAAS,aAAa,MAAM,eAChD;AACA,YAAM;AAAA,QACJ,WAAW;AAAA,MACb;AACA,yBAAmB,CAAC;AACpB,0BAAoB;AAAA,IACtB;AACA,qBAAiB,KAAK;AAAA,MACpB,MAAM;AAAA,MACN,YAAY,SAAS;AAAA,IACvB,CAAC;AACD,yBAAqB,SAAS;AAAA,EAChC;AAEA,MAAI,iBAAiB,SAAS,GAAG;AAC/B,UAAM;AAAA,MACJ,WAAW;AAAA,IACb;AAAA,EACF;AACF;AAEA,eAAsB,8BACpB,WACA,eASA;AACA,QAAM,YACJ,UAAU,kBAAkB,SACxB,MAAM,sBAAsB,SAAS,IACrC,MAAM,UAAU,cAAc;AACpC,QAAM,UAMD,CAAC;AACN,MAAI,mBAGC,CAAC;AACN,MAAI,4BAA4B;AAChC,MAAI,oBAAoB;AAExB,WAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;AACxD,UAAM,WAAW,UAAU,KAAK;AAEhC,QAAI,aAAa,UAAa,SAAS,KAAK,KAAK,MAAM,IAAI;AACzD;AAAA,IACF;AACA,QACE,iBAAiB,SAAS,KAC1B,oBAAoB,SAAS,aAAa,eAC1C;AACA,cAAQ,KAAK;AAAA,QACX,oBAAoB;AAAA,QACpB,WAAW;AAAA,MACb,CAAC;AACD,yBAAmB,CAAC;AACpB,0BAAoB;AAAA,IACtB;AACA,QAAI,iBAAiB,WAAW,GAAG;AACjC,kCAA4B;AAAA,IAC9B;AACA,qBAAiB,KAAK,QAAQ;AAC9B,yBAAqB,SAAS;AAAA,EAChC;AAEA,MAAI,iBAAiB,SAAS,GAAG;AAC/B,YAAQ,KAAK;AAAA,MACX,oBAAoB;AAAA,MACpB,WAAW;AAAA,IACb,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,eAAsB,kBACpB,QACiB;AACjB,QAAM,QAAkB,CAAC;AAEzB,mBAAiB,SAAS,QAAQ;AAChC,UAAM,KAAK,KAAK;AAAA,EAClB;AAEA,SAAO,MAAM,KAAK,EAAE;AACtB;AAEO,SAAS,mBACd,WAIQ;AACR,SAAO,UAAU,OAAO,CAAC,KAAK,aAAa,MAAM,SAAS,YAAY,CAAC;AACzE;AAEA,eAAsB,6BACpB,WACA,aACiB;AACjB,MAAI,YAAY,WAAW,GAAG;AAC5B,WAAO;AAAA,EACT;AAEA,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,YAAY;AAAA,MACV,OAAO,eAAe,MAAM,UAAU,YAAY,UAAU;AAAA,IAC9D;AAAA,EACF;AAEA,SAAO,QACJ;AAAA,IAAQ,CAAC,aACR,SAAS,UAAU,IAAI,CAAC,aAAa,SAAS,IAAI;AAAA,EACpD,EACC,KAAK,GAAG,EACR,KAAK;AACV;AAEA,eAAe,sBACb,WAMA;AACA,QAAM,UAAU,MAAM,QAAQ;AAAA,KAC3B,MAAM,UAAU,gBAAgB,GAAG;AAAA,MAClC,OAAO,eAAe,MAAM,UAAU,YAAY,UAAU;AAAA,IAC9D;AAAA,EACF;AAEA,SAAO,QACJ,KAAK,CAAC,MAAM,UAAU,KAAK,aAAa,MAAM,UAAU,EACxD,QAAQ,CAAC,aAAa,SAAS,SAAS;AAC7C;","names":[]}

package/dist/text/serial/generation.d.ts

@@ -0,0 +1,48 @@
+import { ReaderTextStream } from '../reader/types.js';
+import { SerialGenerationOptions, GenerateSerialOptions, SerialProgressSink, BuildSerialTopologyOptions, BuildSerialSummaryOptions } from './options.js';
+import { Serial } from './topology.js';
+import '../reader/attention/core.js';
+import '../reader/chunk-batch/types.js';
+import '../../runtime/common/language.js';
+import '../../external/llm/client/core.js';
+import '../../external/llm/context.js';
+import '../../external/llm/cache.js';
+import '../../external/llm/types.js';
+import 'ai';
+import '../../document/types.js';
+import '../reader/segment/types.js';
+import '../../runtime/common/llm-scope.js';
+import '../../document/directory/types.js';
+import '../source/meta.js';
+import 'zod';
+import '../source/toc.js';
+import '../source/types.js';
+import '../../document/database.js';
+import 'sqlite3';
+import '../../document/text-streams/serial.js';
+import '../../document/text-streams/draft.js';
+import '../../document/text-streams/types.js';
+import '../../document/stores/chunk.js';
+import '../../document/stores/types.js';
+import '../../document/stores/fragment-group.js';
+import '../../document/stores/graph-build-parameter.js';
+import '../../document/stores/mention-link.js';
+import '../../document/stores/mention.js';
+import '../../document/stores/object-metadata.js';
+import '../../document/stores/reading-edge.js';
+import '../../document/stores/serial.js';
+import '../../document/stores/snake/chunk.js';
+import '../../document/stores/snake/core.js';
+import '../../document/stores/snake/edge.js';
+
+declare class SerialGeneration {
+    #private;
+    constructor(options: SerialGenerationOptions);
+    generate(stream: ReaderTextStream, options: GenerateSerialOptions, progressTracker?: SerialProgressSink): Promise<Serial>;
+    create(stream: ReaderTextStream, options: GenerateSerialOptions): Promise<Serial>;
+    generateInto(serialId: number, stream: ReaderTextStream, options: GenerateSerialOptions, progressTracker?: SerialProgressSink): Promise<Serial>;
+    buildTopologyInto(serialId: number, options: BuildSerialTopologyOptions, progressTracker?: SerialProgressSink): Promise<void>;
+    buildSummary(serialId: number, options?: BuildSerialSummaryOptions): Promise<Serial>;
+}
+
+export { SerialGeneration };