yom-format 0.1.0

package/src/yom/generated/yomu/yom/search-term-ref.ts

@@ -0,0 +1,134 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+import * as flatbuffers from 'flatbuffers';
+
+
+
+export class SearchTermRef implements flatbuffers.IUnpackableObject<SearchTermRefT> {
+  bb: flatbuffers.ByteBuffer|null = null;
+  bb_pos = 0;
+  __init(i:number, bb:flatbuffers.ByteBuffer):SearchTermRef {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+}
+
+static getRootAsSearchTermRef(bb:flatbuffers.ByteBuffer, obj?:SearchTermRef):SearchTermRef {
+  return (obj || new SearchTermRef()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+static getSizePrefixedRootAsSearchTermRef(bb:flatbuffers.ByteBuffer, obj?:SearchTermRef):SearchTermRef {
+  bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH);
+  return (obj || new SearchTermRef()).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+}
+
+term():string|null
+term(optionalEncoding:flatbuffers.Encoding):string|Uint8Array|null
+term(optionalEncoding?:any):string|Uint8Array|null {
+  const offset = this.bb!.__offset(this.bb_pos, 4);
+  return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null;
+}
+
+dictionaryOffset():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 6);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+dictionaryLength():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 8);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+postingsOffset():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 10);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+postingsLength():bigint {
+  const offset = this.bb!.__offset(this.bb_pos, 12);
+  return offset ? this.bb!.readUint64(this.bb_pos + offset) : BigInt('0');
+}
+
+static startSearchTermRef(builder:flatbuffers.Builder) {
+  builder.startObject(5);
+}
+
+static addTerm(builder:flatbuffers.Builder, termOffset:flatbuffers.Offset) {
+  builder.addFieldOffset(0, termOffset, 0);
+}
+
+static addDictionaryOffset(builder:flatbuffers.Builder, dictionaryOffset:bigint) {
+  builder.addFieldInt64(1, dictionaryOffset, BigInt('0'));
+}
+
+static addDictionaryLength(builder:flatbuffers.Builder, dictionaryLength:bigint) {
+  builder.addFieldInt64(2, dictionaryLength, BigInt('0'));
+}
+
+static addPostingsOffset(builder:flatbuffers.Builder, postingsOffset:bigint) {
+  builder.addFieldInt64(3, postingsOffset, BigInt('0'));
+}
+
+static addPostingsLength(builder:flatbuffers.Builder, postingsLength:bigint) {
+  builder.addFieldInt64(4, postingsLength, BigInt('0'));
+}
+
+static endSearchTermRef(builder:flatbuffers.Builder):flatbuffers.Offset {
+  const offset = builder.endObject();
+  return offset;
+}
+
+static createSearchTermRef(builder:flatbuffers.Builder, termOffset:flatbuffers.Offset, dictionaryOffset:bigint, dictionaryLength:bigint, postingsOffset:bigint, postingsLength:bigint):flatbuffers.Offset {
+  SearchTermRef.startSearchTermRef(builder);
+  SearchTermRef.addTerm(builder, termOffset);
+  SearchTermRef.addDictionaryOffset(builder, dictionaryOffset);
+  SearchTermRef.addDictionaryLength(builder, dictionaryLength);
+  SearchTermRef.addPostingsOffset(builder, postingsOffset);
+  SearchTermRef.addPostingsLength(builder, postingsLength);
+  return SearchTermRef.endSearchTermRef(builder);
+}
+
+unpack(): SearchTermRefT {
+  return new SearchTermRefT(
+    this.term(),
+    this.dictionaryOffset(),
+    this.dictionaryLength(),
+    this.postingsOffset(),
+    this.postingsLength()
+  );
+}
+
+
+unpackTo(_o: SearchTermRefT): void {
+  _o.term = this.term();
+  _o.dictionaryOffset = this.dictionaryOffset();
+  _o.dictionaryLength = this.dictionaryLength();
+  _o.postingsOffset = this.postingsOffset();
+  _o.postingsLength = this.postingsLength();
+}
+}
+
+export class SearchTermRefT implements flatbuffers.IGeneratedObject {
+constructor(
+  public term: string|Uint8Array|null = null,
+  public dictionaryOffset: bigint = BigInt('0'),
+  public dictionaryLength: bigint = BigInt('0'),
+  public postingsOffset: bigint = BigInt('0'),
+  public postingsLength: bigint = BigInt('0')
+){}
+
+
+pack(builder:flatbuffers.Builder): flatbuffers.Offset {
+  const term = (this.term !== null ? builder.createString(this.term!) : 0);
+
+  return SearchTermRef.createSearchTermRef(builder,
+    term,
+    this.dictionaryOffset,
+    this.dictionaryLength,
+    this.postingsOffset,
+    this.postingsLength
+  );
+}
+}

package/src/yom/generated/yomu/yom.ts

@@ -0,0 +1,9 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any, @typescript-eslint/no-non-null-assertion */
+
+export { OCRBlock, OCRBlockT } from './yom/ocrblock.js';
+export { OCRLayout, OCRLayoutT } from './yom/ocrlayout.js';
+export { OCRLine, OCRLineT } from './yom/ocrline.js';
+export { OCRPage, OCRPageT } from './yom/ocrpage.js';
+export { OCRToken, OCRTokenT } from './yom/ocrtoken.js';

package/src/yom/hash.ts

@@ -0,0 +1,20 @@
+/** SHA-256 over `data`, returned as raw bytes (matches CryptoKit's `SHA256.hash` used natively). */
+export async function sha256(data: Uint8Array): Promise<Uint8Array> {
+  const digest = await crypto.subtle.digest("SHA-256", toArrayBuffer(data));
+  return new Uint8Array(digest);
+}
+
+export function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}
+
+// `crypto.subtle.digest` requires an ArrayBuffer-backed view; a Uint8Array
+// sliced from a larger buffer (e.g. via `subarray`) may have a non-zero
+// `byteOffset`, so copy defensively rather than pass `data.buffer` directly.
+function toArrayBuffer(data: Uint8Array): ArrayBuffer {
+  return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
+}

package/src/yom/header.ts

@@ -0,0 +1,132 @@
+import { YOM_HASH_LENGTH, YOM_HEADER_SIZE } from "./constants.js";
+
+/**
+ * Fixed binary layout shared by the YOM v1 header (4096 bytes, zero padded)
+ * and its footer mirror (the same fields, unpadded, appended at the end of
+ * the file). Mirrors the Swift implementation in
+ * `swift/Sources/YOMFormat/YOMHeader.swift` byte-for-byte:
+ * little-endian `magic(4) | formatVersion(4) | indexOffset(8) |
+ * indexCompressedLength(8) | indexUncompressedLength(8) | indexHash(32) |
+ * fileHash(32)`, 96 bytes total. Both `indexHash` and `fileHash` are zeroed
+ * out of the header/footer bytes fed into the whole-file SHA-256
+ * calculation (see `zeroingHashes`); the real digests are patched in only
+ * after that hash has been computed.
+ */
+export const HEADER_FIELDS_SIZE = 4 + 4 + 8 + 8 + 8 + YOM_HASH_LENGTH + YOM_HASH_LENGTH;
+
+export interface YOMHeader {
+  magic: Uint8Array;
+  formatVersion: number;
+  indexOffset: bigint;
+  indexCompressedLength: bigint;
+  indexUncompressedLength: bigint;
+  indexHash: Uint8Array;
+  fileHash: Uint8Array;
+}
+
+export class YOMHeaderParseError extends Error {
+  constructor() {
+    super("YOM header truncated");
+  }
+}
+
+/** The fixed 96-byte field block, identical for header and footer. */
+export function encodeHeaderFields(header: YOMHeader): Uint8Array {
+  const out = new Uint8Array(HEADER_FIELDS_SIZE);
+  const view = new DataView(out.buffer);
+  let cursor = 0;
+  out.set(header.magic, cursor);
+  cursor += 4;
+  view.setUint32(cursor, header.formatVersion, true);
+  cursor += 4;
+  view.setBigUint64(cursor, header.indexOffset, true);
+  cursor += 8;
+  view.setBigUint64(cursor, header.indexCompressedLength, true);
+  cursor += 8;
+  view.setBigUint64(cursor, header.indexUncompressedLength, true);
+  cursor += 8;
+  out.set(header.indexHash, cursor);
+  cursor += YOM_HASH_LENGTH;
+  out.set(header.fileHash, cursor);
+  cursor += YOM_HASH_LENGTH;
+  return out;
+}
+
+/** The full 4096-byte, zero-padded header. */
+export function encodeHeader(header: YOMHeader): Uint8Array {
+  const fields = encodeHeaderFields(header);
+  const out = new Uint8Array(YOM_HEADER_SIZE);
+  out.set(fields, 0);
+  return out;
+}
+
+/** The unpadded footer mirror (just the field block). */
+export function encodeFooter(header: YOMHeader): Uint8Array {
+  return encodeHeaderFields(header);
+}
+
+/**
+ * Parses the fixed 96-byte field block from the front of `data`. Does NOT
+ * validate magic/version/semantic correctness -- callers (reader,
+ * validator) decide what to do with the parsed values.
+ */
+export function parseHeaderFields(data: Uint8Array): YOMHeader {
+  if (data.length < HEADER_FIELDS_SIZE) throw new YOMHeaderParseError();
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  let cursor = 0;
+  const magic = data.slice(cursor, cursor + 4);
+  cursor += 4;
+  const formatVersion = view.getUint32(cursor, true);
+  cursor += 4;
+  const indexOffset = view.getBigUint64(cursor, true);
+  cursor += 8;
+  const indexCompressedLength = view.getBigUint64(cursor, true);
+  cursor += 8;
+  const indexUncompressedLength = view.getBigUint64(cursor, true);
+  cursor += 8;
+  const indexHash = data.slice(cursor, cursor + YOM_HASH_LENGTH);
+  cursor += YOM_HASH_LENGTH;
+  const fileHash = data.slice(cursor, cursor + YOM_HASH_LENGTH);
+  cursor += YOM_HASH_LENGTH;
+  return {
+    magic,
+    formatVersion,
+    indexOffset,
+    indexCompressedLength,
+    indexUncompressedLength,
+    indexHash,
+    fileHash,
+  };
+}
+
+/**
+ * A copy with both hash fields zeroed, used as the input to the whole-file
+ * hash calculation (see module doc comment).
+ */
+export function zeroingHashes(header: YOMHeader): YOMHeader {
+  return {
+    ...header,
+    indexHash: new Uint8Array(YOM_HASH_LENGTH),
+    fileHash: new Uint8Array(YOM_HASH_LENGTH),
+  };
+}
+
+export function headersEqual(a: YOMHeader, b: YOMHeader): boolean {
+  return (
+    bytesEqual(a.magic, b.magic) &&
+    a.formatVersion === b.formatVersion &&
+    a.indexOffset === b.indexOffset &&
+    a.indexCompressedLength === b.indexCompressedLength &&
+    a.indexUncompressedLength === b.indexUncompressedLength &&
+    bytesEqual(a.indexHash, b.indexHash) &&
+    bytesEqual(a.fileHash, b.fileHash)
+  );
+}
+
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false;
+  }
+  return true;
+}

package/src/yom/reader.ts

@@ -0,0 +1,163 @@
+import type { MainIndex } from "./generated/yomu/yom/main-index.js";
+import type { PageEntry } from "./generated/yomu/yom/page-entry.js";
+import { Compression } from "./generated/yomu/yom/compression.js";
+import { YOM_FORMAT_VERSION, YOM_HEADER_SIZE, YOM_MAGIC } from "./constants.js";
+import { parseHeaderFields } from "./header.js";
+import { bytesEqual, sha256 } from "./hash.js";
+import { zstdDecompress } from "./zstd.js";
+import { getVerifiedRootAsMainIndex } from "./flatbufferVerify.js";
+import { enforceBlobSizePolicy } from "./blobLimits.js";
+import {
+  HttpRangeByteSource,
+  MemoryByteSource,
+  type ByteSource,
+  type HttpRangeOptions,
+} from "./byteSource.js";
+import { RangeNotSupportedError } from "./errors.js";
+
+/**
+ * Opens a YOM v1 packfile for random-access reading over a {@link ByteSource}
+ * -- either an in-memory buffer ({@link MemoryByteSource}) or an HTTP server
+ * that honours `Range` requests ({@link HttpRangeByteSource}). Ports the Swift
+ * implementation in `swift/Sources/YOMFormat/YOMPackfileReader.swift`.
+ *
+ * `open`/`openSource`/`openURL` read only the fixed header and the main
+ * FlatBuffers index up front -- never the blob area. The index is
+ * bounds-verified (Gap 3) before traversal; each blob read enforces the
+ * reader-side size caps (Gap 2) before decompressing, then verifies the
+ * blob's stored SHA-256 as the final gate.
+ *
+ * Because reads may hit the network, all blob accessors are `Promise`-based.
+ * `pageImage`/`thumbnailImage` never invoke zstd: page and thumbnail blobs are
+ * always `.none` compression per `requiredCompression`.
+ */
+export class YOMReaderError extends Error {}
+
+export class YOMPackfileReader {
+  readonly mainIndex: MainIndex;
+  private readonly source: ByteSource;
+  // Retained so the index ByteBuffer that `mainIndex` reads from stays alive.
+  private readonly indexBytes: Uint8Array;
+
+  private constructor(source: ByteSource, indexBytes: Uint8Array, mainIndex: MainIndex) {
+    this.source = source;
+    this.indexBytes = indexBytes;
+    this.mainIndex = mainIndex;
+  }
+
+  /** Opens from an in-memory buffer (wraps a {@link MemoryByteSource}). */
+  static async open(bytes: Uint8Array): Promise<YOMPackfileReader> {
+    return YOMPackfileReader.openSource(new MemoryByteSource(bytes));
+  }
+
+  /** Opens over any {@link ByteSource}. Reads only the header + index. */
+  static async openSource(source: ByteSource): Promise<YOMPackfileReader> {
+    if (source.size < YOM_HEADER_SIZE) throw new YOMReaderError("file too small");
+
+    const headerBytes = await source.read(0, YOM_HEADER_SIZE);
+    const header = parseHeaderFields(headerBytes);
+    if (!bytesEqual(header.magic, YOM_MAGIC)) throw new YOMReaderError("bad magic");
+    if (header.formatVersion !== YOM_FORMAT_VERSION) {
+      throw new YOMReaderError(`unsupported format version ${header.formatVersion}`);
+    }
+
+    const indexOffset = Number(header.indexOffset);
+    const indexLength = Number(header.indexUncompressedLength);
+    if (indexOffset < YOM_HEADER_SIZE || indexOffset + indexLength > source.size) {
+      throw new YOMReaderError("index out of bounds");
+    }
+
+    const indexBytes = await source.read(indexOffset, indexLength);
+    // Cheap root + vector-length bounds verify on the hot open path (Gap 3).
+    const mainIndex = getVerifiedRootAsMainIndex(indexBytes, false);
+
+    return new YOMPackfileReader(source, indexBytes, mainIndex);
+  }
+
+  /**
+   * Opens a `.yom` over HTTP using `Range` requests -- fetching only the
+   * header + index up front, then each blob on demand. When the server does
+   * not support ranges, throws `RangeNotSupportedError` unless
+   * `fallbackToFullFetch` is set, in which case it does a single full `GET`
+   * and reads from memory.
+   */
+  static async openURL(
+    url: string,
+    opts: HttpRangeOptions & { fallbackToFullFetch?: boolean } = {},
+  ): Promise<YOMPackfileReader> {
+    const { fallbackToFullFetch, ...rangeOpts } = opts;
+    try {
+      const source = await HttpRangeByteSource.create(url, rangeOpts);
+      return await YOMPackfileReader.openSource(source);
+    } catch (err) {
+      if (fallbackToFullFetch && err instanceof RangeNotSupportedError) {
+        const fetchFn = rangeOpts.fetch ?? ((input: string, init?: RequestInit) => fetch(input, init));
+        const res = await fetchFn(url, { headers: rangeOpts.headers, signal: rangeOpts.signal });
+        if (!res.ok) throw new YOMReaderError(`full fetch failed with status ${res.status} for ${url}`);
+        const bytes = new Uint8Array(await res.arrayBuffer());
+        return YOMPackfileReader.openSource(new MemoryByteSource(bytes));
+      }
+      throw err;
+    }
+  }
+
+  get pageCount(): number {
+    return this.mainIndex.pagesLength();
+  }
+
+  /** The book title stored in the package's metadata, or `""` if absent. */
+  get title(): string {
+    const book = this.mainIndex.book();
+    const title = book?.title();
+    return typeof title === "string" ? title : "";
+  }
+
+  page(index: number): PageEntry {
+    const page = this.mainIndex.pages(index);
+    if (!page) throw new YOMReaderError("page index out of range");
+    return page;
+  }
+
+  /**
+   * Reads exactly one blob's byte range, enforcing the reader-side size caps
+   * (Gap 2) before decompressing, then verifying its stored hash against the
+   * decompressed (original) bytes -- the same convention the writer uses when
+   * hashing.
+   */
+  async readBlob(blobIndex: number): Promise<Uint8Array> {
+    if (blobIndex < 0 || blobIndex >= this.mainIndex.blobsLength()) {
+      throw new YOMReaderError("blob index out of range");
+    }
+    const blob = this.mainIndex.blobs(blobIndex);
+    if (!blob) throw new YOMReaderError("blob index out of range");
+
+    const start = Number(blob.offset());
+    const compressedLength = Number(blob.compressedLength());
+    const uncompressedLength = Number(blob.uncompressedLength());
+    if (start < 0 || compressedLength < 0 || start + compressedLength > this.source.size) {
+      throw new YOMReaderError("blob out of bounds");
+    }
+    enforceBlobSizePolicy(blob.kind(), compressedLength, uncompressedLength);
+
+    const rawBytes = await this.source.read(start, compressedLength);
+    const original =
+      blob.compression() === Compression.None ? rawBytes : await zstdDecompress(rawBytes, uncompressedLength);
+
+    const actualHash = await sha256(original);
+    const expectedHash = blob.hashArray();
+    if (!expectedHash || !bytesEqual(actualHash, expectedHash)) {
+      throw new YOMReaderError("blob hash mismatch");
+    }
+    return original;
+  }
+
+  /** Never invokes zstd -- page image blobs are always `.none` by policy. */
+  async pageImage(pageIndex: number): Promise<Uint8Array> {
+    return this.readBlob(this.page(pageIndex).pageImageBlobIndex());
+  }
+
+  /** Never invokes zstd -- thumbnail blobs are always `.none` by policy. */
+  async thumbnailImage(pageIndex: number): Promise<Uint8Array> {
+    return this.readBlob(this.page(pageIndex).thumbnailBlobIndex());
+  }
+}

package/src/yom/searchIndex.ts

@@ -0,0 +1,178 @@
+/**
+ * Builds YOM's search dictionary + postings blobs over normalized text
+ * (`textNormalizer.ts` output), using character bigrams. Ports the Swift
+ * implementation in `swift/Sources/YOMFormat/YOMSearchIndexBuilder.swift`.
+ *
+ * **Dictionary blob**: the concatenated UTF-8 bytes of every term, in sorted
+ * order. Redundant with `SearchTermRef.term` in the main index but is a
+ * deliberately simple, schema-valid choice (see the Swift source's doc
+ * comment) -- the format only requires the byte range exist, not that it
+ * hold any particular content.
+ *
+ * **Postings blob**: for each term, a delta+varint-encoded sequence of
+ * `(page, position)` hits sorted by page then position, where `position` is
+ * the *code-point* offset within that page's normalized text. Never
+ * compressed (`.none`) per the fixed policy.
+ *
+ * Iterates by Unicode code point (`Array.from`), the closest JS analogue of
+ * Swift's extended-grapheme-cluster `Character` iteration -- the two only
+ * diverge for combining marks/emoji sequences, which NFKC-normalized
+ * Japanese book text does not produce in practice.
+ */
+export interface SearchTerm {
+  term: string;
+  dictionaryOffset: number;
+  dictionaryLength: number;
+  postingsOffset: number;
+  postingsLength: number;
+}
+
+export interface SearchIndexResult {
+  dictionaryBlob: Uint8Array;
+  postingsBlob: Uint8Array;
+  searchTerms: SearchTerm[];
+}
+
+export interface Hit {
+  page: number;
+  position: number;
+}
+
+export function buildSearchIndex(normalizedPageTexts: string[]): SearchIndexResult {
+  const postingsByTerm = new Map<string, Hit[]>();
+
+  normalizedPageTexts.forEach((text, pageIndex) => {
+    const characters = Array.from(text);
+    if (characters.length < 2) return;
+    for (let index = 0; index <= characters.length - 2; index++) {
+      const gram = characters[index] + characters[index + 1];
+      let hits = postingsByTerm.get(gram);
+      if (!hits) {
+        hits = [];
+        postingsByTerm.set(gram, hits);
+      }
+      hits.push({ page: pageIndex, position: index });
+    }
+  });
+
+  const encoder = new TextEncoder();
+  const dictionaryChunks: Uint8Array[] = [];
+  const postingsChunks: Uint8Array[] = [];
+  const searchTerms: SearchTerm[] = [];
+  let dictionaryCursor = 0;
+  let postingsCursor = 0;
+
+  const terms = Array.from(postingsByTerm.keys()).sort();
+  for (const term of terms) {
+    const hits = postingsByTerm.get(term)!.sort((a, b) => (a.page === b.page ? a.position - b.position : a.page - b.page));
+
+    const termBytes = encoder.encode(term);
+    dictionaryChunks.push(termBytes);
+    const dictionaryOffset = dictionaryCursor;
+    dictionaryCursor += termBytes.length;
+
+    const postingsBytes = encodePostings(hits);
+    postingsChunks.push(postingsBytes);
+    const postingsOffset = postingsCursor;
+    postingsCursor += postingsBytes.length;
+
+    searchTerms.push({
+      term,
+      dictionaryOffset,
+      dictionaryLength: termBytes.length,
+      postingsOffset,
+      postingsLength: postingsBytes.length,
+    });
+  }
+
+  return {
+    dictionaryBlob: concatBytes(dictionaryChunks, dictionaryCursor),
+    postingsBlob: concatBytes(postingsChunks, postingsCursor),
+    searchTerms,
+  };
+}
+
+/** Decodes one term's postings byte range back into `(page, position)` hits. */
+export function decodePostings(blob: Uint8Array): Hit[] {
+  const hits: Hit[] = [];
+  let offset = 0;
+  let previousPage = 0;
+  let previousPosition = 0;
+  let isFirst = true;
+
+  while (offset < blob.length) {
+    const first = readVarint(blob, offset);
+    offset = first.nextOffset;
+    const second = readVarint(blob, offset);
+    offset = second.nextOffset;
+
+    if (isFirst) {
+      previousPage = first.value;
+      previousPosition = second.value;
+      isFirst = false;
+    } else if (first.value === 0) {
+      previousPosition += second.value;
+    } else {
+      previousPage += first.value;
+      previousPosition = second.value;
+    }
+    hits.push({ page: previousPage, position: previousPosition });
+  }
+  return hits;
+}
+
+function encodePostings(hits: Hit[]): Uint8Array {
+  const bytes: number[] = [];
+  let previousPage = 0;
+  let previousPosition = 0;
+  hits.forEach((hit, index) => {
+    if (index === 0) {
+      writeVarint(hit.page, bytes);
+      writeVarint(hit.position, bytes);
+    } else if (hit.page === previousPage) {
+      writeVarint(0, bytes);
+      writeVarint(hit.position - previousPosition, bytes);
+    } else {
+      writeVarint(hit.page - previousPage, bytes);
+      writeVarint(hit.position, bytes);
+    }
+    previousPage = hit.page;
+    previousPosition = hit.position;
+  });
+  return Uint8Array.from(bytes);
+}
+
+function writeVarint(value: number, out: number[]): void {
+  let remaining = value >>> 0;
+  while (true) {
+    let byte = remaining & 0x7f;
+    remaining >>>= 7;
+    if (remaining !== 0) byte |= 0x80;
+    out.push(byte);
+    if (remaining === 0) break;
+  }
+}
+
+function readVarint(bytes: Uint8Array, offset: number): { value: number; nextOffset: number } {
+  let result = 0;
+  let shift = 0;
+  let cursor = offset;
+  while (true) {
+    const byte = bytes[cursor];
+    cursor += 1;
+    result |= (byte & 0x7f) << shift;
+    if ((byte & 0x80) === 0) break;
+    shift += 7;
+  }
+  return { value: result >>> 0, nextOffset: cursor };
+}
+
+function concatBytes(chunks: Uint8Array[], totalLength: number): Uint8Array {
+  const out = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+}