turbodata 0.1.0

package/src/msgpack.ts

@@ -0,0 +1,46 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// msgpack decoder for TopicMetadata.Metadata. Uses @msgpack/msgpack with
+// useBigInt64 so int64 values round-trip exactly.
+
+import { decode } from "@msgpack/msgpack";
+
+/**
+ * Decode a msgpack-encoded map into a Map<string, unknown>.
+ *
+ * - `useBigInt64: true` so any int64 values round-trip exactly as bigint.
+ *   Smaller integers (int32 and below) decode as number.
+ * - The Go writer always msgpack-encodes a `map[string]any`. We coerce the
+ *   decoded plain object into a `Map` for an unambiguous string-keyed view.
+ */
+export function decodeMetadata(buf: Uint8Array): Map<string, unknown> {
+  const decoded = decode(buf, { useBigInt64: true });
+  const out = new Map<string, unknown>();
+  if (decoded === null || typeof decoded !== "object") {
+    return out;
+  }
+  if (decoded instanceof Map) {
+    for (const [k, v] of decoded as Map<unknown, unknown>) {
+      if (typeof k === "string") {
+        out.set(k, v);
+      }
+    }
+    return out;
+  }
+  for (const k of Object.keys(decoded as Record<string, unknown>)) {
+    out.set(k, (decoded as Record<string, unknown>)[k]);
+  }
+  return out;
+}

package/src/multi_reader.ts

@@ -0,0 +1,337 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// MultiReader: present several single-file Readers as one time-ordered stream.
+// Mirrors go/multi_reader.go and py/turbodata/multi_reader.py.
+//
+// readMessages merges every reader's stream in timestamp order; sample resolves
+// floor messages across the union of readers. Both operate entirely in each
+// Reader's exposed-name space. Topic-name collisions across files are the
+// caller's responsibility: configure per-Reader topicRemap so that names meant
+// to union share an exposed name and names meant to stay distinct do not.
+
+import { MessageHeap } from "./iterators/message_heap.js";
+import type { ReadOptions } from "./read_options.js";
+import {
+  Reader,
+  SampleValidationError,
+  type Message,
+  type SampleOptions,
+  type SampleQuery,
+  type SampleResult,
+  type TopicBound,
+} from "./reader.js";
+
+/**
+ * Thrown by MultiReader.sample/readMessages under videoDecodable when a queried
+ * video topic is provided by more than one reader whose time ranges overlap.
+ * Decodable multi-file video requires each video topic's source files to be
+ * time-disjoint; merging overlapping decodable sources would interleave frames
+ * from different GOP chains into an undecodable stream. Mirrors go
+ * ErrVideoSourcesOverlap / py VideoSourcesOverlapError.
+ */
+export class VideoSourcesOverlapError extends Error {
+  readonly topic: string;
+  readonly first: readonly [bigint, bigint];
+  readonly second: readonly [bigint, bigint];
+  constructor(
+    topic: string,
+    first: readonly [bigint, bigint],
+    second: readonly [bigint, bigint],
+  ) {
+    super(
+      `turbodata: video topic "${topic}" sources [${first[0]},${first[1]}] and [${second[0]},${second[1]}] overlap in time`,
+    );
+    this.name = "VideoSourcesOverlapError";
+    this.topic = topic;
+    this.first = first;
+    this.second = second;
+  }
+}
+
+export class MultiReader {
+  private readonly readers: Reader[];
+
+  /**
+   * Group readers into one merged view. At least one reader is required.
+   * Per-file behavior (remap, etc.) lives on each underlying Reader.
+   */
+  constructor(readers: Reader[]) {
+    if (readers.length === 0) {
+      throw new Error("turbodata: MultiReader requires at least one reader");
+    }
+    this.readers = readers;
+  }
+
+  /**
+   * Read every reader's messages as one merged, time-ordered stream. Options
+   * pass through to each underlying Reader.readMessages unchanged, so order,
+   * time bounds, topic filtering (by exposed name), and strategy apply per file
+   * before the merge.
+   *
+   * When videoDecodable is set, any in-scope video topic supplied by more than
+   * one reader must have time-disjoint source ranges; overlapping sources throw
+   * VideoSourcesOverlapError (merging them would interleave frames from
+   * different GOP chains into an undecodable stream). As with the single
+   * Reader, all I/O (and this check) is deferred to the first iteration step.
+   */
+  readMessages(opts: ReadOptions = {}): AsyncIterableIterator<Message> {
+    const readers = this.readers;
+    const reverse = opts.order === "reverse-time";
+    const videoDecodable = opts.videoDecodable === true;
+    const wantedNames = opts.topicNames;
+
+    let initialized = false;
+    let exhausted = false;
+    const subs: AsyncIterableIterator<Message>[] = [];
+    const heap = new MessageHeap<{ subIdx: number; msg: Message }>(reverse);
+
+    const init = async (): Promise<void> => {
+      for (const r of readers) {
+        subs.push(r.readMessages(opts));
+      }
+
+      if (videoDecodable) {
+        const bounds = await Promise.all(readers.map((r) => r.topicBounds()));
+        enforceVideoDisjoint(readScopeTopics(wantedNames, bounds), bounds);
+      }
+
+      // Prime the heap with one message from each reader's stream.
+      for (let i = 0; i < subs.length; i++) {
+        const res = await subs[i]!.next();
+        if (res.done === true) {
+          continue;
+        }
+        heap.push(res.value.timestamp, { subIdx: i, msg: res.value });
+      }
+    };
+
+    const iter: AsyncIterableIterator<Message> = {
+      [Symbol.asyncIterator]() {
+        return iter;
+      },
+      async next(): Promise<IteratorResult<Message>> {
+        if (exhausted) {
+          return { value: undefined, done: true };
+        }
+        if (!initialized) {
+          await init();
+          initialized = true;
+        }
+        const top = heap.pop();
+        if (top === undefined) {
+          exhausted = true;
+          return { value: undefined, done: true };
+        }
+        const out = top.value.msg;
+        const nm = await subs[top.value.subIdx]!.next();
+        if (nm.done !== true) {
+          heap.push(nm.value.timestamp, {
+            subIdx: top.value.subIdx,
+            msg: nm.value,
+          });
+        }
+        return { value: out, done: false };
+      },
+      async return(): Promise<IteratorResult<Message>> {
+        exhausted = true;
+        for (const s of subs) {
+          if (s.return !== undefined) {
+            await s.return();
+          }
+        }
+        return { value: undefined, done: true };
+      },
+    };
+
+    return iter;
+  }
+
+  /**
+   * Resolve floor messages across every reader. For each
+   * (queries[i].topic, queries[i].timestamps[j]) pair, out[i][j] is the floor
+   * result with the latest found timestamp among all readers that hold the
+   * topic (by exposed name) - the true floor over the union. Ties resolve
+   * arbitrarily.
+   *
+   * Preconditions, validated before any data I/O (aggregated into a single
+   * SampleValidationError):
+   *   - each queries[i].timestamps must be strictly increasing
+   *   - each queries[i].topic must be unique across all i
+   *   - each queries[i].topic must exist in at least one reader
+   *
+   * When videoDecodable is set, a video topic supplied by more than one reader
+   * must have time-disjoint source ranges; overlapping sources throw
+   * VideoSourcesOverlapError.
+   */
+  async sample(
+    queries: SampleQuery[],
+    opts: SampleOptions = {},
+  ): Promise<SampleResult[][]> {
+    const videoDecodable = opts.videoDecodable === true;
+    const bounds = await Promise.all(this.readers.map((r) => r.topicBounds()));
+
+    validateMultiSampleQueries(queries, bounds);
+
+    if (videoDecodable) {
+      enforceVideoDisjoint(
+        queries.map((q) => q.topic),
+        bounds,
+      );
+    }
+
+    const out: SampleResult[][] = queries.map((q) =>
+      q.timestamps.map(() => ({
+        found: false,
+        timestamp: 0n,
+        data: new Uint8Array(0),
+        isVideo: false,
+        frames: [],
+        resetDecoder: false,
+      })),
+    );
+
+    // Dispatch to each reader only the queries whose topic it holds (avoids
+    // the engine's unknown-topic error), then merge by latest found floor.
+    for (let ri = 0; ri < this.readers.length; ri++) {
+      const subset: SampleQuery[] = [];
+      const origIdx: number[] = [];
+      for (let qi = 0; qi < queries.length; qi++) {
+        if (bounds[ri]!.has(queries[qi]!.topic)) {
+          subset.push(queries[qi]!);
+          origIdx.push(qi);
+        }
+      }
+      if (subset.length === 0) {
+        continue;
+      }
+      const res = await this.readers[ri]!.sample(subset, opts);
+      for (let sub = 0; sub < origIdx.length; sub++) {
+        const qi = origIdx[sub]!;
+        const row = res[sub]!;
+        for (let j = 0; j < row.length; j++) {
+          const cand = row[j]!;
+          if (!cand.found) {
+            continue;
+          }
+          const cur = out[qi]![j]!;
+          if (!cur.found || cand.timestamp > cur.timestamp) {
+            out[qi]![j] = cand;
+          }
+        }
+      }
+    }
+    return out;
+  }
+}
+
+/**
+ * The exposed topic names a readMessages call covers: the requested
+ * topicNames, or every exposed topic across all readers when unset.
+ */
+function readScopeTopics(
+  topicNames: string[] | undefined,
+  bounds: Map<string, TopicBound>[],
+): string[] {
+  if (topicNames !== undefined && topicNames.length > 0) {
+    return topicNames;
+  }
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const b of bounds) {
+    for (const name of b.keys()) {
+      if (!seen.has(name)) {
+        seen.add(name);
+        out.push(name);
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Mirror the single-reader checks but treat a topic as known if any reader
+ * holds it. All violations are aggregated into one SampleValidationError.
+ * Mirrors go validateMultiSampleQueries.
+ */
+function validateMultiSampleQueries(
+  queries: SampleQuery[],
+  bounds: Map<string, TopicBound>[],
+): void {
+  const seen = new Map<string, number>();
+  const violations: string[] = [];
+  for (let i = 0; i < queries.length; i++) {
+    const q = queries[i]!;
+    const known = bounds.some((b) => b.has(q.topic));
+    if (!known) {
+      violations.push(
+        `queries[${i}]: unknown topic "${q.topic}" (not in any reader)`,
+      );
+    }
+    const prev = seen.get(q.topic);
+    if (prev !== undefined) {
+      violations.push(
+        `queries[${i}]: duplicate topic "${q.topic}" already used by queries[${prev}]`,
+      );
+    } else {
+      seen.set(q.topic, i);
+    }
+    for (let j = 1; j < q.timestamps.length; j++) {
+      if (q.timestamps[j]! <= q.timestamps[j - 1]!) {
+        violations.push(
+          `queries[${i}].timestamps not strictly increasing at position ${j} (${q.timestamps[j]} <= ${q.timestamps[j - 1]})`,
+        );
+      }
+    }
+  }
+  if (violations.length > 0) {
+    throw new SampleValidationError(violations);
+  }
+}
+
+/**
+ * Throw VideoSourcesOverlapError when any of the given exposed video topics is
+ * provided by readers with overlapping inclusive time ranges. Topics that are
+ * not video, or are held by fewer than two readers, are skipped. Mirrors go
+ * enforceVideoDisjoint.
+ */
+function enforceVideoDisjoint(
+  topics: string[],
+  bounds: Map<string, TopicBound>[],
+): void {
+  for (const topic of topics) {
+    const ranges: [bigint, bigint][] = [];
+    let isVideo = false;
+    for (const b of bounds) {
+      const bound = b.get(topic);
+      if (bound === undefined) {
+        continue;
+      }
+      if (bound.isVideo) {
+        isVideo = true;
+      }
+      ranges.push([bound.minTs, bound.maxTs]);
+    }
+    if (!isVideo || ranges.length < 2) {
+      continue;
+    }
+    for (let a = 0; a < ranges.length; a++) {
+      for (let c = a + 1; c < ranges.length; c++) {
+        if (ranges[a]![0] <= ranges[c]![1] && ranges[c]![0] <= ranges[a]![1]) {
+          throw new VideoSourcesOverlapError(topic, ranges[a]!, ranges[c]!);
+        }
+      }
+    }
+  }
+}

package/src/read_fetcher.ts

@@ -0,0 +1,74 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Read fetcher: executes ReadOps with bounded concurrency. Mirrors
+// go/read_fetcher.go (promise-pool instead of goroutines + token channel).
+
+import type { ReadOp } from "./read_planner.js";
+import type { ReadSource } from "./read_source.js";
+
+/**
+ * Execute ops against rs with at most maxConcurrency in-flight reads. Returns
+ * one Uint8Array per op, in input order. Throws on the first error
+ * encountered; in-flight reads may still settle in the background.
+ */
+export async function fetchAll(
+  rs: ReadSource,
+  ops: ReadOp[],
+  maxConcurrency: number,
+): Promise<Uint8Array[]> {
+  if (ops.length === 0) {
+    return [];
+  }
+  const workers = maxConcurrency > 0 ? maxConcurrency : 1;
+  const out = new Array<Uint8Array>(ops.length);
+
+  let nextIdx = 0;
+  let firstErr: unknown = undefined;
+
+  const runOne = async (): Promise<void> => {
+    while (firstErr === undefined) {
+      const i = nextIdx++;
+      if (i >= ops.length) {
+        return;
+      }
+      const op = ops[i]!;
+      try {
+        const buf = await rs.read(op.offset, op.length);
+        if (BigInt(buf.byteLength) !== op.length) {
+          throw new Error(
+            `fetcher: short read for op ${i}: got ${buf.byteLength} want ${op.length}`,
+          );
+        }
+        out[i] = buf;
+      } catch (err) {
+        if (firstErr === undefined) {
+          firstErr = err;
+        }
+        return;
+      }
+    }
+  };
+
+  const pool: Promise<void>[] = new Array(Math.min(workers, ops.length));
+  for (let w = 0; w < pool.length; w++) {
+    pool[w] = runOne();
+  }
+  await Promise.all(pool);
+
+  if (firstErr !== undefined) {
+    throw firstErr;
+  }
+  return out;
+}

package/src/read_options.ts

@@ -0,0 +1,58 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Read options for Reader.readMessages(). Mirrors go/read_option.go.
+
+import type { ReadStrategy } from "./read_strategy.js";
+
+export type Order = "time" | "reverse-time";
+
+export interface ReadOptions {
+  /** When set, only messages from these topic names are emitted. */
+  topicNames?: string[];
+  /** int64. Inclusive lower bound on message timestamp. Default: 0n. */
+  startTimestamp?: bigint;
+  /** int64. Inclusive upper bound on message timestamp. Default: int64-max. */
+  endTimestamp?: bigint;
+  /** Iteration order. Default: "time". */
+  order?: Order;
+  /**
+   * When set, switches Reader.readMessages onto the cost-aware path. Without
+   * a strategy, the reader uses the default lazy path (one chunk at a time).
+   */
+  strategy?: ReadStrategy;
+  /**
+   * int64. Bytes to read speculatively from the file tail when loading the
+   * summary. When the (footer + compressed summary) fits within the prefetch
+   * window, summary loading costs one read instead of two. Default: 0n
+   * (footer-only read).
+   */
+  tailPrefetch?: bigint;
+  /**
+   * If true, the iterator hands out a freshly-allocated Uint8Array per
+   * message (safe to retain). If false (default), `data` aliases an internal
+   * reusable buffer that becomes invalid on the next iteration step.
+   */
+  copy?: boolean;
+  /**
+   * If true, for any video topic in scope the effective per-group
+   * startTimestamp is snapped back to the latest key frame whose timestamp is
+   * <= startTimestamp, so the emitted sequence can be fed to a decoder cold.
+   * Non-video topics are unaffected. Default: false.
+   */
+  videoDecodable?: boolean;
+}
+
+/** Default value for endTimestamp: matches Go's math.MaxInt64. */
+export const MAX_INT64 = 0x7fffffffffffffffn;

package/src/read_planner.ts

@@ -0,0 +1,153 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Read planner: groups Ranges into ReadOps according to a ReadStrategy.
+// Mirrors go/read_planner.go.
+
+import type { ReadStrategy } from "./read_strategy.js";
+
+export interface Range {
+  /** int64. Absolute file offset. */
+  offset: bigint;
+  /** int64. Length in bytes. */
+  length: bigint;
+}
+
+export interface ReadOp {
+  offset: bigint;
+  length: bigint;
+}
+
+export interface RangeLocation {
+  /** Index into the produced ReadOp[]. */
+  opIndex: number;
+  /** Byte offset within that op's buffer (must fit in number; ops are bounded). */
+  inOpOff: number;
+  /** Length of this original range, in bytes. */
+  length: number;
+}
+
+interface CoalesceGroup {
+  offset: bigint;
+  end: bigint; // exclusive
+  members: number[]; // indices into input ranges, in offset order
+}
+
+/**
+ * Plan groups Ranges into ReadOps.
+ *
+ * Precondition: ranges must be in non-decreasing offset order. All call sites
+ * in this package satisfy this by construction (writer lays out groups and
+ * chunks at monotonically increasing file offsets).
+ *
+ * Algorithm:
+ *  1. Coalesce: merge adjacent ranges whose gap is < coalesceGap.
+ *  2. Split: a coalesced op whose length exceeds splitThreshold is sliced at
+ *     internal range boundaries via greedy packing. A single range larger
+ *     than splitThreshold stays as one oversize op (atomic units never break).
+ */
+export function plan(
+  ranges: Range[],
+  s: ReadStrategy,
+): { ops: ReadOp[]; locations: RangeLocation[] } {
+  const locations: RangeLocation[] = new Array(ranges.length);
+  if (ranges.length === 0) {
+    return { ops: [], locations };
+  }
+
+  const groups: CoalesceGroup[] = [];
+  for (let idx = 0; idx < ranges.length; idx++) {
+    const r = ranges[idx]!;
+    if (groups.length === 0) {
+      groups.push({
+        offset: r.offset,
+        end: r.offset + r.length,
+        members: [idx],
+      });
+      continue;
+    }
+    const cur = groups[groups.length - 1]!;
+    const gap = r.offset - cur.end;
+    if (gap >= 0n && gap < s.coalesceGap) {
+      const newEnd = r.offset + r.length;
+      if (newEnd > cur.end) {
+        cur.end = newEnd;
+      }
+      cur.members.push(idx);
+      continue;
+    }
+    groups.push({
+      offset: r.offset,
+      end: r.offset + r.length,
+      members: [idx],
+    });
+  }
+
+  const ops: ReadOp[] = [];
+  const threshold = s.splitThreshold;
+  const splitDisabled = threshold <= 0n;
+
+  for (const g of groups) {
+    if (splitDisabled || g.end - g.offset <= threshold) {
+      const opIndex = ops.length;
+      ops.push({ offset: g.offset, length: g.end - g.offset });
+      for (const mi of g.members) {
+        const r = ranges[mi]!;
+        locations[mi] = {
+          opIndex,
+          inOpOff: Number(r.offset - g.offset),
+          length: Number(r.length),
+        };
+      }
+      continue;
+    }
+
+    // Greedy split at range boundaries.
+    let opStartIdx = 0;
+    let opStartOffset = ranges[g.members[0]!]!.offset;
+    let opEndOffset =
+      ranges[g.members[0]!]!.offset + ranges[g.members[0]!]!.length;
+
+    const flushOp = (lastMember: number): void => {
+      const opIndex = ops.length;
+      ops.push({ offset: opStartOffset, length: opEndOffset - opStartOffset });
+      for (let k = opStartIdx; k <= lastMember; k++) {
+        const mi = g.members[k]!;
+        const r = ranges[mi]!;
+        locations[mi] = {
+          opIndex,
+          inOpOff: Number(r.offset - opStartOffset),
+          length: Number(r.length),
+        };
+      }
+    };
+
+    for (let k = 1; k < g.members.length; k++) {
+      const mi = g.members[k]!;
+      const r = ranges[mi]!;
+      const nextEnd = r.offset + r.length;
+      if (nextEnd - opStartOffset > threshold) {
+        flushOp(k - 1);
+        opStartIdx = k;
+        opStartOffset = r.offset;
+        opEndOffset = nextEnd;
+        continue;
+      }
+      opEndOffset = nextEnd;
+    }
+    flushOp(g.members.length - 1);
+  }
+
+  return { ops, locations };
+}

package/src/read_source.ts

@@ -0,0 +1,33 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * ReadSource is the storage abstraction the Reader uses. It is intentionally
+ * minimal: every Go ReadAt/Read+Seek call site can be expressed as `read(off,
+ * len)`.
+ *
+ * Concurrency contract: implementations MUST be safe for concurrent `read`
+ * calls. The cost-aware reader path issues many in-flight reads bounded only
+ * by `maxConcurrency`.
+ */
+export interface ReadSource {
+  /** Total length of the underlying file/blob/resource, in bytes. */
+  size(): Promise<bigint>;
+
+  /**
+   * Read exactly `length` bytes starting at `offset`. Implementations should
+   * throw if fewer bytes are available; the Reader assumes a full read.
+   */
+  read(offset: bigint, length: bigint): Promise<Uint8Array>;
+}