turbodata 0.1.0

package/src/sort_and_filter.ts

@@ -0,0 +1,167 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// sortAndFilter: pure version that turns a decoded IndexChunk into a flat,
+// timestamp-ordered list of kept (topicId, timestamp, offsetInChunk) plus
+// per-message lengths. Mirrors go/sort_and_filter.go.
+//
+// The "sort" is by per-message OffsetInChunk; the writer interleaves messages
+// in timestamp order, so offset order == timestamp order within a chunk. We
+// preserve that invariant rather than re-sorting by timestamp explicitly.
+
+import { MessageHeap } from "./iterators/message_heap.js";
+import type { IndexChunk, MessageIndex, TopicIndex } from "./types.js";
+
+export interface MessageRef {
+  timestamp: bigint;
+  topicId: number;
+  offsetInChunk: bigint;
+}
+
+interface ItemWithTopic {
+  topicId: number;
+  mi: MessageIndex;
+}
+
+export function sortAndFilter(
+  topicIndexes: TopicIndex[],
+  totalLen: bigint,
+  topicIds: Set<number>,
+  startTimestamp: bigint,
+  endTimestamp: bigint,
+  videoDecodable = false,
+): { msgs: MessageRef[]; lens: bigint[] } {
+  let total = 0;
+  for (const ti of topicIndexes) {
+    total += ti.messageIndexes.length;
+  }
+  if (total === 0) {
+    return { msgs: [], lens: [] };
+  }
+
+  const sorted: ItemWithTopic[] = new Array(total);
+  let writeIdx = 0;
+
+  if (topicIndexes.length === 1) {
+    // Fast path: single topic, messages are already in offset order.
+    const ti = topicIndexes[0]!;
+    for (const mi of ti.messageIndexes) {
+      sorted[writeIdx++] = { topicId: ti.id, mi };
+    }
+  } else {
+    // K-way merge by offsetInChunk across topics, via a min-heap.
+    // Mirrors go/sort_and_filter.go's MessageIndexHeap-based merge: O(N log K)
+    // regardless of how many topics share a group.
+    const heap = new MessageHeap<{ topicSlot: number; cursor: number }>(false);
+    for (let t = 0; t < topicIndexes.length; t++) {
+      const mis = topicIndexes[t]!.messageIndexes;
+      if (mis.length === 0) {
+        continue;
+      }
+      heap.push(mis[0]!.offsetInChunk, { topicSlot: t, cursor: 0 });
+    }
+    while (heap.size > 0) {
+      const top = heap.pop()!;
+      const { topicSlot, cursor } = top.value;
+      const ti = topicIndexes[topicSlot]!;
+      sorted[writeIdx++] = { topicId: ti.id, mi: ti.messageIndexes[cursor]! };
+      const next = cursor + 1;
+      if (next < ti.messageIndexes.length) {
+        heap.push(ti.messageIndexes[next]!.offsetInChunk, {
+          topicSlot,
+          cursor: next,
+        });
+      }
+    }
+  }
+
+  // Per-entry lengths from offset deltas.
+  const allLens = new Array<bigint>(sorted.length);
+  for (let i = 0; i < sorted.length - 1; i++) {
+    allLens[i] = sorted[i + 1]!.mi.offsetInChunk - sorted[i]!.mi.offsetInChunk;
+  }
+  allLens[allLens.length - 1] =
+    totalLen - sorted[sorted.length - 1]!.mi.offsetInChunk;
+
+  // For video-decodable groups, snap the lower bound back to the anchoring
+  // key frame so the kept sequence is decodable cold.
+  let effectiveStart = startTimestamp;
+  if (videoDecodable) {
+    effectiveStart = keyFrameStart(topicIndexes, startTimestamp);
+  }
+
+  // Filter by topic + timestamp range.
+  const msgs: MessageRef[] = [];
+  const lens: bigint[] = [];
+  for (let i = 0; i < sorted.length; i++) {
+    const item = sorted[i]!;
+    if (!topicIds.has(item.topicId)) {
+      continue;
+    }
+    const ts = item.mi.timestamp;
+    if (ts < effectiveStart || ts > endTimestamp) {
+      continue;
+    }
+    msgs.push({
+      timestamp: ts,
+      topicId: item.topicId,
+      offsetInChunk: item.mi.offsetInChunk,
+    });
+    lens.push(allLens[i]!);
+  }
+  return { msgs, lens };
+}
+
+/**
+ * Timestamp of the latest key frame whose timestamp is <= startTimestamp, or
+ * startTimestamp unchanged if no such key frame exists in this chunk. Mirrors
+ * py key_frame_start / go keyFrameStart.
+ *
+ * Video-decodable groups always hold exactly one topic. keyFrameIndexes are
+ * ascending positions into that topic's timestamp-ordered messageIndexes.
+ */
+export function keyFrameStart(
+  topicIndexes: TopicIndex[],
+  startTimestamp: bigint,
+): bigint {
+  if (topicIndexes.length === 0) {
+    return startTimestamp;
+  }
+  const ti = topicIndexes[0]!;
+  let anchor = startTimestamp;
+  for (const kfIdx of ti.keyFrameIndexes) {
+    const ts = ti.messageIndexes[kfIdx]!.timestamp;
+    if (ts > startTimestamp) {
+      break;
+    }
+    anchor = ts;
+  }
+  return anchor;
+}
+
+/** Convenience: sort+filter directly from a decoded IndexChunk. */
+export function sortAndFilterChunk(
+  chunk: IndexChunk,
+  topicIds: Set<number>,
+  startTimestamp: bigint,
+  endTimestamp: bigint,
+): { msgs: MessageRef[]; lens: bigint[] } {
+  return sortAndFilter(
+    chunk.topicIndexes,
+    chunk.uncompressedLen,
+    topicIds,
+    startTimestamp,
+    endTimestamp,
+  );
+}

package/src/sources/blob.ts

@@ -0,0 +1,52 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// BlobReadSource: ReadSource backed by a Blob (or File, which extends Blob).
+//
+// Blob.slice() is synchronous, so read concurrency is implicit at the browser
+// level — the only async work is decoding to Uint8Array via arrayBuffer().
+
+import type { ReadSource } from "../read_source.js";
+
+export class BlobReadSource implements ReadSource {
+  private readonly blob: Blob;
+
+  constructor(blob: Blob) {
+    this.blob = blob;
+  }
+
+  async size(): Promise<bigint> {
+    return BigInt(this.blob.size);
+  }
+
+  async read(offset: bigint, length: bigint): Promise<Uint8Array> {
+    // Blob.slice takes number offsets. We accept that browsers can't host a
+    // single Blob > 2^53 bytes anyway; clamp via Number() and validate.
+    const start = Number(offset);
+    const end = Number(offset + length);
+    if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end)) {
+      throw new Error(
+        `BlobReadSource.read: offset/length exceed Number.MAX_SAFE_INTEGER (offset=${offset}, length=${length})`,
+      );
+    }
+    const slice = this.blob.slice(start, end);
+    const buf = await slice.arrayBuffer();
+    if (buf.byteLength !== Number(length)) {
+      throw new Error(
+        `BlobReadSource.read: short read at offset ${offset}, expected ${length}, got ${buf.byteLength}`,
+      );
+    }
+    return new Uint8Array(buf);
+  }
+}

package/src/sources/http.ts

@@ -0,0 +1,121 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// HttpRangeReadSource: ReadSource backed by HTTP Range requests.
+//
+// The browser's fetch caps concurrent connections per origin (~6 on HTTP/1.1,
+// effectively unlimited mux on HTTP/2). The Reader's cost-aware path bounds
+// concurrency to `maxConcurrency`; HTTP-level limits are an additional cap.
+//
+// `size()` is resolved lazily via a HEAD (or GET Range: bytes=0-0). We prefer
+// HEAD because most static-file servers respond cheaply, but fall back to a
+// 1-byte GET if the server forbids HEAD.
+
+import type { ReadSource } from "../read_source.js";
+
+export interface HttpRangeReadSourceOptions {
+  /** Custom fetch implementation (e.g. for tests or auth headers). */
+  fetch?: typeof fetch;
+  /** Extra request init applied to every request (e.g. credentials, headers). */
+  init?: RequestInit;
+}
+
+export class HttpRangeReadSource implements ReadSource {
+  private readonly url: string;
+  private readonly fetchImpl: typeof fetch;
+  private readonly init: RequestInit;
+  private cachedSize: bigint | undefined;
+
+  constructor(url: string, opts: HttpRangeReadSourceOptions = {}) {
+    this.url = url;
+    this.fetchImpl = opts.fetch ?? fetch.bind(globalThis);
+    this.init = opts.init ?? {};
+  }
+
+  async size(): Promise<bigint> {
+    if (this.cachedSize !== undefined) {
+      return this.cachedSize;
+    }
+    // Try HEAD first.
+    let resp: Response;
+    try {
+      resp = await this.fetchImpl(this.url, { ...this.init, method: "HEAD" });
+    } catch (err) {
+      throw new Error(
+        `HttpRangeReadSource: HEAD failed for ${this.url}: ${(err as Error).message}`,
+      );
+    }
+    if (resp.ok) {
+      const cl = resp.headers.get("content-length");
+      if (cl !== null) {
+        const v = BigInt(cl);
+        this.cachedSize = v;
+        return v;
+      }
+    }
+    // Fallback: a 1-byte ranged GET, parse Content-Range total.
+    const probe = await this.fetchImpl(this.url, {
+      ...this.init,
+      method: "GET",
+      headers: { ...(this.init.headers ?? {}), Range: "bytes=0-0" },
+    });
+    if (!probe.ok && probe.status !== 206) {
+      throw new Error(
+        `HttpRangeReadSource: probe GET failed: ${probe.status} ${probe.statusText}`,
+      );
+    }
+    const cr = probe.headers.get("content-range");
+    if (cr === null) {
+      throw new Error(
+        `HttpRangeReadSource: server returned no Content-Length or Content-Range`,
+      );
+    }
+    // Format: "bytes 0-0/12345"
+    const m = /\/(\d+)$/.exec(cr);
+    if (m === null) {
+      throw new Error(`HttpRangeReadSource: malformed Content-Range: ${cr}`);
+    }
+    const v = BigInt(m[1]!);
+    this.cachedSize = v;
+    // Drain the probe body so the connection can be reused.
+    await probe.arrayBuffer();
+    return v;
+  }
+
+  async read(offset: bigint, length: bigint): Promise<Uint8Array> {
+    if (length <= 0n) {
+      return new Uint8Array(0);
+    }
+    const end = offset + length - 1n;
+    const headers = new Headers(this.init.headers);
+    headers.set("Range", `bytes=${offset.toString()}-${end.toString()}`);
+    const resp = await this.fetchImpl(this.url, {
+      ...this.init,
+      method: "GET",
+      headers,
+    });
+    if (!resp.ok && resp.status !== 206) {
+      throw new Error(
+        `HttpRangeReadSource: GET range ${offset}-${end} failed: ${resp.status} ${resp.statusText}`,
+      );
+    }
+    const buf = new Uint8Array(await resp.arrayBuffer());
+    if (BigInt(buf.byteLength) !== length) {
+      throw new Error(
+        `HttpRangeReadSource: short read at ${offset}, expected ${length}, got ${buf.byteLength}`,
+      );
+    }
+    return buf;
+  }
+}

package/src/types.ts

@@ -0,0 +1,110 @@
+// Copyright 2026 Wanjia He
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Domain types for turbodata. Mirrors go/types.go.
+//
+// Numeric type convention (matches @mcap/core):
+//   - Go int64 -> bigint
+//   - Go uint8 / uint16 / uint32 -> number
+
+export interface Footer {
+  /** uncompressed: int64. Length of the (zstd-compressed) summary block. */
+  summaryLen: bigint;
+  /** Always equals "7URB0" for valid files. */
+  magic: Uint8Array;
+}
+
+export interface Summary {
+  topicsInfos: TopicsInfo[];
+}
+
+export interface TopicsInfo {
+  topicMetadatas: TopicMetadata[];
+  indexChunkInfoList: IndexChunkInfo[];
+  /**
+   * int64. Total length of all index chunks for this topic group, in bytes
+   * (compressed). Used by the reader to compute the length of the *last*
+   * index chunk in the group.
+   */
+  totalLen: bigint;
+}
+
+export interface TopicMetadata {
+  /** uint16 */
+  id: number;
+  name: string;
+  /**
+   * Decoded msgpack map. Int64 values are decoded as bigint (via
+   * `useBigInt64: true`); smaller integers stay as number.
+   */
+  metadata: Map<string, unknown>;
+  /** uint32. Number of messages written into this topic. */
+  messageCount: number;
+}
+
+export interface IndexChunkInfo {
+  /** int64 */
+  startTimestamp: bigint;
+  /** int64 */
+  endTimestamp: bigint;
+  /** int64. Absolute file offset where the (compressed) index chunk starts. */
+  offset: bigint;
+}
+
+export interface IndexChunk {
+  topicIndexes: TopicIndex[];
+  /** int64. Absolute file offset of this index chunk's data. */
+  chunkOffset: bigint;
+  /** int64. On-disk (possibly compressed) data length in bytes. */
+  chunkLen: bigint;
+  /** int64. Decompressed data length, used to compute the last message's length. */
+  uncompressedLen: bigint;
+}
+
+export interface TopicIndex {
+  /** uint16 */
+  id: number;
+  messageIndexes: MessageIndex[];
+  /**
+   * uint32 entries: ascending positions into `messageIndexes` marking key
+   * frames. Consumed by the video-decodable read/sample paths to anchor GOPs.
+   */
+  keyFrameIndexes: Uint32Array;
+}
+
+export interface MessageIndex {
+  /** int64 */
+  timestamp: bigint;
+  /** int64. Offset relative to the start of the (uncompressed) data chunk. */
+  offsetInChunk: bigint;
+}
+
+/** The 5 magic bytes that close every turbodata file: "7URB0". */
+export const MAGIC: Uint8Array = new Uint8Array([0x37, 0x55, 0x52, 0x42, 0x30]);
+
+/** Footer is fixed at 13 bytes: 8 (summaryLen) + 5 (magic). */
+export const FOOTER_LEN = 13;
+
+// Common per-topic schema metadata keys. SCHEMA_DATA holds raw schema bytes.
+export const META_KEY_SCHEMA_NAME = "schema_name";
+export const META_KEY_SCHEMA_ENCODING = "schema_encoding";
+export const META_KEY_SCHEMA_DATA = "schema_data";
+
+// Internal per-topic metadata keys used to carry write-format control flags
+// alongside caller-supplied metadata. The __td_ prefix namespaces them away
+// from user keys. The reader reads these back to drive decompression and
+// video-aware sampling.
+export const META_KEY_COMPRESSED = "__td_is_compressed";
+export const META_KEY_VIDEO = "__td_is_video";
+export const META_KEY_CHUNK_CONFIG = "__td_chunk_config";