turbodata 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +27 -0
- package/README.md +215 -0
- package/dist/compression.d.ts +8 -0
- package/dist/compression.d.ts.map +1 -0
- package/dist/compression.js +19 -0
- package/dist/compression.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -0
- package/dist/io.d.ts +37 -0
- package/dist/io.d.ts.map +1 -0
- package/dist/io.js +174 -0
- package/dist/io.js.map +1 -0
- package/dist/iterators/message_heap.d.ts +17 -0
- package/dist/iterators/message_heap.d.ts.map +1 -0
- package/dist/iterators/message_heap.js +83 -0
- package/dist/iterators/message_heap.js.map +1 -0
- package/dist/iterators/preloaded_topics_group_iterator.d.ts +38 -0
- package/dist/iterators/preloaded_topics_group_iterator.d.ts.map +1 -0
- package/dist/iterators/preloaded_topics_group_iterator.js +109 -0
- package/dist/iterators/preloaded_topics_group_iterator.js.map +1 -0
- package/dist/iterators/topics_group_iterator.d.ts +45 -0
- package/dist/iterators/topics_group_iterator.d.ts.map +1 -0
- package/dist/iterators/topics_group_iterator.js +132 -0
- package/dist/iterators/topics_group_iterator.js.map +1 -0
- package/dist/loaded_bytes.d.ts +12 -0
- package/dist/loaded_bytes.d.ts.map +1 -0
- package/dist/loaded_bytes.js +43 -0
- package/dist/loaded_bytes.js.map +1 -0
- package/dist/msgpack.d.ts +10 -0
- package/dist/msgpack.d.ts.map +1 -0
- package/dist/msgpack.js +44 -0
- package/dist/msgpack.js.map +1 -0
- package/dist/multi_reader.d.ts +56 -0
- package/dist/multi_reader.d.ts.map +1 -0
- package/dist/multi_reader.js +281 -0
- package/dist/multi_reader.js.map +1 -0
- package/dist/read_fetcher.d.ts +9 -0
- package/dist/read_fetcher.d.ts.map +1 -0
- package/dist/read_fetcher.js +59 -0
- package/dist/read_fetcher.js.map +1 -0
- package/dist/read_options.d.ts +40 -0
- package/dist/read_options.d.ts.map +1 -0
- package/dist/read_options.js +16 -0
- package/dist/read_options.js.map +1 -0
- package/dist/read_planner.d.ts +37 -0
- package/dist/read_planner.d.ts.map +1 -0
- package/dist/read_planner.js +110 -0
- package/dist/read_planner.js.map +1 -0
- package/dist/read_source.d.ts +19 -0
- package/dist/read_source.d.ts.map +1 -0
- package/dist/read_source.js +15 -0
- package/dist/read_source.js.map +1 -0
- package/dist/read_strategy.d.ts +52 -0
- package/dist/read_strategy.d.ts.map +1 -0
- package/dist/read_strategy.js +70 -0
- package/dist/read_strategy.js.map +1 -0
- package/dist/reader.d.ts +209 -0
- package/dist/reader.d.ts.map +1 -0
- package/dist/reader.js +649 -0
- package/dist/reader.js.map +1 -0
- package/dist/sample.d.ts +41 -0
- package/dist/sample.d.ts.map +1 -0
- package/dist/sample.js +528 -0
- package/dist/sample.js.map +1 -0
- package/dist/sort_and_filter.d.ts +25 -0
- package/dist/sort_and_filter.d.ts.map +1 -0
- package/dist/sort_and_filter.js +126 -0
- package/dist/sort_and_filter.js.map +1 -0
- package/dist/sources/blob.d.ts +8 -0
- package/dist/sources/blob.d.ts.map +1 -0
- package/dist/sources/blob.js +38 -0
- package/dist/sources/blob.js.map +1 -0
- package/dist/sources/http.d.ts +17 -0
- package/dist/sources/http.d.ts.map +1 -0
- package/dist/sources/http.js +90 -0
- package/dist/sources/http.js.map +1 -0
- package/dist/types.d.ts +75 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +29 -0
- package/dist/types.js.map +1 -0
- package/package.json +61 -0
- package/src/compression.ts +28 -0
- package/src/index.ts +70 -0
- package/src/io.ts +214 -0
- package/src/iterators/message_heap.ts +104 -0
- package/src/iterators/preloaded_topics_group_iterator.ts +143 -0
- package/src/iterators/topics_group_iterator.ts +180 -0
- package/src/loaded_bytes.ts +61 -0
- package/src/msgpack.ts +46 -0
- package/src/multi_reader.ts +337 -0
- package/src/read_fetcher.ts +74 -0
- package/src/read_options.ts +58 -0
- package/src/read_planner.ts +153 -0
- package/src/read_source.ts +33 -0
- package/src/read_strategy.ts +111 -0
- package/src/reader.ts +910 -0
- package/src/sample.ts +715 -0
- package/src/sort_and_filter.ts +167 -0
- package/src/sources/blob.ts +52 -0
- package/src/sources/http.ts +121 -0
- package/src/types.ts +110 -0
package/src/sample.ts
ADDED
|
@@ -0,0 +1,715 @@
|
|
|
1
|
+
// Copyright 2026 Wanjia He
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
|
|
15
|
+
// Sample engine: floor-message lookup at concrete timestamps.
|
|
16
|
+
//
|
|
17
|
+
// Ports go/internal/iter/sample.go (via py/turbodata/_sample.py, the no-video
|
|
18
|
+
// reference). Two concurrent I/O waves:
|
|
19
|
+
//
|
|
20
|
+
// Phase A: fetch the candidate index chunks per (topic, timestamp), decode
|
|
21
|
+
// them, advance each query's forward cursor, and re-queue fallbacks
|
|
22
|
+
// against chunk c-1 when the candidate chunk has no message of the
|
|
23
|
+
// topic with ts <= T. Loop until nothing is pending. Bounded by the
|
|
24
|
+
// chunk count.
|
|
25
|
+
//
|
|
26
|
+
// Phase B: fetch data ranges (chunk-level for compressed groups, per-message
|
|
27
|
+
// for uncompressed groups) in one concurrent wave, decompress, and
|
|
28
|
+
// copy the floor bytes into the result slots.
|
|
29
|
+
//
|
|
30
|
+
// When videoDecodable is set, a video topic instead yields a decoder-ready GOP
|
|
31
|
+
// sequence (frames + resetDecoder) per the SampleResult contract; without it a
|
|
32
|
+
// video topic is sampled like any other topic (its floor frame in `data`).
|
|
33
|
+
|
|
34
|
+
import type { Decompressor } from "./compression.js";
|
|
35
|
+
import { BinaryReader, readIndexChunk } from "./io.js";
|
|
36
|
+
import { LoadedBytes } from "./loaded_bytes.js";
|
|
37
|
+
import { fetchAll } from "./read_fetcher.js";
|
|
38
|
+
import { plan, type Range } from "./read_planner.js";
|
|
39
|
+
import type { ReadSource } from "./read_source.js";
|
|
40
|
+
import type { ReadStrategy } from "./read_strategy.js";
|
|
41
|
+
import {
|
|
42
|
+
META_KEY_COMPRESSED,
|
|
43
|
+
META_KEY_VIDEO,
|
|
44
|
+
type IndexChunk,
|
|
45
|
+
type Summary,
|
|
46
|
+
type TopicIndex,
|
|
47
|
+
} from "./types.js";
|
|
48
|
+
|
|
49
|
+
/** One engine-level query: floor message of `topic` at each `timestamps[j]`. */
|
|
50
|
+
export interface SampleSpec {
|
|
51
|
+
topic: string;
|
|
52
|
+
/** Assumed strictly increasing (the Reader validates this). */
|
|
53
|
+
timestamps: bigint[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* One coded video frame surfaced in a SampleHit's GOP prefix or incremental
|
|
58
|
+
* tail. Structurally identical to the public turbodata Frame.
|
|
59
|
+
*/
|
|
60
|
+
export interface SampleFrame {
|
|
61
|
+
timestamp: bigint;
|
|
62
|
+
isKeyFrame: boolean;
|
|
63
|
+
data: Uint8Array;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* One engine-level result. `found` is false when no message with ts <= the
|
|
68
|
+
* requested timestamp exists for the topic. `data` is an owned copy.
|
|
69
|
+
*
|
|
70
|
+
* For video-decodable video topics, `data` stays empty and the frame bytes
|
|
71
|
+
* live in `frames` (target = last element); `isVideo` is true and
|
|
72
|
+
* `resetDecoder` flags GOP boundaries. See the public SampleResult contract.
|
|
73
|
+
*/
|
|
74
|
+
export interface SampleHit {
|
|
75
|
+
found: boolean;
|
|
76
|
+
timestamp: bigint;
|
|
77
|
+
data: Uint8Array;
|
|
78
|
+
isVideo: boolean;
|
|
79
|
+
frames: SampleFrame[];
|
|
80
|
+
resetDecoder: boolean;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
interface PendingItem {
|
|
84
|
+
/** Index into the spec's `timestamps`. */
|
|
85
|
+
tIdx: number;
|
|
86
|
+
/** Candidate chunk index within the query's group. */
|
|
87
|
+
chunkIdx: number;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
interface ResolvedItem {
|
|
91
|
+
tIdx: number;
|
|
92
|
+
groupIdx: number;
|
|
93
|
+
chunkIdx: number;
|
|
94
|
+
msgOffInChunk: bigint;
|
|
95
|
+
msgLen: bigint;
|
|
96
|
+
timestamp: bigint;
|
|
97
|
+
// Video-only: positions into the topic's messageIndexes in the cached
|
|
98
|
+
// IndexChunk. keyframeMsgIdx is the anchor key frame for the GOP that
|
|
99
|
+
// contains the target; targetMsgIdx is the target's position.
|
|
100
|
+
keyframeMsgIdx: number;
|
|
101
|
+
targetMsgIdx: number;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
interface QueryState {
|
|
105
|
+
spec: SampleSpec;
|
|
106
|
+
groupIdx: number;
|
|
107
|
+
topicId: number;
|
|
108
|
+
isCompressed: boolean;
|
|
109
|
+
isVideo: boolean;
|
|
110
|
+
results: SampleHit[];
|
|
111
|
+
pending: PendingItem[];
|
|
112
|
+
resolved: ResolvedItem[];
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* A decoded index chunk plus per-topic message lengths (computed once when the
|
|
117
|
+
* chunk is first decoded). Messages from different topics within a chunk are
|
|
118
|
+
* interleaved by offset, so per-topic lengths can't be derived from a single
|
|
119
|
+
* topic's MessageIndexes alone.
|
|
120
|
+
*/
|
|
121
|
+
interface ChunkCache {
|
|
122
|
+
ic: IndexChunk;
|
|
123
|
+
perTopicLens: Map<number, bigint[]>;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const EMPTY = new Uint8Array(0);
|
|
127
|
+
|
|
128
|
+
/** Compose the (group, chunk) cache key. */
|
|
129
|
+
function chunkKey(group: number, chunk: number): string {
|
|
130
|
+
return `${group},${chunk}`;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function cmpBigint(a: bigint, b: bigint): number {
|
|
134
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Resolve floor messages for the given specs. Returns hits shaped like the
|
|
139
|
+
* input: out[i][j] corresponds to specs[i].timestamps[j].
|
|
140
|
+
*/
|
|
141
|
+
export async function sampleMessages(
|
|
142
|
+
rs: ReadSource,
|
|
143
|
+
summary: Summary,
|
|
144
|
+
specs: SampleSpec[],
|
|
145
|
+
strategy: ReadStrategy,
|
|
146
|
+
decompress: Decompressor,
|
|
147
|
+
videoDecodable = false,
|
|
148
|
+
): Promise<SampleHit[][]> {
|
|
149
|
+
if (specs.length === 0) {
|
|
150
|
+
return [];
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const states: QueryState[] = specs.map((spec) => {
|
|
154
|
+
const qs: QueryState = {
|
|
155
|
+
spec,
|
|
156
|
+
groupIdx: 0,
|
|
157
|
+
topicId: 0,
|
|
158
|
+
isCompressed: false,
|
|
159
|
+
isVideo: false,
|
|
160
|
+
results: spec.timestamps.map(() => ({
|
|
161
|
+
found: false,
|
|
162
|
+
timestamp: 0n,
|
|
163
|
+
data: EMPTY,
|
|
164
|
+
isVideo: false,
|
|
165
|
+
frames: [],
|
|
166
|
+
resetDecoder: false,
|
|
167
|
+
})),
|
|
168
|
+
pending: [],
|
|
169
|
+
resolved: [],
|
|
170
|
+
};
|
|
171
|
+
resolveTopic(qs, summary, videoDecodable);
|
|
172
|
+
return qs;
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
// Assign initial candidates: per query, walk the strictly-increasing
|
|
176
|
+
// timestamps alongside the group's IndexChunkInfoList with one forward
|
|
177
|
+
// cursor. The candidate is the chunk with the largest StartTimestamp <= T.
|
|
178
|
+
for (const qs of states) {
|
|
179
|
+
const chunks = summary.topicsInfos[qs.groupIdx]!.indexChunkInfoList;
|
|
180
|
+
if (chunks.length === 0) {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
let c = -1;
|
|
184
|
+
for (let t = 0; t < qs.spec.timestamps.length; t++) {
|
|
185
|
+
const T = qs.spec.timestamps[t]!;
|
|
186
|
+
while (c + 1 < chunks.length && chunks[c + 1]!.startTimestamp <= T) {
|
|
187
|
+
c++;
|
|
188
|
+
}
|
|
189
|
+
if (c < 0) {
|
|
190
|
+
continue; // T precedes every chunk; results[t].found stays false.
|
|
191
|
+
}
|
|
192
|
+
qs.pending.push({ tIdx: t, chunkIdx: c });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const cache = new Map<string, ChunkCache>();
|
|
197
|
+
|
|
198
|
+
// ---- Phase A ----
|
|
199
|
+
while (states.some((qs) => qs.pending.length > 0)) {
|
|
200
|
+
const needs = gatherChunkNeeds(summary, states, cache);
|
|
201
|
+
if (needs.length > 0) {
|
|
202
|
+
await fetchAndDecodeIndexChunks(rs, strategy, decompress, needs, cache);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
let anyProgress = false;
|
|
206
|
+
for (const qs of states) {
|
|
207
|
+
if (qs.pending.length === 0) {
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
const still: PendingItem[] = [];
|
|
211
|
+
let i = 0;
|
|
212
|
+
while (i < qs.pending.length) {
|
|
213
|
+
// Group the contiguous run of pending items sharing one candidate chunk.
|
|
214
|
+
let j = i;
|
|
215
|
+
while (
|
|
216
|
+
j < qs.pending.length &&
|
|
217
|
+
qs.pending[j]!.chunkIdx === qs.pending[i]!.chunkIdx
|
|
218
|
+
) {
|
|
219
|
+
j++;
|
|
220
|
+
}
|
|
221
|
+
const c = qs.pending[i]!.chunkIdx;
|
|
222
|
+
const cc = cache.get(chunkKey(qs.groupIdx, c));
|
|
223
|
+
if (cc === undefined) {
|
|
224
|
+
for (let k = i; k < j; k++) {
|
|
225
|
+
still.push(qs.pending[k]!);
|
|
226
|
+
}
|
|
227
|
+
i = j;
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
anyProgress = true;
|
|
231
|
+
const fallback = resolveItemsInChunk(qs, c, cc, qs.pending.slice(i, j));
|
|
232
|
+
for (const f of fallback) {
|
|
233
|
+
still.push(f);
|
|
234
|
+
}
|
|
235
|
+
i = j;
|
|
236
|
+
}
|
|
237
|
+
qs.pending = still;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (!anyProgress && states.some((qs) => qs.pending.length > 0)) {
|
|
241
|
+
// gatherChunkNeeds should always surface a fetchable chunk for any
|
|
242
|
+
// pending item; reaching here would indicate an internal bug.
|
|
243
|
+
throw new Error(
|
|
244
|
+
"sample: phase A made no progress with pending items remaining",
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// ---- Phase B ----
|
|
250
|
+
await runPhaseB(rs, strategy, decompress, states, cache);
|
|
251
|
+
|
|
252
|
+
// Mark every result of a video query (including not-found ones) so callers
|
|
253
|
+
// can branch on isVideo. Mirrors the py/go engine's collect step.
|
|
254
|
+
for (const qs of states) {
|
|
255
|
+
if (qs.isVideo) {
|
|
256
|
+
for (const h of qs.results) {
|
|
257
|
+
h.isVideo = true;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
return states.map((qs) => qs.results);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Locate the topic's group index, topic id, compression flag, and (when video
|
|
267
|
+
* decoding is requested) video flag in the summary. Assumes the caller
|
|
268
|
+
* validated that the topic exists.
|
|
269
|
+
*/
|
|
270
|
+
function resolveTopic(
|
|
271
|
+
qs: QueryState,
|
|
272
|
+
summary: Summary,
|
|
273
|
+
videoDecodable: boolean,
|
|
274
|
+
): void {
|
|
275
|
+
for (let gi = 0; gi < summary.topicsInfos.length; gi++) {
|
|
276
|
+
const ti = summary.topicsInfos[gi]!;
|
|
277
|
+
for (const tm of ti.topicMetadatas) {
|
|
278
|
+
if (tm.name === qs.spec.topic) {
|
|
279
|
+
qs.groupIdx = gi;
|
|
280
|
+
qs.topicId = tm.id;
|
|
281
|
+
qs.isCompressed =
|
|
282
|
+
ti.topicMetadatas[0]!.metadata.get(META_KEY_COMPRESSED) === true;
|
|
283
|
+
// Video decoding is opt-in. Without it, a video topic is sampled like
|
|
284
|
+
// any other topic (its floor frame in `data`).
|
|
285
|
+
qs.isVideo =
|
|
286
|
+
videoDecodable &&
|
|
287
|
+
ti.topicMetadatas[0]!.metadata.get(META_KEY_VIDEO) === true;
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
interface ChunkNeed {
|
|
295
|
+
key: string;
|
|
296
|
+
offset: bigint;
|
|
297
|
+
length: bigint;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Returns the not-yet-cached candidate chunks, sorted by file offset. */
|
|
301
|
+
function gatherChunkNeeds(
|
|
302
|
+
summary: Summary,
|
|
303
|
+
states: QueryState[],
|
|
304
|
+
cache: Map<string, ChunkCache>,
|
|
305
|
+
): ChunkNeed[] {
|
|
306
|
+
const seen = new Set<string>();
|
|
307
|
+
const out: ChunkNeed[] = [];
|
|
308
|
+
for (const qs of states) {
|
|
309
|
+
for (const p of qs.pending) {
|
|
310
|
+
const key = chunkKey(qs.groupIdx, p.chunkIdx);
|
|
311
|
+
if (cache.has(key) || seen.has(key)) {
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
seen.add(key);
|
|
315
|
+
const ti = summary.topicsInfos[qs.groupIdx]!;
|
|
316
|
+
const list = ti.indexChunkInfoList;
|
|
317
|
+
const info = list[p.chunkIdx]!;
|
|
318
|
+
// The on-disk length of index chunk c: the gap to the next chunk's
|
|
319
|
+
// offset, or (for the last chunk) the group's total index length closes
|
|
320
|
+
// it. Mirrors go IndexChunkLen / py _gather_chunk_needs.
|
|
321
|
+
let length: bigint;
|
|
322
|
+
if (p.chunkIdx < list.length - 1) {
|
|
323
|
+
length = list[p.chunkIdx + 1]!.offset - info.offset;
|
|
324
|
+
} else {
|
|
325
|
+
length = ti.totalLen - info.offset + list[0]!.offset;
|
|
326
|
+
}
|
|
327
|
+
out.push({ key, offset: info.offset, length });
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
out.sort((a, b) => cmpBigint(a.offset, b.offset));
|
|
331
|
+
return out;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
async function fetchAndDecodeIndexChunks(
|
|
335
|
+
rs: ReadSource,
|
|
336
|
+
strategy: ReadStrategy,
|
|
337
|
+
decompress: Decompressor,
|
|
338
|
+
needs: ChunkNeed[],
|
|
339
|
+
cache: Map<string, ChunkCache>,
|
|
340
|
+
): Promise<void> {
|
|
341
|
+
const ranges: Range[] = needs.map((n) => ({
|
|
342
|
+
offset: n.offset,
|
|
343
|
+
length: n.length,
|
|
344
|
+
}));
|
|
345
|
+
const { ops, locations } = plan(ranges, strategy);
|
|
346
|
+
const bufs = await fetchAll(rs, ops, strategy.maxConcurrency);
|
|
347
|
+
const loaded = new LoadedBytes(ranges, locations, bufs);
|
|
348
|
+
for (const n of needs) {
|
|
349
|
+
const raw = loaded.get(n.offset);
|
|
350
|
+
if (raw === undefined) {
|
|
351
|
+
throw new Error(`sample: missing index chunk at offset ${n.offset}`);
|
|
352
|
+
}
|
|
353
|
+
const ic = readIndexChunk(new BinaryReader(decompress(raw)));
|
|
354
|
+
cache.set(n.key, buildChunkCache(ic));
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Derive per-message lengths for every topic in a decoded index chunk. Flatten
|
|
360
|
+
* all topics' MessageIndexes, sort by offset (== timestamp order within a
|
|
361
|
+
* chunk), take successive offset deltas (UncompressedLen closing the last),
|
|
362
|
+
* then project the lengths back onto each topic's MessageIndexes by index.
|
|
363
|
+
*/
|
|
364
|
+
function buildChunkCache(ic: IndexChunk): ChunkCache {
|
|
365
|
+
const items: { topicId: number; offset: bigint; idx: number }[] = [];
|
|
366
|
+
for (const ti of ic.topicIndexes) {
|
|
367
|
+
for (let i = 0; i < ti.messageIndexes.length; i++) {
|
|
368
|
+
items.push({
|
|
369
|
+
topicId: ti.id,
|
|
370
|
+
offset: ti.messageIndexes[i]!.offsetInChunk,
|
|
371
|
+
idx: i,
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
items.sort((a, b) => cmpBigint(a.offset, b.offset));
|
|
376
|
+
|
|
377
|
+
const perTopicLens = new Map<number, bigint[]>();
|
|
378
|
+
for (const ti of ic.topicIndexes) {
|
|
379
|
+
perTopicLens.set(ti.id, new Array<bigint>(ti.messageIndexes.length).fill(0n));
|
|
380
|
+
}
|
|
381
|
+
for (let i = 0; i < items.length - 1; i++) {
|
|
382
|
+
const it = items[i]!;
|
|
383
|
+
perTopicLens.get(it.topicId)![it.idx] = items[i + 1]!.offset - it.offset;
|
|
384
|
+
}
|
|
385
|
+
if (items.length > 0) {
|
|
386
|
+
const last = items[items.length - 1]!;
|
|
387
|
+
perTopicLens.get(last.topicId)![last.idx] = ic.uncompressedLen - last.offset;
|
|
388
|
+
}
|
|
389
|
+
return { ic, perTopicLens };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function findTopicIndex(
|
|
393
|
+
ic: IndexChunk,
|
|
394
|
+
topicId: number,
|
|
395
|
+
): TopicIndex | undefined {
|
|
396
|
+
for (const ti of ic.topicIndexes) {
|
|
397
|
+
if (ti.id === topicId) {
|
|
398
|
+
return ti;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return undefined;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Resolve a contiguous run of pending items that all share candidate chunk c.
|
|
406
|
+
* Items are in strictly-increasing T order, so a single forward cursor over
|
|
407
|
+
* the topic's MessageIndexes assigns each its floor. Items the cursor can't
|
|
408
|
+
* satisfy (topic absent, or its first message in this chunk is > T) become
|
|
409
|
+
* fallbacks against c-1; at c == 0 they are marked not-found.
|
|
410
|
+
*/
|
|
411
|
+
function resolveItemsInChunk(
|
|
412
|
+
qs: QueryState,
|
|
413
|
+
c: number,
|
|
414
|
+
cache: ChunkCache,
|
|
415
|
+
items: PendingItem[],
|
|
416
|
+
): PendingItem[] {
|
|
417
|
+
const fallback: PendingItem[] = [];
|
|
418
|
+
const ti = findTopicIndex(cache.ic, qs.topicId);
|
|
419
|
+
if (ti === undefined || ti.messageIndexes.length === 0) {
|
|
420
|
+
for (const p of items) {
|
|
421
|
+
if (c === 0) {
|
|
422
|
+
qs.results[p.tIdx]!.found = false;
|
|
423
|
+
} else {
|
|
424
|
+
fallback.push({ tIdx: p.tIdx, chunkIdx: c - 1 });
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
return fallback;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const mis = ti.messageIndexes;
|
|
431
|
+
const lens = cache.perTopicLens.get(qs.topicId)!;
|
|
432
|
+
// For video, kfPos tracks the position within keyFrameIndexes of the most
|
|
433
|
+
// recent key frame index <= k. Items in this run are in increasing T (hence
|
|
434
|
+
// increasing k) order, so the cursor only moves forward.
|
|
435
|
+
let kfPos = 0;
|
|
436
|
+
let k = 0;
|
|
437
|
+
for (const p of items) {
|
|
438
|
+
const T = qs.spec.timestamps[p.tIdx]!;
|
|
439
|
+
while (k + 1 < mis.length && mis[k + 1]!.timestamp <= T) {
|
|
440
|
+
k++;
|
|
441
|
+
}
|
|
442
|
+
if (mis[k]!.timestamp > T) {
|
|
443
|
+
if (c === 0) {
|
|
444
|
+
qs.results[p.tIdx]!.found = false;
|
|
445
|
+
} else {
|
|
446
|
+
fallback.push({ tIdx: p.tIdx, chunkIdx: c - 1 });
|
|
447
|
+
}
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
const item: ResolvedItem = {
|
|
451
|
+
tIdx: p.tIdx,
|
|
452
|
+
groupIdx: qs.groupIdx,
|
|
453
|
+
chunkIdx: c,
|
|
454
|
+
msgOffInChunk: mis[k]!.offsetInChunk,
|
|
455
|
+
msgLen: lens[k]!,
|
|
456
|
+
timestamp: mis[k]!.timestamp,
|
|
457
|
+
keyframeMsgIdx: 0,
|
|
458
|
+
targetMsgIdx: 0,
|
|
459
|
+
};
|
|
460
|
+
if (qs.isVideo) {
|
|
461
|
+
const kfs = ti.keyFrameIndexes;
|
|
462
|
+
if (kfs.length === 0 || kfs[0]! > k) {
|
|
463
|
+
// No anchor key frame for this target in this chunk: violates the
|
|
464
|
+
// writer's GOP-integrity invariant. Treat as not found rather than
|
|
465
|
+
// emit an undecodable single frame.
|
|
466
|
+
qs.results[p.tIdx]!.found = false;
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
while (kfPos + 1 < kfs.length && kfs[kfPos + 1]! <= k) {
|
|
470
|
+
kfPos++;
|
|
471
|
+
}
|
|
472
|
+
item.keyframeMsgIdx = kfs[kfPos]!;
|
|
473
|
+
item.targetMsgIdx = k;
|
|
474
|
+
}
|
|
475
|
+
qs.resolved.push(item);
|
|
476
|
+
}
|
|
477
|
+
return fallback;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* Issue every resolved item's data read in one concurrent wave and copy the
|
|
482
|
+
* floor bytes into result slots. Compressed groups register one range per
|
|
483
|
+
* unique chunk (decompressed once, then sliced per message). Uncompressed
|
|
484
|
+
* groups register one range per unique message offset; plan() coalesces
|
|
485
|
+
* neighbors per the strategy.
|
|
486
|
+
*/
|
|
487
|
+
async function runPhaseB(
|
|
488
|
+
rs: ReadSource,
|
|
489
|
+
strategy: ReadStrategy,
|
|
490
|
+
decompress: Decompressor,
|
|
491
|
+
states: QueryState[],
|
|
492
|
+
cache: Map<string, ChunkCache>,
|
|
493
|
+
): Promise<void> {
|
|
494
|
+
const seenChunk = new Set<string>();
|
|
495
|
+
// Multiple query timestamps in one spec can floor to the same message;
|
|
496
|
+
// dedup by absolute offset so we don't register (and read) it twice. The
|
|
497
|
+
// offset is globally unique per message, and a given offset always has the
|
|
498
|
+
// same length, so LoadedBytes keying by offset stays consistent.
|
|
499
|
+
const seenMsg = new Set<bigint>();
|
|
500
|
+
// Video: dedupe per (group, chunk, keyframe). All queries that share a GOP
|
|
501
|
+
// register a single range from the keyframe to the furthest target across
|
|
502
|
+
// those queries; shorter-target queries slice less of the same loaded bytes.
|
|
503
|
+
// Maps the (group, chunk, keyframe) key -> [startInChunk, endInChunkExcl].
|
|
504
|
+
const videoExtents = new Map<
|
|
505
|
+
string,
|
|
506
|
+
{ groupIdx: number; chunkIdx: number; start: bigint; endExcl: bigint }
|
|
507
|
+
>();
|
|
508
|
+
|
|
509
|
+
const ranges: Range[] = [];
|
|
510
|
+
for (const qs of states) {
|
|
511
|
+
for (const r of qs.resolved) {
|
|
512
|
+
const cc = cache.get(chunkKey(r.groupIdx, r.chunkIdx))!;
|
|
513
|
+
const ic = cc.ic;
|
|
514
|
+
if (qs.isCompressed) {
|
|
515
|
+
const k = chunkKey(r.groupIdx, r.chunkIdx);
|
|
516
|
+
if (seenChunk.has(k)) {
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
seenChunk.add(k);
|
|
520
|
+
ranges.push({ offset: ic.chunkOffset, length: ic.chunkLen });
|
|
521
|
+
} else if (qs.isVideo) {
|
|
522
|
+
// Video groups are single-topic by writer invariant.
|
|
523
|
+
const ti = ic.topicIndexes[0]!;
|
|
524
|
+
const mis = ti.messageIndexes;
|
|
525
|
+
const lens = cc.perTopicLens.get(qs.topicId)!;
|
|
526
|
+
const start = mis[r.keyframeMsgIdx]!.offsetInChunk;
|
|
527
|
+
const endExcl = mis[r.targetMsgIdx]!.offsetInChunk + lens[r.targetMsgIdx]!;
|
|
528
|
+
const vk = `${r.groupIdx},${r.chunkIdx},${r.keyframeMsgIdx}`;
|
|
529
|
+
const ve = videoExtents.get(vk);
|
|
530
|
+
if (ve !== undefined) {
|
|
531
|
+
if (endExcl > ve.endExcl) {
|
|
532
|
+
ve.endExcl = endExcl;
|
|
533
|
+
}
|
|
534
|
+
} else {
|
|
535
|
+
videoExtents.set(vk, {
|
|
536
|
+
groupIdx: r.groupIdx,
|
|
537
|
+
chunkIdx: r.chunkIdx,
|
|
538
|
+
start,
|
|
539
|
+
endExcl,
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
} else {
|
|
543
|
+
const off = ic.chunkOffset + r.msgOffInChunk;
|
|
544
|
+
if (seenMsg.has(off)) {
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
seenMsg.add(off);
|
|
548
|
+
ranges.push({ offset: off, length: r.msgLen });
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
for (const ve of videoExtents.values()) {
|
|
553
|
+
const ic = cache.get(chunkKey(ve.groupIdx, ve.chunkIdx))!.ic;
|
|
554
|
+
ranges.push({
|
|
555
|
+
offset: ic.chunkOffset + ve.start,
|
|
556
|
+
length: ve.endExcl - ve.start,
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
if (ranges.length === 0) {
|
|
560
|
+
return;
|
|
561
|
+
}
|
|
562
|
+
ranges.sort((a, b) => cmpBigint(a.offset, b.offset));
|
|
563
|
+
|
|
564
|
+
const { ops, locations } = plan(ranges, strategy);
|
|
565
|
+
const bufs = await fetchAll(rs, ops, strategy.maxConcurrency);
|
|
566
|
+
const loaded = new LoadedBytes(ranges, locations, bufs);
|
|
567
|
+
|
|
568
|
+
// Compressed: group resolved items by chunk so each chunk decompresses once.
|
|
569
|
+
const byChunk = new Map<string, { qs: QueryState; r: ResolvedItem }[]>();
|
|
570
|
+
for (const qs of states) {
|
|
571
|
+
if (!qs.isCompressed) {
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
for (const r of qs.resolved) {
|
|
575
|
+
const k = chunkKey(r.groupIdx, r.chunkIdx);
|
|
576
|
+
let arr = byChunk.get(k);
|
|
577
|
+
if (arr === undefined) {
|
|
578
|
+
arr = [];
|
|
579
|
+
byChunk.set(k, arr);
|
|
580
|
+
}
|
|
581
|
+
arr.push({ qs, r });
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
for (const [k, group] of byChunk) {
|
|
585
|
+
const ic = cache.get(k)!.ic;
|
|
586
|
+
const compressed = loaded.get(ic.chunkOffset)!;
|
|
587
|
+
const decompressed = decompress(compressed);
|
|
588
|
+
for (const { qs, r } of group) {
|
|
589
|
+
const start = Number(r.msgOffInChunk);
|
|
590
|
+
const end = start + Number(r.msgLen);
|
|
591
|
+
// .slice copies, so the result owns its bytes.
|
|
592
|
+
qs.results[r.tIdx] = {
|
|
593
|
+
found: true,
|
|
594
|
+
timestamp: r.timestamp,
|
|
595
|
+
data: decompressed.slice(start, end),
|
|
596
|
+
isVideo: false,
|
|
597
|
+
frames: [],
|
|
598
|
+
resetDecoder: false,
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// Uncompressed non-video: per-message copy.
|
|
604
|
+
for (const qs of states) {
|
|
605
|
+
if (qs.isCompressed || qs.isVideo) {
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
608
|
+
for (const r of qs.resolved) {
|
|
609
|
+
const ic = cache.get(chunkKey(r.groupIdx, r.chunkIdx))!.ic;
|
|
610
|
+
const raw = loaded.get(ic.chunkOffset + r.msgOffInChunk)!;
|
|
611
|
+
qs.results[r.tIdx] = {
|
|
612
|
+
found: true,
|
|
613
|
+
timestamp: r.timestamp,
|
|
614
|
+
data: new Uint8Array(raw), // copy: detach from the shared op buffer.
|
|
615
|
+
isVideo: false,
|
|
616
|
+
frames: [],
|
|
617
|
+
resetDecoder: false,
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
// Video: incremental frames + resetDecoder.
|
|
623
|
+
for (const qs of states) {
|
|
624
|
+
if (!qs.isVideo) {
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
materializeVideo(qs, cache, loaded);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Walk one query's resolved items in target-timestamp order and emit
|
|
633
|
+
* incremental frames + resetDecoder per the public SampleResult contract.
|
|
634
|
+
* Mirrors py _materialize_video.
|
|
635
|
+
*/
|
|
636
|
+
function materializeVideo(
|
|
637
|
+
qs: QueryState,
|
|
638
|
+
cache: Map<string, ChunkCache>,
|
|
639
|
+
loaded: LoadedBytes,
|
|
640
|
+
): void {
|
|
641
|
+
if (qs.resolved.length === 0) {
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
644
|
+
// Resolved items can arrive out of tIdx order if Phase A required fallbacks;
|
|
645
|
+
// sort so the row's processing order matches the caller's.
|
|
646
|
+
const resolved = [...qs.resolved].sort((a, b) => a.tIdx - b.tIdx);
|
|
647
|
+
|
|
648
|
+
// Decoder-state continuity is per-row, scoped to a single GOP (= same chunk
|
|
649
|
+
// + same keyframeMsgIdx).
|
|
650
|
+
let havePrev = false;
|
|
651
|
+
let prevGroupIdx = 0;
|
|
652
|
+
let prevChunkIdx = 0;
|
|
653
|
+
let prevKeyframeIdx = 0;
|
|
654
|
+
let prevTargetMsgIdx = 0;
|
|
655
|
+
|
|
656
|
+
for (const r of resolved) {
|
|
657
|
+
const cc = cache.get(chunkKey(r.groupIdx, r.chunkIdx))!;
|
|
658
|
+
const ti = cc.ic.topicIndexes[0]!;
|
|
659
|
+
const mis = ti.messageIndexes;
|
|
660
|
+
const lens = cc.perTopicLens.get(qs.topicId)!;
|
|
661
|
+
|
|
662
|
+
const sameGop =
|
|
663
|
+
havePrev &&
|
|
664
|
+
prevGroupIdx === r.groupIdx &&
|
|
665
|
+
prevChunkIdx === r.chunkIdx &&
|
|
666
|
+
prevKeyframeIdx === r.keyframeMsgIdx;
|
|
667
|
+
|
|
668
|
+
let startIdx: number;
|
|
669
|
+
let reset: boolean;
|
|
670
|
+
if (!sameGop) {
|
|
671
|
+
startIdx = r.keyframeMsgIdx;
|
|
672
|
+
reset = true;
|
|
673
|
+
} else {
|
|
674
|
+
startIdx = prevTargetMsgIdx + 1;
|
|
675
|
+
reset = false;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
// The Phase B range starts at the GOP's keyframe in this chunk and is long
|
|
679
|
+
// enough to cover the furthest target across queries in this GOP.
|
|
680
|
+
const raw = loaded.get(
|
|
681
|
+
cc.ic.chunkOffset + mis[r.keyframeMsgIdx]!.offsetInChunk,
|
|
682
|
+
)!;
|
|
683
|
+
const baseInChunk = mis[r.keyframeMsgIdx]!.offsetInChunk;
|
|
684
|
+
|
|
685
|
+
const frames: SampleFrame[] = [];
|
|
686
|
+
if (startIdx <= r.targetMsgIdx) {
|
|
687
|
+
for (let i = startIdx; i <= r.targetMsgIdx; i++) {
|
|
688
|
+
const frameStart = Number(mis[i]!.offsetInChunk - baseInChunk);
|
|
689
|
+
const frameLen = Number(lens[i]!);
|
|
690
|
+
frames.push({
|
|
691
|
+
timestamp: mis[i]!.timestamp,
|
|
692
|
+
// keyframeMsgIdx is the greatest key frame index <= targetMsgIdx, so
|
|
693
|
+
// it is the only key frame in range.
|
|
694
|
+
isKeyFrame: i === r.keyframeMsgIdx,
|
|
695
|
+
data: raw.slice(frameStart, frameStart + frameLen),
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
// data stays empty for video; target is the last frame.
|
|
699
|
+
qs.results[r.tIdx]!.timestamp = frames[frames.length - 1]!.timestamp;
|
|
700
|
+
} else {
|
|
701
|
+
// Same target as the previous in-row result: nothing new to feed.
|
|
702
|
+
qs.results[r.tIdx]!.timestamp = mis[r.targetMsgIdx]!.timestamp;
|
|
703
|
+
}
|
|
704
|
+
qs.results[r.tIdx]!.found = true;
|
|
705
|
+
qs.results[r.tIdx]!.data = EMPTY;
|
|
706
|
+
qs.results[r.tIdx]!.frames = frames;
|
|
707
|
+
qs.results[r.tIdx]!.resetDecoder = reset;
|
|
708
|
+
|
|
709
|
+
havePrev = true;
|
|
710
|
+
prevGroupIdx = r.groupIdx;
|
|
711
|
+
prevChunkIdx = r.chunkIdx;
|
|
712
|
+
prevKeyframeIdx = r.keyframeMsgIdx;
|
|
713
|
+
prevTargetMsgIdx = r.targetMsgIdx;
|
|
714
|
+
}
|
|
715
|
+
}
|