turbodata 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +27 -0
- package/README.md +215 -0
- package/dist/compression.d.ts +8 -0
- package/dist/compression.d.ts.map +1 -0
- package/dist/compression.js +19 -0
- package/dist/compression.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -0
- package/dist/io.d.ts +37 -0
- package/dist/io.d.ts.map +1 -0
- package/dist/io.js +174 -0
- package/dist/io.js.map +1 -0
- package/dist/iterators/message_heap.d.ts +17 -0
- package/dist/iterators/message_heap.d.ts.map +1 -0
- package/dist/iterators/message_heap.js +83 -0
- package/dist/iterators/message_heap.js.map +1 -0
- package/dist/iterators/preloaded_topics_group_iterator.d.ts +38 -0
- package/dist/iterators/preloaded_topics_group_iterator.d.ts.map +1 -0
- package/dist/iterators/preloaded_topics_group_iterator.js +109 -0
- package/dist/iterators/preloaded_topics_group_iterator.js.map +1 -0
- package/dist/iterators/topics_group_iterator.d.ts +45 -0
- package/dist/iterators/topics_group_iterator.d.ts.map +1 -0
- package/dist/iterators/topics_group_iterator.js +132 -0
- package/dist/iterators/topics_group_iterator.js.map +1 -0
- package/dist/loaded_bytes.d.ts +12 -0
- package/dist/loaded_bytes.d.ts.map +1 -0
- package/dist/loaded_bytes.js +43 -0
- package/dist/loaded_bytes.js.map +1 -0
- package/dist/msgpack.d.ts +10 -0
- package/dist/msgpack.d.ts.map +1 -0
- package/dist/msgpack.js +44 -0
- package/dist/msgpack.js.map +1 -0
- package/dist/multi_reader.d.ts +56 -0
- package/dist/multi_reader.d.ts.map +1 -0
- package/dist/multi_reader.js +281 -0
- package/dist/multi_reader.js.map +1 -0
- package/dist/read_fetcher.d.ts +9 -0
- package/dist/read_fetcher.d.ts.map +1 -0
- package/dist/read_fetcher.js +59 -0
- package/dist/read_fetcher.js.map +1 -0
- package/dist/read_options.d.ts +40 -0
- package/dist/read_options.d.ts.map +1 -0
- package/dist/read_options.js +16 -0
- package/dist/read_options.js.map +1 -0
- package/dist/read_planner.d.ts +37 -0
- package/dist/read_planner.d.ts.map +1 -0
- package/dist/read_planner.js +110 -0
- package/dist/read_planner.js.map +1 -0
- package/dist/read_source.d.ts +19 -0
- package/dist/read_source.d.ts.map +1 -0
- package/dist/read_source.js +15 -0
- package/dist/read_source.js.map +1 -0
- package/dist/read_strategy.d.ts +52 -0
- package/dist/read_strategy.d.ts.map +1 -0
- package/dist/read_strategy.js +70 -0
- package/dist/read_strategy.js.map +1 -0
- package/dist/reader.d.ts +209 -0
- package/dist/reader.d.ts.map +1 -0
- package/dist/reader.js +649 -0
- package/dist/reader.js.map +1 -0
- package/dist/sample.d.ts +41 -0
- package/dist/sample.d.ts.map +1 -0
- package/dist/sample.js +528 -0
- package/dist/sample.js.map +1 -0
- package/dist/sort_and_filter.d.ts +25 -0
- package/dist/sort_and_filter.d.ts.map +1 -0
- package/dist/sort_and_filter.js +126 -0
- package/dist/sort_and_filter.js.map +1 -0
- package/dist/sources/blob.d.ts +8 -0
- package/dist/sources/blob.d.ts.map +1 -0
- package/dist/sources/blob.js +38 -0
- package/dist/sources/blob.js.map +1 -0
- package/dist/sources/http.d.ts +17 -0
- package/dist/sources/http.d.ts.map +1 -0
- package/dist/sources/http.js +90 -0
- package/dist/sources/http.js.map +1 -0
- package/dist/types.d.ts +75 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +29 -0
- package/dist/types.js.map +1 -0
- package/package.json +61 -0
- package/src/compression.ts +28 -0
- package/src/index.ts +70 -0
- package/src/io.ts +214 -0
- package/src/iterators/message_heap.ts +104 -0
- package/src/iterators/preloaded_topics_group_iterator.ts +143 -0
- package/src/iterators/topics_group_iterator.ts +180 -0
- package/src/loaded_bytes.ts +61 -0
- package/src/msgpack.ts +46 -0
- package/src/multi_reader.ts +337 -0
- package/src/read_fetcher.ts +74 -0
- package/src/read_options.ts +58 -0
- package/src/read_planner.ts +153 -0
- package/src/read_source.ts +33 -0
- package/src/read_strategy.ts +111 -0
- package/src/reader.ts +910 -0
- package/src/sample.ts +715 -0
- package/src/sort_and_filter.ts +167 -0
- package/src/sources/blob.ts +52 -0
- package/src/sources/http.ts +121 -0
- package/src/types.ts +110 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Range, RangeLocation } from "./read_planner.js";
|
|
2
|
+
export declare class LoadedBytes {
|
|
3
|
+
private readonly buffers;
|
|
4
|
+
private readonly index;
|
|
5
|
+
constructor(ranges: Range[], locations: RangeLocation[], buffers: Uint8Array[]);
|
|
6
|
+
/**
|
|
7
|
+
* Return the slice for the range registered at offset. The returned
|
|
8
|
+
* Uint8Array aliases the underlying buffer; callers must not mutate it.
|
|
9
|
+
*/
|
|
10
|
+
get(offset: bigint): Uint8Array | undefined;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=loaded_bytes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loaded_bytes.d.ts","sourceRoot":"","sources":["../src/loaded_bytes.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAQ9D,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;gBAGjD,MAAM,EAAE,KAAK,EAAE,EACf,SAAS,EAAE,aAAa,EAAE,EAC1B,OAAO,EAAE,UAAU,EAAE;IAevB;;;OAGG;IACH,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;CAQ5C"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// Copyright 2026 Wanjia He
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
export class LoadedBytes {
|
|
15
|
+
buffers;
|
|
16
|
+
index;
|
|
17
|
+
constructor(ranges, locations, buffers) {
|
|
18
|
+
this.buffers = buffers;
|
|
19
|
+
this.index = new Map();
|
|
20
|
+
for (let i = 0; i < ranges.length; i++) {
|
|
21
|
+
const r = ranges[i];
|
|
22
|
+
const loc = locations[i];
|
|
23
|
+
this.index.set(r.offset, {
|
|
24
|
+
bufferIdx: loc.opIndex,
|
|
25
|
+
inBufOff: loc.inOpOff,
|
|
26
|
+
length: loc.length,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Return the slice for the range registered at offset. The returned
|
|
32
|
+
* Uint8Array aliases the underlying buffer; callers must not mutate it.
|
|
33
|
+
*/
|
|
34
|
+
get(offset) {
|
|
35
|
+
const loc = this.index.get(offset);
|
|
36
|
+
if (loc === undefined) {
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
const buf = this.buffers[loc.bufferIdx];
|
|
40
|
+
return buf.subarray(loc.inBufOff, loc.inBufOff + loc.length);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=loaded_bytes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loaded_bytes.js","sourceRoot":"","sources":["../src/loaded_bytes.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAcjC,MAAM,OAAO,WAAW;IACL,OAAO,CAAe;IACtB,KAAK,CAA6B;IAEnD,YACE,MAAe,EACf,SAA0B,EAC1B,OAAqB;QAErB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE;gBACvB,SAAS,EAAE,GAAG,CAAC,OAAO;gBACtB,QAAQ,EAAE,GAAG,CAAC,OAAO;gBACrB,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,MAAc;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QACzC,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decode a msgpack-encoded map into a Map<string, unknown>.
|
|
3
|
+
*
|
|
4
|
+
* - `useBigInt64: true` so any int64 values round-trip exactly as bigint.
|
|
5
|
+
* Smaller integers (int32 and below) decode as number.
|
|
6
|
+
* - The Go writer always msgpack-encodes a `map[string]any`. We coerce the
|
|
7
|
+
* decoded plain object into a `Map` for an unambiguous string-keyed view.
|
|
8
|
+
*/
|
|
9
|
+
export declare function decodeMetadata(buf: Uint8Array): Map<string, unknown>;
|
|
10
|
+
//# sourceMappingURL=msgpack.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"msgpack.d.ts","sourceRoot":"","sources":["../src/msgpack.ts"],"names":[],"mappings":"AAmBA;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAkBpE"}
|
package/dist/msgpack.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Copyright 2026 Wanjia He
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
// msgpack decoder for TopicMetadata.Metadata. Uses @msgpack/msgpack with
|
|
15
|
+
// useBigInt64 so int64 values round-trip exactly.
|
|
16
|
+
import { decode } from "@msgpack/msgpack";
|
|
17
|
+
/**
|
|
18
|
+
* Decode a msgpack-encoded map into a Map<string, unknown>.
|
|
19
|
+
*
|
|
20
|
+
* - `useBigInt64: true` so any int64 values round-trip exactly as bigint.
|
|
21
|
+
* Smaller integers (int32 and below) decode as number.
|
|
22
|
+
* - The Go writer always msgpack-encodes a `map[string]any`. We coerce the
|
|
23
|
+
* decoded plain object into a `Map` for an unambiguous string-keyed view.
|
|
24
|
+
*/
|
|
25
|
+
export function decodeMetadata(buf) {
|
|
26
|
+
const decoded = decode(buf, { useBigInt64: true });
|
|
27
|
+
const out = new Map();
|
|
28
|
+
if (decoded === null || typeof decoded !== "object") {
|
|
29
|
+
return out;
|
|
30
|
+
}
|
|
31
|
+
if (decoded instanceof Map) {
|
|
32
|
+
for (const [k, v] of decoded) {
|
|
33
|
+
if (typeof k === "string") {
|
|
34
|
+
out.set(k, v);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return out;
|
|
38
|
+
}
|
|
39
|
+
for (const k of Object.keys(decoded)) {
|
|
40
|
+
out.set(k, decoded[k]);
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=msgpack.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"msgpack.js","sourceRoot":"","sources":["../src/msgpack.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAEjC,yEAAyE;AACzE,kDAAkD;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,GAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAmB,CAAC;IACvC,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,OAAO,YAAY,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAgC,EAAE,CAAC;YACtD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC1B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAkC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAG,OAAmC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { ReadOptions } from "./read_options.js";
|
|
2
|
+
import { Reader, type Message, type SampleOptions, type SampleQuery, type SampleResult } from "./reader.js";
|
|
3
|
+
/**
|
|
4
|
+
* Thrown by MultiReader.sample/readMessages under videoDecodable when a queried
|
|
5
|
+
* video topic is provided by more than one reader whose time ranges overlap.
|
|
6
|
+
* Decodable multi-file video requires each video topic's source files to be
|
|
7
|
+
* time-disjoint; merging overlapping decodable sources would interleave frames
|
|
8
|
+
* from different GOP chains into an undecodable stream. Mirrors go
|
|
9
|
+
* ErrVideoSourcesOverlap / py VideoSourcesOverlapError.
|
|
10
|
+
*/
|
|
11
|
+
export declare class VideoSourcesOverlapError extends Error {
|
|
12
|
+
readonly topic: string;
|
|
13
|
+
readonly first: readonly [bigint, bigint];
|
|
14
|
+
readonly second: readonly [bigint, bigint];
|
|
15
|
+
constructor(topic: string, first: readonly [bigint, bigint], second: readonly [bigint, bigint]);
|
|
16
|
+
}
|
|
17
|
+
export declare class MultiReader {
|
|
18
|
+
private readonly readers;
|
|
19
|
+
/**
|
|
20
|
+
* Group readers into one merged view. At least one reader is required.
|
|
21
|
+
* Per-file behavior (remap, etc.) lives on each underlying Reader.
|
|
22
|
+
*/
|
|
23
|
+
constructor(readers: Reader[]);
|
|
24
|
+
/**
|
|
25
|
+
* Read every reader's messages as one merged, time-ordered stream. Options
|
|
26
|
+
* pass through to each underlying Reader.readMessages unchanged, so order,
|
|
27
|
+
* time bounds, topic filtering (by exposed name), and strategy apply per file
|
|
28
|
+
* before the merge.
|
|
29
|
+
*
|
|
30
|
+
* When videoDecodable is set, any in-scope video topic supplied by more than
|
|
31
|
+
* one reader must have time-disjoint source ranges; overlapping sources throw
|
|
32
|
+
* VideoSourcesOverlapError (merging them would interleave frames from
|
|
33
|
+
* different GOP chains into an undecodable stream). As with the single
|
|
34
|
+
* Reader, all I/O (and this check) is deferred to the first iteration step.
|
|
35
|
+
*/
|
|
36
|
+
readMessages(opts?: ReadOptions): AsyncIterableIterator<Message>;
|
|
37
|
+
/**
|
|
38
|
+
* Resolve floor messages across every reader. For each
|
|
39
|
+
* (queries[i].topic, queries[i].timestamps[j]) pair, out[i][j] is the floor
|
|
40
|
+
* result with the latest found timestamp among all readers that hold the
|
|
41
|
+
* topic (by exposed name) - the true floor over the union. Ties resolve
|
|
42
|
+
* arbitrarily.
|
|
43
|
+
*
|
|
44
|
+
* Preconditions, validated before any data I/O (aggregated into a single
|
|
45
|
+
* SampleValidationError):
|
|
46
|
+
* - each queries[i].timestamps must be strictly increasing
|
|
47
|
+
* - each queries[i].topic must be unique across all i
|
|
48
|
+
* - each queries[i].topic must exist in at least one reader
|
|
49
|
+
*
|
|
50
|
+
* When videoDecodable is set, a video topic supplied by more than one reader
|
|
51
|
+
* must have time-disjoint source ranges; overlapping sources throw
|
|
52
|
+
* VideoSourcesOverlapError.
|
|
53
|
+
*/
|
|
54
|
+
sample(queries: SampleQuery[], opts?: SampleOptions): Promise<SampleResult[][]>;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=multi_reader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi_reader.d.ts","sourceRoot":"","sources":["../src/multi_reader.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,MAAM,EAEN,KAAK,OAAO,EACZ,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,YAAY,EAElB,MAAM,aAAa,CAAC;AAErB;;;;;;;GAOG;AACH,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEzC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;CAUpC;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAW;IAEnC;;;OAGG;gBACS,OAAO,EAAE,MAAM,EAAE;IAO7B;;;;;;;;;;;OAWG;IACH,YAAY,CAAC,IAAI,GAAE,WAAgB,GAAG,qBAAqB,CAAC,OAAO,CAAC;IAwEpE;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CACV,OAAO,EAAE,WAAW,EAAE,EACtB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;CAwD7B"}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
// Copyright 2026 Wanjia He
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
// MultiReader: present several single-file Readers as one time-ordered stream.
|
|
15
|
+
// Mirrors go/multi_reader.go and py/turbodata/multi_reader.py.
|
|
16
|
+
//
|
|
17
|
+
// readMessages merges every reader's stream in timestamp order; sample resolves
|
|
18
|
+
// floor messages across the union of readers. Both operate entirely in each
|
|
19
|
+
// Reader's exposed-name space. Topic-name collisions across files are the
|
|
20
|
+
// caller's responsibility: configure per-Reader topicRemap so that names meant
|
|
21
|
+
// to union share an exposed name and names meant to stay distinct do not.
|
|
22
|
+
import { MessageHeap } from "./iterators/message_heap.js";
|
|
23
|
+
import { SampleValidationError, } from "./reader.js";
|
|
24
|
+
/**
|
|
25
|
+
* Thrown by MultiReader.sample/readMessages under videoDecodable when a queried
|
|
26
|
+
* video topic is provided by more than one reader whose time ranges overlap.
|
|
27
|
+
* Decodable multi-file video requires each video topic's source files to be
|
|
28
|
+
* time-disjoint; merging overlapping decodable sources would interleave frames
|
|
29
|
+
* from different GOP chains into an undecodable stream. Mirrors go
|
|
30
|
+
* ErrVideoSourcesOverlap / py VideoSourcesOverlapError.
|
|
31
|
+
*/
|
|
32
|
+
export class VideoSourcesOverlapError extends Error {
|
|
33
|
+
topic;
|
|
34
|
+
first;
|
|
35
|
+
second;
|
|
36
|
+
constructor(topic, first, second) {
|
|
37
|
+
super(`turbodata: video topic "${topic}" sources [${first[0]},${first[1]}] and [${second[0]},${second[1]}] overlap in time`);
|
|
38
|
+
this.name = "VideoSourcesOverlapError";
|
|
39
|
+
this.topic = topic;
|
|
40
|
+
this.first = first;
|
|
41
|
+
this.second = second;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export class MultiReader {
|
|
45
|
+
readers;
|
|
46
|
+
/**
|
|
47
|
+
* Group readers into one merged view. At least one reader is required.
|
|
48
|
+
* Per-file behavior (remap, etc.) lives on each underlying Reader.
|
|
49
|
+
*/
|
|
50
|
+
constructor(readers) {
|
|
51
|
+
if (readers.length === 0) {
|
|
52
|
+
throw new Error("turbodata: MultiReader requires at least one reader");
|
|
53
|
+
}
|
|
54
|
+
this.readers = readers;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Read every reader's messages as one merged, time-ordered stream. Options
|
|
58
|
+
* pass through to each underlying Reader.readMessages unchanged, so order,
|
|
59
|
+
* time bounds, topic filtering (by exposed name), and strategy apply per file
|
|
60
|
+
* before the merge.
|
|
61
|
+
*
|
|
62
|
+
* When videoDecodable is set, any in-scope video topic supplied by more than
|
|
63
|
+
* one reader must have time-disjoint source ranges; overlapping sources throw
|
|
64
|
+
* VideoSourcesOverlapError (merging them would interleave frames from
|
|
65
|
+
* different GOP chains into an undecodable stream). As with the single
|
|
66
|
+
* Reader, all I/O (and this check) is deferred to the first iteration step.
|
|
67
|
+
*/
|
|
68
|
+
readMessages(opts = {}) {
|
|
69
|
+
const readers = this.readers;
|
|
70
|
+
const reverse = opts.order === "reverse-time";
|
|
71
|
+
const videoDecodable = opts.videoDecodable === true;
|
|
72
|
+
const wantedNames = opts.topicNames;
|
|
73
|
+
let initialized = false;
|
|
74
|
+
let exhausted = false;
|
|
75
|
+
const subs = [];
|
|
76
|
+
const heap = new MessageHeap(reverse);
|
|
77
|
+
const init = async () => {
|
|
78
|
+
for (const r of readers) {
|
|
79
|
+
subs.push(r.readMessages(opts));
|
|
80
|
+
}
|
|
81
|
+
if (videoDecodable) {
|
|
82
|
+
const bounds = await Promise.all(readers.map((r) => r.topicBounds()));
|
|
83
|
+
enforceVideoDisjoint(readScopeTopics(wantedNames, bounds), bounds);
|
|
84
|
+
}
|
|
85
|
+
// Prime the heap with one message from each reader's stream.
|
|
86
|
+
for (let i = 0; i < subs.length; i++) {
|
|
87
|
+
const res = await subs[i].next();
|
|
88
|
+
if (res.done === true) {
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
heap.push(res.value.timestamp, { subIdx: i, msg: res.value });
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
const iter = {
|
|
95
|
+
[Symbol.asyncIterator]() {
|
|
96
|
+
return iter;
|
|
97
|
+
},
|
|
98
|
+
async next() {
|
|
99
|
+
if (exhausted) {
|
|
100
|
+
return { value: undefined, done: true };
|
|
101
|
+
}
|
|
102
|
+
if (!initialized) {
|
|
103
|
+
await init();
|
|
104
|
+
initialized = true;
|
|
105
|
+
}
|
|
106
|
+
const top = heap.pop();
|
|
107
|
+
if (top === undefined) {
|
|
108
|
+
exhausted = true;
|
|
109
|
+
return { value: undefined, done: true };
|
|
110
|
+
}
|
|
111
|
+
const out = top.value.msg;
|
|
112
|
+
const nm = await subs[top.value.subIdx].next();
|
|
113
|
+
if (nm.done !== true) {
|
|
114
|
+
heap.push(nm.value.timestamp, {
|
|
115
|
+
subIdx: top.value.subIdx,
|
|
116
|
+
msg: nm.value,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return { value: out, done: false };
|
|
120
|
+
},
|
|
121
|
+
async return() {
|
|
122
|
+
exhausted = true;
|
|
123
|
+
for (const s of subs) {
|
|
124
|
+
if (s.return !== undefined) {
|
|
125
|
+
await s.return();
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return { value: undefined, done: true };
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
return iter;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Resolve floor messages across every reader. For each
|
|
135
|
+
* (queries[i].topic, queries[i].timestamps[j]) pair, out[i][j] is the floor
|
|
136
|
+
* result with the latest found timestamp among all readers that hold the
|
|
137
|
+
* topic (by exposed name) - the true floor over the union. Ties resolve
|
|
138
|
+
* arbitrarily.
|
|
139
|
+
*
|
|
140
|
+
* Preconditions, validated before any data I/O (aggregated into a single
|
|
141
|
+
* SampleValidationError):
|
|
142
|
+
* - each queries[i].timestamps must be strictly increasing
|
|
143
|
+
* - each queries[i].topic must be unique across all i
|
|
144
|
+
* - each queries[i].topic must exist in at least one reader
|
|
145
|
+
*
|
|
146
|
+
* When videoDecodable is set, a video topic supplied by more than one reader
|
|
147
|
+
* must have time-disjoint source ranges; overlapping sources throw
|
|
148
|
+
* VideoSourcesOverlapError.
|
|
149
|
+
*/
|
|
150
|
+
async sample(queries, opts = {}) {
|
|
151
|
+
const videoDecodable = opts.videoDecodable === true;
|
|
152
|
+
const bounds = await Promise.all(this.readers.map((r) => r.topicBounds()));
|
|
153
|
+
validateMultiSampleQueries(queries, bounds);
|
|
154
|
+
if (videoDecodable) {
|
|
155
|
+
enforceVideoDisjoint(queries.map((q) => q.topic), bounds);
|
|
156
|
+
}
|
|
157
|
+
const out = queries.map((q) => q.timestamps.map(() => ({
|
|
158
|
+
found: false,
|
|
159
|
+
timestamp: 0n,
|
|
160
|
+
data: new Uint8Array(0),
|
|
161
|
+
isVideo: false,
|
|
162
|
+
frames: [],
|
|
163
|
+
resetDecoder: false,
|
|
164
|
+
})));
|
|
165
|
+
// Dispatch to each reader only the queries whose topic it holds (avoids
|
|
166
|
+
// the engine's unknown-topic error), then merge by latest found floor.
|
|
167
|
+
for (let ri = 0; ri < this.readers.length; ri++) {
|
|
168
|
+
const subset = [];
|
|
169
|
+
const origIdx = [];
|
|
170
|
+
for (let qi = 0; qi < queries.length; qi++) {
|
|
171
|
+
if (bounds[ri].has(queries[qi].topic)) {
|
|
172
|
+
subset.push(queries[qi]);
|
|
173
|
+
origIdx.push(qi);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
if (subset.length === 0) {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const res = await this.readers[ri].sample(subset, opts);
|
|
180
|
+
for (let sub = 0; sub < origIdx.length; sub++) {
|
|
181
|
+
const qi = origIdx[sub];
|
|
182
|
+
const row = res[sub];
|
|
183
|
+
for (let j = 0; j < row.length; j++) {
|
|
184
|
+
const cand = row[j];
|
|
185
|
+
if (!cand.found) {
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const cur = out[qi][j];
|
|
189
|
+
if (!cur.found || cand.timestamp > cur.timestamp) {
|
|
190
|
+
out[qi][j] = cand;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return out;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* The exposed topic names a readMessages call covers: the requested
|
|
200
|
+
* topicNames, or every exposed topic across all readers when unset.
|
|
201
|
+
*/
|
|
202
|
+
function readScopeTopics(topicNames, bounds) {
|
|
203
|
+
if (topicNames !== undefined && topicNames.length > 0) {
|
|
204
|
+
return topicNames;
|
|
205
|
+
}
|
|
206
|
+
const seen = new Set();
|
|
207
|
+
const out = [];
|
|
208
|
+
for (const b of bounds) {
|
|
209
|
+
for (const name of b.keys()) {
|
|
210
|
+
if (!seen.has(name)) {
|
|
211
|
+
seen.add(name);
|
|
212
|
+
out.push(name);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Mirror the single-reader checks but treat a topic as known if any reader
|
|
220
|
+
* holds it. All violations are aggregated into one SampleValidationError.
|
|
221
|
+
* Mirrors go validateMultiSampleQueries.
|
|
222
|
+
*/
|
|
223
|
+
function validateMultiSampleQueries(queries, bounds) {
|
|
224
|
+
const seen = new Map();
|
|
225
|
+
const violations = [];
|
|
226
|
+
for (let i = 0; i < queries.length; i++) {
|
|
227
|
+
const q = queries[i];
|
|
228
|
+
const known = bounds.some((b) => b.has(q.topic));
|
|
229
|
+
if (!known) {
|
|
230
|
+
violations.push(`queries[${i}]: unknown topic "${q.topic}" (not in any reader)`);
|
|
231
|
+
}
|
|
232
|
+
const prev = seen.get(q.topic);
|
|
233
|
+
if (prev !== undefined) {
|
|
234
|
+
violations.push(`queries[${i}]: duplicate topic "${q.topic}" already used by queries[${prev}]`);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
seen.set(q.topic, i);
|
|
238
|
+
}
|
|
239
|
+
for (let j = 1; j < q.timestamps.length; j++) {
|
|
240
|
+
if (q.timestamps[j] <= q.timestamps[j - 1]) {
|
|
241
|
+
violations.push(`queries[${i}].timestamps not strictly increasing at position ${j} (${q.timestamps[j]} <= ${q.timestamps[j - 1]})`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (violations.length > 0) {
|
|
246
|
+
throw new SampleValidationError(violations);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Throw VideoSourcesOverlapError when any of the given exposed video topics is
|
|
251
|
+
* provided by readers with overlapping inclusive time ranges. Topics that are
|
|
252
|
+
* not video, or are held by fewer than two readers, are skipped. Mirrors go
|
|
253
|
+
* enforceVideoDisjoint.
|
|
254
|
+
*/
|
|
255
|
+
function enforceVideoDisjoint(topics, bounds) {
|
|
256
|
+
for (const topic of topics) {
|
|
257
|
+
const ranges = [];
|
|
258
|
+
let isVideo = false;
|
|
259
|
+
for (const b of bounds) {
|
|
260
|
+
const bound = b.get(topic);
|
|
261
|
+
if (bound === undefined) {
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (bound.isVideo) {
|
|
265
|
+
isVideo = true;
|
|
266
|
+
}
|
|
267
|
+
ranges.push([bound.minTs, bound.maxTs]);
|
|
268
|
+
}
|
|
269
|
+
if (!isVideo || ranges.length < 2) {
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
for (let a = 0; a < ranges.length; a++) {
|
|
273
|
+
for (let c = a + 1; c < ranges.length; c++) {
|
|
274
|
+
if (ranges[a][0] <= ranges[c][1] && ranges[c][0] <= ranges[a][1]) {
|
|
275
|
+
throw new VideoSourcesOverlapError(topic, ranges[a], ranges[c]);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
//# sourceMappingURL=multi_reader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi_reader.js","sourceRoot":"","sources":["../src/multi_reader.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAEjC,+EAA+E;AAC/E,+DAA+D;AAC/D,EAAE;AACF,gFAAgF;AAChF,4EAA4E;AAC5E,0EAA0E;AAC1E,+EAA+E;AAC/E,0EAA0E;AAE1E,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE1D,OAAO,EAEL,qBAAqB,GAMtB,MAAM,aAAa,CAAC;AAErB;;;;;;;GAOG;AACH,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACxC,KAAK,CAAS;IACd,KAAK,CAA4B;IACjC,MAAM,CAA4B;IAC3C,YACE,KAAa,EACb,KAAgC,EAChC,MAAiC;QAEjC,KAAK,CACH,2BAA2B,KAAK,cAAc,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,mBAAmB,CACtH,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,MAAM,OAAO,WAAW;IACL,OAAO,CAAW;IAEnC;;;OAGG;IACH,YAAY,OAAiB;QAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;OAWG;IACH,YAAY,CAAC,OAAoB,EAAE;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,KAAK,cAAc,CAAC;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEpC,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,IAAI,GAAqC,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,WAAW,CAAmC,OAAO,CAAC,CAAC;QAExE,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;YACrC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBACtE,oBAAoB,CAAC,eAAe,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;YACrE,CAAC;YAED,6DAA6D;YAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBACtB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,IAAI,GAAmC;YAC3C,CAAC,MAAM,CAAC,aAAa,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,KAAK,CAAC,IAAI;gBACR,IAAI,SAAS,EAAE,CAAC;oBACd,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAC1C,CAAC;gBACD,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,MAAM,IAAI,EAAE,CAAC;oBACb,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,IAAI,CAAC;oBACjB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAC1C,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;gBAC1B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE;wBAC5B,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM;wBACxB,GAAG,EAAE,EAAE,CAAC,KAAK;qBACd,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACrC,CAAC;YACD,KAAK,CAAC,MAAM;gBACV,SAAS,GAAG,IAAI,CAAC;gBACjB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;oBACrB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;wBAC3B,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC1C,CAAC;SACF,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,MAAM,CACV,OAAsB,EACtB,OAAsB,EAAE;QAExB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE3E,0BAA0B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE5C,IAAI,cAAc,EAAE,CAAC;YACnB,oBAAoB,CAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAC3B,MAAM,CACP,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9C,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACtB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,EAAE;YACb,IAAI,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC;YACvB,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,EAAE;YACV,YAAY,EAAE,KAAK;SACpB,CAAC,CAAC,CACJ,CAAC;QAEF,wEAAwE;QACxE,uEAAuE;QACvE,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAChD,MAAM,MAAM,GAAkB,EAAE,CAAC;YACjC,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,EAAE,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAE,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE,CAAC,CAAC;oBAC1B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACzD,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;oBACrB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;wBAChB,SAAS;oBACX,CAAC;oBACD,MAAM,GAAG,GAAG,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAE,CAAC;oBACzB,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;wBACjD,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,eAAe,CACtB,UAAgC,EAChC,MAAiC;IAEjC,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,0BAA0B,CACjC,OAAsB,EACtB,MAAiC;IAEjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,UAAU,CAAC,IAAI,CACb,WAAW,CAAC,qBAAqB,CAAC,CAAC,KAAK,uBAAuB,CAChE,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,UAAU,CAAC,IAAI,CACb,WAAW,CAAC,uBAAuB,CAAC,CAAC,KAAK,6BAA6B,IAAI,GAAG,CAC/E,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAE,EAAE,CAAC;gBAC7C,UAAU,CAAC,IAAI,CACb,WAAW,CAAC,oDAAoD,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CACnH,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,MAAgB,EAChB,MAAiC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAuB,EAAE,CAAC;QACtC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAClB,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,SAAS;QACX,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrE,MAAM,IAAI,wBAAwB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ReadOp } from "./read_planner.js";
|
|
2
|
+
import type { ReadSource } from "./read_source.js";
|
|
3
|
+
/**
|
|
4
|
+
* Execute ops against rs with at most maxConcurrency in-flight reads. Returns
|
|
5
|
+
* one Uint8Array per op, in input order. Throws on the first error
|
|
6
|
+
* encountered; in-flight reads may still settle in the background.
|
|
7
|
+
*/
|
|
8
|
+
export declare function fetchAll(rs: ReadSource, ops: ReadOp[], maxConcurrency: number): Promise<Uint8Array[]>;
|
|
9
|
+
//# sourceMappingURL=read_fetcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read_fetcher.d.ts","sourceRoot":"","sources":["../src/read_fetcher.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,MAAM,EAAE,EACb,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,UAAU,EAAE,CAAC,CA4CvB"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Copyright 2026 Wanjia He
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
/**
|
|
15
|
+
* Execute ops against rs with at most maxConcurrency in-flight reads. Returns
|
|
16
|
+
* one Uint8Array per op, in input order. Throws on the first error
|
|
17
|
+
* encountered; in-flight reads may still settle in the background.
|
|
18
|
+
*/
|
|
19
|
+
export async function fetchAll(rs, ops, maxConcurrency) {
|
|
20
|
+
if (ops.length === 0) {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
const workers = maxConcurrency > 0 ? maxConcurrency : 1;
|
|
24
|
+
const out = new Array(ops.length);
|
|
25
|
+
let nextIdx = 0;
|
|
26
|
+
let firstErr = undefined;
|
|
27
|
+
const runOne = async () => {
|
|
28
|
+
while (firstErr === undefined) {
|
|
29
|
+
const i = nextIdx++;
|
|
30
|
+
if (i >= ops.length) {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const op = ops[i];
|
|
34
|
+
try {
|
|
35
|
+
const buf = await rs.read(op.offset, op.length);
|
|
36
|
+
if (BigInt(buf.byteLength) !== op.length) {
|
|
37
|
+
throw new Error(`fetcher: short read for op ${i}: got ${buf.byteLength} want ${op.length}`);
|
|
38
|
+
}
|
|
39
|
+
out[i] = buf;
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
if (firstErr === undefined) {
|
|
43
|
+
firstErr = err;
|
|
44
|
+
}
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
const pool = new Array(Math.min(workers, ops.length));
|
|
50
|
+
for (let w = 0; w < pool.length; w++) {
|
|
51
|
+
pool[w] = runOne();
|
|
52
|
+
}
|
|
53
|
+
await Promise.all(pool);
|
|
54
|
+
if (firstErr !== undefined) {
|
|
55
|
+
throw firstErr;
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=read_fetcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read_fetcher.js","sourceRoot":"","sources":["../src/read_fetcher.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAQjC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAc,EACd,GAAa,EACb,cAAsB;IAEtB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,OAAO,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAa,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAY,SAAS,CAAC;IAElC,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO;YACT,CAAC;YACD,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;gBAChD,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC;oBACzC,MAAM,IAAI,KAAK,CACb,8BAA8B,CAAC,SAAS,GAAG,CAAC,UAAU,SAAS,EAAE,CAAC,MAAM,EAAE,CAC3E,CAAC;gBACJ,CAAC;gBACD,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YACf,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,QAAQ,GAAG,GAAG,CAAC;gBACjB,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,IAAI,GAAoB,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC;IACrB,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAExB,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { ReadStrategy } from "./read_strategy.js";
|
|
2
|
+
export type Order = "time" | "reverse-time";
|
|
3
|
+
export interface ReadOptions {
|
|
4
|
+
/** When set, only messages from these topic names are emitted. */
|
|
5
|
+
topicNames?: string[];
|
|
6
|
+
/** int64. Inclusive lower bound on message timestamp. Default: 0n. */
|
|
7
|
+
startTimestamp?: bigint;
|
|
8
|
+
/** int64. Inclusive upper bound on message timestamp. Default: int64-max. */
|
|
9
|
+
endTimestamp?: bigint;
|
|
10
|
+
/** Iteration order. Default: "time". */
|
|
11
|
+
order?: Order;
|
|
12
|
+
/**
|
|
13
|
+
* When set, switches Reader.readMessages onto the cost-aware path. Without
|
|
14
|
+
* a strategy, the reader uses the default lazy path (one chunk at a time).
|
|
15
|
+
*/
|
|
16
|
+
strategy?: ReadStrategy;
|
|
17
|
+
/**
|
|
18
|
+
* int64. Bytes to read speculatively from the file tail when loading the
|
|
19
|
+
* summary. When the (footer + compressed summary) fits within the prefetch
|
|
20
|
+
* window, summary loading costs one read instead of two. Default: 0n
|
|
21
|
+
* (footer-only read).
|
|
22
|
+
*/
|
|
23
|
+
tailPrefetch?: bigint;
|
|
24
|
+
/**
|
|
25
|
+
* If true, the iterator hands out a freshly-allocated Uint8Array per
|
|
26
|
+
* message (safe to retain). If false (default), `data` aliases an internal
|
|
27
|
+
* reusable buffer that becomes invalid on the next iteration step.
|
|
28
|
+
*/
|
|
29
|
+
copy?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* If true, for any video topic in scope the effective per-group
|
|
32
|
+
* startTimestamp is snapped back to the latest key frame whose timestamp is
|
|
33
|
+
* <= startTimestamp, so the emitted sequence can be fed to a decoder cold.
|
|
34
|
+
* Non-video topics are unaffected. Default: false.
|
|
35
|
+
*/
|
|
36
|
+
videoDecodable?: boolean;
|
|
37
|
+
}
|
|
38
|
+
/** Default value for endTimestamp: matches Go's math.MaxInt64. */
|
|
39
|
+
export declare const MAX_INT64 = 9223372036854775807n;
|
|
40
|
+
//# sourceMappingURL=read_options.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read_options.d.ts","sourceRoot":"","sources":["../src/read_options.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,cAAc,CAAC;AAE5C,MAAM,WAAW,WAAW;IAC1B,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd;;;OAGG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,kEAAkE;AAClE,eAAO,MAAM,SAAS,uBAAsB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Copyright 2026 Wanjia He
|
|
2
|
+
//
|
|
3
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
// you may not use this file except in compliance with the License.
|
|
5
|
+
// You may obtain a copy of the License at
|
|
6
|
+
//
|
|
7
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
//
|
|
9
|
+
// Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
// See the License for the specific language governing permissions and
|
|
13
|
+
// limitations under the License.
|
|
14
|
+
/** Default value for endTimestamp: matches Go's math.MaxInt64. */
|
|
15
|
+
export const MAX_INT64 = 0x7fffffffffffffffn;
|
|
16
|
+
//# sourceMappingURL=read_options.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"read_options.js","sourceRoot":"","sources":["../src/read_options.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AA4CjC,kEAAkE;AAClE,MAAM,CAAC,MAAM,SAAS,GAAG,mBAAmB,CAAC"}
|