turbodata 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/LICENSE +201 -0
  2. package/NOTICE +27 -0
  3. package/README.md +215 -0
  4. package/dist/compression.d.ts +8 -0
  5. package/dist/compression.d.ts.map +1 -0
  6. package/dist/compression.js +19 -0
  7. package/dist/compression.js.map +1 -0
  8. package/dist/index.d.ts +16 -0
  9. package/dist/index.d.ts.map +1 -0
  10. package/dist/index.js +23 -0
  11. package/dist/index.js.map +1 -0
  12. package/dist/io.d.ts +37 -0
  13. package/dist/io.d.ts.map +1 -0
  14. package/dist/io.js +174 -0
  15. package/dist/io.js.map +1 -0
  16. package/dist/iterators/message_heap.d.ts +17 -0
  17. package/dist/iterators/message_heap.d.ts.map +1 -0
  18. package/dist/iterators/message_heap.js +83 -0
  19. package/dist/iterators/message_heap.js.map +1 -0
  20. package/dist/iterators/preloaded_topics_group_iterator.d.ts +38 -0
  21. package/dist/iterators/preloaded_topics_group_iterator.d.ts.map +1 -0
  22. package/dist/iterators/preloaded_topics_group_iterator.js +109 -0
  23. package/dist/iterators/preloaded_topics_group_iterator.js.map +1 -0
  24. package/dist/iterators/topics_group_iterator.d.ts +45 -0
  25. package/dist/iterators/topics_group_iterator.d.ts.map +1 -0
  26. package/dist/iterators/topics_group_iterator.js +132 -0
  27. package/dist/iterators/topics_group_iterator.js.map +1 -0
  28. package/dist/loaded_bytes.d.ts +12 -0
  29. package/dist/loaded_bytes.d.ts.map +1 -0
  30. package/dist/loaded_bytes.js +43 -0
  31. package/dist/loaded_bytes.js.map +1 -0
  32. package/dist/msgpack.d.ts +10 -0
  33. package/dist/msgpack.d.ts.map +1 -0
  34. package/dist/msgpack.js +44 -0
  35. package/dist/msgpack.js.map +1 -0
  36. package/dist/multi_reader.d.ts +56 -0
  37. package/dist/multi_reader.d.ts.map +1 -0
  38. package/dist/multi_reader.js +281 -0
  39. package/dist/multi_reader.js.map +1 -0
  40. package/dist/read_fetcher.d.ts +9 -0
  41. package/dist/read_fetcher.d.ts.map +1 -0
  42. package/dist/read_fetcher.js +59 -0
  43. package/dist/read_fetcher.js.map +1 -0
  44. package/dist/read_options.d.ts +40 -0
  45. package/dist/read_options.d.ts.map +1 -0
  46. package/dist/read_options.js +16 -0
  47. package/dist/read_options.js.map +1 -0
  48. package/dist/read_planner.d.ts +37 -0
  49. package/dist/read_planner.d.ts.map +1 -0
  50. package/dist/read_planner.js +110 -0
  51. package/dist/read_planner.js.map +1 -0
  52. package/dist/read_source.d.ts +19 -0
  53. package/dist/read_source.d.ts.map +1 -0
  54. package/dist/read_source.js +15 -0
  55. package/dist/read_source.js.map +1 -0
  56. package/dist/read_strategy.d.ts +52 -0
  57. package/dist/read_strategy.d.ts.map +1 -0
  58. package/dist/read_strategy.js +70 -0
  59. package/dist/read_strategy.js.map +1 -0
  60. package/dist/reader.d.ts +209 -0
  61. package/dist/reader.d.ts.map +1 -0
  62. package/dist/reader.js +649 -0
  63. package/dist/reader.js.map +1 -0
  64. package/dist/sample.d.ts +41 -0
  65. package/dist/sample.d.ts.map +1 -0
  66. package/dist/sample.js +528 -0
  67. package/dist/sample.js.map +1 -0
  68. package/dist/sort_and_filter.d.ts +25 -0
  69. package/dist/sort_and_filter.d.ts.map +1 -0
  70. package/dist/sort_and_filter.js +126 -0
  71. package/dist/sort_and_filter.js.map +1 -0
  72. package/dist/sources/blob.d.ts +8 -0
  73. package/dist/sources/blob.d.ts.map +1 -0
  74. package/dist/sources/blob.js +38 -0
  75. package/dist/sources/blob.js.map +1 -0
  76. package/dist/sources/http.d.ts +17 -0
  77. package/dist/sources/http.d.ts.map +1 -0
  78. package/dist/sources/http.js +90 -0
  79. package/dist/sources/http.js.map +1 -0
  80. package/dist/types.d.ts +75 -0
  81. package/dist/types.d.ts.map +1 -0
  82. package/dist/types.js +29 -0
  83. package/dist/types.js.map +1 -0
  84. package/package.json +61 -0
  85. package/src/compression.ts +28 -0
  86. package/src/index.ts +70 -0
  87. package/src/io.ts +214 -0
  88. package/src/iterators/message_heap.ts +104 -0
  89. package/src/iterators/preloaded_topics_group_iterator.ts +143 -0
  90. package/src/iterators/topics_group_iterator.ts +180 -0
  91. package/src/loaded_bytes.ts +61 -0
  92. package/src/msgpack.ts +46 -0
  93. package/src/multi_reader.ts +337 -0
  94. package/src/read_fetcher.ts +74 -0
  95. package/src/read_options.ts +58 -0
  96. package/src/read_planner.ts +153 -0
  97. package/src/read_source.ts +33 -0
  98. package/src/read_strategy.ts +111 -0
  99. package/src/reader.ts +910 -0
  100. package/src/sample.ts +715 -0
  101. package/src/sort_and_filter.ts +167 -0
  102. package/src/sources/blob.ts +52 -0
  103. package/src/sources/http.ts +121 -0
  104. package/src/types.ts +110 -0
@@ -0,0 +1,12 @@
1
+ import type { Range, RangeLocation } from "./read_planner.js";
2
+ export declare class LoadedBytes {
3
+ private readonly buffers;
4
+ private readonly index;
5
+ constructor(ranges: Range[], locations: RangeLocation[], buffers: Uint8Array[]);
6
+ /**
7
+ * Return the slice for the range registered at offset. The returned
8
+ * Uint8Array aliases the underlying buffer; callers must not mutate it.
9
+ */
10
+ get(offset: bigint): Uint8Array | undefined;
11
+ }
12
+ //# sourceMappingURL=loaded_bytes.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"loaded_bytes.d.ts","sourceRoot":"","sources":["../src/loaded_bytes.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAQ9D,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;gBAGjD,MAAM,EAAE,KAAK,EAAE,EACf,SAAS,EAAE,aAAa,EAAE,EAC1B,OAAO,EAAE,UAAU,EAAE;IAevB;;;OAGG;IACH,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;CAQ5C"}
@@ -0,0 +1,43 @@
1
+ // Copyright 2026 Wanjia He
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ export class LoadedBytes {
15
+ buffers;
16
+ index;
17
+ constructor(ranges, locations, buffers) {
18
+ this.buffers = buffers;
19
+ this.index = new Map();
20
+ for (let i = 0; i < ranges.length; i++) {
21
+ const r = ranges[i];
22
+ const loc = locations[i];
23
+ this.index.set(r.offset, {
24
+ bufferIdx: loc.opIndex,
25
+ inBufOff: loc.inOpOff,
26
+ length: loc.length,
27
+ });
28
+ }
29
+ }
30
+ /**
31
+ * Return the slice for the range registered at offset. The returned
32
+ * Uint8Array aliases the underlying buffer; callers must not mutate it.
33
+ */
34
+ get(offset) {
35
+ const loc = this.index.get(offset);
36
+ if (loc === undefined) {
37
+ return undefined;
38
+ }
39
+ const buf = this.buffers[loc.bufferIdx];
40
+ return buf.subarray(loc.inBufOff, loc.inBufOff + loc.length);
41
+ }
42
+ }
43
+ //# sourceMappingURL=loaded_bytes.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"loaded_bytes.js","sourceRoot":"","sources":["../src/loaded_bytes.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAcjC,MAAM,OAAO,WAAW;IACL,OAAO,CAAe;IACtB,KAAK,CAA6B;IAEnD,YACE,MAAe,EACf,SAA0B,EAC1B,OAAqB;QAErB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE;gBACvB,SAAS,EAAE,GAAG,CAAC,OAAO;gBACtB,QAAQ,EAAE,GAAG,CAAC,OAAO;gBACrB,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,MAAc;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QACzC,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;CACF"}
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Decode a msgpack-encoded map into a Map<string, unknown>.
3
+ *
4
+ * - `useBigInt64: true` so any int64 values round-trip exactly as bigint.
5
+ * Smaller integers (int32 and below) decode as number.
6
+ * - The Go writer always msgpack-encodes a `map[string]any`. We coerce the
7
+ * decoded plain object into a `Map` for an unambiguous string-keyed view.
8
+ */
9
+ export declare function decodeMetadata(buf: Uint8Array): Map<string, unknown>;
10
+ //# sourceMappingURL=msgpack.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"msgpack.d.ts","sourceRoot":"","sources":["../src/msgpack.ts"],"names":[],"mappings":"AAmBA;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAkBpE"}
@@ -0,0 +1,44 @@
1
+ // Copyright 2026 Wanjia He
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ // msgpack decoder for TopicMetadata.Metadata. Uses @msgpack/msgpack with
15
+ // useBigInt64 so int64 values round-trip exactly.
16
+ import { decode } from "@msgpack/msgpack";
17
+ /**
18
+ * Decode a msgpack-encoded map into a Map<string, unknown>.
19
+ *
20
+ * - `useBigInt64: true` so any int64 values round-trip exactly as bigint.
21
+ * Smaller integers (int32 and below) decode as number.
22
+ * - The Go writer always msgpack-encodes a `map[string]any`. We coerce the
23
+ * decoded plain object into a `Map` for an unambiguous string-keyed view.
24
+ */
25
+ export function decodeMetadata(buf) {
26
+ const decoded = decode(buf, { useBigInt64: true });
27
+ const out = new Map();
28
+ if (decoded === null || typeof decoded !== "object") {
29
+ return out;
30
+ }
31
+ if (decoded instanceof Map) {
32
+ for (const [k, v] of decoded) {
33
+ if (typeof k === "string") {
34
+ out.set(k, v);
35
+ }
36
+ }
37
+ return out;
38
+ }
39
+ for (const k of Object.keys(decoded)) {
40
+ out.set(k, decoded[k]);
41
+ }
42
+ return out;
43
+ }
44
+ //# sourceMappingURL=msgpack.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"msgpack.js","sourceRoot":"","sources":["../src/msgpack.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAEjC,yEAAyE;AACzE,kDAAkD;AAElD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,GAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAmB,CAAC;IACvC,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,OAAO,YAAY,GAAG,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAgC,EAAE,CAAC;YACtD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC1B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAkC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAG,OAAmC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,56 @@
1
+ import type { ReadOptions } from "./read_options.js";
2
+ import { Reader, type Message, type SampleOptions, type SampleQuery, type SampleResult } from "./reader.js";
3
+ /**
4
+ * Thrown by MultiReader.sample/readMessages under videoDecodable when a queried
5
+ * video topic is provided by more than one reader whose time ranges overlap.
6
+ * Decodable multi-file video requires each video topic's source files to be
7
+ * time-disjoint; merging overlapping decodable sources would interleave frames
8
+ * from different GOP chains into an undecodable stream. Mirrors go
9
+ * ErrVideoSourcesOverlap / py VideoSourcesOverlapError.
10
+ */
11
+ export declare class VideoSourcesOverlapError extends Error {
12
+ readonly topic: string;
13
+ readonly first: readonly [bigint, bigint];
14
+ readonly second: readonly [bigint, bigint];
15
+ constructor(topic: string, first: readonly [bigint, bigint], second: readonly [bigint, bigint]);
16
+ }
17
+ export declare class MultiReader {
18
+ private readonly readers;
19
+ /**
20
+ * Group readers into one merged view. At least one reader is required.
21
+ * Per-file behavior (remap, etc.) lives on each underlying Reader.
22
+ */
23
+ constructor(readers: Reader[]);
24
+ /**
25
+ * Read every reader's messages as one merged, time-ordered stream. Options
26
+ * pass through to each underlying Reader.readMessages unchanged, so order,
27
+ * time bounds, topic filtering (by exposed name), and strategy apply per file
28
+ * before the merge.
29
+ *
30
+ * When videoDecodable is set, any in-scope video topic supplied by more than
31
+ * one reader must have time-disjoint source ranges; overlapping sources throw
32
+ * VideoSourcesOverlapError (merging them would interleave frames from
33
+ * different GOP chains into an undecodable stream). As with the single
34
+ * Reader, all I/O (and this check) is deferred to the first iteration step.
35
+ */
36
+ readMessages(opts?: ReadOptions): AsyncIterableIterator<Message>;
37
+ /**
38
+ * Resolve floor messages across every reader. For each
39
+ * (queries[i].topic, queries[i].timestamps[j]) pair, out[i][j] is the floor
40
+ * result with the latest found timestamp among all readers that hold the
41
+ * topic (by exposed name) - the true floor over the union. Ties resolve
42
+ * arbitrarily.
43
+ *
44
+ * Preconditions, validated before any data I/O (aggregated into a single
45
+ * SampleValidationError):
46
+ * - each queries[i].timestamps must be strictly increasing
47
+ * - each queries[i].topic must be unique across all i
48
+ * - each queries[i].topic must exist in at least one reader
49
+ *
50
+ * When videoDecodable is set, a video topic supplied by more than one reader
51
+ * must have time-disjoint source ranges; overlapping sources throw
52
+ * VideoSourcesOverlapError.
53
+ */
54
+ sample(queries: SampleQuery[], opts?: SampleOptions): Promise<SampleResult[][]>;
55
+ }
56
+ //# sourceMappingURL=multi_reader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"multi_reader.d.ts","sourceRoot":"","sources":["../src/multi_reader.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,MAAM,EAEN,KAAK,OAAO,EACZ,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,YAAY,EAElB,MAAM,aAAa,CAAC;AAErB;;;;;;;GAOG;AACH,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEzC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;CAUpC;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAW;IAEnC;;;OAGG;gBACS,OAAO,EAAE,MAAM,EAAE;IAO7B;;;;;;;;;;;OAWG;IACH,YAAY,CAAC,IAAI,GAAE,WAAgB,GAAG,qBAAqB,CAAC,OAAO,CAAC;IAwEpE;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CACV,OAAO,EAAE,WAAW,EAAE,EACtB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;CAwD7B"}
@@ -0,0 +1,281 @@
1
+ // Copyright 2026 Wanjia He
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ // MultiReader: present several single-file Readers as one time-ordered stream.
15
+ // Mirrors go/multi_reader.go and py/turbodata/multi_reader.py.
16
+ //
17
+ // readMessages merges every reader's stream in timestamp order; sample resolves
18
+ // floor messages across the union of readers. Both operate entirely in each
19
+ // Reader's exposed-name space. Topic-name collisions across files are the
20
+ // caller's responsibility: configure per-Reader topicRemap so that names meant
21
+ // to union share an exposed name and names meant to stay distinct do not.
22
+ import { MessageHeap } from "./iterators/message_heap.js";
23
+ import { SampleValidationError, } from "./reader.js";
24
+ /**
25
+ * Thrown by MultiReader.sample/readMessages under videoDecodable when a queried
26
+ * video topic is provided by more than one reader whose time ranges overlap.
27
+ * Decodable multi-file video requires each video topic's source files to be
28
+ * time-disjoint; merging overlapping decodable sources would interleave frames
29
+ * from different GOP chains into an undecodable stream. Mirrors go
30
+ * ErrVideoSourcesOverlap / py VideoSourcesOverlapError.
31
+ */
32
+ export class VideoSourcesOverlapError extends Error {
33
+ topic;
34
+ first;
35
+ second;
36
+ constructor(topic, first, second) {
37
+ super(`turbodata: video topic "${topic}" sources [${first[0]},${first[1]}] and [${second[0]},${second[1]}] overlap in time`);
38
+ this.name = "VideoSourcesOverlapError";
39
+ this.topic = topic;
40
+ this.first = first;
41
+ this.second = second;
42
+ }
43
+ }
44
+ export class MultiReader {
45
+ readers;
46
+ /**
47
+ * Group readers into one merged view. At least one reader is required.
48
+ * Per-file behavior (remap, etc.) lives on each underlying Reader.
49
+ */
50
+ constructor(readers) {
51
+ if (readers.length === 0) {
52
+ throw new Error("turbodata: MultiReader requires at least one reader");
53
+ }
54
+ this.readers = readers;
55
+ }
56
+ /**
57
+ * Read every reader's messages as one merged, time-ordered stream. Options
58
+ * pass through to each underlying Reader.readMessages unchanged, so order,
59
+ * time bounds, topic filtering (by exposed name), and strategy apply per file
60
+ * before the merge.
61
+ *
62
+ * When videoDecodable is set, any in-scope video topic supplied by more than
63
+ * one reader must have time-disjoint source ranges; overlapping sources throw
64
+ * VideoSourcesOverlapError (merging them would interleave frames from
65
+ * different GOP chains into an undecodable stream). As with the single
66
+ * Reader, all I/O (and this check) is deferred to the first iteration step.
67
+ */
68
+ readMessages(opts = {}) {
69
+ const readers = this.readers;
70
+ const reverse = opts.order === "reverse-time";
71
+ const videoDecodable = opts.videoDecodable === true;
72
+ const wantedNames = opts.topicNames;
73
+ let initialized = false;
74
+ let exhausted = false;
75
+ const subs = [];
76
+ const heap = new MessageHeap(reverse);
77
+ const init = async () => {
78
+ for (const r of readers) {
79
+ subs.push(r.readMessages(opts));
80
+ }
81
+ if (videoDecodable) {
82
+ const bounds = await Promise.all(readers.map((r) => r.topicBounds()));
83
+ enforceVideoDisjoint(readScopeTopics(wantedNames, bounds), bounds);
84
+ }
85
+ // Prime the heap with one message from each reader's stream.
86
+ for (let i = 0; i < subs.length; i++) {
87
+ const res = await subs[i].next();
88
+ if (res.done === true) {
89
+ continue;
90
+ }
91
+ heap.push(res.value.timestamp, { subIdx: i, msg: res.value });
92
+ }
93
+ };
94
+ const iter = {
95
+ [Symbol.asyncIterator]() {
96
+ return iter;
97
+ },
98
+ async next() {
99
+ if (exhausted) {
100
+ return { value: undefined, done: true };
101
+ }
102
+ if (!initialized) {
103
+ await init();
104
+ initialized = true;
105
+ }
106
+ const top = heap.pop();
107
+ if (top === undefined) {
108
+ exhausted = true;
109
+ return { value: undefined, done: true };
110
+ }
111
+ const out = top.value.msg;
112
+ const nm = await subs[top.value.subIdx].next();
113
+ if (nm.done !== true) {
114
+ heap.push(nm.value.timestamp, {
115
+ subIdx: top.value.subIdx,
116
+ msg: nm.value,
117
+ });
118
+ }
119
+ return { value: out, done: false };
120
+ },
121
+ async return() {
122
+ exhausted = true;
123
+ for (const s of subs) {
124
+ if (s.return !== undefined) {
125
+ await s.return();
126
+ }
127
+ }
128
+ return { value: undefined, done: true };
129
+ },
130
+ };
131
+ return iter;
132
+ }
133
+ /**
134
+ * Resolve floor messages across every reader. For each
135
+ * (queries[i].topic, queries[i].timestamps[j]) pair, out[i][j] is the floor
136
+ * result with the latest found timestamp among all readers that hold the
137
+ * topic (by exposed name) - the true floor over the union. Ties resolve
138
+ * arbitrarily.
139
+ *
140
+ * Preconditions, validated before any data I/O (aggregated into a single
141
+ * SampleValidationError):
142
+ * - each queries[i].timestamps must be strictly increasing
143
+ * - each queries[i].topic must be unique across all i
144
+ * - each queries[i].topic must exist in at least one reader
145
+ *
146
+ * When videoDecodable is set, a video topic supplied by more than one reader
147
+ * must have time-disjoint source ranges; overlapping sources throw
148
+ * VideoSourcesOverlapError.
149
+ */
150
+ async sample(queries, opts = {}) {
151
+ const videoDecodable = opts.videoDecodable === true;
152
+ const bounds = await Promise.all(this.readers.map((r) => r.topicBounds()));
153
+ validateMultiSampleQueries(queries, bounds);
154
+ if (videoDecodable) {
155
+ enforceVideoDisjoint(queries.map((q) => q.topic), bounds);
156
+ }
157
+ const out = queries.map((q) => q.timestamps.map(() => ({
158
+ found: false,
159
+ timestamp: 0n,
160
+ data: new Uint8Array(0),
161
+ isVideo: false,
162
+ frames: [],
163
+ resetDecoder: false,
164
+ })));
165
+ // Dispatch to each reader only the queries whose topic it holds (avoids
166
+ // the engine's unknown-topic error), then merge by latest found floor.
167
+ for (let ri = 0; ri < this.readers.length; ri++) {
168
+ const subset = [];
169
+ const origIdx = [];
170
+ for (let qi = 0; qi < queries.length; qi++) {
171
+ if (bounds[ri].has(queries[qi].topic)) {
172
+ subset.push(queries[qi]);
173
+ origIdx.push(qi);
174
+ }
175
+ }
176
+ if (subset.length === 0) {
177
+ continue;
178
+ }
179
+ const res = await this.readers[ri].sample(subset, opts);
180
+ for (let sub = 0; sub < origIdx.length; sub++) {
181
+ const qi = origIdx[sub];
182
+ const row = res[sub];
183
+ for (let j = 0; j < row.length; j++) {
184
+ const cand = row[j];
185
+ if (!cand.found) {
186
+ continue;
187
+ }
188
+ const cur = out[qi][j];
189
+ if (!cur.found || cand.timestamp > cur.timestamp) {
190
+ out[qi][j] = cand;
191
+ }
192
+ }
193
+ }
194
+ }
195
+ return out;
196
+ }
197
+ }
198
+ /**
199
+ * The exposed topic names a readMessages call covers: the requested
200
+ * topicNames, or every exposed topic across all readers when unset.
201
+ */
202
+ function readScopeTopics(topicNames, bounds) {
203
+ if (topicNames !== undefined && topicNames.length > 0) {
204
+ return topicNames;
205
+ }
206
+ const seen = new Set();
207
+ const out = [];
208
+ for (const b of bounds) {
209
+ for (const name of b.keys()) {
210
+ if (!seen.has(name)) {
211
+ seen.add(name);
212
+ out.push(name);
213
+ }
214
+ }
215
+ }
216
+ return out;
217
+ }
218
+ /**
219
+ * Mirror the single-reader checks but treat a topic as known if any reader
220
+ * holds it. All violations are aggregated into one SampleValidationError.
221
+ * Mirrors go validateMultiSampleQueries.
222
+ */
223
+ function validateMultiSampleQueries(queries, bounds) {
224
+ const seen = new Map();
225
+ const violations = [];
226
+ for (let i = 0; i < queries.length; i++) {
227
+ const q = queries[i];
228
+ const known = bounds.some((b) => b.has(q.topic));
229
+ if (!known) {
230
+ violations.push(`queries[${i}]: unknown topic "${q.topic}" (not in any reader)`);
231
+ }
232
+ const prev = seen.get(q.topic);
233
+ if (prev !== undefined) {
234
+ violations.push(`queries[${i}]: duplicate topic "${q.topic}" already used by queries[${prev}]`);
235
+ }
236
+ else {
237
+ seen.set(q.topic, i);
238
+ }
239
+ for (let j = 1; j < q.timestamps.length; j++) {
240
+ if (q.timestamps[j] <= q.timestamps[j - 1]) {
241
+ violations.push(`queries[${i}].timestamps not strictly increasing at position ${j} (${q.timestamps[j]} <= ${q.timestamps[j - 1]})`);
242
+ }
243
+ }
244
+ }
245
+ if (violations.length > 0) {
246
+ throw new SampleValidationError(violations);
247
+ }
248
+ }
249
+ /**
250
+ * Throw VideoSourcesOverlapError when any of the given exposed video topics is
251
+ * provided by readers with overlapping inclusive time ranges. Topics that are
252
+ * not video, or are held by fewer than two readers, are skipped. Mirrors go
253
+ * enforceVideoDisjoint.
254
+ */
255
+ function enforceVideoDisjoint(topics, bounds) {
256
+ for (const topic of topics) {
257
+ const ranges = [];
258
+ let isVideo = false;
259
+ for (const b of bounds) {
260
+ const bound = b.get(topic);
261
+ if (bound === undefined) {
262
+ continue;
263
+ }
264
+ if (bound.isVideo) {
265
+ isVideo = true;
266
+ }
267
+ ranges.push([bound.minTs, bound.maxTs]);
268
+ }
269
+ if (!isVideo || ranges.length < 2) {
270
+ continue;
271
+ }
272
+ for (let a = 0; a < ranges.length; a++) {
273
+ for (let c = a + 1; c < ranges.length; c++) {
274
+ if (ranges[a][0] <= ranges[c][1] && ranges[c][0] <= ranges[a][1]) {
275
+ throw new VideoSourcesOverlapError(topic, ranges[a], ranges[c]);
276
+ }
277
+ }
278
+ }
279
+ }
280
+ }
281
+ //# sourceMappingURL=multi_reader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"multi_reader.js","sourceRoot":"","sources":["../src/multi_reader.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAEjC,+EAA+E;AAC/E,+DAA+D;AAC/D,EAAE;AACF,gFAAgF;AAChF,4EAA4E;AAC5E,0EAA0E;AAC1E,+EAA+E;AAC/E,0EAA0E;AAE1E,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE1D,OAAO,EAEL,qBAAqB,GAMtB,MAAM,aAAa,CAAC;AAErB;;;;;;;GAOG;AACH,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACxC,KAAK,CAAS;IACd,KAAK,CAA4B;IACjC,MAAM,CAA4B;IAC3C,YACE,KAAa,EACb,KAAgC,EAChC,MAAiC;QAEjC,KAAK,CACH,2BAA2B,KAAK,cAAc,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,mBAAmB,CACtH,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,MAAM,OAAO,WAAW;IACL,OAAO,CAAW;IAEnC;;;OAGG;IACH,YAAY,OAAiB;QAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;;;;;;;;;;OAWG;IACH,YAAY,CAAC,OAAoB,EAAE;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,KAAK,cAAc,CAAC;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;QAEpC,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,IAAI,GAAqC,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,WAAW,CAAmC,OAAO,CAAC,CAAC;QAExE,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;YACrC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBACtE,oBAAoB,CAAC,eAAe,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;YACrE,CAAC;YAED,6DAA6D;YAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBACtB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,IAAI,GAAmC;YAC3C,CAAC,MAAM,CAAC,aAAa,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC;YACD,KAAK,CAAC,IAAI;gBACR,IAAI,SAAS,EAAE,CAAC;oBACd,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAC1C,CAAC;gBACD,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,MAAM,IAAI,EAAE,CAAC;oBACb,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBACtB,SAAS,GAAG,IAAI,CAAC;oBACjB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAC1C,CAAC;gBACD,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;gBAC1B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE;wBAC5B,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM;wBACxB,GAAG,EAAE,EAAE,CAAC,KAAK;qBACd,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACrC,CAAC;YACD,KAAK,CAAC,MAAM;gBACV,SAAS,GAAG,IAAI,CAAC;gBACjB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;oBACrB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;wBAC3B,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;oBACnB,CAAC;gBACH,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC1C,CAAC;SACF,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,MAAM,CACV,OAAsB,EACtB,OAAsB,EAAE;QAExB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE3E,0BAA0B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE5C,IAAI,cAAc,EAAE,CAAC;YACnB,oBAAoB,CAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAC3B,MAAM,CACP,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9C,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACtB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,EAAE;YACb,IAAI,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC;YACvB,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,EAAE;YACV,YAAY,EAAE,KAAK;SACpB,CAAC,CAAC,CACJ,CAAC;QAEF,wEAAwE;QACxE,uEAAuE;QACvE,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAChD,MAAM,MAAM,GAAkB,EAAE,CAAC;YACjC,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,EAAE,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAE,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE,CAAC,CAAC;oBAC1B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACzD,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;oBACrB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;wBAChB,SAAS;oBACX,CAAC;oBACD,MAAM,GAAG,GAAG,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAE,CAAC;oBACzB,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;wBACjD,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,eAAe,CACtB,UAAgC,EAChC,MAAiC;IAEjC,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,0BAA0B,CACjC,OAAsB,EACtB,MAAiC;IAEjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,UAAU,CAAC,IAAI,CACb,WAAW,CAAC,qBAAqB,CAAC,CAAC,KAAK,uBAAuB,CAChE,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,UAAU,CAAC,IAAI,CACb,WAAW,CAAC,uBAAuB,CAAC,CAAC,KAAK,6BAA6B,IAAI,GAAG,CAC/E,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAE,EAAE,CAAC;gBAC7C,UAAU,CAAC,IAAI,CACb,WAAW,CAAC,oDAAoD,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CACnH,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,qBAAqB,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,MAAgB,EAChB,MAAiC;IAEjC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAuB,EAAE,CAAC;QACtC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAClB,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,SAAS;QACX,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrE,MAAM,IAAI,wBAAwB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,9 @@
1
+ import type { ReadOp } from "./read_planner.js";
2
+ import type { ReadSource } from "./read_source.js";
3
+ /**
4
+ * Execute ops against rs with at most maxConcurrency in-flight reads. Returns
5
+ * one Uint8Array per op, in input order. Throws on the first error
6
+ * encountered; in-flight reads may still settle in the background.
7
+ */
8
+ export declare function fetchAll(rs: ReadSource, ops: ReadOp[], maxConcurrency: number): Promise<Uint8Array[]>;
9
+ //# sourceMappingURL=read_fetcher.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read_fetcher.d.ts","sourceRoot":"","sources":["../src/read_fetcher.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,MAAM,EAAE,EACb,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,UAAU,EAAE,CAAC,CA4CvB"}
@@ -0,0 +1,59 @@
1
+ // Copyright 2026 Wanjia He
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ /**
15
+ * Execute ops against rs with at most maxConcurrency in-flight reads. Returns
16
+ * one Uint8Array per op, in input order. Throws on the first error
17
+ * encountered; in-flight reads may still settle in the background.
18
+ */
19
+ export async function fetchAll(rs, ops, maxConcurrency) {
20
+ if (ops.length === 0) {
21
+ return [];
22
+ }
23
+ const workers = maxConcurrency > 0 ? maxConcurrency : 1;
24
+ const out = new Array(ops.length);
25
+ let nextIdx = 0;
26
+ let firstErr = undefined;
27
+ const runOne = async () => {
28
+ while (firstErr === undefined) {
29
+ const i = nextIdx++;
30
+ if (i >= ops.length) {
31
+ return;
32
+ }
33
+ const op = ops[i];
34
+ try {
35
+ const buf = await rs.read(op.offset, op.length);
36
+ if (BigInt(buf.byteLength) !== op.length) {
37
+ throw new Error(`fetcher: short read for op ${i}: got ${buf.byteLength} want ${op.length}`);
38
+ }
39
+ out[i] = buf;
40
+ }
41
+ catch (err) {
42
+ if (firstErr === undefined) {
43
+ firstErr = err;
44
+ }
45
+ return;
46
+ }
47
+ }
48
+ };
49
+ const pool = new Array(Math.min(workers, ops.length));
50
+ for (let w = 0; w < pool.length; w++) {
51
+ pool[w] = runOne();
52
+ }
53
+ await Promise.all(pool);
54
+ if (firstErr !== undefined) {
55
+ throw firstErr;
56
+ }
57
+ return out;
58
+ }
59
+ //# sourceMappingURL=read_fetcher.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read_fetcher.js","sourceRoot":"","sources":["../src/read_fetcher.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AAQjC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAc,EACd,GAAa,EACb,cAAsB;IAEtB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,OAAO,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAa,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAY,SAAS,CAAC;IAElC,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO;YACT,CAAC;YACD,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;gBAChD,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC;oBACzC,MAAM,IAAI,KAAK,CACb,8BAA8B,CAAC,SAAS,GAAG,CAAC,UAAU,SAAS,EAAE,CAAC,MAAM,EAAE,CAC3E,CAAC;gBACJ,CAAC;gBACD,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YACf,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,QAAQ,GAAG,GAAG,CAAC;gBACjB,CAAC;gBACD,OAAO;YACT,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,IAAI,GAAoB,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC;IACrB,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAExB,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,40 @@
1
+ import type { ReadStrategy } from "./read_strategy.js";
2
+ export type Order = "time" | "reverse-time";
3
+ export interface ReadOptions {
4
+ /** When set, only messages from these topic names are emitted. */
5
+ topicNames?: string[];
6
+ /** int64. Inclusive lower bound on message timestamp. Default: 0n. */
7
+ startTimestamp?: bigint;
8
+ /** int64. Inclusive upper bound on message timestamp. Default: int64-max. */
9
+ endTimestamp?: bigint;
10
+ /** Iteration order. Default: "time". */
11
+ order?: Order;
12
+ /**
13
+ * When set, switches Reader.readMessages onto the cost-aware path. Without
14
+ * a strategy, the reader uses the default lazy path (one chunk at a time).
15
+ */
16
+ strategy?: ReadStrategy;
17
+ /**
18
+ * int64. Bytes to read speculatively from the file tail when loading the
19
+ * summary. When the (footer + compressed summary) fits within the prefetch
20
+ * window, summary loading costs one read instead of two. Default: 0n
21
+ * (footer-only read).
22
+ */
23
+ tailPrefetch?: bigint;
24
+ /**
25
+ * If true, the iterator hands out a freshly-allocated Uint8Array per
26
+ * message (safe to retain). If false (default), `data` aliases an internal
27
+ * reusable buffer that becomes invalid on the next iteration step.
28
+ */
29
+ copy?: boolean;
30
+ /**
31
+ * If true, for any video topic in scope the effective per-group
32
+ * startTimestamp is snapped back to the latest key frame whose timestamp is
33
+ * <= startTimestamp, so the emitted sequence can be fed to a decoder cold.
34
+ * Non-video topics are unaffected. Default: false.
35
+ */
36
+ videoDecodable?: boolean;
37
+ }
38
+ /** Default value for endTimestamp: matches Go's math.MaxInt64. */
39
+ export declare const MAX_INT64 = 9223372036854775807n;
40
+ //# sourceMappingURL=read_options.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read_options.d.ts","sourceRoot":"","sources":["../src/read_options.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,cAAc,CAAC;AAE5C,MAAM,WAAW,WAAW;IAC1B,kEAAkE;IAClE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wCAAwC;IACxC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd;;;OAGG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,kEAAkE;AAClE,eAAO,MAAM,SAAS,uBAAsB,CAAC"}
@@ -0,0 +1,16 @@
1
+ // Copyright 2026 Wanjia He
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+ /** Default value for endTimestamp: matches Go's math.MaxInt64. */
15
+ export const MAX_INT64 = 0x7fffffffffffffffn;
16
+ //# sourceMappingURL=read_options.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read_options.js","sourceRoot":"","sources":["../src/read_options.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B,EAAE;AACF,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,EAAE;AACF,iDAAiD;AACjD,EAAE;AACF,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AA4CjC,kEAAkE;AAClE,MAAM,CAAC,MAAM,SAAS,GAAG,mBAAmB,CAAC"}