yggio 0.0.16 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,58 @@
1
+ 'use strict'
2
+
3
+ // Boundary cost of the Arrow exchange of Serie and ChunkedSerie: in from an
4
+ // IPC stream, out through `intoArrowReader().intoIpc()`, and the row-by-row
5
+ // `asJs` read for comparison, on a tenth of the rows. The row count is
6
+ // YGGIO_BENCH_ROWS (default 200000).
7
+
8
+ const { performance } = require('node:perf_hooks')
9
+
10
+ const arrow = require('apache-arrow')
11
+
12
+ const { BatchReader, ChunkedSerie, Serie } = require('yggio')
13
+
14
+ const rows = Number.parseInt(process.env.YGGIO_BENCH_ROWS ?? '200000', 10)
15
+ if (!Number.isSafeInteger(rows) || rows <= 0) {
16
+ throw new RangeError('YGGIO_BENCH_ROWS must be a positive safe integer')
17
+ }
18
+ const asJsRows = Math.max(1, Math.floor(rows / 10))
19
+ const BATCHES = 8
20
+
21
+ const ids = Float64Array.from({ length: rows }, (_, index) => index)
22
+ const labels = Array.from({ length: rows }, (_, index) => `row-${index % 1000}`)
23
+ const table = arrow.tableFromArrays({ id: ids, label: labels })
24
+ const stream = Buffer.from(arrow.tableToIPC(table, 'stream'))
25
+ const batched = (() => {
26
+ const size = Math.ceil(rows / BATCHES)
27
+ const parts = []
28
+ for (let offset = 0; offset < rows; offset += size) {
29
+ parts.push(table.slice(offset, Math.min(rows, offset + size)).batches[0])
30
+ }
31
+ return Buffer.from(arrow.tableToIPC(new arrow.Table(parts), 'stream'))
32
+ })()
33
+
34
+ function benchmark(name, count, operation) {
35
+ operation()
36
+ const runs = []
37
+ for (let round = 0; round < 5; round += 1) {
38
+ const started = performance.now()
39
+ operation()
40
+ runs.push(performance.now() - started)
41
+ }
42
+ runs.sort((a, b) => a - b)
43
+ console.log(`${name.padEnd(44)} ${((runs[2] * 1e6) / count).toFixed(1).padStart(10)} ns/row`)
44
+ }
45
+
46
+ const serie = Serie.fromArrowReader(BatchReader.fromIpc(stream))
47
+ const chunked = ChunkedSerie.fromArrowReader(BatchReader.fromIpc(batched))
48
+
49
+ benchmark('Serie.fromArrowReader(fromIpc)', rows, () =>
50
+ Serie.fromArrowReader(BatchReader.fromIpc(stream)).length,
51
+ )
52
+ benchmark('ChunkedSerie.fromArrowReader(fromIpc)', rows, () =>
53
+ ChunkedSerie.fromArrowReader(BatchReader.fromIpc(batched)).length,
54
+ )
55
+ benchmark('Serie.intoArrowReader().intoIpc()', rows, () => serie.intoArrowReader().intoIpc())
56
+ benchmark('ChunkedSerie.intoArrowReader().intoIpc()', rows, () => chunked.intoArrowReader().intoIpc())
57
+ const head = serie.child('id').slice(0, asJsRows)
58
+ benchmark(`Serie.asJs() over ${asJsRows} rows`, asJsRows, () => head.asJs())
@@ -106,7 +106,8 @@ async function main() {
106
106
  // An XML document is one root element, so a record of several entries
107
107
  // travels under one name there and as itself everywhere else.
108
108
  const rooted = (fixture) => (name === 'xml' ? { data: fixture } : fixture)
109
- const encoded = format.dumps(value)
109
+ const document = rooted(value)
110
+ const encoded = format.dumps(document)
110
111
  const text = encoded.toString('utf8')
111
112
  const padded = Buffer.concat([Buffer.from('xx'), encoded, Buffer.from('yy')])
112
113
  const view = new DataView(
@@ -124,12 +125,12 @@ async function main() {
124
125
  )
125
126
  measure(`${name}/string`, encoded.length, 100, () => format.loads(text))
126
127
  measure(`${name}/offset_view`, encoded.length, 100, () => format.loads(view))
127
- measure(`${name}/reader_path`, encoded.length, 100, () =>
128
+ await measureAsync(`${name}/reader_path`, encoded.length, 100, () =>
128
129
  format.load(pathToFileURL(file)),
129
130
  )
130
- measure(`${name}/vector_emit`, encoded.length, 100, () => format.dumps(value))
131
+ measure(`${name}/vector_emit`, encoded.length, 100, () => format.dumps(document))
131
132
  measure(`${name}/formatted_emit`, encoded.length, 100, () =>
132
- format.dumps(value, { indent: 2 }),
133
+ format.dumps(document, { indent: 2 }),
133
134
  )
134
135
  measure(`${name}/explicit_limits`, encoded.length, 100, () =>
135
136
  format.loads(encoded, {
@@ -139,13 +140,15 @@ async function main() {
139
140
  maxDocuments: 1,
140
141
  }),
141
142
  )
142
- measure(`${name}/writer_path`, encoded.length, 100, () => format.dump(value, file))
143
+ await measureAsync(`${name}/writer_path`, encoded.length, 100, () =>
144
+ format.dump(document, file),
145
+ )
143
146
  await measureAsync(`${name}/reader_stream`, encoded.length, 25, () =>
144
147
  format.load(Readable.from([encoded])),
145
148
  )
146
149
  await measureAsync(`${name}/writer_stream`, encoded.length, 25, () =>
147
150
  format.dump(
148
- value,
151
+ document,
149
152
  new Writable({ write(_chunk, _encoding, done) { done() } }),
150
153
  ),
151
154
  )
@@ -12,7 +12,6 @@ const {
12
12
  MimeType,
13
13
  Scalar,
14
14
  Serie,
15
- SerieReader,
16
15
  Side,
17
16
  SpillOptions,
18
17
  Plan,
@@ -29,15 +28,24 @@ if (!Number.isSafeInteger(iterations) || iterations <= 0) {
29
28
  throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
30
29
  }
31
30
 
32
- function benchmark(name, operation) {
31
+ // The addon's finalizers run when the event loop turns, so a run of rows
32
+ // that never yields keeps every native value it made until the end - tens
33
+ // of gigabytes over the whole file. Each row yields once after its timed
34
+ // loop, outside the timing, and the next starts from a released heap.
35
+ const settled = () => new Promise((resolve) => setImmediate(resolve))
36
+
37
+ async function benchmark(name, operation) {
33
38
  for (let index = 0; index < Math.min(iterations, 1_000); index += 1) operation()
34
39
  const started = performance.now()
35
40
  for (let index = 0; index < iterations; index += 1) operation()
36
41
  const elapsed = performance.now() - started
37
42
  const rate = Math.round((iterations * 1_000) / elapsed)
38
43
  console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
44
+ await settled()
39
45
  }
40
46
 
47
+ async function main() {
48
+
41
49
  const id = fields.int32('id', { nullable: false, metadata: { source: 'event' } })
42
50
  const digestBits = fields.int64('digest', { nullable: false })
43
51
  const unsignedDigest = arrow.vectorFromArray([2n ** 64n - 1n], new arrow.Uint64())
@@ -113,30 +121,30 @@ const sideScalar = side.scalar('BUY')
113
121
  const kind = DataType.from('marketdatakind')
114
122
  const kindScalar = kind.scalar('OPER')
115
123
 
116
- benchmark('version/native_parts', () => new Version(5, 0, 300))
117
- benchmark('version/native_parse', () => Version.fromStr('5.0.300'))
118
- benchmark('version/native_parts_text_patch', () => new Version(1, 0, '-rc1'))
119
- benchmark('version/native_parse_text_patch', () => Version.fromStr('1.0-rc1'))
120
- benchmark('version/patch', () => release.patch)
121
- benchmark('version/compare', () => earlierRelease.compare(release))
122
- benchmark('version/compare_text_patch', () => candidate.compare(laterCandidate))
123
- benchmark('version/stable_hash', () => release.stableHash())
124
- benchmark('version/clone', () => release.clone())
125
- benchmark('version/into_scalar', () => Scalar.from(release))
126
- benchmark('version/field_into_scalar', () => Scalar.from('5.0.300', { field: releaseField }))
127
- benchmark('version/scalar_as_js', () => releaseScalar.asJs())
128
- benchmark('figi/into_scalar', () => figi.scalar('BBG000BLNQ16'))
129
- benchmark('figi/field_into_scalar', () => Scalar.from('BBG000BLNQ16', { field: figiField }))
130
- benchmark('figi/scalar_as_js', () => figiScalar.asJs())
131
- benchmark('side/member_lookup', () => Side.SELL)
132
- benchmark('side/into_scalar', () => side.scalar('SELL'))
133
- benchmark('side/scalar_as_js', () => sideScalar.asJs())
134
- benchmark('marketdatakind/member_lookup', () => MarketDataKind.OPER)
135
- benchmark('marketdatakind/into_scalar', () => kind.scalar('OPER'))
136
- benchmark('marketdatakind/scalar_as_js', () => kindScalar.asJs())
124
+ await benchmark('version/native_parts', () => new Version(5, 0, 300))
125
+ await benchmark('version/native_parse', () => Version.fromStr('5.0.300'))
126
+ await benchmark('version/native_parts_text_patch', () => new Version(1, 0, '-rc1'))
127
+ await benchmark('version/native_parse_text_patch', () => Version.fromStr('1.0-rc1'))
128
+ await benchmark('version/patch', () => release.patch)
129
+ await benchmark('version/compare', () => earlierRelease.compare(release))
130
+ await benchmark('version/compare_text_patch', () => candidate.compare(laterCandidate))
131
+ await benchmark('version/stable_hash', () => release.stableHash())
132
+ await benchmark('version/clone', () => release.clone())
133
+ await benchmark('version/into_scalar', () => Scalar.from(release))
134
+ await benchmark('version/field_into_scalar', () => Scalar.from('5.0.300', { field: releaseField }))
135
+ await benchmark('version/scalar_as_js', () => releaseScalar.asJs())
136
+ await benchmark('figi/into_scalar', () => figi.scalar('BBG000BLNQ16'))
137
+ await benchmark('figi/field_into_scalar', () => Scalar.from('BBG000BLNQ16', { field: figiField }))
138
+ await benchmark('figi/scalar_as_js', () => figiScalar.asJs())
139
+ await benchmark('side/member_lookup', () => Side.SELL)
140
+ await benchmark('side/into_scalar', () => side.scalar('SELL'))
141
+ await benchmark('side/scalar_as_js', () => sideScalar.asJs())
142
+ await benchmark('marketdatakind/member_lookup', () => MarketDataKind.OPER)
143
+ await benchmark('marketdatakind/into_scalar', () => kind.scalar('OPER'))
144
+ await benchmark('marketdatakind/scalar_as_js', () => kindScalar.asJs())
137
145
 
138
- benchmark('schema/from_fields', () => DataType.fromFields([id, name]))
139
- benchmark('serie/from_arrow_array_bits', () =>
146
+ await benchmark('schema/from_fields', () => DataType.fromFields([id, name]))
147
+ await benchmark('serie/from_arrow_array_bits', () =>
140
148
  Serie.fromArrowArray(unsignedDigest, digestBits, { representation: 'bits' }),
141
149
  )
142
150
  // A chunked column crosses as copied IPC, one batch per Data or per table
@@ -150,21 +158,20 @@ const chunkedTable = new arrow.Table([
150
158
  const heldChunked = ChunkedSerie.fromArrowArray(chunkedVector)
151
159
  const heldChunkedTable = ChunkedSerie.fromArrowBatch(chunkedTable)
152
160
  const chunkedWide = fields.float64('item', { nullable: false })
153
- benchmark('chunked_serie/from_arrow_array', () => ChunkedSerie.fromArrowArray(chunkedVector))
154
- benchmark('chunked_serie/from_arrow_batch_table', () =>
161
+ await benchmark('chunked_serie/from_arrow_array', () => ChunkedSerie.fromArrowArray(chunkedVector))
162
+ await benchmark('chunked_serie/from_arrow_batch_table', () =>
155
163
  ChunkedSerie.fromArrowBatch(chunkedTable),
156
164
  )
157
- benchmark('chunked_serie/cast', () => heldChunked.cast(chunkedWide))
158
- benchmark('chunked_serie/into_serie', () => heldChunked.intoSerie())
159
- benchmark('chunked_serie/into_arrow_array', () => heldChunked.intoArrowArray())
160
- benchmark('chunked_serie/into_arrow_table', () => heldChunkedTable.intoArrowTable())
161
- benchmark('chunked_serie/child', () => heldChunkedTable.child('id'))
165
+ await benchmark('chunked_serie/cast', () => heldChunked.cast(chunkedWide))
166
+ await benchmark('chunked_serie/into_serie', () => heldChunked.intoSerie())
167
+ await benchmark('chunked_serie/into_arrow_array', () => heldChunked.intoArrowArray())
168
+ await benchmark('chunked_serie/into_arrow_table', () => heldChunkedTable.intoArrowTable())
169
+ await benchmark('chunked_serie/child', () => heldChunkedTable.child('id'))
162
170
  // A held record column streams with no plan compiled; a cast is one plan
163
171
  // over every record the reader holds, applied as the cast is asked for.
164
172
  const heldRecords = Serie.fromArrowBatch(chunkedTable)
165
173
  const heldRecordsWide = fields.struct('row', [fields.float64('id')], { nullable: false })
166
- benchmark('serie_reader/from_serie', () => SerieReader.fromSerie(heldRecords))
167
- benchmark('serie_reader/cast', () => SerieReader.fromSerie(heldRecords).cast(heldRecordsWide))
174
+ await benchmark('serie/cast_records', () => heldRecords.cast(heldRecordsWide))
168
175
  // Ordering, uniqueness and grouping over a 64-row primitive column, a
169
176
  // 64-row text column and a record of both; every argument serie is built
170
177
  // once, so a row measures the verb and its crossing alone. An `as*` write
@@ -195,31 +202,31 @@ const orderMask = Serie.fromScalars(
195
202
  Array.from({ length: orderRows }, () => true),
196
203
  )
197
204
  const descending = { descending: true }
198
- benchmark('serie/sort_indices', () => orderPrices.sortIndices())
199
- benchmark('serie/sort_indices_utf8', () => orderVenues.sortIndices(descending))
200
- benchmark('serie/is_sorted', () => orderPrices.isSorted())
201
- benchmark('serie/is_unique', () => orderVenues.isUnique())
202
- benchmark('serie/unique_count', () => orderVenues.uniqueCount())
203
- benchmark('serie/memory_size', () => orderPrices.memorySize())
204
- benchmark('serie/into_sorted', () => orderPrices.intoSorted())
205
- benchmark('serie/into_unique', () => orderVenues.intoUnique())
206
- benchmark('serie/into_reversed', () => orderPrices.intoReversed())
207
- benchmark('serie/into_taken', () => orderPrices.intoTaken(orderIndices))
208
- benchmark('serie/into_filtered', () => orderPrices.intoFiltered(orderMask))
209
- benchmark('serie/partition_by', () => orderPrices.partitionBy(orderVenues))
205
+ await benchmark('serie/sort_indices', () => orderPrices.sortIndices())
206
+ await benchmark('serie/sort_indices_utf8', () => orderVenues.sortIndices(descending))
207
+ await benchmark('serie/is_sorted', () => orderPrices.isSorted())
208
+ await benchmark('serie/is_unique', () => orderVenues.isUnique())
209
+ await benchmark('serie/unique_count', () => orderVenues.uniqueCount())
210
+ await benchmark('serie/memory_size', () => orderPrices.memorySize())
211
+ await benchmark('serie/into_sorted', () => orderPrices.intoSorted())
212
+ await benchmark('serie/into_unique', () => orderVenues.intoUnique())
213
+ await benchmark('serie/into_reversed', () => orderPrices.intoReversed())
214
+ await benchmark('serie/into_taken', () => orderPrices.intoTaken(orderIndices))
215
+ await benchmark('serie/into_filtered', () => orderPrices.intoFiltered(orderMask))
216
+ await benchmark('serie/partition_by', () => orderPrices.partitionBy(orderVenues))
210
217
  const sortedInPlace = orderPrices.intoSorted()
211
- benchmark('serie/as_sorted', () => sortedInPlace.asSorted())
218
+ await benchmark('serie/as_sorted', () => sortedInPlace.asSorted())
212
219
  const uniqueInPlace = orderVenues.intoUnique()
213
- benchmark('serie/as_unique', () => uniqueInPlace.asUnique())
220
+ await benchmark('serie/as_unique', () => uniqueInPlace.asUnique())
214
221
  const reversedInPlace = orderPrices.intoSorted()
215
- benchmark('serie/as_reversed', () => reversedInPlace.asReversed())
222
+ await benchmark('serie/as_reversed', () => reversedInPlace.asReversed())
216
223
  const takenInPlace = orderPrices.intoSorted()
217
- benchmark('serie/as_taken', () => takenInPlace.asTaken(orderIndices))
224
+ await benchmark('serie/as_taken', () => takenInPlace.asTaken(orderIndices))
218
225
  const filteredInPlace = orderPrices.intoSorted()
219
- benchmark('serie/as_filtered', () => filteredInPlace.asFiltered(orderMask))
220
- benchmark('serie/window', () => orderPrices.window(8, 32))
226
+ await benchmark('serie/as_filtered', () => filteredInPlace.asFiltered(orderMask))
227
+ await benchmark('serie/window', () => orderPrices.window(8, 32))
221
228
  // By key: the keys read once by the core's `order by` rule, then one take.
222
- benchmark('serie/into_sort_by', () => orderQuotes.intoSortBy('venue, price desc'))
229
+ await benchmark('serie/into_sort_by', () => orderQuotes.intoSortBy('venue, price desc'))
223
230
  // A join of the quotes against their three venues, the smaller side hashed.
224
231
  const orderCities = Serie.fromScalars(
225
232
  fields.struct('venue', [fields.utf8('venue'), fields.utf8('city')], { nullable: false }),
@@ -232,7 +239,7 @@ const orderCities = Serie.fromScalars(
232
239
  if (orderQuotes.joinWith(orderCities, 'venue').length !== orderRows) {
233
240
  throw new Error('the join fixture matches every quote once')
234
241
  }
235
- benchmark('serie/join_with', () => orderQuotes.joinWith(orderCities, 'venue'))
242
+ await benchmark('serie/join_with', () => orderQuotes.joinWith(orderCities, 'venue'))
236
243
  // A spill writes the column once to a private file and maps it back; the one
237
244
  // row written back after it copies the buffer to the heap and drops the
238
245
  // mapping, so every call spills again and holds one file at most.
@@ -245,7 +252,7 @@ spilling.set(0, spilledBack)
245
252
  if (!spilledOnce || spilling.isSpilled()) {
246
253
  throw new Error('the spill fixture spills and comes back to the heap')
247
254
  }
248
- benchmark('serie/spill', () => {
255
+ await benchmark('serie/spill', () => {
249
256
  spilling.spill(spillEverything)
250
257
  spilling.set(0, spilledBack)
251
258
  })
@@ -253,14 +260,14 @@ benchmark('serie/spill', () => {
253
260
  // whose write-back drops the mapping, so every call holds one file at most. A
254
261
  // spilled copy is not timed here: each holds its file until the collector
255
262
  // drops it, which a loop outruns.
256
- benchmark('serie/as_spilled', () => {
263
+ await benchmark('serie/as_spilled', () => {
257
264
  spilling.asSpilled(spillEverything).set(0, spilledBack)
258
265
  })
259
266
  // A constant column holds its one value once, whatever its length: the cost
260
267
  // is the value's proof, not the rows.
261
268
  const litField = fields.utf8('venue', { nullable: false })
262
269
  const litValue = Scalar.from('XNAS')
263
- benchmark('serie/lit', () => Serie.lit(litField, litValue, 1_000_000))
270
+ await benchmark('serie/lit', () => Serie.lit(litField, litValue, 1_000_000))
264
271
  // A window reads and writes through the serie it holds at each call.
265
272
  const windowed = orderPrices.intoSorted()
266
273
  const window = windowed.window(8, 32)
@@ -276,62 +283,62 @@ const windowKeys = orderVenues.slice(8, 32)
276
283
  const windowSource = orderPrices.window(0, 32)
277
284
  const windowRows = Array.from({ length: 4 }, (_, index) => Scalar.from(index))
278
285
  const windowValue = Scalar.from(7)
279
- benchmark('window_serie/null_count', () => window.nullCount())
280
- benchmark('window_serie/scalar', () => window.scalar(3))
281
- benchmark('window_serie/rows', () => window.rows())
282
- benchmark('window_serie/memory_size', () => window.memorySize())
283
- benchmark('window_serie/is_sorted', () => window.isSorted())
284
- benchmark('window_serie/is_unique', () => window.isUnique())
285
- benchmark('window_serie/unique_count', () => window.uniqueCount())
286
- benchmark('window_serie/sort_indices', () => window.sortIndices())
287
- benchmark('window_serie/window', () => window.window(4, 8))
288
- benchmark('window_serie/into_serie', () => window.intoSerie())
289
- benchmark('window_serie/into_sorted', () => window.intoSorted(descending))
290
- benchmark('window_serie/into_unique', () => window.intoUnique())
291
- benchmark('window_serie/into_reversed', () => window.intoReversed())
292
- benchmark('window_serie/into_taken', () => window.intoTaken(windowIndices))
293
- benchmark('window_serie/into_filtered', () => window.intoFiltered(windowMask))
294
- benchmark('window_serie/partition_by', () => window.partitionBy(windowKeys))
295
- benchmark('window_serie/set', () => window.set(3, windowValue))
296
- benchmark('window_serie/fill', () => window.fill(windowValue))
297
- benchmark('window_serie/swap', () => window.swap(0, 31))
298
- benchmark('window_serie/copy_from', () => window.copyFrom(windowSource))
299
- benchmark('window_serie/splice', () => window.splice(4, 8, windowRows))
300
- benchmark('window_serie/as_sorted', () => window.asSorted())
301
- benchmark('window_serie/as_reversed', () => window.asReversed())
302
- benchmark('window_serie/as_taken', () => window.asTaken(windowIndices))
286
+ await benchmark('window_serie/null_count', () => window.nullCount())
287
+ await benchmark('window_serie/scalar', () => window.scalar(3))
288
+ await benchmark('window_serie/rows', () => window.rows())
289
+ await benchmark('window_serie/memory_size', () => window.memorySize())
290
+ await benchmark('window_serie/is_sorted', () => window.isSorted())
291
+ await benchmark('window_serie/is_unique', () => window.isUnique())
292
+ await benchmark('window_serie/unique_count', () => window.uniqueCount())
293
+ await benchmark('window_serie/sort_indices', () => window.sortIndices())
294
+ await benchmark('window_serie/window', () => window.window(4, 8))
295
+ await benchmark('window_serie/into_serie', () => window.intoSerie())
296
+ await benchmark('window_serie/into_sorted', () => window.intoSorted(descending))
297
+ await benchmark('window_serie/into_unique', () => window.intoUnique())
298
+ await benchmark('window_serie/into_reversed', () => window.intoReversed())
299
+ await benchmark('window_serie/into_taken', () => window.intoTaken(windowIndices))
300
+ await benchmark('window_serie/into_filtered', () => window.intoFiltered(windowMask))
301
+ await benchmark('window_serie/partition_by', () => window.partitionBy(windowKeys))
302
+ await benchmark('window_serie/set', () => window.set(3, windowValue))
303
+ await benchmark('window_serie/fill', () => window.fill(windowValue))
304
+ await benchmark('window_serie/swap', () => window.swap(0, 31))
305
+ await benchmark('window_serie/copy_from', () => window.copyFrom(windowSource))
306
+ await benchmark('window_serie/splice', () => window.splice(4, 8, windowRows))
307
+ await benchmark('window_serie/as_sorted', () => window.asSorted())
308
+ await benchmark('window_serie/as_reversed', () => window.asReversed())
309
+ await benchmark('window_serie/as_taken', () => window.asTaken(windowIndices))
303
310
  // Across the chunks: chunk by chunk where a chunk can answer alone, through
304
311
  // the one join where the rows must be seen together.
305
312
  const orderChunked = ChunkedSerie.fromSeries(
306
313
  [orderPrices.slice(0, 32), orderPrices.slice(32, 32)],
307
314
  orderPrices.field,
308
315
  )
309
- benchmark('chunked_serie/sort_indices', () => orderChunked.sortIndices())
310
- benchmark('chunked_serie/is_sorted', () => orderChunked.isSorted())
311
- benchmark('chunked_serie/is_unique', () => orderChunked.isUnique())
312
- benchmark('chunked_serie/unique_count', () => orderChunked.uniqueCount())
313
- benchmark('chunked_serie/memory_size', () => orderChunked.memorySize())
314
- benchmark('chunked_serie/into_sorted', () => orderChunked.intoSorted())
315
- benchmark('chunked_serie/into_unique', () => orderChunked.intoUnique())
316
- benchmark('chunked_serie/into_reversed', () => orderChunked.intoReversed())
317
- benchmark('chunked_serie/into_taken', () => orderChunked.intoTaken(orderIndices))
318
- benchmark('chunked_serie/into_filtered', () => orderChunked.intoFiltered(orderMask))
319
- benchmark('chunked_serie/partition_by', () => orderChunked.partitionBy(orderVenues))
316
+ await benchmark('chunked_serie/sort_indices', () => orderChunked.sortIndices())
317
+ await benchmark('chunked_serie/is_sorted', () => orderChunked.isSorted())
318
+ await benchmark('chunked_serie/is_unique', () => orderChunked.isUnique())
319
+ await benchmark('chunked_serie/unique_count', () => orderChunked.uniqueCount())
320
+ await benchmark('chunked_serie/memory_size', () => orderChunked.memorySize())
321
+ await benchmark('chunked_serie/into_sorted', () => orderChunked.intoSorted())
322
+ await benchmark('chunked_serie/into_unique', () => orderChunked.intoUnique())
323
+ await benchmark('chunked_serie/into_reversed', () => orderChunked.intoReversed())
324
+ await benchmark('chunked_serie/into_taken', () => orderChunked.intoTaken(orderIndices))
325
+ await benchmark('chunked_serie/into_filtered', () => orderChunked.intoFiltered(orderMask))
326
+ await benchmark('chunked_serie/partition_by', () => orderChunked.partitionBy(orderVenues))
320
327
  const orderChunkedVenues = ChunkedSerie.fromSeries(
321
328
  [orderVenues.slice(0, 32), orderVenues.slice(32, 32)],
322
329
  orderVenues.field,
323
330
  )
324
- benchmark('chunked_serie/partition_by_chunked_keys', () =>
331
+ await benchmark('chunked_serie/partition_by_chunked_keys', () =>
325
332
  orderChunked.partitionBy(orderChunkedVenues),
326
333
  )
327
334
  const chunkedInPlace = orderChunked.intoSorted()
328
- benchmark('chunked_serie/as_sorted', () => chunkedInPlace.asSorted())
329
- benchmark('chunked_serie/as_unique', () => chunkedInPlace.asUnique())
330
- benchmark('chunked_serie/as_reversed', () => chunkedInPlace.asReversed())
335
+ await benchmark('chunked_serie/as_sorted', () => chunkedInPlace.asSorted())
336
+ await benchmark('chunked_serie/as_unique', () => chunkedInPlace.asUnique())
337
+ await benchmark('chunked_serie/as_reversed', () => chunkedInPlace.asReversed())
331
338
  const chunkedTakenInPlace = orderChunked.intoSorted()
332
- benchmark('chunked_serie/as_taken', () => chunkedTakenInPlace.asTaken(orderIndices))
339
+ await benchmark('chunked_serie/as_taken', () => chunkedTakenInPlace.asTaken(orderIndices))
333
340
  const chunkedFilteredInPlace = orderChunked.intoSorted()
334
- benchmark('chunked_serie/as_filtered', () => chunkedFilteredInPlace.asFiltered(orderMask))
341
+ await benchmark('chunked_serie/as_filtered', () => chunkedFilteredInPlace.asFiltered(orderMask))
335
342
  // Windows by key over 64 quotes a minute apart, their venues in sorted runs
336
343
  // of 16: held windows and their records, chunks, and a stream drained window
337
344
  // by window. A lent window's record is read through the windows of its call,
@@ -347,135 +354,128 @@ const tickedChunked = ChunkedSerie.fromSeries(
347
354
  [ticked.slice(0, 32), ticked.slice(32, 32)],
348
355
  ticked.field,
349
356
  )
350
- const lent = ticked.windowBy('venue')[1][1]
351
- const lentLast = ticked.windowBy('minutes(ts, 1)').at(-1)[1]
352
357
  function drained(windows) {
353
358
  let rows = 0
354
- for (const window of windows) {
355
- for (const piece of window) rows += piece.length
356
- }
359
+ for (const window of windows) rows += window.rows.length
357
360
  return rows
358
361
  }
359
362
  // The rows of every partition a stream is cut into, drained pair by pair.
360
363
  function partitionRows(partitions) {
361
364
  let rows = 0
362
- for (const [, chunked] of partitions) rows += chunked.length
365
+ for (const partition of partitions) rows += partition.rows.length
363
366
  return rows
364
367
  }
365
368
  if (
366
- partitionRows(SerieReader.fromChunked(tickedChunked).partitionBy('venue')) !== orderRows ||
369
+ partitionRows(tickedChunked.partitionBy('venue')) !== orderRows ||
367
370
  ticked.windowBy('venue').length !== 4 ||
368
371
  tickedChunked.windowBy('venue').length !== 4 ||
369
- drained(SerieReader.fromSerie(ticked).windowBy('venue', true)) !== orderRows ||
370
- lentLast.staticValues.get('windownum').asJs() !== orderRows - 1
372
+ drained(ticked.windowBy('venue', true)) !== orderRows
371
373
  ) {
372
374
  throw new Error('the window_by fixtures cut what they claim')
373
375
  }
374
- benchmark('serie/window_by_period', () => ticked.windowBy('minutes(ts, 15)'))
375
- benchmark('serie/window_by_in_order', () => ticked.windowBy('venue'))
376
- benchmark('serie/window_by_sorted_gathered', () => orderQuotes.windowBy('venue', true))
377
- benchmark('window_serie/window_by', () => ticked.window(8, 32).windowBy('venue'))
378
- benchmark('window_serie/window_by_lent', () => lent.windowBy('minutes(ts, 15)'))
379
- benchmark('window_serie/static_values', () => lent.staticValues)
380
- benchmark('window_serie/static_values_last_of_64', () => lentLast.staticValues)
381
- benchmark('chunked_serie/window_by', () => tickedChunked.windowBy('venue'))
382
- benchmark('chunked_serie/window_by_sorted', () => tickedChunked.windowBy('venue', true))
383
- benchmark('serie_reader/window_by_drained', () =>
384
- drained(SerieReader.fromSerie(ticked).windowBy('venue')),
376
+ await benchmark('serie/window_by_period', () => ticked.windowBy('minutes(ts, 15)'))
377
+ await benchmark('serie/window_by_in_order', () => ticked.windowBy('venue'))
378
+ await benchmark('serie/window_by_sorted_gathered', () => orderQuotes.windowBy('venue', true))
379
+ await benchmark('window_serie/window_by', () => ticked.window(8, 32).windowBy('venue'))
380
+ await benchmark('chunked_serie/window_by', () => tickedChunked.windowBy('venue'))
381
+ await benchmark('chunked_serie/window_by_sorted', () => tickedChunked.windowBy('venue', true))
382
+ await benchmark('serie/window_by_drained', () =>
383
+ drained(ticked.windowBy('venue')),
385
384
  )
386
- benchmark('serie_reader/window_by_sorted_drained', () =>
387
- drained(SerieReader.fromSerie(ticked).windowBy('venue', true)),
385
+ await benchmark('serie/window_by_sorted_drained', () =>
386
+ drained(ticked.windowBy('venue', true)),
388
387
  )
389
388
  // A stream of two chunks cut by venue: every partition held to the end and
390
389
  // closed in key order, then one open at a time on one thread, each closed as
391
390
  // the next venue arrives.
392
- benchmark('serie_reader/partition_by_drained', () =>
393
- partitionRows(SerieReader.fromChunked(tickedChunked).partitionBy('venue')),
394
- )
395
- benchmark('serie_reader/partition_by_bounded_drained', () =>
396
- partitionRows(
397
- SerieReader.fromChunked(tickedChunked).partitionBy('venue', { maxOpen: 1, threads: 1 }),
398
- ),
391
+ await benchmark('chunked_serie/partition_by_drained', () =>
392
+ partitionRows(tickedChunked.partitionBy('venue')),
399
393
  )
400
- benchmark('schema/map_of', () => fields.mapOf('labels', 'utf8', 'int32'))
401
- benchmark('schema/time_infer_time32', () => DataType.time('ms'))
402
- benchmark('schema/time_infer_time64', () => DataType.time('ns'))
403
- benchmark('schema/variant_dense_2', () => DataType.variant([id, name]))
404
- benchmark('schema/fixed_ascii', () => DataType.fixedAscii(3))
405
- benchmark('schema/string_parameters', () =>
394
+ await benchmark('schema/map_of', () => fields.mapOf('labels', 'utf8', 'int32'))
395
+ await benchmark('schema/time_infer_time32', () => DataType.time('ms'))
396
+ await benchmark('schema/time_infer_time64', () => DataType.time('ns'))
397
+ await benchmark('schema/variant_dense_2', () => DataType.variant([id, name]))
398
+ await benchmark('schema/fixed_ascii', () => DataType.fixedAscii(3))
399
+ await benchmark('schema/string_parameters', () =>
406
400
  DataType.string({ charset: 'windows-1252', max: 32 }),
407
401
  )
408
- benchmark('schema/bytes_parameters', () => DataType.bytes({ max: 16 }))
409
- benchmark('schema/string_parameters_get', () => latin.stringParameters)
410
- benchmark('schema/fixed_byte_width', () => tenor.fixedByteWidth)
411
- benchmark('schema/ccy', () => DataType.from('ccy'))
412
- benchmark('schema/ccy_field', () => fields.ccy('ccy'))
413
- benchmark('schema/fixed_ascii_field', () => fields.fixedAscii('tenor', 8))
414
- benchmark('schema/string_field', () =>
402
+ await benchmark('schema/bytes_parameters', () => DataType.bytes({ max: 16 }))
403
+ await benchmark('schema/string_parameters_get', () => latin.stringParameters)
404
+ await benchmark('schema/fixed_byte_width', () => tenor.fixedByteWidth)
405
+ await benchmark('schema/ccy', () => DataType.from('ccy'))
406
+ await benchmark('schema/ccy_field', () => fields.ccy('ccy'))
407
+ await benchmark('schema/fixed_ascii_field', () => fields.fixedAscii('tenor', 8))
408
+ await benchmark('schema/string_field', () =>
415
409
  fields.string('note', { charset: 'windows-1252', max: 32 }),
416
410
  )
417
- benchmark('schema/bytes_field', () => fields.bytes('payload', { max: 16 }))
418
- benchmark('schema/time_field_infer_time32', () => fields.time('clock', 'ms'))
419
- benchmark('schema/time_field_infer_time64', () => fields.time('clock', 'ns'))
420
- benchmark('schema/from_json', () => DataType.fromJSON(structuralJson))
421
- benchmark('schema/into_field_native', () => intoField(rowField))
422
- benchmark('schema/into_field_class_cached', () => intoField(BenchRow))
423
- benchmark('schema/into_field_renamed', () => intoField(BenchRow, 'row'))
424
- benchmark('schema/metadata_ignored_equals', () => struct.equals(struct, false))
425
- benchmark('term/equals', () => term.equals(term))
426
- benchmark('term/compare', () => term.compare(term))
427
- benchmark('term/stable_hash', () => term.stableHash())
428
- benchmark('term/clone', () => term.clone())
429
- benchmark('expression/equals', () => expression.equals(expression))
430
- benchmark('expression/compare', () => expression.compare(expression))
431
- benchmark('expression/stable_hash', () => expression.stableHash())
432
- benchmark('expression/clone', () => expression.clone())
433
- benchmark('plan/equals', () => plan.equals(plan))
434
- benchmark('plan/compare', () => plan.compare(plan))
435
- benchmark('plan/stable_hash', () => plan.stableHash())
436
- benchmark('plan/clone', () => plan.clone())
437
- benchmark('iceberg/partition_spec_equals', () => icebergSpec.equals(icebergSpec))
438
- benchmark('iceberg/partition_spec_compare', () => icebergSpec.compare(icebergSpec))
439
- benchmark('iceberg/partition_spec_stable_hash', () => icebergSpec.stableHash())
440
- benchmark('iceberg/partition_spec_clone', () => icebergSpec.clone())
441
- benchmark('iceberg/partition_spec_from_json', () =>
411
+ await benchmark('schema/bytes_field', () => fields.bytes('payload', { max: 16 }))
412
+ await benchmark('schema/time_field_infer_time32', () => fields.time('clock', 'ms'))
413
+ await benchmark('schema/time_field_infer_time64', () => fields.time('clock', 'ns'))
414
+ await benchmark('schema/from_json', () => DataType.fromJSON(structuralJson))
415
+ await benchmark('schema/into_field_native', () => intoField(rowField))
416
+ await benchmark('schema/into_field_class_cached', () => intoField(BenchRow))
417
+ await benchmark('schema/into_field_renamed', () => intoField(BenchRow, 'row'))
418
+ await benchmark('schema/metadata_ignored_equals', () => struct.equals(struct, false))
419
+ await benchmark('term/equals', () => term.equals(term))
420
+ await benchmark('term/compare', () => term.compare(term))
421
+ await benchmark('term/stable_hash', () => term.stableHash())
422
+ await benchmark('term/clone', () => term.clone())
423
+ await benchmark('expression/equals', () => expression.equals(expression))
424
+ await benchmark('expression/compare', () => expression.compare(expression))
425
+ await benchmark('expression/stable_hash', () => expression.stableHash())
426
+ await benchmark('expression/clone', () => expression.clone())
427
+ await benchmark('plan/equals', () => plan.equals(plan))
428
+ await benchmark('plan/compare', () => plan.compare(plan))
429
+ await benchmark('plan/stable_hash', () => plan.stableHash())
430
+ await benchmark('plan/clone', () => plan.clone())
431
+ await benchmark('iceberg/partition_spec_equals', () => icebergSpec.equals(icebergSpec))
432
+ await benchmark('iceberg/partition_spec_compare', () => icebergSpec.compare(icebergSpec))
433
+ await benchmark('iceberg/partition_spec_stable_hash', () => icebergSpec.stableHash())
434
+ await benchmark('iceberg/partition_spec_clone', () => icebergSpec.clone())
435
+ await benchmark('iceberg/partition_spec_from_json', () =>
442
436
  icebergApi.PartitionSpec.fromJSON(icebergSpecDocument),
443
437
  )
444
- benchmark('iceberg/partition_spec_into_json', () => icebergSpec.intoJSON())
445
- benchmark('schema/diff_first_wide_struct_1024', () =>
438
+ await benchmark('iceberg/partition_spec_into_json', () => icebergSpec.intoJSON())
439
+ await benchmark('schema/diff_first_wide_struct_1024', () =>
446
440
  wideLeft.showDiffs(wideRight, false).next(),
447
441
  )
448
- benchmark('schema/protocol_view', () => property.iceberg)
449
- benchmark('schema/protocol_view_get', () => iceberg.get('doc'))
450
- benchmark('schema/protocol_property_get', () =>
442
+ await benchmark('schema/protocol_view', () => property.iceberg)
443
+ await benchmark('schema/protocol_view_get', () => iceberg.get('doc'))
444
+ await benchmark('schema/protocol_property_get', () =>
451
445
  property.getProperty('iceberg', 'doc'),
452
446
  )
453
- benchmark('schema/protocol_view_set', () => iceberg.set('doc', 'closing price'))
454
- benchmark('schema/protocol_view_entries', () => iceberg.entries())
455
- benchmark('schema/partition_field_names', () => partitioned.partitionFieldNames())
447
+ await benchmark('schema/protocol_view_set', () => iceberg.set('doc', 'closing price'))
448
+ await benchmark('schema/protocol_view_entries', () => iceberg.entries())
449
+ await benchmark('schema/partition_field_names', () => partitioned.partitionFieldNames())
456
450
  // The `by` declarations: a partition declaration materialized on a struct
457
451
  // root, read back, and the typed list on a protocol view.
458
452
  const declaring = Field.from('row: struct<venue: utf8 not null, ts: timestamp(us) not null> not null')
459
453
  const declared = declaring.withPartitionBy(['venue', 'days(ts)'])
460
454
  const sortView = declaring.clone().sort
461
455
  sortView.by = ['venue', 'ts desc']
462
- benchmark('schema/with_partition_by', () => declaring.withPartitionBy(['venue', 'days(ts)']))
463
- benchmark('schema/partition_by', () => declared.partitionBy())
464
- benchmark('schema/protocol_by_get', () => sortView.by)
465
- benchmark('schema/protocol_by_set', () => {
456
+ await benchmark('schema/with_partition_by', () => declaring.withPartitionBy(['venue', 'days(ts)']))
457
+ await benchmark('schema/partition_by', () => declared.partitionBy())
458
+ await benchmark('schema/protocol_by_get', () => sortView.by)
459
+ await benchmark('schema/protocol_by_set', () => {
466
460
  sortView.by = ['venue', 'ts desc']
467
461
  })
468
- benchmark('schema/protocol_remove_by', () => declared.clone().partition.removeBy())
469
- benchmark('schema/without_partition_fields', () =>
462
+ await benchmark('schema/protocol_remove_by', () => declared.clone().partition.removeBy())
463
+ await benchmark('schema/without_partition_fields', () =>
470
464
  partitioned.withoutPartitionFields(),
471
465
  )
472
- benchmark('schema/string_vocabulary_prebuilt', () =>
466
+ await benchmark('schema/string_vocabulary_prebuilt', () =>
473
467
  StringEnum.fromLogicalName('ccy'),
474
468
  )
475
- benchmark('schema/string_vocabulary_enum', () => ccys.intoEnum('ccy'))
476
- benchmark('schema/mime_known_parse', () => MimeType.fromString(knownMime))
477
- benchmark('schema/mime_custom_parse', () => MimeType.fromString(customMime))
478
- benchmark('schema/media_compound_parse', () => MediaType.fromString(compoundMedia))
479
- benchmark('schema/media_header_inference', () =>
469
+ await benchmark('schema/string_vocabulary_enum', () => ccys.intoEnum('ccy'))
470
+ await benchmark('schema/mime_known_parse', () => MimeType.fromString(knownMime))
471
+ await benchmark('schema/mime_custom_parse', () => MimeType.fromString(customMime))
472
+ await benchmark('schema/media_compound_parse', () => MediaType.fromString(compoundMedia))
473
+ await benchmark('schema/media_header_inference', () =>
480
474
  MediaType.fromContentHeaders(contentType, contentEncoding),
481
475
  )
476
+ }
477
+
478
+ main().catch((error) => {
479
+ console.error(error)
480
+ process.exitCode = 1
481
+ })