yggio 0.0.16 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,386 +12,379 @@ const {
12
12
  MimeType,
13
13
  RecordOptions,
14
14
  Serie,
15
- SerieReader,
16
15
  fields,
17
16
  iceberg,
18
17
  } = require('yggio')
19
18
 
20
- // A record crossing is far more work than a schema lookup, so this target
21
- // counts in thousands rather than tens of thousands by default. It also keeps
22
- // Parquet footer and projected-WKB Scalar crossings beside the record surface.
23
- // Smoke mode keeps fixtures and loops small enough for package verification.
24
- const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
25
- const iterations = Number.parseInt(
26
- smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '200'),
27
- 10,
28
- )
29
- if (!Number.isSafeInteger(iterations) || iterations <= 0) {
30
- throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
31
- }
32
- const warmups = smoke ? 0 : Math.min(iterations, 20)
33
- const filter = process.env.YGGIO_BENCH_FILTER
19
+ async function main() {
34
20
 
35
- function selected(name) {
36
- return filter === undefined || name.includes(filter)
37
- }
21
+ // A record crossing is far more work than a schema lookup, so this target
22
+ // counts in thousands rather than tens of thousands by default. It also keeps
23
+ // Parquet footer and projected-WKB Scalar crossings beside the record surface.
24
+ // Smoke mode keeps fixtures and loops small enough for package verification.
25
+ const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
26
+ const iterations = Number.parseInt(
27
+ smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '200'),
28
+ 10,
29
+ )
30
+ if (!Number.isSafeInteger(iterations) || iterations <= 0) {
31
+ throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
32
+ }
33
+ const warmups = smoke ? 0 : Math.min(iterations, 20)
34
+ const filter = process.env.YGGIO_BENCH_FILTER
38
35
 
39
- function benchmark(name, operation) {
40
- if (!selected(name)) return
41
- for (let index = 0; index < warmups; index += 1) operation()
42
- const started = performance.now()
43
- for (let index = 0; index < iterations; index += 1) operation()
44
- const elapsed = performance.now() - started
45
- const rate = Math.round((iterations * 1_000) / elapsed)
46
- console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
47
- }
36
+ function selected(name) {
37
+ return filter === undefined || name.includes(filter)
38
+ }
48
39
 
49
- const rows = smoke ? 64 : 10_000
50
- const writeBatchSize = smoke ? 32 : 1_024
51
- // A cadence counts whole batches, so the committed paths are fed batches small
52
- // enough that one publication spans several: 100-row batches, ten to a
53
- // publication, so 10,000 rows publish ten times.
54
- const commitBatchRowSize = smoke ? 8 : 100
55
- const commitBatchNum = smoke ? 2 : 10
56
- // A plain array of bigints, not a `BigInt64Array`: Apache Arrow JS reads a
57
- // typed array as a column that cannot be null, and the declared root below says
58
- // every column can be.
59
- const ids = new Array(rows)
60
- const symbols = new Array(rows)
61
- const venues = new Array(rows)
62
- for (let index = 0; index < rows; index += 1) {
63
- ids[index] = BigInt(index)
64
- symbols[index] = index % 2 === 0 ? 'AAPL' : 'MSFT'
65
- venues[index] = index % 3 === 0 ? 'XNAS' : 'XNYS'
66
- }
67
-
68
- // Every fixture is built once, outside the measured loops: the numbers report
69
- // the boundary crossing, not Apache Arrow JS building a column.
70
- const table = new arrow.Table({
71
- id: arrow.vectorFromArray(ids, new arrow.Int64()),
72
- symbol: arrow.vectorFromArray(symbols, new arrow.Utf8()),
73
- venue: arrow.vectorFromArray(venues, new arrow.Utf8()),
74
- })
75
- const ipc = arrow.tableToIPC(table)
76
- const commitBatches = []
77
- for (let start = 0; start < rows; start += commitBatchRowSize) {
78
- commitBatches.push(...table.slice(start, start + commitBatchRowSize).batches)
79
- }
80
- const commitIpc = arrow.tableToIPC(new arrow.Table(commitBatches))
81
- const schema = fields.struct(
82
- 'row',
83
- [Field.from('id: int64'), Field.from('symbol: utf8'), Field.from('venue: utf8')],
84
- { nullable: false },
85
- )
86
- const wanted = fields.struct('row', [Field.from('id: int64')], { nullable: false })
40
+ async function benchmark(name, operation) {
41
+ if (!selected(name)) return
42
+ for (let index = 0; index < warmups; index += 1) {
43
+ const result = operation()
44
+ if (result instanceof Promise) await result
45
+ }
46
+ const started = performance.now()
47
+ for (let index = 0; index < iterations; index += 1) {
48
+ const result = operation()
49
+ if (result instanceof Promise) await result
50
+ }
51
+ const elapsed = performance.now() - started
52
+ const rate = Math.round((iterations * 1_000) / elapsed)
53
+ console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
54
+ }
87
55
 
88
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-bench-'))
89
- const stream = new IOBase(path.join(root, 'trades.arrows'))
90
- stream.overwriteArrowReader(BatchReader.from(ipc))
91
- const parquet = new IOBase(path.join(root, 'trades.parquet'))
92
- parquet.overwriteArrowTable(table, parquet.recordOptions().withField(schema))
93
- const wkbPoint = Buffer.allocUnsafe(21)
94
- wkbPoint.writeUInt8(1, 0)
95
- wkbPoint.writeUInt32LE(1, 1)
96
- wkbPoint.writeDoubleLE(1, 5)
97
- wkbPoint.writeDoubleLE(2, 13)
98
- const geospatial = new IOBase(path.join(root, 'shapes.parquet'))
99
- geospatial.overwriteArrowTable(
100
- new arrow.Table({
101
- shape: arrow.vectorFromArray(new Array(rows).fill(wkbPoint), new arrow.Binary()),
102
- }),
103
- )
56
+ const rows = smoke ? 64 : 10_000
57
+ const writeBatchSize = smoke ? 32 : 1_024
58
+ // A cadence counts whole batches, so the committed paths are fed batches small
59
+ // enough that one publication spans several: 100-row batches, ten to a
60
+ // publication, so 10,000 rows publish ten times.
61
+ const commitBatchRowSize = smoke ? 8 : 100
62
+ const commitBatchNum = smoke ? 2 : 10
63
+ // A plain array of bigints, not a `BigInt64Array`: Apache Arrow JS reads a
64
+ // typed array as a column that cannot be null, and the declared root below says
65
+ // every column can be.
66
+ const ids = new Array(rows)
67
+ const symbols = new Array(rows)
68
+ const venues = new Array(rows)
69
+ for (let index = 0; index < rows; index += 1) {
70
+ ids[index] = BigInt(index)
71
+ symbols[index] = index % 2 === 0 ? 'AAPL' : 'MSFT'
72
+ venues[index] = index % 3 === 0 ? 'XNAS' : 'XNYS'
73
+ }
104
74
 
105
- const memory = IOBase.fromBytes(stream.readBytes())
106
- memory.mediaType = MimeType.ARROW_STREAM
75
+ // Every fixture is built once, outside the measured loops: the numbers report
76
+ // the boundary crossing, not Apache Arrow JS building a column.
77
+ const table = new arrow.Table({
78
+ id: arrow.vectorFromArray(ids, new arrow.Int64()),
79
+ symbol: arrow.vectorFromArray(symbols, new arrow.Utf8()),
80
+ venue: arrow.vectorFromArray(venues, new arrow.Utf8()),
81
+ })
82
+ const ipc = arrow.tableToIPC(table)
83
+ const commitBatches = []
84
+ for (let start = 0; start < rows; start += commitBatchRowSize) {
85
+ commitBatches.push(...table.slice(start, start + commitBatchRowSize).batches)
86
+ }
87
+ const commitIpc = arrow.tableToIPC(new arrow.Table(commitBatches))
88
+ const schema = fields.struct(
89
+ 'row',
90
+ [Field.from('id: int64'), Field.from('symbol: utf8'), Field.from('venue: utf8')],
91
+ { nullable: false },
92
+ )
93
+ const wanted = fields.struct('row', [Field.from('id: int64')], { nullable: false })
107
94
 
108
- benchmark('records/reader_from_ipc', () => BatchReader.fromIpc(ipc))
109
- benchmark('records/reader_from_arrow_table', () => BatchReader.from(table))
110
- benchmark('records/record_options', () => memory.recordOptions())
111
- const protocolOptions = memory.recordOptions().withField(schema).withBatchRowSize(writeBatchSize)
112
- benchmark('records/record_options_equals', () => protocolOptions.equals(protocolOptions))
113
- benchmark('records/record_options_compare', () => protocolOptions.compare(protocolOptions))
114
- benchmark('records/record_options_stable_hash', () => protocolOptions.stableHash())
115
- benchmark('records/record_options_clone', () => protocolOptions.clone())
116
- const avroOptions = RecordOptions.from('benchmark.avro')
117
- const avroSyncMarker = Buffer.from('0123456789abcdef')
118
- benchmark('records/avro_option_values', () => [
119
- avroOptions.blockCodec,
120
- avroOptions.syncMarker,
121
- ])
122
- benchmark('records/avro_option_setters', () => {
123
- avroOptions.blockCodec = 'deflate'
124
- avroOptions.syncMarker = avroSyncMarker
125
- })
126
- benchmark('records/read_field_ipc', () => memory.readArrowField())
127
- benchmark('records/read_parquet_statistics', () => parquet.readParquetStatistics())
128
- benchmark('records/read_parquet_geospatial_statistics', () =>
129
- geospatial.readParquetGeospatialStatistics('shape'),
130
- )
131
- const memoryWanted = memory.recordOptions().withField(wanted)
132
- const parquetWanted = parquet.recordOptions().withField(wanted)
133
- benchmark('records/read_ipc_into_ipc', () => memory.readArrowReader().intoIpc())
134
- benchmark('records/read_ipc_into_arrow', () => memory.readArrowReader().intoTable())
135
- benchmark('records/read_ipc_pushdown', () => memory.readArrowReader(memoryWanted).intoIpc())
136
- benchmark('records/read_parquet_into_ipc', () => parquet.readArrowReader().intoIpc())
137
- benchmark('records/read_parquet_pushdown', () => parquet.readArrowReader(parquetWanted).intoIpc())
95
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-bench-'))
96
+ const stream = new IOBase(path.join(root, 'trades.arrows'))
97
+ await stream.overwriteArrowReader(BatchReader.from(ipc))
98
+ const parquet = new IOBase(path.join(root, 'trades.parquet'))
99
+ await parquet.overwriteArrowTable(table, parquet.recordOptions().withField(schema))
100
+ const wkbPoint = Buffer.allocUnsafe(21)
101
+ wkbPoint.writeUInt8(1, 0)
102
+ wkbPoint.writeUInt32LE(1, 1)
103
+ wkbPoint.writeDoubleLE(1, 5)
104
+ wkbPoint.writeDoubleLE(2, 13)
105
+ const geospatial = new IOBase(path.join(root, 'shapes.parquet'))
106
+ await geospatial.overwriteArrowTable(
107
+ new arrow.Table({
108
+ shape: arrow.vectorFromArray(new Array(rows).fill(wkbPoint), new arrow.Binary()),
109
+ }),
110
+ )
138
111
 
139
- // Each explicit representation adapter is benchmarked under every intent. A
140
- // fresh in-memory handle keeps append/merge iterations bounded instead of
141
- // growing one benchmark fixture forever.
142
- const records = new Array(rows)
143
- for (let index = 0; index < rows; index += 1) {
144
- records[index] = { id: ids[index], symbol: symbols[index], venue: venues[index] }
145
- }
146
- const storedBytes = stream.readBytes()
147
- function stored() {
148
- const handle = IOBase.fromBytes(storedBytes)
149
- handle.mediaType = MimeType.ARROW_STREAM
150
- return handle
151
- }
152
- function keyed(handle) {
153
- return handle.recordOptions().withMergeBy(['id'])
154
- }
155
- function committed(handle, intent) {
156
- let options = handle
157
- .recordOptions()
158
- .withField(schema)
159
- .withBatchRowSize(commitBatchRowSize)
160
- .withCommitBatchNum(commitBatchNum)
161
- if (intent === 'merge') options = options.withMergeBy(['id'])
162
- return options
163
- }
112
+ const memory = IOBase.fromBytes(stream.readBytes())
113
+ memory.mediaType = MimeType.ARROW_STREAM
164
114
 
165
- benchmark('records/overwrite_arrow_reader', () =>
166
- stored().overwriteArrowReader(BatchReader.fromIpc(ipc)),
167
- )
168
- benchmark('records/append_arrow_reader', () =>
169
- stored().appendArrowReader(BatchReader.fromIpc(ipc)),
170
- )
171
- benchmark('records/merge_arrow_reader', () => {
172
- const handle = stored()
173
- handle.mergeArrowReader(BatchReader.fromIpc(ipc), keyed(handle))
174
- })
175
- benchmark('records/overwrite_arrow_reader_commit_batches', () => {
176
- const handle = stored()
177
- handle.overwriteArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'overwrite'))
178
- })
179
- benchmark('records/append_arrow_reader_commit_batches', () => {
180
- const handle = stored()
181
- handle.appendArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'append'))
182
- })
183
- benchmark('records/merge_arrow_reader_commit_batches', () => {
184
- const handle = stored()
185
- handle.mergeArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'merge'))
186
- })
115
+ await benchmark('records/reader_from_ipc', () => BatchReader.fromIpc(ipc))
116
+ await benchmark('records/reader_from_arrow_table', () => BatchReader.from(table))
117
+ await benchmark('records/record_options', () => memory.recordOptions())
118
+ const protocolOptions = memory.recordOptions().withField(schema).withBatchRowSize(writeBatchSize)
119
+ await benchmark('records/record_options_equals', () => protocolOptions.equals(protocolOptions))
120
+ await benchmark('records/record_options_compare', () => protocolOptions.compare(protocolOptions))
121
+ await benchmark('records/record_options_stable_hash', () => protocolOptions.stableHash())
122
+ await benchmark('records/record_options_clone', () => protocolOptions.clone())
123
+ const avroOptions = RecordOptions.from('benchmark.avro')
124
+ const avroSyncMarker = Buffer.from('0123456789abcdef')
125
+ await benchmark('records/avro_option_values', () => [
126
+ avroOptions.blockCodec,
127
+ avroOptions.syncMarker,
128
+ ])
129
+ await benchmark('records/avro_option_setters', () => {
130
+ avroOptions.blockCodec = 'deflate'
131
+ avroOptions.syncMarker = avroSyncMarker
132
+ })
133
+ await benchmark('records/read_field_ipc', () => memory.readArrowField())
134
+ await benchmark('records/read_parquet_statistics', () => parquet.readParquetStatistics())
135
+ await benchmark('records/read_parquet_geospatial_statistics', () =>
136
+ geospatial.readParquetGeospatialStatistics('shape'),
137
+ )
138
+ const memoryWanted = memory.recordOptions().withField(wanted)
139
+ const parquetWanted = parquet.recordOptions().withField(wanted)
140
+ await benchmark('records/read_ipc_into_ipc', () => memory.readArrowReader().intoIpc())
141
+ await benchmark('records/read_ipc_into_arrow', () => memory.readArrowReader().intoTable())
142
+ await benchmark('records/read_ipc_pushdown', () => memory.readArrowReader(memoryWanted).intoIpc())
143
+ await benchmark('records/read_parquet_into_ipc', () => parquet.readArrowReader().intoIpc())
144
+ await benchmark('records/read_parquet_pushdown', () => parquet.readArrowReader(parquetWanted).intoIpc())
187
145
 
188
- benchmark('records/overwrite_arrow_table', () => stored().overwriteArrowTable(table))
189
- benchmark('records/append_arrow_table', () => stored().appendArrowTable(table))
190
- benchmark('records/merge_arrow_table', () => {
191
- const handle = stored()
192
- handle.mergeArrowTable(table, keyed(handle))
193
- })
146
+ // Each explicit representation adapter is benchmarked under every intent. A
147
+ // fresh in-memory handle keeps append/merge iterations bounded instead of
148
+ // growing one benchmark fixture forever.
149
+ const records = new Array(rows)
150
+ for (let index = 0; index < rows; index += 1) {
151
+ records[index] = { id: ids[index], symbol: symbols[index], venue: venues[index] }
152
+ }
153
+ const storedBytes = stream.readBytes()
154
+ function stored() {
155
+ const handle = IOBase.fromBytes(storedBytes)
156
+ handle.mediaType = MimeType.ARROW_STREAM
157
+ return handle
158
+ }
159
+ function keyed(handle) {
160
+ return handle.recordOptions().withMergeBy(['id'])
161
+ }
162
+ function committed(handle, intent) {
163
+ let options = handle
164
+ .recordOptions()
165
+ .withField(schema)
166
+ .withBatchRowSize(commitBatchRowSize)
167
+ .withCommitBatchNum(commitBatchNum)
168
+ if (intent === 'merge') options = options.withMergeBy(['id'])
169
+ return options
170
+ }
194
171
 
195
- benchmark('records/overwrite_arrow_batch', () =>
196
- stored().overwriteArrowBatch(table.batches[0]),
197
- )
198
- benchmark('records/append_arrow_batch', () =>
199
- stored().appendArrowBatch(table.batches[0]),
200
- )
201
- benchmark('records/merge_arrow_batch', () => {
202
- const handle = stored()
203
- handle.mergeArrowBatch(table.batches[0], keyed(handle))
204
- })
172
+ await benchmark('records/overwrite_arrow_reader', () =>
173
+ stored().overwriteArrowReader(BatchReader.fromIpc(ipc)),
174
+ )
175
+ await benchmark('records/append_arrow_reader', () =>
176
+ stored().appendArrowReader(BatchReader.fromIpc(ipc)),
177
+ )
178
+ await benchmark('records/merge_arrow_reader', async () => {
179
+ const handle = stored()
180
+ await handle.mergeArrowReader(BatchReader.fromIpc(ipc), keyed(handle))
181
+ })
182
+ await benchmark('records/overwrite_arrow_reader_commit_batches', async () => {
183
+ const handle = stored()
184
+ await handle.overwriteArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'overwrite'))
185
+ })
186
+ await benchmark('records/append_arrow_reader_commit_batches', async () => {
187
+ const handle = stored()
188
+ await handle.appendArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'append'))
189
+ })
190
+ await benchmark('records/merge_arrow_reader_commit_batches', async () => {
191
+ const handle = stored()
192
+ await handle.mergeArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'merge'))
193
+ })
205
194
 
206
- benchmark('records/overwrite_records', () => stored().overwriteRecords(records))
207
- function* recordGenerator() {
208
- yield* records
209
- }
210
- benchmark('records/overwrite_record_generator', () =>
211
- stored().overwriteRecords(recordGenerator()),
212
- )
213
- benchmark('records/overwrite_record_generator_commit_batches', () => {
214
- const handle = stored()
215
- handle.overwriteRecords(recordGenerator(), committed(handle, 'overwrite'))
216
- })
217
- benchmark('records/append_record_generator_commit_batches', () => {
218
- const handle = stored()
219
- handle.appendRecords(recordGenerator(), committed(handle, 'append'))
220
- })
221
- benchmark('records/merge_record_generator_commit_batches', () => {
222
- const handle = stored()
223
- handle.mergeRecords(recordGenerator(), committed(handle, 'merge'))
224
- })
225
- benchmark('records/append_records', () => stored().appendRecords(records))
226
- benchmark('records/merge_records', () => {
227
- const handle = stored()
228
- handle.mergeRecords(records, keyed(handle))
229
- })
195
+ await benchmark('records/overwrite_arrow_table', () => stored().overwriteArrowTable(table))
196
+ await benchmark('records/append_arrow_table', () => stored().appendArrowTable(table))
197
+ await benchmark('records/merge_arrow_table', async () => {
198
+ const handle = stored()
199
+ await handle.mergeArrowTable(table, keyed(handle))
200
+ })
230
201
 
231
- // The configurable surface adds only mode dispatch; keep each public input
232
- // shape visible so benchmark regressions cannot hide in a binding adapter.
233
- for (const mode of ['overwrite', 'append', 'merge']) {
234
- benchmark(`records/write_arrow_reader/${mode}`, () => {
202
+ await benchmark('records/overwrite_arrow_batch', () =>
203
+ stored().overwriteArrowBatch(table.batches[0]),
204
+ )
205
+ await benchmark('records/append_arrow_batch', () =>
206
+ stored().appendArrowBatch(table.batches[0]),
207
+ )
208
+ await benchmark('records/merge_arrow_batch', async () => {
235
209
  const handle = stored()
236
- handle.writeArrowReader(
237
- BatchReader.fromIpc(ipc),
238
- mode,
239
- mode === 'merge' ? keyed(handle) : undefined,
240
- )
210
+ await handle.mergeArrowBatch(table.batches[0], keyed(handle))
241
211
  })
242
- benchmark(`records/write_arrow_table/${mode}`, () => {
212
+
213
+ await benchmark('records/overwrite_records', () => stored().overwriteRecords(records))
214
+ function* recordGenerator() {
215
+ yield* records
216
+ }
217
+ await benchmark('records/overwrite_record_generator', () =>
218
+ stored().overwriteRecords(recordGenerator()),
219
+ )
220
+ await benchmark('records/overwrite_record_generator_commit_batches', async () => {
243
221
  const handle = stored()
244
- handle.writeArrowTable(
245
- table,
246
- mode,
247
- mode === 'merge' ? keyed(handle) : undefined,
248
- )
222
+ await handle.overwriteRecords(recordGenerator(), committed(handle, 'overwrite'))
249
223
  })
250
- benchmark(`records/write_arrow_batch/${mode}`, () => {
224
+ await benchmark('records/append_record_generator_commit_batches', async () => {
251
225
  const handle = stored()
252
- handle.writeArrowBatch(
253
- table.batches[0],
254
- mode,
255
- mode === 'merge' ? keyed(handle) : undefined,
256
- )
226
+ await handle.appendRecords(recordGenerator(), committed(handle, 'append'))
257
227
  })
258
- benchmark(`records/write_records/${mode}`, () => {
228
+ await benchmark('records/merge_record_generator_commit_batches', async () => {
259
229
  const handle = stored()
260
- handle.writeRecords(
261
- records,
262
- mode,
263
- mode === 'merge' ? keyed(handle) : undefined,
264
- )
230
+ await handle.mergeRecords(recordGenerator(), committed(handle, 'merge'))
265
231
  })
266
- }
267
-
268
- // The Serie doors: a held column crosses as the one record batch it is, with
269
- // no IPC and no copy, a stream as itself; the Arrow JS table is the IPC bridge
270
- // every other Arrow JS write pays, then the same native door.
271
- const heldRows = Serie.fromArrowBatch(table, schema)
272
- benchmark('records/read_serie', () => {
273
- for (const batch of memory.readSerie()) batch.length
274
- })
275
- for (const mode of ['overwrite', 'append', 'merge']) {
276
- benchmark(`records/write_serie/${mode}`, () => {
232
+ await benchmark('records/append_records', () => stored().appendRecords(records))
233
+ await benchmark('records/merge_records', async () => {
277
234
  const handle = stored()
278
- handle.writeSerie(heldRows, mode, mode === 'merge' ? keyed(handle) : undefined)
235
+ await handle.mergeRecords(records, keyed(handle))
279
236
  })
280
- }
281
- benchmark('records/overwrite_serie_reader', () =>
282
- stored().overwriteSerie(SerieReader.fromSerie(heldRows)),
283
- )
284
- benchmark('records/overwrite_serie_arrow_table', () => stored().overwriteSerie(table))
285
237
 
286
- // One commit writes a data file, a manifest, a manifest list, and a metadata
287
- // document, so an Iceberg append is measured separately from a plain write.
288
- const iced = iceberg.assignFieldIds(schema)
289
- const lake = path.join(root, 'lake')
290
- const iceTable = iceberg.IcebergTable.create(lake, iced)
291
- iceTable.append(ipc)
292
- const iceFile = iceTable.dataFiles()[0]
293
- const icePlan = iceTable.plan()
294
- const icePartitionField = iceberg.PartitionSpec.identity(iced, ['venue']).fields[0]
295
- const iceSnapshot = iceTable.currentSnapshot
296
- const iceManifest = iceTable.manifests()[0]
297
- iceTable.createTag('benchmark-value-protocol', iceSnapshot.snapshotId)
298
- const iceSnapshotRef = iceTable.removeRef('benchmark-value-protocol')
299
- // One live file has no compaction group, so this bounded report is produced
300
- // without another snapshot or data rewrite.
301
- const iceCompaction = iceTable.compact()
302
- const iceOptions = new iceberg.IcebergOptions({
303
- commitRetries: 4,
304
- targetFileSize: 512 * 1024 * 1024,
305
- readParallelism: 4,
306
- dataMimeType: MimeType.AVRO,
307
- })
238
+ // The configurable surface adds only mode dispatch; keep each public input
239
+ // shape visible so benchmark regressions cannot hide in a binding adapter.
240
+ for (const mode of ['overwrite', 'append', 'merge']) {
241
+ await benchmark(`records/write_arrow_reader/${mode}`, async () => {
242
+ const handle = stored()
243
+ await handle.writeArrowReader(
244
+ BatchReader.fromIpc(ipc),
245
+ mode,
246
+ mode === 'merge' ? keyed(handle) : undefined,
247
+ )
248
+ })
249
+ await benchmark(`records/write_arrow_table/${mode}`, async () => {
250
+ const handle = stored()
251
+ await handle.writeArrowTable(
252
+ table,
253
+ mode,
254
+ mode === 'merge' ? keyed(handle) : undefined,
255
+ )
256
+ })
257
+ await benchmark(`records/write_arrow_batch/${mode}`, async () => {
258
+ const handle = stored()
259
+ await handle.writeArrowBatch(
260
+ table.batches[0],
261
+ mode,
262
+ mode === 'merge' ? keyed(handle) : undefined,
263
+ )
264
+ })
265
+ await benchmark(`records/write_records/${mode}`, async () => {
266
+ const handle = stored()
267
+ await handle.writeRecords(
268
+ records,
269
+ mode,
270
+ mode === 'merge' ? keyed(handle) : undefined,
271
+ )
272
+ })
273
+ }
308
274
 
309
- // What the widened write costs at the boundary. Each sample commits into its
310
- // own fresh table, because a growing manifest list would otherwise time the
311
- // table's history rather than the write - the same rule the Python Iceberg
312
- // benchmark follows.
313
- const iceRows = Array.from({ length: 64 }, (_, index) => ({
314
- id: BigInt(index),
315
- venue: 'XNAS',
316
- price: index / 4,
317
- }))
318
- let iceBenchIndex = 0
319
- const freshIceTable = () =>
320
- iceberg.IcebergTable.create(path.join(root, 'bench', `t${iceBenchIndex++}`), iced)
321
- benchmark('iceberg/append_arrow_ipc', () => freshIceTable().append(ipc))
322
- benchmark('iceberg/append_rows', () => freshIceTable().append(iceRows))
275
+ // The Serie doors: a held column crosses as the one record batch it is, with
276
+ // no IPC and no copy, a stream as itself; the Arrow JS table is the IPC bridge
277
+ // every other Arrow JS write pays, then the same native door.
278
+ const heldRows = Serie.fromArrowBatch(table, schema)
279
+ await benchmark('records/read_serie', () => {
280
+ for (const batch of memory.readSerie()) batch.length
281
+ })
282
+ for (const mode of ['overwrite', 'append', 'merge']) {
283
+ await benchmark(`records/write_serie/${mode}`, async () => {
284
+ const handle = stored()
285
+ await handle.writeSerie(heldRows, mode, mode === 'merge' ? keyed(handle) : undefined)
286
+ })
287
+ }
288
+ await benchmark('records/overwrite_serie', () => stored().overwriteSerie(heldRows))
289
+ await benchmark('records/overwrite_serie_arrow_table', () => stored().overwriteSerie(table))
323
290
 
324
- benchmark('iceberg/scan_into_ipc', () => iceTable.scan().intoIpc())
325
- benchmark('iceberg/scan_pushdown', () =>
326
- iceTable.scan(fields.struct('row', [iced.dtype.getFieldAt(0)], { nullable: false })).intoIpc(),
327
- )
328
- benchmark('iceberg/data_files', () => iceTable.dataFiles())
329
- benchmark('iceberg/data_file_equals', () => iceFile.equals(iceFile))
330
- benchmark('iceberg/data_file_compare', () => iceFile.compare(iceFile))
331
- benchmark('iceberg/data_file_stable_hash', () => iceFile.stableHash())
332
- benchmark('iceberg/data_file_clone', () => iceFile.clone())
333
- benchmark('iceberg/scan_plan_equals', () => icePlan.equals(icePlan))
334
- benchmark('iceberg/scan_plan_compare', () => icePlan.compare(icePlan))
335
- benchmark('iceberg/scan_plan_stable_hash', () => icePlan.stableHash())
336
- benchmark('iceberg/scan_plan_clone', () => icePlan.clone())
337
- benchmark('iceberg/partition_field_equals', () => icePartitionField.equals(icePartitionField))
338
- benchmark('iceberg/partition_field_compare', () => icePartitionField.compare(icePartitionField))
339
- benchmark('iceberg/partition_field_stable_hash', () => icePartitionField.stableHash())
340
- benchmark('iceberg/partition_field_clone', () => icePartitionField.clone())
341
- benchmark('iceberg/snapshot_equals', () => iceSnapshot.equals(iceSnapshot))
342
- benchmark('iceberg/snapshot_compare', () => iceSnapshot.compare(iceSnapshot))
343
- benchmark('iceberg/snapshot_stable_hash', () => iceSnapshot.stableHash())
344
- benchmark('iceberg/snapshot_clone', () => iceSnapshot.clone())
345
- benchmark('iceberg/snapshot_ref_equals', () => iceSnapshotRef.equals(iceSnapshotRef))
346
- benchmark('iceberg/snapshot_ref_compare', () => iceSnapshotRef.compare(iceSnapshotRef))
347
- benchmark('iceberg/snapshot_ref_stable_hash', () => iceSnapshotRef.stableHash())
348
- benchmark('iceberg/snapshot_ref_clone', () => iceSnapshotRef.clone())
349
- benchmark('iceberg/manifest_file_equals', () => iceManifest.equals(iceManifest))
350
- benchmark('iceberg/manifest_file_compare', () => iceManifest.compare(iceManifest))
351
- benchmark('iceberg/manifest_file_stable_hash', () => iceManifest.stableHash())
352
- benchmark('iceberg/manifest_file_clone', () => iceManifest.clone())
353
- benchmark('iceberg/compaction_equals', () => iceCompaction.equals(iceCompaction))
354
- benchmark('iceberg/compaction_compare', () => iceCompaction.compare(iceCompaction))
355
- benchmark('iceberg/compaction_stable_hash', () => iceCompaction.stableHash())
356
- benchmark('iceberg/compaction_clone', () => iceCompaction.clone())
357
- benchmark('iceberg/options_equals', () => iceOptions.equals(iceOptions))
358
- benchmark('iceberg/options_compare', () => iceOptions.compare(iceOptions))
359
- benchmark('iceberg/options_stable_hash', () => iceOptions.stableHash())
360
- benchmark('iceberg/options_clone', () => iceOptions.clone())
361
- benchmark('iceberg/open', () => iceberg.IcebergTable.open(lake))
291
+ // One commit writes a data file, a manifest, a manifest list, and a metadata
292
+ // document, so an Iceberg append is measured separately from a plain write.
293
+ const iced = iceberg.assignFieldIds(schema)
294
+ const lake = path.join(root, 'lake')
295
+ const iceTable = iceberg.IcebergTable.create(lake, iced)
296
+ await iceTable.append(ipc)
297
+ const iceFile = iceTable.dataFiles()[0]
298
+ const icePlan = iceTable.plan()
299
+ const icePartitionField = iceberg.PartitionSpec.identity(iced, ['venue']).fields[0]
300
+ const iceSnapshot = iceTable.currentSnapshot
301
+ const iceManifest = iceTable.manifests()[0]
302
+ iceTable.createTag('benchmark-value-protocol', iceSnapshot.snapshotId)
303
+ const iceSnapshotRef = iceTable.removeRef('benchmark-value-protocol')
304
+ // One live file has no compaction group, so this bounded report is produced
305
+ // without another snapshot or data rewrite.
306
+ const iceCompaction = iceTable.compact()
307
+ const iceOptions = new iceberg.IcebergOptions({
308
+ commitRetries: 4,
309
+ targetFileSize: 512 * 1024 * 1024,
310
+ readParallelism: 4,
311
+ dataMimeType: MimeType.AVRO,
312
+ })
362
313
 
363
- // A catalog append crosses the whole boundary a caller with only rows and a
364
- // name uses: resolve the dotted name against the warehouse, locate the table
365
- // there, and commit one snapshot. The rows stay small so the number reports
366
- // that path rather than Parquet encoding. A create descends through existing
367
- // namespaces only, so the namespace is made first.
368
- const catalog = new iceberg.IcebergCatalog('lake', path.join(root, 'warehouse'))
369
- catalog.namespaces().create('bench')
370
- const catalogRows = arrow.tableToIPC(
371
- new arrow.Table({
372
- id: arrow.vectorFromArray([1n, 2n, 3n, 4n], new arrow.Int64()),
373
- }),
374
- )
375
- catalog.tables().append('bench.trades', catalogRows)
376
- benchmark('iceberg/catalog_append', () => catalog.tables().append('bench.trades', catalogRows))
314
+ // What the widened write costs at the boundary. Each sample commits into its
315
+ // own fresh table, because a growing manifest list would otherwise time the
316
+ // table's history rather than the write - the same rule the Python Iceberg
317
+ // benchmark follows.
318
+ const iceRows = Array.from({ length: 64 }, (_, index) => ({
319
+ id: BigInt(index),
320
+ venue: 'XNAS',
321
+ price: index / 4,
322
+ }))
323
+ let iceBenchIndex = 0
324
+ const freshIceTable = () =>
325
+ iceberg.IcebergTable.create(path.join(root, 'bench', `t${iceBenchIndex++}`), iced)
326
+ await benchmark('iceberg/append_arrow_ipc', () => freshIceTable().append(ipc))
327
+ await benchmark('iceberg/append_rows', () => freshIceTable().append(iceRows))
377
328
 
378
- async function benchmarkAsync(name, operation) {
379
- if (!selected(name)) return
380
- const asyncWarmups = smoke ? 0 : Math.min(iterations, 5)
381
- for (let index = 0; index < asyncWarmups; index += 1) await operation()
382
- const started = performance.now()
383
- for (let index = 0; index < iterations; index += 1) await operation()
384
- const elapsed = performance.now() - started
385
- const rate = Math.round((iterations * 1_000) / elapsed)
386
- console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
387
- }
329
+ await benchmark('iceberg/scan_into_ipc', () => iceTable.scan().intoIpc())
330
+ await benchmark('iceberg/scan_pushdown', () =>
331
+ iceTable.scan(fields.struct('row', [iced.dtype.getFieldAt(0)], { nullable: false })).intoIpc(),
332
+ )
333
+ await benchmark('iceberg/data_files', () => iceTable.dataFiles())
334
+ await benchmark('iceberg/data_file_equals', () => iceFile.equals(iceFile))
335
+ await benchmark('iceberg/data_file_compare', () => iceFile.compare(iceFile))
336
+ await benchmark('iceberg/data_file_stable_hash', () => iceFile.stableHash())
337
+ await benchmark('iceberg/data_file_clone', () => iceFile.clone())
338
+ await benchmark('iceberg/scan_plan_equals', () => icePlan.equals(icePlan))
339
+ await benchmark('iceberg/scan_plan_compare', () => icePlan.compare(icePlan))
340
+ await benchmark('iceberg/scan_plan_stable_hash', () => icePlan.stableHash())
341
+ await benchmark('iceberg/scan_plan_clone', () => icePlan.clone())
342
+ await benchmark('iceberg/partition_field_equals', () => icePartitionField.equals(icePartitionField))
343
+ await benchmark('iceberg/partition_field_compare', () => icePartitionField.compare(icePartitionField))
344
+ await benchmark('iceberg/partition_field_stable_hash', () => icePartitionField.stableHash())
345
+ await benchmark('iceberg/partition_field_clone', () => icePartitionField.clone())
346
+ await benchmark('iceberg/snapshot_equals', () => iceSnapshot.equals(iceSnapshot))
347
+ await benchmark('iceberg/snapshot_compare', () => iceSnapshot.compare(iceSnapshot))
348
+ await benchmark('iceberg/snapshot_stable_hash', () => iceSnapshot.stableHash())
349
+ await benchmark('iceberg/snapshot_clone', () => iceSnapshot.clone())
350
+ await benchmark('iceberg/snapshot_ref_equals', () => iceSnapshotRef.equals(iceSnapshotRef))
351
+ await benchmark('iceberg/snapshot_ref_compare', () => iceSnapshotRef.compare(iceSnapshotRef))
352
+ await benchmark('iceberg/snapshot_ref_stable_hash', () => iceSnapshotRef.stableHash())
353
+ await benchmark('iceberg/snapshot_ref_clone', () => iceSnapshotRef.clone())
354
+ await benchmark('iceberg/manifest_file_equals', () => iceManifest.equals(iceManifest))
355
+ await benchmark('iceberg/manifest_file_compare', () => iceManifest.compare(iceManifest))
356
+ await benchmark('iceberg/manifest_file_stable_hash', () => iceManifest.stableHash())
357
+ await benchmark('iceberg/manifest_file_clone', () => iceManifest.clone())
358
+ await benchmark('iceberg/compaction_equals', () => iceCompaction.equals(iceCompaction))
359
+ await benchmark('iceberg/compaction_compare', () => iceCompaction.compare(iceCompaction))
360
+ await benchmark('iceberg/compaction_stable_hash', () => iceCompaction.stableHash())
361
+ await benchmark('iceberg/compaction_clone', () => iceCompaction.clone())
362
+ await benchmark('iceberg/options_equals', () => iceOptions.equals(iceOptions))
363
+ await benchmark('iceberg/options_compare', () => iceOptions.compare(iceOptions))
364
+ await benchmark('iceberg/options_stable_hash', () => iceOptions.stableHash())
365
+ await benchmark('iceberg/options_clone', () => iceOptions.clone())
366
+ await benchmark('iceberg/open', () => iceberg.IcebergTable.open(lake))
388
367
 
389
- async function* asyncRecordGenerator() {
390
- yield* records
391
- }
368
+ // A catalog append crosses the whole boundary a caller with only rows and a
369
+ // name uses: resolve the dotted name against the warehouse, locate the table
370
+ // there, and commit one snapshot. The rows stay small so the number reports
371
+ // that path rather than Parquet encoding. A create descends through existing
372
+ // namespaces only, so the namespace is made first.
373
+ const catalog = new iceberg.IcebergCatalog('lake', path.join(root, 'warehouse'))
374
+ catalog.namespaces().create('bench')
375
+ const catalogRows = arrow.tableToIPC(
376
+ new arrow.Table({
377
+ id: arrow.vectorFromArray([1n, 2n, 3n, 4n], new arrow.Int64()),
378
+ }),
379
+ )
380
+ catalog.tables().append('bench.trades', catalogRows)
381
+ await benchmark('iceberg/catalog_append', () => catalog.tables().append('bench.trades', catalogRows))
382
+
383
+ async function* asyncRecordGenerator() {
384
+ yield* records
385
+ }
392
386
 
393
- async function finish() {
394
- await benchmarkAsync('records/overwrite_async_records_commit_batches', async () => {
387
+ await benchmark('records/overwrite_async_records_commit_batches', async () => {
395
388
  const handle = stored()
396
389
  await handle.overwriteRecords(
397
390
  asyncRecordGenerator(),
@@ -401,8 +394,7 @@ async function finish() {
401
394
  fs.rmSync(root, { recursive: true, force: true })
402
395
  }
403
396
 
404
- finish().catch((error) => {
405
- fs.rmSync(root, { recursive: true, force: true })
397
+ main().catch((error) => {
406
398
  console.error(error)
407
399
  process.exitCode = 1
408
400
  })