yggio 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +334 -0
- package/README.md +43 -35
- package/benchmarks/expression.js +44 -0
- package/benchmarks/fix.js +13 -1
- package/benchmarks/fix_parse.js +51 -0
- package/benchmarks/fix_view.js +45 -0
- package/benchmarks/holder/io.js +129 -107
- package/benchmarks/holder.js +247 -228
- package/benchmarks/js_thread_block.js +144 -0
- package/benchmarks/media.js +341 -349
- package/benchmarks/serie_arrow.js +58 -0
- package/benchmarks/text.js +9 -6
- package/benchmarks/types.js +185 -185
- package/binding.d.ts +488 -68
- package/binding.js +362 -53
- package/fields.js +2 -2
- package/index.d.ts +1730 -78
- package/index.js +62 -8
- package/package.json +10 -5
- package/records.js +116 -127
- package/values.js +160 -0
package/benchmarks/media.js
CHANGED
|
@@ -12,386 +12,379 @@ const {
|
|
|
12
12
|
MimeType,
|
|
13
13
|
RecordOptions,
|
|
14
14
|
Serie,
|
|
15
|
-
SerieReader,
|
|
16
15
|
fields,
|
|
17
16
|
iceberg,
|
|
18
17
|
} = require('yggio')
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
// counts in thousands rather than tens of thousands by default. It also keeps
|
|
22
|
-
// Parquet footer and projected-WKB Scalar crossings beside the record surface.
|
|
23
|
-
// Smoke mode keeps fixtures and loops small enough for package verification.
|
|
24
|
-
const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
|
|
25
|
-
const iterations = Number.parseInt(
|
|
26
|
-
smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '200'),
|
|
27
|
-
10,
|
|
28
|
-
)
|
|
29
|
-
if (!Number.isSafeInteger(iterations) || iterations <= 0) {
|
|
30
|
-
throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
|
|
31
|
-
}
|
|
32
|
-
const warmups = smoke ? 0 : Math.min(iterations, 20)
|
|
33
|
-
const filter = process.env.YGGIO_BENCH_FILTER
|
|
19
|
+
async function main() {
|
|
34
20
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
21
|
+
// A record crossing is far more work than a schema lookup, so this target
|
|
22
|
+
// counts in thousands rather than tens of thousands by default. It also keeps
|
|
23
|
+
// Parquet footer and projected-WKB Scalar crossings beside the record surface.
|
|
24
|
+
// Smoke mode keeps fixtures and loops small enough for package verification.
|
|
25
|
+
const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
|
|
26
|
+
const iterations = Number.parseInt(
|
|
27
|
+
smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '200'),
|
|
28
|
+
10,
|
|
29
|
+
)
|
|
30
|
+
if (!Number.isSafeInteger(iterations) || iterations <= 0) {
|
|
31
|
+
throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
|
|
32
|
+
}
|
|
33
|
+
const warmups = smoke ? 0 : Math.min(iterations, 20)
|
|
34
|
+
const filter = process.env.YGGIO_BENCH_FILTER
|
|
38
35
|
|
|
39
|
-
function
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
const started = performance.now()
|
|
43
|
-
for (let index = 0; index < iterations; index += 1) operation()
|
|
44
|
-
const elapsed = performance.now() - started
|
|
45
|
-
const rate = Math.round((iterations * 1_000) / elapsed)
|
|
46
|
-
console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
|
|
47
|
-
}
|
|
36
|
+
function selected(name) {
|
|
37
|
+
return filter === undefined || name.includes(filter)
|
|
38
|
+
}
|
|
48
39
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
symbols[index] = index % 2 === 0 ? 'AAPL' : 'MSFT'
|
|
65
|
-
venues[index] = index % 3 === 0 ? 'XNAS' : 'XNYS'
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// Every fixture is built once, outside the measured loops: the numbers report
|
|
69
|
-
// the boundary crossing, not Apache Arrow JS building a column.
|
|
70
|
-
const table = new arrow.Table({
|
|
71
|
-
id: arrow.vectorFromArray(ids, new arrow.Int64()),
|
|
72
|
-
symbol: arrow.vectorFromArray(symbols, new arrow.Utf8()),
|
|
73
|
-
venue: arrow.vectorFromArray(venues, new arrow.Utf8()),
|
|
74
|
-
})
|
|
75
|
-
const ipc = arrow.tableToIPC(table)
|
|
76
|
-
const commitBatches = []
|
|
77
|
-
for (let start = 0; start < rows; start += commitBatchRowSize) {
|
|
78
|
-
commitBatches.push(...table.slice(start, start + commitBatchRowSize).batches)
|
|
79
|
-
}
|
|
80
|
-
const commitIpc = arrow.tableToIPC(new arrow.Table(commitBatches))
|
|
81
|
-
const schema = fields.struct(
|
|
82
|
-
'row',
|
|
83
|
-
[Field.from('id: int64'), Field.from('symbol: utf8'), Field.from('venue: utf8')],
|
|
84
|
-
{ nullable: false },
|
|
85
|
-
)
|
|
86
|
-
const wanted = fields.struct('row', [Field.from('id: int64')], { nullable: false })
|
|
40
|
+
async function benchmark(name, operation) {
|
|
41
|
+
if (!selected(name)) return
|
|
42
|
+
for (let index = 0; index < warmups; index += 1) {
|
|
43
|
+
const result = operation()
|
|
44
|
+
if (result instanceof Promise) await result
|
|
45
|
+
}
|
|
46
|
+
const started = performance.now()
|
|
47
|
+
for (let index = 0; index < iterations; index += 1) {
|
|
48
|
+
const result = operation()
|
|
49
|
+
if (result instanceof Promise) await result
|
|
50
|
+
}
|
|
51
|
+
const elapsed = performance.now() - started
|
|
52
|
+
const rate = Math.round((iterations * 1_000) / elapsed)
|
|
53
|
+
console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
|
|
54
|
+
}
|
|
87
55
|
|
|
88
|
-
const
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
new
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
56
|
+
const rows = smoke ? 64 : 10_000
|
|
57
|
+
const writeBatchSize = smoke ? 32 : 1_024
|
|
58
|
+
// A cadence counts whole batches, so the committed paths are fed batches small
|
|
59
|
+
// enough that one publication spans several: 100-row batches, ten to a
|
|
60
|
+
// publication, so 10,000 rows publish ten times.
|
|
61
|
+
const commitBatchRowSize = smoke ? 8 : 100
|
|
62
|
+
const commitBatchNum = smoke ? 2 : 10
|
|
63
|
+
// A plain array of bigints, not a `BigInt64Array`: Apache Arrow JS reads a
|
|
64
|
+
// typed array as a column that cannot be null, and the declared root below says
|
|
65
|
+
// every column can be.
|
|
66
|
+
const ids = new Array(rows)
|
|
67
|
+
const symbols = new Array(rows)
|
|
68
|
+
const venues = new Array(rows)
|
|
69
|
+
for (let index = 0; index < rows; index += 1) {
|
|
70
|
+
ids[index] = BigInt(index)
|
|
71
|
+
symbols[index] = index % 2 === 0 ? 'AAPL' : 'MSFT'
|
|
72
|
+
venues[index] = index % 3 === 0 ? 'XNAS' : 'XNYS'
|
|
73
|
+
}
|
|
104
74
|
|
|
105
|
-
|
|
106
|
-
|
|
75
|
+
// Every fixture is built once, outside the measured loops: the numbers report
|
|
76
|
+
// the boundary crossing, not Apache Arrow JS building a column.
|
|
77
|
+
const table = new arrow.Table({
|
|
78
|
+
id: arrow.vectorFromArray(ids, new arrow.Int64()),
|
|
79
|
+
symbol: arrow.vectorFromArray(symbols, new arrow.Utf8()),
|
|
80
|
+
venue: arrow.vectorFromArray(venues, new arrow.Utf8()),
|
|
81
|
+
})
|
|
82
|
+
const ipc = arrow.tableToIPC(table)
|
|
83
|
+
const commitBatches = []
|
|
84
|
+
for (let start = 0; start < rows; start += commitBatchRowSize) {
|
|
85
|
+
commitBatches.push(...table.slice(start, start + commitBatchRowSize).batches)
|
|
86
|
+
}
|
|
87
|
+
const commitIpc = arrow.tableToIPC(new arrow.Table(commitBatches))
|
|
88
|
+
const schema = fields.struct(
|
|
89
|
+
'row',
|
|
90
|
+
[Field.from('id: int64'), Field.from('symbol: utf8'), Field.from('venue: utf8')],
|
|
91
|
+
{ nullable: false },
|
|
92
|
+
)
|
|
93
|
+
const wanted = fields.struct('row', [Field.from('id: int64')], { nullable: false })
|
|
107
94
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
avroOptions.syncMarker = avroSyncMarker
|
|
125
|
-
})
|
|
126
|
-
benchmark('records/read_field_ipc', () => memory.readArrowField())
|
|
127
|
-
benchmark('records/read_parquet_statistics', () => parquet.readParquetStatistics())
|
|
128
|
-
benchmark('records/read_parquet_geospatial_statistics', () =>
|
|
129
|
-
geospatial.readParquetGeospatialStatistics('shape'),
|
|
130
|
-
)
|
|
131
|
-
const memoryWanted = memory.recordOptions().withField(wanted)
|
|
132
|
-
const parquetWanted = parquet.recordOptions().withField(wanted)
|
|
133
|
-
benchmark('records/read_ipc_into_ipc', () => memory.readArrowReader().intoIpc())
|
|
134
|
-
benchmark('records/read_ipc_into_arrow', () => memory.readArrowReader().intoTable())
|
|
135
|
-
benchmark('records/read_ipc_pushdown', () => memory.readArrowReader(memoryWanted).intoIpc())
|
|
136
|
-
benchmark('records/read_parquet_into_ipc', () => parquet.readArrowReader().intoIpc())
|
|
137
|
-
benchmark('records/read_parquet_pushdown', () => parquet.readArrowReader(parquetWanted).intoIpc())
|
|
95
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-bench-'))
|
|
96
|
+
const stream = new IOBase(path.join(root, 'trades.arrows'))
|
|
97
|
+
await stream.overwriteArrowReader(BatchReader.from(ipc))
|
|
98
|
+
const parquet = new IOBase(path.join(root, 'trades.parquet'))
|
|
99
|
+
await parquet.overwriteArrowTable(table, parquet.recordOptions().withField(schema))
|
|
100
|
+
const wkbPoint = Buffer.allocUnsafe(21)
|
|
101
|
+
wkbPoint.writeUInt8(1, 0)
|
|
102
|
+
wkbPoint.writeUInt32LE(1, 1)
|
|
103
|
+
wkbPoint.writeDoubleLE(1, 5)
|
|
104
|
+
wkbPoint.writeDoubleLE(2, 13)
|
|
105
|
+
const geospatial = new IOBase(path.join(root, 'shapes.parquet'))
|
|
106
|
+
await geospatial.overwriteArrowTable(
|
|
107
|
+
new arrow.Table({
|
|
108
|
+
shape: arrow.vectorFromArray(new Array(rows).fill(wkbPoint), new arrow.Binary()),
|
|
109
|
+
}),
|
|
110
|
+
)
|
|
138
111
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
// growing one benchmark fixture forever.
|
|
142
|
-
const records = new Array(rows)
|
|
143
|
-
for (let index = 0; index < rows; index += 1) {
|
|
144
|
-
records[index] = { id: ids[index], symbol: symbols[index], venue: venues[index] }
|
|
145
|
-
}
|
|
146
|
-
const storedBytes = stream.readBytes()
|
|
147
|
-
function stored() {
|
|
148
|
-
const handle = IOBase.fromBytes(storedBytes)
|
|
149
|
-
handle.mediaType = MimeType.ARROW_STREAM
|
|
150
|
-
return handle
|
|
151
|
-
}
|
|
152
|
-
function keyed(handle) {
|
|
153
|
-
return handle.recordOptions().withMergeBy(['id'])
|
|
154
|
-
}
|
|
155
|
-
function committed(handle, intent) {
|
|
156
|
-
let options = handle
|
|
157
|
-
.recordOptions()
|
|
158
|
-
.withField(schema)
|
|
159
|
-
.withBatchRowSize(commitBatchRowSize)
|
|
160
|
-
.withCommitBatchNum(commitBatchNum)
|
|
161
|
-
if (intent === 'merge') options = options.withMergeBy(['id'])
|
|
162
|
-
return options
|
|
163
|
-
}
|
|
112
|
+
const memory = IOBase.fromBytes(stream.readBytes())
|
|
113
|
+
memory.mediaType = MimeType.ARROW_STREAM
|
|
164
114
|
|
|
165
|
-
benchmark('records/
|
|
166
|
-
|
|
167
|
-
)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
)
|
|
171
|
-
benchmark('records/
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
benchmark('records/
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
benchmark('records/
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
})
|
|
183
|
-
benchmark('records/
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
115
|
+
await benchmark('records/reader_from_ipc', () => BatchReader.fromIpc(ipc))
|
|
116
|
+
await benchmark('records/reader_from_arrow_table', () => BatchReader.from(table))
|
|
117
|
+
await benchmark('records/record_options', () => memory.recordOptions())
|
|
118
|
+
const protocolOptions = memory.recordOptions().withField(schema).withBatchRowSize(writeBatchSize)
|
|
119
|
+
await benchmark('records/record_options_equals', () => protocolOptions.equals(protocolOptions))
|
|
120
|
+
await benchmark('records/record_options_compare', () => protocolOptions.compare(protocolOptions))
|
|
121
|
+
await benchmark('records/record_options_stable_hash', () => protocolOptions.stableHash())
|
|
122
|
+
await benchmark('records/record_options_clone', () => protocolOptions.clone())
|
|
123
|
+
const avroOptions = RecordOptions.from('benchmark.avro')
|
|
124
|
+
const avroSyncMarker = Buffer.from('0123456789abcdef')
|
|
125
|
+
await benchmark('records/avro_option_values', () => [
|
|
126
|
+
avroOptions.blockCodec,
|
|
127
|
+
avroOptions.syncMarker,
|
|
128
|
+
])
|
|
129
|
+
await benchmark('records/avro_option_setters', () => {
|
|
130
|
+
avroOptions.blockCodec = 'deflate'
|
|
131
|
+
avroOptions.syncMarker = avroSyncMarker
|
|
132
|
+
})
|
|
133
|
+
await benchmark('records/read_field_ipc', () => memory.readArrowField())
|
|
134
|
+
await benchmark('records/read_parquet_statistics', () => parquet.readParquetStatistics())
|
|
135
|
+
await benchmark('records/read_parquet_geospatial_statistics', () =>
|
|
136
|
+
geospatial.readParquetGeospatialStatistics('shape'),
|
|
137
|
+
)
|
|
138
|
+
const memoryWanted = memory.recordOptions().withField(wanted)
|
|
139
|
+
const parquetWanted = parquet.recordOptions().withField(wanted)
|
|
140
|
+
await benchmark('records/read_ipc_into_ipc', () => memory.readArrowReader().intoIpc())
|
|
141
|
+
await benchmark('records/read_ipc_into_arrow', () => memory.readArrowReader().intoTable())
|
|
142
|
+
await benchmark('records/read_ipc_pushdown', () => memory.readArrowReader(memoryWanted).intoIpc())
|
|
143
|
+
await benchmark('records/read_parquet_into_ipc', () => parquet.readArrowReader().intoIpc())
|
|
144
|
+
await benchmark('records/read_parquet_pushdown', () => parquet.readArrowReader(parquetWanted).intoIpc())
|
|
187
145
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
benchmark
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
}
|
|
146
|
+
// Each explicit representation adapter is benchmarked under every intent. A
|
|
147
|
+
// fresh in-memory handle keeps append/merge iterations bounded instead of
|
|
148
|
+
// growing one benchmark fixture forever.
|
|
149
|
+
const records = new Array(rows)
|
|
150
|
+
for (let index = 0; index < rows; index += 1) {
|
|
151
|
+
records[index] = { id: ids[index], symbol: symbols[index], venue: venues[index] }
|
|
152
|
+
}
|
|
153
|
+
const storedBytes = stream.readBytes()
|
|
154
|
+
function stored() {
|
|
155
|
+
const handle = IOBase.fromBytes(storedBytes)
|
|
156
|
+
handle.mediaType = MimeType.ARROW_STREAM
|
|
157
|
+
return handle
|
|
158
|
+
}
|
|
159
|
+
function keyed(handle) {
|
|
160
|
+
return handle.recordOptions().withMergeBy(['id'])
|
|
161
|
+
}
|
|
162
|
+
function committed(handle, intent) {
|
|
163
|
+
let options = handle
|
|
164
|
+
.recordOptions()
|
|
165
|
+
.withField(schema)
|
|
166
|
+
.withBatchRowSize(commitBatchRowSize)
|
|
167
|
+
.withCommitBatchNum(commitBatchNum)
|
|
168
|
+
if (intent === 'merge') options = options.withMergeBy(['id'])
|
|
169
|
+
return options
|
|
170
|
+
}
|
|
194
171
|
|
|
195
|
-
benchmark('records/
|
|
196
|
-
|
|
197
|
-
)
|
|
198
|
-
benchmark('records/
|
|
199
|
-
|
|
200
|
-
)
|
|
201
|
-
benchmark('records/
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
})
|
|
172
|
+
await benchmark('records/overwrite_arrow_reader', () =>
|
|
173
|
+
stored().overwriteArrowReader(BatchReader.fromIpc(ipc)),
|
|
174
|
+
)
|
|
175
|
+
await benchmark('records/append_arrow_reader', () =>
|
|
176
|
+
stored().appendArrowReader(BatchReader.fromIpc(ipc)),
|
|
177
|
+
)
|
|
178
|
+
await benchmark('records/merge_arrow_reader', async () => {
|
|
179
|
+
const handle = stored()
|
|
180
|
+
await handle.mergeArrowReader(BatchReader.fromIpc(ipc), keyed(handle))
|
|
181
|
+
})
|
|
182
|
+
await benchmark('records/overwrite_arrow_reader_commit_batches', async () => {
|
|
183
|
+
const handle = stored()
|
|
184
|
+
await handle.overwriteArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'overwrite'))
|
|
185
|
+
})
|
|
186
|
+
await benchmark('records/append_arrow_reader_commit_batches', async () => {
|
|
187
|
+
const handle = stored()
|
|
188
|
+
await handle.appendArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'append'))
|
|
189
|
+
})
|
|
190
|
+
await benchmark('records/merge_arrow_reader_commit_batches', async () => {
|
|
191
|
+
const handle = stored()
|
|
192
|
+
await handle.mergeArrowReader(BatchReader.fromIpc(commitIpc), committed(handle, 'merge'))
|
|
193
|
+
})
|
|
205
194
|
|
|
206
|
-
benchmark('records/
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
)
|
|
213
|
-
benchmark('records/overwrite_record_generator_commit_batches', () => {
|
|
214
|
-
const handle = stored()
|
|
215
|
-
handle.overwriteRecords(recordGenerator(), committed(handle, 'overwrite'))
|
|
216
|
-
})
|
|
217
|
-
benchmark('records/append_record_generator_commit_batches', () => {
|
|
218
|
-
const handle = stored()
|
|
219
|
-
handle.appendRecords(recordGenerator(), committed(handle, 'append'))
|
|
220
|
-
})
|
|
221
|
-
benchmark('records/merge_record_generator_commit_batches', () => {
|
|
222
|
-
const handle = stored()
|
|
223
|
-
handle.mergeRecords(recordGenerator(), committed(handle, 'merge'))
|
|
224
|
-
})
|
|
225
|
-
benchmark('records/append_records', () => stored().appendRecords(records))
|
|
226
|
-
benchmark('records/merge_records', () => {
|
|
227
|
-
const handle = stored()
|
|
228
|
-
handle.mergeRecords(records, keyed(handle))
|
|
229
|
-
})
|
|
195
|
+
await benchmark('records/overwrite_arrow_table', () => stored().overwriteArrowTable(table))
|
|
196
|
+
await benchmark('records/append_arrow_table', () => stored().appendArrowTable(table))
|
|
197
|
+
await benchmark('records/merge_arrow_table', async () => {
|
|
198
|
+
const handle = stored()
|
|
199
|
+
await handle.mergeArrowTable(table, keyed(handle))
|
|
200
|
+
})
|
|
230
201
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
benchmark(
|
|
202
|
+
await benchmark('records/overwrite_arrow_batch', () =>
|
|
203
|
+
stored().overwriteArrowBatch(table.batches[0]),
|
|
204
|
+
)
|
|
205
|
+
await benchmark('records/append_arrow_batch', () =>
|
|
206
|
+
stored().appendArrowBatch(table.batches[0]),
|
|
207
|
+
)
|
|
208
|
+
await benchmark('records/merge_arrow_batch', async () => {
|
|
235
209
|
const handle = stored()
|
|
236
|
-
handle.
|
|
237
|
-
BatchReader.fromIpc(ipc),
|
|
238
|
-
mode,
|
|
239
|
-
mode === 'merge' ? keyed(handle) : undefined,
|
|
240
|
-
)
|
|
210
|
+
await handle.mergeArrowBatch(table.batches[0], keyed(handle))
|
|
241
211
|
})
|
|
242
|
-
|
|
212
|
+
|
|
213
|
+
await benchmark('records/overwrite_records', () => stored().overwriteRecords(records))
|
|
214
|
+
function* recordGenerator() {
|
|
215
|
+
yield* records
|
|
216
|
+
}
|
|
217
|
+
await benchmark('records/overwrite_record_generator', () =>
|
|
218
|
+
stored().overwriteRecords(recordGenerator()),
|
|
219
|
+
)
|
|
220
|
+
await benchmark('records/overwrite_record_generator_commit_batches', async () => {
|
|
243
221
|
const handle = stored()
|
|
244
|
-
handle.
|
|
245
|
-
table,
|
|
246
|
-
mode,
|
|
247
|
-
mode === 'merge' ? keyed(handle) : undefined,
|
|
248
|
-
)
|
|
222
|
+
await handle.overwriteRecords(recordGenerator(), committed(handle, 'overwrite'))
|
|
249
223
|
})
|
|
250
|
-
benchmark(
|
|
224
|
+
await benchmark('records/append_record_generator_commit_batches', async () => {
|
|
251
225
|
const handle = stored()
|
|
252
|
-
handle.
|
|
253
|
-
table.batches[0],
|
|
254
|
-
mode,
|
|
255
|
-
mode === 'merge' ? keyed(handle) : undefined,
|
|
256
|
-
)
|
|
226
|
+
await handle.appendRecords(recordGenerator(), committed(handle, 'append'))
|
|
257
227
|
})
|
|
258
|
-
benchmark(
|
|
228
|
+
await benchmark('records/merge_record_generator_commit_batches', async () => {
|
|
259
229
|
const handle = stored()
|
|
260
|
-
handle.
|
|
261
|
-
records,
|
|
262
|
-
mode,
|
|
263
|
-
mode === 'merge' ? keyed(handle) : undefined,
|
|
264
|
-
)
|
|
230
|
+
await handle.mergeRecords(recordGenerator(), committed(handle, 'merge'))
|
|
265
231
|
})
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
// The Serie doors: a held column crosses as the one record batch it is, with
|
|
269
|
-
// no IPC and no copy, a stream as itself; the Arrow JS table is the IPC bridge
|
|
270
|
-
// every other Arrow JS write pays, then the same native door.
|
|
271
|
-
const heldRows = Serie.fromArrowBatch(table, schema)
|
|
272
|
-
benchmark('records/read_serie', () => {
|
|
273
|
-
for (const batch of memory.readSerie()) batch.length
|
|
274
|
-
})
|
|
275
|
-
for (const mode of ['overwrite', 'append', 'merge']) {
|
|
276
|
-
benchmark(`records/write_serie/${mode}`, () => {
|
|
232
|
+
await benchmark('records/append_records', () => stored().appendRecords(records))
|
|
233
|
+
await benchmark('records/merge_records', async () => {
|
|
277
234
|
const handle = stored()
|
|
278
|
-
handle.
|
|
235
|
+
await handle.mergeRecords(records, keyed(handle))
|
|
279
236
|
})
|
|
280
|
-
}
|
|
281
|
-
benchmark('records/overwrite_serie_reader', () =>
|
|
282
|
-
stored().overwriteSerie(SerieReader.fromSerie(heldRows)),
|
|
283
|
-
)
|
|
284
|
-
benchmark('records/overwrite_serie_arrow_table', () => stored().overwriteSerie(table))
|
|
285
237
|
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
238
|
+
// The configurable surface adds only mode dispatch; keep each public input
|
|
239
|
+
// shape visible so benchmark regressions cannot hide in a binding adapter.
|
|
240
|
+
for (const mode of ['overwrite', 'append', 'merge']) {
|
|
241
|
+
await benchmark(`records/write_arrow_reader/${mode}`, async () => {
|
|
242
|
+
const handle = stored()
|
|
243
|
+
await handle.writeArrowReader(
|
|
244
|
+
BatchReader.fromIpc(ipc),
|
|
245
|
+
mode,
|
|
246
|
+
mode === 'merge' ? keyed(handle) : undefined,
|
|
247
|
+
)
|
|
248
|
+
})
|
|
249
|
+
await benchmark(`records/write_arrow_table/${mode}`, async () => {
|
|
250
|
+
const handle = stored()
|
|
251
|
+
await handle.writeArrowTable(
|
|
252
|
+
table,
|
|
253
|
+
mode,
|
|
254
|
+
mode === 'merge' ? keyed(handle) : undefined,
|
|
255
|
+
)
|
|
256
|
+
})
|
|
257
|
+
await benchmark(`records/write_arrow_batch/${mode}`, async () => {
|
|
258
|
+
const handle = stored()
|
|
259
|
+
await handle.writeArrowBatch(
|
|
260
|
+
table.batches[0],
|
|
261
|
+
mode,
|
|
262
|
+
mode === 'merge' ? keyed(handle) : undefined,
|
|
263
|
+
)
|
|
264
|
+
})
|
|
265
|
+
await benchmark(`records/write_records/${mode}`, async () => {
|
|
266
|
+
const handle = stored()
|
|
267
|
+
await handle.writeRecords(
|
|
268
|
+
records,
|
|
269
|
+
mode,
|
|
270
|
+
mode === 'merge' ? keyed(handle) : undefined,
|
|
271
|
+
)
|
|
272
|
+
})
|
|
273
|
+
}
|
|
308
274
|
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
})
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
benchmark('
|
|
275
|
+
// The Serie doors: a held column crosses as the one record batch it is, with
|
|
276
|
+
// no IPC and no copy, a stream as itself; the Arrow JS table is the IPC bridge
|
|
277
|
+
// every other Arrow JS write pays, then the same native door.
|
|
278
|
+
const heldRows = Serie.fromArrowBatch(table, schema)
|
|
279
|
+
await benchmark('records/read_serie', () => {
|
|
280
|
+
for (const batch of memory.readSerie()) batch.length
|
|
281
|
+
})
|
|
282
|
+
for (const mode of ['overwrite', 'append', 'merge']) {
|
|
283
|
+
await benchmark(`records/write_serie/${mode}`, async () => {
|
|
284
|
+
const handle = stored()
|
|
285
|
+
await handle.writeSerie(heldRows, mode, mode === 'merge' ? keyed(handle) : undefined)
|
|
286
|
+
})
|
|
287
|
+
}
|
|
288
|
+
await benchmark('records/overwrite_serie', () => stored().overwriteSerie(heldRows))
|
|
289
|
+
await benchmark('records/overwrite_serie_arrow_table', () => stored().overwriteSerie(table))
|
|
323
290
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
)
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
benchmark('iceberg/snapshot_ref_compare', () => iceSnapshotRef.compare(iceSnapshotRef))
|
|
347
|
-
benchmark('iceberg/snapshot_ref_stable_hash', () => iceSnapshotRef.stableHash())
|
|
348
|
-
benchmark('iceberg/snapshot_ref_clone', () => iceSnapshotRef.clone())
|
|
349
|
-
benchmark('iceberg/manifest_file_equals', () => iceManifest.equals(iceManifest))
|
|
350
|
-
benchmark('iceberg/manifest_file_compare', () => iceManifest.compare(iceManifest))
|
|
351
|
-
benchmark('iceberg/manifest_file_stable_hash', () => iceManifest.stableHash())
|
|
352
|
-
benchmark('iceberg/manifest_file_clone', () => iceManifest.clone())
|
|
353
|
-
benchmark('iceberg/compaction_equals', () => iceCompaction.equals(iceCompaction))
|
|
354
|
-
benchmark('iceberg/compaction_compare', () => iceCompaction.compare(iceCompaction))
|
|
355
|
-
benchmark('iceberg/compaction_stable_hash', () => iceCompaction.stableHash())
|
|
356
|
-
benchmark('iceberg/compaction_clone', () => iceCompaction.clone())
|
|
357
|
-
benchmark('iceberg/options_equals', () => iceOptions.equals(iceOptions))
|
|
358
|
-
benchmark('iceberg/options_compare', () => iceOptions.compare(iceOptions))
|
|
359
|
-
benchmark('iceberg/options_stable_hash', () => iceOptions.stableHash())
|
|
360
|
-
benchmark('iceberg/options_clone', () => iceOptions.clone())
|
|
361
|
-
benchmark('iceberg/open', () => iceberg.IcebergTable.open(lake))
|
|
291
|
+
// One commit writes a data file, a manifest, a manifest list, and a metadata
|
|
292
|
+
// document, so an Iceberg append is measured separately from a plain write.
|
|
293
|
+
const iced = iceberg.assignFieldIds(schema)
|
|
294
|
+
const lake = path.join(root, 'lake')
|
|
295
|
+
const iceTable = iceberg.IcebergTable.create(lake, iced)
|
|
296
|
+
await iceTable.append(ipc)
|
|
297
|
+
const iceFile = iceTable.dataFiles()[0]
|
|
298
|
+
const icePlan = iceTable.plan()
|
|
299
|
+
const icePartitionField = iceberg.PartitionSpec.identity(iced, ['venue']).fields[0]
|
|
300
|
+
const iceSnapshot = iceTable.currentSnapshot
|
|
301
|
+
const iceManifest = iceTable.manifests()[0]
|
|
302
|
+
iceTable.createTag('benchmark-value-protocol', iceSnapshot.snapshotId)
|
|
303
|
+
const iceSnapshotRef = iceTable.removeRef('benchmark-value-protocol')
|
|
304
|
+
// One live file has no compaction group, so this bounded report is produced
|
|
305
|
+
// without another snapshot or data rewrite.
|
|
306
|
+
const iceCompaction = iceTable.compact()
|
|
307
|
+
const iceOptions = new iceberg.IcebergOptions({
|
|
308
|
+
commitRetries: 4,
|
|
309
|
+
targetFileSize: 512 * 1024 * 1024,
|
|
310
|
+
readParallelism: 4,
|
|
311
|
+
dataMimeType: MimeType.AVRO,
|
|
312
|
+
})
|
|
362
313
|
|
|
363
|
-
//
|
|
364
|
-
//
|
|
365
|
-
//
|
|
366
|
-
//
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
)
|
|
375
|
-
|
|
376
|
-
benchmark('iceberg/
|
|
314
|
+
// What the widened write costs at the boundary. Each sample commits into its
|
|
315
|
+
// own fresh table, because a growing manifest list would otherwise time the
|
|
316
|
+
// table's history rather than the write - the same rule the Python Iceberg
|
|
317
|
+
// benchmark follows.
|
|
318
|
+
const iceRows = Array.from({ length: 64 }, (_, index) => ({
|
|
319
|
+
id: BigInt(index),
|
|
320
|
+
venue: 'XNAS',
|
|
321
|
+
price: index / 4,
|
|
322
|
+
}))
|
|
323
|
+
let iceBenchIndex = 0
|
|
324
|
+
const freshIceTable = () =>
|
|
325
|
+
iceberg.IcebergTable.create(path.join(root, 'bench', `t${iceBenchIndex++}`), iced)
|
|
326
|
+
await benchmark('iceberg/append_arrow_ipc', () => freshIceTable().append(ipc))
|
|
327
|
+
await benchmark('iceberg/append_rows', () => freshIceTable().append(iceRows))
|
|
377
328
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
329
|
+
await benchmark('iceberg/scan_into_ipc', () => iceTable.scan().intoIpc())
|
|
330
|
+
await benchmark('iceberg/scan_pushdown', () =>
|
|
331
|
+
iceTable.scan(fields.struct('row', [iced.dtype.getFieldAt(0)], { nullable: false })).intoIpc(),
|
|
332
|
+
)
|
|
333
|
+
await benchmark('iceberg/data_files', () => iceTable.dataFiles())
|
|
334
|
+
await benchmark('iceberg/data_file_equals', () => iceFile.equals(iceFile))
|
|
335
|
+
await benchmark('iceberg/data_file_compare', () => iceFile.compare(iceFile))
|
|
336
|
+
await benchmark('iceberg/data_file_stable_hash', () => iceFile.stableHash())
|
|
337
|
+
await benchmark('iceberg/data_file_clone', () => iceFile.clone())
|
|
338
|
+
await benchmark('iceberg/scan_plan_equals', () => icePlan.equals(icePlan))
|
|
339
|
+
await benchmark('iceberg/scan_plan_compare', () => icePlan.compare(icePlan))
|
|
340
|
+
await benchmark('iceberg/scan_plan_stable_hash', () => icePlan.stableHash())
|
|
341
|
+
await benchmark('iceberg/scan_plan_clone', () => icePlan.clone())
|
|
342
|
+
await benchmark('iceberg/partition_field_equals', () => icePartitionField.equals(icePartitionField))
|
|
343
|
+
await benchmark('iceberg/partition_field_compare', () => icePartitionField.compare(icePartitionField))
|
|
344
|
+
await benchmark('iceberg/partition_field_stable_hash', () => icePartitionField.stableHash())
|
|
345
|
+
await benchmark('iceberg/partition_field_clone', () => icePartitionField.clone())
|
|
346
|
+
await benchmark('iceberg/snapshot_equals', () => iceSnapshot.equals(iceSnapshot))
|
|
347
|
+
await benchmark('iceberg/snapshot_compare', () => iceSnapshot.compare(iceSnapshot))
|
|
348
|
+
await benchmark('iceberg/snapshot_stable_hash', () => iceSnapshot.stableHash())
|
|
349
|
+
await benchmark('iceberg/snapshot_clone', () => iceSnapshot.clone())
|
|
350
|
+
await benchmark('iceberg/snapshot_ref_equals', () => iceSnapshotRef.equals(iceSnapshotRef))
|
|
351
|
+
await benchmark('iceberg/snapshot_ref_compare', () => iceSnapshotRef.compare(iceSnapshotRef))
|
|
352
|
+
await benchmark('iceberg/snapshot_ref_stable_hash', () => iceSnapshotRef.stableHash())
|
|
353
|
+
await benchmark('iceberg/snapshot_ref_clone', () => iceSnapshotRef.clone())
|
|
354
|
+
await benchmark('iceberg/manifest_file_equals', () => iceManifest.equals(iceManifest))
|
|
355
|
+
await benchmark('iceberg/manifest_file_compare', () => iceManifest.compare(iceManifest))
|
|
356
|
+
await benchmark('iceberg/manifest_file_stable_hash', () => iceManifest.stableHash())
|
|
357
|
+
await benchmark('iceberg/manifest_file_clone', () => iceManifest.clone())
|
|
358
|
+
await benchmark('iceberg/compaction_equals', () => iceCompaction.equals(iceCompaction))
|
|
359
|
+
await benchmark('iceberg/compaction_compare', () => iceCompaction.compare(iceCompaction))
|
|
360
|
+
await benchmark('iceberg/compaction_stable_hash', () => iceCompaction.stableHash())
|
|
361
|
+
await benchmark('iceberg/compaction_clone', () => iceCompaction.clone())
|
|
362
|
+
await benchmark('iceberg/options_equals', () => iceOptions.equals(iceOptions))
|
|
363
|
+
await benchmark('iceberg/options_compare', () => iceOptions.compare(iceOptions))
|
|
364
|
+
await benchmark('iceberg/options_stable_hash', () => iceOptions.stableHash())
|
|
365
|
+
await benchmark('iceberg/options_clone', () => iceOptions.clone())
|
|
366
|
+
await benchmark('iceberg/open', () => iceberg.IcebergTable.open(lake))
|
|
388
367
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
368
|
+
// A catalog append crosses the whole boundary a caller with only rows and a
|
|
369
|
+
// name uses: resolve the dotted name against the warehouse, locate the table
|
|
370
|
+
// there, and commit one snapshot. The rows stay small so the number reports
|
|
371
|
+
// that path rather than Parquet encoding. A create descends through existing
|
|
372
|
+
// namespaces only, so the namespace is made first.
|
|
373
|
+
const catalog = new iceberg.IcebergCatalog('lake', path.join(root, 'warehouse'))
|
|
374
|
+
catalog.namespaces().create('bench')
|
|
375
|
+
const catalogRows = arrow.tableToIPC(
|
|
376
|
+
new arrow.Table({
|
|
377
|
+
id: arrow.vectorFromArray([1n, 2n, 3n, 4n], new arrow.Int64()),
|
|
378
|
+
}),
|
|
379
|
+
)
|
|
380
|
+
catalog.tables().append('bench.trades', catalogRows)
|
|
381
|
+
await benchmark('iceberg/catalog_append', () => catalog.tables().append('bench.trades', catalogRows))
|
|
382
|
+
|
|
383
|
+
async function* asyncRecordGenerator() {
|
|
384
|
+
yield* records
|
|
385
|
+
}
|
|
392
386
|
|
|
393
|
-
async
|
|
394
|
-
await benchmarkAsync('records/overwrite_async_records_commit_batches', async () => {
|
|
387
|
+
await benchmark('records/overwrite_async_records_commit_batches', async () => {
|
|
395
388
|
const handle = stored()
|
|
396
389
|
await handle.overwriteRecords(
|
|
397
390
|
asyncRecordGenerator(),
|
|
@@ -401,8 +394,7 @@ async function finish() {
|
|
|
401
394
|
fs.rmSync(root, { recursive: true, force: true })
|
|
402
395
|
}
|
|
403
396
|
|
|
404
|
-
|
|
405
|
-
fs.rmSync(root, { recursive: true, force: true })
|
|
397
|
+
main().catch((error) => {
|
|
406
398
|
console.error(error)
|
|
407
399
|
process.exitCode = 1
|
|
408
400
|
})
|