yggio 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +334 -0
- package/README.md +43 -35
- package/benchmarks/expression.js +44 -0
- package/benchmarks/fix.js +13 -1
- package/benchmarks/fix_parse.js +51 -0
- package/benchmarks/fix_view.js +45 -0
- package/benchmarks/holder/io.js +129 -107
- package/benchmarks/holder.js +247 -228
- package/benchmarks/js_thread_block.js +144 -0
- package/benchmarks/media.js +341 -349
- package/benchmarks/serie_arrow.js +58 -0
- package/benchmarks/text.js +9 -6
- package/benchmarks/types.js +185 -185
- package/binding.d.ts +488 -68
- package/binding.js +362 -53
- package/fields.js +2 -2
- package/index.d.ts +1730 -78
- package/index.js +62 -8
- package/package.json +10 -5
- package/records.js +116 -127
- package/values.js +160 -0
package/benchmarks/holder.js
CHANGED
|
@@ -26,264 +26,283 @@ const path = require('node:path')
|
|
|
26
26
|
const arrow = require('apache-arrow')
|
|
27
27
|
const { IOBase } = require('yggio')
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
throw new RangeError(
|
|
37
|
-
'YGGIO_BENCH_ITERATIONS must be a positive safe integer',
|
|
29
|
+
async function main() {
|
|
30
|
+
|
|
31
|
+
// A byte round trip is far more work than a path lookup, so this target counts
|
|
32
|
+
// in thousands rather than tens of thousands by default.
|
|
33
|
+
const iterations = Number.parseInt(
|
|
34
|
+
process.env.YGGIO_BENCH_ITERATIONS ?? '2000',
|
|
35
|
+
10,
|
|
38
36
|
)
|
|
39
|
-
|
|
37
|
+
if (!Number.isSafeInteger(iterations) || iterations <= 0) {
|
|
38
|
+
throw new RangeError(
|
|
39
|
+
'YGGIO_BENCH_ITERATIONS must be a positive safe integer',
|
|
40
|
+
)
|
|
41
|
+
}
|
|
40
42
|
|
|
41
|
-
function benchmark(name, operation) {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
async function benchmark(name, operation) {
|
|
44
|
+
for (let index = 0; index < Math.min(iterations, 100); index += 1) {
|
|
45
|
+
const result = operation()
|
|
46
|
+
if (result instanceof Promise) await result
|
|
47
|
+
}
|
|
48
|
+
const started = performance.now()
|
|
49
|
+
for (let index = 0; index < iterations; index += 1) {
|
|
50
|
+
const result = operation()
|
|
51
|
+
if (result instanceof Promise) await result
|
|
52
|
+
}
|
|
53
|
+
const elapsed = performance.now() - started
|
|
54
|
+
const rate = Math.round((iterations * 1_000) / elapsed)
|
|
55
|
+
console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
|
|
56
|
+
}
|
|
49
57
|
|
|
50
|
-
function input(location) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
function input(location) {
|
|
59
|
+
const descriptor = fs.openSync(location, 'r')
|
|
60
|
+
let position = 0n
|
|
61
|
+
let closed = false
|
|
62
|
+
const readAt = (offset, length) => {
|
|
63
|
+
const bytes = Buffer.alloc(Number(length))
|
|
64
|
+
return bytes.subarray(
|
|
65
|
+
0,
|
|
66
|
+
fs.readSync(descriptor, bytes, 0, bytes.length, Number(offset)),
|
|
67
|
+
)
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
read(length) {
|
|
71
|
+
const bytes = readAt(position, length)
|
|
72
|
+
position += BigInt(bytes.length)
|
|
73
|
+
return bytes
|
|
74
|
+
},
|
|
75
|
+
readAt,
|
|
76
|
+
seek(offset, whence) {
|
|
77
|
+
const base =
|
|
78
|
+
whence === 'current'
|
|
79
|
+
? position
|
|
80
|
+
: whence === 'end'
|
|
81
|
+
? fs.fstatSync(descriptor, { bigint: true }).size
|
|
82
|
+
: 0n
|
|
83
|
+
position = base + offset
|
|
84
|
+
return position
|
|
85
|
+
},
|
|
86
|
+
tell: () => position,
|
|
87
|
+
close() {
|
|
88
|
+
if (!closed) fs.closeSync(descriptor)
|
|
89
|
+
closed = true
|
|
90
|
+
},
|
|
91
|
+
get closed() {
|
|
92
|
+
return closed
|
|
93
|
+
},
|
|
94
|
+
}
|
|
60
95
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
96
|
+
|
|
97
|
+
function output(location, append) {
|
|
98
|
+
const descriptor = fs.openSync(location, append ? 'a' : 'w')
|
|
99
|
+
let position = append ? fs.fstatSync(descriptor, { bigint: true }).size : 0n
|
|
100
|
+
let closed = false
|
|
101
|
+
return {
|
|
102
|
+
write(bytes) {
|
|
103
|
+
const written = fs.writeSync(descriptor, bytes)
|
|
104
|
+
position += BigInt(written)
|
|
105
|
+
return BigInt(written)
|
|
106
|
+
},
|
|
107
|
+
tell: () => position,
|
|
108
|
+
flush: () => fs.fsyncSync(descriptor),
|
|
109
|
+
close() {
|
|
110
|
+
if (!closed) fs.closeSync(descriptor)
|
|
111
|
+
closed = true
|
|
112
|
+
},
|
|
113
|
+
get closed() {
|
|
114
|
+
return closed
|
|
115
|
+
},
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const domain = {}
|
|
120
|
+
|
|
121
|
+
// The complete synchronous handler under measurement, directly over node:fs.
|
|
122
|
+
const handler = {
|
|
123
|
+
typeName: 'local',
|
|
124
|
+
domain,
|
|
125
|
+
equals(other) {
|
|
126
|
+
return other?.domain === domain
|
|
66
127
|
},
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
return
|
|
128
|
+
normalizePath: path.normalize,
|
|
129
|
+
fileInfo(location) {
|
|
130
|
+
const stats = fs.statSync(location, {
|
|
131
|
+
throwIfNoEntry: false,
|
|
132
|
+
bigint: true,
|
|
133
|
+
})
|
|
134
|
+
if (stats === undefined) return { path: location, kind: 'not-found' }
|
|
135
|
+
if (stats.isDirectory())
|
|
136
|
+
return { path: location, kind: 'directory', mtimeNs: stats.mtimeNs }
|
|
137
|
+
return {
|
|
138
|
+
path: location,
|
|
139
|
+
kind: 'file',
|
|
140
|
+
size: stats.size,
|
|
141
|
+
mtimeNs: stats.mtimeNs,
|
|
142
|
+
}
|
|
77
143
|
},
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
144
|
+
list(selector) {
|
|
145
|
+
return fs
|
|
146
|
+
.readdirSync(selector.baseDir, {
|
|
147
|
+
recursive: selector.recursive,
|
|
148
|
+
withFileTypes: true,
|
|
149
|
+
})
|
|
150
|
+
.map((entry) => {
|
|
151
|
+
const full = path.join(entry.parentPath ?? entry.path, entry.name)
|
|
152
|
+
return this.fileInfo(full)
|
|
153
|
+
})
|
|
82
154
|
},
|
|
83
|
-
|
|
84
|
-
|
|
155
|
+
createDir(location, recursive) {
|
|
156
|
+
fs.mkdirSync(location, { recursive })
|
|
85
157
|
},
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const written = fs.writeSync(descriptor, bytes)
|
|
96
|
-
position += BigInt(written)
|
|
97
|
-
return BigInt(written)
|
|
158
|
+
deleteDir: fs.rmdirSync,
|
|
159
|
+
deleteDirContents(location, missingDirOk) {
|
|
160
|
+
if (!fs.existsSync(location) && missingDirOk) return
|
|
161
|
+
for (const name of fs.readdirSync(location)) {
|
|
162
|
+
fs.rmSync(path.join(location, name), { recursive: true })
|
|
163
|
+
}
|
|
164
|
+
},
|
|
165
|
+
deleteRootDirContents() {
|
|
166
|
+
throw new Error('Unsupported: local root deletion')
|
|
98
167
|
},
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
168
|
+
deleteFile: fs.unlinkSync,
|
|
169
|
+
copyFile: fs.copyFileSync,
|
|
170
|
+
move: fs.renameSync,
|
|
171
|
+
openInputFile: input,
|
|
172
|
+
openInputStream: input,
|
|
173
|
+
openOutputStream(location) {
|
|
174
|
+
return output(location, false)
|
|
104
175
|
},
|
|
105
|
-
|
|
106
|
-
return
|
|
176
|
+
openAppendStream(location) {
|
|
177
|
+
return output(location, true)
|
|
107
178
|
},
|
|
108
179
|
}
|
|
109
|
-
}
|
|
110
180
|
|
|
111
|
-
|
|
181
|
+
// Every fixture is built once, outside the measured loops, so the numbers
|
|
182
|
+
// report the boundary crossing rather than the corpus.
|
|
183
|
+
const PAYLOAD_BYTES = 64 * 1_024
|
|
184
|
+
const RANGE_BYTES = 4_096
|
|
112
185
|
|
|
113
|
-
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
normalizePath: path.normalize,
|
|
121
|
-
fileInfo(location) {
|
|
122
|
-
const stats = fs.statSync(location, {
|
|
123
|
-
throwIfNoEntry: false,
|
|
124
|
-
bigint: true,
|
|
125
|
-
})
|
|
126
|
-
if (stats === undefined) return { path: location, kind: 'not-found' }
|
|
127
|
-
if (stats.isDirectory())
|
|
128
|
-
return { path: location, kind: 'directory', mtimeNs: stats.mtimeNs }
|
|
129
|
-
return {
|
|
130
|
-
path: location,
|
|
131
|
-
kind: 'file',
|
|
132
|
-
size: stats.size,
|
|
133
|
-
mtimeNs: stats.mtimeNs,
|
|
134
|
-
}
|
|
135
|
-
},
|
|
136
|
-
list(selector) {
|
|
137
|
-
return fs
|
|
138
|
-
.readdirSync(selector.baseDir, {
|
|
139
|
-
recursive: selector.recursive,
|
|
140
|
-
withFileTypes: true,
|
|
141
|
-
})
|
|
142
|
-
.map((entry) => {
|
|
143
|
-
const full = path.join(entry.parentPath ?? entry.path, entry.name)
|
|
144
|
-
return this.fileInfo(full)
|
|
145
|
-
})
|
|
146
|
-
},
|
|
147
|
-
createDir(location, recursive) {
|
|
148
|
-
fs.mkdirSync(location, { recursive })
|
|
149
|
-
},
|
|
150
|
-
deleteDir: fs.rmdirSync,
|
|
151
|
-
deleteDirContents(location, missingDirOk) {
|
|
152
|
-
if (!fs.existsSync(location) && missingDirOk) return
|
|
153
|
-
for (const name of fs.readdirSync(location)) {
|
|
154
|
-
fs.rmSync(path.join(location, name), { recursive: true })
|
|
186
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-fs-bench-'))
|
|
187
|
+
const lake = path.join(root, 'lake')
|
|
188
|
+
for (const year of ['2024', '2025']) {
|
|
189
|
+
for (const month of ['01', '02']) {
|
|
190
|
+
const leaf = path.join(lake, `year=${year}`, `month=${month}`)
|
|
191
|
+
fs.mkdirSync(leaf, { recursive: true })
|
|
192
|
+
fs.writeFileSync(path.join(leaf, 'part-0.parquet'), 'parquet')
|
|
155
193
|
}
|
|
156
|
-
}
|
|
157
|
-
deleteRootDirContents() {
|
|
158
|
-
throw new Error('Unsupported: local root deletion')
|
|
159
|
-
},
|
|
160
|
-
deleteFile: fs.unlinkSync,
|
|
161
|
-
copyFile: fs.copyFileSync,
|
|
162
|
-
move: fs.renameSync,
|
|
163
|
-
openInputFile: input,
|
|
164
|
-
openInputStream: input,
|
|
165
|
-
openOutputStream(location) {
|
|
166
|
-
return output(location, false)
|
|
167
|
-
},
|
|
168
|
-
openAppendStream(location) {
|
|
169
|
-
return output(location, true)
|
|
170
|
-
},
|
|
171
|
-
}
|
|
194
|
+
}
|
|
172
195
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
const
|
|
196
|
+
const payload = Buffer.alloc(PAYLOAD_BYTES, 'symbol,price\n')
|
|
197
|
+
const source = path.join(root, 'source.bin')
|
|
198
|
+
fs.writeFileSync(source, payload)
|
|
199
|
+
const wrapperSink = path.join(root, 'sink-wrapper.bin')
|
|
200
|
+
const directSink = path.join(root, 'sink-direct.bin')
|
|
177
201
|
|
|
178
|
-
const
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
for (
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
fs.writeFileSync(path.join(leaf, 'part-0.parquet'), 'parquet')
|
|
202
|
+
const rows = 10_000
|
|
203
|
+
const ids = new Array(rows)
|
|
204
|
+
const symbols = new Array(rows)
|
|
205
|
+
for (let index = 0; index < rows; index += 1) {
|
|
206
|
+
ids[index] = BigInt(index)
|
|
207
|
+
symbols[index] = index % 2 === 0 ? 'AAPL' : 'MSFT'
|
|
185
208
|
}
|
|
186
|
-
|
|
209
|
+
const table = new arrow.Table({
|
|
210
|
+
id: arrow.vectorFromArray(ids, new arrow.Int64()),
|
|
211
|
+
symbol: arrow.vectorFromArray(symbols, new arrow.Utf8()),
|
|
212
|
+
})
|
|
213
|
+
const wrapperRecords = path.join(root, 'wrapper.arrows')
|
|
214
|
+
const directRecords = path.join(root, 'direct.arrows')
|
|
215
|
+
// Settle the fixture before measuring reads.
|
|
216
|
+
const records = IOBase.fromFs(handler, wrapperRecords)
|
|
217
|
+
await records.overwriteArrowTable(table)
|
|
218
|
+
records.flush()
|
|
219
|
+
await new IOBase(directRecords).overwriteArrowTable(table)
|
|
187
220
|
|
|
188
|
-
const
|
|
189
|
-
const
|
|
190
|
-
|
|
191
|
-
const wrapperSink = path.join(root, 'sink-wrapper.bin')
|
|
192
|
-
const directSink = path.join(root, 'sink-direct.bin')
|
|
221
|
+
const handle = IOBase.fromFs(handler, source)
|
|
222
|
+
const folder = IOBase.fromFs(handler, lake)
|
|
223
|
+
const localFolder = new IOBase(lake)
|
|
193
224
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
for (let index = 0; index < rows; index += 1) {
|
|
198
|
-
ids[index] = BigInt(index)
|
|
199
|
-
symbols[index] = index % 2 === 0 ? 'AAPL' : 'MSFT'
|
|
200
|
-
}
|
|
201
|
-
const table = new arrow.Table({
|
|
202
|
-
id: arrow.vectorFromArray(ids, new arrow.Int64()),
|
|
203
|
-
symbol: arrow.vectorFromArray(symbols, new arrow.Utf8()),
|
|
204
|
-
})
|
|
205
|
-
const wrapperRecords = path.join(root, 'wrapper.arrows')
|
|
206
|
-
const directRecords = path.join(root, 'direct.arrows')
|
|
207
|
-
// Settle the fixture before measuring reads.
|
|
208
|
-
const records = IOBase.fromFs(handler, wrapperRecords)
|
|
209
|
-
records.overwriteArrowTable(table)
|
|
210
|
-
records.flush()
|
|
211
|
-
new IOBase(directRecords).overwriteArrowTable(table)
|
|
225
|
+
// Construction: what holding a handler costs beside naming a local path.
|
|
226
|
+
await benchmark('fs/handle_from_path', () => IOBase.fromFs(handler, source))
|
|
227
|
+
await benchmark('local/handle_from_path', () => new IOBase(source))
|
|
212
228
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
const
|
|
229
|
+
// Output bytes are forwarded to the handler's writer as they arrive.
|
|
230
|
+
await benchmark('fs/write_bytes', () => {
|
|
231
|
+
const sink = IOBase.fromFs(handler, wrapperSink)
|
|
232
|
+
sink.writeBytes(payload)
|
|
233
|
+
sink.flush()
|
|
234
|
+
})
|
|
235
|
+
await benchmark('node:fs/write_bytes', () => fs.writeFileSync(directSink, payload))
|
|
216
236
|
|
|
217
|
-
|
|
218
|
-
benchmark('fs/
|
|
219
|
-
benchmark('local/handle_from_path', () => new IOBase(source))
|
|
237
|
+
await benchmark('fs/read_bytes', () => IOBase.fromFs(handler, source).readBytes())
|
|
238
|
+
await benchmark('node:fs/read_bytes', () => fs.readFileSync(source))
|
|
220
239
|
|
|
221
|
-
//
|
|
222
|
-
benchmark('fs/
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
240
|
+
// The row that says whether a range stayed a range.
|
|
241
|
+
await benchmark('fs/read_range_4k', () => handle.readRangeBytes(0, RANGE_BYTES))
|
|
242
|
+
await benchmark('node:fs/read_range_4k', () => {
|
|
243
|
+
const stream = input(source)
|
|
244
|
+
try {
|
|
245
|
+
return stream.readAt(0n, BigInt(RANGE_BYTES))
|
|
246
|
+
} finally {
|
|
247
|
+
stream.close()
|
|
248
|
+
}
|
|
249
|
+
})
|
|
228
250
|
|
|
229
|
-
benchmark('fs/
|
|
230
|
-
benchmark('node:fs/
|
|
251
|
+
await benchmark('fs/size', () => IOBase.fromFs(handler, source).size())
|
|
252
|
+
await benchmark('node:fs/size', () => fs.statSync(source).size)
|
|
231
253
|
|
|
232
|
-
|
|
233
|
-
benchmark('
|
|
234
|
-
benchmark('node:fs/
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
return stream.readAt(0n, BigInt(RANGE_BYTES))
|
|
238
|
-
} finally {
|
|
239
|
-
stream.close()
|
|
240
|
-
}
|
|
241
|
-
})
|
|
254
|
+
await benchmark('fs/list_children', () => folder.iterdir())
|
|
255
|
+
await benchmark('local/list_children', () => localFolder.iterdir())
|
|
256
|
+
await benchmark('node:fs/list_children', () =>
|
|
257
|
+
fs.readdirSync(lake, { withFileTypes: true }),
|
|
258
|
+
)
|
|
242
259
|
|
|
243
|
-
benchmark('fs/
|
|
244
|
-
benchmark('node:fs/
|
|
260
|
+
await benchmark('fs/glob_parquet', () => folder.rglob('*.parquet'))
|
|
261
|
+
await benchmark('node:fs/glob_parquet', () =>
|
|
262
|
+
fs
|
|
263
|
+
.readdirSync(lake, { recursive: true })
|
|
264
|
+
.filter((name) => name.endsWith('.parquet')),
|
|
265
|
+
)
|
|
245
266
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
)
|
|
267
|
+
// Records: the wrapper against the same core code over a mapped local file.
|
|
268
|
+
// A record read is thousands of times the work of a path lookup, so it counts
|
|
269
|
+
// in hundreds rather than thousands.
|
|
270
|
+
const recordIterations = Math.max(1, Math.floor(iterations / 20))
|
|
251
271
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
)
|
|
272
|
+
async function recordBenchmark(name, operation) {
|
|
273
|
+
for (let index = 0; index < Math.min(recordIterations, 10); index += 1) {
|
|
274
|
+
const result = operation()
|
|
275
|
+
if (result instanceof Promise) await result
|
|
276
|
+
}
|
|
277
|
+
const started = performance.now()
|
|
278
|
+
for (let index = 0; index < recordIterations; index += 1) {
|
|
279
|
+
const result = operation()
|
|
280
|
+
if (result instanceof Promise) await result
|
|
281
|
+
}
|
|
282
|
+
const elapsed = performance.now() - started
|
|
283
|
+
const rate = Math.round((recordIterations * rows * 1_000) / elapsed)
|
|
284
|
+
console.log(`${name}: ${rate.toLocaleString('en-US')} rows/second`)
|
|
285
|
+
}
|
|
258
286
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
287
|
+
await recordBenchmark('fs/read_records', () =>
|
|
288
|
+
IOBase.fromFs(handler, wrapperRecords).readArrowReader().intoTable(),
|
|
289
|
+
)
|
|
290
|
+
await recordBenchmark('local/read_records', () =>
|
|
291
|
+
new IOBase(directRecords).readArrowReader().intoTable(),
|
|
292
|
+
)
|
|
293
|
+
await recordBenchmark('fs/write_records', async () => {
|
|
294
|
+
const sink = IOBase.fromFs(handler, wrapperRecords)
|
|
295
|
+
await sink.overwriteArrowTable(table)
|
|
296
|
+
sink.flush()
|
|
297
|
+
})
|
|
298
|
+
await recordBenchmark('local/write_records', () =>
|
|
299
|
+
new IOBase(directRecords).overwriteArrowTable(table),
|
|
300
|
+
)
|
|
263
301
|
|
|
264
|
-
|
|
265
|
-
for (let index = 0; index < Math.min(recordIterations, 10); index += 1)
|
|
266
|
-
operation()
|
|
267
|
-
const started = performance.now()
|
|
268
|
-
for (let index = 0; index < recordIterations; index += 1) operation()
|
|
269
|
-
const elapsed = performance.now() - started
|
|
270
|
-
const rate = Math.round((recordIterations * rows * 1_000) / elapsed)
|
|
271
|
-
console.log(`${name}: ${rate.toLocaleString('en-US')} rows/second`)
|
|
302
|
+
fs.rmSync(root, { recursive: true, force: true })
|
|
272
303
|
}
|
|
273
304
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
recordBenchmark('local/read_records', () =>
|
|
278
|
-
new IOBase(directRecords).readArrowReader().intoTable(),
|
|
279
|
-
)
|
|
280
|
-
recordBenchmark('fs/write_records', () => {
|
|
281
|
-
const sink = IOBase.fromFs(handler, wrapperRecords)
|
|
282
|
-
sink.overwriteArrowTable(table)
|
|
283
|
-
sink.flush()
|
|
305
|
+
main().catch((error) => {
|
|
306
|
+
console.error(error)
|
|
307
|
+
process.exitCode = 1
|
|
284
308
|
})
|
|
285
|
-
recordBenchmark('local/write_records', () =>
|
|
286
|
-
new IOBase(directRecords).overwriteArrowTable(table),
|
|
287
|
-
)
|
|
288
|
-
|
|
289
|
-
fs.rmSync(root, { recursive: true, force: true })
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// How long each synchronous addon door blocks the JS thread. Each door is
|
|
4
|
+
// called 3 times (median reported) while a setInterval(1ms) ticker runs; the
|
|
5
|
+
// longest gap between ticks across the calls is the blocking evidence.
|
|
6
|
+
// Numbers come from whatever addon is built (the debug one by default).
|
|
7
|
+
// Sizes are shrunk with YGGIO_BLOCK_SCALE (default 1).
|
|
8
|
+
|
|
9
|
+
const fs = require('node:fs')
|
|
10
|
+
const { pathToFileURL } = require('node:url')
|
|
11
|
+
const os = require('node:os')
|
|
12
|
+
const path = require('node:path')
|
|
13
|
+
const { performance } = require('node:perf_hooks')
|
|
14
|
+
|
|
15
|
+
const { fix, json, gzip, zstd, Workbook, Sheet, ai } = require('yggio')
|
|
16
|
+
|
|
17
|
+
const scale = Number(process.env.YGGIO_BLOCK_SCALE ?? '1')
|
|
18
|
+
const MESSAGES = Math.round(100_000 * scale)
|
|
19
|
+
// the debug addon writes ~1.5 ms/message: 100k x 3 runs would not fit the budget
|
|
20
|
+
const WRITER_MESSAGES = Math.round(10_000 * scale)
|
|
21
|
+
const JSON_MB = Math.max(1, Math.round(50 * scale))
|
|
22
|
+
const CODING_MB = Math.max(1, Math.round(64 * scale))
|
|
23
|
+
const ROWS = Math.round(100_000 * scale)
|
|
24
|
+
const LIMITS = { maxNodes: 1e9, maxInputBytes: 1e10 } // the default node limit refuses 50 MB
|
|
25
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-block-'))
|
|
26
|
+
|
|
27
|
+
function median(values) {
|
|
28
|
+
return [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function sleep(ms) {
|
|
32
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function measure(door, today, size, prepare, call, note = '') {
|
|
36
|
+
const calls = []
|
|
37
|
+
const gaps = []
|
|
38
|
+
for (let round = 0; round < 3; round += 1) {
|
|
39
|
+
const state = prepare()
|
|
40
|
+
let last = performance.now()
|
|
41
|
+
let maxGap = 0
|
|
42
|
+
const ticker = setInterval(() => {
|
|
43
|
+
const now = performance.now()
|
|
44
|
+
maxGap = Math.max(maxGap, now - last)
|
|
45
|
+
last = now
|
|
46
|
+
}, 1)
|
|
47
|
+
await sleep(30) // let the ticker settle
|
|
48
|
+
last = performance.now()
|
|
49
|
+
maxGap = 0
|
|
50
|
+
const started = performance.now()
|
|
51
|
+
await call(state)
|
|
52
|
+
const elapsed = performance.now() - started
|
|
53
|
+
// a tick is only observable after the call returns: the gap is the block
|
|
54
|
+
await sleep(5)
|
|
55
|
+
maxGap = Math.max(maxGap, performance.now() - last)
|
|
56
|
+
clearInterval(ticker)
|
|
57
|
+
calls.push(elapsed)
|
|
58
|
+
gaps.push(maxGap)
|
|
59
|
+
}
|
|
60
|
+
const row = {
|
|
61
|
+
door,
|
|
62
|
+
today_returns: today,
|
|
63
|
+
size,
|
|
64
|
+
call_ms: Math.round(median(calls) * 10) / 10,
|
|
65
|
+
max_tick_gap_ms: Math.round(Math.max(...gaps) * 10) / 10,
|
|
66
|
+
note: note || 'synchronous: ticker starved for the whole call',
|
|
67
|
+
}
|
|
68
|
+
rows.push(row)
|
|
69
|
+
console.error(JSON.stringify(row))
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const rows = []
|
|
73
|
+
|
|
74
|
+
async function main() {
|
|
75
|
+
console.error(`addon: ${process.env.YGGIO_ADDON ?? 'built in node/ (debug unless release)'}; scale ${scale}`)
|
|
76
|
+
const SEED = path.join(__dirname, '..', '..', 'config', 'seeds', 'fix.parquet')
|
|
77
|
+
const options = new fix.MarketMessagesOptions(fix.FixRegistry.fromHandle(SEED))
|
|
78
|
+
const lines = Array.from({ length: MESSAGES }, (_, index) =>
|
|
79
|
+
Buffer.from(
|
|
80
|
+
`8=FIX.4.4|35=8|37=O1|11=C${index + 1}|17=E${index + 1}|150=F|39=2|40=2|59=1|54=1|55=AAPL|15=USD|` +
|
|
81
|
+
'32=10|31=101|30=XNAS|60=20261030-10:00:00|10=0|',
|
|
82
|
+
),
|
|
83
|
+
)
|
|
84
|
+
const handler = () => new fix.MarketMessages(options).fromLines(lines)
|
|
85
|
+
|
|
86
|
+
await measure('MarketMessages.intoSerie().length', 'JsSerie', `${MESSAGES} messages`, handler, async (m) => (await m.intoSerie()).length,
|
|
87
|
+
'intoSerie reads the stream on a worker thread and answers a Promise')
|
|
88
|
+
const writerLines = lines.slice(0, WRITER_MESSAGES)
|
|
89
|
+
await measure('MarketMessages.intoWriter', 'number', `${WRITER_MESSAGES} messages (shrunk from 100k, debug addon)`, () => new fix.MarketMessages(options).fromLines(writerLines), async (m) => {
|
|
90
|
+
let bytes = 0
|
|
91
|
+
await m.intoWriter({ write(chunk) { bytes += chunk.length } })
|
|
92
|
+
}, 'intoWriter reads and formats on a worker thread; the sink is called on the JS thread')
|
|
93
|
+
|
|
94
|
+
// 3. text codec path doors over a ~JSON_MB file
|
|
95
|
+
const jsonPath = path.join(tmp, 'big.json')
|
|
96
|
+
const record = { id: 1, venue: 'XNAS', symbol: 'AAPL', note: 'x'.repeat(60) }
|
|
97
|
+
const per = JSON.stringify(record).length + 1
|
|
98
|
+
const items = Math.ceil((JSON_MB * 1024 * 1024) / per)
|
|
99
|
+
const array = Array.from({ length: items }, (_, i) => ({ ...record, id: i }))
|
|
100
|
+
fs.writeFileSync(jsonPath, JSON.stringify(array))
|
|
101
|
+
const jsonSize = `${(fs.statSync(jsonPath).size / 1048576).toFixed(1)} MB json file`
|
|
102
|
+
await measure('json.load(path)', 'Promise', jsonSize, () => null, async () => await json.load(pathToFileURL(jsonPath), LIMITS), 'Promise: the native decodes off the JS thread')
|
|
103
|
+
const decoded = array
|
|
104
|
+
await measure('json.dump(value, path)', 'Promise', jsonSize, () => null, async () => await json.dump(decoded, pathToFileURL(path.join(tmp, 'out.json'))), 'Promise: the native encodes and writes off the JS thread')
|
|
105
|
+
|
|
106
|
+
// 4. coding
|
|
107
|
+
const payload = Buffer.from('{"id":1,"venue":"XNAS"}\n'.repeat(Math.ceil((CODING_MB * 1048576) / 24))).subarray(0, CODING_MB * 1048576)
|
|
108
|
+
for (const [name, coding] of [['gzip', gzip], ['zstd', zstd]]) {
|
|
109
|
+
const encoded = coding.dumps(payload)
|
|
110
|
+
await measure(`${name}.dumps`, 'Buffer', `${CODING_MB} MB`, () => null, () => coding.dumps(payload))
|
|
111
|
+
await measure(`${name}.loads`, 'Buffer', `${CODING_MB} MB decoded (${(encoded.length / 1048576).toFixed(2)} MB encoded)`, () => null, () => coding.loads(encoded))
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// 5. Excel
|
|
115
|
+
const sheet = new Sheet('Trades')
|
|
116
|
+
for (let r = 0; r < ROWS; r += 1) {
|
|
117
|
+
for (let c = 0; c < 5; c += 1) sheet.setCell([r, c], c === 0 ? `S${r}` : r * c + 0.5)
|
|
118
|
+
}
|
|
119
|
+
const workbook = new Workbook()
|
|
120
|
+
workbook.insertSheet(sheet)
|
|
121
|
+
const bytes = workbook.intoBytes()
|
|
122
|
+
await measure('Workbook.fromBytes', 'Workbook', `${ROWS} rows x 5 cols (${(bytes.length / 1048576).toFixed(1)} MB xlsx)`, () => null, () => Workbook.fromBytes(bytes),
|
|
123
|
+
'open indexes the archive only; sheets parse lazily on first access')
|
|
124
|
+
await measure('Workbook.fromBytes + sheet parse', 'Workbook', `${ROWS} rows x 5 cols`, () => null, () => Workbook.fromBytes(bytes).sheetAt(0).size,
|
|
125
|
+
'open plus first sheet access (the parse)')
|
|
126
|
+
await measure('Workbook.intoBytes', 'Buffer', `${ROWS} rows x 5 cols`, () => null, () => workbook.intoBytes())
|
|
127
|
+
|
|
128
|
+
// 6. ai
|
|
129
|
+
const model = ai.Model.card('org/big:q4')
|
|
130
|
+
const file = path.join(tmp, 'weights.bin')
|
|
131
|
+
fs.writeFileSync(file, Buffer.alloc(CODING_MB * 1048576, 7))
|
|
132
|
+
let n = 0
|
|
133
|
+
await measure('ai.ModelStore.put', '["created"|"updated"|"unchanged", Manifest]', `1 file of ${CODING_MB} MB`, () => null, () => {
|
|
134
|
+
const store = new ai.ModelStore(path.join(tmp, `models${n++}`))
|
|
135
|
+
store.put(model, 'org/big', 'r1', [['weights.bin', file]])
|
|
136
|
+
}, 'local copy plus digest; synchronous')
|
|
137
|
+
rows.push({ door: 'ai.Agent.run / runWith', today_returns: 'Promise<Reply>', size: 'n/a', call_ms: null, max_tick_gap_ms: null, note: 'already a Promise on a worker thread; needs a model, not measured' })
|
|
138
|
+
rows.push({ door: 'Lab.train / Weights.save', today_returns: 'n/a', size: 'n/a', call_ms: null, max_tick_gap_ms: null, note: 'no such door in node/index.d.ts' })
|
|
139
|
+
|
|
140
|
+
fs.rmSync(tmp, { recursive: true, force: true })
|
|
141
|
+
console.log(JSON.stringify({ file: 'node/benchmarks/js_thread_block.js', rows }, null, 1))
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
main().then(() => process.exit(0), (error) => { console.error(error); process.exit(1) })
|