yggio 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +334 -0
- package/README.md +45 -38
- package/benchmarks/expression.js +44 -0
- package/benchmarks/fix.js +13 -1
- package/benchmarks/fix_parse.js +51 -0
- package/benchmarks/fix_view.js +45 -0
- package/benchmarks/holder/io.js +129 -107
- package/benchmarks/holder.js +247 -228
- package/benchmarks/js_thread_block.js +144 -0
- package/benchmarks/media.js +341 -349
- package/benchmarks/serie_arrow.js +58 -0
- package/benchmarks/text.js +9 -6
- package/benchmarks/types.js +185 -186
- package/binding.d.ts +566 -197
- package/binding.js +753 -786
- package/fields.js +45 -49
- package/index.d.ts +2240 -594
- package/index.js +71 -14
- package/package.json +10 -5
- package/properties.js +13 -1
- package/records.js +169 -159
- package/values.js +160 -0
package/benchmarks/holder/io.js
CHANGED
|
@@ -7,117 +7,139 @@ const path = require('node:path')
|
|
|
7
7
|
const arrow = require('apache-arrow')
|
|
8
8
|
const { IOBase, MimeType, Timezone, Uri, Url } = require('yggio')
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
const iterations = Number.parseInt(
|
|
12
|
-
smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '20000'),
|
|
13
|
-
10,
|
|
14
|
-
)
|
|
15
|
-
if (!Number.isSafeInteger(iterations) || iterations <= 0) {
|
|
16
|
-
throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
|
|
17
|
-
}
|
|
18
|
-
const warmups = smoke ? 0 : Math.min(iterations, 1_000)
|
|
10
|
+
async function main() {
|
|
19
11
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
}
|
|
12
|
+
const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
|
|
13
|
+
const iterations = Number.parseInt(
|
|
14
|
+
smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '20000'),
|
|
15
|
+
10,
|
|
16
|
+
)
|
|
17
|
+
if (!Number.isSafeInteger(iterations) || iterations <= 0) {
|
|
18
|
+
throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
|
|
19
|
+
}
|
|
20
|
+
const warmups = smoke ? 0 : Math.min(iterations, 1_000)
|
|
28
21
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
22
|
+
async function benchmark(name, operation, count = iterations) {
|
|
23
|
+
for (let index = 0; index < Math.min(warmups, count); index += 1) {
|
|
24
|
+
const result = operation()
|
|
25
|
+
if (result instanceof Promise) await result
|
|
26
|
+
}
|
|
27
|
+
const started = performance.now()
|
|
28
|
+
for (let index = 0; index < count; index += 1) {
|
|
29
|
+
const result = operation()
|
|
30
|
+
if (result instanceof Promise) await result
|
|
31
|
+
}
|
|
32
|
+
const elapsed = performance.now() - started
|
|
33
|
+
const rate = Math.round((count * 1_000) / elapsed)
|
|
34
|
+
console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
|
|
37
35
|
}
|
|
38
|
-
}
|
|
39
36
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
})
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
const zone = Timezone.fromString('America/New_York')
|
|
51
|
-
const epoch = 1_720_000_000
|
|
52
|
-
const dimensions = new IOBase(path.join(path.dirname(root), 'dimensions.arrows'))
|
|
53
|
-
dimensions.overwriteArrowTable(
|
|
54
|
-
new arrow.Table({
|
|
55
|
-
id: arrow.vectorFromArray([1n, 2n, 3n], new arrow.Int64()),
|
|
56
|
-
venue: arrow.vectorFromArray(['XNAS', 'XNYS', 'XLON'], new arrow.Utf8()),
|
|
57
|
-
}),
|
|
58
|
-
)
|
|
59
|
-
// Open state is the caching contract: repeated metadata getters below measure
|
|
60
|
-
// the JavaScript/native boundary and the core cache, never repeated decoding.
|
|
61
|
-
dimensions.open()
|
|
62
|
-
const structured = IOBase.fromBytes()
|
|
63
|
-
structured.mediaType = MimeType.JSON
|
|
64
|
-
const structuredValue = {
|
|
65
|
-
quantity: 2,
|
|
66
|
-
symbol: 'AAPL',
|
|
67
|
-
}
|
|
68
|
-
structured.writeScalar(structuredValue)
|
|
37
|
+
// One small Hive-partitioned lake, prepared outside every measured loop so the
|
|
38
|
+
// numbers report the boundary crossing rather than the fixture.
|
|
39
|
+
const root = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-bench-')), 'lake')
|
|
40
|
+
for (const year of ['2024', '2025']) {
|
|
41
|
+
for (const month of ['01', '02']) {
|
|
42
|
+
const leaf = path.join(root, `year=${year}`, `month=${month}`)
|
|
43
|
+
fs.mkdirSync(leaf, { recursive: true })
|
|
44
|
+
fs.writeFileSync(path.join(leaf, 'part-0.parquet'), 'parquet')
|
|
45
|
+
}
|
|
46
|
+
}
|
|
69
47
|
|
|
70
|
-
|
|
71
|
-
const
|
|
72
|
-
const
|
|
73
|
-
const
|
|
48
|
+
const lake = new IOBase(root)
|
|
49
|
+
const leaf = lake.joinpath('year=2024', 'month=01', 'part-0.parquet')
|
|
50
|
+
const memory = IOBase.fromBytes(Buffer.alloc(64 * 1_024))
|
|
51
|
+
const cachedMemory = IOBase.fromBytes(Buffer.alloc(64 * 1_024)).buffered({
|
|
52
|
+
pageSize: 4_096,
|
|
53
|
+
maxBytes: 64 * 1_024,
|
|
54
|
+
})
|
|
55
|
+
const payload = Buffer.from('symbol,price\n')
|
|
56
|
+
const url = Url.fromString('file:///lake/year=2024/month=01/part-0.parquet')
|
|
57
|
+
const uri = Uri.fromString('s3://warehouse/year=2024/month=01')
|
|
58
|
+
const zone = Timezone.fromString('America/New_York')
|
|
59
|
+
const epoch = 1_720_000_000
|
|
60
|
+
const dimensions = new IOBase(path.join(path.dirname(root), 'dimensions.arrows'))
|
|
61
|
+
await dimensions.overwriteArrowTable(
|
|
62
|
+
new arrow.Table({
|
|
63
|
+
id: arrow.vectorFromArray([1n, 2n, 3n], new arrow.Int64()),
|
|
64
|
+
venue: arrow.vectorFromArray(['XNAS', 'XNYS', 'XLON'], new arrow.Utf8()),
|
|
65
|
+
}),
|
|
66
|
+
)
|
|
67
|
+
// Open state is the caching contract: repeated metadata getters below measure
|
|
68
|
+
// the JavaScript/native boundary and the core cache, never repeated decoding.
|
|
69
|
+
dimensions.open()
|
|
70
|
+
const structured = IOBase.fromBytes()
|
|
71
|
+
structured.mediaType = MimeType.JSON
|
|
72
|
+
const structuredValue = {
|
|
73
|
+
quantity: 2,
|
|
74
|
+
symbol: 'AAPL',
|
|
75
|
+
}
|
|
76
|
+
structured.writeScalar(structuredValue)
|
|
74
77
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
benchmark('io/
|
|
81
|
-
benchmark('io/
|
|
82
|
-
benchmark('io/
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
benchmark('io/
|
|
87
|
-
benchmark('io/
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
benchmark('io/
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
benchmark('io/
|
|
106
|
-
benchmark('io/
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
benchmark('io/
|
|
110
|
-
benchmark('io/
|
|
111
|
-
benchmark('io/
|
|
112
|
-
benchmark('
|
|
113
|
-
benchmark('
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
benchmark('
|
|
118
|
-
benchmark('
|
|
119
|
-
benchmark('
|
|
120
|
-
benchmark('
|
|
78
|
+
// UTF-8 content, because the text answer decodes what it reads.
|
|
79
|
+
const text = IOBase.fromBytes(Buffer.from('symbol,price\n'.repeat(1_024)))
|
|
80
|
+
const appended = IOBase.fromBytes()
|
|
81
|
+
const line = Buffer.from('AAPL,1\n')
|
|
82
|
+
|
|
83
|
+
await benchmark('io/handle_from_path', () => new IOBase(root))
|
|
84
|
+
await benchmark('io/handle_from_url', () => IOBase.from(url))
|
|
85
|
+
await benchmark('io/join_child', () => lake.joinpath('year=2024', 'month=01'))
|
|
86
|
+
await benchmark('io/list_children', () => lake.iterdir())
|
|
87
|
+
await benchmark('io/glob_fixed_prefix', () => lake.glob('year=2024/**/*.parquet'))
|
|
88
|
+
await benchmark('io/children_where', () => lake.childrenWhere({ year: '2024' }))
|
|
89
|
+
await benchmark('io/partitions', () => leaf.partitions)
|
|
90
|
+
await benchmark('io/read_leaf_bytes', () => leaf.readBytes())
|
|
91
|
+
// A folder streams its leaves through one core stream the iterator holds: a
|
|
92
|
+
// first chunk inside the first leaf costs the listing and one open, the drain
|
|
93
|
+
// one open per leaf. An iterator left after its first chunk holds its leaf's
|
|
94
|
+
// open file until the GC collects it - `ByteIterator` has no `return()` - so
|
|
95
|
+
// the first-chunk row runs a bounded count, below any descriptor limit,
|
|
96
|
+
// rather than one open file per iteration of the whole run.
|
|
97
|
+
const FIRST_CHUNK_ITERATIONS = Math.min(iterations, 256)
|
|
98
|
+
await benchmark(
|
|
99
|
+
'io/folder_stream_first_chunk',
|
|
100
|
+
() => lake.pstreamBytes(0, 4).next(),
|
|
101
|
+
FIRST_CHUNK_ITERATIONS,
|
|
102
|
+
)
|
|
103
|
+
await benchmark('io/folder_stream_drain', () => {
|
|
104
|
+
let bytes = 0
|
|
105
|
+
for (const chunk of lake.pstreamBytes()) bytes += chunk.length
|
|
106
|
+
return bytes
|
|
107
|
+
})
|
|
108
|
+
await benchmark('io/memory_read_range_4k', () => memory.readRangeBytes(0, 4_096))
|
|
109
|
+
await benchmark('io/buffered_read_range_4k_hit', () => cachedMemory.readRangeBytes(0, 4_096))
|
|
110
|
+
// The inferring entry points over those same natives: the delta is what the
|
|
111
|
+
// boundary's type dispatch costs, nothing else.
|
|
112
|
+
await benchmark('io/memory_read_range_inferred_4k', () => memory.readRange(0, 4_096))
|
|
113
|
+
await benchmark('io/memory_read_range_text_4k', () => text.readRange(0, 4_096, { text: true }))
|
|
114
|
+
await benchmark('io/memory_append_bytes', () => appended.appendBytes(line))
|
|
115
|
+
await benchmark('io/memory_append_inferred', () => appended.append('AAPL,1\n'))
|
|
116
|
+
await benchmark('io/buffered_idempotent_redirect', () => cachedMemory.buffered({
|
|
117
|
+
pageSize: 4_096,
|
|
118
|
+
maxBytes: 64 * 1_024,
|
|
119
|
+
}))
|
|
120
|
+
await benchmark('io/memory_pwrite', () => memory.pwrite(0, payload))
|
|
121
|
+
await benchmark('io/read_scalar', () => structured.readScalar())
|
|
122
|
+
await benchmark('io/read_exact_scalar', () => structured.readScalar({ scalar: true }))
|
|
123
|
+
await benchmark('io/write_scalar', () => structured.writeScalar(structuredValue))
|
|
124
|
+
await benchmark('io/kind', () => dimensions.kind())
|
|
125
|
+
await benchmark('io/is_io', () => dimensions.isIo())
|
|
126
|
+
await benchmark('io/row_size_open_cached', () => dimensions.rowSize())
|
|
127
|
+
await benchmark('io/column_size_open_cached', () => dimensions.columnSize())
|
|
128
|
+
await benchmark('uri/url_parts', () => url.parts)
|
|
129
|
+
await benchmark('uri/url_parents', () => url.parents)
|
|
130
|
+
await benchmark('uri/url_match', () => url.match('lake/**/part-?.parquet'))
|
|
131
|
+
await benchmark('uri/url_partitions', () => url.partitions)
|
|
132
|
+
await benchmark('uri/url_relative_to', () => url.relativeTo('file:///lake'))
|
|
133
|
+
await benchmark('uri/join_path', () => uri.joinPath('day=01', 'part-0.parquet'))
|
|
134
|
+
await benchmark('timezone/parse_alias', () => Timezone.fromString('US/Eastern'))
|
|
135
|
+
await benchmark('timezone/offset_at', () => zone.offsetAt(epoch))
|
|
136
|
+
await benchmark('timezone/abbreviation_at', () => zone.abbreviationAt(epoch))
|
|
121
137
|
|
|
122
|
-
dimensions.close()
|
|
123
|
-
fs.rmSync(path.dirname(root), { recursive: true, force: true })
|
|
138
|
+
dimensions.close()
|
|
139
|
+
fs.rmSync(path.dirname(root), { recursive: true, force: true })
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
main().catch((error) => {
|
|
143
|
+
console.error(error)
|
|
144
|
+
process.exitCode = 1
|
|
145
|
+
})
|