yggio 0.0.16 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,117 +7,139 @@ const path = require('node:path')
7
7
  const arrow = require('apache-arrow')
8
8
  const { IOBase, MimeType, Timezone, Uri, Url } = require('yggio')
9
9
 
10
- const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
11
- const iterations = Number.parseInt(
12
- smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '20000'),
13
- 10,
14
- )
15
- if (!Number.isSafeInteger(iterations) || iterations <= 0) {
16
- throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
17
- }
18
- const warmups = smoke ? 0 : Math.min(iterations, 1_000)
10
+ async function main() {
19
11
 
20
- function benchmark(name, operation) {
21
- for (let index = 0; index < warmups; index += 1) operation()
22
- const started = performance.now()
23
- for (let index = 0; index < iterations; index += 1) operation()
24
- const elapsed = performance.now() - started
25
- const rate = Math.round((iterations * 1_000) / elapsed)
26
- console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
27
- }
12
+ const smoke = process.env.YGGIO_BENCH_SMOKE === '1'
13
+ const iterations = Number.parseInt(
14
+ smoke ? '1' : (process.env.YGGIO_BENCH_ITERATIONS ?? '20000'),
15
+ 10,
16
+ )
17
+ if (!Number.isSafeInteger(iterations) || iterations <= 0) {
18
+ throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
19
+ }
20
+ const warmups = smoke ? 0 : Math.min(iterations, 1_000)
28
21
 
29
- // One small Hive-partitioned lake, prepared outside every measured loop so the
30
- // numbers report the boundary crossing rather than the fixture.
31
- const root = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-bench-')), 'lake')
32
- for (const year of ['2024', '2025']) {
33
- for (const month of ['01', '02']) {
34
- const leaf = path.join(root, `year=${year}`, `month=${month}`)
35
- fs.mkdirSync(leaf, { recursive: true })
36
- fs.writeFileSync(path.join(leaf, 'part-0.parquet'), 'parquet')
22
+ async function benchmark(name, operation, count = iterations) {
23
+ for (let index = 0; index < Math.min(warmups, count); index += 1) {
24
+ const result = operation()
25
+ if (result instanceof Promise) await result
26
+ }
27
+ const started = performance.now()
28
+ for (let index = 0; index < count; index += 1) {
29
+ const result = operation()
30
+ if (result instanceof Promise) await result
31
+ }
32
+ const elapsed = performance.now() - started
33
+ const rate = Math.round((count * 1_000) / elapsed)
34
+ console.log(`${name}: ${rate.toLocaleString('en-US')} operations/second`)
37
35
  }
38
- }
39
36
 
40
- const lake = new IOBase(root)
41
- const leaf = lake.joinpath('year=2024', 'month=01', 'part-0.parquet')
42
- const memory = IOBase.fromBytes(Buffer.alloc(64 * 1_024))
43
- const cachedMemory = IOBase.fromBytes(Buffer.alloc(64 * 1_024)).buffered({
44
- pageSize: 4_096,
45
- maxBytes: 64 * 1_024,
46
- })
47
- const payload = Buffer.from('symbol,price\n')
48
- const url = Url.fromString('file:///lake/year=2024/month=01/part-0.parquet')
49
- const uri = Uri.fromString('s3://warehouse/year=2024/month=01')
50
- const zone = Timezone.fromString('America/New_York')
51
- const epoch = 1_720_000_000
52
- const dimensions = new IOBase(path.join(path.dirname(root), 'dimensions.arrows'))
53
- dimensions.overwriteArrowTable(
54
- new arrow.Table({
55
- id: arrow.vectorFromArray([1n, 2n, 3n], new arrow.Int64()),
56
- venue: arrow.vectorFromArray(['XNAS', 'XNYS', 'XLON'], new arrow.Utf8()),
57
- }),
58
- )
59
- // Open state is the caching contract: repeated metadata getters below measure
60
- // the JavaScript/native boundary and the core cache, never repeated decoding.
61
- dimensions.open()
62
- const structured = IOBase.fromBytes()
63
- structured.mediaType = MimeType.JSON
64
- const structuredValue = {
65
- quantity: 2,
66
- symbol: 'AAPL',
67
- }
68
- structured.writeScalar(structuredValue)
37
+ // One small Hive-partitioned lake, prepared outside every measured loop so the
38
+ // numbers report the boundary crossing rather than the fixture.
39
+ const root = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-bench-')), 'lake')
40
+ for (const year of ['2024', '2025']) {
41
+ for (const month of ['01', '02']) {
42
+ const leaf = path.join(root, `year=${year}`, `month=${month}`)
43
+ fs.mkdirSync(leaf, { recursive: true })
44
+ fs.writeFileSync(path.join(leaf, 'part-0.parquet'), 'parquet')
45
+ }
46
+ }
69
47
 
70
- // UTF-8 content, because the text answer decodes what it reads.
71
- const text = IOBase.fromBytes(Buffer.from('symbol,price\n'.repeat(1_024)))
72
- const appended = IOBase.fromBytes()
73
- const line = Buffer.from('AAPL,1\n')
48
+ const lake = new IOBase(root)
49
+ const leaf = lake.joinpath('year=2024', 'month=01', 'part-0.parquet')
50
+ const memory = IOBase.fromBytes(Buffer.alloc(64 * 1_024))
51
+ const cachedMemory = IOBase.fromBytes(Buffer.alloc(64 * 1_024)).buffered({
52
+ pageSize: 4_096,
53
+ maxBytes: 64 * 1_024,
54
+ })
55
+ const payload = Buffer.from('symbol,price\n')
56
+ const url = Url.fromString('file:///lake/year=2024/month=01/part-0.parquet')
57
+ const uri = Uri.fromString('s3://warehouse/year=2024/month=01')
58
+ const zone = Timezone.fromString('America/New_York')
59
+ const epoch = 1_720_000_000
60
+ const dimensions = new IOBase(path.join(path.dirname(root), 'dimensions.arrows'))
61
+ await dimensions.overwriteArrowTable(
62
+ new arrow.Table({
63
+ id: arrow.vectorFromArray([1n, 2n, 3n], new arrow.Int64()),
64
+ venue: arrow.vectorFromArray(['XNAS', 'XNYS', 'XLON'], new arrow.Utf8()),
65
+ }),
66
+ )
67
+ // Open state is the caching contract: repeated metadata getters below measure
68
+ // the JavaScript/native boundary and the core cache, never repeated decoding.
69
+ dimensions.open()
70
+ const structured = IOBase.fromBytes()
71
+ structured.mediaType = MimeType.JSON
72
+ const structuredValue = {
73
+ quantity: 2,
74
+ symbol: 'AAPL',
75
+ }
76
+ structured.writeScalar(structuredValue)
74
77
 
75
- benchmark('io/handle_from_path', () => new IOBase(root))
76
- benchmark('io/handle_from_url', () => IOBase.from(url))
77
- benchmark('io/join_child', () => lake.joinpath('year=2024', 'month=01'))
78
- benchmark('io/list_children', () => lake.iterdir())
79
- benchmark('io/glob_fixed_prefix', () => lake.glob('year=2024/**/*.parquet'))
80
- benchmark('io/children_where', () => lake.childrenWhere({ year: '2024' }))
81
- benchmark('io/partitions', () => leaf.partitions)
82
- benchmark('io/read_leaf_bytes', () => leaf.readBytes())
83
- // A folder streams its leaves through one core stream the iterator holds: a
84
- // first chunk inside the first leaf costs the listing and one open, the drain
85
- // one open per leaf.
86
- benchmark('io/folder_stream_first_chunk', () => lake.pstreamBytes(0, 4).next())
87
- benchmark('io/folder_stream_drain', () => {
88
- let bytes = 0
89
- for (const chunk of lake.pstreamBytes()) bytes += chunk.length
90
- return bytes
91
- })
92
- benchmark('io/memory_read_range_4k', () => memory.readRangeBytes(0, 4_096))
93
- benchmark('io/buffered_read_range_4k_hit', () => cachedMemory.readRangeBytes(0, 4_096))
94
- // The inferring entry points over those same natives: the delta is what the
95
- // boundary's type dispatch costs, nothing else.
96
- benchmark('io/memory_read_range_inferred_4k', () => memory.readRange(0, 4_096))
97
- benchmark('io/memory_read_range_text_4k', () => text.readRange(0, 4_096, { text: true }))
98
- benchmark('io/memory_append_bytes', () => appended.appendBytes(line))
99
- benchmark('io/memory_append_inferred', () => appended.append('AAPL,1\n'))
100
- benchmark('io/buffered_idempotent_redirect', () => cachedMemory.buffered({
101
- pageSize: 4_096,
102
- maxBytes: 64 * 1_024,
103
- }))
104
- benchmark('io/memory_pwrite', () => memory.pwrite(0, payload))
105
- benchmark('io/read_scalar', () => structured.readScalar())
106
- benchmark('io/read_exact_scalar', () => structured.readScalar({ scalar: true }))
107
- benchmark('io/write_scalar', () => structured.writeScalar(structuredValue))
108
- benchmark('io/kind', () => dimensions.kind())
109
- benchmark('io/is_io', () => dimensions.isIo())
110
- benchmark('io/row_size_open_cached', () => dimensions.rowSize())
111
- benchmark('io/column_size_open_cached', () => dimensions.columnSize())
112
- benchmark('uri/url_parts', () => url.parts)
113
- benchmark('uri/url_parents', () => url.parents)
114
- benchmark('uri/url_match', () => url.match('lake/**/part-?.parquet'))
115
- benchmark('uri/url_partitions', () => url.partitions)
116
- benchmark('uri/url_relative_to', () => url.relativeTo('file:///lake'))
117
- benchmark('uri/join_path', () => uri.joinPath('day=01', 'part-0.parquet'))
118
- benchmark('timezone/parse_alias', () => Timezone.fromString('US/Eastern'))
119
- benchmark('timezone/offset_at', () => zone.offsetAt(epoch))
120
- benchmark('timezone/abbreviation_at', () => zone.abbreviationAt(epoch))
78
+ // UTF-8 content, because the text answer decodes what it reads.
79
+ const text = IOBase.fromBytes(Buffer.from('symbol,price\n'.repeat(1_024)))
80
+ const appended = IOBase.fromBytes()
81
+ const line = Buffer.from('AAPL,1\n')
82
+
83
+ await benchmark('io/handle_from_path', () => new IOBase(root))
84
+ await benchmark('io/handle_from_url', () => IOBase.from(url))
85
+ await benchmark('io/join_child', () => lake.joinpath('year=2024', 'month=01'))
86
+ await benchmark('io/list_children', () => lake.iterdir())
87
+ await benchmark('io/glob_fixed_prefix', () => lake.glob('year=2024/**/*.parquet'))
88
+ await benchmark('io/children_where', () => lake.childrenWhere({ year: '2024' }))
89
+ await benchmark('io/partitions', () => leaf.partitions)
90
+ await benchmark('io/read_leaf_bytes', () => leaf.readBytes())
91
+ // A folder streams its leaves through one core stream the iterator holds: a
92
+ // first chunk inside the first leaf costs the listing and one open, the drain
93
+ // one open per leaf. An iterator left after its first chunk holds its leaf's
94
+ // open file until the GC collects it - `ByteIterator` has no `return()` - so
95
+ // the first-chunk row runs a bounded count, below any descriptor limit,
96
+ // rather than one open file per iteration of the whole run.
97
+ const FIRST_CHUNK_ITERATIONS = Math.min(iterations, 256)
98
+ await benchmark(
99
+ 'io/folder_stream_first_chunk',
100
+ () => lake.pstreamBytes(0, 4).next(),
101
+ FIRST_CHUNK_ITERATIONS,
102
+ )
103
+ await benchmark('io/folder_stream_drain', () => {
104
+ let bytes = 0
105
+ for (const chunk of lake.pstreamBytes()) bytes += chunk.length
106
+ return bytes
107
+ })
108
+ await benchmark('io/memory_read_range_4k', () => memory.readRangeBytes(0, 4_096))
109
+ await benchmark('io/buffered_read_range_4k_hit', () => cachedMemory.readRangeBytes(0, 4_096))
110
+ // The inferring entry points over those same natives: the delta is what the
111
+ // boundary's type dispatch costs, nothing else.
112
+ await benchmark('io/memory_read_range_inferred_4k', () => memory.readRange(0, 4_096))
113
+ await benchmark('io/memory_read_range_text_4k', () => text.readRange(0, 4_096, { text: true }))
114
+ await benchmark('io/memory_append_bytes', () => appended.appendBytes(line))
115
+ await benchmark('io/memory_append_inferred', () => appended.append('AAPL,1\n'))
116
+ await benchmark('io/buffered_idempotent_redirect', () => cachedMemory.buffered({
117
+ pageSize: 4_096,
118
+ maxBytes: 64 * 1_024,
119
+ }))
120
+ await benchmark('io/memory_pwrite', () => memory.pwrite(0, payload))
121
+ await benchmark('io/read_scalar', () => structured.readScalar())
122
+ await benchmark('io/read_exact_scalar', () => structured.readScalar({ scalar: true }))
123
+ await benchmark('io/write_scalar', () => structured.writeScalar(structuredValue))
124
+ await benchmark('io/kind', () => dimensions.kind())
125
+ await benchmark('io/is_io', () => dimensions.isIo())
126
+ await benchmark('io/row_size_open_cached', () => dimensions.rowSize())
127
+ await benchmark('io/column_size_open_cached', () => dimensions.columnSize())
128
+ await benchmark('uri/url_parts', () => url.parts)
129
+ await benchmark('uri/url_parents', () => url.parents)
130
+ await benchmark('uri/url_match', () => url.match('lake/**/part-?.parquet'))
131
+ await benchmark('uri/url_partitions', () => url.partitions)
132
+ await benchmark('uri/url_relative_to', () => url.relativeTo('file:///lake'))
133
+ await benchmark('uri/join_path', () => uri.joinPath('day=01', 'part-0.parquet'))
134
+ await benchmark('timezone/parse_alias', () => Timezone.fromString('US/Eastern'))
135
+ await benchmark('timezone/offset_at', () => zone.offsetAt(epoch))
136
+ await benchmark('timezone/abbreviation_at', () => zone.abbreviationAt(epoch))
121
137
 
122
- dimensions.close()
123
- fs.rmSync(path.dirname(root), { recursive: true, force: true })
138
+ dimensions.close()
139
+ fs.rmSync(path.dirname(root), { recursive: true, force: true })
140
+ }
141
+
142
+ main().catch((error) => {
143
+ console.error(error)
144
+ process.exitCode = 1
145
+ })