yggio 0.0.0-stage → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +395 -2
- package/benchmarks/coding.js +32 -0
- package/benchmarks/filesystem.js +201 -0
- package/benchmarks/fix.js +347 -0
- package/benchmarks/hashing/txhash.js +96 -0
- package/benchmarks/hashing/xxhash.js +152 -0
- package/benchmarks/holder/io.js +123 -0
- package/benchmarks/holder.js +289 -0
- package/benchmarks/http.js +73 -0
- package/benchmarks/logging.js +56 -0
- package/benchmarks/media/text.js +128 -0
- package/benchmarks/media.js +408 -0
- package/benchmarks/text.js +297 -0
- package/benchmarks/types/defaults.js +70 -0
- package/benchmarks/types.js +482 -0
- package/benchmarks/warehouse.js +117 -0
- package/binding.d.ts +5411 -0
- package/binding.js +7047 -0
- package/defaults.js +128 -0
- package/fields.js +385 -0
- package/index.d.ts +10265 -0
- package/index.js +669 -0
- package/package.json +77 -4
- package/properties.js +147 -0
- package/records.js +1336 -0
- package/values.js +147 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
202
|
+
|
package/README.md
CHANGED
|
@@ -1,3 +1,396 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Yggio for JavaScript
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
```javascript
|
|
4
|
+
const { json, toml, yaml } = require('yggio')
|
|
5
|
+
|
|
6
|
+
const bytes = json.dumps({ symbol: 'AAPL', price: 224.62 })
|
|
7
|
+
const trade = json.loads(bytes)
|
|
8
|
+
const config = toml.loads('symbol = "AAPL"\nprice = 224.62\n')
|
|
9
|
+
const documents = yaml.loadsAll(Buffer.from('---\nsymbol: AAPL\n---\nsymbol: MSFT\n'))
|
|
10
|
+
|
|
11
|
+
console.assert(Buffer.isBuffer(bytes))
|
|
12
|
+
console.assert(trade.symbol === 'AAPL')
|
|
13
|
+
console.assert(config.symbol === 'AAPL')
|
|
14
|
+
console.assert(documents[1].symbol === 'MSFT')
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
JSON, YAML, and TOML emit only each format's natural shapes, with no private
|
|
18
|
+
tags. Exact decimals and binary values use ordinary strings where needed;
|
|
19
|
+
values with no natural spelling are refused. Pass `{ field }` while decoding
|
|
20
|
+
to restore exact widths and types in Rust. A source string is document content;
|
|
21
|
+
use a `file:` URL or descriptor for a location. Native path I/O avoids an
|
|
22
|
+
existence probe and JavaScript whole-file staging. Streams preserve
|
|
23
|
+
backpressure and remain caller-owned.
|
|
24
|
+
|
|
25
|
+
```javascript
|
|
26
|
+
const { Scalar, fields, json } = require('yggio')
|
|
27
|
+
|
|
28
|
+
const price = Scalar.decimal(123456789012345678901234n, 4)
|
|
29
|
+
const field = fields.decimal256('price', 40, 4, { nullable: false })
|
|
30
|
+
const restored = json.loads(json.dumps(price), { field })
|
|
31
|
+
|
|
32
|
+
console.assert(restored.equals(price))
|
|
33
|
+
// Inference picks the narrowest width; the field restores the declared one.
|
|
34
|
+
console.assert(price.kind === 'd128')
|
|
35
|
+
console.assert(restored.kind === 'd256')
|
|
36
|
+
console.assert(typeof price.stableHash() === 'bigint')
|
|
37
|
+
console.assert(price.clone().compare(price) === 0)
|
|
38
|
+
console.assert(price.intoJson() === '"12345678901234567890.1234"')
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The width, unit, scale and zone are named on the type, and `DataType.scalar`
|
|
42
|
+
and `Field.scalar` read a value under it - `fields.datetime64('at', 'ns',
|
|
43
|
+
'UTC').scalar(1700000000123456789n)`, `new DataType('date32').scalar(19723)`.
|
|
44
|
+
`Scalar.from` infers when no type is named. Three factories remain, each for
|
|
45
|
+
something JavaScript cannot otherwise say: `decimal(coefficient, scale = 0)`,
|
|
46
|
+
because there is no decimal type; `duration(count, unit)`, which picks
|
|
47
|
+
`duration32` or `duration64` from the count itself; and `float(value, width =
|
|
48
|
+
64)`, because `100` and `100.0` are the same `Number`, so an integral float
|
|
49
|
+
has no literal and the type would read it as an integer. Plain objects become
|
|
50
|
+
sorted named Records; JavaScript `Map` remains a Mapping.
|
|
51
|
+
Immutable native values expose `equals`, `compare`, `stableHash`, and `clone`
|
|
52
|
+
whenever the Rust value has those semantics. JavaScript has no object hash
|
|
53
|
+
protocol, so `stableHash()` is the explicit deterministic `bigint`; JavaScript
|
|
54
|
+
`===` remains reference identity. `Expression` and `Statement` expose the
|
|
55
|
+
same native total structural order rather than a binding-side approximation.
|
|
56
|
+
`RecordOptions` exposes the same four methods over its encoding variant and
|
|
57
|
+
every current core setting; a clone is detached, so later mutation changes
|
|
58
|
+
only that copy's equality, order, and hash.
|
|
59
|
+
`Scalar.add`, `subtract`, `multiply`, `divide`, `remainder`, `negate`, and
|
|
60
|
+
`absolute` are checked native numeric operations. A plain JavaScript operand crosses
|
|
61
|
+
`Scalar.from` exactly once before the operation; containers and text are not
|
|
62
|
+
silently concatenated or coerced. Invalid operands throw `TypeError` with
|
|
63
|
+
`ERR_YGGIO_INVALID_ARITHMETIC`; overflow, zero division, and inexact decimal
|
|
64
|
+
results throw `RangeError` with distinct `ERR_YGGIO_ARITHMETIC_OVERFLOW`,
|
|
65
|
+
`ERR_YGGIO_DIVISION_BY_ZERO`, and `ERR_YGGIO_INEXACT_ARITHMETIC` codes.
|
|
66
|
+
Expression methods with the same names build
|
|
67
|
+
lazy native expression nodes instead of evaluating them; an `Expression` stays
|
|
68
|
+
native, a string parses as expression text, and another JavaScript value is
|
|
69
|
+
inferred once as a literal. Avro `fingerprint` follows Parsing Canonical Form,
|
|
70
|
+
while `equals`, `compare`, and `stableHash` retain logical annotations,
|
|
71
|
+
defaults, aliases, and extension attributes.
|
|
72
|
+
|
|
73
|
+
```javascript
|
|
74
|
+
const { avro } = require('yggio')
|
|
75
|
+
|
|
76
|
+
const schema = { type: 'array', items: 'long' }
|
|
77
|
+
const bytes = avro.dumps([[1, 2], [3]], schema, { source: 'readme' })
|
|
78
|
+
const blocks = avro.blocks(bytes, {
|
|
79
|
+
maxDepth: 8,
|
|
80
|
+
maxInputBytes: 1_048_576,
|
|
81
|
+
maxNodes: 10_000,
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
console.assert(blocks.metadata.source === 'readme')
|
|
85
|
+
console.assert(blocks.next().value.rows().length === 2)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The frozen `avro` namespace exposes native schemas, whole containers,
|
|
89
|
+
single-object framing, and a fused lazy compressed-block iterator. Optional
|
|
90
|
+
reader resolution and all decode limits cross in the same options object;
|
|
91
|
+
natural values still pass through the shared native `Scalar` conversion. A
|
|
92
|
+
whole decoded container is a detached `{schema, metadata, rows}` JavaScript
|
|
93
|
+
projection for natural destructuring and JSON use; only its native `schema`
|
|
94
|
+
member carries value protocols. Compressed blocks and their iterator are
|
|
95
|
+
one-shot operational objects.
|
|
96
|
+
|
|
97
|
+
```javascript
|
|
98
|
+
const { Readable, Writable } = require('node:stream')
|
|
99
|
+
const { json } = require('yggio')
|
|
100
|
+
|
|
101
|
+
async function main() {
|
|
102
|
+
const value = await json.load(Readable.from(['{"id":', '42}']))
|
|
103
|
+
await json.dump(value, new Writable({
|
|
104
|
+
write(chunk, _encoding, done) {
|
|
105
|
+
process.stdout.write(chunk, done)
|
|
106
|
+
},
|
|
107
|
+
}))
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
main()
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
`loadAll` redirects an async byte source to the lazy `loadAllStream` iterator;
|
|
114
|
+
`dumpAll` redirects a writable destination to `dumpAllStream`. Single-document
|
|
115
|
+
stream reads use one bounded buffer because Node's async reader cannot be
|
|
116
|
+
passed to synchronous Rust `Read`; multi-document reads and writes stay
|
|
117
|
+
incremental. That async boundary retains only document framing and
|
|
118
|
+
Node/WHATWG backpressure: each complete JSON Lines row or YAML document still
|
|
119
|
+
crosses the native codec for syntax, values, and local byte positions, with
|
|
120
|
+
parity tests against buffered core decoding. Iterators stop after their first
|
|
121
|
+
read or parse failure. Buffer, ArrayBuffer, SharedArrayBuffer, DataView, and
|
|
122
|
+
typed-array offsets are all preserved exactly.
|
|
123
|
+
|
|
124
|
+
TOML is deliberately single-document and exposes only `loads`, `load`,
|
|
125
|
+
`dumps`, `dump`, `loadStream`, and `dumpStream`. Generic `codec` operations
|
|
126
|
+
infer `.toml` paths in Rust or accept explicit `format: 'toml'` for content.
|
|
127
|
+
|
|
128
|
+
```javascript
|
|
129
|
+
const { DataType, Field, MediaType, MimeType, Uri } = require('yggio')
|
|
130
|
+
|
|
131
|
+
const type = DataType.fromString('struct<symbol:string,price:decimal(18,4)>')
|
|
132
|
+
const coarseClock = DataType.time('milliseconds')
|
|
133
|
+
const preciseClock = DataType.time('nanoseconds')
|
|
134
|
+
const field = new Field('trade', type, false, { source: 'book' })
|
|
135
|
+
field.setParquetFieldId(17)
|
|
136
|
+
field.setLocation('s3://warehouse/trades/data.arrow')
|
|
137
|
+
field.setProperty('postgres', 'type', 'jsonb')
|
|
138
|
+
const file = Uri.fromPath('C:\\data\\trades.arrow')
|
|
139
|
+
const encoded = Uri.fromString('https://example.test/trades.csv.gz')
|
|
140
|
+
const partition = Uri.fromString('s3://warehouse/trades').joinPath('day=2026-08-23')
|
|
141
|
+
|
|
142
|
+
console.assert(Field.fromString(field.toString()).equals(field))
|
|
143
|
+
console.assert(coarseClock.id === 'time32' && coarseClock.kind === 'temporal')
|
|
144
|
+
console.assert(preciseClock.id === 'time64' && preciseClock.kind === 'temporal')
|
|
145
|
+
console.assert(field.parquetFieldId === 17)
|
|
146
|
+
console.assert(field.get('PARQUET:field_id') === '17')
|
|
147
|
+
console.assert(field.location.scheme === 's3')
|
|
148
|
+
console.assert(field.getProperty('postgres', 'type') === 'jsonb')
|
|
149
|
+
console.assert(file.toString() === 'file:///C:/data/trades.arrow')
|
|
150
|
+
console.assert(file.intoPath() === 'C:/data/trades.arrow')
|
|
151
|
+
console.assert(partition.toString() === 's3://warehouse/trades/day=2026-08-23')
|
|
152
|
+
console.assert(encoded.mediaType.base.equals(MimeType.fromString('text/csv')))
|
|
153
|
+
encoded.setMediaType(MediaType.fromParts('application/json', ['application/gzip']))
|
|
154
|
+
console.assert(encoded.fileName === 'trades.json.gz')
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The Node-API package exposes native `DataType`, `Field`, `IOBase`, `MimeType`,
|
|
158
|
+
`MediaType`, `Timezone`, `Uri`, `Url`, `Urn`, and `Scalar` values, the record pair
|
|
159
|
+
`BatchReader` and `RecordOptions`, and the frozen `avro` and `iceberg`
|
|
160
|
+
namespaces over all of them.
|
|
161
|
+
Recursive parsing, validation, Arrow casting, structured scalar conversion, codec
|
|
162
|
+
conversion, ordering, stable hashing, metadata, and path normalization stay in
|
|
163
|
+
Rust.
|
|
164
|
+
`DataType.time(unit)` and `fields.time(name, unit)` require an explicit unit;
|
|
165
|
+
seconds/milliseconds select Time32 and microseconds/nanoseconds select Time64.
|
|
166
|
+
They accept the same native unit aliases as datatype parsing and reject
|
|
167
|
+
interval layouts.
|
|
168
|
+
|
|
169
|
+
```javascript
|
|
170
|
+
const { fields, intoField } = require('yggio')
|
|
171
|
+
|
|
172
|
+
let builds = 0
|
|
173
|
+
class Trade {
|
|
174
|
+
static get intoStructField() {
|
|
175
|
+
builds += 1
|
|
176
|
+
return fields.struct('Trade', [
|
|
177
|
+
fields.int64('id'),
|
|
178
|
+
fields.utf8('symbol'),
|
|
179
|
+
fields.decimal128('price', 18, 4),
|
|
180
|
+
], { nullable: false })
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const trade = intoField(Trade)
|
|
185
|
+
console.assert(intoField(new Trade()) === trade)
|
|
186
|
+
console.assert(builds === 1)
|
|
187
|
+
console.assert(trade.dtype.id === 'struct' && trade.dtype.kind === 'nested')
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
`intoField(value, name?)` is the one dynamic field converter. A class exposes
|
|
191
|
+
its root through the actual static getter `intoStructField`; the loader validates
|
|
192
|
+
both its property descriptor and its non-null native Struct `Field` result, then
|
|
193
|
+
memoizes that result per class. Passing a different `name` returns a renamed
|
|
194
|
+
clone and leaves the cached root untouched.
|
|
195
|
+
|
|
196
|
+
Apache Arrow JS conversion is an explicit copied IPC boundary, and a column
|
|
197
|
+
is what crosses it: a `Serie`. `Serie.fromArrowArray`, `Serie.fromArrowBatch`
|
|
198
|
+
and `Serie.fromArrowReader` land Arrow JS data under its own field or cast it
|
|
199
|
+
once into the one you pass, and `intoArrowScalar`, `intoArrowArray` and
|
|
200
|
+
`intoArrowBatch` hand a column back. A column is one value - `serie.intoScalar()`,
|
|
201
|
+
or `serie.scalar(0)` for one row - and values become a column through
|
|
202
|
+
`Serie.fromScalars(field, rows)`; `Scalar` has no Arrow door of its own. A
|
|
203
|
+
`BatchReader` read yields Arrow JS batches, and a write consumes a native
|
|
204
|
+
reader, Arrow JS table/batch/reader, IPC bytes, named columns, or plain rows
|
|
205
|
+
through that same reader path. `serie.cast` casts a column in hand,
|
|
206
|
+
`SerieReader.fromArrowReader` casts a stream batch by batch under one plan,
|
|
207
|
+
`SerieReader.fromSerie` streams a held column as the one record serie it is,
|
|
208
|
+
and `ArrowCastPlan.compile` holds one cast for every column of a layout.
|
|
209
|
+
Each takes `{ safe, representation }`, an absent answer taking the core's
|
|
210
|
+
default; whether a value may be absent is the target field's nullability. Run
|
|
211
|
+
`npm run bench:records` for the copied-IPC read, projection, cast, and write
|
|
212
|
+
paths.
|
|
213
|
+
|
|
214
|
+
```javascript
|
|
215
|
+
const { Serie, fields } = require('yggio')
|
|
216
|
+
|
|
217
|
+
const payload = fields.struct('payload', [
|
|
218
|
+
fields.int32('count', { nullable: false }),
|
|
219
|
+
fields.utf8('note', { nullable: true }),
|
|
220
|
+
], { nullable: false })
|
|
221
|
+
const value = payload.defaultJSValue()
|
|
222
|
+
|
|
223
|
+
console.assert(value[0] === 0 && value[1] === null)
|
|
224
|
+
console.assert(payload.defaultJSHint().constructor === Array)
|
|
225
|
+
console.assert(Serie.fromDefault(fields.int32('id', { nullable: false })).intoArrowScalar() === 0)
|
|
226
|
+
console.assert(
|
|
227
|
+
fields.uint8('small').intoSchemeCompat('spark').dtype.id === 'int16',
|
|
228
|
+
)
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Defaults are computed by the Rust core and projected into natural JavaScript
|
|
232
|
+
values. A Struct is one ordered array with one value per child; its cached hint
|
|
233
|
+
names `Array` without materializing a default value or a second Field model.
|
|
234
|
+
Hint identity is wrapper-local, not a global interning promise for equivalent
|
|
235
|
+
fields.
|
|
236
|
+
Arrow scalars use one copied one-row IPC projection into Apache Arrow JS, and
|
|
237
|
+
unsupported Arrow JS layouts fail without introducing a second scalar model.
|
|
238
|
+
Run `npm run bench:defaults` for JavaScript defaults, cached hints,
|
|
239
|
+
compatibility normalization, and Arrow-scalar IPC materialization.
|
|
240
|
+
|
|
241
|
+
```javascript
|
|
242
|
+
const { Field, MediaType, MimeType } = require('yggio')
|
|
243
|
+
|
|
244
|
+
const media = MediaType.fromParts(
|
|
245
|
+
MimeType.CSV,
|
|
246
|
+
new Set([MimeType.GZIP, MimeType.ZSTD]),
|
|
247
|
+
)
|
|
248
|
+
const field = new Field('payload', 'binary', false, {
|
|
249
|
+
'HTTPS:Content-Type': 'text/csv; charset=utf-8',
|
|
250
|
+
})
|
|
251
|
+
field.setMediaType(media)
|
|
252
|
+
|
|
253
|
+
console.assert(field.mimeType.equals(MimeType.CSV))
|
|
254
|
+
console.assert(field.contentEncoding === 'gzip, zstd')
|
|
255
|
+
console.assert(field.get('HTTPS:CONTENT-TYPE') === 'text/csv')
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
`MimeType` exposes every native known value as a frozen class constant and
|
|
259
|
+
accepts custom validated MIME names. `MediaType()` defaults to octet-stream;
|
|
260
|
+
its iterable adapters consume once and `encodings` returns a detached Array.
|
|
261
|
+
HTTP and HTTPS Field input shares canonical lowercase `http:*` Arrow metadata.
|
|
262
|
+
Raw headers preserve parameters, while typed MIME/media, exact unsigned
|
|
263
|
+
content length (`bigint`), and absolute HTTP Location accessors remain native.
|
|
264
|
+
Media pair updates are atomic and cache-aware.
|
|
265
|
+
|
|
266
|
+
```javascript
|
|
267
|
+
const { DataType, Field, fields } = require('yggio')
|
|
268
|
+
|
|
269
|
+
const id = fields.int32('id', { nullable: false })
|
|
270
|
+
const tags = fields.serie('tags', fields.utf8('item'))
|
|
271
|
+
const row = DataType.fromFields([id, tags])
|
|
272
|
+
|
|
273
|
+
console.assert(id instanceof Field)
|
|
274
|
+
console.assert(tags.dtype.id === 'serie')
|
|
275
|
+
console.assert(tags.dtype.kind === 'nested')
|
|
276
|
+
console.assert(row.id === 'struct')
|
|
277
|
+
// The list spellings the serie layouts had before still read as them.
|
|
278
|
+
console.assert(DataType.from('list<utf8>').equals(DataType.from('serie<utf8>')))
|
|
279
|
+
console.assert(id.showDiff(id) === '✓ equal')
|
|
280
|
+
console.assert([...id.showDiffs(fields.int64('id'))].length > 0)
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
The `fields` namespace covers every datatype variant and returns the same
|
|
284
|
+
generic native `Field`; TypeScript retains the exact kind/value view. Equality
|
|
285
|
+
can ignore recursive metadata with `equals(other, false)`, while `showDiffs`
|
|
286
|
+
returns a UTF-8 iterator and `showDiff` joins its terminal-readable lines.
|
|
287
|
+
|
|
288
|
+
Finite alternatives can use `DataType.variant(fields)` or
|
|
289
|
+
`fields.denseUnion(name, fields, options)`. Both assign dense Arrow Union type
|
|
290
|
+
IDs in declaration order and retain the ordinary `union` kind; union values
|
|
291
|
+
use the explicit `{ typeId, value }` shape. Bare `DataType.variant()` and
|
|
292
|
+
`fields.variant(name, options)` are different - the parenthesis disambiguates:
|
|
293
|
+
they build the first-class self-describing Variant datatype, which the Parquet
|
|
294
|
+
writer stores as its `VARIANT` logical type. PostgreSQL `json/jsonb` remains an
|
|
295
|
+
explicit adapter format, not an alias for either.
|
|
296
|
+
|
|
297
|
+
```javascript
|
|
298
|
+
const fs = require('node:fs')
|
|
299
|
+
const os = require('node:os')
|
|
300
|
+
const path = require('node:path')
|
|
301
|
+
const arrow = require('apache-arrow')
|
|
302
|
+
const { Field, IOBase, fields, iceberg } = require('yggio')
|
|
303
|
+
|
|
304
|
+
// A location is absolute: a handle is a URL, and a relative name names no root.
|
|
305
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'yggio-'))
|
|
306
|
+
|
|
307
|
+
const schema = iceberg.assignFieldIds(
|
|
308
|
+
fields.struct('row', [Field.from('id: int64'), Field.from('venue: utf8')], {
|
|
309
|
+
nullable: false,
|
|
310
|
+
}),
|
|
311
|
+
)
|
|
312
|
+
const rows = new arrow.Table({
|
|
313
|
+
id: arrow.vectorFromArray([1n, 2n], new arrow.Int64()),
|
|
314
|
+
venue: arrow.vectorFromArray(['XNAS', 'XNYS'], new arrow.Utf8()),
|
|
315
|
+
})
|
|
316
|
+
|
|
317
|
+
// A handle reads and writes records through the encoding its name declares.
|
|
318
|
+
const file = new IOBase(path.join(root, 'trades.parquet'))
|
|
319
|
+
const written = file.overwriteArrowTable(rows)
|
|
320
|
+
console.assert(written.toString() === 'read 2 rows, wrote 2, skipped 0')
|
|
321
|
+
console.assert(file.readArrowReader().intoTable().numRows === 2)
|
|
322
|
+
console.assert(file.isIo())
|
|
323
|
+
console.assert(file.rowSize === 2)
|
|
324
|
+
console.assert(file.columnSize === 2)
|
|
325
|
+
|
|
326
|
+
// An Iceberg table is a folder, and a folder is all it ever touches.
|
|
327
|
+
const table = iceberg.IcebergTable.create(path.join(root, 'trades'), schema, ['venue'])
|
|
328
|
+
table.append(rows)
|
|
329
|
+
console.assert(table.currentSnapshot.operation === 'append')
|
|
330
|
+
console.assert(table.dataFiles().length === 2)
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
`PartitionSpec`, `PartitionField`, `Snapshot`, `SnapshotRef`, `ManifestFile`,
|
|
334
|
+
`DataFile`, and `Compaction` are immutable native values with `equals`,
|
|
335
|
+
`compare`, `stableHash`, and `clone`. A `DataFile`'s partition spec
|
|
336
|
+
only names its tuple positions and is not part of the core file identity;
|
|
337
|
+
cloning retains that projection context. `PartitionSpec.fromJSON` accepts the
|
|
338
|
+
v1 field array or v2 object, while `intoJSON` and the standard `toJSON` hook
|
|
339
|
+
emit the v2 object through the shared native `Scalar`. `IcebergOptions` exposes
|
|
340
|
+
the same four value methods over its explicitly configured settings; a clone
|
|
341
|
+
is detached, and mutation naturally changes equality, order, and hash.
|
|
342
|
+
`ScanPlan` is an immutable bounded report with the same four methods. Its
|
|
343
|
+
complete identity, in comparison order, is `(recordCount, filesPlanned,
|
|
344
|
+
filesSkipped, manifestsRead, manifestsSkipped)`; physical task paths are not
|
|
345
|
+
part of the public report.
|
|
346
|
+
The other classes retain their complete Rust-core values, including snapshot
|
|
347
|
+
v3 lineage and manifest partition summaries, so their protocols never compare
|
|
348
|
+
a lossy JavaScript projection. Exact 64-bit identifiers cross as `bigint`.
|
|
349
|
+
|
|
350
|
+
`BatchReader` is the primitive record shape: `readArrowReader` returns one and
|
|
351
|
+
the `overwriteArrowReader`/`appendArrowReader`/`mergeArrowReader` methods consume
|
|
352
|
+
one. Table, record-batch, and record methods infer their input then redirect to
|
|
353
|
+
those primitives. Each Arrow JS crossing is copied IPC because Arrow JS has no
|
|
354
|
+
C Data consumer. The encoding is never an argument: `recordOptions()` derives
|
|
355
|
+
it from the handle's media type, covering Arrow IPC and Apache Parquet.
|
|
356
|
+
|
|
357
|
+
Configured intent uses `writeArrowReader`, `writeArrowTable`,
|
|
358
|
+
`writeArrowBatch`, or `writeRecords` with `(input, mode, options?)`.
|
|
359
|
+
The required mode is `'overwrite'`, `'append'`, or `'merge'` and is checked
|
|
360
|
+
before a one-shot reader, exporter, or iterable is inspected.
|
|
361
|
+
|
|
362
|
+
Every record write answers an `IOResult` - an async record write a promise of
|
|
363
|
+
one - stating the rows it read from its source (`readRows`), wrote
|
|
364
|
+
(`writtenRows`) and skipped (`skippedRows`: kept out by the options' `where`,
|
|
365
|
+
or cut off a last batch by a bound), as the core counted them. A write cut
|
|
366
|
+
into several commits answers their sum; `add` sums two results the same way.
|
|
367
|
+
|
|
368
|
+
`isIo()` is the general capability check: byte values and tabular media return
|
|
369
|
+
true, while a container holding neither returns false. `rowSize` and
|
|
370
|
+
`columnSize` are lazy metadata getters for the whole logical media, independent
|
|
371
|
+
of projections and read limits. Successful answers are cached by the Rust core
|
|
372
|
+
only while a handle is open, invalidated by writes through that handle, and
|
|
373
|
+
computed fresh after `close()`; JavaScript keeps no parallel count or schema.
|
|
374
|
+
Counts saturate at `Number.MAX_SAFE_INTEGER` rather than becoming imprecise.
|
|
375
|
+
|
|
376
|
+
Record iterables cross in bounded IPC chunks (`options.batchRowSize`, or
|
|
377
|
+
65,536 rows), each chunk one batch. Synchronous iterables are pulled lazily through one native reader. With
|
|
378
|
+
no `options.commitBatchNum`, async iterables spool those bounded chunks to a
|
|
379
|
+
private temporary file before the one native write, preserving the
|
|
380
|
+
destination's own cadence while bounding memory; the spool is removed on
|
|
381
|
+
success or failure.
|
|
382
|
+
|
|
383
|
+
A positive `options.commitBatchNum` publishes every that many whole batches,
|
|
384
|
+
then the remainder; a cadence never cuts a batch, so a chunk is the unit it
|
|
385
|
+
counts.
|
|
386
|
+
The async path alternates one awaited chunk with one opaque Rust-session push,
|
|
387
|
+
so every complete cadence is visible before the next source pull and a later
|
|
388
|
+
failure drops only the incomplete cadence. Global row and byte limits remain
|
|
389
|
+
one operation-wide budget. A zero limit does not inspect the source: append is
|
|
390
|
+
a synchronous no-op answering the empty `IOResult`, overwrite publishes an
|
|
391
|
+
explicitly typed empty value, and a limited merge is rejected.
|
|
392
|
+
|
|
393
|
+
Every page of the [documentation](https://platob.github.io/yggio/) shows its
|
|
394
|
+
operation in Rust, Python and JavaScript, so field conversion, typed rows,
|
|
395
|
+
copied IPC interoperability and codec format inference are each documented on
|
|
396
|
+
the page owning the vocabulary they belong to.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// A smoke-sized view of the native coding boundary. Increase the iteration
|
|
4
|
+
// count explicitly when collecting publication numbers.
|
|
5
|
+
|
|
6
|
+
const { performance } = require('node:perf_hooks')
|
|
7
|
+
const { gzip, zlib, zstd } = require('yggio')
|
|
8
|
+
|
|
9
|
+
const iterations = Number.parseInt(process.env.YGGIO_BENCH_ITERATIONS ?? '200', 10)
|
|
10
|
+
if (!Number.isSafeInteger(iterations) || iterations <= 0) {
|
|
11
|
+
throw new RangeError('YGGIO_BENCH_ITERATIONS must be a positive safe integer')
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const payload = Buffer.from('{"id":1,"venue":"XNAS"}\n'.repeat(100))
|
|
15
|
+
const cases = [
|
|
16
|
+
['gzip', gzip],
|
|
17
|
+
['zlib', zlib],
|
|
18
|
+
['zstd', zstd],
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
for (const [name, coding] of cases) {
|
|
22
|
+
const encoded = coding.dumps(payload)
|
|
23
|
+
for (const [direction, operation] of [
|
|
24
|
+
['encode', () => coding.dumps(payload)],
|
|
25
|
+
['decode', () => coding.loads(encoded)],
|
|
26
|
+
]) {
|
|
27
|
+
const started = performance.now()
|
|
28
|
+
for (let index = 0; index < iterations; index += 1) operation()
|
|
29
|
+
const nanoseconds = ((performance.now() - started) * 1_000_000) / iterations
|
|
30
|
+
console.log(`${name} ${direction}: ${nanoseconds.toFixed(1)} ns/op`)
|
|
31
|
+
}
|
|
32
|
+
}
|