yggio 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +334 -0
- package/README.md +43 -35
- package/benchmarks/expression.js +44 -0
- package/benchmarks/fix.js +13 -1
- package/benchmarks/fix_parse.js +51 -0
- package/benchmarks/fix_view.js +45 -0
- package/benchmarks/holder/io.js +129 -107
- package/benchmarks/holder.js +247 -228
- package/benchmarks/js_thread_block.js +144 -0
- package/benchmarks/media.js +341 -349
- package/benchmarks/serie_arrow.js +58 -0
- package/benchmarks/text.js +9 -6
- package/benchmarks/types.js +185 -185
- package/binding.d.ts +488 -68
- package/binding.js +362 -53
- package/fields.js +2 -2
- package/index.d.ts +1730 -78
- package/index.js +62 -8
- package/package.json +10 -5
- package/records.js +116 -127
- package/values.js +160 -0
package/binding.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// this loader only adapts JavaScript symbols and built-in collection inputs.
|
|
5
5
|
const fs = require('node:fs')
|
|
6
6
|
const { fileURLToPath, URL } = require('node:url')
|
|
7
|
-
const { types: utilTypes } = require('node:util')
|
|
7
|
+
const { promisify, types: utilTypes } = require('node:util')
|
|
8
8
|
const binding = require('./index.js')
|
|
9
9
|
const {
|
|
10
10
|
arrow,
|
|
@@ -15,6 +15,11 @@ const {
|
|
|
15
15
|
arrowVectorChunksIntoIPC,
|
|
16
16
|
arrowVectorFromIPC,
|
|
17
17
|
arrowVectorIntoIPC,
|
|
18
|
+
arrowBatchFromNative,
|
|
19
|
+
arrowBatchIntoNative,
|
|
20
|
+
arrowVectorFromNative,
|
|
21
|
+
arrowVectorIntoNative,
|
|
22
|
+
arrowVectorChunksIntoNative,
|
|
18
23
|
} = require('./values.js')
|
|
19
24
|
|
|
20
25
|
// Native methods taking serde_json::Scalar recurse through caller-owned JS
|
|
@@ -929,7 +934,13 @@ function sourceFilePath(value) {
|
|
|
929
934
|
return fileURLToPath(value)
|
|
930
935
|
}
|
|
931
936
|
|
|
932
|
-
|
|
937
|
+
const readAsync = promisify(fs.read)
|
|
938
|
+
const writeAsync = promisify(fs.write)
|
|
939
|
+
|
|
940
|
+
async function readDescriptorBounded(
|
|
941
|
+
descriptor,
|
|
942
|
+
maximum = DEFAULT_MAX_STREAM_BYTES,
|
|
943
|
+
) {
|
|
933
944
|
const chunks = []
|
|
934
945
|
let total = 0
|
|
935
946
|
while (true) {
|
|
@@ -938,7 +949,13 @@ function readDescriptorBounded(descriptor, maximum = DEFAULT_MAX_STREAM_BYTES) {
|
|
|
938
949
|
throw new RangeError(`input exceeds the ${maximum}-byte input limit`)
|
|
939
950
|
}
|
|
940
951
|
const chunk = Buffer.allocUnsafe(capacity)
|
|
941
|
-
const
|
|
952
|
+
const { bytesRead: length } = await readAsync(
|
|
953
|
+
descriptor,
|
|
954
|
+
chunk,
|
|
955
|
+
0,
|
|
956
|
+
capacity,
|
|
957
|
+
null,
|
|
958
|
+
)
|
|
942
959
|
if (length === 0) break
|
|
943
960
|
total += length
|
|
944
961
|
if (total > maximum) {
|
|
@@ -953,11 +970,11 @@ function readDescriptorBounded(descriptor, maximum = DEFAULT_MAX_STREAM_BYTES) {
|
|
|
953
970
|
|
|
954
971
|
function readSource(source, maximum = DEFAULT_MAX_STREAM_BYTES) {
|
|
955
972
|
if (typeof source === 'number') {
|
|
956
|
-
return { content:
|
|
973
|
+
return { content: null, path: null, descriptor: source }
|
|
957
974
|
}
|
|
958
975
|
const path = sourceFilePath(source)
|
|
959
|
-
if (path !== null) return { content: null, path }
|
|
960
|
-
return { content: toNativeContent(source), path: null }
|
|
976
|
+
if (path !== null) return { content: null, path, descriptor: null }
|
|
977
|
+
return { content: toNativeContent(source), path: null, descriptor: null }
|
|
961
978
|
}
|
|
962
979
|
|
|
963
980
|
function formatFor(source, options, fallback) {
|
|
@@ -1425,6 +1442,20 @@ Object.defineProperties(Scalar.prototype, {
|
|
|
1425
1442
|
},
|
|
1426
1443
|
})
|
|
1427
1444
|
|
|
1445
|
+
// The `copy` an Arrow export reads: `undefined` the default rule - lend a
|
|
1446
|
+
// buffer only where the column alone holds it - `true` copy everything,
|
|
1447
|
+
// `false` lend everything a write cannot fault on.
|
|
1448
|
+
function copyOption(options) {
|
|
1449
|
+
if (options === null || typeof options !== 'object') {
|
|
1450
|
+
throw new TypeError('Arrow export options must be an object')
|
|
1451
|
+
}
|
|
1452
|
+
const { copy } = options
|
|
1453
|
+
if (copy !== undefined && typeof copy !== 'boolean') {
|
|
1454
|
+
throw new TypeError(`copy must be a boolean, got ${typeof copy}`)
|
|
1455
|
+
}
|
|
1456
|
+
return copy
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1428
1459
|
// A serie is the native value; the conversions it needs from JavaScript -
|
|
1429
1460
|
// rows typed through the column's own field, Arrow JS through IPC - and the
|
|
1430
1461
|
// leaf classes nested columns are handed out as are defined here, over the
|
|
@@ -1436,10 +1467,13 @@ const nativeSerie = Object.freeze({
|
|
|
1436
1467
|
lit: NativeSerie._litNative.bind(NativeSerie),
|
|
1437
1468
|
fromArrowArray: NativeSerie._fromArrowArrayIpcNative.bind(NativeSerie),
|
|
1438
1469
|
fromArrowBatch: NativeSerie._fromArrowBatchIpcNative.bind(NativeSerie),
|
|
1470
|
+
fromArrowArrayData: NativeSerie._fromArrowArrayDataNative.bind(NativeSerie),
|
|
1471
|
+
fromArrowBatchData: NativeSerie._fromArrowBatchDataNative.bind(NativeSerie),
|
|
1439
1472
|
empty: NativeSerie._emptyNative.bind(NativeSerie),
|
|
1440
1473
|
withCapacity: NativeSerie._withCapacityNative.bind(NativeSerie),
|
|
1441
1474
|
fromArrowReader: NativeSerie._fromArrowReaderNative.bind(NativeSerie),
|
|
1442
1475
|
fromSortedSpans: NativeSerie._fromSortedSpansNative.bind(NativeSerie),
|
|
1476
|
+
fromSpillLease: NativeSerie._fromSpillLeaseNative.bind(NativeSerie),
|
|
1443
1477
|
...takePrivates(NativeSerie, {
|
|
1444
1478
|
intoChunks: '_intoChunksNative',
|
|
1445
1479
|
intoCast: '_intoCastNative',
|
|
@@ -1455,8 +1489,8 @@ const nativeSerie = Object.freeze({
|
|
|
1455
1489
|
setCell: '_setCellNative',
|
|
1456
1490
|
cast: '_castNative',
|
|
1457
1491
|
intoArrowScalar: '_intoArrowScalarIpcNative',
|
|
1458
|
-
|
|
1459
|
-
|
|
1492
|
+
intoArrowArrayData: '_intoArrowArrayDataNative',
|
|
1493
|
+
intoArrowBatchData: '_intoArrowBatchDataNative',
|
|
1460
1494
|
equals: '_equalsNative',
|
|
1461
1495
|
compare: '_compareNative',
|
|
1462
1496
|
offsets: '_offsetsNative',
|
|
@@ -1621,11 +1655,14 @@ const Serie = publicNativeClass(
|
|
|
1621
1655
|
'_litNative',
|
|
1622
1656
|
'_fromArrowArrayIpcNative',
|
|
1623
1657
|
'_fromArrowBatchIpcNative',
|
|
1658
|
+
'_fromArrowArrayDataNative',
|
|
1659
|
+
'_fromArrowBatchDataNative',
|
|
1624
1660
|
'_emptyNative',
|
|
1625
1661
|
'_withCapacityNative',
|
|
1626
1662
|
'_fromArrowReaderNative',
|
|
1627
1663
|
'_fromSourceNative',
|
|
1628
1664
|
'_fromSortedSpansNative',
|
|
1665
|
+
'_fromSpillLeaseNative',
|
|
1629
1666
|
]),
|
|
1630
1667
|
(args) => (args[0] == null ? [] : [serieValues(args[0], undefined, 'Serie values')]),
|
|
1631
1668
|
)
|
|
@@ -1801,12 +1838,23 @@ Object.defineProperties(Serie, {
|
|
|
1801
1838
|
fromArrowArray: {
|
|
1802
1839
|
configurable: true,
|
|
1803
1840
|
value(vector, field, options) {
|
|
1841
|
+
// One `Data` crosses by its buffers, each copied once; a vector of
|
|
1842
|
+
// several chunks, or a layout the native side reads no buffers of,
|
|
1843
|
+
// as IPC.
|
|
1844
|
+
const crossed = arrowVectorIntoNative(vector, 'Serie.fromArrowArray input')
|
|
1804
1845
|
return describedSerie(
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1846
|
+
crossed === null
|
|
1847
|
+
? nativeSerie.fromArrowArray(
|
|
1848
|
+
arrowVectorIntoIPC(vector, 'Serie.fromArrowArray input'),
|
|
1849
|
+
optionalField(field),
|
|
1850
|
+
...castOptionArgs(options),
|
|
1851
|
+
)
|
|
1852
|
+
: nativeSerie.fromArrowArrayData(
|
|
1853
|
+
crossed.schema,
|
|
1854
|
+
crossed.data,
|
|
1855
|
+
optionalField(field),
|
|
1856
|
+
...castOptionArgs(options),
|
|
1857
|
+
),
|
|
1810
1858
|
)
|
|
1811
1859
|
},
|
|
1812
1860
|
},
|
|
@@ -1815,6 +1863,17 @@ Object.defineProperties(Serie, {
|
|
|
1815
1863
|
fromArrowBatch: {
|
|
1816
1864
|
configurable: true,
|
|
1817
1865
|
value(batch, root, options) {
|
|
1866
|
+
const crossed = arrowBatchIntoNative(batch, 'Serie.fromArrowBatch input')
|
|
1867
|
+
if (crossed !== null) {
|
|
1868
|
+
return describedSerie(
|
|
1869
|
+
nativeSerie.fromArrowBatchData(
|
|
1870
|
+
crossed.schema,
|
|
1871
|
+
crossed.data,
|
|
1872
|
+
optionalField(root),
|
|
1873
|
+
...castOptionArgs(options),
|
|
1874
|
+
),
|
|
1875
|
+
)
|
|
1876
|
+
}
|
|
1818
1877
|
const bytes = arrow().isArrowTable(batch)
|
|
1819
1878
|
? arrowTableIntoIPC(batch, 'Serie.fromArrowBatch input')
|
|
1820
1879
|
: arrowBatchIntoIPC(batch, 'Serie.fromArrowBatch input')
|
|
@@ -1972,18 +2031,18 @@ Object.defineProperties(Serie.prototype, {
|
|
|
1972
2031
|
},
|
|
1973
2032
|
intoArrowArray: {
|
|
1974
2033
|
configurable: true,
|
|
1975
|
-
value() {
|
|
1976
|
-
return
|
|
1977
|
-
Reflect.apply(nativeSerie.
|
|
2034
|
+
value(options = {}) {
|
|
2035
|
+
return arrowVectorFromNative(
|
|
2036
|
+
Reflect.apply(nativeSerie.intoArrowArrayData, this, [copyOption(options)]),
|
|
1978
2037
|
'Serie.intoArrowArray output',
|
|
1979
2038
|
)
|
|
1980
2039
|
},
|
|
1981
2040
|
},
|
|
1982
2041
|
intoArrowBatch: {
|
|
1983
2042
|
configurable: true,
|
|
1984
|
-
value() {
|
|
1985
|
-
return
|
|
1986
|
-
Reflect.apply(nativeSerie.
|
|
2043
|
+
value(options = {}) {
|
|
2044
|
+
return arrowBatchFromNative(
|
|
2045
|
+
Reflect.apply(nativeSerie.intoArrowBatchData, this, [copyOption(options)]),
|
|
1987
2046
|
'Serie.intoArrowBatch output',
|
|
1988
2047
|
)
|
|
1989
2048
|
},
|
|
@@ -2513,6 +2572,7 @@ const nativeChunkedSerie = Object.freeze({
|
|
|
2513
2572
|
fromSerie: NativeChunkedSerie._fromSerieNative.bind(NativeChunkedSerie),
|
|
2514
2573
|
fromSeries: NativeChunkedSerie._fromSeriesNative.bind(NativeChunkedSerie),
|
|
2515
2574
|
fromArrowArray: NativeChunkedSerie._fromArrowArrayIpcNative.bind(NativeChunkedSerie),
|
|
2575
|
+
fromArrowArrayData: NativeChunkedSerie._fromArrowArrayDataNative.bind(NativeChunkedSerie),
|
|
2516
2576
|
fromArrowBatch: NativeChunkedSerie._fromArrowBatchIpcNative.bind(NativeChunkedSerie),
|
|
2517
2577
|
fromArrowReader: NativeChunkedSerie._fromArrowReaderNative.bind(NativeChunkedSerie),
|
|
2518
2578
|
...takePrivates(NativeChunkedSerie, {
|
|
@@ -2523,7 +2583,7 @@ const nativeChunkedSerie = Object.freeze({
|
|
|
2523
2583
|
pushChunk: '_pushChunkNative',
|
|
2524
2584
|
intoSerie: '_intoSerieNative',
|
|
2525
2585
|
cast: '_castNative',
|
|
2526
|
-
|
|
2586
|
+
intoArrowArrayData: '_intoArrowArrayDataNative',
|
|
2527
2587
|
equals: '_equalsNative',
|
|
2528
2588
|
compare: '_compareNative',
|
|
2529
2589
|
sortIndices: '_sortIndicesNative',
|
|
@@ -2553,6 +2613,7 @@ const ChunkedSerie = publicNativeClass(
|
|
|
2553
2613
|
'_emptyNative',
|
|
2554
2614
|
'_fromSerieNative',
|
|
2555
2615
|
'_fromSeriesNative',
|
|
2616
|
+
'_fromArrowArrayDataNative',
|
|
2556
2617
|
'_fromArrowArrayIpcNative',
|
|
2557
2618
|
'_fromArrowBatchIpcNative',
|
|
2558
2619
|
'_fromArrowReaderNative',
|
|
@@ -2605,6 +2666,17 @@ Object.defineProperties(ChunkedSerie, {
|
|
|
2605
2666
|
fromArrowArray: {
|
|
2606
2667
|
configurable: true,
|
|
2607
2668
|
value(vector, field, options) {
|
|
2669
|
+
// Each `Data` crosses by its buffers, copied once; an empty one, or a
|
|
2670
|
+
// layout the native side reads no buffers of, sends the vector as IPC.
|
|
2671
|
+
const crossed = arrowVectorChunksIntoNative(vector, 'ChunkedSerie.fromArrowArray input')
|
|
2672
|
+
if (crossed !== null) {
|
|
2673
|
+
return nativeChunkedSerie.fromArrowArrayData(
|
|
2674
|
+
crossed.schema,
|
|
2675
|
+
crossed.chunks,
|
|
2676
|
+
optionalField(field),
|
|
2677
|
+
...castOptionArgs(options),
|
|
2678
|
+
)
|
|
2679
|
+
}
|
|
2608
2680
|
return nativeChunkedSerie.fromArrowArray(
|
|
2609
2681
|
arrowVectorChunksIntoIPC(vector, 'ChunkedSerie.fromArrowArray input'),
|
|
2610
2682
|
optionalField(field),
|
|
@@ -2698,9 +2770,9 @@ Object.defineProperties(ChunkedSerie.prototype, {
|
|
|
2698
2770
|
},
|
|
2699
2771
|
intoArrowArray: {
|
|
2700
2772
|
configurable: true,
|
|
2701
|
-
value() {
|
|
2702
|
-
return
|
|
2703
|
-
Reflect.apply(nativeChunkedSerie.
|
|
2773
|
+
value(options = {}) {
|
|
2774
|
+
return arrowVectorFromNative(
|
|
2775
|
+
Reflect.apply(nativeChunkedSerie.intoArrowArrayData, this, [copyOption(options)]),
|
|
2704
2776
|
'ChunkedSerie.intoArrowArray output',
|
|
2705
2777
|
)
|
|
2706
2778
|
},
|
|
@@ -2910,6 +2982,18 @@ function asSpanSerie(value) {
|
|
|
2910
2982
|
}
|
|
2911
2983
|
|
|
2912
2984
|
// Non-overlapping spans sorted one span in memory at a time.
|
|
2985
|
+
// The column a lease file holds, mapped zero copy: the path another reader
|
|
2986
|
+
// - a worker thread, another process - was handed by `intoSpillLease`.
|
|
2987
|
+
Object.defineProperty(Serie, 'fromSpillLease', {
|
|
2988
|
+
configurable: true,
|
|
2989
|
+
value(path, folder) {
|
|
2990
|
+
if (typeof path !== 'string') {
|
|
2991
|
+
throw new TypeError('Serie.fromSpillLease takes the path a SpillLease answers')
|
|
2992
|
+
}
|
|
2993
|
+
return describedSerie(nativeSerie.fromSpillLease(path, folder))
|
|
2994
|
+
},
|
|
2995
|
+
})
|
|
2996
|
+
|
|
2913
2997
|
Object.defineProperty(Serie, 'fromSortedSpans', {
|
|
2914
2998
|
configurable: true,
|
|
2915
2999
|
value(spans, by) {
|
|
@@ -2953,9 +3037,11 @@ const nativeCastPlan = Object.freeze({
|
|
|
2953
3037
|
compile: NativeCastPlan._compileNative.bind(NativeCastPlan),
|
|
2954
3038
|
apply: NativeCastPlan.prototype._applyNative,
|
|
2955
3039
|
applyChunked: NativeCastPlan.prototype._applyChunkedNative,
|
|
3040
|
+
applyRejecting: NativeCastPlan.prototype._applyRejectingNative,
|
|
2956
3041
|
})
|
|
2957
3042
|
delete NativeCastPlan.prototype._applyNative
|
|
2958
3043
|
delete NativeCastPlan.prototype._applyChunkedNative
|
|
3044
|
+
delete NativeCastPlan.prototype._applyRejectingNative
|
|
2959
3045
|
const CastPlan = publicNativeClass(
|
|
2960
3046
|
NativeCastPlan,
|
|
2961
3047
|
'CastPlan',
|
|
@@ -2998,6 +3084,23 @@ Object.defineProperty(CastPlan.prototype, 'apply', {
|
|
|
2998
3084
|
},
|
|
2999
3085
|
})
|
|
3000
3086
|
|
|
3087
|
+
// A record plan over a native reader: the accepted rows as a reader, and the
|
|
3088
|
+
// rows it cannot honour reported, `{ rejectsLimit }` defaulting to 1000.
|
|
3089
|
+
Object.defineProperty(CastPlan.prototype, 'applyRejecting', {
|
|
3090
|
+
configurable: true,
|
|
3091
|
+
value(reader, options = {}) {
|
|
3092
|
+
if (!(reader instanceof binding.BatchReader)) {
|
|
3093
|
+
throw new TypeError('CastPlan.applyRejecting takes a BatchReader')
|
|
3094
|
+
}
|
|
3095
|
+
closedOptions(options, ['rejectsLimit'], 'CastPlan.applyRejecting')
|
|
3096
|
+
const { rejectsLimit } = options
|
|
3097
|
+
if (rejectsLimit !== undefined && (!Number.isInteger(rejectsLimit) || rejectsLimit < 0)) {
|
|
3098
|
+
throw new TypeError(`rejectsLimit must be a non-negative integer, got ${rejectsLimit}`)
|
|
3099
|
+
}
|
|
3100
|
+
return Reflect.apply(nativeCastPlan.applyRejecting, this, [reader, rejectsLimit])
|
|
3101
|
+
},
|
|
3102
|
+
})
|
|
3103
|
+
|
|
3001
3104
|
// The bound a column stays resident under, the budget the process's holders
|
|
3002
3105
|
// stay resident under together, and the local folder they spill to. The
|
|
3003
3106
|
// constructor reads `{ byteSize, folder, budget }`, each absent or null the
|
|
@@ -3342,9 +3445,9 @@ function nativeLoadsAll(content, format, options) {
|
|
|
3342
3445
|
: decoded.map((value) => fromTransport(value))
|
|
3343
3446
|
}
|
|
3344
3447
|
|
|
3345
|
-
function nativeLoadPath(path, format, options) {
|
|
3448
|
+
async function nativeLoadPath(path, format, options) {
|
|
3346
3449
|
options = checkedOptions(options)
|
|
3347
|
-
const decoded = nativeFormat(format).loadPath(
|
|
3450
|
+
const decoded = await nativeFormat(format).loadPath(
|
|
3348
3451
|
path,
|
|
3349
3452
|
codecLimits(options),
|
|
3350
3453
|
...loadArguments(format, options),
|
|
@@ -3353,12 +3456,12 @@ function nativeLoadPath(path, format, options) {
|
|
|
3353
3456
|
return options.scalar === true ? decoded : fromTransport(decoded)
|
|
3354
3457
|
}
|
|
3355
3458
|
|
|
3356
|
-
function nativeLoadAllPath(path, format, options) {
|
|
3459
|
+
async function nativeLoadAllPath(path, format, options) {
|
|
3357
3460
|
options = checkedOptions(options)
|
|
3358
3461
|
if (format === FORMAT_JSON || format === FORMAT_JSON_LINES) {
|
|
3359
3462
|
refuseFillingForJson(format, options)
|
|
3360
3463
|
}
|
|
3361
|
-
const decoded = nativeFormat(format).loadAllPath(
|
|
3464
|
+
const decoded = await nativeFormat(format).loadAllPath(
|
|
3362
3465
|
path,
|
|
3363
3466
|
codecLimits(options),
|
|
3364
3467
|
options.field,
|
|
@@ -3407,9 +3510,9 @@ function nativeDumpAll(values, format, options) {
|
|
|
3407
3510
|
)
|
|
3408
3511
|
}
|
|
3409
3512
|
|
|
3410
|
-
function nativeDumpPath(value, path, format, options) {
|
|
3513
|
+
async function nativeDumpPath(value, path, format, options) {
|
|
3411
3514
|
options = checkedOptions(options)
|
|
3412
|
-
nativeFormat(format).dumpPath(
|
|
3515
|
+
await nativeFormat(format).dumpPath(
|
|
3413
3516
|
value,
|
|
3414
3517
|
path,
|
|
3415
3518
|
options.maxDepth,
|
|
@@ -3419,9 +3522,9 @@ function nativeDumpPath(value, path, format, options) {
|
|
|
3419
3522
|
)
|
|
3420
3523
|
}
|
|
3421
3524
|
|
|
3422
|
-
function nativeDumpAllPath(values, path, format, options) {
|
|
3525
|
+
async function nativeDumpAllPath(values, path, format, options) {
|
|
3423
3526
|
options = checkedOptions(options)
|
|
3424
|
-
nativeFormat(format).dumpAllPath(
|
|
3527
|
+
await nativeFormat(format).dumpAllPath(
|
|
3425
3528
|
boundedValues(values),
|
|
3426
3529
|
path,
|
|
3427
3530
|
options.maxDepth,
|
|
@@ -3473,16 +3576,27 @@ function destinationPath(destination) {
|
|
|
3473
3576
|
return fileURLToPath(destination)
|
|
3474
3577
|
}
|
|
3475
3578
|
|
|
3579
|
+
async function writeDescriptor(descriptor, bytes) {
|
|
3580
|
+
let offset = 0
|
|
3581
|
+
while (offset < bytes.length) {
|
|
3582
|
+
const { bytesWritten } = await writeAsync(
|
|
3583
|
+
descriptor,
|
|
3584
|
+
bytes,
|
|
3585
|
+
offset,
|
|
3586
|
+
bytes.length - offset,
|
|
3587
|
+
null,
|
|
3588
|
+
)
|
|
3589
|
+
offset += bytesWritten
|
|
3590
|
+
}
|
|
3591
|
+
}
|
|
3592
|
+
|
|
3476
3593
|
function writeDestination(destination, bytes) {
|
|
3477
3594
|
if (destination === undefined || destination === null) return bytes
|
|
3478
|
-
if (typeof destination === '
|
|
3479
|
-
|
|
3480
|
-
return undefined
|
|
3481
|
-
}
|
|
3595
|
+
if (typeof destination === 'number') return writeDescriptor(destination, bytes)
|
|
3596
|
+
if (typeof destination === 'string') return fs.promises.writeFile(destination, bytes)
|
|
3482
3597
|
if (destination instanceof URL) {
|
|
3483
3598
|
destinationPath(destination)
|
|
3484
|
-
fs.
|
|
3485
|
-
return undefined
|
|
3599
|
+
return fs.promises.writeFile(destination, bytes)
|
|
3486
3600
|
}
|
|
3487
3601
|
throw new TypeError('destination must be a path, file URL, or descriptor')
|
|
3488
3602
|
}
|
|
@@ -3900,6 +4014,14 @@ function singleDocumentMethods(format) {
|
|
|
3900
4014
|
nativeLoads(await readStream(source, options), format, options))()
|
|
3901
4015
|
}
|
|
3902
4016
|
const input = readSource(source, codecInputLimit(options))
|
|
4017
|
+
if (input.descriptor !== null) {
|
|
4018
|
+
return (async () =>
|
|
4019
|
+
nativeLoads(
|
|
4020
|
+
await readDescriptorBounded(input.descriptor, codecInputLimit(options)),
|
|
4021
|
+
format,
|
|
4022
|
+
options,
|
|
4023
|
+
))()
|
|
4024
|
+
}
|
|
3903
4025
|
return input.path === null
|
|
3904
4026
|
? nativeLoads(input.content, format, options)
|
|
3905
4027
|
: nativeLoadPath(input.path, format, options)
|
|
@@ -3916,10 +4038,7 @@ function singleDocumentMethods(format) {
|
|
|
3916
4038
|
return writeStream(destination, nativeDumps(value, format, options))
|
|
3917
4039
|
}
|
|
3918
4040
|
const path = destinationPath(destination)
|
|
3919
|
-
if (path !== null)
|
|
3920
|
-
nativeDumpPath(value, path, format, options)
|
|
3921
|
-
return undefined
|
|
3922
|
-
}
|
|
4041
|
+
if (path !== null) return nativeDumpPath(value, path, format, options)
|
|
3923
4042
|
return writeDestination(destination, nativeDumps(value, format, options))
|
|
3924
4043
|
},
|
|
3925
4044
|
async loadStream(stream, options) {
|
|
@@ -3946,6 +4065,14 @@ function fixedCodec(format, multiFormat) {
|
|
|
3946
4065
|
: yamlDocumentStream(source, options)
|
|
3947
4066
|
}
|
|
3948
4067
|
const input = readSource(source, codecInputLimit(options))
|
|
4068
|
+
if (input.descriptor !== null) {
|
|
4069
|
+
return (async () =>
|
|
4070
|
+
nativeLoadsAll(
|
|
4071
|
+
await readDescriptorBounded(input.descriptor, codecInputLimit(options)),
|
|
4072
|
+
multiFormat,
|
|
4073
|
+
options,
|
|
4074
|
+
))()
|
|
4075
|
+
}
|
|
3949
4076
|
return input.path === null
|
|
3950
4077
|
? nativeLoadsAll(input.content, multiFormat, options)
|
|
3951
4078
|
: nativeLoadAllPath(input.path, multiFormat, options)
|
|
@@ -3965,8 +4092,7 @@ function fixedCodec(format, multiFormat) {
|
|
|
3965
4092
|
}
|
|
3966
4093
|
const path = destinationPath(destination)
|
|
3967
4094
|
if (path !== null) {
|
|
3968
|
-
nativeDumpAllPath(values, path, multiFormat, options)
|
|
3969
|
-
return undefined
|
|
4095
|
+
return nativeDumpAllPath(values, path, multiFormat, options)
|
|
3970
4096
|
}
|
|
3971
4097
|
return writeDestination(
|
|
3972
4098
|
destination,
|
|
@@ -3995,6 +4121,21 @@ const codec = Object.freeze({
|
|
|
3995
4121
|
if (isReadable(source)) return codec.fromStream(source, options)
|
|
3996
4122
|
options = checkedOptions(options)
|
|
3997
4123
|
const input = readSource(source, codecInputLimit(options))
|
|
4124
|
+
if (input.descriptor !== null) {
|
|
4125
|
+
return (async () => {
|
|
4126
|
+
const content = await readDescriptorBounded(
|
|
4127
|
+
input.descriptor,
|
|
4128
|
+
codecInputLimit(options),
|
|
4129
|
+
)
|
|
4130
|
+
if (options.format === undefined) {
|
|
4131
|
+
return nativeLoadsInferred(content, options)
|
|
4132
|
+
}
|
|
4133
|
+
const format = formatFor(null, options)
|
|
4134
|
+
return format === FORMAT_JSON_LINES
|
|
4135
|
+
? nativeLoadsAll(content, format, options)
|
|
4136
|
+
: nativeLoads(content, format, options)
|
|
4137
|
+
})()
|
|
4138
|
+
}
|
|
3998
4139
|
if (input.path === null && options.format === undefined) {
|
|
3999
4140
|
return nativeLoadsInferred(input.content, options)
|
|
4000
4141
|
}
|
|
@@ -4020,12 +4161,9 @@ const codec = Object.freeze({
|
|
|
4020
4161
|
const path = destinationPath(destination)
|
|
4021
4162
|
const format = formatFor(path, options, FORMAT_JSON)
|
|
4022
4163
|
if (path !== null) {
|
|
4023
|
-
|
|
4024
|
-
nativeDumpAllPath(value, path, format, options)
|
|
4025
|
-
|
|
4026
|
-
nativeDumpPath(value, path, format, options)
|
|
4027
|
-
}
|
|
4028
|
-
return undefined
|
|
4164
|
+
return format === FORMAT_JSON_LINES
|
|
4165
|
+
? nativeDumpAllPath(value, path, format, options)
|
|
4166
|
+
: nativeDumpPath(value, path, format, options)
|
|
4029
4167
|
}
|
|
4030
4168
|
const bytes =
|
|
4031
4169
|
format === FORMAT_JSON_LINES
|
|
@@ -4706,13 +4844,138 @@ for (const Clause of [NativeTerm, NativeFilter, NativeSelector]) {
|
|
|
4706
4844
|
})
|
|
4707
4845
|
}
|
|
4708
4846
|
|
|
4847
|
+
// The order desk over one log (`yggio_market::Desk`): `Desk.open` opens its
|
|
4848
|
+
// session as `connect` does and reads the desk's own keys beside it; every
|
|
4849
|
+
// door that writes or reads the log or a register answers a promise from the
|
|
4850
|
+
// libuv pool under the desk's one lock, and a read on this thread is refused
|
|
4851
|
+
// by name while one runs. The loader spells what JavaScript has of its own: a
|
|
4852
|
+
// limit as one object, the answer and every row as frozen plain objects. A
|
|
4853
|
+
// refused order is a judgement and resolves, `refused` true.
|
|
4854
|
+
{
|
|
4855
|
+
const NativeDesk = binding.Desk
|
|
4856
|
+
delete binding.DeskAnswer
|
|
4857
|
+
const nativeOpen = binding._deskOpenNative
|
|
4858
|
+
delete binding._deskOpenNative
|
|
4859
|
+
const nativeSubmit = takePrivate(NativeDesk, '_submitNative')
|
|
4860
|
+
const nativeMassCancel = takePrivate(NativeDesk, '_massCancelNative')
|
|
4861
|
+
const nativeCancel = takePrivate(NativeDesk, '_cancelNative')
|
|
4862
|
+
const nativeReplace = takePrivate(NativeDesk, '_replaceNative')
|
|
4863
|
+
const nativeSetLimit = takePrivate(NativeDesk, '_setLimitNative')
|
|
4864
|
+
const nativeSetLimits = takePrivate(NativeDesk, '_setLimitsNative')
|
|
4865
|
+
const nativeLiftLimit = takePrivate(NativeDesk, '_liftLimitNative')
|
|
4866
|
+
const nativeKill = takePrivate(NativeDesk, '_killNative')
|
|
4867
|
+
const OPEN_KEYS = ['options', 'warehouse', 'dialect', 'log', 'walked', 'elements', 'limits', 'seats', 'breaches', 'resumeNs', 'record']
|
|
4868
|
+
const LIMIT_KEYS = ['kind', 'scopekind', 'scope', 'bound', 'currency', 'window', 'severity']
|
|
4869
|
+
|
|
4870
|
+
// One row a native Serie holds, as its named record.
|
|
4871
|
+
const frozenRow = (serie) => Object.freeze(serie.asJs()[0])
|
|
4872
|
+
const answerOf = (native) => {
|
|
4873
|
+
const refusal = native.refusal
|
|
4874
|
+
return Object.freeze({
|
|
4875
|
+
row: frozenRow(native.row),
|
|
4876
|
+
version: native.version,
|
|
4877
|
+
refused: native.refused,
|
|
4878
|
+
refusal: refusal == null ? undefined : Object.freeze({ path: refusal[0], reason: refusal[1] }),
|
|
4879
|
+
held: native.held,
|
|
4880
|
+
checks: Object.freeze(native.checks.asJs().map((row) => Object.freeze(row))),
|
|
4881
|
+
walkasof: native.walkasof,
|
|
4882
|
+
})
|
|
4883
|
+
}
|
|
4884
|
+
|
|
4885
|
+
// No URL is the Yggio engine, over `warehouse` or the process's own; every
|
|
4886
|
+
// key that is not the desk's is a session property, as `connect` reads it.
|
|
4887
|
+
NativeDesk.open = async function open(url, options) {
|
|
4888
|
+
if (options !== undefined && options !== null && !isPlainObject(options)) {
|
|
4889
|
+
throw new TypeError(`Desk.open options must be a plain object of ${OPEN_KEYS.join(', ')} and properties`)
|
|
4890
|
+
}
|
|
4891
|
+
const { options: messages, warehouse, dialect, log, walked, elements, limits, seats, breaches, resumeNs, record, ...properties } =
|
|
4892
|
+
options ?? {}
|
|
4893
|
+
if (seats !== undefined && seats !== null && !(seats instanceof NativeSerie)) {
|
|
4894
|
+
throw new TypeError('Desk.open seats must be a native Serie of the seats register rows')
|
|
4895
|
+
}
|
|
4896
|
+
const location = url instanceof URL || url instanceof Url ? url.toString() : url
|
|
4897
|
+
return nativeOpen(location, messages, warehouse, dialect, log, walked, elements, limits, seats, breaches, resumeNs, record, properties)
|
|
4898
|
+
}
|
|
4899
|
+
|
|
4900
|
+
Object.defineProperties(NativeDesk.prototype, {
|
|
4901
|
+
submit: {
|
|
4902
|
+
configurable: true,
|
|
4903
|
+
async value(order, at, seen) {
|
|
4904
|
+
return answerOf(await nativeSubmit.call(this, order, at, seen))
|
|
4905
|
+
},
|
|
4906
|
+
},
|
|
4907
|
+
// The request built from the version seen by the core, asked by `by`: a refusal resolves.
|
|
4908
|
+
cancel: {
|
|
4909
|
+
configurable: true,
|
|
4910
|
+
async value(seen, by, clordid, at) {
|
|
4911
|
+
return answerOf(await nativeCancel.call(this, seen, by, clordid, at))
|
|
4912
|
+
},
|
|
4913
|
+
},
|
|
4914
|
+
replace: {
|
|
4915
|
+
configurable: true,
|
|
4916
|
+
async value(seen, patch, by, clordid, at) {
|
|
4917
|
+
return answerOf(await nativeReplace.call(this, seen, patch, by, clordid, at))
|
|
4918
|
+
},
|
|
4919
|
+
},
|
|
4920
|
+
// One answer a chain the q scopes, in chain order: a cancel each.
|
|
4921
|
+
massCancel: {
|
|
4922
|
+
configurable: true,
|
|
4923
|
+
async value(request, at) {
|
|
4924
|
+
return Object.freeze((await nativeMassCancel.call(this, request, at)).map(answerOf))
|
|
4925
|
+
},
|
|
4926
|
+
},
|
|
4927
|
+
setLimit: {
|
|
4928
|
+
configurable: true,
|
|
4929
|
+
async value(limit, by, at, seen) {
|
|
4930
|
+
if (!isPlainObject(limit)) throw new TypeError(`Desk.setLimit takes a limit object of ${LIMIT_KEYS.join(', ')}`)
|
|
4931
|
+
const { kind, scopekind, scope, bound, currency, window, severity } = closedOptions(limit, LIMIT_KEYS, 'Desk.setLimit')
|
|
4932
|
+
return frozenRow(
|
|
4933
|
+
await nativeSetLimit.call(this, kind, scopekind, scope, bound, currency, window, severity, by, at, seen),
|
|
4934
|
+
)
|
|
4935
|
+
},
|
|
4936
|
+
},
|
|
4937
|
+
// All or none, one register commit: each limit a plain object as setLimit takes.
|
|
4938
|
+
setLimits: {
|
|
4939
|
+
configurable: true,
|
|
4940
|
+
async value(limits, by, at) {
|
|
4941
|
+
if (!Array.isArray(limits)) throw new TypeError('Desk.setLimits takes an array of limit objects')
|
|
4942
|
+
const parts = limits.map((limit) => {
|
|
4943
|
+
if (!isPlainObject(limit)) throw new TypeError(`Desk.setLimits takes limit objects of ${LIMIT_KEYS.join(', ')}`)
|
|
4944
|
+
return closedOptions(limit, LIMIT_KEYS, 'Desk.setLimits')
|
|
4945
|
+
})
|
|
4946
|
+
return Object.freeze((await nativeSetLimits.call(this, parts, by, at)).map(frozenRow))
|
|
4947
|
+
},
|
|
4948
|
+
},
|
|
4949
|
+
liftLimit: {
|
|
4950
|
+
configurable: true,
|
|
4951
|
+
async value(crossuuid, by, at, seen) {
|
|
4952
|
+
return frozenRow(await nativeLiftLimit.call(this, crossuuid, by, at, seen))
|
|
4953
|
+
},
|
|
4954
|
+
},
|
|
4955
|
+
kill: {
|
|
4956
|
+
configurable: true,
|
|
4957
|
+
async value(scope, key, by, at) {
|
|
4958
|
+
return frozenRow(await nativeKill.call(this, scope, key, by, at))
|
|
4959
|
+
},
|
|
4960
|
+
},
|
|
4961
|
+
})
|
|
4962
|
+
}
|
|
4963
|
+
|
|
4709
4964
|
// One Arrow surface per applier: `applyArrowReader` keeps a native reader
|
|
4710
4965
|
// lazy, `applyArrowBatch` and `applyArrowTable` keep the Apache Arrow JS
|
|
4711
4966
|
// holder they were given, and `applyArrow` infers which of the three it has.
|
|
4712
4967
|
function defineArrowAppliers(Owner, label, { records = true } = {}) {
|
|
4713
4968
|
const nativeReader = takePrivate(Owner, '_applyArrowReaderNative')
|
|
4714
4969
|
const nativeBatch = takePrivate(Owner, '_applyArrowBatchNative')
|
|
4970
|
+
const nativeSerie = takePrivate(Owner, '_applySerieNative')
|
|
4715
4971
|
const properties = {
|
|
4972
|
+
applySerie: {
|
|
4973
|
+
configurable: true,
|
|
4974
|
+
value(serie) {
|
|
4975
|
+
if (!(serie instanceof NativeSerie)) throw new TypeError(`${label}.applySerie takes a native Serie`)
|
|
4976
|
+
return describedSerie(nativeSerie.call(this, serie))
|
|
4977
|
+
},
|
|
4978
|
+
},
|
|
4716
4979
|
applyArrowReader: {
|
|
4717
4980
|
configurable: true,
|
|
4718
4981
|
value(reader) {
|
|
@@ -5034,12 +5297,31 @@ for (const name of [
|
|
|
5034
5297
|
// that result onto the standard protocol, without prefetching or collecting.
|
|
5035
5298
|
if (binding.ByteIterator) {
|
|
5036
5299
|
const nativeNext = binding.ByteIterator.prototype.next
|
|
5300
|
+
const nativeClose = takePrivate(binding.ByteIterator, '_closeNative')
|
|
5037
5301
|
binding.ByteIterator.prototype.next = function next() {
|
|
5038
5302
|
const bytes = nativeNext.call(this)
|
|
5039
5303
|
return bytes === null
|
|
5040
5304
|
? { value: undefined, done: true }
|
|
5041
5305
|
: { value: bytes, done: false }
|
|
5042
5306
|
}
|
|
5307
|
+
// Leaving a `for...of` early - `break`, `return`, a throw - calls
|
|
5308
|
+
// `return()`: the stream lets go of its open leaf at once, not at the GC.
|
|
5309
|
+
Object.defineProperty(binding.ByteIterator.prototype, 'return', {
|
|
5310
|
+
configurable: true,
|
|
5311
|
+
writable: true,
|
|
5312
|
+
value: function release(value) {
|
|
5313
|
+
nativeClose.call(this)
|
|
5314
|
+
return { value, done: true }
|
|
5315
|
+
},
|
|
5316
|
+
})
|
|
5317
|
+
if (typeof Symbol.dispose === 'symbol') {
|
|
5318
|
+
Object.defineProperty(binding.ByteIterator.prototype, Symbol.dispose, {
|
|
5319
|
+
configurable: true,
|
|
5320
|
+
value: function dispose() {
|
|
5321
|
+
nativeClose.call(this)
|
|
5322
|
+
},
|
|
5323
|
+
})
|
|
5324
|
+
}
|
|
5043
5325
|
Object.defineProperty(binding.ByteIterator.prototype, Symbol.iterator, {
|
|
5044
5326
|
configurable: true,
|
|
5045
5327
|
value: function bytes() {
|
|
@@ -5125,6 +5407,7 @@ for (const name of [
|
|
|
5125
5407
|
'readRangeBytes',
|
|
5126
5408
|
'writeBytes',
|
|
5127
5409
|
'createBytes',
|
|
5410
|
+
'storeAddressed',
|
|
5128
5411
|
'writeText',
|
|
5129
5412
|
'pwrite',
|
|
5130
5413
|
'appendBytes',
|
|
@@ -5350,7 +5633,7 @@ MarketMessage.fromFixRecord = function fromFixRecord(field, value, options) {
|
|
|
5350
5633
|
MarketMessage.fromRow = function fromRow(schema, row, options) {
|
|
5351
5634
|
return NativeMarketMessage.fromRow(schema, asScalar(row), options)
|
|
5352
5635
|
}
|
|
5353
|
-
for (const name of ['fromLine', 'fromFixMessage', 'fromUllinkLine', 'fromFixmlMessage']) {
|
|
5636
|
+
for (const name of ['fromLine', 'fromFixMessage', 'fromUllinkLine', 'fromFixmlMessage', 'fromDialectMessage']) {
|
|
5354
5637
|
const native = NativeMarketMessage[name]
|
|
5355
5638
|
MarketMessage[name] = {
|
|
5356
5639
|
[name](body, options) {
|
|
@@ -5460,6 +5743,13 @@ function asLine(value) {
|
|
|
5460
5743
|
}
|
|
5461
5744
|
// A stage reads the stream it takes, so the failure kept for that stream
|
|
5462
5745
|
// is the one its own stream throws.
|
|
5746
|
+
// An audit's parameters cross as the record a plan binds.
|
|
5747
|
+
{
|
|
5748
|
+
const native = takePrivate(NativeMarketMessages, '_intoAuditNative')
|
|
5749
|
+
NativeMarketMessages.prototype.intoAudit = function intoAudit(report, viewer, params) {
|
|
5750
|
+
return native.call(this, report, viewer, suppliedParameters(params))
|
|
5751
|
+
}
|
|
5752
|
+
}
|
|
5463
5753
|
for (const name of ['intoLifecycle', 'intoLineage']) {
|
|
5464
5754
|
const native = NativeMarketMessages.prototype[name]
|
|
5465
5755
|
NativeMarketMessages.prototype[name] = {
|
|
@@ -5535,6 +5825,7 @@ const fix = Object.freeze({
|
|
|
5535
5825
|
MarketMessages: NativeMarketMessages,
|
|
5536
5826
|
MsgType: binding.MsgType,
|
|
5537
5827
|
Underlyings: binding.Underlyings,
|
|
5828
|
+
Plugins: binding.Plugins,
|
|
5538
5829
|
crateFields: binding.fixCrateFields,
|
|
5539
5830
|
underlyingField: binding.fixUnderlyingField,
|
|
5540
5831
|
registryField: binding.fixRegistryField,
|
|
@@ -5544,7 +5835,16 @@ const fix = Object.freeze({
|
|
|
5544
5835
|
pluginField: binding.fixPluginField,
|
|
5545
5836
|
pluginSide: binding.fixPluginSide,
|
|
5546
5837
|
reports: binding.fixReports,
|
|
5838
|
+
report: binding.fixReport,
|
|
5839
|
+
analyses: binding.fixAnalyses,
|
|
5840
|
+
blotters: binding.fixBlotters,
|
|
5841
|
+
messagesWhere: binding.fixMessagesWhere,
|
|
5842
|
+
messagesField: binding.fixMessagesField,
|
|
5843
|
+
partyFromText: binding.fixPartyFromText,
|
|
5844
|
+
reportFor: ((native) => (name, viewer, params) => native(name, viewer, suppliedParameters(params)))(binding._fixReportForNative),
|
|
5845
|
+
reportExplain: binding.fixReportExplain,
|
|
5547
5846
|
rules: binding.fixRules,
|
|
5847
|
+
kindMembers: binding.fixKindMembers,
|
|
5548
5848
|
})
|
|
5549
5849
|
|
|
5550
5850
|
// The FIX values are reached through the namespace and nowhere else, so a
|
|
@@ -5557,6 +5857,7 @@ for (const name of [
|
|
|
5557
5857
|
'FixRegistry',
|
|
5558
5858
|
'MsgType',
|
|
5559
5859
|
'Underlyings',
|
|
5860
|
+
'Plugins',
|
|
5560
5861
|
'JsFixFieldIterator',
|
|
5561
5862
|
'JsMarketMessage',
|
|
5562
5863
|
'JsMarketMessagesOptions',
|
|
@@ -5564,6 +5865,7 @@ for (const name of [
|
|
|
5564
5865
|
'JsFixRegistry',
|
|
5565
5866
|
'JsMsgType',
|
|
5566
5867
|
'JsUnderlyings',
|
|
5868
|
+
'JsPlugins',
|
|
5567
5869
|
'fixCrateFields',
|
|
5568
5870
|
'fixUnderlyingField',
|
|
5569
5871
|
'fixRegistryField',
|
|
@@ -5573,7 +5875,14 @@ for (const name of [
|
|
|
5573
5875
|
'fixPluginField',
|
|
5574
5876
|
'fixPluginSide',
|
|
5575
5877
|
'fixReports',
|
|
5878
|
+
'fixReport',
|
|
5879
|
+
'fixAnalyses',
|
|
5880
|
+
'fixBlotters',
|
|
5881
|
+
'fixMessagesWhere',
|
|
5882
|
+
'fixPartyFromText',
|
|
5883
|
+
'_fixReportForNative',
|
|
5576
5884
|
'fixRules',
|
|
5885
|
+
'fixKindMembers',
|
|
5577
5886
|
]) {
|
|
5578
5887
|
delete binding[name]
|
|
5579
5888
|
}
|
|
@@ -6511,10 +6820,10 @@ for (const name of [
|
|
|
6511
6820
|
})
|
|
6512
6821
|
|
|
6513
6822
|
const Workbook = publicNativeClass(NativeWorkbook, 'Workbook', new Set())
|
|
6514
|
-
Object.defineProperty(Workbook.prototype, Symbol.
|
|
6823
|
+
Object.defineProperty(Workbook.prototype, Symbol.asyncIterator, {
|
|
6515
6824
|
configurable: true,
|
|
6516
|
-
value() {
|
|
6517
|
-
|
|
6825
|
+
async *value() {
|
|
6826
|
+
yield* await this.sheets()
|
|
6518
6827
|
},
|
|
6519
6828
|
})
|
|
6520
6829
|
|