yggio 0.0.15 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -203,9 +203,8 @@ or `serie.scalar(0)` for one row - and values become a column through
203
203
  `BatchReader` read yields Arrow JS batches, and a write consumes a native
204
204
  reader, Arrow JS table/batch/reader, IPC bytes, named columns, or plain rows
205
205
  through that same reader path. `serie.cast` casts a column in hand,
206
- `SerieReader.fromArrowReader` casts a stream batch by batch under one plan,
207
- `SerieReader.fromSerie` streams a held column as the one record serie it is,
208
- and `ArrowCastPlan.compile` holds one cast for every column of a layout.
206
+ `Serie.fromArrowReader` keeps a reader external as a lazy Serie, cast batch by batch under one plan; `Serie.intoChunks` walks any serie one record column per batch;
207
+ and `CastPlan.compile` holds one cast for every column of a layout.
209
208
  Each takes `{ safe, representation }`, an absent answer taking the core's
210
209
  default; whether a value may be absent is the target field's nullability. Run
211
210
  `npm run bench:records` for the copied-IPC read, projection, cast, and write
@@ -207,7 +207,6 @@ benchmark('serie/into_reversed', () => orderPrices.intoReversed())
207
207
  benchmark('serie/into_taken', () => orderPrices.intoTaken(orderIndices))
208
208
  benchmark('serie/into_filtered', () => orderPrices.intoFiltered(orderMask))
209
209
  benchmark('serie/partition_by', () => orderPrices.partitionBy(orderVenues))
210
- benchmark('serie/partition_by_paths', () => orderQuotes.partitionByPaths(['venue', 'price']))
211
210
  const sortedInPlace = orderPrices.intoSorted()
212
211
  benchmark('serie/as_sorted', () => sortedInPlace.asSorted())
213
212
  const uniqueInPlace = orderVenues.intoUnique()
package/binding.d.ts CHANGED
@@ -1,6 +1,9 @@
1
1
  export {
2
2
  ai,
3
3
  distance,
4
+ mysql,
5
+ Connection,
6
+ Cursor,
4
7
  Arn,
5
8
  BatchReader,
6
9
  CellRange,
@@ -30,15 +33,14 @@ export {
30
33
  Plan,
31
34
  ProtocolField,
32
35
  RecordOptions,
33
- StreamSerie,
36
+ SerieChunks,
34
37
  KeySerie,
35
38
  KeySeries,
36
39
  StreamKeySerie,
37
40
  Selector,
38
- StreamChunkedSerie,
39
41
  SpillOptions,
40
42
  Parallelism,
41
- ArrowCastPlan,
43
+ CastPlan,
42
44
  StringEnum,
43
45
  Term,
44
46
  TextLine,
@@ -96,6 +98,8 @@ export {
96
98
  import type {
97
99
  Arn,
98
100
  BatchReader,
101
+ Connection,
102
+ Cursor,
99
103
  Bound,
100
104
  BoundSelector,
101
105
  ByteIterator,
@@ -127,17 +131,16 @@ import type {
127
131
  ProtocolField,
128
132
  Plan,
129
133
  RecordOptions,
130
- StreamSerie,
134
+ SerieChunks,
131
135
  KeySerie,
132
136
  KeySeries,
133
137
  StreamKeySerie,
134
138
  Selector,
135
139
  Serie as NativeSerie,
136
- StreamChunkedSerie,
137
140
  SpillOptions,
138
141
  SpillStats,
139
142
  WindowSerie as NativeWindowSerie,
140
- ArrowCastPlan,
143
+ CastPlan,
141
144
  JoinOptionsInput,
142
145
  PartitionOptionsInput,
143
146
  StringParametersInput,
@@ -689,7 +692,15 @@ export type BytesDataTypeId =
689
692
 
690
693
  /** Core compatibility targets supported by DataType and Field projection. */
691
694
  export type CompatibilityScheme =
692
- 'arrow' | 'spark' | 'polars' | 'pandas' | 'iceberg' | 'doris'
695
+ | 'arrow'
696
+ | 'spark'
697
+ | 'polars'
698
+ | 'pandas'
699
+ | 'iceberg'
700
+ | 'postgres'
701
+ | 'sqlserver'
702
+ | 'doris'
703
+ | 'mysql'
693
704
 
694
705
  /** Required intent for a generic record-write entry point. */
695
706
  export type IOMode = 'overwrite' | 'append' | 'merge' | 'readonly' | 'random'
@@ -795,7 +806,7 @@ declare module './index' {
795
806
  /** Native rows in, native rows out: a `Scalar` sequence of records, or plain objects. */
796
807
  interface RecordAppliers {
797
808
  /** The native rows this publishes, bound once against `schema` or the first record. */
798
- applyRecords(rows: Scalar | Iterable<unknown>, schema?: FieldLike | null): StreamSerie
809
+ applyRecords(rows: Scalar | Iterable<unknown>, schema?: FieldLike | null): Serie
799
810
  }
800
811
 
801
812
  interface Term {
@@ -869,13 +880,7 @@ declare module './index' {
869
880
  toJSON(): unknown
870
881
  }
871
882
 
872
- interface StreamSerie extends Iterator<Scalar>, Iterable<Scalar> {
873
- /** The next row, or done once every row was yielded. */
874
- next(): IteratorResult<Scalar>
875
- /** Every remaining row, as an array. */
876
- collect(): Scalar[]
877
- [Symbol.iterator](): StreamSerie
878
- }
883
+ interface SerieChunks extends IterableIterator<Serie> {}
879
884
 
880
885
  interface StringEnum {
881
886
  /**
@@ -966,7 +971,13 @@ declare module './index' {
966
971
  }
967
972
 
968
973
  namespace Serie {
969
- function from(value: unknown, field?: FieldLike | null, options?: ArrowCastOptions): Serie
974
+ /**
975
+ * Sort a serie whose chunks arrive as non-overlapping spans, one span
976
+ * in memory at a time: `by` is bound and checked before the first span
977
+ * is pulled, then each span is sorted on its own as the output advances.
978
+ */
979
+ function fromSortedSpans(spans: Iterable<Serie | ChunkedSerie | BatchReader>, by: OrderingKeys): Serie
980
+ function from(value: unknown, field?: FieldLike | null, options?: CastOptions): Serie
970
981
  /** The column `field` types `rows` into, each through the field's own contract. */
971
982
  function fromScalars(field: Field | string, rows: Iterable<unknown>): Serie
972
983
  /** The empty column of `field`. */
@@ -990,7 +1001,7 @@ declare module './index' {
990
1001
  function fromArrowArray(
991
1002
  vector: ArrowVector,
992
1003
  field?: Field | string,
993
- options?: ArrowCastOptions,
1004
+ options?: CastOptions,
994
1005
  ): Serie
995
1006
  /**
996
1007
  * One Apache Arrow JS record batch or table as the record column of its
@@ -999,16 +1010,20 @@ declare module './index' {
999
1010
  function fromArrowBatch(
1000
1011
  batch: ArrowRecordBatch | ArrowTable,
1001
1012
  root?: Field | string,
1002
- options?: ArrowCastOptions,
1013
+ options?: CastOptions,
1003
1014
  ): Serie
1004
1015
  /**
1005
- * Drain a native reader into the record column of its rows, every batch
1006
- * cast by one plan; the reader is consumed.
1016
+ * A native reader as one lazy record column of its rows: of its own
1017
+ * schema, or cast into `root` by one plan compiled here, and nothing
1018
+ * pulled at the call. Each batch is cast as a read pulls it -
1019
+ * `intoChunks` one record column per batch - a refused batch thrown by
1020
+ * the pull that reaches it, after the batches before it. The reader is
1021
+ * consumed.
1007
1022
  */
1008
1023
  function fromArrowReader(
1009
1024
  reader: BatchReader,
1010
1025
  root?: Field | string,
1011
- options?: ArrowCastOptions,
1026
+ options?: CastOptions,
1012
1027
  ): Serie
1013
1028
  }
1014
1029
 
@@ -1057,7 +1072,7 @@ declare module './index' {
1057
1072
  * This column under `field` - a DataType as its required `value` field -
1058
1073
  * cast once; a run is refused.
1059
1074
  */
1060
- cast(field: Field | DataType | string, options?: ArrowCastOptions): Serie
1075
+ cast(field: Field | DataType | string, options?: CastOptions): Serie
1061
1076
  /** This one-row column as the Apache Arrow JS value of its row. */
1062
1077
  intoArrowScalar(): unknown
1063
1078
  /** This column as one Apache Arrow JS vector. */
@@ -1094,7 +1109,11 @@ declare module './index' {
1094
1109
  */
1095
1110
  intoFiltered(mask: SerieArgument): Serie
1096
1111
  /** Native key partitions; the payload retains its own layout. */
1097
- partitionBy(by: WindowKey): KeySeries
1112
+ partitionBy(by: WindowKey, options?: PartitionOptionsInput | null): KeySeries
1113
+ /** This serie walked one record column per batch. */
1114
+ intoChunks(): SerieChunks
1115
+ /** This serie cast into `field` by one plan, lazily where it is lazy. */
1116
+ intoCast(field: Field | DataType | string, options?: CastOptions): Serie
1098
1117
  /**
1099
1118
  * Sort the rows in place under `options` and answer this serie, so calls
1100
1119
  * chain: a primitive column holding its buffer alone sorts where it
@@ -1235,101 +1254,14 @@ declare module './index' {
1235
1254
  asSortBy(by: OrderingKeys): this
1236
1255
  }
1237
1256
 
1238
- namespace StreamChunkedSerie {
1239
- /**
1240
- * Read a native reader's batches as record series: of its own schema,
1241
- * named `row`, or cast into `root` by one plan compiled here. The reader
1242
- * is consumed.
1243
- */
1244
- function fromArrowReader(
1245
- reader: BatchReader,
1246
- root?: Field | string,
1247
- options?: ArrowCastOptions,
1248
- ): StreamChunkedSerie
1249
- /**
1250
- * Read one held column as a stream of the one record serie it is: a
1251
- * record column as it stands, any other column as the one child of a
1252
- * record named `row`. A run, and a record column with an absent row,
1253
- * are refused.
1254
- */
1255
- function fromSerie(serie: Serie): StreamChunkedSerie
1256
- /**
1257
- * Read a held chunked column as the stream of its chunks, one record
1258
- * serie per chunk: a record's chunks as they stand, any other field's
1259
- * each the one child of a record named `row`. Nothing is cast or copied;
1260
- * a record chunk with an absent row is refused.
1261
- */
1262
- function fromChunked(chunked: ChunkedSerie): StreamChunkedSerie
1263
- /**
1264
- * Sort a stream whose chunks arrive as non-overlapping spans, one span
1265
- * in memory at a time: `by` is bound and checked before the first span
1266
- * is pulled, then the first span is pulled for its root field and each
1267
- * span is sorted on its own as the output advances.
1268
- */
1269
- function fromSortedSpans(
1270
- spans: Iterable<StreamChunkedSerie | ChunkedSerie | Serie | BatchReader>,
1271
- by: OrderingKeys,
1272
- ): StreamChunkedSerie
1273
- }
1274
-
1275
- interface StreamChunkedSerie extends Iterable<Serie> {
1276
- readonly schema: import('apache-arrow').Schema
1277
- readNextBatch(): ArrowRecordBatch | null
1278
- readAll(): ArrowTable
1279
- /** One record serie per batch, each cast as it is pulled. */
1280
- [Symbol.iterator](): Generator<Serie, void, undefined>
1281
- /**
1282
- * Every record this reader yields cast into `field` by one plan: a
1283
- * record root, or a column - a DataType as its required `value` field -
1284
- * as the one child of a record named `row`. This reader's own root
1285
- * answers it as it stands; held records are cast here, once each, and a
1286
- * stream's batches as they are pulled. The reader is consumed.
1287
- */
1288
- cast(field: Field | DataType | string, options?: ArrowCastOptions): StreamChunkedSerie
1289
- /** Adjacent key contexts and native payloads. */
1290
- windowBy(by: WindowKey, sorted?: boolean | null): StreamKeySerie
1291
- /** Native key partitions; the payload retains its own layout. */
1292
- partitionBy(by: WindowKey, options?: PartitionOptionsInput | null): StreamKeySerie
1293
- intoSorted(options?: SortOptions | null): StreamChunkedSerie
1294
- /**
1295
- * Every record this reader yields in the order the `order by` keys of
1296
- * `by` state, bound against the root before any batch is pulled, then
1297
- * drained and merged as `intoSorted` is. The reader is consumed, a
1298
- * refused key included.
1299
- */
1300
- intoSortBy(by: OrderingKeys): StreamChunkedSerie
1301
- /**
1302
- * This stream joined with `other` - a Serie, a ChunkedSerie, or another
1303
- * StreamChunkedSerie, consumed too - on `by`, under `how` (`inner` when absent
1304
- * or null): a stream of the output, one probe batch joined at a time,
1305
- * the held side built and hashed first, `Serie.joinWith`'s rules for the
1306
- * columns. `how` and `options` are read before anything is consumed;
1307
- * this reader is consumed, a refused key included.
1308
- */
1309
- joinWith(
1310
- other: Serie | ChunkedSerie | StreamChunkedSerie,
1311
- by: JoinKeys,
1312
- how?: JoinHow | null,
1313
- options?: JoinOptionsInput | null,
1314
- ): StreamChunkedSerie
1315
- /**
1316
- * Spill the records this reader holds in place under the bound `options`
1317
- * states, as `spill` does, and answer this reader; a stream holds none
1318
- * and is untouched. Refused once the reader was consumed.
1319
- */
1320
- asSpilled(options?: SpillOptions | null): this
1321
- /**
1322
- * This reader with the records it holds spilled under the bound
1323
- * `options` states, handed over as a new reader: a stream moves rather
1324
- * than copies, so this reader is consumed.
1325
- */
1326
- intoSpilled(options?: SpillOptions | null): StreamChunkedSerie
1327
- }
1328
-
1329
1257
  namespace ChunkedSerie {
1330
1258
  /** The chunked serie of no chunks under `field`. */
1331
1259
  function empty(field: Field | string): ChunkedSerie
1332
- /** One held column as its one chunk, sharing its buffers; a run is refused. */
1260
+ /**
1261
+ * The rows held apart as the chunks they are: a chunked serie's own, a
1262
+ * lazy stream's batches one chunk each, an empty column no chunk, any
1263
+ * other column its one chunk, sharing its buffers; a run is refused.
1264
+ */
1333
1265
  function fromSerie(serie: Serie): ChunkedSerie
1334
1266
  /**
1335
1267
  * Held columns as chunks under one field. With no field they are one
@@ -1341,7 +1273,7 @@ declare module './index' {
1341
1273
  function fromSeries(
1342
1274
  chunks: Iterable<Serie>,
1343
1275
  field?: Field | string,
1344
- options?: ArrowCastOptions,
1276
+ options?: CastOptions,
1345
1277
  ): ChunkedSerie
1346
1278
  /**
1347
1279
  * One Apache Arrow JS vector as one chunk per `Data` it holds: of its
@@ -1351,7 +1283,7 @@ declare module './index' {
1351
1283
  function fromArrowArray(
1352
1284
  vector: ArrowVector,
1353
1285
  field?: Field | string,
1354
- options?: ArrowCastOptions,
1286
+ options?: CastOptions,
1355
1287
  ): ChunkedSerie
1356
1288
  /**
1357
1289
  * One Apache Arrow JS table as one record chunk per batch, and one
@@ -1361,7 +1293,7 @@ declare module './index' {
1361
1293
  function fromArrowBatch(
1362
1294
  batch: ArrowRecordBatch | ArrowTable,
1363
1295
  root?: Field | string,
1364
- options?: ArrowCastOptions,
1296
+ options?: CastOptions,
1365
1297
  ): ChunkedSerie
1366
1298
  /**
1367
1299
  * Drain a native reader, one record chunk per batch, every batch cast
@@ -1370,7 +1302,7 @@ declare module './index' {
1370
1302
  function fromArrowReader(
1371
1303
  reader: BatchReader,
1372
1304
  root?: Field | string,
1373
- options?: ArrowCastOptions,
1305
+ options?: CastOptions,
1374
1306
  ): ChunkedSerie
1375
1307
  }
1376
1308
 
@@ -1389,14 +1321,14 @@ declare module './index' {
1389
1321
  * Append one column as a chunk: under the field as it stands, any other
1390
1322
  * cast into it; a refusal leaves the chunks as they were.
1391
1323
  */
1392
- pushChunk(chunk: Serie, options?: ArrowCastOptions): void
1324
+ pushChunk(chunk: Serie, options?: CastOptions): void
1393
1325
  /** Every row as one column: the one join. */
1394
1326
  intoSerie(): Serie
1395
1327
  /**
1396
1328
  * Every chunk under `field` - a DataType as its required `value` field -
1397
1329
  * by one plan.
1398
1330
  */
1399
- cast(field: Field | DataType | string, options?: ArrowCastOptions): ChunkedSerie
1331
+ cast(field: Field | DataType | string, options?: CastOptions): ChunkedSerie
1400
1332
  /**
1401
1333
  * One Apache Arrow JS vector holding one `Data` per chunk. Arrow JS
1402
1334
  * holds no vector of no `Data`, so a chunked serie of no chunk is a
@@ -1486,7 +1418,7 @@ declare module './index' {
1486
1418
  const NEVER: bigint
1487
1419
  }
1488
1420
 
1489
- namespace ArrowCastPlan {
1421
+ namespace CastPlan {
1490
1422
  /**
1491
1423
  * Compile the cast from `source` to `target` once. An Apache Arrow JS
1492
1424
  * schema, table or batch names its source by its schema, imported as the
@@ -1495,17 +1427,17 @@ declare module './index' {
1495
1427
  function compile(
1496
1428
  source: Field | string | ArrowSchema | ArrowTable | ArrowRecordBatch,
1497
1429
  target: Field | string,
1498
- options?: ArrowCastOptions,
1499
- ): ArrowCastPlan
1430
+ options?: CastOptions,
1431
+ ): CastPlan
1500
1432
  }
1501
1433
 
1502
- interface ArrowCastPlan {
1434
+ interface CastPlan {
1503
1435
  /** Cast one column laid out as `source`. */
1504
1436
  apply(serie: Serie): Serie
1505
1437
  /** Cast every chunk of a chunked column laid out as `source`, kept apart. */
1506
1438
  apply(chunked: ChunkedSerie): ChunkedSerie
1507
1439
  /** The two cast answers this plan was compiled under. */
1508
- readonly options: Readonly<Required<ArrowCastOptions>>
1440
+ readonly options: Readonly<Required<CastOptions>>
1509
1441
  }
1510
1442
 
1511
1443
  interface Xxh32 {
@@ -4441,7 +4373,7 @@ declare module './index' {
4441
4373
  readSerie(options?: RecordOptionsInput | RecordProperties | null, properties?: RecordProperties | null): Serie
4442
4374
  /**
4443
4375
  * Write rows in any shape they are held - a Serie, a ChunkedSerie, a
4444
- * StreamChunkedSerie (consumed), or any value `BatchReader.from` accepts -
4376
+ * keyed or windowed serie, or any value `BatchReader.from` accepts -
4445
4377
  * under `mode`, `overwrite` when absent. A structured text document
4446
4378
  * takes `overwrite` alone, and of the options the declared field alone.
4447
4379
  */
@@ -4674,7 +4606,7 @@ export type Representation = 'value' | 'bits'
4674
4606
  * column refuses one by path. An absent answer takes the core's default:
4675
4607
  * safe, `"value"`.
4676
4608
  */
4677
- export interface ArrowCastOptions {
4609
+ export interface CastOptions {
4678
4610
  safe?: boolean
4679
4611
  representation?: Representation
4680
4612
  }
@@ -4740,7 +4672,7 @@ export type JoinKeys =
4740
4672
  * `full`.
4741
4673
  */
4742
4674
  export type JoinHow = 'inner' | 'left' | 'right' | 'full' | 'outer' | 'semi' | 'anti' | (string & {})
4743
- export type NativeSerieInput = Serie | ChunkedSerie | WindowSerie | StreamSerie | StreamChunkedSerie | KeySerie | KeySeries | StreamKeySerie
4675
+ export type NativeSerieInput = Serie | ChunkedSerie | WindowSerie | KeySerie | KeySeries | StreamKeySerie
4744
4676
  export type WindowKey = Selector | Term | string | readonly (Term | string)[] | FieldPath | readonly FieldPath[] | NativeSerieInput | ArrowVector | ArrowTable | ArrowRecordBatch
4745
4677
 
4746
4678
  /** The field paths a record partitions by: one path, its text, or an iterable of them. */
@@ -5500,9 +5432,26 @@ export interface FileSystemHandler {
5500
5432
  // The key context and payload travel separately through native stream adapters.
5501
5433
  declare module './index' {
5502
5434
  interface KeySerie { readonly rows: Serie }
5435
+ /** Iterating, `length` and `get` drain a lazy collection once; its stream's failure throws. */
5503
5436
  interface KeySeries extends Iterable<KeySerie> { [Symbol.iterator](): IterableIterator<KeySerie> }
5504
5437
  interface StreamKeySerie extends IterableIterator<KeySerie> {
5505
5438
  next(): IteratorResult<KeySerie>
5506
5439
  [Symbol.iterator](): StreamKeySerie
5507
5440
  }
5508
5441
  }
5442
+ /** What `connect` reads beside the URL: the Yggio engine's warehouse, the dialect text is read in, and every other key a session (no URL) or engine (a URL) property. */
5443
+ export type ConnectOptions = { warehouse?: Warehouse; dialect?: 'ansi' | 'doris' | 'mysql' | 'datafusion' } & Record<string, string | number | boolean | Warehouse | undefined>
5444
+ /** Open a session on the libuv pool: no URL the Yggio engine over `warehouse` (the process's SystemWarehouse by default), a URL the engine its scheme's crate answers - `mysql://` through yggio-mysql. */
5445
+ export declare function connect(url?: string | NodeURL | Url | null, options?: ConnectOptions): Promise<Connection>
5446
+ declare module './index' {
5447
+ interface Connection {
5448
+ /** Write `data` - whatever `BatchReader.from` reads - into the table `table` names by `mode`, creating it where absent (Connection::write_serie). */
5449
+ writeSerie(table: ObjectPathInput, data: BatchSource, mode?: 'overwrite' | 'append' | 'merge'): Promise<IOResult>
5450
+ }
5451
+ interface Cursor extends AsyncIterable<ArrowRecordBatch> {
5452
+ /** Run SQL text (the session's dialect) or a Plan with one record of parameters; settles with this cursor. An aborted `signal` rejects with its reason and cancels the session's statement. */
5453
+ execute(operation: string | Plan, parameters?: Scalar | Record<string, unknown> | null, options?: { signal?: AbortSignal }): Promise<Cursor>
5454
+ /** The last statement's rows drained on the libuv pool into one Apache Arrow JS table, once. */
5455
+ fetchArrowTable(): Promise<ArrowTable>
5456
+ }
5457
+ }