rowpipe 2.0.0 → 2.2.0

package/dist/writers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/writers/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAE1B,MAAM,QAAQ,GAAkC;IAC9C,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;QAC5B,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC;KAClE;IACD,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;QAC5B,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAChC,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,CAAC;KAC/E;IACD,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAChC,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC;KAC9E;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC;QAChC,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC;KACvE;IACD,EAAE,EAAE;QACF,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC;KACvE;IACD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC;QAC/B,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC;KACtE;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC;KAC1D;IACD,KAAK,EAAE;QACL,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;QAC5C,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC;KAC3D;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC;KACnE;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,MAAM,SAAS,GAAG,yBAAyB,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACpE,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC9D,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,MAAyB,EACzB,UAA+C,EAAE;IAEjD,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAE3C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5D,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IACpD,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,KAAK,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,+BAA+B,MAAM,6BAA6B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrG,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/writers/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,cAAc,eAAe,CAAC;AAC9B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAE1B,MAAM,QAAQ,GAAkC;IAC9C,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;QAC5B,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC;KAClE;IACD,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;QAC5B,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAChC,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,CAAC;KAC/E;IACD,GAAG,EAAE;QACH,IAAI,EAAE,KAAK;QACX,UAAU,EAAE,CAAC,MAAM,CAAC;QACpB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAChC,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC;KAC9E;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC;QAChC,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC;KACvE;IACD,EAAE,EAAE;QACF,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,CAAC,KAAK,CAAC;QACnB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC;KACvE;IACD,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC;QAC/B,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC;KACtE;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC;KAC1D;IACD,KAAK,EAAE;QACL,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;QAC5C,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC;KAC3D;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC;KACnE;IACD,EAAE,EAAE;QACF,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;QAC1C,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;YAChC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,IAAI,oBAAoB,CAAC,wDAAwD,CAAC,CAAC;YAC3F,CAAC;YACD,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;KACF;IACD,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,EAAE;QACd,YAAY,EAAE,GAAG,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;YAChC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,IAAI,oBAAoB,CAAC,wDAAwD,CAAC,CAAC;YAC3F,CAAC;YACD,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;KACF;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,IAAI,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAAG,yBAAyB,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACpE,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC9D,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,MAAyB,EACzB,UAA+C,EAAE;IAEjD,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAE3C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QAC5D,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC;IACpD,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,KAAK,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,+BAA+B,MAAM,6BAA6B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrG,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC"}

package/dist/writers/json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/writers/json.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,qBAAa,UAAW,YAAW,aAAa;IAC9C,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM;IAIrC,OAAO,CAAC,eAAe;YAST,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/writers/json.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjF,qBAAa,UAAW,YAAW,aAAa;IAC9C,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM;IAIrC,OAAO,CAAC,eAAe;YAQT,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}

package/dist/writers/json.js

@@ -1,4 +1,5 @@
 import { closeWritableStream, openCompressedWriteStream } from "../utils/compression.js";
+import { safeJsonStringify } from "../utils/formatting.js";
 export class JSONWriter {
     output;
     createdStream;
@@ -6,8 +7,7 @@ export class JSONWriter {
         this.output = output;
     }
     getOutputStream(opts) {
-        const gzip = opts?.gzip;
-        const stream = openCompressedWriteStream(this.output, { gzip });
+        const stream = openCompressedWriteStream(this.output, opts);
         if (typeof this.output === "string") {
             this.createdStream = stream;
         }
@@ -26,14 +26,15 @@ export class JSONWriter {
             for (const row of batch.rows) {
                 const prefix = isFirstRow ? "  " : ",\n  ";
                 isFirstRow = false;
-                await this.writeChunk(outStream, `${prefix}${JSON.stringify(row)}`);
+                await this.writeChunk(outStream, `${prefix}${safeJsonStringify(row)}`);
             }
         }
         await this.writeChunk(outStream, "\n]\n");
     }
     async close() {
-        if (this.createdStream) {
-            await closeWritableStream(this.createdStream);
+        const s = this.createdStream || (typeof this.output !== "string" ? this.output : undefined);
+        if (s && s !== process.stdout) {
+            await closeWritableStream(s);
         }
     }
 }

package/dist/writers/json.js.map

@@ -1 +1 @@
-{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/writers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAIzF,MAAM,OAAO,UAAU;IACb,MAAM,CAAoB;IAC1B,aAAa,CAAY;IAEjC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,CAAC;QACxB,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,UAAU,GAAG,IAAI,CAAC;QAEtB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAExC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3C,UAAU,GAAG,KAAK,CAAC;gBACnB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/writers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAIzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,OAAO,UAAU;IACb,MAAM,CAAoB;IAC1B,aAAa,CAAY;IAEjC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5D,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,UAAU,GAAG,IAAI,CAAC;QAEtB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAExC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3C,UAAU,GAAG,KAAK,CAAC;gBACnB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,CAAC,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC5F,IAAI,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/writers/jsonl.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/writers/jsonl.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,qBAAa,WAAY,YAAW,aAAa;IAC/C,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM;IAIrC,OAAO,CAAC,eAAe;YAST,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}
+{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/writers/jsonl.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,qBAAa,WAAY,YAAW,aAAa;IAC/C,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM;IAIrC,OAAO,CAAC,eAAe;YAQT,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}

package/dist/writers/jsonl.js

@@ -1,4 +1,5 @@
 import { closeWritableStream, openCompressedWriteStream } from "../utils/compression.js";
+import { safeJsonStringify } from "../utils/formatting.js";
 export class JSONLWriter {
     output;
     createdStream;
@@ -6,8 +7,7 @@ export class JSONLWriter {
         this.output = output;
     }
     getOutputStream(opts) {
-        const gzip = opts?.gzip;
-        const stream = openCompressedWriteStream(this.output, { gzip });
+        const stream = openCompressedWriteStream(this.output, opts);
         if (typeof this.output === "string") {
             this.createdStream = stream;
         }
@@ -23,7 +23,7 @@ export class JSONLWriter {
         for await (const batch of dataStream) {
             let chunkText = "";
             for (const row of batch.rows) {
-                chunkText += JSON.stringify(row) + "\n";
+                chunkText += safeJsonStringify(row) + "\n";
             }
             if (chunkText.length > 0) {
                 await this.writeChunk(outStream, chunkText);
@@ -31,8 +31,9 @@ export class JSONLWriter {
         }
     }
     async close() {
-        if (this.createdStream) {
-            await closeWritableStream(this.createdStream);
+        const s = this.createdStream || (typeof this.output !== "string" ? this.output : undefined);
+        if (s && s !== process.stdout) {
+            await closeWritableStream(s);
         }
     }
 }

package/dist/writers/jsonl.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/writers/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAIzF,MAAM,OAAO,WAAW;IACd,MAAM,CAAoB;IAC1B,aAAa,CAAY;IAEjC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,CAAC;QACxB,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAEhD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YAC1C,CAAC;YACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/writers/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAI3D,MAAM,OAAO,WAAW;IACd,MAAM,CAAoB;IAC1B,aAAa,CAAY;IAEjC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5D,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAEhD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YAC7C,CAAC;YACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,CAAC,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC5F,IAAI,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/writers/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,aAAa;IAClD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,qBAA0B;IAK1E,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,UAAU;YAUJ,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAsFrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,aAAa;IAClD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,qBAA0B;IAK1E,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,UAAU;YAUJ,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAsFrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}

package/dist/writers/markdown.js

@@ -12,8 +12,8 @@ export class MarkdownWriter {
         this.options = { ...options };
     }
     getOutputStream(opts) {
-        const gzip = opts?.gzip ?? this.options.gzip;
-        const stream = openCompressedWriteStream(this.output, { gzip });
+        const merged = { ...this.options, ...opts };
+        const stream = openCompressedWriteStream(this.output, merged);
         if (typeof this.output === "string") {
             this.createdStream = stream;
         }
@@ -103,8 +103,9 @@ export class MarkdownWriter {
         }
     }
     async close() {
-        if (this.createdStream) {
-            await closeWritableStream(this.createdStream);
+        const s = this.createdStream || (typeof this.output !== "string" ? this.output : undefined);
+        if (s && s !== process.stdout) {
+            await closeWritableStream(s);
         }
     }
 }

package/dist/writers/markdown.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAQzF;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAoB;IAC1B,OAAO,CAAwB;IAC/B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAAiC,EAAE;QACxE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,UAAU,CAAC,KAAc;QAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,GAAG;aACP,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAA0B;YAC3C,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC;QAC7D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,gBAAgB;YAChB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YAC/D,OAAO;QACT,CAAC;QAED,mCAAmC;QACnC,MAAM,SAAS,GAAa,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACpD,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;oBACzB,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBACvB,CAAC;YACH,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,MAAM,YAAY,GAAc,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YACvC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,IAAI,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACvC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CACtE,CAAC;QACF,MAAM,UAAU,GAAG,KAAK,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAEtD,6CAA6C;QAC7C,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5C,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,MAAM,aAAa,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAE5D,IAAI,MAAM,GAAG,UAAU,GAAG,aAAa,CAAC;QAExC,sBAAsB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACnC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAC5E,CAAC;YACF,MAAM,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAQzF;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAoB;IAC1B,OAAO,CAAwB;IAC/B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAAiC,EAAE;QACxE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9D,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,UAAU,CAAC,KAAc;QAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,GAAG;aACP,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAA0B;YAC3C,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC;QAC7D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,gBAAgB;YAChB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YAC/D,OAAO;QACT,CAAC;QAED,mCAAmC;QACnC,MAAM,SAAS,GAAa,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACpD,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;oBACzB,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBACvB,CAAC;YACH,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,MAAM,YAAY,GAAc,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YACvC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,IAAI,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACvC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CACtE,CAAC;QACF,MAAM,UAAU,GAAG,KAAK,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAEtD,6CAA6C;QAC7C,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5C,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,MAAM,aAAa,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAE5D,IAAI,MAAM,GAAG,UAAU,GAAG,aAAa,CAAC;QAExC,sBAAsB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACnC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAC5E,CAAC;YACF,MAAM,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,CAAC,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC5F,IAAI,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/writers/parquet.d.ts

@@ -2,7 +2,7 @@ import type { Writable } from "node:stream";
 import type { DataStream, TabularWriter, WriterOptions } from "../core/types.js";
 export interface ParquetWriterOptions extends WriterOptions {
     schema?: Record<string, any>;
-    compression?: "UNCOMPRESSED" | "SNAPPY" | "GZIP";
+    compression?: "UNCOMPRESSED" | "SNAPPY" | "GZIP" | "ZSTD" | "BROTLI" | string;
 }
 /**
  * Streaming Apache Parquet Writer.

package/dist/writers/parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/writers/parquet.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAG5C,OAAO,KAAK,EAAE,UAAU,EAAO,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtF,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,WAAW,CAAC,EAAE,cAAc,GAAG,QAAQ,GAAG,MAAM,CAAC;CAClD;AAiED;;;GAGG;AACH,qBAAa,aAAc,YAAW,aAAa;IACjD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,MAAM,CAAC,CAAM;gBAET,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,oBAAyB;IAKnE,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAS7B"}
+{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/writers/parquet.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAG5C,OAAO,KAAK,EAAE,UAAU,EAAO,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtF,MAAM,WAAW,oBAAqB,SAAQ,aAAa;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,WAAW,CAAC,EAAE,cAAc,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC/E;AAiED;;;GAGG;AACH,qBAAa,aAAc,YAAW,aAAa;IACjD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,MAAM,CAAC,CAAM;gBAET,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,oBAAyB;IAKnE,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAS7B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rowpipe",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "description": "Stream-first tabular data toolkit for CSV, TSV, JSON, JSONL, XLSX, Apache Parquet, and Markdown",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -19,8 +19,11 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "bench": "tsx benchmarks/memory-bench.ts",
+    "benchmark:db": "tsx benchmarks/db-bench.ts",
     "benchmark:diff": "tsx benchmarks/diff-bench.ts",
-    "benchmark:files": "tsx benchmarks/files-bench.ts"
+    "benchmark:files": "tsx benchmarks/files-bench.ts",
+    "benchmark:ops": "tsx benchmarks/operations-bench.ts",
+    "benchmark:expr": "tsx benchmarks/expression-bench.ts"
   },
   "keywords": [
     "csv",
@@ -41,15 +44,21 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
+    "@bokuweb/zstd-wasm": "^0.0.27",
     "commander": "^13.1.0",
+    "fzstd": "^0.1.1",
     "hyparquet": "^1.30.1",
     "hyparquet-compressors": "^1.1.1",
+    "mysql2": "^3.24.4",
     "parquetjs-lite": "^0.8.7",
+    "pg": "^8.23.0",
+    "pg-query-stream": "^4.17.0",
     "read-excel-file": "^9.3.10",
     "write-excel-file": "^4.1.1"
   },
   "devDependencies": {
     "@types/node": "^22.13.9",
+    "@types/pg": "^8.23.1",
     "tsx": "^4.19.3",
     "typescript": "^5.8.2",
     "vitest": "^3.0.7"

package/skills/rowpipe/SKILL.md

@@ -1,11 +1,11 @@
 ---
 name: rowpipe
-description: Stream-first tabular data toolkit for inspecting, analyzing, transforming, validating, converting, mapping, and reducing large datasets across CSV, TSV, JSON, JSONL, XLSX, Apache Parquet, and Markdown formats with bounded O(1) memory and extreme throughput (~500,000 rows/s).
+description: Stream-first tabular data toolkit for inspecting, analyzing, transforming, validating, converting, mapping, reducing, and streaming large datasets across PostgreSQL, MySQL, SQLite, CSV, TSV, JSON, JSONL, XLSX, Apache Parquet, and Markdown formats with bounded O(1) memory and extreme throughput (~500,000 rows/s).
 ---
 
 # Rowpipe: Stream-First Tabular Data Toolkit
 
-Rowpipe is a high-performance, stream-first command-line toolkit and Node.js library for tabular data processing. It processes gigabyte-scale datasets with bounded $O(1)$ memory usage, backpressure, batching, and zero-`eval` safe expression execution.
+Rowpipe is a high-performance, stream-first command-line toolkit and Node.js library for tabular data processing. It processes gigabyte-scale datasets with bounded $O(1)$ memory usage, backpressure, batching, pushdown optimization, and zero-`eval` safe expression execution.
 
 ---
 
@@ -13,6 +13,10 @@ Rowpipe is a high-performance, stream-first command-line toolkit and Node.js lib
 
 | Task | Command Syntax | Description |
 |---|---|---|
+| **Database** | `rowpipe db <url> [--table <table>] [--query <sql>]` | Stream directly from PostgreSQL, MySQL, and SQLite databases with cursor batching and pushdown |
+| **DB Tables** | `rowpipe db <url> --tables [--json]` | List all tables in the target database |
+| **DB Schema** | `rowpipe db <url> --schema <table> [--json]` | Inspect database table columns, types, nullability, and primary keys |
+| **File to DB** | `rowpipe <file> --to-db <url> --to-table <table> [--create-table]` | Stream load files directly into database tables with batch inserts and transactions |
 | **Inspect** | `rowpipe inspect <file> [--sheet <name>] [--json]` | Inspect row count, column types, null %, and HLL uniqueness |
 | **Convert** | `rowpipe convert <input> <output> [--all-sheets]` | Stream convert between CSV, TSV, JSON, JSONL, XLSX, Parquet, Markdown |
 | **Schema** | `rowpipe schema <file> [--full] [--sample <n>]` | Infer column types and semantic patterns (email, url, uuid) |
@@ -20,12 +24,21 @@ Rowpipe is a high-performance, stream-first command-line toolkit and Node.js lib
 | **Map** | `rowpipe map <file> "<col1=expr1>" "<col2=expr2>"` | Derive and compute columns per row with arithmetic/JSON dot-notation |
 | **Filter** | `rowpipe filter <file> "<expression>"` | Filter rows using safe expression engine and pipe syntax (`\|`) |
 | **Reduce** | `rowpipe reduce <file> "<agg1=func(col)>" [--by <cols>]` | Global & group-by streaming aggregations (`sum`, `avg`, `min`, `max`, `count`, `countDistinct`, `stddev`) |
+| **Limit / Head** | `rowpipe limit <file> <n>` / `rowpipe head <file> [-n <n>]` | Emit first N rows with early upstream stream cancellation |
+| **Offset** | `rowpipe offset <file> <n>` | Skip first N rows without scanning remainder |
+| **Tail** | `rowpipe tail <file> [-n <n>]` | Emit last N rows using bounded $O(N)$ circular ring buffer |
+| **Sort** | `rowpipe sort <file> --by <cols> [--memory-limit <m>]` | Multi-column typed external merge sort with disk run spilling and $K$-way min-heap |
+| **Top** | `rowpipe top <file> --by <cols> [-n <n>]` | Extract Top-K largest/smallest records using bounded $O(K)$ heap |
+| **Unique** | `rowpipe unique <file> [--by <cols>] [--keep first\|last]` | Deduplicate rows by key columns or entire rows with spillable hash key store |
+| **Count** | `rowpipe count <file> [--by <cols>] [--distinct <col>]` | Ultra-fast row total, distinct count (HyperLogLog), or group-count |
+| **Group** | `rowpipe group <file> --by <cols> [--count] [--sum <c>]` | Group tabular streams with hash accumulation and multi-column aggregations |
+| **Explain** | `rowpipe explain <file\|db_url> [pipeline flags]` | Visualize execution pipeline, database pushdown, and memory bounds |
 | **Select** | `rowpipe select <file> <col1,col2,...>` | Project subset of columns in order |
 | **Rename** | `rowpipe rename <file> <old1=new1> <old2=new2>` | Rename columns in stream |
 | **Cast** | `rowpipe cast <file> <col1:type> [--on-error null]` | Stream cast types (`integer`, `number`, `boolean`, `date`) |
 | **Sample** | `rowpipe sample <file> --rows <n> [--seed <s>]` | Reservoir sampling with bounded memory |
 | **Validate** | `rowpipe validate <file> --schema <schema.json>` | Validate stream against JSON schema definition |
-| **Diff** | `rowpipe diff <left> <right> --key <cols> [opts]` | Stream compare datasets by key across formats with bounded RAM and disk spilling |
+| **Diff** | `rowpipe diff <left> <right> --key <cols> [opts]` | Stream compare datasets by key across formats and databases with bounded RAM |
 | **Files** | `rowpipe files <path> [--include <glob>] [opts]` | Stream filesystem directory as tabular records with bounded memory, globs, hashes, and MIME |
 
 ---