rowpipe 2.2.1 → 2.10.2

package/dist/readers/json.js

@@ -1,5 +1,6 @@
 import { openDecompressedReadStream } from "../utils/compression.js";
 import { ParseError } from "../core/errors.js";
+import { RowErrorHandler } from "../core/error-handler.js";
 /**
  * Incremental streaming JSON parser for top-level arrays [ {...}, ... ]
  * and nested array paths (e.g. data.results).
@@ -26,6 +27,11 @@ export class JSONReader {
         const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
         const targetPath = mergedOptions.path ?? "";
         const stream = this.getInputStream(mergedOptions);
+        const errorHandler = new RowErrorHandler({
+            strategy: mergedOptions.onError,
+            badRowsLog: mergedOptions.badRowsLog,
+            sourceFile: mergedOptions.filePath,
+        });
         let buffer = "";
         let cursor = 0;
         let byteOffset = 0;
@@ -42,154 +48,172 @@ export class JSONReader {
         let rowIndex = 0;
         // Helper for path navigation if targetPath is set
         const targetPathParts = targetPath ? targetPath.split(".") : [];
-        for await (const chunk of stream) {
-            const chunkStr = typeof chunk === "string" ? chunk : chunk.toString("utf8");
-            buffer += chunkStr;
-            while (cursor < buffer.length) {
-                const char = buffer[cursor];
-                if (isEscaped) {
-                    isEscaped = false;
-                    cursor++;
-                    byteOffset++;
-                    continue;
-                }
-                if (char === "\\") {
-                    isEscaped = true;
-                    cursor++;
-                    byteOffset++;
-                    continue;
-                }
-                if (char === '"') {
-                    inString = !inString;
-                    cursor++;
-                    byteOffset++;
-                    continue;
-                }
-                if (inString) {
-                    cursor++;
-                    byteOffset++;
-                    continue;
-                }
-                // We are outside of any string literal
-                if (!foundTargetArray) {
-                    // If we need to find target path (e.g. data.results)
-                    // Look for keys and match path parts
-                    if (char === "{") {
+        try {
+            for await (const chunk of stream) {
+                const chunkStr = typeof chunk === "string" ? chunk : chunk.toString("utf8");
+                buffer += chunkStr;
+                while (cursor < buffer.length) {
+                    const char = buffer[cursor];
+                    if (isEscaped) {
+                        isEscaped = false;
                         cursor++;
                         byteOffset++;
                         continue;
                     }
-                    else if (char === "[") {
-                        // Check if current path matches targetPath
-                        // If path matched or we hit array at root
-                        foundTargetArray = true;
-                        inArray = true;
+                    if (char === "\\") {
+                        isEscaped = true;
                         cursor++;
                         byteOffset++;
                         continue;
                     }
-                    else if (char === ":") {
-                        // Scan backwards from ':' to find previous string key
-                        let keyEnd = cursor - 1;
-                        while (keyEnd >= 0 && /\s/.test(buffer[keyEnd]))
-                            keyEnd--;
-                        if (keyEnd >= 0 && buffer[keyEnd] === '"') {
-                            let keyStart = keyEnd - 1;
-                            while (keyStart >= 0 && buffer[keyStart] !== '"')
-                                keyStart--;
-                            if (keyStart >= 0) {
-                                const keyName = buffer.slice(keyStart + 1, keyEnd);
-                                currentPath.push(keyName);
-                                if (currentPath.join(".") === targetPath ||
-                                    keyName === targetPathParts[targetPathParts.length - 1]) {
-                                    // Next non-whitespace should be '['
-                                    // Let it continue to find '['
-                                }
-                            }
-                        }
+                    if (char === '"') {
+                        inString = !inString;
                         cursor++;
                         byteOffset++;
                         continue;
                     }
-                    cursor++;
-                    byteOffset++;
-                    continue;
-                }
-                // Now we are streaming inside the target array
-                if (!inArray) {
-                    if (char === "[") {
-                        inArray = true;
+                    if (inString) {
                         cursor++;
                         byteOffset++;
                         continue;
                     }
-                }
-                else {
-                    // Inside array: looking for objects '{ ... }'
-                    if (char === "{") {
-                        if (objectDepth === 0) {
-                            objectStartIndex = cursor;
+                    // We are outside of any string literal
+                    if (!foundTargetArray) {
+                        // If we need to find target path (e.g. data.results)
+                        // Look for keys and match path parts
+                        if (char === "{") {
+                            cursor++;
+                            byteOffset++;
+                            continue;
                         }
-                        objectDepth++;
-                    }
-                    else if (char === "}") {
-                        objectDepth--;
-                        if (objectDepth === 0 && objectStartIndex !== -1) {
-                            // We have captured a complete JSON object!
-                            const objectStr = buffer.slice(objectStartIndex, cursor + 1);
-                            let parsedRow;
-                            try {
-                                parsedRow = JSON.parse(objectStr);
+                        else if (char === "[") {
+                            // Check if current path matches targetPath
+                            // If path matched or we hit array at root
+                            foundTargetArray = true;
+                            inArray = true;
+                            cursor++;
+                            byteOffset++;
+                            continue;
+                        }
+                        else if (char === ":") {
+                            // Scan backwards from ':' to find previous string key
+                            let keyEnd = cursor - 1;
+                            while (keyEnd >= 0 && /\s/.test(buffer[keyEnd]))
+                                keyEnd--;
+                            if (keyEnd >= 0 && buffer[keyEnd] === '"') {
+                                let keyStart = keyEnd - 1;
+                                while (keyStart >= 0 && buffer[keyStart] !== '"')
+                                    keyStart--;
+                                if (keyStart >= 0) {
+                                    const keyName = buffer.slice(keyStart + 1, keyEnd);
+                                    currentPath.push(keyName);
+                                    if (currentPath.join(".") === targetPath ||
+                                        keyName === targetPathParts[targetPathParts.length - 1]) {
+                                        // Next non-whitespace should be '['
+                                        // Let it continue to find '['
+                                    }
+                                }
                             }
-                            catch (err) {
-                                throw new ParseError(`Malformed JSON object in array: ${err.message}`, {
-                                    file: mergedOptions.filePath,
-                                    row: rowIndex + 1,
-                                    byteOffset,
-                                });
+                            cursor++;
+                            byteOffset++;
+                            continue;
+                        }
+                        cursor++;
+                        byteOffset++;
+                        continue;
+                    }
+                    // Now we are streaming inside the target array
+                    if (!inArray) {
+                        if (char === "[") {
+                            inArray = true;
+                            cursor++;
+                            byteOffset++;
+                            continue;
+                        }
+                    }
+                    else {
+                        // Inside array: looking for objects '{ ... }'
+                        if (char === "{") {
+                            if (objectDepth === 0) {
+                                objectStartIndex = cursor;
                             }
-                            if (typeof parsedRow === "object" &&
-                                parsedRow !== null &&
-                                !Array.isArray(parsedRow)) {
-                                rowsInCurrentBatch.push(parsedRow);
-                                rowIndex++;
-                                if (rowsInCurrentBatch.length >= batchSize) {
-                                    yield {
-                                        rows: rowsInCurrentBatch,
-                                        offset: globalOffset,
-                                    };
-                                    globalOffset += rowsInCurrentBatch.length;
-                                    rowsInCurrentBatch = [];
+                            objectDepth++;
+                        }
+                        else if (char === "}") {
+                            objectDepth--;
+                            if (objectDepth === 0 && objectStartIndex !== -1) {
+                                // We have captured a complete JSON object!
+                                const objectStr = buffer.slice(objectStartIndex, cursor + 1);
+                                let parsedRow;
+                                try {
+                                    parsedRow = JSON.parse(objectStr);
+                                }
+                                catch (err) {
+                                    const parseErr = new ParseError(`Malformed JSON object in array: ${err.message}`, {
+                                        file: mergedOptions.filePath,
+                                        row: rowIndex + 1,
+                                        byteOffset,
+                                    });
+                                    errorHandler.handle(parseErr, {
+                                        file: mergedOptions.filePath,
+                                        row: rowIndex + 1,
+                                        byteOffset,
+                                        raw: objectStr,
+                                    });
+                                    if (mergedOptions.onParseError) {
+                                        mergedOptions.onParseError(err, objectStr);
+                                    }
+                                    objectStartIndex = -1;
+                                    cursor++;
+                                    byteOffset++;
+                                    continue;
                                 }
+                                if (typeof parsedRow === "object" &&
+                                    parsedRow !== null &&
+                                    !Array.isArray(parsedRow)) {
+                                    rowsInCurrentBatch.push(parsedRow);
+                                    rowIndex++;
+                                    if (rowsInCurrentBatch.length >= batchSize) {
+                                        yield {
+                                            rows: rowsInCurrentBatch,
+                                            offset: globalOffset,
+                                        };
+                                        globalOffset += rowsInCurrentBatch.length;
+                                        rowsInCurrentBatch = [];
+                                    }
+                                }
+                                objectStartIndex = -1;
                             }
-                            objectStartIndex = -1;
                         }
+                        else if (char === "]" && objectDepth === 0) {
+                            // End of array reached
+                            inArray = false;
+                            break;
+                        }
+                        cursor++;
+                        byteOffset++;
                     }
-                    else if (char === "]" && objectDepth === 0) {
-                        // End of array reached
-                        inArray = false;
-                        break;
-                    }
-                    cursor++;
-                    byteOffset++;
+                }
+                // Compact buffer while preserving active object if in the middle of parsing
+                if (objectStartIndex !== -1) {
+                    buffer = buffer.slice(objectStartIndex);
+                    cursor = cursor - objectStartIndex;
+                    objectStartIndex = 0;
+                }
+                else {
+                    buffer = buffer.slice(cursor);
+                    cursor = 0;
                 }
             }
-            // Compact buffer while preserving active object if in the middle of parsing
-            if (objectStartIndex !== -1) {
-                buffer = buffer.slice(objectStartIndex);
-                cursor = cursor - objectStartIndex;
-                objectStartIndex = 0;
-            }
-            else {
-                buffer = buffer.slice(cursor);
-                cursor = 0;
+            if (rowsInCurrentBatch.length > 0) {
+                yield {
+                    rows: rowsInCurrentBatch,
+                    offset: globalOffset,
+                };
             }
         }
-        if (rowsInCurrentBatch.length > 0) {
-            yield {
-                rows: rowsInCurrentBatch,
-                offset: globalOffset,
-            };
+        finally {
+            errorHandler.close();
         }
     }
     async inspect(options) {

package/dist/readers/json.js.map

@@ -1 +1 @@
-{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/readers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAe/C;;;;GAIG;AACH,MAAM,OAAO,UAAU;IACb,KAAK,CAAoB;IACzB,OAAO,CAAoB;IAEnC,YAAY,KAAwB,EAAE,UAA6B,EAAE;QACnE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAoB;QACzC,OAAO,0BAA0B,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,OAAuB;QACjC,MAAM,aAAa,GAAsB;YACvC,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAElD,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,gBAAgB,GAAG,CAAC,UAAU,CAAC,CAAC,oCAAoC;QACxE,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,wCAAwC;QACxC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAE1B,IAAI,kBAAkB,GAAU,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,kDAAkD;QAClD,MAAM,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEhE,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,MAAM,QAAQ,GACZ,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,KAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACzE,MAAM,IAAI,QAAQ,CAAC;YAEnB,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBAE5B,IAAI,SAAS,EAAE,CAAC;oBACd,SAAS,GAAG,KAAK,CAAC;oBAClB,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;oBACb,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAClB,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;oBACb,SAAS;gBACX,CAAC;gBAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;oBACjB,QAAQ,GAAG,CAAC,QAAQ,CAAC;oBACrB,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;oBACb,SAAS;gBACX,CAAC;gBAED,IAAI,QAAQ,EAAE,CAAC;oBACb,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;oBACb,SAAS;gBACX,CAAC;gBAED,uCAAuC;gBACvC,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACtB,qDAAqD;oBACrD,qCAAqC;oBACrC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACjB,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;yBAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACxB,2CAA2C;wBAC3C,0CAA0C;wBAC1C,gBAAgB,GAAG,IAAI,CAAC;wBACxB,OAAO,GAAG,IAAI,CAAC;wBACf,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;yBAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACxB,sDAAsD;wBACtD,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,CAAC;wBACxB,OAAO,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC;4BAAE,MAAM,EAAE,CAAC;wBAC3D,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC;4BAC1C,IAAI,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;4BAC1B,OAAO,QAAQ,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,GAAG;gCAAE,QAAQ,EAAE,CAAC;4BAC7D,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gCAClB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;gCACnD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gCAC1B,IACE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,UAAU;oCACpC,OAAO,KAAK,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,EACvD,CAAC;oCACD,oCAAoC;oCACpC,8BAA8B;gCAChC,CAAC;4BACH,CAAC;wBACH,CAAC;wBACD,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;oBACD,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;oBACb,SAAS;gBACX,CAAC;gBAED,+CAA+C;gBAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACjB,OAAO,GAAG,IAAI,CAAC;wBACf,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,8CAA8C;oBAC9C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACjB,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;4BACtB,gBAAgB,GAAG,MAAM,CAAC;wBAC5B,CAAC;wBACD,WAAW,EAAE,CAAC;oBAChB,CAAC;yBAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACxB,WAAW,EAAE,CAAC;wBACd,IAAI,WAAW,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,CAAC,EAAE,CAAC;4BACjD,2CAA2C;4BAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;4BAC7D,IAAI,SAAkB,CAAC;4BACvB,IAAI,CAAC;gCACH,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;4BACpC,CAAC;4BAAC,OAAO,GAAG,EAAE,CAAC;gCACb,MAAM,IAAI,UAAU,CAClB,mCAAoC,GAAa,CAAC,OAAO,EAAE,EAC3D;oCACE,IAAI,EAAE,aAAa,CAAC,QAAQ;oCAC5B,GAAG,EAAE,QAAQ,GAAG,CAAC;oCACjB,UAAU;iCACX,CACF,CAAC;4BACJ,CAAC;4BAED,IACE,OAAO,SAAS,KAAK,QAAQ;gCAC7B,SAAS,KAAK,IAAI;gCAClB,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EACzB,CAAC;gCACD,kBAAkB,CAAC,IAAI,CAAC,SAAgB,CAAC,CAAC;gCAC1C,QAAQ,EAAE,CAAC;gCAEX,IAAI,kBAAkB,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oCAC3C,MAAM;wCACJ,IAAI,EAAE,kBAAkB;wCACxB,MAAM,EAAE,YAAY;qCACrB,CAAC;oCACF,YAAY,IAAI,kBAAkB,CAAC,MAAM,CAAC;oCAC1C,kBAAkB,GAAG,EAAE,CAAC;gCAC1B,CAAC;4BACH,CAAC;4BAED,gBAAgB,GAAG,CAAC,CAAC,CAAC;wBACxB,CAAC;oBACH,CAAC;yBAAM,IAAI,IAAI,KAAK,GAAG,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;wBAC7C,uBAAuB;wBACvB,OAAO,GAAG,KAAK,CAAC;wBAChB,MAAM;oBACR,CAAC;oBAED,MAAM,EAAE,CAAC;oBACT,UAAU,EAAE,CAAC;gBACf,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,IAAI,gBAAgB,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC5B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBACxC,MAAM,GAAG,MAAM,GAAG,gBAAgB,CAAC;gBACnC,gBAAgB,GAAG,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9B,MAAM,GAAG,CAAC,CAAC;YACb,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM;gBACJ,IAAI,EAAE,kBAAkB;gBACxB,MAAM,EAAE,YAAY;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAuB;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACrE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,CAAC;YACD,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,CAAC;QAED,OAAO;YACL,MAAM,EAAE,MAAM;YACd,QAAQ;YACR,WAAW,EAAE,WAAW,CAAC,MAAM;YAC/B,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI;gBACJ,IAAI,EAAE,QAAQ;gBACd,cAAc,EAAE,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/readers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAe3D;;;;GAIG;AACH,MAAM,OAAO,UAAU;IACb,KAAK,CAAoB;IACzB,OAAO,CAAoB;IAEnC,YAAY,KAAwB,EAAE,UAA6B,EAAE;QACnE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAoB;QACzC,OAAO,0BAA0B,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,OAAuB;QACjC,MAAM,aAAa,GAAsB;YACvC,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAElD,MAAM,YAAY,GAAG,IAAI,eAAe,CAAC;YACvC,QAAQ,EAAE,aAAa,CAAC,OAAO;YAC/B,UAAU,EAAE,aAAa,CAAC,UAAU;YACpC,UAAU,EAAE,aAAa,CAAC,QAAQ;SACnC,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,gBAAgB,GAAG,CAAC,UAAU,CAAC,CAAC,oCAAoC;QACxE,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,wCAAwC;QACxC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;QAE1B,IAAI,kBAAkB,GAAU,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,kDAAkD;QAClD,MAAM,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEhE,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,MAAM,QAAQ,GACZ,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,KAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACzE,MAAM,IAAI,QAAQ,CAAC;gBAEnB,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;oBAE5B,IAAI,SAAS,EAAE,CAAC;wBACd,SAAS,GAAG,KAAK,CAAC;wBAClB,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;oBAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBAClB,SAAS,GAAG,IAAI,CAAC;wBACjB,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;oBAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;wBACjB,QAAQ,GAAG,CAAC,QAAQ,CAAC;wBACrB,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;oBAED,IAAI,QAAQ,EAAE,CAAC;wBACb,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;oBAED,uCAAuC;oBACvC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACtB,qDAAqD;wBACrD,qCAAqC;wBACrC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACjB,MAAM,EAAE,CAAC;4BACT,UAAU,EAAE,CAAC;4BACb,SAAS;wBACX,CAAC;6BAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACxB,2CAA2C;4BAC3C,0CAA0C;4BAC1C,gBAAgB,GAAG,IAAI,CAAC;4BACxB,OAAO,GAAG,IAAI,CAAC;4BACf,MAAM,EAAE,CAAC;4BACT,UAAU,EAAE,CAAC;4BACb,SAAS;wBACX,CAAC;6BAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACxB,sDAAsD;4BACtD,IAAI,MAAM,GAAG,MAAM,GAAG,CAAC,CAAC;4BACxB,OAAO,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC;gCAAE,MAAM,EAAE,CAAC;4BAC3D,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC;gCAC1C,IAAI,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;gCAC1B,OAAO,QAAQ,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,GAAG;oCAAE,QAAQ,EAAE,CAAC;gCAC7D,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;oCAClB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;oCACnD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oCAC1B,IACE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,UAAU;wCACpC,OAAO,KAAK,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,EACvD,CAAC;wCACD,oCAAoC;wCACpC,8BAA8B;oCAChC,CAAC;gCACH,CAAC;4BACH,CAAC;4BACD,MAAM,EAAE,CAAC;4BACT,UAAU,EAAE,CAAC;4BACb,SAAS;wBACX,CAAC;wBACD,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;wBACb,SAAS;oBACX,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACjB,OAAO,GAAG,IAAI,CAAC;4BACf,MAAM,EAAE,CAAC;4BACT,UAAU,EAAE,CAAC;4BACb,SAAS;wBACX,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,8CAA8C;wBAC9C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACjB,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;gCACtB,gBAAgB,GAAG,MAAM,CAAC;4BAC5B,CAAC;4BACD,WAAW,EAAE,CAAC;wBAChB,CAAC;6BAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACxB,WAAW,EAAE,CAAC;4BACd,IAAI,WAAW,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,CAAC,EAAE,CAAC;gCACjD,2CAA2C;gCAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gCAC7D,IAAI,SAAkB,CAAC;gCACvB,IAAI,CAAC;oCACH,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gCACpC,CAAC;gCAAC,OAAO,GAAG,EAAE,CAAC;oCACb,MAAM,QAAQ,GAAG,IAAI,UAAU,CAC7B,mCAAoC,GAAa,CAAC,OAAO,EAAE,EAC3D;wCACE,IAAI,EAAE,aAAa,CAAC,QAAQ;wCAC5B,GAAG,EAAE,QAAQ,GAAG,CAAC;wCACjB,UAAU;qCACX,CACF,CAAC;oCACF,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE;wCAC5B,IAAI,EAAE,aAAa,CAAC,QAAQ;wCAC5B,GAAG,EAAE,QAAQ,GAAG,CAAC;wCACjB,UAAU;wCACV,GAAG,EAAE,SAAS;qCACf,CAAC,CAAC;oCACH,IAAI,aAAa,CAAC,YAAY,EAAE,CAAC;wCAC/B,aAAa,CAAC,YAAY,CAAC,GAAY,EAAE,SAAS,CAAC,CAAC;oCACtD,CAAC;oCACD,gBAAgB,GAAG,CAAC,CAAC,CAAC;oCACtB,MAAM,EAAE,CAAC;oCACT,UAAU,EAAE,CAAC;oCACb,SAAS;gCACX,CAAC;gCAED,IACE,OAAO,SAAS,KAAK,QAAQ;oCAC7B,SAAS,KAAK,IAAI;oCAClB,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EACzB,CAAC;oCACD,kBAAkB,CAAC,IAAI,CAAC,SAAgB,CAAC,CAAC;oCAC1C,QAAQ,EAAE,CAAC;oCAEX,IAAI,kBAAkB,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;wCAC3C,MAAM;4CACJ,IAAI,EAAE,kBAAkB;4CACxB,MAAM,EAAE,YAAY;yCACrB,CAAC;wCACF,YAAY,IAAI,kBAAkB,CAAC,MAAM,CAAC;wCAC1C,kBAAkB,GAAG,EAAE,CAAC;oCAC1B,CAAC;gCACH,CAAC;gCAED,gBAAgB,GAAG,CAAC,CAAC,CAAC;4BACxB,CAAC;wBACH,CAAC;6BAAM,IAAI,IAAI,KAAK,GAAG,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;4BAC7C,uBAAuB;4BACvB,OAAO,GAAG,KAAK,CAAC;4BAChB,MAAM;wBACR,CAAC;wBAED,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,4EAA4E;gBAC5E,IAAI,gBAAgB,KAAK,CAAC,CAAC,EAAE,CAAC;oBAC5B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;oBACxC,MAAM,GAAG,MAAM,GAAG,gBAAgB,CAAC;oBACnC,gBAAgB,GAAG,CAAC,CAAC;gBACvB,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBAC9B,MAAM,GAAG,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM;oBACJ,IAAI,EAAE,kBAAkB;oBACxB,MAAM,EAAE,YAAY;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAuB;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACrE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,CAAC;YACD,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,CAAC;QAED,OAAO;YACL,MAAM,EAAE,MAAM;YACd,QAAQ;YACR,WAAW,EAAE,WAAW,CAAC,MAAM;YAC/B,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI;gBACJ,IAAI,EAAE,QAAQ;gBACd,cAAc,EAAE,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF"}

package/dist/readers/jsonl.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/readers/jsonl.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAEV,UAAU,EACV,kBAAkB,EAClB,aAAa,EAEb,aAAa,EACd,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,kBAAmB,SAAQ,aAAa;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,WAAY,YAAW,aAAa;IAC/C,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,OAAO,CAAqB;gBAExB,KAAK,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAQtE,OAAO,CAAC,cAAc;IAIf,IAAI,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,UAAU;IAmE1C,OAAO,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAsBpE"}
+{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/readers/jsonl.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAG5C,OAAO,KAAK,EAEV,UAAU,EACV,kBAAkB,EAClB,aAAa,EAEb,aAAa,EACd,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,kBAAmB,SAAQ,aAAa;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,WAAY,YAAW,aAAa;IAC/C,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,OAAO,CAAqB;gBAExB,KAAK,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,kBAAuB;IAQtE,OAAO,CAAC,cAAc;IAIf,IAAI,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,UAAU;IAsF1C,OAAO,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAsBpE"}

package/dist/readers/jsonl.js

@@ -1,6 +1,7 @@
 import { openDecompressedReadStream } from "../utils/compression.js";
 import { createInterface } from "node:readline";
 import { ParseError } from "../core/errors.js";
+import { RowErrorHandler } from "../core/error-handler.js";
 export class JSONLReader {
     input;
     options;
@@ -25,47 +26,66 @@ export class JSONLReader {
             input: stream,
             crlfDelay: Infinity,
         });
+        const errorHandler = new RowErrorHandler({
+            strategy: mergedOptions.onError,
+            badRowsLog: mergedOptions.badRowsLog,
+            sourceFile: mergedOptions.filePath,
+        });
         let lineNumber = 0;
         let rowsInCurrentBatch = [];
         let globalOffset = 0;
-        for await (const line of rl) {
-            lineNumber++;
-            const trimmed = line.trim();
-            if (!trimmed)
-                continue;
-            let parsed;
-            try {
-                parsed = JSON.parse(trimmed);
-            }
-            catch (err) {
-                throw new ParseError(`Invalid JSON on line ${lineNumber}: ${err.message}`, {
-                    file: mergedOptions.filePath,
-                    line: lineNumber,
-                    row: lineNumber,
-                });
-            }
-            if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-                throw new ParseError(`Expected JSON object on line ${lineNumber}, got ${Array.isArray(parsed) ? "array" : typeof parsed}`, {
-                    file: mergedOptions.filePath,
-                    line: lineNumber,
-                    row: lineNumber,
-                });
+        try {
+            for await (const line of rl) {
+                lineNumber++;
+                const trimmed = line.trim();
+                if (!trimmed)
+                    continue;
+                let parsed;
+                try {
+                    parsed = JSON.parse(trimmed);
+                }
+                catch (err) {
+                    errorHandler.handle(new ParseError(`Invalid JSON on line ${lineNumber}: ${err.message}`, {
+                        file: mergedOptions.filePath,
+                        line: lineNumber,
+                        row: lineNumber,
+                    }), { row: lineNumber, raw: trimmed, file: mergedOptions.filePath });
+                    if (mergedOptions.onParseError) {
+                        mergedOptions.onParseError(err, trimmed);
+                    }
+                    continue;
+                }
+                if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+                    const err = new ParseError(`Expected JSON object on line ${lineNumber}, got ${Array.isArray(parsed) ? "array" : typeof parsed}`, {
+                        file: mergedOptions.filePath,
+                        line: lineNumber,
+                        row: lineNumber,
+                    });
+                    errorHandler.handle(err, { row: lineNumber, raw: trimmed, file: mergedOptions.filePath });
+                    if (mergedOptions.onParseError) {
+                        mergedOptions.onParseError(err, trimmed);
+                    }
+                    continue;
+                }
+                rowsInCurrentBatch.push(parsed);
+                if (rowsInCurrentBatch.length >= batchSize) {
+                    yield {
+                        rows: rowsInCurrentBatch,
+                        offset: globalOffset,
+                    };
+                    globalOffset += rowsInCurrentBatch.length;
+                    rowsInCurrentBatch = [];
+                }
             }
-            rowsInCurrentBatch.push(parsed);
-            if (rowsInCurrentBatch.length >= batchSize) {
+            if (rowsInCurrentBatch.length > 0) {
                 yield {
                     rows: rowsInCurrentBatch,
                     offset: globalOffset,
                 };
-                globalOffset += rowsInCurrentBatch.length;
-                rowsInCurrentBatch = [];
             }
         }
-        if (rowsInCurrentBatch.length > 0) {
-            yield {
-                rows: rowsInCurrentBatch,
-                offset: globalOffset,
-            };
+        finally {
+            errorHandler.close();
         }
     }
     async inspect(options) {

package/dist/readers/jsonl.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/readers/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAc/C,MAAM,OAAO,WAAW;IACd,KAAK,CAAoB;IACzB,OAAO,CAAqB;IAEpC,YAAY,KAAwB,EAAE,UAA8B,EAAE;QACpE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAoB;QACzC,OAAO,0BAA0B,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,OAAuB;QACjC,MAAM,aAAa,GAAuB;YACxC,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAElD,MAAM,EAAE,GAAG,eAAe,CAAC;YACzB,KAAK,EAAE,MAAM;YACb,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,kBAAkB,GAAU,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YAC5B,UAAU,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,UAAU,CAClB,wBAAwB,UAAU,KAAM,GAAa,CAAC,OAAO,EAAE,EAC/D;oBACE,IAAI,EAAE,aAAa,CAAC,QAAQ;oBAC5B,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB,CACF,CAAC;YACJ,CAAC;YAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3E,MAAM,IAAI,UAAU,CAClB,gCAAgC,UAAU,SAAS,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,MAAM,EAAE,EACpG;oBACE,IAAI,EAAE,aAAa,CAAC,QAAQ;oBAC5B,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB,CACF,CAAC;YACJ,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC,MAAa,CAAC,CAAC;YAEvC,IAAI,kBAAkB,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;gBAC3C,MAAM;oBACJ,IAAI,EAAE,kBAAkB;oBACxB,MAAM,EAAE,YAAY;iBACrB,CAAC;gBACF,YAAY,IAAI,kBAAkB,CAAC,MAAM,CAAC;gBAC1C,kBAAkB,GAAG,EAAE,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM;gBACJ,IAAI,EAAE,kBAAkB;gBACxB,MAAM,EAAE,YAAY;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAuB;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACrE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,CAAC;YACD,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,CAAC;QAED,OAAO;YACL,MAAM,EAAE,OAAO;YACf,QAAQ;YACR,WAAW,EAAE,WAAW,CAAC,MAAM;YAC/B,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI;gBACJ,IAAI,EAAE,QAAQ;gBACd,cAAc,EAAE,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/readers/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAc3D,MAAM,OAAO,WAAW;IACd,KAAK,CAAoB;IACzB,OAAO,CAAqB;IAEpC,YAAY,KAAwB,EAAE,UAA8B,EAAE;QACpE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAoB;QACzC,OAAO,0BAA0B,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,OAAuB;QACjC,MAAM,aAAa,GAAuB;YACxC,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAElD,MAAM,EAAE,GAAG,eAAe,CAAC;YACzB,KAAK,EAAE,MAAM;YACb,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,IAAI,eAAe,CAAC;YACvC,QAAQ,EAAE,aAAa,CAAC,OAAO;YAC/B,UAAU,EAAE,aAAa,CAAC,UAAU;YACpC,UAAU,EAAE,aAAa,CAAC,QAAQ;SACnC,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,kBAAkB,GAAU,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;gBAC5B,UAAU,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO;oBAAE,SAAS;gBAEvB,IAAI,MAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,YAAY,CAAC,MAAM,CACjB,IAAI,UAAU,CAAC,wBAAwB,UAAU,KAAM,GAAa,CAAC,OAAO,EAAE,EAAE;wBAC9E,IAAI,EAAE,aAAa,CAAC,QAAQ;wBAC5B,IAAI,EAAE,UAAU;wBAChB,GAAG,EAAE,UAAU;qBAChB,CAAC,EACF,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,CAAC,QAAQ,EAAE,CAChE,CAAC;oBACF,IAAI,aAAa,CAAC,YAAY,EAAE,CAAC;wBAC/B,aAAa,CAAC,YAAY,CAAC,GAAY,EAAE,OAAO,CAAC,CAAC;oBACpD,CAAC;oBACD,SAAS;gBACX,CAAC;gBAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3E,MAAM,GAAG,GAAG,IAAI,UAAU,CACxB,gCAAgC,UAAU,SAAS,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,MAAM,EAAE,EACpG;wBACE,IAAI,EAAE,aAAa,CAAC,QAAQ;wBAC5B,IAAI,EAAE,UAAU;wBAChB,GAAG,EAAE,UAAU;qBAChB,CACF,CAAC;oBACF,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAC1F,IAAI,aAAa,CAAC,YAAY,EAAE,CAAC;wBAC/B,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBAC3C,CAAC;oBACD,SAAS;gBACX,CAAC;gBAED,kBAAkB,CAAC,IAAI,CAAC,MAAa,CAAC,CAAC;gBAEvC,IAAI,kBAAkB,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC3C,MAAM;wBACJ,IAAI,EAAE,kBAAkB;wBACxB,MAAM,EAAE,YAAY;qBACrB,CAAC;oBACF,YAAY,IAAI,kBAAkB,CAAC,MAAM,CAAC;oBAC1C,kBAAkB,GAAG,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM;oBACJ,IAAI,EAAE,kBAAkB;oBACxB,MAAM,EAAE,YAAY;iBACrB,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAuB;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,WAAW,GAAa,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACrE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvE,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,CAAC;YACD,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,CAAC;QAED,OAAO;YACL,MAAM,EAAE,OAAO;YACf,QAAQ;YACR,WAAW,EAAE,WAAW,CAAC,MAAM;YAC/B,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI;gBACJ,IAAI,EAAE,QAAQ;gBACd,cAAc,EAAE,CAAC;aAClB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF"}

package/dist/transforms/concat.d.ts

@@ -0,0 +1,26 @@
+import type { DataStream, TabularReader } from "../core/types.js";
+export interface ConcatSource {
+    reader: TabularReader;
+    label?: string;
+}
+export interface ConcatOptions {
+    sourceCol?: string;
+    align?: "union" | "intersect";
+    batchSize?: number;
+}
+/**
+ * Streaming concatenation reader that combines multiple TabularReaders sequentially
+ * into a single unified DataStream with zero extra memory overhead and optional source tagging.
+ */
+export declare class ConcatReader implements TabularReader {
+    private sources;
+    private options;
+    constructor(sources: Array<TabularReader | ConcatSource>, options?: ConcatOptions);
+    read(): DataStream;
+    close(): Promise<void>;
+}
+/**
+ * Helper to concatenate multiple tabular readers with options.
+ */
+export declare function concatReaders(sources: Array<TabularReader | ConcatSource>, options?: ConcatOptions): TabularReader;
+//# sourceMappingURL=concat.d.ts.map

package/dist/transforms/concat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"concat.d.ts","sourceRoot":"","sources":["../../src/transforms/concat.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAa,UAAU,EAAO,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,GAAG,WAAW,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,qBAAa,YAAa,YAAW,aAAa;IAChD,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,OAAO,CAAgB;gBAEnB,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,YAAY,CAAC,EAAE,OAAO,GAAE,aAAkB;IAS9E,IAAI,IAAI,UAAU;IAgCnB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAO7B;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,KAAK,CAAC,aAAa,GAAG,YAAY,CAAC,EAC5C,OAAO,GAAE,aAAkB,GAC1B,aAAa,CAEf"}

package/dist/transforms/concat.js

@@ -0,0 +1,57 @@
+import { InvalidArgumentError } from "../core/errors.js";
+/**
+ * Streaming concatenation reader that combines multiple TabularReaders sequentially
+ * into a single unified DataStream with zero extra memory overhead and optional source tagging.
+ */
+export class ConcatReader {
+    sources;
+    options;
+    constructor(sources, options = {}) {
+        if (sources.length === 0) {
+            throw new InvalidArgumentError("ConcatReader requires at least one reader or source");
+        }
+        this.sources = sources.map((s) => ("reader" in s ? s : { reader: s }));
+        this.options = options;
+    }
+    async *read() {
+        const sourceCol = this.options.sourceCol;
+        const batchSize = this.options.batchSize || 1000;
+        let outRows = [];
+        let offset = 0;
+        for (let i = 0; i < this.sources.length; i++) {
+            const src = this.sources[i];
+            const label = src.label || `source_${i + 1}`;
+            for await (const batch of src.reader.read()) {
+                for (const row of batch.rows) {
+                    const outRow = { ...row };
+                    if (sourceCol) {
+                        outRow[sourceCol] = label;
+                    }
+                    outRows.push(outRow);
+                    if (outRows.length >= batchSize) {
+                        yield { rows: outRows, offset };
+                        offset += outRows.length;
+                        outRows = [];
+                    }
+                }
+            }
+        }
+        if (outRows.length > 0) {
+            yield { rows: outRows, offset };
+        }
+    }
+    async close() {
+        for (const src of this.sources) {
+            if (src.reader.close) {
+                await src.reader.close();
+            }
+        }
+    }
+}
+/**
+ * Helper to concatenate multiple tabular readers with options.
+ */
+export function concatReaders(sources, options = {}) {
+    return new ConcatReader(sources, options);
+}
+//# sourceMappingURL=concat.js.map

package/dist/transforms/concat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"concat.js","sourceRoot":"","sources":["../../src/transforms/concat.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAazD;;;GAGG;AACH,MAAM,OAAO,YAAY;IACf,OAAO,CAAiB;IACxB,OAAO,CAAgB;IAE/B,YAAY,OAA4C,EAAE,UAAyB,EAAE;QACnF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,oBAAoB,CAAC,qDAAqD,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,CAAC,IAAI;QACT,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;QACjD,IAAI,OAAO,GAAU,EAAE,CAAC;QACxB,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAE7C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC5C,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBAC7B,MAAM,MAAM,GAAQ,EAAE,GAAG,GAAG,EAAE,CAAC;oBAC/B,IAAI,SAAS,EAAE,CAAC;wBACd,MAAM,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC;oBAC5B,CAAC;oBAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACrB,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;wBAChC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;wBAChC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;wBACzB,OAAO,GAAG,EAAE,CAAC;oBACf,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAClC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC/B,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,OAA4C,EAC5C,UAAyB,EAAE;IAE3B,OAAO,IAAI,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC5C,CAAC"}

package/dist/transforms/explode.d.ts

@@ -0,0 +1,14 @@
+import type { TransformFunction } from "../core/types.js";
+export interface ExplodeOptions {
+    column: string;
+    delimiter?: string | RegExp;
+    trim?: boolean;
+    dropEmpty?: boolean;
+    preserveNullAndEmpty?: boolean;
+}
+/**
+ * High-performance streaming Explode transform.
+ * Expands arrays or delimited string values into multiple rows (1 -> N).
+ */
+export declare function explodeRows(columnOrOptions: string | ExplodeOptions, options?: Partial<ExplodeOptions>): TransformFunction;
+//# sourceMappingURL=explode.d.ts.map

package/dist/transforms/explode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"explode.d.ts","sourceRoot":"","sources":["../../src/transforms/explode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEtF,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,eAAe,EAAE,MAAM,GAAG,cAAc,EACxC,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAChC,iBAAiB,CAiFnB"}