rowpipe 1.0.0

package/src/readers/json.ts

@@ -0,0 +1,256 @@
+import { createReadStream } from "node:fs";
+import type { Readable } from "node:stream";
+import { ParseError } from "../core/errors.js";
+import type {
+  DataBatch,
+  DataStream,
+  InspectionMetadata,
+  ReaderOptions,
+  Row,
+  TabularReader,
+} from "../core/types.js";
+
+export interface JSONReaderOptions extends ReaderOptions {
+  path?: string;
+  filePath?: string;
+}
+
+/**
+ * Incremental streaming JSON parser for top-level arrays [ {...}, ... ]
+ * and nested array paths (e.g. data.results).
+ * Never loads the full JSON file into memory.
+ */
+export class JSONReader implements TabularReader {
+  private input: Readable | string;
+  private options: JSONReaderOptions;
+
+  constructor(input: Readable | string, options: JSONReaderOptions = {}) {
+    this.input = input;
+    this.options = { ...options };
+    if (typeof input === "string") {
+      this.options.filePath = input;
+    }
+  }
+
+  private getInputStream(): Readable {
+    if (typeof this.input === "string") {
+      return createReadStream(this.input);
+    }
+    return this.input;
+  }
+
+  async *read(options?: ReaderOptions): DataStream {
+    const mergedOptions: JSONReaderOptions = {
+      ...this.options,
+      ...options,
+    };
+    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
+    const targetPath = mergedOptions.path; // e.g. "data.results"
+    const stream = this.getInputStream();
+
+    let buffer = "";
+    let cursor = 0;
+    let byteOffset = 0;
+    let foundTargetArray = !targetPath; // if no path, we look for first '['
+    let currentPath: string[] = [];
+
+    // State machine for array item scanning
+    let inArray = false;
+    let inString = false;
+    let isEscaped = false;
+    let objectDepth = 0;
+    let objectStartIndex = -1;
+
+    let rowsInCurrentBatch: Row[] = [];
+    let globalOffset = 0;
+    let rowIndex = 0;
+
+    // Helper for path navigation if targetPath is set
+    const targetPathParts = targetPath ? targetPath.split(".") : [];
+
+    for await (const chunk of stream) {
+      const chunkStr =
+        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8");
+      buffer += chunkStr;
+
+      while (cursor < buffer.length) {
+        const char = buffer[cursor];
+
+        if (isEscaped) {
+          isEscaped = false;
+          cursor++;
+          byteOffset++;
+          continue;
+        }
+
+        if (char === "\\") {
+          isEscaped = true;
+          cursor++;
+          byteOffset++;
+          continue;
+        }
+
+        if (char === '"') {
+          inString = !inString;
+          cursor++;
+          byteOffset++;
+          continue;
+        }
+
+        if (inString) {
+          cursor++;
+          byteOffset++;
+          continue;
+        }
+
+        // We are outside of any string literal
+        if (!foundTargetArray) {
+          // If we need to find target path (e.g. data.results)
+          // Look for keys and match path parts
+          if (char === "{") {
+            cursor++;
+            byteOffset++;
+            continue;
+          } else if (char === "[") {
+            // Check if current path matches targetPath
+            // If path matched or we hit array at root
+            foundTargetArray = true;
+            inArray = true;
+            cursor++;
+            byteOffset++;
+            continue;
+          } else if (char === ":") {
+            // Scan backwards from ':' to find previous string key
+            let keyEnd = cursor - 1;
+            while (keyEnd >= 0 && /\s/.test(buffer[keyEnd]!)) keyEnd--;
+            if (keyEnd >= 0 && buffer[keyEnd] === '"') {
+              let keyStart = keyEnd - 1;
+              while (keyStart >= 0 && buffer[keyStart] !== '"') keyStart--;
+              if (keyStart >= 0) {
+                const keyName = buffer.slice(keyStart + 1, keyEnd);
+                currentPath.push(keyName);
+                if (
+                  currentPath.join(".") === targetPath ||
+                  keyName === targetPathParts[targetPathParts.length - 1]
+                ) {
+                  // Next non-whitespace should be '['
+                  // Let it continue to find '['
+                }
+              }
+            }
+            cursor++;
+            byteOffset++;
+            continue;
+          }
+          cursor++;
+          byteOffset++;
+          continue;
+        }
+
+        // Now we are streaming inside the target array
+        if (!inArray) {
+          if (char === "[") {
+            inArray = true;
+            cursor++;
+            byteOffset++;
+            continue;
+          }
+        } else {
+          // Inside array: looking for objects '{ ... }'
+          if (char === "{") {
+            if (objectDepth === 0) {
+              objectStartIndex = cursor;
+            }
+            objectDepth++;
+          } else if (char === "}") {
+            objectDepth--;
+            if (objectDepth === 0 && objectStartIndex !== -1) {
+              // We have captured a complete JSON object!
+              const objectStr = buffer.slice(objectStartIndex, cursor + 1);
+              let parsedRow: unknown;
+              try {
+                parsedRow = JSON.parse(objectStr);
+              } catch (err) {
+                throw new ParseError(
+                  `Malformed JSON object in array: ${(err as Error).message}`,
+                  {
+                    file: mergedOptions.filePath,
+                    row: rowIndex + 1,
+                    byteOffset,
+                  }
+                );
+              }
+
+              if (
+                typeof parsedRow === "object" &&
+                parsedRow !== null &&
+                !Array.isArray(parsedRow)
+              ) {
+                rowsInCurrentBatch.push(parsedRow as Row);
+                rowIndex++;
+
+                if (rowsInCurrentBatch.length >= batchSize) {
+                  yield {
+                    rows: rowsInCurrentBatch,
+                    offset: globalOffset,
+                  };
+                  globalOffset += rowsInCurrentBatch.length;
+                  rowsInCurrentBatch = [];
+                }
+              }
+
+              objectStartIndex = -1;
+            }
+          } else if (char === "]" && objectDepth === 0) {
+            // End of array reached
+            inArray = false;
+            break;
+          }
+
+          cursor++;
+          byteOffset++;
+        }
+      }
+
+      // Compact buffer while preserving active object if in the middle of parsing
+      if (objectStartIndex !== -1) {
+        buffer = buffer.slice(objectStartIndex);
+        cursor = cursor - objectStartIndex;
+        objectStartIndex = 0;
+      } else {
+        buffer = buffer.slice(cursor);
+        cursor = 0;
+      }
+    }
+
+    if (rowsInCurrentBatch.length > 0) {
+      yield {
+        rows: rowsInCurrentBatch,
+        offset: globalOffset,
+      };
+    }
+  }
+
+  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
+    let rowCount = 0;
+    let columnNames: string[] = [];
+
+    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
+      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
+        columnNames = Object.keys(batch.rows[0]);
+      }
+      rowCount += batch.rows.length;
+    }
+
+    return {
+      format: "JSON",
+      rowCount,
+      columnCount: columnNames.length,
+      columns: columnNames.map((name) => ({
+        name,
+        type: "string",
+        nullPercentage: 0,
+      })),
+    };
+  }
+}

package/src/readers/jsonl.ts

@@ -0,0 +1,126 @@
+import { createReadStream } from "node:fs";
+import { createInterface } from "node:readline";
+import type { Readable } from "node:stream";
+import { ParseError } from "../core/errors.js";
+import type {
+  DataBatch,
+  DataStream,
+  InspectionMetadata,
+  ReaderOptions,
+  Row,
+  TabularReader,
+} from "../core/types.js";
+
+export interface JSONLReaderOptions extends ReaderOptions {
+  filePath?: string;
+}
+
+export class JSONLReader implements TabularReader {
+  private input: Readable | string;
+  private options: JSONLReaderOptions;
+
+  constructor(input: Readable | string, options: JSONLReaderOptions = {}) {
+    this.input = input;
+    this.options = { ...options };
+    if (typeof input === "string") {
+      this.options.filePath = input;
+    }
+  }
+
+  private getInputStream(): Readable {
+    if (typeof this.input === "string") {
+      return createReadStream(this.input);
+    }
+    return this.input;
+  }
+
+  async *read(options?: ReaderOptions): DataStream {
+    const mergedOptions: JSONLReaderOptions = {
+      ...this.options,
+      ...options,
+    };
+    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
+    const stream = this.getInputStream();
+
+    const rl = createInterface({
+      input: stream,
+      crlfDelay: Infinity,
+    });
+
+    let lineNumber = 0;
+    let rowsInCurrentBatch: Row[] = [];
+    let globalOffset = 0;
+
+    for await (const line of rl) {
+      lineNumber++;
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(trimmed);
+      } catch (err) {
+        throw new ParseError(
+          `Invalid JSON on line ${lineNumber}: ${(err as Error).message}`,
+          {
+            file: mergedOptions.filePath,
+            line: lineNumber,
+            row: lineNumber,
+          }
+        );
+      }
+
+      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        throw new ParseError(
+          `Expected JSON object on line ${lineNumber}, got ${Array.isArray(parsed) ? "array" : typeof parsed}`,
+          {
+            file: mergedOptions.filePath,
+            line: lineNumber,
+            row: lineNumber,
+          }
+        );
+      }
+
+      rowsInCurrentBatch.push(parsed as Row);
+
+      if (rowsInCurrentBatch.length >= batchSize) {
+        yield {
+          rows: rowsInCurrentBatch,
+          offset: globalOffset,
+        };
+        globalOffset += rowsInCurrentBatch.length;
+        rowsInCurrentBatch = [];
+      }
+    }
+
+    if (rowsInCurrentBatch.length > 0) {
+      yield {
+        rows: rowsInCurrentBatch,
+        offset: globalOffset,
+      };
+    }
+  }
+
+  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
+    let rowCount = 0;
+    let columnNames: string[] = [];
+
+    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
+      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
+        columnNames = Object.keys(batch.rows[0]);
+      }
+      rowCount += batch.rows.length;
+    }
+
+    return {
+      format: "JSONL",
+      rowCount,
+      columnCount: columnNames.length,
+      columns: columnNames.map((name) => ({
+        name,
+        type: "string",
+        nullPercentage: 0,
+      })),
+    };
+  }
+}

package/src/readers/xlsx.ts

@@ -0,0 +1,182 @@
+import type { Readable } from "node:stream";
+import readXlsxFile from "read-excel-file/node";
+import { InvalidArgumentError, ParseError } from "../core/errors.js";
+import type {
+  DataBatch,
+  DataStream,
+  InspectionMetadata,
+  ReaderOptions,
+  Row,
+  SheetMetadata,
+  TabularReader,
+} from "../core/types.js";
+
+export interface XLSXReaderOptions extends ReaderOptions {
+  sheet?: string | number;
+  filePath?: string;
+}
+
+export class XLSXReader implements TabularReader {
+  private input: Readable | string;
+  private options: XLSXReaderOptions;
+
+  constructor(input: Readable | string, options: XLSXReaderOptions = {}) {
+    this.input = input;
+    this.options = { ...options };
+    if (typeof input === "string") {
+      this.options.filePath = input;
+    }
+  }
+
+  async getParsedSheets(): Promise<Array<{ sheet: string; data: unknown[][] }>> {
+    try {
+      const result = await (readXlsxFile as unknown as (input: unknown, options?: unknown) => Promise<unknown>)(
+        this.input,
+        { getSheets: true }
+      );
+      if (Array.isArray(result)) {
+        if (result.length > 0 && typeof result[0] === "object" && result[0] !== null && "sheet" in result[0]) {
+          return result as Array<{ sheet: string; data: unknown[][] }>;
+        }
+        return [{ sheet: "Sheet1", data: result as unknown[][] }];
+      }
+      return [];
+    } catch (err) {
+      throw new ParseError(`Failed to parse XLSX: ${(err as Error).message}`, {
+        file: this.options.filePath,
+      });
+    }
+  }
+
+  async *read(options?: ReaderOptions): DataStream {
+    const mergedOptions: XLSXReaderOptions = {
+      ...this.options,
+      ...options,
+    };
+    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
+    const targetSheet = mergedOptions.sheet;
+
+    const sheets = await this.getParsedSheets();
+    if (sheets.length === 0) {
+      return;
+    }
+
+    let selectedSheet: { sheet: string; data: unknown[][] } | undefined;
+
+    if (targetSheet === undefined || targetSheet === null) {
+      selectedSheet = sheets[0];
+    } else if (typeof targetSheet === "number") {
+      selectedSheet = sheets[targetSheet - 1];
+    } else {
+      selectedSheet = sheets.find(
+        (s) => s.sheet.toLowerCase() === String(targetSheet).toLowerCase()
+      );
+    }
+
+    if (!selectedSheet) {
+      throw new InvalidArgumentError(
+        `Sheet "${targetSheet}" was not found in the workbook. Available sheets: ${sheets.map((s) => s.sheet).join(", ")}`
+      );
+    }
+
+    const grid = selectedSheet.data;
+    if (grid.length === 0) return;
+
+    let headers: string[] | null = null;
+    let rowsInCurrentBatch: Row[] = [];
+    let globalOffset = 0;
+
+    for (let r = 0; r < grid.length; r++) {
+      const rowCells = grid[r] || [];
+      if (rowCells.length === 0 || rowCells.every((c) => c === null || c === undefined || c === "")) {
+        continue;
+      }
+
+      if (headers === null) {
+        headers = rowCells.map((c, i) => {
+          const str = c !== null && c !== undefined ? String(c).trim() : "";
+          return str.length > 0 ? str : `col_${i + 1}`;
+        });
+        continue;
+      }
+
+      const rowObj: Row = {};
+      for (let i = 0; i < headers.length; i++) {
+        const key = headers[i] || `col_${i + 1}`;
+        const cellVal = rowCells[i];
+        rowObj[key] = cellVal !== null && cellVal !== undefined ? cellVal : "";
+      }
+
+      rowsInCurrentBatch.push(rowObj);
+
+      if (rowsInCurrentBatch.length >= batchSize) {
+        yield {
+          rows: rowsInCurrentBatch,
+          offset: globalOffset,
+        };
+        globalOffset += rowsInCurrentBatch.length;
+        rowsInCurrentBatch = [];
+      }
+    }
+
+    if (rowsInCurrentBatch.length > 0) {
+      yield {
+        rows: rowsInCurrentBatch,
+        offset: globalOffset,
+      };
+    }
+  }
+
+  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
+    const sheets = await this.getParsedSheets();
+    const targetSheet = options?.sheet ?? this.options.sheet;
+
+    const sheetSummaries: SheetMetadata[] = sheets.map((s) => {
+      const nonHeaderRows = Math.max(0, s.data.length - 1);
+      const headerRow = s.data[0] || [];
+      const colHeaders = headerRow.map((c, i) =>
+        c !== null && c !== undefined ? String(c).trim() : `col_${i + 1}`
+      );
+
+      return {
+        name: s.sheet,
+        rowCount: nonHeaderRows,
+        columnCount: colHeaders.length,
+        columns: colHeaders,
+      };
+    });
+
+    let selectedSheet = sheets[0];
+    if (targetSheet !== undefined && targetSheet !== null) {
+      selectedSheet =
+        typeof targetSheet === "number"
+          ? sheets[targetSheet - 1]
+          : sheets.find((s) => s.sheet.toLowerCase() === String(targetSheet).toLowerCase());
+    }
+
+    let columns: Array<{ name: string; type: any; nullPercentage: number }> = [];
+    let rowCount = 0;
+
+    if (selectedSheet && selectedSheet.data.length > 0) {
+      const headerRow = selectedSheet.data[0] || [];
+      const colHeaders = headerRow.map((c, i) =>
+        c !== null && c !== undefined ? String(c).trim() : `col_${i + 1}`
+      );
+      rowCount = Math.max(0, selectedSheet.data.length - 1);
+      columns = colHeaders.map((name) => ({
+        name,
+        type: "string",
+        nullPercentage: 0,
+      }));
+    }
+
+    return {
+      format: "XLSX",
+      rowCount,
+      columnCount: columns.length,
+      columns,
+      sheetsCount: sheets.length,
+      sheets: sheetSummaries,
+    };
+  }
+}