rowpipe 1.0.0 → 1.2.0

package/src/readers/json.ts

@@ -1,256 +0,0 @@
-import { createReadStream } from "node:fs";
-import type { Readable } from "node:stream";
-import { ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  TabularReader,
-} from "../core/types.js";
-
-export interface JSONReaderOptions extends ReaderOptions {
-  path?: string;
-  filePath?: string;
-}
-
-/**
- * Incremental streaming JSON parser for top-level arrays [ {...}, ... ]
- * and nested array paths (e.g. data.results).
- * Never loads the full JSON file into memory.
- */
-export class JSONReader implements TabularReader {
-  private input: Readable | string;
-  private options: JSONReaderOptions;
-
-  constructor(input: Readable | string, options: JSONReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  private getInputStream(): Readable {
-    if (typeof this.input === "string") {
-      return createReadStream(this.input);
-    }
-    return this.input;
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: JSONReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const targetPath = mergedOptions.path; // e.g. "data.results"
-    const stream = this.getInputStream();
-
-    let buffer = "";
-    let cursor = 0;
-    let byteOffset = 0;
-    let foundTargetArray = !targetPath; // if no path, we look for first '['
-    let currentPath: string[] = [];
-
-    // State machine for array item scanning
-    let inArray = false;
-    let inString = false;
-    let isEscaped = false;
-    let objectDepth = 0;
-    let objectStartIndex = -1;
-
-    let rowsInCurrentBatch: Row[] = [];
-    let globalOffset = 0;
-    let rowIndex = 0;
-
-    // Helper for path navigation if targetPath is set
-    const targetPathParts = targetPath ? targetPath.split(".") : [];
-
-    for await (const chunk of stream) {
-      const chunkStr =
-        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8");
-      buffer += chunkStr;
-
-      while (cursor < buffer.length) {
-        const char = buffer[cursor];
-
-        if (isEscaped) {
-          isEscaped = false;
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        if (char === "\\") {
-          isEscaped = true;
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        if (char === '"') {
-          inString = !inString;
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        if (inString) {
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        // We are outside of any string literal
-        if (!foundTargetArray) {
-          // If we need to find target path (e.g. data.results)
-          // Look for keys and match path parts
-          if (char === "{") {
-            cursor++;
-            byteOffset++;
-            continue;
-          } else if (char === "[") {
-            // Check if current path matches targetPath
-            // If path matched or we hit array at root
-            foundTargetArray = true;
-            inArray = true;
-            cursor++;
-            byteOffset++;
-            continue;
-          } else if (char === ":") {
-            // Scan backwards from ':' to find previous string key
-            let keyEnd = cursor - 1;
-            while (keyEnd >= 0 && /\s/.test(buffer[keyEnd]!)) keyEnd--;
-            if (keyEnd >= 0 && buffer[keyEnd] === '"') {
-              let keyStart = keyEnd - 1;
-              while (keyStart >= 0 && buffer[keyStart] !== '"') keyStart--;
-              if (keyStart >= 0) {
-                const keyName = buffer.slice(keyStart + 1, keyEnd);
-                currentPath.push(keyName);
-                if (
-                  currentPath.join(".") === targetPath ||
-                  keyName === targetPathParts[targetPathParts.length - 1]
-                ) {
-                  // Next non-whitespace should be '['
-                  // Let it continue to find '['
-                }
-              }
-            }
-            cursor++;
-            byteOffset++;
-            continue;
-          }
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        // Now we are streaming inside the target array
-        if (!inArray) {
-          if (char === "[") {
-            inArray = true;
-            cursor++;
-            byteOffset++;
-            continue;
-          }
-        } else {
-          // Inside array: looking for objects '{ ... }'
-          if (char === "{") {
-            if (objectDepth === 0) {
-              objectStartIndex = cursor;
-            }
-            objectDepth++;
-          } else if (char === "}") {
-            objectDepth--;
-            if (objectDepth === 0 && objectStartIndex !== -1) {
-              // We have captured a complete JSON object!
-              const objectStr = buffer.slice(objectStartIndex, cursor + 1);
-              let parsedRow: unknown;
-              try {
-                parsedRow = JSON.parse(objectStr);
-              } catch (err) {
-                throw new ParseError(
-                  `Malformed JSON object in array: ${(err as Error).message}`,
-                  {
-                    file: mergedOptions.filePath,
-                    row: rowIndex + 1,
-                    byteOffset,
-                  }
-                );
-              }
-
-              if (
-                typeof parsedRow === "object" &&
-                parsedRow !== null &&
-                !Array.isArray(parsedRow)
-              ) {
-                rowsInCurrentBatch.push(parsedRow as Row);
-                rowIndex++;
-
-                if (rowsInCurrentBatch.length >= batchSize) {
-                  yield {
-                    rows: rowsInCurrentBatch,
-                    offset: globalOffset,
-                  };
-                  globalOffset += rowsInCurrentBatch.length;
-                  rowsInCurrentBatch = [];
-                }
-              }
-
-              objectStartIndex = -1;
-            }
-          } else if (char === "]" && objectDepth === 0) {
-            // End of array reached
-            inArray = false;
-            break;
-          }
-
-          cursor++;
-          byteOffset++;
-        }
-      }
-
-      // Compact buffer while preserving active object if in the middle of parsing
-      if (objectStartIndex !== -1) {
-        buffer = buffer.slice(objectStartIndex);
-        cursor = cursor - objectStartIndex;
-        objectStartIndex = 0;
-      } else {
-        buffer = buffer.slice(cursor);
-        cursor = 0;
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    let rowCount = 0;
-    let columnNames: string[] = [];
-
-    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
-      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
-        columnNames = Object.keys(batch.rows[0]);
-      }
-      rowCount += batch.rows.length;
-    }
-
-    return {
-      format: "JSON",
-      rowCount,
-      columnCount: columnNames.length,
-      columns: columnNames.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      })),
-    };
-  }
-}

package/src/readers/jsonl.ts

@@ -1,126 +0,0 @@
-import { createReadStream } from "node:fs";
-import { createInterface } from "node:readline";
-import type { Readable } from "node:stream";
-import { ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  TabularReader,
-} from "../core/types.js";
-
-export interface JSONLReaderOptions extends ReaderOptions {
-  filePath?: string;
-}
-
-export class JSONLReader implements TabularReader {
-  private input: Readable | string;
-  private options: JSONLReaderOptions;
-
-  constructor(input: Readable | string, options: JSONLReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  private getInputStream(): Readable {
-    if (typeof this.input === "string") {
-      return createReadStream(this.input);
-    }
-    return this.input;
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: JSONLReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const stream = this.getInputStream();
-
-    const rl = createInterface({
-      input: stream,
-      crlfDelay: Infinity,
-    });
-
-    let lineNumber = 0;
-    let rowsInCurrentBatch: Row[] = [];
-    let globalOffset = 0;
-
-    for await (const line of rl) {
-      lineNumber++;
-      const trimmed = line.trim();
-      if (!trimmed) continue;
-
-      let parsed: unknown;
-      try {
-        parsed = JSON.parse(trimmed);
-      } catch (err) {
-        throw new ParseError(
-          `Invalid JSON on line ${lineNumber}: ${(err as Error).message}`,
-          {
-            file: mergedOptions.filePath,
-            line: lineNumber,
-            row: lineNumber,
-          }
-        );
-      }
-
-      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-        throw new ParseError(
-          `Expected JSON object on line ${lineNumber}, got ${Array.isArray(parsed) ? "array" : typeof parsed}`,
-          {
-            file: mergedOptions.filePath,
-            line: lineNumber,
-            row: lineNumber,
-          }
-        );
-      }
-
-      rowsInCurrentBatch.push(parsed as Row);
-
-      if (rowsInCurrentBatch.length >= batchSize) {
-        yield {
-          rows: rowsInCurrentBatch,
-          offset: globalOffset,
-        };
-        globalOffset += rowsInCurrentBatch.length;
-        rowsInCurrentBatch = [];
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    let rowCount = 0;
-    let columnNames: string[] = [];
-
-    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
-      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
-        columnNames = Object.keys(batch.rows[0]);
-      }
-      rowCount += batch.rows.length;
-    }
-
-    return {
-      format: "JSONL",
-      rowCount,
-      columnCount: columnNames.length,
-      columns: columnNames.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      })),
-    };
-  }
-}

package/src/readers/xlsx.ts

@@ -1,182 +0,0 @@
-import type { Readable } from "node:stream";
-import readXlsxFile from "read-excel-file/node";
-import { InvalidArgumentError, ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  SheetMetadata,
-  TabularReader,
-} from "../core/types.js";
-
-export interface XLSXReaderOptions extends ReaderOptions {
-  sheet?: string | number;
-  filePath?: string;
-}
-
-export class XLSXReader implements TabularReader {
-  private input: Readable | string;
-  private options: XLSXReaderOptions;
-
-  constructor(input: Readable | string, options: XLSXReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  async getParsedSheets(): Promise<Array<{ sheet: string; data: unknown[][] }>> {
-    try {
-      const result = await (readXlsxFile as unknown as (input: unknown, options?: unknown) => Promise<unknown>)(
-        this.input,
-        { getSheets: true }
-      );
-      if (Array.isArray(result)) {
-        if (result.length > 0 && typeof result[0] === "object" && result[0] !== null && "sheet" in result[0]) {
-          return result as Array<{ sheet: string; data: unknown[][] }>;
-        }
-        return [{ sheet: "Sheet1", data: result as unknown[][] }];
-      }
-      return [];
-    } catch (err) {
-      throw new ParseError(`Failed to parse XLSX: ${(err as Error).message}`, {
-        file: this.options.filePath,
-      });
-    }
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: XLSXReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const targetSheet = mergedOptions.sheet;
-
-    const sheets = await this.getParsedSheets();
-    if (sheets.length === 0) {
-      return;
-    }
-
-    let selectedSheet: { sheet: string; data: unknown[][] } | undefined;
-
-    if (targetSheet === undefined || targetSheet === null) {
-      selectedSheet = sheets[0];
-    } else if (typeof targetSheet === "number") {
-      selectedSheet = sheets[targetSheet - 1];
-    } else {
-      selectedSheet = sheets.find(
-        (s) => s.sheet.toLowerCase() === String(targetSheet).toLowerCase()
-      );
-    }
-
-    if (!selectedSheet) {
-      throw new InvalidArgumentError(
-        `Sheet "${targetSheet}" was not found in the workbook. Available sheets: ${sheets.map((s) => s.sheet).join(", ")}`
-      );
-    }
-
-    const grid = selectedSheet.data;
-    if (grid.length === 0) return;
-
-    let headers: string[] | null = null;
-    let rowsInCurrentBatch: Row[] = [];
-    let globalOffset = 0;
-
-    for (let r = 0; r < grid.length; r++) {
-      const rowCells = grid[r] || [];
-      if (rowCells.length === 0 || rowCells.every((c) => c === null || c === undefined || c === "")) {
-        continue;
-      }
-
-      if (headers === null) {
-        headers = rowCells.map((c, i) => {
-          const str = c !== null && c !== undefined ? String(c).trim() : "";
-          return str.length > 0 ? str : `col_${i + 1}`;
-        });
-        continue;
-      }
-
-      const rowObj: Row = {};
-      for (let i = 0; i < headers.length; i++) {
-        const key = headers[i] || `col_${i + 1}`;
-        const cellVal = rowCells[i];
-        rowObj[key] = cellVal !== null && cellVal !== undefined ? cellVal : "";
-      }
-
-      rowsInCurrentBatch.push(rowObj);
-
-      if (rowsInCurrentBatch.length >= batchSize) {
-        yield {
-          rows: rowsInCurrentBatch,
-          offset: globalOffset,
-        };
-        globalOffset += rowsInCurrentBatch.length;
-        rowsInCurrentBatch = [];
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    const sheets = await this.getParsedSheets();
-    const targetSheet = options?.sheet ?? this.options.sheet;
-
-    const sheetSummaries: SheetMetadata[] = sheets.map((s) => {
-      const nonHeaderRows = Math.max(0, s.data.length - 1);
-      const headerRow = s.data[0] || [];
-      const colHeaders = headerRow.map((c, i) =>
-        c !== null && c !== undefined ? String(c).trim() : `col_${i + 1}`
-      );
-
-      return {
-        name: s.sheet,
-        rowCount: nonHeaderRows,
-        columnCount: colHeaders.length,
-        columns: colHeaders,
-      };
-    });
-
-    let selectedSheet = sheets[0];
-    if (targetSheet !== undefined && targetSheet !== null) {
-      selectedSheet =
-        typeof targetSheet === "number"
-          ? sheets[targetSheet - 1]
-          : sheets.find((s) => s.sheet.toLowerCase() === String(targetSheet).toLowerCase());
-    }
-
-    let columns: Array<{ name: string; type: any; nullPercentage: number }> = [];
-    let rowCount = 0;
-
-    if (selectedSheet && selectedSheet.data.length > 0) {
-      const headerRow = selectedSheet.data[0] || [];
-      const colHeaders = headerRow.map((c, i) =>
-        c !== null && c !== undefined ? String(c).trim() : `col_${i + 1}`
-      );
-      rowCount = Math.max(0, selectedSheet.data.length - 1);
-      columns = colHeaders.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      }));
-    }
-
-    return {
-      format: "XLSX",
-      rowCount,
-      columnCount: columns.length,
-      columns,
-      sheetsCount: sheets.length,
-      sheets: sheetSummaries,
-    };
-  }
-}