rowpipe 1.0.0 → 1.1.0

package/src/readers/csv.ts

@@ -1,302 +0,0 @@
-import { createReadStream } from "node:fs";
-import type { Readable } from "node:stream";
-import { ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  TabularReader,
-} from "../core/types.js";
-
-export interface CSVReaderOptions extends ReaderOptions {
-  delimiter?: string;
-  header?: boolean;
-  filePath?: string;
-}
-
-/**
- * Streaming CSV Reader supporting RFC 4180 quotes, multiline cells,
- * custom & auto-detected delimiters, BOM removal, and exact error reporting.
- */
-export class CSVReader implements TabularReader {
-  private input: Readable | string;
-  private options: CSVReaderOptions;
-
-  constructor(input: Readable | string, options: CSVReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  private getInputStream(): Readable {
-    if (typeof this.input === "string") {
-      return createReadStream(this.input);
-    }
-    return this.input;
-  }
-
-  /**
-   * Auto-detects delimiter by scanning candidate delimiters in a sample string.
-   */
-  public static detectDelimiter(sample: string): string {
-    const candidates = [",", "\t", ";", "|"];
-    const firstLine = sample.split(/\r?\n/)[0] || "";
-
-    let bestDelimiter = ",";
-    let maxCount = -1;
-
-    for (const cand of candidates) {
-      let count = 0;
-      let inQuotes = false;
-      for (let i = 0; i < firstLine.length; i++) {
-        const ch = firstLine[i];
-        if (ch === '"') {
-          inQuotes = !inQuotes;
-        } else if (ch === cand && !inQuotes) {
-          count++;
-        }
-      }
-      if (count > maxCount) {
-        maxCount = count;
-        bestDelimiter = cand;
-      }
-    }
-
-    return maxCount > 0 ? bestDelimiter : ",";
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: CSVReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const hasHeader = mergedOptions.header !== false;
-    let delimiter = mergedOptions.delimiter;
-
-    const stream = this.getInputStream();
-
-    let buffer = "";
-    let headers: string[] | null = null;
-    let isFirstChunk = true;
-    let rowsInCurrentBatch: Row[] = [];
-    let globalRowOffset = 0;
-    let rowNumber = 0;
-    let byteOffset = 0;
-    let currentField = "";
-    let currentRowFields: string[] = [];
-    let inQuotes = false;
-    let cursor = 0;
-
-    for await (const chunk of stream) {
-      const chunkStr: string =
-        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8");
-
-      if (isFirstChunk) {
-        isFirstChunk = false;
-        // Remove UTF-8 BOM if present
-        if (chunkStr.charCodeAt(0) === 0xfeff) {
-          buffer = chunkStr.slice(1);
-        } else {
-          buffer = chunkStr;
-        }
-
-        // Auto-detect delimiter if not specified
-        if (!delimiter) {
-          delimiter = CSVReader.detectDelimiter(buffer);
-        }
-      } else {
-        buffer += chunkStr;
-      }
-
-      const activeDelimiter = delimiter ?? ",";
-      const delimLen = activeDelimiter.length;
-
-      while (cursor < buffer.length) {
-        const char = buffer[cursor];
-
-        if (inQuotes) {
-          if (char === '"') {
-            if (cursor + 1 < buffer.length && buffer[cursor + 1] === '"') {
-              currentField += '"';
-              cursor += 2;
-              byteOffset += 2;
-              continue;
-            } else {
-              inQuotes = false;
-              cursor++;
-              byteOffset++;
-              continue;
-            }
-          } else {
-            currentField += char;
-            cursor++;
-            byteOffset++;
-            continue;
-          }
-        } else {
-          if (char === '"') {
-            if (currentField.length === 0) {
-              inQuotes = true;
-              cursor++;
-              byteOffset++;
-              continue;
-            } else {
-              throw new ParseError(
-                `Unexpected quote inside unquoted field at byte offset ${byteOffset}`,
-                {
-                  file: mergedOptions.filePath,
-                  row: rowNumber + 1,
-                  column: currentRowFields.length + 1,
-                  byteOffset,
-                }
-              );
-            }
-          }
-
-          // Check for delimiter
-          if (buffer.startsWith(activeDelimiter, cursor)) {
-            currentRowFields.push(currentField);
-            currentField = "";
-            cursor += delimLen;
-            byteOffset += delimLen;
-            continue;
-          }
-
-          // Check for CRLF / LF newline
-          if (char === "\r" || char === "\n") {
-            currentRowFields.push(currentField);
-            currentField = "";
-
-            if (char === "\r" && cursor + 1 < buffer.length && buffer[cursor + 1] === "\n") {
-              cursor += 2;
-              byteOffset += 2;
-            } else {
-              cursor++;
-              byteOffset++;
-            }
-
-            // Skip entirely empty row if it's trailing whitespace
-            if (
-              currentRowFields.length === 1 &&
-              currentRowFields[0]?.trim() === "" &&
-              cursor >= buffer.length
-            ) {
-              currentRowFields = [];
-              continue;
-            }
-
-            if (hasHeader && headers === null) {
-              headers = currentRowFields.map((h, i) =>
-                h.trim().length > 0 ? h.trim() : `col_${i + 1}`
-              );
-              currentRowFields = [];
-              continue;
-            }
-
-            if (headers === null) {
-              headers = currentRowFields.map((_, i) => `col_${i + 1}`);
-            }
-
-            // Construct Row object
-            const rowObj: Row = {};
-            for (let i = 0; i < headers.length; i++) {
-              const key = headers[i] || `col_${i + 1}`;
-              const val = currentRowFields[i] ?? "";
-              rowObj[key] = val;
-            }
-
-            rowNumber++;
-            rowsInCurrentBatch.push(rowObj);
-            currentRowFields = [];
-
-            if (rowsInCurrentBatch.length >= batchSize) {
-              yield {
-                rows: rowsInCurrentBatch,
-                offset: globalRowOffset,
-              };
-              globalRowOffset += rowsInCurrentBatch.length;
-              rowsInCurrentBatch = [];
-            }
-            continue;
-          }
-
-          currentField += char;
-          cursor++;
-          byteOffset++;
-        }
-      }
-
-      // Compact processed buffer to keep memory bounded
-      buffer = buffer.slice(cursor);
-      cursor = 0;
-    }
-
-    // Process remainder if any
-    if (inQuotes) {
-      throw new ParseError("Unclosed quote at end of CSV stream", {
-        file: mergedOptions.filePath,
-        row: rowNumber + 1,
-        byteOffset,
-      });
-    }
-
-    if (currentField.length > 0 || currentRowFields.length > 0) {
-      currentRowFields.push(currentField);
-
-      if (hasHeader && headers === null) {
-        headers = currentRowFields.map((h, i) =>
-          h.trim().length > 0 ? h.trim() : `col_${i + 1}`
-        );
-      } else {
-        if (headers === null) {
-          headers = currentRowFields.map((_, i) => `col_${i + 1}`);
-        }
-
-        const rowObj: Row = {};
-        for (let i = 0; i < headers.length; i++) {
-          const key = headers[i] || `col_${i + 1}`;
-          const val = currentRowFields[i] ?? "";
-          rowObj[key] = val;
-        }
-
-        rowNumber++;
-        rowsInCurrentBatch.push(rowObj);
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalRowOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    let rowCount = 0;
-    let columnNames: string[] = [];
-
-    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
-      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
-        columnNames = Object.keys(batch.rows[0]);
-      }
-      rowCount += batch.rows.length;
-    }
-
-    return {
-      format: "CSV",
-      rowCount,
-      columnCount: columnNames.length,
-      columns: columnNames.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      })),
-    };
-  }
-}

package/src/readers/index.ts

@@ -1,88 +0,0 @@
-import type { Readable } from "node:stream";
-import { InvalidArgumentError } from "../core/errors.js";
-import type { FormatAdapter, ReaderOptions, TabularReader } from "../core/types.js";
-import { CSVReader } from "./csv.js";
-import { JSONReader } from "./json.js";
-import { JSONLReader } from "./jsonl.js";
-import { XLSXReader } from "./xlsx.js";
-
-export * from "./csv.js";
-export * from "./json.js";
-export * from "./jsonl.js";
-export * from "./xlsx.js";
-
-const adapters: Record<string, FormatAdapter> = {
-  csv: {
-    name: "CSV",
-    extensions: [".csv", ".tsv", ".txt"],
-    createReader: (input, options) => new CSVReader(input, options),
-    createWriter: () => {
-      throw new Error("Writer adapter called from reader");
-    },
-  },
-  json: {
-    name: "JSON",
-    extensions: [".json"],
-    createReader: (input, options) => new JSONReader(input, options),
-    createWriter: () => {
-      throw new Error("Writer adapter called from reader");
-    },
-  },
-  jsonl: {
-    name: "JSONL",
-    extensions: [".jsonl", ".ndjson", ".ldjson"],
-    createReader: (input, options) => new JSONLReader(input, options),
-    createWriter: () => {
-      throw new Error("Writer adapter called from reader");
-    },
-  },
-  xlsx: {
-    name: "XLSX",
-    extensions: [".xlsx", ".xlsm"],
-    createReader: (input, options) => new XLSXReader(input, options),
-    createWriter: () => {
-      throw new Error("Writer adapter called from reader");
-    },
-  },
-};
-
-/**
- * Infers format name from file path or extension.
- */
-export function inferFormatFromPath(filePath: string): string | null {
-  const cleanPath = filePath.toLowerCase().replace(/\.gz$/, "");
-  for (const [format, adapter] of Object.entries(adapters)) {
-    if (adapter.extensions.some((ext) => cleanPath.endsWith(ext))) {
-      return format;
-    }
-  }
-  return null;
-}
-
-/**
- * Creates an appropriate TabularReader for input and format.
- */
-export function createReader(
-  input: Readable | string,
-  options: ReaderOptions & { format?: string } = {}
-): TabularReader {
-  let format = options.format?.toLowerCase();
-
-  if (!format && typeof input === "string" && input !== "-") {
-    format = inferFormatFromPath(input) ?? undefined;
-  }
-
-  if (!format) {
-    // Default to CSV if cannot infer
-    format = "csv";
-  }
-
-  const adapter = adapters[format];
-  if (!adapter) {
-    throw new InvalidArgumentError(
-      `Unsupported input format: "${format}". Supported formats are: ${Object.keys(adapters).join(", ")}`
-    );
-  }
-
-  return adapter.createReader(input, options);
-}

package/src/readers/json.ts

@@ -1,256 +0,0 @@
-import { createReadStream } from "node:fs";
-import type { Readable } from "node:stream";
-import { ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  TabularReader,
-} from "../core/types.js";
-
-export interface JSONReaderOptions extends ReaderOptions {
-  path?: string;
-  filePath?: string;
-}
-
-/**
- * Incremental streaming JSON parser for top-level arrays [ {...}, ... ]
- * and nested array paths (e.g. data.results).
- * Never loads the full JSON file into memory.
- */
-export class JSONReader implements TabularReader {
-  private input: Readable | string;
-  private options: JSONReaderOptions;
-
-  constructor(input: Readable | string, options: JSONReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  private getInputStream(): Readable {
-    if (typeof this.input === "string") {
-      return createReadStream(this.input);
-    }
-    return this.input;
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: JSONReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const targetPath = mergedOptions.path; // e.g. "data.results"
-    const stream = this.getInputStream();
-
-    let buffer = "";
-    let cursor = 0;
-    let byteOffset = 0;
-    let foundTargetArray = !targetPath; // if no path, we look for first '['
-    let currentPath: string[] = [];
-
-    // State machine for array item scanning
-    let inArray = false;
-    let inString = false;
-    let isEscaped = false;
-    let objectDepth = 0;
-    let objectStartIndex = -1;
-
-    let rowsInCurrentBatch: Row[] = [];
-    let globalOffset = 0;
-    let rowIndex = 0;
-
-    // Helper for path navigation if targetPath is set
-    const targetPathParts = targetPath ? targetPath.split(".") : [];
-
-    for await (const chunk of stream) {
-      const chunkStr =
-        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8");
-      buffer += chunkStr;
-
-      while (cursor < buffer.length) {
-        const char = buffer[cursor];
-
-        if (isEscaped) {
-          isEscaped = false;
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        if (char === "\\") {
-          isEscaped = true;
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        if (char === '"') {
-          inString = !inString;
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        if (inString) {
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        // We are outside of any string literal
-        if (!foundTargetArray) {
-          // If we need to find target path (e.g. data.results)
-          // Look for keys and match path parts
-          if (char === "{") {
-            cursor++;
-            byteOffset++;
-            continue;
-          } else if (char === "[") {
-            // Check if current path matches targetPath
-            // If path matched or we hit array at root
-            foundTargetArray = true;
-            inArray = true;
-            cursor++;
-            byteOffset++;
-            continue;
-          } else if (char === ":") {
-            // Scan backwards from ':' to find previous string key
-            let keyEnd = cursor - 1;
-            while (keyEnd >= 0 && /\s/.test(buffer[keyEnd]!)) keyEnd--;
-            if (keyEnd >= 0 && buffer[keyEnd] === '"') {
-              let keyStart = keyEnd - 1;
-              while (keyStart >= 0 && buffer[keyStart] !== '"') keyStart--;
-              if (keyStart >= 0) {
-                const keyName = buffer.slice(keyStart + 1, keyEnd);
-                currentPath.push(keyName);
-                if (
-                  currentPath.join(".") === targetPath ||
-                  keyName === targetPathParts[targetPathParts.length - 1]
-                ) {
-                  // Next non-whitespace should be '['
-                  // Let it continue to find '['
-                }
-              }
-            }
-            cursor++;
-            byteOffset++;
-            continue;
-          }
-          cursor++;
-          byteOffset++;
-          continue;
-        }
-
-        // Now we are streaming inside the target array
-        if (!inArray) {
-          if (char === "[") {
-            inArray = true;
-            cursor++;
-            byteOffset++;
-            continue;
-          }
-        } else {
-          // Inside array: looking for objects '{ ... }'
-          if (char === "{") {
-            if (objectDepth === 0) {
-              objectStartIndex = cursor;
-            }
-            objectDepth++;
-          } else if (char === "}") {
-            objectDepth--;
-            if (objectDepth === 0 && objectStartIndex !== -1) {
-              // We have captured a complete JSON object!
-              const objectStr = buffer.slice(objectStartIndex, cursor + 1);
-              let parsedRow: unknown;
-              try {
-                parsedRow = JSON.parse(objectStr);
-              } catch (err) {
-                throw new ParseError(
-                  `Malformed JSON object in array: ${(err as Error).message}`,
-                  {
-                    file: mergedOptions.filePath,
-                    row: rowIndex + 1,
-                    byteOffset,
-                  }
-                );
-              }
-
-              if (
-                typeof parsedRow === "object" &&
-                parsedRow !== null &&
-                !Array.isArray(parsedRow)
-              ) {
-                rowsInCurrentBatch.push(parsedRow as Row);
-                rowIndex++;
-
-                if (rowsInCurrentBatch.length >= batchSize) {
-                  yield {
-                    rows: rowsInCurrentBatch,
-                    offset: globalOffset,
-                  };
-                  globalOffset += rowsInCurrentBatch.length;
-                  rowsInCurrentBatch = [];
-                }
-              }
-
-              objectStartIndex = -1;
-            }
-          } else if (char === "]" && objectDepth === 0) {
-            // End of array reached
-            inArray = false;
-            break;
-          }
-
-          cursor++;
-          byteOffset++;
-        }
-      }
-
-      // Compact buffer while preserving active object if in the middle of parsing
-      if (objectStartIndex !== -1) {
-        buffer = buffer.slice(objectStartIndex);
-        cursor = cursor - objectStartIndex;
-        objectStartIndex = 0;
-      } else {
-        buffer = buffer.slice(cursor);
-        cursor = 0;
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    let rowCount = 0;
-    let columnNames: string[] = [];
-
-    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
-      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
-        columnNames = Object.keys(batch.rows[0]);
-      }
-      rowCount += batch.rows.length;
-    }
-
-    return {
-      format: "JSON",
-      rowCount,
-      columnCount: columnNames.length,
-      columns: columnNames.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      })),
-    };
-  }
-}