rowpipe 1.0.0

package/src/core/pipeline.ts

@@ -0,0 +1,150 @@
+import type {
+  Aggregator,
+  DataBatch,
+  DataStream,
+  ProgressCallback,
+  ReaderOptions,
+  Row,
+  TabularReader,
+  TabularWriter,
+  TransformFunction,
+  WriterOptions,
+} from "./types.js";
+import { batchesToRows } from "./batch.js";
+
+export class Pipeline {
+  private stream: DataStream;
+  private progressCallbacks: ProgressCallback[] = [];
+
+  constructor(source: DataStream | TabularReader, options?: ReaderOptions) {
+    if ("read" in source && typeof source.read === "function") {
+      this.stream = source.read(options);
+    } else {
+      this.stream = source as DataStream;
+    }
+  }
+
+  /**
+   * Chains a transform function to the pipeline.
+   */
+  pipe(transform: TransformFunction): this {
+    this.stream = transform(this.stream);
+    return this;
+  }
+
+  /**
+   * Adds a progress listener.
+   */
+  onProgress(callback: ProgressCallback): this {
+    this.progressCallbacks.push(callback);
+    return this;
+  }
+
+  /**
+   * Wraps the current stream with a progress tracking generator if callbacks are registered.
+   */
+  private getTrackedStream(): DataStream {
+    if (this.progressCallbacks.length === 0) {
+      return this.stream;
+    }
+
+    const callbacks = this.progressCallbacks;
+    const stream = this.stream;
+
+    return (async function* () {
+      let rowsProcessed = 0;
+      const startTime = Date.now();
+      let lastReport = startTime;
+
+      for await (const batch of stream) {
+        rowsProcessed += batch.rows.length;
+        const now = Date.now();
+
+        // Throttle progress notifications to ~100ms intervals
+        if (now - lastReport >= 100) {
+          const elapsedSec = (now - startTime) / 1000 || 0.001;
+          const info = {
+            rowsProcessed,
+            elapsedMs: now - startTime,
+            rowsPerSecond: Math.round(rowsProcessed / elapsedSec),
+          };
+          for (const cb of callbacks) {
+            cb(info);
+          }
+          lastReport = now;
+        }
+
+        yield batch;
+      }
+
+      // Final progress callback
+      const finalNow = Date.now();
+      const finalElapsedSec = (finalNow - startTime) / 1000 || 0.001;
+      const finalInfo = {
+        rowsProcessed,
+        elapsedMs: finalNow - startTime,
+        rowsPerSecond: Math.round(rowsProcessed / finalElapsedSec),
+      };
+      for (const cb of callbacks) {
+        cb(finalInfo);
+      }
+    })();
+  }
+
+  /**
+   * Pipes the stream into a TabularWriter.
+   */
+  async to(writer: TabularWriter, options?: WriterOptions): Promise<void> {
+    const tracked = this.getTrackedStream();
+    await writer.write(tracked, options);
+    if (writer.close) {
+      await writer.close();
+    }
+  }
+
+  /**
+   * Consumes the stream using an Aggregator.
+   */
+  async reduce<T>(aggregator: Aggregator<T>): Promise<T> {
+    const tracked = this.getTrackedStream();
+    for await (const row of batchesToRows(tracked)) {
+      aggregator.add(row);
+    }
+    return aggregator.result();
+  }
+
+  /**
+   * Returns an AsyncIterable of DataBatches.
+   */
+  batches(): DataStream {
+    return this.getTrackedStream();
+  }
+
+  /**
+   * Returns an AsyncIterable of individual rows.
+   */
+  rows(): AsyncIterable<Row> {
+    return batchesToRows(this.getTrackedStream());
+  }
+
+  /**
+   * Collects all rows up to a maximum limit (default 10,000 to prevent unbounded memory).
+   */
+  async toArray(maxRows = 10000): Promise<Row[]> {
+    const results: Row[] = [];
+    for await (const row of this.rows()) {
+      results.push(row);
+      if (results.length >= maxRows) {
+        break;
+      }
+    }
+    return results;
+  }
+}
+
+export function createPipeline(
+  source: DataStream | TabularReader,
+  options?: ReaderOptions
+): Pipeline {
+  return new Pipeline(source, options);
+}

package/src/core/types.ts

@@ -0,0 +1,110 @@
+import type { Readable, Writable } from "node:stream";
+
+export type Row = Record<string, unknown>;
+
+export interface DataBatch {
+  rows: Row[];
+  offset: number;
+}
+
+export type DataStream = AsyncIterable<DataBatch>;
+
+export type ColumnType =
+  | "string"
+  | "integer"
+  | "number"
+  | "boolean"
+  | "date"
+  | "datetime"
+  | "null"
+  | "mixed";
+
+export type SemanticType =
+  | "email"
+  | "url"
+  | "uuid"
+  | "ipv4"
+  | "ipv6"
+  | "country-code"
+  | "currency"
+  | "phone";
+
+export interface ReaderOptions {
+  batchSize?: number;
+  header?: boolean;
+  delimiter?: string;
+  path?: string;
+  sheet?: string | number;
+  maxRows?: number;
+  filePath?: string;
+}
+
+export interface WriterOptions {
+  delimiter?: string;
+  header?: boolean;
+  sheet?: string;
+}
+
+export interface SheetMetadata {
+  name: string;
+  rowCount: number;
+  columnCount?: number;
+  columns?: string[];
+}
+
+export interface InspectionMetadata {
+  format: string;
+  sizeBytes?: number;
+  rowCount?: number;
+  columnCount?: number;
+  columns?: Array<{
+    name: string;
+    type: ColumnType;
+    nullPercentage: number;
+    approxUniquePercentage?: number;
+  }>;
+  sheetsCount?: number;
+  sheets?: SheetMetadata[];
+}
+
+export interface TabularReader {
+  read(options?: ReaderOptions): DataStream;
+  inspect?(options?: ReaderOptions): Promise<InspectionMetadata>;
+  close?(): Promise<void>;
+}
+
+export interface TabularWriter {
+  write(stream: DataStream, options?: WriterOptions): Promise<void>;
+  close?(): Promise<void>;
+}
+
+export interface FormatAdapter {
+  name: string;
+  extensions: string[];
+  createReader(
+    input: Readable | string,
+    options?: ReaderOptions
+  ): TabularReader;
+  createWriter(
+    output: Writable | string,
+    options?: WriterOptions
+  ): TabularWriter;
+}
+
+export type TransformFunction = (stream: DataStream) => DataStream;
+
+export interface Aggregator<T> {
+  add(row: Row): void;
+  merge?(other: Aggregator<T>): void;
+  result(): T;
+}
+
+export interface ProgressCallbackInfo {
+  rowsProcessed: number;
+  bytesRead?: number;
+  elapsedMs: number;
+  rowsPerSecond: number;
+  megabytesPerSecond?: number;
+}
+
+export type ProgressCallback = (info: ProgressCallbackInfo) => void;

package/src/index.ts

@@ -0,0 +1,52 @@
+// Core
+export * from "./core/types.js";
+export * from "./core/errors.js";
+export * from "./core/batch.js";
+export * from "./core/pipeline.js";
+
+// Readers
+export {
+  CSVReader,
+  type CSVReaderOptions,
+  JSONReader,
+  type JSONReaderOptions,
+  JSONLReader,
+  type JSONLReaderOptions,
+  XLSXReader,
+  type XLSXReaderOptions,
+  createReader,
+  inferFormatFromPath as inferReaderFormatFromPath,
+} from "./readers/index.js";
+
+// Writers
+export {
+  CSVWriter,
+  type CSVWriterOptions,
+  JSONWriter,
+  JSONLWriter,
+  XLSXWriter,
+  type XLSXWriterOptions,
+  createWriter,
+  inferFormatFromPath as inferWriterFormatFromPath,
+} from "./writers/index.js";
+
+// Transforms
+export * from "./transforms/select.js";
+export * from "./transforms/rename.js";
+export * from "./transforms/cast.js";
+export * from "./transforms/sample.js";
+export * from "./transforms/filter.js";
+export * from "./transforms/expression.js";
+export * from "./transforms/map.js";
+
+// Analytics
+export * from "./analytics/stats.js";
+export * from "./analytics/reduce.js";
+export * from "./analytics/schema-inference.js";
+export * from "./analytics/semantic-types.js";
+export * from "./analytics/validator.js";
+
+// Utilities
+export * from "./utils/compression.js";
+export * from "./utils/formatting.js";
+export * from "./utils/progress.js";

package/src/readers/csv.ts

@@ -0,0 +1,302 @@
+import { createReadStream } from "node:fs";
+import type { Readable } from "node:stream";
+import { ParseError } from "../core/errors.js";
+import type {
+  DataBatch,
+  DataStream,
+  InspectionMetadata,
+  ReaderOptions,
+  Row,
+  TabularReader,
+} from "../core/types.js";
+
+export interface CSVReaderOptions extends ReaderOptions {
+  delimiter?: string;
+  header?: boolean;
+  filePath?: string;
+}
+
+/**
+ * Streaming CSV Reader supporting RFC 4180 quotes, multiline cells,
+ * custom & auto-detected delimiters, BOM removal, and exact error reporting.
+ */
+export class CSVReader implements TabularReader {
+  private input: Readable | string;
+  private options: CSVReaderOptions;
+
+  constructor(input: Readable | string, options: CSVReaderOptions = {}) {
+    this.input = input;
+    this.options = { ...options };
+    if (typeof input === "string") {
+      this.options.filePath = input;
+    }
+  }
+
+  private getInputStream(): Readable {
+    if (typeof this.input === "string") {
+      return createReadStream(this.input);
+    }
+    return this.input;
+  }
+
+  /**
+   * Auto-detects delimiter by scanning candidate delimiters in a sample string.
+   */
+  public static detectDelimiter(sample: string): string {
+    const candidates = [",", "\t", ";", "|"];
+    const firstLine = sample.split(/\r?\n/)[0] || "";
+
+    let bestDelimiter = ",";
+    let maxCount = -1;
+
+    for (const cand of candidates) {
+      let count = 0;
+      let inQuotes = false;
+      for (let i = 0; i < firstLine.length; i++) {
+        const ch = firstLine[i];
+        if (ch === '"') {
+          inQuotes = !inQuotes;
+        } else if (ch === cand && !inQuotes) {
+          count++;
+        }
+      }
+      if (count > maxCount) {
+        maxCount = count;
+        bestDelimiter = cand;
+      }
+    }
+
+    return maxCount > 0 ? bestDelimiter : ",";
+  }
+
+  async *read(options?: ReaderOptions): DataStream {
+    const mergedOptions: CSVReaderOptions = {
+      ...this.options,
+      ...options,
+    };
+    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
+    const hasHeader = mergedOptions.header !== false;
+    let delimiter = mergedOptions.delimiter;
+
+    const stream = this.getInputStream();
+
+    let buffer = "";
+    let headers: string[] | null = null;
+    let isFirstChunk = true;
+    let rowsInCurrentBatch: Row[] = [];
+    let globalRowOffset = 0;
+    let rowNumber = 0;
+    let byteOffset = 0;
+    let currentField = "";
+    let currentRowFields: string[] = [];
+    let inQuotes = false;
+    let cursor = 0;
+
+    for await (const chunk of stream) {
+      const chunkStr: string =
+        typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8");
+
+      if (isFirstChunk) {
+        isFirstChunk = false;
+        // Remove UTF-8 BOM if present
+        if (chunkStr.charCodeAt(0) === 0xfeff) {
+          buffer = chunkStr.slice(1);
+        } else {
+          buffer = chunkStr;
+        }
+
+        // Auto-detect delimiter if not specified
+        if (!delimiter) {
+          delimiter = CSVReader.detectDelimiter(buffer);
+        }
+      } else {
+        buffer += chunkStr;
+      }
+
+      const activeDelimiter = delimiter ?? ",";
+      const delimLen = activeDelimiter.length;
+
+      while (cursor < buffer.length) {
+        const char = buffer[cursor];
+
+        if (inQuotes) {
+          if (char === '"') {
+            if (cursor + 1 < buffer.length && buffer[cursor + 1] === '"') {
+              currentField += '"';
+              cursor += 2;
+              byteOffset += 2;
+              continue;
+            } else {
+              inQuotes = false;
+              cursor++;
+              byteOffset++;
+              continue;
+            }
+          } else {
+            currentField += char;
+            cursor++;
+            byteOffset++;
+            continue;
+          }
+        } else {
+          if (char === '"') {
+            if (currentField.length === 0) {
+              inQuotes = true;
+              cursor++;
+              byteOffset++;
+              continue;
+            } else {
+              throw new ParseError(
+                `Unexpected quote inside unquoted field at byte offset ${byteOffset}`,
+                {
+                  file: mergedOptions.filePath,
+                  row: rowNumber + 1,
+                  column: currentRowFields.length + 1,
+                  byteOffset,
+                }
+              );
+            }
+          }
+
+          // Check for delimiter
+          if (buffer.startsWith(activeDelimiter, cursor)) {
+            currentRowFields.push(currentField);
+            currentField = "";
+            cursor += delimLen;
+            byteOffset += delimLen;
+            continue;
+          }
+
+          // Check for CRLF / LF newline
+          if (char === "\r" || char === "\n") {
+            currentRowFields.push(currentField);
+            currentField = "";
+
+            if (char === "\r" && cursor + 1 < buffer.length && buffer[cursor + 1] === "\n") {
+              cursor += 2;
+              byteOffset += 2;
+            } else {
+              cursor++;
+              byteOffset++;
+            }
+
+            // Skip entirely empty row if it's trailing whitespace
+            if (
+              currentRowFields.length === 1 &&
+              currentRowFields[0]?.trim() === "" &&
+              cursor >= buffer.length
+            ) {
+              currentRowFields = [];
+              continue;
+            }
+
+            if (hasHeader && headers === null) {
+              headers = currentRowFields.map((h, i) =>
+                h.trim().length > 0 ? h.trim() : `col_${i + 1}`
+              );
+              currentRowFields = [];
+              continue;
+            }
+
+            if (headers === null) {
+              headers = currentRowFields.map((_, i) => `col_${i + 1}`);
+            }
+
+            // Construct Row object
+            const rowObj: Row = {};
+            for (let i = 0; i < headers.length; i++) {
+              const key = headers[i] || `col_${i + 1}`;
+              const val = currentRowFields[i] ?? "";
+              rowObj[key] = val;
+            }
+
+            rowNumber++;
+            rowsInCurrentBatch.push(rowObj);
+            currentRowFields = [];
+
+            if (rowsInCurrentBatch.length >= batchSize) {
+              yield {
+                rows: rowsInCurrentBatch,
+                offset: globalRowOffset,
+              };
+              globalRowOffset += rowsInCurrentBatch.length;
+              rowsInCurrentBatch = [];
+            }
+            continue;
+          }
+
+          currentField += char;
+          cursor++;
+          byteOffset++;
+        }
+      }
+
+      // Compact processed buffer to keep memory bounded
+      buffer = buffer.slice(cursor);
+      cursor = 0;
+    }
+
+    // Process remainder if any
+    if (inQuotes) {
+      throw new ParseError("Unclosed quote at end of CSV stream", {
+        file: mergedOptions.filePath,
+        row: rowNumber + 1,
+        byteOffset,
+      });
+    }
+
+    if (currentField.length > 0 || currentRowFields.length > 0) {
+      currentRowFields.push(currentField);
+
+      if (hasHeader && headers === null) {
+        headers = currentRowFields.map((h, i) =>
+          h.trim().length > 0 ? h.trim() : `col_${i + 1}`
+        );
+      } else {
+        if (headers === null) {
+          headers = currentRowFields.map((_, i) => `col_${i + 1}`);
+        }
+
+        const rowObj: Row = {};
+        for (let i = 0; i < headers.length; i++) {
+          const key = headers[i] || `col_${i + 1}`;
+          const val = currentRowFields[i] ?? "";
+          rowObj[key] = val;
+        }
+
+        rowNumber++;
+        rowsInCurrentBatch.push(rowObj);
+      }
+    }
+
+    if (rowsInCurrentBatch.length > 0) {
+      yield {
+        rows: rowsInCurrentBatch,
+        offset: globalRowOffset,
+      };
+    }
+  }
+
+  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
+    let rowCount = 0;
+    let columnNames: string[] = [];
+
+    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
+      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
+        columnNames = Object.keys(batch.rows[0]);
+      }
+      rowCount += batch.rows.length;
+    }
+
+    return {
+      format: "CSV",
+      rowCount,
+      columnCount: columnNames.length,
+      columns: columnNames.map((name) => ({
+        name,
+        type: "string",
+        nullPercentage: 0,
+      })),
+    };
+  }
+}

package/src/readers/index.ts

@@ -0,0 +1,88 @@
+import type { Readable } from "node:stream";
+import { InvalidArgumentError } from "../core/errors.js";
+import type { FormatAdapter, ReaderOptions, TabularReader } from "../core/types.js";
+import { CSVReader } from "./csv.js";
+import { JSONReader } from "./json.js";
+import { JSONLReader } from "./jsonl.js";
+import { XLSXReader } from "./xlsx.js";
+
+export * from "./csv.js";
+export * from "./json.js";
+export * from "./jsonl.js";
+export * from "./xlsx.js";
+
+const adapters: Record<string, FormatAdapter> = {
+  csv: {
+    name: "CSV",
+    extensions: [".csv", ".tsv", ".txt"],
+    createReader: (input, options) => new CSVReader(input, options),
+    createWriter: () => {
+      throw new Error("Writer adapter called from reader");
+    },
+  },
+  json: {
+    name: "JSON",
+    extensions: [".json"],
+    createReader: (input, options) => new JSONReader(input, options),
+    createWriter: () => {
+      throw new Error("Writer adapter called from reader");
+    },
+  },
+  jsonl: {
+    name: "JSONL",
+    extensions: [".jsonl", ".ndjson", ".ldjson"],
+    createReader: (input, options) => new JSONLReader(input, options),
+    createWriter: () => {
+      throw new Error("Writer adapter called from reader");
+    },
+  },
+  xlsx: {
+    name: "XLSX",
+    extensions: [".xlsx", ".xlsm"],
+    createReader: (input, options) => new XLSXReader(input, options),
+    createWriter: () => {
+      throw new Error("Writer adapter called from reader");
+    },
+  },
+};
+
+/**
+ * Infers format name from file path or extension.
+ */
+export function inferFormatFromPath(filePath: string): string | null {
+  const cleanPath = filePath.toLowerCase().replace(/\.gz$/, "");
+  for (const [format, adapter] of Object.entries(adapters)) {
+    if (adapter.extensions.some((ext) => cleanPath.endsWith(ext))) {
+      return format;
+    }
+  }
+  return null;
+}
+
+/**
+ * Creates an appropriate TabularReader for input and format.
+ */
+export function createReader(
+  input: Readable | string,
+  options: ReaderOptions & { format?: string } = {}
+): TabularReader {
+  let format = options.format?.toLowerCase();
+
+  if (!format && typeof input === "string" && input !== "-") {
+    format = inferFormatFromPath(input) ?? undefined;
+  }
+
+  if (!format) {
+    // Default to CSV if cannot infer
+    format = "csv";
+  }
+
+  const adapter = adapters[format];
+  if (!adapter) {
+    throw new InvalidArgumentError(
+      `Unsupported input format: "${format}". Supported formats are: ${Object.keys(adapters).join(", ")}`
+    );
+  }
+
+  return adapter.createReader(input, options);
+}