rowpipe 1.0.0

package/src/transforms/filter.ts

@@ -0,0 +1,38 @@
+import type { DataStream, Row, TransformFunction } from "../core/types.js";
+import { compileExpression } from "./expression.js";
+
+/**
+ * Creates a transform that filters rows based on a safe compiled filter expression.
+ */
+export function filterRows(
+  predicateOrExpression: string | ((row: Row) => boolean)
+): TransformFunction {
+  const predicate =
+    typeof predicateOrExpression === "function"
+      ? predicateOrExpression
+      : compileExpression(predicateOrExpression);
+
+  return function (stream: DataStream): DataStream {
+    return (async function* () {
+      for await (const batch of stream) {
+        const rows = batch.rows;
+        const len = rows.length;
+        const filteredRows: Row[] = [];
+
+        for (let i = 0; i < len; i++) {
+          const row = rows[i]!;
+          if (predicate(row)) {
+            filteredRows.push(row);
+          }
+        }
+
+        if (filteredRows.length > 0) {
+          yield {
+            rows: filteredRows,
+            offset: batch.offset,
+          };
+        }
+      }
+    })();
+  };
+}

package/src/transforms/map.ts

@@ -0,0 +1,99 @@
+import { InvalidArgumentError } from "../core/errors.js";
+import type { DataBatch, DataStream, Row, TransformFunction } from "../core/types.js";
+import { compileValueExpression } from "./expression.js";
+
+/**
+ * Parses CLI map argument specs into a column-to-expression dictionary.
+ * Supports formats: "profit = revenue - cost", "tax=revenue*0.2"
+ */
+export function parseMapSpecs(specs: string[]): Record<string, string> {
+  const result: Record<string, string> = {};
+
+  for (const spec of specs) {
+    const trimmed = spec.trim();
+    if (!trimmed) continue;
+
+    const eqIdx = trimmed.indexOf("=");
+    if (eqIdx === -1) {
+      throw new InvalidArgumentError(
+        `Invalid map specification "${spec}". Expected format "column_name = expression"`
+      );
+    }
+
+    const colName = trimmed.slice(0, eqIdx).trim();
+    const expr = trimmed.slice(eqIdx + 1).trim();
+
+    if (!colName) {
+      throw new InvalidArgumentError(
+        `Missing target column name in map specification "${spec}"`
+      );
+    }
+    if (!expr) {
+      throw new InvalidArgumentError(
+        `Missing expression in map specification "${spec}"`
+      );
+    }
+
+    result[colName] = expr;
+  }
+
+  return result;
+}
+
+export type RowMapper = (row: Row) => Row;
+
+/**
+ * Creates a high-performance streaming transform that computes/derives columns per row.
+ */
+export function mapRows(
+  specs: Record<string, string> | RowMapper
+): TransformFunction {
+  if (typeof specs === "function") {
+    const customMapper = specs;
+    return async function* (stream: DataStream): DataStream {
+      for await (const batch of stream) {
+        const len = batch.rows.length;
+        const mappedRows: Row[] = new Array(len);
+        for (let i = 0; i < len; i++) {
+          mappedRows[i] = customMapper(batch.rows[i]!);
+        }
+        yield {
+          rows: mappedRows,
+          offset: batch.offset,
+        };
+      }
+    };
+  }
+
+  // Pre-compile all AST expression evaluators once before processing stream
+  const compiledEntries = Object.entries(specs).map(([targetCol, exprStr]) => ({
+    targetCol,
+    evaluator: compileValueExpression(exprStr),
+  }));
+
+  return async function* (stream: DataStream): DataStream {
+    for await (const batch of stream) {
+      const rowCount = batch.rows.length;
+      const mappedRows: Row[] = new Array(rowCount);
+      const compiledLen = compiledEntries.length;
+
+      for (let i = 0; i < rowCount; i++) {
+        const srcRow = batch.rows[i]!;
+        // Clone row to avoid mutating original
+        const outRow: Row = { ...srcRow };
+
+        for (let j = 0; j < compiledLen; j++) {
+          const entry = compiledEntries[j]!;
+          outRow[entry.targetCol] = entry.evaluator(srcRow);
+        }
+
+        mappedRows[i] = outRow;
+      }
+
+      yield {
+        rows: mappedRows,
+        offset: batch.offset,
+      };
+    }
+  };
+}

package/src/transforms/rename.ts

@@ -0,0 +1,47 @@
+import type { DataStream, Row, TransformFunction } from "../core/types.js";
+
+/**
+ * Creates a transform that renames specified columns in streaming batches.
+ */
+export function renameColumns(
+  mapping: Record<string, string>
+): TransformFunction {
+  return function (stream: DataStream): DataStream {
+    return (async function* () {
+      for await (const batch of stream) {
+        const renamedRows: Row[] = new Array(batch.rows.length);
+
+        for (let i = 0; i < batch.rows.length; i++) {
+          const row = batch.rows[i]!;
+          const newRow: Row = {};
+
+          for (const [key, val] of Object.entries(row)) {
+            const targetKey = mapping[key] || key;
+            newRow[targetKey] = val;
+          }
+
+          renamedRows[i] = newRow;
+        }
+
+        yield {
+          rows: renamedRows,
+          offset: batch.offset,
+        };
+      }
+    })();
+  };
+}
+
+/**
+ * Parses command line rename specs like ["username=name", "signup_date=created_at"].
+ */
+export function parseRenameSpecs(specs: string[]): Record<string, string> {
+  const mapping: Record<string, string> = {};
+  for (const spec of specs) {
+    const parts = spec.split("=");
+    if (parts.length === 2 && parts[0] && parts[1]) {
+      mapping[parts[0].trim()] = parts[1].trim();
+    }
+  }
+  return mapping;
+}

package/src/transforms/sample.ts

@@ -0,0 +1,57 @@
+import type { DataStream, Row, TransformFunction } from "../core/types.js";
+
+/**
+ * Creates a deterministic PRNG using Mulberry32 algorithm.
+ */
+function createPrng(seed: number): () => number {
+  let s = seed >>> 0;
+  return function () {
+    let t = (s += 0x6d2b79f5);
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+export interface SampleOptions {
+  rows: number;
+  seed?: number;
+}
+
+/**
+ * Reservoir sampling transform with bounded O(k) memory.
+ */
+export function sampleRows(options: SampleOptions): TransformFunction {
+  const k = Math.max(1, options.rows);
+  const randomFn =
+    options.seed !== undefined ? createPrng(options.seed) : Math.random;
+
+  return function (stream: DataStream): DataStream {
+    return (async function* () {
+      const reservoir: Row[] = [];
+      let totalRowsSeen = 0;
+
+      for await (const batch of stream) {
+        for (const row of batch.rows) {
+          if (totalRowsSeen < k) {
+            reservoir.push(row);
+          } else {
+            // Pick a random integer from 0 to totalRowsSeen
+            const j = Math.floor(randomFn() * (totalRowsSeen + 1));
+            if (j < k) {
+              reservoir[j] = row;
+            }
+          }
+          totalRowsSeen++;
+        }
+      }
+
+      if (reservoir.length > 0) {
+        yield {
+          rows: reservoir,
+          offset: 0,
+        };
+      }
+    })();
+  };
+}

package/src/transforms/select.ts

@@ -0,0 +1,64 @@
+import type { DataBatch, DataStream, Row, TransformFunction } from "../core/types.js";
+
+/**
+ * Creates a transform that projects only the specified columns in order.
+ */
+export function selectColumns(columns: string[]): TransformFunction {
+  const columnSet = columns.map((c) => c.trim()).filter((c) => c.length > 0);
+  const len = columnSet.length;
+
+  let projectRow: (row: Row) => Row;
+  if (len === 1) {
+    const c0 = columnSet[0]!;
+    projectRow = (row) => ({ [c0]: row[c0] !== undefined ? row[c0] : "" });
+  } else if (len === 2) {
+    const c0 = columnSet[0]!, c1 = columnSet[1]!;
+    projectRow = (row) => ({
+      [c0]: row[c0] !== undefined ? row[c0] : "",
+      [c1]: row[c1] !== undefined ? row[c1] : "",
+    });
+  } else if (len === 3) {
+    const c0 = columnSet[0]!, c1 = columnSet[1]!, c2 = columnSet[2]!;
+    projectRow = (row) => ({
+      [c0]: row[c0] !== undefined ? row[c0] : "",
+      [c1]: row[c1] !== undefined ? row[c1] : "",
+      [c2]: row[c2] !== undefined ? row[c2] : "",
+    });
+  } else if (len === 4) {
+    const c0 = columnSet[0]!, c1 = columnSet[1]!, c2 = columnSet[2]!, c3 = columnSet[3]!;
+    projectRow = (row) => ({
+      [c0]: row[c0] !== undefined ? row[c0] : "",
+      [c1]: row[c1] !== undefined ? row[c1] : "",
+      [c2]: row[c2] !== undefined ? row[c2] : "",
+      [c3]: row[c3] !== undefined ? row[c3] : "",
+    });
+  } else {
+    projectRow = (row) => {
+      const newRow: Row = {};
+      for (let j = 0; j < len; j++) {
+        const col = columnSet[j]!;
+        newRow[col] = row[col] !== undefined ? row[col] : "";
+      }
+      return newRow;
+    };
+  }
+
+  return function (stream: DataStream): DataStream {
+    return (async function* () {
+      for await (const batch of stream) {
+        const rows = batch.rows;
+        const rowCount = rows.length;
+        const projectedRows: Row[] = new Array(rowCount);
+
+        for (let i = 0; i < rowCount; i++) {
+          projectedRows[i] = projectRow(rows[i]!);
+        }
+
+        yield {
+          rows: projectedRows,
+          offset: batch.offset,
+        };
+      }
+    })();
+  };
+}

package/src/utils/compression.ts

@@ -0,0 +1,43 @@
+import { createReadStream, createWriteStream } from "node:fs";
+import type { Readable, Writable } from "node:stream";
+import { createGzip, createGunzip } from "node:zlib";
+
+/**
+ * Checks if a path or filename indicates gzip compression.
+ */
+export function isGzipPath(filePath: string): boolean {
+  return filePath.toLowerCase().endsWith(".gz");
+}
+
+/**
+ * Creates a readable stream, automatically piping through gunzip if the path ends with .gz.
+ */
+export function openReadableStream(filePath: string): Readable {
+  if (filePath === "-") {
+    return process.stdin;
+  }
+
+  const rawStream = createReadStream(filePath);
+  if (isGzipPath(filePath)) {
+    const gunzip = createGunzip();
+    return rawStream.pipe(gunzip);
+  }
+  return rawStream;
+}
+
+/**
+ * Creates a writable stream, automatically piping through gzip if the path ends with .gz.
+ */
+export function openWritableStream(filePath: string): Writable {
+  if (filePath === "-") {
+    return process.stdout;
+  }
+
+  const fileStream = createWriteStream(filePath);
+  if (isGzipPath(filePath)) {
+    const gzip = createGzip();
+    gzip.pipe(fileStream);
+    return gzip;
+  }
+  return fileStream;
+}

package/src/utils/formatting.ts

@@ -0,0 +1,75 @@
+export function formatNumber(num: number | null | undefined): string {
+  if (num === null || num === undefined || Number.isNaN(num)) {
+    return "-";
+  }
+  return new Intl.NumberFormat("en-US").format(num);
+}
+
+export function formatDecimal(num: number | null | undefined, digits = 2): string {
+  if (num === null || num === undefined || Number.isNaN(num)) {
+    return "-";
+  }
+  return new Intl.NumberFormat("en-US", {
+    minimumFractionDigits: 0,
+    maximumFractionDigits: digits,
+  }).format(num);
+}
+
+export function formatBytes(bytes: number | null | undefined): string {
+  if (bytes === null || bytes === undefined || bytes === 0) return "0 B";
+  const k = 1024;
+  const sizes = ["B", "KB", "MB", "GB", "TB"];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return `${Number.parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`;
+}
+
+export function formatTable(
+  headers: string[],
+  rows: string[][],
+  alignments?: Array<"left" | "right">
+): string {
+  if (headers.length === 0) return "";
+
+  const colWidths = headers.map((h) => h.length);
+
+  for (const row of rows) {
+    for (let i = 0; i < headers.length; i++) {
+      const cell = row[i] || "";
+      if (cell.length > (colWidths[i] || 0)) {
+        colWidths[i] = cell.length;
+      }
+    }
+  }
+
+  const formatCell = (text: string, width: number, align: "left" | "right") => {
+    if (align === "right") {
+      return text.padStart(width);
+    }
+    return text.padEnd(width);
+  };
+
+  const headerLine = headers
+    .map((h, i) =>
+      formatCell(h, colWidths[i]!, alignments?.[i] || "left")
+    )
+    .join("   ");
+
+  const rowLines = rows.map((row) =>
+    headers
+      .map((_, i) =>
+        formatCell(row[i] || "", colWidths[i]!, alignments?.[i] || "left")
+      )
+      .join("   ")
+  );
+
+  return [headerLine, ...rowLines].join("\n");
+}
+
+export function logMemoryDebug(): void {
+  if (process.env["ROWPIPE_DEBUG_MEMORY"] === "1") {
+    const mem = process.memoryUsage();
+    process.stderr.write(
+      `\n[ROWPIPE DEBUG MEMORY]\n  Peak RSS:  ${formatBytes(mem.rss)}\n  Heap Used: ${formatBytes(mem.heapUsed)}\n  Heap Total: ${formatBytes(mem.heapTotal)}\n  External:  ${formatBytes(mem.external)}\n\n`
+    );
+  }
+}

package/src/utils/progress.ts

@@ -0,0 +1,37 @@
+import type { ProgressCallbackInfo } from "../core/types.js";
+import { formatNumber } from "./formatting.js";
+
+export interface ProgressReporterOptions {
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export class ProgressReporter {
+  private isEnabled: boolean;
+  private hasReported = false;
+
+  constructor(options: ProgressReporterOptions = {}) {
+    this.isEnabled =
+      !options.quiet &&
+      !options.noProgress &&
+      Boolean(process.stderr.isTTY);
+  }
+
+  update(info: ProgressCallbackInfo): void {
+    if (!this.isEnabled) return;
+    this.hasReported = true;
+
+    const rows = formatNumber(info.rowsProcessed);
+    const speed = formatNumber(info.rowsPerSecond);
+    const timeSec = (info.elapsedMs / 1000).toFixed(1);
+
+    const line = `\r\x1b[2K  Processed ${rows} rows | ${timeSec}s | ${speed} rows/s`;
+    process.stderr.write(line);
+  }
+
+  done(): void {
+    if (this.isEnabled && this.hasReported) {
+      process.stderr.write("\r\x1b[2K");
+    }
+  }
+}

package/src/writers/csv.ts

@@ -0,0 +1,106 @@
+import { createWriteStream } from "node:fs";
+import type { Writable } from "node:stream";
+import type { DataStream, TabularWriter, WriterOptions } from "../core/types.js";
+
+export interface CSVWriterOptions extends WriterOptions {
+  delimiter?: string;
+  header?: boolean;
+}
+
+export class CSVWriter implements TabularWriter {
+  private output: Writable | string;
+  private options: CSVWriterOptions;
+  private createdStream?: Writable;
+
+  constructor(output: Writable | string, options: CSVWriterOptions = {}) {
+    this.output = output;
+    this.options = { ...options };
+  }
+
+  private getOutputStream(): Writable {
+    if (typeof this.output === "string") {
+      this.createdStream = createWriteStream(this.output);
+      return this.createdStream;
+    }
+    return this.output;
+  }
+
+  private escapeField(value: unknown, delimiter: string): string {
+    if (value === null || value === undefined) {
+      return "";
+    }
+    const str = String(value);
+    const mustQuote =
+      str.includes(delimiter) ||
+      str.includes('"') ||
+      str.includes("\n") ||
+      str.includes("\r");
+
+    if (mustQuote) {
+      return `"${str.replace(/"/g, '""')}"`;
+    }
+    return str;
+  }
+
+  private async writeChunk(stream: Writable, data: string): Promise<void> {
+    if (!stream.write(data)) {
+      await new Promise<void>((resolve) => stream.once("drain", resolve));
+    }
+  }
+
+  async write(dataStream: DataStream, options?: WriterOptions): Promise<void> {
+    const mergedOptions: CSVWriterOptions = {
+      ...this.options,
+      ...options,
+    };
+    const delimiter = mergedOptions.delimiter ?? ",";
+    const writeHeader = mergedOptions.header !== false;
+
+    const outStream = this.getOutputStream();
+    let headers: string[] | null = null;
+
+    for await (const batch of dataStream) {
+      if (batch.rows.length === 0) continue;
+
+      if (headers === null) {
+        // Collect all distinct keys from first batch to form header
+        const keySet = new Set<string>();
+        for (const row of batch.rows) {
+          for (const key of Object.keys(row)) {
+            keySet.add(key);
+          }
+        }
+        headers = Array.from(keySet);
+
+        if (writeHeader && headers.length > 0) {
+          const headerLine =
+            headers.map((h) => this.escapeField(h, delimiter)).join(delimiter) +
+            "\n";
+          await this.writeChunk(outStream, headerLine);
+        }
+      }
+
+      let chunkText = "";
+      for (const row of batch.rows) {
+        const line =
+          headers
+            .map((h) => this.escapeField(row[h], delimiter))
+            .join(delimiter) + "\n";
+        chunkText += line;
+      }
+
+      if (chunkText.length > 0) {
+        await this.writeChunk(outStream, chunkText);
+      }
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.createdStream) {
+      await new Promise<void>((resolve, reject) => {
+        this.createdStream!.end(() => resolve());
+        this.createdStream!.on("error", reject);
+      });
+    }
+  }
+}

package/src/writers/index.ts

@@ -0,0 +1,87 @@
+import type { Writable } from "node:stream";
+import { InvalidArgumentError } from "../core/errors.js";
+import type { FormatAdapter, TabularWriter, WriterOptions } from "../core/types.js";
+import { CSVWriter } from "./csv.js";
+import { JSONWriter } from "./json.js";
+import { JSONLWriter } from "./jsonl.js";
+import { XLSXWriter } from "./xlsx.js";
+
+export * from "./csv.js";
+export * from "./json.js";
+export * from "./jsonl.js";
+export * from "./xlsx.js";
+
+const adapters: Record<string, FormatAdapter> = {
+  csv: {
+    name: "CSV",
+    extensions: [".csv", ".tsv", ".txt"],
+    createReader: () => {
+      throw new Error("Reader adapter called from writer");
+    },
+    createWriter: (output, options) => new CSVWriter(output, options),
+  },
+  json: {
+    name: "JSON",
+    extensions: [".json"],
+    createReader: () => {
+      throw new Error("Reader adapter called from writer");
+    },
+    createWriter: (output) => new JSONWriter(output),
+  },
+  jsonl: {
+    name: "JSONL",
+    extensions: [".jsonl", ".ndjson", ".ldjson"],
+    createReader: () => {
+      throw new Error("Reader adapter called from writer");
+    },
+    createWriter: (output) => new JSONLWriter(output),
+  },
+  xlsx: {
+    name: "XLSX",
+    extensions: [".xlsx"],
+    createReader: () => {
+      throw new Error("Reader adapter called from writer");
+    },
+    createWriter: (output, options) => new XLSXWriter(output, options),
+  },
+};
+
+/**
+ * Infers format name from file path or extension.
+ */
+export function inferFormatFromPath(filePath: string): string | null {
+  const cleanPath = filePath.toLowerCase().replace(/\.gz$/, "");
+  for (const [format, adapter] of Object.entries(adapters)) {
+    if (adapter.extensions.some((ext) => cleanPath.endsWith(ext))) {
+      return format;
+    }
+  }
+  return null;
+}
+
+/**
+ * Creates an appropriate TabularWriter for output and format.
+ */
+export function createWriter(
+  output: Writable | string,
+  options: WriterOptions & { format?: string } = {}
+): TabularWriter {
+  let format = options.format?.toLowerCase();
+
+  if (!format && typeof output === "string" && output !== "-") {
+    format = inferFormatFromPath(output) ?? undefined;
+  }
+
+  if (!format) {
+    format = "csv";
+  }
+
+  const adapter = adapters[format];
+  if (!adapter) {
+    throw new InvalidArgumentError(
+      `Unsupported output format: "${format}". Supported formats are: ${Object.keys(adapters).join(", ")}`
+    );
+  }
+
+  return adapter.createWriter(output, options);
+}