rowpipe 1.0.0

package/src/cli/commands/filter.ts

@@ -0,0 +1,58 @@
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { filterRows } from "../../transforms/filter.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface FilterCommandOptions {
+  from?: string;
+  to?: string;
+  sheet?: string;
+  delimiter?: string;
+  batchSize?: string | number;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function filterCommand(
+  inputPath = "-",
+  expression: string,
+  options: FilterCommandOptions = {}
+): Promise<void> {
+  const effectiveBatchSize = Number(options.batchSize) || 1000;
+  const progress = new ProgressReporter(options);
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferReaderFormat(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    delimiter: options.delimiter,
+    batchSize: effectiveBatchSize,
+    filePath: inputPath,
+  });
+
+  const writer = createWriter(process.stdout, {
+    format: toFormat,
+    delimiter: options.delimiter,
+  });
+
+  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+  pipeline.pipe(filterRows(expression));
+  pipeline.onProgress((info) => progress.update(info));
+
+  await pipeline.to(writer);
+  progress.done();
+  logMemoryDebug();
+}

package/src/cli/commands/inspect.ts

@@ -0,0 +1,193 @@
+import { stat } from "node:fs/promises";
+import { DatasetStatsAggregator } from "../../analytics/stats.js";
+import { SchemaInferenceAggregator } from "../../analytics/schema-inference.js";
+import { InvalidArgumentError } from "../../core/errors.js";
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath } from "../../readers/index.js";
+import { openReadableStream } from "../../utils/compression.js";
+import {
+  formatBytes,
+  formatDecimal,
+  formatNumber,
+  formatTable,
+  logMemoryDebug,
+} from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface InspectCommandOptions {
+  sheet?: string;
+  path?: string;
+  delimiter?: string;
+  json?: boolean;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function inspectCommand(
+  inputPath = "-",
+  options: InspectCommandOptions = {}
+): Promise<void> {
+  const progress = new ProgressReporter(options);
+
+  let fileSize: number | undefined;
+  if (inputPath !== "-") {
+    try {
+      const fileStat = await stat(inputPath);
+      fileSize = fileStat.size;
+    } catch {
+      // Ignore if cannot stat
+    }
+  }
+
+  const format =
+    inputPath !== "-" ? inferFormatFromPath(inputPath) || "CSV" : "CSV";
+
+  const stream = openReadableStream(inputPath);
+  const reader = createReader(stream, {
+    format,
+    sheet: options.sheet,
+    path: options.path,
+    delimiter: options.delimiter,
+    filePath: inputPath,
+  });
+
+  // If XLSX without a specific --sheet requested, show rich workbook & sheet breakdown
+  if (format === "xlsx" && !options.sheet && reader.inspect) {
+    const meta = await reader.inspect();
+    if (options.json) {
+      process.stdout.write(
+        JSON.stringify(
+          {
+            file: inputPath === "-" ? "stdin" : inputPath,
+            format: "XLSX",
+            sizeBytes: fileSize,
+            sizeFormatted: fileSize !== undefined ? formatBytes(fileSize) : undefined,
+            sheetsCount: meta.sheets?.length ?? 0,
+            sheets: meta.sheets,
+          },
+          null,
+          2
+        ) + "\n"
+      );
+      logMemoryDebug();
+      return;
+    }
+
+    process.stdout.write(`File: ${inputPath === "-" ? "stdin" : inputPath}\n`);
+    process.stdout.write(`Format: XLSX\n`);
+    if (fileSize !== undefined) {
+      process.stdout.write(`Size: ${formatBytes(fileSize)}\n`);
+    }
+
+    if (meta.sheets && meta.sheets.length > 0) {
+      process.stdout.write(`Sheets: ${meta.sheets.length}\n\n`);
+      const sheetRows = meta.sheets.map((s) => {
+        const previewCols = (s.columns || []).slice(0, 5).join(", ") + (s.columns && s.columns.length > 5 ? ", ..." : "");
+        return [
+          s.name,
+          formatNumber(s.rowCount),
+          s.columnCount !== undefined ? String(s.columnCount) : "-",
+          previewCols,
+        ];
+      });
+
+      process.stdout.write(
+        formatTable(
+          ["SHEET", "ROWS", "COLUMNS", "PREVIEW COLUMNS"],
+          sheetRows,
+          ["left", "right", "right", "left"]
+        ) + "\n\nTip: Run with --sheet <name> to inspect a specific sheet in full detail.\n"
+      );
+    }
+    logMemoryDebug();
+    return;
+  }
+
+  // General inspection across stream
+  const schemaAgg = new SchemaInferenceAggregator();
+  const statsAgg = new DatasetStatsAggregator();
+
+  const pipeline = createPipeline(reader);
+  pipeline.onProgress((info) => progress.update(info));
+
+  for await (const row of pipeline.rows()) {
+    schemaAgg.add(row);
+    statsAgg.add(row);
+  }
+
+  progress.done();
+
+  const schemaRes = schemaAgg.result();
+  const statsRes = statsAgg.result();
+
+  const totalRows = schemaRes.totalRowsScanned;
+  const columnCount = schemaRes.columns.length;
+
+  const columnDetails = schemaRes.columns.map((col) => {
+    const nullPct =
+      col.sampleCount > 0 ? (col.nullCount / col.sampleCount) * 100 : 0;
+    const colStats = statsRes.columns[col.name];
+    const distinctCount =
+      colStats?.numeric?.approxDistinct ?? colStats?.string?.approxDistinct ?? 0;
+    const uniquePct =
+      totalRows > 0 ? Math.min(100, (distinctCount / totalRows) * 100) : 0;
+
+    return {
+      name: col.name,
+      type: col.type,
+      nullPct,
+      uniquePct,
+      semantic: col.semantic,
+    };
+  });
+
+  if (options.json) {
+    const resultJson = {
+      file: inputPath === "-" ? "stdin" : inputPath,
+      format: format.toUpperCase(),
+      sheet: options.sheet,
+      sizeBytes: fileSize,
+      sizeFormatted: fileSize !== undefined ? formatBytes(fileSize) : undefined,
+      rows: totalRows,
+      columnsCount: columnCount,
+      columns: columnDetails.map((c) => ({
+        name: c.name,
+        type: c.type,
+        nullPercentage: Math.round(c.nullPct * 10) / 10,
+        approxUniquePercentage: Math.round(c.uniquePct * 10) / 10,
+        semantic: c.semantic,
+      })),
+    };
+    process.stdout.write(JSON.stringify(resultJson, null, 2) + "\n");
+  } else {
+    process.stdout.write(`File: ${inputPath === "-" ? "stdin" : inputPath}\n`);
+    process.stdout.write(`Format: ${format.toUpperCase()}`);
+    if (options.sheet) {
+      process.stdout.write(` (Sheet: ${options.sheet})`);
+    }
+    process.stdout.write("\n");
+
+    if (fileSize !== undefined) {
+      process.stdout.write(`Size: ${formatBytes(fileSize)}\n`);
+    }
+    process.stdout.write(`Rows: ${formatNumber(totalRows)}\n`);
+    process.stdout.write(`Columns: ${columnCount}\n\n`);
+
+    const tableRows = columnDetails.map((c) => [
+      c.name,
+      c.type,
+      `${formatDecimal(c.nullPct, 1)}%`,
+      `${formatDecimal(c.uniquePct, 1)}%`,
+    ]);
+
+    process.stdout.write(
+      formatTable(
+        ["COLUMN", "TYPE", "NULL", "UNIQUE*"],
+        tableRows,
+        ["left", "left", "right", "right"]
+      ) + "\n\n* approximate\n"
+    );
+  }
+
+  logMemoryDebug();
+}

package/src/cli/commands/map.ts

@@ -0,0 +1,66 @@
+import { InvalidArgumentError } from "../../core/errors.js";
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { mapRows, parseMapSpecs } from "../../transforms/map.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface MapCommandOptions {
+  from?: string;
+  to?: string;
+  sheet?: string;
+  delimiter?: string;
+  batchSize?: string | number;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function mapCommand(
+  inputPath = "-",
+  specs: string[],
+  options: MapCommandOptions = {}
+): Promise<void> {
+  if (!specs || specs.length === 0) {
+    throw new InvalidArgumentError(
+      "At least one map specification is required (e.g. rowpipe map users.csv 'full_name=concat(first, \" \", last)')"
+    );
+  }
+
+  const effectiveBatchSize = Number(options.batchSize) || 1000;
+  const progress = new ProgressReporter(options);
+  const parsedSpecs = parseMapSpecs(specs);
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferReaderFormat(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    delimiter: options.delimiter,
+    batchSize: effectiveBatchSize,
+    filePath: inputPath,
+  });
+
+  const writer = createWriter(process.stdout, {
+    format: toFormat,
+    delimiter: options.delimiter,
+  });
+
+  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+  pipeline.pipe(mapRows(parsedSpecs));
+  pipeline.onProgress((info) => progress.update(info));
+
+  await pipeline.to(writer);
+  progress.done();
+  logMemoryDebug();
+}

package/src/cli/commands/reduce.ts

@@ -0,0 +1,74 @@
+import { InvalidArgumentError } from "../../core/errors.js";
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { parseReduceSpecs, reduceRows } from "../../analytics/reduce.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface ReduceCommandOptions {
+  by?: string;
+  from?: string;
+  to?: string;
+  sheet?: string;
+  delimiter?: string;
+  batchSize?: string | number;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function reduceCommand(
+  inputPath = "-",
+  specs: string[],
+  options: ReduceCommandOptions = {}
+): Promise<void> {
+  if (!specs || specs.length === 0) {
+    throw new InvalidArgumentError(
+      "At least one aggregation specification is required (e.g. rowpipe reduce sales.csv 'total=sum(revenue)' --by country)"
+    );
+  }
+
+  const effectiveBatchSize = Number(options.batchSize) || 1000;
+  const progress = new ProgressReporter(options);
+  const parsedSpecs = parseReduceSpecs(specs);
+
+  const byCols = options.by
+    ? options.by
+        .split(",")
+        .map((c) => c.trim())
+        .filter(Boolean)
+    : undefined;
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferReaderFormat(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    delimiter: options.delimiter,
+    batchSize: effectiveBatchSize,
+    filePath: inputPath,
+  });
+
+  const writer = createWriter(process.stdout, {
+    format: toFormat,
+    delimiter: options.delimiter,
+  });
+
+  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+  pipeline.pipe(reduceRows({ by: byCols, aggregations: parsedSpecs }));
+  pipeline.onProgress((info) => progress.update(info));
+
+  await pipeline.to(writer);
+  progress.done();
+  logMemoryDebug();
+}

package/src/cli/commands/rename.ts

@@ -0,0 +1,59 @@
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { parseRenameSpecs, renameColumns } from "../../transforms/rename.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface RenameCommandOptions {
+  from?: string;
+  to?: string;
+  sheet?: string;
+  delimiter?: string;
+  batchSize?: string | number;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function renameCommand(
+  inputPath = "-",
+  specs: string[],
+  options: RenameCommandOptions = {}
+): Promise<void> {
+  const effectiveBatchSize = Number(options.batchSize) || 1000;
+  const progress = new ProgressReporter(options);
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferReaderFormat(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+  const mapping = parseRenameSpecs(specs);
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    delimiter: options.delimiter,
+    batchSize: effectiveBatchSize,
+    filePath: inputPath,
+  });
+
+  const writer = createWriter(process.stdout, {
+    format: toFormat,
+    delimiter: options.delimiter,
+  });
+
+  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+  pipeline.pipe(renameColumns(mapping));
+  pipeline.onProgress((info) => progress.update(info));
+
+  await pipeline.to(writer);
+  progress.done();
+  logMemoryDebug();
+}

package/src/cli/commands/sample.ts

@@ -0,0 +1,61 @@
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { sampleRows } from "../../transforms/sample.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface SampleCommandOptions {
+  rows?: string | number;
+  seed?: string | number;
+  from?: string;
+  to?: string;
+  sheet?: string;
+  delimiter?: string;
+  batchSize?: string | number;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function sampleCommand(
+  inputPath = "-",
+  options: SampleCommandOptions = {}
+): Promise<void> {
+  const sampleCount = options.rows !== undefined ? Number(options.rows) : 1000;
+  const seed = options.seed !== undefined ? Number(options.seed) : undefined;
+  const effectiveBatchSize = Number(options.batchSize) || 1000;
+  const progress = new ProgressReporter(options);
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferReaderFormat(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    delimiter: options.delimiter,
+    batchSize: effectiveBatchSize,
+    filePath: inputPath,
+  });
+
+  const writer = createWriter(process.stdout, {
+    format: toFormat,
+    delimiter: options.delimiter,
+  });
+
+  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+  pipeline.pipe(sampleRows({ rows: sampleCount, seed }));
+  pipeline.onProgress((info) => progress.update(info));
+
+  await pipeline.to(writer);
+  progress.done();
+  logMemoryDebug();
+}

package/src/cli/commands/schema.ts

@@ -0,0 +1,86 @@
+import { SchemaInferenceAggregator } from "../../analytics/schema-inference.js";
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath } from "../../readers/index.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { formatTable, logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface SchemaCommandOptions {
+  from?: string;
+  sheet?: string;
+  path?: string;
+  delimiter?: string;
+  sample?: string | number;
+  full?: boolean;
+  json?: boolean;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function schemaCommand(
+  inputPath = "-",
+  options: SchemaCommandOptions = {}
+): Promise<void> {
+  const progress = new ProgressReporter(options);
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferFormatFromPath(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const sampleSize = options.full
+    ? Number.POSITIVE_INFINITY
+    : options.sample !== undefined
+      ? Number(options.sample)
+      : 10000;
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    path: options.path,
+    delimiter: options.delimiter,
+    filePath: inputPath,
+  });
+
+  const schemaAgg = new SchemaInferenceAggregator({ sample: sampleSize });
+  const pipeline = createPipeline(reader);
+  pipeline.onProgress((info) => progress.update(info));
+
+  for await (const row of pipeline.rows()) {
+    schemaAgg.add(row);
+  }
+
+  progress.done();
+  const res = schemaAgg.result();
+
+  if (options.json) {
+    process.stdout.write(JSON.stringify(res, null, 2) + "\n");
+  } else {
+    const tableRows = res.columns.map((col) => {
+      let typeDisplay = col.type as string;
+      if (col.semantic) {
+        typeDisplay += ` (${col.semantic})`;
+      }
+      return [
+        col.name,
+        typeDisplay,
+        col.nullable ? "true" : "false",
+        `${col.confidence}%`,
+      ];
+    });
+
+    process.stdout.write(
+      formatTable(
+        ["COLUMN", "TYPE", "NULLABLE", "CONFIDENCE"],
+        tableRows,
+        ["left", "left", "left", "right"]
+      ) + "\n"
+    );
+  }
+
+  logMemoryDebug();
+}

package/src/cli/commands/select.ts

@@ -0,0 +1,59 @@
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { selectColumns } from "../../transforms/select.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+
+export interface SelectCommandOptions {
+  from?: string;
+  to?: string;
+  sheet?: string;
+  delimiter?: string;
+  batchSize?: string | number;
+  quiet?: boolean;
+  noProgress?: boolean;
+}
+
+export async function selectCommand(
+  inputPath = "-",
+  columnsArg: string,
+  options: SelectCommandOptions = {}
+): Promise<void> {
+  const effectiveBatchSize = Number(options.batchSize) || 1000;
+  const progress = new ProgressReporter(options);
+
+  let fromFormat = options.from?.toLowerCase();
+  if (!fromFormat && inputPath !== "-") {
+    fromFormat = inferReaderFormat(inputPath) ?? undefined;
+  }
+  if (!fromFormat) {
+    fromFormat = "csv";
+  }
+
+  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+  const columns = columnsArg.split(",").map((c) => c.trim());
+
+  const inputStream = openReadableStream(inputPath);
+  const reader = createReader(inputStream, {
+    format: fromFormat,
+    sheet: options.sheet,
+    delimiter: options.delimiter,
+    batchSize: effectiveBatchSize,
+    filePath: inputPath,
+  });
+
+  const writer = createWriter(process.stdout, {
+    format: toFormat,
+    delimiter: options.delimiter,
+  });
+
+  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+  pipeline.pipe(selectColumns(columns));
+  pipeline.onProgress((info) => progress.update(info));
+
+  await pipeline.to(writer);
+  progress.done();
+  logMemoryDebug();
+}