rowpipe 1.0.0 → 1.1.0

package/src/cli/commands/convert.ts

@@ -1,128 +0,0 @@
-import { mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { XLSXReader } from "../../readers/xlsx.js";
-import { createWriter, inferFormatFromPath as inferWriterFormat } from "../../writers/index.js";
-import { openReadableStream, openWritableStream } from "../../utils/compression.js";
-import { formatNumber, logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface ConvertCommandOptions {
-  from?: string;
-  to?: string;
-  sheet?: string;
-  allSheets?: boolean;
-  outDir?: string;
-  path?: string;
-  delimiter?: string;
-  header?: boolean;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function convertCommand(
-  inputPath = "-",
-  outputPath?: string,
-  options: ConvertCommandOptions = {}
-): Promise<void> {
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-
-  // Determine input format
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  // Handle multi-sheet export if --all-sheets is specified
-  if (options.allSheets && fromFormat === "xlsx" && inputPath !== "-") {
-    const outDir = options.outDir || outputPath || "./";
-    const toFormat = (options.to?.toLowerCase() || "csv").replace(/^\./, "");
-
-    await mkdir(outDir, { recursive: true });
-
-    const xlsxReader = new XLSXReader(inputPath);
-    const sheets = await xlsxReader.getParsedSheets();
-
-    if (sheets.length === 0) {
-      process.stderr.write("No sheets found in workbook.\n");
-      return;
-    }
-
-    process.stderr.write(`Exporting ${sheets.length} sheets to "${outDir}"...\n`);
-
-    for (const sheet of sheets) {
-      const sheetName = sheet.sheet;
-      const targetFileName = `${sheetName}.${toFormat}`;
-      const targetFilePath = join(outDir, targetFileName);
-
-      const sheetReader = createReader(inputPath, {
-        format: "xlsx",
-        sheet: sheetName,
-        batchSize: effectiveBatchSize,
-        filePath: inputPath,
-      });
-
-      const sheetWriter = createWriter(targetFilePath, {
-        format: toFormat,
-        delimiter: options.delimiter,
-        header: options.header,
-      });
-
-      const pipeline = createPipeline(sheetReader, { batchSize: effectiveBatchSize });
-      await pipeline.to(sheetWriter);
-
-      const rowCount = Math.max(0, sheet.data.length - 1);
-      process.stderr.write(
-        `  ✓ Sheet "${sheetName}" -> ${targetFilePath} (${formatNumber(rowCount)} rows)\n`
-      );
-    }
-
-    process.stderr.write("Done.\n");
-    logMemoryDebug();
-    return;
-  }
-
-  // Determine output format
-  let toFormat = options.to?.toLowerCase();
-  if (!toFormat && outputPath && outputPath !== "-") {
-    toFormat = inferWriterFormat(outputPath) ?? undefined;
-  }
-  if (!toFormat) {
-    toFormat = "csv";
-  }
-
-  const progress = new ProgressReporter(options);
-  const effectiveOutput = outputPath || "-";
-
-  const inputStream = openReadableStream(inputPath);
-  const outputStream = openWritableStream(effectiveOutput);
-
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    header: options.header,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(outputStream, {
-    format: toFormat,
-    delimiter: options.delimiter,
-    header: options.header,
-    sheet: options.sheet,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}

package/src/cli/commands/filter.ts

@@ -1,58 +0,0 @@
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { createWriter } from "../../writers/index.js";
-import { filterRows } from "../../transforms/filter.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface FilterCommandOptions {
-  from?: string;
-  to?: string;
-  sheet?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function filterCommand(
-  inputPath = "-",
-  expression: string,
-  options: FilterCommandOptions = {}
-): Promise<void> {
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    delimiter: options.delimiter,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(process.stdout, {
-    format: toFormat,
-    delimiter: options.delimiter,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.pipe(filterRows(expression));
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}

package/src/cli/commands/inspect.ts

@@ -1,193 +0,0 @@
-import { stat } from "node:fs/promises";
-import { DatasetStatsAggregator } from "../../analytics/stats.js";
-import { SchemaInferenceAggregator } from "../../analytics/schema-inference.js";
-import { InvalidArgumentError } from "../../core/errors.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath } from "../../readers/index.js";
-import { openReadableStream } from "../../utils/compression.js";
-import {
-  formatBytes,
-  formatDecimal,
-  formatNumber,
-  formatTable,
-  logMemoryDebug,
-} from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface InspectCommandOptions {
-  sheet?: string;
-  path?: string;
-  delimiter?: string;
-  json?: boolean;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function inspectCommand(
-  inputPath = "-",
-  options: InspectCommandOptions = {}
-): Promise<void> {
-  const progress = new ProgressReporter(options);
-
-  let fileSize: number | undefined;
-  if (inputPath !== "-") {
-    try {
-      const fileStat = await stat(inputPath);
-      fileSize = fileStat.size;
-    } catch {
-      // Ignore if cannot stat
-    }
-  }
-
-  const format =
-    inputPath !== "-" ? inferFormatFromPath(inputPath) || "CSV" : "CSV";
-
-  const stream = openReadableStream(inputPath);
-  const reader = createReader(stream, {
-    format,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    filePath: inputPath,
-  });
-
-  // If XLSX without a specific --sheet requested, show rich workbook & sheet breakdown
-  if (format === "xlsx" && !options.sheet && reader.inspect) {
-    const meta = await reader.inspect();
-    if (options.json) {
-      process.stdout.write(
-        JSON.stringify(
-          {
-            file: inputPath === "-" ? "stdin" : inputPath,
-            format: "XLSX",
-            sizeBytes: fileSize,
-            sizeFormatted: fileSize !== undefined ? formatBytes(fileSize) : undefined,
-            sheetsCount: meta.sheets?.length ?? 0,
-            sheets: meta.sheets,
-          },
-          null,
-          2
-        ) + "\n"
-      );
-      logMemoryDebug();
-      return;
-    }
-
-    process.stdout.write(`File: ${inputPath === "-" ? "stdin" : inputPath}\n`);
-    process.stdout.write(`Format: XLSX\n`);
-    if (fileSize !== undefined) {
-      process.stdout.write(`Size: ${formatBytes(fileSize)}\n`);
-    }
-
-    if (meta.sheets && meta.sheets.length > 0) {
-      process.stdout.write(`Sheets: ${meta.sheets.length}\n\n`);
-      const sheetRows = meta.sheets.map((s) => {
-        const previewCols = (s.columns || []).slice(0, 5).join(", ") + (s.columns && s.columns.length > 5 ? ", ..." : "");
-        return [
-          s.name,
-          formatNumber(s.rowCount),
-          s.columnCount !== undefined ? String(s.columnCount) : "-",
-          previewCols,
-        ];
-      });
-
-      process.stdout.write(
-        formatTable(
-          ["SHEET", "ROWS", "COLUMNS", "PREVIEW COLUMNS"],
-          sheetRows,
-          ["left", "right", "right", "left"]
-        ) + "\n\nTip: Run with --sheet <name> to inspect a specific sheet in full detail.\n"
-      );
-    }
-    logMemoryDebug();
-    return;
-  }
-
-  // General inspection across stream
-  const schemaAgg = new SchemaInferenceAggregator();
-  const statsAgg = new DatasetStatsAggregator();
-
-  const pipeline = createPipeline(reader);
-  pipeline.onProgress((info) => progress.update(info));
-
-  for await (const row of pipeline.rows()) {
-    schemaAgg.add(row);
-    statsAgg.add(row);
-  }
-
-  progress.done();
-
-  const schemaRes = schemaAgg.result();
-  const statsRes = statsAgg.result();
-
-  const totalRows = schemaRes.totalRowsScanned;
-  const columnCount = schemaRes.columns.length;
-
-  const columnDetails = schemaRes.columns.map((col) => {
-    const nullPct =
-      col.sampleCount > 0 ? (col.nullCount / col.sampleCount) * 100 : 0;
-    const colStats = statsRes.columns[col.name];
-    const distinctCount =
-      colStats?.numeric?.approxDistinct ?? colStats?.string?.approxDistinct ?? 0;
-    const uniquePct =
-      totalRows > 0 ? Math.min(100, (distinctCount / totalRows) * 100) : 0;
-
-    return {
-      name: col.name,
-      type: col.type,
-      nullPct,
-      uniquePct,
-      semantic: col.semantic,
-    };
-  });
-
-  if (options.json) {
-    const resultJson = {
-      file: inputPath === "-" ? "stdin" : inputPath,
-      format: format.toUpperCase(),
-      sheet: options.sheet,
-      sizeBytes: fileSize,
-      sizeFormatted: fileSize !== undefined ? formatBytes(fileSize) : undefined,
-      rows: totalRows,
-      columnsCount: columnCount,
-      columns: columnDetails.map((c) => ({
-        name: c.name,
-        type: c.type,
-        nullPercentage: Math.round(c.nullPct * 10) / 10,
-        approxUniquePercentage: Math.round(c.uniquePct * 10) / 10,
-        semantic: c.semantic,
-      })),
-    };
-    process.stdout.write(JSON.stringify(resultJson, null, 2) + "\n");
-  } else {
-    process.stdout.write(`File: ${inputPath === "-" ? "stdin" : inputPath}\n`);
-    process.stdout.write(`Format: ${format.toUpperCase()}`);
-    if (options.sheet) {
-      process.stdout.write(` (Sheet: ${options.sheet})`);
-    }
-    process.stdout.write("\n");
-
-    if (fileSize !== undefined) {
-      process.stdout.write(`Size: ${formatBytes(fileSize)}\n`);
-    }
-    process.stdout.write(`Rows: ${formatNumber(totalRows)}\n`);
-    process.stdout.write(`Columns: ${columnCount}\n\n`);
-
-    const tableRows = columnDetails.map((c) => [
-      c.name,
-      c.type,
-      `${formatDecimal(c.nullPct, 1)}%`,
-      `${formatDecimal(c.uniquePct, 1)}%`,
-    ]);
-
-    process.stdout.write(
-      formatTable(
-        ["COLUMN", "TYPE", "NULL", "UNIQUE*"],
-        tableRows,
-        ["left", "left", "right", "right"]
-      ) + "\n\n* approximate\n"
-    );
-  }
-
-  logMemoryDebug();
-}

package/src/cli/commands/map.ts

@@ -1,66 +0,0 @@
-import { InvalidArgumentError } from "../../core/errors.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { createWriter } from "../../writers/index.js";
-import { mapRows, parseMapSpecs } from "../../transforms/map.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface MapCommandOptions {
-  from?: string;
-  to?: string;
-  sheet?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function mapCommand(
-  inputPath = "-",
-  specs: string[],
-  options: MapCommandOptions = {}
-): Promise<void> {
-  if (!specs || specs.length === 0) {
-    throw new InvalidArgumentError(
-      "At least one map specification is required (e.g. rowpipe map users.csv 'full_name=concat(first, \" \", last)')"
-    );
-  }
-
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-  const progress = new ProgressReporter(options);
-  const parsedSpecs = parseMapSpecs(specs);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    delimiter: options.delimiter,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(process.stdout, {
-    format: toFormat,
-    delimiter: options.delimiter,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.pipe(mapRows(parsedSpecs));
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}

package/src/cli/commands/reduce.ts

@@ -1,74 +0,0 @@
-import { InvalidArgumentError } from "../../core/errors.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { createWriter } from "../../writers/index.js";
-import { parseReduceSpecs, reduceRows } from "../../analytics/reduce.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface ReduceCommandOptions {
-  by?: string;
-  from?: string;
-  to?: string;
-  sheet?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function reduceCommand(
-  inputPath = "-",
-  specs: string[],
-  options: ReduceCommandOptions = {}
-): Promise<void> {
-  if (!specs || specs.length === 0) {
-    throw new InvalidArgumentError(
-      "At least one aggregation specification is required (e.g. rowpipe reduce sales.csv 'total=sum(revenue)' --by country)"
-    );
-  }
-
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-  const progress = new ProgressReporter(options);
-  const parsedSpecs = parseReduceSpecs(specs);
-
-  const byCols = options.by
-    ? options.by
-        .split(",")
-        .map((c) => c.trim())
-        .filter(Boolean)
-    : undefined;
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    delimiter: options.delimiter,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(process.stdout, {
-    format: toFormat,
-    delimiter: options.delimiter,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.pipe(reduceRows({ by: byCols, aggregations: parsedSpecs }));
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}

package/src/cli/commands/rename.ts

@@ -1,59 +0,0 @@
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { createWriter } from "../../writers/index.js";
-import { parseRenameSpecs, renameColumns } from "../../transforms/rename.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface RenameCommandOptions {
-  from?: string;
-  to?: string;
-  sheet?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function renameCommand(
-  inputPath = "-",
-  specs: string[],
-  options: RenameCommandOptions = {}
-): Promise<void> {
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
-  const mapping = parseRenameSpecs(specs);
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    delimiter: options.delimiter,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(process.stdout, {
-    format: toFormat,
-    delimiter: options.delimiter,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.pipe(renameColumns(mapping));
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}

package/src/cli/commands/sample.ts

@@ -1,61 +0,0 @@
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { createWriter } from "../../writers/index.js";
-import { sampleRows } from "../../transforms/sample.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface SampleCommandOptions {
-  rows?: string | number;
-  seed?: string | number;
-  from?: string;
-  to?: string;
-  sheet?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function sampleCommand(
-  inputPath = "-",
-  options: SampleCommandOptions = {}
-): Promise<void> {
-  const sampleCount = options.rows !== undefined ? Number(options.rows) : 1000;
-  const seed = options.seed !== undefined ? Number(options.seed) : undefined;
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    delimiter: options.delimiter,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(process.stdout, {
-    format: toFormat,
-    delimiter: options.delimiter,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.pipe(sampleRows({ rows: sampleCount, seed }));
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}