rowpipe 1.0.0 → 1.2.0

package/src/cli/commands/stats.ts

@@ -1,100 +0,0 @@
-import { DatasetStatsAggregator } from "../../analytics/stats.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath } from "../../readers/index.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { formatDecimal, formatNumber, logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface StatsCommandOptions {
-  column?: string;
-  fast?: boolean;
-  exact?: boolean;
-  json?: boolean;
-  from?: string;
-  sheet?: string;
-  path?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function statsCommand(
-  inputPath = "-",
-  options: StatsCommandOptions = {}
-): Promise<void> {
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferFormatFromPath(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    batchSize: Number(options.batchSize) || 2000,
-    filePath: inputPath,
-  });
-
-  const statsAgg = new DatasetStatsAggregator({ column: options.column });
-  const pipeline = createPipeline(reader);
-  pipeline.onProgress((info) => progress.update(info));
-
-  for await (const row of pipeline.rows()) {
-    statsAgg.add(row);
-  }
-
-  progress.done();
-  const res = statsAgg.result();
-
-  if (options.json) {
-    process.stdout.write(JSON.stringify(res, null, 2) + "\n");
-  } else {
-    process.stdout.write(`Total Rows: ${formatNumber(res.totalRows)}\n\n`);
-
-    for (const [colName, colStat] of Object.entries(res.columns)) {
-      process.stdout.write(`${colName} (${colStat.type})\n`);
-      process.stdout.write("----------------------------------------\n");
-
-      if (colStat.type === "numeric" && colStat.numeric) {
-        const n = colStat.numeric;
-        process.stdout.write(`  count        ${formatNumber(n.count)}\n`);
-        process.stdout.write(`  null         ${formatNumber(n.nullCount)}\n`);
-        process.stdout.write(`  min          ${formatDecimal(n.min)}\n`);
-        process.stdout.write(`  max          ${formatDecimal(n.max)}\n`);
-        process.stdout.write(`  sum          ${formatDecimal(n.sum)}\n`);
-        process.stdout.write(`  mean         ${formatDecimal(n.mean)}\n`);
-        process.stdout.write(`  stddev       ${formatDecimal(n.stddev)}\n`);
-        process.stdout.write(`  variance     ${formatDecimal(n.variance)}\n`);
-        process.stdout.write(`  distinct*    ~${formatNumber(n.approxDistinct)}\n\n`);
-      } else if (colStat.type === "string" && colStat.string) {
-        const s = colStat.string;
-        process.stdout.write(`  count        ${formatNumber(s.count)}\n`);
-        process.stdout.write(`  null         ${formatNumber(s.nullCount)}\n`);
-        process.stdout.write(`  empty        ${formatNumber(s.emptyCount)}\n`);
-        process.stdout.write(`  min length   ${formatNumber(s.minLength)}\n`);
-        process.stdout.write(`  max length   ${formatNumber(s.maxLength)}\n`);
-        process.stdout.write(`  avg length   ${formatDecimal(s.avgLength, 1)}\n`);
-        process.stdout.write(`  distinct*    ~${formatNumber(s.approxDistinct)}\n`);
-        if (s.topValues.length > 0) {
-          process.stdout.write(`  top values:\n`);
-          for (const tv of s.topValues) {
-            process.stdout.write(`    - "${tv.value}": ${formatNumber(tv.count)}\n`);
-          }
-        }
-        process.stdout.write("\n");
-      }
-    }
-
-    process.stdout.write("* approximate\n");
-  }
-
-  logMemoryDebug();
-}

package/src/cli/commands/validate.ts

@@ -1,123 +0,0 @@
-import { readFile } from "node:fs/promises";
-import {
-  type SchemaValidatorAggregator,
-  type ValidationSchemaDefinition,
-  SchemaValidatorAggregator as Validator,
-} from "../../analytics/validator.js";
-import { InvalidArgumentError, ValidationError } from "../../core/errors.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath } from "../../readers/index.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { formatNumber, logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface ValidateCommandOptions {
-  schema: string;
-  from?: string;
-  sheet?: string;
-  path?: string;
-  delimiter?: string;
-  json?: boolean;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function validateCommand(
-  inputPath = "-",
-  options: ValidateCommandOptions
-): Promise<void> {
-  if (!options.schema) {
-    throw new InvalidArgumentError("Missing required option: --schema <path_to_schema.json>");
-  }
-
-  const progress = new ProgressReporter(options);
-
-  // Read and parse schema file
-  let schemaContent: string;
-  try {
-    schemaContent = await readFile(options.schema, "utf8");
-  } catch (err) {
-    throw new InvalidArgumentError(
-      `Failed to read schema file at "${options.schema}": ${(err as Error).message}`
-    );
-  }
-
-  let schemaDef: ValidationSchemaDefinition;
-  try {
-    schemaDef = JSON.parse(schemaContent);
-  } catch (err) {
-    throw new InvalidArgumentError(
-      `Invalid JSON in schema file "${options.schema}": ${(err as Error).message}`
-    );
-  }
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferFormatFromPath(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    filePath: inputPath,
-  });
-
-  const validator = new Validator(schemaDef);
-  const pipeline = createPipeline(reader);
-  pipeline.onProgress((info) => progress.update(info));
-
-  for await (const row of pipeline.rows()) {
-    validator.add(row);
-  }
-
-  progress.done();
-  const report = validator.result();
-
-  if (options.json) {
-    process.stdout.write(JSON.stringify(report, null, 2) + "\n");
-  } else {
-    process.stdout.write(`${formatNumber(report.totalRows)} rows scanned\n`);
-    process.stdout.write(`${formatNumber(report.validRows)} valid\n`);
-    process.stdout.write(`${formatNumber(report.invalidRows)} invalid\n`);
-
-    if (report.violations.length > 0) {
-      process.stdout.write("\nErrors\n\n");
-      const groupedByCol = new Map<string, typeof report.violations>();
-      for (const v of report.violations) {
-        if (!groupedByCol.has(v.column)) groupedByCol.set(v.column, []);
-        groupedByCol.get(v.column)!.push(v);
-      }
-
-      for (const [col, violations] of groupedByCol.entries()) {
-        process.stdout.write(`${col}\n`);
-        for (const v of violations) {
-          process.stdout.write(`  ${v.rule}\n`);
-          process.stdout.write(`  ${formatNumber(v.violationsCount)} violations\n\n`);
-        }
-      }
-    }
-  }
-
-  logMemoryDebug();
-
-  if (!report.isValid) {
-    throw new ValidationError(
-      `Schema validation failed: ${report.invalidRows} invalid rows found`,
-      report.violations.map((v) => ({
-        column: v.column,
-        expected: v.rule,
-        count: v.violationsCount,
-        examples: v.sampleInvalidValues,
-      })),
-      report.totalRows,
-      report.validRows,
-      report.invalidRows
-    );
-  }
-}

package/src/cli/index.ts

@@ -1,285 +0,0 @@
-#!/usr/bin/env node
-
-import { Command } from "commander";
-import { RowpipeError } from "../core/errors.js";
-import { castCommand } from "./commands/cast.js";
-import { convertCommand } from "./commands/convert.js";
-import { filterCommand } from "./commands/filter.js";
-import { inspectCommand } from "./commands/inspect.js";
-import { mapCommand } from "./commands/map.js";
-import { reduceCommand } from "./commands/reduce.js";
-import { renameCommand } from "./commands/rename.js";
-import { sampleCommand } from "./commands/sample.js";
-import { schemaCommand } from "./commands/schema.js";
-import { selectCommand } from "./commands/select.js";
-import { statsCommand } from "./commands/stats.js";
-import { validateCommand } from "./commands/validate.js";
-
-// Handle broken pipe gracefully when piping to head/less
-process.stdout.on("error", (err: unknown) => {
-  if ((err as NodeJS.ErrnoException).code === "EPIPE") {
-    process.exit(0);
-  }
-});
-
-const program = new Command();
-
-program
-  .name("rowpipe")
-  .description("Stream-first tabular data toolkit for CSV, JSON, JSONL, and XLSX")
-  .version("1.0.0");
-
-// Global options
-program
-  .option("--batch-size <number>", "Processing batch size in rows", "1000")
-  .option("--quiet", "Suppress non-data output and progress")
-  .option("--no-progress", "Disable real-time progress bar");
-
-// 1. inspect
-program
-  .command("inspect [input]")
-  .description("Inspect format, row count, columns, and sheet summary")
-  .option("--sheet <sheet>", "Target worksheet name or index for XLSX")
-  .option("--path <path>", "Nested object path for JSON (e.g. data.results)")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output metadata as machine-readable JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await inspectCommand(input, opts);
-  });
-
-// 2. convert
-program
-  .command("convert [input] [output]")
-  .description("Stream convert tabular datasets across formats")
-  .option("--from <format>", "Input format (csv, json, jsonl, xlsx)")
-  .option("--to <format>", "Output format (csv, json, jsonl, xlsx)")
-  .option("--sheet <sheet>", "Worksheet name or index for XLSX")
-  .option("--all-sheets", "Export all worksheets in the workbook to individual files")
-  .option("--out-dir <dir>", "Output directory for --all-sheets export")
-  .option("--path <path>", "Nested path for JSON array")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--no-header", "Disable writing or reading headers in CSV")
-  .action(async (input = "-", output, cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await convertCommand(input, output, opts);
-  });
-
-// 3. schema
-program
-  .command("schema [input]")
-  .description("Infer column types, nullability, and semantic annotations")
-  .option("--sample <rows>", "Number of rows to sample for inference", "10000")
-  .option("--full", "Scan full stream for exact schema inference")
-  .option("--from <format>", "Input format")
-  .option("--sheet <sheet>", "Worksheet name or index for XLSX")
-  .option("--path <path>", "Nested path for JSON")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output schema as machine-readable JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await schemaCommand(input, opts);
-  });
-
-// 4. stats
-program
-  .command("stats [input]")
-  .description("Compute streaming statistics (Welford numeric stats and HLL distinct counts)")
-  .option("--column <name>", "Compute statistics only for specific column")
-  .option("--fast", "Use approximate algorithms for distinct counts")
-  .option("--exact", "Use exact counts where available")
-  .option("--from <format>", "Input format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output stats as machine-readable JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await statsCommand(input, opts);
-  });
-
-// 5. select
-program
-  .command("select [input] [columns]")
-  .description("Select a subset of columns in stream")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (inputOrCols, maybeCols, cmdOptions) => {
-    let input = "-";
-    let columns = "";
-
-    if (maybeCols !== undefined) {
-      input = inputOrCols || "-";
-      columns = maybeCols;
-    } else {
-      columns = inputOrCols || "";
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await selectCommand(input, columns, opts);
-  });
-
-// 6. rename
-program
-  .command("rename <args...>")
-  .description("Rename columns in stream (e.g. rowpipe rename users.csv old=new)")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes("=")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await renameCommand(input, specs, opts);
-  });
-
-// 7. cast
-program
-  .command("cast <args...>")
-  .description("Cast column types in stream (e.g. rowpipe cast users.csv age:number active:boolean)")
-  .option("--on-error <behavior>", "Error behavior: fail, null, keep, skip-row", "null")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes(":")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await castCommand(input, specs, opts);
-  });
-
-// 8. filter
-program
-  .command("filter [input] [expression]")
-  .description("Filter rows using safe expression engine (e.g. rowpipe filter users.csv 'age > 30')")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (inputOrExpr, maybeExpr, cmdOptions) => {
-    let input = "-";
-    let expression = "";
-
-    if (maybeExpr !== undefined) {
-      input = inputOrExpr || "-";
-      expression = maybeExpr;
-    } else {
-      expression = inputOrExpr || "";
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await filterCommand(input, expression, opts);
-  });
-
-// 9. map
-program
-  .command("map <args...>")
-  .description("Transform and derive columns per row (e.g. rowpipe map sales.csv 'profit=revenue-cost')")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes("=")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await mapCommand(input, specs, opts);
-  });
-
-// 10. reduce
-program
-  .command("reduce <args...>")
-  .description("Aggregate and group dataset (e.g. rowpipe reduce sales.csv 'total=sum(revenue)' --by country)")
-  .option("--by <columns>", "Group by columns (comma-separated, e.g. country,category)")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes("=")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await reduceCommand(input, specs, opts);
-  });
-
-// 11. sample
-program
-  .command("sample [input]")
-  .description("Reservoir sample rows with bounded memory")
-  .option("--rows <number>", "Number of rows to sample", "1000")
-  .option("--seed <number>", "Deterministic random seed")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await sampleCommand(input, opts);
-  });
-
-// 10. validate
-program
-  .command("validate [input]")
-  .description("Validate stream against schema definition")
-  .requiredOption("--schema <file>", "Path to JSON schema file")
-  .option("--from <format>", "Input format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output validation report as JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await validateCommand(input, opts);
-  });
-
-async function main() {
-  try {
-    await program.parseAsync(process.argv);
-  } catch (err: unknown) {
-    if (err instanceof RowpipeError) {
-      process.stderr.write(`\nError: ${err.message}\n`);
-      process.exit(err.exitCode);
-    }
-    process.stderr.write(`\nUnexpected error: ${(err as Error).message}\n`);
-    if (process.env["DEBUG"]) {
-      process.stderr.write(`${(err as Error).stack}\n`);
-    }
-    process.exit(1);
-  }
-}
-
-main();

package/src/core/batch.ts

@@ -1,81 +0,0 @@
-import type { DataBatch, DataStream, Row } from "./types.js";
-
-/**
- * Packs an AsyncIterable of individual rows into DataBatch chunks with a given batchSize.
- */
-export async function* rowsToBatches(
-  rows: AsyncIterable<Row>,
-  batchSize = 1000
-): AsyncGenerator<DataBatch> {
-  const effectiveBatchSize = Math.max(1, batchSize);
-  let currentRows: Row[] = [];
-  let totalOffset = 0;
-
-  for await (const row of rows) {
-    currentRows.push(row);
-    if (currentRows.length >= effectiveBatchSize) {
-      yield {
-        rows: currentRows,
-        offset: totalOffset,
-      };
-      totalOffset += currentRows.length;
-      currentRows = [];
-    }
-  }
-
-  if (currentRows.length > 0) {
-    yield {
-      rows: currentRows,
-      offset: totalOffset,
-    };
-  }
-}
-
-/**
- * Unpacks DataBatches into individual rows.
- */
-export async function* batchesToRows(stream: DataStream): AsyncGenerator<Row> {
-  for await (const batch of stream) {
-    for (const row of batch.rows) {
-      yield row;
-    }
-  }
-}
-
-/**
- * Applies a mapping function to each row in batches, maintaining batch structure
- * and filtering out null/undefined results.
- */
-export async function* batchMap(
-  stream: DataStream,
-  fn: (row: Row, globalIndex: number) => Row | null | undefined,
-  targetBatchSize = 1000
-): AsyncGenerator<DataBatch> {
-  let pendingRows: Row[] = [];
-  let currentOffset = 0;
-  let globalIndex = 0;
-
-  for await (const batch of stream) {
-    for (const row of batch.rows) {
-      const transformed = fn(row, globalIndex++);
-      if (transformed !== null && transformed !== undefined) {
-        pendingRows.push(transformed);
-        if (pendingRows.length >= targetBatchSize) {
-          yield {
-            rows: pendingRows,
-            offset: currentOffset,
-          };
-          currentOffset += pendingRows.length;
-          pendingRows = [];
-        }
-      }
-    }
-  }
-
-  if (pendingRows.length > 0) {
-    yield {
-      rows: pendingRows,
-      offset: currentOffset,
-    };
-  }
-}

package/src/core/errors.ts

@@ -1,81 +0,0 @@
-export class RowpipeError extends Error {
-  public readonly exitCode: number;
-
-  constructor(message: string, exitCode = 1) {
-    super(message);
-    this.name = "RowpipeError";
-    this.exitCode = exitCode;
-    Object.setPrototypeOf(this, new.target.prototype);
-  }
-}
-
-export class InvalidArgumentError extends RowpipeError {
-  constructor(message: string) {
-    super(message, 2);
-    this.name = "InvalidArgumentError";
-  }
-}
-
-export interface ParseErrorContext {
-  file?: string;
-  sheet?: string;
-  row?: number;
-  column?: string | number;
-  byteOffset?: number;
-  line?: number;
-  cell?: string;
-}
-
-export class ParseError extends RowpipeError {
-  public readonly context: ParseErrorContext;
-
-  constructor(message: string, context: ParseErrorContext = {}) {
-    let detailedMsg = `Parse error: ${message}`;
-    const parts: string[] = [];
-
-    if (context.file) parts.push(`File: ${context.file}`);
-    if (context.sheet) parts.push(`Sheet: ${context.sheet}`);
-    if (context.row !== undefined) parts.push(`Row: ${context.row}`);
-    if (context.column !== undefined) parts.push(`Column: ${context.column}`);
-    if (context.cell) parts.push(`Cell: ${context.cell}`);
-    if (context.byteOffset !== undefined) parts.push(`Byte offset: ${context.byteOffset}`);
-
-    if (parts.length > 0) {
-      detailedMsg += `\n  Context: ${parts.join(", ")}`;
-    }
-
-    super(detailedMsg, 3);
-    this.name = "ParseError";
-    this.context = context;
-  }
-}
-
-export interface ValidationViolation {
-  column: string;
-  expected: string;
-  actual?: string;
-  count: number;
-  examples?: unknown[];
-}
-
-export class ValidationError extends RowpipeError {
-  public readonly violations: ValidationViolation[];
-  public readonly totalRows: number;
-  public readonly validRows: number;
-  public readonly invalidRows: number;
-
-  constructor(
-    message: string,
-    violations: ValidationViolation[] = [],
-    totalRows = 0,
-    validRows = 0,
-    invalidRows = 0
-  ) {
-    super(message, 4);
-    this.name = "ValidationError";
-    this.violations = violations;
-    this.totalRows = totalRows;
-    this.validRows = validRows;
-    this.invalidRows = invalidRows;
-  }
-}