rowpipe 1.0.0 → 1.1.0

package/src/cli/commands/schema.ts

@@ -1,86 +0,0 @@
-import { SchemaInferenceAggregator } from "../../analytics/schema-inference.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath } from "../../readers/index.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { formatTable, logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface SchemaCommandOptions {
-  from?: string;
-  sheet?: string;
-  path?: string;
-  delimiter?: string;
-  sample?: string | number;
-  full?: boolean;
-  json?: boolean;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function schemaCommand(
-  inputPath = "-",
-  options: SchemaCommandOptions = {}
-): Promise<void> {
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferFormatFromPath(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const sampleSize = options.full
-    ? Number.POSITIVE_INFINITY
-    : options.sample !== undefined
-      ? Number(options.sample)
-      : 10000;
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    filePath: inputPath,
-  });
-
-  const schemaAgg = new SchemaInferenceAggregator({ sample: sampleSize });
-  const pipeline = createPipeline(reader);
-  pipeline.onProgress((info) => progress.update(info));
-
-  for await (const row of pipeline.rows()) {
-    schemaAgg.add(row);
-  }
-
-  progress.done();
-  const res = schemaAgg.result();
-
-  if (options.json) {
-    process.stdout.write(JSON.stringify(res, null, 2) + "\n");
-  } else {
-    const tableRows = res.columns.map((col) => {
-      let typeDisplay = col.type as string;
-      if (col.semantic) {
-        typeDisplay += ` (${col.semantic})`;
-      }
-      return [
-        col.name,
-        typeDisplay,
-        col.nullable ? "true" : "false",
-        `${col.confidence}%`,
-      ];
-    });
-
-    process.stdout.write(
-      formatTable(
-        ["COLUMN", "TYPE", "NULLABLE", "CONFIDENCE"],
-        tableRows,
-        ["left", "left", "left", "right"]
-      ) + "\n"
-    );
-  }
-
-  logMemoryDebug();
-}

package/src/cli/commands/select.ts

@@ -1,59 +0,0 @@
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
-import { createWriter } from "../../writers/index.js";
-import { selectColumns } from "../../transforms/select.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface SelectCommandOptions {
-  from?: string;
-  to?: string;
-  sheet?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function selectCommand(
-  inputPath = "-",
-  columnsArg: string,
-  options: SelectCommandOptions = {}
-): Promise<void> {
-  const effectiveBatchSize = Number(options.batchSize) || 1000;
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferReaderFormat(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
-  const columns = columnsArg.split(",").map((c) => c.trim());
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    delimiter: options.delimiter,
-    batchSize: effectiveBatchSize,
-    filePath: inputPath,
-  });
-
-  const writer = createWriter(process.stdout, {
-    format: toFormat,
-    delimiter: options.delimiter,
-  });
-
-  const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
-  pipeline.pipe(selectColumns(columns));
-  pipeline.onProgress((info) => progress.update(info));
-
-  await pipeline.to(writer);
-  progress.done();
-  logMemoryDebug();
-}

package/src/cli/commands/stats.ts

@@ -1,100 +0,0 @@
-import { DatasetStatsAggregator } from "../../analytics/stats.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath } from "../../readers/index.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { formatDecimal, formatNumber, logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface StatsCommandOptions {
-  column?: string;
-  fast?: boolean;
-  exact?: boolean;
-  json?: boolean;
-  from?: string;
-  sheet?: string;
-  path?: string;
-  delimiter?: string;
-  batchSize?: string | number;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function statsCommand(
-  inputPath = "-",
-  options: StatsCommandOptions = {}
-): Promise<void> {
-  const progress = new ProgressReporter(options);
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferFormatFromPath(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    batchSize: Number(options.batchSize) || 2000,
-    filePath: inputPath,
-  });
-
-  const statsAgg = new DatasetStatsAggregator({ column: options.column });
-  const pipeline = createPipeline(reader);
-  pipeline.onProgress((info) => progress.update(info));
-
-  for await (const row of pipeline.rows()) {
-    statsAgg.add(row);
-  }
-
-  progress.done();
-  const res = statsAgg.result();
-
-  if (options.json) {
-    process.stdout.write(JSON.stringify(res, null, 2) + "\n");
-  } else {
-    process.stdout.write(`Total Rows: ${formatNumber(res.totalRows)}\n\n`);
-
-    for (const [colName, colStat] of Object.entries(res.columns)) {
-      process.stdout.write(`${colName} (${colStat.type})\n`);
-      process.stdout.write("----------------------------------------\n");
-
-      if (colStat.type === "numeric" && colStat.numeric) {
-        const n = colStat.numeric;
-        process.stdout.write(`  count        ${formatNumber(n.count)}\n`);
-        process.stdout.write(`  null         ${formatNumber(n.nullCount)}\n`);
-        process.stdout.write(`  min          ${formatDecimal(n.min)}\n`);
-        process.stdout.write(`  max          ${formatDecimal(n.max)}\n`);
-        process.stdout.write(`  sum          ${formatDecimal(n.sum)}\n`);
-        process.stdout.write(`  mean         ${formatDecimal(n.mean)}\n`);
-        process.stdout.write(`  stddev       ${formatDecimal(n.stddev)}\n`);
-        process.stdout.write(`  variance     ${formatDecimal(n.variance)}\n`);
-        process.stdout.write(`  distinct*    ~${formatNumber(n.approxDistinct)}\n\n`);
-      } else if (colStat.type === "string" && colStat.string) {
-        const s = colStat.string;
-        process.stdout.write(`  count        ${formatNumber(s.count)}\n`);
-        process.stdout.write(`  null         ${formatNumber(s.nullCount)}\n`);
-        process.stdout.write(`  empty        ${formatNumber(s.emptyCount)}\n`);
-        process.stdout.write(`  min length   ${formatNumber(s.minLength)}\n`);
-        process.stdout.write(`  max length   ${formatNumber(s.maxLength)}\n`);
-        process.stdout.write(`  avg length   ${formatDecimal(s.avgLength, 1)}\n`);
-        process.stdout.write(`  distinct*    ~${formatNumber(s.approxDistinct)}\n`);
-        if (s.topValues.length > 0) {
-          process.stdout.write(`  top values:\n`);
-          for (const tv of s.topValues) {
-            process.stdout.write(`    - "${tv.value}": ${formatNumber(tv.count)}\n`);
-          }
-        }
-        process.stdout.write("\n");
-      }
-    }
-
-    process.stdout.write("* approximate\n");
-  }
-
-  logMemoryDebug();
-}

package/src/cli/commands/validate.ts

@@ -1,123 +0,0 @@
-import { readFile } from "node:fs/promises";
-import {
-  type SchemaValidatorAggregator,
-  type ValidationSchemaDefinition,
-  SchemaValidatorAggregator as Validator,
-} from "../../analytics/validator.js";
-import { InvalidArgumentError, ValidationError } from "../../core/errors.js";
-import { createPipeline } from "../../core/pipeline.js";
-import { createReader, inferFormatFromPath } from "../../readers/index.js";
-import { openReadableStream } from "../../utils/compression.js";
-import { formatNumber, logMemoryDebug } from "../../utils/formatting.js";
-import { ProgressReporter } from "../../utils/progress.js";
-
-export interface ValidateCommandOptions {
-  schema: string;
-  from?: string;
-  sheet?: string;
-  path?: string;
-  delimiter?: string;
-  json?: boolean;
-  quiet?: boolean;
-  noProgress?: boolean;
-}
-
-export async function validateCommand(
-  inputPath = "-",
-  options: ValidateCommandOptions
-): Promise<void> {
-  if (!options.schema) {
-    throw new InvalidArgumentError("Missing required option: --schema <path_to_schema.json>");
-  }
-
-  const progress = new ProgressReporter(options);
-
-  // Read and parse schema file
-  let schemaContent: string;
-  try {
-    schemaContent = await readFile(options.schema, "utf8");
-  } catch (err) {
-    throw new InvalidArgumentError(
-      `Failed to read schema file at "${options.schema}": ${(err as Error).message}`
-    );
-  }
-
-  let schemaDef: ValidationSchemaDefinition;
-  try {
-    schemaDef = JSON.parse(schemaContent);
-  } catch (err) {
-    throw new InvalidArgumentError(
-      `Invalid JSON in schema file "${options.schema}": ${(err as Error).message}`
-    );
-  }
-
-  let fromFormat = options.from?.toLowerCase();
-  if (!fromFormat && inputPath !== "-") {
-    fromFormat = inferFormatFromPath(inputPath) ?? undefined;
-  }
-  if (!fromFormat) {
-    fromFormat = "csv";
-  }
-
-  const inputStream = openReadableStream(inputPath);
-  const reader = createReader(inputStream, {
-    format: fromFormat,
-    sheet: options.sheet,
-    path: options.path,
-    delimiter: options.delimiter,
-    filePath: inputPath,
-  });
-
-  const validator = new Validator(schemaDef);
-  const pipeline = createPipeline(reader);
-  pipeline.onProgress((info) => progress.update(info));
-
-  for await (const row of pipeline.rows()) {
-    validator.add(row);
-  }
-
-  progress.done();
-  const report = validator.result();
-
-  if (options.json) {
-    process.stdout.write(JSON.stringify(report, null, 2) + "\n");
-  } else {
-    process.stdout.write(`${formatNumber(report.totalRows)} rows scanned\n`);
-    process.stdout.write(`${formatNumber(report.validRows)} valid\n`);
-    process.stdout.write(`${formatNumber(report.invalidRows)} invalid\n`);
-
-    if (report.violations.length > 0) {
-      process.stdout.write("\nErrors\n\n");
-      const groupedByCol = new Map<string, typeof report.violations>();
-      for (const v of report.violations) {
-        if (!groupedByCol.has(v.column)) groupedByCol.set(v.column, []);
-        groupedByCol.get(v.column)!.push(v);
-      }
-
-      for (const [col, violations] of groupedByCol.entries()) {
-        process.stdout.write(`${col}\n`);
-        for (const v of violations) {
-          process.stdout.write(`  ${v.rule}\n`);
-          process.stdout.write(`  ${formatNumber(v.violationsCount)} violations\n\n`);
-        }
-      }
-    }
-  }
-
-  logMemoryDebug();
-
-  if (!report.isValid) {
-    throw new ValidationError(
-      `Schema validation failed: ${report.invalidRows} invalid rows found`,
-      report.violations.map((v) => ({
-        column: v.column,
-        expected: v.rule,
-        count: v.violationsCount,
-        examples: v.sampleInvalidValues,
-      })),
-      report.totalRows,
-      report.validRows,
-      report.invalidRows
-    );
-  }
-}

package/src/cli/index.ts

@@ -1,285 +0,0 @@
-#!/usr/bin/env node
-
-import { Command } from "commander";
-import { RowpipeError } from "../core/errors.js";
-import { castCommand } from "./commands/cast.js";
-import { convertCommand } from "./commands/convert.js";
-import { filterCommand } from "./commands/filter.js";
-import { inspectCommand } from "./commands/inspect.js";
-import { mapCommand } from "./commands/map.js";
-import { reduceCommand } from "./commands/reduce.js";
-import { renameCommand } from "./commands/rename.js";
-import { sampleCommand } from "./commands/sample.js";
-import { schemaCommand } from "./commands/schema.js";
-import { selectCommand } from "./commands/select.js";
-import { statsCommand } from "./commands/stats.js";
-import { validateCommand } from "./commands/validate.js";
-
-// Handle broken pipe gracefully when piping to head/less
-process.stdout.on("error", (err: unknown) => {
-  if ((err as NodeJS.ErrnoException).code === "EPIPE") {
-    process.exit(0);
-  }
-});
-
-const program = new Command();
-
-program
-  .name("rowpipe")
-  .description("Stream-first tabular data toolkit for CSV, JSON, JSONL, and XLSX")
-  .version("1.0.0");
-
-// Global options
-program
-  .option("--batch-size <number>", "Processing batch size in rows", "1000")
-  .option("--quiet", "Suppress non-data output and progress")
-  .option("--no-progress", "Disable real-time progress bar");
-
-// 1. inspect
-program
-  .command("inspect [input]")
-  .description("Inspect format, row count, columns, and sheet summary")
-  .option("--sheet <sheet>", "Target worksheet name or index for XLSX")
-  .option("--path <path>", "Nested object path for JSON (e.g. data.results)")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output metadata as machine-readable JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await inspectCommand(input, opts);
-  });
-
-// 2. convert
-program
-  .command("convert [input] [output]")
-  .description("Stream convert tabular datasets across formats")
-  .option("--from <format>", "Input format (csv, json, jsonl, xlsx)")
-  .option("--to <format>", "Output format (csv, json, jsonl, xlsx)")
-  .option("--sheet <sheet>", "Worksheet name or index for XLSX")
-  .option("--all-sheets", "Export all worksheets in the workbook to individual files")
-  .option("--out-dir <dir>", "Output directory for --all-sheets export")
-  .option("--path <path>", "Nested path for JSON array")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--no-header", "Disable writing or reading headers in CSV")
-  .action(async (input = "-", output, cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await convertCommand(input, output, opts);
-  });
-
-// 3. schema
-program
-  .command("schema [input]")
-  .description("Infer column types, nullability, and semantic annotations")
-  .option("--sample <rows>", "Number of rows to sample for inference", "10000")
-  .option("--full", "Scan full stream for exact schema inference")
-  .option("--from <format>", "Input format")
-  .option("--sheet <sheet>", "Worksheet name or index for XLSX")
-  .option("--path <path>", "Nested path for JSON")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output schema as machine-readable JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await schemaCommand(input, opts);
-  });
-
-// 4. stats
-program
-  .command("stats [input]")
-  .description("Compute streaming statistics (Welford numeric stats and HLL distinct counts)")
-  .option("--column <name>", "Compute statistics only for specific column")
-  .option("--fast", "Use approximate algorithms for distinct counts")
-  .option("--exact", "Use exact counts where available")
-  .option("--from <format>", "Input format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output stats as machine-readable JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await statsCommand(input, opts);
-  });
-
-// 5. select
-program
-  .command("select [input] [columns]")
-  .description("Select a subset of columns in stream")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (inputOrCols, maybeCols, cmdOptions) => {
-    let input = "-";
-    let columns = "";
-
-    if (maybeCols !== undefined) {
-      input = inputOrCols || "-";
-      columns = maybeCols;
-    } else {
-      columns = inputOrCols || "";
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await selectCommand(input, columns, opts);
-  });
-
-// 6. rename
-program
-  .command("rename <args...>")
-  .description("Rename columns in stream (e.g. rowpipe rename users.csv old=new)")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes("=")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await renameCommand(input, specs, opts);
-  });
-
-// 7. cast
-program
-  .command("cast <args...>")
-  .description("Cast column types in stream (e.g. rowpipe cast users.csv age:number active:boolean)")
-  .option("--on-error <behavior>", "Error behavior: fail, null, keep, skip-row", "null")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes(":")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await castCommand(input, specs, opts);
-  });
-
-// 8. filter
-program
-  .command("filter [input] [expression]")
-  .description("Filter rows using safe expression engine (e.g. rowpipe filter users.csv 'age > 30')")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (inputOrExpr, maybeExpr, cmdOptions) => {
-    let input = "-";
-    let expression = "";
-
-    if (maybeExpr !== undefined) {
-      input = inputOrExpr || "-";
-      expression = maybeExpr;
-    } else {
-      expression = inputOrExpr || "";
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await filterCommand(input, expression, opts);
-  });
-
-// 9. map
-program
-  .command("map <args...>")
-  .description("Transform and derive columns per row (e.g. rowpipe map sales.csv 'profit=revenue-cost')")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes("=")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await mapCommand(input, specs, opts);
-  });
-
-// 10. reduce
-program
-  .command("reduce <args...>")
-  .description("Aggregate and group dataset (e.g. rowpipe reduce sales.csv 'total=sum(revenue)' --by country)")
-  .option("--by <columns>", "Group by columns (comma-separated, e.g. country,category)")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (args, cmdOptions) => {
-    let input = "-";
-    let specs: string[] = [];
-
-    if (args[0] && !args[0].includes("=")) {
-      input = args[0];
-      specs = args.slice(1);
-    } else {
-      specs = args;
-    }
-
-    const opts = { ...program.opts(), ...cmdOptions };
-    await reduceCommand(input, specs, opts);
-  });
-
-// 11. sample
-program
-  .command("sample [input]")
-  .description("Reservoir sample rows with bounded memory")
-  .option("--rows <number>", "Number of rows to sample", "1000")
-  .option("--seed <number>", "Deterministic random seed")
-  .option("--from <format>", "Input format")
-  .option("--to <format>", "Output format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await sampleCommand(input, opts);
-  });
-
-// 10. validate
-program
-  .command("validate [input]")
-  .description("Validate stream against schema definition")
-  .requiredOption("--schema <file>", "Path to JSON schema file")
-  .option("--from <format>", "Input format")
-  .option("--sheet <sheet>", "Worksheet name for XLSX")
-  .option("--delimiter <delim>", "Custom CSV delimiter")
-  .option("--json", "Output validation report as JSON")
-  .action(async (input = "-", cmdOptions) => {
-    const opts = { ...program.opts(), ...cmdOptions };
-    await validateCommand(input, opts);
-  });
-
-async function main() {
-  try {
-    await program.parseAsync(process.argv);
-  } catch (err: unknown) {
-    if (err instanceof RowpipeError) {
-      process.stderr.write(`\nError: ${err.message}\n`);
-      process.exit(err.exitCode);
-    }
-    process.stderr.write(`\nUnexpected error: ${(err as Error).message}\n`);
-    if (process.env["DEBUG"]) {
-      process.stderr.write(`${(err as Error).stack}\n`);
-    }
-    process.exit(1);
-  }
-}
-
-main();