rowpipe 1.0.0 → 1.1.0

package/benchmarks/memory-bench.ts

@@ -1,109 +0,0 @@
-import { createPipeline } from "../src/core/pipeline.js";
-import { filterRows } from "../src/transforms/filter.js";
-import { selectColumns } from "../src/transforms/select.js";
-import { castColumns } from "../src/transforms/cast.js";
-import { DatasetStatsAggregator } from "../src/analytics/stats.js";
-import type { DataBatch, DataStream, Row } from "../src/core/types.js";
-import { formatBytes, formatNumber } from "../src/utils/formatting.js";
-
-/**
- * Generator producing synthetic data batches without buffering.
- */
-async function* generateSyntheticStream(
-  totalRows: number,
-  batchSize = 2000
-): DataStream {
-  let offset = 0;
-  while (offset < totalRows) {
-    const currentBatchSize = Math.min(batchSize, totalRows - offset);
-    const rows: Row[] = new Array(currentBatchSize);
-
-    for (let i = 0; i < currentBatchSize; i++) {
-      const id = offset + i;
-      rows[i] = {
-        id: String(id),
-        name: `User_${id}`,
-        age: String(20 + (id % 50)),
-        email: `user${id}@example.com`,
-        revenue: String((id % 1000) * 1.5),
-        country: id % 3 === 0 ? "TR" : id % 3 === 1 ? "US" : "DE",
-      };
-    }
-
-    yield {
-      rows,
-      offset,
-    };
-    offset += currentBatchSize;
-  }
-}
-
-async function runBenchmark(totalRows: number): Promise<void> {
-  process.stdout.write(`\n======================================================\n`);
-  process.stdout.write(`Running Benchmark with ${formatNumber(totalRows)} synthetic rows...\n`);
-  process.stdout.write(`Pipeline: Input -> Filter(age > 25) -> Select(id, name, age, revenue) -> Cast(revenue:number) -> Stats\n`);
-  process.stdout.write(`------------------------------------------------------\n`);
-
-  const initialMem = process.memoryUsage();
-  let peakRss = initialMem.rss;
-  let peakHeap = initialMem.heapUsed;
-
-  const startTime = Date.now();
-  let processedRows = 0;
-
-  const rawStream = generateSyntheticStream(totalRows, 2000);
-
-  const pipeline = createPipeline(rawStream)
-    .pipe(filterRows("age > 25"))
-    .pipe(selectColumns(["id", "name", "age", "revenue"]))
-    .pipe(castColumns({ revenue: "number" }));
-
-  const statsAgg = new DatasetStatsAggregator();
-
-  for await (const batch of pipeline.batches()) {
-    processedRows += batch.rows.length;
-    for (const row of batch.rows) {
-      statsAgg.add(row);
-    }
-
-    const currentMem = process.memoryUsage();
-    if (currentMem.rss > peakRss) peakRss = currentMem.rss;
-    if (currentMem.heapUsed > peakHeap) peakHeap = currentMem.heapUsed;
-
-    if (processedRows % 250000 === 0 || processedRows === totalRows) {
-      const elapsed = (Date.now() - startTime) / 1000 || 0.001;
-      const speed = Math.round(processedRows / elapsed);
-      process.stdout.write(
-        `  Processed: ${formatNumber(processedRows)} rows | Elapsed: ${elapsed.toFixed(1)}s | RSS: ${formatBytes(currentMem.rss)} | Heap: ${formatBytes(currentMem.heapUsed)} | Speed: ${formatNumber(speed)} rows/s\n`
-      );
-    }
-  }
-
-  const totalTimeMs = Date.now() - startTime;
-  const elapsedSec = totalTimeMs / 1000 || 0.001;
-  const rowsPerSec = Math.round(totalRows / elapsedSec);
-  // Estimate ~70 bytes per row
-  const estimatedMB = (totalRows * 70) / (1024 * 1024);
-  const mbPerSec = (estimatedMB / elapsedSec).toFixed(1);
-
-  const results = statsAgg.result();
-
-  process.stdout.write(`------------------------------------------------------\n`);
-  process.stdout.write(`Benchmark Results:\n`);
-  process.stdout.write(`  Total Input Rows:  ${formatNumber(totalRows)}\n`);
-  process.stdout.write(`  Filtered Rows:     ${formatNumber(processedRows)}\n`);
-  process.stdout.write(`  Execution Time:    ${elapsedSec.toFixed(2)}s\n`);
-  process.stdout.write(`  Throughput:        ${formatNumber(rowsPerSec)} rows/s (~${mbPerSec} MB/s)\n`);
-  process.stdout.write(`  Initial RSS:       ${formatBytes(initialMem.rss)}\n`);
-  process.stdout.write(`  Peak RSS:          ${formatBytes(peakRss)}\n`);
-  process.stdout.write(`  Peak Heap Used:    ${formatBytes(peakHeap)}\n`);
-  process.stdout.write(`  Memory Growth:     Bounded ($O(1)$ constant memory)\n`);
-  process.stdout.write(`======================================================\n`);
-}
-
-async function main() {
-  await runBenchmark(100_000);
-  await runBenchmark(1_000_000);
-}
-
-main().catch(console.error);

package/src/analytics/reduce.ts

@@ -1,388 +0,0 @@
-import { InvalidArgumentError } from "../core/errors.js";
-import type { Aggregator, DataBatch, DataStream, Row, TransformFunction } from "../core/types.js";
-import { compileValueExpression } from "../transforms/expression.js";
-import { HyperLogLog } from "./stats.js";
-
-export type AggregationFunction =
-  | "sum"
-  | "avg"
-  | "mean"
-  | "min"
-  | "max"
-  | "count"
-  | "countdistinct"
-  | "stddev"
-  | "variance"
-  | "first"
-  | "last";
-
-export interface AggregationSpec {
-  targetField: string;
-  func: AggregationFunction;
-  sourceExpr?: string;
-}
-
-/**
- * Parses CLI reduce specs into structured AggregationSpec objects.
- * Supports syntax: "total_rev = sum(revenue)", "avg_margin = avg(margin)", "orders = count()"
- */
-export function parseReduceSpecs(specs: string[]): AggregationSpec[] {
-  const result: AggregationSpec[] = [];
-
-  for (const spec of specs) {
-    const trimmed = spec.trim();
-    if (!trimmed) continue;
-
-    const eqIdx = trimmed.indexOf("=");
-    if (eqIdx === -1) {
-      throw new InvalidArgumentError(
-        `Invalid reduce specification "${spec}". Expected format "target_column = func(source_expr)"`
-      );
-    }
-
-    const targetField = trimmed.slice(0, eqIdx).trim();
-    const rhs = trimmed.slice(eqIdx + 1).trim();
-
-    const match = rhs.match(/^([a-zA-Z_]+)\s*\((.*)\)$/);
-    if (!match) {
-      throw new InvalidArgumentError(
-        `Invalid aggregation function call in "${spec}". Expected format "func(column)" e.g. "sum(revenue)" or "count()"`
-      );
-    }
-
-    const rawFunc = match[1]!.toLowerCase();
-    const sourceExpr = match[2]!.trim() || undefined;
-
-    const validFuncs = [
-      "sum",
-      "avg",
-      "mean",
-      "min",
-      "max",
-      "count",
-      "countdistinct",
-      "stddev",
-      "variance",
-      "first",
-      "last",
-    ];
-
-    if (!validFuncs.includes(rawFunc)) {
-      throw new InvalidArgumentError(
-        `Unknown aggregation function "${rawFunc}" in "${spec}". Supported functions: ${validFuncs.join(", ")}`
-      );
-    }
-
-    result.push({
-      targetField,
-      func: rawFunc as AggregationFunction,
-      sourceExpr,
-    });
-  }
-
-  return result;
-}
-
-interface Accumulator {
-  add(val: unknown): void;
-  result(): unknown;
-}
-
-class SumAccumulator implements Accumulator {
-  private sum = 0;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      this.sum += num;
-    }
-  }
-  result(): number {
-    return this.sum;
-  }
-}
-
-class AvgAccumulator implements Accumulator {
-  private sum = 0;
-  private count = 0;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      this.sum += num;
-      this.count++;
-    }
-  }
-  result(): number | null {
-    return this.count > 0 ? this.sum / this.count : null;
-  }
-}
-
-class MinAccumulator implements Accumulator {
-  private min: number | null = null;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      if (this.min === null || num < this.min) this.min = num;
-    }
-  }
-  result(): number | null {
-    return this.min;
-  }
-}
-
-class MaxAccumulator implements Accumulator {
-  private max: number | null = null;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      if (this.max === null || num > this.max) this.max = num;
-    }
-  }
-  result(): number | null {
-    return this.max;
-  }
-}
-
-class CountAccumulator implements Accumulator {
-  private count = 0;
-  private hasSource: boolean;
-  constructor(hasSource: boolean) {
-    this.hasSource = hasSource;
-  }
-  add(val: unknown): void {
-    if (!this.hasSource) {
-      this.count++;
-    } else if (val !== null && val !== undefined && val !== "") {
-      this.count++;
-    }
-  }
-  result(): number {
-    return this.count;
-  }
-}
-
-class CountDistinctAccumulator implements Accumulator {
-  private hll = new HyperLogLog();
-  add(val: unknown): void {
-    if (val !== null && val !== undefined && val !== "") {
-      this.hll.add(val);
-    }
-  }
-  result(): number {
-    return this.hll.count();
-  }
-}
-
-class WelfordAccumulator implements Accumulator {
-  private count = 0;
-  private mean = 0;
-  private M2 = 0;
-  private mode: "stddev" | "variance";
-
-  constructor(mode: "stddev" | "variance") {
-    this.mode = mode;
-  }
-
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (Number.isNaN(num)) return;
-
-    this.count++;
-    const delta = num - this.mean;
-    this.mean += delta / this.count;
-    const delta2 = num - this.mean;
-    this.M2 += delta * delta2;
-  }
-
-  result(): number | null {
-    if (this.count <= 1) return this.count === 1 ? 0 : null;
-    const variance = this.M2 / (this.count - 1);
-    return this.mode === "stddev" ? Math.sqrt(variance) : variance;
-  }
-}
-
-class FirstAccumulator implements Accumulator {
-  private value: unknown = null;
-  private found = false;
-  add(val: unknown): void {
-    if (!this.found && val !== null && val !== undefined && val !== "") {
-      this.value = val;
-      this.found = true;
-    }
-  }
-  result(): unknown {
-    return this.value;
-  }
-}
-
-class LastAccumulator implements Accumulator {
-  private value: unknown = null;
-  add(val: unknown): void {
-    if (val !== null && val !== undefined && val !== "") {
-      this.value = val;
-    }
-  }
-  result(): unknown {
-    return this.value;
-  }
-}
-
-function createAccumulatorFor(func: AggregationFunction, hasSource: boolean): Accumulator {
-  switch (func) {
-    case "sum":
-      return new SumAccumulator();
-    case "avg":
-    case "mean":
-      return new AvgAccumulator();
-    case "min":
-      return new MinAccumulator();
-    case "max":
-      return new MaxAccumulator();
-    case "count":
-      return new CountAccumulator(hasSource);
-    case "countdistinct":
-      return new CountDistinctAccumulator();
-    case "stddev":
-      return new WelfordAccumulator("stddev");
-    case "variance":
-      return new WelfordAccumulator("variance");
-    case "first":
-      return new FirstAccumulator();
-    case "last":
-      return new LastAccumulator();
-  }
-}
-
-interface CompiledAggregation {
-  targetField: string;
-  func: AggregationFunction;
-  evaluator?: (row: Row) => unknown;
-  createAccumulator: () => Accumulator;
-}
-
-export interface ReduceOptions {
-  by?: string[];
-  aggregations: AggregationSpec[] | string[];
-}
-
-/**
- * Streaming Reduce & Group-by Aggregator.
- */
-export class ReduceAggregator implements Aggregator<Row[]> {
-  private byCols: string[];
-  private compiled: CompiledAggregation[];
-  private globalAccs?: Accumulator[];
-  private groupMap?: Map<string, { groupValues: Record<string, unknown>; accs: Accumulator[] }>;
-
-  constructor(options: ReduceOptions) {
-    this.byCols = options.by && options.by.length > 0 ? options.by : [];
-
-    const rawSpecs: AggregationSpec[] =
-      typeof options.aggregations[0] === "string"
-        ? parseReduceSpecs(options.aggregations as string[])
-        : (options.aggregations as AggregationSpec[]);
-
-    this.compiled = rawSpecs.map((spec) => ({
-      targetField: spec.targetField,
-      func: spec.func,
-      evaluator: spec.sourceExpr ? compileValueExpression(spec.sourceExpr) : undefined,
-      createAccumulator: () => createAccumulatorFor(spec.func, Boolean(spec.sourceExpr)),
-    }));
-
-    if (this.byCols.length === 0) {
-      this.globalAccs = this.compiled.map((c) => c.createAccumulator());
-    } else {
-      this.groupMap = new Map();
-    }
-  }
-
-  add(row: Row): void {
-    if (this.globalAccs) {
-      // Global reduction
-      for (let i = 0; i < this.compiled.length; i++) {
-        const item = this.compiled[i]!;
-        const val = item.evaluator ? item.evaluator(row) : null;
-        this.globalAccs[i]!.add(val);
-      }
-    } else if (this.groupMap) {
-      // Group-by reduction
-      let key = "";
-      const groupValues: Record<string, unknown> = {};
-
-      for (let i = 0; i < this.byCols.length; i++) {
-        const col = this.byCols[i]!;
-        const val = row[col] ?? null;
-        groupValues[col] = val;
-        key += (i > 0 ? "\x1f" : "") + String(val ?? "");
-      }
-
-      let group = this.groupMap.get(key);
-      if (!group) {
-        group = {
-          groupValues,
-          accs: this.compiled.map((c) => c.createAccumulator()),
-        };
-        this.groupMap.set(key, group);
-      }
-
-      for (let i = 0; i < this.compiled.length; i++) {
-        const item = this.compiled[i]!;
-        const val = item.evaluator ? item.evaluator(row) : null;
-        group.accs[i]!.add(val);
-      }
-    }
-  }
-
-  result(): Row[] {
-    if (this.globalAccs) {
-      const summaryRow: Row = {};
-      for (let i = 0; i < this.compiled.length; i++) {
-        const item = this.compiled[i]!;
-        summaryRow[item.targetField] = this.globalAccs[i]!.result();
-      }
-      return [summaryRow];
-    }
-
-    if (this.groupMap) {
-      const rows: Row[] = [];
-      for (const group of this.groupMap.values()) {
-        const outRow: Row = { ...group.groupValues };
-        for (let i = 0; i < this.compiled.length; i++) {
-          const item = this.compiled[i]!;
-          outRow[item.targetField] = group.accs[i]!.result();
-        }
-        rows.push(outRow);
-      }
-      return rows;
-    }
-
-    return [];
-  }
-}
-
-/**
- * Creates a high-performance streaming transform that reduces/aggregates rows.
- */
-export function reduceRows(options: ReduceOptions): TransformFunction {
-  return async function* (stream: DataStream): DataStream {
-    const aggregator = new ReduceAggregator(options);
-
-    for await (const batch of stream) {
-      const len = batch.rows.length;
-      for (let i = 0; i < len; i++) {
-        aggregator.add(batch.rows[i]!);
-      }
-    }
-
-    const finalRows = aggregator.result();
-
-    yield {
-      rows: finalRows,
-      offset: 0,
-    };
-  };
-}

package/src/analytics/schema-inference.ts

@@ -1,217 +0,0 @@
-import type { Aggregator, ColumnType, Row, SemanticType } from "../core/types.js";
-import { detectSemanticType } from "./semantic-types.js";
-
-export interface ColumnInferenceResult {
-  name: string;
-  type: ColumnType;
-  nullable: boolean;
-  confidence: number;
-  semantic?: SemanticType;
-  semanticConfidence?: number;
-  typeBreakdown: Record<string, number>;
-  sampleCount: number;
-  nullCount: number;
-}
-
-export interface InferredSchemaResult {
-  totalRowsScanned: number;
-  columns: ColumnInferenceResult[];
-}
-
-const DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/;
-const DATETIME_REGEX = /^\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/;
-
-/**
- * Tests primitive type for an individual value.
- */
-export function classifyPrimitiveType(value: unknown): ColumnType {
-  if (value === null || value === undefined || value === "") {
-    return "null";
-  }
-
-  if (typeof value === "boolean") return "boolean";
-  if (typeof value === "number") {
-    return Number.isInteger(value) ? "integer" : "number";
-  }
-
-  const str = String(value).trim();
-
-  // Boolean strings
-  if (["true", "false"].includes(str.toLowerCase())) {
-    return "boolean";
-  }
-
-  // Integer regex
-  if (/^-?\d+$/.test(str)) {
-    return "integer";
-  }
-
-  // Floating point regex
-  if (/^-?\d+\.\d+$/.test(str) || /^-?\d+(?:\.\d+)?e[+-]?\d+$/i.test(str)) {
-    return "number";
-  }
-
-  // Date / Datetime
-  if (DATE_REGEX.test(str)) {
-    const d = new Date(str);
-    if (!Number.isNaN(d.getTime())) return "date";
-  }
-
-  if (DATETIME_REGEX.test(str)) {
-    const d = new Date(str);
-    if (!Number.isNaN(d.getTime())) return "datetime";
-  }
-
-  return "string";
-}
-
-export class SchemaInferenceAggregator implements Aggregator<InferredSchemaResult> {
-  private totalRows = 0;
-  private maxSample: number;
-  private columnStats = new Map<
-    string,
-    {
-      counts: Record<ColumnType, number>;
-      semanticCounts: Record<string, number>;
-      nullCount: number;
-      totalCount: number;
-    }
-  >();
-
-  constructor(options?: { sample?: number }) {
-    this.maxSample = options?.sample ?? Number.POSITIVE_INFINITY;
-  }
-
-  add(row: Row): void {
-    if (this.totalRows >= this.maxSample) return;
-    this.totalRows++;
-
-    for (const [col, val] of Object.entries(row)) {
-      if (!this.columnStats.has(col)) {
-        this.columnStats.set(col, {
-          counts: {
-            string: 0,
-            integer: 0,
-            number: 0,
-            boolean: 0,
-            date: 0,
-            datetime: 0,
-            null: 0,
-            mixed: 0,
-          },
-          semanticCounts: {},
-          nullCount: 0,
-          totalCount: 0,
-        });
-      }
-
-      const stat = this.columnStats.get(col)!;
-      stat.totalCount++;
-
-      const pType = classifyPrimitiveType(val);
-      stat.counts[pType]++;
-      if (pType === "null") {
-        stat.nullCount++;
-      } else {
-        const sem = detectSemanticType(val);
-        if (sem) {
-          stat.semanticCounts[sem] = (stat.semanticCounts[sem] || 0) + 1;
-        }
-      }
-    }
-  }
-
-  result(): InferredSchemaResult {
-    const columns: ColumnInferenceResult[] = [];
-
-    for (const [col, stat] of this.columnStats.entries()) {
-      const nonNullCount = stat.totalCount - stat.nullCount;
-      const nullable = stat.nullCount > 0;
-
-      if (nonNullCount === 0) {
-        columns.push({
-          name: col,
-          type: "null",
-          nullable: true,
-          confidence: 100,
-          typeBreakdown: { null: 100 },
-          sampleCount: stat.totalCount,
-          nullCount: stat.nullCount,
-        });
-        continue;
-      }
-
-      // Calculate confidence percentages
-      const breakdown: Record<string, number> = {};
-      let dominantType: ColumnType = "string";
-      let maxPercentage = 0;
-
-      const candidates: ColumnType[] = [
-        "integer",
-        "number",
-        "boolean",
-        "datetime",
-        "date",
-        "string",
-      ];
-
-      for (const t of candidates) {
-        const pct = (stat.counts[t] / nonNullCount) * 100;
-        if (pct > 0) {
-          breakdown[t] = Math.round(pct * 10) / 10;
-        }
-        if (pct > maxPercentage) {
-          maxPercentage = pct;
-          dominantType = t;
-        }
-      }
-
-      // If integer + number together make 100%, consider number
-      if (
-        dominantType === "integer" &&
-        stat.counts["number"] > 0 &&
-        (stat.counts["integer"] + stat.counts["number"]) / nonNullCount >= 0.95
-      ) {
-        dominantType = "number";
-        maxPercentage = ((stat.counts["integer"] + stat.counts["number"]) / nonNullCount) * 100;
-      }
-
-      let inferredType: ColumnType = dominantType;
-      let confidence = Math.round(maxPercentage * 10) / 10;
-
-      if (maxPercentage < 90 && Object.keys(breakdown).length > 1) {
-        inferredType = "mixed";
-      }
-
-      // Semantic type detection
-      let semantic: SemanticType | undefined;
-      let semanticConfidence: number | undefined;
-
-      for (const [sem, count] of Object.entries(stat.semanticCounts)) {
-        const semPct = (count / nonNullCount) * 100;
-        if (semPct >= 80) {
-          semantic = sem as SemanticType;
-          semanticConfidence = Math.round(semPct * 10) / 10;
-          break;
-        }
-      }
-
-      columns.push({
-        name: col,
-        type: inferredType,
-        nullable,
-        confidence,
-        semantic,
-        semanticConfidence,
-        typeBreakdown: breakdown,
-        sampleCount: stat.totalCount,
-        nullCount: stat.nullCount,
-      });
-    }
-
-    return {
-      totalRowsScanned: this.totalRows,
-      columns,
-    };
-  }
-}