rowpipe 1.0.0 → 1.2.0

package/src/analytics/reduce.ts

@@ -1,388 +0,0 @@
-import { InvalidArgumentError } from "../core/errors.js";
-import type { Aggregator, DataBatch, DataStream, Row, TransformFunction } from "../core/types.js";
-import { compileValueExpression } from "../transforms/expression.js";
-import { HyperLogLog } from "./stats.js";
-
-export type AggregationFunction =
-  | "sum"
-  | "avg"
-  | "mean"
-  | "min"
-  | "max"
-  | "count"
-  | "countdistinct"
-  | "stddev"
-  | "variance"
-  | "first"
-  | "last";
-
-export interface AggregationSpec {
-  targetField: string;
-  func: AggregationFunction;
-  sourceExpr?: string;
-}
-
-/**
- * Parses CLI reduce specs into structured AggregationSpec objects.
- * Supports syntax: "total_rev = sum(revenue)", "avg_margin = avg(margin)", "orders = count()"
- */
-export function parseReduceSpecs(specs: string[]): AggregationSpec[] {
-  const result: AggregationSpec[] = [];
-
-  for (const spec of specs) {
-    const trimmed = spec.trim();
-    if (!trimmed) continue;
-
-    const eqIdx = trimmed.indexOf("=");
-    if (eqIdx === -1) {
-      throw new InvalidArgumentError(
-        `Invalid reduce specification "${spec}". Expected format "target_column = func(source_expr)"`
-      );
-    }
-
-    const targetField = trimmed.slice(0, eqIdx).trim();
-    const rhs = trimmed.slice(eqIdx + 1).trim();
-
-    const match = rhs.match(/^([a-zA-Z_]+)\s*\((.*)\)$/);
-    if (!match) {
-      throw new InvalidArgumentError(
-        `Invalid aggregation function call in "${spec}". Expected format "func(column)" e.g. "sum(revenue)" or "count()"`
-      );
-    }
-
-    const rawFunc = match[1]!.toLowerCase();
-    const sourceExpr = match[2]!.trim() || undefined;
-
-    const validFuncs = [
-      "sum",
-      "avg",
-      "mean",
-      "min",
-      "max",
-      "count",
-      "countdistinct",
-      "stddev",
-      "variance",
-      "first",
-      "last",
-    ];
-
-    if (!validFuncs.includes(rawFunc)) {
-      throw new InvalidArgumentError(
-        `Unknown aggregation function "${rawFunc}" in "${spec}". Supported functions: ${validFuncs.join(", ")}`
-      );
-    }
-
-    result.push({
-      targetField,
-      func: rawFunc as AggregationFunction,
-      sourceExpr,
-    });
-  }
-
-  return result;
-}
-
-interface Accumulator {
-  add(val: unknown): void;
-  result(): unknown;
-}
-
-class SumAccumulator implements Accumulator {
-  private sum = 0;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      this.sum += num;
-    }
-  }
-  result(): number {
-    return this.sum;
-  }
-}
-
-class AvgAccumulator implements Accumulator {
-  private sum = 0;
-  private count = 0;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      this.sum += num;
-      this.count++;
-    }
-  }
-  result(): number | null {
-    return this.count > 0 ? this.sum / this.count : null;
-  }
-}
-
-class MinAccumulator implements Accumulator {
-  private min: number | null = null;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      if (this.min === null || num < this.min) this.min = num;
-    }
-  }
-  result(): number | null {
-    return this.min;
-  }
-}
-
-class MaxAccumulator implements Accumulator {
-  private max: number | null = null;
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (!Number.isNaN(num)) {
-      if (this.max === null || num > this.max) this.max = num;
-    }
-  }
-  result(): number | null {
-    return this.max;
-  }
-}
-
-class CountAccumulator implements Accumulator {
-  private count = 0;
-  private hasSource: boolean;
-  constructor(hasSource: boolean) {
-    this.hasSource = hasSource;
-  }
-  add(val: unknown): void {
-    if (!this.hasSource) {
-      this.count++;
-    } else if (val !== null && val !== undefined && val !== "") {
-      this.count++;
-    }
-  }
-  result(): number {
-    return this.count;
-  }
-}
-
-class CountDistinctAccumulator implements Accumulator {
-  private hll = new HyperLogLog();
-  add(val: unknown): void {
-    if (val !== null && val !== undefined && val !== "") {
-      this.hll.add(val);
-    }
-  }
-  result(): number {
-    return this.hll.count();
-  }
-}
-
-class WelfordAccumulator implements Accumulator {
-  private count = 0;
-  private mean = 0;
-  private M2 = 0;
-  private mode: "stddev" | "variance";
-
-  constructor(mode: "stddev" | "variance") {
-    this.mode = mode;
-  }
-
-  add(val: unknown): void {
-    if (val === null || val === undefined || val === "") return;
-    const num = typeof val === "number" ? val : Number(val);
-    if (Number.isNaN(num)) return;
-
-    this.count++;
-    const delta = num - this.mean;
-    this.mean += delta / this.count;
-    const delta2 = num - this.mean;
-    this.M2 += delta * delta2;
-  }
-
-  result(): number | null {
-    if (this.count <= 1) return this.count === 1 ? 0 : null;
-    const variance = this.M2 / (this.count - 1);
-    return this.mode === "stddev" ? Math.sqrt(variance) : variance;
-  }
-}
-
-class FirstAccumulator implements Accumulator {
-  private value: unknown = null;
-  private found = false;
-  add(val: unknown): void {
-    if (!this.found && val !== null && val !== undefined && val !== "") {
-      this.value = val;
-      this.found = true;
-    }
-  }
-  result(): unknown {
-    return this.value;
-  }
-}
-
-class LastAccumulator implements Accumulator {
-  private value: unknown = null;
-  add(val: unknown): void {
-    if (val !== null && val !== undefined && val !== "") {
-      this.value = val;
-    }
-  }
-  result(): unknown {
-    return this.value;
-  }
-}
-
-function createAccumulatorFor(func: AggregationFunction, hasSource: boolean): Accumulator {
-  switch (func) {
-    case "sum":
-      return new SumAccumulator();
-    case "avg":
-    case "mean":
-      return new AvgAccumulator();
-    case "min":
-      return new MinAccumulator();
-    case "max":
-      return new MaxAccumulator();
-    case "count":
-      return new CountAccumulator(hasSource);
-    case "countdistinct":
-      return new CountDistinctAccumulator();
-    case "stddev":
-      return new WelfordAccumulator("stddev");
-    case "variance":
-      return new WelfordAccumulator("variance");
-    case "first":
-      return new FirstAccumulator();
-    case "last":
-      return new LastAccumulator();
-  }
-}
-
-interface CompiledAggregation {
-  targetField: string;
-  func: AggregationFunction;
-  evaluator?: (row: Row) => unknown;
-  createAccumulator: () => Accumulator;
-}
-
-export interface ReduceOptions {
-  by?: string[];
-  aggregations: AggregationSpec[] | string[];
-}
-
-/**
- * Streaming Reduce & Group-by Aggregator.
- */
-export class ReduceAggregator implements Aggregator<Row[]> {
-  private byCols: string[];
-  private compiled: CompiledAggregation[];
-  private globalAccs?: Accumulator[];
-  private groupMap?: Map<string, { groupValues: Record<string, unknown>; accs: Accumulator[] }>;
-
-  constructor(options: ReduceOptions) {
-    this.byCols = options.by && options.by.length > 0 ? options.by : [];
-
-    const rawSpecs: AggregationSpec[] =
-      typeof options.aggregations[0] === "string"
-        ? parseReduceSpecs(options.aggregations as string[])
-        : (options.aggregations as AggregationSpec[]);
-
-    this.compiled = rawSpecs.map((spec) => ({
-      targetField: spec.targetField,
-      func: spec.func,
-      evaluator: spec.sourceExpr ? compileValueExpression(spec.sourceExpr) : undefined,
-      createAccumulator: () => createAccumulatorFor(spec.func, Boolean(spec.sourceExpr)),
-    }));
-
-    if (this.byCols.length === 0) {
-      this.globalAccs = this.compiled.map((c) => c.createAccumulator());
-    } else {
-      this.groupMap = new Map();
-    }
-  }
-
-  add(row: Row): void {
-    if (this.globalAccs) {
-      // Global reduction
-      for (let i = 0; i < this.compiled.length; i++) {
-        const item = this.compiled[i]!;
-        const val = item.evaluator ? item.evaluator(row) : null;
-        this.globalAccs[i]!.add(val);
-      }
-    } else if (this.groupMap) {
-      // Group-by reduction
-      let key = "";
-      const groupValues: Record<string, unknown> = {};
-
-      for (let i = 0; i < this.byCols.length; i++) {
-        const col = this.byCols[i]!;
-        const val = row[col] ?? null;
-        groupValues[col] = val;
-        key += (i > 0 ? "\x1f" : "") + String(val ?? "");
-      }
-
-      let group = this.groupMap.get(key);
-      if (!group) {
-        group = {
-          groupValues,
-          accs: this.compiled.map((c) => c.createAccumulator()),
-        };
-        this.groupMap.set(key, group);
-      }
-
-      for (let i = 0; i < this.compiled.length; i++) {
-        const item = this.compiled[i]!;
-        const val = item.evaluator ? item.evaluator(row) : null;
-        group.accs[i]!.add(val);
-      }
-    }
-  }
-
-  result(): Row[] {
-    if (this.globalAccs) {
-      const summaryRow: Row = {};
-      for (let i = 0; i < this.compiled.length; i++) {
-        const item = this.compiled[i]!;
-        summaryRow[item.targetField] = this.globalAccs[i]!.result();
-      }
-      return [summaryRow];
-    }
-
-    if (this.groupMap) {
-      const rows: Row[] = [];
-      for (const group of this.groupMap.values()) {
-        const outRow: Row = { ...group.groupValues };
-        for (let i = 0; i < this.compiled.length; i++) {
-          const item = this.compiled[i]!;
-          outRow[item.targetField] = group.accs[i]!.result();
-        }
-        rows.push(outRow);
-      }
-      return rows;
-    }
-
-    return [];
-  }
-}
-
-/**
- * Creates a high-performance streaming transform that reduces/aggregates rows.
- */
-export function reduceRows(options: ReduceOptions): TransformFunction {
-  return async function* (stream: DataStream): DataStream {
-    const aggregator = new ReduceAggregator(options);
-
-    for await (const batch of stream) {
-      const len = batch.rows.length;
-      for (let i = 0; i < len; i++) {
-        aggregator.add(batch.rows[i]!);
-      }
-    }
-
-    const finalRows = aggregator.result();
-
-    yield {
-      rows: finalRows,
-      offset: 0,
-    };
-  };
-}

package/src/analytics/schema-inference.ts

@@ -1,217 +0,0 @@
-import type { Aggregator, ColumnType, Row, SemanticType } from "../core/types.js";
-import { detectSemanticType } from "./semantic-types.js";
-
-export interface ColumnInferenceResult {
-  name: string;
-  type: ColumnType;
-  nullable: boolean;
-  confidence: number;
-  semantic?: SemanticType;
-  semanticConfidence?: number;
-  typeBreakdown: Record<string, number>;
-  sampleCount: number;
-  nullCount: number;
-}
-
-export interface InferredSchemaResult {
-  totalRowsScanned: number;
-  columns: ColumnInferenceResult[];
-}
-
-const DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/;
-const DATETIME_REGEX = /^\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/;
-
-/**
- * Tests primitive type for an individual value.
- */
-export function classifyPrimitiveType(value: unknown): ColumnType {
-  if (value === null || value === undefined || value === "") {
-    return "null";
-  }
-
-  if (typeof value === "boolean") return "boolean";
-  if (typeof value === "number") {
-    return Number.isInteger(value) ? "integer" : "number";
-  }
-
-  const str = String(value).trim();
-
-  // Boolean strings
-  if (["true", "false"].includes(str.toLowerCase())) {
-    return "boolean";
-  }
-
-  // Integer regex
-  if (/^-?\d+$/.test(str)) {
-    return "integer";
-  }
-
-  // Floating point regex
-  if (/^-?\d+\.\d+$/.test(str) || /^-?\d+(?:\.\d+)?e[+-]?\d+$/i.test(str)) {
-    return "number";
-  }
-
-  // Date / Datetime
-  if (DATE_REGEX.test(str)) {
-    const d = new Date(str);
-    if (!Number.isNaN(d.getTime())) return "date";
-  }
-
-  if (DATETIME_REGEX.test(str)) {
-    const d = new Date(str);
-    if (!Number.isNaN(d.getTime())) return "datetime";
-  }
-
-  return "string";
-}
-
-export class SchemaInferenceAggregator implements Aggregator<InferredSchemaResult> {
-  private totalRows = 0;
-  private maxSample: number;
-  private columnStats = new Map<
-    string,
-    {
-      counts: Record<ColumnType, number>;
-      semanticCounts: Record<string, number>;
-      nullCount: number;
-      totalCount: number;
-    }
-  >();
-
-  constructor(options?: { sample?: number }) {
-    this.maxSample = options?.sample ?? Number.POSITIVE_INFINITY;
-  }
-
-  add(row: Row): void {
-    if (this.totalRows >= this.maxSample) return;
-    this.totalRows++;
-
-    for (const [col, val] of Object.entries(row)) {
-      if (!this.columnStats.has(col)) {
-        this.columnStats.set(col, {
-          counts: {
-            string: 0,
-            integer: 0,
-            number: 0,
-            boolean: 0,
-            date: 0,
-            datetime: 0,
-            null: 0,
-            mixed: 0,
-          },
-          semanticCounts: {},
-          nullCount: 0,
-          totalCount: 0,
-        });
-      }
-
-      const stat = this.columnStats.get(col)!;
-      stat.totalCount++;
-
-      const pType = classifyPrimitiveType(val);
-      stat.counts[pType]++;
-      if (pType === "null") {
-        stat.nullCount++;
-      } else {
-        const sem = detectSemanticType(val);
-        if (sem) {
-          stat.semanticCounts[sem] = (stat.semanticCounts[sem] || 0) + 1;
-        }
-      }
-    }
-  }
-
-  result(): InferredSchemaResult {
-    const columns: ColumnInferenceResult[] = [];
-
-    for (const [col, stat] of this.columnStats.entries()) {
-      const nonNullCount = stat.totalCount - stat.nullCount;
-      const nullable = stat.nullCount > 0;
-
-      if (nonNullCount === 0) {
-        columns.push({
-          name: col,
-          type: "null",
-          nullable: true,
-          confidence: 100,
-          typeBreakdown: { null: 100 },
-          sampleCount: stat.totalCount,
-          nullCount: stat.nullCount,
-        });
-        continue;
-      }
-
-      // Calculate confidence percentages
-      const breakdown: Record<string, number> = {};
-      let dominantType: ColumnType = "string";
-      let maxPercentage = 0;
-
-      const candidates: ColumnType[] = [
-        "integer",
-        "number",
-        "boolean",
-        "datetime",
-        "date",
-        "string",
-      ];
-
-      for (const t of candidates) {
-        const pct = (stat.counts[t] / nonNullCount) * 100;
-        if (pct > 0) {
-          breakdown[t] = Math.round(pct * 10) / 10;
-        }
-        if (pct > maxPercentage) {
-          maxPercentage = pct;
-          dominantType = t;
-        }
-      }
-
-      // If integer + number together make 100%, consider number
-      if (
-        dominantType === "integer" &&
-        stat.counts["number"] > 0 &&
-        (stat.counts["integer"] + stat.counts["number"]) / nonNullCount >= 0.95
-      ) {
-        dominantType = "number";
-        maxPercentage = ((stat.counts["integer"] + stat.counts["number"]) / nonNullCount) * 100;
-      }
-
-      let inferredType: ColumnType = dominantType;
-      let confidence = Math.round(maxPercentage * 10) / 10;
-
-      if (maxPercentage < 90 && Object.keys(breakdown).length > 1) {
-        inferredType = "mixed";
-      }
-
-      // Semantic type detection
-      let semantic: SemanticType | undefined;
-      let semanticConfidence: number | undefined;
-
-      for (const [sem, count] of Object.entries(stat.semanticCounts)) {
-        const semPct = (count / nonNullCount) * 100;
-        if (semPct >= 80) {
-          semantic = sem as SemanticType;
-          semanticConfidence = Math.round(semPct * 10) / 10;
-          break;
-        }
-      }
-
-      columns.push({
-        name: col,
-        type: inferredType,
-        nullable,
-        confidence,
-        semantic,
-        semanticConfidence,
-        typeBreakdown: breakdown,
-        sampleCount: stat.totalCount,
-        nullCount: stat.nullCount,
-      });
-    }
-
-    return {
-      totalRowsScanned: this.totalRows,
-      columns,
-    };
-  }
-}

package/src/analytics/semantic-types.ts

@@ -1,36 +0,0 @@
-import type { SemanticType } from "../core/types.js";
-
-const EMAIL_REGEX = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/;
-const URL_REGEX = /^https?:\/\/[^\s/$.?#].[^\s]*$/i;
-const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-const IPV4_REGEX = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
-const IPV6_REGEX = /^(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}$/i;
-const PHONE_REGEX = /^\+?[0-9\s\-().]{7,20}$/;
-
-const ISO_COUNTRY_CODES = new Set([
-  "US", "GB", "DE", "FR", "TR", "CA", "AU", "JP", "CN", "IN", "BR", "IT", "ES", "NL", "SE", "CH", "PL", "MX", "KR", "RU", "ZA", "EG", "SA", "AE", "SG", "NZ", "IE", "NO", "DK", "FI", "PT", "GR", "AT", "BE", "IL", "AR", "CL", "CO", "PE", "TH", "MY", "ID", "PH", "VN", "PK", "NG", "KE", "ZA"
-]);
-
-const ISO_CURRENCIES = new Set([
-  "USD", "EUR", "GBP", "TRY", "JPY", "CAD", "AUD", "CHF", "CNY", "INR", "BRL", "RUB", "KRW", "SEK", "NOK", "MXN", "SGD", "NZD", "HKD", "PLN", "ZAR", "AED", "SAR", "THB", "IDR"
-]);
-
-/**
- * Tests if a string value matches a specific semantic pattern.
- */
-export function detectSemanticType(value: unknown): SemanticType | null {
-  if (value === null || value === undefined) return null;
-  const str = String(value).trim();
-  if (str.length === 0) return null;
-
-  if (UUID_REGEX.test(str)) return "uuid";
-  if (EMAIL_REGEX.test(str)) return "email";
-  if (URL_REGEX.test(str)) return "url";
-  if (IPV4_REGEX.test(str)) return "ipv4";
-  if (IPV6_REGEX.test(str)) return "ipv6";
-  if (str.length >= 2 && str.length <= 3 && ISO_COUNTRY_CODES.has(str.toUpperCase())) return "country-code";
-  if (str.length === 3 && ISO_CURRENCIES.has(str.toUpperCase())) return "currency";
-  if (PHONE_REGEX.test(str) && /\d{4,}/.test(str) && (str.startsWith("+") || str.includes("-") || str.includes("("))) return "phone";
-
-  return null;
-}