rowpipe 1.0.0

package/dist/analytics/stats.d.ts

@@ -0,0 +1,102 @@
+import type { Aggregator, Row } from "../core/types.js";
+/**
+ * HyperLogLog (HLL) distinct count estimator with bounded O(1) memory.
+ * Uses m = 1024 registers (p = 10), standard error ~3.25%.
+ * Hardware Math.clz32 for instantaneous zero counting.
+ */
+export declare class HyperLogLog {
+    private p;
+    private m;
+    private registers;
+    private alphaMM;
+    constructor(p?: number);
+    add(value: unknown): void;
+    merge(other: HyperLogLog): void;
+    count(): number;
+}
+export interface NumericColumnStats {
+    count: number;
+    nullCount: number;
+    min: number | null;
+    max: number | null;
+    sum: number;
+    mean: number | null;
+    variance: number | null;
+    stddev: number | null;
+    approxDistinct: number;
+}
+export interface StringColumnStats {
+    count: number;
+    nullCount: number;
+    emptyCount: number;
+    minLength: number | null;
+    maxLength: number | null;
+    avgLength: number | null;
+    emptyCountRatio?: number;
+    approxDistinct: number;
+    topValues: Array<{
+        value: string;
+        count: number;
+    }>;
+}
+export interface ColumnStatsResult {
+    column: string;
+    type: "numeric" | "string";
+    numeric?: NumericColumnStats;
+    string?: StringColumnStats;
+}
+export interface DatasetStatsResult {
+    totalRows: number;
+    columns: Record<string, ColumnStatsResult>;
+}
+/**
+ * Online Welford statistics collector for a numeric column.
+ */
+export declare class NumericStatsCollector {
+    count: number;
+    nullCount: number;
+    min: number | null;
+    max: number | null;
+    sum: number;
+    mean: number;
+    M2: number;
+    hll: HyperLogLog;
+    add(val: unknown): void;
+    merge(other: NumericStatsCollector): void;
+    result(): NumericColumnStats;
+}
+/**
+ * Online statistics collector for a string column.
+ */
+export declare class StringStatsCollector {
+    count: number;
+    nullCount: number;
+    emptyCount: number;
+    minLength: number | null;
+    maxLength: number | null;
+    totalLength: number;
+    hll: HyperLogLog;
+    private frequencyMap;
+    private maxTrackedValues;
+    add(val: unknown): void;
+    result(): StringColumnStats;
+}
+/**
+ * Dataset-wide Aggregator collecting statistics across all columns in stream.
+ */
+export declare class DatasetStatsAggregator implements Aggregator<DatasetStatsResult> {
+    private totalRows;
+    private targetColumn?;
+    private numericCollectors;
+    private stringCollectors;
+    private columnTypeHints;
+    private handlers;
+    private initialized;
+    constructor(options?: {
+        column?: string;
+    });
+    private initHandler;
+    add(row: Row): void;
+    result(): DatasetStatsResult;
+}
+//# sourceMappingURL=stats.d.ts.map

package/dist/analytics/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/analytics/stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AAsBxD;;;;GAIG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,CAAC,CAAS;IAClB,OAAO,CAAC,CAAC,CAAS;IAClB,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,OAAO,CAAS;gBAEZ,CAAC,SAAK;IAYlB,GAAG,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAuBzB,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAQ/B,KAAK,IAAI,MAAM;CAmBhB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC3B,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAC7B,MAAM,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,qBAAa,qBAAqB;IACzB,KAAK,SAAK;IACV,SAAS,SAAK;IACd,GAAG,EAAE,MAAM,GAAG,IAAI,CAAQ;IAC1B,GAAG,EAAE,MAAM,GAAG,IAAI,CAAQ;IAC1B,GAAG,SAAK;IACR,IAAI,SAAK;IACT,EAAE,SAAK;IACP,GAAG,cAAqB;IAE/B,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IA0BvB,KAAK,CAAC,KAAK,EAAE,qBAAqB,GAAG,IAAI;IA+BzC,MAAM,IAAI,kBAAkB;CAgB7B;AAED;;GAEG;AACH,qBAAa,oBAAoB;IACxB,KAAK,SAAK;IACV,SAAS,SAAK;IACd,UAAU,SAAK;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAQ;IAChC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAQ;IAChC,WAAW,SAAK;IAChB,GAAG,cAAqB;IAC/B,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,gBAAgB,CAAQ;IAEhC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;IAwBvB,MAAM,IAAI,iBAAiB;CAiB5B;AASD;;GAEG;AACH,qBAAa,sBAAuB,YAAW,UAAU,CAAC,kBAAkB,CAAC;IAC3E,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAA4C;IACrE,OAAO,CAAC,gBAAgB,CAA2C;IACnE,OAAO,CAAC,eAAe,CAAmE;IAC1F,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,WAAW,CAAS;gBAEhB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;IAIzC,OAAO,CAAC,WAAW;IAiBnB,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI;IAoCnB,MAAM,IAAI,kBAAkB;CA4B7B"}

package/dist/analytics/stats.js

@@ -0,0 +1,307 @@
+/**
+ * Fast 32-bit FNV-1a hash function with 32-bit avalanche mixing.
+ * Zero BigInt allocations, runs at native CPU register speed.
+ */
+function hash32(str) {
+    let h = 0x811c9dc5;
+    const len = str.length;
+    for (let i = 0; i < len; i++) {
+        h = (h ^ str.charCodeAt(i)) >>> 0;
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    // Avalanche mixing
+    h ^= h >>> 16;
+    h = Math.imul(h, 0x85ebca6b) >>> 0;
+    h ^= h >>> 13;
+    h = Math.imul(h, 0xc2b2ae35) >>> 0;
+    h ^= h >>> 16;
+    return h >>> 0;
+}
+/**
+ * HyperLogLog (HLL) distinct count estimator with bounded O(1) memory.
+ * Uses m = 1024 registers (p = 10), standard error ~3.25%.
+ * Hardware Math.clz32 for instantaneous zero counting.
+ */
+export class HyperLogLog {
+    p;
+    m;
+    registers;
+    alphaMM;
+    constructor(p = 10) {
+        this.p = p;
+        this.m = 1 << p;
+        this.registers = new Uint8Array(this.m);
+        // Alpha constant calculation
+        if (this.m === 16)
+            this.alphaMM = 0.673 * this.m * this.m;
+        else if (this.m === 32)
+            this.alphaMM = 0.697 * this.m * this.m;
+        else if (this.m === 64)
+            this.alphaMM = 0.709 * this.m * this.m;
+        else
+            this.alphaMM = (0.7213 / (1 + 1.079 / this.m)) * this.m * this.m;
+    }
+    add(value) {
+        if (value === null || value === undefined)
+            return;
+        const str = String(value);
+        const hash = hash32(str);
+        // Low p bits for register index
+        const index = hash & (this.m - 1);
+        // Remaining (32 - p) bits for leading zeros
+        const w = hash >>> this.p;
+        // Single-cycle CPU instruction Math.clz32 for leading zero count
+        let leadingZeros;
+        if (w === 0) {
+            leadingZeros = 32 - this.p + 1;
+        }
+        else {
+            leadingZeros = Math.clz32(w) - this.p + 1;
+        }
+        if (leadingZeros > this.registers[index]) {
+            this.registers[index] = leadingZeros;
+        }
+    }
+    merge(other) {
+        for (let i = 0; i < this.m; i++) {
+            if (other.registers[i] > this.registers[i]) {
+                this.registers[i] = other.registers[i];
+            }
+        }
+    }
+    count() {
+        let sum = 0;
+        let zeros = 0;
+        for (let i = 0; i < this.m; i++) {
+            const val = this.registers[i];
+            sum += 2 ** -val;
+            if (val === 0)
+                zeros++;
+        }
+        let estimate = this.alphaMM / sum;
+        // Small range correction (Linear Counting)
+        if (estimate <= 2.5 * this.m && zeros > 0) {
+            estimate = this.m * Math.log(this.m / zeros);
+        }
+        return Math.round(estimate);
+    }
+}
+/**
+ * Online Welford statistics collector for a numeric column.
+ */
+export class NumericStatsCollector {
+    count = 0;
+    nullCount = 0;
+    min = null;
+    max = null;
+    sum = 0;
+    mean = 0;
+    M2 = 0; // sum of squared differences from the mean
+    hll = new HyperLogLog();
+    add(val) {
+        if (val === null || val === undefined || val === "") {
+            this.nullCount++;
+            return;
+        }
+        const num = typeof val === "number" ? val : Number(val);
+        if (Number.isNaN(num)) {
+            this.nullCount++;
+            return;
+        }
+        this.count++;
+        this.sum += num;
+        this.hll.add(num);
+        if (this.min === null || num < this.min)
+            this.min = num;
+        if (this.max === null || num > this.max)
+            this.max = num;
+        // Welford's online algorithm
+        const delta = num - this.mean;
+        this.mean += delta / this.count;
+        const delta2 = num - this.mean;
+        this.M2 += delta * delta2;
+    }
+    merge(other) {
+        if (other.count === 0) {
+            this.nullCount += other.nullCount;
+            return;
+        }
+        if (this.count === 0) {
+            this.count = other.count;
+            this.nullCount = other.nullCount;
+            this.min = other.min;
+            this.max = other.max;
+            this.sum = other.sum;
+            this.mean = other.mean;
+            this.M2 = other.M2;
+            this.hll.merge(other.hll);
+            return;
+        }
+        const totalCount = this.count + other.count;
+        const delta = other.mean - this.mean;
+        this.mean = (this.count * this.mean + other.count * other.mean) / totalCount;
+        this.M2 = this.M2 + other.M2 + (delta * delta * this.count * other.count) / totalCount;
+        this.count = totalCount;
+        this.nullCount += other.nullCount;
+        this.sum += other.sum;
+        if (this.min === null || (other.min !== null && other.min < this.min))
+            this.min = other.min;
+        if (this.max === null || (other.max !== null && other.max > this.max))
+            this.max = other.max;
+        this.hll.merge(other.hll);
+    }
+    result() {
+        const variance = this.count > 1 ? this.M2 / (this.count - 1) : this.count === 1 ? 0 : null;
+        const stddev = variance !== null ? Math.sqrt(variance) : null;
+        return {
+            count: this.count,
+            nullCount: this.nullCount,
+            min: this.min,
+            max: this.max,
+            sum: this.sum,
+            mean: this.count > 0 ? this.mean : null,
+            variance,
+            stddev,
+            approxDistinct: this.hll.count(),
+        };
+    }
+}
+/**
+ * Online statistics collector for a string column.
+ */
+export class StringStatsCollector {
+    count = 0;
+    nullCount = 0;
+    emptyCount = 0;
+    minLength = null;
+    maxLength = null;
+    totalLength = 0;
+    hll = new HyperLogLog();
+    frequencyMap = new Map();
+    maxTrackedValues = 1000;
+    add(val) {
+        if (val === null || val === undefined) {
+            this.nullCount++;
+            return;
+        }
+        const str = String(val);
+        if (str === "") {
+            this.emptyCount++;
+        }
+        this.count++;
+        this.totalLength += str.length;
+        this.hll.add(str);
+        if (this.minLength === null || str.length < this.minLength)
+            this.minLength = str.length;
+        if (this.maxLength === null || str.length > this.maxLength)
+            this.maxLength = str.length;
+        // Track top values bounded
+        if (this.frequencyMap.size < this.maxTrackedValues || this.frequencyMap.has(str)) {
+            this.frequencyMap.set(str, (this.frequencyMap.get(str) || 0) + 1);
+        }
+    }
+    result() {
+        const sorted = Array.from(this.frequencyMap.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 5)
+            .map(([value, count]) => ({ value, count }));
+        return {
+            count: this.count,
+            nullCount: this.nullCount,
+            emptyCount: this.emptyCount,
+            minLength: this.minLength,
+            maxLength: this.maxLength,
+            avgLength: this.count > 0 ? this.totalLength / this.count : null,
+            approxDistinct: this.hll.count(),
+            topValues: sorted,
+        };
+    }
+}
+/**
+ * Dataset-wide Aggregator collecting statistics across all columns in stream.
+ */
+export class DatasetStatsAggregator {
+    totalRows = 0;
+    targetColumn;
+    numericCollectors = new Map();
+    stringCollectors = new Map();
+    columnTypeHints = new Map();
+    handlers = [];
+    initialized = false;
+    constructor(options) {
+        this.targetColumn = options?.column;
+    }
+    initHandler(col) {
+        let numCollector = this.numericCollectors.get(col);
+        let strCollector = this.stringCollectors.get(col);
+        let hint = this.columnTypeHints.get(col);
+        if (!numCollector) {
+            numCollector = new NumericStatsCollector();
+            strCollector = new StringStatsCollector();
+            hint = { numericVotes: 0, totalVotes: 0 };
+            this.numericCollectors.set(col, numCollector);
+            this.stringCollectors.set(col, strCollector);
+            this.columnTypeHints.set(col, hint);
+        }
+        return { col, numCollector: numCollector, strCollector: strCollector, hint: hint };
+    }
+    add(row) {
+        this.totalRows++;
+        if (!this.initialized) {
+            const columns = this.targetColumn ? [this.targetColumn] : Object.keys(row);
+            this.handlers = new Array(columns.length);
+            for (let i = 0; i < columns.length; i++) {
+                this.handlers[i] = this.initHandler(columns[i]);
+            }
+            this.initialized = true;
+        }
+        const handlers = this.handlers;
+        const len = handlers.length;
+        for (let i = 0; i < len; i++) {
+            const h = handlers[i];
+            const val = row[h.col];
+            h.strCollector.add(val);
+            if (val !== null && val !== undefined && val !== "") {
+                h.hint.totalVotes++;
+                const num = typeof val === "number" ? val : Number(val);
+                if (!Number.isNaN(num) && typeof val !== "boolean") {
+                    h.hint.numericVotes++;
+                    h.numCollector.add(num);
+                }
+                else {
+                    h.numCollector.add(null);
+                }
+            }
+            else {
+                h.numCollector.add(null);
+            }
+        }
+    }
+    result() {
+        const columns = {};
+        for (const [col, hint] of this.columnTypeHints.entries()) {
+            const isNumeric = hint.totalVotes > 0 && hint.numericVotes / hint.totalVotes >= 0.8;
+            const numCollector = this.numericCollectors.get(col);
+            const strCollector = this.stringCollectors.get(col);
+            if (isNumeric) {
+                columns[col] = {
+                    column: col,
+                    type: "numeric",
+                    numeric: numCollector.result(),
+                };
+            }
+            else {
+                columns[col] = {
+                    column: col,
+                    type: "string",
+                    string: strCollector.result(),
+                };
+            }
+        }
+        return {
+            totalRows: this.totalRows,
+            columns,
+        };
+    }
+}
+//# sourceMappingURL=stats.js.map

package/dist/analytics/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/analytics/stats.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,SAAS,MAAM,CAAC,GAAW;IACzB,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IACD,mBAAmB;IACnB,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACd,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACd,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACd,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,WAAW;IACd,CAAC,CAAS;IACV,CAAC,CAAS;IACV,SAAS,CAAa;IACtB,OAAO,CAAS;IAExB,YAAY,CAAC,GAAG,EAAE;QAChB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACX,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAExC,6BAA6B;QAC7B,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE;YAAE,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;aACrD,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE;YAAE,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;aAC1D,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE;YAAE,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;;YAC1D,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,GAAG,CAAC,KAAc;QAChB,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAClD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAEzB,gCAAgC;QAChC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClC,4CAA4C;QAC5C,MAAM,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QAE1B,iEAAiE;QACjE,IAAI,YAAoB,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,YAAY,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAE,EAAE,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAkB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,EAAE,CAAC;gBAC7C,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;YAC1C,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;YAC/B,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;YACjB,IAAI,GAAG,KAAK,CAAC;gBAAE,KAAK,EAAE,CAAC;QACzB,CAAC;QAED,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC;QAElC,2CAA2C;QAC3C,IAAI,QAAQ,IAAI,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC1C,QAAQ,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC/C,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;CACF;AAsCD;;GAEG;AACH,MAAM,OAAO,qBAAqB;IACzB,KAAK,GAAG,CAAC,CAAC;IACV,SAAS,GAAG,CAAC,CAAC;IACd,GAAG,GAAkB,IAAI,CAAC;IAC1B,GAAG,GAAkB,IAAI,CAAC;IAC1B,GAAG,GAAG,CAAC,CAAC;IACR,IAAI,GAAG,CAAC,CAAC;IACT,EAAE,GAAG,CAAC,CAAC,CAAC,2CAA2C;IACnD,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;IAE/B,GAAG,CAAC,GAAY;QACd,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACpD,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxD,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElB,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG;YAAE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACxD,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG;YAAE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QAExD,6BAA6B;QAC7B,MAAM,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;QAC9B,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAChC,MAAM,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,CAAC,EAAE,IAAI,KAAK,GAAG,MAAM,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAA4B;QAChC,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;YAClC,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YACzB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;YACjC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YACrB,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YACrB,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YACrB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACvB,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QAErC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;QAC7E,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC;QACvF,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;QAEtB,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QAC5F,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QAC5F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM;QACJ,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3F,MAAM,MAAM,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE9D,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,IAAI,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACvC,QAAQ;YACR,MAAM;YACN,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE;SACjC,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,oBAAoB;IACxB,KAAK,GAAG,CAAC,CAAC;IACV,SAAS,GAAG,CAAC,CAAC;IACd,UAAU,GAAG,CAAC,CAAC;IACf,SAAS,GAAkB,IAAI,CAAC;IAChC,SAAS,GAAkB,IAAI,CAAC;IAChC,WAAW,GAAG,CAAC,CAAC;IAChB,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;IACvB,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,gBAAgB,GAAG,IAAI,CAAC;IAEhC,GAAG,CAAC,GAAY;QACd,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtC,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACf,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,MAAM,CAAC;QAC/B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElB,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC;QACxF,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC;QAExF,2BAA2B;QAC3B,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACjF,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,MAAM;QACJ,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;aACnD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAE/C,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;YAChE,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE;YAChC,SAAS,EAAE,MAAM;SAClB,CAAC;IACJ,CAAC;CACF;AASD;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACzB,SAAS,GAAG,CAAC,CAAC;IACd,YAAY,CAAU;IACtB,iBAAiB,GAAG,IAAI,GAAG,EAAiC,CAAC;IAC7D,gBAAgB,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC3D,eAAe,GAAG,IAAI,GAAG,EAAwD,CAAC;IAClF,QAAQ,GAAoB,EAAE,CAAC;IAC/B,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,OAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,MAAM,CAAC;IACtC,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,IAAI,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzC,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,YAAY,GAAG,IAAI,qBAAqB,EAAE,CAAC;YAC3C,YAAY,GAAG,IAAI,oBAAoB,EAAE,CAAC;YAC1C,IAAI,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YAC9C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YAC7C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,YAAa,EAAE,YAAY,EAAE,YAAa,EAAE,IAAI,EAAE,IAAK,EAAE,CAAC;IACxF,CAAC;IAED,GAAG,CAAC,GAAQ;QACV,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3E,IAAI,CAAC,QAAQ,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YACnD,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAEvB,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExB,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;gBACpD,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACxD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC;oBACnD,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC1B,CAAC;qBAAM,CAAC;oBACN,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM;QACJ,MAAM,OAAO,GAAsC,EAAE,CAAC;QAEtD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,GAAG,CAAC;YACpF,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YACtD,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YAErD,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,GAAG;oBACb,MAAM,EAAE,GAAG;oBACX,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE;iBAC/B,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,GAAG;oBACb,MAAM,EAAE,GAAG;oBACX,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE;iBAC9B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO;SACR,CAAC;IACJ,CAAC;CACF"}

package/dist/analytics/validator.d.ts

@@ -0,0 +1,32 @@
+import type { Aggregator, Row } from "../core/types.js";
+export interface ColumnSchemaRule {
+    type?: string;
+    nullable?: boolean;
+    format?: string;
+}
+export type ValidationSchemaDefinition = Record<string, ColumnSchemaRule>;
+export interface ColumnViolationSummary {
+    column: string;
+    rule: string;
+    violationsCount: number;
+    sampleInvalidValues: unknown[];
+}
+export interface ValidationReport {
+    totalRows: number;
+    validRows: number;
+    invalidRows: number;
+    isValid: boolean;
+    violations: ColumnViolationSummary[];
+}
+export declare class SchemaValidatorAggregator implements Aggregator<ValidationReport> {
+    private schema;
+    private totalRows;
+    private validRows;
+    private invalidRows;
+    private violationsMap;
+    constructor(schema: ValidationSchemaDefinition);
+    private addViolation;
+    add(row: Row): void;
+    result(): ValidationReport;
+}
+//# sourceMappingURL=validator.d.ts.map

package/dist/analytics/validator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validator.d.ts","sourceRoot":"","sources":["../../src/analytics/validator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AAIxD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,0BAA0B,GAAG,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;AAE1E,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,OAAO,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,sBAAsB,EAAE,CAAC;CACtC;AAED,qBAAa,yBAA0B,YAAW,UAAU,CAAC,gBAAgB,CAAC;IAC5E,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,aAAa,CAA6C;gBAEtD,MAAM,EAAE,0BAA0B;IAI9C,OAAO,CAAC,YAAY;IAkBpB,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI;IAoEnB,MAAM,IAAI,gBAAgB;CAS3B"}

package/dist/analytics/validator.js

@@ -0,0 +1,96 @@
+import { classifyPrimitiveType } from "./schema-inference.js";
+import { detectSemanticType } from "./semantic-types.js";
+export class SchemaValidatorAggregator {
+    schema;
+    totalRows = 0;
+    validRows = 0;
+    invalidRows = 0;
+    violationsMap = new Map();
+    constructor(schema) {
+        this.schema = schema;
+    }
+    addViolation(column, rule, value) {
+        const key = `${column}::${rule}`;
+        if (!this.violationsMap.has(key)) {
+            this.violationsMap.set(key, {
+                column,
+                rule,
+                violationsCount: 0,
+                sampleInvalidValues: [],
+            });
+        }
+        const v = this.violationsMap.get(key);
+        v.violationsCount++;
+        if (v.sampleInvalidValues.length < 5) {
+            v.sampleInvalidValues.push(value);
+        }
+    }
+    add(row) {
+        this.totalRows++;
+        let rowHasError = false;
+        for (const [colName, rule] of Object.entries(this.schema)) {
+            const val = row[colName];
+            const isNull = val === null || val === undefined || val === "";
+            // Nullability check
+            if (isNull) {
+                if (rule.nullable === false) {
+                    this.addViolation(colName, "null not allowed", val);
+                    rowHasError = true;
+                }
+                continue;
+            }
+            // Type check
+            if (rule.type) {
+                const actualType = classifyPrimitiveType(val);
+                const expectedType = rule.type.toLowerCase();
+                let matchesType = false;
+                if (expectedType === "string") {
+                    matchesType = true;
+                }
+                else if (expectedType === "integer" || expectedType === "int") {
+                    matchesType = actualType === "integer";
+                }
+                else if (expectedType === "number" || expectedType === "float") {
+                    matchesType = actualType === "integer" || actualType === "number";
+                }
+                else if (expectedType === "boolean" || expectedType === "bool") {
+                    matchesType = actualType === "boolean";
+                }
+                else if (expectedType === "date") {
+                    matchesType = actualType === "date" || actualType === "datetime";
+                }
+                else if (expectedType === "datetime") {
+                    matchesType = actualType === "datetime";
+                }
+                if (!matchesType) {
+                    this.addViolation(colName, `expected ${rule.type}`, val);
+                    rowHasError = true;
+                }
+            }
+            // Format check (semantic type)
+            if (rule.format) {
+                const detected = detectSemanticType(val);
+                if (detected !== rule.format.toLowerCase()) {
+                    this.addViolation(colName, `expected format ${rule.format}`, val);
+                    rowHasError = true;
+                }
+            }
+        }
+        if (rowHasError) {
+            this.invalidRows++;
+        }
+        else {
+            this.validRows++;
+        }
+    }
+    result() {
+        return {
+            totalRows: this.totalRows,
+            validRows: this.validRows,
+            invalidRows: this.invalidRows,
+            isValid: this.invalidRows === 0,
+            violations: Array.from(this.violationsMap.values()),
+        };
+    }
+}
+//# sourceMappingURL=validator.js.map

package/dist/analytics/validator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"validator.js","sourceRoot":"","sources":["../../src/analytics/validator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAyBzD,MAAM,OAAO,yBAAyB;IAC5B,MAAM,CAA6B;IACnC,SAAS,GAAG,CAAC,CAAC;IACd,SAAS,GAAG,CAAC,CAAC;IACd,WAAW,GAAG,CAAC,CAAC;IAChB,aAAa,GAAG,IAAI,GAAG,EAAkC,CAAC;IAElE,YAAY,MAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,YAAY,CAAC,MAAc,EAAE,IAAY,EAAE,KAAc;QAC/D,MAAM,GAAG,GAAG,GAAG,MAAM,KAAK,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC1B,MAAM;gBACN,IAAI;gBACJ,eAAe,EAAE,CAAC;gBAClB,mBAAmB,EAAE,EAAE;aACxB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QACvC,CAAC,CAAC,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,GAAG,CAAC,GAAQ;QACV,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,WAAW,GAAG,KAAK,CAAC;QAExB,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,MAAM,GAAG,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,CAAC;YAE/D,oBAAoB;YACpB,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;oBAC5B,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;oBACpD,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,aAAa;YACb,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,MAAM,UAAU,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAE7C,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,IAAI,YAAY,KAAK,QAAQ,EAAE,CAAC;oBAC9B,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;qBAAM,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,KAAK,EAAE,CAAC;oBAChE,WAAW,GAAG,UAAU,KAAK,SAAS,CAAC;gBACzC,CAAC;qBAAM,IAAI,YAAY,KAAK,QAAQ,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;oBACjE,WAAW,GAAG,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,QAAQ,CAAC;gBACpE,CAAC;qBAAM,IAAI,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;oBACjE,WAAW,GAAG,UAAU,KAAK,SAAS,CAAC;gBACzC,CAAC;qBAAM,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;oBACnC,WAAW,GAAG,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,UAAU,CAAC;gBACnE,CAAC;qBAAM,IAAI,YAAY,KAAK,UAAU,EAAE,CAAC;oBACvC,WAAW,GAAG,UAAU,KAAK,UAAU,CAAC;gBAC1C,CAAC;gBAED,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,IAAI,CAAC,YAAY,CACf,OAAO,EACP,YAAY,IAAI,CAAC,IAAI,EAAE,EACvB,GAAG,CACJ,CAAC;oBACF,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACzC,IAAI,QAAQ,KAAK,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC3C,IAAI,CAAC,YAAY,CACf,OAAO,EACP,mBAAmB,IAAI,CAAC,MAAM,EAAE,EAChC,GAAG,CACJ,CAAC;oBACF,WAAW,GAAG,IAAI,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAED,MAAM;QACJ,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,WAAW,KAAK,CAAC;YAC/B,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;CACF"}

package/dist/cli/commands/cast.d.ts

@@ -0,0 +1,13 @@
+import { type CastOnError } from "../../transforms/cast.js";
+export interface CastCommandOptions {
+    onError?: CastOnError;
+    from?: string;
+    to?: string;
+    sheet?: string;
+    delimiter?: string;
+    batchSize?: string | number;
+    quiet?: boolean;
+    noProgress?: boolean;
+}
+export declare function castCommand(inputPath: string | undefined, specs: string[], options?: CastCommandOptions): Promise<void>;
+//# sourceMappingURL=cast.d.ts.map

package/dist/cli/commands/cast.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cast.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/cast.ts"],"names":[],"mappings":"AAGA,OAAO,EAAe,KAAK,WAAW,EAAkB,MAAM,0BAA0B,CAAC;AAKzF,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,WAAW,CAC/B,SAAS,oBAAM,EACf,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAoCf"}

package/dist/cli/commands/cast.js

@@ -0,0 +1,39 @@
+import { createPipeline } from "../../core/pipeline.js";
+import { createReader, inferFormatFromPath as inferReaderFormat } from "../../readers/index.js";
+import { createWriter } from "../../writers/index.js";
+import { castColumns, parseCastSpecs } from "../../transforms/cast.js";
+import { openReadableStream } from "../../utils/compression.js";
+import { logMemoryDebug } from "../../utils/formatting.js";
+import { ProgressReporter } from "../../utils/progress.js";
+export async function castCommand(inputPath = "-", specs, options = {}) {
+    const effectiveBatchSize = Number(options.batchSize) || 1000;
+    const progress = new ProgressReporter(options);
+    let fromFormat = options.from?.toLowerCase();
+    if (!fromFormat && inputPath !== "-") {
+        fromFormat = inferReaderFormat(inputPath) ?? undefined;
+    }
+    if (!fromFormat) {
+        fromFormat = "csv";
+    }
+    const toFormat = options.to?.toLowerCase() || (fromFormat === "xlsx" ? "csv" : fromFormat);
+    const mapping = parseCastSpecs(specs);
+    const inputStream = openReadableStream(inputPath);
+    const reader = createReader(inputStream, {
+        format: fromFormat,
+        sheet: options.sheet,
+        delimiter: options.delimiter,
+        batchSize: effectiveBatchSize,
+        filePath: inputPath,
+    });
+    const writer = createWriter(process.stdout, {
+        format: toFormat,
+        delimiter: options.delimiter,
+    });
+    const pipeline = createPipeline(reader, { batchSize: effectiveBatchSize });
+    pipeline.pipe(castColumns(mapping, { onError: options.onError }));
+    pipeline.onProgress((info) => progress.update(info));
+    await pipeline.to(writer);
+    progress.done();
+    logMemoryDebug();
+}
+//# sourceMappingURL=cast.js.map

package/dist/cli/commands/cast.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cast.js","sourceRoot":"","sources":["../../../src/cli/commands/cast.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,mBAAmB,IAAI,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAoB,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAa3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAS,GAAG,GAAG,EACf,KAAe,EACf,UAA8B,EAAE;IAEhC,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC7C,IAAI,CAAC,UAAU,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACrC,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,EAAE;QACvC,MAAM,EAAE,UAAU;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,SAAS;KACpB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;QAC1C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC3E,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAClE,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChB,cAAc,EAAE,CAAC;AACnB,CAAC"}

package/dist/cli/commands/convert.d.ts

@@ -0,0 +1,15 @@
+export interface ConvertCommandOptions {
+    from?: string;
+    to?: string;
+    sheet?: string;
+    allSheets?: boolean;
+    outDir?: string;
+    path?: string;
+    delimiter?: string;
+    header?: boolean;
+    batchSize?: string | number;
+    quiet?: boolean;
+    noProgress?: boolean;
+}
+export declare function convertCommand(inputPath?: string, outputPath?: string, options?: ConvertCommandOptions): Promise<void>;
+//# sourceMappingURL=convert.d.ts.map

package/dist/cli/commands/convert.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/convert.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,qBAAqB;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,cAAc,CAClC,SAAS,SAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,IAAI,CAAC,CAmGf"}