rowpipe 1.0.0 → 1.1.0

package/src/writers/json.ts

@@ -1,52 +0,0 @@
-import { createWriteStream } from "node:fs";
-import type { Writable } from "node:stream";
-import type { DataStream, TabularWriter, WriterOptions } from "../core/types.js";
-
-export class JSONWriter implements TabularWriter {
-  private output: Writable | string;
-  private createdStream?: Writable;
-
-  constructor(output: Writable | string) {
-    this.output = output;
-  }
-
-  private getOutputStream(): Writable {
-    if (typeof this.output === "string") {
-      this.createdStream = createWriteStream(this.output);
-      return this.createdStream;
-    }
-    return this.output;
-  }
-
-  private async writeChunk(stream: Writable, data: string): Promise<void> {
-    if (!stream.write(data)) {
-      await new Promise<void>((resolve) => stream.once("drain", resolve));
-    }
-  }
-
-  async write(dataStream: DataStream, _options?: WriterOptions): Promise<void> {
-    const outStream = this.getOutputStream();
-    let isFirstRow = true;
-
-    await this.writeChunk(outStream, "[\n");
-
-    for await (const batch of dataStream) {
-      for (const row of batch.rows) {
-        const prefix = isFirstRow ? "  " : ",\n  ";
-        isFirstRow = false;
-        await this.writeChunk(outStream, `${prefix}${JSON.stringify(row)}`);
-      }
-    }
-
-    await this.writeChunk(outStream, "\n]\n");
-  }
-
-  async close(): Promise<void> {
-    if (this.createdStream) {
-      await new Promise<void>((resolve, reject) => {
-        this.createdStream!.end(() => resolve());
-        this.createdStream!.on("error", reject);
-      });
-    }
-  }
-}

package/src/writers/jsonl.ts

@@ -1,49 +0,0 @@
-import { createWriteStream } from "node:fs";
-import type { Writable } from "node:stream";
-import type { DataStream, TabularWriter, WriterOptions } from "../core/types.js";
-
-export class JSONLWriter implements TabularWriter {
-  private output: Writable | string;
-  private createdStream?: Writable;
-
-  constructor(output: Writable | string) {
-    this.output = output;
-  }
-
-  private getOutputStream(): Writable {
-    if (typeof this.output === "string") {
-      this.createdStream = createWriteStream(this.output);
-      return this.createdStream;
-    }
-    return this.output;
-  }
-
-  private async writeChunk(stream: Writable, data: string): Promise<void> {
-    if (!stream.write(data)) {
-      await new Promise<void>((resolve) => stream.once("drain", resolve));
-    }
-  }
-
-  async write(dataStream: DataStream, _options?: WriterOptions): Promise<void> {
-    const outStream = this.getOutputStream();
-
-    for await (const batch of dataStream) {
-      let chunkText = "";
-      for (const row of batch.rows) {
-        chunkText += JSON.stringify(row) + "\n";
-      }
-      if (chunkText.length > 0) {
-        await this.writeChunk(outStream, chunkText);
-      }
-    }
-  }
-
-  async close(): Promise<void> {
-    if (this.createdStream) {
-      await new Promise<void>((resolve, reject) => {
-        this.createdStream!.end(() => resolve());
-        this.createdStream!.on("error", reject);
-      });
-    }
-  }
-}

package/src/writers/xlsx.ts

@@ -1,87 +0,0 @@
-import type { Writable } from "node:stream";
-import writeXlsxFile from "write-excel-file/node";
-import type { DataStream, TabularWriter, WriterOptions } from "../core/types.js";
-
-export interface XLSXWriterOptions extends WriterOptions {
-  sheet?: string;
-}
-
-export class XLSXWriter implements TabularWriter {
-  private output: Writable | string;
-  private options: XLSXWriterOptions;
-
-  constructor(output: Writable | string, options: XLSXWriterOptions = {}) {
-    this.output = output;
-    this.options = { ...options };
-  }
-
-  async write(dataStream: DataStream, options?: WriterOptions): Promise<void> {
-    const mergedOptions: XLSXWriterOptions = {
-      ...this.options,
-      ...options,
-    };
-    const sheetName = mergedOptions.sheet || "Sheet1";
-
-    let headers: string[] | null = null;
-    const grid: Array<Array<{ value?: string | number | boolean | Date; fontWeight?: string }>> = [];
-
-    for await (const batch of dataStream) {
-      if (batch.rows.length === 0) continue;
-
-      if (headers === null) {
-        const keySet = new Set<string>();
-        for (const row of batch.rows) {
-          for (const key of Object.keys(row)) {
-            keySet.add(key);
-          }
-        }
-        headers = Array.from(keySet);
-
-        // Header row
-        grid.push(
-          headers.map((h) => ({
-            value: h,
-            fontWeight: "bold",
-          }))
-        );
-      }
-
-      for (const row of batch.rows) {
-        grid.push(
-          headers.map((h) => {
-            const val = row[h];
-            if (val === null || val === undefined || val === "") {
-              return { value: undefined };
-            }
-            if (typeof val === "number" || typeof val === "boolean" || typeof val === "string") {
-              return { value: val };
-            }
-            if (val instanceof Date) {
-              return { value: val };
-            }
-            return { value: String(val) };
-          })
-        );
-      }
-    }
-
-    if (grid.length === 0) {
-      grid.push([{ value: "" }]);
-    }
-
-    const writerInstance = (writeXlsxFile as unknown as (data: unknown, opts: unknown) => {
-      toFile: (path: string) => Promise<void>;
-      toStream: (stream: unknown) => Promise<void>;
-    })(grid, { sheet: sheetName });
-
-    if (typeof this.output === "string") {
-      await writerInstance.toFile(this.output);
-    } else {
-      await writerInstance.toStream(this.output);
-    }
-  }
-
-  async close(): Promise<void> {
-    // Handled on write
-  }
-}

package/tests/analytics.test.ts

@@ -1,119 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { NumericStatsCollector, HyperLogLog, DatasetStatsAggregator } from "../src/analytics/stats.js";
-import { SchemaInferenceAggregator } from "../src/analytics/schema-inference.js";
-import { detectSemanticType } from "../src/analytics/semantic-types.js";
-import { SchemaValidatorAggregator } from "../src/analytics/validator.js";
-import type { Row } from "../src/core/types.js";
-
-describe("Welford Numeric Statistics", () => {
-  it("should calculate exact mean, variance, stddev, sum, min, max", () => {
-    const collector = new NumericStatsCollector();
-    const values = [10, 20, 30, 40, 50]; // mean = 30, variance = 250, stddev = 15.811...
-
-    for (const val of values) {
-      collector.add(val);
-    }
-
-    const res = collector.result();
-    expect(res.count).toBe(5);
-    expect(res.nullCount).toBe(0);
-    expect(res.min).toBe(10);
-    expect(res.max).toBe(50);
-    expect(res.sum).toBe(150);
-    expect(res.mean).toBe(30);
-    expect(res.variance).toBe(250);
-    expect(res.stddev).toBeCloseTo(15.811388, 4);
-  });
-
-  it("should handle null and missing values properly", () => {
-    const collector = new NumericStatsCollector();
-    collector.add(10);
-    collector.add(null);
-    collector.add("");
-    collector.add(30);
-
-    const res = collector.result();
-    expect(res.count).toBe(2);
-    expect(res.nullCount).toBe(2);
-    expect(res.mean).toBe(20);
-  });
-});
-
-describe("HyperLogLog Distinct Estimator", () => {
-  it("should estimate cardinality accurately with bounded memory", () => {
-    const hll = new HyperLogLog(10);
-    const uniqueCount = 5000;
-
-    for (let i = 0; i < uniqueCount; i++) {
-      hll.add(`user_${i}`);
-    }
-
-    const estimated = hll.count();
-    // With p=10, error is typically within ~5%
-    const errorRatio = Math.abs(estimated - uniqueCount) / uniqueCount;
-    expect(errorRatio).toBeLessThan(0.08);
-  });
-});
-
-describe("Schema Inference", () => {
-  it("should infer types, nullability, and confidence", () => {
-    const rows: Row[] = [
-      { id: "1", name: "Alice", email: "alice@test.com", age: "30", created_at: "2026-01-01" },
-      { id: "2", name: "Bob", email: "bob@test.com", age: "25", created_at: "2026-01-02" },
-      { id: "3", name: "Charlie", email: null, age: "40", created_at: "2026-01-03" },
-    ];
-
-    const agg = new SchemaInferenceAggregator();
-    for (const row of rows) agg.add(row);
-
-    const schema = agg.result();
-    const colMap = new Map(schema.columns.map((c) => [c.name, c]));
-
-    expect(colMap.get("id")?.type).toBe("integer");
-    expect(colMap.get("id")?.nullable).toBe(false);
-
-    expect(colMap.get("email")?.type).toBe("string");
-    expect(colMap.get("email")?.nullable).toBe(true);
-    expect(colMap.get("email")?.semantic).toBe("email");
-
-    expect(colMap.get("age")?.type).toBe("integer");
-    expect(colMap.get("created_at")?.type).toBe("date");
-  });
-});
-
-describe("Semantic Type Detector", () => {
-  it("should detect emails, urls, uuids, and ips", () => {
-    expect(detectSemanticType("user@example.com")).toBe("email");
-    expect(detectSemanticType("https://google.com/path")).toBe("url");
-    expect(detectSemanticType("123e4567-e89b-12d3-a456-426614174000")).toBe("uuid");
-    expect(detectSemanticType("192.168.1.1")).toBe("ipv4");
-    expect(detectSemanticType("USD")).toBe("currency");
-    expect(detectSemanticType("TR")).toBe("country-code");
-  });
-});
-
-describe("Schema Validator", () => {
-  it("should validate stream and report violations", () => {
-    const schema = {
-      id: { type: "integer", nullable: false },
-      email: { type: "string", nullable: false, format: "email" },
-      age: { type: "integer", nullable: true },
-    };
-
-    const rows: Row[] = [
-      { id: 1, email: "valid@test.com", age: 30 },
-      { id: 2, email: null, age: 25 }, // email null violation
-      { id: "bad_id", email: "bad@test.com", age: "not_a_num" }, // id and age type violations
-    ];
-
-    const validator = new SchemaValidatorAggregator(schema);
-    for (const row of rows) validator.add(row);
-
-    const report = validator.result();
-    expect(report.totalRows).toBe(3);
-    expect(report.validRows).toBe(1);
-    expect(report.invalidRows).toBe(2);
-    expect(report.isValid).toBe(false);
-    expect(report.violations.length).toBeGreaterThanOrEqual(3);
-  });
-});

package/tests/cli-integration.test.ts

@@ -1,187 +0,0 @@
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { exec } from "node:child_process";
-import { promisify } from "node:util";
-import { promises as fs } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import writeXlsxFile from "write-excel-file/node";
-
-const execAsync = promisify(exec);
-const cliPath = join(process.cwd(), "dist/cli/index.js");
-
-describe("CLI Integration Tests", { timeout: 30000 }, () => {
-  const tempDir = join(tmpdir(), `rowpipe_cli_test_${Date.now()}`);
-  const sampleCsv = join(tempDir, "sample.csv");
-  const sampleSchema = join(tempDir, "sample.schema.json");
-  const sampleXlsx = join(tempDir, "workbook.xlsx");
-
-  beforeAll(async () => {
-    await fs.mkdir(tempDir, { recursive: true });
-
-    // Build project to ensure dist is up to date
-    await execAsync(`export PATH=$PATH:/usr/local/bin:/opt/homebrew/bin:~/.nvm/versions/node/$(ls ~/.nvm/versions/node 2>/dev/null | tail -n 1)/bin; npm run build`);
-
-    // Create test CSV
-    const csvContent = `id,name,age,email,revenue\n1,Alice,30,alice@test.com,150.50\n2,Bob,17,bob@test.com,50.00\n3,Charlie,45,charlie@test.com,500.25\n4,David,15,david@test.com,10.00\n`;
-    await fs.writeFile(sampleCsv, csvContent, "utf8");
-
-    // Create test schema
-    const schemaContent = JSON.stringify({
-      id: { type: "integer", nullable: false },
-      name: { type: "string", nullable: false },
-      age: { type: "integer", nullable: false },
-      email: { type: "string", nullable: false, format: "email" },
-      revenue: { type: "number", nullable: false },
-    });
-    await fs.writeFile(sampleSchema, schemaContent, "utf8");
-
-    // Create multi-sheet XLSX workbook
-    const usersSheet = [
-      [{ value: "id", fontWeight: "bold" }, { value: "name", fontWeight: "bold" }, { value: "email", fontWeight: "bold" }],
-      [{ value: 1 }, { value: "Alice" }, { value: "alice@example.com" }],
-      [{ value: 2 }, { value: "Bob" }, { value: "bob@example.com" }],
-    ];
-    const ordersSheet = [
-      [{ value: "order_id", fontWeight: "bold" }, { value: "user_id", fontWeight: "bold" }, { value: "total", fontWeight: "bold" }],
-      [{ value: "ORD-101" }, { value: 1 }, { value: 99.9 }],
-      [{ value: "ORD-102" }, { value: 2 }, { value: 149.5 }],
-      [{ value: "ORD-103" }, { value: 1 }, { value: 25.0 }],
-    ];
-
-    await (writeXlsxFile as unknown as (sheets: unknown[], opts?: unknown) => { toFile: (path: string) => Promise<void> })(
-      [
-        { data: usersSheet, sheet: "Users" },
-        { data: ordersSheet, sheet: "Orders" },
-      ]
-    ).toFile(sampleXlsx);
-  });
-
-  afterAll(async () => {
-    await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
-  });
-
-  it("should inspect CSV file", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} inspect ${sampleCsv}`);
-    expect(stdout).toContain("Rows: 4");
-    expect(stdout).toContain("Columns: 5");
-    expect(stdout).toContain("email");
-  });
-
-  it("should inspect with --json output", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} inspect ${sampleCsv} --json`);
-    const json = JSON.parse(stdout);
-    expect(json.rows).toBe(4);
-    expect(json.columnsCount).toBe(5);
-  });
-
-  it("should convert CSV to JSONL", async () => {
-    const outputJsonl = join(tempDir, "output.jsonl");
-    await execAsync(`node ${cliPath} convert ${sampleCsv} ${outputJsonl}`);
-
-    const content = await fs.readFile(outputJsonl, "utf8");
-    const lines = content.trim().split("\n");
-    expect(lines.length).toBe(4);
-    expect(JSON.parse(lines[0]!).name).toBe("Alice");
-  });
-
-  it("should infer schema", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} schema ${sampleCsv}`);
-    expect(stdout).toContain("COLUMN");
-    expect(stdout).toContain("id");
-    expect(stdout).toContain("integer");
-    expect(stdout).toContain("email");
-  });
-
-  it("should calculate statistics", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} stats ${sampleCsv}`);
-    expect(stdout).toContain("Total Rows: 4");
-    expect(stdout).toContain("revenue (numeric)");
-    expect(stdout).toContain("mean");
-  });
-
-  it("should filter stream", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} filter ${sampleCsv} "age >= 18"`);
-    const lines = stdout.trim().split("\n");
-    expect(lines.length).toBe(3);
-    expect(stdout).toContain("Alice");
-    expect(stdout).toContain("Charlie");
-    expect(stdout).not.toContain("David");
-  });
-
-  it("should select columns", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} select ${sampleCsv} id,email`);
-    const lines = stdout.trim().split("\n");
-    expect(lines[0]).toBe("id,email");
-    expect(lines[1]).toBe("1,alice@test.com");
-  });
-
-  it("should execute Unix pipe chain", async () => {
-    const cmd = `cat ${sampleCsv} | node ${cliPath} filter - "age >= 18" | node ${cliPath} select - id,name,age | node ${cliPath} convert - --from csv --to jsonl`;
-    const { stdout } = await execAsync(cmd);
-    const lines = stdout.trim().split("\n");
-    expect(lines.length).toBe(2);
-
-    const row1 = JSON.parse(lines[0]!);
-    expect(row1).toEqual({ id: "1", name: "Alice", age: "30" });
-    const row2 = JSON.parse(lines[1]!);
-    expect(row2).toEqual({ id: "3", name: "Charlie", age: "45" });
-  });
-
-  it("should validate CSV against schema", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} validate ${sampleCsv} --schema ${sampleSchema}`);
-    expect(stdout).toContain("4 rows scanned");
-    expect(stdout).toContain("4 valid");
-    expect(stdout).toContain("0 invalid");
-  });
-
-  it("should inspect multi-sheet XLSX workbook", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} inspect ${sampleXlsx}`);
-    expect(stdout).toContain("Sheets: 2");
-    expect(stdout).toContain("Users");
-    expect(stdout).toContain("Orders");
-  });
-
-  it("should inspect specific XLSX sheet", async () => {
-    const { stdout } = await execAsync(`node ${cliPath} inspect ${sampleXlsx} --sheet Orders`);
-    expect(stdout).toContain("order_id");
-    expect(stdout).toContain("total");
-    expect(stdout).toContain("Rows: 3");
-  });
-
-  it("should convert specific XLSX sheet to CSV", async () => {
-    const ordersCsv = join(tempDir, "orders.csv");
-    await execAsync(`node ${cliPath} convert ${sampleXlsx} ${ordersCsv} --sheet Orders`);
-    const content = await fs.readFile(ordersCsv, "utf8");
-    expect(content).toContain("order_id,user_id,total");
-    expect(content).toContain("ORD-101");
-  });
-
-  it("should export all sheets from workbook using --all-sheets", async () => {
-    const exportDir = join(tempDir, "all_exported");
-    await execAsync(`node ${cliPath} convert ${sampleXlsx} --all-sheets --out-dir ${exportDir} --to csv`);
-
-    const usersContent = await fs.readFile(join(exportDir, "Users.csv"), "utf8");
-    expect(usersContent).toContain("Alice");
-
-    const ordersContent = await fs.readFile(join(exportDir, "Orders.csv"), "utf8");
-    expect(ordersContent).toContain("ORD-101");
-  });
-
-  it("should execute rowpipe map to derive new columns", async () => {
-    const { stdout } = await execAsync(
-      `node ${cliPath} map ${sampleCsv} "is_adult=age >= 30" "tax=revenue * 0.20"`
-    );
-    expect(stdout).toContain("is_adult");
-    expect(stdout).toContain("tax");
-    expect(stdout).toContain("30.1");
-  });
-
-  it("should execute rowpipe reduce to aggregate stream", async () => {
-    const { stdout } = await execAsync(
-      `node ${cliPath} reduce ${sampleCsv} "total_rev=sum(revenue)" "avg_age=avg(age)" "total=count()"`
-    );
-    expect(stdout).toContain("total_rev,avg_age,total");
-    expect(stdout).toContain("710.75,26.75,4");
-  });
-});
-