rowpipe 1.0.0 → 1.2.0

package/tests/map-reduce.test.ts

@@ -1,223 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { rowsToBatches } from "../src/core/batch.js";
-import type { DataBatch, Row } from "../src/core/types.js";
-import { mapRows, parseMapSpecs } from "../src/transforms/map.js";
-import { parseReduceSpecs, reduceRows, ReduceAggregator } from "../src/analytics/reduce.js";
-import { filterRows } from "../src/transforms/filter.js";
-
-async function collectRows(stream: AsyncIterable<DataBatch>): Promise<Row[]> {
-  const results: Row[] = [];
-  for await (const batch of stream) {
-    results.push(...batch.rows);
-  }
-  return results;
-}
-
-describe("Map Transform (Row-level calculations & derivations)", () => {
-  it("should parse map specifications correctly", () => {
-    const specs = [
-      "profit = revenue - cost",
-      "tax=revenue * 0.20",
-      "user_city = payload.user.city",
-    ];
-    const parsed = parseMapSpecs(specs);
-    expect(parsed).toEqual({
-      profit: "revenue - cost",
-      tax: "revenue * 0.20",
-      user_city: "payload.user.city",
-    });
-  });
-
-  it("should compute mathematical and percentage formulas per row", async () => {
-    const rows: Row[] = [
-      { id: 1, revenue: 1000, cost: 600 },
-      { id: 2, revenue: 2500, cost: 1500 },
-    ];
-
-    const stream = mapRows({
-      profit: "revenue - cost",
-      tax: "revenue * 0.20",
-      margin: "((revenue - cost) / revenue) * 100",
-    })(rowsToBatches(rows));
-
-    const results = await collectRows(stream);
-    expect(results).toEqual([
-      { id: 1, revenue: 1000, cost: 600, profit: 400, tax: 200, margin: 40 },
-      { id: 2, revenue: 2500, cost: 1500, profit: 1000, tax: 500, margin: 40 },
-    ]);
-  });
-
-  it("should support JSON dot-notation, pipe syntax, and date extraction in map", async () => {
-    const rows: Row[] = [
-      {
-        id: 1,
-        created_at: "2026-09-16",
-        email: "  ADMIN@ROWPIPE.IO ",
-        payload: JSON.stringify({ user: { city: "Istanbul" }, items: [{ price: 99 }] }),
-      },
-    ];
-
-    const stream = mapRows({
-      city: "payload.user.city",
-      first_item_price: "payload.items.0.price",
-      clean_email: "email | lower | trim",
-      year: "created_at | year",
-    })(rowsToBatches(rows));
-
-    const results = await collectRows(stream);
-    expect(results[0]?.city).toBe("Istanbul");
-    expect(results[0]?.first_item_price).toBe(99);
-    expect(results[0]?.clean_email).toBe("admin@rowpipe.io");
-    expect(results[0]?.year).toBe(2026);
-  });
-
-  it("should support custom function mapper in mapRows", async () => {
-    const rows: Row[] = [{ a: 1 }, { a: 2 }];
-    const stream = mapRows((row) => ({ ...row, doubled: (Number(row.a) || 0) * 2 }))(rowsToBatches(rows));
-    const results = await collectRows(stream);
-    expect(results).toEqual([
-      { a: 1, doubled: 2 },
-      { a: 2, doubled: 4 },
-    ]);
-  });
-});
-
-describe("Reduce Transform (Global and Group-By Aggregations)", () => {
-  it("should parse reduce specifications correctly", () => {
-    const specs = [
-      "total_rev = sum(revenue)",
-      "avg_margin = avg(margin)",
-      "total_orders = count()",
-      "unique_users = countDistinct(user_id)",
-      "min_val = min(price)",
-      "max_val = max(price)",
-    ];
-    const parsed = parseReduceSpecs(specs);
-    expect(parsed).toEqual([
-      { targetField: "total_rev", func: "sum", sourceExpr: "revenue" },
-      { targetField: "avg_margin", func: "avg", sourceExpr: "margin" },
-      { targetField: "total_orders", func: "count", sourceExpr: undefined },
-      { targetField: "unique_users", func: "countdistinct", sourceExpr: "user_id" },
-      { targetField: "min_val", func: "min", sourceExpr: "price" },
-      { targetField: "max_val", func: "max", sourceExpr: "price" },
-    ]);
-  });
-
-  it("should compute global aggregations across stream", async () => {
-    const rows: Row[] = [
-      { id: 1, user_id: "u1", revenue: 100, price: 50 },
-      { id: 2, user_id: "u2", revenue: 200, price: 100 },
-      { id: 3, user_id: "u1", revenue: 300, price: 150 },
-      { id: 4, user_id: "u3", revenue: 400, price: 200 },
-    ];
-
-    const stream = reduceRows({
-      aggregations: [
-        "total_revenue = sum(revenue)",
-        "avg_price = avg(price)",
-        "min_price = min(price)",
-        "max_price = max(price)",
-        "order_count = count()",
-        "unique_users = countDistinct(user_id)",
-      ],
-    })(rowsToBatches(rows));
-
-    const results = await collectRows(stream);
-    expect(results.length).toBe(1);
-    expect(results[0]).toEqual({
-      total_revenue: 1000,
-      avg_price: 125,
-      min_price: 50,
-      max_price: 200,
-      order_count: 4,
-      unique_users: 3,
-    });
-  });
-
-  it("should perform streaming group-by aggregations (--by country,category)", async () => {
-    const rows: Row[] = [
-      { country: "TR", category: "Electronics", revenue: 1000, cost: 600 },
-      { country: "TR", category: "Electronics", revenue: 2000, cost: 1200 },
-      { country: "TR", category: "Books", revenue: 300, cost: 100 },
-      { country: "US", category: "Electronics", revenue: 5000, cost: 3000 },
-      { country: "US", category: "Books", revenue: 800, cost: 400 },
-    ];
-
-    const stream = reduceRows({
-      by: ["country", "category"],
-      aggregations: [
-        "total_revenue = sum(revenue)",
-        "total_cost = sum(cost)",
-        "item_count = count()",
-      ],
-    })(rowsToBatches(rows));
-
-    const results = await collectRows(stream);
-    expect(results.length).toBe(4);
-
-    const trElectronics = results.find((r) => r.country === "TR" && r.category === "Electronics");
-    expect(trElectronics).toEqual({
-      country: "TR",
-      category: "Electronics",
-      total_revenue: 3000,
-      total_cost: 1800,
-      item_count: 2,
-    });
-
-    const usBooks = results.find((r) => r.country === "US" && r.category === "Books");
-    expect(usBooks).toEqual({
-      country: "US",
-      category: "Books",
-      total_revenue: 800,
-      total_cost: 400,
-      item_count: 1,
-    });
-  });
-
-  it("should execute full Map -> Filter -> Reduce pipeline smoothly", async () => {
-    const rows: Row[] = [
-      { id: 1, country: "TR", revenue: 1000, cost: 600 },
-      { id: 2, country: "TR", revenue: 500, cost: 450 },  // margin 10% (will be filtered out)
-      { id: 3, country: "US", revenue: 3000, cost: 1500 }, // margin 50%
-      { id: 4, country: "US", revenue: 2000, cost: 1600 }, // margin 20%
-    ];
-
-    // 1. Map: derive profit and margin
-    const mapped = mapRows({
-      profit: "revenue - cost",
-      margin: "((revenue - cost) / revenue) * 100",
-    })(rowsToBatches(rows));
-
-    // 2. Filter: only margin >= 20
-    const filtered = filterRows("margin >= 20")(mapped);
-
-    // 3. Reduce: group by country
-    const reduced = reduceRows({
-      by: ["country"],
-      aggregations: [
-        "total_profit = sum(profit)",
-        "avg_margin = avg(margin)",
-        "count = count()",
-      ],
-    })(filtered);
-
-    const results = await collectRows(reduced);
-    expect(results.length).toBe(2);
-
-    const tr = results.find((r) => r.country === "TR");
-    expect(tr).toEqual({
-      country: "TR",
-      total_profit: 400,
-      avg_margin: 40,
-      count: 1,
-    });
-
-    const us = results.find((r) => r.country === "US");
-    expect(us).toEqual({
-      country: "US",
-      total_profit: 1900,
-      avg_margin: 35,
-      count: 2,
-    });
-  });
-});

package/tests/readers-writers.test.ts

@@ -1,189 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { Readable, Writable } from "node:stream";
-import { CSVReader } from "../src/readers/csv.js";
-import { CSVWriter } from "../src/writers/csv.js";
-import { JSONLReader } from "../src/readers/jsonl.js";
-import { JSONLWriter } from "../src/writers/jsonl.js";
-import { JSONReader } from "../src/readers/json.js";
-import { JSONWriter } from "../src/writers/json.js";
-import { XLSXReader } from "../src/readers/xlsx.js";
-import { XLSXWriter } from "../src/writers/xlsx.js";
-import { batchesToRows, rowsToBatches } from "../src/core/batch.js";
-import type { DataBatch, Row } from "../src/core/types.js";
-import { promises as fs } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-
-async function collectRows(stream: AsyncIterable<DataBatch>): Promise<Row[]> {
-  const results: Row[] = [];
-  for await (const batch of stream) {
-    results.push(...batch.rows);
-  }
-  return results;
-}
-
-describe("CSV Reader & Writer", () => {
-  it("should read standard CSV with headers", async () => {
-    const csvData = `id,name,age\n1,Alice,30\n2,Bob,25\n`;
-    const reader = new CSVReader(Readable.from(csvData));
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([
-      { id: "1", name: "Alice", age: "30" },
-      { id: "2", name: "Bob", age: "25" },
-    ]);
-  });
-
-  it("should handle custom delimiter and quotes with commas and newlines", async () => {
-    const csvData = `"id";"name";"notes"\n"1";"Smith, John";"Line 1\nLine 2"\n"2";"Jane";"Simple"\n`;
-    const reader = new CSVReader(Readable.from(csvData), { delimiter: ";" });
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([
-      { id: "1", name: "Smith, John", notes: "Line 1\nLine 2" },
-      { id: "2", name: "Jane", notes: "Simple" },
-    ]);
-  });
-
-  it("should auto-detect delimiter when not specified", async () => {
-    const tsvData = `id\tname\tage\n1\tAlice\t30\n2\tBob\t25\n`;
-    const reader = new CSVReader(Readable.from(tsvData));
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([
-      { id: "1", name: "Alice", age: "30" },
-      { id: "2", name: "Bob", age: "25" },
-    ]);
-  });
-
-  it("should handle UTF-8 BOM", async () => {
-    const bomCsv = `\uFEFFid,name\n1,Alice\n`;
-    const reader = new CSVReader(Readable.from(bomCsv));
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([{ id: "1", name: "Alice" }]);
-  });
-
-  it("should write CSV with proper escaping", async () => {
-    const rows: Row[] = [
-      { id: 1, name: "Alice, Bob", notes: 'He said "Hello"' },
-      { id: 2, name: "Charlie", notes: "Line1\nLine2" },
-    ];
-
-    let output = "";
-    const writable = new Writable({
-      write(chunk, _encoding, callback) {
-        output += chunk.toString();
-        callback();
-      },
-    });
-
-    const writer = new CSVWriter(writable);
-    await writer.write(rowsToBatches(rows));
-
-    expect(output).toContain('"Alice, Bob"');
-    expect(output).toContain('"He said ""Hello"""');
-    expect(output).toContain('"Line1\nLine2"');
-  });
-});
-
-describe("JSONL Reader & Writer", () => {
-  it("should stream read and write JSONL", async () => {
-    const jsonlData = `{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n`;
-    const reader = new JSONLReader(Readable.from(jsonlData));
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([
-      { id: 1, name: "Alice" },
-      { id: 2, name: "Bob" },
-    ]);
-
-    let output = "";
-    const writable = new Writable({
-      write(chunk, _encoding, callback) {
-        output += chunk.toString();
-        callback();
-      },
-    });
-
-    const writer = new JSONLWriter(writable);
-    await writer.write(rowsToBatches(rows));
-
-    expect(output).toBe(`{"id":1,"name":"Alice"}\n{"id":2,"name":"Bob"}\n`);
-  });
-});
-
-describe("JSON Reader & Writer", () => {
-  it("should streaming parse top-level JSON array", async () => {
-    const jsonData = `[\n  {"id": 1, "name": "Alice"},\n  {"id": 2, "name": "Bob"}\n]`;
-    const reader = new JSONReader(Readable.from(jsonData));
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([
-      { id: 1, name: "Alice" },
-      { id: 2, name: "Bob" },
-    ]);
-  });
-
-  it("should streaming parse nested path in JSON", async () => {
-    const jsonData = `{\n  "status": "ok",\n  "data": {\n    "results": [\n      {"id": 1, "title": "First"},\n      {"id": 2, "title": "Second"}\n    ]\n  }\n}`;
-    const reader = new JSONReader(Readable.from(jsonData), {
-      path: "data.results",
-    });
-    const rows = await collectRows(reader.read());
-
-    expect(rows).toEqual([
-      { id: 1, title: "First" },
-      { id: 2, title: "Second" },
-    ]);
-  });
-
-  it("should write JSON array", async () => {
-    const rows: Row[] = [
-      { id: 1, name: "Alice" },
-      { id: 2, name: "Bob" },
-    ];
-
-    let output = "";
-    const writable = new Writable({
-      write(chunk, _encoding, callback) {
-        output += chunk.toString();
-        callback();
-      },
-    });
-
-    const writer = new JSONWriter(writable);
-    await writer.write(rowsToBatches(rows));
-
-    const parsed = JSON.parse(output);
-    expect(parsed).toEqual(rows);
-  });
-});
-
-describe("XLSX Reader & Writer", () => {
-  it("should write and read back XLSX workbook", async () => {
-    const tempFile = join(tmpdir(), `rowpipe_test_${Date.now()}.xlsx`);
-    const initialRows: Row[] = [
-      { id: 1, name: "Alice", score: 95.5 },
-      { id: 2, name: "Bob", score: 82.0 },
-    ];
-
-    const writer = new XLSXWriter(tempFile, { sheet: "Students" });
-    await writer.write(rowsToBatches(initialRows));
-    await writer.close();
-
-    const reader = new XLSXReader(tempFile, { sheet: "Students" });
-    const rows = await collectRows(reader.read());
-
-    expect(rows.length).toBe(2);
-    expect(rows[0]!["name"]).toBe("Alice");
-    expect(rows[1]!["name"]).toBe("Bob");
-
-    // Inspect sheets
-    const meta = await reader.inspect();
-    expect(meta.sheets?.length).toBeGreaterThanOrEqual(1);
-    expect(meta.sheets?.[0]?.name).toBe("Students");
-
-    await fs.unlink(tempFile).catch(() => {});
-  });
-});

package/tests/transforms.test.ts

@@ -1,199 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { selectColumns } from "../src/transforms/select.js";
-import { renameColumns, parseRenameSpecs } from "../src/transforms/rename.js";
-import { castColumns, parseCastSpecs } from "../src/transforms/cast.js";
-import { sampleRows } from "../src/transforms/sample.js";
-import { filterRows } from "../src/transforms/filter.js";
-import { compileExpression } from "../src/transforms/expression.js";
-import { rowsToBatches } from "../src/core/batch.js";
-import type { DataBatch, Row } from "../src/core/types.js";
-
-async function collectRows(stream: AsyncIterable<DataBatch>): Promise<Row[]> {
-  const results: Row[] = [];
-  for await (const batch of stream) {
-    results.push(...batch.rows);
-  }
-  return results;
-}
-
-describe("Select Transform", () => {
-  it("should project selected columns", async () => {
-    const rows: Row[] = [
-      { id: 1, name: "Alice", email: "alice@example.com", age: 30 },
-      { id: 2, name: "Bob", email: "bob@example.com", age: 25 },
-    ];
-
-    const stream = selectColumns(["id", "email"])(rowsToBatches(rows));
-    const results = await collectRows(stream);
-
-    expect(results).toEqual([
-      { id: 1, email: "alice@example.com" },
-      { id: 2, email: "bob@example.com" },
-    ]);
-  });
-});
-
-describe("Rename Transform", () => {
-  it("should rename columns and parse rename specs", async () => {
-    const specs = ["username=name", "user_age=age"];
-    const mapping = parseRenameSpecs(specs);
-    expect(mapping).toEqual({ username: "name", user_age: "age" });
-
-    const rows: Row[] = [
-      { username: "alice", user_age: 30, country: "TR" },
-    ];
-
-    const stream = renameColumns(mapping)(rowsToBatches(rows));
-    const results = await collectRows(stream);
-
-    expect(results).toEqual([
-      { name: "alice", age: 30, country: "TR" },
-    ]);
-  });
-});
-
-describe("Cast Transform", () => {
-  it("should cast columns according to specs", async () => {
-    const specs = parseCastSpecs(["age:integer", "revenue:number", "active:boolean"]);
-    expect(specs).toEqual({
-      age: "integer",
-      revenue: "number",
-      active: "boolean",
-    });
-
-    const rows: Row[] = [
-      { id: "1", age: "32", revenue: "1250.50", active: "true" },
-      { id: "2", age: "invalid", revenue: "200", active: "0" },
-    ];
-
-    const stream = castColumns(specs, { onError: "null" })(rowsToBatches(rows));
-    const results = await collectRows(stream);
-
-    expect(results).toEqual([
-      { id: "1", age: 32, revenue: 1250.5, active: true },
-      { id: "2", age: null, revenue: 200, active: false },
-    ]);
-  });
-
-  it("should support skip-row error handling", async () => {
-    const specs = { age: "integer" };
-    const rows: Row[] = [
-      { id: "1", age: "30" },
-      { id: "2", age: "invalid" },
-      { id: "3", age: "40" },
-    ];
-
-    const stream = castColumns(specs, { onError: "skip-row" })(rowsToBatches(rows));
-    const results = await collectRows(stream);
-
-    expect(results).toEqual([
-      { id: "1", age: 30 },
-      { id: "3", age: 40 },
-    ]);
-  });
-});
-
-describe("Sample Transform", () => {
-  it("should sample k rows deterministically with seed", async () => {
-    const rows: Row[] = [];
-    for (let i = 0; i < 1000; i++) {
-      rows.push({ id: i, val: `row_${i}` });
-    }
-
-    const stream1 = sampleRows({ rows: 10, seed: 42 })(rowsToBatches(rows));
-    const sample1 = await collectRows(stream1);
-
-    const stream2 = sampleRows({ rows: 10, seed: 42 })(rowsToBatches(rows));
-    const sample2 = await collectRows(stream2);
-
-    expect(sample1.length).toBe(10);
-    expect(sample1).toEqual(sample2);
-  });
-});
-
-describe("Filter Transform & Expression Parser", () => {
-  it("should evaluate comparisons and logical operators", () => {
-    const fn1 = compileExpression('age > 30 && country == "TR"');
-    expect(fn1({ age: 35, country: "TR" })).toBe(true);
-    expect(fn1({ age: 25, country: "TR" })).toBe(false);
-    expect(fn1({ age: 35, country: "US" })).toBe(false);
-
-    const fn2 = compileExpression('revenue >= 1000 || is_vip == true');
-    expect(fn2({ revenue: 500, is_vip: true })).toBe(true);
-    expect(fn2({ revenue: 1200, is_vip: false })).toBe(true);
-    expect(fn2({ revenue: 800, is_vip: false })).toBe(false);
-  });
-
-  it("should evaluate standard and pipe-syntax functions", () => {
-    // 1. Pipe syntax for string transforms: email | lower | trim | endsWith(".com")
-    const fn1 = compileExpression('email | lower | trim | endsWith(".com")');
-    expect(fn1({ email: "  ADMIN@CORP.COM  " })).toBe(true);
-    expect(fn1({ email: "user@corp.org" })).toBe(false);
-
-    // 2. Date & Time functions with pipe: created_at | year == 2026
-    const fn2 = compileExpression('created_at | year == 2026 && created_at | month >= 6');
-    expect(fn2({ created_at: "2026-09-16" })).toBe(true);
-    expect(fn2({ created_at: "2025-09-16" })).toBe(false);
-
-    // 3. Set & Range functions: country | in("TR", "US", "DE") and age | between(18, 65)
-    const fn3 = compileExpression('country | in("TR", "US", "DE") && age | between(18, 65)');
-    expect(fn3({ country: "TR", age: 30 })).toBe(true);
-    expect(fn3({ country: "FR", age: 30 })).toBe(false);
-    expect(fn3({ country: "US", age: 70 })).toBe(false);
-
-    // 4. String splitIndex and substring with pipe: email | splitIndex("@", 1) == "gmail.com"
-    const fn4 = compileExpression('email | splitIndex("@", 1) == "gmail.com"');
-    expect(fn4({ email: "john.doe@gmail.com" })).toBe(true);
-    expect(fn4({ email: "john.doe@yahoo.com" })).toBe(false);
-
-    // 5. Type inspection: email | isEmail
-    const fn5 = compileExpression('email | isEmail');
-    expect(fn5({ email: "test@example.com" })).toBe(true);
-    expect(fn5({ email: "not-an-email" })).toBe(false);
-
-    // 6. JSON nested access: direct dot-notation & jsonGet fallback
-    const fn6 = compileExpression('payload.user.city == "Istanbul"');
-    expect(fn6({ payload: JSON.stringify({ user: { city: "Istanbul" } }) })).toBe(true);
-    expect(fn6({ payload: JSON.stringify({ user: { city: "Ankara" } }) })).toBe(false);
-
-    // Direct nested object access with array index and pipe
-    const fn6b = compileExpression('payload.items.0.price > 50 && payload.user.email | lower | endsWith("@corp.com")');
-    expect(
-      fn6b({
-        payload: {
-          items: [{ price: 99.9 }],
-          user: { email: "AHMET@CORP.COM" },
-        },
-      })
-    ).toBe(true);
-    expect(
-      fn6b({
-        payload: {
-          items: [{ price: 20 }],
-          user: { email: "AHMET@CORP.COM" },
-        },
-      })
-    ).toBe(false);
-
-    // 7. Math functions: price | clamp(10, 100)
-    const fn7 = compileExpression('price | clamp(10, 100) == 50');
-    expect(fn7({ price: 50 })).toBe(true);
-    expect(fn7({ price: 200 })).toBe(false);
-  });
-
-  it("should filter stream using filterRows", async () => {
-    const rows: Row[] = [
-      { id: 1, age: 35, email: "alice@test.com" },
-      { id: 2, age: 20, email: "bob@test.org" },
-      { id: 3, age: 45, email: "charlie@test.com" },
-    ];
-
-    const stream = filterRows('age | between(30, 50) && email | endsWith(".com")')(rowsToBatches(rows));
-    const results = await collectRows(stream);
-
-    expect(results).toEqual([
-      { id: 1, age: 35, email: "alice@test.com" },
-      { id: 3, age: 45, email: "charlie@test.com" },
-    ]);
-  });
-});

package/tsconfig.json

@@ -1,21 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "lib": ["ES2022"],
-    "types": ["node"],
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "noUncheckedIndexedAccess": true
-  },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules", "dist", "tests", "benchmarks"]
-}

package/vitest.config.ts

@@ -1,9 +0,0 @@
-import { defineConfig } from "vitest/config";
-
-export default defineConfig({
-  test: {
-    globals: true,
-    testTimeout: 30000,
-    include: ["tests/**/*.test.ts"],
-  },
-});