rowpipe 1.0.0 → 1.1.0

package/src/readers/jsonl.ts

@@ -1,126 +0,0 @@
-import { createReadStream } from "node:fs";
-import { createInterface } from "node:readline";
-import type { Readable } from "node:stream";
-import { ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  TabularReader,
-} from "../core/types.js";
-
-export interface JSONLReaderOptions extends ReaderOptions {
-  filePath?: string;
-}
-
-export class JSONLReader implements TabularReader {
-  private input: Readable | string;
-  private options: JSONLReaderOptions;
-
-  constructor(input: Readable | string, options: JSONLReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  private getInputStream(): Readable {
-    if (typeof this.input === "string") {
-      return createReadStream(this.input);
-    }
-    return this.input;
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: JSONLReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const stream = this.getInputStream();
-
-    const rl = createInterface({
-      input: stream,
-      crlfDelay: Infinity,
-    });
-
-    let lineNumber = 0;
-    let rowsInCurrentBatch: Row[] = [];
-    let globalOffset = 0;
-
-    for await (const line of rl) {
-      lineNumber++;
-      const trimmed = line.trim();
-      if (!trimmed) continue;
-
-      let parsed: unknown;
-      try {
-        parsed = JSON.parse(trimmed);
-      } catch (err) {
-        throw new ParseError(
-          `Invalid JSON on line ${lineNumber}: ${(err as Error).message}`,
-          {
-            file: mergedOptions.filePath,
-            line: lineNumber,
-            row: lineNumber,
-          }
-        );
-      }
-
-      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-        throw new ParseError(
-          `Expected JSON object on line ${lineNumber}, got ${Array.isArray(parsed) ? "array" : typeof parsed}`,
-          {
-            file: mergedOptions.filePath,
-            line: lineNumber,
-            row: lineNumber,
-          }
-        );
-      }
-
-      rowsInCurrentBatch.push(parsed as Row);
-
-      if (rowsInCurrentBatch.length >= batchSize) {
-        yield {
-          rows: rowsInCurrentBatch,
-          offset: globalOffset,
-        };
-        globalOffset += rowsInCurrentBatch.length;
-        rowsInCurrentBatch = [];
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    let rowCount = 0;
-    let columnNames: string[] = [];
-
-    for await (const batch of this.read({ ...options, batchSize: 5000 })) {
-      if (columnNames.length === 0 && batch.rows.length > 0 && batch.rows[0]) {
-        columnNames = Object.keys(batch.rows[0]);
-      }
-      rowCount += batch.rows.length;
-    }
-
-    return {
-      format: "JSONL",
-      rowCount,
-      columnCount: columnNames.length,
-      columns: columnNames.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      })),
-    };
-  }
-}

package/src/readers/xlsx.ts

@@ -1,182 +0,0 @@
-import type { Readable } from "node:stream";
-import readXlsxFile from "read-excel-file/node";
-import { InvalidArgumentError, ParseError } from "../core/errors.js";
-import type {
-  DataBatch,
-  DataStream,
-  InspectionMetadata,
-  ReaderOptions,
-  Row,
-  SheetMetadata,
-  TabularReader,
-} from "../core/types.js";
-
-export interface XLSXReaderOptions extends ReaderOptions {
-  sheet?: string | number;
-  filePath?: string;
-}
-
-export class XLSXReader implements TabularReader {
-  private input: Readable | string;
-  private options: XLSXReaderOptions;
-
-  constructor(input: Readable | string, options: XLSXReaderOptions = {}) {
-    this.input = input;
-    this.options = { ...options };
-    if (typeof input === "string") {
-      this.options.filePath = input;
-    }
-  }
-
-  async getParsedSheets(): Promise<Array<{ sheet: string; data: unknown[][] }>> {
-    try {
-      const result = await (readXlsxFile as unknown as (input: unknown, options?: unknown) => Promise<unknown>)(
-        this.input,
-        { getSheets: true }
-      );
-      if (Array.isArray(result)) {
-        if (result.length > 0 && typeof result[0] === "object" && result[0] !== null && "sheet" in result[0]) {
-          return result as Array<{ sheet: string; data: unknown[][] }>;
-        }
-        return [{ sheet: "Sheet1", data: result as unknown[][] }];
-      }
-      return [];
-    } catch (err) {
-      throw new ParseError(`Failed to parse XLSX: ${(err as Error).message}`, {
-        file: this.options.filePath,
-      });
-    }
-  }
-
-  async *read(options?: ReaderOptions): DataStream {
-    const mergedOptions: XLSXReaderOptions = {
-      ...this.options,
-      ...options,
-    };
-    const batchSize = Math.max(1, mergedOptions.batchSize ?? 1000);
-    const targetSheet = mergedOptions.sheet;
-
-    const sheets = await this.getParsedSheets();
-    if (sheets.length === 0) {
-      return;
-    }
-
-    let selectedSheet: { sheet: string; data: unknown[][] } | undefined;
-
-    if (targetSheet === undefined || targetSheet === null) {
-      selectedSheet = sheets[0];
-    } else if (typeof targetSheet === "number") {
-      selectedSheet = sheets[targetSheet - 1];
-    } else {
-      selectedSheet = sheets.find(
-        (s) => s.sheet.toLowerCase() === String(targetSheet).toLowerCase()
-      );
-    }
-
-    if (!selectedSheet) {
-      throw new InvalidArgumentError(
-        `Sheet "${targetSheet}" was not found in the workbook. Available sheets: ${sheets.map((s) => s.sheet).join(", ")}`
-      );
-    }
-
-    const grid = selectedSheet.data;
-    if (grid.length === 0) return;
-
-    let headers: string[] | null = null;
-    let rowsInCurrentBatch: Row[] = [];
-    let globalOffset = 0;
-
-    for (let r = 0; r < grid.length; r++) {
-      const rowCells = grid[r] || [];
-      if (rowCells.length === 0 || rowCells.every((c) => c === null || c === undefined || c === "")) {
-        continue;
-      }
-
-      if (headers === null) {
-        headers = rowCells.map((c, i) => {
-          const str = c !== null && c !== undefined ? String(c).trim() : "";
-          return str.length > 0 ? str : `col_${i + 1}`;
-        });
-        continue;
-      }
-
-      const rowObj: Row = {};
-      for (let i = 0; i < headers.length; i++) {
-        const key = headers[i] || `col_${i + 1}`;
-        const cellVal = rowCells[i];
-        rowObj[key] = cellVal !== null && cellVal !== undefined ? cellVal : "";
-      }
-
-      rowsInCurrentBatch.push(rowObj);
-
-      if (rowsInCurrentBatch.length >= batchSize) {
-        yield {
-          rows: rowsInCurrentBatch,
-          offset: globalOffset,
-        };
-        globalOffset += rowsInCurrentBatch.length;
-        rowsInCurrentBatch = [];
-      }
-    }
-
-    if (rowsInCurrentBatch.length > 0) {
-      yield {
-        rows: rowsInCurrentBatch,
-        offset: globalOffset,
-      };
-    }
-  }
-
-  async inspect(options?: ReaderOptions): Promise<InspectionMetadata> {
-    const sheets = await this.getParsedSheets();
-    const targetSheet = options?.sheet ?? this.options.sheet;
-
-    const sheetSummaries: SheetMetadata[] = sheets.map((s) => {
-      const nonHeaderRows = Math.max(0, s.data.length - 1);
-      const headerRow = s.data[0] || [];
-      const colHeaders = headerRow.map((c, i) =>
-        c !== null && c !== undefined ? String(c).trim() : `col_${i + 1}`
-      );
-
-      return {
-        name: s.sheet,
-        rowCount: nonHeaderRows,
-        columnCount: colHeaders.length,
-        columns: colHeaders,
-      };
-    });
-
-    let selectedSheet = sheets[0];
-    if (targetSheet !== undefined && targetSheet !== null) {
-      selectedSheet =
-        typeof targetSheet === "number"
-          ? sheets[targetSheet - 1]
-          : sheets.find((s) => s.sheet.toLowerCase() === String(targetSheet).toLowerCase());
-    }
-
-    let columns: Array<{ name: string; type: any; nullPercentage: number }> = [];
-    let rowCount = 0;
-
-    if (selectedSheet && selectedSheet.data.length > 0) {
-      const headerRow = selectedSheet.data[0] || [];
-      const colHeaders = headerRow.map((c, i) =>
-        c !== null && c !== undefined ? String(c).trim() : `col_${i + 1}`
-      );
-      rowCount = Math.max(0, selectedSheet.data.length - 1);
-      columns = colHeaders.map((name) => ({
-        name,
-        type: "string",
-        nullPercentage: 0,
-      }));
-    }
-
-    return {
-      format: "XLSX",
-      rowCount,
-      columnCount: columns.length,
-      columns,
-      sheetsCount: sheets.length,
-      sheets: sheetSummaries,
-    };
-  }
-}

package/src/transforms/cast.ts

@@ -1,212 +0,0 @@
-import { InvalidArgumentError } from "../core/errors.js";
-import type { DataStream, Row, TransformFunction } from "../core/types.js";
-
-export type CastOnError = "fail" | "null" | "keep" | "skip-row";
-
-export interface CastOptions {
-  onError?: CastOnError;
-}
-
-/**
- * Casts a single value to the target type.
- */
-export function castValue(
-  value: unknown,
-  targetType: string,
-  onError: CastOnError = "null"
-): { value: unknown; success: boolean } {
-  if (value === null || value === undefined || value === "") {
-    return { value: null, success: true };
-  }
-
-  const str = String(value).trim();
-  const lowerType = targetType.toLowerCase();
-
-  try {
-    switch (lowerType) {
-      case "string":
-        return { value: String(value), success: true };
-
-      case "integer":
-      case "int": {
-        if (!/^-?\d+$/.test(str)) {
-          throw new Error(`Cannot cast "${str}" to integer`);
-        }
-        const parsedInt = Number.parseInt(str, 10);
-        if (Number.isNaN(parsedInt)) {
-          throw new Error(`Cannot cast "${str}" to integer`);
-        }
-        return { value: parsedInt, success: true };
-      }
-
-      case "number":
-      case "float":
-      case "double": {
-        const parsedNum = Number.parseFloat(str);
-        if (Number.isNaN(parsedNum)) {
-          throw new Error(`Cannot cast "${str}" to number`);
-        }
-        return { value: parsedNum, success: true };
-      }
-
-      case "boolean":
-      case "bool": {
-        const lower = str.toLowerCase();
-        if (["true", "1", "yes", "t", "y"].includes(lower)) {
-          return { value: true, success: true };
-        }
-        if (["false", "0", "no", "f", "n"].includes(lower)) {
-          return { value: false, success: true };
-        }
-        throw new Error(`Cannot cast "${str}" to boolean`);
-      }
-
-      case "date": {
-        const d = new Date(str);
-        if (Number.isNaN(d.getTime())) {
-          throw new Error(`Cannot cast "${str}" to date`);
-        }
-        return { value: d.toISOString().split("T")[0], success: true };
-      }
-
-      case "datetime": {
-        const d = new Date(str);
-        if (Number.isNaN(d.getTime())) {
-          throw new Error(`Cannot cast "${str}" to datetime`);
-        }
-        return { value: d.toISOString(), success: true };
-      }
-
-      case "json": {
-        const parsed = JSON.parse(str);
-        return { value: parsed, success: true };
-      }
-
-      default:
-        throw new Error(`Unknown target type "${targetType}"`);
-    }
-  } catch (err) {
-    if (onError === "fail") {
-      throw new InvalidArgumentError(
-        `Failed to cast value "${str}" to type "${targetType}": ${(err as Error).message}`
-      );
-    }
-    if (onError === "keep") {
-      return { value, success: false };
-    }
-    // "null" or "skip-row"
-    return { value: null, success: false };
-  }
-}
-
-/**
- * Creates a transform that casts specified columns to target types.
- */
-export function castColumns(
-  typeSpecs: Record<string, string>,
-  options: CastOptions = {}
-): TransformFunction {
-  const onError = options.onError || "null";
-
-  const compiled = Object.entries(typeSpecs).map(([col, targetType]) => {
-    const lower = targetType.toLowerCase();
-
-    // Fast-path specialized caster functions
-    if (lower === "number" || lower === "float" || lower === "double") {
-      return {
-        col,
-        cast: (val: unknown) => {
-          if (val === null || val === undefined || val === "") return { value: null, success: true };
-          const num = typeof val === "number" ? val : Number(val);
-          if (!Number.isNaN(num)) return { value: num, success: true };
-          if (onError === "fail") throw new InvalidArgumentError(`Cannot cast "${String(val)}" to number`);
-          if (onError === "keep") return { value: val, success: false };
-          return { value: null, success: false };
-        },
-      };
-    }
-
-    if (lower === "integer" || lower === "int") {
-      return {
-        col,
-        cast: (val: unknown) => {
-          if (val === null || val === undefined || val === "") return { value: null, success: true };
-          const str = String(val).trim();
-          if (!/^-?\d+$/.test(str)) {
-            if (onError === "fail") throw new InvalidArgumentError(`Cannot cast "${str}" to integer`);
-            if (onError === "keep") return { value: val, success: false };
-            return { value: null, success: false };
-          }
-          const num = Number.parseInt(str, 10);
-          return { value: num, success: true };
-        },
-      };
-    }
-
-    if (lower === "string") {
-      return {
-        col,
-        cast: (val: unknown) => ({ value: val !== null && val !== undefined ? String(val) : "", success: true }),
-      };
-    }
-
-    return {
-      col,
-      cast: (val: unknown) => castValue(val, targetType, onError),
-    };
-  });
-
-  const compiledLen = compiled.length;
-
-  return function (stream: DataStream): DataStream {
-    return (async function* () {
-      for await (const batch of stream) {
-        const rows = batch.rows;
-        const rowCount = rows.length;
-        const transformedRows: Row[] = [];
-
-        for (let i = 0; i < rowCount; i++) {
-          const row = rows[i]!;
-          let skipRow = false;
-          const newRow: Row = { ...row };
-
-          for (let j = 0; j < compiledLen; j++) {
-            const entry = compiled[j]!;
-            const col = entry.col;
-            if (col in newRow) {
-              const res = entry.cast(newRow[col]);
-              if (!res.success && onError === "skip-row") {
-                skipRow = true;
-                break;
-              }
-              newRow[col] = res.value;
-            }
-          }
-
-          if (!skipRow) {
-            transformedRows.push(newRow);
-          }
-        }
-
-        yield {
-          rows: transformedRows,
-          offset: batch.offset,
-        };
-      }
-    })();
-  };
-}
-
-/**
- * Parses command line cast specs like ["age:integer", "revenue:number"].
- */
-export function parseCastSpecs(specs: string[]): Record<string, string> {
-  const mapping: Record<string, string> = {};
-  for (const spec of specs) {
-    const parts = spec.split(":");
-    if (parts.length === 2 && parts[0] && parts[1]) {
-      mapping[parts[0].trim()] = parts[1].trim();
-    }
-  }
-  return mapping;
-}