rowpipe 1.0.0 → 1.1.0

package/src/core/batch.ts

@@ -1,81 +0,0 @@
-import type { DataBatch, DataStream, Row } from "./types.js";
-
-/**
- * Packs an AsyncIterable of individual rows into DataBatch chunks with a given batchSize.
- */
-export async function* rowsToBatches(
-  rows: AsyncIterable<Row>,
-  batchSize = 1000
-): AsyncGenerator<DataBatch> {
-  const effectiveBatchSize = Math.max(1, batchSize);
-  let currentRows: Row[] = [];
-  let totalOffset = 0;
-
-  for await (const row of rows) {
-    currentRows.push(row);
-    if (currentRows.length >= effectiveBatchSize) {
-      yield {
-        rows: currentRows,
-        offset: totalOffset,
-      };
-      totalOffset += currentRows.length;
-      currentRows = [];
-    }
-  }
-
-  if (currentRows.length > 0) {
-    yield {
-      rows: currentRows,
-      offset: totalOffset,
-    };
-  }
-}
-
-/**
- * Unpacks DataBatches into individual rows.
- */
-export async function* batchesToRows(stream: DataStream): AsyncGenerator<Row> {
-  for await (const batch of stream) {
-    for (const row of batch.rows) {
-      yield row;
-    }
-  }
-}
-
-/**
- * Applies a mapping function to each row in batches, maintaining batch structure
- * and filtering out null/undefined results.
- */
-export async function* batchMap(
-  stream: DataStream,
-  fn: (row: Row, globalIndex: number) => Row | null | undefined,
-  targetBatchSize = 1000
-): AsyncGenerator<DataBatch> {
-  let pendingRows: Row[] = [];
-  let currentOffset = 0;
-  let globalIndex = 0;
-
-  for await (const batch of stream) {
-    for (const row of batch.rows) {
-      const transformed = fn(row, globalIndex++);
-      if (transformed !== null && transformed !== undefined) {
-        pendingRows.push(transformed);
-        if (pendingRows.length >= targetBatchSize) {
-          yield {
-            rows: pendingRows,
-            offset: currentOffset,
-          };
-          currentOffset += pendingRows.length;
-          pendingRows = [];
-        }
-      }
-    }
-  }
-
-  if (pendingRows.length > 0) {
-    yield {
-      rows: pendingRows,
-      offset: currentOffset,
-    };
-  }
-}

package/src/core/errors.ts

@@ -1,81 +0,0 @@
-export class RowpipeError extends Error {
-  public readonly exitCode: number;
-
-  constructor(message: string, exitCode = 1) {
-    super(message);
-    this.name = "RowpipeError";
-    this.exitCode = exitCode;
-    Object.setPrototypeOf(this, new.target.prototype);
-  }
-}
-
-export class InvalidArgumentError extends RowpipeError {
-  constructor(message: string) {
-    super(message, 2);
-    this.name = "InvalidArgumentError";
-  }
-}
-
-export interface ParseErrorContext {
-  file?: string;
-  sheet?: string;
-  row?: number;
-  column?: string | number;
-  byteOffset?: number;
-  line?: number;
-  cell?: string;
-}
-
-export class ParseError extends RowpipeError {
-  public readonly context: ParseErrorContext;
-
-  constructor(message: string, context: ParseErrorContext = {}) {
-    let detailedMsg = `Parse error: ${message}`;
-    const parts: string[] = [];
-
-    if (context.file) parts.push(`File: ${context.file}`);
-    if (context.sheet) parts.push(`Sheet: ${context.sheet}`);
-    if (context.row !== undefined) parts.push(`Row: ${context.row}`);
-    if (context.column !== undefined) parts.push(`Column: ${context.column}`);
-    if (context.cell) parts.push(`Cell: ${context.cell}`);
-    if (context.byteOffset !== undefined) parts.push(`Byte offset: ${context.byteOffset}`);
-
-    if (parts.length > 0) {
-      detailedMsg += `\n  Context: ${parts.join(", ")}`;
-    }
-
-    super(detailedMsg, 3);
-    this.name = "ParseError";
-    this.context = context;
-  }
-}
-
-export interface ValidationViolation {
-  column: string;
-  expected: string;
-  actual?: string;
-  count: number;
-  examples?: unknown[];
-}
-
-export class ValidationError extends RowpipeError {
-  public readonly violations: ValidationViolation[];
-  public readonly totalRows: number;
-  public readonly validRows: number;
-  public readonly invalidRows: number;
-
-  constructor(
-    message: string,
-    violations: ValidationViolation[] = [],
-    totalRows = 0,
-    validRows = 0,
-    invalidRows = 0
-  ) {
-    super(message, 4);
-    this.name = "ValidationError";
-    this.violations = violations;
-    this.totalRows = totalRows;
-    this.validRows = validRows;
-    this.invalidRows = invalidRows;
-  }
-}

package/src/core/pipeline.ts

@@ -1,150 +0,0 @@
-import type {
-  Aggregator,
-  DataBatch,
-  DataStream,
-  ProgressCallback,
-  ReaderOptions,
-  Row,
-  TabularReader,
-  TabularWriter,
-  TransformFunction,
-  WriterOptions,
-} from "./types.js";
-import { batchesToRows } from "./batch.js";
-
-export class Pipeline {
-  private stream: DataStream;
-  private progressCallbacks: ProgressCallback[] = [];
-
-  constructor(source: DataStream | TabularReader, options?: ReaderOptions) {
-    if ("read" in source && typeof source.read === "function") {
-      this.stream = source.read(options);
-    } else {
-      this.stream = source as DataStream;
-    }
-  }
-
-  /**
-   * Chains a transform function to the pipeline.
-   */
-  pipe(transform: TransformFunction): this {
-    this.stream = transform(this.stream);
-    return this;
-  }
-
-  /**
-   * Adds a progress listener.
-   */
-  onProgress(callback: ProgressCallback): this {
-    this.progressCallbacks.push(callback);
-    return this;
-  }
-
-  /**
-   * Wraps the current stream with a progress tracking generator if callbacks are registered.
-   */
-  private getTrackedStream(): DataStream {
-    if (this.progressCallbacks.length === 0) {
-      return this.stream;
-    }
-
-    const callbacks = this.progressCallbacks;
-    const stream = this.stream;
-
-    return (async function* () {
-      let rowsProcessed = 0;
-      const startTime = Date.now();
-      let lastReport = startTime;
-
-      for await (const batch of stream) {
-        rowsProcessed += batch.rows.length;
-        const now = Date.now();
-
-        // Throttle progress notifications to ~100ms intervals
-        if (now - lastReport >= 100) {
-          const elapsedSec = (now - startTime) / 1000 || 0.001;
-          const info = {
-            rowsProcessed,
-            elapsedMs: now - startTime,
-            rowsPerSecond: Math.round(rowsProcessed / elapsedSec),
-          };
-          for (const cb of callbacks) {
-            cb(info);
-          }
-          lastReport = now;
-        }
-
-        yield batch;
-      }
-
-      // Final progress callback
-      const finalNow = Date.now();
-      const finalElapsedSec = (finalNow - startTime) / 1000 || 0.001;
-      const finalInfo = {
-        rowsProcessed,
-        elapsedMs: finalNow - startTime,
-        rowsPerSecond: Math.round(rowsProcessed / finalElapsedSec),
-      };
-      for (const cb of callbacks) {
-        cb(finalInfo);
-      }
-    })();
-  }
-
-  /**
-   * Pipes the stream into a TabularWriter.
-   */
-  async to(writer: TabularWriter, options?: WriterOptions): Promise<void> {
-    const tracked = this.getTrackedStream();
-    await writer.write(tracked, options);
-    if (writer.close) {
-      await writer.close();
-    }
-  }
-
-  /**
-   * Consumes the stream using an Aggregator.
-   */
-  async reduce<T>(aggregator: Aggregator<T>): Promise<T> {
-    const tracked = this.getTrackedStream();
-    for await (const row of batchesToRows(tracked)) {
-      aggregator.add(row);
-    }
-    return aggregator.result();
-  }
-
-  /**
-   * Returns an AsyncIterable of DataBatches.
-   */
-  batches(): DataStream {
-    return this.getTrackedStream();
-  }
-
-  /**
-   * Returns an AsyncIterable of individual rows.
-   */
-  rows(): AsyncIterable<Row> {
-    return batchesToRows(this.getTrackedStream());
-  }
-
-  /**
-   * Collects all rows up to a maximum limit (default 10,000 to prevent unbounded memory).
-   */
-  async toArray(maxRows = 10000): Promise<Row[]> {
-    const results: Row[] = [];
-    for await (const row of this.rows()) {
-      results.push(row);
-      if (results.length >= maxRows) {
-        break;
-      }
-    }
-    return results;
-  }
-}
-
-export function createPipeline(
-  source: DataStream | TabularReader,
-  options?: ReaderOptions
-): Pipeline {
-  return new Pipeline(source, options);
-}

package/src/core/types.ts

@@ -1,110 +0,0 @@
-import type { Readable, Writable } from "node:stream";
-
-export type Row = Record<string, unknown>;
-
-export interface DataBatch {
-  rows: Row[];
-  offset: number;
-}
-
-export type DataStream = AsyncIterable<DataBatch>;
-
-export type ColumnType =
-  | "string"
-  | "integer"
-  | "number"
-  | "boolean"
-  | "date"
-  | "datetime"
-  | "null"
-  | "mixed";
-
-export type SemanticType =
-  | "email"
-  | "url"
-  | "uuid"
-  | "ipv4"
-  | "ipv6"
-  | "country-code"
-  | "currency"
-  | "phone";
-
-export interface ReaderOptions {
-  batchSize?: number;
-  header?: boolean;
-  delimiter?: string;
-  path?: string;
-  sheet?: string | number;
-  maxRows?: number;
-  filePath?: string;
-}
-
-export interface WriterOptions {
-  delimiter?: string;
-  header?: boolean;
-  sheet?: string;
-}
-
-export interface SheetMetadata {
-  name: string;
-  rowCount: number;
-  columnCount?: number;
-  columns?: string[];
-}
-
-export interface InspectionMetadata {
-  format: string;
-  sizeBytes?: number;
-  rowCount?: number;
-  columnCount?: number;
-  columns?: Array<{
-    name: string;
-    type: ColumnType;
-    nullPercentage: number;
-    approxUniquePercentage?: number;
-  }>;
-  sheetsCount?: number;
-  sheets?: SheetMetadata[];
-}
-
-export interface TabularReader {
-  read(options?: ReaderOptions): DataStream;
-  inspect?(options?: ReaderOptions): Promise<InspectionMetadata>;
-  close?(): Promise<void>;
-}
-
-export interface TabularWriter {
-  write(stream: DataStream, options?: WriterOptions): Promise<void>;
-  close?(): Promise<void>;
-}
-
-export interface FormatAdapter {
-  name: string;
-  extensions: string[];
-  createReader(
-    input: Readable | string,
-    options?: ReaderOptions
-  ): TabularReader;
-  createWriter(
-    output: Writable | string,
-    options?: WriterOptions
-  ): TabularWriter;
-}
-
-export type TransformFunction = (stream: DataStream) => DataStream;
-
-export interface Aggregator<T> {
-  add(row: Row): void;
-  merge?(other: Aggregator<T>): void;
-  result(): T;
-}
-
-export interface ProgressCallbackInfo {
-  rowsProcessed: number;
-  bytesRead?: number;
-  elapsedMs: number;
-  rowsPerSecond: number;
-  megabytesPerSecond?: number;
-}
-
-export type ProgressCallback = (info: ProgressCallbackInfo) => void;

package/src/index.ts

@@ -1,52 +0,0 @@
-// Core
-export * from "./core/types.js";
-export * from "./core/errors.js";
-export * from "./core/batch.js";
-export * from "./core/pipeline.js";
-
-// Readers
-export {
-  CSVReader,
-  type CSVReaderOptions,
-  JSONReader,
-  type JSONReaderOptions,
-  JSONLReader,
-  type JSONLReaderOptions,
-  XLSXReader,
-  type XLSXReaderOptions,
-  createReader,
-  inferFormatFromPath as inferReaderFormatFromPath,
-} from "./readers/index.js";
-
-// Writers
-export {
-  CSVWriter,
-  type CSVWriterOptions,
-  JSONWriter,
-  JSONLWriter,
-  XLSXWriter,
-  type XLSXWriterOptions,
-  createWriter,
-  inferFormatFromPath as inferWriterFormatFromPath,
-} from "./writers/index.js";
-
-// Transforms
-export * from "./transforms/select.js";
-export * from "./transforms/rename.js";
-export * from "./transforms/cast.js";
-export * from "./transforms/sample.js";
-export * from "./transforms/filter.js";
-export * from "./transforms/expression.js";
-export * from "./transforms/map.js";
-
-// Analytics
-export * from "./analytics/stats.js";
-export * from "./analytics/reduce.js";
-export * from "./analytics/schema-inference.js";
-export * from "./analytics/semantic-types.js";
-export * from "./analytics/validator.js";
-
-// Utilities
-export * from "./utils/compression.js";
-export * from "./utils/formatting.js";
-export * from "./utils/progress.js";