wn-turso 0.1.0

package/dist/pipeline/pipeline-builder.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Fluent Pipeline builder API
+ */
+import type { PipelineSource, PipelineSink, PipelineResult } from './types.js';
+/**
+ * Fluent pipeline builder for data transfer operations
+ *
+ * @example
+ * ```typescript
+ * const result = await Pipeline
+ *   .from(tursoSource(sourceConfig, 'synsets'))
+ *   .filter(row => row.language === 'en')
+ *   .extend(row => ({
+ *     cached_count: 0,
+ *     custom_score: computeScore(row),
+ *   }))
+ *   .to(tursoSink(destConfig, 'synsets_working'));
+ * ```
+ */
+export declare class Pipeline<T> {
+    private source;
+    private operations;
+    private constructor();
+    /**
+     * Create a new pipeline from a source
+     */
+    static from<T>(source: PipelineSource<T>): Pipeline<T>;
+    /**
+     * Filter rows based on a predicate
+     */
+    filter(predicate: (row: T) => boolean): Pipeline<T>;
+    /**
+     * Map rows to a new shape
+     */
+    map<U>(fn: (row: T) => U): Pipeline<U>;
+    /**
+     * Transform rows, returning null to skip a row
+     */
+    transform<U>(fn: (row: T) => U | null | undefined): Pipeline<U>;
+    /**
+     * Async transform for operations that need to await
+     */
+    transformAsync<U>(fn: (row: T) => Promise<U | null | undefined>): Pipeline<U>;
+    /**
+     * Extend rows with additional properties
+     * Useful for adding columns to a "working DB" schema
+     */
+    extend<Ext extends Record<string, any>>(fn: (row: T) => Ext): Pipeline<T & Ext>;
+    /**
+     * Async extend for operations that need to await
+     */
+    extendAsync<Ext extends Record<string, any>>(fn: (row: T) => Promise<Ext>): Pipeline<T & Ext>;
+    /**
+     * Batch rows into arrays
+     */
+    batch(size: number): Pipeline<T[]>;
+    /**
+     * Side effect for each row (logging, metrics)
+     */
+    tap(fn: (row: T) => void): Pipeline<T>;
+    /**
+     * Async side effect
+     */
+    tapAsync(fn: (row: T) => Promise<void>): Pipeline<T>;
+    /**
+     * Take first n rows
+     */
+    take(n: number): Pipeline<T>;
+    /**
+     * Skip first n rows
+     */
+    skip(n: number): Pipeline<T>;
+    /**
+     * Deduplicate rows by key
+     */
+    distinct<K>(keyFn: (row: T) => K): Pipeline<T>;
+    /**
+     * Build the pipeline as an async iterable
+     * Useful for manual iteration or custom sinks
+     */
+    build(): AsyncIterable<T>;
+    /**
+     * Execute the pipeline and write to a sink
+     */
+    to(sink: PipelineSink<T>): Promise<PipelineResult>;
+    /**
+     * Execute the pipeline and collect results into an array
+     */
+    toArray(): Promise<T[]>;
+    /**
+     * Count the rows that pass through the pipeline
+     */
+    count(): Promise<number>;
+    /**
+     * Execute a function for each row
+     */
+    forEach(fn: (row: T) => void): Promise<void>;
+    /**
+     * Execute an async function for each row
+     */
+    forEachAsync(fn: (row: T) => Promise<void>): Promise<void>;
+    /**
+     * Get the source count (if available)
+     */
+    sourceCount(): Promise<number | undefined>;
+}
+//# sourceMappingURL=pipeline-builder.d.ts.map

package/dist/pipeline/pipeline-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-builder.d.ts","sourceRoot":"","sources":["../../src/pipeline/pipeline-builder.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EAGf,MAAM,YAAY,CAAC;AAGpB;;;;;;;;;;;;;;GAcG;AACH,qBAAa,QAAQ,CAAC,CAAC;IACrB,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,UAAU,CAA4B;IAE9C,OAAO;IAIP;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAItD;;OAEG;IACH,MAAM,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC;IAKnD;;OAEG;IACH,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAKtC;;OAEG;IACH,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC;IAK/D;;OAEG;IACH,cAAc,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAK7E;;;OAGG;IACH,MAAM,CAAC,GAAG,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACpC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,GAAG,GAClB,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC;IAKpB;;OAEG;IACH,WAAW,CAAC,GAAG,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,GAC3B,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC;IAKpB;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,EAAE,CAAC;IAKlC;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC;IAKtC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAKpD;;OAEG;IACH,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC;IAK5B;;OAEG;IACH,IAAI,CAAC,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC;IAK5B;;OAEG;IACH,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAK9C;;;OAGG;IACH,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC;IAUzB;;OAEG;IACG,EAAE,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC;IAKxD;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAW7B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAW9B;;OAEG;IACG,OAAO,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAQlD;;OAEG;IACG,YAAY,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAQhE;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;CAGjD"}

package/dist/pipeline/pipeline-builder.js

@@ -0,0 +1,181 @@
+/**
+ * Fluent Pipeline builder API
+ */
+import * as operators from './operators.js';
+/**
+ * Fluent pipeline builder for data transfer operations
+ *
+ * @example
+ * ```typescript
+ * const result = await Pipeline
+ *   .from(tursoSource(sourceConfig, 'synsets'))
+ *   .filter(row => row.language === 'en')
+ *   .extend(row => ({
+ *     cached_count: 0,
+ *     custom_score: computeScore(row),
+ *   }))
+ *   .to(tursoSink(destConfig, 'synsets_working'));
+ * ```
+ */
+export class Pipeline {
+    source;
+    operations = [];
+    constructor(source) {
+        this.source = source;
+    }
+    /**
+     * Create a new pipeline from a source
+     */
+    static from(source) {
+        return new Pipeline(source);
+    }
+    /**
+     * Filter rows based on a predicate
+     */
+    filter(predicate) {
+        this.operations.push(operators.filter(predicate));
+        return this;
+    }
+    /**
+     * Map rows to a new shape
+     */
+    map(fn) {
+        this.operations.push(operators.map(fn));
+        return this;
+    }
+    /**
+     * Transform rows, returning null to skip a row
+     */
+    transform(fn) {
+        this.operations.push(operators.transform(fn));
+        return this;
+    }
+    /**
+     * Async transform for operations that need to await
+     */
+    transformAsync(fn) {
+        this.operations.push(operators.transformAsync(fn));
+        return this;
+    }
+    /**
+     * Extend rows with additional properties
+     * Useful for adding columns to a "working DB" schema
+     */
+    extend(fn) {
+        this.operations.push(operators.extend(fn));
+        return this;
+    }
+    /**
+     * Async extend for operations that need to await
+     */
+    extendAsync(fn) {
+        this.operations.push(operators.extendAsync(fn));
+        return this;
+    }
+    /**
+     * Batch rows into arrays
+     */
+    batch(size) {
+        this.operations.push(operators.batch(size));
+        return this;
+    }
+    /**
+     * Side effect for each row (logging, metrics)
+     */
+    tap(fn) {
+        this.operations.push(operators.tap(fn));
+        return this;
+    }
+    /**
+     * Async side effect
+     */
+    tapAsync(fn) {
+        this.operations.push(operators.tapAsync(fn));
+        return this;
+    }
+    /**
+     * Take first n rows
+     */
+    take(n) {
+        this.operations.push(operators.take(n));
+        return this;
+    }
+    /**
+     * Skip first n rows
+     */
+    skip(n) {
+        this.operations.push(operators.skip(n));
+        return this;
+    }
+    /**
+     * Deduplicate rows by key
+     */
+    distinct(keyFn) {
+        this.operations.push(operators.distinct(keyFn));
+        return this;
+    }
+    /**
+     * Build the pipeline as an async iterable
+     * Useful for manual iteration or custom sinks
+     */
+    build() {
+        let stream = this.source.read();
+        for (const op of this.operations) {
+            stream = op(stream);
+        }
+        return stream;
+    }
+    /**
+     * Execute the pipeline and write to a sink
+     */
+    async to(sink) {
+        const stream = this.build();
+        return sink.write(stream);
+    }
+    /**
+     * Execute the pipeline and collect results into an array
+     */
+    async toArray() {
+        const results = [];
+        const stream = this.build();
+        for await (const row of stream) {
+            results.push(row);
+        }
+        return results;
+    }
+    /**
+     * Count the rows that pass through the pipeline
+     */
+    async count() {
+        let count = 0;
+        const stream = this.build();
+        for await (const _row of stream) {
+            count++;
+        }
+        return count;
+    }
+    /**
+     * Execute a function for each row
+     */
+    async forEach(fn) {
+        const stream = this.build();
+        for await (const row of stream) {
+            fn(row);
+        }
+    }
+    /**
+     * Execute an async function for each row
+     */
+    async forEachAsync(fn) {
+        const stream = this.build();
+        for await (const row of stream) {
+            await fn(row);
+        }
+    }
+    /**
+     * Get the source count (if available)
+     */
+    async sourceCount() {
+        return this.source.count?.();
+    }
+}

package/dist/pipeline/sink.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Pipeline sinks for writing data
+ */
+import type { Kysely } from 'kysely';
+import type { TursoDatabaseConfig } from '../config.js';
+import type { PipelineSink, SinkOptions } from './types.js';
+/**
+ * Create a pipeline sink for a Turso database
+ */
+export declare function tursoSink<T extends Record<string, any>>(config: TursoDatabaseConfig, table: string, options?: SinkOptions): PipelineSink<T>;
+/**
+ * Create a pipeline sink from a Kysely database instance
+ * (for use with existing connections)
+ */
+export declare function kyselySink<T extends Record<string, any>>(db: Kysely<any>, table: string, options?: SinkOptions): PipelineSink<T>;
+/**
+ * Create a sink that collects items into an array (for testing)
+ */
+export declare function arraySink<T>(): PipelineSink<T> & {
+    items: T[];
+};
+//# sourceMappingURL=sink.d.ts.map

package/dist/pipeline/sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sink.d.ts","sourceRoot":"","sources":["../../src/pipeline/sink.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAkB,MAAM,YAAY,CAAC;AAG5E;;GAEG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACrD,MAAM,EAAE,mBAAmB,EAC3B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,WAAgB,GACxB,YAAY,CAAC,CAAC,CAAC,CAkBjB;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACtD,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,WAAgB,GACxB,YAAY,CAAC,CAAC,CAAC,CAQjB;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,GAAG;IAAE,KAAK,EAAE,CAAC,EAAE,CAAA;CAAE,CAyB/D"}

package/dist/pipeline/sink.js

@@ -0,0 +1,63 @@
+/**
+ * Pipeline sinks for writing data
+ */
+import { TursoDatabase } from '../database/turso-database.js';
+import { writeBatches } from './streams.js';
+/**
+ * Create a pipeline sink for a Turso database
+ */
+export function tursoSink(config, table, options = {}) {
+    return {
+        name: `turso:${config.url}/${table}`,
+        async write(data) {
+            const db = new TursoDatabase({
+                ...config,
+                readonly: false, // Ensure we can write
+            });
+            await db.initialize();
+            try {
+                return await writeBatches(db.getDatabase(), table, data, options);
+            }
+            finally {
+                await db.close();
+            }
+        },
+    };
+}
+/**
+ * Create a pipeline sink from a Kysely database instance
+ * (for use with existing connections)
+ */
+export function kyselySink(db, table, options = {}) {
+    return {
+        name: `kysely:${table}`,
+        async write(data) {
+            return writeBatches(db, table, data, options);
+        },
+    };
+}
+/**
+ * Create a sink that collects items into an array (for testing)
+ */
+export function arraySink() {
+    const items = [];
+    const startTime = Date.now();
+    return {
+        name: 'array',
+        items,
+        async write(data) {
+            let processed = 0;
+            for await (const item of data) {
+                items.push(item);
+                processed++;
+            }
+            return {
+                processed,
+                inserted: processed,
+                skipped: 0,
+                errors: 0,
+                duration: Date.now() - startTime,
+            };
+        },
+    };
+}

package/dist/pipeline/source.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Pipeline sources for reading data
+ */
+import type { Kysely } from 'kysely';
+import type { TursoDatabaseConfig } from '../config.js';
+import type { PipelineSource, SourceOptions } from './types.js';
+/**
+ * Create a pipeline source from a Turso database
+ */
+export declare function tursoSource<T extends Record<string, any>>(config: TursoDatabaseConfig, table: string, options?: SourceOptions<T>): PipelineSource<T>;
+/**
+ * Create a pipeline source from a Kysely database instance
+ * (for use with existing connections)
+ */
+export declare function kyselySource<T extends Record<string, any>>(db: Kysely<any>, table: string, options?: SourceOptions<T>): PipelineSource<T>;
+/**
+ * Create a source from an array (for testing)
+ */
+export declare function arraySource<T>(items: T[], name?: string): PipelineSource<T>;
+//# sourceMappingURL=source.d.ts.map

package/dist/pipeline/source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../src/pipeline/source.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAErC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAGhE;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACvD,MAAM,EAAE,mBAAmB,EAC3B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAa,CAAC,CAAC,CAAM,GAC7B,cAAc,CAAC,CAAC,CAAC,CA6BnB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACxD,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAa,CAAC,CAAC,CAAM,GAC7B,cAAc,CAAC,CAAC,CAAC,CAYnB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAC3B,KAAK,EAAE,CAAC,EAAE,EACV,IAAI,GAAE,MAAgB,GACrB,cAAc,CAAC,CAAC,CAAC,CAcnB"}

package/dist/pipeline/source.js

@@ -0,0 +1,66 @@
+/**
+ * Pipeline sources for reading data
+ */
+import { TursoDatabase } from '../database/turso-database.js';
+import { streamTable, countRows } from './streams.js';
+/**
+ * Create a pipeline source from a Turso database
+ */
+export function tursoSource(config, table, options = {}) {
+    let db = null;
+    return {
+        name: `turso:${config.url}/${table}`,
+        async *read() {
+            db = new TursoDatabase(config);
+            await db.initialize();
+            try {
+                yield* streamTable(db.getDatabase(), table, options);
+            }
+            finally {
+                await db.close();
+                db = null;
+            }
+        },
+        async count() {
+            const tempDb = new TursoDatabase(config);
+            await tempDb.initialize();
+            try {
+                return await countRows(tempDb.getDatabase(), table, options);
+            }
+            finally {
+                await tempDb.close();
+            }
+        },
+    };
+}
+/**
+ * Create a pipeline source from a Kysely database instance
+ * (for use with existing connections)
+ */
+export function kyselySource(db, table, options = {}) {
+    return {
+        name: `kysely:${table}`,
+        async *read() {
+            yield* streamTable(db, table, options);
+        },
+        async count() {
+            return countRows(db, table, options);
+        },
+    };
+}
+/**
+ * Create a source from an array (for testing)
+ */
+export function arraySource(items, name = 'array') {
+    return {
+        name,
+        async *read() {
+            for (const item of items) {
+                yield item;
+            }
+        },
+        async count() {
+            return items.length;
+        },
+    };
+}

package/dist/pipeline/streams.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Low-level streaming utilities for pipeline operations
+ */
+import type { Kysely } from 'kysely';
+import type { SourceOptions, SinkOptions, PipelineResult } from './types.js';
+/**
+ * Stream rows from a table as an async generator
+ */
+export declare function streamTable<T>(db: Kysely<any>, table: string, options?: SourceOptions<T>): AsyncGenerator<T>;
+/**
+ * Write rows in batches to a table
+ */
+export declare function writeBatches<T extends Record<string, any>>(db: Kysely<any>, table: string, rows: AsyncIterable<T>, options?: SinkOptions): Promise<PipelineResult>;
+/**
+ * Count rows in a table with optional filter
+ */
+export declare function countRows<T>(db: Kysely<any>, table: string, options?: SourceOptions<T>): Promise<number>;
+//# sourceMappingURL=streams.d.ts.map

package/dist/pipeline/streams.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../src/pipeline/streams.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE7E;;GAEG;AACH,wBAAuB,WAAW,CAAC,CAAC,EAClC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAa,CAAC,CAAC,CAAM,GAC7B,cAAc,CAAC,CAAC,CAAC,CA8CnB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9D,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EACf,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EACtB,OAAO,GAAE,WAAgB,GACxB,OAAO,CAAC,cAAc,CAAC,CAiEzB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAC/B,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EACf,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,aAAa,CAAC,CAAC,CAAM,GAC7B,OAAO,CAAC,MAAM,CAAC,CAWjB"}

package/dist/pipeline/streams.js

@@ -0,0 +1,115 @@
+/**
+ * Low-level streaming utilities for pipeline operations
+ */
+/**
+ * Stream rows from a table as an async generator
+ */
+export async function* streamTable(db, table, options = {}) {
+    const { batchSize = 1000, where, orderBy, limit, offset = 0 } = options;
+    let currentOffset = offset;
+    let hasMore = true;
+    let totalFetched = 0;
+    while (hasMore) {
+        let query = db.selectFrom(table).selectAll();
+        if (where) {
+            query = where(query);
+        }
+        if (orderBy) {
+            query = query.orderBy(orderBy);
+        }
+        // Apply pagination
+        query = query.limit(batchSize).offset(currentOffset);
+        const rows = await query.execute();
+        if (rows.length === 0) {
+            hasMore = false;
+            break;
+        }
+        for (const row of rows) {
+            yield row;
+            totalFetched++;
+            // Check if we've hit the limit
+            if (limit !== undefined && totalFetched >= limit) {
+                hasMore = false;
+                break;
+            }
+        }
+        // If we got fewer rows than batch size, we're done
+        if (rows.length < batchSize) {
+            hasMore = false;
+        }
+        currentOffset += batchSize;
+    }
+}
+/**
+ * Write rows in batches to a table
+ */
+export async function writeBatches(db, table, rows, options = {}) {
+    const { batchSize = 100, onConflict = 'error', onProgress } = options;
+    const startTime = Date.now();
+    let processed = 0;
+    let inserted = 0;
+    let skipped = 0;
+    let errors = 0;
+    let batch = [];
+    const flushBatch = async () => {
+        if (batch.length === 0)
+            return;
+        try {
+            let query = db.insertInto(table).values(batch);
+            if (onConflict === 'ignore') {
+                query = query.onConflict((oc) => oc.doNothing());
+            }
+            else if (onConflict === 'replace') {
+                // For replace, we need to know the primary key columns
+                // This is a simplified version - full implementation would need schema introspection
+                query = query.onConflict((oc) => oc.doNothing());
+            }
+            const result = await query.execute();
+            inserted += batch.length;
+        }
+        catch (error) {
+            if (onConflict === 'error') {
+                throw error;
+            }
+            // For ignore/replace, count as errors
+            errors += batch.length;
+        }
+        batch = [];
+    };
+    for await (const row of rows) {
+        batch.push(row);
+        processed++;
+        if (batch.length >= batchSize) {
+            await flushBatch();
+            if (onProgress) {
+                onProgress({
+                    current: processed,
+                    processed,
+                    skipped,
+                    errors,
+                });
+            }
+        }
+    }
+    // Flush remaining rows
+    await flushBatch();
+    return {
+        processed,
+        inserted,
+        skipped,
+        errors,
+        duration: Date.now() - startTime,
+    };
+}
+/**
+ * Count rows in a table with optional filter
+ */
+export async function countRows(db, table, options = {}) {
+    const { where } = options;
+    let query = db.selectFrom(table).select((eb) => eb.fn.count('*').as('count'));
+    if (where) {
+        query = where(query);
+    }
+    const result = await query.executeTakeFirst();
+    return Number(result?.count ?? 0);
+}