rowpipe 2.1.0 → 2.2.0

package/dist/db/adapters/mysql.d.ts

@@ -0,0 +1,16 @@
+import type { DataBatch } from "../../core/types.js";
+import type { DatabaseAdapter, DatabaseConnectionConfig, DatabaseDialect, DatabaseReadOptions, DatabaseWriteOptions, DatabaseWriteResult, TableSchemaMetadata } from "../types.js";
+export declare class MysqlDatabaseAdapter implements DatabaseAdapter {
+    readonly dialect: DatabaseDialect;
+    readonly sanitizedUrl: string;
+    private connectionConfig;
+    private pool;
+    constructor(config: DatabaseConnectionConfig);
+    stream(options: DatabaseReadOptions): AsyncIterable<DataBatch>;
+    write(input: AsyncIterable<DataBatch>, options: DatabaseWriteOptions): Promise<DatabaseWriteResult>;
+    getTables(): Promise<string[]>;
+    getTableSchema(table: string): Promise<TableSchemaMetadata>;
+    executeRaw(sql: string, params?: unknown[]): Promise<unknown>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=mysql.d.ts.map

package/dist/db/adapters/mysql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mysql.d.ts","sourceRoot":"","sources":["../../../src/db/adapters/mysql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAO,MAAM,qBAAqB,CAAC;AAE1D,OAAO,KAAK,EACV,eAAe,EACf,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EAEnB,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAErB,qBAAa,oBAAqB,YAAW,eAAe;IAC1D,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAW;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,gBAAgB,CAA2B;IACnD,OAAO,CAAC,IAAI,CAAa;gBAEb,MAAM,EAAE,wBAAwB;IAkBrC,MAAM,CAAC,OAAO,EAAE,mBAAmB,GAAG,aAAa,CAAC,SAAS,CAAC;IA4D/D,KAAK,CACT,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,EAC/B,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC;IAwHzB,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAW9B,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAiC3D,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAK7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}

package/dist/db/adapters/mysql.js

@@ -0,0 +1,230 @@
+import mysql from "mysql2/promise";
+import { escapeIdentifier, escapeQualifiedTable, generateCreateTableSql, mysqlTypeToRowpipe } from "../mapping.js";
+export class MysqlDatabaseAdapter {
+    dialect = "mysql";
+    sanitizedUrl;
+    connectionConfig;
+    pool;
+    constructor(config) {
+        this.sanitizedUrl = config.sanitizedUrl;
+        this.connectionConfig = config;
+        this.pool = mysql.createPool({
+            host: config.host || "localhost",
+            port: config.port || 3306,
+            user: config.user || "root",
+            password: config.password,
+            database: config.database,
+            waitForConnections: true,
+            connectionLimit: 10,
+            queueLimit: 0,
+            supportBigNumbers: true,
+            bigNumberStrings: true,
+            dateStrings: true,
+        });
+    }
+    async *stream(options) {
+        const batchSize = Math.max(1, options.batchSize || 1000);
+        const sql = options.query || (options.table ? `SELECT * FROM ${escapeQualifiedTable(options.table, "mysql")}` : "");
+        if (!sql) {
+            throw new Error("Either table or query must be specified for MySQL stream");
+        }
+        const conn = await this.pool.getConnection();
+        try {
+            // mysql2 underlying connection stream
+            const rawConn = conn.connection;
+            const queryStream = rawConn.query(sql, options.params || []).stream({ highWaterMark: batchSize });
+            let currentBatch = [];
+            let offset = 0;
+            const onAbort = () => {
+                queryStream.destroy();
+            };
+            if (options.signal) {
+                options.signal.addEventListener("abort", onAbort, { once: true });
+            }
+            try {
+                for await (const row of queryStream) {
+                    if (options.signal?.aborted) {
+                        break;
+                    }
+                    currentBatch.push(row);
+                    if (currentBatch.length >= batchSize) {
+                        yield {
+                            rows: currentBatch,
+                            offset,
+                        };
+                        offset += currentBatch.length;
+                        currentBatch = [];
+                    }
+                }
+                if (currentBatch.length > 0 && !options.signal?.aborted) {
+                    yield {
+                        rows: currentBatch,
+                        offset,
+                    };
+                }
+            }
+            finally {
+                if (options.signal) {
+                    options.signal.removeEventListener("abort", onAbort);
+                }
+                queryStream.destroy();
+            }
+        }
+        finally {
+            conn.release();
+        }
+    }
+    async write(input, options) {
+        const startTime = Date.now();
+        let rowsWritten = 0;
+        let batchesWritten = 0;
+        let tableCreated = false;
+        const table = options.table;
+        if (!table) {
+            throw new Error("Target table name is required for database write");
+        }
+        const escapedTable = escapeQualifiedTable(table, "mysql");
+        const conn = await this.pool.getConnection();
+        try {
+            if (options.truncate) {
+                await conn.query(`TRUNCATE TABLE ${escapedTable};`);
+            }
+            const useTx = options.transaction !== false;
+            let insertColumns = [];
+            for await (const batch of input) {
+                if (batch.rows.length === 0)
+                    continue;
+                // Auto create table from first batch if requested
+                if (options.createTable && !tableCreated) {
+                    const firstRow = batch.rows[0];
+                    const inferredSchema = {};
+                    for (const [key, val] of Object.entries(firstRow)) {
+                        if (typeof val === "number") {
+                            inferredSchema[key] = Number.isInteger(val) ? "integer" : "number";
+                        }
+                        else if (typeof val === "bigint") {
+                            inferredSchema[key] = "bigint";
+                        }
+                        else if (typeof val === "boolean") {
+                            inferredSchema[key] = "boolean";
+                        }
+                        else {
+                            inferredSchema[key] = "string";
+                        }
+                    }
+                    const ddl = generateCreateTableSql(table, inferredSchema, "mysql", options.conflictColumns);
+                    if (options.dryRun) {
+                        process.stdout.write(`\n-- Dry-run DDL:\n${ddl}\n`);
+                        return { rowsWritten: 0, batchesWritten: 0, durationMs: Date.now() - startTime, tableCreated: true };
+                    }
+                    await conn.query(ddl);
+                    tableCreated = true;
+                }
+                if (insertColumns.length === 0) {
+                    insertColumns = Object.keys(batch.rows[0]);
+                }
+                if (useTx) {
+                    await conn.beginTransaction();
+                }
+                try {
+                    const escapedCols = insertColumns.map((c) => escapeIdentifier(c, "mysql")).join(", ");
+                    const MAX_PARAMS = 50000;
+                    const chunkSize = Math.max(1, Math.floor(MAX_PARAMS / insertColumns.length));
+                    for (let i = 0; i < batch.rows.length; i += chunkSize) {
+                        const rowChunk = batch.rows.slice(i, i + chunkSize);
+                        const rowPlaceholder = `(${insertColumns.map(() => "?").join(", ")})`;
+                        const allPlaceholders = rowChunk.map(() => rowPlaceholder).join(", ");
+                        const flatParams = [];
+                        for (const row of rowChunk) {
+                            for (const col of insertColumns) {
+                                const v = row[col];
+                                if (typeof v === "object" && v !== null && !(v instanceof Uint8Array) && !(v instanceof Date)) {
+                                    flatParams.push(JSON.stringify(v));
+                                }
+                                else {
+                                    flatParams.push(v === undefined ? null : v);
+                                }
+                            }
+                            rowsWritten++;
+                        }
+                        let sql = `INSERT INTO ${escapedTable} (${escapedCols}) VALUES ${allPlaceholders}`;
+                        if (options.upsert) {
+                            const updateCols = insertColumns
+                                .map((c) => `${escapeIdentifier(c, "mysql")} = VALUES(${escapeIdentifier(c, "mysql")})`)
+                                .join(", ");
+                            sql += ` ON DUPLICATE KEY UPDATE ${updateCols}`;
+                        }
+                        await conn.query(sql, flatParams);
+                    }
+                    if (useTx) {
+                        await conn.commit();
+                    }
+                }
+                catch (err) {
+                    if (useTx) {
+                        try {
+                            await conn.rollback();
+                        }
+                        catch {
+                            // Ignore rollback errors
+                        }
+                    }
+                    throw err;
+                }
+                batchesWritten++;
+            }
+        }
+        finally {
+            conn.release();
+        }
+        return {
+            rowsWritten,
+            batchesWritten,
+            durationMs: Date.now() - startTime,
+            tableCreated,
+        };
+    }
+    async getTables() {
+        const [rows] = await this.pool.query(`SELECT table_name
+       FROM information_schema.tables
+       WHERE table_schema = DATABASE()
+         AND table_type = 'BASE TABLE'
+       ORDER BY table_name`);
+        return rows.map((r) => r.table_name || r.TABLE_NAME);
+    }
+    async getTableSchema(table) {
+        const [rows] = await this.pool.query(`SELECT column_name, column_type, is_nullable, column_key, column_default
+       FROM information_schema.columns
+       WHERE table_schema = DATABASE() AND table_name = ?
+       ORDER BY ordinal_position`, [table]);
+        const primaryKeys = [];
+        const columns = rows.map((r) => {
+            const colName = r.column_name || r.COLUMN_NAME;
+            const colType = r.column_type || r.COLUMN_TYPE;
+            const isPk = (r.column_key || r.COLUMN_KEY) === "PRI";
+            if (isPk)
+                primaryKeys.push(colName);
+            return {
+                name: colName,
+                dbType: colType,
+                rowpipeType: mysqlTypeToRowpipe(colType),
+                nullable: (r.is_nullable || r.IS_NULLABLE) === "YES",
+                isPrimaryKey: isPk,
+                defaultValue: r.column_default || r.COLUMN_DEFAULT,
+            };
+        });
+        return {
+            table,
+            columns,
+            primaryKeys,
+        };
+    }
+    async executeRaw(sql, params) {
+        const [result] = await this.pool.query(sql, params);
+        return result;
+    }
+    async close() {
+        await this.pool.end();
+    }
+}
+//# sourceMappingURL=mysql.js.map

package/dist/db/adapters/mysql.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mysql.js","sourceRoot":"","sources":["../../../src/db/adapters/mysql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,gBAAgB,CAAC;AAEnC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAYnH,MAAM,OAAO,oBAAoB;IACtB,OAAO,GAAoB,OAAO,CAAC;IACnC,YAAY,CAAS;IACtB,gBAAgB,CAA2B;IAC3C,IAAI,CAAa;IAEzB,YAAY,MAAgC;QAC1C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC;YAC3B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,WAAW;YAChC,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,IAAI;YACzB,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM;YAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,kBAAkB,EAAE,IAAI;YACxB,eAAe,EAAE,EAAE;YACnB,UAAU,EAAE,CAAC;YACb,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,OAA4B;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpH,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAE7C,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,OAAO,GAAS,IAAY,CAAC,UAAU,CAAC;YAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC;YAElG,IAAI,YAAY,GAAU,EAAE,CAAC;YAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,MAAM,OAAO,GAAG,GAAG,EAAE;gBACnB,WAAW,CAAC,OAAO,EAAE,CAAC;YACxB,CAAC,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;oBACpC,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;wBAC5B,MAAM;oBACR,CAAC;oBAED,YAAY,CAAC,IAAI,CAAC,GAAU,CAAC,CAAC;oBAE9B,IAAI,YAAY,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;wBACrC,MAAM;4BACJ,IAAI,EAAE,YAAY;4BAClB,MAAM;yBACP,CAAC;wBACF,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;wBAC9B,YAAY,GAAG,EAAE,CAAC;oBACpB,CAAC;gBACH,CAAC;gBAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBACxD,MAAM;wBACJ,IAAI,EAAE,YAAY;wBAClB,MAAM;qBACP,CAAC;gBACJ,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBACvD,CAAC;gBACD,WAAW,CAAC,OAAO,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CACT,KAA+B,EAC/B,OAA6B;QAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAE7C,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,YAAY,GAAG,CAAC,CAAC;YACtD,CAAC;YAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC;YAC5C,IAAI,aAAa,GAAa,EAAE,CAAC;YAEjC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAChC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEtC,kDAAkD;gBAClD,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,YAAY,EAAE,CAAC;oBACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;oBAChC,MAAM,cAAc,GAAwB,EAAE,CAAC;oBAC/C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAClD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;4BAC5B,cAAc,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;wBACrE,CAAC;6BAAM,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;4BACnC,cAAc,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;wBACjC,CAAC;6BAAM,IAAI,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC;4BACpC,cAAc,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;wBAClC,CAAC;6BAAM,CAAC;4BACN,cAAc,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;wBACjC,CAAC;oBACH,CAAC;oBACD,MAAM,GAAG,GAAG,sBAAsB,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;oBAC5F,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;wBACpD,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;oBACvG,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACtB,YAAY,GAAG,IAAI,CAAC;gBACtB,CAAC;gBAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC/B,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;gBAC9C,CAAC;gBAED,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAChC,CAAC;gBAED,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtF,MAAM,UAAU,GAAG,KAAK,CAAC;oBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;oBAE7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;wBACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;wBACpD,MAAM,cAAc,GAAG,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;wBACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBACtE,MAAM,UAAU,GAAc,EAAE,CAAC;wBAEjC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;4BAC3B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gCAChC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;gCACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;oCAC9F,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;gCACrC,CAAC;qCAAM,CAAC;oCACN,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gCAC9C,CAAC;4BACH,CAAC;4BACD,WAAW,EAAE,CAAC;wBAChB,CAAC;wBAED,IAAI,GAAG,GAAG,eAAe,YAAY,KAAK,WAAW,YAAY,eAAe,EAAE,CAAC;wBAEnF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;4BACnB,MAAM,UAAU,GAAG,aAAa;iCAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC;iCACvF,IAAI,CAAC,IAAI,CAAC,CAAC;4BACd,GAAG,IAAI,4BAA4B,UAAU,EAAE,CAAC;wBAClD,CAAC;wBAED,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBACpC,CAAC;oBAED,IAAI,KAAK,EAAE,CAAC;wBACV,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;oBACtB,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,KAAK,EAAE,CAAC;wBACV,IAAI,CAAC;4BACH,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACxB,CAAC;wBAAC,MAAM,CAAC;4BACP,yBAAyB;wBAC3B,CAAC;oBACH,CAAC;oBACD,MAAM,GAAG,CAAC;gBACZ,CAAC;gBAED,cAAc,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;QAED,OAAO;YACL,WAAW;YACX,cAAc;YACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,YAAY;SACb,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,CAAC,IAAI,CAAC,GAAQ,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CACvC;;;;2BAIqB,CACtB,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa;QAChC,MAAM,CAAC,IAAI,CAAC,GAAQ,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CACvC;;;iCAG2B,EAC3B,CAAC,KAAK,CAAC,CACR,CAAC;QAEF,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,OAAO,GAA0B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE;YACzD,MAAM,OAAO,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,CAAC;YAC/C,MAAM,OAAO,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,CAAC;YAC/C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,KAAK,CAAC;YACtD,IAAI,IAAI;gBAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEpC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,OAAO;gBACf,WAAW,EAAE,kBAAkB,CAAC,OAAO,CAAC;gBACxC,QAAQ,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,CAAC,KAAK,KAAK;gBACpD,YAAY,EAAE,IAAI;gBAClB,YAAY,EAAE,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,cAAc;aACnD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,KAAK;YACL,OAAO;YACP,WAAW;SACZ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,MAAkB;QAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACpD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;CACF"}

package/dist/db/adapters/postgres.d.ts

@@ -0,0 +1,17 @@
+import type { DataBatch } from "../../core/types.js";
+import type { DatabaseAdapter, DatabaseConnectionConfig, DatabaseDialect, DatabaseReadOptions, DatabaseWriteOptions, DatabaseWriteResult, TableSchemaMetadata } from "../types.js";
+export declare class PostgresDatabaseAdapter implements DatabaseAdapter {
+    readonly dialect: DatabaseDialect;
+    readonly sanitizedUrl: string;
+    private client;
+    private isConnected;
+    constructor(config: DatabaseConnectionConfig);
+    private ensureConnected;
+    stream(options: DatabaseReadOptions): AsyncIterable<DataBatch>;
+    write(input: AsyncIterable<DataBatch>, options: DatabaseWriteOptions): Promise<DatabaseWriteResult>;
+    getTables(): Promise<string[]>;
+    getTableSchema(table: string): Promise<TableSchemaMetadata>;
+    executeRaw(sql: string, params?: unknown[]): Promise<unknown>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=postgres.d.ts.map

package/dist/db/adapters/postgres.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"postgres.d.ts","sourceRoot":"","sources":["../../../src/db/adapters/postgres.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAO,MAAM,qBAAqB,CAAC;AAE1D,OAAO,KAAK,EACV,eAAe,EACf,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EAEnB,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAErB,qBAAa,uBAAwB,YAAW,eAAe;IAC7D,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAc;IAC/C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,WAAW,CAAS;gBAEhB,MAAM,EAAE,wBAAwB;YAQ9B,eAAe;IAOtB,MAAM,CAAC,OAAO,EAAE,mBAAmB,GAAG,aAAa,CAAC,SAAS,CAAC;IAwD/D,KAAK,CACT,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,EAC/B,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC;IA+HzB,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAY9B,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAgD3D,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAK7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}

package/dist/db/adapters/postgres.js

@@ -0,0 +1,239 @@
+import pg from "pg";
+import QueryStream from "pg-query-stream";
+import { escapeIdentifier, escapeQualifiedTable, generateCreateTableSql, postgresTypeToRowpipe } from "../mapping.js";
+export class PostgresDatabaseAdapter {
+    dialect = "postgres";
+    sanitizedUrl;
+    client;
+    isConnected = false;
+    constructor(config) {
+        this.sanitizedUrl = config.sanitizedUrl;
+        this.client = new pg.Client({
+            connectionString: config.url,
+            ssl: config.ssl,
+        });
+    }
+    async ensureConnected() {
+        if (!this.isConnected) {
+            await this.client.connect();
+            this.isConnected = true;
+        }
+    }
+    async *stream(options) {
+        await this.ensureConnected();
+        const batchSize = Math.max(1, options.batchSize || 1000);
+        const sql = options.query || (options.table ? `SELECT * FROM ${escapeQualifiedTable(options.table, "postgres")}` : "");
+        if (!sql) {
+            throw new Error("Either table or query must be specified for PostgreSQL stream");
+        }
+        const params = options.params || [];
+        const queryStream = new QueryStream(sql, params, { batchSize });
+        const stream = this.client.query(queryStream);
+        let currentBatch = [];
+        let offset = 0;
+        const onAbort = () => {
+            stream.destroy();
+        };
+        if (options.signal) {
+            options.signal.addEventListener("abort", onAbort, { once: true });
+        }
+        try {
+            for await (const row of stream) {
+                if (options.signal?.aborted) {
+                    break;
+                }
+                currentBatch.push(row);
+                if (currentBatch.length >= batchSize) {
+                    yield {
+                        rows: currentBatch,
+                        offset,
+                    };
+                    offset += currentBatch.length;
+                    currentBatch = [];
+                }
+            }
+            if (currentBatch.length > 0 && !options.signal?.aborted) {
+                yield {
+                    rows: currentBatch,
+                    offset,
+                };
+            }
+        }
+        finally {
+            if (options.signal) {
+                options.signal.removeEventListener("abort", onAbort);
+            }
+            stream.destroy();
+        }
+    }
+    async write(input, options) {
+        await this.ensureConnected();
+        const startTime = Date.now();
+        let rowsWritten = 0;
+        let batchesWritten = 0;
+        let tableCreated = false;
+        const table = options.table;
+        if (!table) {
+            throw new Error("Target table name is required for database write");
+        }
+        const escapedTable = escapeQualifiedTable(table, "postgres");
+        if (options.truncate) {
+            await this.client.query(`TRUNCATE TABLE ${escapedTable};`);
+        }
+        const useTx = options.transaction !== false;
+        let insertColumns = [];
+        for await (const batch of input) {
+            if (batch.rows.length === 0)
+                continue;
+            // Auto create table from first batch if requested
+            if (options.createTable && !tableCreated) {
+                const firstRow = batch.rows[0];
+                const inferredSchema = {};
+                for (const [key, val] of Object.entries(firstRow)) {
+                    if (typeof val === "number") {
+                        inferredSchema[key] = Number.isInteger(val) ? "integer" : "number";
+                    }
+                    else if (typeof val === "bigint") {
+                        inferredSchema[key] = "bigint";
+                    }
+                    else if (typeof val === "boolean") {
+                        inferredSchema[key] = "boolean";
+                    }
+                    else {
+                        inferredSchema[key] = "string";
+                    }
+                }
+                const ddl = generateCreateTableSql(table, inferredSchema, "postgres", options.conflictColumns);
+                if (options.dryRun) {
+                    process.stdout.write(`\n-- Dry-run DDL:\n${ddl}\n`);
+                    return { rowsWritten: 0, batchesWritten: 0, durationMs: Date.now() - startTime, tableCreated: true };
+                }
+                await this.client.query(ddl);
+                tableCreated = true;
+            }
+            if (insertColumns.length === 0) {
+                insertColumns = Object.keys(batch.rows[0]);
+            }
+            if (useTx) {
+                await this.client.query("BEGIN;");
+            }
+            try {
+                const escapedCols = insertColumns.map((c) => escapeIdentifier(c, "postgres")).join(", ");
+                const MAX_PARAMS = 50000;
+                const chunkSize = Math.max(1, Math.floor(MAX_PARAMS / insertColumns.length));
+                for (let i = 0; i < batch.rows.length; i += chunkSize) {
+                    const rowChunk = batch.rows.slice(i, i + chunkSize);
+                    const valuesPlaceholderRows = [];
+                    const flatParams = [];
+                    let paramIdx = 1;
+                    for (const row of rowChunk) {
+                        const placeholders = [];
+                        for (const col of insertColumns) {
+                            const v = row[col];
+                            placeholders.push(`$${paramIdx++}`);
+                            if (typeof v === "object" && v !== null && !(v instanceof Uint8Array) && !(v instanceof Date)) {
+                                flatParams.push(JSON.stringify(v));
+                            }
+                            else {
+                                flatParams.push(v === undefined ? null : v);
+                            }
+                        }
+                        valuesPlaceholderRows.push(`(${placeholders.join(", ")})`);
+                        rowsWritten++;
+                    }
+                    let sql = `INSERT INTO ${escapedTable} (${escapedCols}) VALUES ${valuesPlaceholderRows.join(", ")}`;
+                    if (options.upsert && options.conflictColumns && options.conflictColumns.length > 0) {
+                        const conflictCols = options.conflictColumns.map((c) => escapeIdentifier(c, "postgres")).join(", ");
+                        const updateCols = insertColumns
+                            .filter((c) => !options.conflictColumns.includes(c))
+                            .map((c) => `${escapeIdentifier(c, "postgres")} = EXCLUDED.${escapeIdentifier(c, "postgres")}`)
+                            .join(", ");
+                        if (updateCols.length > 0) {
+                            sql += ` ON CONFLICT (${conflictCols}) DO UPDATE SET ${updateCols}`;
+                        }
+                        else {
+                            sql += ` ON CONFLICT (${conflictCols}) DO NOTHING`;
+                        }
+                    }
+                    await this.client.query(sql, flatParams);
+                }
+                if (useTx) {
+                    await this.client.query("COMMIT;");
+                }
+            }
+            catch (err) {
+                if (useTx) {
+                    try {
+                        await this.client.query("ROLLBACK;");
+                    }
+                    catch {
+                        // Ignore rollback errors
+                    }
+                }
+                throw err;
+            }
+            batchesWritten++;
+        }
+        return {
+            rowsWritten,
+            batchesWritten,
+            durationMs: Date.now() - startTime,
+            tableCreated,
+        };
+    }
+    async getTables() {
+        await this.ensureConnected();
+        const res = await this.client.query(`SELECT table_schema || '.' || table_name AS full_name
+       FROM information_schema.tables
+       WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
+         AND table_type = 'BASE TABLE'
+       ORDER BY table_schema, table_name`);
+        return res.rows.map((r) => r.full_name);
+    }
+    async getTableSchema(table) {
+        await this.ensureConnected();
+        let schemaName = "public";
+        let tableName = table;
+        if (table.includes(".")) {
+            const parts = table.split(".");
+            schemaName = parts[0];
+            tableName = parts[1];
+        }
+        const res = await this.client.query(`SELECT column_name, udt_name, data_type, is_nullable, column_default
+       FROM information_schema.columns
+       WHERE table_schema = $1 AND table_name = $2
+       ORDER BY ordinal_position`, [schemaName, tableName]);
+        const pkRes = await this.client.query(`SELECT kcu.column_name
+       FROM information_schema.table_constraints tc
+       JOIN information_schema.key_column_usage kcu
+         ON tc.constraint_name = kcu.constraint_name
+         AND tc.table_schema = kcu.table_schema
+       WHERE tc.constraint_type = 'PRIMARY KEY'
+         AND tc.table_schema = $1 AND tc.table_name = $2`, [schemaName, tableName]);
+        const primaryKeys = pkRes.rows.map((r) => r.column_name);
+        const columns = res.rows.map((r) => ({
+            name: r.column_name,
+            dbType: r.udt_name || r.data_type,
+            rowpipeType: postgresTypeToRowpipe(r.udt_name || r.data_type),
+            nullable: r.is_nullable === "YES",
+            isPrimaryKey: primaryKeys.includes(r.column_name),
+            defaultValue: r.column_default,
+        }));
+        return {
+            table,
+            columns,
+            primaryKeys,
+        };
+    }
+    async executeRaw(sql, params) {
+        await this.ensureConnected();
+        return this.client.query(sql, params);
+    }
+    async close() {
+        if (this.isConnected) {
+            await this.client.end();
+            this.isConnected = false;
+        }
+    }
+}
+//# sourceMappingURL=postgres.js.map

package/dist/db/adapters/postgres.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"postgres.js","sourceRoot":"","sources":["../../../src/db/adapters/postgres.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,WAAW,MAAM,iBAAiB,CAAC;AAE1C,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAYtH,MAAM,OAAO,uBAAuB;IACzB,OAAO,GAAoB,UAAU,CAAC;IACtC,YAAY,CAAS;IACtB,MAAM,CAAY;IAClB,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,MAAgC;QAC1C,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC;YAC1B,gBAAgB,EAAE,MAAM,CAAC,GAAG;YAC5B,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,OAA4B;QACxC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvH,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAU,EAAE,CAAC;QAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC;QAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;oBAC5B,MAAM;gBACR,CAAC;gBAED,YAAY,CAAC,IAAI,CAAC,GAAU,CAAC,CAAC;gBAE9B,IAAI,YAAY,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,MAAM;wBACJ,IAAI,EAAE,YAAY;wBAClB,MAAM;qBACP,CAAC;oBACF,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;oBAC9B,YAAY,GAAG,EAAE,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACxD,MAAM;oBACJ,IAAI,EAAE,YAAY;oBAClB,MAAM;iBACP,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CACT,KAA+B,EAC/B,OAA6B;QAE7B,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE7D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,YAAY,GAAG,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC;QAC5C,IAAI,aAAa,GAAa,EAAE,CAAC;QAEjC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,kDAAkD;YAClD,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAChC,MAAM,cAAc,GAAwB,EAAE,CAAC;gBAC/C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAClD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;wBAC5B,cAAc,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;oBACrE,CAAC;yBAAM,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;wBACnC,cAAc,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;oBACjC,CAAC;yBAAM,IAAI,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC;wBACpC,cAAc,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;oBAClC,CAAC;yBAAM,CAAC;wBACN,cAAc,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;oBACjC,CAAC;gBACH,CAAC;gBACD,MAAM,GAAG,GAAG,sBAAsB,CAAC,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;gBAC/F,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC;oBACpD,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;gBACvG,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,YAAY,GAAG,IAAI,CAAC;YACtB,CAAC;YAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzF,MAAM,UAAU,GAAG,KAAK,CAAC;gBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;gBAE7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;oBACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;oBACpD,MAAM,qBAAqB,GAAa,EAAE,CAAC;oBAC3C,MAAM,UAAU,GAAc,EAAE,CAAC;oBACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;oBAEjB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;wBAC3B,MAAM,YAAY,GAAa,EAAE,CAAC;wBAClC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;4BAChC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;4BACnB,YAAY,CAAC,IAAI,CAAC,IAAI,QAAQ,EAAE,EAAE,CAAC,CAAC;4BACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;gCAC9F,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;4BACrC,CAAC;iCAAM,CAAC;gCACN,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;4BAC9C,CAAC;wBACH,CAAC;wBACD,qBAAqB,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC3D,WAAW,EAAE,CAAC;oBAChB,CAAC;oBAED,IAAI,GAAG,GAAG,eAAe,YAAY,KAAK,WAAW,YAAY,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAEpG,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACpF,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBACpG,MAAM,UAAU,GAAG,aAAa;6BAC7B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,eAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;6BACpD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,UAAU,CAAC,eAAe,gBAAgB,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;6BAC9F,IAAI,CAAC,IAAI,CAAC,CAAC;wBAEd,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC1B,GAAG,IAAI,iBAAiB,YAAY,mBAAmB,UAAU,EAAE,CAAC;wBACtE,CAAC;6BAAM,CAAC;4BACN,GAAG,IAAI,iBAAiB,YAAY,cAAc,CAAC;wBACrD,CAAC;oBACH,CAAC;oBAED,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC3C,CAAC;gBAED,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,KAAK,EAAE,CAAC;oBACV,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;oBACvC,CAAC;oBAAC,MAAM,CAAC;wBACP,yBAAyB;oBAC3B,CAAC;gBACH,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,cAAc,EAAE,CAAC;QACnB,CAAC;QAED,OAAO;YACL,WAAW;YACX,cAAc;YACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,YAAY;SACb,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CACjC;;;;yCAImC,CACpC,CAAC;QACF,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAa;QAChC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC/B,UAAU,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACxB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CACjC;;;iCAG2B,EAC3B,CAAC,UAAU,EAAE,SAAS,CAAC,CACxB,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CACnC;;;;;;yDAMmD,EACnD,CAAC,UAAU,EAAE,SAAS,CAAC,CACxB,CAAC;QAEF,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAE9D,MAAM,OAAO,GAA0B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAC/D,IAAI,EAAE,CAAC,CAAC,WAAW;YACnB,MAAM,EAAE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS;YACjC,WAAW,EAAE,qBAAqB,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC;YAC7D,QAAQ,EAAE,CAAC,CAAC,WAAW,KAAK,KAAK;YACjC,YAAY,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;YACjD,YAAY,EAAE,CAAC,CAAC,cAAc;SAC/B,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,KAAK;YACL,OAAO;YACP,WAAW;SACZ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,MAAkB;QAC9C,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACxB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC;CACF"}

package/dist/db/adapters/sqlite.d.ts

@@ -0,0 +1,18 @@
+import type { DataBatch, Row } from "../../core/types.js";
+import type { DatabaseAdapter, DatabaseConnectionConfig, DatabaseDialect, DatabaseReadOptions, DatabaseWriteOptions, DatabaseWriteResult, TableSchemaMetadata } from "../types.js";
+export declare class SqliteDatabaseAdapter implements DatabaseAdapter {
+    readonly dialect: DatabaseDialect;
+    readonly config: DatabaseConnectionConfig;
+    readonly sanitizedUrl: string;
+    private db;
+    private isClosed;
+    constructor(config: DatabaseConnectionConfig);
+    stream(options: DatabaseReadOptions): AsyncIterable<DataBatch>;
+    write(input: AsyncIterable<DataBatch>, options: DatabaseWriteOptions): Promise<DatabaseWriteResult>;
+    insertBatch(table: string, rows: Row[], options?: DatabaseWriteOptions): Promise<DatabaseWriteResult>;
+    getTables(): Promise<string[]>;
+    getTableSchema(table: string): Promise<TableSchemaMetadata>;
+    executeRaw(sql: string, params?: unknown[]): Promise<unknown>;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=sqlite.d.ts.map

package/dist/db/adapters/sqlite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../src/db/adapters/sqlite.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,KAAK,EACV,eAAe,EACf,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EAEnB,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAYrB,qBAAa,qBAAsB,YAAW,eAAe;IAC3D,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAY;IAC7C,QAAQ,CAAC,MAAM,EAAE,wBAAwB,CAAC;IAC1C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,OAAO,CAAC,EAAE,CAAe;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,wBAAwB;IAOrC,MAAM,CAAC,OAAO,EAAE,mBAAmB,GAAG,aAAa,CAAC,SAAS,CAAC;IA4C/D,KAAK,CACT,KAAK,EAAE,aAAa,CAAC,SAAS,CAAC,EAC/B,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC;IAyGzB,WAAW,CACf,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,GAAG,EAAE,EACX,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,mBAAmB,CAAC;IAOzB,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAQ9B,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IA6B3D,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAQ7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAS7B"}