velocious 1.0.534 → 1.0.536

package/src/database/drivers/mysql/index.js

@@ -14,6 +14,7 @@ import Options from "./options.js"
 import mysql from "mysql"
 import query from "./query.js"
 import QueryParser from "./query-parser.js"
+import streamQuery from "./query-stream.js"
 import RemoveIndex from "./sql/remove-index.js"
 import Table from "./table.js"
 import StructureSql from "./structure-sql.js"
@@ -320,19 +321,32 @@ export default class VelociousDatabaseDriversMysql extends Base{
    * @returns {import("../base.js").RetryableDatabaseErrorResult} - Retry info.
    */
   retryableDatabaseError(error) {
-    const errorCode = /** @type {?} */ (error).code
-    const message = error.message || ""
-    const shouldRetry = (
-      errorCode == "ECONNREFUSED" ||
-      message.includes("ECONNREFUSED") ||
-      message.includes("connect ECONNREFUSED") ||
-      message.includes("PROTOCOL_CONNECTION_LOST") ||
-      message.includes("Connection lost")
-    )
+    /** @type {Error | undefined} */
+    let currentError = error
+    let shouldReconnect = false
+
+    while (currentError) {
+      const errorCode = "code" in currentError && typeof currentError.code == "string" ? currentError.code : undefined
+      const message = currentError.message || ""
+
+      if (errorCode == "ER_CHECKREAD" || message.includes("Record has changed since last read")) {
+        return {retry: true, reconnect: false, waitMs: 50}
+      }
+
+      shouldReconnect ||= (
+        errorCode == "ECONNREFUSED" ||
+        message.includes("ECONNREFUSED") ||
+        message.includes("connect ECONNREFUSED") ||
+        message.includes("PROTOCOL_CONNECTION_LOST") ||
+        message.includes("Connection lost")
+      )
+
+      currentError = currentError.cause instanceof Error ? currentError.cause : undefined
+    }
 
     return {
-      retry: shouldRetry,
-      reconnect: shouldRetry,
+      retry: shouldReconnect,
+      reconnect: shouldReconnect,
       waitMs: 50
     }
   }
@@ -358,6 +372,20 @@ export default class VelociousDatabaseDriversMysql extends Base{
     }
   }
 
+  /**
+   * Streams the rows of `sql` from a dedicated pooled connection using the MySQL cursor, so a
+   * large result set is read incrementally instead of being buffered. Overrides the base
+   * buffered fallback with true server-side streaming.
+   * @param {string} sql - SQL string to stream.
+   * @yields {Record<string, unknown>} - The result rows, one at a time.
+   */
+  async *queryStream(sql) {
+    if (!this.pool) await this.connect()
+    if (!this.pool) throw new Error("MySQL pool failed to initialize")
+
+    yield* streamQuery(this.pool, sql)
+  }
+
   /**
    * Executes a mutation with affected-row metadata.
    * @param {string} sql - Mutation SQL.

package/src/database/drivers/mysql/query-stream.js

@@ -0,0 +1,43 @@
+import {PassThrough} from "node:stream"
+
+/**
+ * Streams the rows of `sql` from a dedicated pooled connection, yielding row objects one at a
+ * time so an arbitrarily large result set is never buffered in memory. The `mysql` package's
+ * query stream is a `readable-stream` polyfill that is not async-iterable, so it is piped through
+ * a native {@link PassThrough} (which is) — `pipe` preserves backpressure, pausing the source
+ * connection when the consumer falls behind. The connection is released back to the pool on
+ * normal completion, and destroyed if iteration is aborted (a `break`/`throw` out of the
+ * consuming `for await`) so a half-drained connection is never handed back to the pool.
+ * @param {import("mysql").Pool} pool - Pool to check a streaming connection out of.
+ * @param {string} sql - SQL string to stream.
+ * @yields {Record<string, unknown>} - The result rows, one at a time.
+ */
+export default async function* streamQuery(pool, sql) {
+  const connection = await new Promise((resolve, reject) => {
+    pool.getConnection((error, pooledConnection) => {
+      if (error) reject(error)
+      else resolve(pooledConnection)
+    })
+  })
+  let completed = false
+
+  try {
+    const sourceStream = connection.query(sql).stream()
+    const rowStream = new PassThrough({objectMode: true})
+
+    sourceStream.on("error", (/** @type {unknown} */ error) => rowStream.destroy(error instanceof Error ? error : new Error(String(error))))
+    sourceStream.pipe(rowStream)
+
+    for await (const row of rowStream) {
+      yield row
+    }
+
+    completed = true
+  } finally {
+    if (completed) {
+      connection.release()
+    } else {
+      connection.destroy()
+    }
+  }
+}

package/src/database/tenants/data-copier.js

@@ -2,6 +2,7 @@
 
 const DEFAULT_INSERT_CHUNK_SIZE = 100
 const DEFAULT_QUERY_CHUNK_SIZE = 500
+const DEFAULT_STREAM_BATCH_SIZE = 1000
 
 /**
  * Splits an array into chunks of at most `chunkSize` items.
@@ -130,6 +131,35 @@ export default class DataCopier {
    * @returns {Promise<Map<string, Record<string, unknown>[]>>} - Loaded rows grouped by table name.
    */
   async loadRows(db, keyValue) {
+    return (await this.traversePlan(db, keyValue)).rowsByTableName
+  }
+
+  /**
+   * Loads only the ids for `keyValue` for every table in the plan from `db`, using the same
+   * parent/child traversal as {@link DataCopier#loadRows} but selecting just the id column.
+   * Callers that only need to compare row membership — for example verifying a tenant already
+   * holds every default row before a cleanup delete — should use this instead of loadRows so
+   * they never materialise full rows; for large tenants that is the difference between a few
+   * kilobytes of ids and gigabytes of row data.
+   * @param {import("../drivers/base.js").default} db - Source or target database to traverse.
+   * @param {string} keyValue - Tenant key selecting the root plan rows.
+   * @returns {Promise<Map<string, string[]>>} - Loaded ids grouped by table name.
+   */
+  async loadRowIds(db, keyValue) {
+    return (await this.traversePlan(db, keyValue, [this.idColumn])).idsByTableName
+  }
+
+  /**
+   * Traverses the table plan for `keyValue`, querying each table by its tenant key column or
+   * (for child tables) by the ids already selected for its parent, and returns both the ids
+   * and the loaded rows grouped by table name. `selectColumns` bounds the columns each query
+   * selects; pass `[idColumn]` for an id-only traversal, or omit it to load full rows.
+   * @param {import("../drivers/base.js").default} db - Source or target database to traverse.
+   * @param {string} keyValue - Tenant key selecting the root plan rows.
+   * @param {string[]} [selectColumns] - Columns to select; defaults to every column.
+   * @returns {Promise<{idsByTableName: Map<string, string[]>, rowsByTableName: Map<string, Record<string, unknown>[]>}>} - Ids and rows grouped by table name.
+   */
+  async traversePlan(db, keyValue, selectColumns) {
     /** @type {Map<string, string[]>} */
     const idsByTableName = new Map()
     /** @type {Map<string, Record<string, unknown>[]>} */
@@ -142,6 +172,7 @@ export default class DataCopier {
         rows = await this.queryRowsByColumn({
           columnName: tableConfig.keyColumn,
           db,
+          selectColumns,
           tableName: tableConfig.tableName,
           values: [keyValue]
         })
@@ -157,6 +188,7 @@ export default class DataCopier {
         rows = await this.queryRowsByColumn({
           columnName: tableConfig.parentColumn,
           db,
+          selectColumns,
           tableName: tableConfig.tableName,
           values: idsByTableName.get(tableConfig.parentTableName) || []
         })
@@ -170,15 +202,184 @@ export default class DataCopier {
       rowsByTableName.set(tableConfig.tableName, rows)
     }
 
-    return rowsByTableName
+    return {idsByTableName, rowsByTableName}
+  }
+
+  /**
+   * The plan tables referenced as a parent by some child entry. Their ids must be retained
+   * while streaming so their children can be scoped; leaf tables — typically the high-volume
+   * ones — are never in this set and so are never accumulated in memory.
+   * @returns {Set<string>} - Table names that are a parent of another plan entry.
+   */
+  parentTableNames() {
+    /** @type {Set<string>} */
+    const names = new Set()
+
+    for (const tableConfig of this.tablePlan) {
+      if (tableConfig.parentTableName) {
+        names.add(tableConfig.parentTableName)
+      }
+    }
+
+    return names
+  }
+
+  /**
+   * Streams every plan table's source rows scoped to `keyValue` in bounded `batchSize` batches,
+   * following parent/child chaining. Each table is read through a real
+   * {@link import("../drivers/base.js").default#queryStream} cursor, so a large table is never
+   * buffered; only the ids of tables that are themselves a parent are retained (to scope their
+   * children). `selectColumns` bounds each row's projection — pass `[idColumn]` for a light
+   * id-only scan, or omit it to stream full rows — and must include the id column for any table
+   * that has children. Memory stays bounded to one batch plus the retained parent-table ids.
+   * @param {import("../drivers/base.js").default} db - Source database to stream.
+   * @param {string} keyValue - Tenant key selecting the root plan rows.
+   * @param {{batchSize: number, selectColumns?: string[]}} options - Batch size and optional projection.
+   * @yields {{rows: Record<string, unknown>[], tableName: string}} - Successive row batches per table.
+   */
+  async *streamPlanSourceBatches(db, keyValue, {batchSize, selectColumns}) {
+    /** @type {Map<string, string[]>} */
+    const retainedIdsByTableName = new Map()
+    const parentTableNames = this.parentTableNames()
+
+    for (const tableConfig of this.tablePlan) {
+      /** @type {string} */
+      let scopeColumn
+      /** @type {string[]} */
+      let scopeValues
+
+      if (tableConfig.keyColumn) {
+        scopeColumn = tableConfig.keyColumn
+        scopeValues = [keyValue]
+      } else {
+        if (!tableConfig.parentColumn || !tableConfig.parentTableName) {
+          throw new Error(`Expected keyColumn or parentTableName+parentColumn for table ${tableConfig.tableName} in the tenant table plan.`)
+        }
+
+        if (!retainedIdsByTableName.has(tableConfig.parentTableName)) {
+          throw new Error(`Tenant table plan entry ${tableConfig.tableName} references parent table ${tableConfig.parentTableName}, which has not been streamed; parent tables must appear before their children in the plan.`)
+        }
+
+        scopeColumn = tableConfig.parentColumn
+        scopeValues = retainedIdsByTableName.get(tableConfig.parentTableName) || []
+      }
+
+      /** @type {string[] | null} */
+      const retainedIds = parentTableNames.has(tableConfig.tableName) ? [] : null
+      const quotedTable = db.quoteTable(tableConfig.tableName)
+      const quotedScope = db.quoteColumn(scopeColumn)
+      const selectList = selectColumns
+        ? selectColumns.map((column) => `${quotedTable}.${db.quoteColumn(column)}`).join(", ")
+        : `${quotedTable}.*`
+      /** @type {Record<string, unknown>[]} */
+      let batch = []
+
+      for (const scopeChunk of chunks(uniqueStrings(scopeValues), this.queryChunkSize)) {
+        const sql = `SELECT ${selectList} FROM ${quotedTable} WHERE ${quotedScope} IN (${this.quotedValuesSql(db, scopeChunk)})`
+
+        for await (const row of db.queryStream(sql)) {
+          batch.push(row)
+
+          if (retainedIds) {
+            retainedIds.push(String(row[this.idColumn]))
+          }
+
+          if (batch.length >= batchSize) {
+            yield {rows: batch, tableName: tableConfig.tableName}
+            batch = []
+          }
+        }
+      }
+
+      if (batch.length > 0) {
+        yield {rows: batch, tableName: tableConfig.tableName}
+      }
+
+      retainedIdsByTableName.set(tableConfig.tableName, retainedIds || [])
+    }
+  }
+
+  /**
+   * Returns which of `ids` currently exist in `tableName` in `db`. `ids` is one already-bounded
+   * batch, so it is probed with a single `IN (...)` lookup.
+   * @param {{db: import("../drivers/base.js").default, ids: string[], tableName: string}} args - Database, ids to probe, and table.
+   * @returns {Promise<Set<string>>} - The subset of `ids` present in the table.
+   */
+  async queryExistingIds({db, ids, tableName}) {
+    if (ids.length <= 0) {
+      return new Set()
+    }
+
+    const quotedTable = db.quoteTable(tableName)
+    const quotedId = db.quoteColumn(this.idColumn)
+    const rows = await this.executeQuietQuery(db, `SELECT ${quotedTable}.${quotedId} FROM ${quotedTable} WHERE ${quotedId} IN (${this.quotedValuesSql(db, ids)})`)
+
+    return new Set(rows.map((row) => String(row[this.idColumn])))
+  }
+
+  /**
+   * Streams the source ids for `keyValue` and returns the first table found to be missing rows in
+   * the target, with that batch's missing ids — stopping at the first shortfall instead of
+   * enumerating every missing row. Callers verifying a tenant already holds every source row (for
+   * example before deleting the source copies) treat an empty result as the go-ahead and any entry
+   * as a hard stop; failing fast keeps memory bounded to a single batch even when the target is
+   * far behind.
+   * @param {string} keyValue - Tenant key selecting the source rows to check.
+   * @param {{batchSize?: number}} [options] - Streaming batch size.
+   * @returns {Promise<Map<string, string[]>>} - The first table missing rows and that batch's missing ids, or empty when the target holds everything.
+   */
+  async findMissingRowIds(keyValue, {batchSize = DEFAULT_STREAM_BATCH_SIZE} = {}) {
+    /** @type {Map<string, string[]>} */
+    const missingByTableName = new Map()
+
+    for await (const {rows, tableName} of this.streamPlanSourceBatches(this.sourceDb, keyValue, {batchSize, selectColumns: [this.idColumn]})) {
+      const ids = rows.map((row) => String(row[this.idColumn]))
+      const existingIds = await this.queryExistingIds({db: this.targetDb, ids, tableName})
+      const missingIds = ids.filter((id) => !existingIds.has(id))
+
+      if (missingIds.length > 0) {
+        missingByTableName.set(tableName, missingIds)
+        break
+      }
+    }
+
+    return missingByTableName
+  }
+
+  /**
+   * Streams the source rows for `keyValue` and copies into the target, batch by batch, only the
+   * rows missing there. Because the full rows travel in the stream, each batch's missing rows are
+   * inserted as it arrives — nothing is accumulated, and no second source query runs while the
+   * source connection is held by the stream — so a table with large columns stays bounded to one
+   * batch even when the target is empty. Intended for single-table plans (or plans whose per-table,
+   * parent-first insert order is foreign-key safe).
+   * @param {string} keyValue - Tenant key selecting the source rows to reconcile.
+   * @param {{batchSize?: number}} [options] - Streaming batch size.
+   * @returns {Promise<number>} - The number of rows copied into the target.
+   */
+  async copyMissingRows(keyValue, {batchSize = DEFAULT_STREAM_BATCH_SIZE} = {}) {
+    let copiedCount = 0
+
+    for await (const {rows, tableName} of this.streamPlanSourceBatches(this.sourceDb, keyValue, {batchSize})) {
+      const existingIds = await this.queryExistingIds({db: this.targetDb, ids: rows.map((row) => String(row[this.idColumn])), tableName})
+      const missingRows = rows.filter((row) => !existingIds.has(String(row[this.idColumn])))
+
+      if (missingRows.length > 0) {
+        await this.insertTargetRows(new Map([[tableName, missingRows]]))
+        copiedCount += missingRows.length
+      }
+    }
+
+    return copiedCount
   }
 
   /**
-   * Selects all rows of `tableName` in `db` whose `columnName` is in `values`, chunked.
-   * @param {{columnName: string, db: import("../drivers/base.js").default, tableName: string, values: string[]}} args - Table, column, database, and values for the chunked lookup.
+   * Selects `tableName` rows in `db` whose `columnName` is in `values`, chunked. `selectColumns`
+   * bounds the projection and defaults to every column.
+   * @param {{columnName: string, db: import("../drivers/base.js").default, selectColumns?: string[], tableName: string, values: string[]}} args - Table, column, projection, database, and values for the chunked lookup.
    * @returns {Promise<Record<string, unknown>[]>} - Rows matching the supplied column values.
    */
-  async queryRowsByColumn({columnName, db, tableName, values}) {
+  async queryRowsByColumn({columnName, db, selectColumns, tableName, values}) {
     const normalizedValues = uniqueStrings(values)
 
     if (normalizedValues.length <= 0) {
@@ -188,9 +389,12 @@ export default class DataCopier {
     const rows = []
     const quotedTable = db.quoteTable(tableName)
     const quotedColumn = db.quoteColumn(columnName)
+    const selectList = selectColumns
+      ? selectColumns.map((column) => `${quotedTable}.${db.quoteColumn(column)}`).join(", ")
+      : `${quotedTable}.*`
 
     for (const valuesChunk of chunks(normalizedValues, this.queryChunkSize)) {
-      const sql = `SELECT ${quotedTable}.* FROM ${quotedTable} WHERE ${quotedColumn} IN (${this.quotedValuesSql(db, valuesChunk)})`
+      const sql = `SELECT ${selectList} FROM ${quotedTable} WHERE ${quotedColumn} IN (${this.quotedValuesSql(db, valuesChunk)})`
 
       rows.push(...await this.executeQuietQuery(db, sql))
     }