velocious 1.0.698 → 1.0.700

package/build/database/drivers/sqlite/index.js

@@ -110,42 +110,34 @@ export default class VelociousDatabaseDriversSqliteNode extends Base {
   }
 
   /**
-   * Layers a filesystem lock directory on top of the in-process waiter
-   * queue so SQLite deployments with multiple Node processes writing to
-   * the same database file see consistent advisory-lock mutual exclusion
-   * across processes, not just within a single process.
+   * The filesystem lock directory is this driver's driver-native tier on
+   * top of the shared process-local pre-filter tier from the base driver,
+   * so SQLite deployments with multiple Node processes writing to the same
+   * database file see consistent advisory-lock mutual exclusion across
+   * processes, not just within a single process.
+   * @returns {boolean} - Always true for the Node SQLite driver.
+   */
+  supportsDriverNativeAdvisoryLock() {
+    return true
+  }
+
+  /**
+   * Filesystem lock directory as the driver-native tier on top of the
+   * shared process-local pre-filter tier from the base driver, so SQLite
+   * deployments with multiple Node processes writing to the same database
+   * file see consistent advisory-lock mutual exclusion across processes,
+   * not just within a single process.
    *
-   * The in-process queue from the shared SQLite base class is still used
-   * for the fast intra-process path (no polling, waiters wake each other
-   * through the `Set<string>` + waiter queue); the filesystem lock is
-   * only checked once the in-process queue has granted the caller, so
-   * typical single-process traffic pays at most two `fs.mkdir` calls
-   * (create and remove) per critical section.
+   * The base driver's process-local tier serializes same-process callers
+   * first (no filesystem round trip for uncontended traffic); this hook
+   * is only reached once the tier has granted the caller its turn, and its
+   * result is what confirms cross-process exclusion for SQLite.
    * @param {string} name - Lock name.
    * @param {{timeoutMs?: number | null}} [args] - Optional timeout in milliseconds; `null`, `undefined`, or negative blocks forever.
    * @returns {Promise<boolean>} - Whether the advisory lock was acquired.
    */
   async _acquireAdvisoryLock(name, {timeoutMs} = {}) {
-    const deadline = typeof timeoutMs === "number" && timeoutMs >= 0 ? Date.now() + timeoutMs : null
-    const remainingForInProcess = deadline !== null ? Math.max(0, deadline - Date.now()) : null
-    const inProcessAcquired = await super._acquireAdvisoryLock(name, {timeoutMs: remainingForInProcess})
-
-    if (!inProcessAcquired) return false
-
-    try {
-      const remainingForFile = deadline !== null ? Math.max(0, deadline - Date.now()) : null
-      const fileAcquired = await this._acquireAdvisoryLockFile(name, {timeoutMs: remainingForFile})
-
-      if (!fileAcquired) {
-        await super._releaseAdvisoryLock(name)
-        return false
-      }
-    } catch (error) {
-      await super._releaseAdvisoryLock(name)
-      throw error
-    }
-
-    return true
+    return await this._acquireAdvisoryLockFile(name, {timeoutMs})
   }
 
   /**
@@ -154,53 +146,40 @@ export default class VelociousDatabaseDriversSqliteNode extends Base {
    * @returns {Promise<boolean>} - Whether the advisory lock was acquired immediately.
    */
   async _tryAcquireAdvisoryLock(name) {
-    const inProcessAcquired = await super._tryAcquireAdvisoryLock(name)
-
-    if (!inProcessAcquired) return false
-
-    try {
-      const fileAcquired = await this._tryAcquireAdvisoryLockFile(name)
-
-      if (!fileAcquired) {
-        await super._releaseAdvisoryLock(name)
-        return false
-      }
-    } catch (error) {
-      await super._releaseAdvisoryLock(name)
-      throw error
-    }
-
-    return true
+    return await this._tryAcquireAdvisoryLockFile(name)
   }
 
   /**
-   * Releases the lock only if **this** driver instance acquired it, both
-   * in the shared in-process owner table and in the on-disk lock
-   * directory. A caller that tries to release a lock it never acquired
-   * (or that was already released by another driver) gets `false` back
-   * and the filesystem state is left alone so we never delete somebody
-   * else's lock directory.
+   * Releases the on-disk lock directory for a lock this driver instance
+   * acquired through the base driver's two-tier chain. A caller that tries
+   * to release a lock it never acquired (or that was already released by
+   * another driver) gets `false` back and the filesystem state is left
+   * alone so we never delete somebody else's lock directory.
    * @param {string} name - Lock name.
    * @returns {Promise<boolean>} - Whether the advisory lock was released.
    */
   async _releaseAdvisoryLock(name) {
-    const inProcessReleased = await super._releaseAdvisoryLock(name)
+    const lockPath = this._advisoryLockPath(name)
 
-    if (inProcessReleased) {
-      await this._releaseAdvisoryLockFile(name)
+    try {
+      await fs.stat(lockPath)
+    } catch {
+      return false
     }
 
-    return inProcessReleased
+    await this._releaseAdvisoryLockFile(name)
+
+    return true
   }
 
   /**
-   * Runs is advisory lock held.
+   * Filesystem-tier view for the base driver's `isAdvisoryLockHeld`, which
+   * answers from the process-local tier first and only asks this hook when
+   * the tier has no owner for the name.
    * @param {string} name - Lock name.
-   * @returns {Promise<boolean>} - Whether the advisory lock is held.
+   * @returns {Promise<boolean>} - Whether the advisory lock file is held.
    */
-  async isAdvisoryLockHeld(name) {
-    if (await super.isAdvisoryLockHeld(name)) return true
-
+  async _isAdvisoryLockHeldNative(name) {
     return await this._isAdvisoryLockFileHeld(name)
   }
 

package/build/database/migrator.js

@@ -4,10 +4,24 @@ import {digg} from "diggerize"
 import * as inflection from "inflection"
 import Logger from "../logger.js"
 import MigrationsLedger from "./migrations-ledger.js"
+import AdvisoryLockRunner from "./advisory-lock-runner.js"
 import {NotImplementedError} from "./migration/index.js"
 import { migrationExecutionPhase, migrationRunsInExecutionPhase } from "./migration-execution-phase.js"
 import restArgsError from "../utils/rest-args-error.js"
 
+/**
+ * Builds the advisory-lock name that serialises migrations for one database identifier.
+ *
+ * The name is derived from the database identifier only (not the migration) so that
+ * concurrent processes queue on one lock while applying the pending set, instead of
+ * racing on individual DDL statements.
+ * @param {string} dbIdentifier - Database identifier being migrated.
+ * @returns {string} - Advisory lock name.
+ */
+function migrationsAdvisoryLockName(dbIdentifier) {
+  return `velocious:migrations:${dbIdentifier}`
+}
+
 export default class VelociousDatabaseMigrator {
   /**
    * Migrations versions.
@@ -49,8 +63,11 @@ export default class VelociousDatabaseMigrator {
    * @returns {Promise<void>} - Resolves when complete.
    */
   async prepare() {
+    console.error("[MIG-DEBUG] prepare() start")
     await this.createMigrationsTable()
+    console.error("[MIG-DEBUG] createMigrationsTable done")
     await this.loadMigrationsVersions()
+    console.error("[MIG-DEBUG] prepare() done")
   }
 
   /**
@@ -115,6 +132,7 @@ export default class VelociousDatabaseMigrator {
    */
   async migrateFiles(files, importCallback) {
     let appliedCount = 0
+    console.error("[MIG-DEBUG] migrateFiles count=" + files.length)
 
     await this.configuration.ensureConnections({databaseIdentifiers: this.databaseIdentifiers, name: "Database migrator: migrate files"}, async () => {
       for (const migration of files) {
@@ -278,10 +296,29 @@ export default class VelociousDatabaseMigrator {
 
     if (!environmentHandler || Object.keys(filteredDbs).length == 0) return
 
+    // Serialise the framework-schema bootstrap across processes. This creates the attachments
+    // and background-jobs tables, and it runs once per process outside the per-migration loop:
+    // without this lock, two processes bootstrapping one database race on CREATE TABLE /
+    // CREATE INDEX and one dies mid-boot (duplicate object/index).
+    for (const [dbIdentifier, db] of Object.entries(filteredDbs)) {
+      const runner = new AdvisoryLockRunner({
+        configuration: this.configuration,
+        connectionProvider: () => db,
+        databaseIdentifier: dbIdentifier
+      })
+
+      await runner.withAdvisoryLock(
+        migrationsAdvisoryLockName(dbIdentifier),
+        async () => {
+          await environmentHandler.ensureFrameworkSchema({dbs: {[dbIdentifier]: db}})
+        },
+        {dedicatedConnection: true}
+      )
+    }
+
     // Ensure Velocious' own framework schema before the structure dump. The dump is
     // gated to enabled environments, but migration-enabled databases must include
     // framework tables so `db:migrate` and schema:load produce a complete database.
-    await environmentHandler.ensureFrameworkSchema({dbs: filteredDbs})
     await environmentHandler.afterMigrations({dbs: filteredDbs})
   }
 
@@ -487,6 +524,7 @@ export default class VelociousDatabaseMigrator {
    */
   async runMigrationFile({migration, requireMigration, direction = "up"}) {
     if (!this.configuration) throw new Error("No configuration set")
+    console.error("[MIG-DEBUG] runMigrationFile file=" + (migration && migration.file))
     if (!this.configuration.isDatabasePoolInitialized()) await this.configuration.initializeDatabasePool()
     if (!this.migrationsVersions) await this.loadMigrationsVersions()
 
@@ -542,67 +580,99 @@ export default class VelociousDatabaseMigrator {
         continue
       }
 
-      if (direction == "up") {
-        if (this.hasRunMigrationVersion(dbIdentifier, migration.date)) {
-          this.logger.debug(`${dbIdentifier} has already run migration ${migration.file}`)
-          continue
-        }
-      } else if (direction == "down") {
-        if (!this.hasRunMigrationVersion(dbIdentifier, migration.date)) {
-          this.logger.debug(`${dbIdentifier} hasn't run migration ${migration.file}`)
-          continue
-        }
-      } else {
-        throw new Error(`Unknown direction: ${direction}`)
-      }
-
-      this.logger.debug(`Running migration on ${dbIdentifier}: ${migration.file}`, {migrationDatabaseIdentifiers})
-
-      applied = true
       const db = dbs[dbIdentifier]
-      const MigrationClass = migrationClass
-      const migrationInstance = new MigrationClass({
+      const dateString = `${digg(migration, "date")}`
+
+      // Serialise the check-then-apply across processes. Without this, two shards can both
+      // observe the migration as unapplied and both execute its DDL; the loser dies with the
+      // driver's duplicate-object error mid-boot.
+      //
+      // The version is re-read *inside* the lock: while this process waited, another process
+      // may already have applied the migration, in which case this one must become a no-op.
+      console.error("[MIG-DEBUG] acquiring lock db=" + dbIdentifier + " file=" + migration.file)
+      const runner = new AdvisoryLockRunner({
         configuration: this.configuration,
-        databaseIdentifier: dbIdentifier,
-        db
+        connectionProvider: () => db,
+        databaseIdentifier: dbIdentifier
       })
-      const dateString = `${digg(migration, "date")}`
 
-      if (direction == "up") {
-        try {
-          await migrationInstance.change()
-        } catch (changeError) {
-          if (changeError instanceof NotImplementedError) {
+      const appliedInLock = await runner.withAdvisoryLock(
+        migrationsAdvisoryLockName(dbIdentifier),
+        async () => {
+          // Re-read the applied set from the database while holding the lock. The version
+          // list was loaded during prepare(), before any other process may have finished
+          // applying migrations, so trusting that cached snapshot here would let this
+          // process re-run DDL that another process has already committed.
+          await this.loadMigrationsVersionsForDatabase({dbIdentifier, db})
+
+          if (direction == "up") {
+            if (this.hasRunMigrationVersion(dbIdentifier, migration.date)) {
+              this.logger.debug(`${dbIdentifier} has already run migration ${migration.file}`)
+              return false
+            }
+          } else if (direction == "down") {
+            if (!this.hasRunMigrationVersion(dbIdentifier, migration.date)) {
+              this.logger.debug(`${dbIdentifier} hasn't run migration ${migration.file}`)
+              return false
+            }
+          } else {
+            throw new Error(`Unknown direction: ${direction}`)
+          }
+
+          this.logger.debug(`Running migration on ${dbIdentifier}: ${migration.file}`, {migrationDatabaseIdentifiers})
+
+          const MigrationClass = migrationClass
+          const migrationInstance = new MigrationClass({
+            configuration: this.configuration,
+            databaseIdentifier: dbIdentifier,
+            db
+          })
+
+          if (direction == "up") {
             try {
-              await migrationInstance.up()
-            } catch (upError) {
-              if (upError instanceof NotImplementedError) {
-                throw new Error(`'change' or 'up' didn't exist on migration: ${migration.file}`, {cause: upError})
+              await migrationInstance.change()
+            } catch (changeError) {
+              if (changeError instanceof NotImplementedError) {
+                try {
+                  await migrationInstance.up()
+                } catch (upError) {
+                  if (upError instanceof NotImplementedError) {
+                    throw new Error(`'change' or 'up' didn't exist on migration: ${migration.file}`, {cause: upError})
+                  } else {
+                    throw upError
+                  }
+                }
               } else {
-                throw upError
+                throw changeError
               }
             }
-          } else {
-            throw changeError
-          }
-        }
 
-        await MigrationsLedger.recordVersion(db, dateString)
-      } else if (direction == "down") {
-        try {
-          await migrationInstance.down()
-        } catch (downError) {
-          if (downError instanceof NotImplementedError) {
-            throw new Error(`'down' didn't exist on migration: ${migration.file} or migrating down with a change method isn't currently supported`, {cause: downError})
+            await MigrationsLedger.recordVersion(db, dateString)
           } else {
-            throw downError
+            try {
+              await migrationInstance.down()
+            } catch (downError) {
+              if (downError instanceof NotImplementedError) {
+                throw new Error(`'down' didn't exist on migration: ${migration.file} or migrating down with a change method isn't currently supported`, {cause: downError})
+              } else {
+                throw downError
+              }
+            }
+
+            await MigrationsLedger.removeVersion(db, dateString)
           }
-        }
 
-        await MigrationsLedger.removeVersion(db, dateString)
-      } else {
-        throw new Error(`Unknown direction: ${direction}`)
-      }
+          return true
+        },
+        // Pin the lock to one physical session: MSSQL/explicit advisory locks are
+        // session-scoped, and a pooled connection can route acquire and release to
+        // different sessions, which would let two processes both "hold" the lock.
+        {dedicatedConnection: true}
+      )
+
+      if (!appliedInLock) continue
+
+      applied = true
     }
 
     return applied

package/build/database/record/attachments/store.js

@@ -15,6 +15,10 @@ const ATTACHMENT_OWNER_INDEX_NAME = "index_velocious_attachments_on_record_type_
 const ATTACHMENT_RECORD_ID_DIGEST_LENGTH = 64
 const ATTACHMENT_RECORD_ID_DIGEST_MIGRATION_BATCH_SIZE = 100
 const ATTACHMENT_SCHEMA_LOCK_NAME = "velocious-attachments-schema"
+/** SQL Server deadlock victim — raised when concurrent sessions deadlock on the same catalog reads. */
+const MSSQL_DEADLOCK_VICTIM_ERROR_NUMBER = 1205
+/** Retry budget for transient catalog deadlocks while bootstrapping attachment schema. */
+const ATTACHMENT_SCHEMA_MAX_ATTEMPTS = 3
 
 /**
  * Stores by configuration.
@@ -158,6 +162,52 @@ export function recordAttachmentsStoreForModel(model) {
 /**
  * Attachment persistence store.
  */
+/**
+ * Returns whether an error means a concurrent process created the same schema object.
+ *
+ * Drivers report this differently: SQLite raises "index ... already exists", MSSQL raises
+ * "There is already an object named ..." (2714), PostgreSQL raises DuplicateTable (42P07) or
+ * DuplicateObject (42710). Matching on the message keeps the check driver-agnostic without
+ * hiding unrelated failures.
+ * @param {unknown} error - Thrown error.
+ * @returns {boolean} - Whether this is a concurrent-creation collision.
+ */
+function isConcurrentSchemaCreationError(error) {
+  if (!error) return false
+
+  const code = typeof error === "object" && "number" in error ? Number(error.number) : NaN
+
+  // MSSQL 2714 (duplicate object), 1913 (duplicate index); PostgreSQL 42P07 / 42710.
+  if (code === 2714 || code === 1913 || code === 42) return true
+
+  const message = error instanceof Error ? error.message : String(error)
+
+  return /already exists/i.test(message) || /already an object named/i.test(message) || /duplicate (table|object|key)/i.test(message)
+}
+
+/**
+ * Whether an error is a retryable database deadlock victim.
+ *
+ * Matched by numeric error number, never by message text: SQL Server raises 1205 when a
+ * concurrent session wins the lock and this one is chosen as the victim. The victim's
+ * transaction has already rolled back, so the operation is safe to retry.
+ * @param {unknown} error - Thrown error.
+ * @returns {boolean} - Whether this is a retryable deadlock victim.
+ */
+function isDeadlockVictimError(error) {
+  let current = error
+
+  while (current instanceof Error) {
+    const errorNumber = /** @type {{number?: unknown}} */ (current).number
+
+    if (typeof errorNumber === "number" && errorNumber === MSSQL_DEADLOCK_VICTIM_ERROR_NUMBER) return true
+
+    current = current.cause
+  }
+
+  return false
+}
+
 export default class RecordAttachmentsStore {
   /**
    * Runs constructor.
@@ -237,7 +287,40 @@ export default class RecordAttachmentsStore {
     table.bigint("updated_at_ms", {null: false})
     table.addIndex(new TableIndex(["record_type", "record_id_digest"], {name: ATTACHMENT_OWNER_INDEX_NAME}))
 
-    await db.createTable(table)
+    let attempt = 0
+
+    while (true) {
+      attempt++
+
+      try {
+        await db.createTable(table)
+
+        break
+      } catch (error) {
+        // A concurrent process may win the create between our tableExists() check and this
+        // call. createTable guards the table with ifNotExists but NOT its indexes, so the
+        // loser fails with a duplicate-object error. Treat that as "someone else bootstrapped
+        // it": reconcile through the upgrade path instead of failing the caller.
+        if (isConcurrentSchemaCreationError(error)) {
+          await this.ensureAttachmentStoreSchema({db})
+
+          return
+        }
+
+        // Concurrent boots also deadlock on the INFORMATION_SCHEMA metadata scan that
+        // getTableByName performs on SQL Server. The victim transaction rolled back cleanly,
+        // so retrying it is safe. Without this the loser takes down its caller even though
+        // another process is building the very table it needs.
+        if (isDeadlockVictimError(error) && attempt < ATTACHMENT_SCHEMA_MAX_ATTEMPTS) {
+          db.clearSchemaCache()
+
+          continue
+        }
+
+        throw error
+      }
+    }
+
     this._driverColumnsAvailable = true
     this._contentBase64Nullable = true
     this._schemaReadyGeneration = this._schemaCacheGeneration(db)

package/build/environment-handlers/base.js

@@ -36,6 +36,19 @@ export default class VelociousEnvironmentHandlerBase {
    */
   resolveTestSharedTransactionPoolType({configuredPoolType}) { return configuredPoolType }
 
+  /**
+   * Creates the file-tier advisory-lock backend (STC-213). Only filesystem
+   * capable environments (the Node handler) can provide this tier; browser
+   * and Expo configurations have no `fs`, so the default refuses loudly
+   * instead of half-working when a deployment explicitly enables
+   * `advisoryLocks.backends.file.enabled`.
+   * @param {{directory: string, leaseMs: number, renewIntervalMs: number}} _args - Backend options.
+   * @returns {Promise<import("../database/advisory-lock-file-backend.js").default>} - Resolves with the created backend in filesystem-capable handlers.
+   */
+  async createAdvisoryLockFileBackend(_args) {
+    throw new Error("The advisory-lock file tier requires a filesystem-capable environment handler (Node)")
+  }
+
   /**
    * Node test runtimes may replace a physical child connection with a broker
    * proxy. Other environments never participate in this test-only protocol.

package/build/environment-handlers/node.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 import "../database/annotations-async-hooks.js"
+import AdvisoryLockFileBackend from "../database/advisory-lock-file-backend.js"
 import Base from "./base.js"
 import BackgroundJobsClient from "../background-jobs/client.js"
 import SqlBackgroundJobsAdapter from "../background-jobs/sql-adapter.js"
@@ -106,6 +107,16 @@ export default class VelociousEnvironmentHandlerNode extends Base{
     return AsyncTrackedMultiConnectionPool
   }
 
+  /**
+   * Creates the filesystem-backed advisory-lock backend for the file tier
+   * (STC-213). The Node handler is the only filesystem-capable environment.
+   * @param {{directory: string, leaseMs: number, renewIntervalMs: number}} args - Backend options.
+   * @returns {Promise<import("../database/advisory-lock-file-backend.js").default>} - Resolves with the created backend.
+   */
+  async createAdvisoryLockFileBackend(args) {
+    return new AdvisoryLockFileBackend(args)
+  }
+
   /**
    * Creates a test-only child proxy when TestRunner supplied an active broker.
    * @param {{DriverClass: typeof import("../database/drivers/base.js").default, config: import("../configuration-types.js").DatabaseConfigurationType, configuration: import("../configuration.js").default, databaseIdentifier: string, reuseKey?: string}} args - Connection details.