velocious 1.0.664 → 1.0.666

package/src/background-jobs/client.js

@@ -1,7 +1,8 @@
 // @ts-check
 
-import timeout from "awaitery/build/timeout.js"
+import timeout, {TimeoutError} from "awaitery/build/timeout.js"
 import configurationResolver from "../configuration-resolver.js"
+import BackgroundJobEnqueueAcknowledgementTimeoutError from "./enqueue-acknowledgement-timeout-error.js"
 import BackgroundJobsSocketRequest from "./socket-request.js"
 import { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
 
@@ -57,17 +58,36 @@ export default class BackgroundJobsClient {
       ...(producerInvocationId ? {producerInvocationId} : {}),
       ...(producerProof ? {producerProof} : {})
     }
-    const acknowledgement = {explicitlyRejected: false, generationFenced: false, requestSent: false}
+    /**
+     * Creates safe observations for one attempt without retaining request data.
+     * @param {object} args - Attempt identity.
+     * @param {"initial" | "owned_replay"} args.attemptKind - Initial attempt or owned replay.
+     * @param {number} args.attemptNumber - One-based attempt number.
+     * @returns {import("./enqueue-acknowledgement-timeout-error.js").BackgroundJobEnqueueAttempt} - Mutable attempt observations.
+     */
+    const newAttemptObservation = ({attemptKind, attemptNumber}) => ({
+      acknowledgementWaitElapsedMs: 0,
+      attemptElapsedMs: 0,
+      attemptKind,
+      attemptNumber,
+      explicitlyRejected: false,
+      generationFenced: false,
+      requestSent: false
+    })
     /**
      * Sends one enqueue attempt. An owned caller may replay this exact message
      * once when transport acknowledgement remains ambiguous after send.
-     * @param {{explicitlyRejected: boolean, generationFenced: boolean, requestSent: boolean} | undefined} attemptAcknowledgement - First-attempt observations.
+     * @param {import("./enqueue-acknowledgement-timeout-error.js").BackgroundJobEnqueueAttempt} attemptObservation - Mutable attempt observations.
+     * @param {Readonly<Array<import("./enqueue-acknowledgement-timeout-error.js").BackgroundJobEnqueueAttempt>>} previousAttempts - Earlier timed-out attempts.
      * @returns {Promise<string>} - Job id.
      */
-    const enqueueAttempt = async (attemptAcknowledgement) => {
+    const enqueueAttempt = async (attemptObservation, previousAttempts) => {
+      const attemptStartedAtMs = Date.now()
       const request = await this._request()
       const requestAbortController = new AbortController()
       const timeoutErrorMessage = `Background job enqueue acknowledgement timed out after ${this.enqueueTimeoutMs}ms`
+      /** @type {number | undefined} */
+      let requestSentAtMs
       /**
        * Resolves the pre-send phase when the mutation has entered the socket.
        * @type {() => void}
@@ -99,10 +119,9 @@ export default class BackgroundJobsClient {
         signal: requestAbortController.signal,
         onConnect: (jsonSocket) => {
           jsonSocket.send(message)
-          if (attemptAcknowledgement) {
-            attemptAcknowledgement.generationFenced = Boolean(request.generationId)
-            attemptAcknowledgement.requestSent = true
-          }
+          attemptObservation.generationFenced = Boolean(request.generationId)
+          attemptObservation.requestSent = true
+          requestSentAtMs = Date.now()
           markRequestSent()
         },
         onMessage: ({message, resolve, reject}) => {
@@ -112,7 +131,7 @@ export default class BackgroundJobsClient {
           }
 
           if (message?.type === "enqueue-error") {
-            if (attemptAcknowledgement) attemptAcknowledgement.explicitlyRejected = true
+            attemptObservation.explicitlyRejected = true
             reject(new Error(message.error || "Failed to enqueue job"))
           }
         }
@@ -120,16 +139,43 @@ export default class BackgroundJobsClient {
 
       await withEnqueueTimeout(async () => await Promise.race([requestSent, requestPromise]))
 
-      return await withEnqueueTimeout(async () => await requestPromise)
+      try {
+        return await withEnqueueTimeout(async () => await requestPromise)
+      } catch (error) {
+        if (!(error instanceof TimeoutError)) throw error
+        if (requestSentAtMs === undefined) throw new Error("Background job enqueue acknowledgement wait started before the request was sent", {cause: error})
+
+        const timedOutAtMs = Date.now()
+        // The fired timer proves its logical deadline even when the adjustable wall clock reports a shorter interval.
+        const acknowledgementWaitElapsedMs = Math.max(this.enqueueTimeoutMs, timedOutAtMs - requestSentAtMs)
+
+        attemptObservation.acknowledgementWaitElapsedMs = acknowledgementWaitElapsedMs
+        attemptObservation.attemptElapsedMs = Math.max(acknowledgementWaitElapsedMs, timedOutAtMs - attemptStartedAtMs)
+
+        throw new BackgroundJobEnqueueAcknowledgementTimeoutError({
+          acknowledgementTimeoutMs: this.enqueueTimeoutMs,
+          attemptHistory: [...previousAttempts, attemptObservation],
+          cause: error,
+          generationId: request.generationId,
+          jobName,
+          producerInvocationId,
+          producerProofPresent: Boolean(producerProof)
+        })
+      }
     }
 
+    const initialAttempt = newAttemptObservation({attemptKind: "initial", attemptNumber: 1})
+
     try {
-      return await enqueueAttempt(acknowledgement)
+      return await enqueueAttempt(initialAttempt, [])
     } catch (error) {
-      if (!producerInvocationId || !producerProof || !acknowledgement.generationFenced || !acknowledgement.requestSent || acknowledgement.explicitlyRejected) throw error
-    }
+      if (!producerInvocationId || !producerProof || !initialAttempt.generationFenced || !initialAttempt.requestSent || initialAttempt.explicitlyRejected) throw error
 
-    return await enqueueAttempt(undefined)
+      const previousAttempts = error instanceof BackgroundJobEnqueueAcknowledgementTimeoutError ? error.attemptHistory : []
+      const replayAttempt = newAttemptObservation({attemptKind: "owned_replay", attemptNumber: 2})
+
+      return await enqueueAttempt(replayAttempt, previousAttempts)
+    }
   }
 
   /**

package/src/background-jobs/enqueue-acknowledgement-timeout-error.js

@@ -0,0 +1,50 @@
+// @ts-check
+
+import {TimeoutError} from "awaitery/build/timeout.js"
+
+/**
+ * @typedef {object} BackgroundJobEnqueueAttempt
+ * @property {number} acknowledgementWaitElapsedMs - Time spent waiting for the acknowledgement after send.
+ * @property {number} attemptElapsedMs - Total attempt time including connection and generation fencing.
+ * @property {"initial" | "owned_replay"} attemptKind - Initial attempt or the one eligible owned replay.
+ * @property {number} attemptNumber - One-based attempt number.
+ * @property {boolean} explicitlyRejected - Whether the main explicitly rejected the enqueue.
+ * @property {boolean} generationFenced - Whether the configured generation accepted the connection before send.
+ * @property {boolean} requestSent - Whether the enqueue request entered the socket.
+ */
+
+/** Safe typed failure for an ambiguous post-send enqueue acknowledgement timeout. */
+export default class BackgroundJobEnqueueAcknowledgementTimeoutError extends TimeoutError {
+  /**
+   * Builds an enqueue acknowledgement timeout error without request payload or connection details.
+   * @param {object} args - Safe timeout context.
+   * @param {number} args.acknowledgementTimeoutMs - Configured post-send acknowledgement deadline.
+   * @param {Array<BackgroundJobEnqueueAttempt>} args.attemptHistory - Safe observations for timed-out attempts.
+   * @param {TimeoutError} [args.cause] - Original Awaitery timeout.
+   * @param {string} [args.generationId] - Accepted release generation identity.
+   * @param {string} args.jobName - Resolved job class name.
+   * @param {string} [args.producerInvocationId] - Owned enqueue invocation identity.
+   * @param {boolean} args.producerProofPresent - Whether the request carried an internal producer proof.
+   */
+  constructor({acknowledgementTimeoutMs, attemptHistory, cause, generationId, jobName, producerInvocationId, producerProofPresent}) {
+    super(`Background job enqueue acknowledgement timed out after ${acknowledgementTimeoutMs}ms`, cause ? {cause} : undefined)
+
+    this.name = "BackgroundJobEnqueueAcknowledgementTimeoutError"
+    /** @type {"BACKGROUND_JOB_ENQUEUE_ACKNOWLEDGEMENT_TIMEOUT"} */
+    this.code = "BACKGROUND_JOB_ENQUEUE_ACKNOWLEDGEMENT_TIMEOUT"
+    this.acknowledgementTimeoutMs = acknowledgementTimeoutMs
+    this.attemptHistory = Object.freeze(attemptHistory.map((attempt) => Object.freeze({
+      acknowledgementWaitElapsedMs: attempt.acknowledgementWaitElapsedMs,
+      attemptElapsedMs: attempt.attemptElapsedMs,
+      attemptKind: attempt.attemptKind,
+      attemptNumber: attempt.attemptNumber,
+      explicitlyRejected: attempt.explicitlyRejected,
+      generationFenced: attempt.generationFenced,
+      requestSent: attempt.requestSent
+    })))
+    this.generationId = generationId
+    this.jobName = jobName
+    this.producerInvocationId = producerInvocationId
+    this.producerProofPresent = producerProofPresent
+  }
+}

package/src/sync/sync-publisher.js

@@ -4,6 +4,8 @@ import Configuration from "../configuration.js"
 import Logger from "../logger.js"
 import {scalarModelPrimaryKeyValue} from "../utils/model-primary-key.js"
 import restArgsError from "../utils/rest-args-error.js"
+import sha256Hex from "../utils/sha256-hex.js"
+import stableJsonStringify from "../utils/stable-json.js"
 
 import {declaredSyncScopeAttributes} from "./sync-scope-attributes.js"
 import {deliverDeclaredBroadcasts, upsertSyncRow} from "./sync-change-fanout.js"
@@ -211,7 +213,8 @@ export default class SyncPublisher {
 
       await record.connection().afterCommit(async () => {
         try {
-          const syncRow = await this.upsertPublishedSyncRow(attributes, operationScope)
+          const scopeColumnNames = resourceConfig.scopePlan.map(({columnName}) => columnName)
+          const syncRow = await this.upsertPublishedSyncRow(attributes, operationScope, scopeColumnNames)
 
           await this.broadcaster()({
             body: {
@@ -362,22 +365,50 @@ export default class SyncPublisher {
   /**
    * Upserts the published server-origin sync row for a resource identity:
    * server-origin rows carry a null actor column (no device to echo the
-   * change back to), so repeated server changes to one resource reuse and
-   * re-sequence one feed row.
+   * change back to), so repeated server changes to one complete resource and
+   * scope identity reuse and re-sequence one feed row. A database advisory
+   * lock serializes reconciliation plus the shared upsert because unique
+   * constraints containing nullable actor/scope columns do not enforce this
+   * identity portably across supported databases. Reconciliation retains the
+   * row with the newest feed sequence (then lowest id for a deterministic tie)
+   * and removes older matching server-origin rows before applying the current
+   * mutation.
    * @param {Record<string, ReturnType<typeof JSON.parse>>} attributes - Snapshotted sync row attributes.
    * @param {ReturnType<typeof JSON.parse>} syncModel - Operation-bound or static Sync model interface.
+   * @param {string[]} scopeColumnNames - Persisted scope columns participating in the complete identity.
    * @returns {Promise<ReturnType<typeof JSON.parse>>} Upserted sync row.
    */
-  async upsertPublishedSyncRow(attributes, syncModel = this.config.syncModel) {
-    const existingSync = await syncModel
-      .where({
-        [this.config.actorForeignKeyColumn]: null,
-        resource_id: attributes.resource_id,
-        resource_type: attributes.resource_type
-      })
-      .first()
+  async upsertPublishedSyncRow(attributes, syncModel = this.config.syncModel, scopeColumnNames = []) {
+    /** @type {Record<string, ReturnType<typeof JSON.parse>>} */
+    const identity = {
+      [this.config.actorForeignKeyColumn]: null,
+      resource_id: attributes.resource_id,
+      resource_type: attributes.resource_type
+    }
+
+    for (const columnName of scopeColumnNames) {
+      if (!Object.hasOwn(attributes, columnName)) {
+        throw new Error(`Published sync row identity is missing the scope column ${columnName}`)
+      }
+
+      identity[columnName] = attributes[columnName]
+    }
+
+    return await this.config.syncModel.withAdvisoryLock(syncPublisherIdentityLockName(identity), async () => {
+      const matchingSyncs = await syncModel
+        .where(identity)
+        .toArray()
+
+      matchingSyncs.sort(comparePublishedSyncRowsByRecency)
+
+      const [existingSync, ...duplicateSyncs] = matchingSyncs
+
+      for (const duplicateSync of duplicateSyncs) {
+        await duplicateSync.destroy()
+      }
 
-    return await upsertSyncRow({attributes, existingSync, syncModel})
+      return await upsertSyncRow({attributes, existingSync, syncModel})
+    }, {dedicatedConnection: true})
   }
 
   /**
@@ -434,6 +465,40 @@ export default class SyncPublisher {
   }
 }
 
+/**
+ * Returns a deterministic, MySQL-safe advisory-lock name for one complete
+ * server-origin publisher identity. Stable JSON preserves null actor/scope
+ * components distinctly from strings, and the truncated SHA-256 digest keeps
+ * the final name below MySQL/MariaDB's 64-character `GET_LOCK` limit.
+ * @param {Record<string, ReturnType<typeof JSON.parse>>} identity - Complete sync-row identity in column form.
+ * @returns {string} Advisory-lock name.
+ */
+function syncPublisherIdentityLockName(identity) {
+  const hash = sha256Hex(stableJsonStringify(identity)).slice(0, 32)
+
+  return `vsp:${hash}`
+}
+
+/**
+ * Orders matching published rows by the feed's public recency contract so
+ * legacy duplicates have one deterministic survivor. Server sequences are
+ * positive and monotonic; a legacy null sequence is older than any assigned
+ * sequence, and the immutable row id breaks otherwise-equal ties.
+ * @param {ReturnType<typeof JSON.parse>} left - First matching sync row.
+ * @param {ReturnType<typeof JSON.parse>} right - Second matching sync row.
+ * @returns {number} Sort comparison with the canonical survivor first.
+ */
+function comparePublishedSyncRowsByRecency(left, right) {
+  const leftSequence = left.serverSequence()
+  const rightSequence = right.serverSequence()
+
+  if (leftSequence === null && rightSequence !== null) return 1
+  if (leftSequence !== null && rightSequence === null) return -1
+  if (leftSequence !== rightSequence) return rightSequence - leftSequence
+
+  return left.id().localeCompare(right.id())
+}
+
 /**
  * Resolves a model class's active publish declaration from `static sync`.
  * Opted-out (`publish: false`) and undeclared models resolve to null; every