velocious 1.0.721 → 1.0.722

package/src/sync/sync-realtime-bridge.js

@@ -5,10 +5,46 @@ import {mergeRemoteRequestContext} from "../remote-request-context.js"
 
 import SyncApiClient from "./sync-api-client.js"
 import {VELOCIOUS_SYNC_CHANNEL} from "./sync-channel-name.js"
+import {cursorFromSyncEntry, isCursorAfter} from "./sync-cursor.js"
 
 /** @typedef {import("../configuration-types.js").VelociousSyncRealtimeChannelDescriptor} VelociousSyncRealtimeChannelDescriptor */
 /** @typedef {import("../configuration-types.js").VelociousSyncRealtimeWebsocketClient} VelociousSyncRealtimeWebsocketClient */
 
+/**
+ * The persisted scope row the bridge carries so it can advance the scope's cursor
+ * on applied pushes. Mirrors the framework-owned scope store row shape.
+ * @typedef {object} SyncRealtimeScopeRow
+ * @property {Record<string, ReturnType<typeof JSON.parse>>} conditions - Scope attribute conditions.
+ * @property {string | null} cursorPayload - Persisted cursor JSON payload.
+ * @property {string} id - Scope row id.
+ * @property {string | null} resourceType - Scope resource type.
+ * @property {string} scopeDigest - Fixed-size scope digest.
+ * @property {string} state - Scope state.
+ * @property {string} storeIdentity - Physical store identity.
+ */
+
+/**
+ * One derived channel subscription carrying its owning scope row (null for legacy
+ * app channels) and the in-memory cursor watermark advanced by applied pushes.
+ * @typedef {object} DerivedSyncChannel
+ * @property {string} channel - Channel name.
+ * @property {import("./sync-api-client-types.js").SyncCursor} cursor - In-memory cursor watermark.
+ * @property {string | null} resourceType - Default resource type for pushed changes.
+ * @property {SyncRealtimeScopeRow | null} scopeRow - Owning scope row, or null for legacy channels.
+ * @property {import("../configuration-types.js").VelociousSyncRealtimeSubscription} subscription - Subscription handle.
+ */
+
+/**
+ * One derived channel to subscribe, carrying its owning scope row and cursor for
+ * framework sync channel subscriptions (null scope row for legacy app channels).
+ * @typedef {object} DerivedSyncChannelDescriptor
+ * @property {string} channel - Channel name.
+ * @property {import("./sync-api-client-types.js").SyncCursor} cursor - In-memory cursor watermark (null for legacy channels).
+ * @property {Record<string, ReturnType<typeof JSON.parse>>} params - Subscribe params (without the framework-injected auth token).
+ * @property {string | null} resourceType - Default resource type for pushed changes.
+ * @property {SyncRealtimeScopeRow | null} scopeRow - Owning scope row, or null for legacy channels.
+ */
+
 /**
  * Derived realtime push bridge for the sync client. Subscribes every declared
  * pull scope to the framework sync channel ({@link VELOCIOUS_SYNC_CHANNEL})
@@ -29,7 +65,7 @@ export default class SyncRealtimeBridge {
    */
   constructor({syncClient}) {
     this.syncClient = syncClient
-    /** @type {Array<{channel: string, resourceType: string | null, subscription: import("../configuration-types.js").VelociousSyncRealtimeSubscription}>} */
+    /** @type {Array<DerivedSyncChannel>} */
     this._channels = []
     /** @type {VelociousSyncRealtimeWebsocketClient | null} */
     this._client = null
@@ -40,8 +76,6 @@ export default class SyncRealtimeBridge {
     /** @type {number} Subscription generation - bumped by unsubscribe so in-flight subscribes detect they became stale. */
     this._generation = 0
     /** @type {Promise<void> | null} */
-    this._scheduledPull = null
-    /** @type {Promise<void> | null} */
     this._subscribePromise = null
     /** @type {"subscribed" | "subscribing" | "unsubscribed"} */
     this._state = "unsubscribed"
@@ -83,7 +117,7 @@ export default class SyncRealtimeBridge {
 
     this._state = "subscribing"
 
-    /** @type {Array<{channel: string, resourceType: string | null, subscription: import("../configuration-types.js").VelociousSyncRealtimeSubscription}>} */
+    /** @type {Array<DerivedSyncChannel>} */
     const channels = []
     /** @type {VelociousSyncRealtimeWebsocketClient | null} */
     let client = null
@@ -154,13 +188,19 @@ export default class SyncRealtimeBridge {
           label: "Sync client request context",
           params: {...channelDescriptor.params, authenticationToken}
         })
-        const subscription = client.subscribeChannel(channelDescriptor.channel, {
-          onMessage: (body) => this.enqueueApply({body, generation, resourceType}),
-          onResume: () => this.schedulePull(generation),
-          params
-        })
+        /** @type {DerivedSyncChannel} */
+        const channel = {
+          channel: channelDescriptor.channel,
+          cursor: channelDescriptor.cursor,
+          resourceType,
+          scopeRow: channelDescriptor.scopeRow,
+          subscription: client.subscribeChannel(channelDescriptor.channel, {
+            onMessage: (body) => this.enqueueApply({body, channel, generation}),
+            params
+          })
+        }
 
-        channels.push({channel: channelDescriptor.channel, resourceType, subscription})
+        channels.push(channel)
       }
 
       await Promise.all(channels.map(({subscription}) => subscription.waitForReady({signal})))
@@ -175,7 +215,6 @@ export default class SyncRealtimeBridge {
       this._client = client
       this._ownsClient = ownsClient
       this._state = "subscribed"
-      this.schedulePull(generation)
     } catch (error) {
       await teardown()
 
@@ -225,12 +264,11 @@ export default class SyncRealtimeBridge {
   }
 
   /**
-   * Awaits all enqueued message applies and any scheduled pull (tests, shutdown flows).
+   * Awaits all enqueued message applies (tests, shutdown flows).
    * @returns {Promise<void>}
    */
   async waitForApplied() {
     await this._applyPromise
-    if (this._scheduledPull) await this._scheduledPull
   }
 
   /**
@@ -266,33 +304,44 @@ export default class SyncRealtimeBridge {
    * model-level static realtime declarations and the config channels callback.
    * Fails loudly when nothing is subscribable.
    * @param {ReturnType<typeof JSON.parse>} context - App context passed to the deprecated channels callback.
-   * @returns {Promise<Array<VelociousSyncRealtimeChannelDescriptor>>} Channel descriptors.
+   * @returns {Promise<Array<DerivedSyncChannelDescriptor>>} Channel descriptors.
    */
   async channelDescriptors(context) {
     const realtime = this.realtimeConfiguration()
-    /** @type {Array<VelociousSyncRealtimeChannelDescriptor>} */
+    /** @type {Array<DerivedSyncChannelDescriptor>} */
     const channelDescriptors = []
 
     for (const scopeRow of await this.syncClient.scopeStore().activeScopes()) {
+      const cursorPayload = await this.syncClient.scopeStore().loadCursor(scopeRow)
+      const cursor = SyncApiClient.syncCursorFromPayload(cursorPayload)
+
       channelDescriptors.push({
         channel: VELOCIOUS_SYNC_CHANNEL,
+        cursor,
         params: {
           conditions: this.attributeNamedConditions(scopeRow),
           resourceType: scopeRow.resourceType,
           // Only the all-types scope carries the type list; a type-declared scope needs none.
-          ...(scopeRow.resourceType === null ? {resourceTypes: this.syncClient.userScopeResourceTypes()} : {})
-        }
+          ...(scopeRow.resourceType === null ? {resourceTypes: this.syncClient.userScopeResourceTypes()} : {}),
+          // Reuse the changes-endpoint cursor fields so the server replays the
+          // catch-up backlog from the scope's persisted cursor on subscribe.
+          ...SyncApiClient.cursorPayload("after", cursor)
+        },
+        resourceType: scopeRow.resourceType,
+        scopeRow
       })
     }
 
     for (const [resourceType, resourceConfig] of Object.entries(this.syncClient.config.resources)) {
       if (!resourceConfig.realtime) continue
 
-      channelDescriptors.push({channel: resourceConfig.realtime.channel, params: resourceConfig.realtime.params, resourceType})
+      channelDescriptors.push({channel: resourceConfig.realtime.channel, cursor: null, params: resourceConfig.realtime.params ?? {}, resourceType, scopeRow: null})
     }
 
     if (realtime?.channels) {
-      channelDescriptors.push(...await realtime.channels(context))
+      const legacyChannels = await realtime.channels(context)
+
+      channelDescriptors.push(...legacyChannels.map((descriptor) => ({channel: descriptor.channel, cursor: null, params: descriptor.params ?? {}, resourceType: descriptor.resourceType ?? null, scopeRow: null})))
     }
 
     if (channelDescriptors.length === 0) {
@@ -341,15 +390,15 @@ export default class SyncRealtimeBridge {
   /**
    * Chains one pushed message onto the serialized apply queue so changes apply
    * in arrival order; failures go to the sync client's error reporting.
-   * @param {{body: ReturnType<typeof JSON.parse>, generation?: number, resourceType: string | null}} args - Message args.
+   * @param {{body: ReturnType<typeof JSON.parse>, channel: DerivedSyncChannel, generation?: number}} args - Message args.
    * @returns {void}
    */
-  enqueueApply({body, generation = this._generation, resourceType}) {
+  enqueueApply({body, channel, generation = this._generation}) {
     this._applyPromise = this._applyPromise.then(async () => {
       if (generation !== this._generation) return
 
       try {
-        await this.applyMessage({body, resourceType})
+        await this.applyMessage({body, channel})
       } catch (error) {
         this.syncClient.reportError(/** @type {Error} */ (error))
       }
@@ -359,11 +408,12 @@ export default class SyncRealtimeBridge {
   /**
    * Applies one pushed message through the derived resource applier: drops
    * own-device messages by echo origin, defaults the channel's resourceType onto
-   * envelopes without one, and fails loudly on unknown resource types.
-   * @param {{body: ReturnType<typeof JSON.parse>, resourceType: string | null}} args - Message args.
+   * envelopes without one, fails loudly on unknown resource types, and advances
+   * the owning scope's cursor so a later replay resumes after the applied rows.
+   * @param {{body: ReturnType<typeof JSON.parse>, channel: DerivedSyncChannel}} args - Message args.
    * @returns {Promise<void>}
    */
-  async applyMessage({body, resourceType}) {
+  async applyMessage({body, channel}) {
     if (!body || typeof body !== "object" || Array.isArray(body)) {
       throw new Error(`Realtime sync messages must be envelope objects, got: ${JSON.stringify(body)}`)
     }
@@ -382,44 +432,40 @@ export default class SyncRealtimeBridge {
     // Coalesce record-change events across the pushed batch so it triggers one live-query re-run.
     await recordChanges.batch(async () => {
       for (const syncPayload of syncPayloads) {
-        const sync = SyncApiClient.syncEnvelopeFromPayload({resourceType, ...syncPayload})
+        const sync = SyncApiClient.syncEnvelopeFromPayload({resourceType: channel.resourceType, ...syncPayload})
 
         await applySync(sync)
       }
     })
+
+    await this.advanceScopeCursor(channel, syncPayloads)
   }
 
   /**
-   * Schedules a coalesced background pull closing offline gaps after
-   * (re)subscription readiness. Resumes arriving while a pull is already
-   * scheduled or in flight coalesce into that pull instead of stacking.
-   * @param {number} [generation] - Subscription generation owning the resume/readiness callback.
-   * @returns {void}
+   * Advances the owning scope's persisted cursor to the newest applied row,
+   * monotonically (never backwards) so an out-of-order or duplicate push cannot
+   * regress the cursor and re-deliver already-applied rows on a later replay.
+   * Legacy channels with no owning scope are skipped.
+   * @param {DerivedSyncChannel} channel - Channel owning the scope.
+   * @param {Array<ReturnType<typeof JSON.parse>>} syncPayloads - Applied sync entries.
+   * @returns {Promise<void>}
    */
-  schedulePull(generation = this._generation) {
-    if (generation !== this._generation) return
-    if (this.realtimeConfiguration()?.pullOnReconnect === false) return
+  async advanceScopeCursor(channel, syncPayloads) {
+    if (!channel.scopeRow) return
 
-    const coordinatorRun = this.syncClient.requestCoordinatorSync("realtime")
+    let latestCursor = channel.cursor
 
-    if (coordinatorRun) {
-      this._scheduledPull ||= coordinatorRun.finally(() => {
-        this._scheduledPull = null
-      })
-      return
+    for (const syncPayload of syncPayloads) {
+      const entry = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (syncPayload)
+      const cursor = cursorFromSyncEntry(entry)
+
+      if (cursor && isCursorAfter(cursor, latestCursor)) latestCursor = cursor
     }
 
-    this._scheduledPull ||= (async () => {
-      try {
-        await this.syncClient.pull()
-      } catch (error) {
-        if (!this.syncClient.isLifecycleAbort(error)) {
-          this.syncClient.reportError(/** @type {Error} */ (error))
-        }
-      } finally {
-        this._scheduledPull = null
-      }
-    })()
+    if (latestCursor && latestCursor !== channel.cursor) {
+      channel.cursor = latestCursor
+      await this.syncClient.scopeStore().saveCursor(channel.scopeRow, latestCursor)
+    }
   }
 }
 

package/src/sync/sync-websocket-channel.js

@@ -2,10 +2,53 @@
 
 import VelociousWebsocketChannel from "../http-server/websocket-channel.js"
 import {VELOCIOUS_SYNC_CHANNEL} from "./sync-channel-name.js"
+import {cursorFromSyncEntry, isCursorAfter} from "./sync-cursor.js"
+import SyncModelChangeFeedService from "./sync-model-change-feed-service.js"
 
 /** Configurations whose framework sync channel has already been registered. */
 const registeredConfigurations = new WeakSet()
 
+/** Default catch-up replay page size when the subscriber sends no `limit` param. */
+const DEFAULT_REPLAY_LIMIT = 1000
+
+/**
+ * Serializes a cursor into the change-feed request params under the given prefix.
+ * @param {"after" | "upTo"} prefix - Request field prefix.
+ * @param {import("./sync-api-client-types.js").SyncCursor} cursor - Cursor to serialize.
+ * @returns {Record<string, string | number | null>} Serialized cursor params.
+ */
+function cursorParams(prefix, cursor) {
+  if (!cursor) return {}
+
+  return {
+    [`${prefix}Id`]: cursor.id,
+    ...(cursor.serverSequence !== null ? {[`${prefix}ServerSequence`]: cursor.serverSequence} : {}),
+    [`${prefix}UpdatedAt`]: cursor.updatedAt
+  }
+}
+
+/**
+ * Reads the after-cursor the subscriber declared in its subscribe params, or null
+ * when it sent none (a fresh subscription replays the whole scope).
+ * @param {Record<string, ReturnType<typeof JSON.parse>>} params - Subscribe params.
+ * @returns {import("./sync-api-client-types.js").SyncCursor} Declared after-cursor, or null.
+ */
+function cursorFromParams(params) {
+  const updatedAt = typeof params.afterUpdatedAt === "string" ? params.afterUpdatedAt : null
+
+  if (!updatedAt) return null
+
+  const serverSequence = params.afterServerSequence === null || params.afterServerSequence === undefined
+    ? null
+    : Number(params.afterServerSequence)
+
+  return {
+    id: params.afterId === null || params.afterId === undefined ? null : String(params.afterId),
+    serverSequence,
+    updatedAt
+  }
+}
+
 /**
  * Framework-owned websocket channel for synced resources
  * ({@link VELOCIOUS_SYNC_CHANNEL}).
@@ -25,6 +68,14 @@ export default class SyncWebsocketChannel extends VelociousWebsocketChannel {
    */
   _scope = null
 
+  /**
+   * Cursor of the last sync entry delivered to this subscription, advanced by
+   * live broadcasts and catch-up replay so a resume replays only the rows missed
+   * while the socket was down.
+   * @type {import("./sync-api-client-types.js").SyncCursor}
+   */
+  _deliveredCursor = null
+
   /**
    * Registers the framework sync channel on a configuration declaring a
    * `sync.api` block (guarded so repeated server boots with the same
@@ -169,13 +220,17 @@ export default class SyncWebsocketChannel extends VelociousWebsocketChannel {
   async deliverBroadcast(body, meta) {
     if (!this._isUserScope()) {
       this.sendMessage(body, meta)
+      this._trackDeliveredCursor(body)
 
       return
     }
 
     const deliverableBody = await this._userScopeDeliverableBody(body)
 
-    if (deliverableBody !== null) this.sendMessage(deliverableBody, meta)
+    if (deliverableBody !== null) {
+      this.sendMessage(deliverableBody, meta)
+      this._trackDeliveredCursor(deliverableBody)
+    }
   }
 
   /**
@@ -245,6 +300,103 @@ export default class SyncWebsocketChannel extends VelociousWebsocketChannel {
     return deliverableSyncs[0]
   }
 
+  /**
+   * Replays the catch-up backlog to this subscriber once its subscription is
+   * authorized: pages the scoped change feed from the cursor the client declared
+   * in its subscribe params and sends each page as a standard sync envelope, then
+   * live broadcasts continue from there. A fresh subscription with no cursor
+   * replays the whole authorized scope.
+   * @returns {Promise<void>}
+   */
+  async subscribed() {
+    if (!this._scope) return
+
+    await this._replayFrom(cursorFromParams(this.params))
+  }
+
+  /**
+   * Replays rows missed while the socket was down, resuming from the cursor of
+   * the last entry this subscription delivered instead of re-reading the whole
+   * scope from the subscribe-time cursor.
+   * @returns {Promise<void>}
+   */
+  async onResume() {
+    if (!this._scope) return
+
+    await this._replayFrom(this._deliveredCursor ?? cursorFromParams(this.params))
+  }
+
+  /**
+   * Pages the scoped change feed from a cursor and delivers each page to this
+   * subscriber as a `{echoOrigin, syncs}` envelope. The snapshot high-water is
+   * resolved once and pinned across pages so rows arriving mid-replay are left to
+   * the live broadcast path instead of looping the replay. The delivered cursor
+   * advances so a later resume resumes from here.
+   * @param {import("./sync-api-client-types.js").SyncCursor} afterCursor - Cursor to replay after.
+   * @returns {Promise<void>}
+   */
+  async _replayFrom(afterCursor) {
+    const configuration = this.session.configuration
+    const scope = this._scope
+
+    await configuration.ensureConnections({name: `${VELOCIOUS_SYNC_CHANNEL} catch-up replay`}, async () => {
+      const resource = await this.buildSyncResource()
+      const limit = this._replayLimit(resource)
+      /** @type {import("./sync-api-client-types.js").SyncCursor} */
+      let upToCursor = null
+
+      while (true) {
+        const page = await resource.changeFeedService({
+          params: {...this.params, ...cursorParams("after", afterCursor), ...(upToCursor ? cursorParams("upTo", upToCursor) : {})},
+          scope
+        }).changes()
+
+        if (!upToCursor) upToCursor = page.upToCursor
+
+        if (page.syncs.length === 0) break
+
+        await this.deliverBroadcast({echoOrigin: null, syncs: page.syncs})
+
+        afterCursor = page.nextCursor
+        if (afterCursor) this._deliveredCursor = afterCursor
+
+        if (page.syncs.length < limit) break
+      }
+    })
+  }
+
+  /**
+   * Resolves the catch-up replay page size from the subscriber's `limit` param
+   * (defaulting to {@link DEFAULT_REPLAY_LIMIT}).
+   * @param {import("./sync-resource-base.js").default} resource - Authorizing sync resource.
+   * @returns {number} Replay page size.
+   */
+  _replayLimit(resource) {
+    return new SyncModelChangeFeedService({modelClass: resource.syncModelClass(), params: this.params}).normalizedLimit(this.params.limit)
+  }
+
+  /**
+   * Advances the delivered-cursor watermark from a delivered envelope so a resume
+   * resumes after the newest row actually sent to this subscriber. Broadcasts may
+   * arrive out of order across resources, so each entry only advances the
+   * watermark when it is later than the current one.
+   * @param {import("../http-server/websocket-channel.js").WebsocketJsonValue} body - Delivered envelope.
+   * @returns {void}
+   */
+  _trackDeliveredCursor(body) {
+    if (!body || typeof body !== "object" || Array.isArray(body)) return
+
+    const envelope = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (body)
+    const syncs = Array.isArray(envelope.syncs) ? envelope.syncs : [envelope]
+
+    for (const entry of syncs) {
+      const syncEntry = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (entry)
+      const cursor = cursorFromSyncEntry(syncEntry)
+
+      if (cursor && isCursorAfter(cursor, this._deliveredCursor)) this._deliveredCursor = cursor
+    }
+  }
+
   /**
    * Returns the authorized scope for debug snapshots.
    * @returns {Record<string, ReturnType<typeof JSON.parse>>} Debug-safe subscription details.