stripe-experiment-sync 1.0.17 → 1.0.19

package/dist/index.cjs

@@ -46,7 +46,7 @@ var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
 // package.json
 var package_default = {
   name: "stripe-experiment-sync",
-  version: "1.0.17",
+  version: "1.0.19",
   private: false,
   description: "Stripe Sync Engine to sync Stripe data to Postgres",
   type: "module",
@@ -523,7 +523,8 @@ var PostgresClient = class {
       `UPDATE "${this.config.schema}"."_sync_obj_runs" o
        SET status = 'error',
            error_message = 'Auto-cancelled: stale (no update in 5 min)',
-           completed_at = now()
+           completed_at = now(),
+           page_cursor = NULL
        WHERE o."_account_id" = $1
          AND o.status = 'running'
          AND o.updated_at < now() - interval '5 minutes'`,
@@ -630,15 +631,17 @@ var PostgresClient = class {
   /**
    * Create object run entries for a sync run.
    * All objects start as 'pending'.
+   *
+   * @param resourceNames - Database resource names (e.g. 'products', 'customers', NOT 'product', 'customer')
    */
-  async createObjectRuns(accountId, runStartedAt, objects) {
-    if (objects.length === 0) return;
-    const values = objects.map((_, i) => `($1, $2, $${i + 3})`).join(", ");
+  async createObjectRuns(accountId, runStartedAt, resourceNames) {
+    if (resourceNames.length === 0) return;
+    const values = resourceNames.map((_, i) => `($1, $2, $${i + 3})`).join(", ");
     await this.query(
       `INSERT INTO "${this.config.schema}"."_sync_obj_runs" ("_account_id", run_started_at, object)
        VALUES ${values}
        ON CONFLICT ("_account_id", run_started_at, object) DO NOTHING`,
-      [accountId, runStartedAt, ...objects]
+      [accountId, runStartedAt, ...resourceNames]
     );
   }
   /**
@@ -667,7 +670,7 @@ var PostgresClient = class {
    */
   async getObjectRun(accountId, runStartedAt, object) {
     const result = await this.query(
-      `SELECT object, status, processed_count, cursor
+      `SELECT object, status, processed_count, cursor, page_cursor
        FROM "${this.config.schema}"."_sync_obj_runs"
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object]
@@ -678,7 +681,8 @@ var PostgresClient = class {
       object: row.object,
       status: row.status,
       processedCount: row.processed_count,
-      cursor: row.cursor
+      cursor: row.cursor,
+      pageCursor: row.page_cursor
     };
   }
   /**
@@ -693,6 +697,23 @@ var PostgresClient = class {
       [accountId, runStartedAt, object, count]
     );
   }
+  /**
+   * Update the pagination page_cursor used for backfills using Stripe list calls.
+   */
+  async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
+    await this.query(
+      `UPDATE "${this.config.schema}"."_sync_obj_runs"
+       SET page_cursor = $4, updated_at = now()
+       WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
+      [accountId, runStartedAt, object, pageCursor]
+    );
+  }
+  /**
+   * Clear the pagination page_cursor for an object sync.
+   */
+  async clearObjectPageCursor(accountId, runStartedAt, object) {
+    await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
+  }
   /**
    * Update the cursor for an object sync.
    * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -725,9 +746,10 @@ var PostgresClient = class {
   }
   /**
    * Get the highest cursor from previous syncs for an object type.
-   * This considers completed, error, AND running runs to ensure recovery syncs
-   * don't re-process data that was already synced before a crash.
-   * A 'running' status with a cursor means the process was killed mid-sync.
+   * Uses only completed object runs.
+   * - During the initial backfill we page through history, but we also update the cursor as we go.
+   *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+   *   too early and only ever fetch the newest page (breaking the historical backfill).
    *
    * Handles two cursor formats:
    * - Numeric: compared as bigint for correct ordering
@@ -742,11 +764,31 @@ var PostgresClient = class {
        FROM "${this.config.schema}"."_sync_obj_runs" o
        WHERE o."_account_id" = $1
          AND o.object = $2
-         AND o.cursor IS NOT NULL`,
+         AND o.cursor IS NOT NULL
+         AND o.status = 'complete'`,
       [accountId, object]
     );
     return result.rows[0]?.cursor ?? null;
   }
+  /**
+   * Get the highest cursor from previous syncs for an object type, excluding the current run.
+   */
+  async getLastCursorBeforeRun(accountId, object, runStartedAt) {
+    const result = await this.query(
+      `SELECT CASE
+         WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
+         ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
+       END as cursor
+       FROM "${this.config.schema}"."_sync_obj_runs" o
+       WHERE o."_account_id" = $1
+         AND o.object = $2
+         AND o.cursor IS NOT NULL
+         AND o.status = 'complete'
+         AND o.run_started_at < $3`,
+      [accountId, object, runStartedAt]
+    );
+    return result.rows[0]?.cursor ?? null;
+  }
   /**
    * Delete all sync runs and object runs for an account.
    * Useful for testing or resetting sync state.
@@ -767,7 +809,7 @@ var PostgresClient = class {
   async completeObjectSync(accountId, runStartedAt, object) {
     await this.query(
       `UPDATE "${this.config.schema}"."_sync_obj_runs"
-       SET status = 'complete', completed_at = now()
+       SET status = 'complete', completed_at = now(), page_cursor = NULL
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object]
     );
@@ -783,7 +825,7 @@ var PostgresClient = class {
   async failObjectSync(accountId, runStartedAt, object, errorMessage) {
     await this.query(
       `UPDATE "${this.config.schema}"."_sync_obj_runs"
-       SET status = 'error', error_message = $4, completed_at = now()
+       SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object, errorMessage]
     );
@@ -2084,57 +2126,74 @@ var StripeSync = class {
    * ```
    */
   async processNext(object, params) {
-    await this.getCurrentAccount();
-    const accountId = await this.getAccountId();
-    const resourceName = this.getResourceName(object);
-    let runStartedAt;
-    if (params?.runStartedAt) {
-      runStartedAt = params.runStartedAt;
-    } else {
-      const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
-      runStartedAt = runKey.runStartedAt;
-    }
-    await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
-    const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
-    if (objRun?.status === "complete" || objRun?.status === "error") {
-      return {
-        processed: 0,
-        hasMore: false,
-        runStartedAt
-      };
-    }
-    if (objRun?.status === "pending") {
-      const started = await this.postgresClient.tryStartObjectSync(
-        accountId,
-        runStartedAt,
-        resourceName
-      );
-      if (!started) {
+    try {
+      await this.getCurrentAccount();
+      const accountId = await this.getAccountId();
+      const resourceName = this.getResourceName(object);
+      let runStartedAt;
+      if (params?.runStartedAt) {
+        runStartedAt = params.runStartedAt;
+      } else {
+        const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
+        runStartedAt = runKey.runStartedAt;
+      }
+      await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
+      const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
+      if (objRun?.status === "complete" || objRun?.status === "error") {
         return {
           processed: 0,
-          hasMore: true,
+          hasMore: false,
           runStartedAt
         };
       }
-    }
-    let cursor = null;
-    if (!params?.created) {
-      if (objRun?.cursor) {
-        cursor = objRun.cursor;
-      } else {
-        const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
+      if (objRun?.status === "pending") {
+        const started = await this.postgresClient.tryStartObjectSync(
+          accountId,
+          runStartedAt,
+          resourceName
+        );
+        if (!started) {
+          return {
+            processed: 0,
+            hasMore: true,
+            runStartedAt
+          };
+        }
+      }
+      let cursor = null;
+      if (!params?.created) {
+        const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
+          accountId,
+          resourceName,
+          runStartedAt
+        );
         cursor = lastCursor ?? null;
       }
+      const result = await this.fetchOnePage(
+        object,
+        accountId,
+        resourceName,
+        runStartedAt,
+        cursor,
+        objRun?.pageCursor ?? null,
+        params
+      );
+      return result;
+    } catch (error) {
+      throw this.appendMigrationHint(error);
     }
-    const result = await this.fetchOnePage(
-      object,
-      accountId,
-      resourceName,
-      runStartedAt,
-      cursor,
-      params
-    );
-    return result;
+  }
+  appendMigrationHint(error) {
+    const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
+    const withHint = (message) => message.includes(hint) ? message : `${hint}
+${message}`;
+    if (error instanceof Error) {
+      const { stack } = error;
+      error.message = withHint(error.message);
+      if (stack) error.stack = stack;
+      return error;
+    }
+    return new Error(withHint(String(error)));
   }
   /**
    * Get the database resource name for a SyncObject type
@@ -2166,7 +2225,7 @@ var StripeSync = class {
    * Uses resourceRegistry for DRY list/upsert operations.
    * Uses the observable sync system for tracking progress.
    */
-  async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
+  async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
     const limit = 100;
     if (object === "payment_method" || object === "tax_id") {
       this.config.logger?.warn(`processNext for ${object} requires customer context`);
@@ -2194,7 +2253,16 @@ var StripeSync = class {
           listParams.created = created;
         }
       }
+      if (pageCursor) {
+        listParams.starting_after = pageCursor;
+      }
       const response = await config.listFn(listParams);
+      if (response.data.length === 0 && response.has_more) {
+        const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
+        this.config.logger?.warn(message);
+        await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
+        return { processed: 0, hasMore: false, runStartedAt };
+      }
       if (response.data.length > 0) {
         this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
         await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
@@ -2215,6 +2283,15 @@ var StripeSync = class {
             String(maxCreated)
           );
         }
+        const lastId = response.data[response.data.length - 1].id;
+        if (response.has_more) {
+          await this.postgresClient.updateObjectPageCursor(
+            accountId,
+            runStartedAt,
+            resourceName,
+            lastId
+          );
+        }
       }
       if (!response.has_more) {
         await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
@@ -2363,31 +2440,43 @@ var StripeSync = class {
    * This is used by workers and background processes that should cooperate.
    *
    * @param triggeredBy - What triggered this sync (for observability)
+   * @param objectFilter - Optional specific object to sync (e.g. 'payment_intent'). If 'all' or undefined, syncs all objects.
    * @returns Run key and list of objects to sync
    */
-  async joinOrCreateSyncRun(triggeredBy = "worker") {
+  async joinOrCreateSyncRun(triggeredBy = "worker", objectFilter) {
     await this.getCurrentAccount();
     const accountId = await this.getAccountId();
     const result = await this.postgresClient.getOrCreateSyncRun(accountId, triggeredBy);
+    const objects = objectFilter === "all" || objectFilter === void 0 ? this.getSupportedSyncObjects() : [objectFilter];
     if (!result) {
       const activeRun = await this.postgresClient.getActiveSyncRun(accountId);
       if (!activeRun) {
         throw new Error("Failed to get or create sync run");
       }
+      await this.postgresClient.createObjectRuns(
+        activeRun.accountId,
+        activeRun.runStartedAt,
+        objects.map((obj) => this.getResourceName(obj))
+      );
       return {
         runKey: { accountId: activeRun.accountId, runStartedAt: activeRun.runStartedAt },
-        objects: this.getSupportedSyncObjects()
+        objects
       };
     }
     const { accountId: runAccountId, runStartedAt } = result;
+    await this.postgresClient.createObjectRuns(
+      runAccountId,
+      runStartedAt,
+      objects.map((obj) => this.getResourceName(obj))
+    );
     return {
       runKey: { accountId: runAccountId, runStartedAt },
-      objects: this.getSupportedSyncObjects()
+      objects
     };
   }
   async processUntilDone(params) {
     const { object } = params ?? { object: "all" };
-    const { runKey } = await this.joinOrCreateSyncRun("processUntilDone");
+    const { runKey } = await this.joinOrCreateSyncRun("processUntilDone", object);
     return this.processUntilDoneWithRun(runKey.runStartedAt, object, params);
   }
   /**

package/dist/index.d.cts

@@ -137,8 +137,10 @@ declare class PostgresClient {
     /**
      * Create object run entries for a sync run.
      * All objects start as 'pending'.
+     *
+     * @param resourceNames - Database resource names (e.g. 'products', 'customers', NOT 'product', 'customer')
      */
-    createObjectRuns(accountId: string, runStartedAt: Date, objects: string[]): Promise<void>;
+    createObjectRuns(accountId: string, runStartedAt: Date, resourceNames: string[]): Promise<void>;
     /**
      * Try to start an object sync (respects max_concurrent).
      * Returns true if claimed, false if already running or at concurrency limit.
@@ -155,12 +157,21 @@ declare class PostgresClient {
         status: string;
         processedCount: number;
         cursor: string | null;
+        pageCursor: string | null;
     } | null>;
     /**
      * Update progress for an object sync.
      * Also touches updated_at for stale detection.
      */
     incrementObjectProgress(accountId: string, runStartedAt: Date, object: string, count: number): Promise<void>;
+    /**
+     * Update the pagination page_cursor used for backfills using Stripe list calls.
+     */
+    updateObjectPageCursor(accountId: string, runStartedAt: Date, object: string, pageCursor: string | null): Promise<void>;
+    /**
+     * Clear the pagination page_cursor for an object sync.
+     */
+    clearObjectPageCursor(accountId: string, runStartedAt: Date, object: string): Promise<void>;
     /**
      * Update the cursor for an object sync.
      * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -170,15 +181,20 @@ declare class PostgresClient {
     updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
     /**
      * Get the highest cursor from previous syncs for an object type.
-     * This considers completed, error, AND running runs to ensure recovery syncs
-     * don't re-process data that was already synced before a crash.
-     * A 'running' status with a cursor means the process was killed mid-sync.
+     * Uses only completed object runs.
+     * - During the initial backfill we page through history, but we also update the cursor as we go.
+     *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+     *   too early and only ever fetch the newest page (breaking the historical backfill).
      *
      * Handles two cursor formats:
      * - Numeric: compared as bigint for correct ordering
      * - Composite cursors: compared as strings with COLLATE "C"
      */
     getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
+    /**
+     * Get the highest cursor from previous syncs for an object type, excluding the current run.
+     */
+    getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
     /**
      * Delete all sync runs and object runs for an account.
      * Useful for testing or resetting sync state.
@@ -585,6 +601,7 @@ declare class StripeSync {
      * ```
      */
     processNext(object: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>, params?: ProcessNextParams): Promise<ProcessNextResult>;
+    private appendMigrationHint;
     /**
      * Get the database resource name for a SyncObject type
      */
@@ -621,9 +638,10 @@ declare class StripeSync {
      * This is used by workers and background processes that should cooperate.
      *
      * @param triggeredBy - What triggered this sync (for observability)
+     * @param objectFilter - Optional specific object to sync (e.g. 'payment_intent'). If 'all' or undefined, syncs all objects.
      * @returns Run key and list of objects to sync
      */
-    joinOrCreateSyncRun(triggeredBy?: string): Promise<{
+    joinOrCreateSyncRun(triggeredBy?: string, objectFilter?: SyncObject): Promise<{
         runKey: RunKey;
         objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
     }>;

package/dist/index.d.ts

@@ -137,8 +137,10 @@ declare class PostgresClient {
     /**
      * Create object run entries for a sync run.
      * All objects start as 'pending'.
+     *
+     * @param resourceNames - Database resource names (e.g. 'products', 'customers', NOT 'product', 'customer')
      */
-    createObjectRuns(accountId: string, runStartedAt: Date, objects: string[]): Promise<void>;
+    createObjectRuns(accountId: string, runStartedAt: Date, resourceNames: string[]): Promise<void>;
     /**
      * Try to start an object sync (respects max_concurrent).
      * Returns true if claimed, false if already running or at concurrency limit.
@@ -155,12 +157,21 @@ declare class PostgresClient {
         status: string;
         processedCount: number;
         cursor: string | null;
+        pageCursor: string | null;
     } | null>;
     /**
      * Update progress for an object sync.
      * Also touches updated_at for stale detection.
      */
     incrementObjectProgress(accountId: string, runStartedAt: Date, object: string, count: number): Promise<void>;
+    /**
+     * Update the pagination page_cursor used for backfills using Stripe list calls.
+     */
+    updateObjectPageCursor(accountId: string, runStartedAt: Date, object: string, pageCursor: string | null): Promise<void>;
+    /**
+     * Clear the pagination page_cursor for an object sync.
+     */
+    clearObjectPageCursor(accountId: string, runStartedAt: Date, object: string): Promise<void>;
     /**
      * Update the cursor for an object sync.
      * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -170,15 +181,20 @@ declare class PostgresClient {
     updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
     /**
      * Get the highest cursor from previous syncs for an object type.
-     * This considers completed, error, AND running runs to ensure recovery syncs
-     * don't re-process data that was already synced before a crash.
-     * A 'running' status with a cursor means the process was killed mid-sync.
+     * Uses only completed object runs.
+     * - During the initial backfill we page through history, but we also update the cursor as we go.
+     *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+     *   too early and only ever fetch the newest page (breaking the historical backfill).
      *
      * Handles two cursor formats:
      * - Numeric: compared as bigint for correct ordering
      * - Composite cursors: compared as strings with COLLATE "C"
      */
     getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
+    /**
+     * Get the highest cursor from previous syncs for an object type, excluding the current run.
+     */
+    getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
     /**
      * Delete all sync runs and object runs for an account.
      * Useful for testing or resetting sync state.
@@ -585,6 +601,7 @@ declare class StripeSync {
      * ```
      */
     processNext(object: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>, params?: ProcessNextParams): Promise<ProcessNextResult>;
+    private appendMigrationHint;
     /**
      * Get the database resource name for a SyncObject type
      */
@@ -621,9 +638,10 @@ declare class StripeSync {
      * This is used by workers and background processes that should cooperate.
      *
      * @param triggeredBy - What triggered this sync (for observability)
+     * @param objectFilter - Optional specific object to sync (e.g. 'payment_intent'). If 'all' or undefined, syncs all objects.
      * @returns Run key and list of objects to sync
      */
-    joinOrCreateSyncRun(triggeredBy?: string): Promise<{
+    joinOrCreateSyncRun(triggeredBy?: string, objectFilter?: SyncObject): Promise<{
         runKey: RunKey;
         objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
     }>;

package/dist/index.js

@@ -5,8 +5,8 @@ import {
   createStripeWebSocketClient,
   hashApiKey,
   runMigrations
-} from "./chunk-TV67ZOCK.js";
-import "./chunk-57SXDCMH.js";
+} from "./chunk-XKBCLBFT.js";
+import "./chunk-CMGFQCD7.js";
 export {
   PostgresClient,
   StripeSync,

package/dist/migrations/0061_add_page_cursor.sql

@@ -0,0 +1,3 @@
+-- Add page_cursor column for pagination state within a single sync run.
+-- This is used to store the starting_after ID for backfills using Stripe list calls.
+ALTER TABLE "stripe"."_sync_obj_runs" ADD COLUMN IF NOT EXISTS page_cursor text;

package/dist/supabase/index.cjs

@@ -54,7 +54,7 @@ var workerFunctionCode = stripe_worker_default;
 // package.json
 var package_default = {
   name: "stripe-experiment-sync",
-  version: "1.0.17",
+  version: "1.0.19",
   private: false,
   description: "Stripe Sync Engine to sync Stripe data to Postgres",
   type: "module",

package/dist/supabase/index.js

@@ -9,8 +9,8 @@ import {
   uninstall,
   webhookFunctionCode,
   workerFunctionCode
-} from "../chunk-I7IFXSAU.js";
-import "../chunk-57SXDCMH.js";
+} from "../chunk-4P6TAP7L.js";
+import "../chunk-CMGFQCD7.js";
 export {
   INSTALLATION_ERROR_SUFFIX,
   INSTALLATION_INSTALLED_SUFFIX,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stripe-experiment-sync",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "private": false,
   "description": "Stripe Sync Engine to sync Stripe data to Postgres",
   "type": "module",