stripe-experiment-sync 1.0.18 → 1.0.19

package/README.md

@@ -202,6 +202,9 @@ Supported objects: `all`, `charge`, `checkout_sessions`, `credit_note`, `custome
 
 The sync engine tracks cursors per account and resource, enabling incremental syncing that resumes after interruptions.
 
+For paged backfills, the engine keeps a separate per-run pagination cursor (`page_cursor`) while the
+incremental cursor continues to track the highest `created` timestamp.
+
 > **Tip:** For large Stripe accounts (>10,000 objects), loop through date ranges day-by-day to avoid timeouts.
 
 ## Account Management

package/dist/{chunk-M5TTM27L.js → chunk-4P6TAP7L.js}

@@ -1,6 +1,6 @@
 import {
   package_default
-} from "./chunk-XO57OJUE.js";
+} from "./chunk-CMGFQCD7.js";
 
 // src/supabase/supabase.ts
 import { SupabaseManagementAPI } from "supabase-management-js";

package/dist/{chunk-XO57OJUE.js → chunk-CMGFQCD7.js}

@@ -1,7 +1,7 @@
 // package.json
 var package_default = {
   name: "stripe-experiment-sync",
-  version: "1.0.18",
+  version: "1.0.19",
   private: false,
   description: "Stripe Sync Engine to sync Stripe data to Postgres",
   type: "module",

package/dist/{chunk-HV64EIZU.js → chunk-HZ2OIPQ5.js}

@@ -2,11 +2,11 @@ import {
   StripeSync,
   createStripeWebSocketClient,
   runMigrations
-} from "./chunk-K4JQHI7Y.js";
+} from "./chunk-XKBCLBFT.js";
 import {
   install,
   uninstall
-} from "./chunk-M5TTM27L.js";
+} from "./chunk-4P6TAP7L.js";
 
 // src/cli/config.ts
 import dotenv from "dotenv";

package/dist/{chunk-K4JQHI7Y.js → chunk-XKBCLBFT.js}

@@ -1,6 +1,6 @@
 import {
   package_default
-} from "./chunk-XO57OJUE.js";
+} from "./chunk-CMGFQCD7.js";
 
 // src/stripeSync.ts
 import Stripe3 from "stripe";
@@ -398,7 +398,8 @@ var PostgresClient = class {
       `UPDATE "${this.config.schema}"."_sync_obj_runs" o
        SET status = 'error',
            error_message = 'Auto-cancelled: stale (no update in 5 min)',
-           completed_at = now()
+           completed_at = now(),
+           page_cursor = NULL
        WHERE o."_account_id" = $1
          AND o.status = 'running'
          AND o.updated_at < now() - interval '5 minutes'`,
@@ -544,7 +545,7 @@ var PostgresClient = class {
    */
   async getObjectRun(accountId, runStartedAt, object) {
     const result = await this.query(
-      `SELECT object, status, processed_count, cursor
+      `SELECT object, status, processed_count, cursor, page_cursor
        FROM "${this.config.schema}"."_sync_obj_runs"
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object]
@@ -555,7 +556,8 @@ var PostgresClient = class {
       object: row.object,
       status: row.status,
       processedCount: row.processed_count,
-      cursor: row.cursor
+      cursor: row.cursor,
+      pageCursor: row.page_cursor
     };
   }
   /**
@@ -570,6 +572,23 @@ var PostgresClient = class {
       [accountId, runStartedAt, object, count]
     );
   }
+  /**
+   * Update the pagination page_cursor used for backfills using Stripe list calls.
+   */
+  async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
+    await this.query(
+      `UPDATE "${this.config.schema}"."_sync_obj_runs"
+       SET page_cursor = $4, updated_at = now()
+       WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
+      [accountId, runStartedAt, object, pageCursor]
+    );
+  }
+  /**
+   * Clear the pagination page_cursor for an object sync.
+   */
+  async clearObjectPageCursor(accountId, runStartedAt, object) {
+    await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
+  }
   /**
    * Update the cursor for an object sync.
    * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -602,9 +621,10 @@ var PostgresClient = class {
   }
   /**
    * Get the highest cursor from previous syncs for an object type.
-   * This considers completed, error, AND running runs to ensure recovery syncs
-   * don't re-process data that was already synced before a crash.
-   * A 'running' status with a cursor means the process was killed mid-sync.
+   * Uses only completed object runs.
+   * - During the initial backfill we page through history, but we also update the cursor as we go.
+   *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+   *   too early and only ever fetch the newest page (breaking the historical backfill).
    *
    * Handles two cursor formats:
    * - Numeric: compared as bigint for correct ordering
@@ -619,11 +639,31 @@ var PostgresClient = class {
        FROM "${this.config.schema}"."_sync_obj_runs" o
        WHERE o."_account_id" = $1
          AND o.object = $2
-         AND o.cursor IS NOT NULL`,
+         AND o.cursor IS NOT NULL
+         AND o.status = 'complete'`,
       [accountId, object]
     );
     return result.rows[0]?.cursor ?? null;
   }
+  /**
+   * Get the highest cursor from previous syncs for an object type, excluding the current run.
+   */
+  async getLastCursorBeforeRun(accountId, object, runStartedAt) {
+    const result = await this.query(
+      `SELECT CASE
+         WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
+         ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
+       END as cursor
+       FROM "${this.config.schema}"."_sync_obj_runs" o
+       WHERE o."_account_id" = $1
+         AND o.object = $2
+         AND o.cursor IS NOT NULL
+         AND o.status = 'complete'
+         AND o.run_started_at < $3`,
+      [accountId, object, runStartedAt]
+    );
+    return result.rows[0]?.cursor ?? null;
+  }
   /**
    * Delete all sync runs and object runs for an account.
    * Useful for testing or resetting sync state.
@@ -644,7 +684,7 @@ var PostgresClient = class {
   async completeObjectSync(accountId, runStartedAt, object) {
     await this.query(
       `UPDATE "${this.config.schema}"."_sync_obj_runs"
-       SET status = 'complete', completed_at = now()
+       SET status = 'complete', completed_at = now(), page_cursor = NULL
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object]
     );
@@ -660,7 +700,7 @@ var PostgresClient = class {
   async failObjectSync(accountId, runStartedAt, object, errorMessage) {
     await this.query(
       `UPDATE "${this.config.schema}"."_sync_obj_runs"
-       SET status = 'error', error_message = $4, completed_at = now()
+       SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object, errorMessage]
     );
@@ -1961,57 +2001,74 @@ var StripeSync = class {
    * ```
    */
   async processNext(object, params) {
-    await this.getCurrentAccount();
-    const accountId = await this.getAccountId();
-    const resourceName = this.getResourceName(object);
-    let runStartedAt;
-    if (params?.runStartedAt) {
-      runStartedAt = params.runStartedAt;
-    } else {
-      const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
-      runStartedAt = runKey.runStartedAt;
-    }
-    await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
-    const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
-    if (objRun?.status === "complete" || objRun?.status === "error") {
-      return {
-        processed: 0,
-        hasMore: false,
-        runStartedAt
-      };
-    }
-    if (objRun?.status === "pending") {
-      const started = await this.postgresClient.tryStartObjectSync(
-        accountId,
-        runStartedAt,
-        resourceName
-      );
-      if (!started) {
+    try {
+      await this.getCurrentAccount();
+      const accountId = await this.getAccountId();
+      const resourceName = this.getResourceName(object);
+      let runStartedAt;
+      if (params?.runStartedAt) {
+        runStartedAt = params.runStartedAt;
+      } else {
+        const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
+        runStartedAt = runKey.runStartedAt;
+      }
+      await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
+      const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
+      if (objRun?.status === "complete" || objRun?.status === "error") {
         return {
           processed: 0,
-          hasMore: true,
+          hasMore: false,
           runStartedAt
         };
       }
-    }
-    let cursor = null;
-    if (!params?.created) {
-      if (objRun?.cursor) {
-        cursor = objRun.cursor;
-      } else {
-        const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
+      if (objRun?.status === "pending") {
+        const started = await this.postgresClient.tryStartObjectSync(
+          accountId,
+          runStartedAt,
+          resourceName
+        );
+        if (!started) {
+          return {
+            processed: 0,
+            hasMore: true,
+            runStartedAt
+          };
+        }
+      }
+      let cursor = null;
+      if (!params?.created) {
+        const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
+          accountId,
+          resourceName,
+          runStartedAt
+        );
         cursor = lastCursor ?? null;
       }
+      const result = await this.fetchOnePage(
+        object,
+        accountId,
+        resourceName,
+        runStartedAt,
+        cursor,
+        objRun?.pageCursor ?? null,
+        params
+      );
+      return result;
+    } catch (error) {
+      throw this.appendMigrationHint(error);
     }
-    const result = await this.fetchOnePage(
-      object,
-      accountId,
-      resourceName,
-      runStartedAt,
-      cursor,
-      params
-    );
-    return result;
+  }
+  appendMigrationHint(error) {
+    const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
+    const withHint = (message) => message.includes(hint) ? message : `${hint}
+${message}`;
+    if (error instanceof Error) {
+      const { stack } = error;
+      error.message = withHint(error.message);
+      if (stack) error.stack = stack;
+      return error;
+    }
+    return new Error(withHint(String(error)));
   }
   /**
    * Get the database resource name for a SyncObject type
@@ -2043,7 +2100,7 @@ var StripeSync = class {
    * Uses resourceRegistry for DRY list/upsert operations.
    * Uses the observable sync system for tracking progress.
    */
-  async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
+  async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
     const limit = 100;
     if (object === "payment_method" || object === "tax_id") {
       this.config.logger?.warn(`processNext for ${object} requires customer context`);
@@ -2071,7 +2128,16 @@ var StripeSync = class {
           listParams.created = created;
         }
       }
+      if (pageCursor) {
+        listParams.starting_after = pageCursor;
+      }
       const response = await config.listFn(listParams);
+      if (response.data.length === 0 && response.has_more) {
+        const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
+        this.config.logger?.warn(message);
+        await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
+        return { processed: 0, hasMore: false, runStartedAt };
+      }
       if (response.data.length > 0) {
         this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
         await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
@@ -2092,6 +2158,15 @@ var StripeSync = class {
             String(maxCreated)
           );
         }
+        const lastId = response.data[response.data.length - 1].id;
+        if (response.has_more) {
+          await this.postgresClient.updateObjectPageCursor(
+            accountId,
+            runStartedAt,
+            resourceName,
+            lastId
+          );
+        }
       }
       if (!response.has_more) {
         await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);

package/dist/cli/index.cjs

@@ -33,7 +33,7 @@ var import_commander = require("commander");
 // package.json
 var package_default = {
   name: "stripe-experiment-sync",
-  version: "1.0.18",
+  version: "1.0.19",
   private: false,
   description: "Stripe Sync Engine to sync Stripe data to Postgres",
   type: "module",
@@ -580,7 +580,8 @@ var PostgresClient = class {
       `UPDATE "${this.config.schema}"."_sync_obj_runs" o
        SET status = 'error',
            error_message = 'Auto-cancelled: stale (no update in 5 min)',
-           completed_at = now()
+           completed_at = now(),
+           page_cursor = NULL
        WHERE o."_account_id" = $1
          AND o.status = 'running'
          AND o.updated_at < now() - interval '5 minutes'`,
@@ -726,7 +727,7 @@ var PostgresClient = class {
    */
   async getObjectRun(accountId, runStartedAt, object) {
     const result = await this.query(
-      `SELECT object, status, processed_count, cursor
+      `SELECT object, status, processed_count, cursor, page_cursor
        FROM "${this.config.schema}"."_sync_obj_runs"
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object]
@@ -737,7 +738,8 @@ var PostgresClient = class {
       object: row.object,
       status: row.status,
       processedCount: row.processed_count,
-      cursor: row.cursor
+      cursor: row.cursor,
+      pageCursor: row.page_cursor
     };
   }
   /**
@@ -752,6 +754,23 @@ var PostgresClient = class {
       [accountId, runStartedAt, object, count]
     );
   }
+  /**
+   * Update the pagination page_cursor used for backfills using Stripe list calls.
+   */
+  async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
+    await this.query(
+      `UPDATE "${this.config.schema}"."_sync_obj_runs"
+       SET page_cursor = $4, updated_at = now()
+       WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
+      [accountId, runStartedAt, object, pageCursor]
+    );
+  }
+  /**
+   * Clear the pagination page_cursor for an object sync.
+   */
+  async clearObjectPageCursor(accountId, runStartedAt, object) {
+    await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
+  }
   /**
    * Update the cursor for an object sync.
    * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -784,9 +803,10 @@ var PostgresClient = class {
   }
   /**
    * Get the highest cursor from previous syncs for an object type.
-   * This considers completed, error, AND running runs to ensure recovery syncs
-   * don't re-process data that was already synced before a crash.
-   * A 'running' status with a cursor means the process was killed mid-sync.
+   * Uses only completed object runs.
+   * - During the initial backfill we page through history, but we also update the cursor as we go.
+   *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+   *   too early and only ever fetch the newest page (breaking the historical backfill).
    *
    * Handles two cursor formats:
    * - Numeric: compared as bigint for correct ordering
@@ -801,11 +821,31 @@ var PostgresClient = class {
        FROM "${this.config.schema}"."_sync_obj_runs" o
        WHERE o."_account_id" = $1
          AND o.object = $2
-         AND o.cursor IS NOT NULL`,
+         AND o.cursor IS NOT NULL
+         AND o.status = 'complete'`,
       [accountId, object]
     );
     return result.rows[0]?.cursor ?? null;
   }
+  /**
+   * Get the highest cursor from previous syncs for an object type, excluding the current run.
+   */
+  async getLastCursorBeforeRun(accountId, object, runStartedAt) {
+    const result = await this.query(
+      `SELECT CASE
+         WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
+         ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
+       END as cursor
+       FROM "${this.config.schema}"."_sync_obj_runs" o
+       WHERE o."_account_id" = $1
+         AND o.object = $2
+         AND o.cursor IS NOT NULL
+         AND o.status = 'complete'
+         AND o.run_started_at < $3`,
+      [accountId, object, runStartedAt]
+    );
+    return result.rows[0]?.cursor ?? null;
+  }
   /**
    * Delete all sync runs and object runs for an account.
    * Useful for testing or resetting sync state.
@@ -826,7 +866,7 @@ var PostgresClient = class {
   async completeObjectSync(accountId, runStartedAt, object) {
     await this.query(
       `UPDATE "${this.config.schema}"."_sync_obj_runs"
-       SET status = 'complete', completed_at = now()
+       SET status = 'complete', completed_at = now(), page_cursor = NULL
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object]
     );
@@ -842,7 +882,7 @@ var PostgresClient = class {
   async failObjectSync(accountId, runStartedAt, object, errorMessage) {
     await this.query(
       `UPDATE "${this.config.schema}"."_sync_obj_runs"
-       SET status = 'error', error_message = $4, completed_at = now()
+       SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
        WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
       [accountId, runStartedAt, object, errorMessage]
     );
@@ -2143,57 +2183,74 @@ var StripeSync = class {
    * ```
    */
   async processNext(object, params) {
-    await this.getCurrentAccount();
-    const accountId = await this.getAccountId();
-    const resourceName = this.getResourceName(object);
-    let runStartedAt;
-    if (params?.runStartedAt) {
-      runStartedAt = params.runStartedAt;
-    } else {
-      const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
-      runStartedAt = runKey.runStartedAt;
-    }
-    await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
-    const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
-    if (objRun?.status === "complete" || objRun?.status === "error") {
-      return {
-        processed: 0,
-        hasMore: false,
-        runStartedAt
-      };
-    }
-    if (objRun?.status === "pending") {
-      const started = await this.postgresClient.tryStartObjectSync(
-        accountId,
-        runStartedAt,
-        resourceName
-      );
-      if (!started) {
+    try {
+      await this.getCurrentAccount();
+      const accountId = await this.getAccountId();
+      const resourceName = this.getResourceName(object);
+      let runStartedAt;
+      if (params?.runStartedAt) {
+        runStartedAt = params.runStartedAt;
+      } else {
+        const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
+        runStartedAt = runKey.runStartedAt;
+      }
+      await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
+      const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
+      if (objRun?.status === "complete" || objRun?.status === "error") {
         return {
           processed: 0,
-          hasMore: true,
+          hasMore: false,
           runStartedAt
         };
       }
-    }
-    let cursor = null;
-    if (!params?.created) {
-      if (objRun?.cursor) {
-        cursor = objRun.cursor;
-      } else {
-        const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
+      if (objRun?.status === "pending") {
+        const started = await this.postgresClient.tryStartObjectSync(
+          accountId,
+          runStartedAt,
+          resourceName
+        );
+        if (!started) {
+          return {
+            processed: 0,
+            hasMore: true,
+            runStartedAt
+          };
+        }
+      }
+      let cursor = null;
+      if (!params?.created) {
+        const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
+          accountId,
+          resourceName,
+          runStartedAt
+        );
         cursor = lastCursor ?? null;
       }
+      const result = await this.fetchOnePage(
+        object,
+        accountId,
+        resourceName,
+        runStartedAt,
+        cursor,
+        objRun?.pageCursor ?? null,
+        params
+      );
+      return result;
+    } catch (error) {
+      throw this.appendMigrationHint(error);
     }
-    const result = await this.fetchOnePage(
-      object,
-      accountId,
-      resourceName,
-      runStartedAt,
-      cursor,
-      params
-    );
-    return result;
+  }
+  appendMigrationHint(error) {
+    const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
+    const withHint = (message) => message.includes(hint) ? message : `${hint}
+${message}`;
+    if (error instanceof Error) {
+      const { stack } = error;
+      error.message = withHint(error.message);
+      if (stack) error.stack = stack;
+      return error;
+    }
+    return new Error(withHint(String(error)));
   }
   /**
    * Get the database resource name for a SyncObject type
@@ -2225,7 +2282,7 @@ var StripeSync = class {
    * Uses resourceRegistry for DRY list/upsert operations.
    * Uses the observable sync system for tracking progress.
    */
-  async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
+  async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
     const limit = 100;
     if (object === "payment_method" || object === "tax_id") {
       this.config.logger?.warn(`processNext for ${object} requires customer context`);
@@ -2253,7 +2310,16 @@ var StripeSync = class {
           listParams.created = created;
         }
       }
+      if (pageCursor) {
+        listParams.starting_after = pageCursor;
+      }
       const response = await config.listFn(listParams);
+      if (response.data.length === 0 && response.has_more) {
+        const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
+        this.config.logger?.warn(message);
+        await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
+        return { processed: 0, hasMore: false, runStartedAt };
+      }
       if (response.data.length > 0) {
         this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
         await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
@@ -2274,6 +2340,15 @@ var StripeSync = class {
             String(maxCreated)
           );
         }
+        const lastId = response.data[response.data.length - 1].id;
+        if (response.has_more) {
+          await this.postgresClient.updateObjectPageCursor(
+            accountId,
+            runStartedAt,
+            resourceName,
+            lastId
+          );
+        }
       }
       if (!response.has_more) {
         await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);

package/dist/cli/index.js

@@ -5,12 +5,12 @@ import {
   migrateCommand,
   syncCommand,
   uninstallCommand
-} from "../chunk-HV64EIZU.js";
-import "../chunk-K4JQHI7Y.js";
-import "../chunk-M5TTM27L.js";
+} from "../chunk-HZ2OIPQ5.js";
+import "../chunk-XKBCLBFT.js";
+import "../chunk-4P6TAP7L.js";
 import {
   package_default
-} from "../chunk-XO57OJUE.js";
+} from "../chunk-CMGFQCD7.js";
 
 // src/cli/index.ts
 import { Command } from "commander";