stripe-experiment-sync 1.0.18 → 1.0.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli/lib.cjs CHANGED
@@ -117,7 +117,7 @@ async function loadConfig(options) {
117
117
  // package.json
118
118
  var package_default = {
119
119
  name: "stripe-experiment-sync",
120
- version: "1.0.18",
120
+ version: "1.0.19",
121
121
  private: false,
122
122
  description: "Stripe Sync Engine to sync Stripe data to Postgres",
123
123
  type: "module",
@@ -594,7 +594,8 @@ var PostgresClient = class {
594
594
  `UPDATE "${this.config.schema}"."_sync_obj_runs" o
595
595
  SET status = 'error',
596
596
  error_message = 'Auto-cancelled: stale (no update in 5 min)',
597
- completed_at = now()
597
+ completed_at = now(),
598
+ page_cursor = NULL
598
599
  WHERE o."_account_id" = $1
599
600
  AND o.status = 'running'
600
601
  AND o.updated_at < now() - interval '5 minutes'`,
@@ -740,7 +741,7 @@ var PostgresClient = class {
740
741
  */
741
742
  async getObjectRun(accountId, runStartedAt, object) {
742
743
  const result = await this.query(
743
- `SELECT object, status, processed_count, cursor
744
+ `SELECT object, status, processed_count, cursor, page_cursor
744
745
  FROM "${this.config.schema}"."_sync_obj_runs"
745
746
  WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
746
747
  [accountId, runStartedAt, object]
@@ -751,7 +752,8 @@ var PostgresClient = class {
751
752
  object: row.object,
752
753
  status: row.status,
753
754
  processedCount: row.processed_count,
754
- cursor: row.cursor
755
+ cursor: row.cursor,
756
+ pageCursor: row.page_cursor
755
757
  };
756
758
  }
757
759
  /**
@@ -766,6 +768,23 @@ var PostgresClient = class {
766
768
  [accountId, runStartedAt, object, count]
767
769
  );
768
770
  }
771
+ /**
772
+ * Update the pagination page_cursor used for backfills using Stripe list calls.
773
+ */
774
+ async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
775
+ await this.query(
776
+ `UPDATE "${this.config.schema}"."_sync_obj_runs"
777
+ SET page_cursor = $4, updated_at = now()
778
+ WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
779
+ [accountId, runStartedAt, object, pageCursor]
780
+ );
781
+ }
782
+ /**
783
+ * Clear the pagination page_cursor for an object sync.
784
+ */
785
+ async clearObjectPageCursor(accountId, runStartedAt, object) {
786
+ await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
787
+ }
769
788
  /**
770
789
  * Update the cursor for an object sync.
771
790
  * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -798,9 +817,10 @@ var PostgresClient = class {
798
817
  }
799
818
  /**
800
819
  * Get the highest cursor from previous syncs for an object type.
801
- * This considers completed, error, AND running runs to ensure recovery syncs
802
- * don't re-process data that was already synced before a crash.
803
- * A 'running' status with a cursor means the process was killed mid-sync.
820
+ * Uses only completed object runs.
821
+ * - During the initial backfill we page through history, but we also update the cursor as we go.
822
+ * If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
823
+ * too early and only ever fetch the newest page (breaking the historical backfill).
804
824
  *
805
825
  * Handles two cursor formats:
806
826
  * - Numeric: compared as bigint for correct ordering
@@ -815,11 +835,31 @@ var PostgresClient = class {
815
835
  FROM "${this.config.schema}"."_sync_obj_runs" o
816
836
  WHERE o."_account_id" = $1
817
837
  AND o.object = $2
818
- AND o.cursor IS NOT NULL`,
838
+ AND o.cursor IS NOT NULL
839
+ AND o.status = 'complete'`,
819
840
  [accountId, object]
820
841
  );
821
842
  return result.rows[0]?.cursor ?? null;
822
843
  }
844
+ /**
845
+ * Get the highest cursor from previous syncs for an object type, excluding the current run.
846
+ */
847
+ async getLastCursorBeforeRun(accountId, object, runStartedAt) {
848
+ const result = await this.query(
849
+ `SELECT CASE
850
+ WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
851
+ ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
852
+ END as cursor
853
+ FROM "${this.config.schema}"."_sync_obj_runs" o
854
+ WHERE o."_account_id" = $1
855
+ AND o.object = $2
856
+ AND o.cursor IS NOT NULL
857
+ AND o.status = 'complete'
858
+ AND o.run_started_at < $3`,
859
+ [accountId, object, runStartedAt]
860
+ );
861
+ return result.rows[0]?.cursor ?? null;
862
+ }
823
863
  /**
824
864
  * Delete all sync runs and object runs for an account.
825
865
  * Useful for testing or resetting sync state.
@@ -840,7 +880,7 @@ var PostgresClient = class {
840
880
  async completeObjectSync(accountId, runStartedAt, object) {
841
881
  await this.query(
842
882
  `UPDATE "${this.config.schema}"."_sync_obj_runs"
843
- SET status = 'complete', completed_at = now()
883
+ SET status = 'complete', completed_at = now(), page_cursor = NULL
844
884
  WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
845
885
  [accountId, runStartedAt, object]
846
886
  );
@@ -856,7 +896,7 @@ var PostgresClient = class {
856
896
  async failObjectSync(accountId, runStartedAt, object, errorMessage) {
857
897
  await this.query(
858
898
  `UPDATE "${this.config.schema}"."_sync_obj_runs"
859
- SET status = 'error', error_message = $4, completed_at = now()
899
+ SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
860
900
  WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
861
901
  [accountId, runStartedAt, object, errorMessage]
862
902
  );
@@ -2157,57 +2197,74 @@ var StripeSync = class {
2157
2197
  * ```
2158
2198
  */
2159
2199
  async processNext(object, params) {
2160
- await this.getCurrentAccount();
2161
- const accountId = await this.getAccountId();
2162
- const resourceName = this.getResourceName(object);
2163
- let runStartedAt;
2164
- if (params?.runStartedAt) {
2165
- runStartedAt = params.runStartedAt;
2166
- } else {
2167
- const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
2168
- runStartedAt = runKey.runStartedAt;
2169
- }
2170
- await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
2171
- const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
2172
- if (objRun?.status === "complete" || objRun?.status === "error") {
2173
- return {
2174
- processed: 0,
2175
- hasMore: false,
2176
- runStartedAt
2177
- };
2178
- }
2179
- if (objRun?.status === "pending") {
2180
- const started = await this.postgresClient.tryStartObjectSync(
2181
- accountId,
2182
- runStartedAt,
2183
- resourceName
2184
- );
2185
- if (!started) {
2200
+ try {
2201
+ await this.getCurrentAccount();
2202
+ const accountId = await this.getAccountId();
2203
+ const resourceName = this.getResourceName(object);
2204
+ let runStartedAt;
2205
+ if (params?.runStartedAt) {
2206
+ runStartedAt = params.runStartedAt;
2207
+ } else {
2208
+ const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
2209
+ runStartedAt = runKey.runStartedAt;
2210
+ }
2211
+ await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
2212
+ const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
2213
+ if (objRun?.status === "complete" || objRun?.status === "error") {
2186
2214
  return {
2187
2215
  processed: 0,
2188
- hasMore: true,
2216
+ hasMore: false,
2189
2217
  runStartedAt
2190
2218
  };
2191
2219
  }
2192
- }
2193
- let cursor = null;
2194
- if (!params?.created) {
2195
- if (objRun?.cursor) {
2196
- cursor = objRun.cursor;
2197
- } else {
2198
- const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
2220
+ if (objRun?.status === "pending") {
2221
+ const started = await this.postgresClient.tryStartObjectSync(
2222
+ accountId,
2223
+ runStartedAt,
2224
+ resourceName
2225
+ );
2226
+ if (!started) {
2227
+ return {
2228
+ processed: 0,
2229
+ hasMore: true,
2230
+ runStartedAt
2231
+ };
2232
+ }
2233
+ }
2234
+ let cursor = null;
2235
+ if (!params?.created) {
2236
+ const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
2237
+ accountId,
2238
+ resourceName,
2239
+ runStartedAt
2240
+ );
2199
2241
  cursor = lastCursor ?? null;
2200
2242
  }
2243
+ const result = await this.fetchOnePage(
2244
+ object,
2245
+ accountId,
2246
+ resourceName,
2247
+ runStartedAt,
2248
+ cursor,
2249
+ objRun?.pageCursor ?? null,
2250
+ params
2251
+ );
2252
+ return result;
2253
+ } catch (error) {
2254
+ throw this.appendMigrationHint(error);
2201
2255
  }
2202
- const result = await this.fetchOnePage(
2203
- object,
2204
- accountId,
2205
- resourceName,
2206
- runStartedAt,
2207
- cursor,
2208
- params
2209
- );
2210
- return result;
2256
+ }
2257
+ appendMigrationHint(error) {
2258
+ const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
2259
+ const withHint = (message) => message.includes(hint) ? message : `${hint}
2260
+ ${message}`;
2261
+ if (error instanceof Error) {
2262
+ const { stack } = error;
2263
+ error.message = withHint(error.message);
2264
+ if (stack) error.stack = stack;
2265
+ return error;
2266
+ }
2267
+ return new Error(withHint(String(error)));
2211
2268
  }
2212
2269
  /**
2213
2270
  * Get the database resource name for a SyncObject type
@@ -2239,7 +2296,7 @@ var StripeSync = class {
2239
2296
  * Uses resourceRegistry for DRY list/upsert operations.
2240
2297
  * Uses the observable sync system for tracking progress.
2241
2298
  */
2242
- async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
2299
+ async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
2243
2300
  const limit = 100;
2244
2301
  if (object === "payment_method" || object === "tax_id") {
2245
2302
  this.config.logger?.warn(`processNext for ${object} requires customer context`);
@@ -2267,7 +2324,16 @@ var StripeSync = class {
2267
2324
  listParams.created = created;
2268
2325
  }
2269
2326
  }
2327
+ if (pageCursor) {
2328
+ listParams.starting_after = pageCursor;
2329
+ }
2270
2330
  const response = await config.listFn(listParams);
2331
+ if (response.data.length === 0 && response.has_more) {
2332
+ const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
2333
+ this.config.logger?.warn(message);
2334
+ await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
2335
+ return { processed: 0, hasMore: false, runStartedAt };
2336
+ }
2271
2337
  if (response.data.length > 0) {
2272
2338
  this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
2273
2339
  await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
@@ -2288,6 +2354,15 @@ var StripeSync = class {
2288
2354
  String(maxCreated)
2289
2355
  );
2290
2356
  }
2357
+ const lastId = response.data[response.data.length - 1].id;
2358
+ if (response.has_more) {
2359
+ await this.postgresClient.updateObjectPageCursor(
2360
+ accountId,
2361
+ runStartedAt,
2362
+ resourceName,
2363
+ lastId
2364
+ );
2365
+ }
2291
2366
  }
2292
2367
  if (!response.has_more) {
2293
2368
  await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
package/dist/cli/lib.js CHANGED
@@ -6,10 +6,10 @@ import {
6
6
  migrateCommand,
7
7
  syncCommand,
8
8
  uninstallCommand
9
- } from "../chunk-HV64EIZU.js";
10
- import "../chunk-K4JQHI7Y.js";
11
- import "../chunk-M5TTM27L.js";
12
- import "../chunk-XO57OJUE.js";
9
+ } from "../chunk-HZ2OIPQ5.js";
10
+ import "../chunk-XKBCLBFT.js";
11
+ import "../chunk-4P6TAP7L.js";
12
+ import "../chunk-CMGFQCD7.js";
13
13
  export {
14
14
  backfillCommand,
15
15
  createTunnel,
package/dist/index.cjs CHANGED
@@ -46,7 +46,7 @@ var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
46
46
  // package.json
47
47
  var package_default = {
48
48
  name: "stripe-experiment-sync",
49
- version: "1.0.18",
49
+ version: "1.0.19",
50
50
  private: false,
51
51
  description: "Stripe Sync Engine to sync Stripe data to Postgres",
52
52
  type: "module",
@@ -523,7 +523,8 @@ var PostgresClient = class {
523
523
  `UPDATE "${this.config.schema}"."_sync_obj_runs" o
524
524
  SET status = 'error',
525
525
  error_message = 'Auto-cancelled: stale (no update in 5 min)',
526
- completed_at = now()
526
+ completed_at = now(),
527
+ page_cursor = NULL
527
528
  WHERE o."_account_id" = $1
528
529
  AND o.status = 'running'
529
530
  AND o.updated_at < now() - interval '5 minutes'`,
@@ -669,7 +670,7 @@ var PostgresClient = class {
669
670
  */
670
671
  async getObjectRun(accountId, runStartedAt, object) {
671
672
  const result = await this.query(
672
- `SELECT object, status, processed_count, cursor
673
+ `SELECT object, status, processed_count, cursor, page_cursor
673
674
  FROM "${this.config.schema}"."_sync_obj_runs"
674
675
  WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
675
676
  [accountId, runStartedAt, object]
@@ -680,7 +681,8 @@ var PostgresClient = class {
680
681
  object: row.object,
681
682
  status: row.status,
682
683
  processedCount: row.processed_count,
683
- cursor: row.cursor
684
+ cursor: row.cursor,
685
+ pageCursor: row.page_cursor
684
686
  };
685
687
  }
686
688
  /**
@@ -695,6 +697,23 @@ var PostgresClient = class {
695
697
  [accountId, runStartedAt, object, count]
696
698
  );
697
699
  }
700
+ /**
701
+ * Update the pagination page_cursor used for backfills using Stripe list calls.
702
+ */
703
+ async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
704
+ await this.query(
705
+ `UPDATE "${this.config.schema}"."_sync_obj_runs"
706
+ SET page_cursor = $4, updated_at = now()
707
+ WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
708
+ [accountId, runStartedAt, object, pageCursor]
709
+ );
710
+ }
711
+ /**
712
+ * Clear the pagination page_cursor for an object sync.
713
+ */
714
+ async clearObjectPageCursor(accountId, runStartedAt, object) {
715
+ await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
716
+ }
698
717
  /**
699
718
  * Update the cursor for an object sync.
700
719
  * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -727,9 +746,10 @@ var PostgresClient = class {
727
746
  }
728
747
  /**
729
748
  * Get the highest cursor from previous syncs for an object type.
730
- * This considers completed, error, AND running runs to ensure recovery syncs
731
- * don't re-process data that was already synced before a crash.
732
- * A 'running' status with a cursor means the process was killed mid-sync.
749
+ * Uses only completed object runs.
750
+ * - During the initial backfill we page through history, but we also update the cursor as we go.
751
+ * If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
752
+ * too early and only ever fetch the newest page (breaking the historical backfill).
733
753
  *
734
754
  * Handles two cursor formats:
735
755
  * - Numeric: compared as bigint for correct ordering
@@ -744,11 +764,31 @@ var PostgresClient = class {
744
764
  FROM "${this.config.schema}"."_sync_obj_runs" o
745
765
  WHERE o."_account_id" = $1
746
766
  AND o.object = $2
747
- AND o.cursor IS NOT NULL`,
767
+ AND o.cursor IS NOT NULL
768
+ AND o.status = 'complete'`,
748
769
  [accountId, object]
749
770
  );
750
771
  return result.rows[0]?.cursor ?? null;
751
772
  }
773
+ /**
774
+ * Get the highest cursor from previous syncs for an object type, excluding the current run.
775
+ */
776
+ async getLastCursorBeforeRun(accountId, object, runStartedAt) {
777
+ const result = await this.query(
778
+ `SELECT CASE
779
+ WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
780
+ ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
781
+ END as cursor
782
+ FROM "${this.config.schema}"."_sync_obj_runs" o
783
+ WHERE o."_account_id" = $1
784
+ AND o.object = $2
785
+ AND o.cursor IS NOT NULL
786
+ AND o.status = 'complete'
787
+ AND o.run_started_at < $3`,
788
+ [accountId, object, runStartedAt]
789
+ );
790
+ return result.rows[0]?.cursor ?? null;
791
+ }
752
792
  /**
753
793
  * Delete all sync runs and object runs for an account.
754
794
  * Useful for testing or resetting sync state.
@@ -769,7 +809,7 @@ var PostgresClient = class {
769
809
  async completeObjectSync(accountId, runStartedAt, object) {
770
810
  await this.query(
771
811
  `UPDATE "${this.config.schema}"."_sync_obj_runs"
772
- SET status = 'complete', completed_at = now()
812
+ SET status = 'complete', completed_at = now(), page_cursor = NULL
773
813
  WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
774
814
  [accountId, runStartedAt, object]
775
815
  );
@@ -785,7 +825,7 @@ var PostgresClient = class {
785
825
  async failObjectSync(accountId, runStartedAt, object, errorMessage) {
786
826
  await this.query(
787
827
  `UPDATE "${this.config.schema}"."_sync_obj_runs"
788
- SET status = 'error', error_message = $4, completed_at = now()
828
+ SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
789
829
  WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
790
830
  [accountId, runStartedAt, object, errorMessage]
791
831
  );
@@ -2086,57 +2126,74 @@ var StripeSync = class {
2086
2126
  * ```
2087
2127
  */
2088
2128
  async processNext(object, params) {
2089
- await this.getCurrentAccount();
2090
- const accountId = await this.getAccountId();
2091
- const resourceName = this.getResourceName(object);
2092
- let runStartedAt;
2093
- if (params?.runStartedAt) {
2094
- runStartedAt = params.runStartedAt;
2095
- } else {
2096
- const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
2097
- runStartedAt = runKey.runStartedAt;
2098
- }
2099
- await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
2100
- const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
2101
- if (objRun?.status === "complete" || objRun?.status === "error") {
2102
- return {
2103
- processed: 0,
2104
- hasMore: false,
2105
- runStartedAt
2106
- };
2107
- }
2108
- if (objRun?.status === "pending") {
2109
- const started = await this.postgresClient.tryStartObjectSync(
2110
- accountId,
2111
- runStartedAt,
2112
- resourceName
2113
- );
2114
- if (!started) {
2129
+ try {
2130
+ await this.getCurrentAccount();
2131
+ const accountId = await this.getAccountId();
2132
+ const resourceName = this.getResourceName(object);
2133
+ let runStartedAt;
2134
+ if (params?.runStartedAt) {
2135
+ runStartedAt = params.runStartedAt;
2136
+ } else {
2137
+ const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
2138
+ runStartedAt = runKey.runStartedAt;
2139
+ }
2140
+ await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
2141
+ const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
2142
+ if (objRun?.status === "complete" || objRun?.status === "error") {
2115
2143
  return {
2116
2144
  processed: 0,
2117
- hasMore: true,
2145
+ hasMore: false,
2118
2146
  runStartedAt
2119
2147
  };
2120
2148
  }
2121
- }
2122
- let cursor = null;
2123
- if (!params?.created) {
2124
- if (objRun?.cursor) {
2125
- cursor = objRun.cursor;
2126
- } else {
2127
- const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
2149
+ if (objRun?.status === "pending") {
2150
+ const started = await this.postgresClient.tryStartObjectSync(
2151
+ accountId,
2152
+ runStartedAt,
2153
+ resourceName
2154
+ );
2155
+ if (!started) {
2156
+ return {
2157
+ processed: 0,
2158
+ hasMore: true,
2159
+ runStartedAt
2160
+ };
2161
+ }
2162
+ }
2163
+ let cursor = null;
2164
+ if (!params?.created) {
2165
+ const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
2166
+ accountId,
2167
+ resourceName,
2168
+ runStartedAt
2169
+ );
2128
2170
  cursor = lastCursor ?? null;
2129
2171
  }
2172
+ const result = await this.fetchOnePage(
2173
+ object,
2174
+ accountId,
2175
+ resourceName,
2176
+ runStartedAt,
2177
+ cursor,
2178
+ objRun?.pageCursor ?? null,
2179
+ params
2180
+ );
2181
+ return result;
2182
+ } catch (error) {
2183
+ throw this.appendMigrationHint(error);
2130
2184
  }
2131
- const result = await this.fetchOnePage(
2132
- object,
2133
- accountId,
2134
- resourceName,
2135
- runStartedAt,
2136
- cursor,
2137
- params
2138
- );
2139
- return result;
2185
+ }
2186
+ appendMigrationHint(error) {
2187
+ const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
2188
+ const withHint = (message) => message.includes(hint) ? message : `${hint}
2189
+ ${message}`;
2190
+ if (error instanceof Error) {
2191
+ const { stack } = error;
2192
+ error.message = withHint(error.message);
2193
+ if (stack) error.stack = stack;
2194
+ return error;
2195
+ }
2196
+ return new Error(withHint(String(error)));
2140
2197
  }
2141
2198
  /**
2142
2199
  * Get the database resource name for a SyncObject type
@@ -2168,7 +2225,7 @@ var StripeSync = class {
2168
2225
  * Uses resourceRegistry for DRY list/upsert operations.
2169
2226
  * Uses the observable sync system for tracking progress.
2170
2227
  */
2171
- async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
2228
+ async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
2172
2229
  const limit = 100;
2173
2230
  if (object === "payment_method" || object === "tax_id") {
2174
2231
  this.config.logger?.warn(`processNext for ${object} requires customer context`);
@@ -2196,7 +2253,16 @@ var StripeSync = class {
2196
2253
  listParams.created = created;
2197
2254
  }
2198
2255
  }
2256
+ if (pageCursor) {
2257
+ listParams.starting_after = pageCursor;
2258
+ }
2199
2259
  const response = await config.listFn(listParams);
2260
+ if (response.data.length === 0 && response.has_more) {
2261
+ const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
2262
+ this.config.logger?.warn(message);
2263
+ await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
2264
+ return { processed: 0, hasMore: false, runStartedAt };
2265
+ }
2200
2266
  if (response.data.length > 0) {
2201
2267
  this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
2202
2268
  await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
@@ -2217,6 +2283,15 @@ var StripeSync = class {
2217
2283
  String(maxCreated)
2218
2284
  );
2219
2285
  }
2286
+ const lastId = response.data[response.data.length - 1].id;
2287
+ if (response.has_more) {
2288
+ await this.postgresClient.updateObjectPageCursor(
2289
+ accountId,
2290
+ runStartedAt,
2291
+ resourceName,
2292
+ lastId
2293
+ );
2294
+ }
2220
2295
  }
2221
2296
  if (!response.has_more) {
2222
2297
  await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
package/dist/index.d.cts CHANGED
@@ -157,12 +157,21 @@ declare class PostgresClient {
157
157
  status: string;
158
158
  processedCount: number;
159
159
  cursor: string | null;
160
+ pageCursor: string | null;
160
161
  } | null>;
161
162
  /**
162
163
  * Update progress for an object sync.
163
164
  * Also touches updated_at for stale detection.
164
165
  */
165
166
  incrementObjectProgress(accountId: string, runStartedAt: Date, object: string, count: number): Promise<void>;
167
+ /**
168
+ * Update the pagination page_cursor used for backfills using Stripe list calls.
169
+ */
170
+ updateObjectPageCursor(accountId: string, runStartedAt: Date, object: string, pageCursor: string | null): Promise<void>;
171
+ /**
172
+ * Clear the pagination page_cursor for an object sync.
173
+ */
174
+ clearObjectPageCursor(accountId: string, runStartedAt: Date, object: string): Promise<void>;
166
175
  /**
167
176
  * Update the cursor for an object sync.
168
177
  * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -172,15 +181,20 @@ declare class PostgresClient {
172
181
  updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
173
182
  /**
174
183
  * Get the highest cursor from previous syncs for an object type.
175
- * This considers completed, error, AND running runs to ensure recovery syncs
176
- * don't re-process data that was already synced before a crash.
177
- * A 'running' status with a cursor means the process was killed mid-sync.
184
+ * Uses only completed object runs.
185
+ * - During the initial backfill we page through history, but we also update the cursor as we go.
186
+ * If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
187
+ * too early and only ever fetch the newest page (breaking the historical backfill).
178
188
  *
179
189
  * Handles two cursor formats:
180
190
  * - Numeric: compared as bigint for correct ordering
181
191
  * - Composite cursors: compared as strings with COLLATE "C"
182
192
  */
183
193
  getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
194
+ /**
195
+ * Get the highest cursor from previous syncs for an object type, excluding the current run.
196
+ */
197
+ getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
184
198
  /**
185
199
  * Delete all sync runs and object runs for an account.
186
200
  * Useful for testing or resetting sync state.
@@ -587,6 +601,7 @@ declare class StripeSync {
587
601
  * ```
588
602
  */
589
603
  processNext(object: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>, params?: ProcessNextParams): Promise<ProcessNextResult>;
604
+ private appendMigrationHint;
590
605
  /**
591
606
  * Get the database resource name for a SyncObject type
592
607
  */