stripe-experiment-sync 1.0.18 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/{chunk-M5TTM27L.js → chunk-4P6TAP7L.js} +1 -1
- package/dist/{chunk-XO57OJUE.js → chunk-CMGFQCD7.js} +1 -1
- package/dist/{chunk-HV64EIZU.js → chunk-HZ2OIPQ5.js} +2 -2
- package/dist/{chunk-K4JQHI7Y.js → chunk-XKBCLBFT.js} +129 -54
- package/dist/cli/index.cjs +129 -54
- package/dist/cli/index.js +4 -4
- package/dist/cli/lib.cjs +129 -54
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +129 -54
- package/dist/index.d.cts +18 -3
- package/dist/index.d.ts +18 -3
- package/dist/index.js +2 -2
- package/dist/migrations/0061_add_page_cursor.sql +3 -0
- package/dist/supabase/index.cjs +1 -1
- package/dist/supabase/index.js +2 -2
- package/package.json +1 -1
package/dist/cli/lib.cjs
CHANGED
|
@@ -117,7 +117,7 @@ async function loadConfig(options) {
|
|
|
117
117
|
// package.json
|
|
118
118
|
var package_default = {
|
|
119
119
|
name: "stripe-experiment-sync",
|
|
120
|
-
version: "1.0.
|
|
120
|
+
version: "1.0.19",
|
|
121
121
|
private: false,
|
|
122
122
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
123
123
|
type: "module",
|
|
@@ -594,7 +594,8 @@ var PostgresClient = class {
|
|
|
594
594
|
`UPDATE "${this.config.schema}"."_sync_obj_runs" o
|
|
595
595
|
SET status = 'error',
|
|
596
596
|
error_message = 'Auto-cancelled: stale (no update in 5 min)',
|
|
597
|
-
completed_at = now()
|
|
597
|
+
completed_at = now(),
|
|
598
|
+
page_cursor = NULL
|
|
598
599
|
WHERE o."_account_id" = $1
|
|
599
600
|
AND o.status = 'running'
|
|
600
601
|
AND o.updated_at < now() - interval '5 minutes'`,
|
|
@@ -740,7 +741,7 @@ var PostgresClient = class {
|
|
|
740
741
|
*/
|
|
741
742
|
async getObjectRun(accountId, runStartedAt, object) {
|
|
742
743
|
const result = await this.query(
|
|
743
|
-
`SELECT object, status, processed_count, cursor
|
|
744
|
+
`SELECT object, status, processed_count, cursor, page_cursor
|
|
744
745
|
FROM "${this.config.schema}"."_sync_obj_runs"
|
|
745
746
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
746
747
|
[accountId, runStartedAt, object]
|
|
@@ -751,7 +752,8 @@ var PostgresClient = class {
|
|
|
751
752
|
object: row.object,
|
|
752
753
|
status: row.status,
|
|
753
754
|
processedCount: row.processed_count,
|
|
754
|
-
cursor: row.cursor
|
|
755
|
+
cursor: row.cursor,
|
|
756
|
+
pageCursor: row.page_cursor
|
|
755
757
|
};
|
|
756
758
|
}
|
|
757
759
|
/**
|
|
@@ -766,6 +768,23 @@ var PostgresClient = class {
|
|
|
766
768
|
[accountId, runStartedAt, object, count]
|
|
767
769
|
);
|
|
768
770
|
}
|
|
771
|
+
/**
|
|
772
|
+
* Update the pagination page_cursor used for backfills using Stripe list calls.
|
|
773
|
+
*/
|
|
774
|
+
async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
|
|
775
|
+
await this.query(
|
|
776
|
+
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
777
|
+
SET page_cursor = $4, updated_at = now()
|
|
778
|
+
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
779
|
+
[accountId, runStartedAt, object, pageCursor]
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Clear the pagination page_cursor for an object sync.
|
|
784
|
+
*/
|
|
785
|
+
async clearObjectPageCursor(accountId, runStartedAt, object) {
|
|
786
|
+
await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
|
|
787
|
+
}
|
|
769
788
|
/**
|
|
770
789
|
* Update the cursor for an object sync.
|
|
771
790
|
* Only updates if the new cursor is higher than the existing one (cursors should never decrease).
|
|
@@ -798,9 +817,10 @@ var PostgresClient = class {
|
|
|
798
817
|
}
|
|
799
818
|
/**
|
|
800
819
|
* Get the highest cursor from previous syncs for an object type.
|
|
801
|
-
*
|
|
802
|
-
*
|
|
803
|
-
*
|
|
820
|
+
* Uses only completed object runs.
|
|
821
|
+
* - During the initial backfill we page through history, but we also update the cursor as we go.
|
|
822
|
+
* If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
|
|
823
|
+
* too early and only ever fetch the newest page (breaking the historical backfill).
|
|
804
824
|
*
|
|
805
825
|
* Handles two cursor formats:
|
|
806
826
|
* - Numeric: compared as bigint for correct ordering
|
|
@@ -815,11 +835,31 @@ var PostgresClient = class {
|
|
|
815
835
|
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
816
836
|
WHERE o."_account_id" = $1
|
|
817
837
|
AND o.object = $2
|
|
818
|
-
AND o.cursor IS NOT NULL
|
|
838
|
+
AND o.cursor IS NOT NULL
|
|
839
|
+
AND o.status = 'complete'`,
|
|
819
840
|
[accountId, object]
|
|
820
841
|
);
|
|
821
842
|
return result.rows[0]?.cursor ?? null;
|
|
822
843
|
}
|
|
844
|
+
/**
|
|
845
|
+
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
846
|
+
*/
|
|
847
|
+
async getLastCursorBeforeRun(accountId, object, runStartedAt) {
|
|
848
|
+
const result = await this.query(
|
|
849
|
+
`SELECT CASE
|
|
850
|
+
WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
|
|
851
|
+
ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
|
|
852
|
+
END as cursor
|
|
853
|
+
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
854
|
+
WHERE o."_account_id" = $1
|
|
855
|
+
AND o.object = $2
|
|
856
|
+
AND o.cursor IS NOT NULL
|
|
857
|
+
AND o.status = 'complete'
|
|
858
|
+
AND o.run_started_at < $3`,
|
|
859
|
+
[accountId, object, runStartedAt]
|
|
860
|
+
);
|
|
861
|
+
return result.rows[0]?.cursor ?? null;
|
|
862
|
+
}
|
|
823
863
|
/**
|
|
824
864
|
* Delete all sync runs and object runs for an account.
|
|
825
865
|
* Useful for testing or resetting sync state.
|
|
@@ -840,7 +880,7 @@ var PostgresClient = class {
|
|
|
840
880
|
async completeObjectSync(accountId, runStartedAt, object) {
|
|
841
881
|
await this.query(
|
|
842
882
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
843
|
-
SET status = 'complete', completed_at = now()
|
|
883
|
+
SET status = 'complete', completed_at = now(), page_cursor = NULL
|
|
844
884
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
845
885
|
[accountId, runStartedAt, object]
|
|
846
886
|
);
|
|
@@ -856,7 +896,7 @@ var PostgresClient = class {
|
|
|
856
896
|
async failObjectSync(accountId, runStartedAt, object, errorMessage) {
|
|
857
897
|
await this.query(
|
|
858
898
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
859
|
-
SET status = 'error', error_message = $4, completed_at = now()
|
|
899
|
+
SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
|
|
860
900
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
861
901
|
[accountId, runStartedAt, object, errorMessage]
|
|
862
902
|
);
|
|
@@ -2157,57 +2197,74 @@ var StripeSync = class {
|
|
|
2157
2197
|
* ```
|
|
2158
2198
|
*/
|
|
2159
2199
|
async processNext(object, params) {
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
processed: 0,
|
|
2175
|
-
hasMore: false,
|
|
2176
|
-
runStartedAt
|
|
2177
|
-
};
|
|
2178
|
-
}
|
|
2179
|
-
if (objRun?.status === "pending") {
|
|
2180
|
-
const started = await this.postgresClient.tryStartObjectSync(
|
|
2181
|
-
accountId,
|
|
2182
|
-
runStartedAt,
|
|
2183
|
-
resourceName
|
|
2184
|
-
);
|
|
2185
|
-
if (!started) {
|
|
2200
|
+
try {
|
|
2201
|
+
await this.getCurrentAccount();
|
|
2202
|
+
const accountId = await this.getAccountId();
|
|
2203
|
+
const resourceName = this.getResourceName(object);
|
|
2204
|
+
let runStartedAt;
|
|
2205
|
+
if (params?.runStartedAt) {
|
|
2206
|
+
runStartedAt = params.runStartedAt;
|
|
2207
|
+
} else {
|
|
2208
|
+
const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
|
|
2209
|
+
runStartedAt = runKey.runStartedAt;
|
|
2210
|
+
}
|
|
2211
|
+
await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
|
|
2212
|
+
const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
|
|
2213
|
+
if (objRun?.status === "complete" || objRun?.status === "error") {
|
|
2186
2214
|
return {
|
|
2187
2215
|
processed: 0,
|
|
2188
|
-
hasMore:
|
|
2216
|
+
hasMore: false,
|
|
2189
2217
|
runStartedAt
|
|
2190
2218
|
};
|
|
2191
2219
|
}
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2220
|
+
if (objRun?.status === "pending") {
|
|
2221
|
+
const started = await this.postgresClient.tryStartObjectSync(
|
|
2222
|
+
accountId,
|
|
2223
|
+
runStartedAt,
|
|
2224
|
+
resourceName
|
|
2225
|
+
);
|
|
2226
|
+
if (!started) {
|
|
2227
|
+
return {
|
|
2228
|
+
processed: 0,
|
|
2229
|
+
hasMore: true,
|
|
2230
|
+
runStartedAt
|
|
2231
|
+
};
|
|
2232
|
+
}
|
|
2233
|
+
}
|
|
2234
|
+
let cursor = null;
|
|
2235
|
+
if (!params?.created) {
|
|
2236
|
+
const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
|
|
2237
|
+
accountId,
|
|
2238
|
+
resourceName,
|
|
2239
|
+
runStartedAt
|
|
2240
|
+
);
|
|
2199
2241
|
cursor = lastCursor ?? null;
|
|
2200
2242
|
}
|
|
2243
|
+
const result = await this.fetchOnePage(
|
|
2244
|
+
object,
|
|
2245
|
+
accountId,
|
|
2246
|
+
resourceName,
|
|
2247
|
+
runStartedAt,
|
|
2248
|
+
cursor,
|
|
2249
|
+
objRun?.pageCursor ?? null,
|
|
2250
|
+
params
|
|
2251
|
+
);
|
|
2252
|
+
return result;
|
|
2253
|
+
} catch (error) {
|
|
2254
|
+
throw this.appendMigrationHint(error);
|
|
2201
2255
|
}
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2256
|
+
}
|
|
2257
|
+
appendMigrationHint(error) {
|
|
2258
|
+
const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
|
|
2259
|
+
const withHint = (message) => message.includes(hint) ? message : `${hint}
|
|
2260
|
+
${message}`;
|
|
2261
|
+
if (error instanceof Error) {
|
|
2262
|
+
const { stack } = error;
|
|
2263
|
+
error.message = withHint(error.message);
|
|
2264
|
+
if (stack) error.stack = stack;
|
|
2265
|
+
return error;
|
|
2266
|
+
}
|
|
2267
|
+
return new Error(withHint(String(error)));
|
|
2211
2268
|
}
|
|
2212
2269
|
/**
|
|
2213
2270
|
* Get the database resource name for a SyncObject type
|
|
@@ -2239,7 +2296,7 @@ var StripeSync = class {
|
|
|
2239
2296
|
* Uses resourceRegistry for DRY list/upsert operations.
|
|
2240
2297
|
* Uses the observable sync system for tracking progress.
|
|
2241
2298
|
*/
|
|
2242
|
-
async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
|
|
2299
|
+
async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
|
|
2243
2300
|
const limit = 100;
|
|
2244
2301
|
if (object === "payment_method" || object === "tax_id") {
|
|
2245
2302
|
this.config.logger?.warn(`processNext for ${object} requires customer context`);
|
|
@@ -2267,7 +2324,16 @@ var StripeSync = class {
|
|
|
2267
2324
|
listParams.created = created;
|
|
2268
2325
|
}
|
|
2269
2326
|
}
|
|
2327
|
+
if (pageCursor) {
|
|
2328
|
+
listParams.starting_after = pageCursor;
|
|
2329
|
+
}
|
|
2270
2330
|
const response = await config.listFn(listParams);
|
|
2331
|
+
if (response.data.length === 0 && response.has_more) {
|
|
2332
|
+
const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
|
|
2333
|
+
this.config.logger?.warn(message);
|
|
2334
|
+
await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
|
|
2335
|
+
return { processed: 0, hasMore: false, runStartedAt };
|
|
2336
|
+
}
|
|
2271
2337
|
if (response.data.length > 0) {
|
|
2272
2338
|
this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
|
|
2273
2339
|
await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
|
|
@@ -2288,6 +2354,15 @@ var StripeSync = class {
|
|
|
2288
2354
|
String(maxCreated)
|
|
2289
2355
|
);
|
|
2290
2356
|
}
|
|
2357
|
+
const lastId = response.data[response.data.length - 1].id;
|
|
2358
|
+
if (response.has_more) {
|
|
2359
|
+
await this.postgresClient.updateObjectPageCursor(
|
|
2360
|
+
accountId,
|
|
2361
|
+
runStartedAt,
|
|
2362
|
+
resourceName,
|
|
2363
|
+
lastId
|
|
2364
|
+
);
|
|
2365
|
+
}
|
|
2291
2366
|
}
|
|
2292
2367
|
if (!response.has_more) {
|
|
2293
2368
|
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
package/dist/cli/lib.js
CHANGED
|
@@ -6,10 +6,10 @@ import {
|
|
|
6
6
|
migrateCommand,
|
|
7
7
|
syncCommand,
|
|
8
8
|
uninstallCommand
|
|
9
|
-
} from "../chunk-
|
|
10
|
-
import "../chunk-
|
|
11
|
-
import "../chunk-
|
|
12
|
-
import "../chunk-
|
|
9
|
+
} from "../chunk-HZ2OIPQ5.js";
|
|
10
|
+
import "../chunk-XKBCLBFT.js";
|
|
11
|
+
import "../chunk-4P6TAP7L.js";
|
|
12
|
+
import "../chunk-CMGFQCD7.js";
|
|
13
13
|
export {
|
|
14
14
|
backfillCommand,
|
|
15
15
|
createTunnel,
|
package/dist/index.cjs
CHANGED
|
@@ -46,7 +46,7 @@ var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
|
46
46
|
// package.json
|
|
47
47
|
var package_default = {
|
|
48
48
|
name: "stripe-experiment-sync",
|
|
49
|
-
version: "1.0.
|
|
49
|
+
version: "1.0.19",
|
|
50
50
|
private: false,
|
|
51
51
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
52
52
|
type: "module",
|
|
@@ -523,7 +523,8 @@ var PostgresClient = class {
|
|
|
523
523
|
`UPDATE "${this.config.schema}"."_sync_obj_runs" o
|
|
524
524
|
SET status = 'error',
|
|
525
525
|
error_message = 'Auto-cancelled: stale (no update in 5 min)',
|
|
526
|
-
completed_at = now()
|
|
526
|
+
completed_at = now(),
|
|
527
|
+
page_cursor = NULL
|
|
527
528
|
WHERE o."_account_id" = $1
|
|
528
529
|
AND o.status = 'running'
|
|
529
530
|
AND o.updated_at < now() - interval '5 minutes'`,
|
|
@@ -669,7 +670,7 @@ var PostgresClient = class {
|
|
|
669
670
|
*/
|
|
670
671
|
async getObjectRun(accountId, runStartedAt, object) {
|
|
671
672
|
const result = await this.query(
|
|
672
|
-
`SELECT object, status, processed_count, cursor
|
|
673
|
+
`SELECT object, status, processed_count, cursor, page_cursor
|
|
673
674
|
FROM "${this.config.schema}"."_sync_obj_runs"
|
|
674
675
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
675
676
|
[accountId, runStartedAt, object]
|
|
@@ -680,7 +681,8 @@ var PostgresClient = class {
|
|
|
680
681
|
object: row.object,
|
|
681
682
|
status: row.status,
|
|
682
683
|
processedCount: row.processed_count,
|
|
683
|
-
cursor: row.cursor
|
|
684
|
+
cursor: row.cursor,
|
|
685
|
+
pageCursor: row.page_cursor
|
|
684
686
|
};
|
|
685
687
|
}
|
|
686
688
|
/**
|
|
@@ -695,6 +697,23 @@ var PostgresClient = class {
|
|
|
695
697
|
[accountId, runStartedAt, object, count]
|
|
696
698
|
);
|
|
697
699
|
}
|
|
700
|
+
/**
|
|
701
|
+
* Update the pagination page_cursor used for backfills using Stripe list calls.
|
|
702
|
+
*/
|
|
703
|
+
async updateObjectPageCursor(accountId, runStartedAt, object, pageCursor) {
|
|
704
|
+
await this.query(
|
|
705
|
+
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
706
|
+
SET page_cursor = $4, updated_at = now()
|
|
707
|
+
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
708
|
+
[accountId, runStartedAt, object, pageCursor]
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Clear the pagination page_cursor for an object sync.
|
|
713
|
+
*/
|
|
714
|
+
async clearObjectPageCursor(accountId, runStartedAt, object) {
|
|
715
|
+
await this.updateObjectPageCursor(accountId, runStartedAt, object, null);
|
|
716
|
+
}
|
|
698
717
|
/**
|
|
699
718
|
* Update the cursor for an object sync.
|
|
700
719
|
* Only updates if the new cursor is higher than the existing one (cursors should never decrease).
|
|
@@ -727,9 +746,10 @@ var PostgresClient = class {
|
|
|
727
746
|
}
|
|
728
747
|
/**
|
|
729
748
|
* Get the highest cursor from previous syncs for an object type.
|
|
730
|
-
*
|
|
731
|
-
*
|
|
732
|
-
*
|
|
749
|
+
* Uses only completed object runs.
|
|
750
|
+
* - During the initial backfill we page through history, but we also update the cursor as we go.
|
|
751
|
+
* If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
|
|
752
|
+
* too early and only ever fetch the newest page (breaking the historical backfill).
|
|
733
753
|
*
|
|
734
754
|
* Handles two cursor formats:
|
|
735
755
|
* - Numeric: compared as bigint for correct ordering
|
|
@@ -744,11 +764,31 @@ var PostgresClient = class {
|
|
|
744
764
|
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
745
765
|
WHERE o."_account_id" = $1
|
|
746
766
|
AND o.object = $2
|
|
747
|
-
AND o.cursor IS NOT NULL
|
|
767
|
+
AND o.cursor IS NOT NULL
|
|
768
|
+
AND o.status = 'complete'`,
|
|
748
769
|
[accountId, object]
|
|
749
770
|
);
|
|
750
771
|
return result.rows[0]?.cursor ?? null;
|
|
751
772
|
}
|
|
773
|
+
/**
|
|
774
|
+
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
775
|
+
*/
|
|
776
|
+
async getLastCursorBeforeRun(accountId, object, runStartedAt) {
|
|
777
|
+
const result = await this.query(
|
|
778
|
+
`SELECT CASE
|
|
779
|
+
WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
|
|
780
|
+
ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
|
|
781
|
+
END as cursor
|
|
782
|
+
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
783
|
+
WHERE o."_account_id" = $1
|
|
784
|
+
AND o.object = $2
|
|
785
|
+
AND o.cursor IS NOT NULL
|
|
786
|
+
AND o.status = 'complete'
|
|
787
|
+
AND o.run_started_at < $3`,
|
|
788
|
+
[accountId, object, runStartedAt]
|
|
789
|
+
);
|
|
790
|
+
return result.rows[0]?.cursor ?? null;
|
|
791
|
+
}
|
|
752
792
|
/**
|
|
753
793
|
* Delete all sync runs and object runs for an account.
|
|
754
794
|
* Useful for testing or resetting sync state.
|
|
@@ -769,7 +809,7 @@ var PostgresClient = class {
|
|
|
769
809
|
async completeObjectSync(accountId, runStartedAt, object) {
|
|
770
810
|
await this.query(
|
|
771
811
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
772
|
-
SET status = 'complete', completed_at = now()
|
|
812
|
+
SET status = 'complete', completed_at = now(), page_cursor = NULL
|
|
773
813
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
774
814
|
[accountId, runStartedAt, object]
|
|
775
815
|
);
|
|
@@ -785,7 +825,7 @@ var PostgresClient = class {
|
|
|
785
825
|
async failObjectSync(accountId, runStartedAt, object, errorMessage) {
|
|
786
826
|
await this.query(
|
|
787
827
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
788
|
-
SET status = 'error', error_message = $4, completed_at = now()
|
|
828
|
+
SET status = 'error', error_message = $4, completed_at = now(), page_cursor = NULL
|
|
789
829
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
790
830
|
[accountId, runStartedAt, object, errorMessage]
|
|
791
831
|
);
|
|
@@ -2086,57 +2126,74 @@ var StripeSync = class {
|
|
|
2086
2126
|
* ```
|
|
2087
2127
|
*/
|
|
2088
2128
|
async processNext(object, params) {
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
processed: 0,
|
|
2104
|
-
hasMore: false,
|
|
2105
|
-
runStartedAt
|
|
2106
|
-
};
|
|
2107
|
-
}
|
|
2108
|
-
if (objRun?.status === "pending") {
|
|
2109
|
-
const started = await this.postgresClient.tryStartObjectSync(
|
|
2110
|
-
accountId,
|
|
2111
|
-
runStartedAt,
|
|
2112
|
-
resourceName
|
|
2113
|
-
);
|
|
2114
|
-
if (!started) {
|
|
2129
|
+
try {
|
|
2130
|
+
await this.getCurrentAccount();
|
|
2131
|
+
const accountId = await this.getAccountId();
|
|
2132
|
+
const resourceName = this.getResourceName(object);
|
|
2133
|
+
let runStartedAt;
|
|
2134
|
+
if (params?.runStartedAt) {
|
|
2135
|
+
runStartedAt = params.runStartedAt;
|
|
2136
|
+
} else {
|
|
2137
|
+
const { runKey } = await this.joinOrCreateSyncRun(params?.triggeredBy ?? "processNext");
|
|
2138
|
+
runStartedAt = runKey.runStartedAt;
|
|
2139
|
+
}
|
|
2140
|
+
await this.postgresClient.createObjectRuns(accountId, runStartedAt, [resourceName]);
|
|
2141
|
+
const objRun = await this.postgresClient.getObjectRun(accountId, runStartedAt, resourceName);
|
|
2142
|
+
if (objRun?.status === "complete" || objRun?.status === "error") {
|
|
2115
2143
|
return {
|
|
2116
2144
|
processed: 0,
|
|
2117
|
-
hasMore:
|
|
2145
|
+
hasMore: false,
|
|
2118
2146
|
runStartedAt
|
|
2119
2147
|
};
|
|
2120
2148
|
}
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2149
|
+
if (objRun?.status === "pending") {
|
|
2150
|
+
const started = await this.postgresClient.tryStartObjectSync(
|
|
2151
|
+
accountId,
|
|
2152
|
+
runStartedAt,
|
|
2153
|
+
resourceName
|
|
2154
|
+
);
|
|
2155
|
+
if (!started) {
|
|
2156
|
+
return {
|
|
2157
|
+
processed: 0,
|
|
2158
|
+
hasMore: true,
|
|
2159
|
+
runStartedAt
|
|
2160
|
+
};
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
let cursor = null;
|
|
2164
|
+
if (!params?.created) {
|
|
2165
|
+
const lastCursor = await this.postgresClient.getLastCursorBeforeRun(
|
|
2166
|
+
accountId,
|
|
2167
|
+
resourceName,
|
|
2168
|
+
runStartedAt
|
|
2169
|
+
);
|
|
2128
2170
|
cursor = lastCursor ?? null;
|
|
2129
2171
|
}
|
|
2172
|
+
const result = await this.fetchOnePage(
|
|
2173
|
+
object,
|
|
2174
|
+
accountId,
|
|
2175
|
+
resourceName,
|
|
2176
|
+
runStartedAt,
|
|
2177
|
+
cursor,
|
|
2178
|
+
objRun?.pageCursor ?? null,
|
|
2179
|
+
params
|
|
2180
|
+
);
|
|
2181
|
+
return result;
|
|
2182
|
+
} catch (error) {
|
|
2183
|
+
throw this.appendMigrationHint(error);
|
|
2130
2184
|
}
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2185
|
+
}
|
|
2186
|
+
appendMigrationHint(error) {
|
|
2187
|
+
const hint = "Error occurred. Make sure you are up to date with DB migrations which can sometimes help with this. Details:";
|
|
2188
|
+
const withHint = (message) => message.includes(hint) ? message : `${hint}
|
|
2189
|
+
${message}`;
|
|
2190
|
+
if (error instanceof Error) {
|
|
2191
|
+
const { stack } = error;
|
|
2192
|
+
error.message = withHint(error.message);
|
|
2193
|
+
if (stack) error.stack = stack;
|
|
2194
|
+
return error;
|
|
2195
|
+
}
|
|
2196
|
+
return new Error(withHint(String(error)));
|
|
2140
2197
|
}
|
|
2141
2198
|
/**
|
|
2142
2199
|
* Get the database resource name for a SyncObject type
|
|
@@ -2168,7 +2225,7 @@ var StripeSync = class {
|
|
|
2168
2225
|
* Uses resourceRegistry for DRY list/upsert operations.
|
|
2169
2226
|
* Uses the observable sync system for tracking progress.
|
|
2170
2227
|
*/
|
|
2171
|
-
async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, params) {
|
|
2228
|
+
async fetchOnePage(object, accountId, resourceName, runStartedAt, cursor, pageCursor, params) {
|
|
2172
2229
|
const limit = 100;
|
|
2173
2230
|
if (object === "payment_method" || object === "tax_id") {
|
|
2174
2231
|
this.config.logger?.warn(`processNext for ${object} requires customer context`);
|
|
@@ -2196,7 +2253,16 @@ var StripeSync = class {
|
|
|
2196
2253
|
listParams.created = created;
|
|
2197
2254
|
}
|
|
2198
2255
|
}
|
|
2256
|
+
if (pageCursor) {
|
|
2257
|
+
listParams.starting_after = pageCursor;
|
|
2258
|
+
}
|
|
2199
2259
|
const response = await config.listFn(listParams);
|
|
2260
|
+
if (response.data.length === 0 && response.has_more) {
|
|
2261
|
+
const message = `Stripe returned has_more=true with empty page for ${resourceName}. Aborting to avoid infinite loop.`;
|
|
2262
|
+
this.config.logger?.warn(message);
|
|
2263
|
+
await this.postgresClient.failObjectSync(accountId, runStartedAt, resourceName, message);
|
|
2264
|
+
return { processed: 0, hasMore: false, runStartedAt };
|
|
2265
|
+
}
|
|
2200
2266
|
if (response.data.length > 0) {
|
|
2201
2267
|
this.config.logger?.info(`processNext: upserting ${response.data.length} ${resourceName}`);
|
|
2202
2268
|
await config.upsertFn(response.data, accountId, params?.backfillRelatedEntities);
|
|
@@ -2217,6 +2283,15 @@ var StripeSync = class {
|
|
|
2217
2283
|
String(maxCreated)
|
|
2218
2284
|
);
|
|
2219
2285
|
}
|
|
2286
|
+
const lastId = response.data[response.data.length - 1].id;
|
|
2287
|
+
if (response.has_more) {
|
|
2288
|
+
await this.postgresClient.updateObjectPageCursor(
|
|
2289
|
+
accountId,
|
|
2290
|
+
runStartedAt,
|
|
2291
|
+
resourceName,
|
|
2292
|
+
lastId
|
|
2293
|
+
);
|
|
2294
|
+
}
|
|
2220
2295
|
}
|
|
2221
2296
|
if (!response.has_more) {
|
|
2222
2297
|
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
package/dist/index.d.cts
CHANGED
|
@@ -157,12 +157,21 @@ declare class PostgresClient {
|
|
|
157
157
|
status: string;
|
|
158
158
|
processedCount: number;
|
|
159
159
|
cursor: string | null;
|
|
160
|
+
pageCursor: string | null;
|
|
160
161
|
} | null>;
|
|
161
162
|
/**
|
|
162
163
|
* Update progress for an object sync.
|
|
163
164
|
* Also touches updated_at for stale detection.
|
|
164
165
|
*/
|
|
165
166
|
incrementObjectProgress(accountId: string, runStartedAt: Date, object: string, count: number): Promise<void>;
|
|
167
|
+
/**
|
|
168
|
+
* Update the pagination page_cursor used for backfills using Stripe list calls.
|
|
169
|
+
*/
|
|
170
|
+
updateObjectPageCursor(accountId: string, runStartedAt: Date, object: string, pageCursor: string | null): Promise<void>;
|
|
171
|
+
/**
|
|
172
|
+
* Clear the pagination page_cursor for an object sync.
|
|
173
|
+
*/
|
|
174
|
+
clearObjectPageCursor(accountId: string, runStartedAt: Date, object: string): Promise<void>;
|
|
166
175
|
/**
|
|
167
176
|
* Update the cursor for an object sync.
|
|
168
177
|
* Only updates if the new cursor is higher than the existing one (cursors should never decrease).
|
|
@@ -172,15 +181,20 @@ declare class PostgresClient {
|
|
|
172
181
|
updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
|
|
173
182
|
/**
|
|
174
183
|
* Get the highest cursor from previous syncs for an object type.
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
184
|
+
* Uses only completed object runs.
|
|
185
|
+
* - During the initial backfill we page through history, but we also update the cursor as we go.
|
|
186
|
+
* If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
|
|
187
|
+
* too early and only ever fetch the newest page (breaking the historical backfill).
|
|
178
188
|
*
|
|
179
189
|
* Handles two cursor formats:
|
|
180
190
|
* - Numeric: compared as bigint for correct ordering
|
|
181
191
|
* - Composite cursors: compared as strings with COLLATE "C"
|
|
182
192
|
*/
|
|
183
193
|
getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
|
|
194
|
+
/**
|
|
195
|
+
* Get the highest cursor from previous syncs for an object type, excluding the current run.
|
|
196
|
+
*/
|
|
197
|
+
getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
|
|
184
198
|
/**
|
|
185
199
|
* Delete all sync runs and object runs for an account.
|
|
186
200
|
* Useful for testing or resetting sync state.
|
|
@@ -587,6 +601,7 @@ declare class StripeSync {
|
|
|
587
601
|
* ```
|
|
588
602
|
*/
|
|
589
603
|
processNext(object: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>, params?: ProcessNextParams): Promise<ProcessNextResult>;
|
|
604
|
+
private appendMigrationHint;
|
|
590
605
|
/**
|
|
591
606
|
* Get the database resource name for a SyncObject type
|
|
592
607
|
*/
|