stripe-experiment-sync 1.0.9-beta.1765909347 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-2CYYWBTD.js → chunk-62FKHVHJ.js} +27 -486
- package/dist/{chunk-SI3VFP3M.js → chunk-AHNO3EMD.js} +36 -9
- package/dist/{chunk-DBJCCGXP.js → chunk-FZQ4B7VZ.js} +12 -29
- package/dist/{chunk-3W3CERIG.js → chunk-VEEV6P4R.js} +1 -3
- package/dist/cli/index.cjs +82 -527
- package/dist/cli/index.js +12 -8
- package/dist/cli/lib.cjs +74 -523
- package/dist/cli/lib.d.cts +1 -2
- package/dist/cli/lib.d.ts +1 -2
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +27 -490
- package/dist/index.d.cts +4 -107
- package/dist/index.d.ts +4 -107
- package/dist/index.js +2 -6
- package/dist/supabase/index.cjs +36 -11
- package/dist/supabase/index.d.cts +4 -2
- package/dist/supabase/index.d.ts +4 -2
- package/dist/supabase/index.js +2 -2
- package/package.json +1 -3
- package/dist/migrations/0059_sigma_subscription_item_change_events_v2_beta.sql +0 -61
- package/dist/migrations/0060_sigma_exchange_rates_from_usd.sql +0 -38
package/dist/cli/index.cjs
CHANGED
|
@@ -33,7 +33,7 @@ var import_commander = require("commander");
|
|
|
33
33
|
// package.json
|
|
34
34
|
var package_default = {
|
|
35
35
|
name: "stripe-experiment-sync",
|
|
36
|
-
version: "1.0.
|
|
36
|
+
version: "1.0.10",
|
|
37
37
|
private: false,
|
|
38
38
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
39
39
|
type: "module",
|
|
@@ -73,7 +73,6 @@ var package_default = {
|
|
|
73
73
|
dotenv: "^16.4.7",
|
|
74
74
|
express: "^4.18.2",
|
|
75
75
|
inquirer: "^12.3.0",
|
|
76
|
-
papaparse: "5.4.1",
|
|
77
76
|
pg: "^8.16.3",
|
|
78
77
|
"pg-node-migrations": "0.0.8",
|
|
79
78
|
stripe: "^17.7.0",
|
|
@@ -85,7 +84,6 @@ var package_default = {
|
|
|
85
84
|
"@types/express": "^4.17.21",
|
|
86
85
|
"@types/inquirer": "^9.0.7",
|
|
87
86
|
"@types/node": "^24.10.1",
|
|
88
|
-
"@types/papaparse": "5.3.16",
|
|
89
87
|
"@types/pg": "^8.15.5",
|
|
90
88
|
"@types/ws": "^8.5.13",
|
|
91
89
|
"@types/yesql": "^4.1.4",
|
|
@@ -129,7 +127,6 @@ async function loadConfig(options) {
|
|
|
129
127
|
config.stripeApiKey = options.stripeKey || process.env.STRIPE_API_KEY || "";
|
|
130
128
|
config.ngrokAuthToken = options.ngrokToken || process.env.NGROK_AUTH_TOKEN || "";
|
|
131
129
|
config.databaseUrl = options.databaseUrl || process.env.DATABASE_URL || "";
|
|
132
|
-
config.enableSigmaSync = options.enableSigmaSync ?? (process.env.ENABLE_SIGMA_SYNC !== void 0 ? process.env.ENABLE_SIGMA_SYNC === "true" : void 0);
|
|
133
130
|
const questions = [];
|
|
134
131
|
if (!config.stripeApiKey) {
|
|
135
132
|
questions.push({
|
|
@@ -141,8 +138,8 @@ async function loadConfig(options) {
|
|
|
141
138
|
if (!input || input.trim() === "") {
|
|
142
139
|
return "Stripe API key is required";
|
|
143
140
|
}
|
|
144
|
-
if (!input.startsWith("sk_")
|
|
145
|
-
return 'Stripe API key should start with "sk_"
|
|
141
|
+
if (!input.startsWith("sk_")) {
|
|
142
|
+
return 'Stripe API key should start with "sk_"';
|
|
146
143
|
}
|
|
147
144
|
return true;
|
|
148
145
|
}
|
|
@@ -165,80 +162,23 @@ async function loadConfig(options) {
|
|
|
165
162
|
}
|
|
166
163
|
});
|
|
167
164
|
}
|
|
168
|
-
if (config.enableSigmaSync === void 0) {
|
|
169
|
-
questions.push({
|
|
170
|
-
type: "confirm",
|
|
171
|
-
name: "enableSigmaSync",
|
|
172
|
-
message: "Enable Sigma sync? (Requires Sigma access in Stripe API key)",
|
|
173
|
-
default: false
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
165
|
if (questions.length > 0) {
|
|
177
|
-
console.log(import_chalk.default.yellow("\nMissing configuration. Please provide:"));
|
|
166
|
+
console.log(import_chalk.default.yellow("\nMissing required configuration. Please provide:"));
|
|
178
167
|
const answers = await import_inquirer.default.prompt(questions);
|
|
179
168
|
Object.assign(config, answers);
|
|
180
169
|
}
|
|
181
|
-
if (config.enableSigmaSync === void 0) {
|
|
182
|
-
config.enableSigmaSync = false;
|
|
183
|
-
}
|
|
184
170
|
return config;
|
|
185
171
|
}
|
|
186
172
|
|
|
187
173
|
// src/stripeSync.ts
|
|
188
|
-
var
|
|
174
|
+
var import_stripe2 = __toESM(require("stripe"), 1);
|
|
189
175
|
var import_yesql2 = require("yesql");
|
|
190
176
|
|
|
191
177
|
// src/database/postgres.ts
|
|
192
178
|
var import_pg = __toESM(require("pg"), 1);
|
|
193
179
|
var import_yesql = require("yesql");
|
|
194
|
-
|
|
195
|
-
// src/database/QueryUtils.ts
|
|
196
|
-
var QueryUtils = class _QueryUtils {
|
|
197
|
-
constructor() {
|
|
198
|
-
}
|
|
199
|
-
static quoteIdent(name) {
|
|
200
|
-
return `"${name}"`;
|
|
201
|
-
}
|
|
202
|
-
static quotedList(names) {
|
|
203
|
-
return names.map(_QueryUtils.quoteIdent).join(", ");
|
|
204
|
-
}
|
|
205
|
-
static buildInsertParts(columns) {
|
|
206
|
-
const columnsSql = columns.map((c) => _QueryUtils.quoteIdent(c.column)).join(", ");
|
|
207
|
-
const valuesSql = columns.map((c, i) => {
|
|
208
|
-
const placeholder = `$${i + 1}`;
|
|
209
|
-
return `${placeholder}::${c.pgType}`;
|
|
210
|
-
}).join(", ");
|
|
211
|
-
const params = columns.map((c) => c.value);
|
|
212
|
-
return { columnsSql, valuesSql, params };
|
|
213
|
-
}
|
|
214
|
-
static buildRawJsonUpsertQuery(schema, table, columns, conflictTarget) {
|
|
215
|
-
const { columnsSql, valuesSql, params } = _QueryUtils.buildInsertParts(columns);
|
|
216
|
-
const conflictSql = _QueryUtils.quotedList(conflictTarget);
|
|
217
|
-
const tsParamIdx = columns.findIndex((c) => c.column === "_last_synced_at") + 1;
|
|
218
|
-
if (tsParamIdx <= 0) {
|
|
219
|
-
throw new Error("buildRawJsonUpsertQuery requires _last_synced_at column");
|
|
220
|
-
}
|
|
221
|
-
const sql3 = `
|
|
222
|
-
INSERT INTO ${_QueryUtils.quoteIdent(schema)}.${_QueryUtils.quoteIdent(table)} (${columnsSql})
|
|
223
|
-
VALUES (${valuesSql})
|
|
224
|
-
ON CONFLICT (${conflictSql})
|
|
225
|
-
DO UPDATE SET
|
|
226
|
-
"_raw_data" = EXCLUDED."_raw_data",
|
|
227
|
-
"_last_synced_at" = $${tsParamIdx},
|
|
228
|
-
"_account_id" = EXCLUDED."_account_id"
|
|
229
|
-
WHERE ${_QueryUtils.quoteIdent(table)}."_last_synced_at" IS NULL
|
|
230
|
-
OR ${_QueryUtils.quoteIdent(table)}."_last_synced_at" < $${tsParamIdx}
|
|
231
|
-
RETURNING *
|
|
232
|
-
`;
|
|
233
|
-
return { sql: sql3, params };
|
|
234
|
-
}
|
|
235
|
-
};
|
|
236
|
-
|
|
237
|
-
// src/database/postgres.ts
|
|
238
180
|
var ORDERED_STRIPE_TABLES = [
|
|
239
|
-
"exchange_rates_from_usd",
|
|
240
181
|
"subscription_items",
|
|
241
|
-
"subscription_item_change_events_v2_beta",
|
|
242
182
|
"subscriptions",
|
|
243
183
|
"subscription_schedules",
|
|
244
184
|
"checkout_session_line_items",
|
|
@@ -308,7 +248,7 @@ var PostgresClient = class {
|
|
|
308
248
|
}
|
|
309
249
|
return results.flatMap((it) => it.rows);
|
|
310
250
|
}
|
|
311
|
-
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp
|
|
251
|
+
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp) {
|
|
312
252
|
const timestamp = syncTimestamp || (/* @__PURE__ */ new Date()).toISOString();
|
|
313
253
|
if (!entries.length) return [];
|
|
314
254
|
const chunkSize = 5;
|
|
@@ -343,33 +283,20 @@ var PostgresClient = class {
|
|
|
343
283
|
const prepared = (0, import_yesql.pg)(upsertSql, { useNullForMissing: true })(cleansed);
|
|
344
284
|
queries.push(this.pool.query(prepared.text, prepared.values));
|
|
345
285
|
} else {
|
|
346
|
-
const
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
];
|
|
361
|
-
for (const c of columns) {
|
|
362
|
-
if (c.value === void 0) {
|
|
363
|
-
throw new Error(`Missing required value for ${table}.${c.column}`);
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
const { sql: upsertSql, params } = QueryUtils.buildRawJsonUpsertQuery(
|
|
367
|
-
this.config.schema,
|
|
368
|
-
table,
|
|
369
|
-
columns,
|
|
370
|
-
conflictTarget
|
|
371
|
-
);
|
|
372
|
-
queries.push(this.pool.query(upsertSql, params));
|
|
286
|
+
const rawData = JSON.stringify(entry);
|
|
287
|
+
const upsertSql = `
|
|
288
|
+
INSERT INTO "${this.config.schema}"."${table}" ("_raw_data", "_last_synced_at", "_account_id")
|
|
289
|
+
VALUES ($1::jsonb, $2, $3)
|
|
290
|
+
ON CONFLICT (id)
|
|
291
|
+
DO UPDATE SET
|
|
292
|
+
"_raw_data" = EXCLUDED."_raw_data",
|
|
293
|
+
"_last_synced_at" = $2,
|
|
294
|
+
"_account_id" = EXCLUDED."_account_id"
|
|
295
|
+
WHERE "${table}"."_last_synced_at" IS NULL
|
|
296
|
+
OR "${table}"."_last_synced_at" < $2
|
|
297
|
+
RETURNING *
|
|
298
|
+
`;
|
|
299
|
+
queries.push(this.pool.query(upsertSql, [rawData, timestamp, accountId]));
|
|
373
300
|
}
|
|
374
301
|
});
|
|
375
302
|
results.push(...await Promise.all(queries));
|
|
@@ -769,12 +696,7 @@ var PostgresClient = class {
|
|
|
769
696
|
} else {
|
|
770
697
|
await this.query(
|
|
771
698
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
772
|
-
SET cursor =
|
|
773
|
-
WHEN cursor IS NULL THEN $4
|
|
774
|
-
WHEN (cursor COLLATE "C") < ($4::text COLLATE "C") THEN $4
|
|
775
|
-
ELSE cursor
|
|
776
|
-
END,
|
|
777
|
-
updated_at = now()
|
|
699
|
+
SET cursor = $4, updated_at = now()
|
|
778
700
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
779
701
|
[accountId, runStartedAt, object, cursor]
|
|
780
702
|
);
|
|
@@ -785,17 +707,10 @@ var PostgresClient = class {
|
|
|
785
707
|
* This considers completed, error, AND running runs to ensure recovery syncs
|
|
786
708
|
* don't re-process data that was already synced before a crash.
|
|
787
709
|
* A 'running' status with a cursor means the process was killed mid-sync.
|
|
788
|
-
*
|
|
789
|
-
* Handles two cursor formats:
|
|
790
|
-
* - Numeric: compared as bigint for correct ordering
|
|
791
|
-
* - Composite cursors: compared as strings with COLLATE "C"
|
|
792
710
|
*/
|
|
793
711
|
async getLastCompletedCursor(accountId, object) {
|
|
794
712
|
const result = await this.query(
|
|
795
|
-
`SELECT
|
|
796
|
-
WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
|
|
797
|
-
ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
|
|
798
|
-
END as cursor
|
|
713
|
+
`SELECT MAX(o.cursor::bigint)::text as cursor
|
|
799
714
|
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
800
715
|
WHERE o."_account_id" = $1
|
|
801
716
|
AND o.object = $2
|
|
@@ -1080,269 +995,6 @@ function hashApiKey(apiKey) {
|
|
|
1080
995
|
return (0, import_crypto.createHash)("sha256").update(apiKey).digest("hex");
|
|
1081
996
|
}
|
|
1082
997
|
|
|
1083
|
-
// src/sigma/sigmaApi.ts
|
|
1084
|
-
var import_papaparse = __toESM(require("papaparse"), 1);
|
|
1085
|
-
var import_stripe2 = __toESM(require("stripe"), 1);
|
|
1086
|
-
var STRIPE_FILES_BASE = "https://files.stripe.com/v1";
|
|
1087
|
-
function sleep2(ms) {
|
|
1088
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1089
|
-
}
|
|
1090
|
-
function parseCsvObjects(csv) {
|
|
1091
|
-
const input = csv.replace(/^\uFEFF/, "");
|
|
1092
|
-
const parsed = import_papaparse.default.parse(input, {
|
|
1093
|
-
header: true,
|
|
1094
|
-
skipEmptyLines: "greedy"
|
|
1095
|
-
});
|
|
1096
|
-
if (parsed.errors.length > 0) {
|
|
1097
|
-
throw new Error(`Failed to parse Sigma CSV: ${parsed.errors[0]?.message ?? "unknown error"}`);
|
|
1098
|
-
}
|
|
1099
|
-
return parsed.data.filter((row) => row && Object.keys(row).length > 0).map(
|
|
1100
|
-
(row) => Object.fromEntries(
|
|
1101
|
-
Object.entries(row).map(([k, v]) => [k, v == null || v === "" ? null : String(v)])
|
|
1102
|
-
)
|
|
1103
|
-
);
|
|
1104
|
-
}
|
|
1105
|
-
function normalizeSigmaTimestampToIso(value) {
|
|
1106
|
-
const v = value.trim();
|
|
1107
|
-
if (!v) return null;
|
|
1108
|
-
const hasExplicitTz = /z$|[+-]\d{2}:?\d{2}$/i.test(v);
|
|
1109
|
-
const isoish = v.includes("T") ? v : v.replace(" ", "T");
|
|
1110
|
-
const candidate = hasExplicitTz ? isoish : `${isoish}Z`;
|
|
1111
|
-
const d = new Date(candidate);
|
|
1112
|
-
if (Number.isNaN(d.getTime())) return null;
|
|
1113
|
-
return d.toISOString();
|
|
1114
|
-
}
|
|
1115
|
-
async function fetchStripeText(url, apiKey, options) {
|
|
1116
|
-
const res = await fetch(url, {
|
|
1117
|
-
...options,
|
|
1118
|
-
headers: {
|
|
1119
|
-
...options.headers ?? {},
|
|
1120
|
-
Authorization: `Bearer ${apiKey}`
|
|
1121
|
-
}
|
|
1122
|
-
});
|
|
1123
|
-
const text = await res.text();
|
|
1124
|
-
if (!res.ok) {
|
|
1125
|
-
throw new Error(`Sigma file download error (${res.status}) for ${url}: ${text}`);
|
|
1126
|
-
}
|
|
1127
|
-
return text;
|
|
1128
|
-
}
|
|
1129
|
-
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1130
|
-
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1131
|
-
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1132
|
-
const stripe = new import_stripe2.default(params.apiKey);
|
|
1133
|
-
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1134
|
-
sql: params.sql
|
|
1135
|
-
});
|
|
1136
|
-
const queryRunId = created.id;
|
|
1137
|
-
const start = Date.now();
|
|
1138
|
-
let current = created;
|
|
1139
|
-
while (current.status === "running") {
|
|
1140
|
-
if (Date.now() - start > pollTimeoutMs) {
|
|
1141
|
-
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1142
|
-
}
|
|
1143
|
-
await sleep2(pollIntervalMs);
|
|
1144
|
-
current = await stripe.rawRequest(
|
|
1145
|
-
"GET",
|
|
1146
|
-
`/v1/sigma/query_runs/${queryRunId}`,
|
|
1147
|
-
{}
|
|
1148
|
-
);
|
|
1149
|
-
}
|
|
1150
|
-
if (current.status !== "succeeded") {
|
|
1151
|
-
throw new Error(
|
|
1152
|
-
`Sigma query run did not succeed (status=${current.status}) id=${queryRunId} error=${JSON.stringify(
|
|
1153
|
-
current.error
|
|
1154
|
-
)}`
|
|
1155
|
-
);
|
|
1156
|
-
}
|
|
1157
|
-
const fileId = current.result?.file;
|
|
1158
|
-
if (!fileId) {
|
|
1159
|
-
throw new Error(`Sigma query run succeeded but result.file is missing (id=${queryRunId})`);
|
|
1160
|
-
}
|
|
1161
|
-
const csv = await fetchStripeText(
|
|
1162
|
-
`${STRIPE_FILES_BASE}/files/${fileId}/contents`,
|
|
1163
|
-
params.apiKey,
|
|
1164
|
-
{ method: "GET" }
|
|
1165
|
-
);
|
|
1166
|
-
return { queryRunId, fileId, csv };
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
|
-
// src/sigma/sigmaIngestionConfigs.ts
|
|
1170
|
-
var SIGMA_INGESTION_CONFIGS = {
|
|
1171
|
-
subscription_item_change_events_v2_beta: {
|
|
1172
|
-
sigmaTable: "subscription_item_change_events_v2_beta",
|
|
1173
|
-
destinationTable: "subscription_item_change_events_v2_beta",
|
|
1174
|
-
pageSize: 1e4,
|
|
1175
|
-
cursor: {
|
|
1176
|
-
version: 1,
|
|
1177
|
-
columns: [
|
|
1178
|
-
{ column: "event_timestamp", type: "timestamp" },
|
|
1179
|
-
{ column: "event_type", type: "string" },
|
|
1180
|
-
{ column: "subscription_item_id", type: "string" }
|
|
1181
|
-
]
|
|
1182
|
-
},
|
|
1183
|
-
upsert: {
|
|
1184
|
-
conflictTarget: ["_account_id", "event_timestamp", "event_type", "subscription_item_id"],
|
|
1185
|
-
extraColumns: [
|
|
1186
|
-
{ column: "event_timestamp", pgType: "timestamptz", entryKey: "event_timestamp" },
|
|
1187
|
-
{ column: "event_type", pgType: "text", entryKey: "event_type" },
|
|
1188
|
-
{ column: "subscription_item_id", pgType: "text", entryKey: "subscription_item_id" }
|
|
1189
|
-
]
|
|
1190
|
-
}
|
|
1191
|
-
},
|
|
1192
|
-
exchange_rates_from_usd: {
|
|
1193
|
-
sigmaTable: "exchange_rates_from_usd",
|
|
1194
|
-
destinationTable: "exchange_rates_from_usd",
|
|
1195
|
-
pageSize: 1e4,
|
|
1196
|
-
cursor: {
|
|
1197
|
-
version: 1,
|
|
1198
|
-
columns: [
|
|
1199
|
-
{ column: "date", type: "string" },
|
|
1200
|
-
{ column: "sell_currency", type: "string" }
|
|
1201
|
-
]
|
|
1202
|
-
},
|
|
1203
|
-
upsert: {
|
|
1204
|
-
conflictTarget: ["_account_id", "date", "sell_currency"],
|
|
1205
|
-
extraColumns: [
|
|
1206
|
-
{ column: "date", pgType: "date", entryKey: "date" },
|
|
1207
|
-
{ column: "sell_currency", pgType: "text", entryKey: "sell_currency" }
|
|
1208
|
-
]
|
|
1209
|
-
}
|
|
1210
|
-
}
|
|
1211
|
-
};
|
|
1212
|
-
|
|
1213
|
-
// src/sigma/sigmaIngestion.ts
|
|
1214
|
-
var SIGMA_CURSOR_DELIM = "";
|
|
1215
|
-
function escapeSigmaSqlStringLiteral(value) {
|
|
1216
|
-
return value.replace(/'/g, "''");
|
|
1217
|
-
}
|
|
1218
|
-
function formatSigmaTimestampForSqlLiteral(date) {
|
|
1219
|
-
return date.toISOString().replace("T", " ").replace("Z", "");
|
|
1220
|
-
}
|
|
1221
|
-
function decodeSigmaCursorValues(spec, cursor) {
|
|
1222
|
-
const prefix = `v${spec.version}${SIGMA_CURSOR_DELIM}`;
|
|
1223
|
-
if (!cursor.startsWith(prefix)) {
|
|
1224
|
-
throw new Error(
|
|
1225
|
-
`Unrecognized Sigma cursor format (expected prefix ${JSON.stringify(prefix)}): ${cursor}`
|
|
1226
|
-
);
|
|
1227
|
-
}
|
|
1228
|
-
const parts = cursor.split(SIGMA_CURSOR_DELIM);
|
|
1229
|
-
const expected = 1 + spec.columns.length;
|
|
1230
|
-
if (parts.length !== expected) {
|
|
1231
|
-
throw new Error(`Malformed Sigma cursor: expected ${expected} parts, got ${parts.length}`);
|
|
1232
|
-
}
|
|
1233
|
-
return parts.slice(1);
|
|
1234
|
-
}
|
|
1235
|
-
function encodeSigmaCursor(spec, values) {
|
|
1236
|
-
if (values.length !== spec.columns.length) {
|
|
1237
|
-
throw new Error(
|
|
1238
|
-
`Cannot encode Sigma cursor: expected ${spec.columns.length} values, got ${values.length}`
|
|
1239
|
-
);
|
|
1240
|
-
}
|
|
1241
|
-
for (const v of values) {
|
|
1242
|
-
if (v.includes(SIGMA_CURSOR_DELIM)) {
|
|
1243
|
-
throw new Error("Cannot encode Sigma cursor: value contains delimiter character");
|
|
1244
|
-
}
|
|
1245
|
-
}
|
|
1246
|
-
return [`v${spec.version}`, ...values].join(SIGMA_CURSOR_DELIM);
|
|
1247
|
-
}
|
|
1248
|
-
function sigmaSqlLiteralForCursorValue(spec, rawValue) {
|
|
1249
|
-
switch (spec.type) {
|
|
1250
|
-
case "timestamp": {
|
|
1251
|
-
const d = new Date(rawValue);
|
|
1252
|
-
if (Number.isNaN(d.getTime())) {
|
|
1253
|
-
throw new Error(`Invalid timestamp cursor value for ${spec.column}: ${rawValue}`);
|
|
1254
|
-
}
|
|
1255
|
-
return `timestamp '${formatSigmaTimestampForSqlLiteral(d)}'`;
|
|
1256
|
-
}
|
|
1257
|
-
case "number": {
|
|
1258
|
-
if (!/^-?\d+(\.\d+)?$/.test(rawValue)) {
|
|
1259
|
-
throw new Error(`Invalid numeric cursor value for ${spec.column}: ${rawValue}`);
|
|
1260
|
-
}
|
|
1261
|
-
return rawValue;
|
|
1262
|
-
}
|
|
1263
|
-
case "string":
|
|
1264
|
-
return `'${escapeSigmaSqlStringLiteral(rawValue)}'`;
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
function buildSigmaCursorWhereClause(spec, cursorValues) {
|
|
1268
|
-
if (cursorValues.length !== spec.columns.length) {
|
|
1269
|
-
throw new Error(
|
|
1270
|
-
`Cannot build Sigma cursor predicate: expected ${spec.columns.length} values, got ${cursorValues.length}`
|
|
1271
|
-
);
|
|
1272
|
-
}
|
|
1273
|
-
const cols = spec.columns.map((c) => c.column);
|
|
1274
|
-
const lits = spec.columns.map((c, i) => sigmaSqlLiteralForCursorValue(c, cursorValues[i] ?? ""));
|
|
1275
|
-
const ors = [];
|
|
1276
|
-
for (let i = 0; i < cols.length; i++) {
|
|
1277
|
-
const ands = [];
|
|
1278
|
-
for (let j = 0; j < i; j++) {
|
|
1279
|
-
ands.push(`${cols[j]} = ${lits[j]}`);
|
|
1280
|
-
}
|
|
1281
|
-
ands.push(`${cols[i]} > ${lits[i]}`);
|
|
1282
|
-
ors.push(`(${ands.join(" AND ")})`);
|
|
1283
|
-
}
|
|
1284
|
-
return ors.join(" OR ");
|
|
1285
|
-
}
|
|
1286
|
-
function buildSigmaQuery(config, cursor) {
|
|
1287
|
-
const select = config.select === void 0 || config.select === "*" ? "*" : config.select.join(", ");
|
|
1288
|
-
const whereParts = [];
|
|
1289
|
-
if (config.additionalWhere) {
|
|
1290
|
-
whereParts.push(`(${config.additionalWhere})`);
|
|
1291
|
-
}
|
|
1292
|
-
if (cursor) {
|
|
1293
|
-
const values = decodeSigmaCursorValues(config.cursor, cursor);
|
|
1294
|
-
const predicate = buildSigmaCursorWhereClause(config.cursor, values);
|
|
1295
|
-
whereParts.push(`(${predicate})`);
|
|
1296
|
-
}
|
|
1297
|
-
const whereClause = whereParts.length > 0 ? `WHERE ${whereParts.join(" AND ")}` : "";
|
|
1298
|
-
const orderBy = config.cursor.columns.map((c) => c.column).join(", ");
|
|
1299
|
-
return [
|
|
1300
|
-
`SELECT ${select} FROM ${config.sigmaTable}`,
|
|
1301
|
-
whereClause,
|
|
1302
|
-
`ORDER BY ${orderBy} ASC`,
|
|
1303
|
-
`LIMIT ${config.pageSize}`
|
|
1304
|
-
].filter(Boolean).join(" ");
|
|
1305
|
-
}
|
|
1306
|
-
function defaultSigmaRowToEntry(config, row) {
|
|
1307
|
-
const out = { ...row };
|
|
1308
|
-
for (const col of config.cursor.columns) {
|
|
1309
|
-
const raw = row[col.column];
|
|
1310
|
-
if (raw == null) {
|
|
1311
|
-
throw new Error(`Sigma row missing required cursor column: ${col.column}`);
|
|
1312
|
-
}
|
|
1313
|
-
if (col.type === "timestamp") {
|
|
1314
|
-
const normalized = normalizeSigmaTimestampToIso(raw);
|
|
1315
|
-
if (!normalized) {
|
|
1316
|
-
throw new Error(`Sigma row has invalid timestamp for ${col.column}: ${raw}`);
|
|
1317
|
-
}
|
|
1318
|
-
out[col.column] = normalized;
|
|
1319
|
-
} else if (col.type === "string") {
|
|
1320
|
-
const v = raw.trim();
|
|
1321
|
-
if (!v) {
|
|
1322
|
-
throw new Error(`Sigma row has empty string for required cursor column: ${col.column}`);
|
|
1323
|
-
}
|
|
1324
|
-
out[col.column] = v;
|
|
1325
|
-
} else {
|
|
1326
|
-
const v = raw.trim();
|
|
1327
|
-
if (!v) {
|
|
1328
|
-
throw new Error(`Sigma row has empty value for required cursor column: ${col.column}`);
|
|
1329
|
-
}
|
|
1330
|
-
out[col.column] = v;
|
|
1331
|
-
}
|
|
1332
|
-
}
|
|
1333
|
-
return out;
|
|
1334
|
-
}
|
|
1335
|
-
function sigmaCursorFromEntry(config, entry) {
|
|
1336
|
-
const values = config.cursor.columns.map((c) => {
|
|
1337
|
-
const raw = entry[c.column];
|
|
1338
|
-
if (raw == null) {
|
|
1339
|
-
throw new Error(`Cannot build cursor: entry missing ${c.column}`);
|
|
1340
|
-
}
|
|
1341
|
-
return String(raw);
|
|
1342
|
-
});
|
|
1343
|
-
return encodeSigmaCursor(config.cursor, values);
|
|
1344
|
-
}
|
|
1345
|
-
|
|
1346
998
|
// src/stripeSync.ts
|
|
1347
999
|
function getUniqueIds(entries, key) {
|
|
1348
1000
|
const set = new Set(
|
|
@@ -1353,7 +1005,7 @@ function getUniqueIds(entries, key) {
|
|
|
1353
1005
|
var StripeSync = class {
|
|
1354
1006
|
constructor(config) {
|
|
1355
1007
|
this.config = config;
|
|
1356
|
-
const baseStripe = new
|
|
1008
|
+
const baseStripe = new import_stripe2.default(config.stripeSecretKey, {
|
|
1357
1009
|
// https://github.com/stripe/stripe-node#configuration
|
|
1358
1010
|
// @ts-ignore
|
|
1359
1011
|
apiVersion: config.stripeApiVersion,
|
|
@@ -1754,17 +1406,6 @@ var StripeSync = class {
|
|
|
1754
1406
|
listFn: (p) => this.stripe.checkout.sessions.list(p),
|
|
1755
1407
|
upsertFn: (items, id) => this.upsertCheckoutSessions(items, id),
|
|
1756
1408
|
supportsCreatedFilter: true
|
|
1757
|
-
},
|
|
1758
|
-
// Sigma-backed resources
|
|
1759
|
-
subscription_item_change_events_v2_beta: {
|
|
1760
|
-
order: 18,
|
|
1761
|
-
supportsCreatedFilter: false,
|
|
1762
|
-
sigma: SIGMA_INGESTION_CONFIGS.subscription_item_change_events_v2_beta
|
|
1763
|
-
},
|
|
1764
|
-
exchange_rates_from_usd: {
|
|
1765
|
-
order: 19,
|
|
1766
|
-
supportsCreatedFilter: false,
|
|
1767
|
-
sigma: SIGMA_INGESTION_CONFIGS.exchange_rates_from_usd
|
|
1768
1409
|
}
|
|
1769
1410
|
};
|
|
1770
1411
|
async processEvent(event) {
|
|
@@ -1797,13 +1438,7 @@ var StripeSync = class {
|
|
|
1797
1438
|
* Order is determined by the `order` field in resourceRegistry.
|
|
1798
1439
|
*/
|
|
1799
1440
|
getSupportedSyncObjects() {
|
|
1800
|
-
|
|
1801
|
-
if (!this.config.enableSigmaSync) {
|
|
1802
|
-
return all.filter(
|
|
1803
|
-
(o) => o !== "subscription_item_change_events_v2_beta" && o !== "exchange_rates_from_usd"
|
|
1804
|
-
);
|
|
1805
|
-
}
|
|
1806
|
-
return all;
|
|
1441
|
+
return Object.entries(this.resourceRegistry).sort(([, a], [, b]) => a.order - b.order).map(([key]) => key);
|
|
1807
1442
|
}
|
|
1808
1443
|
// Event handler methods
|
|
1809
1444
|
async handleChargeEvent(event, accountId) {
|
|
@@ -1882,7 +1517,7 @@ var StripeSync = class {
|
|
|
1882
1517
|
);
|
|
1883
1518
|
await this.upsertProducts([product], accountId, this.getSyncTimestamp(event, refetched));
|
|
1884
1519
|
} catch (err) {
|
|
1885
|
-
if (err instanceof
|
|
1520
|
+
if (err instanceof import_stripe2.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1886
1521
|
const product = event.data.object;
|
|
1887
1522
|
await this.deleteProduct(product.id);
|
|
1888
1523
|
} else {
|
|
@@ -1902,7 +1537,7 @@ var StripeSync = class {
|
|
|
1902
1537
|
);
|
|
1903
1538
|
await this.upsertPrices([price], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1904
1539
|
} catch (err) {
|
|
1905
|
-
if (err instanceof
|
|
1540
|
+
if (err instanceof import_stripe2.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1906
1541
|
const price = event.data.object;
|
|
1907
1542
|
await this.deletePrice(price.id);
|
|
1908
1543
|
} else {
|
|
@@ -1922,7 +1557,7 @@ var StripeSync = class {
|
|
|
1922
1557
|
);
|
|
1923
1558
|
await this.upsertPlans([plan], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1924
1559
|
} catch (err) {
|
|
1925
|
-
if (err instanceof
|
|
1560
|
+
if (err instanceof import_stripe2.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1926
1561
|
const plan = event.data.object;
|
|
1927
1562
|
await this.deletePlan(plan.id);
|
|
1928
1563
|
} else {
|
|
@@ -2169,10 +1804,10 @@ var StripeSync = class {
|
|
|
2169
1804
|
let cursor = null;
|
|
2170
1805
|
if (!params?.created) {
|
|
2171
1806
|
if (objRun?.cursor) {
|
|
2172
|
-
cursor = objRun.cursor;
|
|
1807
|
+
cursor = parseInt(objRun.cursor);
|
|
2173
1808
|
} else {
|
|
2174
1809
|
const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
|
|
2175
|
-
cursor = lastCursor
|
|
1810
|
+
cursor = lastCursor ? parseInt(lastCursor) : null;
|
|
2176
1811
|
}
|
|
2177
1812
|
}
|
|
2178
1813
|
const result = await this.fetchOnePage(
|
|
@@ -2227,18 +1862,9 @@ var StripeSync = class {
|
|
|
2227
1862
|
throw new Error(`Unsupported object type for processNext: ${object}`);
|
|
2228
1863
|
}
|
|
2229
1864
|
try {
|
|
2230
|
-
if (config.sigma) {
|
|
2231
|
-
return await this.fetchOneSigmaPage(
|
|
2232
|
-
accountId,
|
|
2233
|
-
resourceName,
|
|
2234
|
-
runStartedAt,
|
|
2235
|
-
cursor,
|
|
2236
|
-
config.sigma
|
|
2237
|
-
);
|
|
2238
|
-
}
|
|
2239
1865
|
const listParams = { limit };
|
|
2240
1866
|
if (config.supportsCreatedFilter) {
|
|
2241
|
-
const created = params?.created ?? (cursor
|
|
1867
|
+
const created = params?.created ?? (cursor ? { gte: cursor } : void 0);
|
|
2242
1868
|
if (created) {
|
|
2243
1869
|
listParams.created = created;
|
|
2244
1870
|
}
|
|
@@ -2283,97 +1909,6 @@ var StripeSync = class {
|
|
|
2283
1909
|
throw error;
|
|
2284
1910
|
}
|
|
2285
1911
|
}
|
|
2286
|
-
async getSigmaFallbackCursorFromDestination(accountId, sigmaConfig) {
|
|
2287
|
-
const cursorCols = sigmaConfig.cursor.columns;
|
|
2288
|
-
const selectCols = cursorCols.map((c) => `"${c.column}"`).join(", ");
|
|
2289
|
-
const orderBy = cursorCols.map((c) => `"${c.column}" DESC`).join(", ");
|
|
2290
|
-
const result = await this.postgresClient.query(
|
|
2291
|
-
`SELECT ${selectCols}
|
|
2292
|
-
FROM "stripe"."${sigmaConfig.destinationTable}"
|
|
2293
|
-
WHERE "_account_id" = $1
|
|
2294
|
-
ORDER BY ${orderBy}
|
|
2295
|
-
LIMIT 1`,
|
|
2296
|
-
[accountId]
|
|
2297
|
-
);
|
|
2298
|
-
if (result.rows.length === 0) return null;
|
|
2299
|
-
const row = result.rows[0];
|
|
2300
|
-
const entryForCursor = {};
|
|
2301
|
-
for (const c of cursorCols) {
|
|
2302
|
-
const v = row[c.column];
|
|
2303
|
-
if (v == null) {
|
|
2304
|
-
throw new Error(
|
|
2305
|
-
`Sigma fallback cursor query returned null for ${sigmaConfig.destinationTable}.${c.column}`
|
|
2306
|
-
);
|
|
2307
|
-
}
|
|
2308
|
-
if (c.type === "timestamp") {
|
|
2309
|
-
const d = v instanceof Date ? v : new Date(String(v));
|
|
2310
|
-
if (Number.isNaN(d.getTime())) {
|
|
2311
|
-
throw new Error(
|
|
2312
|
-
`Sigma fallback cursor query returned invalid timestamp for ${sigmaConfig.destinationTable}.${c.column}: ${String(
|
|
2313
|
-
v
|
|
2314
|
-
)}`
|
|
2315
|
-
);
|
|
2316
|
-
}
|
|
2317
|
-
entryForCursor[c.column] = d.toISOString();
|
|
2318
|
-
} else {
|
|
2319
|
-
entryForCursor[c.column] = String(v);
|
|
2320
|
-
}
|
|
2321
|
-
}
|
|
2322
|
-
return sigmaCursorFromEntry(sigmaConfig, entryForCursor);
|
|
2323
|
-
}
|
|
2324
|
-
async fetchOneSigmaPage(accountId, resourceName, runStartedAt, cursor, sigmaConfig) {
|
|
2325
|
-
if (!this.config.stripeSecretKey) {
|
|
2326
|
-
throw new Error("Sigma sync requested but stripeSecretKey is not configured.");
|
|
2327
|
-
}
|
|
2328
|
-
if (resourceName !== sigmaConfig.destinationTable) {
|
|
2329
|
-
throw new Error(
|
|
2330
|
-
`Sigma sync config mismatch: resourceName=${resourceName} destinationTable=${sigmaConfig.destinationTable}`
|
|
2331
|
-
);
|
|
2332
|
-
}
|
|
2333
|
-
const effectiveCursor = cursor ?? await this.getSigmaFallbackCursorFromDestination(accountId, sigmaConfig);
|
|
2334
|
-
const sigmaSql = buildSigmaQuery(sigmaConfig, effectiveCursor);
|
|
2335
|
-
this.config.logger?.info(
|
|
2336
|
-
{ object: resourceName, pageSize: sigmaConfig.pageSize, hasCursor: Boolean(effectiveCursor) },
|
|
2337
|
-
"Sigma sync: running query"
|
|
2338
|
-
);
|
|
2339
|
-
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
2340
|
-
apiKey: this.config.stripeSecretKey,
|
|
2341
|
-
sql: sigmaSql,
|
|
2342
|
-
logger: this.config.logger
|
|
2343
|
-
});
|
|
2344
|
-
const rows = parseCsvObjects(csv);
|
|
2345
|
-
if (rows.length === 0) {
|
|
2346
|
-
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2347
|
-
return { processed: 0, hasMore: false, runStartedAt };
|
|
2348
|
-
}
|
|
2349
|
-
const entries = rows.map(
|
|
2350
|
-
(row) => defaultSigmaRowToEntry(sigmaConfig, row)
|
|
2351
|
-
);
|
|
2352
|
-
this.config.logger?.info(
|
|
2353
|
-
{ object: resourceName, rows: entries.length, queryRunId, fileId },
|
|
2354
|
-
"Sigma sync: upserting rows"
|
|
2355
|
-
);
|
|
2356
|
-
await this.postgresClient.upsertManyWithTimestampProtection(
|
|
2357
|
-
entries,
|
|
2358
|
-
resourceName,
|
|
2359
|
-
accountId,
|
|
2360
|
-
void 0,
|
|
2361
|
-
sigmaConfig.upsert
|
|
2362
|
-
);
|
|
2363
|
-
await this.postgresClient.incrementObjectProgress(
|
|
2364
|
-
accountId,
|
|
2365
|
-
runStartedAt,
|
|
2366
|
-
resourceName,
|
|
2367
|
-
entries.length
|
|
2368
|
-
);
|
|
2369
|
-
const newCursor = sigmaCursorFromEntry(sigmaConfig, entries[entries.length - 1]);
|
|
2370
|
-
await this.postgresClient.updateObjectCursor(accountId, runStartedAt, resourceName, newCursor);
|
|
2371
|
-
const hasMore = rows.length === sigmaConfig.pageSize;
|
|
2372
|
-
if (!hasMore) {
|
|
2373
|
-
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2374
|
-
}
|
|
2375
|
-
return { processed: entries.length, hasMore, runStartedAt };
|
|
2376
|
-
}
|
|
2377
1912
|
/**
|
|
2378
1913
|
* Process all pages for all (or specified) object types until complete.
|
|
2379
1914
|
*
|
|
@@ -2502,12 +2037,6 @@ var StripeSync = class {
|
|
|
2502
2037
|
case "checkout_sessions":
|
|
2503
2038
|
results.checkoutSessions = result;
|
|
2504
2039
|
break;
|
|
2505
|
-
case "subscription_item_change_events_v2_beta":
|
|
2506
|
-
results.subscriptionItemChangeEventsV2Beta = result;
|
|
2507
|
-
break;
|
|
2508
|
-
case "exchange_rates_from_usd":
|
|
2509
|
-
results.exchangeRatesFromUsd = result;
|
|
2510
|
-
break;
|
|
2511
2040
|
}
|
|
2512
2041
|
}
|
|
2513
2042
|
}
|
|
@@ -4108,7 +3637,7 @@ var stripe_setup_default = "import { StripeSync, runMigrations } from 'npm:strip
|
|
|
4108
3637
|
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
4109
3638
|
|
|
4110
3639
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-worker.ts
|
|
4111
|
-
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron
|
|
3640
|
+
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
4112
3641
|
|
|
4113
3642
|
// src/supabase/edge-function-code.ts
|
|
4114
3643
|
var setupFunctionCode = stripe_setup_default;
|
|
@@ -4116,7 +3645,7 @@ var webhookFunctionCode = stripe_webhook_default;
|
|
|
4116
3645
|
var workerFunctionCode = stripe_worker_default;
|
|
4117
3646
|
|
|
4118
3647
|
// src/supabase/supabase.ts
|
|
4119
|
-
var
|
|
3648
|
+
var import_stripe3 = __toESM(require("stripe"), 1);
|
|
4120
3649
|
var STRIPE_SCHEMA_COMMENT_PREFIX = "stripe-sync";
|
|
4121
3650
|
var INSTALLATION_STARTED_SUFFIX = "installation:started";
|
|
4122
3651
|
var INSTALLATION_ERROR_SUFFIX = "installation:error";
|
|
@@ -4182,8 +3711,29 @@ var SupabaseSetupClient = class {
|
|
|
4182
3711
|
}
|
|
4183
3712
|
/**
|
|
4184
3713
|
* Setup pg_cron job to invoke worker function
|
|
3714
|
+
* @param intervalSeconds - How often to run the worker (default: 60 seconds)
|
|
4185
3715
|
*/
|
|
4186
|
-
async setupPgCronJob() {
|
|
3716
|
+
async setupPgCronJob(intervalSeconds = 60) {
|
|
3717
|
+
if (!Number.isInteger(intervalSeconds) || intervalSeconds < 1) {
|
|
3718
|
+
throw new Error(`Invalid interval: ${intervalSeconds}. Must be a positive integer.`);
|
|
3719
|
+
}
|
|
3720
|
+
let schedule;
|
|
3721
|
+
if (intervalSeconds < 60) {
|
|
3722
|
+
schedule = `${intervalSeconds} seconds`;
|
|
3723
|
+
} else if (intervalSeconds % 60 === 0) {
|
|
3724
|
+
const minutes = intervalSeconds / 60;
|
|
3725
|
+
if (minutes < 60) {
|
|
3726
|
+
schedule = `*/${minutes} * * * *`;
|
|
3727
|
+
} else {
|
|
3728
|
+
throw new Error(
|
|
3729
|
+
`Invalid interval: ${intervalSeconds}. Intervals >= 3600 seconds (1 hour) are not supported. Use a value between 1-3599 seconds.`
|
|
3730
|
+
);
|
|
3731
|
+
}
|
|
3732
|
+
} else {
|
|
3733
|
+
throw new Error(
|
|
3734
|
+
`Invalid interval: ${intervalSeconds}. Must be either 1-59 seconds or a multiple of 60 (e.g., 60, 120, 180).`
|
|
3735
|
+
);
|
|
3736
|
+
}
|
|
4187
3737
|
const serviceRoleKey = await this.getServiceRoleKey();
|
|
4188
3738
|
const escapedServiceRoleKey = serviceRoleKey.replace(/'/g, "''");
|
|
4189
3739
|
const sql3 = `
|
|
@@ -4211,11 +3761,11 @@ var SupabaseSetupClient = class {
|
|
|
4211
3761
|
SELECT 1 FROM cron.job WHERE jobname = 'stripe-sync-scheduler'
|
|
4212
3762
|
);
|
|
4213
3763
|
|
|
4214
|
-
-- Create job to invoke worker
|
|
3764
|
+
-- Create job to invoke worker at configured interval
|
|
4215
3765
|
-- Worker reads from pgmq, enqueues objects if empty, and processes sync work
|
|
4216
3766
|
SELECT cron.schedule(
|
|
4217
3767
|
'stripe-sync-worker',
|
|
4218
|
-
'
|
|
3768
|
+
'${schedule}',
|
|
4219
3769
|
$$
|
|
4220
3770
|
SELECT net.http_post(
|
|
4221
3771
|
url := 'https://${this.projectRef}.${this.projectBaseUrl}/functions/v1/stripe-worker',
|
|
@@ -4376,7 +3926,7 @@ var SupabaseSetupClient = class {
|
|
|
4376
3926
|
* Removes all Edge Functions, secrets, database resources, and Stripe webhooks
|
|
4377
3927
|
*/
|
|
4378
3928
|
async uninstall(stripeSecretKey) {
|
|
4379
|
-
const stripe = new
|
|
3929
|
+
const stripe = new import_stripe3.default(stripeSecretKey, { apiVersion: "2025-02-24.acacia" });
|
|
4380
3930
|
try {
|
|
4381
3931
|
try {
|
|
4382
3932
|
const webhookResult = await this.runSQL(`
|
|
@@ -4443,7 +3993,7 @@ var SupabaseSetupClient = class {
|
|
|
4443
3993
|
`from 'npm:stripe-experiment-sync@${version}'`
|
|
4444
3994
|
);
|
|
4445
3995
|
}
|
|
4446
|
-
async install(stripeKey, packageVersion) {
|
|
3996
|
+
async install(stripeKey, packageVersion, workerIntervalSeconds) {
|
|
4447
3997
|
const trimmedStripeKey = stripeKey.trim();
|
|
4448
3998
|
if (!trimmedStripeKey.startsWith("sk_") && !trimmedStripeKey.startsWith("rk_")) {
|
|
4449
3999
|
throw new Error('Stripe key should start with "sk_" or "rk_"');
|
|
@@ -4467,7 +4017,7 @@ var SupabaseSetupClient = class {
|
|
|
4467
4017
|
if (!setupResult.success) {
|
|
4468
4018
|
throw new Error(`Setup failed: ${setupResult.error}`);
|
|
4469
4019
|
}
|
|
4470
|
-
await this.setupPgCronJob();
|
|
4020
|
+
await this.setupPgCronJob(workerIntervalSeconds);
|
|
4471
4021
|
await this.updateInstallationComment(
|
|
4472
4022
|
`${STRIPE_SCHEMA_COMMENT_PREFIX} v${package_default.version} ${INSTALLATION_INSTALLED_SUFFIX}`
|
|
4473
4023
|
);
|
|
@@ -4480,14 +4030,20 @@ var SupabaseSetupClient = class {
|
|
|
4480
4030
|
}
|
|
4481
4031
|
};
|
|
4482
4032
|
async function install(params) {
|
|
4483
|
-
const {
|
|
4033
|
+
const {
|
|
4034
|
+
supabaseAccessToken,
|
|
4035
|
+
supabaseProjectRef,
|
|
4036
|
+
stripeKey,
|
|
4037
|
+
packageVersion,
|
|
4038
|
+
workerIntervalSeconds
|
|
4039
|
+
} = params;
|
|
4484
4040
|
const client = new SupabaseSetupClient({
|
|
4485
4041
|
accessToken: supabaseAccessToken,
|
|
4486
4042
|
projectRef: supabaseProjectRef,
|
|
4487
4043
|
projectBaseUrl: params.baseProjectUrl,
|
|
4488
4044
|
managementApiBaseUrl: params.baseManagementApiUrl
|
|
4489
4045
|
});
|
|
4490
|
-
await client.install(stripeKey, packageVersion);
|
|
4046
|
+
await client.install(stripeKey, packageVersion, workerIntervalSeconds);
|
|
4491
4047
|
}
|
|
4492
4048
|
async function uninstall(params) {
|
|
4493
4049
|
const { supabaseAccessToken, supabaseProjectRef, stripeKey } = params;
|
|
@@ -4522,9 +4078,7 @@ var VALID_SYNC_OBJECTS = [
|
|
|
4522
4078
|
"credit_note",
|
|
4523
4079
|
"early_fraud_warning",
|
|
4524
4080
|
"refund",
|
|
4525
|
-
"checkout_sessions"
|
|
4526
|
-
"subscription_item_change_events_v2_beta",
|
|
4527
|
-
"exchange_rates_from_usd"
|
|
4081
|
+
"checkout_sessions"
|
|
4528
4082
|
];
|
|
4529
4083
|
async function backfillCommand(options, entityName) {
|
|
4530
4084
|
let stripeSync = null;
|
|
@@ -4553,8 +4107,8 @@ async function backfillCommand(options, entityName) {
|
|
|
4553
4107
|
if (!input || input.trim() === "") {
|
|
4554
4108
|
return "Stripe API key is required";
|
|
4555
4109
|
}
|
|
4556
|
-
if (!input.startsWith("sk_")
|
|
4557
|
-
return 'Stripe API key should start with "sk_"
|
|
4110
|
+
if (!input.startsWith("sk_")) {
|
|
4111
|
+
return 'Stripe API key should start with "sk_"';
|
|
4558
4112
|
}
|
|
4559
4113
|
return true;
|
|
4560
4114
|
}
|
|
@@ -4611,7 +4165,6 @@ async function backfillCommand(options, entityName) {
|
|
|
4611
4165
|
stripeSync = new StripeSync({
|
|
4612
4166
|
databaseUrl: config.databaseUrl,
|
|
4613
4167
|
stripeSecretKey: config.stripeApiKey,
|
|
4614
|
-
enableSigmaSync: process.env.ENABLE_SIGMA_SYNC === "true",
|
|
4615
4168
|
stripeApiVersion: process.env.STRIPE_API_VERSION || "2020-08-27",
|
|
4616
4169
|
autoExpandLists: process.env.AUTO_EXPAND_LISTS === "true",
|
|
4617
4170
|
backfillRelatedEntities: process.env.BACKFILL_RELATED_ENTITIES !== "false",
|
|
@@ -4775,7 +4328,6 @@ Mode: ${modeLabel}`));
|
|
|
4775
4328
|
stripeSync = new StripeSync({
|
|
4776
4329
|
databaseUrl: config.databaseUrl,
|
|
4777
4330
|
stripeSecretKey: config.stripeApiKey,
|
|
4778
|
-
enableSigmaSync: config.enableSigmaSync,
|
|
4779
4331
|
stripeApiVersion: process.env.STRIPE_API_VERSION || "2020-08-27",
|
|
4780
4332
|
autoExpandLists: process.env.AUTO_EXPAND_LISTS === "true",
|
|
4781
4333
|
backfillRelatedEntities: process.env.BACKFILL_RELATED_ENTITIES !== "false",
|
|
@@ -4923,8 +4475,7 @@ async function installCommand(options) {
|
|
|
4923
4475
|
mask: "*",
|
|
4924
4476
|
validate: (input) => {
|
|
4925
4477
|
if (!input.trim()) return "Stripe key is required";
|
|
4926
|
-
if (!input.startsWith("sk_")
|
|
4927
|
-
return 'Stripe key should start with "sk_" or "rk_"';
|
|
4478
|
+
if (!input.startsWith("sk_")) return 'Stripe key should start with "sk_"';
|
|
4928
4479
|
return true;
|
|
4929
4480
|
}
|
|
4930
4481
|
});
|
|
@@ -4943,7 +4494,8 @@ async function installCommand(options) {
|
|
|
4943
4494
|
supabaseAccessToken: accessToken,
|
|
4944
4495
|
supabaseProjectRef: projectRef,
|
|
4945
4496
|
stripeKey,
|
|
4946
|
-
packageVersion: options.packageVersion
|
|
4497
|
+
packageVersion: options.packageVersion,
|
|
4498
|
+
workerIntervalSeconds: options.workerInterval
|
|
4947
4499
|
});
|
|
4948
4500
|
console.log(import_chalk3.default.cyan("\n\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501"));
|
|
4949
4501
|
console.log(import_chalk3.default.cyan.bold(" Installation Complete!"));
|
|
@@ -4994,8 +4546,7 @@ async function uninstallCommand(options) {
|
|
|
4994
4546
|
mask: "*",
|
|
4995
4547
|
validate: (input) => {
|
|
4996
4548
|
if (!input.trim()) return "Stripe key is required";
|
|
4997
|
-
if (!input.startsWith("sk_")
|
|
4998
|
-
return 'Stripe key should start with "sk_" or "rk_"';
|
|
4549
|
+
if (!input.startsWith("sk_")) return 'Stripe key should start with "sk_"';
|
|
4999
4550
|
return true;
|
|
5000
4551
|
}
|
|
5001
4552
|
});
|
|
@@ -5042,12 +4593,11 @@ program.command("migrate").description("Run database migrations only").option("-
|
|
|
5042
4593
|
databaseUrl: options.databaseUrl
|
|
5043
4594
|
});
|
|
5044
4595
|
});
|
|
5045
|
-
program.command("start").description("Start Stripe sync").option("--stripe-key <key>", "Stripe API key (or STRIPE_API_KEY env)").option("--ngrok-token <token>", "ngrok auth token (or NGROK_AUTH_TOKEN env)").option("--database-url <url>", "Postgres DATABASE_URL (or DATABASE_URL env)").
|
|
4596
|
+
program.command("start").description("Start Stripe sync").option("--stripe-key <key>", "Stripe API key (or STRIPE_API_KEY env)").option("--ngrok-token <token>", "ngrok auth token (or NGROK_AUTH_TOKEN env)").option("--database-url <url>", "Postgres DATABASE_URL (or DATABASE_URL env)").action(async (options) => {
|
|
5046
4597
|
await syncCommand({
|
|
5047
4598
|
stripeKey: options.stripeKey,
|
|
5048
4599
|
ngrokToken: options.ngrokToken,
|
|
5049
|
-
databaseUrl: options.databaseUrl
|
|
5050
|
-
enableSigmaSync: options.sigma
|
|
4600
|
+
databaseUrl: options.databaseUrl
|
|
5051
4601
|
});
|
|
5052
4602
|
});
|
|
5053
4603
|
program.command("backfill <entityName>").description("Backfill a specific entity type from Stripe (e.g., customer, invoice, product)").option("--stripe-key <key>", "Stripe API key (or STRIPE_API_KEY env)").option("--database-url <url>", "Postgres DATABASE_URL (or DATABASE_URL env)").action(async (entityName, options) => {
|
|
@@ -5063,12 +4613,17 @@ var supabase = program.command("supabase").description("Supabase Edge Functions
|
|
|
5063
4613
|
supabase.command("install").description("Install Stripe sync to Supabase Edge Functions").option("--token <token>", "Supabase access token (or SUPABASE_ACCESS_TOKEN env)").option("--project <ref>", "Supabase project ref (or SUPABASE_PROJECT_REF env)").option("--stripe-key <key>", "Stripe API key (or STRIPE_API_KEY env)").option(
|
|
5064
4614
|
"--package-version <version>",
|
|
5065
4615
|
"Package version to install (e.g., 1.0.8-beta.1, defaults to latest)"
|
|
4616
|
+
).option(
|
|
4617
|
+
"--worker-interval <seconds>",
|
|
4618
|
+
"Worker interval in seconds (defaults to 60)",
|
|
4619
|
+
(val) => parseInt(val, 10)
|
|
5066
4620
|
).action(async (options) => {
|
|
5067
4621
|
await installCommand({
|
|
5068
4622
|
supabaseAccessToken: options.token,
|
|
5069
4623
|
supabaseProjectRef: options.project,
|
|
5070
4624
|
stripeKey: options.stripeKey,
|
|
5071
|
-
packageVersion: options.packageVersion
|
|
4625
|
+
packageVersion: options.packageVersion,
|
|
4626
|
+
workerInterval: options.workerInterval
|
|
5072
4627
|
});
|
|
5073
4628
|
});
|
|
5074
4629
|
supabase.command("uninstall").description("Uninstall Stripe sync from Supabase Edge Functions").option("--token <token>", "Supabase access token (or SUPABASE_ACCESS_TOKEN env)").option("--project <ref>", "Supabase project ref (or SUPABASE_PROJECT_REF env)").option("--stripe-key <key>", "Stripe API key (or STRIPE_API_KEY env)").action(async (options) => {
|