stripe-experiment-sync 1.0.8 → 1.0.9-beta.1765909347
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-J7BH3XD6.js → chunk-2CYYWBTD.js} +486 -27
- package/dist/{chunk-CKWVW2JK.js → chunk-3W3CERIG.js} +3 -1
- package/dist/{chunk-YH6KRZDQ.js → chunk-DBJCCGXP.js} +28 -10
- package/dist/{chunk-X2OQQCC2.js → chunk-SI3VFP3M.js} +16 -5
- package/dist/cli/index.cjs +534 -45
- package/dist/cli/index.js +9 -8
- package/dist/cli/lib.cjs +529 -41
- package/dist/cli/lib.d.cts +2 -0
- package/dist/cli/lib.d.ts +2 -0
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +490 -27
- package/dist/index.d.cts +107 -4
- package/dist/index.d.ts +107 -4
- package/dist/index.js +6 -2
- package/dist/migrations/0059_sigma_subscription_item_change_events_v2_beta.sql +61 -0
- package/dist/migrations/0060_sigma_exchange_rates_from_usd.sql +38 -0
- package/dist/supabase/index.cjs +18 -5
- package/dist/supabase/index.js +2 -2
- package/package.json +3 -1
package/dist/index.cjs
CHANGED
|
@@ -35,6 +35,8 @@ __export(index_exports, {
|
|
|
35
35
|
VERSION: () => VERSION,
|
|
36
36
|
createStripeWebSocketClient: () => createStripeWebSocketClient,
|
|
37
37
|
hashApiKey: () => hashApiKey,
|
|
38
|
+
normalizeSigmaTimestampToIso: () => normalizeSigmaTimestampToIso,
|
|
39
|
+
parseCsvObjects: () => parseCsvObjects,
|
|
38
40
|
runMigrations: () => runMigrations
|
|
39
41
|
});
|
|
40
42
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -46,7 +48,7 @@ var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
|
46
48
|
// package.json
|
|
47
49
|
var package_default = {
|
|
48
50
|
name: "stripe-experiment-sync",
|
|
49
|
-
version: "1.0.
|
|
51
|
+
version: "1.0.9-beta.1765909347",
|
|
50
52
|
private: false,
|
|
51
53
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
52
54
|
type: "module",
|
|
@@ -86,6 +88,7 @@ var package_default = {
|
|
|
86
88
|
dotenv: "^16.4.7",
|
|
87
89
|
express: "^4.18.2",
|
|
88
90
|
inquirer: "^12.3.0",
|
|
91
|
+
papaparse: "5.4.1",
|
|
89
92
|
pg: "^8.16.3",
|
|
90
93
|
"pg-node-migrations": "0.0.8",
|
|
91
94
|
stripe: "^17.7.0",
|
|
@@ -97,6 +100,7 @@ var package_default = {
|
|
|
97
100
|
"@types/express": "^4.17.21",
|
|
98
101
|
"@types/inquirer": "^9.0.7",
|
|
99
102
|
"@types/node": "^24.10.1",
|
|
103
|
+
"@types/papaparse": "5.3.16",
|
|
100
104
|
"@types/pg": "^8.15.5",
|
|
101
105
|
"@types/ws": "^8.5.13",
|
|
102
106
|
"@types/yesql": "^4.1.4",
|
|
@@ -126,14 +130,60 @@ var package_default = {
|
|
|
126
130
|
};
|
|
127
131
|
|
|
128
132
|
// src/stripeSync.ts
|
|
129
|
-
var
|
|
133
|
+
var import_stripe3 = __toESM(require("stripe"), 1);
|
|
130
134
|
var import_yesql2 = require("yesql");
|
|
131
135
|
|
|
132
136
|
// src/database/postgres.ts
|
|
133
137
|
var import_pg = __toESM(require("pg"), 1);
|
|
134
138
|
var import_yesql = require("yesql");
|
|
139
|
+
|
|
140
|
+
// src/database/QueryUtils.ts
|
|
141
|
+
var QueryUtils = class _QueryUtils {
|
|
142
|
+
constructor() {
|
|
143
|
+
}
|
|
144
|
+
static quoteIdent(name) {
|
|
145
|
+
return `"${name}"`;
|
|
146
|
+
}
|
|
147
|
+
static quotedList(names) {
|
|
148
|
+
return names.map(_QueryUtils.quoteIdent).join(", ");
|
|
149
|
+
}
|
|
150
|
+
static buildInsertParts(columns) {
|
|
151
|
+
const columnsSql = columns.map((c) => _QueryUtils.quoteIdent(c.column)).join(", ");
|
|
152
|
+
const valuesSql = columns.map((c, i) => {
|
|
153
|
+
const placeholder = `$${i + 1}`;
|
|
154
|
+
return `${placeholder}::${c.pgType}`;
|
|
155
|
+
}).join(", ");
|
|
156
|
+
const params = columns.map((c) => c.value);
|
|
157
|
+
return { columnsSql, valuesSql, params };
|
|
158
|
+
}
|
|
159
|
+
static buildRawJsonUpsertQuery(schema, table, columns, conflictTarget) {
|
|
160
|
+
const { columnsSql, valuesSql, params } = _QueryUtils.buildInsertParts(columns);
|
|
161
|
+
const conflictSql = _QueryUtils.quotedList(conflictTarget);
|
|
162
|
+
const tsParamIdx = columns.findIndex((c) => c.column === "_last_synced_at") + 1;
|
|
163
|
+
if (tsParamIdx <= 0) {
|
|
164
|
+
throw new Error("buildRawJsonUpsertQuery requires _last_synced_at column");
|
|
165
|
+
}
|
|
166
|
+
const sql3 = `
|
|
167
|
+
INSERT INTO ${_QueryUtils.quoteIdent(schema)}.${_QueryUtils.quoteIdent(table)} (${columnsSql})
|
|
168
|
+
VALUES (${valuesSql})
|
|
169
|
+
ON CONFLICT (${conflictSql})
|
|
170
|
+
DO UPDATE SET
|
|
171
|
+
"_raw_data" = EXCLUDED."_raw_data",
|
|
172
|
+
"_last_synced_at" = $${tsParamIdx},
|
|
173
|
+
"_account_id" = EXCLUDED."_account_id"
|
|
174
|
+
WHERE ${_QueryUtils.quoteIdent(table)}."_last_synced_at" IS NULL
|
|
175
|
+
OR ${_QueryUtils.quoteIdent(table)}."_last_synced_at" < $${tsParamIdx}
|
|
176
|
+
RETURNING *
|
|
177
|
+
`;
|
|
178
|
+
return { sql: sql3, params };
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
// src/database/postgres.ts
|
|
135
183
|
var ORDERED_STRIPE_TABLES = [
|
|
184
|
+
"exchange_rates_from_usd",
|
|
136
185
|
"subscription_items",
|
|
186
|
+
"subscription_item_change_events_v2_beta",
|
|
137
187
|
"subscriptions",
|
|
138
188
|
"subscription_schedules",
|
|
139
189
|
"checkout_session_line_items",
|
|
@@ -203,7 +253,7 @@ var PostgresClient = class {
|
|
|
203
253
|
}
|
|
204
254
|
return results.flatMap((it) => it.rows);
|
|
205
255
|
}
|
|
206
|
-
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp) {
|
|
256
|
+
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp, upsertOptions) {
|
|
207
257
|
const timestamp = syncTimestamp || (/* @__PURE__ */ new Date()).toISOString();
|
|
208
258
|
if (!entries.length) return [];
|
|
209
259
|
const chunkSize = 5;
|
|
@@ -238,20 +288,33 @@ var PostgresClient = class {
|
|
|
238
288
|
const prepared = (0, import_yesql.pg)(upsertSql, { useNullForMissing: true })(cleansed);
|
|
239
289
|
queries.push(this.pool.query(prepared.text, prepared.values));
|
|
240
290
|
} else {
|
|
241
|
-
const
|
|
242
|
-
const
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
291
|
+
const conflictTarget = upsertOptions?.conflictTarget ?? ["id"];
|
|
292
|
+
const extraColumns = upsertOptions?.extraColumns ?? [];
|
|
293
|
+
if (!conflictTarget.length) {
|
|
294
|
+
throw new Error(`Invalid upsert config for ${table}: conflictTarget must be non-empty`);
|
|
295
|
+
}
|
|
296
|
+
const columns = [
|
|
297
|
+
{ column: "_raw_data", pgType: "jsonb", value: JSON.stringify(entry) },
|
|
298
|
+
...extraColumns.map((c) => ({
|
|
299
|
+
column: c.column,
|
|
300
|
+
pgType: c.pgType,
|
|
301
|
+
value: entry[c.entryKey]
|
|
302
|
+
})),
|
|
303
|
+
{ column: "_last_synced_at", pgType: "timestamptz", value: timestamp },
|
|
304
|
+
{ column: "_account_id", pgType: "text", value: accountId }
|
|
305
|
+
];
|
|
306
|
+
for (const c of columns) {
|
|
307
|
+
if (c.value === void 0) {
|
|
308
|
+
throw new Error(`Missing required value for ${table}.${c.column}`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
const { sql: upsertSql, params } = QueryUtils.buildRawJsonUpsertQuery(
|
|
312
|
+
this.config.schema,
|
|
313
|
+
table,
|
|
314
|
+
columns,
|
|
315
|
+
conflictTarget
|
|
316
|
+
);
|
|
317
|
+
queries.push(this.pool.query(upsertSql, params));
|
|
255
318
|
}
|
|
256
319
|
});
|
|
257
320
|
results.push(...await Promise.all(queries));
|
|
@@ -651,7 +714,12 @@ var PostgresClient = class {
|
|
|
651
714
|
} else {
|
|
652
715
|
await this.query(
|
|
653
716
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
654
|
-
SET cursor =
|
|
717
|
+
SET cursor = CASE
|
|
718
|
+
WHEN cursor IS NULL THEN $4
|
|
719
|
+
WHEN (cursor COLLATE "C") < ($4::text COLLATE "C") THEN $4
|
|
720
|
+
ELSE cursor
|
|
721
|
+
END,
|
|
722
|
+
updated_at = now()
|
|
655
723
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
656
724
|
[accountId, runStartedAt, object, cursor]
|
|
657
725
|
);
|
|
@@ -662,10 +730,17 @@ var PostgresClient = class {
|
|
|
662
730
|
* This considers completed, error, AND running runs to ensure recovery syncs
|
|
663
731
|
* don't re-process data that was already synced before a crash.
|
|
664
732
|
* A 'running' status with a cursor means the process was killed mid-sync.
|
|
733
|
+
*
|
|
734
|
+
* Handles two cursor formats:
|
|
735
|
+
* - Numeric: compared as bigint for correct ordering
|
|
736
|
+
* - Composite cursors: compared as strings with COLLATE "C"
|
|
665
737
|
*/
|
|
666
738
|
async getLastCompletedCursor(accountId, object) {
|
|
667
739
|
const result = await this.query(
|
|
668
|
-
`SELECT
|
|
740
|
+
`SELECT CASE
|
|
741
|
+
WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
|
|
742
|
+
ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
|
|
743
|
+
END as cursor
|
|
669
744
|
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
670
745
|
WHERE o."_account_id" = $1
|
|
671
746
|
AND o.object = $2
|
|
@@ -950,6 +1025,269 @@ function hashApiKey(apiKey) {
|
|
|
950
1025
|
return (0, import_crypto.createHash)("sha256").update(apiKey).digest("hex");
|
|
951
1026
|
}
|
|
952
1027
|
|
|
1028
|
+
// src/sigma/sigmaApi.ts
|
|
1029
|
+
var import_papaparse = __toESM(require("papaparse"), 1);
|
|
1030
|
+
var import_stripe2 = __toESM(require("stripe"), 1);
|
|
1031
|
+
var STRIPE_FILES_BASE = "https://files.stripe.com/v1";
|
|
1032
|
+
function sleep2(ms) {
|
|
1033
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1034
|
+
}
|
|
1035
|
+
function parseCsvObjects(csv) {
|
|
1036
|
+
const input = csv.replace(/^\uFEFF/, "");
|
|
1037
|
+
const parsed = import_papaparse.default.parse(input, {
|
|
1038
|
+
header: true,
|
|
1039
|
+
skipEmptyLines: "greedy"
|
|
1040
|
+
});
|
|
1041
|
+
if (parsed.errors.length > 0) {
|
|
1042
|
+
throw new Error(`Failed to parse Sigma CSV: ${parsed.errors[0]?.message ?? "unknown error"}`);
|
|
1043
|
+
}
|
|
1044
|
+
return parsed.data.filter((row) => row && Object.keys(row).length > 0).map(
|
|
1045
|
+
(row) => Object.fromEntries(
|
|
1046
|
+
Object.entries(row).map(([k, v]) => [k, v == null || v === "" ? null : String(v)])
|
|
1047
|
+
)
|
|
1048
|
+
);
|
|
1049
|
+
}
|
|
1050
|
+
function normalizeSigmaTimestampToIso(value) {
|
|
1051
|
+
const v = value.trim();
|
|
1052
|
+
if (!v) return null;
|
|
1053
|
+
const hasExplicitTz = /z$|[+-]\d{2}:?\d{2}$/i.test(v);
|
|
1054
|
+
const isoish = v.includes("T") ? v : v.replace(" ", "T");
|
|
1055
|
+
const candidate = hasExplicitTz ? isoish : `${isoish}Z`;
|
|
1056
|
+
const d = new Date(candidate);
|
|
1057
|
+
if (Number.isNaN(d.getTime())) return null;
|
|
1058
|
+
return d.toISOString();
|
|
1059
|
+
}
|
|
1060
|
+
async function fetchStripeText(url, apiKey, options) {
|
|
1061
|
+
const res = await fetch(url, {
|
|
1062
|
+
...options,
|
|
1063
|
+
headers: {
|
|
1064
|
+
...options.headers ?? {},
|
|
1065
|
+
Authorization: `Bearer ${apiKey}`
|
|
1066
|
+
}
|
|
1067
|
+
});
|
|
1068
|
+
const text = await res.text();
|
|
1069
|
+
if (!res.ok) {
|
|
1070
|
+
throw new Error(`Sigma file download error (${res.status}) for ${url}: ${text}`);
|
|
1071
|
+
}
|
|
1072
|
+
return text;
|
|
1073
|
+
}
|
|
1074
|
+
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1075
|
+
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1076
|
+
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1077
|
+
const stripe = new import_stripe2.default(params.apiKey);
|
|
1078
|
+
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1079
|
+
sql: params.sql
|
|
1080
|
+
});
|
|
1081
|
+
const queryRunId = created.id;
|
|
1082
|
+
const start = Date.now();
|
|
1083
|
+
let current = created;
|
|
1084
|
+
while (current.status === "running") {
|
|
1085
|
+
if (Date.now() - start > pollTimeoutMs) {
|
|
1086
|
+
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1087
|
+
}
|
|
1088
|
+
await sleep2(pollIntervalMs);
|
|
1089
|
+
current = await stripe.rawRequest(
|
|
1090
|
+
"GET",
|
|
1091
|
+
`/v1/sigma/query_runs/${queryRunId}`,
|
|
1092
|
+
{}
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
if (current.status !== "succeeded") {
|
|
1096
|
+
throw new Error(
|
|
1097
|
+
`Sigma query run did not succeed (status=${current.status}) id=${queryRunId} error=${JSON.stringify(
|
|
1098
|
+
current.error
|
|
1099
|
+
)}`
|
|
1100
|
+
);
|
|
1101
|
+
}
|
|
1102
|
+
const fileId = current.result?.file;
|
|
1103
|
+
if (!fileId) {
|
|
1104
|
+
throw new Error(`Sigma query run succeeded but result.file is missing (id=${queryRunId})`);
|
|
1105
|
+
}
|
|
1106
|
+
const csv = await fetchStripeText(
|
|
1107
|
+
`${STRIPE_FILES_BASE}/files/${fileId}/contents`,
|
|
1108
|
+
params.apiKey,
|
|
1109
|
+
{ method: "GET" }
|
|
1110
|
+
);
|
|
1111
|
+
return { queryRunId, fileId, csv };
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
// src/sigma/sigmaIngestionConfigs.ts
|
|
1115
|
+
var SIGMA_INGESTION_CONFIGS = {
|
|
1116
|
+
subscription_item_change_events_v2_beta: {
|
|
1117
|
+
sigmaTable: "subscription_item_change_events_v2_beta",
|
|
1118
|
+
destinationTable: "subscription_item_change_events_v2_beta",
|
|
1119
|
+
pageSize: 1e4,
|
|
1120
|
+
cursor: {
|
|
1121
|
+
version: 1,
|
|
1122
|
+
columns: [
|
|
1123
|
+
{ column: "event_timestamp", type: "timestamp" },
|
|
1124
|
+
{ column: "event_type", type: "string" },
|
|
1125
|
+
{ column: "subscription_item_id", type: "string" }
|
|
1126
|
+
]
|
|
1127
|
+
},
|
|
1128
|
+
upsert: {
|
|
1129
|
+
conflictTarget: ["_account_id", "event_timestamp", "event_type", "subscription_item_id"],
|
|
1130
|
+
extraColumns: [
|
|
1131
|
+
{ column: "event_timestamp", pgType: "timestamptz", entryKey: "event_timestamp" },
|
|
1132
|
+
{ column: "event_type", pgType: "text", entryKey: "event_type" },
|
|
1133
|
+
{ column: "subscription_item_id", pgType: "text", entryKey: "subscription_item_id" }
|
|
1134
|
+
]
|
|
1135
|
+
}
|
|
1136
|
+
},
|
|
1137
|
+
exchange_rates_from_usd: {
|
|
1138
|
+
sigmaTable: "exchange_rates_from_usd",
|
|
1139
|
+
destinationTable: "exchange_rates_from_usd",
|
|
1140
|
+
pageSize: 1e4,
|
|
1141
|
+
cursor: {
|
|
1142
|
+
version: 1,
|
|
1143
|
+
columns: [
|
|
1144
|
+
{ column: "date", type: "string" },
|
|
1145
|
+
{ column: "sell_currency", type: "string" }
|
|
1146
|
+
]
|
|
1147
|
+
},
|
|
1148
|
+
upsert: {
|
|
1149
|
+
conflictTarget: ["_account_id", "date", "sell_currency"],
|
|
1150
|
+
extraColumns: [
|
|
1151
|
+
{ column: "date", pgType: "date", entryKey: "date" },
|
|
1152
|
+
{ column: "sell_currency", pgType: "text", entryKey: "sell_currency" }
|
|
1153
|
+
]
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
};
|
|
1157
|
+
|
|
1158
|
+
// src/sigma/sigmaIngestion.ts
|
|
1159
|
+
var SIGMA_CURSOR_DELIM = "";
|
|
1160
|
+
function escapeSigmaSqlStringLiteral(value) {
|
|
1161
|
+
return value.replace(/'/g, "''");
|
|
1162
|
+
}
|
|
1163
|
+
function formatSigmaTimestampForSqlLiteral(date) {
|
|
1164
|
+
return date.toISOString().replace("T", " ").replace("Z", "");
|
|
1165
|
+
}
|
|
1166
|
+
function decodeSigmaCursorValues(spec, cursor) {
|
|
1167
|
+
const prefix = `v${spec.version}${SIGMA_CURSOR_DELIM}`;
|
|
1168
|
+
if (!cursor.startsWith(prefix)) {
|
|
1169
|
+
throw new Error(
|
|
1170
|
+
`Unrecognized Sigma cursor format (expected prefix ${JSON.stringify(prefix)}): ${cursor}`
|
|
1171
|
+
);
|
|
1172
|
+
}
|
|
1173
|
+
const parts = cursor.split(SIGMA_CURSOR_DELIM);
|
|
1174
|
+
const expected = 1 + spec.columns.length;
|
|
1175
|
+
if (parts.length !== expected) {
|
|
1176
|
+
throw new Error(`Malformed Sigma cursor: expected ${expected} parts, got ${parts.length}`);
|
|
1177
|
+
}
|
|
1178
|
+
return parts.slice(1);
|
|
1179
|
+
}
|
|
1180
|
+
function encodeSigmaCursor(spec, values) {
|
|
1181
|
+
if (values.length !== spec.columns.length) {
|
|
1182
|
+
throw new Error(
|
|
1183
|
+
`Cannot encode Sigma cursor: expected ${spec.columns.length} values, got ${values.length}`
|
|
1184
|
+
);
|
|
1185
|
+
}
|
|
1186
|
+
for (const v of values) {
|
|
1187
|
+
if (v.includes(SIGMA_CURSOR_DELIM)) {
|
|
1188
|
+
throw new Error("Cannot encode Sigma cursor: value contains delimiter character");
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
return [`v${spec.version}`, ...values].join(SIGMA_CURSOR_DELIM);
|
|
1192
|
+
}
|
|
1193
|
+
function sigmaSqlLiteralForCursorValue(spec, rawValue) {
|
|
1194
|
+
switch (spec.type) {
|
|
1195
|
+
case "timestamp": {
|
|
1196
|
+
const d = new Date(rawValue);
|
|
1197
|
+
if (Number.isNaN(d.getTime())) {
|
|
1198
|
+
throw new Error(`Invalid timestamp cursor value for ${spec.column}: ${rawValue}`);
|
|
1199
|
+
}
|
|
1200
|
+
return `timestamp '${formatSigmaTimestampForSqlLiteral(d)}'`;
|
|
1201
|
+
}
|
|
1202
|
+
case "number": {
|
|
1203
|
+
if (!/^-?\d+(\.\d+)?$/.test(rawValue)) {
|
|
1204
|
+
throw new Error(`Invalid numeric cursor value for ${spec.column}: ${rawValue}`);
|
|
1205
|
+
}
|
|
1206
|
+
return rawValue;
|
|
1207
|
+
}
|
|
1208
|
+
case "string":
|
|
1209
|
+
return `'${escapeSigmaSqlStringLiteral(rawValue)}'`;
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
function buildSigmaCursorWhereClause(spec, cursorValues) {
|
|
1213
|
+
if (cursorValues.length !== spec.columns.length) {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
`Cannot build Sigma cursor predicate: expected ${spec.columns.length} values, got ${cursorValues.length}`
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
const cols = spec.columns.map((c) => c.column);
|
|
1219
|
+
const lits = spec.columns.map((c, i) => sigmaSqlLiteralForCursorValue(c, cursorValues[i] ?? ""));
|
|
1220
|
+
const ors = [];
|
|
1221
|
+
for (let i = 0; i < cols.length; i++) {
|
|
1222
|
+
const ands = [];
|
|
1223
|
+
for (let j = 0; j < i; j++) {
|
|
1224
|
+
ands.push(`${cols[j]} = ${lits[j]}`);
|
|
1225
|
+
}
|
|
1226
|
+
ands.push(`${cols[i]} > ${lits[i]}`);
|
|
1227
|
+
ors.push(`(${ands.join(" AND ")})`);
|
|
1228
|
+
}
|
|
1229
|
+
return ors.join(" OR ");
|
|
1230
|
+
}
|
|
1231
|
+
function buildSigmaQuery(config, cursor) {
|
|
1232
|
+
const select = config.select === void 0 || config.select === "*" ? "*" : config.select.join(", ");
|
|
1233
|
+
const whereParts = [];
|
|
1234
|
+
if (config.additionalWhere) {
|
|
1235
|
+
whereParts.push(`(${config.additionalWhere})`);
|
|
1236
|
+
}
|
|
1237
|
+
if (cursor) {
|
|
1238
|
+
const values = decodeSigmaCursorValues(config.cursor, cursor);
|
|
1239
|
+
const predicate = buildSigmaCursorWhereClause(config.cursor, values);
|
|
1240
|
+
whereParts.push(`(${predicate})`);
|
|
1241
|
+
}
|
|
1242
|
+
const whereClause = whereParts.length > 0 ? `WHERE ${whereParts.join(" AND ")}` : "";
|
|
1243
|
+
const orderBy = config.cursor.columns.map((c) => c.column).join(", ");
|
|
1244
|
+
return [
|
|
1245
|
+
`SELECT ${select} FROM ${config.sigmaTable}`,
|
|
1246
|
+
whereClause,
|
|
1247
|
+
`ORDER BY ${orderBy} ASC`,
|
|
1248
|
+
`LIMIT ${config.pageSize}`
|
|
1249
|
+
].filter(Boolean).join(" ");
|
|
1250
|
+
}
|
|
1251
|
+
function defaultSigmaRowToEntry(config, row) {
|
|
1252
|
+
const out = { ...row };
|
|
1253
|
+
for (const col of config.cursor.columns) {
|
|
1254
|
+
const raw = row[col.column];
|
|
1255
|
+
if (raw == null) {
|
|
1256
|
+
throw new Error(`Sigma row missing required cursor column: ${col.column}`);
|
|
1257
|
+
}
|
|
1258
|
+
if (col.type === "timestamp") {
|
|
1259
|
+
const normalized = normalizeSigmaTimestampToIso(raw);
|
|
1260
|
+
if (!normalized) {
|
|
1261
|
+
throw new Error(`Sigma row has invalid timestamp for ${col.column}: ${raw}`);
|
|
1262
|
+
}
|
|
1263
|
+
out[col.column] = normalized;
|
|
1264
|
+
} else if (col.type === "string") {
|
|
1265
|
+
const v = raw.trim();
|
|
1266
|
+
if (!v) {
|
|
1267
|
+
throw new Error(`Sigma row has empty string for required cursor column: ${col.column}`);
|
|
1268
|
+
}
|
|
1269
|
+
out[col.column] = v;
|
|
1270
|
+
} else {
|
|
1271
|
+
const v = raw.trim();
|
|
1272
|
+
if (!v) {
|
|
1273
|
+
throw new Error(`Sigma row has empty value for required cursor column: ${col.column}`);
|
|
1274
|
+
}
|
|
1275
|
+
out[col.column] = v;
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
return out;
|
|
1279
|
+
}
|
|
1280
|
+
function sigmaCursorFromEntry(config, entry) {
|
|
1281
|
+
const values = config.cursor.columns.map((c) => {
|
|
1282
|
+
const raw = entry[c.column];
|
|
1283
|
+
if (raw == null) {
|
|
1284
|
+
throw new Error(`Cannot build cursor: entry missing ${c.column}`);
|
|
1285
|
+
}
|
|
1286
|
+
return String(raw);
|
|
1287
|
+
});
|
|
1288
|
+
return encodeSigmaCursor(config.cursor, values);
|
|
1289
|
+
}
|
|
1290
|
+
|
|
953
1291
|
// src/stripeSync.ts
|
|
954
1292
|
function getUniqueIds(entries, key) {
|
|
955
1293
|
const set = new Set(
|
|
@@ -960,7 +1298,7 @@ function getUniqueIds(entries, key) {
|
|
|
960
1298
|
var StripeSync = class {
|
|
961
1299
|
constructor(config) {
|
|
962
1300
|
this.config = config;
|
|
963
|
-
const baseStripe = new
|
|
1301
|
+
const baseStripe = new import_stripe3.default(config.stripeSecretKey, {
|
|
964
1302
|
// https://github.com/stripe/stripe-node#configuration
|
|
965
1303
|
// @ts-ignore
|
|
966
1304
|
apiVersion: config.stripeApiVersion,
|
|
@@ -1361,6 +1699,17 @@ var StripeSync = class {
|
|
|
1361
1699
|
listFn: (p) => this.stripe.checkout.sessions.list(p),
|
|
1362
1700
|
upsertFn: (items, id) => this.upsertCheckoutSessions(items, id),
|
|
1363
1701
|
supportsCreatedFilter: true
|
|
1702
|
+
},
|
|
1703
|
+
// Sigma-backed resources
|
|
1704
|
+
subscription_item_change_events_v2_beta: {
|
|
1705
|
+
order: 18,
|
|
1706
|
+
supportsCreatedFilter: false,
|
|
1707
|
+
sigma: SIGMA_INGESTION_CONFIGS.subscription_item_change_events_v2_beta
|
|
1708
|
+
},
|
|
1709
|
+
exchange_rates_from_usd: {
|
|
1710
|
+
order: 19,
|
|
1711
|
+
supportsCreatedFilter: false,
|
|
1712
|
+
sigma: SIGMA_INGESTION_CONFIGS.exchange_rates_from_usd
|
|
1364
1713
|
}
|
|
1365
1714
|
};
|
|
1366
1715
|
async processEvent(event) {
|
|
@@ -1393,7 +1742,13 @@ var StripeSync = class {
|
|
|
1393
1742
|
* Order is determined by the `order` field in resourceRegistry.
|
|
1394
1743
|
*/
|
|
1395
1744
|
getSupportedSyncObjects() {
|
|
1396
|
-
|
|
1745
|
+
const all = Object.entries(this.resourceRegistry).sort(([, a], [, b]) => a.order - b.order).map(([key]) => key);
|
|
1746
|
+
if (!this.config.enableSigmaSync) {
|
|
1747
|
+
return all.filter(
|
|
1748
|
+
(o) => o !== "subscription_item_change_events_v2_beta" && o !== "exchange_rates_from_usd"
|
|
1749
|
+
);
|
|
1750
|
+
}
|
|
1751
|
+
return all;
|
|
1397
1752
|
}
|
|
1398
1753
|
// Event handler methods
|
|
1399
1754
|
async handleChargeEvent(event, accountId) {
|
|
@@ -1472,7 +1827,7 @@ var StripeSync = class {
|
|
|
1472
1827
|
);
|
|
1473
1828
|
await this.upsertProducts([product], accountId, this.getSyncTimestamp(event, refetched));
|
|
1474
1829
|
} catch (err) {
|
|
1475
|
-
if (err instanceof
|
|
1830
|
+
if (err instanceof import_stripe3.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1476
1831
|
const product = event.data.object;
|
|
1477
1832
|
await this.deleteProduct(product.id);
|
|
1478
1833
|
} else {
|
|
@@ -1492,7 +1847,7 @@ var StripeSync = class {
|
|
|
1492
1847
|
);
|
|
1493
1848
|
await this.upsertPrices([price], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1494
1849
|
} catch (err) {
|
|
1495
|
-
if (err instanceof
|
|
1850
|
+
if (err instanceof import_stripe3.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1496
1851
|
const price = event.data.object;
|
|
1497
1852
|
await this.deletePrice(price.id);
|
|
1498
1853
|
} else {
|
|
@@ -1512,7 +1867,7 @@ var StripeSync = class {
|
|
|
1512
1867
|
);
|
|
1513
1868
|
await this.upsertPlans([plan], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1514
1869
|
} catch (err) {
|
|
1515
|
-
if (err instanceof
|
|
1870
|
+
if (err instanceof import_stripe3.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1516
1871
|
const plan = event.data.object;
|
|
1517
1872
|
await this.deletePlan(plan.id);
|
|
1518
1873
|
} else {
|
|
@@ -1759,10 +2114,10 @@ var StripeSync = class {
|
|
|
1759
2114
|
let cursor = null;
|
|
1760
2115
|
if (!params?.created) {
|
|
1761
2116
|
if (objRun?.cursor) {
|
|
1762
|
-
cursor =
|
|
2117
|
+
cursor = objRun.cursor;
|
|
1763
2118
|
} else {
|
|
1764
2119
|
const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
|
|
1765
|
-
cursor = lastCursor
|
|
2120
|
+
cursor = lastCursor ?? null;
|
|
1766
2121
|
}
|
|
1767
2122
|
}
|
|
1768
2123
|
const result = await this.fetchOnePage(
|
|
@@ -1817,9 +2172,18 @@ var StripeSync = class {
|
|
|
1817
2172
|
throw new Error(`Unsupported object type for processNext: ${object}`);
|
|
1818
2173
|
}
|
|
1819
2174
|
try {
|
|
2175
|
+
if (config.sigma) {
|
|
2176
|
+
return await this.fetchOneSigmaPage(
|
|
2177
|
+
accountId,
|
|
2178
|
+
resourceName,
|
|
2179
|
+
runStartedAt,
|
|
2180
|
+
cursor,
|
|
2181
|
+
config.sigma
|
|
2182
|
+
);
|
|
2183
|
+
}
|
|
1820
2184
|
const listParams = { limit };
|
|
1821
2185
|
if (config.supportsCreatedFilter) {
|
|
1822
|
-
const created = params?.created ?? (cursor ? { gte: cursor } : void 0);
|
|
2186
|
+
const created = params?.created ?? (cursor && /^\d+$/.test(cursor) ? { gte: Number.parseInt(cursor, 10) } : void 0);
|
|
1823
2187
|
if (created) {
|
|
1824
2188
|
listParams.created = created;
|
|
1825
2189
|
}
|
|
@@ -1864,6 +2228,97 @@ var StripeSync = class {
|
|
|
1864
2228
|
throw error;
|
|
1865
2229
|
}
|
|
1866
2230
|
}
|
|
2231
|
+
async getSigmaFallbackCursorFromDestination(accountId, sigmaConfig) {
|
|
2232
|
+
const cursorCols = sigmaConfig.cursor.columns;
|
|
2233
|
+
const selectCols = cursorCols.map((c) => `"${c.column}"`).join(", ");
|
|
2234
|
+
const orderBy = cursorCols.map((c) => `"${c.column}" DESC`).join(", ");
|
|
2235
|
+
const result = await this.postgresClient.query(
|
|
2236
|
+
`SELECT ${selectCols}
|
|
2237
|
+
FROM "stripe"."${sigmaConfig.destinationTable}"
|
|
2238
|
+
WHERE "_account_id" = $1
|
|
2239
|
+
ORDER BY ${orderBy}
|
|
2240
|
+
LIMIT 1`,
|
|
2241
|
+
[accountId]
|
|
2242
|
+
);
|
|
2243
|
+
if (result.rows.length === 0) return null;
|
|
2244
|
+
const row = result.rows[0];
|
|
2245
|
+
const entryForCursor = {};
|
|
2246
|
+
for (const c of cursorCols) {
|
|
2247
|
+
const v = row[c.column];
|
|
2248
|
+
if (v == null) {
|
|
2249
|
+
throw new Error(
|
|
2250
|
+
`Sigma fallback cursor query returned null for ${sigmaConfig.destinationTable}.${c.column}`
|
|
2251
|
+
);
|
|
2252
|
+
}
|
|
2253
|
+
if (c.type === "timestamp") {
|
|
2254
|
+
const d = v instanceof Date ? v : new Date(String(v));
|
|
2255
|
+
if (Number.isNaN(d.getTime())) {
|
|
2256
|
+
throw new Error(
|
|
2257
|
+
`Sigma fallback cursor query returned invalid timestamp for ${sigmaConfig.destinationTable}.${c.column}: ${String(
|
|
2258
|
+
v
|
|
2259
|
+
)}`
|
|
2260
|
+
);
|
|
2261
|
+
}
|
|
2262
|
+
entryForCursor[c.column] = d.toISOString();
|
|
2263
|
+
} else {
|
|
2264
|
+
entryForCursor[c.column] = String(v);
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
return sigmaCursorFromEntry(sigmaConfig, entryForCursor);
|
|
2268
|
+
}
|
|
2269
|
+
async fetchOneSigmaPage(accountId, resourceName, runStartedAt, cursor, sigmaConfig) {
|
|
2270
|
+
if (!this.config.stripeSecretKey) {
|
|
2271
|
+
throw new Error("Sigma sync requested but stripeSecretKey is not configured.");
|
|
2272
|
+
}
|
|
2273
|
+
if (resourceName !== sigmaConfig.destinationTable) {
|
|
2274
|
+
throw new Error(
|
|
2275
|
+
`Sigma sync config mismatch: resourceName=${resourceName} destinationTable=${sigmaConfig.destinationTable}`
|
|
2276
|
+
);
|
|
2277
|
+
}
|
|
2278
|
+
const effectiveCursor = cursor ?? await this.getSigmaFallbackCursorFromDestination(accountId, sigmaConfig);
|
|
2279
|
+
const sigmaSql = buildSigmaQuery(sigmaConfig, effectiveCursor);
|
|
2280
|
+
this.config.logger?.info(
|
|
2281
|
+
{ object: resourceName, pageSize: sigmaConfig.pageSize, hasCursor: Boolean(effectiveCursor) },
|
|
2282
|
+
"Sigma sync: running query"
|
|
2283
|
+
);
|
|
2284
|
+
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
2285
|
+
apiKey: this.config.stripeSecretKey,
|
|
2286
|
+
sql: sigmaSql,
|
|
2287
|
+
logger: this.config.logger
|
|
2288
|
+
});
|
|
2289
|
+
const rows = parseCsvObjects(csv);
|
|
2290
|
+
if (rows.length === 0) {
|
|
2291
|
+
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2292
|
+
return { processed: 0, hasMore: false, runStartedAt };
|
|
2293
|
+
}
|
|
2294
|
+
const entries = rows.map(
|
|
2295
|
+
(row) => defaultSigmaRowToEntry(sigmaConfig, row)
|
|
2296
|
+
);
|
|
2297
|
+
this.config.logger?.info(
|
|
2298
|
+
{ object: resourceName, rows: entries.length, queryRunId, fileId },
|
|
2299
|
+
"Sigma sync: upserting rows"
|
|
2300
|
+
);
|
|
2301
|
+
await this.postgresClient.upsertManyWithTimestampProtection(
|
|
2302
|
+
entries,
|
|
2303
|
+
resourceName,
|
|
2304
|
+
accountId,
|
|
2305
|
+
void 0,
|
|
2306
|
+
sigmaConfig.upsert
|
|
2307
|
+
);
|
|
2308
|
+
await this.postgresClient.incrementObjectProgress(
|
|
2309
|
+
accountId,
|
|
2310
|
+
runStartedAt,
|
|
2311
|
+
resourceName,
|
|
2312
|
+
entries.length
|
|
2313
|
+
);
|
|
2314
|
+
const newCursor = sigmaCursorFromEntry(sigmaConfig, entries[entries.length - 1]);
|
|
2315
|
+
await this.postgresClient.updateObjectCursor(accountId, runStartedAt, resourceName, newCursor);
|
|
2316
|
+
const hasMore = rows.length === sigmaConfig.pageSize;
|
|
2317
|
+
if (!hasMore) {
|
|
2318
|
+
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2319
|
+
}
|
|
2320
|
+
return { processed: entries.length, hasMore, runStartedAt };
|
|
2321
|
+
}
|
|
1867
2322
|
/**
|
|
1868
2323
|
* Process all pages for all (or specified) object types until complete.
|
|
1869
2324
|
*
|
|
@@ -1992,6 +2447,12 @@ var StripeSync = class {
|
|
|
1992
2447
|
case "checkout_sessions":
|
|
1993
2448
|
results.checkoutSessions = result;
|
|
1994
2449
|
break;
|
|
2450
|
+
case "subscription_item_change_events_v2_beta":
|
|
2451
|
+
results.subscriptionItemChangeEventsV2Beta = result;
|
|
2452
|
+
break;
|
|
2453
|
+
case "exchange_rates_from_usd":
|
|
2454
|
+
results.exchangeRatesFromUsd = result;
|
|
2455
|
+
break;
|
|
1995
2456
|
}
|
|
1996
2457
|
}
|
|
1997
2458
|
}
|
|
@@ -3555,5 +4016,7 @@ var VERSION = package_default.version;
|
|
|
3555
4016
|
VERSION,
|
|
3556
4017
|
createStripeWebSocketClient,
|
|
3557
4018
|
hashApiKey,
|
|
4019
|
+
normalizeSigmaTimestampToIso,
|
|
4020
|
+
parseCsvObjects,
|
|
3558
4021
|
runMigrations
|
|
3559
4022
|
});
|