stripe-experiment-sync 1.0.9-beta.1765909347 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-2CYYWBTD.js → chunk-62FKHVHJ.js} +27 -486
- package/dist/{chunk-SI3VFP3M.js → chunk-AHNO3EMD.js} +36 -9
- package/dist/{chunk-DBJCCGXP.js → chunk-FZQ4B7VZ.js} +12 -29
- package/dist/{chunk-3W3CERIG.js → chunk-VEEV6P4R.js} +1 -3
- package/dist/cli/index.cjs +82 -527
- package/dist/cli/index.js +12 -8
- package/dist/cli/lib.cjs +74 -523
- package/dist/cli/lib.d.cts +1 -2
- package/dist/cli/lib.d.ts +1 -2
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +27 -490
- package/dist/index.d.cts +4 -107
- package/dist/index.d.ts +4 -107
- package/dist/index.js +2 -6
- package/dist/supabase/index.cjs +36 -11
- package/dist/supabase/index.d.cts +4 -2
- package/dist/supabase/index.d.ts +4 -2
- package/dist/supabase/index.js +2 -2
- package/package.json +1 -3
- package/dist/migrations/0059_sigma_subscription_item_change_events_v2_beta.sql +0 -61
- package/dist/migrations/0060_sigma_exchange_rates_from_usd.sql +0 -38
package/dist/index.cjs
CHANGED
|
@@ -35,8 +35,6 @@ __export(index_exports, {
|
|
|
35
35
|
VERSION: () => VERSION,
|
|
36
36
|
createStripeWebSocketClient: () => createStripeWebSocketClient,
|
|
37
37
|
hashApiKey: () => hashApiKey,
|
|
38
|
-
normalizeSigmaTimestampToIso: () => normalizeSigmaTimestampToIso,
|
|
39
|
-
parseCsvObjects: () => parseCsvObjects,
|
|
40
38
|
runMigrations: () => runMigrations
|
|
41
39
|
});
|
|
42
40
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -48,7 +46,7 @@ var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
|
48
46
|
// package.json
|
|
49
47
|
var package_default = {
|
|
50
48
|
name: "stripe-experiment-sync",
|
|
51
|
-
version: "1.0.
|
|
49
|
+
version: "1.0.10",
|
|
52
50
|
private: false,
|
|
53
51
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
54
52
|
type: "module",
|
|
@@ -88,7 +86,6 @@ var package_default = {
|
|
|
88
86
|
dotenv: "^16.4.7",
|
|
89
87
|
express: "^4.18.2",
|
|
90
88
|
inquirer: "^12.3.0",
|
|
91
|
-
papaparse: "5.4.1",
|
|
92
89
|
pg: "^8.16.3",
|
|
93
90
|
"pg-node-migrations": "0.0.8",
|
|
94
91
|
stripe: "^17.7.0",
|
|
@@ -100,7 +97,6 @@ var package_default = {
|
|
|
100
97
|
"@types/express": "^4.17.21",
|
|
101
98
|
"@types/inquirer": "^9.0.7",
|
|
102
99
|
"@types/node": "^24.10.1",
|
|
103
|
-
"@types/papaparse": "5.3.16",
|
|
104
100
|
"@types/pg": "^8.15.5",
|
|
105
101
|
"@types/ws": "^8.5.13",
|
|
106
102
|
"@types/yesql": "^4.1.4",
|
|
@@ -130,60 +126,14 @@ var package_default = {
|
|
|
130
126
|
};
|
|
131
127
|
|
|
132
128
|
// src/stripeSync.ts
|
|
133
|
-
var
|
|
129
|
+
var import_stripe2 = __toESM(require("stripe"), 1);
|
|
134
130
|
var import_yesql2 = require("yesql");
|
|
135
131
|
|
|
136
132
|
// src/database/postgres.ts
|
|
137
133
|
var import_pg = __toESM(require("pg"), 1);
|
|
138
134
|
var import_yesql = require("yesql");
|
|
139
|
-
|
|
140
|
-
// src/database/QueryUtils.ts
|
|
141
|
-
var QueryUtils = class _QueryUtils {
|
|
142
|
-
constructor() {
|
|
143
|
-
}
|
|
144
|
-
static quoteIdent(name) {
|
|
145
|
-
return `"${name}"`;
|
|
146
|
-
}
|
|
147
|
-
static quotedList(names) {
|
|
148
|
-
return names.map(_QueryUtils.quoteIdent).join(", ");
|
|
149
|
-
}
|
|
150
|
-
static buildInsertParts(columns) {
|
|
151
|
-
const columnsSql = columns.map((c) => _QueryUtils.quoteIdent(c.column)).join(", ");
|
|
152
|
-
const valuesSql = columns.map((c, i) => {
|
|
153
|
-
const placeholder = `$${i + 1}`;
|
|
154
|
-
return `${placeholder}::${c.pgType}`;
|
|
155
|
-
}).join(", ");
|
|
156
|
-
const params = columns.map((c) => c.value);
|
|
157
|
-
return { columnsSql, valuesSql, params };
|
|
158
|
-
}
|
|
159
|
-
static buildRawJsonUpsertQuery(schema, table, columns, conflictTarget) {
|
|
160
|
-
const { columnsSql, valuesSql, params } = _QueryUtils.buildInsertParts(columns);
|
|
161
|
-
const conflictSql = _QueryUtils.quotedList(conflictTarget);
|
|
162
|
-
const tsParamIdx = columns.findIndex((c) => c.column === "_last_synced_at") + 1;
|
|
163
|
-
if (tsParamIdx <= 0) {
|
|
164
|
-
throw new Error("buildRawJsonUpsertQuery requires _last_synced_at column");
|
|
165
|
-
}
|
|
166
|
-
const sql3 = `
|
|
167
|
-
INSERT INTO ${_QueryUtils.quoteIdent(schema)}.${_QueryUtils.quoteIdent(table)} (${columnsSql})
|
|
168
|
-
VALUES (${valuesSql})
|
|
169
|
-
ON CONFLICT (${conflictSql})
|
|
170
|
-
DO UPDATE SET
|
|
171
|
-
"_raw_data" = EXCLUDED."_raw_data",
|
|
172
|
-
"_last_synced_at" = $${tsParamIdx},
|
|
173
|
-
"_account_id" = EXCLUDED."_account_id"
|
|
174
|
-
WHERE ${_QueryUtils.quoteIdent(table)}."_last_synced_at" IS NULL
|
|
175
|
-
OR ${_QueryUtils.quoteIdent(table)}."_last_synced_at" < $${tsParamIdx}
|
|
176
|
-
RETURNING *
|
|
177
|
-
`;
|
|
178
|
-
return { sql: sql3, params };
|
|
179
|
-
}
|
|
180
|
-
};
|
|
181
|
-
|
|
182
|
-
// src/database/postgres.ts
|
|
183
135
|
var ORDERED_STRIPE_TABLES = [
|
|
184
|
-
"exchange_rates_from_usd",
|
|
185
136
|
"subscription_items",
|
|
186
|
-
"subscription_item_change_events_v2_beta",
|
|
187
137
|
"subscriptions",
|
|
188
138
|
"subscription_schedules",
|
|
189
139
|
"checkout_session_line_items",
|
|
@@ -253,7 +203,7 @@ var PostgresClient = class {
|
|
|
253
203
|
}
|
|
254
204
|
return results.flatMap((it) => it.rows);
|
|
255
205
|
}
|
|
256
|
-
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp
|
|
206
|
+
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp) {
|
|
257
207
|
const timestamp = syncTimestamp || (/* @__PURE__ */ new Date()).toISOString();
|
|
258
208
|
if (!entries.length) return [];
|
|
259
209
|
const chunkSize = 5;
|
|
@@ -288,33 +238,20 @@ var PostgresClient = class {
|
|
|
288
238
|
const prepared = (0, import_yesql.pg)(upsertSql, { useNullForMissing: true })(cleansed);
|
|
289
239
|
queries.push(this.pool.query(prepared.text, prepared.values));
|
|
290
240
|
} else {
|
|
291
|
-
const
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
];
|
|
306
|
-
for (const c of columns) {
|
|
307
|
-
if (c.value === void 0) {
|
|
308
|
-
throw new Error(`Missing required value for ${table}.${c.column}`);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
const { sql: upsertSql, params } = QueryUtils.buildRawJsonUpsertQuery(
|
|
312
|
-
this.config.schema,
|
|
313
|
-
table,
|
|
314
|
-
columns,
|
|
315
|
-
conflictTarget
|
|
316
|
-
);
|
|
317
|
-
queries.push(this.pool.query(upsertSql, params));
|
|
241
|
+
const rawData = JSON.stringify(entry);
|
|
242
|
+
const upsertSql = `
|
|
243
|
+
INSERT INTO "${this.config.schema}"."${table}" ("_raw_data", "_last_synced_at", "_account_id")
|
|
244
|
+
VALUES ($1::jsonb, $2, $3)
|
|
245
|
+
ON CONFLICT (id)
|
|
246
|
+
DO UPDATE SET
|
|
247
|
+
"_raw_data" = EXCLUDED."_raw_data",
|
|
248
|
+
"_last_synced_at" = $2,
|
|
249
|
+
"_account_id" = EXCLUDED."_account_id"
|
|
250
|
+
WHERE "${table}"."_last_synced_at" IS NULL
|
|
251
|
+
OR "${table}"."_last_synced_at" < $2
|
|
252
|
+
RETURNING *
|
|
253
|
+
`;
|
|
254
|
+
queries.push(this.pool.query(upsertSql, [rawData, timestamp, accountId]));
|
|
318
255
|
}
|
|
319
256
|
});
|
|
320
257
|
results.push(...await Promise.all(queries));
|
|
@@ -714,12 +651,7 @@ var PostgresClient = class {
|
|
|
714
651
|
} else {
|
|
715
652
|
await this.query(
|
|
716
653
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
717
|
-
SET cursor =
|
|
718
|
-
WHEN cursor IS NULL THEN $4
|
|
719
|
-
WHEN (cursor COLLATE "C") < ($4::text COLLATE "C") THEN $4
|
|
720
|
-
ELSE cursor
|
|
721
|
-
END,
|
|
722
|
-
updated_at = now()
|
|
654
|
+
SET cursor = $4, updated_at = now()
|
|
723
655
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
724
656
|
[accountId, runStartedAt, object, cursor]
|
|
725
657
|
);
|
|
@@ -730,17 +662,10 @@ var PostgresClient = class {
|
|
|
730
662
|
* This considers completed, error, AND running runs to ensure recovery syncs
|
|
731
663
|
* don't re-process data that was already synced before a crash.
|
|
732
664
|
* A 'running' status with a cursor means the process was killed mid-sync.
|
|
733
|
-
*
|
|
734
|
-
* Handles two cursor formats:
|
|
735
|
-
* - Numeric: compared as bigint for correct ordering
|
|
736
|
-
* - Composite cursors: compared as strings with COLLATE "C"
|
|
737
665
|
*/
|
|
738
666
|
async getLastCompletedCursor(accountId, object) {
|
|
739
667
|
const result = await this.query(
|
|
740
|
-
`SELECT
|
|
741
|
-
WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
|
|
742
|
-
ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
|
|
743
|
-
END as cursor
|
|
668
|
+
`SELECT MAX(o.cursor::bigint)::text as cursor
|
|
744
669
|
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
745
670
|
WHERE o."_account_id" = $1
|
|
746
671
|
AND o.object = $2
|
|
@@ -1025,269 +950,6 @@ function hashApiKey(apiKey) {
|
|
|
1025
950
|
return (0, import_crypto.createHash)("sha256").update(apiKey).digest("hex");
|
|
1026
951
|
}
|
|
1027
952
|
|
|
1028
|
-
// src/sigma/sigmaApi.ts
|
|
1029
|
-
var import_papaparse = __toESM(require("papaparse"), 1);
|
|
1030
|
-
var import_stripe2 = __toESM(require("stripe"), 1);
|
|
1031
|
-
var STRIPE_FILES_BASE = "https://files.stripe.com/v1";
|
|
1032
|
-
function sleep2(ms) {
|
|
1033
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1034
|
-
}
|
|
1035
|
-
function parseCsvObjects(csv) {
|
|
1036
|
-
const input = csv.replace(/^\uFEFF/, "");
|
|
1037
|
-
const parsed = import_papaparse.default.parse(input, {
|
|
1038
|
-
header: true,
|
|
1039
|
-
skipEmptyLines: "greedy"
|
|
1040
|
-
});
|
|
1041
|
-
if (parsed.errors.length > 0) {
|
|
1042
|
-
throw new Error(`Failed to parse Sigma CSV: ${parsed.errors[0]?.message ?? "unknown error"}`);
|
|
1043
|
-
}
|
|
1044
|
-
return parsed.data.filter((row) => row && Object.keys(row).length > 0).map(
|
|
1045
|
-
(row) => Object.fromEntries(
|
|
1046
|
-
Object.entries(row).map(([k, v]) => [k, v == null || v === "" ? null : String(v)])
|
|
1047
|
-
)
|
|
1048
|
-
);
|
|
1049
|
-
}
|
|
1050
|
-
function normalizeSigmaTimestampToIso(value) {
|
|
1051
|
-
const v = value.trim();
|
|
1052
|
-
if (!v) return null;
|
|
1053
|
-
const hasExplicitTz = /z$|[+-]\d{2}:?\d{2}$/i.test(v);
|
|
1054
|
-
const isoish = v.includes("T") ? v : v.replace(" ", "T");
|
|
1055
|
-
const candidate = hasExplicitTz ? isoish : `${isoish}Z`;
|
|
1056
|
-
const d = new Date(candidate);
|
|
1057
|
-
if (Number.isNaN(d.getTime())) return null;
|
|
1058
|
-
return d.toISOString();
|
|
1059
|
-
}
|
|
1060
|
-
async function fetchStripeText(url, apiKey, options) {
|
|
1061
|
-
const res = await fetch(url, {
|
|
1062
|
-
...options,
|
|
1063
|
-
headers: {
|
|
1064
|
-
...options.headers ?? {},
|
|
1065
|
-
Authorization: `Bearer ${apiKey}`
|
|
1066
|
-
}
|
|
1067
|
-
});
|
|
1068
|
-
const text = await res.text();
|
|
1069
|
-
if (!res.ok) {
|
|
1070
|
-
throw new Error(`Sigma file download error (${res.status}) for ${url}: ${text}`);
|
|
1071
|
-
}
|
|
1072
|
-
return text;
|
|
1073
|
-
}
|
|
1074
|
-
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1075
|
-
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1076
|
-
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1077
|
-
const stripe = new import_stripe2.default(params.apiKey);
|
|
1078
|
-
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1079
|
-
sql: params.sql
|
|
1080
|
-
});
|
|
1081
|
-
const queryRunId = created.id;
|
|
1082
|
-
const start = Date.now();
|
|
1083
|
-
let current = created;
|
|
1084
|
-
while (current.status === "running") {
|
|
1085
|
-
if (Date.now() - start > pollTimeoutMs) {
|
|
1086
|
-
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1087
|
-
}
|
|
1088
|
-
await sleep2(pollIntervalMs);
|
|
1089
|
-
current = await stripe.rawRequest(
|
|
1090
|
-
"GET",
|
|
1091
|
-
`/v1/sigma/query_runs/${queryRunId}`,
|
|
1092
|
-
{}
|
|
1093
|
-
);
|
|
1094
|
-
}
|
|
1095
|
-
if (current.status !== "succeeded") {
|
|
1096
|
-
throw new Error(
|
|
1097
|
-
`Sigma query run did not succeed (status=${current.status}) id=${queryRunId} error=${JSON.stringify(
|
|
1098
|
-
current.error
|
|
1099
|
-
)}`
|
|
1100
|
-
);
|
|
1101
|
-
}
|
|
1102
|
-
const fileId = current.result?.file;
|
|
1103
|
-
if (!fileId) {
|
|
1104
|
-
throw new Error(`Sigma query run succeeded but result.file is missing (id=${queryRunId})`);
|
|
1105
|
-
}
|
|
1106
|
-
const csv = await fetchStripeText(
|
|
1107
|
-
`${STRIPE_FILES_BASE}/files/${fileId}/contents`,
|
|
1108
|
-
params.apiKey,
|
|
1109
|
-
{ method: "GET" }
|
|
1110
|
-
);
|
|
1111
|
-
return { queryRunId, fileId, csv };
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
// src/sigma/sigmaIngestionConfigs.ts
|
|
1115
|
-
var SIGMA_INGESTION_CONFIGS = {
|
|
1116
|
-
subscription_item_change_events_v2_beta: {
|
|
1117
|
-
sigmaTable: "subscription_item_change_events_v2_beta",
|
|
1118
|
-
destinationTable: "subscription_item_change_events_v2_beta",
|
|
1119
|
-
pageSize: 1e4,
|
|
1120
|
-
cursor: {
|
|
1121
|
-
version: 1,
|
|
1122
|
-
columns: [
|
|
1123
|
-
{ column: "event_timestamp", type: "timestamp" },
|
|
1124
|
-
{ column: "event_type", type: "string" },
|
|
1125
|
-
{ column: "subscription_item_id", type: "string" }
|
|
1126
|
-
]
|
|
1127
|
-
},
|
|
1128
|
-
upsert: {
|
|
1129
|
-
conflictTarget: ["_account_id", "event_timestamp", "event_type", "subscription_item_id"],
|
|
1130
|
-
extraColumns: [
|
|
1131
|
-
{ column: "event_timestamp", pgType: "timestamptz", entryKey: "event_timestamp" },
|
|
1132
|
-
{ column: "event_type", pgType: "text", entryKey: "event_type" },
|
|
1133
|
-
{ column: "subscription_item_id", pgType: "text", entryKey: "subscription_item_id" }
|
|
1134
|
-
]
|
|
1135
|
-
}
|
|
1136
|
-
},
|
|
1137
|
-
exchange_rates_from_usd: {
|
|
1138
|
-
sigmaTable: "exchange_rates_from_usd",
|
|
1139
|
-
destinationTable: "exchange_rates_from_usd",
|
|
1140
|
-
pageSize: 1e4,
|
|
1141
|
-
cursor: {
|
|
1142
|
-
version: 1,
|
|
1143
|
-
columns: [
|
|
1144
|
-
{ column: "date", type: "string" },
|
|
1145
|
-
{ column: "sell_currency", type: "string" }
|
|
1146
|
-
]
|
|
1147
|
-
},
|
|
1148
|
-
upsert: {
|
|
1149
|
-
conflictTarget: ["_account_id", "date", "sell_currency"],
|
|
1150
|
-
extraColumns: [
|
|
1151
|
-
{ column: "date", pgType: "date", entryKey: "date" },
|
|
1152
|
-
{ column: "sell_currency", pgType: "text", entryKey: "sell_currency" }
|
|
1153
|
-
]
|
|
1154
|
-
}
|
|
1155
|
-
}
|
|
1156
|
-
};
|
|
1157
|
-
|
|
1158
|
-
// src/sigma/sigmaIngestion.ts
|
|
1159
|
-
var SIGMA_CURSOR_DELIM = "";
|
|
1160
|
-
function escapeSigmaSqlStringLiteral(value) {
|
|
1161
|
-
return value.replace(/'/g, "''");
|
|
1162
|
-
}
|
|
1163
|
-
function formatSigmaTimestampForSqlLiteral(date) {
|
|
1164
|
-
return date.toISOString().replace("T", " ").replace("Z", "");
|
|
1165
|
-
}
|
|
1166
|
-
function decodeSigmaCursorValues(spec, cursor) {
|
|
1167
|
-
const prefix = `v${spec.version}${SIGMA_CURSOR_DELIM}`;
|
|
1168
|
-
if (!cursor.startsWith(prefix)) {
|
|
1169
|
-
throw new Error(
|
|
1170
|
-
`Unrecognized Sigma cursor format (expected prefix ${JSON.stringify(prefix)}): ${cursor}`
|
|
1171
|
-
);
|
|
1172
|
-
}
|
|
1173
|
-
const parts = cursor.split(SIGMA_CURSOR_DELIM);
|
|
1174
|
-
const expected = 1 + spec.columns.length;
|
|
1175
|
-
if (parts.length !== expected) {
|
|
1176
|
-
throw new Error(`Malformed Sigma cursor: expected ${expected} parts, got ${parts.length}`);
|
|
1177
|
-
}
|
|
1178
|
-
return parts.slice(1);
|
|
1179
|
-
}
|
|
1180
|
-
function encodeSigmaCursor(spec, values) {
|
|
1181
|
-
if (values.length !== spec.columns.length) {
|
|
1182
|
-
throw new Error(
|
|
1183
|
-
`Cannot encode Sigma cursor: expected ${spec.columns.length} values, got ${values.length}`
|
|
1184
|
-
);
|
|
1185
|
-
}
|
|
1186
|
-
for (const v of values) {
|
|
1187
|
-
if (v.includes(SIGMA_CURSOR_DELIM)) {
|
|
1188
|
-
throw new Error("Cannot encode Sigma cursor: value contains delimiter character");
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
return [`v${spec.version}`, ...values].join(SIGMA_CURSOR_DELIM);
|
|
1192
|
-
}
|
|
1193
|
-
function sigmaSqlLiteralForCursorValue(spec, rawValue) {
|
|
1194
|
-
switch (spec.type) {
|
|
1195
|
-
case "timestamp": {
|
|
1196
|
-
const d = new Date(rawValue);
|
|
1197
|
-
if (Number.isNaN(d.getTime())) {
|
|
1198
|
-
throw new Error(`Invalid timestamp cursor value for ${spec.column}: ${rawValue}`);
|
|
1199
|
-
}
|
|
1200
|
-
return `timestamp '${formatSigmaTimestampForSqlLiteral(d)}'`;
|
|
1201
|
-
}
|
|
1202
|
-
case "number": {
|
|
1203
|
-
if (!/^-?\d+(\.\d+)?$/.test(rawValue)) {
|
|
1204
|
-
throw new Error(`Invalid numeric cursor value for ${spec.column}: ${rawValue}`);
|
|
1205
|
-
}
|
|
1206
|
-
return rawValue;
|
|
1207
|
-
}
|
|
1208
|
-
case "string":
|
|
1209
|
-
return `'${escapeSigmaSqlStringLiteral(rawValue)}'`;
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
|
-
function buildSigmaCursorWhereClause(spec, cursorValues) {
|
|
1213
|
-
if (cursorValues.length !== spec.columns.length) {
|
|
1214
|
-
throw new Error(
|
|
1215
|
-
`Cannot build Sigma cursor predicate: expected ${spec.columns.length} values, got ${cursorValues.length}`
|
|
1216
|
-
);
|
|
1217
|
-
}
|
|
1218
|
-
const cols = spec.columns.map((c) => c.column);
|
|
1219
|
-
const lits = spec.columns.map((c, i) => sigmaSqlLiteralForCursorValue(c, cursorValues[i] ?? ""));
|
|
1220
|
-
const ors = [];
|
|
1221
|
-
for (let i = 0; i < cols.length; i++) {
|
|
1222
|
-
const ands = [];
|
|
1223
|
-
for (let j = 0; j < i; j++) {
|
|
1224
|
-
ands.push(`${cols[j]} = ${lits[j]}`);
|
|
1225
|
-
}
|
|
1226
|
-
ands.push(`${cols[i]} > ${lits[i]}`);
|
|
1227
|
-
ors.push(`(${ands.join(" AND ")})`);
|
|
1228
|
-
}
|
|
1229
|
-
return ors.join(" OR ");
|
|
1230
|
-
}
|
|
1231
|
-
function buildSigmaQuery(config, cursor) {
|
|
1232
|
-
const select = config.select === void 0 || config.select === "*" ? "*" : config.select.join(", ");
|
|
1233
|
-
const whereParts = [];
|
|
1234
|
-
if (config.additionalWhere) {
|
|
1235
|
-
whereParts.push(`(${config.additionalWhere})`);
|
|
1236
|
-
}
|
|
1237
|
-
if (cursor) {
|
|
1238
|
-
const values = decodeSigmaCursorValues(config.cursor, cursor);
|
|
1239
|
-
const predicate = buildSigmaCursorWhereClause(config.cursor, values);
|
|
1240
|
-
whereParts.push(`(${predicate})`);
|
|
1241
|
-
}
|
|
1242
|
-
const whereClause = whereParts.length > 0 ? `WHERE ${whereParts.join(" AND ")}` : "";
|
|
1243
|
-
const orderBy = config.cursor.columns.map((c) => c.column).join(", ");
|
|
1244
|
-
return [
|
|
1245
|
-
`SELECT ${select} FROM ${config.sigmaTable}`,
|
|
1246
|
-
whereClause,
|
|
1247
|
-
`ORDER BY ${orderBy} ASC`,
|
|
1248
|
-
`LIMIT ${config.pageSize}`
|
|
1249
|
-
].filter(Boolean).join(" ");
|
|
1250
|
-
}
|
|
1251
|
-
function defaultSigmaRowToEntry(config, row) {
|
|
1252
|
-
const out = { ...row };
|
|
1253
|
-
for (const col of config.cursor.columns) {
|
|
1254
|
-
const raw = row[col.column];
|
|
1255
|
-
if (raw == null) {
|
|
1256
|
-
throw new Error(`Sigma row missing required cursor column: ${col.column}`);
|
|
1257
|
-
}
|
|
1258
|
-
if (col.type === "timestamp") {
|
|
1259
|
-
const normalized = normalizeSigmaTimestampToIso(raw);
|
|
1260
|
-
if (!normalized) {
|
|
1261
|
-
throw new Error(`Sigma row has invalid timestamp for ${col.column}: ${raw}`);
|
|
1262
|
-
}
|
|
1263
|
-
out[col.column] = normalized;
|
|
1264
|
-
} else if (col.type === "string") {
|
|
1265
|
-
const v = raw.trim();
|
|
1266
|
-
if (!v) {
|
|
1267
|
-
throw new Error(`Sigma row has empty string for required cursor column: ${col.column}`);
|
|
1268
|
-
}
|
|
1269
|
-
out[col.column] = v;
|
|
1270
|
-
} else {
|
|
1271
|
-
const v = raw.trim();
|
|
1272
|
-
if (!v) {
|
|
1273
|
-
throw new Error(`Sigma row has empty value for required cursor column: ${col.column}`);
|
|
1274
|
-
}
|
|
1275
|
-
out[col.column] = v;
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
return out;
|
|
1279
|
-
}
|
|
1280
|
-
function sigmaCursorFromEntry(config, entry) {
|
|
1281
|
-
const values = config.cursor.columns.map((c) => {
|
|
1282
|
-
const raw = entry[c.column];
|
|
1283
|
-
if (raw == null) {
|
|
1284
|
-
throw new Error(`Cannot build cursor: entry missing ${c.column}`);
|
|
1285
|
-
}
|
|
1286
|
-
return String(raw);
|
|
1287
|
-
});
|
|
1288
|
-
return encodeSigmaCursor(config.cursor, values);
|
|
1289
|
-
}
|
|
1290
|
-
|
|
1291
953
|
// src/stripeSync.ts
|
|
1292
954
|
function getUniqueIds(entries, key) {
|
|
1293
955
|
const set = new Set(
|
|
@@ -1298,7 +960,7 @@ function getUniqueIds(entries, key) {
|
|
|
1298
960
|
var StripeSync = class {
|
|
1299
961
|
constructor(config) {
|
|
1300
962
|
this.config = config;
|
|
1301
|
-
const baseStripe = new
|
|
963
|
+
const baseStripe = new import_stripe2.default(config.stripeSecretKey, {
|
|
1302
964
|
// https://github.com/stripe/stripe-node#configuration
|
|
1303
965
|
// @ts-ignore
|
|
1304
966
|
apiVersion: config.stripeApiVersion,
|
|
@@ -1699,17 +1361,6 @@ var StripeSync = class {
|
|
|
1699
1361
|
listFn: (p) => this.stripe.checkout.sessions.list(p),
|
|
1700
1362
|
upsertFn: (items, id) => this.upsertCheckoutSessions(items, id),
|
|
1701
1363
|
supportsCreatedFilter: true
|
|
1702
|
-
},
|
|
1703
|
-
// Sigma-backed resources
|
|
1704
|
-
subscription_item_change_events_v2_beta: {
|
|
1705
|
-
order: 18,
|
|
1706
|
-
supportsCreatedFilter: false,
|
|
1707
|
-
sigma: SIGMA_INGESTION_CONFIGS.subscription_item_change_events_v2_beta
|
|
1708
|
-
},
|
|
1709
|
-
exchange_rates_from_usd: {
|
|
1710
|
-
order: 19,
|
|
1711
|
-
supportsCreatedFilter: false,
|
|
1712
|
-
sigma: SIGMA_INGESTION_CONFIGS.exchange_rates_from_usd
|
|
1713
1364
|
}
|
|
1714
1365
|
};
|
|
1715
1366
|
async processEvent(event) {
|
|
@@ -1742,13 +1393,7 @@ var StripeSync = class {
|
|
|
1742
1393
|
* Order is determined by the `order` field in resourceRegistry.
|
|
1743
1394
|
*/
|
|
1744
1395
|
getSupportedSyncObjects() {
|
|
1745
|
-
|
|
1746
|
-
if (!this.config.enableSigmaSync) {
|
|
1747
|
-
return all.filter(
|
|
1748
|
-
(o) => o !== "subscription_item_change_events_v2_beta" && o !== "exchange_rates_from_usd"
|
|
1749
|
-
);
|
|
1750
|
-
}
|
|
1751
|
-
return all;
|
|
1396
|
+
return Object.entries(this.resourceRegistry).sort(([, a], [, b]) => a.order - b.order).map(([key]) => key);
|
|
1752
1397
|
}
|
|
1753
1398
|
// Event handler methods
|
|
1754
1399
|
async handleChargeEvent(event, accountId) {
|
|
@@ -1827,7 +1472,7 @@ var StripeSync = class {
|
|
|
1827
1472
|
);
|
|
1828
1473
|
await this.upsertProducts([product], accountId, this.getSyncTimestamp(event, refetched));
|
|
1829
1474
|
} catch (err) {
|
|
1830
|
-
if (err instanceof
|
|
1475
|
+
if (err instanceof import_stripe2.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1831
1476
|
const product = event.data.object;
|
|
1832
1477
|
await this.deleteProduct(product.id);
|
|
1833
1478
|
} else {
|
|
@@ -1847,7 +1492,7 @@ var StripeSync = class {
|
|
|
1847
1492
|
);
|
|
1848
1493
|
await this.upsertPrices([price], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1849
1494
|
} catch (err) {
|
|
1850
|
-
if (err instanceof
|
|
1495
|
+
if (err instanceof import_stripe2.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1851
1496
|
const price = event.data.object;
|
|
1852
1497
|
await this.deletePrice(price.id);
|
|
1853
1498
|
} else {
|
|
@@ -1867,7 +1512,7 @@ var StripeSync = class {
|
|
|
1867
1512
|
);
|
|
1868
1513
|
await this.upsertPlans([plan], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1869
1514
|
} catch (err) {
|
|
1870
|
-
if (err instanceof
|
|
1515
|
+
if (err instanceof import_stripe2.default.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1871
1516
|
const plan = event.data.object;
|
|
1872
1517
|
await this.deletePlan(plan.id);
|
|
1873
1518
|
} else {
|
|
@@ -2114,10 +1759,10 @@ var StripeSync = class {
|
|
|
2114
1759
|
let cursor = null;
|
|
2115
1760
|
if (!params?.created) {
|
|
2116
1761
|
if (objRun?.cursor) {
|
|
2117
|
-
cursor = objRun.cursor;
|
|
1762
|
+
cursor = parseInt(objRun.cursor);
|
|
2118
1763
|
} else {
|
|
2119
1764
|
const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
|
|
2120
|
-
cursor = lastCursor
|
|
1765
|
+
cursor = lastCursor ? parseInt(lastCursor) : null;
|
|
2121
1766
|
}
|
|
2122
1767
|
}
|
|
2123
1768
|
const result = await this.fetchOnePage(
|
|
@@ -2172,18 +1817,9 @@ var StripeSync = class {
|
|
|
2172
1817
|
throw new Error(`Unsupported object type for processNext: ${object}`);
|
|
2173
1818
|
}
|
|
2174
1819
|
try {
|
|
2175
|
-
if (config.sigma) {
|
|
2176
|
-
return await this.fetchOneSigmaPage(
|
|
2177
|
-
accountId,
|
|
2178
|
-
resourceName,
|
|
2179
|
-
runStartedAt,
|
|
2180
|
-
cursor,
|
|
2181
|
-
config.sigma
|
|
2182
|
-
);
|
|
2183
|
-
}
|
|
2184
1820
|
const listParams = { limit };
|
|
2185
1821
|
if (config.supportsCreatedFilter) {
|
|
2186
|
-
const created = params?.created ?? (cursor
|
|
1822
|
+
const created = params?.created ?? (cursor ? { gte: cursor } : void 0);
|
|
2187
1823
|
if (created) {
|
|
2188
1824
|
listParams.created = created;
|
|
2189
1825
|
}
|
|
@@ -2228,97 +1864,6 @@ var StripeSync = class {
|
|
|
2228
1864
|
throw error;
|
|
2229
1865
|
}
|
|
2230
1866
|
}
|
|
2231
|
-
async getSigmaFallbackCursorFromDestination(accountId, sigmaConfig) {
|
|
2232
|
-
const cursorCols = sigmaConfig.cursor.columns;
|
|
2233
|
-
const selectCols = cursorCols.map((c) => `"${c.column}"`).join(", ");
|
|
2234
|
-
const orderBy = cursorCols.map((c) => `"${c.column}" DESC`).join(", ");
|
|
2235
|
-
const result = await this.postgresClient.query(
|
|
2236
|
-
`SELECT ${selectCols}
|
|
2237
|
-
FROM "stripe"."${sigmaConfig.destinationTable}"
|
|
2238
|
-
WHERE "_account_id" = $1
|
|
2239
|
-
ORDER BY ${orderBy}
|
|
2240
|
-
LIMIT 1`,
|
|
2241
|
-
[accountId]
|
|
2242
|
-
);
|
|
2243
|
-
if (result.rows.length === 0) return null;
|
|
2244
|
-
const row = result.rows[0];
|
|
2245
|
-
const entryForCursor = {};
|
|
2246
|
-
for (const c of cursorCols) {
|
|
2247
|
-
const v = row[c.column];
|
|
2248
|
-
if (v == null) {
|
|
2249
|
-
throw new Error(
|
|
2250
|
-
`Sigma fallback cursor query returned null for ${sigmaConfig.destinationTable}.${c.column}`
|
|
2251
|
-
);
|
|
2252
|
-
}
|
|
2253
|
-
if (c.type === "timestamp") {
|
|
2254
|
-
const d = v instanceof Date ? v : new Date(String(v));
|
|
2255
|
-
if (Number.isNaN(d.getTime())) {
|
|
2256
|
-
throw new Error(
|
|
2257
|
-
`Sigma fallback cursor query returned invalid timestamp for ${sigmaConfig.destinationTable}.${c.column}: ${String(
|
|
2258
|
-
v
|
|
2259
|
-
)}`
|
|
2260
|
-
);
|
|
2261
|
-
}
|
|
2262
|
-
entryForCursor[c.column] = d.toISOString();
|
|
2263
|
-
} else {
|
|
2264
|
-
entryForCursor[c.column] = String(v);
|
|
2265
|
-
}
|
|
2266
|
-
}
|
|
2267
|
-
return sigmaCursorFromEntry(sigmaConfig, entryForCursor);
|
|
2268
|
-
}
|
|
2269
|
-
async fetchOneSigmaPage(accountId, resourceName, runStartedAt, cursor, sigmaConfig) {
|
|
2270
|
-
if (!this.config.stripeSecretKey) {
|
|
2271
|
-
throw new Error("Sigma sync requested but stripeSecretKey is not configured.");
|
|
2272
|
-
}
|
|
2273
|
-
if (resourceName !== sigmaConfig.destinationTable) {
|
|
2274
|
-
throw new Error(
|
|
2275
|
-
`Sigma sync config mismatch: resourceName=${resourceName} destinationTable=${sigmaConfig.destinationTable}`
|
|
2276
|
-
);
|
|
2277
|
-
}
|
|
2278
|
-
const effectiveCursor = cursor ?? await this.getSigmaFallbackCursorFromDestination(accountId, sigmaConfig);
|
|
2279
|
-
const sigmaSql = buildSigmaQuery(sigmaConfig, effectiveCursor);
|
|
2280
|
-
this.config.logger?.info(
|
|
2281
|
-
{ object: resourceName, pageSize: sigmaConfig.pageSize, hasCursor: Boolean(effectiveCursor) },
|
|
2282
|
-
"Sigma sync: running query"
|
|
2283
|
-
);
|
|
2284
|
-
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
2285
|
-
apiKey: this.config.stripeSecretKey,
|
|
2286
|
-
sql: sigmaSql,
|
|
2287
|
-
logger: this.config.logger
|
|
2288
|
-
});
|
|
2289
|
-
const rows = parseCsvObjects(csv);
|
|
2290
|
-
if (rows.length === 0) {
|
|
2291
|
-
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2292
|
-
return { processed: 0, hasMore: false, runStartedAt };
|
|
2293
|
-
}
|
|
2294
|
-
const entries = rows.map(
|
|
2295
|
-
(row) => defaultSigmaRowToEntry(sigmaConfig, row)
|
|
2296
|
-
);
|
|
2297
|
-
this.config.logger?.info(
|
|
2298
|
-
{ object: resourceName, rows: entries.length, queryRunId, fileId },
|
|
2299
|
-
"Sigma sync: upserting rows"
|
|
2300
|
-
);
|
|
2301
|
-
await this.postgresClient.upsertManyWithTimestampProtection(
|
|
2302
|
-
entries,
|
|
2303
|
-
resourceName,
|
|
2304
|
-
accountId,
|
|
2305
|
-
void 0,
|
|
2306
|
-
sigmaConfig.upsert
|
|
2307
|
-
);
|
|
2308
|
-
await this.postgresClient.incrementObjectProgress(
|
|
2309
|
-
accountId,
|
|
2310
|
-
runStartedAt,
|
|
2311
|
-
resourceName,
|
|
2312
|
-
entries.length
|
|
2313
|
-
);
|
|
2314
|
-
const newCursor = sigmaCursorFromEntry(sigmaConfig, entries[entries.length - 1]);
|
|
2315
|
-
await this.postgresClient.updateObjectCursor(accountId, runStartedAt, resourceName, newCursor);
|
|
2316
|
-
const hasMore = rows.length === sigmaConfig.pageSize;
|
|
2317
|
-
if (!hasMore) {
|
|
2318
|
-
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2319
|
-
}
|
|
2320
|
-
return { processed: entries.length, hasMore, runStartedAt };
|
|
2321
|
-
}
|
|
2322
1867
|
/**
|
|
2323
1868
|
* Process all pages for all (or specified) object types until complete.
|
|
2324
1869
|
*
|
|
@@ -2447,12 +1992,6 @@ var StripeSync = class {
|
|
|
2447
1992
|
case "checkout_sessions":
|
|
2448
1993
|
results.checkoutSessions = result;
|
|
2449
1994
|
break;
|
|
2450
|
-
case "subscription_item_change_events_v2_beta":
|
|
2451
|
-
results.subscriptionItemChangeEventsV2Beta = result;
|
|
2452
|
-
break;
|
|
2453
|
-
case "exchange_rates_from_usd":
|
|
2454
|
-
results.exchangeRatesFromUsd = result;
|
|
2455
|
-
break;
|
|
2456
1995
|
}
|
|
2457
1996
|
}
|
|
2458
1997
|
}
|
|
@@ -4016,7 +3555,5 @@ var VERSION = package_default.version;
|
|
|
4016
3555
|
VERSION,
|
|
4017
3556
|
createStripeWebSocketClient,
|
|
4018
3557
|
hashApiKey,
|
|
4019
|
-
normalizeSigmaTimestampToIso,
|
|
4020
|
-
parseCsvObjects,
|
|
4021
3558
|
runMigrations
|
|
4022
3559
|
});
|