stripe-experiment-sync 1.0.9-beta.1765909347 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-SI3VFP3M.js → chunk-OLVA37VZ.js} +5 -5
- package/dist/{chunk-2CYYWBTD.js → chunk-PQ2T7XTY.js} +27 -486
- package/dist/{chunk-DBJCCGXP.js → chunk-PWJLHHPY.js} +10 -28
- package/dist/{chunk-3W3CERIG.js → chunk-RR5BGG4F.js} +1 -3
- package/dist/cli/index.cjs +43 -521
- package/dist/cli/index.js +6 -7
- package/dist/cli/lib.cjs +41 -518
- package/dist/cli/lib.d.cts +0 -2
- package/dist/cli/lib.d.ts +0 -2
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +27 -490
- package/dist/index.d.cts +4 -107
- package/dist/index.d.ts +4 -107
- package/dist/index.js +2 -6
- package/dist/supabase/index.cjs +5 -7
- package/dist/supabase/index.js +2 -2
- package/package.json +1 -3
- package/dist/migrations/0059_sigma_subscription_item_change_events_v2_beta.sql +0 -61
- package/dist/migrations/0060_sigma_exchange_rates_from_usd.sql +0 -38
|
@@ -1,62 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
package_default
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-RR5BGG4F.js";
|
|
4
4
|
|
|
5
5
|
// src/stripeSync.ts
|
|
6
|
-
import
|
|
6
|
+
import Stripe2 from "stripe";
|
|
7
7
|
import { pg as sql2 } from "yesql";
|
|
8
8
|
|
|
9
9
|
// src/database/postgres.ts
|
|
10
10
|
import pg from "pg";
|
|
11
11
|
import { pg as sql } from "yesql";
|
|
12
|
-
|
|
13
|
-
// src/database/QueryUtils.ts
|
|
14
|
-
var QueryUtils = class _QueryUtils {
|
|
15
|
-
constructor() {
|
|
16
|
-
}
|
|
17
|
-
static quoteIdent(name) {
|
|
18
|
-
return `"${name}"`;
|
|
19
|
-
}
|
|
20
|
-
static quotedList(names) {
|
|
21
|
-
return names.map(_QueryUtils.quoteIdent).join(", ");
|
|
22
|
-
}
|
|
23
|
-
static buildInsertParts(columns) {
|
|
24
|
-
const columnsSql = columns.map((c) => _QueryUtils.quoteIdent(c.column)).join(", ");
|
|
25
|
-
const valuesSql = columns.map((c, i) => {
|
|
26
|
-
const placeholder = `$${i + 1}`;
|
|
27
|
-
return `${placeholder}::${c.pgType}`;
|
|
28
|
-
}).join(", ");
|
|
29
|
-
const params = columns.map((c) => c.value);
|
|
30
|
-
return { columnsSql, valuesSql, params };
|
|
31
|
-
}
|
|
32
|
-
static buildRawJsonUpsertQuery(schema, table, columns, conflictTarget) {
|
|
33
|
-
const { columnsSql, valuesSql, params } = _QueryUtils.buildInsertParts(columns);
|
|
34
|
-
const conflictSql = _QueryUtils.quotedList(conflictTarget);
|
|
35
|
-
const tsParamIdx = columns.findIndex((c) => c.column === "_last_synced_at") + 1;
|
|
36
|
-
if (tsParamIdx <= 0) {
|
|
37
|
-
throw new Error("buildRawJsonUpsertQuery requires _last_synced_at column");
|
|
38
|
-
}
|
|
39
|
-
const sql3 = `
|
|
40
|
-
INSERT INTO ${_QueryUtils.quoteIdent(schema)}.${_QueryUtils.quoteIdent(table)} (${columnsSql})
|
|
41
|
-
VALUES (${valuesSql})
|
|
42
|
-
ON CONFLICT (${conflictSql})
|
|
43
|
-
DO UPDATE SET
|
|
44
|
-
"_raw_data" = EXCLUDED."_raw_data",
|
|
45
|
-
"_last_synced_at" = $${tsParamIdx},
|
|
46
|
-
"_account_id" = EXCLUDED."_account_id"
|
|
47
|
-
WHERE ${_QueryUtils.quoteIdent(table)}."_last_synced_at" IS NULL
|
|
48
|
-
OR ${_QueryUtils.quoteIdent(table)}."_last_synced_at" < $${tsParamIdx}
|
|
49
|
-
RETURNING *
|
|
50
|
-
`;
|
|
51
|
-
return { sql: sql3, params };
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
// src/database/postgres.ts
|
|
56
12
|
var ORDERED_STRIPE_TABLES = [
|
|
57
|
-
"exchange_rates_from_usd",
|
|
58
13
|
"subscription_items",
|
|
59
|
-
"subscription_item_change_events_v2_beta",
|
|
60
14
|
"subscriptions",
|
|
61
15
|
"subscription_schedules",
|
|
62
16
|
"checkout_session_line_items",
|
|
@@ -126,7 +80,7 @@ var PostgresClient = class {
|
|
|
126
80
|
}
|
|
127
81
|
return results.flatMap((it) => it.rows);
|
|
128
82
|
}
|
|
129
|
-
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp
|
|
83
|
+
async upsertManyWithTimestampProtection(entries, table, accountId, syncTimestamp) {
|
|
130
84
|
const timestamp = syncTimestamp || (/* @__PURE__ */ new Date()).toISOString();
|
|
131
85
|
if (!entries.length) return [];
|
|
132
86
|
const chunkSize = 5;
|
|
@@ -161,33 +115,20 @@ var PostgresClient = class {
|
|
|
161
115
|
const prepared = sql(upsertSql, { useNullForMissing: true })(cleansed);
|
|
162
116
|
queries.push(this.pool.query(prepared.text, prepared.values));
|
|
163
117
|
} else {
|
|
164
|
-
const
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
];
|
|
179
|
-
for (const c of columns) {
|
|
180
|
-
if (c.value === void 0) {
|
|
181
|
-
throw new Error(`Missing required value for ${table}.${c.column}`);
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
const { sql: upsertSql, params } = QueryUtils.buildRawJsonUpsertQuery(
|
|
185
|
-
this.config.schema,
|
|
186
|
-
table,
|
|
187
|
-
columns,
|
|
188
|
-
conflictTarget
|
|
189
|
-
);
|
|
190
|
-
queries.push(this.pool.query(upsertSql, params));
|
|
118
|
+
const rawData = JSON.stringify(entry);
|
|
119
|
+
const upsertSql = `
|
|
120
|
+
INSERT INTO "${this.config.schema}"."${table}" ("_raw_data", "_last_synced_at", "_account_id")
|
|
121
|
+
VALUES ($1::jsonb, $2, $3)
|
|
122
|
+
ON CONFLICT (id)
|
|
123
|
+
DO UPDATE SET
|
|
124
|
+
"_raw_data" = EXCLUDED."_raw_data",
|
|
125
|
+
"_last_synced_at" = $2,
|
|
126
|
+
"_account_id" = EXCLUDED."_account_id"
|
|
127
|
+
WHERE "${table}"."_last_synced_at" IS NULL
|
|
128
|
+
OR "${table}"."_last_synced_at" < $2
|
|
129
|
+
RETURNING *
|
|
130
|
+
`;
|
|
131
|
+
queries.push(this.pool.query(upsertSql, [rawData, timestamp, accountId]));
|
|
191
132
|
}
|
|
192
133
|
});
|
|
193
134
|
results.push(...await Promise.all(queries));
|
|
@@ -587,12 +528,7 @@ var PostgresClient = class {
|
|
|
587
528
|
} else {
|
|
588
529
|
await this.query(
|
|
589
530
|
`UPDATE "${this.config.schema}"."_sync_obj_runs"
|
|
590
|
-
SET cursor =
|
|
591
|
-
WHEN cursor IS NULL THEN $4
|
|
592
|
-
WHEN (cursor COLLATE "C") < ($4::text COLLATE "C") THEN $4
|
|
593
|
-
ELSE cursor
|
|
594
|
-
END,
|
|
595
|
-
updated_at = now()
|
|
531
|
+
SET cursor = $4, updated_at = now()
|
|
596
532
|
WHERE "_account_id" = $1 AND run_started_at = $2 AND object = $3`,
|
|
597
533
|
[accountId, runStartedAt, object, cursor]
|
|
598
534
|
);
|
|
@@ -603,17 +539,10 @@ var PostgresClient = class {
|
|
|
603
539
|
* This considers completed, error, AND running runs to ensure recovery syncs
|
|
604
540
|
* don't re-process data that was already synced before a crash.
|
|
605
541
|
* A 'running' status with a cursor means the process was killed mid-sync.
|
|
606
|
-
*
|
|
607
|
-
* Handles two cursor formats:
|
|
608
|
-
* - Numeric: compared as bigint for correct ordering
|
|
609
|
-
* - Composite cursors: compared as strings with COLLATE "C"
|
|
610
542
|
*/
|
|
611
543
|
async getLastCompletedCursor(accountId, object) {
|
|
612
544
|
const result = await this.query(
|
|
613
|
-
`SELECT
|
|
614
|
-
WHEN BOOL_OR(o.cursor !~ '^\\d+$') THEN MAX(o.cursor COLLATE "C")
|
|
615
|
-
ELSE MAX(CASE WHEN o.cursor ~ '^\\d+$' THEN o.cursor::bigint END)::text
|
|
616
|
-
END as cursor
|
|
545
|
+
`SELECT MAX(o.cursor::bigint)::text as cursor
|
|
617
546
|
FROM "${this.config.schema}"."_sync_obj_runs" o
|
|
618
547
|
WHERE o."_account_id" = $1
|
|
619
548
|
AND o.object = $2
|
|
@@ -898,269 +827,6 @@ function hashApiKey(apiKey) {
|
|
|
898
827
|
return createHash("sha256").update(apiKey).digest("hex");
|
|
899
828
|
}
|
|
900
829
|
|
|
901
|
-
// src/sigma/sigmaApi.ts
|
|
902
|
-
import Papa from "papaparse";
|
|
903
|
-
import Stripe2 from "stripe";
|
|
904
|
-
var STRIPE_FILES_BASE = "https://files.stripe.com/v1";
|
|
905
|
-
function sleep2(ms) {
|
|
906
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
907
|
-
}
|
|
908
|
-
function parseCsvObjects(csv) {
|
|
909
|
-
const input = csv.replace(/^\uFEFF/, "");
|
|
910
|
-
const parsed = Papa.parse(input, {
|
|
911
|
-
header: true,
|
|
912
|
-
skipEmptyLines: "greedy"
|
|
913
|
-
});
|
|
914
|
-
if (parsed.errors.length > 0) {
|
|
915
|
-
throw new Error(`Failed to parse Sigma CSV: ${parsed.errors[0]?.message ?? "unknown error"}`);
|
|
916
|
-
}
|
|
917
|
-
return parsed.data.filter((row) => row && Object.keys(row).length > 0).map(
|
|
918
|
-
(row) => Object.fromEntries(
|
|
919
|
-
Object.entries(row).map(([k, v]) => [k, v == null || v === "" ? null : String(v)])
|
|
920
|
-
)
|
|
921
|
-
);
|
|
922
|
-
}
|
|
923
|
-
function normalizeSigmaTimestampToIso(value) {
|
|
924
|
-
const v = value.trim();
|
|
925
|
-
if (!v) return null;
|
|
926
|
-
const hasExplicitTz = /z$|[+-]\d{2}:?\d{2}$/i.test(v);
|
|
927
|
-
const isoish = v.includes("T") ? v : v.replace(" ", "T");
|
|
928
|
-
const candidate = hasExplicitTz ? isoish : `${isoish}Z`;
|
|
929
|
-
const d = new Date(candidate);
|
|
930
|
-
if (Number.isNaN(d.getTime())) return null;
|
|
931
|
-
return d.toISOString();
|
|
932
|
-
}
|
|
933
|
-
async function fetchStripeText(url, apiKey, options) {
|
|
934
|
-
const res = await fetch(url, {
|
|
935
|
-
...options,
|
|
936
|
-
headers: {
|
|
937
|
-
...options.headers ?? {},
|
|
938
|
-
Authorization: `Bearer ${apiKey}`
|
|
939
|
-
}
|
|
940
|
-
});
|
|
941
|
-
const text = await res.text();
|
|
942
|
-
if (!res.ok) {
|
|
943
|
-
throw new Error(`Sigma file download error (${res.status}) for ${url}: ${text}`);
|
|
944
|
-
}
|
|
945
|
-
return text;
|
|
946
|
-
}
|
|
947
|
-
async function runSigmaQueryAndDownloadCsv(params) {
|
|
948
|
-
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
949
|
-
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
950
|
-
const stripe = new Stripe2(params.apiKey);
|
|
951
|
-
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
952
|
-
sql: params.sql
|
|
953
|
-
});
|
|
954
|
-
const queryRunId = created.id;
|
|
955
|
-
const start = Date.now();
|
|
956
|
-
let current = created;
|
|
957
|
-
while (current.status === "running") {
|
|
958
|
-
if (Date.now() - start > pollTimeoutMs) {
|
|
959
|
-
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
960
|
-
}
|
|
961
|
-
await sleep2(pollIntervalMs);
|
|
962
|
-
current = await stripe.rawRequest(
|
|
963
|
-
"GET",
|
|
964
|
-
`/v1/sigma/query_runs/${queryRunId}`,
|
|
965
|
-
{}
|
|
966
|
-
);
|
|
967
|
-
}
|
|
968
|
-
if (current.status !== "succeeded") {
|
|
969
|
-
throw new Error(
|
|
970
|
-
`Sigma query run did not succeed (status=${current.status}) id=${queryRunId} error=${JSON.stringify(
|
|
971
|
-
current.error
|
|
972
|
-
)}`
|
|
973
|
-
);
|
|
974
|
-
}
|
|
975
|
-
const fileId = current.result?.file;
|
|
976
|
-
if (!fileId) {
|
|
977
|
-
throw new Error(`Sigma query run succeeded but result.file is missing (id=${queryRunId})`);
|
|
978
|
-
}
|
|
979
|
-
const csv = await fetchStripeText(
|
|
980
|
-
`${STRIPE_FILES_BASE}/files/${fileId}/contents`,
|
|
981
|
-
params.apiKey,
|
|
982
|
-
{ method: "GET" }
|
|
983
|
-
);
|
|
984
|
-
return { queryRunId, fileId, csv };
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
// src/sigma/sigmaIngestionConfigs.ts
|
|
988
|
-
var SIGMA_INGESTION_CONFIGS = {
|
|
989
|
-
subscription_item_change_events_v2_beta: {
|
|
990
|
-
sigmaTable: "subscription_item_change_events_v2_beta",
|
|
991
|
-
destinationTable: "subscription_item_change_events_v2_beta",
|
|
992
|
-
pageSize: 1e4,
|
|
993
|
-
cursor: {
|
|
994
|
-
version: 1,
|
|
995
|
-
columns: [
|
|
996
|
-
{ column: "event_timestamp", type: "timestamp" },
|
|
997
|
-
{ column: "event_type", type: "string" },
|
|
998
|
-
{ column: "subscription_item_id", type: "string" }
|
|
999
|
-
]
|
|
1000
|
-
},
|
|
1001
|
-
upsert: {
|
|
1002
|
-
conflictTarget: ["_account_id", "event_timestamp", "event_type", "subscription_item_id"],
|
|
1003
|
-
extraColumns: [
|
|
1004
|
-
{ column: "event_timestamp", pgType: "timestamptz", entryKey: "event_timestamp" },
|
|
1005
|
-
{ column: "event_type", pgType: "text", entryKey: "event_type" },
|
|
1006
|
-
{ column: "subscription_item_id", pgType: "text", entryKey: "subscription_item_id" }
|
|
1007
|
-
]
|
|
1008
|
-
}
|
|
1009
|
-
},
|
|
1010
|
-
exchange_rates_from_usd: {
|
|
1011
|
-
sigmaTable: "exchange_rates_from_usd",
|
|
1012
|
-
destinationTable: "exchange_rates_from_usd",
|
|
1013
|
-
pageSize: 1e4,
|
|
1014
|
-
cursor: {
|
|
1015
|
-
version: 1,
|
|
1016
|
-
columns: [
|
|
1017
|
-
{ column: "date", type: "string" },
|
|
1018
|
-
{ column: "sell_currency", type: "string" }
|
|
1019
|
-
]
|
|
1020
|
-
},
|
|
1021
|
-
upsert: {
|
|
1022
|
-
conflictTarget: ["_account_id", "date", "sell_currency"],
|
|
1023
|
-
extraColumns: [
|
|
1024
|
-
{ column: "date", pgType: "date", entryKey: "date" },
|
|
1025
|
-
{ column: "sell_currency", pgType: "text", entryKey: "sell_currency" }
|
|
1026
|
-
]
|
|
1027
|
-
}
|
|
1028
|
-
}
|
|
1029
|
-
};
|
|
1030
|
-
|
|
1031
|
-
// src/sigma/sigmaIngestion.ts
|
|
1032
|
-
var SIGMA_CURSOR_DELIM = "";
|
|
1033
|
-
function escapeSigmaSqlStringLiteral(value) {
|
|
1034
|
-
return value.replace(/'/g, "''");
|
|
1035
|
-
}
|
|
1036
|
-
function formatSigmaTimestampForSqlLiteral(date) {
|
|
1037
|
-
return date.toISOString().replace("T", " ").replace("Z", "");
|
|
1038
|
-
}
|
|
1039
|
-
function decodeSigmaCursorValues(spec, cursor) {
|
|
1040
|
-
const prefix = `v${spec.version}${SIGMA_CURSOR_DELIM}`;
|
|
1041
|
-
if (!cursor.startsWith(prefix)) {
|
|
1042
|
-
throw new Error(
|
|
1043
|
-
`Unrecognized Sigma cursor format (expected prefix ${JSON.stringify(prefix)}): ${cursor}`
|
|
1044
|
-
);
|
|
1045
|
-
}
|
|
1046
|
-
const parts = cursor.split(SIGMA_CURSOR_DELIM);
|
|
1047
|
-
const expected = 1 + spec.columns.length;
|
|
1048
|
-
if (parts.length !== expected) {
|
|
1049
|
-
throw new Error(`Malformed Sigma cursor: expected ${expected} parts, got ${parts.length}`);
|
|
1050
|
-
}
|
|
1051
|
-
return parts.slice(1);
|
|
1052
|
-
}
|
|
1053
|
-
function encodeSigmaCursor(spec, values) {
|
|
1054
|
-
if (values.length !== spec.columns.length) {
|
|
1055
|
-
throw new Error(
|
|
1056
|
-
`Cannot encode Sigma cursor: expected ${spec.columns.length} values, got ${values.length}`
|
|
1057
|
-
);
|
|
1058
|
-
}
|
|
1059
|
-
for (const v of values) {
|
|
1060
|
-
if (v.includes(SIGMA_CURSOR_DELIM)) {
|
|
1061
|
-
throw new Error("Cannot encode Sigma cursor: value contains delimiter character");
|
|
1062
|
-
}
|
|
1063
|
-
}
|
|
1064
|
-
return [`v${spec.version}`, ...values].join(SIGMA_CURSOR_DELIM);
|
|
1065
|
-
}
|
|
1066
|
-
function sigmaSqlLiteralForCursorValue(spec, rawValue) {
|
|
1067
|
-
switch (spec.type) {
|
|
1068
|
-
case "timestamp": {
|
|
1069
|
-
const d = new Date(rawValue);
|
|
1070
|
-
if (Number.isNaN(d.getTime())) {
|
|
1071
|
-
throw new Error(`Invalid timestamp cursor value for ${spec.column}: ${rawValue}`);
|
|
1072
|
-
}
|
|
1073
|
-
return `timestamp '${formatSigmaTimestampForSqlLiteral(d)}'`;
|
|
1074
|
-
}
|
|
1075
|
-
case "number": {
|
|
1076
|
-
if (!/^-?\d+(\.\d+)?$/.test(rawValue)) {
|
|
1077
|
-
throw new Error(`Invalid numeric cursor value for ${spec.column}: ${rawValue}`);
|
|
1078
|
-
}
|
|
1079
|
-
return rawValue;
|
|
1080
|
-
}
|
|
1081
|
-
case "string":
|
|
1082
|
-
return `'${escapeSigmaSqlStringLiteral(rawValue)}'`;
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
|
-
function buildSigmaCursorWhereClause(spec, cursorValues) {
|
|
1086
|
-
if (cursorValues.length !== spec.columns.length) {
|
|
1087
|
-
throw new Error(
|
|
1088
|
-
`Cannot build Sigma cursor predicate: expected ${spec.columns.length} values, got ${cursorValues.length}`
|
|
1089
|
-
);
|
|
1090
|
-
}
|
|
1091
|
-
const cols = spec.columns.map((c) => c.column);
|
|
1092
|
-
const lits = spec.columns.map((c, i) => sigmaSqlLiteralForCursorValue(c, cursorValues[i] ?? ""));
|
|
1093
|
-
const ors = [];
|
|
1094
|
-
for (let i = 0; i < cols.length; i++) {
|
|
1095
|
-
const ands = [];
|
|
1096
|
-
for (let j = 0; j < i; j++) {
|
|
1097
|
-
ands.push(`${cols[j]} = ${lits[j]}`);
|
|
1098
|
-
}
|
|
1099
|
-
ands.push(`${cols[i]} > ${lits[i]}`);
|
|
1100
|
-
ors.push(`(${ands.join(" AND ")})`);
|
|
1101
|
-
}
|
|
1102
|
-
return ors.join(" OR ");
|
|
1103
|
-
}
|
|
1104
|
-
function buildSigmaQuery(config, cursor) {
|
|
1105
|
-
const select = config.select === void 0 || config.select === "*" ? "*" : config.select.join(", ");
|
|
1106
|
-
const whereParts = [];
|
|
1107
|
-
if (config.additionalWhere) {
|
|
1108
|
-
whereParts.push(`(${config.additionalWhere})`);
|
|
1109
|
-
}
|
|
1110
|
-
if (cursor) {
|
|
1111
|
-
const values = decodeSigmaCursorValues(config.cursor, cursor);
|
|
1112
|
-
const predicate = buildSigmaCursorWhereClause(config.cursor, values);
|
|
1113
|
-
whereParts.push(`(${predicate})`);
|
|
1114
|
-
}
|
|
1115
|
-
const whereClause = whereParts.length > 0 ? `WHERE ${whereParts.join(" AND ")}` : "";
|
|
1116
|
-
const orderBy = config.cursor.columns.map((c) => c.column).join(", ");
|
|
1117
|
-
return [
|
|
1118
|
-
`SELECT ${select} FROM ${config.sigmaTable}`,
|
|
1119
|
-
whereClause,
|
|
1120
|
-
`ORDER BY ${orderBy} ASC`,
|
|
1121
|
-
`LIMIT ${config.pageSize}`
|
|
1122
|
-
].filter(Boolean).join(" ");
|
|
1123
|
-
}
|
|
1124
|
-
function defaultSigmaRowToEntry(config, row) {
|
|
1125
|
-
const out = { ...row };
|
|
1126
|
-
for (const col of config.cursor.columns) {
|
|
1127
|
-
const raw = row[col.column];
|
|
1128
|
-
if (raw == null) {
|
|
1129
|
-
throw new Error(`Sigma row missing required cursor column: ${col.column}`);
|
|
1130
|
-
}
|
|
1131
|
-
if (col.type === "timestamp") {
|
|
1132
|
-
const normalized = normalizeSigmaTimestampToIso(raw);
|
|
1133
|
-
if (!normalized) {
|
|
1134
|
-
throw new Error(`Sigma row has invalid timestamp for ${col.column}: ${raw}`);
|
|
1135
|
-
}
|
|
1136
|
-
out[col.column] = normalized;
|
|
1137
|
-
} else if (col.type === "string") {
|
|
1138
|
-
const v = raw.trim();
|
|
1139
|
-
if (!v) {
|
|
1140
|
-
throw new Error(`Sigma row has empty string for required cursor column: ${col.column}`);
|
|
1141
|
-
}
|
|
1142
|
-
out[col.column] = v;
|
|
1143
|
-
} else {
|
|
1144
|
-
const v = raw.trim();
|
|
1145
|
-
if (!v) {
|
|
1146
|
-
throw new Error(`Sigma row has empty value for required cursor column: ${col.column}`);
|
|
1147
|
-
}
|
|
1148
|
-
out[col.column] = v;
|
|
1149
|
-
}
|
|
1150
|
-
}
|
|
1151
|
-
return out;
|
|
1152
|
-
}
|
|
1153
|
-
function sigmaCursorFromEntry(config, entry) {
|
|
1154
|
-
const values = config.cursor.columns.map((c) => {
|
|
1155
|
-
const raw = entry[c.column];
|
|
1156
|
-
if (raw == null) {
|
|
1157
|
-
throw new Error(`Cannot build cursor: entry missing ${c.column}`);
|
|
1158
|
-
}
|
|
1159
|
-
return String(raw);
|
|
1160
|
-
});
|
|
1161
|
-
return encodeSigmaCursor(config.cursor, values);
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
830
|
// src/stripeSync.ts
|
|
1165
831
|
function getUniqueIds(entries, key) {
|
|
1166
832
|
const set = new Set(
|
|
@@ -1171,7 +837,7 @@ function getUniqueIds(entries, key) {
|
|
|
1171
837
|
var StripeSync = class {
|
|
1172
838
|
constructor(config) {
|
|
1173
839
|
this.config = config;
|
|
1174
|
-
const baseStripe = new
|
|
840
|
+
const baseStripe = new Stripe2(config.stripeSecretKey, {
|
|
1175
841
|
// https://github.com/stripe/stripe-node#configuration
|
|
1176
842
|
// @ts-ignore
|
|
1177
843
|
apiVersion: config.stripeApiVersion,
|
|
@@ -1572,17 +1238,6 @@ var StripeSync = class {
|
|
|
1572
1238
|
listFn: (p) => this.stripe.checkout.sessions.list(p),
|
|
1573
1239
|
upsertFn: (items, id) => this.upsertCheckoutSessions(items, id),
|
|
1574
1240
|
supportsCreatedFilter: true
|
|
1575
|
-
},
|
|
1576
|
-
// Sigma-backed resources
|
|
1577
|
-
subscription_item_change_events_v2_beta: {
|
|
1578
|
-
order: 18,
|
|
1579
|
-
supportsCreatedFilter: false,
|
|
1580
|
-
sigma: SIGMA_INGESTION_CONFIGS.subscription_item_change_events_v2_beta
|
|
1581
|
-
},
|
|
1582
|
-
exchange_rates_from_usd: {
|
|
1583
|
-
order: 19,
|
|
1584
|
-
supportsCreatedFilter: false,
|
|
1585
|
-
sigma: SIGMA_INGESTION_CONFIGS.exchange_rates_from_usd
|
|
1586
1241
|
}
|
|
1587
1242
|
};
|
|
1588
1243
|
async processEvent(event) {
|
|
@@ -1615,13 +1270,7 @@ var StripeSync = class {
|
|
|
1615
1270
|
* Order is determined by the `order` field in resourceRegistry.
|
|
1616
1271
|
*/
|
|
1617
1272
|
getSupportedSyncObjects() {
|
|
1618
|
-
|
|
1619
|
-
if (!this.config.enableSigmaSync) {
|
|
1620
|
-
return all.filter(
|
|
1621
|
-
(o) => o !== "subscription_item_change_events_v2_beta" && o !== "exchange_rates_from_usd"
|
|
1622
|
-
);
|
|
1623
|
-
}
|
|
1624
|
-
return all;
|
|
1273
|
+
return Object.entries(this.resourceRegistry).sort(([, a], [, b]) => a.order - b.order).map(([key]) => key);
|
|
1625
1274
|
}
|
|
1626
1275
|
// Event handler methods
|
|
1627
1276
|
async handleChargeEvent(event, accountId) {
|
|
@@ -1700,7 +1349,7 @@ var StripeSync = class {
|
|
|
1700
1349
|
);
|
|
1701
1350
|
await this.upsertProducts([product], accountId, this.getSyncTimestamp(event, refetched));
|
|
1702
1351
|
} catch (err) {
|
|
1703
|
-
if (err instanceof
|
|
1352
|
+
if (err instanceof Stripe2.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1704
1353
|
const product = event.data.object;
|
|
1705
1354
|
await this.deleteProduct(product.id);
|
|
1706
1355
|
} else {
|
|
@@ -1720,7 +1369,7 @@ var StripeSync = class {
|
|
|
1720
1369
|
);
|
|
1721
1370
|
await this.upsertPrices([price], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1722
1371
|
} catch (err) {
|
|
1723
|
-
if (err instanceof
|
|
1372
|
+
if (err instanceof Stripe2.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1724
1373
|
const price = event.data.object;
|
|
1725
1374
|
await this.deletePrice(price.id);
|
|
1726
1375
|
} else {
|
|
@@ -1740,7 +1389,7 @@ var StripeSync = class {
|
|
|
1740
1389
|
);
|
|
1741
1390
|
await this.upsertPlans([plan], accountId, false, this.getSyncTimestamp(event, refetched));
|
|
1742
1391
|
} catch (err) {
|
|
1743
|
-
if (err instanceof
|
|
1392
|
+
if (err instanceof Stripe2.errors.StripeAPIError && err.code === "resource_missing") {
|
|
1744
1393
|
const plan = event.data.object;
|
|
1745
1394
|
await this.deletePlan(plan.id);
|
|
1746
1395
|
} else {
|
|
@@ -1987,10 +1636,10 @@ var StripeSync = class {
|
|
|
1987
1636
|
let cursor = null;
|
|
1988
1637
|
if (!params?.created) {
|
|
1989
1638
|
if (objRun?.cursor) {
|
|
1990
|
-
cursor = objRun.cursor;
|
|
1639
|
+
cursor = parseInt(objRun.cursor);
|
|
1991
1640
|
} else {
|
|
1992
1641
|
const lastCursor = await this.postgresClient.getLastCompletedCursor(accountId, resourceName);
|
|
1993
|
-
cursor = lastCursor
|
|
1642
|
+
cursor = lastCursor ? parseInt(lastCursor) : null;
|
|
1994
1643
|
}
|
|
1995
1644
|
}
|
|
1996
1645
|
const result = await this.fetchOnePage(
|
|
@@ -2045,18 +1694,9 @@ var StripeSync = class {
|
|
|
2045
1694
|
throw new Error(`Unsupported object type for processNext: ${object}`);
|
|
2046
1695
|
}
|
|
2047
1696
|
try {
|
|
2048
|
-
if (config.sigma) {
|
|
2049
|
-
return await this.fetchOneSigmaPage(
|
|
2050
|
-
accountId,
|
|
2051
|
-
resourceName,
|
|
2052
|
-
runStartedAt,
|
|
2053
|
-
cursor,
|
|
2054
|
-
config.sigma
|
|
2055
|
-
);
|
|
2056
|
-
}
|
|
2057
1697
|
const listParams = { limit };
|
|
2058
1698
|
if (config.supportsCreatedFilter) {
|
|
2059
|
-
const created = params?.created ?? (cursor
|
|
1699
|
+
const created = params?.created ?? (cursor ? { gte: cursor } : void 0);
|
|
2060
1700
|
if (created) {
|
|
2061
1701
|
listParams.created = created;
|
|
2062
1702
|
}
|
|
@@ -2101,97 +1741,6 @@ var StripeSync = class {
|
|
|
2101
1741
|
throw error;
|
|
2102
1742
|
}
|
|
2103
1743
|
}
|
|
2104
|
-
async getSigmaFallbackCursorFromDestination(accountId, sigmaConfig) {
|
|
2105
|
-
const cursorCols = sigmaConfig.cursor.columns;
|
|
2106
|
-
const selectCols = cursorCols.map((c) => `"${c.column}"`).join(", ");
|
|
2107
|
-
const orderBy = cursorCols.map((c) => `"${c.column}" DESC`).join(", ");
|
|
2108
|
-
const result = await this.postgresClient.query(
|
|
2109
|
-
`SELECT ${selectCols}
|
|
2110
|
-
FROM "stripe"."${sigmaConfig.destinationTable}"
|
|
2111
|
-
WHERE "_account_id" = $1
|
|
2112
|
-
ORDER BY ${orderBy}
|
|
2113
|
-
LIMIT 1`,
|
|
2114
|
-
[accountId]
|
|
2115
|
-
);
|
|
2116
|
-
if (result.rows.length === 0) return null;
|
|
2117
|
-
const row = result.rows[0];
|
|
2118
|
-
const entryForCursor = {};
|
|
2119
|
-
for (const c of cursorCols) {
|
|
2120
|
-
const v = row[c.column];
|
|
2121
|
-
if (v == null) {
|
|
2122
|
-
throw new Error(
|
|
2123
|
-
`Sigma fallback cursor query returned null for ${sigmaConfig.destinationTable}.${c.column}`
|
|
2124
|
-
);
|
|
2125
|
-
}
|
|
2126
|
-
if (c.type === "timestamp") {
|
|
2127
|
-
const d = v instanceof Date ? v : new Date(String(v));
|
|
2128
|
-
if (Number.isNaN(d.getTime())) {
|
|
2129
|
-
throw new Error(
|
|
2130
|
-
`Sigma fallback cursor query returned invalid timestamp for ${sigmaConfig.destinationTable}.${c.column}: ${String(
|
|
2131
|
-
v
|
|
2132
|
-
)}`
|
|
2133
|
-
);
|
|
2134
|
-
}
|
|
2135
|
-
entryForCursor[c.column] = d.toISOString();
|
|
2136
|
-
} else {
|
|
2137
|
-
entryForCursor[c.column] = String(v);
|
|
2138
|
-
}
|
|
2139
|
-
}
|
|
2140
|
-
return sigmaCursorFromEntry(sigmaConfig, entryForCursor);
|
|
2141
|
-
}
|
|
2142
|
-
async fetchOneSigmaPage(accountId, resourceName, runStartedAt, cursor, sigmaConfig) {
|
|
2143
|
-
if (!this.config.stripeSecretKey) {
|
|
2144
|
-
throw new Error("Sigma sync requested but stripeSecretKey is not configured.");
|
|
2145
|
-
}
|
|
2146
|
-
if (resourceName !== sigmaConfig.destinationTable) {
|
|
2147
|
-
throw new Error(
|
|
2148
|
-
`Sigma sync config mismatch: resourceName=${resourceName} destinationTable=${sigmaConfig.destinationTable}`
|
|
2149
|
-
);
|
|
2150
|
-
}
|
|
2151
|
-
const effectiveCursor = cursor ?? await this.getSigmaFallbackCursorFromDestination(accountId, sigmaConfig);
|
|
2152
|
-
const sigmaSql = buildSigmaQuery(sigmaConfig, effectiveCursor);
|
|
2153
|
-
this.config.logger?.info(
|
|
2154
|
-
{ object: resourceName, pageSize: sigmaConfig.pageSize, hasCursor: Boolean(effectiveCursor) },
|
|
2155
|
-
"Sigma sync: running query"
|
|
2156
|
-
);
|
|
2157
|
-
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
2158
|
-
apiKey: this.config.stripeSecretKey,
|
|
2159
|
-
sql: sigmaSql,
|
|
2160
|
-
logger: this.config.logger
|
|
2161
|
-
});
|
|
2162
|
-
const rows = parseCsvObjects(csv);
|
|
2163
|
-
if (rows.length === 0) {
|
|
2164
|
-
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2165
|
-
return { processed: 0, hasMore: false, runStartedAt };
|
|
2166
|
-
}
|
|
2167
|
-
const entries = rows.map(
|
|
2168
|
-
(row) => defaultSigmaRowToEntry(sigmaConfig, row)
|
|
2169
|
-
);
|
|
2170
|
-
this.config.logger?.info(
|
|
2171
|
-
{ object: resourceName, rows: entries.length, queryRunId, fileId },
|
|
2172
|
-
"Sigma sync: upserting rows"
|
|
2173
|
-
);
|
|
2174
|
-
await this.postgresClient.upsertManyWithTimestampProtection(
|
|
2175
|
-
entries,
|
|
2176
|
-
resourceName,
|
|
2177
|
-
accountId,
|
|
2178
|
-
void 0,
|
|
2179
|
-
sigmaConfig.upsert
|
|
2180
|
-
);
|
|
2181
|
-
await this.postgresClient.incrementObjectProgress(
|
|
2182
|
-
accountId,
|
|
2183
|
-
runStartedAt,
|
|
2184
|
-
resourceName,
|
|
2185
|
-
entries.length
|
|
2186
|
-
);
|
|
2187
|
-
const newCursor = sigmaCursorFromEntry(sigmaConfig, entries[entries.length - 1]);
|
|
2188
|
-
await this.postgresClient.updateObjectCursor(accountId, runStartedAt, resourceName, newCursor);
|
|
2189
|
-
const hasMore = rows.length === sigmaConfig.pageSize;
|
|
2190
|
-
if (!hasMore) {
|
|
2191
|
-
await this.postgresClient.completeObjectSync(accountId, runStartedAt, resourceName);
|
|
2192
|
-
}
|
|
2193
|
-
return { processed: entries.length, hasMore, runStartedAt };
|
|
2194
|
-
}
|
|
2195
1744
|
/**
|
|
2196
1745
|
* Process all pages for all (or specified) object types until complete.
|
|
2197
1746
|
*
|
|
@@ -2320,12 +1869,6 @@ var StripeSync = class {
|
|
|
2320
1869
|
case "checkout_sessions":
|
|
2321
1870
|
results.checkoutSessions = result;
|
|
2322
1871
|
break;
|
|
2323
|
-
case "subscription_item_change_events_v2_beta":
|
|
2324
|
-
results.subscriptionItemChangeEventsV2Beta = result;
|
|
2325
|
-
break;
|
|
2326
|
-
case "exchange_rates_from_usd":
|
|
2327
|
-
results.exchangeRatesFromUsd = result;
|
|
2328
|
-
break;
|
|
2329
1872
|
}
|
|
2330
1873
|
}
|
|
2331
1874
|
}
|
|
@@ -3886,8 +3429,6 @@ var VERSION = package_default.version;
|
|
|
3886
3429
|
export {
|
|
3887
3430
|
PostgresClient,
|
|
3888
3431
|
hashApiKey,
|
|
3889
|
-
parseCsvObjects,
|
|
3890
|
-
normalizeSigmaTimestampToIso,
|
|
3891
3432
|
StripeSync,
|
|
3892
3433
|
runMigrations,
|
|
3893
3434
|
createStripeWebSocketClient,
|