stripe-experiment-sync 1.0.18 → 1.0.20

package/README.md

@@ -202,6 +202,9 @@ Supported objects: `all`, `charge`, `checkout_sessions`, `credit_note`, `custome
 
 The sync engine tracks cursors per account and resource, enabling incremental syncing that resumes after interruptions.
 
+For paged backfills, the engine keeps a separate per-run pagination cursor (`page_cursor`) while the
+incremental cursor continues to track the highest `created` timestamp.
+
 > **Tip:** For large Stripe accounts (>10,000 objects), loop through date ranges day-by-day to avoid timeouts.
 
 ## Account Management

package/dist/{chunk-HV64EIZU.js → chunk-DFIHCPGH.js}

@@ -1,12 +1,13 @@
 import {
+  SIGMA_INGESTION_CONFIGS,
   StripeSync,
   createStripeWebSocketClient,
   runMigrations
-} from "./chunk-K4JQHI7Y.js";
+} from "./chunk-HHDPFCIA.js";
 import {
   install,
   uninstall
-} from "./chunk-M5TTM27L.js";
+} from "./chunk-UPJNFZXA.js";
 
 // src/cli/config.ts
 import dotenv from "dotenv";
@@ -129,22 +130,23 @@ var VALID_SYNC_OBJECTS = [
   "credit_note",
   "early_fraud_warning",
   "refund",
-  "checkout_sessions",
-  "subscription_item_change_events_v2_beta",
-  "exchange_rates_from_usd"
+  "checkout_sessions"
 ];
 async function backfillCommand(options, entityName) {
   let stripeSync = null;
   try {
-    if (!VALID_SYNC_OBJECTS.includes(entityName)) {
+    dotenv2.config();
+    const enableSigma = options.enableSigma ?? process.env.ENABLE_SIGMA === "true";
+    const sigmaTableNames = enableSigma ? Object.keys(SIGMA_INGESTION_CONFIGS) : [];
+    const validEntities = [...VALID_SYNC_OBJECTS, ...sigmaTableNames];
+    if (!validEntities.includes(entityName)) {
+      const entityList = enableSigma ? `${VALID_SYNC_OBJECTS.join(", ")}, and ${sigmaTableNames.length} sigma tables` : VALID_SYNC_OBJECTS.join(", ");
       console.error(
-        chalk3.red(
-          `Error: Invalid entity name "${entityName}". Valid entities are: ${VALID_SYNC_OBJECTS.join(", ")}`
-        )
+        chalk3.red(`Error: Invalid entity name "${entityName}". Valid entities are: ${entityList}`)
       );
       process.exit(1);
     }
-    dotenv2.config();
+    const isSigmaTable = sigmaTableNames.includes(entityName);
     let stripeApiKey = options.stripeKey || process.env.STRIPE_API_KEY || process.env.STRIPE_SECRET_KEY || "";
     let databaseUrl = options.databaseUrl || process.env.DATABASE_URL || "";
     if (!stripeApiKey || !databaseUrl) {
@@ -197,11 +199,13 @@ async function backfillCommand(options, entityName) {
       ngrokAuthToken: ""
       // Not needed for backfill
     };
-    console.log(chalk3.blue(`Backfilling ${entityName} from Stripe in 'stripe' schema...`));
+    const schemaName = isSigmaTable ? "sigma" : "stripe";
+    console.log(chalk3.blue(`Backfilling ${entityName} from Stripe in '${schemaName}' schema...`));
     console.log(chalk3.gray(`Database: ${config.databaseUrl.replace(/:[^:@]+@/, ":****@")}`));
     try {
       await runMigrations({
-        databaseUrl: config.databaseUrl
+        databaseUrl: config.databaseUrl,
+        enableSigma
       });
     } catch (migrationError) {
       console.error(chalk3.red("Failed to run migrations:"));
@@ -218,18 +222,49 @@ async function backfillCommand(options, entityName) {
     stripeSync = new StripeSync({
       databaseUrl: config.databaseUrl,
       stripeSecretKey: config.stripeApiKey,
-      enableSigma: process.env.ENABLE_SIGMA === "true",
+      enableSigma,
       stripeApiVersion: process.env.STRIPE_API_VERSION || "2020-08-27",
       autoExpandLists: process.env.AUTO_EXPAND_LISTS === "true",
       backfillRelatedEntities: process.env.BACKFILL_RELATED_ENTITIES !== "false",
       poolConfig
     });
-    const result = await stripeSync.processUntilDone({ object: entityName });
-    const totalSynced = Object.values(result).reduce(
-      (sum, syncResult) => sum + (syncResult?.synced || 0),
-      0
-    );
-    console.log(chalk3.green(`\u2713 Backfill complete: ${totalSynced} ${entityName} objects synced`));
+    if (entityName === "all") {
+      const backfill = await stripeSync.processUntilDoneParallel({
+        object: "all",
+        triggeredBy: "cli-backfill",
+        maxParallel: 10,
+        skipInaccessibleSigmaTables: true
+      });
+      const objectCount = Object.keys(backfill.totals).length;
+      console.log(
+        chalk3.green(
+          `\u2713 Backfill complete: ${backfill.totalSynced} rows synced across ${objectCount} objects`
+        )
+      );
+      if (backfill.skipped.length > 0) {
+        console.log(
+          chalk3.yellow(
+            `Skipped ${backfill.skipped.length} Sigma tables without access: ${backfill.skipped.join(", ")}`
+          )
+        );
+      }
+      if (backfill.errors.length > 0) {
+        console.log(chalk3.red(`Backfill finished with ${backfill.errors.length} errors:`));
+        for (const err of backfill.errors) {
+          console.log(chalk3.red(`  - ${err.object}: ${err.message}`));
+        }
+      }
+    } else {
+      const result = await stripeSync.processUntilDone({ object: entityName });
+      const totalSynced = Object.values(result).reduce(
+        (sum, syncResult) => sum + (syncResult?.synced || 0),
+        0
+      );
+      const tableType = isSigmaTable ? "(sigma)" : "";
+      console.log(
+        chalk3.green(`\u2713 Backfill complete: ${totalSynced} ${entityName} ${tableType} rows synced`)
+      );
+    }
     await stripeSync.close();
   } catch (error) {
     if (error instanceof Error) {
@@ -269,11 +304,16 @@ async function migrateCommand(options) {
       ]);
       databaseUrl = answers.databaseUrl;
     }
+    const enableSigma = options.enableSigma ?? process.env.ENABLE_SIGMA === "true";
     console.log(chalk3.blue("Running database migrations in 'stripe' schema..."));
     console.log(chalk3.gray(`Database: ${databaseUrl.replace(/:[^:@]+@/, ":****@")}`));
+    if (enableSigma) {
+      console.log(chalk3.blue("Sigma tables enabled"));
+    }
     try {
       await runMigrations({
-        databaseUrl
+        databaseUrl,
+        enableSigma
       });
       console.log(chalk3.green("\u2713 Migrations completed successfully"));
     } catch (migrationError) {
@@ -365,7 +405,8 @@ Mode: ${modeLabel}`));
     console.log(chalk3.gray(`Database: ${maskedDbUrl}`));
     try {
       await runMigrations({
-        databaseUrl: config.databaseUrl
+        databaseUrl: config.databaseUrl,
+        enableSigma: config.enableSigma
       });
     } catch (migrationError) {
       console.error(chalk3.red("Failed to run migrations:"));
@@ -473,13 +514,38 @@ Starting server on port ${port}...`));
       console.log(chalk3.green(`\u2713 Server started on port ${port}`));
     }
     if (process.env.SKIP_BACKFILL !== "true") {
-      console.log(chalk3.blue("\nStarting initial sync of all Stripe data..."));
-      const syncResult = await stripeSync.processUntilDone();
-      const totalSynced = Object.values(syncResult).reduce(
-        (sum, result) => sum + (result?.synced || 0),
-        0
+      if (!stripeSync) {
+        throw new Error("StripeSync not initialized.");
+      }
+      console.log(chalk3.blue("\nStarting historical backfill (parallel sweep)..."));
+      const backfill = await stripeSync.processUntilDoneParallel({
+        triggeredBy: "cli-historical-backfill",
+        maxParallel: 10,
+        skipInaccessibleSigmaTables: true
+      });
+      const objectCount = Object.keys(backfill.totals).length;
+      console.log(
+        chalk3.green(
+          `\u2713 Historical backfill complete: ${backfill.totalSynced} rows synced across ${objectCount} objects`
+        )
       );
-      console.log(chalk3.green(`\u2713 Sync complete: ${totalSynced} objects synced`));
+      if (backfill.skipped.length > 0) {
+        console.log(
+          chalk3.yellow(
+            `Skipped ${backfill.skipped.length} Sigma tables without access: ${backfill.skipped.join(", ")}`
+          )
+        );
+      }
+      if (backfill.errors.length > 0) {
+        console.log(
+          chalk3.red(
+            `Historical backfill finished with ${backfill.errors.length} errors. See logs above.`
+          )
+        );
+      }
+      console.log(chalk3.blue("\nStarting incremental backfill..."));
+      await stripeSync.processUntilDone();
+      console.log(chalk3.green("\u2713 Incremental backfill complete"));
     } else {
       console.log(chalk3.yellow("\n\u23ED\uFE0F  Skipping initial sync (SKIP_BACKFILL=true)"));
     }
@@ -553,7 +619,8 @@ async function installCommand(options) {
       stripeKey,
       packageVersion: options.packageVersion,
       workerIntervalSeconds: options.workerInterval,
-      supabaseManagementUrl
+      supabaseManagementUrl,
+      enableSigma: options.enableSigma
     });
     console.log(chalk3.cyan("\n\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501"));
     console.log(chalk3.cyan.bold("  Installation Complete!"));