stripe-experiment-sync 1.0.21 → 1.0.23

package/README.md

@@ -39,7 +39,7 @@ const sync = new StripeSync({
 })
 
 // Create a managed webhook - no additional processing needed!
-const webhook = await sync.findOrCreateManagedWebhook('https://example.com/stripe-webhooks')
+const webhook = await sync.webhook.findOrCreateManagedWebhook('https://example.com/stripe-webhooks')
 
 // Cleanup when done (closes PostgreSQL connection pool)
 await sync.close()
@@ -54,7 +54,7 @@ The Stripe Sync Engine automatically manages webhook endpoints and their process
 ```typescript
 // Create or reuse an existing webhook endpoint
 // This webhook will automatically sync all Stripe events to your database
-const webhook = await sync.findOrCreateManagedWebhook('https://example.com/stripe-webhooks')
+const webhook = await sync.webhook.findOrCreateManagedWebhook('https://example.com/stripe-webhooks')
 
 // Create a webhook for specific events
 const webhook = await sync.createManagedWebhook('https://example.com/stripe-webhooks', {
@@ -71,7 +71,7 @@ console.log(webhook.secret) // whsec_xxx
 
 ```typescript
 // List all managed webhooks
-const webhooks = await sync.listManagedWebhooks()
+const webhooks = await sync.webhook.listManagedWebhooks()
 
 // Get a specific webhook
 const webhook = await sync.getManagedWebhook('we_xxx')
@@ -104,7 +104,7 @@ app.post('/stripe-webhooks', async (req, res) => {
   const payload = req.body
 
   try {
-    await sync.processWebhook(payload, signature)
+    await sync.webhook.processWebhook(payload, signature)
     res.status(200).send({ received: true })
   } catch (error) {
     res.status(400).send({ error: error.message })
@@ -132,9 +132,6 @@ await sync.close()
 | `autoExpandLists`               | boolean | Fetch all list items from Stripe (not just the default 10)                                                                                                                                                                                                                                             |
 | `backfillRelatedEntities`       | boolean | Ensure related entities exist for foreign key integrity                                                                                                                                                                                                                                                |
 | `revalidateObjectsViaStripeApi` | Array   | Always fetch latest data from Stripe instead of trusting webhook payload. Possible values: charge, credit_note, customer, dispute, invoice, payment_intent, payment_method, plan, price, product, refund, review, radar.early_fraud_warning, setup_intent, subscription, subscription_schedule, tax_id |
-| `maxRetries`                    | number  | Maximum retry attempts for 429 rate limits. Default: 5                                                                                                                                                                                                                                                 |
-| `initialRetryDelayMs`           | number  | Initial retry delay in milliseconds. Default: 1000                                                                                                                                                                                                                                                     |
-| `maxRetryDelayMs`               | number  | Maximum retry delay in milliseconds. Default: 60000                                                                                                                                                                                                                                                    |
 | `logger`                        | Logger  | Logger instance (pino-compatible)                                                                                                                                                                                                                                                                      |
 
 ## Database Schema
@@ -185,17 +182,14 @@ await sync.syncSingleEntity('prod_xyz')
 ### Backfill Historical Data
 
 ```ts
-// Sync all products created after a date
-await sync.processUntilDone({
-  object: 'product',
-  created: { gte: 1643872333 }, // Unix timestamp
-})
+// Sync all products
+await sync.fullSync(['product'])
 
 // Sync all customers
-await sync.processUntilDone({ object: 'customer' })
+await sync.fullSync(['customer'])
 
 // Sync everything
-await sync.processUntilDone({ object: 'all' })
+await sync.fullSync()
 ```
 
 Supported objects: `all`, `charge`, `checkout_sessions`, `credit_note`, `customer`, `customer_with_entitlements`, `dispute`, `early_fraud_warning`, `invoice`, `payment_intent`, `payment_method`, `plan`, `price`, `product`, `refund`, `setup_intent`, `subscription`, `subscription_schedules`, `tax_id`.
@@ -277,6 +271,21 @@ The install command will:
 3. Create a managed Stripe webhook pointing to your Supabase project
 4. Set up a pg_cron job for automatic background syncing
 
+### Required Permissions
+
+The Supabase access token must have the following Management API permissions:
+
+| Permission                     | Used For                                                     |
+| ------------------------------ | ------------------------------------------------------------ |
+| `projects_read`                | Validate project access and existence                        |
+| `edge_functions_write`         | Deploy Edge Functions (setup, webhook, worker, sigma-worker) |
+| `edge_functions_secrets_write` | Set Stripe API key and configuration secrets                 |
+| `database_write`               | Execute database migrations and schema setup                 |
+| `database_read`                | Check schema existence and verify installation               |
+| `api_gateway_keys_read`        | Retrieve project's anon API key                              |
+
+**Note:** When generating a personal access token or using OAuth2, ensure these permissions are granted. For OAuth2 tokens, these correspond to the fine-grained token permissions on the Management API.
+
 ## CLI Commands
 
 ```bash

package/dist/{chunk-G26S5D5J.js → chunk-5TSB2P3P.js}

@@ -3,11 +3,11 @@ import {
   StripeSync,
   createStripeWebSocketClient,
   runMigrations
-} from "./chunk-ECLPGCY6.js";
+} from "./chunk-K22YTC4W.js";
 import {
   install,
   uninstall
-} from "./chunk-3KTFUZTY.js";
+} from "./chunk-H7FT5PGN.js";
 
 // src/cli/config.ts
 import dotenv from "dotenv";
@@ -111,6 +111,73 @@ Creating ngrok tunnel for port ${port}...`));
 import chalk3 from "chalk";
 import express from "express";
 import dotenv2 from "dotenv";
+async function monitorCommand(options) {
+  try {
+    dotenv2.config();
+    let databaseUrl = options.databaseUrl || process.env.DATABASE_URL || "";
+    if (!databaseUrl) {
+      const inquirer2 = (await import("inquirer")).default;
+      const answers = await inquirer2.prompt([
+        {
+          type: "password",
+          name: "databaseUrl",
+          message: "Enter your Postgres DATABASE_URL:",
+          mask: "*",
+          validate: (input) => {
+            if (!input || input.trim() === "") return "DATABASE_URL is required";
+            if (!input.startsWith("postgres://") && !input.startsWith("postgresql://"))
+              return 'DATABASE_URL should start with "postgres://" or "postgresql://"';
+            return true;
+          }
+        }
+      ]);
+      databaseUrl = answers.databaseUrl;
+    }
+    const poolConfig = {
+      max: 1,
+      connectionString: databaseUrl,
+      keepAlive: true
+    };
+    const stripeSync = await StripeSync.create({
+      databaseUrl,
+      stripeSecretKey: options.stripeKey || process.env.STRIPE_API_KEY || process.env.STRIPE_SECRET_KEY || "sk_placeholder",
+      poolConfig
+    });
+    console.log(chalk3.blue("Monitoring table row counts (Ctrl-C to stop)...\n"));
+    const activeRun = await stripeSync.postgresClient.getActiveSyncRun(stripeSync.accountId);
+    if (!activeRun) {
+      const lastCompleted = await stripeSync.postgresClient.getCompletedRun(
+        stripeSync.accountId,
+        Infinity
+      );
+      if (lastCompleted) {
+        console.log(
+          chalk3.green(
+            `No active sync run. Last completed at ${lastCompleted.runStartedAt.toISOString()}`
+          )
+        );
+      } else {
+        console.log(chalk3.yellow("No active or completed sync runs found."));
+      }
+      await stripeSync.close();
+      return;
+    }
+    const interval = stripeSync.startTableMonitor(2e3, activeRun);
+    const cleanup = () => {
+      clearInterval(interval);
+      stripeSync.close().finally(() => process.exit(0));
+    };
+    process.on("SIGINT", cleanup);
+    process.on("SIGTERM", cleanup);
+    await new Promise(() => {
+    });
+  } catch (error) {
+    if (error instanceof Error) {
+      console.error(chalk3.red(error.message));
+    }
+    process.exit(1);
+  }
+}
 var VALID_SYNC_OBJECTS = [
   "all",
   "customer",
@@ -219,7 +286,7 @@ async function backfillCommand(options, entityName) {
       connectionString: config.databaseUrl,
       keepAlive: true
     };
-    stripeSync = new StripeSync({
+    stripeSync = await StripeSync.create({
       databaseUrl: config.databaseUrl,
       stripeSecretKey: config.stripeApiKey,
       enableSigma,
@@ -229,12 +296,7 @@ async function backfillCommand(options, entityName) {
       poolConfig
     });
     if (entityName === "all") {
-      const backfill = await stripeSync.processUntilDoneParallel({
-        object: "all",
-        triggeredBy: "cli-backfill",
-        maxParallel: 10,
-        skipInaccessibleSigmaTables: true
-      });
+      const backfill = await stripeSync.fullSync();
       const objectCount = Object.keys(backfill.totals).length;
       console.log(
         chalk3.green(
@@ -255,14 +317,19 @@ async function backfillCommand(options, entityName) {
         }
       }
     } else {
-      const result = await stripeSync.processUntilDone({ object: entityName });
-      const totalSynced = Object.values(result).reduce(
-        (sum, syncResult) => sum + (syncResult?.synced || 0),
+      const result = await stripeSync.fullSync(
+        [entityName],
+        true,
+        20,
+        10,
+        true,
         0
       );
       const tableType = isSigmaTable ? "(sigma)" : "";
       console.log(
-        chalk3.green(`\u2713 Backfill complete: ${totalSynced} ${entityName} ${tableType} rows synced`)
+        chalk3.green(
+          `\u2713 Full sync complete: ${result.totalSynced} ${entityName} ${tableType} rows synced`
+        )
       );
     }
     await stripeSync.close();
@@ -337,7 +404,7 @@ async function migrateCommand(options) {
   }
 }
 async function syncCommand(options) {
-  let stripeSync = null;
+  let stripeSync;
   let tunnel = null;
   let server = null;
   let webhookId = null;
@@ -357,7 +424,7 @@ Cleaning up... (signal: ${signal || "manual"})`));
     const keepWebhooksOnShutdown = process.env.KEEP_WEBHOOKS_ON_SHUTDOWN === "true";
     if (webhookId && stripeSync && !keepWebhooksOnShutdown) {
       try {
-        await stripeSync.deleteManagedWebhook(webhookId);
+        await stripeSync.webhook.deleteManagedWebhook(webhookId);
         console.log(chalk3.green("\u2713 Webhook cleanup complete"));
       } catch {
         console.log(chalk3.yellow("\u26A0 Could not delete webhook"));
@@ -420,7 +487,7 @@ Mode: ${modeLabel}`));
       connectionString: config.databaseUrl,
       keepAlive: true
     };
-    stripeSync = new StripeSync({
+    stripeSync = await StripeSync.create({
       databaseUrl: config.databaseUrl,
       stripeSecretKey: config.stripeApiKey,
       enableSigma: config.enableSigma,
@@ -439,7 +506,7 @@ Mode: ${modeLabel}`));
             const payload = JSON.parse(event.event_payload);
             console.log(chalk3.cyan(`\u2190 ${payload.type}`) + chalk3.gray(` (${payload.id})`));
             if (stripeSync) {
-              await stripeSync.processEvent(payload);
+              await stripeSync.webhook.processEvent(payload);
               return {
                 status: 200,
                 event_type: payload.type,
@@ -473,7 +540,9 @@ Mode: ${modeLabel}`));
       tunnel = await createTunnel(port, config.ngrokAuthToken);
       const webhookPath = process.env.WEBHOOK_PATH || "/stripe-webhooks";
       console.log(chalk3.blue("\nCreating Stripe webhook endpoint..."));
-      const webhook = await stripeSync.findOrCreateManagedWebhook(`${tunnel.url}${webhookPath}`);
+      const webhook = await stripeSync.webhook.findOrCreateManagedWebhook(
+        `${tunnel.url}${webhookPath}`
+      );
       webhookId = webhook.id;
       const eventCount = webhook.enabled_events?.length || 0;
       console.log(chalk3.green(`\u2713 Webhook created: ${webhook.id}`));
@@ -494,7 +563,7 @@ Mode: ${modeLabel}`));
           return res.status(400).send({ error: "Missing raw body for signature verification" });
         }
         try {
-          await stripeSync.processWebhook(rawBody, sig);
+          await stripeSync.webhook.processWebhook(rawBody, sig);
           return res.status(200).send({ received: true });
         } catch (error) {
           const errorMessage = error instanceof Error ? error.message : "Unknown error";
@@ -518,11 +587,7 @@ Starting server on port ${port}...`));
         throw new Error("StripeSync not initialized.");
       }
       console.log(chalk3.blue("\nStarting historical backfill (parallel sweep)..."));
-      const backfill = await stripeSync.processUntilDoneParallel({
-        triggeredBy: "cli-historical-backfill",
-        maxParallel: 10,
-        skipInaccessibleSigmaTables: true
-      });
+      const backfill = await stripeSync.fullSync();
       const objectCount = Object.keys(backfill.totals).length;
       console.log(
         chalk3.green(
@@ -543,9 +608,6 @@ Starting server on port ${port}...`));
           )
         );
       }
-      console.log(chalk3.blue("\nStarting incremental backfill..."));
-      await stripeSync.processUntilDone();
-      console.log(chalk3.green("\u2713 Incremental backfill complete"));
     } else {
       console.log(chalk3.yellow("\n\u23ED\uFE0F  Skipping initial sync (SKIP_BACKFILL=true)"));
     }
@@ -562,6 +624,145 @@ Starting server on port ${port}...`));
     process.exit(1);
   }
 }
+async function fullSyncCommand(options) {
+  let stripeSync = null;
+  try {
+    dotenv2.config();
+    const enableSigma = options.enableSigma ?? process.env.ENABLE_SIGMA === "true";
+    const intervalSeconds = options.interval ?? 86400;
+    let stripeApiKey = options.stripeKey || process.env.STRIPE_API_KEY || process.env.STRIPE_SECRET_KEY || "";
+    let databaseUrl = options.databaseUrl || process.env.DATABASE_URL || "";
+    if (!stripeApiKey || !databaseUrl) {
+      const inquirer2 = (await import("inquirer")).default;
+      const questions = [];
+      if (!stripeApiKey) {
+        questions.push({
+          type: "password",
+          name: "stripeApiKey",
+          message: "Enter your Stripe API key:",
+          mask: "*",
+          validate: (input) => {
+            if (!input || input.trim() === "") {
+              return "Stripe API key is required";
+            }
+            if (!input.startsWith("sk_") && !input.startsWith("rk_")) {
+              return 'Stripe API key should start with "sk_" or "rk_"';
+            }
+            return true;
+          }
+        });
+      }
+      if (!databaseUrl) {
+        questions.push({
+          type: "password",
+          name: "databaseUrl",
+          message: "Enter your Postgres DATABASE_URL:",
+          mask: "*",
+          validate: (input) => {
+            if (!input || input.trim() === "") {
+              return "DATABASE_URL is required";
+            }
+            if (!input.startsWith("postgres://") && !input.startsWith("postgresql://")) {
+              return 'DATABASE_URL should start with "postgres://" or "postgresql://"';
+            }
+            return true;
+          }
+        });
+      }
+      if (questions.length > 0) {
+        console.log(chalk3.yellow("\nMissing required configuration. Please provide:"));
+        const answers = await inquirer2.prompt(questions);
+        if (answers.stripeApiKey) stripeApiKey = answers.stripeApiKey;
+        if (answers.databaseUrl) databaseUrl = answers.databaseUrl;
+      }
+    }
+    const config = {
+      stripeApiKey,
+      databaseUrl
+    };
+    console.log(chalk3.blue("\nPerforming full resync of all Stripe data..."));
+    console.log(chalk3.gray(`Database: ${config.databaseUrl.replace(/:[^:@]+@/, ":****@")}`));
+    console.log(chalk3.gray(`Reconciliation interval: ${intervalSeconds}s`));
+    try {
+      await runMigrations({
+        databaseUrl: config.databaseUrl,
+        enableSigma
+      });
+    } catch (migrationError) {
+      console.error(chalk3.red("Failed to run migrations:"));
+      console.error(
+        migrationError instanceof Error ? migrationError.message : String(migrationError)
+      );
+      throw migrationError;
+    }
+    const poolConfig = {
+      max: 10,
+      connectionString: config.databaseUrl,
+      keepAlive: true
+    };
+    stripeSync = await StripeSync.create({
+      databaseUrl: config.databaseUrl,
+      stripeSecretKey: config.stripeApiKey,
+      enableSigma,
+      stripeApiVersion: process.env.STRIPE_API_VERSION || "2020-08-27",
+      autoExpandLists: process.env.AUTO_EXPAND_LISTS === "true",
+      backfillRelatedEntities: process.env.BACKFILL_RELATED_ENTITIES !== "false",
+      poolConfig
+    });
+    const completedRun = await stripeSync.postgresClient.getCompletedRun(
+      stripeSync.accountId,
+      intervalSeconds
+    );
+    if (completedRun) {
+      console.log(
+        chalk3.green(
+          `\u2713 Skipping resync \u2014 a successful run completed at ${completedRun.runStartedAt.toISOString()} (within ${intervalSeconds}s window)`
+        )
+      );
+      await stripeSync.close();
+      return;
+    }
+    const startTime = Date.now();
+    const result = await stripeSync.fullSync(
+      void 0,
+      void 0,
+      options.workerCount,
+      options.rateLimit
+    );
+    const elapsed = ((Date.now() - startTime) / 1e3).toFixed(1);
+    const objectCount = Object.keys(result.totals).length;
+    console.log(
+      chalk3.green(
+        `\u2713 Full resync complete: ${result.totalSynced} rows synced across ${objectCount} objects in ${elapsed}s`
+      )
+    );
+    if (result.skipped.length > 0) {
+      console.log(
+        chalk3.yellow(
+          `Skipped ${result.skipped.length} Sigma tables without access: ${result.skipped.join(", ")}`
+        )
+      );
+    }
+    if (result.errors.length > 0) {
+      console.log(chalk3.red(`Full resync finished with ${result.errors.length} errors:`));
+      for (const err of result.errors) {
+        console.log(chalk3.red(`  - ${err.object}: ${err.message}`));
+      }
+    }
+    await stripeSync.close();
+  } catch (error) {
+    if (error instanceof Error) {
+      console.error(chalk3.red(error.message));
+    }
+    if (stripeSync) {
+      try {
+        await stripeSync.close();
+      } catch {
+      }
+    }
+    process.exit(1);
+  }
+}
 async function installCommand(options) {
   try {
     dotenv2.config();
@@ -611,6 +812,14 @@ async function installCommand(options) {
       }
     }
     console.log(chalk3.blue("\n\u{1F680} Installing Stripe Sync to Supabase Edge Functions...\n"));
+    const tokenSource = options.supabaseAccessToken ? "CLI" : process.env.SUPABASE_ACCESS_TOKEN ? "env" : "prompt";
+    const projectSource = options.supabaseProjectRef ? "CLI" : process.env.SUPABASE_PROJECT_REF ? "env" : "prompt";
+    console.log(
+      chalk3.gray(
+        `Access token source: ${tokenSource} (${accessToken.slice(0, 8)}...${accessToken.slice(-4)})`
+      )
+    );
+    console.log(chalk3.gray(`Project ref source: ${projectSource} (${projectRef})`));
     const supabaseManagementUrl = options.supabaseManagementUrl || process.env.SUPABASE_MANAGEMENT_URL;
     console.log(chalk3.gray("Validating project access..."));
     await install({
@@ -619,8 +828,10 @@ async function installCommand(options) {
       stripeKey,
       packageVersion: options.packageVersion,
       workerIntervalSeconds: options.workerInterval,
+      syncIntervalSeconds: options.syncInterval,
       supabaseManagementUrl,
-      enableSigma: options.enableSigma
+      enableSigma: options.enableSigma,
+      rateLimit: options.rateLimit
     });
     console.log(chalk3.cyan("\n\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501"));
     console.log(chalk3.cyan.bold("  Installation Complete!"));
@@ -699,9 +910,11 @@ async function uninstallCommand(options) {
 export {
   loadConfig,
   createTunnel,
+  monitorCommand,
   backfillCommand,
   migrateCommand,
   syncCommand,
+  fullSyncCommand,
   installCommand,
   uninstallCommand
 };

package/dist/{chunk-OWZ4QNLS.js → chunk-5VGSSPW3.js}

@@ -1,7 +1,7 @@
 // package.json
 var package_default = {
   name: "stripe-experiment-sync",
-  version: "1.0.21",
+  version: "1.0.23",
   private: false,
   description: "Stripe Sync Engine to sync Stripe data to Postgres",
   type: "module",
@@ -26,12 +26,15 @@ var package_default = {
   },
   scripts: {
     clean: "rimraf dist",
-    prebuild: "npm run clean",
+    prebuild: "npm run clean && npm run build:functions",
+    "build:functions": "tsx scripts/build-functions.ts",
     build: "tsup src/index.ts src/supabase/index.ts src/cli/index.ts src/cli/lib.ts --format esm,cjs --dts --shims && cp -r src/database/migrations dist/migrations",
     lint: "eslint src --ext .ts",
     test: "vitest",
     "test:integration": "TEST_POSTGRES_DB_URL=${TEST_POSTGRES_DB_URL:-postgresql://postgres:postgres@localhost:55432/postgres} vitest run src/stripeSync*integration.test.ts",
     "test:e2e": "vitest run --config vitest.e2e.config.ts",
+    start: "pnpm run build && node dist/cli/index.js backfill",
+    "full-sync": "pnpm run build && node dist/cli/index.js full-sync",
     "generate:sigma-schema": "tsx src/sigma/schema/fetch-schema.ts"
   },
   files: [
@@ -42,13 +45,14 @@ var package_default = {
     chalk: "^5.3.0",
     commander: "^12.1.0",
     dotenv: "^16.4.7",
+    esbuild: "^0.27.3",
     express: "^4.18.2",
     inquirer: "^12.3.0",
     papaparse: "5.4.1",
     pg: "^8.16.3",
     "pg-node-migrations": "0.0.8",
     stripe: "^17.7.0",
-    "supabase-management-js": "^0.1.6",
+    "supabase-management-js": "^2.0.2",
     ws: "^8.18.0",
     yesql: "^7.0.0"
   },
@@ -61,6 +65,7 @@ var package_default = {
     "@types/ws": "^8.5.13",
     "@types/yesql": "^4.1.4",
     "@vitest/ui": "^4.0.9",
+    esbuild: "^0.27.2",
     tsx: "^4.19.2",
     vitest: "^3.2.4"
   },