stripe-experiment-sync 1.0.21 → 1.0.23

package/dist/index.js

@@ -1,17 +1,25 @@
 import {
   PostgresClient,
   StripeSync,
+  StripeSyncWorker,
   VERSION,
   createStripeWebSocketClient,
+  embeddedMigrations,
+  getTableName,
   hashApiKey,
-  runMigrations
-} from "./chunk-ECLPGCY6.js";
-import "./chunk-OWZ4QNLS.js";
+  runMigrations,
+  runMigrationsFromContent
+} from "./chunk-K22YTC4W.js";
+import "./chunk-5VGSSPW3.js";
 export {
   PostgresClient,
   StripeSync,
+  StripeSyncWorker,
   VERSION,
   createStripeWebSocketClient,
+  embeddedMigrations,
+  getTableName,
   hashApiKey,
-  runMigrations
+  runMigrations,
+  runMigrationsFromContent
 };

package/dist/migrations/0064_add_created_gte_lte.sql

@@ -0,0 +1,13 @@
+-- Add created_gte / created_lte columns for time-range partitioned parallel sync.
+-- Workers use these to scope their Stripe list calls to a specific created window.
+-- Stored as Unix epoch seconds (INTEGER) to match Stripe's created filter format.
+-- created_gte defaults to 0 for non-chunked rows (required by PK).
+ALTER TABLE "stripe"."_sync_obj_runs" ADD COLUMN IF NOT EXISTS created_gte INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "stripe"."_sync_obj_runs" ADD COLUMN IF NOT EXISTS created_lte INTEGER;
+
+-- Expand PK to include created_gte so multiple time-range chunks of the same object can coexist.
+-- PK constraint kept original name from 0053 (_sync_obj_run_pkey) after table rename in 0057.
+ALTER TABLE "stripe"."_sync_obj_runs" DROP CONSTRAINT IF EXISTS "_sync_obj_runs_pkey";
+ALTER TABLE "stripe"."_sync_obj_runs" DROP CONSTRAINT IF EXISTS "_sync_obj_run_pkey";
+ALTER TABLE "stripe"."_sync_obj_runs"
+  ADD CONSTRAINT "_sync_obj_runs_pkey" PRIMARY KEY ("_account_id", run_started_at, object, created_gte);

package/dist/migrations/0065_add_created_lte_to_pk.sql

@@ -0,0 +1,15 @@
+-- Include created_lte in the PK so chunks with the same created_gte but
+-- different created_lte can coexist.  Requires a NOT NULL default first.
+ALTER TABLE "stripe"."_sync_obj_runs"
+  ALTER COLUMN created_lte SET DEFAULT 0;
+
+UPDATE "stripe"."_sync_obj_runs"
+   SET created_lte = 0
+ WHERE created_lte IS NULL;
+
+ALTER TABLE "stripe"."_sync_obj_runs"
+  ALTER COLUMN created_lte SET NOT NULL;
+
+ALTER TABLE "stripe"."_sync_obj_runs" DROP CONSTRAINT IF EXISTS "_sync_obj_runs_pkey";
+ALTER TABLE "stripe"."_sync_obj_runs"
+  ADD CONSTRAINT "_sync_obj_runs_pkey" PRIMARY KEY ("_account_id", run_started_at, object, created_gte, created_lte);

package/dist/migrations/0066_rate_limits.sql

@@ -0,0 +1,43 @@
+-- Rate limiting table and function for cross-process request throttling.
+-- Used by claimNextTask to cap how many claims/sec hit the database.
+
+CREATE TABLE IF NOT EXISTS "stripe"."_rate_limits" (
+  key TEXT PRIMARY KEY,
+  count INTEGER NOT NULL DEFAULT 0,
+  window_start TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE OR REPLACE FUNCTION "stripe".check_rate_limit(
+  rate_key TEXT,
+  max_requests INTEGER,
+  window_seconds INTEGER
+)
+RETURNS VOID AS $$
+DECLARE
+  now TIMESTAMPTZ := clock_timestamp();
+  window_length INTERVAL := make_interval(secs => window_seconds);
+  current_count INTEGER;
+BEGIN
+  PERFORM pg_advisory_xact_lock(hashtext(rate_key));
+
+  INSERT INTO "stripe"."_rate_limits" (key, count, window_start)
+  VALUES (rate_key, 1, now)
+  ON CONFLICT (key) DO UPDATE
+  SET count = CASE
+                WHEN "_rate_limits".window_start + window_length <= now
+                  THEN 1
+                  ELSE "_rate_limits".count + 1
+              END,
+      window_start = CASE
+                       WHEN "_rate_limits".window_start + window_length <= now
+                         THEN now
+                         ELSE "_rate_limits".window_start
+                     END;
+
+  SELECT count INTO current_count FROM "stripe"."_rate_limits" WHERE key = rate_key;
+
+  IF current_count > max_requests THEN
+    RAISE EXCEPTION 'Rate limit exceeded for %', rate_key;
+  END IF;
+END;
+$$ LANGUAGE plpgsql;

package/dist/migrations/0067_add_priority_to_sync_obj_runs.sql

@@ -0,0 +1,10 @@
+-- Add priority column to _sync_obj_runs for deterministic task ordering.
+-- Priority mirrors the `order` field from resourceRegistry so workers
+-- always process parent resources (products, prices) before children
+-- (subscriptions, invoices).
+
+ALTER TABLE "stripe"."_sync_obj_runs"
+  ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0;
+
+CREATE INDEX IF NOT EXISTS idx_sync_obj_runs_priority
+  ON "stripe"."_sync_obj_runs" ("_account_id", run_started_at, status, priority);

package/dist/migrations/0068_sync_obj_progress_view.sql

@@ -0,0 +1,23 @@
+-- Per-object sync progress view for monitoring.
+-- Defaults to the newest run per account; callers can filter by a specific
+-- run_started_at if needed.
+
+DROP FUNCTION IF EXISTS "stripe"."sync_obj_progress"(TEXT, TIMESTAMPTZ);
+
+CREATE OR REPLACE VIEW "stripe"."sync_obj_progress" AS
+SELECT
+  r."_account_id" AS account_id,
+  r.run_started_at,
+  r.object,
+  ROUND(
+    100.0 * COUNT(*) FILTER (WHERE r.status = 'complete') / NULLIF(COUNT(*), 0),
+    1
+  ) AS pct_complete,
+  COALESCE(SUM(r.processed_count), 0) AS processed
+FROM "stripe"."_sync_obj_runs" r
+WHERE r.run_started_at = (
+  SELECT MAX(s.started_at)
+  FROM "stripe"."_sync_runs" s
+  WHERE s."_account_id" = r."_account_id"
+)
+GROUP BY r."_account_id", r.run_started_at, r.object;

package/dist/supabase/functions/sigma-data-worker/index.js

@@ -0,0 +1,238 @@
+// src/supabase/edge-functions/sigma-data-worker.ts
+import { StripeSync } from "npm:stripe-experiment-sync";
+import postgres from "npm:postgres";
+var BATCH_SIZE = 1;
+var MAX_RUN_AGE_MS = 6 * 60 * 60 * 1e3;
+var jsonResponse = (body, status = 200) => new Response(JSON.stringify(body), {
+  status,
+  headers: { "Content-Type": "application/json" }
+});
+Deno.serve(async (req) => {
+  const authHeader = req.headers.get("Authorization");
+  if (!authHeader?.startsWith("Bearer ")) {
+    return new Response("Unauthorized", { status: 401 });
+  }
+  const token = authHeader.substring(7);
+  const dbUrl = Deno.env.get("SUPABASE_DB_URL");
+  if (!dbUrl) {
+    return jsonResponse({ error: "SUPABASE_DB_URL not set" }, 500);
+  }
+  let sql;
+  let stripeSync;
+  try {
+    sql = postgres(dbUrl, { max: 1, prepare: false });
+  } catch (error) {
+    return jsonResponse(
+      {
+        error: "Failed to create postgres connection",
+        details: error.message,
+        stack: error.stack
+      },
+      500
+    );
+  }
+  try {
+    const vaultResult = await sql`
+      SELECT decrypted_secret
+      FROM vault.decrypted_secrets
+      WHERE name = 'stripe_sigma_worker_secret'
+    `;
+    if (vaultResult.length === 0) {
+      await sql.end();
+      return new Response("Sigma worker secret not configured in vault", { status: 500 });
+    }
+    const storedSecret = vaultResult[0].decrypted_secret;
+    if (token !== storedSecret) {
+      await sql.end();
+      return new Response("Forbidden: Invalid sigma worker secret", { status: 403 });
+    }
+    stripeSync = await StripeSync.create({
+      poolConfig: { connectionString: dbUrl, max: 1 },
+      stripeSecretKey: Deno.env.get("STRIPE_SECRET_KEY"),
+      enableSigma: true,
+      sigmaPageSizeOverride: 1e3
+    });
+  } catch (error) {
+    await sql.end();
+    return jsonResponse(
+      {
+        error: "Failed to create StripeSync",
+        details: error.message,
+        stack: error.stack
+      },
+      500
+    );
+  }
+  try {
+    const accountId = await stripeSync.getAccountId();
+    const sigmaObjects = stripeSync.getSupportedSigmaObjects();
+    if (sigmaObjects.length === 0) {
+      return jsonResponse({ message: "No Sigma objects configured for sync" });
+    }
+    const runResult = await stripeSync.postgresClient.getOrCreateSyncRun(accountId, "sigma-worker");
+    const runStartedAt = runResult?.runStartedAt ?? (await stripeSync.postgresClient.getActiveSyncRun(accountId, "sigma-worker"))?.runStartedAt;
+    if (!runStartedAt) {
+      throw new Error("Failed to get or create sync run for sigma worker");
+    }
+    await stripeSync.postgresClient.query(
+      `UPDATE "stripe"."_sync_obj_runs"
+       SET status = 'error',
+           error_message = 'Legacy sigma worker prefix run (sigma.*); superseded by unprefixed runs',
+           completed_at = now()
+       WHERE "_account_id" = $1
+         AND run_started_at = $2
+         AND object LIKE 'sigma.%'
+         AND status IN ('pending', 'running')`,
+      [accountId, runStartedAt]
+    );
+    const runAgeMs = Date.now() - runStartedAt.getTime();
+    if (runAgeMs > MAX_RUN_AGE_MS) {
+      console.warn(
+        `Sigma worker: run too old (${Math.round(runAgeMs / 1e3 / 60)} min), closing without self-trigger`
+      );
+      await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt);
+      return jsonResponse({
+        message: "Sigma run exceeded max age, closed without processing",
+        runAgeMinutes: Math.round(runAgeMs / 1e3 / 60),
+        selfTriggered: false
+      });
+    }
+    await stripeSync.postgresClient.createObjectRuns(accountId, runStartedAt, sigmaObjects);
+    await stripeSync.postgresClient.ensureSyncRunMaxConcurrent(accountId, runStartedAt, BATCH_SIZE);
+    const runningObjects = await stripeSync.postgresClient.listObjectsByStatus(
+      accountId,
+      runStartedAt,
+      "running",
+      sigmaObjects
+    );
+    const objectsToProcess = runningObjects.slice(0, BATCH_SIZE);
+    let pendingObjects = [];
+    if (objectsToProcess.length === 0) {
+      pendingObjects = await stripeSync.postgresClient.listObjectsByStatus(
+        accountId,
+        runStartedAt,
+        "pending",
+        sigmaObjects
+      );
+      for (const objectKey of pendingObjects) {
+        if (objectsToProcess.length >= BATCH_SIZE) break;
+        const started = await stripeSync.postgresClient.tryStartObjectSync(
+          accountId,
+          runStartedAt,
+          objectKey
+        );
+        if (started) {
+          objectsToProcess.push(objectKey);
+        }
+      }
+    }
+    if (objectsToProcess.length === 0) {
+      if (pendingObjects.length === 0) {
+        console.info("Sigma worker: all objects complete or errored - run finished");
+        return jsonResponse({ message: "Sigma sync run complete", selfTriggered: false });
+      }
+      console.info("Sigma worker: at concurrency limit, will self-trigger", {
+        pendingCount: pendingObjects.length
+      });
+      let selfTriggered2 = false;
+      try {
+        await sql`SELECT stripe.trigger_sigma_worker()`;
+        selfTriggered2 = true;
+      } catch (error) {
+        console.warn("Failed to self-trigger sigma worker:", error.message);
+      }
+      return jsonResponse({
+        message: "At concurrency limit",
+        pendingCount: pendingObjects.length,
+        selfTriggered: selfTriggered2
+      });
+    }
+    const results = [];
+    for (const object of objectsToProcess) {
+      const objectKey = object;
+      try {
+        console.info(`Sigma worker: processing ${object}`);
+        const result = await stripeSync.processNext(object, {
+          runStartedAt,
+          triggeredBy: "sigma-worker"
+        });
+        results.push({
+          object,
+          processed: result.processed,
+          hasMore: result.hasMore,
+          status: "success"
+        });
+        if (result.hasMore) {
+          console.info(
+            `Sigma worker: ${object} has more pages, processed ${result.processed} rows so far`
+          );
+        } else {
+          console.info(`Sigma worker: ${object} complete, processed ${result.processed} rows`);
+        }
+      } catch (error) {
+        console.error(`Sigma worker: error processing ${object}:`, error);
+        await stripeSync.postgresClient.failObjectSync(
+          accountId,
+          runStartedAt,
+          objectKey,
+          error.message ?? "Unknown error"
+        );
+        results.push({
+          object,
+          processed: 0,
+          hasMore: false,
+          status: "error",
+          error: error.message
+        });
+      }
+    }
+    const pendingAfter = await stripeSync.postgresClient.listObjectsByStatus(
+      accountId,
+      runStartedAt,
+      "pending",
+      sigmaObjects
+    );
+    const runningAfter = await stripeSync.postgresClient.listObjectsByStatus(
+      accountId,
+      runStartedAt,
+      "running",
+      sigmaObjects
+    );
+    const remainingMs = MAX_RUN_AGE_MS - (Date.now() - runStartedAt.getTime());
+    const remainingMinutes = Math.round(remainingMs / 1e3 / 60);
+    const shouldSelfTrigger = (pendingAfter.length > 0 || runningAfter.length > 0) && remainingMs > 0;
+    let selfTriggered = false;
+    if (shouldSelfTrigger) {
+      console.info("Sigma worker: more work remains, self-triggering", {
+        pending: pendingAfter.length,
+        running: runningAfter.length,
+        remainingMinutes
+      });
+      try {
+        await sql`SELECT stripe.trigger_sigma_worker()`;
+        selfTriggered = true;
+      } catch (error) {
+        console.warn("Failed to self-trigger sigma worker:", error.message);
+      }
+    } else if (pendingAfter.length > 0 || runningAfter.length > 0) {
+      console.warn("Sigma worker: work remains but run timed out, closing", {
+        pending: pendingAfter.length,
+        running: runningAfter.length
+      });
+      await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt);
+    } else {
+      console.info("Sigma worker: no more work, run complete");
+    }
+    return jsonResponse({
+      results,
+      selfTriggered,
+      remaining: { pending: pendingAfter.length, running: runningAfter.length }
+    });
+  } catch (error) {
+    console.error("Sigma worker error:", error);
+    return jsonResponse({ error: error.message, stack: error.stack }, 500);
+  } finally {
+    if (sql) await sql.end();
+    if (stripeSync) await stripeSync.postgresClient.pool.end();
+  }
+});