stripe-experiment-sync 1.0.18 → 1.0.20

package/dist/index.d.cts

@@ -33,7 +33,7 @@ declare class PostgresClient {
     }>(entries: T[], table: string): Promise<T[]>;
     upsertManyWithTimestampProtection<T extends {
         [Key: string]: any;
-    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
+    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions, schemaOverride?: string): Promise<T[]>;
     private cleanseArrayField;
     findMissingEntries(table: string, ids: string[]): Promise<string[]>;
     upsertAccount(accountData: {
@@ -102,9 +102,10 @@ declare class PostgresClient {
     cancelStaleRuns(accountId: string): Promise<void>;
     /**
      * Get or create a sync run for this account.
-     * Returns existing run if one is active, otherwise creates new one.
+     * Returns existing run if one is active for the given triggeredBy, otherwise creates new one.
      * Auto-cancels stale runs before checking.
      *
+     * @param triggeredBy - Worker type (e.g., 'worker', 'sigma-worker'). Runs are isolated per triggeredBy.
      * @returns RunKey with isNew flag, or null if constraint violation (race condition)
      */
     getOrCreateSyncRun(accountId: string, triggeredBy?: string): Promise<{
@@ -114,8 +115,9 @@ declare class PostgresClient {
     } | null>;
     /**
      * Get the active sync run for an account (if any).
+     * @param triggeredBy - If provided, only returns run matching this triggeredBy value
      */
-    getActiveSyncRun(accountId: string): Promise<{
+    getActiveSyncRun(accountId: string, triggeredBy?: string): Promise<{
         accountId: string;
         runStartedAt: Date;
     } | null>;
@@ -129,6 +131,10 @@ declare class PostgresClient {
         maxConcurrent: number;
         closedAt: Date | null;
     } | null>;
+    /**
+     * Ensure a sync run has at least the requested max_concurrent value.
+     */
+    ensureSyncRunMaxConcurrent(accountId: string, runStartedAt: Date, maxConcurrent: number): Promise<void>;
     /**
      * Close a sync run (mark as done).
      * Status (complete/error) is derived from object run states.
@@ -157,12 +163,21 @@ declare class PostgresClient {
         status: string;
         processedCount: number;
         cursor: string | null;
+        pageCursor: string | null;
     } | null>;
     /**
      * Update progress for an object sync.
      * Also touches updated_at for stale detection.
      */
     incrementObjectProgress(accountId: string, runStartedAt: Date, object: string, count: number): Promise<void>;
+    /**
+     * Update the pagination page_cursor used for backfills using Stripe list calls.
+     */
+    updateObjectPageCursor(accountId: string, runStartedAt: Date, object: string, pageCursor: string | null): Promise<void>;
+    /**
+     * Clear the pagination page_cursor for an object sync.
+     */
+    clearObjectPageCursor(accountId: string, runStartedAt: Date, object: string): Promise<void>;
     /**
      * Update the cursor for an object sync.
      * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -170,17 +185,32 @@ declare class PostgresClient {
      * For non-numeric cursors, just sets the value directly.
      */
     updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
+    setObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
+    /**
+     * List object names for a run by status, optionally filtered to a subset.
+     */
+    listObjectsByStatus(accountId: string, runStartedAt: Date, status: string, objectFilter?: string[]): Promise<string[]>;
     /**
      * Get the highest cursor from previous syncs for an object type.
-     * This considers completed, error, AND running runs to ensure recovery syncs
-     * don't re-process data that was already synced before a crash.
-     * A 'running' status with a cursor means the process was killed mid-sync.
+     * Uses only completed object runs.
+     * - During the initial backfill we page through history, but we also update the cursor as we go.
+     *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+     *   too early and only ever fetch the newest page (breaking the historical backfill).
      *
      * Handles two cursor formats:
      * - Numeric: compared as bigint for correct ordering
      * - Composite cursors: compared as strings with COLLATE "C"
      */
     getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
+    /**
+     * Get the highest cursor from previous syncs for an object type, excluding the current run.
+     */
+    getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
+    /**
+     * Get the most recent cursor for an object run before the given run.
+     * This returns the raw cursor value without interpretation.
+     */
+    getLastObjectCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
     /**
      * Delete all sync runs and object runs for an account.
      * Useful for testing or resetting sync state.
@@ -229,6 +259,12 @@ type SigmaCursorSpec = {
     version: 1;
     columns: SigmaCursorColumnSpec[];
 };
+type SigmaColumnSpec = {
+    name: string;
+    sigmaType: string;
+    pgType: string;
+    primaryKey: boolean;
+};
 type SigmaIngestionConfig = {
     /**
      * The Sigma table name to query (no quoting, no schema).
@@ -244,6 +280,11 @@ type SigmaIngestionConfig = {
      * Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
      */
     cursor: SigmaCursorSpec;
+    /**
+     * Column metadata for the destination table.
+     * If provided, used to dynamically create/reconcile the table.
+     */
+    columns?: SigmaColumnSpec[];
     /** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
     additionalWhere?: string;
     /** Columns to SELECT (defaults to `*`). */
@@ -272,6 +313,16 @@ type StripeSyncConfig = {
      * Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
      */
     enableSigma?: boolean;
+    /**
+     * Optional override for Sigma page size (per query).
+     * Useful for edge function CPU limits; lower values reduce per-invocation work.
+     */
+    sigmaPageSizeOverride?: number;
+    /**
+     * Postgres schema name for Sigma tables.
+     * Default: "sigma"
+     */
+    sigmaSchemaName?: string;
     /** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
     stripeAccountId?: string;
     /** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
@@ -326,7 +377,7 @@ type StripeSyncConfig = {
      */
     maxConcurrentCustomers?: number;
 };
-type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | 'subscription_item_change_events_v2_beta' | 'exchange_rates_from_usd';
+type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions';
 interface Sync {
     synced: number;
 }
@@ -350,6 +401,8 @@ interface SyncBackfill {
     checkoutSessions?: Sync;
     subscriptionItemChangeEventsV2Beta?: Sync;
     exchangeRatesFromUsd?: Sync;
+    /** Sigma-backed results by table name (e.g. subscription_item_change_events_v2_beta). */
+    sigma?: Record<string, Sync>;
 }
 interface SyncParams {
     created?: {
@@ -392,6 +445,8 @@ interface ProcessNextResult {
     hasMore: boolean;
     /** The sync run this processing belongs to */
     runStartedAt: Date;
+    /** Sigma-only: whether this step started a new Sigma query run */
+    startedQuery?: boolean;
 }
 /**
  * Parameters for processNext() including optional run context
@@ -482,7 +537,12 @@ declare class StripeSync {
     private config;
     stripe: Stripe;
     postgresClient: PostgresClient;
+    private readonly resourceRegistry;
+    private get sigmaSchemaName();
     constructor(config: StripeSyncConfig);
+    private buildResourceRegistry;
+    private isSigmaResource;
+    private sigmaResultKey;
     /**
      * Get the Stripe account ID. Delegates to getCurrentAccount() for the actual lookup.
      */
@@ -525,7 +585,6 @@ declare class StripeSync {
     }>;
     processWebhook(payload: Buffer | string, signature: string | undefined): Promise<void>;
     private readonly eventHandlers;
-    private readonly resourceRegistry;
     processEvent(event: Stripe.Event): Promise<void>;
     /**
      * Returns an array of all webhook event types that this sync engine can handle.
@@ -538,6 +597,13 @@ declare class StripeSync {
      * Order is determined by the `order` field in resourceRegistry.
      */
     getSupportedSyncObjects(): Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
+    /**
+     * Get the list of Sigma-backed object types that can be synced.
+     * Only returns sigma objects when enableSigma is true.
+     *
+     * @returns Array of sigma object names (e.g. 'subscription_item_change_events_v2_beta')
+     */
+    getSupportedSigmaObjects(): string[];
     private handleChargeEvent;
     private handleCustomerDeletedEvent;
     private handleCustomerEvent;
@@ -587,6 +653,7 @@ declare class StripeSync {
      * ```
      */
     processNext(object: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>, params?: ProcessNextParams): Promise<ProcessNextResult>;
+    private appendMigrationHint;
     /**
      * Get the database resource name for a SyncObject type
      */
@@ -630,6 +697,22 @@ declare class StripeSync {
         runKey: RunKey;
         objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
     }>;
+    private applySyncBackfillResult;
+    processUntilDoneParallel(params?: SyncParams & {
+        maxParallel?: number;
+        triggeredBy?: string;
+        continueOnError?: boolean;
+        skipInaccessibleSigmaTables?: boolean;
+    }): Promise<{
+        results: SyncBackfill;
+        totals: Record<string, number>;
+        totalSynced: number;
+        skipped: string[];
+        errors: Array<{
+            object: string;
+            message: string;
+        }>;
+    }>;
     processUntilDone(params?: SyncParams): Promise<SyncBackfill>;
     /**
      * Internal implementation of processUntilDone with an existing run.
@@ -748,6 +831,7 @@ type MigrationConfig = {
     databaseUrl: string;
     ssl?: ConnectionOptions;
     logger?: Logger;
+    enableSigma?: boolean;
 };
 declare function runMigrations(config: MigrationConfig): Promise<void>;
 

package/dist/index.d.ts

@@ -33,7 +33,7 @@ declare class PostgresClient {
     }>(entries: T[], table: string): Promise<T[]>;
     upsertManyWithTimestampProtection<T extends {
         [Key: string]: any;
-    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
+    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions, schemaOverride?: string): Promise<T[]>;
     private cleanseArrayField;
     findMissingEntries(table: string, ids: string[]): Promise<string[]>;
     upsertAccount(accountData: {
@@ -102,9 +102,10 @@ declare class PostgresClient {
     cancelStaleRuns(accountId: string): Promise<void>;
     /**
      * Get or create a sync run for this account.
-     * Returns existing run if one is active, otherwise creates new one.
+     * Returns existing run if one is active for the given triggeredBy, otherwise creates new one.
      * Auto-cancels stale runs before checking.
      *
+     * @param triggeredBy - Worker type (e.g., 'worker', 'sigma-worker'). Runs are isolated per triggeredBy.
      * @returns RunKey with isNew flag, or null if constraint violation (race condition)
      */
     getOrCreateSyncRun(accountId: string, triggeredBy?: string): Promise<{
@@ -114,8 +115,9 @@ declare class PostgresClient {
     } | null>;
     /**
      * Get the active sync run for an account (if any).
+     * @param triggeredBy - If provided, only returns run matching this triggeredBy value
      */
-    getActiveSyncRun(accountId: string): Promise<{
+    getActiveSyncRun(accountId: string, triggeredBy?: string): Promise<{
         accountId: string;
         runStartedAt: Date;
     } | null>;
@@ -129,6 +131,10 @@ declare class PostgresClient {
         maxConcurrent: number;
         closedAt: Date | null;
     } | null>;
+    /**
+     * Ensure a sync run has at least the requested max_concurrent value.
+     */
+    ensureSyncRunMaxConcurrent(accountId: string, runStartedAt: Date, maxConcurrent: number): Promise<void>;
     /**
      * Close a sync run (mark as done).
      * Status (complete/error) is derived from object run states.
@@ -157,12 +163,21 @@ declare class PostgresClient {
         status: string;
         processedCount: number;
         cursor: string | null;
+        pageCursor: string | null;
     } | null>;
     /**
      * Update progress for an object sync.
      * Also touches updated_at for stale detection.
      */
     incrementObjectProgress(accountId: string, runStartedAt: Date, object: string, count: number): Promise<void>;
+    /**
+     * Update the pagination page_cursor used for backfills using Stripe list calls.
+     */
+    updateObjectPageCursor(accountId: string, runStartedAt: Date, object: string, pageCursor: string | null): Promise<void>;
+    /**
+     * Clear the pagination page_cursor for an object sync.
+     */
+    clearObjectPageCursor(accountId: string, runStartedAt: Date, object: string): Promise<void>;
     /**
      * Update the cursor for an object sync.
      * Only updates if the new cursor is higher than the existing one (cursors should never decrease).
@@ -170,17 +185,32 @@ declare class PostgresClient {
      * For non-numeric cursors, just sets the value directly.
      */
     updateObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
+    setObjectCursor(accountId: string, runStartedAt: Date, object: string, cursor: string | null): Promise<void>;
+    /**
+     * List object names for a run by status, optionally filtered to a subset.
+     */
+    listObjectsByStatus(accountId: string, runStartedAt: Date, status: string, objectFilter?: string[]): Promise<string[]>;
     /**
      * Get the highest cursor from previous syncs for an object type.
-     * This considers completed, error, AND running runs to ensure recovery syncs
-     * don't re-process data that was already synced before a crash.
-     * A 'running' status with a cursor means the process was killed mid-sync.
+     * Uses only completed object runs.
+     * - During the initial backfill we page through history, but we also update the cursor as we go.
+     *   If we crash mid-backfill and reuse that cursor, we can accidentally switch into incremental mode
+     *   too early and only ever fetch the newest page (breaking the historical backfill).
      *
      * Handles two cursor formats:
      * - Numeric: compared as bigint for correct ordering
      * - Composite cursors: compared as strings with COLLATE "C"
      */
     getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
+    /**
+     * Get the highest cursor from previous syncs for an object type, excluding the current run.
+     */
+    getLastCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
+    /**
+     * Get the most recent cursor for an object run before the given run.
+     * This returns the raw cursor value without interpretation.
+     */
+    getLastObjectCursorBeforeRun(accountId: string, object: string, runStartedAt: Date): Promise<string | null>;
     /**
      * Delete all sync runs and object runs for an account.
      * Useful for testing or resetting sync state.
@@ -229,6 +259,12 @@ type SigmaCursorSpec = {
     version: 1;
     columns: SigmaCursorColumnSpec[];
 };
+type SigmaColumnSpec = {
+    name: string;
+    sigmaType: string;
+    pgType: string;
+    primaryKey: boolean;
+};
 type SigmaIngestionConfig = {
     /**
      * The Sigma table name to query (no quoting, no schema).
@@ -244,6 +280,11 @@ type SigmaIngestionConfig = {
      * Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
      */
     cursor: SigmaCursorSpec;
+    /**
+     * Column metadata for the destination table.
+     * If provided, used to dynamically create/reconcile the table.
+     */
+    columns?: SigmaColumnSpec[];
     /** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
     additionalWhere?: string;
     /** Columns to SELECT (defaults to `*`). */
@@ -272,6 +313,16 @@ type StripeSyncConfig = {
      * Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
      */
     enableSigma?: boolean;
+    /**
+     * Optional override for Sigma page size (per query).
+     * Useful for edge function CPU limits; lower values reduce per-invocation work.
+     */
+    sigmaPageSizeOverride?: number;
+    /**
+     * Postgres schema name for Sigma tables.
+     * Default: "sigma"
+     */
+    sigmaSchemaName?: string;
     /** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
     stripeAccountId?: string;
     /** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
@@ -326,7 +377,7 @@ type StripeSyncConfig = {
      */
     maxConcurrentCustomers?: number;
 };
-type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | 'subscription_item_change_events_v2_beta' | 'exchange_rates_from_usd';
+type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions';
 interface Sync {
     synced: number;
 }
@@ -350,6 +401,8 @@ interface SyncBackfill {
     checkoutSessions?: Sync;
     subscriptionItemChangeEventsV2Beta?: Sync;
     exchangeRatesFromUsd?: Sync;
+    /** Sigma-backed results by table name (e.g. subscription_item_change_events_v2_beta). */
+    sigma?: Record<string, Sync>;
 }
 interface SyncParams {
     created?: {
@@ -392,6 +445,8 @@ interface ProcessNextResult {
     hasMore: boolean;
     /** The sync run this processing belongs to */
     runStartedAt: Date;
+    /** Sigma-only: whether this step started a new Sigma query run */
+    startedQuery?: boolean;
 }
 /**
  * Parameters for processNext() including optional run context
@@ -482,7 +537,12 @@ declare class StripeSync {
     private config;
     stripe: Stripe;
     postgresClient: PostgresClient;
+    private readonly resourceRegistry;
+    private get sigmaSchemaName();
     constructor(config: StripeSyncConfig);
+    private buildResourceRegistry;
+    private isSigmaResource;
+    private sigmaResultKey;
     /**
      * Get the Stripe account ID. Delegates to getCurrentAccount() for the actual lookup.
      */
@@ -525,7 +585,6 @@ declare class StripeSync {
     }>;
     processWebhook(payload: Buffer | string, signature: string | undefined): Promise<void>;
     private readonly eventHandlers;
-    private readonly resourceRegistry;
     processEvent(event: Stripe.Event): Promise<void>;
     /**
      * Returns an array of all webhook event types that this sync engine can handle.
@@ -538,6 +597,13 @@ declare class StripeSync {
      * Order is determined by the `order` field in resourceRegistry.
      */
     getSupportedSyncObjects(): Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
+    /**
+     * Get the list of Sigma-backed object types that can be synced.
+     * Only returns sigma objects when enableSigma is true.
+     *
+     * @returns Array of sigma object names (e.g. 'subscription_item_change_events_v2_beta')
+     */
+    getSupportedSigmaObjects(): string[];
     private handleChargeEvent;
     private handleCustomerDeletedEvent;
     private handleCustomerEvent;
@@ -587,6 +653,7 @@ declare class StripeSync {
      * ```
      */
     processNext(object: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>, params?: ProcessNextParams): Promise<ProcessNextResult>;
+    private appendMigrationHint;
     /**
      * Get the database resource name for a SyncObject type
      */
@@ -630,6 +697,22 @@ declare class StripeSync {
         runKey: RunKey;
         objects: Exclude<SyncObject, 'all' | 'customer_with_entitlements'>[];
     }>;
+    private applySyncBackfillResult;
+    processUntilDoneParallel(params?: SyncParams & {
+        maxParallel?: number;
+        triggeredBy?: string;
+        continueOnError?: boolean;
+        skipInaccessibleSigmaTables?: boolean;
+    }): Promise<{
+        results: SyncBackfill;
+        totals: Record<string, number>;
+        totalSynced: number;
+        skipped: string[];
+        errors: Array<{
+            object: string;
+            message: string;
+        }>;
+    }>;
     processUntilDone(params?: SyncParams): Promise<SyncBackfill>;
     /**
      * Internal implementation of processUntilDone with an existing run.
@@ -748,6 +831,7 @@ type MigrationConfig = {
     databaseUrl: string;
     ssl?: ConnectionOptions;
     logger?: Logger;
+    enableSigma?: boolean;
 };
 declare function runMigrations(config: MigrationConfig): Promise<void>;
 

package/dist/index.js

@@ -5,8 +5,8 @@ import {
   createStripeWebSocketClient,
   hashApiKey,
   runMigrations
-} from "./chunk-K4JQHI7Y.js";
-import "./chunk-XO57OJUE.js";
+} from "./chunk-HHDPFCIA.js";
+import "./chunk-NM6L6EPQ.js";
 export {
   PostgresClient,
   StripeSync,

package/dist/migrations/0061_add_page_cursor.sql

@@ -0,0 +1,3 @@
+-- Add page_cursor column for pagination state within a single sync run.
+-- This is used to store the starting_after ID for backfills using Stripe list calls.
+ALTER TABLE "stripe"."_sync_obj_runs" ADD COLUMN IF NOT EXISTS page_cursor text;

package/dist/migrations/0062_sigma_query_runs.sql

@@ -0,0 +1,8 @@
+-- Allow parallel sync runs per triggered_by (sigma-worker vs stripe-worker)
+ALTER TABLE "stripe"."_sync_runs" DROP CONSTRAINT IF EXISTS one_active_run_per_account;
+ALTER TABLE "stripe"."_sync_runs"
+ADD CONSTRAINT one_active_run_per_account_triggered_by
+EXCLUDE (
+  "_account_id" WITH =,
+  COALESCE(triggered_by, 'default') WITH =
+) WHERE (closed_at IS NULL);