stripe-experiment-sync 1.0.13 → 1.0.15

package/dist/index.d.cts

@@ -6,6 +6,22 @@ type PostgresConfig = {
     schema: string;
     poolConfig: PoolConfig;
 };
+type RawJsonUpsertOptions = {
+    /**
+     * Columns to use as the ON CONFLICT target.
+     * Example: ['id'] for standard Stripe objects, or a composite key for Sigma tables.
+     */
+    conflictTarget: string[];
+    /**
+     * Additional typed columns to insert alongside `_raw_data` (for tables that don't have `id` keys).
+     * Values are read from `entry[entryKey]` and cast to `pgType` in SQL.
+     */
+    extraColumns?: Array<{
+        column: string;
+        pgType: string;
+        entryKey: string;
+    }>;
+};
 declare class PostgresClient {
     private config;
     pool: pg.Pool;
@@ -17,7 +33,7 @@ declare class PostgresClient {
     }>(entries: T[], table: string): Promise<T[]>;
     upsertManyWithTimestampProtection<T extends {
         [Key: string]: any;
-    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string): Promise<T[]>;
+    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
     private cleanseArrayField;
     findMissingEntries(table: string, ids: string[]): Promise<string[]>;
     upsertAccount(accountData: {
@@ -157,6 +173,10 @@ declare class PostgresClient {
      * This considers completed, error, AND running runs to ensure recovery syncs
      * don't re-process data that was already synced before a crash.
      * A 'running' status with a cursor means the process was killed mid-sync.
+     *
+     * Handles two cursor formats:
+     * - Numeric: compared as bigint for correct ordering
+     * - Composite cursors: compared as strings with COLLATE "C"
      */
     getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
     /**
@@ -198,6 +218,38 @@ declare class PostgresClient {
     close(): Promise<void>;
 }
 
+type SigmaCursorColumnType = 'timestamp' | 'string' | 'number';
+type SigmaCursorColumnSpec = {
+    column: string;
+    type: SigmaCursorColumnType;
+};
+type SigmaCursorSpec = {
+    version: 1;
+    columns: SigmaCursorColumnSpec[];
+};
+type SigmaIngestionConfig = {
+    /**
+     * The Sigma table name to query (no quoting, no schema).
+     */
+    sigmaTable: string;
+    /**
+     * Destination Postgres table name (in the `stripe` schema by convention).
+     */
+    destinationTable: string;
+    /** Limit for each Sigma query page. */
+    pageSize: number;
+    /**
+     * Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
+     */
+    cursor: SigmaCursorSpec;
+    /** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
+    additionalWhere?: string;
+    /** Columns to SELECT (defaults to `*`). */
+    select?: '*' | string[];
+    /** Postgres upsert behavior for this table (conflict target and typed columns). */
+    upsert: RawJsonUpsertOptions;
+};
+
 /**
  * Simple logger interface compatible with both pino and console
  */
@@ -212,6 +264,12 @@ type StripeSyncConfig = {
     databaseUrl?: string;
     /** Stripe secret key used to authenticate requests to the Stripe API. Defaults to empty string */
     stripeSecretKey: string;
+    /**
+     * Enables syncing Stripe Sigma (reporting) tables via the Sigma API.
+     *
+     * Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
+     */
+    enableSigma?: boolean;
     /** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
     stripeAccountId?: string;
     /** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
@@ -259,8 +317,14 @@ type StripeSyncConfig = {
      * Default: 500
      */
     retryJitterMs?: number;
+    /**
+     * Maximum number of customers to process concurrently when syncing payment methods.
+     * Lower values reduce API load but increase sync time.
+     * Default: 10
+     */
+    maxConcurrentCustomers?: number;
 };
-type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions';
+type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | 'subscription_item_change_events_v2_beta' | 'exchange_rates_from_usd';
 interface Sync {
     synced: number;
 }
@@ -282,6 +346,8 @@ interface SyncBackfill {
     earlyFraudWarnings?: Sync;
     refunds?: Sync;
     checkoutSessions?: Sync;
+    subscriptionItemChangeEventsV2Beta?: Sync;
+    exchangeRatesFromUsd?: Sync;
 }
 interface SyncParams {
     created?: {
@@ -334,6 +400,44 @@ interface ProcessNextParams extends SyncParams {
     /** Who/what triggered this sync (for observability) */
     triggeredBy?: string;
 }
+/**
+ * Syncable resource configuration
+ */
+type BaseResourceConfig = {
+    /** Backfill order: lower numbers sync first; parents before children for FK dependencies */
+    order: number;
+    /** Whether this resource supports incremental sync via 'created' filter or cursor */
+    supportsCreatedFilter: boolean;
+};
+type StripeListResourceConfig = BaseResourceConfig & {
+    /** Function to list items from Stripe API */
+    listFn: (params: Stripe.PaginationParams & {
+        created?: Stripe.RangeQueryParam;
+    }) => Promise<{
+        data: unknown[];
+        has_more: boolean;
+    }>;
+    /** Function to upsert items to database */
+    upsertFn: (items: unknown[], accountId: string, backfillRelated?: boolean) => Promise<unknown[] | void>;
+    /** discriminator */
+    sigma?: undefined;
+};
+/**
+ * Configuration for Sigma query-backed resources.
+ * Uses Stripe Sigma SQL queries with composite cursor pagination.
+ */
+type SigmaResourceConfig = BaseResourceConfig & {
+    /** Sigma uses composite cursors, not created filter */
+    supportsCreatedFilter: false;
+    /** Sigma ingestion configuration (query, cursor spec, upsert options) */
+    sigma: SigmaIngestionConfig;
+    /** discriminator */
+    listFn?: undefined;
+    /** discriminator */
+    upsertFn?: undefined;
+};
+/** Union of all resource configuration types */
+type ResourceConfig = StripeListResourceConfig | SigmaResourceConfig;
 /**
  * Installation status of the stripe-sync package
  */
@@ -491,6 +595,8 @@ declare class StripeSync {
      * Uses the observable sync system for tracking progress.
      */
     private fetchOnePage;
+    private getSigmaFallbackCursorFromDestination;
+    private fetchOneSigmaPage;
     /**
      * Process all pages for all (or specified) object types until complete.
      *
@@ -624,6 +730,7 @@ declare class StripeSync {
     backfillSubscriptionSchedules: (subscriptionIds: string[], accountId: string) => Promise<void>;
     /**
      * Stripe only sends the first 10 entries by default, the option will actively fetch all entries.
+     * Uses manual pagination - each fetch() gets automatic retry protection.
      */
     private expandEntity;
     private fetchMissingEntities;
@@ -684,4 +791,4 @@ declare function createStripeWebSocketClient(options: StripeWebSocketOptions): P
 
 declare const VERSION: string;
 
-export { type InstallationStatus, type Logger, PostgresClient, type ProcessNextParams, type ProcessNextResult, type RevalidateEntity, StripeSync, type StripeSyncAccountState, type StripeSyncConfig, type StripeSyncState, type StripeWebSocketClient, type StripeWebSocketOptions, type StripeWebhookEvent, type Sync, type SyncBackfill, type SyncEntitlementsParams, type SyncFeaturesParams, type SyncObject, type SyncParams, VERSION, type WebhookProcessingResult, createStripeWebSocketClient, hashApiKey, runMigrations };
+export { type BaseResourceConfig, type InstallationStatus, type Logger, PostgresClient, type ProcessNextParams, type ProcessNextResult, type ResourceConfig, type RevalidateEntity, type SigmaResourceConfig, type StripeListResourceConfig, StripeSync, type StripeSyncAccountState, type StripeSyncConfig, type StripeSyncState, type StripeWebSocketClient, type StripeWebSocketOptions, type StripeWebhookEvent, type Sync, type SyncBackfill, type SyncEntitlementsParams, type SyncFeaturesParams, type SyncObject, type SyncParams, VERSION, type WebhookProcessingResult, createStripeWebSocketClient, hashApiKey, runMigrations };

package/dist/index.d.ts

@@ -6,6 +6,22 @@ type PostgresConfig = {
     schema: string;
     poolConfig: PoolConfig;
 };
+type RawJsonUpsertOptions = {
+    /**
+     * Columns to use as the ON CONFLICT target.
+     * Example: ['id'] for standard Stripe objects, or a composite key for Sigma tables.
+     */
+    conflictTarget: string[];
+    /**
+     * Additional typed columns to insert alongside `_raw_data` (for tables that don't have `id` keys).
+     * Values are read from `entry[entryKey]` and cast to `pgType` in SQL.
+     */
+    extraColumns?: Array<{
+        column: string;
+        pgType: string;
+        entryKey: string;
+    }>;
+};
 declare class PostgresClient {
     private config;
     pool: pg.Pool;
@@ -17,7 +33,7 @@ declare class PostgresClient {
     }>(entries: T[], table: string): Promise<T[]>;
     upsertManyWithTimestampProtection<T extends {
         [Key: string]: any;
-    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string): Promise<T[]>;
+    }>(entries: T[], table: string, accountId: string, syncTimestamp?: string, upsertOptions?: RawJsonUpsertOptions): Promise<T[]>;
     private cleanseArrayField;
     findMissingEntries(table: string, ids: string[]): Promise<string[]>;
     upsertAccount(accountData: {
@@ -157,6 +173,10 @@ declare class PostgresClient {
      * This considers completed, error, AND running runs to ensure recovery syncs
      * don't re-process data that was already synced before a crash.
      * A 'running' status with a cursor means the process was killed mid-sync.
+     *
+     * Handles two cursor formats:
+     * - Numeric: compared as bigint for correct ordering
+     * - Composite cursors: compared as strings with COLLATE "C"
      */
     getLastCompletedCursor(accountId: string, object: string): Promise<string | null>;
     /**
@@ -198,6 +218,38 @@ declare class PostgresClient {
     close(): Promise<void>;
 }
 
+type SigmaCursorColumnType = 'timestamp' | 'string' | 'number';
+type SigmaCursorColumnSpec = {
+    column: string;
+    type: SigmaCursorColumnType;
+};
+type SigmaCursorSpec = {
+    version: 1;
+    columns: SigmaCursorColumnSpec[];
+};
+type SigmaIngestionConfig = {
+    /**
+     * The Sigma table name to query (no quoting, no schema).
+     */
+    sigmaTable: string;
+    /**
+     * Destination Postgres table name (in the `stripe` schema by convention).
+     */
+    destinationTable: string;
+    /** Limit for each Sigma query page. */
+    pageSize: number;
+    /**
+     * Defines the ordering and cursor semantics. The columns must form a total order (i.e. be unique together) or pagination can be incorrect.
+     */
+    cursor: SigmaCursorSpec;
+    /** Optional additional WHERE clause appended with AND (must not include leading WHERE). */
+    additionalWhere?: string;
+    /** Columns to SELECT (defaults to `*`). */
+    select?: '*' | string[];
+    /** Postgres upsert behavior for this table (conflict target and typed columns). */
+    upsert: RawJsonUpsertOptions;
+};
+
 /**
  * Simple logger interface compatible with both pino and console
  */
@@ -212,6 +264,12 @@ type StripeSyncConfig = {
     databaseUrl?: string;
     /** Stripe secret key used to authenticate requests to the Stripe API. Defaults to empty string */
     stripeSecretKey: string;
+    /**
+     * Enables syncing Stripe Sigma (reporting) tables via the Sigma API.
+     *
+     * Default: false (opt-in, so workers don't enqueue Sigma jobs unexpectedly).
+     */
+    enableSigma?: boolean;
     /** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
     stripeAccountId?: string;
     /** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
@@ -259,8 +317,14 @@ type StripeSyncConfig = {
      * Default: 500
      */
     retryJitterMs?: number;
+    /**
+     * Maximum number of customers to process concurrently when syncing payment methods.
+     * Lower values reduce API load but increase sync time.
+     * Default: 10
+     */
+    maxConcurrentCustomers?: number;
 };
-type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions';
+type SyncObject = 'all' | 'customer' | 'customer_with_entitlements' | 'invoice' | 'price' | 'product' | 'subscription' | 'subscription_schedules' | 'setup_intent' | 'payment_method' | 'dispute' | 'charge' | 'payment_intent' | 'plan' | 'tax_id' | 'credit_note' | 'early_fraud_warning' | 'refund' | 'checkout_sessions' | 'subscription_item_change_events_v2_beta' | 'exchange_rates_from_usd';
 interface Sync {
     synced: number;
 }
@@ -282,6 +346,8 @@ interface SyncBackfill {
     earlyFraudWarnings?: Sync;
     refunds?: Sync;
     checkoutSessions?: Sync;
+    subscriptionItemChangeEventsV2Beta?: Sync;
+    exchangeRatesFromUsd?: Sync;
 }
 interface SyncParams {
     created?: {
@@ -334,6 +400,44 @@ interface ProcessNextParams extends SyncParams {
     /** Who/what triggered this sync (for observability) */
     triggeredBy?: string;
 }
+/**
+ * Syncable resource configuration
+ */
+type BaseResourceConfig = {
+    /** Backfill order: lower numbers sync first; parents before children for FK dependencies */
+    order: number;
+    /** Whether this resource supports incremental sync via 'created' filter or cursor */
+    supportsCreatedFilter: boolean;
+};
+type StripeListResourceConfig = BaseResourceConfig & {
+    /** Function to list items from Stripe API */
+    listFn: (params: Stripe.PaginationParams & {
+        created?: Stripe.RangeQueryParam;
+    }) => Promise<{
+        data: unknown[];
+        has_more: boolean;
+    }>;
+    /** Function to upsert items to database */
+    upsertFn: (items: unknown[], accountId: string, backfillRelated?: boolean) => Promise<unknown[] | void>;
+    /** discriminator */
+    sigma?: undefined;
+};
+/**
+ * Configuration for Sigma query-backed resources.
+ * Uses Stripe Sigma SQL queries with composite cursor pagination.
+ */
+type SigmaResourceConfig = BaseResourceConfig & {
+    /** Sigma uses composite cursors, not created filter */
+    supportsCreatedFilter: false;
+    /** Sigma ingestion configuration (query, cursor spec, upsert options) */
+    sigma: SigmaIngestionConfig;
+    /** discriminator */
+    listFn?: undefined;
+    /** discriminator */
+    upsertFn?: undefined;
+};
+/** Union of all resource configuration types */
+type ResourceConfig = StripeListResourceConfig | SigmaResourceConfig;
 /**
  * Installation status of the stripe-sync package
  */
@@ -491,6 +595,8 @@ declare class StripeSync {
      * Uses the observable sync system for tracking progress.
      */
     private fetchOnePage;
+    private getSigmaFallbackCursorFromDestination;
+    private fetchOneSigmaPage;
     /**
      * Process all pages for all (or specified) object types until complete.
      *
@@ -624,6 +730,7 @@ declare class StripeSync {
     backfillSubscriptionSchedules: (subscriptionIds: string[], accountId: string) => Promise<void>;
     /**
      * Stripe only sends the first 10 entries by default, the option will actively fetch all entries.
+     * Uses manual pagination - each fetch() gets automatic retry protection.
      */
     private expandEntity;
     private fetchMissingEntities;
@@ -684,4 +791,4 @@ declare function createStripeWebSocketClient(options: StripeWebSocketOptions): P
 
 declare const VERSION: string;
 
-export { type InstallationStatus, type Logger, PostgresClient, type ProcessNextParams, type ProcessNextResult, type RevalidateEntity, StripeSync, type StripeSyncAccountState, type StripeSyncConfig, type StripeSyncState, type StripeWebSocketClient, type StripeWebSocketOptions, type StripeWebhookEvent, type Sync, type SyncBackfill, type SyncEntitlementsParams, type SyncFeaturesParams, type SyncObject, type SyncParams, VERSION, type WebhookProcessingResult, createStripeWebSocketClient, hashApiKey, runMigrations };
+export { type BaseResourceConfig, type InstallationStatus, type Logger, PostgresClient, type ProcessNextParams, type ProcessNextResult, type ResourceConfig, type RevalidateEntity, type SigmaResourceConfig, type StripeListResourceConfig, StripeSync, type StripeSyncAccountState, type StripeSyncConfig, type StripeSyncState, type StripeWebSocketClient, type StripeWebSocketOptions, type StripeWebhookEvent, type Sync, type SyncBackfill, type SyncEntitlementsParams, type SyncFeaturesParams, type SyncObject, type SyncParams, VERSION, type WebhookProcessingResult, createStripeWebSocketClient, hashApiKey, runMigrations };

package/dist/index.js

@@ -5,8 +5,8 @@ import {
   createStripeWebSocketClient,
   hashApiKey,
   runMigrations
-} from "./chunk-2GSABFXH.js";
-import "./chunk-RCU5ZXAX.js";
+} from "./chunk-GOBVFXU7.js";
+import "./chunk-3KVILTN4.js";
 export {
   PostgresClient,
   StripeSync,

package/dist/migrations/0059_sigma_subscription_item_change_events_v2_beta.sql

@@ -0,0 +1,61 @@
+-- event_timestamp and event_type are not generated columns because they are not immutable. 
+-- Postgres requires generated expressions to be immutable.
+
+CREATE TABLE IF NOT EXISTS "stripe"."subscription_item_change_events_v2_beta" (
+  "_raw_data" jsonb NOT NULL,
+  "_last_synced_at" timestamptz,
+  "_updated_at" timestamptz DEFAULT now(),
+  "_account_id" text NOT NULL,
+
+  "event_timestamp" timestamptz NOT NULL,
+  "event_type" text NOT NULL,
+  "subscription_item_id" text NOT NULL,
+
+  PRIMARY KEY ("_account_id", "event_timestamp", "event_type", "subscription_item_id")
+);
+
+-- Foreign key to stripe.accounts
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  DROP CONSTRAINT IF EXISTS fk_subscription_item_change_events_v2_beta_account;
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD CONSTRAINT fk_subscription_item_change_events_v2_beta_account
+  FOREIGN KEY ("_account_id") REFERENCES "stripe"."accounts" (id);
+
+-- Maintain _updated_at on UPDATE
+DROP TRIGGER IF EXISTS handle_updated_at ON "stripe"."subscription_item_change_events_v2_beta";
+CREATE TRIGGER handle_updated_at
+  BEFORE UPDATE ON "stripe"."subscription_item_change_events_v2_beta"
+  FOR EACH ROW EXECUTE FUNCTION set_updated_at();
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "currency" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'currency', ''))::text) STORED;
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "mrr_change" bigint
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'mrr_change', ''))::bigint) STORED;
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "quantity_change" bigint
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'quantity_change', ''))::bigint) STORED;
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "subscription_id" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'subscription_id', ''))::text) STORED;
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "customer_id" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'customer_id', ''))::text) STORED;
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "price_id" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'price_id', ''))::text) STORED;
+
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "product_id" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'product_id', ''))::text) STORED;
+
+-- Keep as text to avoid non-immutable timestamp casts in a generated column
+ALTER TABLE "stripe"."subscription_item_change_events_v2_beta"
+  ADD COLUMN IF NOT EXISTS "local_event_timestamp" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'local_event_timestamp', ''))::text) STORED;

package/dist/migrations/0060_sigma_exchange_rates_from_usd.sql

@@ -0,0 +1,38 @@
+
+CREATE TABLE IF NOT EXISTS "stripe"."exchange_rates_from_usd" (
+  "_raw_data" jsonb NOT NULL,
+  "_last_synced_at" timestamptz,
+  "_updated_at" timestamptz DEFAULT now(),
+  "_account_id" text NOT NULL,
+
+  "date" date NOT NULL,
+  "sell_currency" text NOT NULL,
+
+  PRIMARY KEY ("_account_id", "date", "sell_currency")
+);
+
+-- Foreign key to stripe.accounts
+ALTER TABLE "stripe"."exchange_rates_from_usd"
+  DROP CONSTRAINT IF EXISTS fk_exchange_rates_from_usd_account;
+ALTER TABLE "stripe"."exchange_rates_from_usd"
+  ADD CONSTRAINT fk_exchange_rates_from_usd_account
+  FOREIGN KEY ("_account_id") REFERENCES "stripe"."accounts" (id);
+
+-- Maintain _updated_at on UPDATE
+DROP TRIGGER IF EXISTS handle_updated_at ON "stripe"."exchange_rates_from_usd";
+CREATE TRIGGER handle_updated_at
+  BEFORE UPDATE ON "stripe"."exchange_rates_from_usd"
+  FOR EACH ROW EXECUTE FUNCTION set_updated_at();
+
+ALTER TABLE "stripe"."exchange_rates_from_usd"
+  ADD COLUMN IF NOT EXISTS "buy_currency_exchange_rates" text
+    GENERATED ALWAYS AS ((NULLIF(_raw_data->>'buy_currency_exchange_rates', ''))::text) STORED;
+
+-- Index on date for efficient range queries
+CREATE INDEX IF NOT EXISTS idx_exchange_rates_from_usd_date
+  ON "stripe"."exchange_rates_from_usd" ("date");
+
+-- Index on sell_currency for filtering by currency
+CREATE INDEX IF NOT EXISTS idx_exchange_rates_from_usd_sell_currency
+  ON "stripe"."exchange_rates_from_usd" ("sell_currency");
+