stripe-experiment-sync 1.0.21 → 1.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-G26S5D5J.js → chunk-6OQC4YJW.js} +2 -2
- package/dist/{chunk-3KTFUZTY.js → chunk-EIBEWIQC.js} +3 -3
- package/dist/{chunk-ECLPGCY6.js → chunk-EZ73ACI4.js} +20 -9
- package/dist/{chunk-OWZ4QNLS.js → chunk-IBYBC6LT.js} +1 -1
- package/dist/cli/index.cjs +22 -11
- package/dist/cli/index.js +4 -4
- package/dist/cli/lib.cjs +22 -11
- package/dist/cli/lib.js +4 -4
- package/dist/index.cjs +20 -9
- package/dist/index.d.cts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -2
- package/dist/supabase/index.cjs +3 -3
- package/dist/supabase/index.js +2 -2
- package/package.json +1 -1
|
@@ -3,11 +3,11 @@ import {
|
|
|
3
3
|
StripeSync,
|
|
4
4
|
createStripeWebSocketClient,
|
|
5
5
|
runMigrations
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-EZ73ACI4.js";
|
|
7
7
|
import {
|
|
8
8
|
install,
|
|
9
9
|
uninstall
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-EIBEWIQC.js";
|
|
11
11
|
|
|
12
12
|
// src/cli/config.ts
|
|
13
13
|
import dotenv from "dotenv";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
package_default
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-IBYBC6LT.js";
|
|
4
4
|
|
|
5
5
|
// src/supabase/supabase.ts
|
|
6
6
|
import { SupabaseManagementAPI } from "supabase-management-js";
|
|
@@ -9,10 +9,10 @@ import { SupabaseManagementAPI } from "supabase-management-js";
|
|
|
9
9
|
var stripe_setup_default = "import { StripeSync, runMigrations, VERSION } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\n// Get management API base URL from environment variable (for testing against localhost/staging)\n// Caller should provide full URL with protocol (e.g., http://localhost:54323 or https://api.supabase.com)\nconst MGMT_API_BASE_RAW = Deno.env.get('MANAGEMENT_API_URL') || 'https://api.supabase.com'\nconst MGMT_API_BASE = MGMT_API_BASE_RAW.match(/^https?:\\/\\//)\n ? MGMT_API_BASE_RAW\n : `https://${MGMT_API_BASE_RAW}`\n\n// Helper to validate accessToken against Management API\nasync function validateAccessToken(projectRef: string, accessToken: string): Promise<boolean> {\n // Try to fetch project details using the access token\n // This validates that the token is valid for the management API\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}`\n const response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n // If we can successfully get the project, the token is valid\n return response.ok\n}\n\n// Helper to delete edge function via Management API\nasync function deleteEdgeFunction(\n projectRef: string,\n functionSlug: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/functions/${functionSlug}`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n throw new Error(`Failed to delete function ${functionSlug}: ${response.status} ${text}`)\n }\n}\n\n// Helper to delete secrets via Management API\nasync function deleteSecret(\n projectRef: string,\n secretName: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/secrets`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify([secretName]),\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n console.warn(`Failed to delete secret ${secretName}: ${response.status} ${text}`)\n }\n}\n\nDeno.serve(async (req) => {\n // Extract project ref from SUPABASE_URL (format: https://{projectRef}.{base})\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n const projectRef = new URL(supabaseUrl).hostname.split('.')[0]\n\n // Validate access token for all requests\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const accessToken = authHeader.substring(7) // Remove 'Bearer '\n const isValid = await validateAccessToken(projectRef, accessToken)\n if (!isValid) {\n return new Response('Forbidden: Invalid access token for this project', { status: 403 })\n }\n\n // Handle GET requests for status\n if (req.method === 'GET') {\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n let sql\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n\n // Query installation status from schema comment\n const commentResult = await sql`\n SELECT obj_description(oid, 'pg_namespace') as comment\n FROM pg_namespace\n WHERE nspname = 'stripe'\n `\n\n const comment = commentResult[0]?.comment || null\n let installationStatus = 'not_installed'\n\n if (comment && comment.includes('stripe-sync')) {\n // Parse installation status from comment\n if (comment.includes('installation:started')) {\n installationStatus = 'installing'\n } else if (comment.includes('installation:error')) {\n installationStatus = 'error'\n } else if (comment.includes('installed')) {\n installationStatus = 'installed'\n }\n }\n\n // Query sync runs (only if schema exists)\n let syncStatus = []\n if (comment) {\n try {\n syncStatus = await sql`\n SELECT DISTINCT ON (account_id)\n account_id, started_at, closed_at, status, error_message,\n total_processed, total_objects, complete_count, error_count,\n running_count, pending_count, triggered_by, max_concurrent\n FROM stripe.sync_runs\n ORDER BY account_id, started_at DESC\n `\n } catch (err) {\n // Ignore errors if sync_runs view doesn't exist yet\n console.warn('sync_runs query failed (may not exist yet):', err)\n }\n }\n\n return new Response(\n JSON.stringify({\n package_version: VERSION,\n installation_status: installationStatus,\n sync_status: syncStatus,\n }),\n {\n status: 200,\n headers: {\n 'Content-Type': 'application/json',\n 'Cache-Control': 'no-cache, no-store, must-revalidate',\n },\n }\n )\n } catch (error) {\n console.error('Status query error:', error)\n return new Response(\n JSON.stringify({\n error: error.message,\n package_version: VERSION,\n installation_status: 'not_installed',\n }),\n {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } finally {\n if (sql) await sql.end()\n }\n }\n\n // Handle DELETE requests for uninstall\n if (req.method === 'DELETE') {\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n // Stripe key is required for uninstall to delete webhooks\n const stripeKey = Deno.env.get('STRIPE_SECRET_KEY')\n if (!stripeKey) {\n throw new Error('STRIPE_SECRET_KEY environment variable is required for uninstall')\n }\n\n // Step 1: Delete Stripe webhooks and clean up database\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 },\n stripeSecretKey: stripeKey,\n })\n\n // Delete all managed webhooks\n const webhooks = await stripeSync.listManagedWebhooks()\n for (const webhook of webhooks) {\n try {\n await stripeSync.deleteManagedWebhook(webhook.id)\n console.log(`Deleted webhook: ${webhook.id}`)\n } catch (err) {\n console.warn(`Could not delete webhook ${webhook.id}:`, err)\n }\n }\n\n // Unschedule pg_cron jobs\n try {\n await stripeSync.postgresClient.query(`\n DO $$\n BEGIN\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sync-worker') THEN\n PERFORM cron.unschedule('stripe-sync-worker');\n END IF;\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sigma-worker') THEN\n PERFORM cron.unschedule('stripe-sigma-worker');\n END IF;\n END $$;\n `)\n } catch (err) {\n console.warn('Could not unschedule pg_cron job:', err)\n }\n\n // Delete vault secrets\n try {\n await stripeSync.postgresClient.query(`\n DELETE FROM vault.secrets\n WHERE name IN ('stripe_sync_worker_secret', 'stripe_sigma_worker_secret')\n `)\n } catch (err) {\n console.warn('Could not delete vault secret:', err)\n }\n\n // Drop Sigma self-trigger function if present\n try {\n await stripeSync.postgresClient.query(`\n DROP FUNCTION IF EXISTS stripe.trigger_sigma_worker();\n `)\n } catch (err) {\n console.warn('Could not drop sigma trigger function:', err)\n }\n\n // Terminate connections holding locks on stripe schema\n try {\n await stripeSync.postgresClient.query(`\n SELECT pg_terminate_backend(pid)\n FROM pg_locks l\n JOIN pg_class c ON l.relation = c.oid\n JOIN pg_namespace n ON c.relnamespace = n.oid\n WHERE n.nspname = 'stripe'\n AND l.pid != pg_backend_pid()\n `)\n } catch (err) {\n console.warn('Could not terminate connections:', err)\n }\n\n // Drop schema with retry\n let dropAttempts = 0\n const maxAttempts = 3\n while (dropAttempts < maxAttempts) {\n try {\n await stripeSync.postgresClient.query('DROP SCHEMA IF EXISTS stripe CASCADE')\n break // Success, exit loop\n } catch (err) {\n dropAttempts++\n if (dropAttempts >= maxAttempts) {\n throw new Error(\n `Failed to drop schema after ${maxAttempts} attempts. ` +\n `There may be active connections or locks on the stripe schema. ` +\n `Error: ${err.message}`\n )\n }\n // Wait 1 second before retrying\n await new Promise((resolve) => setTimeout(resolve, 1000))\n }\n }\n\n await stripeSync.postgresClient.pool.end()\n\n // Step 2: Delete Supabase secrets\n try {\n await deleteSecret(projectRef, 'STRIPE_SECRET_KEY', accessToken)\n } catch (err) {\n console.warn('Could not delete STRIPE_SECRET_KEY secret:', err)\n }\n\n // Step 3: Delete Edge Functions\n try {\n await deleteEdgeFunction(projectRef, 'stripe-setup', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-setup function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-webhook', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-webhook function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-worker', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-worker function:', err)\n }\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Uninstall complete',\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Uninstall error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n }\n\n // Handle POST requests for install\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const enableSigma = (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true'\n await runMigrations({ databaseUrl: dbUrl, enableSigma })\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 }, // Need 2 for advisory lock + queries\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY'),\n })\n\n // Release any stale advisory locks from previous timeouts\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n\n // Construct webhook URL from SUPABASE_URL (available in all Edge Functions)\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n throw new Error('SUPABASE_URL environment variable is not set')\n }\n const webhookUrl = supabaseUrl + '/functions/v1/stripe-webhook'\n\n const webhook = await stripeSync.findOrCreateManagedWebhook(webhookUrl)\n\n await stripeSync.postgresClient.pool.end()\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Setup complete',\n webhookId: webhook.id,\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Setup error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n})\n";
|
|
10
10
|
|
|
11
11
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-webhook.ts
|
|
12
|
-
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
12
|
+
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n partnerId: 'pp_supabase',\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
13
13
|
|
|
14
14
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-worker.ts
|
|
15
|
-
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
15
|
+
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n partnerId: 'pp_supabase',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
16
16
|
|
|
17
17
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/sigma-data-worker.ts
|
|
18
18
|
var sigma_data_worker_default = "/**\n * Stripe Sigma Data Worker.\n *\n * Hourly cron starts a run; self-trigger continues until all objects finish.\n * Progress persists in _sync_runs and _sync_obj_runs across invocations.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst BATCH_SIZE = 1\nconst MAX_RUN_AGE_MS = 6 * 60 * 60 * 1000\nconst jsonResponse = (body: unknown, status = 200) =>\n new Response(JSON.stringify(body), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7)\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return jsonResponse({ error: 'SUPABASE_DB_URL not set' }, 500)\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql: ReturnType<typeof postgres> | undefined\n let stripeSync: StripeSync | undefined\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return jsonResponse(\n {\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n // Validate the token against vault secret\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sigma_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Sigma worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid sigma worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: true,\n sigmaPageSizeOverride: 1000,\n })\n } catch (error) {\n await sql.end()\n return jsonResponse(\n {\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n const accountId = await stripeSync.getAccountId()\n const sigmaObjects = stripeSync.getSupportedSigmaObjects()\n\n if (sigmaObjects.length === 0) {\n return jsonResponse({ message: 'No Sigma objects configured for sync' })\n }\n\n // Get or create sync run for sigma-worker (isolated from stripe-worker)\n const runResult = await stripeSync.postgresClient.getOrCreateSyncRun(accountId, 'sigma-worker')\n const runStartedAt =\n runResult?.runStartedAt ??\n (await stripeSync.postgresClient.getActiveSyncRun(accountId, 'sigma-worker'))?.runStartedAt\n\n if (!runStartedAt) {\n throw new Error('Failed to get or create sync run for sigma worker')\n }\n\n // Legacy cleanup: remove any prefixed sigma object runs that can block concurrency.\n // Previous versions stored objects as \"sigma.<table>\" which no longer matches processNext.\n await stripeSync.postgresClient.query(\n `UPDATE \"stripe\".\"_sync_obj_runs\"\n SET status = 'error',\n error_message = 'Legacy sigma worker prefix run (sigma.*); superseded by unprefixed runs',\n completed_at = now()\n WHERE \"_account_id\" = $1\n AND run_started_at = $2\n AND object LIKE 'sigma.%'\n AND status IN ('pending', 'running')`,\n [accountId, runStartedAt]\n )\n\n // Stop self-triggering after MAX_RUN_AGE_MS.\n const runAgeMs = Date.now() - runStartedAt.getTime()\n if (runAgeMs > MAX_RUN_AGE_MS) {\n console.warn(\n `Sigma worker: run too old (${Math.round(runAgeMs / 1000 / 60)} min), closing without self-trigger`\n )\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n return jsonResponse({\n message: 'Sigma run exceeded max age, closed without processing',\n runAgeMinutes: Math.round(runAgeMs / 1000 / 60),\n selfTriggered: false,\n })\n }\n\n // Create object runs for all sigma objects (idempotent).\n await stripeSync.postgresClient.createObjectRuns(accountId, runStartedAt, sigmaObjects)\n await stripeSync.postgresClient.ensureSyncRunMaxConcurrent(accountId, runStartedAt, BATCH_SIZE)\n\n // Prefer running objects; otherwise claim pending ones.\n const runningObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n const objectsToProcess = runningObjects.slice(0, BATCH_SIZE)\n let pendingObjects: string[] = []\n\n if (objectsToProcess.length === 0) {\n pendingObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n\n for (const objectKey of pendingObjects) {\n if (objectsToProcess.length >= BATCH_SIZE) break\n const started = await stripeSync.postgresClient.tryStartObjectSync(\n accountId,\n runStartedAt,\n objectKey\n )\n if (started) {\n objectsToProcess.push(objectKey)\n }\n }\n }\n\n if (objectsToProcess.length === 0) {\n if (pendingObjects.length === 0) {\n console.info('Sigma worker: all objects complete or errored - run finished')\n return jsonResponse({ message: 'Sigma sync run complete', selfTriggered: false })\n }\n\n console.info('Sigma worker: at concurrency limit, will self-trigger', {\n pendingCount: pendingObjects.length,\n })\n let selfTriggered = false\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n\n return jsonResponse({\n message: 'At concurrency limit',\n pendingCount: pendingObjects.length,\n selfTriggered,\n })\n }\n\n // Process objects sequentially (one lifecycle per invocation).\n const results: Array<Record<string, unknown>> = []\n\n for (const object of objectsToProcess) {\n const objectKey = object\n try {\n console.info(`Sigma worker: processing ${object}`)\n\n // Process one sigma page and upsert results.\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const result = await stripeSync.processNext(object as any, {\n runStartedAt,\n triggeredBy: 'sigma-worker',\n })\n\n results.push({\n object,\n processed: result.processed,\n hasMore: result.hasMore,\n status: 'success',\n })\n\n if (result.hasMore) {\n console.info(\n `Sigma worker: ${object} has more pages, processed ${result.processed} rows so far`\n )\n } else {\n console.info(`Sigma worker: ${object} complete, processed ${result.processed} rows`)\n }\n } catch (error) {\n console.error(`Sigma worker: error processing ${object}:`, error)\n\n // Mark object as failed and move on (no retries)\n await stripeSync.postgresClient.failObjectSync(\n accountId,\n runStartedAt,\n objectKey,\n error.message ?? 'Unknown error'\n )\n\n results.push({\n object,\n processed: 0,\n hasMore: false,\n status: 'error',\n error: error.message,\n })\n }\n }\n\n // Determine if self-trigger is needed\n const pendingAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n const runningAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n // Calculate remaining run time for logging\n const remainingMs = MAX_RUN_AGE_MS - (Date.now() - runStartedAt.getTime())\n const remainingMinutes = Math.round(remainingMs / 1000 / 60)\n\n // Only self-trigger if there are pending or running objects AND run hasn't timed out\n const shouldSelfTrigger =\n (pendingAfter.length > 0 || runningAfter.length > 0) && remainingMs > 0\n\n let selfTriggered = false\n if (shouldSelfTrigger) {\n console.info('Sigma worker: more work remains, self-triggering', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n remainingMinutes,\n })\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n } else if (pendingAfter.length > 0 || runningAfter.length > 0) {\n // Would self-trigger but run timed out\n console.warn('Sigma worker: work remains but run timed out, closing', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n })\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n } else {\n console.info('Sigma worker: no more work, run complete')\n }\n\n return jsonResponse({\n results,\n selfTriggered,\n remaining: { pending: pendingAfter.length, running: runningAfter.length },\n })\n } catch (error) {\n console.error('Sigma worker error:', error)\n return jsonResponse({ error: error.message, stack: error.stack }, 500)\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
package_default
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-IBYBC6LT.js";
|
|
4
4
|
|
|
5
5
|
// src/stripeSync.ts
|
|
6
6
|
import Stripe3 from "stripe";
|
|
@@ -1048,12 +1048,13 @@ function normalizeSigmaTimestampToIso(value) {
|
|
|
1048
1048
|
if (Number.isNaN(d.getTime())) return null;
|
|
1049
1049
|
return d.toISOString();
|
|
1050
1050
|
}
|
|
1051
|
-
function createStripeClient(apiKey) {
|
|
1051
|
+
function createStripeClient(apiKey, partnerId) {
|
|
1052
1052
|
return new Stripe2(apiKey, {
|
|
1053
1053
|
appInfo: {
|
|
1054
1054
|
name: "Stripe Sync Engine",
|
|
1055
1055
|
version: package_default.version,
|
|
1056
|
-
url: package_default.homepage
|
|
1056
|
+
url: package_default.homepage,
|
|
1057
|
+
...partnerId ? { partner_id: partnerId } : {}
|
|
1057
1058
|
}
|
|
1058
1059
|
});
|
|
1059
1060
|
}
|
|
@@ -1072,14 +1073,14 @@ async function fetchStripeText(url, apiKey, options) {
|
|
|
1072
1073
|
return text;
|
|
1073
1074
|
}
|
|
1074
1075
|
async function createSigmaQueryRun(params) {
|
|
1075
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1076
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1076
1077
|
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1077
1078
|
sql: params.sql
|
|
1078
1079
|
});
|
|
1079
1080
|
return { queryRunId: created.id };
|
|
1080
1081
|
}
|
|
1081
1082
|
async function getSigmaQueryRun(params) {
|
|
1082
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1083
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1083
1084
|
const current = await stripe.rawRequest(
|
|
1084
1085
|
"GET",
|
|
1085
1086
|
`/v1/sigma/query_runs/${params.queryRunId}`,
|
|
@@ -1101,7 +1102,11 @@ async function downloadSigmaFile(params) {
|
|
|
1101
1102
|
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1102
1103
|
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1103
1104
|
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1104
|
-
const { queryRunId } = await createSigmaQueryRun({
|
|
1105
|
+
const { queryRunId } = await createSigmaQueryRun({
|
|
1106
|
+
apiKey: params.apiKey,
|
|
1107
|
+
sql: params.sql,
|
|
1108
|
+
partnerId: params.partnerId
|
|
1109
|
+
});
|
|
1105
1110
|
const start = Date.now();
|
|
1106
1111
|
let current = {
|
|
1107
1112
|
id: queryRunId,
|
|
@@ -1114,7 +1119,11 @@ async function runSigmaQueryAndDownloadCsv(params) {
|
|
|
1114
1119
|
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1115
1120
|
}
|
|
1116
1121
|
await sleep2(pollIntervalMs);
|
|
1117
|
-
const next = await getSigmaQueryRun({
|
|
1122
|
+
const next = await getSigmaQueryRun({
|
|
1123
|
+
apiKey: params.apiKey,
|
|
1124
|
+
queryRunId,
|
|
1125
|
+
partnerId: params.partnerId
|
|
1126
|
+
});
|
|
1118
1127
|
current = {
|
|
1119
1128
|
id: queryRunId,
|
|
1120
1129
|
status: next.status,
|
|
@@ -45132,7 +45141,8 @@ var StripeSync = class {
|
|
|
45132
45141
|
appInfo: {
|
|
45133
45142
|
name: "Stripe Sync Engine",
|
|
45134
45143
|
version: package_default.version,
|
|
45135
|
-
url: package_default.homepage
|
|
45144
|
+
url: package_default.homepage,
|
|
45145
|
+
...config.partnerId ? { partner_id: config.partnerId } : {}
|
|
45136
45146
|
}
|
|
45137
45147
|
});
|
|
45138
45148
|
this.stripe = createRetryableStripeClient(baseStripe, {}, config.logger);
|
|
@@ -46188,7 +46198,8 @@ ${message}`;
|
|
|
46188
46198
|
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
46189
46199
|
apiKey: this.config.stripeSecretKey,
|
|
46190
46200
|
sql: sigmaSql,
|
|
46191
|
-
logger: this.config.logger
|
|
46201
|
+
logger: this.config.logger,
|
|
46202
|
+
partnerId: this.config.partnerId
|
|
46192
46203
|
});
|
|
46193
46204
|
const rows = parseCsvObjects(csv);
|
|
46194
46205
|
if (rows.length === 0) {
|
package/dist/cli/index.cjs
CHANGED
|
@@ -33,7 +33,7 @@ var import_commander = require("commander");
|
|
|
33
33
|
// package.json
|
|
34
34
|
var package_default = {
|
|
35
35
|
name: "stripe-experiment-sync",
|
|
36
|
-
version: "1.0.
|
|
36
|
+
version: "1.0.22",
|
|
37
37
|
private: false,
|
|
38
38
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
39
39
|
type: "module",
|
|
@@ -1233,12 +1233,13 @@ function normalizeSigmaTimestampToIso(value) {
|
|
|
1233
1233
|
if (Number.isNaN(d.getTime())) return null;
|
|
1234
1234
|
return d.toISOString();
|
|
1235
1235
|
}
|
|
1236
|
-
function createStripeClient(apiKey) {
|
|
1236
|
+
function createStripeClient(apiKey, partnerId) {
|
|
1237
1237
|
return new import_stripe2.default(apiKey, {
|
|
1238
1238
|
appInfo: {
|
|
1239
1239
|
name: "Stripe Sync Engine",
|
|
1240
1240
|
version: package_default.version,
|
|
1241
|
-
url: package_default.homepage
|
|
1241
|
+
url: package_default.homepage,
|
|
1242
|
+
...partnerId ? { partner_id: partnerId } : {}
|
|
1242
1243
|
}
|
|
1243
1244
|
});
|
|
1244
1245
|
}
|
|
@@ -1257,14 +1258,14 @@ async function fetchStripeText(url, apiKey, options) {
|
|
|
1257
1258
|
return text;
|
|
1258
1259
|
}
|
|
1259
1260
|
async function createSigmaQueryRun(params) {
|
|
1260
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1261
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1261
1262
|
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1262
1263
|
sql: params.sql
|
|
1263
1264
|
});
|
|
1264
1265
|
return { queryRunId: created.id };
|
|
1265
1266
|
}
|
|
1266
1267
|
async function getSigmaQueryRun(params) {
|
|
1267
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1268
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1268
1269
|
const current = await stripe.rawRequest(
|
|
1269
1270
|
"GET",
|
|
1270
1271
|
`/v1/sigma/query_runs/${params.queryRunId}`,
|
|
@@ -1286,7 +1287,11 @@ async function downloadSigmaFile(params) {
|
|
|
1286
1287
|
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1287
1288
|
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1288
1289
|
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1289
|
-
const { queryRunId } = await createSigmaQueryRun({
|
|
1290
|
+
const { queryRunId } = await createSigmaQueryRun({
|
|
1291
|
+
apiKey: params.apiKey,
|
|
1292
|
+
sql: params.sql,
|
|
1293
|
+
partnerId: params.partnerId
|
|
1294
|
+
});
|
|
1290
1295
|
const start = Date.now();
|
|
1291
1296
|
let current = {
|
|
1292
1297
|
id: queryRunId,
|
|
@@ -1299,7 +1304,11 @@ async function runSigmaQueryAndDownloadCsv(params) {
|
|
|
1299
1304
|
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1300
1305
|
}
|
|
1301
1306
|
await sleep2(pollIntervalMs);
|
|
1302
|
-
const next = await getSigmaQueryRun({
|
|
1307
|
+
const next = await getSigmaQueryRun({
|
|
1308
|
+
apiKey: params.apiKey,
|
|
1309
|
+
queryRunId,
|
|
1310
|
+
partnerId: params.partnerId
|
|
1311
|
+
});
|
|
1303
1312
|
current = {
|
|
1304
1313
|
id: queryRunId,
|
|
1305
1314
|
status: next.status,
|
|
@@ -45317,7 +45326,8 @@ var StripeSync = class {
|
|
|
45317
45326
|
appInfo: {
|
|
45318
45327
|
name: "Stripe Sync Engine",
|
|
45319
45328
|
version: package_default.version,
|
|
45320
|
-
url: package_default.homepage
|
|
45329
|
+
url: package_default.homepage,
|
|
45330
|
+
...config.partnerId ? { partner_id: config.partnerId } : {}
|
|
45321
45331
|
}
|
|
45322
45332
|
});
|
|
45323
45333
|
this.stripe = createRetryableStripeClient(baseStripe, {}, config.logger);
|
|
@@ -46373,7 +46383,8 @@ ${message}`;
|
|
|
46373
46383
|
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
46374
46384
|
apiKey: this.config.stripeSecretKey,
|
|
46375
46385
|
sql: sigmaSql,
|
|
46376
|
-
logger: this.config.logger
|
|
46386
|
+
logger: this.config.logger,
|
|
46387
|
+
partnerId: this.config.partnerId
|
|
46377
46388
|
});
|
|
46378
46389
|
const rows = parseCsvObjects(csv);
|
|
46379
46390
|
if (rows.length === 0) {
|
|
@@ -48541,10 +48552,10 @@ var import_supabase_management_js = require("supabase-management-js");
|
|
|
48541
48552
|
var stripe_setup_default = "import { StripeSync, runMigrations, VERSION } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\n// Get management API base URL from environment variable (for testing against localhost/staging)\n// Caller should provide full URL with protocol (e.g., http://localhost:54323 or https://api.supabase.com)\nconst MGMT_API_BASE_RAW = Deno.env.get('MANAGEMENT_API_URL') || 'https://api.supabase.com'\nconst MGMT_API_BASE = MGMT_API_BASE_RAW.match(/^https?:\\/\\//)\n ? MGMT_API_BASE_RAW\n : `https://${MGMT_API_BASE_RAW}`\n\n// Helper to validate accessToken against Management API\nasync function validateAccessToken(projectRef: string, accessToken: string): Promise<boolean> {\n // Try to fetch project details using the access token\n // This validates that the token is valid for the management API\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}`\n const response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n // If we can successfully get the project, the token is valid\n return response.ok\n}\n\n// Helper to delete edge function via Management API\nasync function deleteEdgeFunction(\n projectRef: string,\n functionSlug: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/functions/${functionSlug}`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n throw new Error(`Failed to delete function ${functionSlug}: ${response.status} ${text}`)\n }\n}\n\n// Helper to delete secrets via Management API\nasync function deleteSecret(\n projectRef: string,\n secretName: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/secrets`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify([secretName]),\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n console.warn(`Failed to delete secret ${secretName}: ${response.status} ${text}`)\n }\n}\n\nDeno.serve(async (req) => {\n // Extract project ref from SUPABASE_URL (format: https://{projectRef}.{base})\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n const projectRef = new URL(supabaseUrl).hostname.split('.')[0]\n\n // Validate access token for all requests\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const accessToken = authHeader.substring(7) // Remove 'Bearer '\n const isValid = await validateAccessToken(projectRef, accessToken)\n if (!isValid) {\n return new Response('Forbidden: Invalid access token for this project', { status: 403 })\n }\n\n // Handle GET requests for status\n if (req.method === 'GET') {\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n let sql\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n\n // Query installation status from schema comment\n const commentResult = await sql`\n SELECT obj_description(oid, 'pg_namespace') as comment\n FROM pg_namespace\n WHERE nspname = 'stripe'\n `\n\n const comment = commentResult[0]?.comment || null\n let installationStatus = 'not_installed'\n\n if (comment && comment.includes('stripe-sync')) {\n // Parse installation status from comment\n if (comment.includes('installation:started')) {\n installationStatus = 'installing'\n } else if (comment.includes('installation:error')) {\n installationStatus = 'error'\n } else if (comment.includes('installed')) {\n installationStatus = 'installed'\n }\n }\n\n // Query sync runs (only if schema exists)\n let syncStatus = []\n if (comment) {\n try {\n syncStatus = await sql`\n SELECT DISTINCT ON (account_id)\n account_id, started_at, closed_at, status, error_message,\n total_processed, total_objects, complete_count, error_count,\n running_count, pending_count, triggered_by, max_concurrent\n FROM stripe.sync_runs\n ORDER BY account_id, started_at DESC\n `\n } catch (err) {\n // Ignore errors if sync_runs view doesn't exist yet\n console.warn('sync_runs query failed (may not exist yet):', err)\n }\n }\n\n return new Response(\n JSON.stringify({\n package_version: VERSION,\n installation_status: installationStatus,\n sync_status: syncStatus,\n }),\n {\n status: 200,\n headers: {\n 'Content-Type': 'application/json',\n 'Cache-Control': 'no-cache, no-store, must-revalidate',\n },\n }\n )\n } catch (error) {\n console.error('Status query error:', error)\n return new Response(\n JSON.stringify({\n error: error.message,\n package_version: VERSION,\n installation_status: 'not_installed',\n }),\n {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } finally {\n if (sql) await sql.end()\n }\n }\n\n // Handle DELETE requests for uninstall\n if (req.method === 'DELETE') {\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n // Stripe key is required for uninstall to delete webhooks\n const stripeKey = Deno.env.get('STRIPE_SECRET_KEY')\n if (!stripeKey) {\n throw new Error('STRIPE_SECRET_KEY environment variable is required for uninstall')\n }\n\n // Step 1: Delete Stripe webhooks and clean up database\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 },\n stripeSecretKey: stripeKey,\n })\n\n // Delete all managed webhooks\n const webhooks = await stripeSync.listManagedWebhooks()\n for (const webhook of webhooks) {\n try {\n await stripeSync.deleteManagedWebhook(webhook.id)\n console.log(`Deleted webhook: ${webhook.id}`)\n } catch (err) {\n console.warn(`Could not delete webhook ${webhook.id}:`, err)\n }\n }\n\n // Unschedule pg_cron jobs\n try {\n await stripeSync.postgresClient.query(`\n DO $$\n BEGIN\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sync-worker') THEN\n PERFORM cron.unschedule('stripe-sync-worker');\n END IF;\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sigma-worker') THEN\n PERFORM cron.unschedule('stripe-sigma-worker');\n END IF;\n END $$;\n `)\n } catch (err) {\n console.warn('Could not unschedule pg_cron job:', err)\n }\n\n // Delete vault secrets\n try {\n await stripeSync.postgresClient.query(`\n DELETE FROM vault.secrets\n WHERE name IN ('stripe_sync_worker_secret', 'stripe_sigma_worker_secret')\n `)\n } catch (err) {\n console.warn('Could not delete vault secret:', err)\n }\n\n // Drop Sigma self-trigger function if present\n try {\n await stripeSync.postgresClient.query(`\n DROP FUNCTION IF EXISTS stripe.trigger_sigma_worker();\n `)\n } catch (err) {\n console.warn('Could not drop sigma trigger function:', err)\n }\n\n // Terminate connections holding locks on stripe schema\n try {\n await stripeSync.postgresClient.query(`\n SELECT pg_terminate_backend(pid)\n FROM pg_locks l\n JOIN pg_class c ON l.relation = c.oid\n JOIN pg_namespace n ON c.relnamespace = n.oid\n WHERE n.nspname = 'stripe'\n AND l.pid != pg_backend_pid()\n `)\n } catch (err) {\n console.warn('Could not terminate connections:', err)\n }\n\n // Drop schema with retry\n let dropAttempts = 0\n const maxAttempts = 3\n while (dropAttempts < maxAttempts) {\n try {\n await stripeSync.postgresClient.query('DROP SCHEMA IF EXISTS stripe CASCADE')\n break // Success, exit loop\n } catch (err) {\n dropAttempts++\n if (dropAttempts >= maxAttempts) {\n throw new Error(\n `Failed to drop schema after ${maxAttempts} attempts. ` +\n `There may be active connections or locks on the stripe schema. ` +\n `Error: ${err.message}`\n )\n }\n // Wait 1 second before retrying\n await new Promise((resolve) => setTimeout(resolve, 1000))\n }\n }\n\n await stripeSync.postgresClient.pool.end()\n\n // Step 2: Delete Supabase secrets\n try {\n await deleteSecret(projectRef, 'STRIPE_SECRET_KEY', accessToken)\n } catch (err) {\n console.warn('Could not delete STRIPE_SECRET_KEY secret:', err)\n }\n\n // Step 3: Delete Edge Functions\n try {\n await deleteEdgeFunction(projectRef, 'stripe-setup', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-setup function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-webhook', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-webhook function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-worker', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-worker function:', err)\n }\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Uninstall complete',\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Uninstall error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n }\n\n // Handle POST requests for install\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const enableSigma = (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true'\n await runMigrations({ databaseUrl: dbUrl, enableSigma })\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 }, // Need 2 for advisory lock + queries\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY'),\n })\n\n // Release any stale advisory locks from previous timeouts\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n\n // Construct webhook URL from SUPABASE_URL (available in all Edge Functions)\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n throw new Error('SUPABASE_URL environment variable is not set')\n }\n const webhookUrl = supabaseUrl + '/functions/v1/stripe-webhook'\n\n const webhook = await stripeSync.findOrCreateManagedWebhook(webhookUrl)\n\n await stripeSync.postgresClient.pool.end()\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Setup complete',\n webhookId: webhook.id,\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Setup error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n})\n";
|
|
48542
48553
|
|
|
48543
48554
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-webhook.ts
|
|
48544
|
-
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48555
|
+
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n partnerId: 'pp_supabase',\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48545
48556
|
|
|
48546
48557
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-worker.ts
|
|
48547
|
-
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48558
|
+
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n partnerId: 'pp_supabase',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48548
48559
|
|
|
48549
48560
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/sigma-data-worker.ts
|
|
48550
48561
|
var sigma_data_worker_default = "/**\n * Stripe Sigma Data Worker.\n *\n * Hourly cron starts a run; self-trigger continues until all objects finish.\n * Progress persists in _sync_runs and _sync_obj_runs across invocations.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst BATCH_SIZE = 1\nconst MAX_RUN_AGE_MS = 6 * 60 * 60 * 1000\nconst jsonResponse = (body: unknown, status = 200) =>\n new Response(JSON.stringify(body), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7)\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return jsonResponse({ error: 'SUPABASE_DB_URL not set' }, 500)\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql: ReturnType<typeof postgres> | undefined\n let stripeSync: StripeSync | undefined\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return jsonResponse(\n {\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n // Validate the token against vault secret\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sigma_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Sigma worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid sigma worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: true,\n sigmaPageSizeOverride: 1000,\n })\n } catch (error) {\n await sql.end()\n return jsonResponse(\n {\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n const accountId = await stripeSync.getAccountId()\n const sigmaObjects = stripeSync.getSupportedSigmaObjects()\n\n if (sigmaObjects.length === 0) {\n return jsonResponse({ message: 'No Sigma objects configured for sync' })\n }\n\n // Get or create sync run for sigma-worker (isolated from stripe-worker)\n const runResult = await stripeSync.postgresClient.getOrCreateSyncRun(accountId, 'sigma-worker')\n const runStartedAt =\n runResult?.runStartedAt ??\n (await stripeSync.postgresClient.getActiveSyncRun(accountId, 'sigma-worker'))?.runStartedAt\n\n if (!runStartedAt) {\n throw new Error('Failed to get or create sync run for sigma worker')\n }\n\n // Legacy cleanup: remove any prefixed sigma object runs that can block concurrency.\n // Previous versions stored objects as \"sigma.<table>\" which no longer matches processNext.\n await stripeSync.postgresClient.query(\n `UPDATE \"stripe\".\"_sync_obj_runs\"\n SET status = 'error',\n error_message = 'Legacy sigma worker prefix run (sigma.*); superseded by unprefixed runs',\n completed_at = now()\n WHERE \"_account_id\" = $1\n AND run_started_at = $2\n AND object LIKE 'sigma.%'\n AND status IN ('pending', 'running')`,\n [accountId, runStartedAt]\n )\n\n // Stop self-triggering after MAX_RUN_AGE_MS.\n const runAgeMs = Date.now() - runStartedAt.getTime()\n if (runAgeMs > MAX_RUN_AGE_MS) {\n console.warn(\n `Sigma worker: run too old (${Math.round(runAgeMs / 1000 / 60)} min), closing without self-trigger`\n )\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n return jsonResponse({\n message: 'Sigma run exceeded max age, closed without processing',\n runAgeMinutes: Math.round(runAgeMs / 1000 / 60),\n selfTriggered: false,\n })\n }\n\n // Create object runs for all sigma objects (idempotent).\n await stripeSync.postgresClient.createObjectRuns(accountId, runStartedAt, sigmaObjects)\n await stripeSync.postgresClient.ensureSyncRunMaxConcurrent(accountId, runStartedAt, BATCH_SIZE)\n\n // Prefer running objects; otherwise claim pending ones.\n const runningObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n const objectsToProcess = runningObjects.slice(0, BATCH_SIZE)\n let pendingObjects: string[] = []\n\n if (objectsToProcess.length === 0) {\n pendingObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n\n for (const objectKey of pendingObjects) {\n if (objectsToProcess.length >= BATCH_SIZE) break\n const started = await stripeSync.postgresClient.tryStartObjectSync(\n accountId,\n runStartedAt,\n objectKey\n )\n if (started) {\n objectsToProcess.push(objectKey)\n }\n }\n }\n\n if (objectsToProcess.length === 0) {\n if (pendingObjects.length === 0) {\n console.info('Sigma worker: all objects complete or errored - run finished')\n return jsonResponse({ message: 'Sigma sync run complete', selfTriggered: false })\n }\n\n console.info('Sigma worker: at concurrency limit, will self-trigger', {\n pendingCount: pendingObjects.length,\n })\n let selfTriggered = false\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n\n return jsonResponse({\n message: 'At concurrency limit',\n pendingCount: pendingObjects.length,\n selfTriggered,\n })\n }\n\n // Process objects sequentially (one lifecycle per invocation).\n const results: Array<Record<string, unknown>> = []\n\n for (const object of objectsToProcess) {\n const objectKey = object\n try {\n console.info(`Sigma worker: processing ${object}`)\n\n // Process one sigma page and upsert results.\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const result = await stripeSync.processNext(object as any, {\n runStartedAt,\n triggeredBy: 'sigma-worker',\n })\n\n results.push({\n object,\n processed: result.processed,\n hasMore: result.hasMore,\n status: 'success',\n })\n\n if (result.hasMore) {\n console.info(\n `Sigma worker: ${object} has more pages, processed ${result.processed} rows so far`\n )\n } else {\n console.info(`Sigma worker: ${object} complete, processed ${result.processed} rows`)\n }\n } catch (error) {\n console.error(`Sigma worker: error processing ${object}:`, error)\n\n // Mark object as failed and move on (no retries)\n await stripeSync.postgresClient.failObjectSync(\n accountId,\n runStartedAt,\n objectKey,\n error.message ?? 'Unknown error'\n )\n\n results.push({\n object,\n processed: 0,\n hasMore: false,\n status: 'error',\n error: error.message,\n })\n }\n }\n\n // Determine if self-trigger is needed\n const pendingAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n const runningAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n // Calculate remaining run time for logging\n const remainingMs = MAX_RUN_AGE_MS - (Date.now() - runStartedAt.getTime())\n const remainingMinutes = Math.round(remainingMs / 1000 / 60)\n\n // Only self-trigger if there are pending or running objects AND run hasn't timed out\n const shouldSelfTrigger =\n (pendingAfter.length > 0 || runningAfter.length > 0) && remainingMs > 0\n\n let selfTriggered = false\n if (shouldSelfTrigger) {\n console.info('Sigma worker: more work remains, self-triggering', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n remainingMinutes,\n })\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n } else if (pendingAfter.length > 0 || runningAfter.length > 0) {\n // Would self-trigger but run timed out\n console.warn('Sigma worker: work remains but run timed out, closing', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n })\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n } else {\n console.info('Sigma worker: no more work, run complete')\n }\n\n return jsonResponse({\n results,\n selfTriggered,\n remaining: { pending: pendingAfter.length, running: runningAfter.length },\n })\n } catch (error) {\n console.error('Sigma worker error:', error)\n return jsonResponse({ error: error.message, stack: error.stack }, 500)\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
package/dist/cli/index.js
CHANGED
|
@@ -5,12 +5,12 @@ import {
|
|
|
5
5
|
migrateCommand,
|
|
6
6
|
syncCommand,
|
|
7
7
|
uninstallCommand
|
|
8
|
-
} from "../chunk-
|
|
9
|
-
import "../chunk-
|
|
10
|
-
import "../chunk-
|
|
8
|
+
} from "../chunk-6OQC4YJW.js";
|
|
9
|
+
import "../chunk-EZ73ACI4.js";
|
|
10
|
+
import "../chunk-EIBEWIQC.js";
|
|
11
11
|
import {
|
|
12
12
|
package_default
|
|
13
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-IBYBC6LT.js";
|
|
14
14
|
|
|
15
15
|
// src/cli/index.ts
|
|
16
16
|
import { Command } from "commander";
|
package/dist/cli/lib.cjs
CHANGED
|
@@ -117,7 +117,7 @@ async function loadConfig(options) {
|
|
|
117
117
|
// package.json
|
|
118
118
|
var package_default = {
|
|
119
119
|
name: "stripe-experiment-sync",
|
|
120
|
-
version: "1.0.
|
|
120
|
+
version: "1.0.22",
|
|
121
121
|
private: false,
|
|
122
122
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
123
123
|
type: "module",
|
|
@@ -1247,12 +1247,13 @@ function normalizeSigmaTimestampToIso(value) {
|
|
|
1247
1247
|
if (Number.isNaN(d.getTime())) return null;
|
|
1248
1248
|
return d.toISOString();
|
|
1249
1249
|
}
|
|
1250
|
-
function createStripeClient(apiKey) {
|
|
1250
|
+
function createStripeClient(apiKey, partnerId) {
|
|
1251
1251
|
return new import_stripe2.default(apiKey, {
|
|
1252
1252
|
appInfo: {
|
|
1253
1253
|
name: "Stripe Sync Engine",
|
|
1254
1254
|
version: package_default.version,
|
|
1255
|
-
url: package_default.homepage
|
|
1255
|
+
url: package_default.homepage,
|
|
1256
|
+
...partnerId ? { partner_id: partnerId } : {}
|
|
1256
1257
|
}
|
|
1257
1258
|
});
|
|
1258
1259
|
}
|
|
@@ -1271,14 +1272,14 @@ async function fetchStripeText(url, apiKey, options) {
|
|
|
1271
1272
|
return text;
|
|
1272
1273
|
}
|
|
1273
1274
|
async function createSigmaQueryRun(params) {
|
|
1274
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1275
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1275
1276
|
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1276
1277
|
sql: params.sql
|
|
1277
1278
|
});
|
|
1278
1279
|
return { queryRunId: created.id };
|
|
1279
1280
|
}
|
|
1280
1281
|
async function getSigmaQueryRun(params) {
|
|
1281
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1282
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1282
1283
|
const current = await stripe.rawRequest(
|
|
1283
1284
|
"GET",
|
|
1284
1285
|
`/v1/sigma/query_runs/${params.queryRunId}`,
|
|
@@ -1300,7 +1301,11 @@ async function downloadSigmaFile(params) {
|
|
|
1300
1301
|
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1301
1302
|
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1302
1303
|
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1303
|
-
const { queryRunId } = await createSigmaQueryRun({
|
|
1304
|
+
const { queryRunId } = await createSigmaQueryRun({
|
|
1305
|
+
apiKey: params.apiKey,
|
|
1306
|
+
sql: params.sql,
|
|
1307
|
+
partnerId: params.partnerId
|
|
1308
|
+
});
|
|
1304
1309
|
const start = Date.now();
|
|
1305
1310
|
let current = {
|
|
1306
1311
|
id: queryRunId,
|
|
@@ -1313,7 +1318,11 @@ async function runSigmaQueryAndDownloadCsv(params) {
|
|
|
1313
1318
|
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1314
1319
|
}
|
|
1315
1320
|
await sleep2(pollIntervalMs);
|
|
1316
|
-
const next = await getSigmaQueryRun({
|
|
1321
|
+
const next = await getSigmaQueryRun({
|
|
1322
|
+
apiKey: params.apiKey,
|
|
1323
|
+
queryRunId,
|
|
1324
|
+
partnerId: params.partnerId
|
|
1325
|
+
});
|
|
1317
1326
|
current = {
|
|
1318
1327
|
id: queryRunId,
|
|
1319
1328
|
status: next.status,
|
|
@@ -45331,7 +45340,8 @@ var StripeSync = class {
|
|
|
45331
45340
|
appInfo: {
|
|
45332
45341
|
name: "Stripe Sync Engine",
|
|
45333
45342
|
version: package_default.version,
|
|
45334
|
-
url: package_default.homepage
|
|
45343
|
+
url: package_default.homepage,
|
|
45344
|
+
...config.partnerId ? { partner_id: config.partnerId } : {}
|
|
45335
45345
|
}
|
|
45336
45346
|
});
|
|
45337
45347
|
this.stripe = createRetryableStripeClient(baseStripe, {}, config.logger);
|
|
@@ -46387,7 +46397,8 @@ ${message}`;
|
|
|
46387
46397
|
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
46388
46398
|
apiKey: this.config.stripeSecretKey,
|
|
46389
46399
|
sql: sigmaSql,
|
|
46390
|
-
logger: this.config.logger
|
|
46400
|
+
logger: this.config.logger,
|
|
46401
|
+
partnerId: this.config.partnerId
|
|
46391
46402
|
});
|
|
46392
46403
|
const rows = parseCsvObjects(csv);
|
|
46393
46404
|
if (rows.length === 0) {
|
|
@@ -48555,10 +48566,10 @@ var import_supabase_management_js = require("supabase-management-js");
|
|
|
48555
48566
|
var stripe_setup_default = "import { StripeSync, runMigrations, VERSION } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\n// Get management API base URL from environment variable (for testing against localhost/staging)\n// Caller should provide full URL with protocol (e.g., http://localhost:54323 or https://api.supabase.com)\nconst MGMT_API_BASE_RAW = Deno.env.get('MANAGEMENT_API_URL') || 'https://api.supabase.com'\nconst MGMT_API_BASE = MGMT_API_BASE_RAW.match(/^https?:\\/\\//)\n ? MGMT_API_BASE_RAW\n : `https://${MGMT_API_BASE_RAW}`\n\n// Helper to validate accessToken against Management API\nasync function validateAccessToken(projectRef: string, accessToken: string): Promise<boolean> {\n // Try to fetch project details using the access token\n // This validates that the token is valid for the management API\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}`\n const response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n // If we can successfully get the project, the token is valid\n return response.ok\n}\n\n// Helper to delete edge function via Management API\nasync function deleteEdgeFunction(\n projectRef: string,\n functionSlug: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/functions/${functionSlug}`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n throw new Error(`Failed to delete function ${functionSlug}: ${response.status} ${text}`)\n }\n}\n\n// Helper to delete secrets via Management API\nasync function deleteSecret(\n projectRef: string,\n secretName: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/secrets`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify([secretName]),\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n console.warn(`Failed to delete secret ${secretName}: ${response.status} ${text}`)\n }\n}\n\nDeno.serve(async (req) => {\n // Extract project ref from SUPABASE_URL (format: https://{projectRef}.{base})\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n const projectRef = new URL(supabaseUrl).hostname.split('.')[0]\n\n // Validate access token for all requests\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const accessToken = authHeader.substring(7) // Remove 'Bearer '\n const isValid = await validateAccessToken(projectRef, accessToken)\n if (!isValid) {\n return new Response('Forbidden: Invalid access token for this project', { status: 403 })\n }\n\n // Handle GET requests for status\n if (req.method === 'GET') {\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n let sql\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n\n // Query installation status from schema comment\n const commentResult = await sql`\n SELECT obj_description(oid, 'pg_namespace') as comment\n FROM pg_namespace\n WHERE nspname = 'stripe'\n `\n\n const comment = commentResult[0]?.comment || null\n let installationStatus = 'not_installed'\n\n if (comment && comment.includes('stripe-sync')) {\n // Parse installation status from comment\n if (comment.includes('installation:started')) {\n installationStatus = 'installing'\n } else if (comment.includes('installation:error')) {\n installationStatus = 'error'\n } else if (comment.includes('installed')) {\n installationStatus = 'installed'\n }\n }\n\n // Query sync runs (only if schema exists)\n let syncStatus = []\n if (comment) {\n try {\n syncStatus = await sql`\n SELECT DISTINCT ON (account_id)\n account_id, started_at, closed_at, status, error_message,\n total_processed, total_objects, complete_count, error_count,\n running_count, pending_count, triggered_by, max_concurrent\n FROM stripe.sync_runs\n ORDER BY account_id, started_at DESC\n `\n } catch (err) {\n // Ignore errors if sync_runs view doesn't exist yet\n console.warn('sync_runs query failed (may not exist yet):', err)\n }\n }\n\n return new Response(\n JSON.stringify({\n package_version: VERSION,\n installation_status: installationStatus,\n sync_status: syncStatus,\n }),\n {\n status: 200,\n headers: {\n 'Content-Type': 'application/json',\n 'Cache-Control': 'no-cache, no-store, must-revalidate',\n },\n }\n )\n } catch (error) {\n console.error('Status query error:', error)\n return new Response(\n JSON.stringify({\n error: error.message,\n package_version: VERSION,\n installation_status: 'not_installed',\n }),\n {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } finally {\n if (sql) await sql.end()\n }\n }\n\n // Handle DELETE requests for uninstall\n if (req.method === 'DELETE') {\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n // Stripe key is required for uninstall to delete webhooks\n const stripeKey = Deno.env.get('STRIPE_SECRET_KEY')\n if (!stripeKey) {\n throw new Error('STRIPE_SECRET_KEY environment variable is required for uninstall')\n }\n\n // Step 1: Delete Stripe webhooks and clean up database\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 },\n stripeSecretKey: stripeKey,\n })\n\n // Delete all managed webhooks\n const webhooks = await stripeSync.listManagedWebhooks()\n for (const webhook of webhooks) {\n try {\n await stripeSync.deleteManagedWebhook(webhook.id)\n console.log(`Deleted webhook: ${webhook.id}`)\n } catch (err) {\n console.warn(`Could not delete webhook ${webhook.id}:`, err)\n }\n }\n\n // Unschedule pg_cron jobs\n try {\n await stripeSync.postgresClient.query(`\n DO $$\n BEGIN\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sync-worker') THEN\n PERFORM cron.unschedule('stripe-sync-worker');\n END IF;\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sigma-worker') THEN\n PERFORM cron.unschedule('stripe-sigma-worker');\n END IF;\n END $$;\n `)\n } catch (err) {\n console.warn('Could not unschedule pg_cron job:', err)\n }\n\n // Delete vault secrets\n try {\n await stripeSync.postgresClient.query(`\n DELETE FROM vault.secrets\n WHERE name IN ('stripe_sync_worker_secret', 'stripe_sigma_worker_secret')\n `)\n } catch (err) {\n console.warn('Could not delete vault secret:', err)\n }\n\n // Drop Sigma self-trigger function if present\n try {\n await stripeSync.postgresClient.query(`\n DROP FUNCTION IF EXISTS stripe.trigger_sigma_worker();\n `)\n } catch (err) {\n console.warn('Could not drop sigma trigger function:', err)\n }\n\n // Terminate connections holding locks on stripe schema\n try {\n await stripeSync.postgresClient.query(`\n SELECT pg_terminate_backend(pid)\n FROM pg_locks l\n JOIN pg_class c ON l.relation = c.oid\n JOIN pg_namespace n ON c.relnamespace = n.oid\n WHERE n.nspname = 'stripe'\n AND l.pid != pg_backend_pid()\n `)\n } catch (err) {\n console.warn('Could not terminate connections:', err)\n }\n\n // Drop schema with retry\n let dropAttempts = 0\n const maxAttempts = 3\n while (dropAttempts < maxAttempts) {\n try {\n await stripeSync.postgresClient.query('DROP SCHEMA IF EXISTS stripe CASCADE')\n break // Success, exit loop\n } catch (err) {\n dropAttempts++\n if (dropAttempts >= maxAttempts) {\n throw new Error(\n `Failed to drop schema after ${maxAttempts} attempts. ` +\n `There may be active connections or locks on the stripe schema. ` +\n `Error: ${err.message}`\n )\n }\n // Wait 1 second before retrying\n await new Promise((resolve) => setTimeout(resolve, 1000))\n }\n }\n\n await stripeSync.postgresClient.pool.end()\n\n // Step 2: Delete Supabase secrets\n try {\n await deleteSecret(projectRef, 'STRIPE_SECRET_KEY', accessToken)\n } catch (err) {\n console.warn('Could not delete STRIPE_SECRET_KEY secret:', err)\n }\n\n // Step 3: Delete Edge Functions\n try {\n await deleteEdgeFunction(projectRef, 'stripe-setup', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-setup function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-webhook', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-webhook function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-worker', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-worker function:', err)\n }\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Uninstall complete',\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Uninstall error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n }\n\n // Handle POST requests for install\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const enableSigma = (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true'\n await runMigrations({ databaseUrl: dbUrl, enableSigma })\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 }, // Need 2 for advisory lock + queries\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY'),\n })\n\n // Release any stale advisory locks from previous timeouts\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n\n // Construct webhook URL from SUPABASE_URL (available in all Edge Functions)\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n throw new Error('SUPABASE_URL environment variable is not set')\n }\n const webhookUrl = supabaseUrl + '/functions/v1/stripe-webhook'\n\n const webhook = await stripeSync.findOrCreateManagedWebhook(webhookUrl)\n\n await stripeSync.postgresClient.pool.end()\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Setup complete',\n webhookId: webhook.id,\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Setup error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n})\n";
|
|
48556
48567
|
|
|
48557
48568
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-webhook.ts
|
|
48558
|
-
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48569
|
+
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n partnerId: 'pp_supabase',\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48559
48570
|
|
|
48560
48571
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-worker.ts
|
|
48561
|
-
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48572
|
+
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n partnerId: 'pp_supabase',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48562
48573
|
|
|
48563
48574
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/sigma-data-worker.ts
|
|
48564
48575
|
var sigma_data_worker_default = "/**\n * Stripe Sigma Data Worker.\n *\n * Hourly cron starts a run; self-trigger continues until all objects finish.\n * Progress persists in _sync_runs and _sync_obj_runs across invocations.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst BATCH_SIZE = 1\nconst MAX_RUN_AGE_MS = 6 * 60 * 60 * 1000\nconst jsonResponse = (body: unknown, status = 200) =>\n new Response(JSON.stringify(body), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7)\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return jsonResponse({ error: 'SUPABASE_DB_URL not set' }, 500)\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql: ReturnType<typeof postgres> | undefined\n let stripeSync: StripeSync | undefined\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return jsonResponse(\n {\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n // Validate the token against vault secret\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sigma_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Sigma worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid sigma worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: true,\n sigmaPageSizeOverride: 1000,\n })\n } catch (error) {\n await sql.end()\n return jsonResponse(\n {\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n const accountId = await stripeSync.getAccountId()\n const sigmaObjects = stripeSync.getSupportedSigmaObjects()\n\n if (sigmaObjects.length === 0) {\n return jsonResponse({ message: 'No Sigma objects configured for sync' })\n }\n\n // Get or create sync run for sigma-worker (isolated from stripe-worker)\n const runResult = await stripeSync.postgresClient.getOrCreateSyncRun(accountId, 'sigma-worker')\n const runStartedAt =\n runResult?.runStartedAt ??\n (await stripeSync.postgresClient.getActiveSyncRun(accountId, 'sigma-worker'))?.runStartedAt\n\n if (!runStartedAt) {\n throw new Error('Failed to get or create sync run for sigma worker')\n }\n\n // Legacy cleanup: remove any prefixed sigma object runs that can block concurrency.\n // Previous versions stored objects as \"sigma.<table>\" which no longer matches processNext.\n await stripeSync.postgresClient.query(\n `UPDATE \"stripe\".\"_sync_obj_runs\"\n SET status = 'error',\n error_message = 'Legacy sigma worker prefix run (sigma.*); superseded by unprefixed runs',\n completed_at = now()\n WHERE \"_account_id\" = $1\n AND run_started_at = $2\n AND object LIKE 'sigma.%'\n AND status IN ('pending', 'running')`,\n [accountId, runStartedAt]\n )\n\n // Stop self-triggering after MAX_RUN_AGE_MS.\n const runAgeMs = Date.now() - runStartedAt.getTime()\n if (runAgeMs > MAX_RUN_AGE_MS) {\n console.warn(\n `Sigma worker: run too old (${Math.round(runAgeMs / 1000 / 60)} min), closing without self-trigger`\n )\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n return jsonResponse({\n message: 'Sigma run exceeded max age, closed without processing',\n runAgeMinutes: Math.round(runAgeMs / 1000 / 60),\n selfTriggered: false,\n })\n }\n\n // Create object runs for all sigma objects (idempotent).\n await stripeSync.postgresClient.createObjectRuns(accountId, runStartedAt, sigmaObjects)\n await stripeSync.postgresClient.ensureSyncRunMaxConcurrent(accountId, runStartedAt, BATCH_SIZE)\n\n // Prefer running objects; otherwise claim pending ones.\n const runningObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n const objectsToProcess = runningObjects.slice(0, BATCH_SIZE)\n let pendingObjects: string[] = []\n\n if (objectsToProcess.length === 0) {\n pendingObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n\n for (const objectKey of pendingObjects) {\n if (objectsToProcess.length >= BATCH_SIZE) break\n const started = await stripeSync.postgresClient.tryStartObjectSync(\n accountId,\n runStartedAt,\n objectKey\n )\n if (started) {\n objectsToProcess.push(objectKey)\n }\n }\n }\n\n if (objectsToProcess.length === 0) {\n if (pendingObjects.length === 0) {\n console.info('Sigma worker: all objects complete or errored - run finished')\n return jsonResponse({ message: 'Sigma sync run complete', selfTriggered: false })\n }\n\n console.info('Sigma worker: at concurrency limit, will self-trigger', {\n pendingCount: pendingObjects.length,\n })\n let selfTriggered = false\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n\n return jsonResponse({\n message: 'At concurrency limit',\n pendingCount: pendingObjects.length,\n selfTriggered,\n })\n }\n\n // Process objects sequentially (one lifecycle per invocation).\n const results: Array<Record<string, unknown>> = []\n\n for (const object of objectsToProcess) {\n const objectKey = object\n try {\n console.info(`Sigma worker: processing ${object}`)\n\n // Process one sigma page and upsert results.\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const result = await stripeSync.processNext(object as any, {\n runStartedAt,\n triggeredBy: 'sigma-worker',\n })\n\n results.push({\n object,\n processed: result.processed,\n hasMore: result.hasMore,\n status: 'success',\n })\n\n if (result.hasMore) {\n console.info(\n `Sigma worker: ${object} has more pages, processed ${result.processed} rows so far`\n )\n } else {\n console.info(`Sigma worker: ${object} complete, processed ${result.processed} rows`)\n }\n } catch (error) {\n console.error(`Sigma worker: error processing ${object}:`, error)\n\n // Mark object as failed and move on (no retries)\n await stripeSync.postgresClient.failObjectSync(\n accountId,\n runStartedAt,\n objectKey,\n error.message ?? 'Unknown error'\n )\n\n results.push({\n object,\n processed: 0,\n hasMore: false,\n status: 'error',\n error: error.message,\n })\n }\n }\n\n // Determine if self-trigger is needed\n const pendingAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n const runningAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n // Calculate remaining run time for logging\n const remainingMs = MAX_RUN_AGE_MS - (Date.now() - runStartedAt.getTime())\n const remainingMinutes = Math.round(remainingMs / 1000 / 60)\n\n // Only self-trigger if there are pending or running objects AND run hasn't timed out\n const shouldSelfTrigger =\n (pendingAfter.length > 0 || runningAfter.length > 0) && remainingMs > 0\n\n let selfTriggered = false\n if (shouldSelfTrigger) {\n console.info('Sigma worker: more work remains, self-triggering', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n remainingMinutes,\n })\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n } else if (pendingAfter.length > 0 || runningAfter.length > 0) {\n // Would self-trigger but run timed out\n console.warn('Sigma worker: work remains but run timed out, closing', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n })\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n } else {\n console.info('Sigma worker: no more work, run complete')\n }\n\n return jsonResponse({\n results,\n selfTriggered,\n remaining: { pending: pendingAfter.length, running: runningAfter.length },\n })\n } catch (error) {\n console.error('Sigma worker error:', error)\n return jsonResponse({ error: error.message, stack: error.stack }, 500)\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
package/dist/cli/lib.js
CHANGED
|
@@ -6,10 +6,10 @@ import {
|
|
|
6
6
|
migrateCommand,
|
|
7
7
|
syncCommand,
|
|
8
8
|
uninstallCommand
|
|
9
|
-
} from "../chunk-
|
|
10
|
-
import "../chunk-
|
|
11
|
-
import "../chunk-
|
|
12
|
-
import "../chunk-
|
|
9
|
+
} from "../chunk-6OQC4YJW.js";
|
|
10
|
+
import "../chunk-EZ73ACI4.js";
|
|
11
|
+
import "../chunk-EIBEWIQC.js";
|
|
12
|
+
import "../chunk-IBYBC6LT.js";
|
|
13
13
|
export {
|
|
14
14
|
backfillCommand,
|
|
15
15
|
createTunnel,
|
package/dist/index.cjs
CHANGED
|
@@ -46,7 +46,7 @@ var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
|
46
46
|
// package.json
|
|
47
47
|
var package_default = {
|
|
48
48
|
name: "stripe-experiment-sync",
|
|
49
|
-
version: "1.0.
|
|
49
|
+
version: "1.0.22",
|
|
50
50
|
private: false,
|
|
51
51
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
52
52
|
type: "module",
|
|
@@ -1176,12 +1176,13 @@ function normalizeSigmaTimestampToIso(value) {
|
|
|
1176
1176
|
if (Number.isNaN(d.getTime())) return null;
|
|
1177
1177
|
return d.toISOString();
|
|
1178
1178
|
}
|
|
1179
|
-
function createStripeClient(apiKey) {
|
|
1179
|
+
function createStripeClient(apiKey, partnerId) {
|
|
1180
1180
|
return new import_stripe2.default(apiKey, {
|
|
1181
1181
|
appInfo: {
|
|
1182
1182
|
name: "Stripe Sync Engine",
|
|
1183
1183
|
version: package_default.version,
|
|
1184
|
-
url: package_default.homepage
|
|
1184
|
+
url: package_default.homepage,
|
|
1185
|
+
...partnerId ? { partner_id: partnerId } : {}
|
|
1185
1186
|
}
|
|
1186
1187
|
});
|
|
1187
1188
|
}
|
|
@@ -1200,14 +1201,14 @@ async function fetchStripeText(url, apiKey, options) {
|
|
|
1200
1201
|
return text;
|
|
1201
1202
|
}
|
|
1202
1203
|
async function createSigmaQueryRun(params) {
|
|
1203
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1204
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1204
1205
|
const created = await stripe.rawRequest("POST", "/v1/sigma/query_runs", {
|
|
1205
1206
|
sql: params.sql
|
|
1206
1207
|
});
|
|
1207
1208
|
return { queryRunId: created.id };
|
|
1208
1209
|
}
|
|
1209
1210
|
async function getSigmaQueryRun(params) {
|
|
1210
|
-
const stripe = createStripeClient(params.apiKey);
|
|
1211
|
+
const stripe = createStripeClient(params.apiKey, params.partnerId);
|
|
1211
1212
|
const current = await stripe.rawRequest(
|
|
1212
1213
|
"GET",
|
|
1213
1214
|
`/v1/sigma/query_runs/${params.queryRunId}`,
|
|
@@ -1229,7 +1230,11 @@ async function downloadSigmaFile(params) {
|
|
|
1229
1230
|
async function runSigmaQueryAndDownloadCsv(params) {
|
|
1230
1231
|
const pollTimeoutMs = params.pollTimeoutMs ?? 5 * 60 * 1e3;
|
|
1231
1232
|
const pollIntervalMs = params.pollIntervalMs ?? 2e3;
|
|
1232
|
-
const { queryRunId } = await createSigmaQueryRun({
|
|
1233
|
+
const { queryRunId } = await createSigmaQueryRun({
|
|
1234
|
+
apiKey: params.apiKey,
|
|
1235
|
+
sql: params.sql,
|
|
1236
|
+
partnerId: params.partnerId
|
|
1237
|
+
});
|
|
1233
1238
|
const start = Date.now();
|
|
1234
1239
|
let current = {
|
|
1235
1240
|
id: queryRunId,
|
|
@@ -1242,7 +1247,11 @@ async function runSigmaQueryAndDownloadCsv(params) {
|
|
|
1242
1247
|
throw new Error(`Sigma query run timed out after ${pollTimeoutMs}ms: ${queryRunId}`);
|
|
1243
1248
|
}
|
|
1244
1249
|
await sleep2(pollIntervalMs);
|
|
1245
|
-
const next = await getSigmaQueryRun({
|
|
1250
|
+
const next = await getSigmaQueryRun({
|
|
1251
|
+
apiKey: params.apiKey,
|
|
1252
|
+
queryRunId,
|
|
1253
|
+
partnerId: params.partnerId
|
|
1254
|
+
});
|
|
1246
1255
|
current = {
|
|
1247
1256
|
id: queryRunId,
|
|
1248
1257
|
status: next.status,
|
|
@@ -45260,7 +45269,8 @@ var StripeSync = class {
|
|
|
45260
45269
|
appInfo: {
|
|
45261
45270
|
name: "Stripe Sync Engine",
|
|
45262
45271
|
version: package_default.version,
|
|
45263
|
-
url: package_default.homepage
|
|
45272
|
+
url: package_default.homepage,
|
|
45273
|
+
...config.partnerId ? { partner_id: config.partnerId } : {}
|
|
45264
45274
|
}
|
|
45265
45275
|
});
|
|
45266
45276
|
this.stripe = createRetryableStripeClient(baseStripe, {}, config.logger);
|
|
@@ -46316,7 +46326,8 @@ ${message}`;
|
|
|
46316
46326
|
const { queryRunId, fileId, csv } = await runSigmaQueryAndDownloadCsv({
|
|
46317
46327
|
apiKey: this.config.stripeSecretKey,
|
|
46318
46328
|
sql: sigmaSql,
|
|
46319
|
-
logger: this.config.logger
|
|
46329
|
+
logger: this.config.logger,
|
|
46330
|
+
partnerId: this.config.partnerId
|
|
46320
46331
|
});
|
|
46321
46332
|
const rows = parseCsvObjects(csv);
|
|
46322
46333
|
if (rows.length === 0) {
|
package/dist/index.d.cts
CHANGED
|
@@ -325,6 +325,8 @@ type StripeSyncConfig = {
|
|
|
325
325
|
sigmaSchemaName?: string;
|
|
326
326
|
/** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
|
|
327
327
|
stripeAccountId?: string;
|
|
328
|
+
/** Optional Stripe partner ID embedded in appInfo for telemetry (e.g. "pp_supabase"). */
|
|
329
|
+
partnerId?: string;
|
|
328
330
|
/** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
|
|
329
331
|
stripeWebhookSecret?: string;
|
|
330
332
|
/** Stripe API version for the webhooks, defaults to 2020-08-27 */
|
package/dist/index.d.ts
CHANGED
|
@@ -325,6 +325,8 @@ type StripeSyncConfig = {
|
|
|
325
325
|
sigmaSchemaName?: string;
|
|
326
326
|
/** Stripe account ID. If not provided, will be retrieved from Stripe API. Used as fallback option. */
|
|
327
327
|
stripeAccountId?: string;
|
|
328
|
+
/** Optional Stripe partner ID embedded in appInfo for telemetry (e.g. "pp_supabase"). */
|
|
329
|
+
partnerId?: string;
|
|
328
330
|
/** Stripe webhook signing secret for validating webhook signatures. Required if not using managed webhooks. */
|
|
329
331
|
stripeWebhookSecret?: string;
|
|
330
332
|
/** Stripe API version for the webhooks, defaults to 2020-08-27 */
|
package/dist/index.js
CHANGED
package/dist/supabase/index.cjs
CHANGED
|
@@ -42,10 +42,10 @@ var import_supabase_management_js = require("supabase-management-js");
|
|
|
42
42
|
var stripe_setup_default = "import { StripeSync, runMigrations, VERSION } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\n// Get management API base URL from environment variable (for testing against localhost/staging)\n// Caller should provide full URL with protocol (e.g., http://localhost:54323 or https://api.supabase.com)\nconst MGMT_API_BASE_RAW = Deno.env.get('MANAGEMENT_API_URL') || 'https://api.supabase.com'\nconst MGMT_API_BASE = MGMT_API_BASE_RAW.match(/^https?:\\/\\//)\n ? MGMT_API_BASE_RAW\n : `https://${MGMT_API_BASE_RAW}`\n\n// Helper to validate accessToken against Management API\nasync function validateAccessToken(projectRef: string, accessToken: string): Promise<boolean> {\n // Try to fetch project details using the access token\n // This validates that the token is valid for the management API\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}`\n const response = await fetch(url, {\n method: 'GET',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n // If we can successfully get the project, the token is valid\n return response.ok\n}\n\n// Helper to delete edge function via Management API\nasync function deleteEdgeFunction(\n projectRef: string,\n functionSlug: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/functions/${functionSlug}`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n throw new Error(`Failed to delete function ${functionSlug}: ${response.status} ${text}`)\n }\n}\n\n// Helper to delete secrets via Management API\nasync function deleteSecret(\n projectRef: string,\n secretName: string,\n accessToken: string\n): Promise<void> {\n const url = `${MGMT_API_BASE}/v1/projects/${projectRef}/secrets`\n const response = await fetch(url, {\n method: 'DELETE',\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify([secretName]),\n })\n\n if (!response.ok && response.status !== 404) {\n const text = await response.text()\n console.warn(`Failed to delete secret ${secretName}: ${response.status} ${text}`)\n }\n}\n\nDeno.serve(async (req) => {\n // Extract project ref from SUPABASE_URL (format: https://{projectRef}.{base})\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n const projectRef = new URL(supabaseUrl).hostname.split('.')[0]\n\n // Validate access token for all requests\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const accessToken = authHeader.substring(7) // Remove 'Bearer '\n const isValid = await validateAccessToken(projectRef, accessToken)\n if (!isValid) {\n return new Response('Forbidden: Invalid access token for this project', { status: 403 })\n }\n\n // Handle GET requests for status\n if (req.method === 'GET') {\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n let sql\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n\n // Query installation status from schema comment\n const commentResult = await sql`\n SELECT obj_description(oid, 'pg_namespace') as comment\n FROM pg_namespace\n WHERE nspname = 'stripe'\n `\n\n const comment = commentResult[0]?.comment || null\n let installationStatus = 'not_installed'\n\n if (comment && comment.includes('stripe-sync')) {\n // Parse installation status from comment\n if (comment.includes('installation:started')) {\n installationStatus = 'installing'\n } else if (comment.includes('installation:error')) {\n installationStatus = 'error'\n } else if (comment.includes('installed')) {\n installationStatus = 'installed'\n }\n }\n\n // Query sync runs (only if schema exists)\n let syncStatus = []\n if (comment) {\n try {\n syncStatus = await sql`\n SELECT DISTINCT ON (account_id)\n account_id, started_at, closed_at, status, error_message,\n total_processed, total_objects, complete_count, error_count,\n running_count, pending_count, triggered_by, max_concurrent\n FROM stripe.sync_runs\n ORDER BY account_id, started_at DESC\n `\n } catch (err) {\n // Ignore errors if sync_runs view doesn't exist yet\n console.warn('sync_runs query failed (may not exist yet):', err)\n }\n }\n\n return new Response(\n JSON.stringify({\n package_version: VERSION,\n installation_status: installationStatus,\n sync_status: syncStatus,\n }),\n {\n status: 200,\n headers: {\n 'Content-Type': 'application/json',\n 'Cache-Control': 'no-cache, no-store, must-revalidate',\n },\n }\n )\n } catch (error) {\n console.error('Status query error:', error)\n return new Response(\n JSON.stringify({\n error: error.message,\n package_version: VERSION,\n installation_status: 'not_installed',\n }),\n {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } finally {\n if (sql) await sql.end()\n }\n }\n\n // Handle DELETE requests for uninstall\n if (req.method === 'DELETE') {\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n // Stripe key is required for uninstall to delete webhooks\n const stripeKey = Deno.env.get('STRIPE_SECRET_KEY')\n if (!stripeKey) {\n throw new Error('STRIPE_SECRET_KEY environment variable is required for uninstall')\n }\n\n // Step 1: Delete Stripe webhooks and clean up database\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 },\n stripeSecretKey: stripeKey,\n })\n\n // Delete all managed webhooks\n const webhooks = await stripeSync.listManagedWebhooks()\n for (const webhook of webhooks) {\n try {\n await stripeSync.deleteManagedWebhook(webhook.id)\n console.log(`Deleted webhook: ${webhook.id}`)\n } catch (err) {\n console.warn(`Could not delete webhook ${webhook.id}:`, err)\n }\n }\n\n // Unschedule pg_cron jobs\n try {\n await stripeSync.postgresClient.query(`\n DO $$\n BEGIN\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sync-worker') THEN\n PERFORM cron.unschedule('stripe-sync-worker');\n END IF;\n IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stripe-sigma-worker') THEN\n PERFORM cron.unschedule('stripe-sigma-worker');\n END IF;\n END $$;\n `)\n } catch (err) {\n console.warn('Could not unschedule pg_cron job:', err)\n }\n\n // Delete vault secrets\n try {\n await stripeSync.postgresClient.query(`\n DELETE FROM vault.secrets\n WHERE name IN ('stripe_sync_worker_secret', 'stripe_sigma_worker_secret')\n `)\n } catch (err) {\n console.warn('Could not delete vault secret:', err)\n }\n\n // Drop Sigma self-trigger function if present\n try {\n await stripeSync.postgresClient.query(`\n DROP FUNCTION IF EXISTS stripe.trigger_sigma_worker();\n `)\n } catch (err) {\n console.warn('Could not drop sigma trigger function:', err)\n }\n\n // Terminate connections holding locks on stripe schema\n try {\n await stripeSync.postgresClient.query(`\n SELECT pg_terminate_backend(pid)\n FROM pg_locks l\n JOIN pg_class c ON l.relation = c.oid\n JOIN pg_namespace n ON c.relnamespace = n.oid\n WHERE n.nspname = 'stripe'\n AND l.pid != pg_backend_pid()\n `)\n } catch (err) {\n console.warn('Could not terminate connections:', err)\n }\n\n // Drop schema with retry\n let dropAttempts = 0\n const maxAttempts = 3\n while (dropAttempts < maxAttempts) {\n try {\n await stripeSync.postgresClient.query('DROP SCHEMA IF EXISTS stripe CASCADE')\n break // Success, exit loop\n } catch (err) {\n dropAttempts++\n if (dropAttempts >= maxAttempts) {\n throw new Error(\n `Failed to drop schema after ${maxAttempts} attempts. ` +\n `There may be active connections or locks on the stripe schema. ` +\n `Error: ${err.message}`\n )\n }\n // Wait 1 second before retrying\n await new Promise((resolve) => setTimeout(resolve, 1000))\n }\n }\n\n await stripeSync.postgresClient.pool.end()\n\n // Step 2: Delete Supabase secrets\n try {\n await deleteSecret(projectRef, 'STRIPE_SECRET_KEY', accessToken)\n } catch (err) {\n console.warn('Could not delete STRIPE_SECRET_KEY secret:', err)\n }\n\n // Step 3: Delete Edge Functions\n try {\n await deleteEdgeFunction(projectRef, 'stripe-setup', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-setup function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-webhook', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-webhook function:', err)\n }\n\n try {\n await deleteEdgeFunction(projectRef, 'stripe-worker', accessToken)\n } catch (err) {\n console.warn('Could not delete stripe-worker function:', err)\n }\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Uninstall complete',\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Uninstall error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n }\n\n // Handle POST requests for install\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n let stripeSync = null\n try {\n // Get and validate database URL\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n throw new Error('SUPABASE_DB_URL environment variable is not set')\n }\n // Remove sslmode from connection string (not supported by pg in Deno)\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const enableSigma = (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true'\n await runMigrations({ databaseUrl: dbUrl, enableSigma })\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 2 }, // Need 2 for advisory lock + queries\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY'),\n })\n\n // Release any stale advisory locks from previous timeouts\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n\n // Construct webhook URL from SUPABASE_URL (available in all Edge Functions)\n const supabaseUrl = Deno.env.get('SUPABASE_URL')\n if (!supabaseUrl) {\n throw new Error('SUPABASE_URL environment variable is not set')\n }\n const webhookUrl = supabaseUrl + '/functions/v1/stripe-webhook'\n\n const webhook = await stripeSync.findOrCreateManagedWebhook(webhookUrl)\n\n await stripeSync.postgresClient.pool.end()\n\n return new Response(\n JSON.stringify({\n success: true,\n message: 'Setup complete',\n webhookId: webhook.id,\n }),\n {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n }\n )\n } catch (error) {\n console.error('Setup error:', error)\n // Cleanup on error\n if (stripeSync) {\n try {\n await stripeSync.postgresClient.query('SELECT pg_advisory_unlock_all()')\n await stripeSync.postgresClient.pool.end()\n } catch (cleanupErr) {\n console.warn('Cleanup failed:', cleanupErr)\n }\n }\n return new Response(JSON.stringify({ success: false, error: error.message }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n})\n";
|
|
43
43
|
|
|
44
44
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-webhook.ts
|
|
45
|
-
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
45
|
+
var stripe_webhook_default = "import { StripeSync } from 'npm:stripe-experiment-sync'\n\nDeno.serve(async (req) => {\n if (req.method !== 'POST') {\n return new Response('Method not allowed', { status: 405 })\n }\n\n const sig = req.headers.get('stripe-signature')\n if (!sig) {\n return new Response('Missing stripe-signature header', { status: 400 })\n }\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n const stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n partnerId: 'pp_supabase',\n })\n\n try {\n const rawBody = new Uint8Array(await req.arrayBuffer())\n await stripeSync.processWebhook(rawBody, sig)\n return new Response(JSON.stringify({ received: true }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Webhook processing error:', error)\n const isSignatureError =\n error.message?.includes('signature') || error.type === 'StripeSignatureVerificationError'\n const status = isSignatureError ? 400 : 500\n return new Response(JSON.stringify({ error: error.message }), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
46
46
|
|
|
47
47
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/stripe-worker.ts
|
|
48
|
-
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
48
|
+
var stripe_worker_default = "/**\n * Stripe Sync Worker\n *\n * Triggered by pg_cron at a configurable interval (default: 60 seconds). Uses pgmq for durable work queue.\n *\n * Flow:\n * 1. Read batch of messages from pgmq (qty=10, vt=60s)\n * 2. If queue empty: enqueue all objects (continuous sync)\n * 3. Process messages in parallel (Promise.all):\n * - processNext(object)\n * - Delete message on success\n * - Re-enqueue if hasMore\n * 4. Return results summary\n *\n * Concurrency:\n * - Multiple workers can run concurrently via overlapping pg_cron triggers.\n * - Each worker processes its batch of messages in parallel (Promise.all).\n * - pgmq visibility timeout prevents duplicate message reads across workers.\n * - processNext() is idempotent (uses internal cursor tracking), so duplicate\n * processing on timeout/crash is safe.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst QUEUE_NAME = 'stripe_sync_work'\nconst VISIBILITY_TIMEOUT = 60 // seconds\nconst BATCH_SIZE = 10\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7) // Remove 'Bearer '\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return new Response(JSON.stringify({ error: 'SUPABASE_DB_URL not set' }), { status: 500 })\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql\n let stripeSync\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return new Response(\n JSON.stringify({\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Validate that the token matches the unique worker secret stored in vault\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sync_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: (Deno.env.get('ENABLE_SIGMA') ?? 'false') === 'true',\n partnerId: 'pp_supabase',\n })\n } catch (error) {\n await sql.end()\n return new Response(\n JSON.stringify({\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n }),\n { status: 500, headers: { 'Content-Type': 'application/json' } }\n )\n }\n\n try {\n // Read batch of messages from queue\n const messages = await sql`\n SELECT * FROM pgmq.read(${QUEUE_NAME}::text, ${VISIBILITY_TIMEOUT}::int, ${BATCH_SIZE}::int)\n `\n\n // If queue empty, enqueue all objects for continuous sync\n if (messages.length === 0) {\n // Create sync run to make enqueued work visible (status='pending')\n const { objects } = await stripeSync.joinOrCreateSyncRun('worker')\n const msgs = objects.map((object) => JSON.stringify({ object }))\n\n await sql`\n SELECT pgmq.send_batch(\n ${QUEUE_NAME}::text,\n ${sql.array(msgs)}::jsonb[]\n )\n `\n\n return new Response(JSON.stringify({ enqueued: objects.length, objects }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n }\n\n // Process messages in parallel\n const results = await Promise.all(\n messages.map(async (msg) => {\n const { object } = msg.message as { object: string }\n\n try {\n const result = await stripeSync.processNext(object)\n\n // Delete message on success (cast to bigint to disambiguate overloaded function)\n await sql`SELECT pgmq.delete(${QUEUE_NAME}::text, ${msg.msg_id}::bigint)`\n\n // Re-enqueue if more pages\n if (result.hasMore) {\n await sql`SELECT pgmq.send(${QUEUE_NAME}::text, ${sql.json({ object })}::jsonb)`\n }\n\n return { object, ...result }\n } catch (error) {\n // Log error but continue to next message\n // Message will become visible again after visibility timeout\n console.error(`Error processing ${object}:`, error)\n return {\n object,\n processed: 0,\n hasMore: false,\n error: error.message,\n stack: error.stack,\n }\n }\n })\n )\n\n return new Response(JSON.stringify({ results }), {\n status: 200,\n headers: { 'Content-Type': 'application/json' },\n })\n } catch (error) {\n console.error('Worker error:', error)\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { 'Content-Type': 'application/json' },\n })\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
49
49
|
|
|
50
50
|
// raw-ts:/home/runner/work/sync-engine/sync-engine/packages/sync-engine/src/supabase/edge-functions/sigma-data-worker.ts
|
|
51
51
|
var sigma_data_worker_default = "/**\n * Stripe Sigma Data Worker.\n *\n * Hourly cron starts a run; self-trigger continues until all objects finish.\n * Progress persists in _sync_runs and _sync_obj_runs across invocations.\n */\n\nimport { StripeSync } from 'npm:stripe-experiment-sync'\nimport postgres from 'npm:postgres'\n\nconst BATCH_SIZE = 1\nconst MAX_RUN_AGE_MS = 6 * 60 * 60 * 1000\nconst jsonResponse = (body: unknown, status = 200) =>\n new Response(JSON.stringify(body), {\n status,\n headers: { 'Content-Type': 'application/json' },\n })\n\nDeno.serve(async (req) => {\n const authHeader = req.headers.get('Authorization')\n if (!authHeader?.startsWith('Bearer ')) {\n return new Response('Unauthorized', { status: 401 })\n }\n\n const token = authHeader.substring(7)\n\n const rawDbUrl = Deno.env.get('SUPABASE_DB_URL')\n if (!rawDbUrl) {\n return jsonResponse({ error: 'SUPABASE_DB_URL not set' }, 500)\n }\n const dbUrl = rawDbUrl.replace(/[?&]sslmode=[^&]*/g, '').replace(/[?&]$/, '')\n\n let sql: ReturnType<typeof postgres> | undefined\n let stripeSync: StripeSync | undefined\n\n try {\n sql = postgres(dbUrl, { max: 1, prepare: false })\n } catch (error) {\n return jsonResponse(\n {\n error: 'Failed to create postgres connection',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n // Validate the token against vault secret\n const vaultResult = await sql`\n SELECT decrypted_secret\n FROM vault.decrypted_secrets\n WHERE name = 'stripe_sigma_worker_secret'\n `\n\n if (vaultResult.length === 0) {\n await sql.end()\n return new Response('Sigma worker secret not configured in vault', { status: 500 })\n }\n\n const storedSecret = vaultResult[0].decrypted_secret\n if (token !== storedSecret) {\n await sql.end()\n return new Response('Forbidden: Invalid sigma worker secret', { status: 403 })\n }\n\n stripeSync = new StripeSync({\n poolConfig: { connectionString: dbUrl, max: 1 },\n stripeSecretKey: Deno.env.get('STRIPE_SECRET_KEY')!,\n enableSigma: true,\n sigmaPageSizeOverride: 1000,\n })\n } catch (error) {\n await sql.end()\n return jsonResponse(\n {\n error: 'Failed to create StripeSync',\n details: error.message,\n stack: error.stack,\n },\n 500\n )\n }\n\n try {\n const accountId = await stripeSync.getAccountId()\n const sigmaObjects = stripeSync.getSupportedSigmaObjects()\n\n if (sigmaObjects.length === 0) {\n return jsonResponse({ message: 'No Sigma objects configured for sync' })\n }\n\n // Get or create sync run for sigma-worker (isolated from stripe-worker)\n const runResult = await stripeSync.postgresClient.getOrCreateSyncRun(accountId, 'sigma-worker')\n const runStartedAt =\n runResult?.runStartedAt ??\n (await stripeSync.postgresClient.getActiveSyncRun(accountId, 'sigma-worker'))?.runStartedAt\n\n if (!runStartedAt) {\n throw new Error('Failed to get or create sync run for sigma worker')\n }\n\n // Legacy cleanup: remove any prefixed sigma object runs that can block concurrency.\n // Previous versions stored objects as \"sigma.<table>\" which no longer matches processNext.\n await stripeSync.postgresClient.query(\n `UPDATE \"stripe\".\"_sync_obj_runs\"\n SET status = 'error',\n error_message = 'Legacy sigma worker prefix run (sigma.*); superseded by unprefixed runs',\n completed_at = now()\n WHERE \"_account_id\" = $1\n AND run_started_at = $2\n AND object LIKE 'sigma.%'\n AND status IN ('pending', 'running')`,\n [accountId, runStartedAt]\n )\n\n // Stop self-triggering after MAX_RUN_AGE_MS.\n const runAgeMs = Date.now() - runStartedAt.getTime()\n if (runAgeMs > MAX_RUN_AGE_MS) {\n console.warn(\n `Sigma worker: run too old (${Math.round(runAgeMs / 1000 / 60)} min), closing without self-trigger`\n )\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n return jsonResponse({\n message: 'Sigma run exceeded max age, closed without processing',\n runAgeMinutes: Math.round(runAgeMs / 1000 / 60),\n selfTriggered: false,\n })\n }\n\n // Create object runs for all sigma objects (idempotent).\n await stripeSync.postgresClient.createObjectRuns(accountId, runStartedAt, sigmaObjects)\n await stripeSync.postgresClient.ensureSyncRunMaxConcurrent(accountId, runStartedAt, BATCH_SIZE)\n\n // Prefer running objects; otherwise claim pending ones.\n const runningObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n const objectsToProcess = runningObjects.slice(0, BATCH_SIZE)\n let pendingObjects: string[] = []\n\n if (objectsToProcess.length === 0) {\n pendingObjects = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n\n for (const objectKey of pendingObjects) {\n if (objectsToProcess.length >= BATCH_SIZE) break\n const started = await stripeSync.postgresClient.tryStartObjectSync(\n accountId,\n runStartedAt,\n objectKey\n )\n if (started) {\n objectsToProcess.push(objectKey)\n }\n }\n }\n\n if (objectsToProcess.length === 0) {\n if (pendingObjects.length === 0) {\n console.info('Sigma worker: all objects complete or errored - run finished')\n return jsonResponse({ message: 'Sigma sync run complete', selfTriggered: false })\n }\n\n console.info('Sigma worker: at concurrency limit, will self-trigger', {\n pendingCount: pendingObjects.length,\n })\n let selfTriggered = false\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n\n return jsonResponse({\n message: 'At concurrency limit',\n pendingCount: pendingObjects.length,\n selfTriggered,\n })\n }\n\n // Process objects sequentially (one lifecycle per invocation).\n const results: Array<Record<string, unknown>> = []\n\n for (const object of objectsToProcess) {\n const objectKey = object\n try {\n console.info(`Sigma worker: processing ${object}`)\n\n // Process one sigma page and upsert results.\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const result = await stripeSync.processNext(object as any, {\n runStartedAt,\n triggeredBy: 'sigma-worker',\n })\n\n results.push({\n object,\n processed: result.processed,\n hasMore: result.hasMore,\n status: 'success',\n })\n\n if (result.hasMore) {\n console.info(\n `Sigma worker: ${object} has more pages, processed ${result.processed} rows so far`\n )\n } else {\n console.info(`Sigma worker: ${object} complete, processed ${result.processed} rows`)\n }\n } catch (error) {\n console.error(`Sigma worker: error processing ${object}:`, error)\n\n // Mark object as failed and move on (no retries)\n await stripeSync.postgresClient.failObjectSync(\n accountId,\n runStartedAt,\n objectKey,\n error.message ?? 'Unknown error'\n )\n\n results.push({\n object,\n processed: 0,\n hasMore: false,\n status: 'error',\n error: error.message,\n })\n }\n }\n\n // Determine if self-trigger is needed\n const pendingAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'pending',\n sigmaObjects\n )\n const runningAfter = await stripeSync.postgresClient.listObjectsByStatus(\n accountId,\n runStartedAt,\n 'running',\n sigmaObjects\n )\n\n // Calculate remaining run time for logging\n const remainingMs = MAX_RUN_AGE_MS - (Date.now() - runStartedAt.getTime())\n const remainingMinutes = Math.round(remainingMs / 1000 / 60)\n\n // Only self-trigger if there are pending or running objects AND run hasn't timed out\n const shouldSelfTrigger =\n (pendingAfter.length > 0 || runningAfter.length > 0) && remainingMs > 0\n\n let selfTriggered = false\n if (shouldSelfTrigger) {\n console.info('Sigma worker: more work remains, self-triggering', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n remainingMinutes,\n })\n try {\n await sql`SELECT stripe.trigger_sigma_worker()`\n selfTriggered = true\n } catch (error) {\n console.warn('Failed to self-trigger sigma worker:', error.message)\n }\n } else if (pendingAfter.length > 0 || runningAfter.length > 0) {\n // Would self-trigger but run timed out\n console.warn('Sigma worker: work remains but run timed out, closing', {\n pending: pendingAfter.length,\n running: runningAfter.length,\n })\n await stripeSync.postgresClient.closeSyncRun(accountId, runStartedAt)\n } else {\n console.info('Sigma worker: no more work, run complete')\n }\n\n return jsonResponse({\n results,\n selfTriggered,\n remaining: { pending: pendingAfter.length, running: runningAfter.length },\n })\n } catch (error) {\n console.error('Sigma worker error:', error)\n return jsonResponse({ error: error.message, stack: error.stack }, 500)\n } finally {\n if (sql) await sql.end()\n if (stripeSync) await stripeSync.postgresClient.pool.end()\n }\n})\n";
|
|
@@ -59,7 +59,7 @@ var sigmaWorkerFunctionCode = sigma_data_worker_default;
|
|
|
59
59
|
// package.json
|
|
60
60
|
var package_default = {
|
|
61
61
|
name: "stripe-experiment-sync",
|
|
62
|
-
version: "1.0.
|
|
62
|
+
version: "1.0.22",
|
|
63
63
|
private: false,
|
|
64
64
|
description: "Stripe Sync Engine to sync Stripe data to Postgres",
|
|
65
65
|
type: "module",
|
package/dist/supabase/index.js
CHANGED
|
@@ -10,8 +10,8 @@ import {
|
|
|
10
10
|
uninstall,
|
|
11
11
|
webhookFunctionCode,
|
|
12
12
|
workerFunctionCode
|
|
13
|
-
} from "../chunk-
|
|
14
|
-
import "../chunk-
|
|
13
|
+
} from "../chunk-EIBEWIQC.js";
|
|
14
|
+
import "../chunk-IBYBC6LT.js";
|
|
15
15
|
export {
|
|
16
16
|
INSTALLATION_ERROR_SUFFIX,
|
|
17
17
|
INSTALLATION_INSTALLED_SUFFIX,
|