webhook-gate 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +7 -0
- package/README.md +115 -0
- package/package.json +34 -0
- package/packages/sdk/idempotent.js +142 -0
- package/packages/sdk/index.js +2 -0
- package/packages/sdk/package.json +13 -0
- package/scripts/chaos.js +72 -0
- package/scripts/git-hooks/pre-commit +6 -0
- package/src/config.js +18 -0
- package/src/consumer-demo.js +60 -0
- package/src/db.js +108 -0
- package/src/idempotent.js +143 -0
- package/src/index.js +127 -0
package/.env.example
ADDED
package/README.md
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# WebhookGate
|
|
2
|
+
|
|
3
|
+
WebhookGate guarantees **no duplicate webhook side effects**.
|
|
4
|
+
|
|
5
|
+
https://webhookgate.com
|
|
6
|
+
|
|
7
|
+
It does this by combining:
|
|
8
|
+
- **durable webhook intake and de-duplication** at the gateway layer, and
|
|
9
|
+
- a **consumer-side idempotency SDK** that makes duplicate side effects **structurally impossible** within the consumer.
|
|
10
|
+
|
|
11
|
+
WebhookGate sits in front of webhook consumers and ensures that each `(provider, eventId)` is **accepted exactly once**, even under retries, replays, or noisy providers - and that downstream effects are never duplicated when the consumer uses the SDK.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## What WebhookGate guarantees (MVP)
|
|
16
|
+
|
|
17
|
+
WebhookGate provides a durable webhook “inbox” in front of your consumer.
|
|
18
|
+
|
|
19
|
+
### Gateway guarantees
|
|
20
|
+
|
|
21
|
+
- **Exactly-once acceptance** per `(provider, eventId)`
|
|
22
|
+
- **De-duplication at intake**: repeated deliveries of the same event are not re-accepted
|
|
23
|
+
- **Durable delivery jobs + transport retries**: downstream delivery is retried on network/transport failure
|
|
24
|
+
- **Deterministic Idempotency-Key propagation** on every downstream delivery
|
|
25
|
+
|
|
26
|
+
The gateway delivers events **at-least-once** downstream, always with the same
|
|
27
|
+
Idempotency-Key, even across retries, crashes, or restarts.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Consumer SDK: No duplicate side effects
|
|
32
|
+
|
|
33
|
+
The consumer SDK converts delivery guarantees into **hard correctness**.
|
|
34
|
+
|
|
35
|
+
### What the SDK guarantees
|
|
36
|
+
|
|
37
|
+
- **At-most-once execution** of the handler per Idempotency-Key
|
|
38
|
+
- **Duplicate deliveries never re-run side effects**
|
|
39
|
+
- **Crash-safe behavior**: if the process crashes mid-handler, the handler is never re-entered (row remains processing)
|
|
40
|
+
- **Error behavior (terminal)**: if the handler throws, the idempotency row transitions to 'failed' and is never re-entered automatically
|
|
41
|
+
- **Transactional DB effects** for database operations performed via the provided db client
|
|
42
|
+
|
|
43
|
+
Once a key is successfully claimed, the handler will **never** be executed again — even if the process crashes, restarts, or receives the same event repeatedly.
|
|
44
|
+
|
|
45
|
+
> Result: side effects are never duplicated.
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## Exactly-once effects (clarified)
|
|
50
|
+
|
|
51
|
+
WebhookGate guarantees that your handler runs at most once per Idempotency-Key.
|
|
52
|
+
|
|
53
|
+
If your handler calls external systems (payments, email providers, APIs), those systems must respect idempotency keys to achieve end-to-end exactly-once effects across system boundaries.
|
|
54
|
+
|
|
55
|
+
This design deliberately favors **safety over re-execution**:
|
|
56
|
+
WebhookGate will never risk double-charging, double-emailing, or double-writing.
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
npm install webhookgate-sdk
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The SDK automatically creates the required idempotency tables on first use.
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## What the developer writes (locked API)
|
|
71
|
+
|
|
72
|
+
```js
|
|
73
|
+
import { idempotent } from "webhookgate-sdk";
|
|
74
|
+
|
|
75
|
+
app.post(
|
|
76
|
+
"/webhooks/stripe",
|
|
77
|
+
idempotent(async ({ event, db }) => {
|
|
78
|
+
await chargeCustomer(event); // never executed twice
|
|
79
|
+
await sendReceipt(event); // never executed twice
|
|
80
|
+
})
|
|
81
|
+
);
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Proof: No duplicate side effects (60 seconds)
|
|
87
|
+
|
|
88
|
+
1. Start Postgres
|
|
89
|
+
2. Install dependencies
|
|
90
|
+
npm install
|
|
91
|
+
3. Start the consumer
|
|
92
|
+
npm run consumer
|
|
93
|
+
4. Start the gateway
|
|
94
|
+
npm run dev
|
|
95
|
+
5. Send a replay storm
|
|
96
|
+
npm run chaos -- evt_test_1
|
|
97
|
+
|
|
98
|
+
Result:
|
|
99
|
+
- Gateway receives 50 duplicate events
|
|
100
|
+
- Consumer executes the handler once
|
|
101
|
+
- /stats shows charges = 1
|
|
102
|
+
|
|
103
|
+
Crash test:
|
|
104
|
+
- Start consumer with CRASH_ONCE=true
|
|
105
|
+
- Re-run chaos
|
|
106
|
+
- Restart consumer
|
|
107
|
+
- Charges still = 1
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## When you need production-grade durability
|
|
112
|
+
|
|
113
|
+
WebhookGate adds durable intake, replay protection, and operational safeguards for environments where local correctness is no longer sufficient.
|
|
114
|
+
|
|
115
|
+
Learn more at https://webhookgate.com
|
package/package.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "webhook-gate",
|
|
3
|
+
"version": "1.0.3",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"main": "src/index.js",
|
|
6
|
+
"scripts": {
|
|
7
|
+
"start": "node src/index.js",
|
|
8
|
+
"dev": "nodemon src/index.js",
|
|
9
|
+
"consumer": "node src/consumer-demo.js",
|
|
10
|
+
"chaos": "node scripts/chaos.js"
|
|
11
|
+
},
|
|
12
|
+
"repository": {
|
|
13
|
+
"type": "git",
|
|
14
|
+
"url": "git+https://github.com/webhookgate/webhook-gate.git"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [],
|
|
17
|
+
"author": "",
|
|
18
|
+
"license": "ISC",
|
|
19
|
+
"bugs": {
|
|
20
|
+
"url": "https://github.com/webhookgate/webhook-gate/issues"
|
|
21
|
+
},
|
|
22
|
+
"homepage": "https://github.com/webhookgate/webhook-gate#readme",
|
|
23
|
+
"description": "",
|
|
24
|
+
"dependencies": {
|
|
25
|
+
"better-sqlite3": "^12.5.0",
|
|
26
|
+
"dotenv": "^17.2.3",
|
|
27
|
+
"express": "^5.2.1",
|
|
28
|
+
"pg": "^8.16.3",
|
|
29
|
+
"stripe": "^20.1.0"
|
|
30
|
+
},
|
|
31
|
+
"devDependencies": {
|
|
32
|
+
"nodemon": "^3.1.11"
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { Pool } from "pg";
|
|
2
|
+
|
|
3
|
+
const pool = new Pool({
|
|
4
|
+
host: process.env.PGHOST,
|
|
5
|
+
port: process.env.PGPORT ? Number(process.env.PGPORT) : undefined,
|
|
6
|
+
database: process.env.PGDATABASE,
|
|
7
|
+
user: process.env.PGUSER,
|
|
8
|
+
password: process.env.PGPASSWORD,
|
|
9
|
+
ssl: process.env.PGSSL === "true" ? { rejectUnauthorized: false } : undefined,
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
let _readyPromise = null;
|
|
13
|
+
|
|
14
|
+
async function ensureTables() {
|
|
15
|
+
if (_readyPromise) return _readyPromise;
|
|
16
|
+
|
|
17
|
+
_readyPromise = (async () => {
|
|
18
|
+
await pool.query(`
|
|
19
|
+
CREATE TABLE IF NOT EXISTS webhook_idempotency (
|
|
20
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
21
|
+
status TEXT NOT NULL CHECK (status IN ('processing','done','failed')),
|
|
22
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
23
|
+
completed_at TIMESTAMPTZ
|
|
24
|
+
);
|
|
25
|
+
`);
|
|
26
|
+
|
|
27
|
+
await pool.query(`
|
|
28
|
+
CREATE TABLE IF NOT EXISTS demo_charges (
|
|
29
|
+
id BIGSERIAL PRIMARY KEY,
|
|
30
|
+
idempotency_key TEXT NOT NULL UNIQUE,
|
|
31
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
32
|
+
);
|
|
33
|
+
`);
|
|
34
|
+
})();
|
|
35
|
+
|
|
36
|
+
return _readyPromise;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function getIdempotencyKey(req) {
|
|
40
|
+
// Primary: what our gateway sends
|
|
41
|
+
const k = req.get("Idempotency-Key") || req.get("idempotency-key");
|
|
42
|
+
return k ? String(k).trim() : "";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Locked API:
|
|
47
|
+
* app.post("/webhooks/stripe", idempotent(async ({ req, event, db }) => {}))
|
|
48
|
+
*
|
|
49
|
+
* Guarantees:
|
|
50
|
+
* - first time key seen: handler runs
|
|
51
|
+
* - any repeat of same key: handler does NOT run (returns 200)
|
|
52
|
+
* - crash or error mid-handler: key transitions to 'failed' and is never re-run
|
|
53
|
+
*/
|
|
54
|
+
export function idempotent(handler) {
|
|
55
|
+
return async function idempotentMiddleware(req, res) {
|
|
56
|
+
await ensureTables();
|
|
57
|
+
|
|
58
|
+
const key = getIdempotencyKey(req);
|
|
59
|
+
|
|
60
|
+
if (!key) {
|
|
61
|
+
return res.status(400).json({
|
|
62
|
+
ok: false,
|
|
63
|
+
error: "Missing Idempotency-Key header",
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// IMPORTANT: the SDK assumes req.body already contains the parsed event
|
|
68
|
+
const event = req.body;
|
|
69
|
+
|
|
70
|
+
const client = await pool.connect();
|
|
71
|
+
let inTxn = false;
|
|
72
|
+
try {
|
|
73
|
+
// Atomic "claim"
|
|
74
|
+
// If it inserts: we own execution.
|
|
75
|
+
// If it conflicts: someone already claimed (or finished) => skip.
|
|
76
|
+
const claimed = await client.query(
|
|
77
|
+
`
|
|
78
|
+
INSERT INTO webhook_idempotency (idempotency_key, status)
|
|
79
|
+
VALUES ($1, 'processing')
|
|
80
|
+
ON CONFLICT (idempotency_key) DO NOTHING
|
|
81
|
+
RETURNING idempotency_key
|
|
82
|
+
`,
|
|
83
|
+
[key]
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
if (claimed.rowCount === 0) {
|
|
87
|
+
// Already seen => do nothing (exactly-once effects)
|
|
88
|
+
return res.status(200).json({ ok: true, deduped: true });
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// 2) Transaction boundary for DB effects inside handler
|
|
92
|
+
await client.query("BEGIN");
|
|
93
|
+
inTxn = true;
|
|
94
|
+
|
|
95
|
+
// Run user handler
|
|
96
|
+
await handler({ req, res, event, db: client });
|
|
97
|
+
|
|
98
|
+
// Mark done
|
|
99
|
+
await client.query(
|
|
100
|
+
`
|
|
101
|
+
UPDATE webhook_idempotency
|
|
102
|
+
SET status='done', completed_at=now()
|
|
103
|
+
WHERE idempotency_key=$1
|
|
104
|
+
`,
|
|
105
|
+
[key]
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
await client.query("COMMIT");
|
|
109
|
+
inTxn = false;
|
|
110
|
+
|
|
111
|
+
// If handler already wrote to res, don't double-send.
|
|
112
|
+
if (!res.headersSent) {
|
|
113
|
+
return res.status(200).json({ ok: true, deduped: false });
|
|
114
|
+
}
|
|
115
|
+
} catch (err) {
|
|
116
|
+
// best-effort rollback if handler started a txn
|
|
117
|
+
try {
|
|
118
|
+
if (inTxn) await client.query("ROLLBACK");
|
|
119
|
+
} catch (_) {}
|
|
120
|
+
|
|
121
|
+
// Optional: mark failed (still prevents reruns, which is the contract you chose)
|
|
122
|
+
try {
|
|
123
|
+
await client.query(
|
|
124
|
+
`
|
|
125
|
+
UPDATE webhook_idempotency
|
|
126
|
+
SET status='failed', completed_at=now()
|
|
127
|
+
WHERE idempotency_key=$1 AND status='processing'
|
|
128
|
+
`,
|
|
129
|
+
[key]
|
|
130
|
+
);
|
|
131
|
+
} catch (_) {
|
|
132
|
+
// ignore secondary failure
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (!res.headersSent) {
|
|
136
|
+
return res.status(500).json({ ok: false, error: err?.message || String(err) });
|
|
137
|
+
}
|
|
138
|
+
} finally {
|
|
139
|
+
client.release();
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
}
|
package/scripts/chaos.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// scripts/chaos.js
|
|
2
|
+
import "dotenv/config";
|
|
3
|
+
|
|
4
|
+
const GATEWAY = process.env.GATEWAY_URL || "http://localhost:3000/ingest";
|
|
5
|
+
const TOKEN = process.env.INGEST_TOKEN || "";
|
|
6
|
+
const CONSUMER = process.env.CONSUMER_URL || "http://localhost:4000";
|
|
7
|
+
|
|
8
|
+
async function postIngest({ provider, eventId, payload }) {
|
|
9
|
+
const resp = await fetch(GATEWAY, {
|
|
10
|
+
method: "POST",
|
|
11
|
+
headers: {
|
|
12
|
+
"Content-Type": "application/json",
|
|
13
|
+
...(TOKEN ? { "X-WebhookGate-Token": TOKEN } : {}),
|
|
14
|
+
},
|
|
15
|
+
body: JSON.stringify({ provider, eventId, payload }),
|
|
16
|
+
});
|
|
17
|
+
const json = await resp.json().catch(() => ({}));
|
|
18
|
+
return { status: resp.status, json };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function sleep(ms) {
|
|
22
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function getStats() {
|
|
26
|
+
const resp = await fetch(`${CONSUMER}/stats`);
|
|
27
|
+
const json = await resp.json().catch(() => ({}));
|
|
28
|
+
return { status: resp.status, json };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function main() {
|
|
32
|
+
const provider = "stripe";
|
|
33
|
+
const eventId = process.argv[2] || "evt_test_1";
|
|
34
|
+
|
|
35
|
+
// Burst duplicates concurrently
|
|
36
|
+
const N = 50;
|
|
37
|
+
const results = await Promise.all(
|
|
38
|
+
Array.from({ length: N }).map((_, i) =>
|
|
39
|
+
postIngest({
|
|
40
|
+
provider,
|
|
41
|
+
eventId,
|
|
42
|
+
payload: { n: i, ts: Date.now() },
|
|
43
|
+
})
|
|
44
|
+
)
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
const firstTimeCount = results.filter((r) => r.json?.firstTime).length;
|
|
48
|
+
console.log({ sent: N, firstTimeCount, sample: results[0] });
|
|
49
|
+
|
|
50
|
+
// Fetch consumer stats (give gateway a moment to deliver + retry loop to run)
|
|
51
|
+
let stats;
|
|
52
|
+
for (let attempt = 1; attempt <= 10; attempt++) {
|
|
53
|
+
stats = await getStats().catch(() => null);
|
|
54
|
+
|
|
55
|
+
if (stats && stats.status === 200) {
|
|
56
|
+
// If your consumer increments charges, this prints the proof
|
|
57
|
+
console.log({ consumerStats: stats.json, attempt });
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
await sleep(250);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (!stats || stats.status !== 200) {
|
|
65
|
+
console.log({ consumerStats: null, error: "Failed to fetch /stats from consumer" });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
main().catch((e) => {
|
|
70
|
+
console.error(e);
|
|
71
|
+
process.exit(1);
|
|
72
|
+
});
|
package/src/config.js
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export const PORT = process.env.PORT ? Number(process.env.PORT) : 3000;
|
|
2
|
+
|
|
3
|
+
// Optional shared secret for webhook auth later
|
|
4
|
+
export const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET || "";
|
|
5
|
+
|
|
6
|
+
// Where to forward accepted webhooks
|
|
7
|
+
export const TARGET_URL = process.env.TARGET_URL || "";
|
|
8
|
+
|
|
9
|
+
// Retry loop tuning (MVP defaults)
|
|
10
|
+
export const RETRY_INTERVAL_MS = process.env.RETRY_INTERVAL_MS
|
|
11
|
+
? Number(process.env.RETRY_INTERVAL_MS)
|
|
12
|
+
: 3000;
|
|
13
|
+
|
|
14
|
+
export const DELIVER_TIMEOUT_MS = process.env.DELIVER_TIMEOUT_MS
|
|
15
|
+
? Number(process.env.DELIVER_TIMEOUT_MS)
|
|
16
|
+
: 5000;
|
|
17
|
+
|
|
18
|
+
export const INGEST_TOKEN = process.env.INGEST_TOKEN || "";
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import "dotenv/config";
|
|
2
|
+
import express from "express";
|
|
3
|
+
import { Pool } from "pg";
|
|
4
|
+
import { idempotent } from "./idempotent.js";
|
|
5
|
+
|
|
6
|
+
const app = express();
|
|
7
|
+
app.use(express.json());
|
|
8
|
+
|
|
9
|
+
let crashedOnce = false;
|
|
10
|
+
|
|
11
|
+
const statsPool = new Pool({
|
|
12
|
+
host: process.env.PGHOST,
|
|
13
|
+
port: process.env.PGPORT ? Number(process.env.PGPORT) : undefined,
|
|
14
|
+
database: process.env.PGDATABASE,
|
|
15
|
+
user: process.env.PGUSER,
|
|
16
|
+
password: process.env.PGPASSWORD,
|
|
17
|
+
ssl: process.env.PGSSL === "true" ? { rejectUnauthorized: false } : undefined,
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
async function chargeCustomer({ db, key }) {
|
|
21
|
+
await db.query(
|
|
22
|
+
`INSERT INTO demo_charges (idempotency_key) VALUES ($1)
|
|
23
|
+
ON CONFLICT (idempotency_key) DO NOTHING`,
|
|
24
|
+
[key]
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
async function sendReceipt(_) {}
|
|
28
|
+
|
|
29
|
+
app.post(
|
|
30
|
+
"/webhooks/stripe",
|
|
31
|
+
idempotent(async ({ event, db, req }) => {
|
|
32
|
+
const key = (req.get("Idempotency-Key") || req.get("idempotency-key") || "").trim();
|
|
33
|
+
|
|
34
|
+
await chargeCustomer({ db, key });
|
|
35
|
+
|
|
36
|
+
if (process.env.CRASH_ONCE === "true" && !crashedOnce) {
|
|
37
|
+
crashedOnce = true;
|
|
38
|
+
console.log("[consumer] crashing after charge (CRASH_ONCE=true)");
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
await sendReceipt(event);
|
|
43
|
+
})
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
app.get("/stats", async (_, res) => {
|
|
47
|
+
try {
|
|
48
|
+
const r = await statsPool.query(`SELECT COUNT(*)::int AS charges FROM demo_charges`);
|
|
49
|
+
return res.json({ charges: r.rows[0].charges });
|
|
50
|
+
} catch (e) {
|
|
51
|
+
return res.status(500).json({ ok: false, error: e?.message || String(e) });
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
process.on("SIGINT", async () => {
|
|
56
|
+
await statsPool.end().catch(() => {});
|
|
57
|
+
process.exit(0);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
app.listen(4000, () => console.log("consumer demo on 4000"));
|
package/src/db.js
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import path from "path";
|
|
4
|
+
|
|
5
|
+
const dbPath = process.env.SQLITE_PATH || "./data/webhookgate.sqlite";
|
|
6
|
+
|
|
7
|
+
// ensure ./data exists
|
|
8
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
9
|
+
|
|
10
|
+
const db = new Database(dbPath);
|
|
11
|
+
|
|
12
|
+
// WAL helps reliability under concurrent access
|
|
13
|
+
db.pragma("journal_mode = WAL");
|
|
14
|
+
|
|
15
|
+
db.exec(`
|
|
16
|
+
CREATE TABLE IF NOT EXISTS receipts (
|
|
17
|
+
provider TEXT NOT NULL,
|
|
18
|
+
event_id TEXT NOT NULL,
|
|
19
|
+
received_at INTEGER NOT NULL,
|
|
20
|
+
PRIMARY KEY (provider, event_id)
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS deliveries (
|
|
24
|
+
provider TEXT NOT NULL,
|
|
25
|
+
event_id TEXT NOT NULL,
|
|
26
|
+
target_url TEXT NOT NULL,
|
|
27
|
+
payload_json TEXT NOT NULL,
|
|
28
|
+
|
|
29
|
+
status TEXT NOT NULL DEFAULT 'pending', -- pending | delivered | failed
|
|
30
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
31
|
+
last_error TEXT,
|
|
32
|
+
created_at INTEGER NOT NULL,
|
|
33
|
+
updated_at INTEGER NOT NULL,
|
|
34
|
+
delivered_at INTEGER,
|
|
35
|
+
|
|
36
|
+
PRIMARY KEY (provider, event_id, target_url)
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
CREATE INDEX IF NOT EXISTS idx_deliveries_status_updated
|
|
40
|
+
ON deliveries(status, updated_at);
|
|
41
|
+
`);
|
|
42
|
+
|
|
43
|
+
export function tryMarkReceived(provider, eventId) {
|
|
44
|
+
const stmt = db.prepare(`
|
|
45
|
+
INSERT INTO receipts (provider, event_id, received_at)
|
|
46
|
+
VALUES (?, ?, ?)
|
|
47
|
+
`);
|
|
48
|
+
|
|
49
|
+
try {
|
|
50
|
+
stmt.run(provider, eventId, Date.now());
|
|
51
|
+
return { firstTime: true };
|
|
52
|
+
} catch (err) {
|
|
53
|
+
const code = String(err && err.code);
|
|
54
|
+
if (code.startsWith("SQLITE_CONSTRAINT")) {
|
|
55
|
+
return { firstTime: false };
|
|
56
|
+
}
|
|
57
|
+
throw err;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function upsertDelivery({ provider, eventId, targetUrl, payload }) {
|
|
62
|
+
const now = Date.now();
|
|
63
|
+
const payloadJson = JSON.stringify(payload ?? null);
|
|
64
|
+
|
|
65
|
+
// If delivery already exists, keep the original payload (don’t overwrite).
|
|
66
|
+
const stmt = db.prepare(`
|
|
67
|
+
INSERT INTO deliveries (provider, event_id, target_url, payload_json, status, attempts, created_at, updated_at)
|
|
68
|
+
VALUES (?, ?, ?, ?, 'pending', 0, ?, ?)
|
|
69
|
+
ON CONFLICT(provider, event_id, target_url) DO NOTHING
|
|
70
|
+
`);
|
|
71
|
+
|
|
72
|
+
stmt.run(provider, eventId, targetUrl, payloadJson, now, now);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function getPendingDeliveries(limit = 25) {
|
|
76
|
+
const stmt = db.prepare(`
|
|
77
|
+
SELECT provider, event_id as eventId, target_url as targetUrl, payload_json as payloadJson, attempts
|
|
78
|
+
FROM deliveries
|
|
79
|
+
WHERE status = 'pending'
|
|
80
|
+
ORDER BY updated_at ASC
|
|
81
|
+
LIMIT ?
|
|
82
|
+
`);
|
|
83
|
+
return stmt.all(limit);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function markDelivered(provider, eventId, targetUrl) {
|
|
87
|
+
const now = Date.now();
|
|
88
|
+
const stmt = db.prepare(`
|
|
89
|
+
UPDATE deliveries
|
|
90
|
+
SET status='delivered', delivered_at=?, updated_at=?
|
|
91
|
+
WHERE provider=? AND event_id=? AND target_url=?
|
|
92
|
+
`);
|
|
93
|
+
stmt.run(now, now, provider, eventId, targetUrl);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function markAttemptFailed(provider, eventId, targetUrl, message) {
|
|
97
|
+
const MAX = Number(process.env.MAX_ATTEMPTS || 25);
|
|
98
|
+
const now = Date.now();
|
|
99
|
+
const stmt = db.prepare(`
|
|
100
|
+
UPDATE deliveries
|
|
101
|
+
SET attempts = attempts + 1,
|
|
102
|
+
last_error = ?,
|
|
103
|
+
status = CASE WHEN attempts + 1 >= ? THEN 'failed' ELSE status END,
|
|
104
|
+
updated_at = ?
|
|
105
|
+
WHERE provider=? AND event_id=? AND target_url=? AND status='pending'
|
|
106
|
+
`);
|
|
107
|
+
stmt.run(String(message || "unknown error"), MAX, now, provider, eventId, targetUrl);
|
|
108
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// src/idempotent.js
|
|
2
|
+
import { Pool } from "pg";
|
|
3
|
+
|
|
4
|
+
const pool = new Pool({
|
|
5
|
+
host: process.env.PGHOST,
|
|
6
|
+
port: process.env.PGPORT ? Number(process.env.PGPORT) : undefined,
|
|
7
|
+
database: process.env.PGDATABASE,
|
|
8
|
+
user: process.env.PGUSER,
|
|
9
|
+
password: process.env.PGPASSWORD,
|
|
10
|
+
ssl: process.env.PGSSL === "true" ? { rejectUnauthorized: false } : undefined,
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
let _readyPromise = null;
|
|
14
|
+
|
|
15
|
+
async function ensureTables() {
|
|
16
|
+
if (_readyPromise) return _readyPromise;
|
|
17
|
+
|
|
18
|
+
_readyPromise = (async () => {
|
|
19
|
+
await pool.query(`
|
|
20
|
+
CREATE TABLE IF NOT EXISTS webhook_idempotency (
|
|
21
|
+
idempotency_key TEXT PRIMARY KEY,
|
|
22
|
+
status TEXT NOT NULL CHECK (status IN ('processing','done','failed')),
|
|
23
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
24
|
+
completed_at TIMESTAMPTZ
|
|
25
|
+
);
|
|
26
|
+
`);
|
|
27
|
+
|
|
28
|
+
await pool.query(`
|
|
29
|
+
CREATE TABLE IF NOT EXISTS demo_charges (
|
|
30
|
+
id BIGSERIAL PRIMARY KEY,
|
|
31
|
+
idempotency_key TEXT NOT NULL UNIQUE,
|
|
32
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
33
|
+
);
|
|
34
|
+
`);
|
|
35
|
+
})();
|
|
36
|
+
|
|
37
|
+
return _readyPromise;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function getIdempotencyKey(req) {
|
|
41
|
+
// Primary: what our gateway sends
|
|
42
|
+
const k = req.get("Idempotency-Key") || req.get("idempotency-key");
|
|
43
|
+
return k ? String(k).trim() : "";
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Locked API:
|
|
48
|
+
* app.post("/webhooks/stripe", idempotent(async ({ req, event, db }) => {}))
|
|
49
|
+
*
|
|
50
|
+
* Guarantees:
|
|
51
|
+
* - first time key seen: handler runs
|
|
52
|
+
* - any repeat of same key: handler does NOT run (returns 200)
|
|
53
|
+
* - crash or error mid-handler: key transitions to 'failed' and is never re-run
|
|
54
|
+
*/
|
|
55
|
+
export function idempotent(handler) {
|
|
56
|
+
return async function idempotentMiddleware(req, res) {
|
|
57
|
+
await ensureTables();
|
|
58
|
+
|
|
59
|
+
const key = getIdempotencyKey(req);
|
|
60
|
+
|
|
61
|
+
if (!key) {
|
|
62
|
+
return res.status(400).json({
|
|
63
|
+
ok: false,
|
|
64
|
+
error: "Missing Idempotency-Key header",
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// IMPORTANT: the SDK assumes req.body already contains the parsed event
|
|
69
|
+
const event = req.body;
|
|
70
|
+
|
|
71
|
+
const client = await pool.connect();
|
|
72
|
+
let inTxn = false;
|
|
73
|
+
try {
|
|
74
|
+
// Atomic "claim"
|
|
75
|
+
// If it inserts: we own execution.
|
|
76
|
+
// If it conflicts: someone already claimed (or finished) => skip.
|
|
77
|
+
const claimed = await client.query(
|
|
78
|
+
`
|
|
79
|
+
INSERT INTO webhook_idempotency (idempotency_key, status)
|
|
80
|
+
VALUES ($1, 'processing')
|
|
81
|
+
ON CONFLICT (idempotency_key) DO NOTHING
|
|
82
|
+
RETURNING idempotency_key
|
|
83
|
+
`,
|
|
84
|
+
[key]
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
if (claimed.rowCount === 0) {
|
|
88
|
+
// Already seen => do nothing (exactly-once effects)
|
|
89
|
+
return res.status(200).json({ ok: true, deduped: true });
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// 2) Transaction boundary for DB effects inside handler
|
|
93
|
+
await client.query("BEGIN");
|
|
94
|
+
inTxn = true;
|
|
95
|
+
|
|
96
|
+
// Run user handler
|
|
97
|
+
await handler({ req, res, event, db: client });
|
|
98
|
+
|
|
99
|
+
// Mark done
|
|
100
|
+
await client.query(
|
|
101
|
+
`
|
|
102
|
+
UPDATE webhook_idempotency
|
|
103
|
+
SET status='done', completed_at=now()
|
|
104
|
+
WHERE idempotency_key=$1
|
|
105
|
+
`,
|
|
106
|
+
[key]
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
await client.query("COMMIT");
|
|
110
|
+
inTxn = false;
|
|
111
|
+
|
|
112
|
+
// If handler already wrote to res, don't double-send.
|
|
113
|
+
if (!res.headersSent) {
|
|
114
|
+
return res.status(200).json({ ok: true, deduped: false });
|
|
115
|
+
}
|
|
116
|
+
} catch (err) {
|
|
117
|
+
// best-effort rollback if handler started a txn
|
|
118
|
+
try {
|
|
119
|
+
if (inTxn) await client.query("ROLLBACK");
|
|
120
|
+
} catch (_) {}
|
|
121
|
+
|
|
122
|
+
// Optional: mark failed (still prevents reruns, which is the contract you chose)
|
|
123
|
+
try {
|
|
124
|
+
await client.query(
|
|
125
|
+
`
|
|
126
|
+
UPDATE webhook_idempotency
|
|
127
|
+
SET status='failed', completed_at=now()
|
|
128
|
+
WHERE idempotency_key=$1 AND status='processing'
|
|
129
|
+
`,
|
|
130
|
+
[key]
|
|
131
|
+
);
|
|
132
|
+
} catch (_) {
|
|
133
|
+
// ignore secondary failure
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (!res.headersSent) {
|
|
137
|
+
return res.status(500).json({ ok: false, error: err?.message || String(err) });
|
|
138
|
+
}
|
|
139
|
+
} finally {
|
|
140
|
+
client.release();
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import "dotenv/config";
|
|
2
|
+
import express from "express";
|
|
3
|
+
import {
|
|
4
|
+
PORT,
|
|
5
|
+
TARGET_URL,
|
|
6
|
+
RETRY_INTERVAL_MS,
|
|
7
|
+
DELIVER_TIMEOUT_MS,
|
|
8
|
+
INGEST_TOKEN,
|
|
9
|
+
} from "./config.js";
|
|
10
|
+
import {
|
|
11
|
+
tryMarkReceived,
|
|
12
|
+
upsertDelivery,
|
|
13
|
+
getPendingDeliveries,
|
|
14
|
+
markDelivered,
|
|
15
|
+
markAttemptFailed,
|
|
16
|
+
} from "./db.js";
|
|
17
|
+
|
|
18
|
+
const app = express();
|
|
19
|
+
app.use(express.json());
|
|
20
|
+
|
|
21
|
+
app.get("/health", (_, res) => res.json({ ok: true }));
|
|
22
|
+
|
|
23
|
+
function withTimeout(ms) {
|
|
24
|
+
const controller = new AbortController();
|
|
25
|
+
const t = setTimeout(() => controller.abort(), ms);
|
|
26
|
+
return { signal: controller.signal, cancel: () => clearTimeout(t) };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function deliverOne({ provider, eventId, targetUrl, payload }) {
|
|
30
|
+
const key = `${provider}:${eventId}:${targetUrl}`;
|
|
31
|
+
|
|
32
|
+
const { signal, cancel } = withTimeout(DELIVER_TIMEOUT_MS);
|
|
33
|
+
try {
|
|
34
|
+
const resp = await fetch(targetUrl, {
|
|
35
|
+
method: "POST",
|
|
36
|
+
headers: {
|
|
37
|
+
"Content-Type": "application/json",
|
|
38
|
+
"Idempotency-Key": key,
|
|
39
|
+
"X-WebhookGate-Provider": provider,
|
|
40
|
+
"X-WebhookGate-EventId": eventId,
|
|
41
|
+
},
|
|
42
|
+
body: JSON.stringify(payload ?? null),
|
|
43
|
+
signal,
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
if (!resp.ok) {
|
|
47
|
+
// Semantic failure in consumer; do NOT retry.
|
|
48
|
+
console.log(`[downstream-non2xx] ${key} ${resp.status} ${resp.statusText}`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
markDelivered(provider, eventId, targetUrl);
|
|
52
|
+
console.log(`[deliver-ok] ${key}`);
|
|
53
|
+
return true;
|
|
54
|
+
|
|
55
|
+
} catch (err) {
|
|
56
|
+
markAttemptFailed(provider, eventId, targetUrl, err?.message || String(err));
|
|
57
|
+
console.log(`[deliver-fail] ${key} ${err?.message || String(err)}`);
|
|
58
|
+
return false;
|
|
59
|
+
} finally {
|
|
60
|
+
cancel();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// MVP: send { provider, eventId, payload }
|
|
65
|
+
app.post("/ingest", async (req, res) => {
|
|
66
|
+
const provider = String(req.body?.provider || "");
|
|
67
|
+
const eventId = String(req.body?.eventId || "");
|
|
68
|
+
const payload = req.body?.payload;
|
|
69
|
+
|
|
70
|
+
if (INGEST_TOKEN) {
|
|
71
|
+
const t = String(req.get("X-WebhookGate-Token") || "");
|
|
72
|
+
if (t !== INGEST_TOKEN) {
|
|
73
|
+
return res.status(401).json({ ok: false, error: "Unauthorized" });
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (!provider || !eventId) {
|
|
78
|
+
return res.status(400).json({ ok: false, error: "provider and eventId are required" });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (!TARGET_URL) {
|
|
82
|
+
return res.status(500).json({ ok: false, error: "TARGET_URL is not set" });
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const { firstTime } = tryMarkReceived(provider, eventId);
|
|
86
|
+
|
|
87
|
+
if (!firstTime) {
|
|
88
|
+
console.log(`[dedupe] ${provider} ${eventId}`);
|
|
89
|
+
return res.status(200).json({ ok: true, firstTime: false });
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
console.log(`[accept] ${provider} ${eventId}`);
|
|
93
|
+
|
|
94
|
+
// Store delivery job durably so retries are possible
|
|
95
|
+
upsertDelivery({ provider, eventId, targetUrl: TARGET_URL, payload });
|
|
96
|
+
|
|
97
|
+
// Best-effort immediate delivery attempt
|
|
98
|
+
const delivered = await deliverOne({ provider, eventId, targetUrl: TARGET_URL, payload });
|
|
99
|
+
|
|
100
|
+
// Even if not delivered yet, we return 200 to stop webhook retry storms.
|
|
101
|
+
return res.status(200).json({ ok: true, firstTime: true, delivered });
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// Simple in-process retry loop (MVP)
|
|
105
|
+
let draining = false;
|
|
106
|
+
async function drainOnce() {
|
|
107
|
+
if (draining) return;
|
|
108
|
+
draining = true;
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
const jobs = getPendingDeliveries(25);
|
|
112
|
+
for (const j of jobs) {
|
|
113
|
+
const payload = JSON.parse(j.payloadJson);
|
|
114
|
+
await deliverOne({ provider: j.provider, eventId: j.eventId, targetUrl: j.targetUrl, payload });
|
|
115
|
+
}
|
|
116
|
+
} finally {
|
|
117
|
+
draining = false;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
setInterval(() => {
|
|
122
|
+
drainOnce().catch((e) => console.log(`[drain-error] ${e?.message || String(e)}`));
|
|
123
|
+
}, RETRY_INTERVAL_MS);
|
|
124
|
+
|
|
125
|
+
app.listen(PORT, () => {
|
|
126
|
+
console.log(`WebhookGate listening on ${PORT}`);
|
|
127
|
+
});
|