webpeel 0.20.2 → 0.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/server/app.d.ts +14 -0
  2. package/dist/server/app.js +384 -0
  3. package/dist/server/auth-store.d.ts +27 -0
  4. package/dist/server/auth-store.js +88 -0
  5. package/dist/server/email-service.d.ts +21 -0
  6. package/dist/server/email-service.js +79 -0
  7. package/dist/server/job-queue.d.ts +100 -0
  8. package/dist/server/job-queue.js +145 -0
  9. package/dist/server/logger.d.ts +10 -0
  10. package/dist/server/logger.js +37 -0
  11. package/dist/server/middleware/auth.d.ts +28 -0
  12. package/dist/server/middleware/auth.js +221 -0
  13. package/dist/server/middleware/rate-limit.d.ts +24 -0
  14. package/dist/server/middleware/rate-limit.js +167 -0
  15. package/dist/server/middleware/url-validator.d.ts +15 -0
  16. package/dist/server/middleware/url-validator.js +186 -0
  17. package/dist/server/openapi.yaml +6418 -0
  18. package/dist/server/pg-auth-store.d.ts +132 -0
  19. package/dist/server/pg-auth-store.js +472 -0
  20. package/dist/server/pg-job-queue.d.ts +59 -0
  21. package/dist/server/pg-job-queue.js +375 -0
  22. package/dist/server/premium/domain-intel.d.ts +16 -0
  23. package/dist/server/premium/domain-intel.js +133 -0
  24. package/dist/server/premium/index.d.ts +17 -0
  25. package/dist/server/premium/index.js +35 -0
  26. package/dist/server/premium/swr-cache.d.ts +14 -0
  27. package/dist/server/premium/swr-cache.js +34 -0
  28. package/dist/server/routes/activity.d.ts +6 -0
  29. package/dist/server/routes/activity.js +74 -0
  30. package/dist/server/routes/answer.d.ts +5 -0
  31. package/dist/server/routes/answer.js +125 -0
  32. package/dist/server/routes/ask.d.ts +28 -0
  33. package/dist/server/routes/ask.js +229 -0
  34. package/dist/server/routes/batch.d.ts +6 -0
  35. package/dist/server/routes/batch.js +493 -0
  36. package/dist/server/routes/cli-usage.d.ts +6 -0
  37. package/dist/server/routes/cli-usage.js +127 -0
  38. package/dist/server/routes/compat.d.ts +23 -0
  39. package/dist/server/routes/compat.js +652 -0
  40. package/dist/server/routes/deep-fetch.d.ts +8 -0
  41. package/dist/server/routes/deep-fetch.js +57 -0
  42. package/dist/server/routes/demo.d.ts +24 -0
  43. package/dist/server/routes/demo.js +517 -0
  44. package/dist/server/routes/do.d.ts +8 -0
  45. package/dist/server/routes/do.js +72 -0
  46. package/dist/server/routes/extract.d.ts +8 -0
  47. package/dist/server/routes/extract.js +235 -0
  48. package/dist/server/routes/fetch.d.ts +7 -0
  49. package/dist/server/routes/fetch.js +999 -0
  50. package/dist/server/routes/health.d.ts +7 -0
  51. package/dist/server/routes/health.js +19 -0
  52. package/dist/server/routes/jobs.d.ts +7 -0
  53. package/dist/server/routes/jobs.js +573 -0
  54. package/dist/server/routes/mcp.d.ts +14 -0
  55. package/dist/server/routes/mcp.js +141 -0
  56. package/dist/server/routes/oauth.d.ts +9 -0
  57. package/dist/server/routes/oauth.js +396 -0
  58. package/dist/server/routes/playground.d.ts +17 -0
  59. package/dist/server/routes/playground.js +283 -0
  60. package/dist/server/routes/screenshot.d.ts +22 -0
  61. package/dist/server/routes/screenshot.js +816 -0
  62. package/dist/server/routes/search.d.ts +6 -0
  63. package/dist/server/routes/search.js +303 -0
  64. package/dist/server/routes/session.d.ts +15 -0
  65. package/dist/server/routes/session.js +397 -0
  66. package/dist/server/routes/stats.d.ts +6 -0
  67. package/dist/server/routes/stats.js +71 -0
  68. package/dist/server/routes/stripe.d.ts +15 -0
  69. package/dist/server/routes/stripe.js +294 -0
  70. package/dist/server/routes/users.d.ts +8 -0
  71. package/dist/server/routes/users.js +1671 -0
  72. package/dist/server/routes/watch.d.ts +15 -0
  73. package/dist/server/routes/watch.js +309 -0
  74. package/dist/server/routes/webhooks.d.ts +26 -0
  75. package/dist/server/routes/webhooks.js +170 -0
  76. package/dist/server/routes/youtube.d.ts +6 -0
  77. package/dist/server/routes/youtube.js +130 -0
  78. package/dist/server/sentry.d.ts +13 -0
  79. package/dist/server/sentry.js +38 -0
  80. package/dist/server/types.d.ts +15 -0
  81. package/dist/server/types.js +7 -0
  82. package/dist/server/utils/response.d.ts +44 -0
  83. package/dist/server/utils/response.js +69 -0
  84. package/dist/server/utils/sse.d.ts +22 -0
  85. package/dist/server/utils/sse.js +38 -0
  86. package/package.json +2 -1
@@ -0,0 +1,15 @@
1
+ /**
2
+ * WebPeel Watch REST API
3
+ *
4
+ * POST /v1/watch — Create a new watch
5
+ * GET /v1/watch — List watches for the authenticated account
6
+ * GET /v1/watch/:id — Get a single watch entry
7
+ * POST /v1/watch/:id/check — Manually trigger a content check
8
+ * PATCH /v1/watch/:id — Update a watch (pause/resume/interval)
9
+ * DELETE /v1/watch/:id — Delete a watch
10
+ *
11
+ * All routes require API-key authentication via the global auth middleware.
12
+ */
13
+ import { Router } from 'express';
14
+ import pg from 'pg';
15
+ export declare function createWatchRouter(pool: pg.Pool): Router;
@@ -0,0 +1,309 @@
1
+ /**
2
+ * WebPeel Watch REST API
3
+ *
4
+ * POST /v1/watch — Create a new watch
5
+ * GET /v1/watch — List watches for the authenticated account
6
+ * GET /v1/watch/:id — Get a single watch entry
7
+ * POST /v1/watch/:id/check — Manually trigger a content check
8
+ * PATCH /v1/watch/:id — Update a watch (pause/resume/interval)
9
+ * DELETE /v1/watch/:id — Delete a watch
10
+ *
11
+ * All routes require API-key authentication via the global auth middleware.
12
+ */
13
+ import { Router } from 'express';
14
+ import { WatchManager, computeLineDiff } from '../../core/watch-manager.js';
15
+ /** Rough token estimate: ~4 characters per token (GPT-style approximation). */
16
+ function estimateTokens(text) {
17
+ return Math.ceil(text.length / 4);
18
+ }
19
+ export function createWatchRouter(pool) {
20
+ const router = Router();
21
+ const manager = new WatchManager(pool);
22
+ // ─── Require authentication helper ─────────────────────────────────────────
23
+ function requireAuth(req, res) {
24
+ const accountId = req.auth?.keyInfo?.accountId;
25
+ if (!accountId) {
26
+ res.status(401).json({
27
+ success: false,
28
+ error: { type: 'unauthorized', message: 'API key required. Pass via Authorization: Bearer <key>.', docs: 'https://webpeel.dev/docs/authentication' },
29
+ requestId: req.requestId,
30
+ });
31
+ return null;
32
+ }
33
+ return accountId;
34
+ }
35
+ // ─── POST /v1/watch — create a watch ────────────────────────────────────────
36
+ router.post('/v1/watch', async (req, res) => {
37
+ const accountId = requireAuth(req, res);
38
+ if (!accountId)
39
+ return;
40
+ const { url, webhookUrl, checkIntervalMinutes, intervalMinutes: intervalMinutesAlias, interval, selector } = req.body;
41
+ // Accept interval aliases: checkIntervalMinutes (canonical), intervalMinutes, interval
42
+ const resolvedIntervalInput = checkIntervalMinutes ?? intervalMinutesAlias ?? interval;
43
+ if (!url || typeof url !== 'string') {
44
+ res.status(400).json({
45
+ success: false,
46
+ error: { type: 'invalid_request', message: 'Missing or invalid "url" parameter.' },
47
+ requestId: req.requestId,
48
+ });
49
+ return;
50
+ }
51
+ if (url.length > 2048) {
52
+ res.status(400).json({
53
+ success: false,
54
+ error: { type: 'invalid_url', message: 'URL too long (max 2048 characters).' },
55
+ requestId: req.requestId,
56
+ });
57
+ return;
58
+ }
59
+ try {
60
+ new URL(url);
61
+ }
62
+ catch {
63
+ res.status(400).json({
64
+ success: false,
65
+ error: { type: 'invalid_url', message: 'URL format is invalid.', hint: 'Ensure the URL includes a scheme (https://) and a valid hostname.' },
66
+ requestId: req.requestId,
67
+ });
68
+ return;
69
+ }
70
+ if (webhookUrl !== undefined && (typeof webhookUrl !== 'string' || webhookUrl.length > 2048)) {
71
+ res.status(400).json({ success: false, error: { type: 'invalid_request', message: 'Invalid "webhookUrl".' }, requestId: req.requestId });
72
+ return;
73
+ }
74
+ const intervalMinutes = resolvedIntervalInput !== undefined ? Number(resolvedIntervalInput) : 60;
75
+ if (!Number.isFinite(intervalMinutes) || intervalMinutes < 1 || intervalMinutes > 44640) {
76
+ res.status(400).json({
77
+ success: false,
78
+ error: { type: 'invalid_request', message: '"checkIntervalMinutes" must be between 1 and 44640 (31 days).' },
79
+ requestId: req.requestId,
80
+ });
81
+ return;
82
+ }
83
+ try {
84
+ const entry = await manager.create(accountId, url, {
85
+ webhookUrl: typeof webhookUrl === 'string' ? webhookUrl : undefined,
86
+ checkIntervalMinutes: intervalMinutes,
87
+ selector: typeof selector === 'string' ? selector : undefined,
88
+ });
89
+ res.status(201).json({ ok: true, watch: entry });
90
+ }
91
+ catch (err) {
92
+ console.error('[watch] create error:', err);
93
+ res.status(500).json({
94
+ success: false,
95
+ error: { type: 'internal_error', message: 'Failed to create watch.' },
96
+ requestId: req.requestId,
97
+ });
98
+ }
99
+ });
100
+ // ─── GET /v1/watch — list watches ───────────────────────────────────────────
101
+ router.get('/v1/watch', async (req, res) => {
102
+ const accountId = requireAuth(req, res);
103
+ if (!accountId)
104
+ return;
105
+ try {
106
+ const watches = await manager.list(accountId);
107
+ res.json({ ok: true, watches });
108
+ }
109
+ catch (err) {
110
+ console.error('[watch] list error:', err);
111
+ res.status(500).json({
112
+ success: false,
113
+ error: { type: 'internal_error', message: 'Failed to list watches.' },
114
+ requestId: req.requestId,
115
+ });
116
+ }
117
+ });
118
+ // ─── GET /v1/watch/:id — get a watch ────────────────────────────────────────
119
+ router.get('/v1/watch/:id', async (req, res) => {
120
+ const accountId = requireAuth(req, res);
121
+ if (!accountId)
122
+ return;
123
+ const watchId = req.params['id'];
124
+ try {
125
+ const entry = await manager.get(watchId);
126
+ if (!entry) {
127
+ res.status(404).json({
128
+ success: false,
129
+ error: { type: 'not_found', message: 'Watch not found.' },
130
+ requestId: req.requestId,
131
+ });
132
+ return;
133
+ }
134
+ if (entry.accountId !== accountId) {
135
+ res.status(403).json({ success: false, error: { type: 'forbidden', message: 'Access denied.' }, requestId: req.requestId });
136
+ return;
137
+ }
138
+ res.json({ ok: true, watch: entry });
139
+ }
140
+ catch (err) {
141
+ console.error('[watch] get error:', err);
142
+ res.status(500).json({
143
+ success: false,
144
+ error: { type: 'internal_error', message: 'Failed to get watch.' },
145
+ requestId: req.requestId,
146
+ });
147
+ }
148
+ });
149
+ // ─── POST /v1/watch/:id/check — manual check ─────────────────────────────────
150
+ //
151
+ // Query params:
152
+ // ?diff=true — Return a line-level diff alongside the full content, with
153
+ // token-savings metadata. Default behaviour (no param) is
154
+ // unchanged — returns the raw WatchDiff object.
155
+ router.post('/v1/watch/:id/check', async (req, res) => {
156
+ const accountId = requireAuth(req, res);
157
+ if (!accountId)
158
+ return;
159
+ const watchId = req.params['id'];
160
+ const includeDiff = req.query['diff'] === 'true';
161
+ try {
162
+ const entry = await manager.get(watchId);
163
+ if (!entry) {
164
+ res.status(404).json({
165
+ success: false,
166
+ error: { type: 'not_found', message: 'Watch not found.' },
167
+ requestId: req.requestId,
168
+ });
169
+ return;
170
+ }
171
+ if (entry.accountId !== accountId) {
172
+ res.status(403).json({ success: false, error: { type: 'forbidden', message: 'Access denied.' }, requestId: req.requestId });
173
+ return;
174
+ }
175
+ const watchDiff = await manager.check(watchId);
176
+ if (includeDiff) {
177
+ // Compute line-level diff between previous and current content.
178
+ const lineDiff = computeLineDiff(watchDiff.previousContent, watchDiff.content);
179
+ const fullTokens = estimateTokens(watchDiff.content);
180
+ const diffText = [...lineDiff.added, ...lineDiff.removed].join('\n');
181
+ const diffTokens = estimateTokens(diffText);
182
+ const result = {
183
+ changed: watchDiff.changed,
184
+ content: watchDiff.content,
185
+ diffTokens,
186
+ fullTokens,
187
+ };
188
+ if (lineDiff.changed) {
189
+ result.diff = {
190
+ added: lineDiff.added,
191
+ removed: lineDiff.removed,
192
+ summary: lineDiff.summary,
193
+ changePercent: lineDiff.changePercent,
194
+ };
195
+ }
196
+ res.json({ ok: true, diff: result });
197
+ }
198
+ else {
199
+ res.json({ ok: true, diff: watchDiff });
200
+ }
201
+ }
202
+ catch (err) {
203
+ console.error('[watch] manual check error:', err);
204
+ res.status(500).json({
205
+ success: false,
206
+ error: {
207
+ type: 'check_failed',
208
+ message: err instanceof Error ? err.message : 'Check failed.',
209
+ docs: 'https://webpeel.dev/docs/errors#check_failed',
210
+ },
211
+ requestId: req.requestId,
212
+ });
213
+ }
214
+ });
215
+ // ─── PATCH /v1/watch/:id — update a watch ───────────────────────────────────
216
+ router.patch('/v1/watch/:id', async (req, res) => {
217
+ const accountId = requireAuth(req, res);
218
+ if (!accountId)
219
+ return;
220
+ const watchId = req.params['id'];
221
+ const { status, webhookUrl, checkIntervalMinutes, intervalMinutes: patchIntervalMinutesAlias, interval: patchInterval, selector } = req.body;
222
+ // Accept interval aliases
223
+ const resolvedPatchInterval = checkIntervalMinutes ?? patchIntervalMinutesAlias ?? patchInterval;
224
+ try {
225
+ const existing = await manager.get(watchId);
226
+ if (!existing) {
227
+ res.status(404).json({
228
+ success: false,
229
+ error: { type: 'not_found', message: 'Watch not found.' },
230
+ requestId: req.requestId,
231
+ });
232
+ return;
233
+ }
234
+ if (existing.accountId !== accountId) {
235
+ res.status(403).json({ success: false, error: { type: 'forbidden', message: 'Access denied.' }, requestId: req.requestId });
236
+ return;
237
+ }
238
+ // Handle pause/resume as a convenience.
239
+ if (status === 'paused') {
240
+ await manager.pause(watchId);
241
+ }
242
+ else if (status === 'active') {
243
+ await manager.resume(watchId);
244
+ }
245
+ const updates = {};
246
+ if (webhookUrl !== undefined) {
247
+ updates.webhookUrl = typeof webhookUrl === 'string' ? webhookUrl : undefined;
248
+ }
249
+ if (resolvedPatchInterval !== undefined) {
250
+ const n = Number(resolvedPatchInterval);
251
+ if (!Number.isFinite(n) || n < 1 || n > 44640) {
252
+ res.status(400).json({
253
+ success: false,
254
+ error: { type: 'invalid_request', message: '"checkIntervalMinutes" must be between 1 and 44640.' },
255
+ requestId: req.requestId,
256
+ });
257
+ return;
258
+ }
259
+ updates.checkIntervalMinutes = n;
260
+ }
261
+ if (selector !== undefined) {
262
+ updates.selector = typeof selector === 'string' ? selector : undefined;
263
+ }
264
+ const updated = await manager.update(watchId, updates);
265
+ res.json({ ok: true, watch: updated ?? existing });
266
+ }
267
+ catch (err) {
268
+ console.error('[watch] update error:', err);
269
+ res.status(500).json({
270
+ success: false,
271
+ error: { type: 'internal_error', message: 'Failed to update watch.' },
272
+ requestId: req.requestId,
273
+ });
274
+ }
275
+ });
276
+ // ─── DELETE /v1/watch/:id — delete a watch ───────────────────────────────────
277
+ router.delete('/v1/watch/:id', async (req, res) => {
278
+ const accountId = requireAuth(req, res);
279
+ if (!accountId)
280
+ return;
281
+ const watchId = req.params['id'];
282
+ try {
283
+ const existing = await manager.get(watchId);
284
+ if (!existing) {
285
+ res.status(404).json({
286
+ success: false,
287
+ error: { type: 'not_found', message: 'Watch not found.' },
288
+ requestId: req.requestId,
289
+ });
290
+ return;
291
+ }
292
+ if (existing.accountId !== accountId) {
293
+ res.status(403).json({ success: false, error: { type: 'forbidden', message: 'Access denied.' }, requestId: req.requestId });
294
+ return;
295
+ }
296
+ await manager.delete(watchId);
297
+ res.json({ ok: true, deleted: watchId });
298
+ }
299
+ catch (err) {
300
+ console.error('[watch] delete error:', err);
301
+ res.status(500).json({
302
+ success: false,
303
+ error: { type: 'internal_error', message: 'Failed to delete watch.' },
304
+ requestId: req.requestId,
305
+ });
306
+ }
307
+ });
308
+ return router;
309
+ }
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Webhook delivery with HMAC-SHA256 signing
3
+ * // AUTH: Not a route handler — this is an outbound utility function (HMAC-signed payloads)
4
+ *
5
+ * Sends webhook notifications for job events with retry logic
6
+ */
7
+ import type { WebhookConfig, WebhookDeliveryResult } from '../job-queue.js';
8
+ /**
9
+ * Validate webhook URL — must be HTTPS, not localhost, not private IPs.
10
+ * Throws an error if the URL is invalid.
11
+ */
12
+ export declare function validateWebhookUrl(url: string): void;
13
+ /**
14
+ * Normalize webhook input — accept either a URL string or a WebhookConfig object.
15
+ * When a string URL is passed, defaults to subscribing to all events.
16
+ */
17
+ export declare function normalizeWebhook(webhook: string | WebhookConfig, defaultEvents?: WebhookConfig['events']): WebhookConfig;
18
+ /**
19
+ * Send a webhook notification
20
+ *
21
+ * @param webhook - Webhook configuration or URL string
22
+ * @param event - Event type (started | page | completed | failed)
23
+ * @param payload - Event payload
24
+ * @returns Delivery result with status, or null if the event was skipped
25
+ */
26
+ export declare function sendWebhook(webhook: string | WebhookConfig, event: string, payload: any): Promise<WebhookDeliveryResult | null>;
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Webhook delivery with HMAC-SHA256 signing
3
+ * // AUTH: Not a route handler — this is an outbound utility function (HMAC-signed payloads)
4
+ *
5
+ * Sends webhook notifications for job events with retry logic
6
+ */
7
+ import { createHmac } from 'crypto';
8
+ import { createLogger } from '../logger.js';
9
+ const log = createLogger('webhook');
10
+ /** Maximum payload size before truncation (1MB) */
11
+ const MAX_PAYLOAD_BYTES = 1_000_000;
12
+ /**
13
+ * Validate webhook URL — must be HTTPS, not localhost, not private IPs.
14
+ * Throws an error if the URL is invalid.
15
+ */
16
+ export function validateWebhookUrl(url) {
17
+ let parsed;
18
+ try {
19
+ parsed = new URL(url);
20
+ }
21
+ catch {
22
+ throw new Error(`Invalid webhook URL: ${url}`);
23
+ }
24
+ if (parsed.protocol !== 'https:') {
25
+ throw new Error(`Webhook URL must use HTTPS (got ${parsed.protocol})`);
26
+ }
27
+ const hostname = parsed.hostname.toLowerCase();
28
+ // Block localhost and loopback
29
+ if (hostname === 'localhost' ||
30
+ hostname === '127.0.0.1' ||
31
+ hostname === '::1' ||
32
+ hostname === '0.0.0.0') {
33
+ throw new Error(`Webhook URL must not target localhost or loopback addresses`);
34
+ }
35
+ // Block private IP ranges
36
+ const privateRanges = [
37
+ /^10\./, // 10.0.0.0/8
38
+ /^172\.(1[6-9]|2\d|3[01])\./, // 172.16.0.0/12
39
+ /^192\.168\./, // 192.168.0.0/16
40
+ /^169\.254\./, // 169.254.0.0/16 (link-local)
41
+ /^fc[0-9a-f]{2}:/i, // IPv6 unique local
42
+ /^fe[89ab][0-9a-f]:/i, // IPv6 link-local
43
+ ];
44
+ if (privateRanges.some(re => re.test(hostname))) {
45
+ throw new Error(`Webhook URL must not target private IP ranges`);
46
+ }
47
+ }
48
+ /**
49
+ * Normalize webhook input — accept either a URL string or a WebhookConfig object.
50
+ * When a string URL is passed, defaults to subscribing to all events.
51
+ */
52
+ export function normalizeWebhook(webhook, defaultEvents = ['started', 'page', 'completed', 'failed']) {
53
+ if (typeof webhook === 'string') {
54
+ return {
55
+ url: webhook,
56
+ events: defaultEvents,
57
+ };
58
+ }
59
+ // Ensure events array exists (guard against malformed objects)
60
+ if (!Array.isArray(webhook.events)) {
61
+ return { ...webhook, events: defaultEvents };
62
+ }
63
+ return webhook;
64
+ }
65
+ /**
66
+ * Send a webhook notification
67
+ *
68
+ * @param webhook - Webhook configuration or URL string
69
+ * @param event - Event type (started | page | completed | failed)
70
+ * @param payload - Event payload
71
+ * @returns Delivery result with status, or null if the event was skipped
72
+ */
73
+ export async function sendWebhook(webhook, event, payload) {
74
+ const config = normalizeWebhook(webhook);
75
+ // Check if this event should be sent
76
+ if (!config.events.includes(event)) {
77
+ return null;
78
+ }
79
+ // Validate URL (HTTPS only, no localhost, no private IPs)
80
+ try {
81
+ validateWebhookUrl(config.url);
82
+ }
83
+ catch (err) {
84
+ log.error(`Webhook URL rejected — ${err.message}`);
85
+ return {
86
+ url: config.url,
87
+ delivered: false,
88
+ error: err.message,
89
+ };
90
+ }
91
+ const webhookPayload = {
92
+ event,
93
+ timestamp: new Date().toISOString(),
94
+ data: {
95
+ ...payload,
96
+ ...config.metadata,
97
+ },
98
+ };
99
+ let body = JSON.stringify(webhookPayload);
100
+ // Size limit: if payload > 1MB, truncate data and include a note
101
+ if (Buffer.byteLength(body, 'utf8') > MAX_PAYLOAD_BYTES) {
102
+ const summary = {
103
+ event,
104
+ timestamp: webhookPayload.timestamp,
105
+ data: {
106
+ ...config.metadata,
107
+ _truncated: true,
108
+ _reason: 'Payload exceeded 1MB limit. Use the job ID to fetch full results via the API.',
109
+ jobId: payload.jobId,
110
+ total: payload.total,
111
+ completed: payload.completed,
112
+ failed: payload.failed,
113
+ },
114
+ };
115
+ body = JSON.stringify(summary);
116
+ log.warn(`Webhook payload truncated (>1MB) for event "${event}" to ${config.url}`);
117
+ }
118
+ // Generate HMAC-SHA256 signature if secret is provided
119
+ const headers = {
120
+ 'Content-Type': 'application/json',
121
+ 'User-Agent': 'WebPeel-Webhook/1.0',
122
+ };
123
+ if (config.secret) {
124
+ const signature = createHmac('sha256', config.secret)
125
+ .update(body)
126
+ .digest('hex');
127
+ headers['X-WebPeel-Signature'] = signature;
128
+ }
129
+ // Retry logic: 3 attempts with exponential backoff (1s, 2s, 4s)
130
+ const maxRetries = 3;
131
+ let lastError = null;
132
+ const startTime = Date.now();
133
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
134
+ try {
135
+ const response = await fetch(config.url, {
136
+ method: 'POST',
137
+ headers,
138
+ body,
139
+ signal: AbortSignal.timeout(10000), // 10s timeout
140
+ });
141
+ if (response.ok) {
142
+ const elapsed = Date.now() - startTime;
143
+ log.info(`Webhook delivered to ${config.url} — status ${response.status} — ${elapsed}ms`);
144
+ return {
145
+ url: config.url,
146
+ delivered: true,
147
+ deliveredAt: new Date().toISOString(),
148
+ statusCode: response.status,
149
+ };
150
+ }
151
+ // Non-2xx response — record for retry
152
+ lastError = new Error(`Webhook returned ${response.status}`);
153
+ }
154
+ catch (error) {
155
+ lastError = error;
156
+ }
157
+ // Exponential backoff before next retry: 1s, 2s, 4s
158
+ if (attempt < maxRetries - 1) {
159
+ const delayMs = Math.pow(2, attempt) * 1000;
160
+ await new Promise(resolve => setTimeout(resolve, delayMs));
161
+ }
162
+ }
163
+ // All retries exhausted
164
+ log.error(`Webhook delivery failed to ${config.url} after ${maxRetries} attempts — ${lastError?.message}`);
165
+ return {
166
+ url: config.url,
167
+ delivered: false,
168
+ error: lastError?.message,
169
+ };
170
+ }
@@ -0,0 +1,6 @@
1
+ /**
2
+ * YouTube transcript endpoint
3
+ * GET /v1/youtube?url=<youtube_url>&language=en
4
+ */
5
+ import { Router } from 'express';
6
+ export declare function createYouTubeRouter(): Router;
@@ -0,0 +1,130 @@
1
+ /**
2
+ * YouTube transcript endpoint
3
+ * GET /v1/youtube?url=<youtube_url>&language=en
4
+ */
5
+ import { Router } from 'express';
6
+ import crypto from 'crypto';
7
+ import { getYouTubeTranscript, parseYouTubeUrl } from '../../core/youtube.js';
8
+ export function createYouTubeRouter() {
9
+ const router = Router();
10
+ /**
11
+ * GET /v1/youtube
12
+ * Extract transcript and metadata from a YouTube video.
13
+ *
14
+ * Query params:
15
+ * url - YouTube video URL (required)
16
+ * language - Preferred language code, default "en" (optional)
17
+ *
18
+ * Example:
19
+ * curl "https://api.webpeel.dev/v1/youtube?url=https://youtu.be/dQw4w9WgXcQ"
20
+ */
21
+ router.get('/v1/youtube', async (req, res) => {
22
+ // AUTH: require authentication (global middleware sets req.auth)
23
+ const ytAuthId = req.auth?.keyInfo?.accountId || req.user?.userId;
24
+ if (!ytAuthId) {
25
+ res.status(401).json({
26
+ success: false,
27
+ error: {
28
+ type: 'authentication_required',
29
+ message: 'API key required. Get one at https://app.webpeel.dev/keys',
30
+ hint: 'Pass your API key in the Authorization header: Bearer <key>',
31
+ docs: 'https://webpeel.dev/docs/errors#authentication-required',
32
+ },
33
+ requestId: req.requestId || crypto.randomUUID(),
34
+ });
35
+ return;
36
+ }
37
+ const { url, language } = req.query;
38
+ if (!url || typeof url !== 'string') {
39
+ res.status(400).json({
40
+ success: false,
41
+ error: {
42
+ type: 'invalid_request',
43
+ message: 'Missing or invalid "url" parameter. Pass a YouTube URL: GET /v1/youtube?url=https://youtu.be/VIDEO_ID',
44
+ hint: 'Example: curl "https://api.webpeel.dev/v1/youtube?url=https://youtu.be/dQw4w9WgXcQ"',
45
+ docs: 'https://webpeel.dev/docs/errors#invalid-request',
46
+ },
47
+ requestId: req.requestId || crypto.randomUUID(),
48
+ });
49
+ return;
50
+ }
51
+ const videoId = parseYouTubeUrl(url);
52
+ if (!videoId) {
53
+ res.status(400).json({
54
+ success: false,
55
+ error: {
56
+ type: 'invalid_youtube_url',
57
+ message: 'The provided URL is not a valid YouTube video URL.',
58
+ hint: 'Supported formats: https://www.youtube.com/watch?v=VIDEO_ID, https://youtu.be/VIDEO_ID',
59
+ docs: 'https://webpeel.dev/docs/errors#invalid-youtube-url',
60
+ },
61
+ supported: [
62
+ 'https://www.youtube.com/watch?v=VIDEO_ID',
63
+ 'https://youtu.be/VIDEO_ID',
64
+ 'https://www.youtube.com/embed/VIDEO_ID',
65
+ 'https://m.youtube.com/watch?v=VIDEO_ID',
66
+ ],
67
+ requestId: req.requestId || crypto.randomUUID(),
68
+ });
69
+ return;
70
+ }
71
+ try {
72
+ const lang = typeof language === 'string' ? language : 'en';
73
+ const transcript = await getYouTubeTranscript(url, { language: lang });
74
+ res.json({
75
+ success: true,
76
+ videoId: transcript.videoId,
77
+ title: transcript.title,
78
+ channel: transcript.channel,
79
+ duration: transcript.duration,
80
+ language: transcript.language,
81
+ availableLanguages: transcript.availableLanguages,
82
+ fullText: transcript.fullText,
83
+ segments: transcript.segments,
84
+ url: `https://www.youtube.com/watch?v=${videoId}`,
85
+ });
86
+ }
87
+ catch (error) {
88
+ const message = error?.message ?? 'Failed to extract YouTube transcript';
89
+ if (message.includes('No captions available')) {
90
+ res.status(404).json({
91
+ success: false,
92
+ error: {
93
+ type: 'no_captions',
94
+ message: `No captions are available for this video. The video may not have subtitles.`,
95
+ hint: 'Try a different video or check if captions are enabled on YouTube.',
96
+ docs: 'https://webpeel.dev/docs/errors#no-captions',
97
+ },
98
+ videoId,
99
+ requestId: req.requestId || crypto.randomUUID(),
100
+ });
101
+ return;
102
+ }
103
+ if (message.includes('Not a valid YouTube URL')) {
104
+ res.status(400).json({
105
+ success: false,
106
+ error: {
107
+ type: 'invalid_youtube_url',
108
+ message,
109
+ hint: 'Supported formats: https://www.youtube.com/watch?v=VIDEO_ID, https://youtu.be/VIDEO_ID',
110
+ docs: 'https://webpeel.dev/docs/errors#invalid-youtube-url',
111
+ },
112
+ requestId: req.requestId || crypto.randomUUID(),
113
+ });
114
+ return;
115
+ }
116
+ console.error('[youtube route] Error:', error);
117
+ res.status(500).json({
118
+ success: false,
119
+ error: {
120
+ type: 'extraction_failed',
121
+ message: 'Failed to extract YouTube transcript. The video page may have changed or the video is unavailable.',
122
+ hint: process.env.NODE_ENV !== 'production' ? message : undefined,
123
+ docs: 'https://webpeel.dev/docs/errors#extraction-failed',
124
+ },
125
+ requestId: req.requestId || crypto.randomUUID(),
126
+ });
127
+ }
128
+ });
129
+ return router;
130
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Optional Sentry integration for the API server.
3
+ *
4
+ * Enabled only when SENTRY_DSN is set.
5
+ * This keeps local/self-hosted setups dependency-light by default.
6
+ */
7
+ import type { ErrorRequestHandler, RequestHandler } from 'express';
8
+ export interface SentryHooks {
9
+ enabled: boolean;
10
+ requestHandler?: RequestHandler;
11
+ errorHandler?: ErrorRequestHandler;
12
+ }
13
+ export declare function createSentryHooks(): SentryHooks;