webpeel 0.20.2 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.js +384 -0
- package/dist/server/auth-store.d.ts +27 -0
- package/dist/server/auth-store.js +88 -0
- package/dist/server/email-service.d.ts +21 -0
- package/dist/server/email-service.js +79 -0
- package/dist/server/job-queue.d.ts +100 -0
- package/dist/server/job-queue.js +145 -0
- package/dist/server/logger.d.ts +10 -0
- package/dist/server/logger.js +37 -0
- package/dist/server/middleware/auth.d.ts +28 -0
- package/dist/server/middleware/auth.js +221 -0
- package/dist/server/middleware/rate-limit.d.ts +24 -0
- package/dist/server/middleware/rate-limit.js +167 -0
- package/dist/server/middleware/url-validator.d.ts +15 -0
- package/dist/server/middleware/url-validator.js +186 -0
- package/dist/server/openapi.yaml +6418 -0
- package/dist/server/pg-auth-store.d.ts +132 -0
- package/dist/server/pg-auth-store.js +472 -0
- package/dist/server/pg-job-queue.d.ts +59 -0
- package/dist/server/pg-job-queue.js +375 -0
- package/dist/server/premium/domain-intel.d.ts +16 -0
- package/dist/server/premium/domain-intel.js +133 -0
- package/dist/server/premium/index.d.ts +17 -0
- package/dist/server/premium/index.js +35 -0
- package/dist/server/premium/swr-cache.d.ts +14 -0
- package/dist/server/premium/swr-cache.js +34 -0
- package/dist/server/routes/activity.d.ts +6 -0
- package/dist/server/routes/activity.js +74 -0
- package/dist/server/routes/answer.d.ts +5 -0
- package/dist/server/routes/answer.js +125 -0
- package/dist/server/routes/ask.d.ts +28 -0
- package/dist/server/routes/ask.js +229 -0
- package/dist/server/routes/batch.d.ts +6 -0
- package/dist/server/routes/batch.js +493 -0
- package/dist/server/routes/cli-usage.d.ts +6 -0
- package/dist/server/routes/cli-usage.js +127 -0
- package/dist/server/routes/compat.d.ts +23 -0
- package/dist/server/routes/compat.js +652 -0
- package/dist/server/routes/deep-fetch.d.ts +8 -0
- package/dist/server/routes/deep-fetch.js +57 -0
- package/dist/server/routes/demo.d.ts +24 -0
- package/dist/server/routes/demo.js +517 -0
- package/dist/server/routes/do.d.ts +8 -0
- package/dist/server/routes/do.js +72 -0
- package/dist/server/routes/extract.d.ts +8 -0
- package/dist/server/routes/extract.js +235 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.js +999 -0
- package/dist/server/routes/health.d.ts +7 -0
- package/dist/server/routes/health.js +19 -0
- package/dist/server/routes/jobs.d.ts +7 -0
- package/dist/server/routes/jobs.js +573 -0
- package/dist/server/routes/mcp.d.ts +14 -0
- package/dist/server/routes/mcp.js +141 -0
- package/dist/server/routes/oauth.d.ts +9 -0
- package/dist/server/routes/oauth.js +396 -0
- package/dist/server/routes/playground.d.ts +17 -0
- package/dist/server/routes/playground.js +283 -0
- package/dist/server/routes/screenshot.d.ts +22 -0
- package/dist/server/routes/screenshot.js +816 -0
- package/dist/server/routes/search.d.ts +6 -0
- package/dist/server/routes/search.js +303 -0
- package/dist/server/routes/session.d.ts +15 -0
- package/dist/server/routes/session.js +397 -0
- package/dist/server/routes/stats.d.ts +6 -0
- package/dist/server/routes/stats.js +71 -0
- package/dist/server/routes/stripe.d.ts +15 -0
- package/dist/server/routes/stripe.js +294 -0
- package/dist/server/routes/users.d.ts +8 -0
- package/dist/server/routes/users.js +1671 -0
- package/dist/server/routes/watch.d.ts +15 -0
- package/dist/server/routes/watch.js +309 -0
- package/dist/server/routes/webhooks.d.ts +26 -0
- package/dist/server/routes/webhooks.js +170 -0
- package/dist/server/routes/youtube.d.ts +6 -0
- package/dist/server/routes/youtube.js +130 -0
- package/dist/server/sentry.d.ts +13 -0
- package/dist/server/sentry.js +38 -0
- package/dist/server/types.d.ts +15 -0
- package/dist/server/types.js +7 -0
- package/dist/server/utils/response.d.ts +44 -0
- package/dist/server/utils/response.js +69 -0
- package/dist/server/utils/sse.d.ts +22 -0
- package/dist/server/utils/sse.js +38 -0
- package/package.json +2 -1
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebPeel API Server
|
|
3
|
+
* Express-based REST API for hosted deployments
|
|
4
|
+
*/
|
|
5
|
+
import { Express } from 'express';
|
|
6
|
+
import './types.js';
|
|
7
|
+
export interface ServerConfig {
|
|
8
|
+
port?: number;
|
|
9
|
+
corsOrigins?: string[];
|
|
10
|
+
rateLimitWindowMs?: number;
|
|
11
|
+
usePostgres?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export declare function createApp(config?: ServerConfig): Express;
|
|
14
|
+
export declare function startServer(config?: ServerConfig): void;
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebPeel API Server
|
|
3
|
+
* Express-based REST API for hosted deployments
|
|
4
|
+
*/
|
|
5
|
+
// Force IPv4-first DNS resolution to prevent IPv6 failures in containers
|
|
6
|
+
// (Render's Docker containers can't do IPv6 outbound, causing IANA/Cloudflare sites to fail)
|
|
7
|
+
import dns from 'dns';
|
|
8
|
+
dns.setDefaultResultOrder('ipv4first');
|
|
9
|
+
import express from 'express';
|
|
10
|
+
import './types.js'; // Augments Express.Request with requestId
|
|
11
|
+
import cors from 'cors';
|
|
12
|
+
import { createLogger } from './logger.js';
|
|
13
|
+
const log = createLogger('server');
|
|
14
|
+
import { InMemoryAuthStore } from './auth-store.js';
|
|
15
|
+
import { PostgresAuthStore } from './pg-auth-store.js';
|
|
16
|
+
import { createAuthMiddleware } from './middleware/auth.js';
|
|
17
|
+
import { createRateLimitMiddleware, RateLimiter } from './middleware/rate-limit.js';
|
|
18
|
+
import { createHealthRouter } from './routes/health.js';
|
|
19
|
+
import { createFetchRouter } from './routes/fetch.js';
|
|
20
|
+
import { createSearchRouter } from './routes/search.js';
|
|
21
|
+
import { createUserRouter } from './routes/users.js';
|
|
22
|
+
import { createStripeRouter, createBillingPortalRouter } from './routes/stripe.js';
|
|
23
|
+
import { createOAuthRouter } from './routes/oauth.js';
|
|
24
|
+
import { createStatsRouter } from './routes/stats.js';
|
|
25
|
+
import { createActivityRouter } from './routes/activity.js';
|
|
26
|
+
import { createCLIUsageRouter } from './routes/cli-usage.js';
|
|
27
|
+
import { createJobsRouter } from './routes/jobs.js';
|
|
28
|
+
import { createBatchRouter } from './routes/batch.js';
|
|
29
|
+
import { createAnswerRouter } from './routes/answer.js';
|
|
30
|
+
import { createAskRouter } from './routes/ask.js';
|
|
31
|
+
import { createMcpRouter } from './routes/mcp.js';
|
|
32
|
+
import { createDoRouter } from './routes/do.js';
|
|
33
|
+
import { createYouTubeRouter } from './routes/youtube.js';
|
|
34
|
+
import { createDeepFetchRouter } from './routes/deep-fetch.js';
|
|
35
|
+
import { createWatchRouter } from './routes/watch.js';
|
|
36
|
+
import pg from 'pg';
|
|
37
|
+
import { createScreenshotRouter } from './routes/screenshot.js';
|
|
38
|
+
import { createDemoRouter } from './routes/demo.js';
|
|
39
|
+
import { createPlaygroundRouter } from './routes/playground.js';
|
|
40
|
+
import { createJobQueue } from './job-queue.js';
|
|
41
|
+
import { createCompatRouter } from './routes/compat.js';
|
|
42
|
+
import { createExtractRouter } from './routes/extract.js';
|
|
43
|
+
import { createSessionRouter } from './routes/session.js';
|
|
44
|
+
import { createSentryHooks } from './sentry.js';
|
|
45
|
+
import { warmup, cleanup as cleanupFetcher } from '../core/fetcher.js';
|
|
46
|
+
import { registerPremiumHooks } from './premium/index.js';
|
|
47
|
+
import { readFileSync } from 'fs';
|
|
48
|
+
import { join, dirname } from 'path';
|
|
49
|
+
import { fileURLToPath } from 'url';
|
|
50
|
+
// Resolve path to the OpenAPI spec (works from both src/ and dist/)
|
|
51
|
+
const __dirname_app = dirname(fileURLToPath(import.meta.url));
|
|
52
|
+
let _openApiYaml = null;
|
|
53
|
+
function getOpenApiYaml() {
|
|
54
|
+
if (_openApiYaml !== null)
|
|
55
|
+
return _openApiYaml;
|
|
56
|
+
try {
|
|
57
|
+
// Try src/server/openapi.yaml relative to compiled dist/server/
|
|
58
|
+
const candidates = [
|
|
59
|
+
join(__dirname_app, 'openapi.yaml'),
|
|
60
|
+
join(__dirname_app, '..', '..', 'src', 'server', 'openapi.yaml'),
|
|
61
|
+
];
|
|
62
|
+
for (const candidate of candidates) {
|
|
63
|
+
try {
|
|
64
|
+
_openApiYaml = readFileSync(candidate, 'utf-8');
|
|
65
|
+
return _openApiYaml;
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// try next
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
throw new Error('openapi.yaml not found');
|
|
72
|
+
}
|
|
73
|
+
catch (e) {
|
|
74
|
+
return '# openapi.yaml not found\n';
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
export function createApp(config = {}) {
|
|
78
|
+
const app = express();
|
|
79
|
+
// SECURITY: Trust proxy for Render/production (HTTPS only)
|
|
80
|
+
app.set('trust proxy', 1);
|
|
81
|
+
// ─── Request ID ─────────────────────────────────────────────────────────────
|
|
82
|
+
// Generate a UUID v4 for every request so errors and logs are traceable.
|
|
83
|
+
// Must run before all other middleware so req.requestId is always set.
|
|
84
|
+
app.use((req, res, next) => {
|
|
85
|
+
req.requestId = crypto.randomUUID();
|
|
86
|
+
res.setHeader('X-Request-Id', req.requestId);
|
|
87
|
+
next();
|
|
88
|
+
});
|
|
89
|
+
// Hard server-side timeouts — no request runs longer than this
|
|
90
|
+
app.use((req, res, next) => {
|
|
91
|
+
const path = req.path;
|
|
92
|
+
let timeoutMs = 30000; // 30s default
|
|
93
|
+
if (path.includes('/crawl') || path.includes('/map'))
|
|
94
|
+
timeoutMs = 300000; // 5min for crawls
|
|
95
|
+
else if (path.includes('/batch'))
|
|
96
|
+
timeoutMs = 120000; // 2min for batch
|
|
97
|
+
else if (path.includes('/screenshot'))
|
|
98
|
+
timeoutMs = 60000; // 1min for screenshots
|
|
99
|
+
else if (req.query?.render === 'true')
|
|
100
|
+
timeoutMs = 60000; // 1min for rendered fetches
|
|
101
|
+
req.setTimeout(timeoutMs);
|
|
102
|
+
res.setTimeout(timeoutMs, () => {
|
|
103
|
+
if (!res.headersSent) {
|
|
104
|
+
res.status(504).json({
|
|
105
|
+
success: false,
|
|
106
|
+
error: {
|
|
107
|
+
type: 'timeout',
|
|
108
|
+
message: `Request timed out after ${timeoutMs / 1000}s`,
|
|
109
|
+
hint: 'Try reducing the scope of your request or upgrading your plan for higher limits.',
|
|
110
|
+
docs: 'https://webpeel.dev/docs/errors#timeout',
|
|
111
|
+
},
|
|
112
|
+
metadata: { requestId: req.requestId },
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
next();
|
|
117
|
+
});
|
|
118
|
+
// Optional error tracking (enabled only when SENTRY_DSN is set)
|
|
119
|
+
const sentry = createSentryHooks();
|
|
120
|
+
if (sentry.requestHandler) {
|
|
121
|
+
app.use(sentry.requestHandler);
|
|
122
|
+
}
|
|
123
|
+
// Stripe webhook route MUST come before express.json() to get raw body
|
|
124
|
+
const stripeRouter = createStripeRouter();
|
|
125
|
+
app.use('/v1/webhooks/stripe', express.raw({ type: 'application/json' }), stripeRouter);
|
|
126
|
+
// Middleware
|
|
127
|
+
// SECURITY: Limit request body size to prevent DoS
|
|
128
|
+
app.use(express.json({ limit: '1mb' }));
|
|
129
|
+
// CORS configuration
|
|
130
|
+
// Always allow our own domains + any env-configured origins
|
|
131
|
+
const envOrigins = process.env.CORS_ORIGINS ? process.env.CORS_ORIGINS.split(',').map(s => s.trim()) : [];
|
|
132
|
+
const defaultOrigins = [
|
|
133
|
+
'https://app.webpeel.dev',
|
|
134
|
+
'https://webpeel.dev',
|
|
135
|
+
// Only allow localhost in development (security: prevents credentialed cross-origin from local pages)
|
|
136
|
+
...(process.env.NODE_ENV !== 'production' ? ['http://localhost:3000', 'http://localhost:3001'] : []),
|
|
137
|
+
];
|
|
138
|
+
const corsOrigins = config.corsOrigins || [...new Set([...defaultOrigins, ...envOrigins])];
|
|
139
|
+
app.use(cors({
|
|
140
|
+
origin: (origin, callback) => {
|
|
141
|
+
// Allow requests with no origin (e.g. curl, server-to-server)
|
|
142
|
+
if (!origin)
|
|
143
|
+
return callback(null, true);
|
|
144
|
+
if (corsOrigins.includes(origin))
|
|
145
|
+
return callback(null, origin);
|
|
146
|
+
// Unknown origins: allow (API key clients need cross-origin access) but no credentials
|
|
147
|
+
return callback(null, true);
|
|
148
|
+
},
|
|
149
|
+
// credentials: set conditionally via post-cors middleware below
|
|
150
|
+
credentials: false,
|
|
151
|
+
}));
|
|
152
|
+
// Set Access-Control-Allow-Credentials only for trusted origins
|
|
153
|
+
app.use((req, res, next) => {
|
|
154
|
+
const origin = req.headers.origin;
|
|
155
|
+
if (origin && corsOrigins.includes(origin)) {
|
|
156
|
+
res.setHeader('Access-Control-Allow-Credentials', 'true');
|
|
157
|
+
}
|
|
158
|
+
next();
|
|
159
|
+
});
|
|
160
|
+
// SECURITY: Security headers
|
|
161
|
+
app.disable('x-powered-by');
|
|
162
|
+
app.use((_req, res, next) => {
|
|
163
|
+
res.setHeader('X-Content-Type-Options', 'nosniff');
|
|
164
|
+
res.setHeader('X-Frame-Options', 'DENY');
|
|
165
|
+
res.setHeader('Strict-Transport-Security', 'max-age=63072000; includeSubDomains; preload');
|
|
166
|
+
res.setHeader('X-XSS-Protection', '1; mode=block');
|
|
167
|
+
res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin');
|
|
168
|
+
res.setHeader('Permissions-Policy', 'geolocation=(), microphone=(), camera=()');
|
|
169
|
+
// API-safe CSP: JSON-only API does not need scripts/styles/fonts.
|
|
170
|
+
// Keep this strict to reduce attack surface without affecting API clients.
|
|
171
|
+
res.setHeader('Content-Security-Policy', "default-src 'none'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'");
|
|
172
|
+
// Best-effort removal of Render's origin header (may be re-added by proxy)
|
|
173
|
+
res.removeHeader('x-render-origin-server');
|
|
174
|
+
next();
|
|
175
|
+
});
|
|
176
|
+
// SECURITY: JSON parse error handler
|
|
177
|
+
app.use((err, req, res, next) => {
|
|
178
|
+
if (err instanceof SyntaxError && 'body' in err) {
|
|
179
|
+
res.status(400).json({
|
|
180
|
+
success: false,
|
|
181
|
+
error: {
|
|
182
|
+
type: 'invalid_request',
|
|
183
|
+
message: 'Malformed JSON in request body',
|
|
184
|
+
hint: 'Ensure the request body is valid JSON',
|
|
185
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
186
|
+
},
|
|
187
|
+
requestId: req.requestId,
|
|
188
|
+
});
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
next(err);
|
|
192
|
+
});
|
|
193
|
+
// Auth store - Use PostgreSQL if DATABASE_URL is set, otherwise in-memory
|
|
194
|
+
const usePostgres = config.usePostgres ?? !!process.env.DATABASE_URL;
|
|
195
|
+
const authStore = usePostgres
|
|
196
|
+
? new PostgresAuthStore()
|
|
197
|
+
: new InMemoryAuthStore();
|
|
198
|
+
log.info(`Using ${usePostgres ? 'PostgreSQL' : 'in-memory'} auth store`);
|
|
199
|
+
// PostgreSQL pool for features that need direct DB access (watch, etc.)
|
|
200
|
+
const pool = process.env.DATABASE_URL
|
|
201
|
+
? new pg.Pool({
|
|
202
|
+
connectionString: process.env.DATABASE_URL,
|
|
203
|
+
ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: true } : false,
|
|
204
|
+
})
|
|
205
|
+
: null;
|
|
206
|
+
// Job queue - Use PostgreSQL if DATABASE_URL is set, otherwise in-memory
|
|
207
|
+
const jobQueue = createJobQueue();
|
|
208
|
+
// Rate limiter
|
|
209
|
+
const rateLimiter = new RateLimiter(config.rateLimitWindowMs || 3_600_000); // 1 hour
|
|
210
|
+
// Clean up rate limiter every 5 minutes
|
|
211
|
+
setInterval(() => {
|
|
212
|
+
rateLimiter.cleanup();
|
|
213
|
+
}, 5 * 60 * 1000);
|
|
214
|
+
// Health check MUST be before auth/rate-limit middleware
|
|
215
|
+
// Render hits /health every ~30s; rate-limiting it causes 429 → service marked as failed
|
|
216
|
+
app.use(createHealthRouter());
|
|
217
|
+
// OpenAPI spec — public, no auth required
|
|
218
|
+
app.get('/openapi.yaml', (_req, res) => {
|
|
219
|
+
res.setHeader('Content-Type', 'application/yaml; charset=utf-8');
|
|
220
|
+
res.setHeader('Cache-Control', 'public, max-age=3600');
|
|
221
|
+
res.send(getOpenApiYaml());
|
|
222
|
+
});
|
|
223
|
+
// Redirect /openapi.json to YAML spec (no extra dependency needed)
|
|
224
|
+
app.get('/openapi.json', (_req, res) => {
|
|
225
|
+
res.redirect(301, '/openapi.yaml');
|
|
226
|
+
});
|
|
227
|
+
// Developer-friendly redirect
|
|
228
|
+
app.get('/docs/api', (_req, res) => {
|
|
229
|
+
res.redirect('/openapi.yaml');
|
|
230
|
+
});
|
|
231
|
+
// Demo endpoint — unauthenticated, must be before auth middleware
|
|
232
|
+
app.use(createDemoRouter());
|
|
233
|
+
// Playground endpoint — unauthenticated, CORS-locked to webpeel.dev/localhost
|
|
234
|
+
app.use('/v1/playground', createPlaygroundRouter());
|
|
235
|
+
// Apply auth middleware globally
|
|
236
|
+
app.use(createAuthMiddleware(authStore));
|
|
237
|
+
// Apply rate limiting middleware globally
|
|
238
|
+
app.use(createRateLimitMiddleware(rateLimiter));
|
|
239
|
+
app.use(createCompatRouter(jobQueue));
|
|
240
|
+
app.use(createSessionRouter());
|
|
241
|
+
app.use(createExtractRouter());
|
|
242
|
+
app.use(createDeepFetchRouter());
|
|
243
|
+
if (pool) {
|
|
244
|
+
app.use(createWatchRouter(pool));
|
|
245
|
+
}
|
|
246
|
+
app.use(createFetchRouter(authStore));
|
|
247
|
+
app.use(createScreenshotRouter(authStore));
|
|
248
|
+
app.use(createSearchRouter(authStore));
|
|
249
|
+
app.use(createBillingPortalRouter(pool));
|
|
250
|
+
app.use(createUserRouter());
|
|
251
|
+
app.use(createOAuthRouter());
|
|
252
|
+
app.use(createStatsRouter(authStore));
|
|
253
|
+
app.use(createActivityRouter(authStore));
|
|
254
|
+
app.use(createCLIUsageRouter());
|
|
255
|
+
app.use(createJobsRouter(jobQueue, authStore));
|
|
256
|
+
app.use(createBatchRouter(jobQueue));
|
|
257
|
+
// Deprecation headers for declining endpoints
|
|
258
|
+
app.use('/v1/answer', (_req, res, next) => {
|
|
259
|
+
res.set('Deprecation', 'true');
|
|
260
|
+
res.set('Sunset', '2026-06-01');
|
|
261
|
+
res.set('Link', '</v1/ask>; rel="successor-version"');
|
|
262
|
+
next();
|
|
263
|
+
});
|
|
264
|
+
app.use(createAnswerRouter());
|
|
265
|
+
app.use(createAskRouter());
|
|
266
|
+
app.use('/v1/do', createDoRouter());
|
|
267
|
+
app.use(createYouTubeRouter());
|
|
268
|
+
app.use(createMcpRouter(authStore, pool));
|
|
269
|
+
// 404 handler
|
|
270
|
+
app.use((req, res) => {
|
|
271
|
+
res.status(404).json({
|
|
272
|
+
success: false,
|
|
273
|
+
error: {
|
|
274
|
+
type: 'not_found',
|
|
275
|
+
message: 'Not found',
|
|
276
|
+
docs: 'https://webpeel.dev/docs/api-reference',
|
|
277
|
+
},
|
|
278
|
+
requestId: req.requestId,
|
|
279
|
+
});
|
|
280
|
+
});
|
|
281
|
+
// Sentry error middleware should run before the generic error handler.
|
|
282
|
+
if (sentry.errorHandler) {
|
|
283
|
+
app.use(sentry.errorHandler);
|
|
284
|
+
}
|
|
285
|
+
// Global error response normalizer — ensures ALL errors use the same structured shape.
|
|
286
|
+
// Catches errors thrown via next(err) that may have a flat format {error: string, message: string}.
|
|
287
|
+
// Must run before the generic error handler below.
|
|
288
|
+
app.use((err, req, res, next) => {
|
|
289
|
+
// Skip if error is already in structured format (has error.type or error.message as object)
|
|
290
|
+
if (err && typeof err.error === 'object' && err.error !== null) {
|
|
291
|
+
return next(err);
|
|
292
|
+
}
|
|
293
|
+
// Skip standard Error objects (handled by the generic error handler with Playwright sanitization)
|
|
294
|
+
if (err instanceof Error && !err.hasOwnProperty('statusCode') && !err.hasOwnProperty('status')) {
|
|
295
|
+
return next(err);
|
|
296
|
+
}
|
|
297
|
+
const statusCode = (err && (err.statusCode || err.status)) || 500;
|
|
298
|
+
if (res.headersSent)
|
|
299
|
+
return next(err);
|
|
300
|
+
const requestId = req.requestId || req.headers['x-request-id'] || crypto.randomUUID();
|
|
301
|
+
res.status(statusCode).json({
|
|
302
|
+
success: false,
|
|
303
|
+
error: {
|
|
304
|
+
type: (err && (err.type || err.error)) || 'server_error',
|
|
305
|
+
message: (err && err.message) || 'An unexpected error occurred',
|
|
306
|
+
...((err && err.hint) ? { hint: err.hint } : {}),
|
|
307
|
+
...((err && err.docs) ? { docs: err.docs } : {}),
|
|
308
|
+
},
|
|
309
|
+
requestId,
|
|
310
|
+
});
|
|
311
|
+
});
|
|
312
|
+
// Error handler - SECURITY: sanitize errors in production to prevent leaking
|
|
313
|
+
// Playwright stack traces, internal paths, or other sensitive details.
|
|
314
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
315
|
+
app.use((err, req, res, _next) => {
|
|
316
|
+
log.error('Unhandled error', { message: err.message, stack: err.stack }); // Log full error server-side
|
|
317
|
+
if (res.headersSent)
|
|
318
|
+
return; // Avoid double-send crash
|
|
319
|
+
if (process.env.NODE_ENV === 'production') {
|
|
320
|
+
// Strip Playwright/browser launch errors and stack traces from responses
|
|
321
|
+
const sanitized = (err.message || 'An unexpected error occurred')
|
|
322
|
+
.replace(/browserType\.launch:.*$/s, 'Browser rendering unavailable on this server. Use the CLI with --render for browser-rendered content.')
|
|
323
|
+
.replace(/at\s+\S.*\n?/g, '') // strip "at <location>" stack lines
|
|
324
|
+
.trim() || 'An unexpected error occurred';
|
|
325
|
+
res.status(500).json({
|
|
326
|
+
success: false,
|
|
327
|
+
error: {
|
|
328
|
+
type: 'internal_error',
|
|
329
|
+
message: sanitized,
|
|
330
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
331
|
+
},
|
|
332
|
+
requestId: req.requestId,
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
res.status(500).json({
|
|
337
|
+
success: false,
|
|
338
|
+
error: {
|
|
339
|
+
type: 'internal_error',
|
|
340
|
+
message: err.message || 'An unexpected error occurred',
|
|
341
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
342
|
+
},
|
|
343
|
+
requestId: req.requestId,
|
|
344
|
+
stack: err.stack,
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
});
|
|
348
|
+
return app;
|
|
349
|
+
}
|
|
350
|
+
export function startServer(config = {}) {
|
|
351
|
+
const app = createApp(config);
|
|
352
|
+
const port = config.port || parseInt(process.env.PORT || '3000', 10);
|
|
353
|
+
// Activate premium strategy hooks (SWR cache, domain intelligence, race).
|
|
354
|
+
registerPremiumHooks();
|
|
355
|
+
// Pre-warm browser resources in the background to reduce first-request latency.
|
|
356
|
+
void warmup().catch((error) => {
|
|
357
|
+
log.warn('Browser warmup failed', { error: error instanceof Error ? error.message : String(error) });
|
|
358
|
+
});
|
|
359
|
+
const server = app.listen(port, () => {
|
|
360
|
+
log.info(`WebPeel API server listening on port ${port}`);
|
|
361
|
+
log.info(`Health: http://localhost:${port}/health Fetch: /v1/fetch Search: /v1/search`);
|
|
362
|
+
});
|
|
363
|
+
// Graceful shutdown
|
|
364
|
+
const shutdown = () => {
|
|
365
|
+
log.info('Shutting down gracefully...');
|
|
366
|
+
server.close(() => {
|
|
367
|
+
log.info('Server closed');
|
|
368
|
+
void cleanupFetcher().finally(() => {
|
|
369
|
+
process.exit(0);
|
|
370
|
+
});
|
|
371
|
+
});
|
|
372
|
+
// Force shutdown after 10 seconds
|
|
373
|
+
setTimeout(() => {
|
|
374
|
+
log.error('Forced shutdown after timeout');
|
|
375
|
+
process.exit(1);
|
|
376
|
+
}, 10000);
|
|
377
|
+
};
|
|
378
|
+
process.on('SIGTERM', shutdown);
|
|
379
|
+
process.on('SIGINT', shutdown);
|
|
380
|
+
}
|
|
381
|
+
// Start server if run directly
|
|
382
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
383
|
+
startServer();
|
|
384
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auth store abstraction for API key validation and usage tracking
|
|
3
|
+
* Designed to easily swap from in-memory to PostgreSQL
|
|
4
|
+
*/
|
|
5
|
+
export interface ApiKeyInfo {
|
|
6
|
+
key: string;
|
|
7
|
+
tier: 'free' | 'starter' | 'pro' | 'enterprise' | 'max';
|
|
8
|
+
rateLimit: number;
|
|
9
|
+
accountId?: string;
|
|
10
|
+
createdAt: Date;
|
|
11
|
+
}
|
|
12
|
+
export interface AuthStore {
|
|
13
|
+
validateKey(key: string): Promise<ApiKeyInfo | null>;
|
|
14
|
+
trackUsage(key: string, creditsOrType: number | 'basic' | 'stealth' | 'captcha' | 'search'): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* In-memory auth store for development and self-hosted deployments
|
|
18
|
+
*/
|
|
19
|
+
export declare class InMemoryAuthStore implements AuthStore {
|
|
20
|
+
private keys;
|
|
21
|
+
private usage;
|
|
22
|
+
constructor();
|
|
23
|
+
validateKey(key: string): Promise<ApiKeyInfo | null>;
|
|
24
|
+
trackUsage(key: string, creditsOrType: number | 'basic' | 'stealth' | 'captcha' | 'search'): Promise<void>;
|
|
25
|
+
addKey(keyInfo: ApiKeyInfo): void;
|
|
26
|
+
getUsage(key: string): number;
|
|
27
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Auth store abstraction for API key validation and usage tracking
|
|
3
|
+
* Designed to easily swap from in-memory to PostgreSQL
|
|
4
|
+
*/
|
|
5
|
+
import { timingSafeEqual } from 'crypto';
|
|
6
|
+
/**
|
|
7
|
+
* Validate API key format and strength
|
|
8
|
+
* SECURITY: Enforce minimum complexity
|
|
9
|
+
*/
|
|
10
|
+
function validateKeyFormat(key) {
|
|
11
|
+
// Minimum 32 characters
|
|
12
|
+
if (key.length < 32) {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
// Must contain alphanumeric characters
|
|
16
|
+
if (!/^[a-zA-Z0-9_-]+$/.test(key)) {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Timing-safe key comparison
|
|
23
|
+
* SECURITY: Prevent timing attacks on key validation
|
|
24
|
+
*/
|
|
25
|
+
function timingSafeKeyCompare(a, b) {
|
|
26
|
+
// Ensure equal length for comparison
|
|
27
|
+
if (a.length !== b.length) {
|
|
28
|
+
// Compare against dummy to prevent timing leak
|
|
29
|
+
const dummy = 'x'.repeat(Math.max(a.length, b.length));
|
|
30
|
+
timingSafeEqual(Buffer.from(dummy), Buffer.from(dummy));
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
return timingSafeEqual(Buffer.from(a), Buffer.from(b));
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* In-memory auth store for development and self-hosted deployments
|
|
42
|
+
*/
|
|
43
|
+
export class InMemoryAuthStore {
|
|
44
|
+
keys = new Map();
|
|
45
|
+
usage = new Map();
|
|
46
|
+
constructor() {
|
|
47
|
+
// SECURITY: Demo key only in development mode
|
|
48
|
+
// Removed hardcoded demo key - use addKey() or environment variables
|
|
49
|
+
if (process.env.NODE_ENV === 'development' && process.env.DEMO_KEY) {
|
|
50
|
+
this.addKey({
|
|
51
|
+
key: process.env.DEMO_KEY,
|
|
52
|
+
tier: 'pro',
|
|
53
|
+
rateLimit: 300,
|
|
54
|
+
createdAt: new Date(),
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
async validateKey(key) {
|
|
59
|
+
// Basic validation
|
|
60
|
+
if (!key || typeof key !== 'string') {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
// SECURITY: Timing-safe comparison to prevent timing attacks
|
|
64
|
+
for (const [storedKey, keyInfo] of this.keys.entries()) {
|
|
65
|
+
if (timingSafeKeyCompare(key, storedKey)) {
|
|
66
|
+
return keyInfo;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
// Constant-time operation for invalid key
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
async trackUsage(key, creditsOrType) {
|
|
73
|
+
// For in-memory store, just count everything as 1 credit
|
|
74
|
+
const credits = typeof creditsOrType === 'number' ? creditsOrType : 1;
|
|
75
|
+
const current = this.usage.get(key) || 0;
|
|
76
|
+
this.usage.set(key, current + credits);
|
|
77
|
+
}
|
|
78
|
+
addKey(keyInfo) {
|
|
79
|
+
// SECURITY: Validate key format before adding
|
|
80
|
+
if (!validateKeyFormat(keyInfo.key)) {
|
|
81
|
+
throw new Error('Invalid API key format: must be at least 32 characters, alphanumeric with - or _');
|
|
82
|
+
}
|
|
83
|
+
this.keys.set(keyInfo.key, keyInfo);
|
|
84
|
+
}
|
|
85
|
+
getUsage(key) {
|
|
86
|
+
return this.usage.get(key) || 0;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Email service using Nodemailer (free, no API key required).
|
|
3
|
+
* Configure via environment variables:
|
|
4
|
+
* EMAIL_SMTP_HOST - SMTP server (e.g. smtp.gmail.com)
|
|
5
|
+
* EMAIL_SMTP_PORT - SMTP port (default: 587)
|
|
6
|
+
* EMAIL_SMTP_USER - SMTP username / email address
|
|
7
|
+
* EMAIL_SMTP_PASS - SMTP password (Gmail: use App Password)
|
|
8
|
+
* EMAIL_FROM - From address (default: EMAIL_SMTP_USER)
|
|
9
|
+
*
|
|
10
|
+
* Gmail setup: https://myaccount.google.com/apppasswords
|
|
11
|
+
* Use "App Password" (not your regular password).
|
|
12
|
+
*/
|
|
13
|
+
export interface UsageAlertEmailParams {
|
|
14
|
+
toEmail: string;
|
|
15
|
+
userName?: string;
|
|
16
|
+
usagePercent: number;
|
|
17
|
+
used: number;
|
|
18
|
+
total: number;
|
|
19
|
+
tier: string;
|
|
20
|
+
}
|
|
21
|
+
export declare function sendUsageAlertEmail(params: UsageAlertEmailParams): Promise<boolean>;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Email service using Nodemailer (free, no API key required).
|
|
3
|
+
* Configure via environment variables:
|
|
4
|
+
* EMAIL_SMTP_HOST - SMTP server (e.g. smtp.gmail.com)
|
|
5
|
+
* EMAIL_SMTP_PORT - SMTP port (default: 587)
|
|
6
|
+
* EMAIL_SMTP_USER - SMTP username / email address
|
|
7
|
+
* EMAIL_SMTP_PASS - SMTP password (Gmail: use App Password)
|
|
8
|
+
* EMAIL_FROM - From address (default: EMAIL_SMTP_USER)
|
|
9
|
+
*
|
|
10
|
+
* Gmail setup: https://myaccount.google.com/apppasswords
|
|
11
|
+
* Use "App Password" (not your regular password).
|
|
12
|
+
*/
|
|
13
|
+
import nodemailer from 'nodemailer';
|
|
14
|
+
function createTransport() {
|
|
15
|
+
const host = process.env.EMAIL_SMTP_HOST;
|
|
16
|
+
const user = process.env.EMAIL_SMTP_USER;
|
|
17
|
+
const pass = process.env.EMAIL_SMTP_PASS;
|
|
18
|
+
if (!host || !user || !pass) {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
return nodemailer.createTransport({
|
|
22
|
+
host,
|
|
23
|
+
port: parseInt(process.env.EMAIL_SMTP_PORT || '587'),
|
|
24
|
+
secure: process.env.EMAIL_SMTP_PORT === '465',
|
|
25
|
+
auth: { user, pass },
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
export async function sendUsageAlertEmail(params) {
|
|
29
|
+
const transport = createTransport();
|
|
30
|
+
if (!transport) {
|
|
31
|
+
console.warn('[email] SMTP not configured. Set EMAIL_SMTP_HOST, EMAIL_SMTP_USER, EMAIL_SMTP_PASS to enable email alerts.');
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
const from = process.env.EMAIL_FROM || process.env.EMAIL_SMTP_USER;
|
|
35
|
+
const html = buildUsageAlertHtml(params);
|
|
36
|
+
try {
|
|
37
|
+
await transport.sendMail({
|
|
38
|
+
from: `WebPeel <${from}>`,
|
|
39
|
+
to: params.toEmail,
|
|
40
|
+
subject: `WebPeel: You've used ${params.usagePercent}% of your weekly API limit`,
|
|
41
|
+
html,
|
|
42
|
+
});
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
catch (err) {
|
|
46
|
+
console.error('[email] Failed to send usage alert:', err);
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function buildUsageAlertHtml(params) {
|
|
51
|
+
const { usagePercent, used, total, tier, userName } = params;
|
|
52
|
+
const color = usagePercent >= 90 ? '#ef4444' : usagePercent >= 75 ? '#f59e0b' : '#5865F2';
|
|
53
|
+
return `<!DOCTYPE html>
|
|
54
|
+
<html>
|
|
55
|
+
<head><meta charset="utf-8"><title>WebPeel Usage Alert</title></head>
|
|
56
|
+
<body style="margin:0;padding:0;background:#0A0A0F;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;">
|
|
57
|
+
<div style="max-width:520px;margin:40px auto;background:#111116;border:1px solid #27272a;border-radius:12px;overflow:hidden;">
|
|
58
|
+
<div style="background:${color};padding:4px 24px;"></div>
|
|
59
|
+
<div style="padding:32px 24px;">
|
|
60
|
+
<div style="font-size:24px;font-weight:700;color:#ffffff;margin-bottom:8px;">Usage Alert</div>
|
|
61
|
+
<div style="font-size:15px;color:#a1a1aa;margin-bottom:24px;">
|
|
62
|
+
${userName ? `Hi ${userName}, you've` : "You've"} used <strong style="color:#ffffff;">${usagePercent}%</strong> of your weekly API limit.
|
|
63
|
+
</div>
|
|
64
|
+
<div style="background:#18181b;border-radius:8px;padding:16px;margin-bottom:24px;">
|
|
65
|
+
<div style="font-size:13px;color:#a1a1aa;margin-bottom:8px;">Usage this week</div>
|
|
66
|
+
<div style="height:8px;background:#27272a;border-radius:4px;overflow:hidden;">
|
|
67
|
+
<div style="height:100%;width:${Math.min(usagePercent, 100)}%;background:${color};border-radius:4px;"></div>
|
|
68
|
+
</div>
|
|
69
|
+
<div style="font-size:13px;color:#a1a1aa;margin-top:8px;">${used} / ${total} requests · ${tier} plan</div>
|
|
70
|
+
</div>
|
|
71
|
+
<a href="https://app.webpeel.dev/billing" style="display:inline-block;background:#5865F2;color:#ffffff;padding:12px 24px;border-radius:8px;text-decoration:none;font-weight:600;font-size:14px;">Upgrade Plan →</a>
|
|
72
|
+
<div style="font-size:12px;color:#71717a;margin-top:24px;">
|
|
73
|
+
To disable these alerts, visit <a href="https://app.webpeel.dev/settings" style="color:#5865F2;">Settings</a>.
|
|
74
|
+
</div>
|
|
75
|
+
</div>
|
|
76
|
+
</div>
|
|
77
|
+
</body>
|
|
78
|
+
</html>`;
|
|
79
|
+
}
|