webpeel 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -9
- package/dist/cli.js +97 -6
- package/dist/cli.js.map +1 -1
- package/dist/core/actions.d.ts +28 -0
- package/dist/core/actions.d.ts.map +1 -1
- package/dist/core/actions.js +60 -0
- package/dist/core/actions.js.map +1 -1
- package/dist/core/bm25-filter.d.ts +10 -0
- package/dist/core/bm25-filter.d.ts.map +1 -1
- package/dist/core/bm25-filter.js +40 -0
- package/dist/core/bm25-filter.js.map +1 -1
- package/dist/core/content-pruner.d.ts +12 -5
- package/dist/core/content-pruner.d.ts.map +1 -1
- package/dist/core/content-pruner.js +247 -190
- package/dist/core/content-pruner.js.map +1 -1
- package/dist/core/research.d.ts +67 -0
- package/dist/core/research.d.ts.map +1 -0
- package/dist/core/research.js +254 -0
- package/dist/core/research.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +37 -3
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +107 -2
- package/dist/mcp/server.js.map +1 -1
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.d.ts.map +1 -0
- package/dist/server/app.js +189 -0
- package/dist/server/app.js.map +1 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.d.ts.map +1 -0
- package/dist/server/auth-store.js +89 -0
- package/dist/server/auth-store.js.map +1 -0
- package/dist/server/job-queue.d.ts +93 -0
- package/dist/server/job-queue.d.ts.map +1 -0
- package/dist/server/job-queue.js +144 -0
- package/dist/server/job-queue.js.map +1 -0
- package/dist/server/middleware/auth.d.ts +28 -0
- package/dist/server/middleware/auth.d.ts.map +1 -0
- package/dist/server/middleware/auth.js +183 -0
- package/dist/server/middleware/auth.js.map +1 -0
- package/dist/server/middleware/rate-limit.d.ts +23 -0
- package/dist/server/middleware/rate-limit.d.ts.map +1 -0
- package/dist/server/middleware/rate-limit.js +126 -0
- package/dist/server/middleware/rate-limit.js.map +1 -0
- package/dist/server/middleware/url-validator.d.ts +16 -0
- package/dist/server/middleware/url-validator.d.ts.map +1 -0
- package/dist/server/middleware/url-validator.js +187 -0
- package/dist/server/middleware/url-validator.js.map +1 -0
- package/dist/server/pg-auth-store.d.ts +129 -0
- package/dist/server/pg-auth-store.d.ts.map +1 -0
- package/dist/server/pg-auth-store.js +457 -0
- package/dist/server/pg-auth-store.js.map +1 -0
- package/dist/server/pg-job-queue.d.ts +60 -0
- package/dist/server/pg-job-queue.d.ts.map +1 -0
- package/dist/server/pg-job-queue.js +365 -0
- package/dist/server/pg-job-queue.js.map +1 -0
- package/dist/server/premium/domain-intel.d.ts +17 -0
- package/dist/server/premium/domain-intel.d.ts.map +1 -0
- package/dist/server/premium/domain-intel.js +134 -0
- package/dist/server/premium/domain-intel.js.map +1 -0
- package/dist/server/premium/index.d.ts +18 -0
- package/dist/server/premium/index.d.ts.map +1 -0
- package/dist/server/premium/index.js +36 -0
- package/dist/server/premium/index.js.map +1 -0
- package/dist/server/premium/swr-cache.d.ts +15 -0
- package/dist/server/premium/swr-cache.d.ts.map +1 -0
- package/dist/server/premium/swr-cache.js +35 -0
- package/dist/server/premium/swr-cache.js.map +1 -0
- package/dist/server/routes/activity.d.ts +7 -0
- package/dist/server/routes/activity.d.ts.map +1 -0
- package/dist/server/routes/activity.js +66 -0
- package/dist/server/routes/activity.js.map +1 -0
- package/dist/server/routes/agent.d.ts +12 -0
- package/dist/server/routes/agent.d.ts.map +1 -0
- package/dist/server/routes/agent.js +356 -0
- package/dist/server/routes/agent.js.map +1 -0
- package/dist/server/routes/answer.d.ts +6 -0
- package/dist/server/routes/answer.d.ts.map +1 -0
- package/dist/server/routes/answer.js +124 -0
- package/dist/server/routes/answer.js.map +1 -0
- package/dist/server/routes/batch.d.ts +7 -0
- package/dist/server/routes/batch.d.ts.map +1 -0
- package/dist/server/routes/batch.js +287 -0
- package/dist/server/routes/batch.js.map +1 -0
- package/dist/server/routes/cli-usage.d.ts +7 -0
- package/dist/server/routes/cli-usage.d.ts.map +1 -0
- package/dist/server/routes/cli-usage.js +121 -0
- package/dist/server/routes/cli-usage.js.map +1 -0
- package/dist/server/routes/compat.d.ts +24 -0
- package/dist/server/routes/compat.d.ts.map +1 -0
- package/dist/server/routes/compat.js +651 -0
- package/dist/server/routes/compat.js.map +1 -0
- package/dist/server/routes/extract.d.ts +9 -0
- package/dist/server/routes/extract.d.ts.map +1 -0
- package/dist/server/routes/extract.js +121 -0
- package/dist/server/routes/extract.js.map +1 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.d.ts.map +1 -0
- package/dist/server/routes/fetch.js +537 -0
- package/dist/server/routes/fetch.js.map +1 -0
- package/dist/server/routes/health.d.ts +8 -0
- package/dist/server/routes/health.d.ts.map +1 -0
- package/dist/server/routes/health.js +36 -0
- package/dist/server/routes/health.js.map +1 -0
- package/dist/server/routes/jobs.d.ts +8 -0
- package/dist/server/routes/jobs.d.ts.map +1 -0
- package/dist/server/routes/jobs.js +374 -0
- package/dist/server/routes/jobs.js.map +1 -0
- package/dist/server/routes/mcp.d.ts +16 -0
- package/dist/server/routes/mcp.d.ts.map +1 -0
- package/dist/server/routes/mcp.js +475 -0
- package/dist/server/routes/mcp.js.map +1 -0
- package/dist/server/routes/oauth.d.ts +10 -0
- package/dist/server/routes/oauth.d.ts.map +1 -0
- package/dist/server/routes/oauth.js +296 -0
- package/dist/server/routes/oauth.js.map +1 -0
- package/dist/server/routes/screenshot.d.ts +10 -0
- package/dist/server/routes/screenshot.d.ts.map +1 -0
- package/dist/server/routes/screenshot.js +217 -0
- package/dist/server/routes/screenshot.js.map +1 -0
- package/dist/server/routes/search.d.ts +7 -0
- package/dist/server/routes/search.d.ts.map +1 -0
- package/dist/server/routes/search.js +287 -0
- package/dist/server/routes/search.js.map +1 -0
- package/dist/server/routes/stats.d.ts +7 -0
- package/dist/server/routes/stats.d.ts.map +1 -0
- package/dist/server/routes/stats.js +65 -0
- package/dist/server/routes/stats.js.map +1 -0
- package/dist/server/routes/stripe.d.ts +9 -0
- package/dist/server/routes/stripe.d.ts.map +1 -0
- package/dist/server/routes/stripe.js +233 -0
- package/dist/server/routes/stripe.js.map +1 -0
- package/dist/server/routes/users.d.ts +9 -0
- package/dist/server/routes/users.d.ts.map +1 -0
- package/dist/server/routes/users.js +954 -0
- package/dist/server/routes/users.js.map +1 -0
- package/dist/server/routes/webhooks.d.ts +15 -0
- package/dist/server/routes/webhooks.d.ts.map +1 -0
- package/dist/server/routes/webhooks.js +73 -0
- package/dist/server/routes/webhooks.js.map +1 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.d.ts.map +1 -0
- package/dist/server/sentry.js +39 -0
- package/dist/server/sentry.js.map +1 -0
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +3 -2
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fetch endpoint with caching
|
|
3
|
+
*/
|
|
4
|
+
import { Router } from 'express';
|
|
5
|
+
import { peel } from '../../index.js';
|
|
6
|
+
import { normalizeActions } from '../../core/actions.js';
|
|
7
|
+
import { extractInlineJson } from '../../core/extract-inline.js';
|
|
8
|
+
import { LRUCache } from 'lru-cache';
|
|
9
|
+
import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
|
|
10
|
+
const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
|
|
11
|
+
export function createFetchRouter(authStore) {
|
|
12
|
+
const router = Router();
|
|
13
|
+
// LRU cache: 5 minute TTL, max 1000 entries, 100MB total size
|
|
14
|
+
const cache = new LRUCache({
|
|
15
|
+
max: 1000,
|
|
16
|
+
ttl: 5 * 60 * 1000, // 5 minutes
|
|
17
|
+
maxSize: 100 * 1024 * 1024, // 100MB
|
|
18
|
+
sizeCalculation: (entry) => {
|
|
19
|
+
return JSON.stringify(entry).length;
|
|
20
|
+
},
|
|
21
|
+
});
|
|
22
|
+
router.get('/v1/fetch', async (req, res) => {
|
|
23
|
+
try {
|
|
24
|
+
const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions, maxAge, storeInCache, stream, } = req.query;
|
|
25
|
+
// Validate URL parameter
|
|
26
|
+
if (!url || typeof url !== 'string') {
|
|
27
|
+
res.status(400).json({
|
|
28
|
+
error: 'invalid_request',
|
|
29
|
+
message: 'Missing or invalid "url" parameter. Pass a URL as a query parameter: GET /v1/fetch?url=https://example.com',
|
|
30
|
+
example: 'curl "https://api.webpeel.dev/v1/fetch?url=https://example.com"',
|
|
31
|
+
docs: 'https://webpeel.dev/docs/api-reference#fetch',
|
|
32
|
+
});
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
// SECURITY: Validate URL format and length
|
|
36
|
+
if (url.length > 2048) {
|
|
37
|
+
res.status(400).json({
|
|
38
|
+
error: 'invalid_url',
|
|
39
|
+
message: 'URL too long (max 2048 characters)',
|
|
40
|
+
});
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
const parsed = new URL(url);
|
|
45
|
+
// Normalize URL for consistent caching
|
|
46
|
+
const normalizedUrl = parsed.href;
|
|
47
|
+
// Use normalized URL for cache key
|
|
48
|
+
if (normalizedUrl !== url) {
|
|
49
|
+
// URL was normalized, update for caching
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
res.status(400).json({
|
|
54
|
+
error: 'invalid_url',
|
|
55
|
+
message: 'Invalid URL format',
|
|
56
|
+
});
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
// SECURITY: Validate URL to prevent SSRF attacks
|
|
60
|
+
try {
|
|
61
|
+
validateUrlForSSRF(url);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
if (error instanceof SSRFError) {
|
|
65
|
+
res.status(400).json({
|
|
66
|
+
error: 'forbidden_url',
|
|
67
|
+
message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
|
|
68
|
+
});
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
throw error;
|
|
72
|
+
}
|
|
73
|
+
// Parse actions query param (JSON-encoded array)
|
|
74
|
+
let parsedActions;
|
|
75
|
+
if (actions && typeof actions === 'string') {
|
|
76
|
+
try {
|
|
77
|
+
const raw = JSON.parse(actions);
|
|
78
|
+
parsedActions = normalizeActions(raw);
|
|
79
|
+
}
|
|
80
|
+
catch (e) {
|
|
81
|
+
res.status(400).json({
|
|
82
|
+
error: 'invalid_request',
|
|
83
|
+
message: 'Invalid "actions" parameter: must be a valid JSON array',
|
|
84
|
+
});
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Build cache key (include new parameters)
|
|
89
|
+
const actionsKey = parsedActions ? JSON.stringify(parsedActions) : '';
|
|
90
|
+
const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}:${actionsKey}`;
|
|
91
|
+
// Check cache (with maxAge support)
|
|
92
|
+
const maxAgeMs = maxAge !== undefined ? parseInt(maxAge, 10) : 172800000; // Default 2 days
|
|
93
|
+
const cached = cache.get(cacheKey);
|
|
94
|
+
if (cached && maxAgeMs > 0) {
|
|
95
|
+
const cacheAge = Date.now() - cached.timestamp;
|
|
96
|
+
if (cacheAge < maxAgeMs) {
|
|
97
|
+
res.setHeader('X-Cache', 'HIT');
|
|
98
|
+
res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
|
|
99
|
+
res.json(cached.result);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Parse options
|
|
104
|
+
const isSoftLimited = req.auth?.softLimited === true;
|
|
105
|
+
const hasExtraUsage = req.auth?.extraUsageAvailable === true;
|
|
106
|
+
// Parse tag arrays from comma-separated strings
|
|
107
|
+
const includeTagsArray = includeTags
|
|
108
|
+
? includeTags.split(',').map(t => t.trim()).filter(Boolean)
|
|
109
|
+
: undefined;
|
|
110
|
+
const excludeTagsArray = excludeTags
|
|
111
|
+
? excludeTags.split(',').map(t => t.trim()).filter(Boolean)
|
|
112
|
+
: undefined;
|
|
113
|
+
const languagesArray = languages
|
|
114
|
+
? languages.split(',').map(l => l.trim()).filter(Boolean)
|
|
115
|
+
: undefined;
|
|
116
|
+
// onlyMainContent is a shortcut for common include tags
|
|
117
|
+
const finalIncludeTags = onlyMainContent === 'true'
|
|
118
|
+
? ['main', 'article', '.content', '#content']
|
|
119
|
+
: includeTagsArray;
|
|
120
|
+
// When actions are present, force browser mode (skip HTTP fast path)
|
|
121
|
+
const hasActions = parsedActions && parsedActions.length > 0;
|
|
122
|
+
const shouldRender = hasActions || render === 'true';
|
|
123
|
+
const options = {
|
|
124
|
+
// SOFT LIMIT: When over quota AND no extra usage, force HTTP-only
|
|
125
|
+
// If extra usage is available, allow full functionality
|
|
126
|
+
// Exception: actions always require render
|
|
127
|
+
render: (isSoftLimited && !hasExtraUsage && !hasActions) ? false : shouldRender,
|
|
128
|
+
wait: (isSoftLimited && !hasExtraUsage) ? 0 : (wait ? parseInt(wait, 10) : undefined),
|
|
129
|
+
format: format || 'markdown',
|
|
130
|
+
stream: stream === 'true',
|
|
131
|
+
includeTags: finalIncludeTags,
|
|
132
|
+
excludeTags: excludeTagsArray,
|
|
133
|
+
images: images === 'true',
|
|
134
|
+
actions: parsedActions,
|
|
135
|
+
location: location || languagesArray ? {
|
|
136
|
+
country: location,
|
|
137
|
+
languages: languagesArray,
|
|
138
|
+
} : undefined,
|
|
139
|
+
};
|
|
140
|
+
// Inform the user if their request was degraded
|
|
141
|
+
if (isSoftLimited && !hasExtraUsage && render === 'true' && !hasActions) {
|
|
142
|
+
res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
|
|
143
|
+
}
|
|
144
|
+
// Validate wait parameter
|
|
145
|
+
if (options.wait !== undefined && (isNaN(options.wait) || options.wait < 0 || options.wait > 60000)) {
|
|
146
|
+
res.status(400).json({
|
|
147
|
+
error: 'invalid_request',
|
|
148
|
+
message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
|
|
149
|
+
});
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
// Validate format parameter
|
|
153
|
+
if (!['markdown', 'text', 'html'].includes(options.format || '')) {
|
|
154
|
+
res.status(400).json({
|
|
155
|
+
error: 'invalid_request',
|
|
156
|
+
message: 'Invalid "format" parameter: must be "markdown", "text", or "html"',
|
|
157
|
+
});
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
const shouldStream = options.stream === true;
|
|
161
|
+
if (shouldStream) {
|
|
162
|
+
res.setHeader('X-Stream', 'true');
|
|
163
|
+
if (typeof res.flushHeaders === 'function') {
|
|
164
|
+
res.flushHeaders();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
// Fetch content
|
|
168
|
+
const startTime = Date.now();
|
|
169
|
+
const result = await peel(url, options);
|
|
170
|
+
const elapsed = Date.now() - startTime;
|
|
171
|
+
// Determine fetch type from the result method
|
|
172
|
+
const fetchType = result.method === 'stealth' ? 'stealth' :
|
|
173
|
+
result.method === 'browser' ? 'stealth' : 'basic';
|
|
174
|
+
// Log request to database (PostgreSQL only)
|
|
175
|
+
const pgStore = authStore;
|
|
176
|
+
if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
|
|
177
|
+
// Log to usage_logs table (user_id = accountId from keyInfo)
|
|
178
|
+
pgStore.pool.query(`INSERT INTO usage_logs
|
|
179
|
+
(user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent)
|
|
180
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
|
|
181
|
+
req.auth.keyInfo.accountId,
|
|
182
|
+
'fetch',
|
|
183
|
+
url,
|
|
184
|
+
fetchType,
|
|
185
|
+
elapsed,
|
|
186
|
+
200,
|
|
187
|
+
req.ip || req.socket.remoteAddress,
|
|
188
|
+
req.get('user-agent'),
|
|
189
|
+
]).catch((err) => {
|
|
190
|
+
console.error('Failed to log request to usage_logs:', err);
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
// Track usage (check for trackBurstUsage method to detect PostgresAuthStore)
|
|
194
|
+
if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
|
|
195
|
+
// Track burst usage (always)
|
|
196
|
+
await pgStore.trackBurstUsage(req.auth.keyInfo.key);
|
|
197
|
+
// If soft-limited with extra usage available, charge to extra usage
|
|
198
|
+
if (isSoftLimited && hasExtraUsage) {
|
|
199
|
+
const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200 // PeelResult doesn't include statusCode, assume success
|
|
200
|
+
);
|
|
201
|
+
if (extraResult.success) {
|
|
202
|
+
res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
|
|
203
|
+
res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
// Extra usage failed - fall back to soft limit
|
|
207
|
+
res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
else if (!isSoftLimited) {
|
|
211
|
+
// Normal weekly usage tracking
|
|
212
|
+
await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
|
|
213
|
+
}
|
|
214
|
+
// If soft-limited WITHOUT extra usage, don't track (already over quota)
|
|
215
|
+
}
|
|
216
|
+
// Cache result (unless storeInCache is explicitly false)
|
|
217
|
+
if (storeInCache !== 'false') {
|
|
218
|
+
cache.set(cacheKey, {
|
|
219
|
+
result,
|
|
220
|
+
timestamp: Date.now(),
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
// Add usage headers
|
|
224
|
+
res.setHeader('X-Cache', 'MISS');
|
|
225
|
+
res.setHeader('X-Credits-Used', '1');
|
|
226
|
+
res.setHeader('X-Processing-Time', elapsed.toString());
|
|
227
|
+
res.setHeader('X-Fetch-Type', fetchType);
|
|
228
|
+
res.json(result);
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
const err = error;
|
|
232
|
+
// Log error to database (PostgreSQL only)
|
|
233
|
+
const pgStore = authStore;
|
|
234
|
+
if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
|
|
235
|
+
const url = req.query.url;
|
|
236
|
+
const render = req.query.render === 'true';
|
|
237
|
+
const fetchType = render ? 'stealth' : 'basic';
|
|
238
|
+
pgStore.pool.query(`INSERT INTO usage_logs
|
|
239
|
+
(user_id, endpoint, url, method, status_code, error, ip_address, user_agent)
|
|
240
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
|
|
241
|
+
req.auth.keyInfo.accountId,
|
|
242
|
+
'fetch',
|
|
243
|
+
url,
|
|
244
|
+
fetchType,
|
|
245
|
+
500,
|
|
246
|
+
err.message || 'Unknown error',
|
|
247
|
+
req.ip || req.socket.remoteAddress,
|
|
248
|
+
req.get('user-agent'),
|
|
249
|
+
]).catch((logErr) => {
|
|
250
|
+
console.error('Failed to log error to usage_logs:', logErr);
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
// SECURITY: Sanitize error messages to prevent information disclosure
|
|
254
|
+
if (err.code) {
|
|
255
|
+
// WebPeelError from core library - safe to expose with helpful context
|
|
256
|
+
const safeMessage = err.message.replace(/[<>"']/g, ''); // Remove HTML chars
|
|
257
|
+
const statusCode = err.code === 'TIMEOUT' ? 504
|
|
258
|
+
: err.code === 'BLOCKED' ? 403
|
|
259
|
+
: err.code === 'NETWORK' ? 502
|
|
260
|
+
: 500;
|
|
261
|
+
const hints = {
|
|
262
|
+
TIMEOUT: 'Try increasing timeout with ?wait=10000, or use render=true for JS-heavy sites.',
|
|
263
|
+
BLOCKED: 'This site blocks automated requests. Try adding render=true or use stealth mode (costs 5 credits).',
|
|
264
|
+
NETWORK: 'Could not reach the target URL. Verify the URL is correct and the site is online.',
|
|
265
|
+
};
|
|
266
|
+
res.status(statusCode).json({
|
|
267
|
+
error: err.code,
|
|
268
|
+
message: safeMessage,
|
|
269
|
+
hint: hints[err.code] || undefined,
|
|
270
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
else {
|
|
274
|
+
// Unexpected error - generic message only
|
|
275
|
+
console.error('Fetch error:', err); // Log full error server-side
|
|
276
|
+
res.status(500).json({
|
|
277
|
+
error: 'internal_error',
|
|
278
|
+
message: 'An unexpected error occurred while fetching the URL. If this persists, check https://webpeel.dev/status',
|
|
279
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
});
|
|
284
|
+
// -----------------------------------------------------------------------
|
|
285
|
+
// POST /v1/fetch — same as GET but accepts JSON body with extract param
|
|
286
|
+
// POST /v2/scrape — alias with identical behaviour
|
|
287
|
+
// -----------------------------------------------------------------------
|
|
288
|
+
async function handlePostFetch(req, res) {
|
|
289
|
+
try {
|
|
290
|
+
const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions: rawActions, storeInCache: storeFlag,
|
|
291
|
+
// Inline extraction (BYOK)
|
|
292
|
+
extract, llmProvider, llmApiKey, llmModel,
|
|
293
|
+
// Firecrawl-compatible formats array
|
|
294
|
+
formats, stream, } = req.body;
|
|
295
|
+
// --- Validate URL -------------------------------------------------------
|
|
296
|
+
if (!url || typeof url !== 'string') {
|
|
297
|
+
res.status(400).json({
|
|
298
|
+
error: 'invalid_request',
|
|
299
|
+
message: 'Missing or invalid "url" in request body. Send JSON: { "url": "https://example.com" }',
|
|
300
|
+
example: 'curl -X POST https://api.webpeel.dev/v1/fetch -H "Content-Type: application/json" -d \'{"url":"https://example.com"}\'',
|
|
301
|
+
docs: 'https://webpeel.dev/docs/api-reference#fetch',
|
|
302
|
+
});
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
if (url.length > 2048) {
|
|
306
|
+
res.status(400).json({
|
|
307
|
+
error: 'invalid_url',
|
|
308
|
+
message: 'URL too long (max 2048 characters)',
|
|
309
|
+
});
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
try {
|
|
313
|
+
new URL(url);
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
res.status(400).json({
|
|
317
|
+
error: 'invalid_url',
|
|
318
|
+
message: 'Invalid URL format',
|
|
319
|
+
});
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
try {
|
|
323
|
+
validateUrlForSSRF(url);
|
|
324
|
+
}
|
|
325
|
+
catch (error) {
|
|
326
|
+
if (error instanceof SSRFError) {
|
|
327
|
+
res.status(400).json({
|
|
328
|
+
error: 'forbidden_url',
|
|
329
|
+
message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
|
|
330
|
+
});
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
throw error;
|
|
334
|
+
}
|
|
335
|
+
// --- Parse and normalize actions -----------------------------------------
|
|
336
|
+
let postActions;
|
|
337
|
+
if (rawActions !== undefined) {
|
|
338
|
+
try {
|
|
339
|
+
postActions = normalizeActions(rawActions);
|
|
340
|
+
}
|
|
341
|
+
catch (e) {
|
|
342
|
+
res.status(400).json({
|
|
343
|
+
error: 'invalid_request',
|
|
344
|
+
message: `Invalid "actions" parameter: ${e.message}`,
|
|
345
|
+
});
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
// --- Resolve inline extract from body or Firecrawl-compatible formats ---
|
|
350
|
+
let resolvedExtract = extract;
|
|
351
|
+
if (!resolvedExtract && Array.isArray(formats)) {
|
|
352
|
+
const jsonFormat = formats.find((f) => (typeof f === 'object' && f !== null && f.type === 'json') ||
|
|
353
|
+
(typeof f === 'string' && f === 'json'));
|
|
354
|
+
if (jsonFormat && typeof jsonFormat === 'object' && (jsonFormat.schema || jsonFormat.prompt)) {
|
|
355
|
+
resolvedExtract = {
|
|
356
|
+
schema: jsonFormat.schema,
|
|
357
|
+
prompt: jsonFormat.prompt,
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
// Validate LLM params if extraction is requested
|
|
362
|
+
if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt)) {
|
|
363
|
+
if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
|
|
364
|
+
res.status(400).json({
|
|
365
|
+
error: 'invalid_request',
|
|
366
|
+
message: `"llmProvider" is required for inline extraction and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`,
|
|
367
|
+
});
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
|
|
371
|
+
res.status(400).json({
|
|
372
|
+
error: 'invalid_request',
|
|
373
|
+
message: 'Missing or invalid "llmApiKey" (BYOK required for inline extraction)',
|
|
374
|
+
});
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
// --- Build PeelOptions ---------------------------------------------------
|
|
379
|
+
const isSoftLimited = req.auth?.softLimited === true;
|
|
380
|
+
const hasExtraUsage = req.auth?.extraUsageAvailable === true;
|
|
381
|
+
const includeTagsArray = Array.isArray(includeTags) ? includeTags : undefined;
|
|
382
|
+
const excludeTagsArray = Array.isArray(excludeTags) ? excludeTags : undefined;
|
|
383
|
+
const languagesArray = Array.isArray(languages) ? languages : undefined;
|
|
384
|
+
const finalIncludeTags = onlyMainContent === true
|
|
385
|
+
? ['main', 'article', '.content', '#content']
|
|
386
|
+
: includeTagsArray;
|
|
387
|
+
const resolvedFormat = format || 'markdown';
|
|
388
|
+
if (!['markdown', 'text', 'html'].includes(resolvedFormat)) {
|
|
389
|
+
res.status(400).json({
|
|
390
|
+
error: 'invalid_request',
|
|
391
|
+
message: 'Invalid "format" parameter: must be "markdown", "text", or "html"',
|
|
392
|
+
});
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
const resolvedWait = typeof wait === 'number' ? wait : undefined;
|
|
396
|
+
if (resolvedWait !== undefined && (isNaN(resolvedWait) || resolvedWait < 0 || resolvedWait > 60000)) {
|
|
397
|
+
res.status(400).json({
|
|
398
|
+
error: 'invalid_request',
|
|
399
|
+
message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
|
|
400
|
+
});
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
// When actions are present, force browser mode
|
|
404
|
+
const postHasActions = postActions && postActions.length > 0;
|
|
405
|
+
const postShouldRender = postHasActions || render === true;
|
|
406
|
+
const options = {
|
|
407
|
+
render: (isSoftLimited && !hasExtraUsage && !postHasActions) ? false : postShouldRender,
|
|
408
|
+
wait: (isSoftLimited && !hasExtraUsage) ? 0 : resolvedWait,
|
|
409
|
+
format: resolvedFormat,
|
|
410
|
+
stream: stream === true,
|
|
411
|
+
includeTags: finalIncludeTags,
|
|
412
|
+
excludeTags: excludeTagsArray,
|
|
413
|
+
images: images === true,
|
|
414
|
+
actions: postActions,
|
|
415
|
+
location: location || languagesArray ? {
|
|
416
|
+
country: location,
|
|
417
|
+
languages: languagesArray,
|
|
418
|
+
} : undefined,
|
|
419
|
+
};
|
|
420
|
+
if (isSoftLimited && !hasExtraUsage && render === true && !postHasActions) {
|
|
421
|
+
res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
|
|
422
|
+
}
|
|
423
|
+
const shouldStream = options.stream === true;
|
|
424
|
+
if (shouldStream) {
|
|
425
|
+
res.setHeader('X-Stream', 'true');
|
|
426
|
+
if (typeof res.flushHeaders === 'function') {
|
|
427
|
+
res.flushHeaders();
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
// --- Fetch content -------------------------------------------------------
|
|
431
|
+
const startTime = Date.now();
|
|
432
|
+
const result = await peel(url, options);
|
|
433
|
+
const elapsed = Date.now() - startTime;
|
|
434
|
+
// --- Inline extraction (post-fetch) -------------------------------------
|
|
435
|
+
let jsonData;
|
|
436
|
+
let extractTokensUsed;
|
|
437
|
+
if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt) && llmApiKey) {
|
|
438
|
+
const extractResult = await extractInlineJson(result.content, {
|
|
439
|
+
schema: resolvedExtract.schema,
|
|
440
|
+
prompt: resolvedExtract.prompt,
|
|
441
|
+
llmProvider: llmProvider,
|
|
442
|
+
llmApiKey: llmApiKey.trim(),
|
|
443
|
+
llmModel,
|
|
444
|
+
});
|
|
445
|
+
jsonData = extractResult.data;
|
|
446
|
+
extractTokensUsed = extractResult.tokensUsed;
|
|
447
|
+
}
|
|
448
|
+
// --- Usage tracking (same as GET) ----------------------------------------
|
|
449
|
+
const fetchType = result.method === 'stealth' ? 'stealth' :
|
|
450
|
+
result.method === 'browser' ? 'stealth' : 'basic';
|
|
451
|
+
const pgStore = authStore;
|
|
452
|
+
if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
|
|
453
|
+
pgStore.pool.query(`INSERT INTO usage_logs
|
|
454
|
+
(user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent)
|
|
455
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
|
|
456
|
+
req.auth.keyInfo.accountId,
|
|
457
|
+
'fetch',
|
|
458
|
+
url,
|
|
459
|
+
fetchType,
|
|
460
|
+
elapsed,
|
|
461
|
+
200,
|
|
462
|
+
req.ip || req.socket.remoteAddress,
|
|
463
|
+
req.get('user-agent'),
|
|
464
|
+
]).catch((err) => {
|
|
465
|
+
console.error('Failed to log request to usage_logs:', err);
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
|
|
469
|
+
await pgStore.trackBurstUsage(req.auth.keyInfo.key);
|
|
470
|
+
if (isSoftLimited && hasExtraUsage) {
|
|
471
|
+
const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200);
|
|
472
|
+
if (extraResult.success) {
|
|
473
|
+
res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
|
|
474
|
+
res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
|
|
475
|
+
}
|
|
476
|
+
else {
|
|
477
|
+
res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
else if (!isSoftLimited) {
|
|
481
|
+
await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
// Cache result
|
|
485
|
+
const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}`;
|
|
486
|
+
if (storeFlag !== false) {
|
|
487
|
+
cache.set(cacheKey, { result, timestamp: Date.now() });
|
|
488
|
+
}
|
|
489
|
+
// --- Build response ------------------------------------------------------
|
|
490
|
+
res.setHeader('X-Cache', 'MISS');
|
|
491
|
+
res.setHeader('X-Credits-Used', '1');
|
|
492
|
+
res.setHeader('X-Processing-Time', elapsed.toString());
|
|
493
|
+
res.setHeader('X-Fetch-Type', fetchType);
|
|
494
|
+
const responseBody = { ...result };
|
|
495
|
+
if (jsonData !== undefined) {
|
|
496
|
+
responseBody.json = jsonData;
|
|
497
|
+
}
|
|
498
|
+
if (extractTokensUsed) {
|
|
499
|
+
responseBody.extractTokensUsed = extractTokensUsed;
|
|
500
|
+
}
|
|
501
|
+
res.json(responseBody);
|
|
502
|
+
}
|
|
503
|
+
catch (error) {
|
|
504
|
+
const err = error;
|
|
505
|
+
console.error('POST fetch/scrape error:', err);
|
|
506
|
+
if (err.code) {
|
|
507
|
+
const safeMessage = err.message.replace(/[<>"']/g, '');
|
|
508
|
+
const statusCode = err.code === 'TIMEOUT' ? 504
|
|
509
|
+
: err.code === 'BLOCKED' ? 403
|
|
510
|
+
: err.code === 'NETWORK' ? 502
|
|
511
|
+
: 500;
|
|
512
|
+
const hints = {
|
|
513
|
+
TIMEOUT: 'Try increasing timeout, or set render:true for JS-heavy sites.',
|
|
514
|
+
BLOCKED: 'Site blocks automated requests. Try render:true or stealth mode.',
|
|
515
|
+
NETWORK: 'Could not reach the target URL. Verify it is correct and online.',
|
|
516
|
+
};
|
|
517
|
+
res.status(statusCode).json({
|
|
518
|
+
error: err.code,
|
|
519
|
+
message: safeMessage,
|
|
520
|
+
hint: hints[err.code] || undefined,
|
|
521
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
else {
|
|
525
|
+
res.status(500).json({
|
|
526
|
+
error: 'internal_error',
|
|
527
|
+
message: 'An unexpected error occurred. If this persists, check https://webpeel.dev/status',
|
|
528
|
+
docs: 'https://webpeel.dev/docs/api-reference#errors',
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
router.post('/v1/fetch', handlePostFetch);
|
|
534
|
+
router.post('/v2/scrape', handlePostFetch);
|
|
535
|
+
return router;
|
|
536
|
+
}
|
|
537
|
+
//# sourceMappingURL=fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/server/routes/fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,mBAAmB,GAAwB,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;AAOnF,MAAM,UAAU,iBAAiB,CAAC,SAAoB;IACpD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,8DAA8D;IAC9D,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAqB;QAC7C,GAAG,EAAE,IAAI;QACT,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;QAChC,OAAO,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ;QACpC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;YACzB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC5D,IAAI,CAAC;YACH,MAAM,EACJ,GAAG,EACH,MAAM,EACN,IAAI,EACJ,MAAM,EACN,WAAW,EACX,WAAW,EACX,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,OAAO,EACP,MAAM,EACN,YAAY,EACZ,MAAM,GACP,GAAG,GAAG,CAAC,KAAK,CAAC;YAEd,yBAAyB;YACzB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,4GAA4G;oBACrH,OAAO,EAAE,iEAAiE;oBAC1E,IAAI,EAAE,8CAA8C;iBACrD,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,2CAA2C;YAC3C,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,uCAAuC;gBACvC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC;gBAElC,mCAAmC;gBACnC,IAAI,aAAa,KAAK,GAAG,EAAE,CAAC;oBAC1B,yCAAyC;gBAC3C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oBAAoB;iBAC9B,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,iDAAiD;YACjD,IAAI,CAAC;gBACH,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;oBAC/B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,eAAe;wBACtB,OAAO,EAAE,4DAA4D;qBACtE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,iDAAiD;YACjD,IAAI,aAAuC,CAAC;YAC5C,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC3C,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAChC,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBACxC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,yDAAyD;qBACnE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,2CAA2C;YAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,WAAW,IAAI,WAAW,IAAI,MAAM,IAAI,QAAQ,IAAI,SAAS,IAAI,eAAe,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YAExK,oCAAoC;YACpC,MAAM,QAAQ,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,iBAAiB;YACrG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC;gBAC/C,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;oBAChC,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACrE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACxB,OAAO;gBACT,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,WAAW,KAAK,IAAI,CAAC;YACrD,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,mBAAmB,KAAK,IAAI,CAAC;YAE7D,gDAAgD;YAChD,MAAM,gBAAgB,GAAG,WAAW;gBAClC,CAAC,CAAE,WAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACvE,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,gBAAgB,GAAG,WAAW;gBAClC,CAAC,CAAE,WAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACvE,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,cAAc,GAAG,SAAS;gBAC9B,CAAC,CAAE,SAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACrE,CAAC,CAAC,SAAS,CAAC;YAEd,wDAAwD;YACxD,MAAM,gBAAgB,GAAG,eAAe,KAAK,MAAM;gBACjD,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;gBAC7C,CAAC,CAAC,gBAAgB,CAAC;YAErB,qEAAqE;YACrE,MAAM,UAAU,GAAG,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,UAAU,IAAI,MAAM,KAAK,MAAM,CAAC;YAErD,MAAM,OAAO,GAAgB;gBAC3B,kEAAkE;gBAClE,wDAAwD;gBACxD,2CAA2C;gBAC3C,MAAM,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY;gBAC/E,IAAI,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/F,MAAM,EAAG,MAAuC,IAAI,UAAU;gBAC9D,MAAM,EAAE,MAAM,KAAK,MAAM;gBACzB,WAAW,EAAE,gBAAgB;gBAC7B,WAAW,EAAE,gBAAgB;gBAC7B,MAAM,EAAE,MAAM,KAAK,MAAM;gBACzB,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,QAAQ,IAAI,cAAc,CAAC,CAAC,CAAC;oBACrC,OAAO,EAAE,QAA8B;oBACvC,SAAS,EAAE,cAAc;iBAC1B,CAAC,CAAC,CAAC,SAAS;aACd,CAAC;YAEF,gDAAgD;YAChD,IAAI,aAAa,IAAI,CAAC,aAAa,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxE,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,sDAAsD,CAAC,CAAC;YACtF,CAAC;YAED,0BAA0B;YAC1B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACpG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,yDAAyD;iBACnE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,4BAA4B;YAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,mEAAmE;iBAC7E,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC;YAC7C,IAAI,YAAY,EAAE,CAAC;gBACjB,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClC,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;oBAC3C,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEvC,8CAA8C;YAC9C,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACzC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YAEpD,4CAA4C;YAC5C,MAAM,OAAO,GAAG,SAAgB,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxE,6DAA6D;gBAC7D,OAAO,CAAC,IAAI,CAAC,KAAK,CAChB;;kDAEwC,EACxC;oBACE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS;oBAC1B,OAAO;oBACP,GAAG;oBACH,SAAS;oBACT,OAAO;oBACP,GAAG;oBACH,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa;oBAClC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;iBACtB,CACF,CAAC,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBACnB,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;gBAC7D,CAAC,CAAC,CAAC;YACL,CAAC;YAED,6EAA6E;YAC7E,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;gBAE5E,6BAA6B;gBAC7B,MAAM,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpD,oEAAoE;gBACpE,IAAI,aAAa,IAAI,aAAa,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,eAAe,CAC/C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EACpB,SAAS,EACT,GAAG,EACH,OAAO,EACP,GAAG,CAAC,wDAAwD;qBAC7D,CAAC;oBAEF,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;wBACxB,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1E,GAAG,CAAC,SAAS,CAAC,2BAA2B,EAAE,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,CAAC;yBAAM,CAAC;wBACN,+CAA+C;wBAC/C,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,kDAAkD,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;qBAAM,IAAI,CAAC,aAAa,EAAE,CAAC;oBAC1B,+BAA+B;oBAC/B,MAAM,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC5D,CAAC;gBACD,wEAAwE;YAC1E,CAAC;YAED,yDAAyD;YACzD,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;gBAC7B,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;oBAClB,MAAM;oBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAC;YACL,CAAC;YAED,oBAAoB;YACpB,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEzC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,KAAY,CAAC;YAEzB,0CAA0C;YAC1C,MAAM,OAAO,GAAG,SAAgB,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAa,CAAC;gBACpC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;gBAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;gBAE/C,OAAO,CAAC,IAAI,CAAC,KAAK,CAChB;;kDAEwC,EACxC;oBACE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS;oBAC1B,OAAO;oBACP,GAAG;oBACH,SAAS;oBACT,GAAG;oBACH,GAAG,CAAC,OAAO,IAAI,eAAe;oBAC9B,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa;oBAClC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;iBACtB,CACF,CAAC,KAAK,CAAC,CAAC,MAAW,EAAE,EAAE;oBACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,MAAM,CAAC,CAAC;gBAC9D,CAAC,CAAC,CAAC;YACL,CAAC;YAED,sEAAsE;YACtE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,uEAAuE;gBACvE,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;gBAC5E,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;oBAC7C,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;wBAC9B,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;4BAC9B,CAAC,CAAC,GAAG,CAAC;gBAER,MAAM,KAAK,GAA2B;oBACpC,OAAO,EAAE,iFAAiF;oBAC1F,OAAO,EAAE,oGAAoG;oBAC7G,OAAO,EAAE,mFAAmF;iBAC7F,CAAC;gBAEF,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;oBAC1B,KAAK,EAAE,GAAG,CAAC,IAAI;oBACf,OAAO,EAAE,WAAW;oBACpB,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS;oBAClC,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,0CAA0C;gBAC1C,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,gBAAgB;oBACvB,OAAO,EAAE,yGAAyG;oBAClH,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,wEAAwE;IACxE,mDAAmD;IACnD,0EAA0E;IAE1E,KAAK,UAAU,eAAe,CAAC,GAAY,EAAE,GAAa;QACxD,IAAI,CAAC;YACH,MAAM,EACJ,GAAG,EACH,MAAM,EACN,IAAI,EACJ,MAAM,EACN,WAAW,EACX,WAAW,EACX,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,OAAO,EAAE,UAAU,EACnB,YAAY,EAAE,SAAS;YACvB,2BAA2B;YAC3B,OAAO,EACP,WAAW,EACX,SAAS,EACT,QAAQ;YACR,qCAAqC;YACrC,OAAO,EACP,MAAM,GACP,GAAG,GAAG,CAAC,IAmBP,CAAC;YAEF,2EAA2E;YAC3E,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,uFAAuF;oBAChG,OAAO,EAAE,wHAAwH;oBACjI,IAAI,EAAE,8CAA8C;iBACrD,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oBAAoB;iBAC9B,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;oBAC/B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,eAAe;wBACtB,OAAO,EAAE,4DAA4D;qBACtE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,4EAA4E;YAC5E,IAAI,WAAqC,CAAC;YAC1C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;gBAC7C,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,gCAAiC,CAAW,CAAC,OAAO,EAAE;qBAChE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,2EAA2E;YAC3E,IAAI,eAAe,GAAmC,OAAO,CAAC;YAE9D,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAC7B,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;oBAC1D,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,MAAM,CAAC,CACpD,CAAC;gBACF,IAAI,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC7F,eAAe,GAAG;wBAChB,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,iDAAiD;YACjD,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,WAAW,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,WAAgC,CAAC,EAAE,CAAC;oBACpF,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,uEAAuE,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBACjH,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjF,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,sEAAsE;qBAChF,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,WAAW,KAAK,IAAI,CAAC;YACrD,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,mBAAmB,KAAK,IAAI,CAAC;YAE7D,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9E,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9E,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAExE,MAAM,gBAAgB,GAAG,eAAe,KAAK,IAAI;gBAC/C,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;gBAC7C,CAAC,CAAC,gBAAgB,CAAC;YAErB,MAAM,cAAc,GAAI,MAAuC,IAAI,UAAU,CAAC;YAC9E,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,mEAAmE;iBAC7E,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,YAAY,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACjE,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,KAAK,CAAC,EAAE,CAAC;gBACpG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,yDAAyD;iBACnE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,cAAc,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7D,MAAM,gBAAgB,GAAG,cAAc,IAAI,MAAM,KAAK,IAAI,CAAC;YAE3D,MAAM,OAAO,GAAgB;gBAC3B,MAAM,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB;gBACvF,IAAI,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;gBAC1D,MAAM,EAAE,cAAc;gBACtB,MAAM,EAAE,MAAM,KAAK,IAAI;gBACvB,WAAW,EAAE,gBAAgB;gBAC7B,WAAW,EAAE,gBAAgB;gBAC7B,MAAM,EAAE,MAAM,KAAK,IAAI;gBACvB,OAAO,EAAE,WAAW;gBACpB,QAAQ,EAAE,QAAQ,IAAI,cAAc,CAAC,CAAC,CAAC;oBACrC,OAAO,EAAE,QAAQ;oBACjB,SAAS,EAAE,cAAc;iBAC1B,CAAC,CAAC,CAAC,SAAS;aACd,CAAC;YAEF,IAAI,aAAa,IAAI,CAAC,aAAa,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1E,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,sDAAsD,CAAC,CAAC;YACtF,CAAC;YAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC;YAC7C,IAAI,YAAY,EAAE,CAAC;gBACjB,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClC,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;oBAC3C,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEvC,2EAA2E;YAC3E,IAAI,QAAyC,CAAC;YAC9C,IAAI,iBAAgE,CAAC;YAErE,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACvF,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE;oBAC5D,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,WAAW,EAAE,WAAgC;oBAC7C,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE;oBAC3B,QAAQ;iBACT,CAAC,CAAC;gBACH,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC;gBAC9B,iBAAiB,GAAG,aAAa,CAAC,UAAU,CAAC;YAC/C,CAAC;YAED,4EAA4E;YAC5E,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACzC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YAEpD,MAAM,OAAO,GAAG,SAAgB,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxE,OAAO,CAAC,IAAI,CAAC,KAAK,CAChB;;kDAEwC,EACxC;oBACE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS;oBAC1B,OAAO;oBACP,GAAG;oBACH,SAAS;oBACT,OAAO;oBACP,GAAG;oBACH,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa;oBAClC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;iBACtB,CACF,CAAC,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBACnB,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;gBAC7D,CAAC,CAAC,CAAC;YACL,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;gBAC5E,MAAM,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpD,IAAI,aAAa,IAAI,aAAa,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,eAAe,CAC/C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EACpB,SAAS,EACT,GAAG,EACH,OAAO,EACP,GAAG,CACJ,CAAC;oBACF,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;wBACxB,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1E,GAAG,CAAC,SAAS,CAAC,2BAA2B,EAAE,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,CAAC;yBAAM,CAAC;wBACN,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,kDAAkD,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;qBAAM,IAAI,CAAC,aAAa,EAAE,CAAC;oBAC1B,MAAM,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC;YAED,eAAe;YACf,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,WAAW,IAAI,WAAW,IAAI,MAAM,IAAI,QAAQ,IAAI,SAAS,IAAI,eAAe,IAAI,MAAM,EAAE,CAAC;YAC1J,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBACxB,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACzD,CAAC;YAED,4EAA4E;YAC5E,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEzC,MAAM,YAAY,GAAQ,EAAE,GAAG,MAAM,EAAE,CAAC;YACxC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,YAAY,CAAC,IAAI,GAAG,QAAQ,CAAC;YAC/B,CAAC;YACD,IAAI,iBAAiB,EAAE,CAAC;gBACtB,YAAY,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;YACrD,CAAC;YAED,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,KAAY,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;YAE/C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gBACvD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;oBAC7C,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;wBAC9B,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;4BAC9B,CAAC,CAAC,GAAG,CAAC;gBAER,MAAM,KAAK,GAA2B;oBACpC,OAAO,EAAE,gEAAgE;oBACzE,OAAO,EAAE,kEAAkE;oBAC3E,OAAO,EAAE,kEAAkE;iBAC5E,CAAC;gBAEF,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;oBAC1B,KAAK,EAAE,GAAG,CAAC,IAAI;oBACf,OAAO,EAAE,WAAW;oBACpB,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS;oBAClC,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,gBAAgB;oBACvB,OAAO,EAAE,kFAAkF;oBAC3F,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC1C,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAE3C,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Health check endpoint
|
|
3
|
+
* NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
|
|
4
|
+
* so it's never blocked by rate limiting (Render hits it every ~30s).
|
|
5
|
+
*/
|
|
6
|
+
import { Router } from 'express';
|
|
7
|
+
export declare function createHealthRouter(): Router;
|
|
8
|
+
//# sourceMappingURL=health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../../src/server/routes/health.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAmBpD,wBAAgB,kBAAkB,IAAI,MAAM,CAe3C"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Health check endpoint
|
|
3
|
+
* NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
|
|
4
|
+
* so it's never blocked by rate limiting (Render hits it every ~30s).
|
|
5
|
+
*/
|
|
6
|
+
import { Router } from 'express';
|
|
7
|
+
import { readFileSync } from 'fs';
|
|
8
|
+
import { join, dirname } from 'path';
|
|
9
|
+
import { fileURLToPath } from 'url';
|
|
10
|
+
const startTime = Date.now();
|
|
11
|
+
// Read version from package.json at startup
|
|
12
|
+
let version = '0.0.0';
|
|
13
|
+
try {
|
|
14
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
15
|
+
// Walk up from dist/server/routes/ to project root
|
|
16
|
+
const pkgPath = join(__dirname, '..', '..', '..', 'package.json');
|
|
17
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
|
|
18
|
+
version = pkg.version;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
// Fallback if package.json can't be read
|
|
22
|
+
}
|
|
23
|
+
export function createHealthRouter() {
|
|
24
|
+
const router = Router();
|
|
25
|
+
router.get('/health', (_req, res) => {
|
|
26
|
+
const uptime = Math.floor((Date.now() - startTime) / 1000);
|
|
27
|
+
res.json({
|
|
28
|
+
status: 'healthy',
|
|
29
|
+
version,
|
|
30
|
+
uptime,
|
|
31
|
+
timestamp: new Date().toISOString(),
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
return router;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=health.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"health.js","sourceRoot":"","sources":["../../../src/server/routes/health.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAE7B,4CAA4C;AAC5C,IAAI,OAAO,GAAG,OAAO,CAAC;AACtB,IAAI,CAAC;IACH,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,mDAAmD;IACnD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;AACxB,CAAC;AAAC,MAAM,CAAC;IACP,yCAAyC;AAC3C,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC;QAE3D,GAAG,CAAC,IAAI,CAAC;YACP,MAAM,EAAE,SAAS;YACjB,OAAO;YACP,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Async jobs API - crawl endpoints with SSE support
|
|
3
|
+
*/
|
|
4
|
+
import { Router } from 'express';
|
|
5
|
+
import type { AuthStore } from '../auth-store.js';
|
|
6
|
+
import type { IJobQueue } from '../job-queue.js';
|
|
7
|
+
export declare function createJobsRouter(jobQueue: IJobQueue, authStore: AuthStore): Router;
|
|
8
|
+
//# sourceMappingURL=jobs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../../src/server/routes/jobs.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAKpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAGjD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,GAAG,MAAM,CA4alF"}
|