webpeel 0.12.0 → 0.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +82 -9
  2. package/dist/cli.js +97 -6
  3. package/dist/cli.js.map +1 -1
  4. package/dist/core/actions.d.ts +28 -0
  5. package/dist/core/actions.d.ts.map +1 -1
  6. package/dist/core/actions.js +60 -0
  7. package/dist/core/actions.js.map +1 -1
  8. package/dist/core/bm25-filter.d.ts +10 -0
  9. package/dist/core/bm25-filter.d.ts.map +1 -1
  10. package/dist/core/bm25-filter.js +40 -0
  11. package/dist/core/bm25-filter.js.map +1 -1
  12. package/dist/core/content-pruner.d.ts +12 -5
  13. package/dist/core/content-pruner.d.ts.map +1 -1
  14. package/dist/core/content-pruner.js +247 -190
  15. package/dist/core/content-pruner.js.map +1 -1
  16. package/dist/core/research.d.ts +67 -0
  17. package/dist/core/research.d.ts.map +1 -0
  18. package/dist/core/research.js +254 -0
  19. package/dist/core/research.js.map +1 -0
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +37 -3
  22. package/dist/index.js.map +1 -1
  23. package/dist/mcp/server.js +107 -2
  24. package/dist/mcp/server.js.map +1 -1
  25. package/dist/server/app.d.ts +14 -0
  26. package/dist/server/app.d.ts.map +1 -0
  27. package/dist/server/app.js +189 -0
  28. package/dist/server/app.js.map +1 -0
  29. package/dist/server/auth-store.d.ts +28 -0
  30. package/dist/server/auth-store.d.ts.map +1 -0
  31. package/dist/server/auth-store.js +89 -0
  32. package/dist/server/auth-store.js.map +1 -0
  33. package/dist/server/job-queue.d.ts +93 -0
  34. package/dist/server/job-queue.d.ts.map +1 -0
  35. package/dist/server/job-queue.js +144 -0
  36. package/dist/server/job-queue.js.map +1 -0
  37. package/dist/server/middleware/auth.d.ts +28 -0
  38. package/dist/server/middleware/auth.d.ts.map +1 -0
  39. package/dist/server/middleware/auth.js +183 -0
  40. package/dist/server/middleware/auth.js.map +1 -0
  41. package/dist/server/middleware/rate-limit.d.ts +23 -0
  42. package/dist/server/middleware/rate-limit.d.ts.map +1 -0
  43. package/dist/server/middleware/rate-limit.js +126 -0
  44. package/dist/server/middleware/rate-limit.js.map +1 -0
  45. package/dist/server/middleware/url-validator.d.ts +16 -0
  46. package/dist/server/middleware/url-validator.d.ts.map +1 -0
  47. package/dist/server/middleware/url-validator.js +187 -0
  48. package/dist/server/middleware/url-validator.js.map +1 -0
  49. package/dist/server/pg-auth-store.d.ts +129 -0
  50. package/dist/server/pg-auth-store.d.ts.map +1 -0
  51. package/dist/server/pg-auth-store.js +457 -0
  52. package/dist/server/pg-auth-store.js.map +1 -0
  53. package/dist/server/pg-job-queue.d.ts +60 -0
  54. package/dist/server/pg-job-queue.d.ts.map +1 -0
  55. package/dist/server/pg-job-queue.js +365 -0
  56. package/dist/server/pg-job-queue.js.map +1 -0
  57. package/dist/server/premium/domain-intel.d.ts +17 -0
  58. package/dist/server/premium/domain-intel.d.ts.map +1 -0
  59. package/dist/server/premium/domain-intel.js +134 -0
  60. package/dist/server/premium/domain-intel.js.map +1 -0
  61. package/dist/server/premium/index.d.ts +18 -0
  62. package/dist/server/premium/index.d.ts.map +1 -0
  63. package/dist/server/premium/index.js +36 -0
  64. package/dist/server/premium/index.js.map +1 -0
  65. package/dist/server/premium/swr-cache.d.ts +15 -0
  66. package/dist/server/premium/swr-cache.d.ts.map +1 -0
  67. package/dist/server/premium/swr-cache.js +35 -0
  68. package/dist/server/premium/swr-cache.js.map +1 -0
  69. package/dist/server/routes/activity.d.ts +7 -0
  70. package/dist/server/routes/activity.d.ts.map +1 -0
  71. package/dist/server/routes/activity.js +66 -0
  72. package/dist/server/routes/activity.js.map +1 -0
  73. package/dist/server/routes/agent.d.ts +12 -0
  74. package/dist/server/routes/agent.d.ts.map +1 -0
  75. package/dist/server/routes/agent.js +356 -0
  76. package/dist/server/routes/agent.js.map +1 -0
  77. package/dist/server/routes/answer.d.ts +6 -0
  78. package/dist/server/routes/answer.d.ts.map +1 -0
  79. package/dist/server/routes/answer.js +124 -0
  80. package/dist/server/routes/answer.js.map +1 -0
  81. package/dist/server/routes/batch.d.ts +7 -0
  82. package/dist/server/routes/batch.d.ts.map +1 -0
  83. package/dist/server/routes/batch.js +287 -0
  84. package/dist/server/routes/batch.js.map +1 -0
  85. package/dist/server/routes/cli-usage.d.ts +7 -0
  86. package/dist/server/routes/cli-usage.d.ts.map +1 -0
  87. package/dist/server/routes/cli-usage.js +121 -0
  88. package/dist/server/routes/cli-usage.js.map +1 -0
  89. package/dist/server/routes/compat.d.ts +24 -0
  90. package/dist/server/routes/compat.d.ts.map +1 -0
  91. package/dist/server/routes/compat.js +651 -0
  92. package/dist/server/routes/compat.js.map +1 -0
  93. package/dist/server/routes/extract.d.ts +9 -0
  94. package/dist/server/routes/extract.d.ts.map +1 -0
  95. package/dist/server/routes/extract.js +121 -0
  96. package/dist/server/routes/extract.js.map +1 -0
  97. package/dist/server/routes/fetch.d.ts +7 -0
  98. package/dist/server/routes/fetch.d.ts.map +1 -0
  99. package/dist/server/routes/fetch.js +537 -0
  100. package/dist/server/routes/fetch.js.map +1 -0
  101. package/dist/server/routes/health.d.ts +8 -0
  102. package/dist/server/routes/health.d.ts.map +1 -0
  103. package/dist/server/routes/health.js +36 -0
  104. package/dist/server/routes/health.js.map +1 -0
  105. package/dist/server/routes/jobs.d.ts +8 -0
  106. package/dist/server/routes/jobs.d.ts.map +1 -0
  107. package/dist/server/routes/jobs.js +374 -0
  108. package/dist/server/routes/jobs.js.map +1 -0
  109. package/dist/server/routes/mcp.d.ts +16 -0
  110. package/dist/server/routes/mcp.d.ts.map +1 -0
  111. package/dist/server/routes/mcp.js +475 -0
  112. package/dist/server/routes/mcp.js.map +1 -0
  113. package/dist/server/routes/oauth.d.ts +10 -0
  114. package/dist/server/routes/oauth.d.ts.map +1 -0
  115. package/dist/server/routes/oauth.js +296 -0
  116. package/dist/server/routes/oauth.js.map +1 -0
  117. package/dist/server/routes/screenshot.d.ts +10 -0
  118. package/dist/server/routes/screenshot.d.ts.map +1 -0
  119. package/dist/server/routes/screenshot.js +217 -0
  120. package/dist/server/routes/screenshot.js.map +1 -0
  121. package/dist/server/routes/search.d.ts +7 -0
  122. package/dist/server/routes/search.d.ts.map +1 -0
  123. package/dist/server/routes/search.js +287 -0
  124. package/dist/server/routes/search.js.map +1 -0
  125. package/dist/server/routes/stats.d.ts +7 -0
  126. package/dist/server/routes/stats.d.ts.map +1 -0
  127. package/dist/server/routes/stats.js +65 -0
  128. package/dist/server/routes/stats.js.map +1 -0
  129. package/dist/server/routes/stripe.d.ts +9 -0
  130. package/dist/server/routes/stripe.d.ts.map +1 -0
  131. package/dist/server/routes/stripe.js +233 -0
  132. package/dist/server/routes/stripe.js.map +1 -0
  133. package/dist/server/routes/users.d.ts +9 -0
  134. package/dist/server/routes/users.d.ts.map +1 -0
  135. package/dist/server/routes/users.js +954 -0
  136. package/dist/server/routes/users.js.map +1 -0
  137. package/dist/server/routes/webhooks.d.ts +15 -0
  138. package/dist/server/routes/webhooks.d.ts.map +1 -0
  139. package/dist/server/routes/webhooks.js +73 -0
  140. package/dist/server/routes/webhooks.js.map +1 -0
  141. package/dist/server/sentry.d.ts +14 -0
  142. package/dist/server/sentry.d.ts.map +1 -0
  143. package/dist/server/sentry.js +39 -0
  144. package/dist/server/sentry.js.map +1 -0
  145. package/dist/types.d.ts +13 -0
  146. package/dist/types.d.ts.map +1 -1
  147. package/dist/types.js.map +1 -1
  148. package/package.json +3 -2
@@ -0,0 +1,537 @@
1
+ /**
2
+ * Fetch endpoint with caching
3
+ */
4
+ import { Router } from 'express';
5
+ import { peel } from '../../index.js';
6
+ import { normalizeActions } from '../../core/actions.js';
7
+ import { extractInlineJson } from '../../core/extract-inline.js';
8
+ import { LRUCache } from 'lru-cache';
9
+ import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
10
+ const VALID_LLM_PROVIDERS = ['openai', 'anthropic', 'google'];
11
+ export function createFetchRouter(authStore) {
12
+ const router = Router();
13
+ // LRU cache: 5 minute TTL, max 1000 entries, 100MB total size
14
+ const cache = new LRUCache({
15
+ max: 1000,
16
+ ttl: 5 * 60 * 1000, // 5 minutes
17
+ maxSize: 100 * 1024 * 1024, // 100MB
18
+ sizeCalculation: (entry) => {
19
+ return JSON.stringify(entry).length;
20
+ },
21
+ });
22
+ router.get('/v1/fetch', async (req, res) => {
23
+ try {
24
+ const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions, maxAge, storeInCache, stream, } = req.query;
25
+ // Validate URL parameter
26
+ if (!url || typeof url !== 'string') {
27
+ res.status(400).json({
28
+ error: 'invalid_request',
29
+ message: 'Missing or invalid "url" parameter. Pass a URL as a query parameter: GET /v1/fetch?url=https://example.com',
30
+ example: 'curl "https://api.webpeel.dev/v1/fetch?url=https://example.com"',
31
+ docs: 'https://webpeel.dev/docs/api-reference#fetch',
32
+ });
33
+ return;
34
+ }
35
+ // SECURITY: Validate URL format and length
36
+ if (url.length > 2048) {
37
+ res.status(400).json({
38
+ error: 'invalid_url',
39
+ message: 'URL too long (max 2048 characters)',
40
+ });
41
+ return;
42
+ }
43
+ try {
44
+ const parsed = new URL(url);
45
+ // Normalize URL for consistent caching
46
+ const normalizedUrl = parsed.href;
47
+ // Use normalized URL for cache key
48
+ if (normalizedUrl !== url) {
49
+ // URL was normalized, update for caching
50
+ }
51
+ }
52
+ catch {
53
+ res.status(400).json({
54
+ error: 'invalid_url',
55
+ message: 'Invalid URL format',
56
+ });
57
+ return;
58
+ }
59
+ // SECURITY: Validate URL to prevent SSRF attacks
60
+ try {
61
+ validateUrlForSSRF(url);
62
+ }
63
+ catch (error) {
64
+ if (error instanceof SSRFError) {
65
+ res.status(400).json({
66
+ error: 'forbidden_url',
67
+ message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
68
+ });
69
+ return;
70
+ }
71
+ throw error;
72
+ }
73
+ // Parse actions query param (JSON-encoded array)
74
+ let parsedActions;
75
+ if (actions && typeof actions === 'string') {
76
+ try {
77
+ const raw = JSON.parse(actions);
78
+ parsedActions = normalizeActions(raw);
79
+ }
80
+ catch (e) {
81
+ res.status(400).json({
82
+ error: 'invalid_request',
83
+ message: 'Invalid "actions" parameter: must be a valid JSON array',
84
+ });
85
+ return;
86
+ }
87
+ }
88
+ // Build cache key (include new parameters)
89
+ const actionsKey = parsedActions ? JSON.stringify(parsedActions) : '';
90
+ const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}:${actionsKey}`;
91
+ // Check cache (with maxAge support)
92
+ const maxAgeMs = maxAge !== undefined ? parseInt(maxAge, 10) : 172800000; // Default 2 days
93
+ const cached = cache.get(cacheKey);
94
+ if (cached && maxAgeMs > 0) {
95
+ const cacheAge = Date.now() - cached.timestamp;
96
+ if (cacheAge < maxAgeMs) {
97
+ res.setHeader('X-Cache', 'HIT');
98
+ res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
99
+ res.json(cached.result);
100
+ return;
101
+ }
102
+ }
103
+ // Parse options
104
+ const isSoftLimited = req.auth?.softLimited === true;
105
+ const hasExtraUsage = req.auth?.extraUsageAvailable === true;
106
+ // Parse tag arrays from comma-separated strings
107
+ const includeTagsArray = includeTags
108
+ ? includeTags.split(',').map(t => t.trim()).filter(Boolean)
109
+ : undefined;
110
+ const excludeTagsArray = excludeTags
111
+ ? excludeTags.split(',').map(t => t.trim()).filter(Boolean)
112
+ : undefined;
113
+ const languagesArray = languages
114
+ ? languages.split(',').map(l => l.trim()).filter(Boolean)
115
+ : undefined;
116
+ // onlyMainContent is a shortcut for common include tags
117
+ const finalIncludeTags = onlyMainContent === 'true'
118
+ ? ['main', 'article', '.content', '#content']
119
+ : includeTagsArray;
120
+ // When actions are present, force browser mode (skip HTTP fast path)
121
+ const hasActions = parsedActions && parsedActions.length > 0;
122
+ const shouldRender = hasActions || render === 'true';
123
+ const options = {
124
+ // SOFT LIMIT: When over quota AND no extra usage, force HTTP-only
125
+ // If extra usage is available, allow full functionality
126
+ // Exception: actions always require render
127
+ render: (isSoftLimited && !hasExtraUsage && !hasActions) ? false : shouldRender,
128
+ wait: (isSoftLimited && !hasExtraUsage) ? 0 : (wait ? parseInt(wait, 10) : undefined),
129
+ format: format || 'markdown',
130
+ stream: stream === 'true',
131
+ includeTags: finalIncludeTags,
132
+ excludeTags: excludeTagsArray,
133
+ images: images === 'true',
134
+ actions: parsedActions,
135
+ location: location || languagesArray ? {
136
+ country: location,
137
+ languages: languagesArray,
138
+ } : undefined,
139
+ };
140
+ // Inform the user if their request was degraded
141
+ if (isSoftLimited && !hasExtraUsage && render === 'true' && !hasActions) {
142
+ res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
143
+ }
144
+ // Validate wait parameter
145
+ if (options.wait !== undefined && (isNaN(options.wait) || options.wait < 0 || options.wait > 60000)) {
146
+ res.status(400).json({
147
+ error: 'invalid_request',
148
+ message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
149
+ });
150
+ return;
151
+ }
152
+ // Validate format parameter
153
+ if (!['markdown', 'text', 'html'].includes(options.format || '')) {
154
+ res.status(400).json({
155
+ error: 'invalid_request',
156
+ message: 'Invalid "format" parameter: must be "markdown", "text", or "html"',
157
+ });
158
+ return;
159
+ }
160
+ const shouldStream = options.stream === true;
161
+ if (shouldStream) {
162
+ res.setHeader('X-Stream', 'true');
163
+ if (typeof res.flushHeaders === 'function') {
164
+ res.flushHeaders();
165
+ }
166
+ }
167
+ // Fetch content
168
+ const startTime = Date.now();
169
+ const result = await peel(url, options);
170
+ const elapsed = Date.now() - startTime;
171
+ // Determine fetch type from the result method
172
+ const fetchType = result.method === 'stealth' ? 'stealth' :
173
+ result.method === 'browser' ? 'stealth' : 'basic';
174
+ // Log request to database (PostgreSQL only)
175
+ const pgStore = authStore;
176
+ if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
177
+ // Log to usage_logs table (user_id = accountId from keyInfo)
178
+ pgStore.pool.query(`INSERT INTO usage_logs
179
+ (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent)
180
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
181
+ req.auth.keyInfo.accountId,
182
+ 'fetch',
183
+ url,
184
+ fetchType,
185
+ elapsed,
186
+ 200,
187
+ req.ip || req.socket.remoteAddress,
188
+ req.get('user-agent'),
189
+ ]).catch((err) => {
190
+ console.error('Failed to log request to usage_logs:', err);
191
+ });
192
+ }
193
+ // Track usage (check for trackBurstUsage method to detect PostgresAuthStore)
194
+ if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
195
+ // Track burst usage (always)
196
+ await pgStore.trackBurstUsage(req.auth.keyInfo.key);
197
+ // If soft-limited with extra usage available, charge to extra usage
198
+ if (isSoftLimited && hasExtraUsage) {
199
+ const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200 // PeelResult doesn't include statusCode, assume success
200
+ );
201
+ if (extraResult.success) {
202
+ res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
203
+ res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
204
+ }
205
+ else {
206
+ // Extra usage failed - fall back to soft limit
207
+ res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
208
+ }
209
+ }
210
+ else if (!isSoftLimited) {
211
+ // Normal weekly usage tracking
212
+ await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
213
+ }
214
+ // If soft-limited WITHOUT extra usage, don't track (already over quota)
215
+ }
216
+ // Cache result (unless storeInCache is explicitly false)
217
+ if (storeInCache !== 'false') {
218
+ cache.set(cacheKey, {
219
+ result,
220
+ timestamp: Date.now(),
221
+ });
222
+ }
223
+ // Add usage headers
224
+ res.setHeader('X-Cache', 'MISS');
225
+ res.setHeader('X-Credits-Used', '1');
226
+ res.setHeader('X-Processing-Time', elapsed.toString());
227
+ res.setHeader('X-Fetch-Type', fetchType);
228
+ res.json(result);
229
+ }
230
+ catch (error) {
231
+ const err = error;
232
+ // Log error to database (PostgreSQL only)
233
+ const pgStore = authStore;
234
+ if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
235
+ const url = req.query.url;
236
+ const render = req.query.render === 'true';
237
+ const fetchType = render ? 'stealth' : 'basic';
238
+ pgStore.pool.query(`INSERT INTO usage_logs
239
+ (user_id, endpoint, url, method, status_code, error, ip_address, user_agent)
240
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
241
+ req.auth.keyInfo.accountId,
242
+ 'fetch',
243
+ url,
244
+ fetchType,
245
+ 500,
246
+ err.message || 'Unknown error',
247
+ req.ip || req.socket.remoteAddress,
248
+ req.get('user-agent'),
249
+ ]).catch((logErr) => {
250
+ console.error('Failed to log error to usage_logs:', logErr);
251
+ });
252
+ }
253
+ // SECURITY: Sanitize error messages to prevent information disclosure
254
+ if (err.code) {
255
+ // WebPeelError from core library - safe to expose with helpful context
256
+ const safeMessage = err.message.replace(/[<>"']/g, ''); // Remove HTML chars
257
+ const statusCode = err.code === 'TIMEOUT' ? 504
258
+ : err.code === 'BLOCKED' ? 403
259
+ : err.code === 'NETWORK' ? 502
260
+ : 500;
261
+ const hints = {
262
+ TIMEOUT: 'Try increasing timeout with ?wait=10000, or use render=true for JS-heavy sites.',
263
+ BLOCKED: 'This site blocks automated requests. Try adding render=true or use stealth mode (costs 5 credits).',
264
+ NETWORK: 'Could not reach the target URL. Verify the URL is correct and the site is online.',
265
+ };
266
+ res.status(statusCode).json({
267
+ error: err.code,
268
+ message: safeMessage,
269
+ hint: hints[err.code] || undefined,
270
+ docs: 'https://webpeel.dev/docs/api-reference#errors',
271
+ });
272
+ }
273
+ else {
274
+ // Unexpected error - generic message only
275
+ console.error('Fetch error:', err); // Log full error server-side
276
+ res.status(500).json({
277
+ error: 'internal_error',
278
+ message: 'An unexpected error occurred while fetching the URL. If this persists, check https://webpeel.dev/status',
279
+ docs: 'https://webpeel.dev/docs/api-reference#errors',
280
+ });
281
+ }
282
+ }
283
+ });
284
+ // -----------------------------------------------------------------------
285
+ // POST /v1/fetch — same as GET but accepts JSON body with extract param
286
+ // POST /v2/scrape — alias with identical behaviour
287
+ // -----------------------------------------------------------------------
288
+ async function handlePostFetch(req, res) {
289
+ try {
290
+ const { url, render, wait, format, includeTags, excludeTags, images, location, languages, onlyMainContent, actions: rawActions, storeInCache: storeFlag,
291
+ // Inline extraction (BYOK)
292
+ extract, llmProvider, llmApiKey, llmModel,
293
+ // Firecrawl-compatible formats array
294
+ formats, stream, } = req.body;
295
+ // --- Validate URL -------------------------------------------------------
296
+ if (!url || typeof url !== 'string') {
297
+ res.status(400).json({
298
+ error: 'invalid_request',
299
+ message: 'Missing or invalid "url" in request body. Send JSON: { "url": "https://example.com" }',
300
+ example: 'curl -X POST https://api.webpeel.dev/v1/fetch -H "Content-Type: application/json" -d \'{"url":"https://example.com"}\'',
301
+ docs: 'https://webpeel.dev/docs/api-reference#fetch',
302
+ });
303
+ return;
304
+ }
305
+ if (url.length > 2048) {
306
+ res.status(400).json({
307
+ error: 'invalid_url',
308
+ message: 'URL too long (max 2048 characters)',
309
+ });
310
+ return;
311
+ }
312
+ try {
313
+ new URL(url);
314
+ }
315
+ catch {
316
+ res.status(400).json({
317
+ error: 'invalid_url',
318
+ message: 'Invalid URL format',
319
+ });
320
+ return;
321
+ }
322
+ try {
323
+ validateUrlForSSRF(url);
324
+ }
325
+ catch (error) {
326
+ if (error instanceof SSRFError) {
327
+ res.status(400).json({
328
+ error: 'forbidden_url',
329
+ message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
330
+ });
331
+ return;
332
+ }
333
+ throw error;
334
+ }
335
+ // --- Parse and normalize actions -----------------------------------------
336
+ let postActions;
337
+ if (rawActions !== undefined) {
338
+ try {
339
+ postActions = normalizeActions(rawActions);
340
+ }
341
+ catch (e) {
342
+ res.status(400).json({
343
+ error: 'invalid_request',
344
+ message: `Invalid "actions" parameter: ${e.message}`,
345
+ });
346
+ return;
347
+ }
348
+ }
349
+ // --- Resolve inline extract from body or Firecrawl-compatible formats ---
350
+ let resolvedExtract = extract;
351
+ if (!resolvedExtract && Array.isArray(formats)) {
352
+ const jsonFormat = formats.find((f) => (typeof f === 'object' && f !== null && f.type === 'json') ||
353
+ (typeof f === 'string' && f === 'json'));
354
+ if (jsonFormat && typeof jsonFormat === 'object' && (jsonFormat.schema || jsonFormat.prompt)) {
355
+ resolvedExtract = {
356
+ schema: jsonFormat.schema,
357
+ prompt: jsonFormat.prompt,
358
+ };
359
+ }
360
+ }
361
+ // Validate LLM params if extraction is requested
362
+ if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt)) {
363
+ if (!llmProvider || !VALID_LLM_PROVIDERS.includes(llmProvider)) {
364
+ res.status(400).json({
365
+ error: 'invalid_request',
366
+ message: `"llmProvider" is required for inline extraction and must be one of: ${VALID_LLM_PROVIDERS.join(', ')}`,
367
+ });
368
+ return;
369
+ }
370
+ if (!llmApiKey || typeof llmApiKey !== 'string' || llmApiKey.trim().length === 0) {
371
+ res.status(400).json({
372
+ error: 'invalid_request',
373
+ message: 'Missing or invalid "llmApiKey" (BYOK required for inline extraction)',
374
+ });
375
+ return;
376
+ }
377
+ }
378
+ // --- Build PeelOptions ---------------------------------------------------
379
+ const isSoftLimited = req.auth?.softLimited === true;
380
+ const hasExtraUsage = req.auth?.extraUsageAvailable === true;
381
+ const includeTagsArray = Array.isArray(includeTags) ? includeTags : undefined;
382
+ const excludeTagsArray = Array.isArray(excludeTags) ? excludeTags : undefined;
383
+ const languagesArray = Array.isArray(languages) ? languages : undefined;
384
+ const finalIncludeTags = onlyMainContent === true
385
+ ? ['main', 'article', '.content', '#content']
386
+ : includeTagsArray;
387
+ const resolvedFormat = format || 'markdown';
388
+ if (!['markdown', 'text', 'html'].includes(resolvedFormat)) {
389
+ res.status(400).json({
390
+ error: 'invalid_request',
391
+ message: 'Invalid "format" parameter: must be "markdown", "text", or "html"',
392
+ });
393
+ return;
394
+ }
395
+ const resolvedWait = typeof wait === 'number' ? wait : undefined;
396
+ if (resolvedWait !== undefined && (isNaN(resolvedWait) || resolvedWait < 0 || resolvedWait > 60000)) {
397
+ res.status(400).json({
398
+ error: 'invalid_request',
399
+ message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
400
+ });
401
+ return;
402
+ }
403
+ // When actions are present, force browser mode
404
+ const postHasActions = postActions && postActions.length > 0;
405
+ const postShouldRender = postHasActions || render === true;
406
+ const options = {
407
+ render: (isSoftLimited && !hasExtraUsage && !postHasActions) ? false : postShouldRender,
408
+ wait: (isSoftLimited && !hasExtraUsage) ? 0 : resolvedWait,
409
+ format: resolvedFormat,
410
+ stream: stream === true,
411
+ includeTags: finalIncludeTags,
412
+ excludeTags: excludeTagsArray,
413
+ images: images === true,
414
+ actions: postActions,
415
+ location: location || languagesArray ? {
416
+ country: location,
417
+ languages: languagesArray,
418
+ } : undefined,
419
+ };
420
+ if (isSoftLimited && !hasExtraUsage && render === true && !postHasActions) {
421
+ res.setHeader('X-Degraded', 'render=true downgraded to HTTP-only (quota exceeded)');
422
+ }
423
+ const shouldStream = options.stream === true;
424
+ if (shouldStream) {
425
+ res.setHeader('X-Stream', 'true');
426
+ if (typeof res.flushHeaders === 'function') {
427
+ res.flushHeaders();
428
+ }
429
+ }
430
+ // --- Fetch content -------------------------------------------------------
431
+ const startTime = Date.now();
432
+ const result = await peel(url, options);
433
+ const elapsed = Date.now() - startTime;
434
+ // --- Inline extraction (post-fetch) -------------------------------------
435
+ let jsonData;
436
+ let extractTokensUsed;
437
+ if (resolvedExtract && (resolvedExtract.schema || resolvedExtract.prompt) && llmApiKey) {
438
+ const extractResult = await extractInlineJson(result.content, {
439
+ schema: resolvedExtract.schema,
440
+ prompt: resolvedExtract.prompt,
441
+ llmProvider: llmProvider,
442
+ llmApiKey: llmApiKey.trim(),
443
+ llmModel,
444
+ });
445
+ jsonData = extractResult.data;
446
+ extractTokensUsed = extractResult.tokensUsed;
447
+ }
448
+ // --- Usage tracking (same as GET) ----------------------------------------
449
+ const fetchType = result.method === 'stealth' ? 'stealth' :
450
+ result.method === 'browser' ? 'stealth' : 'basic';
451
+ const pgStore = authStore;
452
+ if (req.auth?.keyInfo?.accountId && typeof pgStore.pool !== 'undefined') {
453
+ pgStore.pool.query(`INSERT INTO usage_logs
454
+ (user_id, endpoint, url, method, processing_time_ms, status_code, ip_address, user_agent)
455
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, [
456
+ req.auth.keyInfo.accountId,
457
+ 'fetch',
458
+ url,
459
+ fetchType,
460
+ elapsed,
461
+ 200,
462
+ req.ip || req.socket.remoteAddress,
463
+ req.get('user-agent'),
464
+ ]).catch((err) => {
465
+ console.error('Failed to log request to usage_logs:', err);
466
+ });
467
+ }
468
+ if (req.auth?.keyInfo?.key && typeof pgStore.trackBurstUsage === 'function') {
469
+ await pgStore.trackBurstUsage(req.auth.keyInfo.key);
470
+ if (isSoftLimited && hasExtraUsage) {
471
+ const extraResult = await pgStore.trackExtraUsage(req.auth.keyInfo.key, fetchType, url, elapsed, 200);
472
+ if (extraResult.success) {
473
+ res.setHeader('X-Extra-Usage-Charged', `$${extraResult.cost.toFixed(4)}`);
474
+ res.setHeader('X-Extra-Usage-New-Balance', extraResult.newBalance.toFixed(2));
475
+ }
476
+ else {
477
+ res.setHeader('X-Degraded', 'Extra usage insufficient, degraded to soft limit');
478
+ }
479
+ }
480
+ else if (!isSoftLimited) {
481
+ await pgStore.trackUsage(req.auth.keyInfo.key, fetchType);
482
+ }
483
+ }
484
+ // Cache result
485
+ const cacheKey = `fetch:${url}:${render}:${wait}:${format}:${includeTags}:${excludeTags}:${images}:${location}:${languages}:${onlyMainContent}:${stream}`;
486
+ if (storeFlag !== false) {
487
+ cache.set(cacheKey, { result, timestamp: Date.now() });
488
+ }
489
+ // --- Build response ------------------------------------------------------
490
+ res.setHeader('X-Cache', 'MISS');
491
+ res.setHeader('X-Credits-Used', '1');
492
+ res.setHeader('X-Processing-Time', elapsed.toString());
493
+ res.setHeader('X-Fetch-Type', fetchType);
494
+ const responseBody = { ...result };
495
+ if (jsonData !== undefined) {
496
+ responseBody.json = jsonData;
497
+ }
498
+ if (extractTokensUsed) {
499
+ responseBody.extractTokensUsed = extractTokensUsed;
500
+ }
501
+ res.json(responseBody);
502
+ }
503
+ catch (error) {
504
+ const err = error;
505
+ console.error('POST fetch/scrape error:', err);
506
+ if (err.code) {
507
+ const safeMessage = err.message.replace(/[<>"']/g, '');
508
+ const statusCode = err.code === 'TIMEOUT' ? 504
509
+ : err.code === 'BLOCKED' ? 403
510
+ : err.code === 'NETWORK' ? 502
511
+ : 500;
512
+ const hints = {
513
+ TIMEOUT: 'Try increasing timeout, or set render:true for JS-heavy sites.',
514
+ BLOCKED: 'Site blocks automated requests. Try render:true or stealth mode.',
515
+ NETWORK: 'Could not reach the target URL. Verify it is correct and online.',
516
+ };
517
+ res.status(statusCode).json({
518
+ error: err.code,
519
+ message: safeMessage,
520
+ hint: hints[err.code] || undefined,
521
+ docs: 'https://webpeel.dev/docs/api-reference#errors',
522
+ });
523
+ }
524
+ else {
525
+ res.status(500).json({
526
+ error: 'internal_error',
527
+ message: 'An unexpected error occurred. If this persists, check https://webpeel.dev/status',
528
+ docs: 'https://webpeel.dev/docs/api-reference#errors',
529
+ });
530
+ }
531
+ }
532
+ }
533
+ router.post('/v1/fetch', handlePostFetch);
534
+ router.post('/v2/scrape', handlePostFetch);
535
+ return router;
536
+ }
537
+ //# sourceMappingURL=fetch.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/server/routes/fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE/E,MAAM,mBAAmB,GAAwB,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;AAOnF,MAAM,UAAU,iBAAiB,CAAC,SAAoB;IACpD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,8DAA8D;IAC9D,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAqB;QAC7C,GAAG,EAAE,IAAI;QACT,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;QAChC,OAAO,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ;QACpC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;YACzB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC5D,IAAI,CAAC;YACH,MAAM,EACJ,GAAG,EACH,MAAM,EACN,IAAI,EACJ,MAAM,EACN,WAAW,EACX,WAAW,EACX,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,OAAO,EACP,MAAM,EACN,YAAY,EACZ,MAAM,GACP,GAAG,GAAG,CAAC,KAAK,CAAC;YAEd,yBAAyB;YACzB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,4GAA4G;oBACrH,OAAO,EAAE,iEAAiE;oBAC1E,IAAI,EAAE,8CAA8C;iBACrD,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,2CAA2C;YAC3C,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,uCAAuC;gBACvC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC;gBAElC,mCAAmC;gBACnC,IAAI,aAAa,KAAK,GAAG,EAAE,CAAC;oBAC1B,yCAAyC;gBAC3C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oBAAoB;iBAC9B,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,iDAAiD;YACjD,IAAI,CAAC;gBACH,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;oBAC/B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,eAAe;wBACtB,OAAO,EAAE,4DAA4D;qBACtE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,iDAAiD;YACjD,IAAI,aAAuC,CAAC;YAC5C,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC3C,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAChC,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBACxC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,yDAAyD;qBACnE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,2CAA2C;YAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,WAAW,IAAI,WAAW,IAAI,MAAM,IAAI,QAAQ,IAAI,SAAS,IAAI,eAAe,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YAExK,oCAAoC;YACpC,MAAM,QAAQ,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,iBAAiB;YACrG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC;gBAC/C,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;oBAChC,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACrE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACxB,OAAO;gBACT,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,WAAW,KAAK,IAAI,CAAC;YACrD,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,mBAAmB,KAAK,IAAI,CAAC;YAE7D,gDAAgD;YAChD,MAAM,gBAAgB,GAAG,WAAW;gBAClC,CAAC,CAAE,WAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACvE,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,gBAAgB,GAAG,WAAW;gBAClC,CAAC,CAAE,WAAsB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACvE,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,cAAc,GAAG,SAAS;gBAC9B,CAAC,CAAE,SAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACrE,CAAC,CAAC,SAAS,CAAC;YAEd,wDAAwD;YACxD,MAAM,gBAAgB,GAAG,eAAe,KAAK,MAAM;gBACjD,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;gBAC7C,CAAC,CAAC,gBAAgB,CAAC;YAErB,qEAAqE;YACrE,MAAM,UAAU,GAAG,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,UAAU,IAAI,MAAM,KAAK,MAAM,CAAC;YAErD,MAAM,OAAO,GAAgB;gBAC3B,kEAAkE;gBAClE,wDAAwD;gBACxD,2CAA2C;gBAC3C,MAAM,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY;gBAC/E,IAAI,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/F,MAAM,EAAG,MAAuC,IAAI,UAAU;gBAC9D,MAAM,EAAE,MAAM,KAAK,MAAM;gBACzB,WAAW,EAAE,gBAAgB;gBAC7B,WAAW,EAAE,gBAAgB;gBAC7B,MAAM,EAAE,MAAM,KAAK,MAAM;gBACzB,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,QAAQ,IAAI,cAAc,CAAC,CAAC,CAAC;oBACrC,OAAO,EAAE,QAA8B;oBACvC,SAAS,EAAE,cAAc;iBAC1B,CAAC,CAAC,CAAC,SAAS;aACd,CAAC;YAEF,gDAAgD;YAChD,IAAI,aAAa,IAAI,CAAC,aAAa,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxE,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,sDAAsD,CAAC,CAAC;YACtF,CAAC;YAED,0BAA0B;YAC1B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACpG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,yDAAyD;iBACnE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,4BAA4B;YAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,mEAAmE;iBAC7E,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC;YAC7C,IAAI,YAAY,EAAE,CAAC;gBACjB,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClC,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;oBAC3C,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEvC,8CAA8C;YAC9C,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACzC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YAEpD,4CAA4C;YAC5C,MAAM,OAAO,GAAG,SAAgB,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxE,6DAA6D;gBAC7D,OAAO,CAAC,IAAI,CAAC,KAAK,CAChB;;kDAEwC,EACxC;oBACE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS;oBAC1B,OAAO;oBACP,GAAG;oBACH,SAAS;oBACT,OAAO;oBACP,GAAG;oBACH,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa;oBAClC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;iBACtB,CACF,CAAC,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBACnB,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;gBAC7D,CAAC,CAAC,CAAC;YACL,CAAC;YAED,6EAA6E;YAC7E,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;gBAE5E,6BAA6B;gBAC7B,MAAM,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpD,oEAAoE;gBACpE,IAAI,aAAa,IAAI,aAAa,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,eAAe,CAC/C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EACpB,SAAS,EACT,GAAG,EACH,OAAO,EACP,GAAG,CAAC,wDAAwD;qBAC7D,CAAC;oBAEF,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;wBACxB,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1E,GAAG,CAAC,SAAS,CAAC,2BAA2B,EAAE,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,CAAC;yBAAM,CAAC;wBACN,+CAA+C;wBAC/C,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,kDAAkD,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;qBAAM,IAAI,CAAC,aAAa,EAAE,CAAC;oBAC1B,+BAA+B;oBAC/B,MAAM,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC5D,CAAC;gBACD,wEAAwE;YAC1E,CAAC;YAED,yDAAyD;YACzD,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;gBAC7B,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;oBAClB,MAAM;oBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAC;YACL,CAAC;YAED,oBAAoB;YACpB,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEzC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,KAAY,CAAC;YAEzB,0CAA0C;YAC1C,MAAM,OAAO,GAAG,SAAgB,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxE,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAa,CAAC;gBACpC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;gBAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;gBAE/C,OAAO,CAAC,IAAI,CAAC,KAAK,CAChB;;kDAEwC,EACxC;oBACE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS;oBAC1B,OAAO;oBACP,GAAG;oBACH,SAAS;oBACT,GAAG;oBACH,GAAG,CAAC,OAAO,IAAI,eAAe;oBAC9B,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa;oBAClC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;iBACtB,CACF,CAAC,KAAK,CAAC,CAAC,MAAW,EAAE,EAAE;oBACtB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,MAAM,CAAC,CAAC;gBAC9D,CAAC,CAAC,CAAC;YACL,CAAC;YAED,sEAAsE;YACtE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,uEAAuE;gBACvE,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;gBAC5E,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;oBAC7C,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;wBAC9B,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;4BAC9B,CAAC,CAAC,GAAG,CAAC;gBAER,MAAM,KAAK,GAA2B;oBACpC,OAAO,EAAE,iFAAiF;oBAC1F,OAAO,EAAE,oGAAoG;oBAC7G,OAAO,EAAE,mFAAmF;iBAC7F,CAAC;gBAEF,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;oBAC1B,KAAK,EAAE,GAAG,CAAC,IAAI;oBACf,OAAO,EAAE,WAAW;oBACpB,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS;oBAClC,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,0CAA0C;gBAC1C,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,gBAAgB;oBACvB,OAAO,EAAE,yGAAyG;oBAClH,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,wEAAwE;IACxE,mDAAmD;IACnD,0EAA0E;IAE1E,KAAK,UAAU,eAAe,CAAC,GAAY,EAAE,GAAa;QACxD,IAAI,CAAC;YACH,MAAM,EACJ,GAAG,EACH,MAAM,EACN,IAAI,EACJ,MAAM,EACN,WAAW,EACX,WAAW,EACX,MAAM,EACN,QAAQ,EACR,SAAS,EACT,eAAe,EACf,OAAO,EAAE,UAAU,EACnB,YAAY,EAAE,SAAS;YACvB,2BAA2B;YAC3B,OAAO,EACP,WAAW,EACX,SAAS,EACT,QAAQ;YACR,qCAAqC;YACrC,OAAO,EACP,MAAM,GACP,GAAG,GAAG,CAAC,IAmBP,CAAC;YAEF,2EAA2E;YAC3E,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,uFAAuF;oBAChG,OAAO,EAAE,wHAAwH;oBACjI,IAAI,EAAE,8CAA8C;iBACrD,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oBAAoB;iBAC9B,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,kBAAkB,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;oBAC/B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,eAAe;wBACtB,OAAO,EAAE,4DAA4D;qBACtE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,4EAA4E;YAC5E,IAAI,WAAqC,CAAC;YAC1C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;gBAC7C,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,gCAAiC,CAAW,CAAC,OAAO,EAAE;qBAChE,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,2EAA2E;YAC3E,IAAI,eAAe,GAAmC,OAAO,CAAC;YAE9D,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAC7B,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;oBAC1D,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,MAAM,CAAC,CACpD,CAAC;gBACF,IAAI,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC7F,eAAe,GAAG;wBAChB,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;qBAC1B,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,iDAAiD;YACjD,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,WAAW,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,WAAgC,CAAC,EAAE,CAAC;oBACpF,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,uEAAuE,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBACjH,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjF,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,sEAAsE;qBAChF,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,WAAW,KAAK,IAAI,CAAC;YACrD,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,mBAAmB,KAAK,IAAI,CAAC;YAE7D,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9E,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9E,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAExE,MAAM,gBAAgB,GAAG,eAAe,KAAK,IAAI;gBAC/C,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;gBAC7C,CAAC,CAAC,gBAAgB,CAAC;YAErB,MAAM,cAAc,GAAI,MAAuC,IAAI,UAAU,CAAC;YAC9E,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,mEAAmE;iBAC7E,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,YAAY,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACjE,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,KAAK,CAAC,EAAE,CAAC;gBACpG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,yDAAyD;iBACnE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,cAAc,GAAG,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7D,MAAM,gBAAgB,GAAG,cAAc,IAAI,MAAM,KAAK,IAAI,CAAC;YAE3D,MAAM,OAAO,GAAgB;gBAC3B,MAAM,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB;gBACvF,IAAI,EAAE,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;gBAC1D,MAAM,EAAE,cAAc;gBACtB,MAAM,EAAE,MAAM,KAAK,IAAI;gBACvB,WAAW,EAAE,gBAAgB;gBAC7B,WAAW,EAAE,gBAAgB;gBAC7B,MAAM,EAAE,MAAM,KAAK,IAAI;gBACvB,OAAO,EAAE,WAAW;gBACpB,QAAQ,EAAE,QAAQ,IAAI,cAAc,CAAC,CAAC,CAAC;oBACrC,OAAO,EAAE,QAAQ;oBACjB,SAAS,EAAE,cAAc;iBAC1B,CAAC,CAAC,CAAC,SAAS;aACd,CAAC;YAEF,IAAI,aAAa,IAAI,CAAC,aAAa,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1E,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,sDAAsD,CAAC,CAAC;YACtF,CAAC;YAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC;YAC7C,IAAI,YAAY,EAAE,CAAC;gBACjB,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAClC,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;oBAC3C,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEvC,2EAA2E;YAC3E,IAAI,QAAyC,CAAC;YAC9C,IAAI,iBAAgE,CAAC;YAErE,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACvF,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,OAAO,EAAE;oBAC5D,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,WAAW,EAAE,WAAgC;oBAC7C,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE;oBAC3B,QAAQ;iBACT,CAAC,CAAC;gBACH,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC;gBAC9B,iBAAiB,GAAG,aAAa,CAAC,UAAU,CAAC;YAC/C,CAAC;YAED,4EAA4E;YAC5E,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACzC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YAEpD,MAAM,OAAO,GAAG,SAAgB,CAAC;YACjC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACxE,OAAO,CAAC,IAAI,CAAC,KAAK,CAChB;;kDAEwC,EACxC;oBACE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS;oBAC1B,OAAO;oBACP,GAAG;oBACH,SAAS;oBACT,OAAO;oBACP,GAAG;oBACH,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa;oBAClC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;iBACtB,CACF,CAAC,KAAK,CAAC,CAAC,GAAQ,EAAE,EAAE;oBACnB,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,GAAG,CAAC,CAAC;gBAC7D,CAAC,CAAC,CAAC;YACL,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;gBAC5E,MAAM,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpD,IAAI,aAAa,IAAI,aAAa,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,eAAe,CAC/C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EACpB,SAAS,EACT,GAAG,EACH,OAAO,EACP,GAAG,CACJ,CAAC;oBACF,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;wBACxB,GAAG,CAAC,SAAS,CAAC,uBAAuB,EAAE,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC1E,GAAG,CAAC,SAAS,CAAC,2BAA2B,EAAE,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChF,CAAC;yBAAM,CAAC;wBACN,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,kDAAkD,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;qBAAM,IAAI,CAAC,aAAa,EAAE,CAAC;oBAC1B,MAAM,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC5D,CAAC;YACH,CAAC;YAED,eAAe;YACf,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,WAAW,IAAI,WAAW,IAAI,MAAM,IAAI,QAAQ,IAAI,SAAS,IAAI,eAAe,IAAI,MAAM,EAAE,CAAC;YAC1J,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBACxB,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACzD,CAAC;YAED,4EAA4E;YAC5E,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEzC,MAAM,YAAY,GAAQ,EAAE,GAAG,MAAM,EAAE,CAAC;YACxC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,YAAY,CAAC,IAAI,GAAG,QAAQ,CAAC;YAC/B,CAAC;YACD,IAAI,iBAAiB,EAAE,CAAC;gBACtB,YAAY,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;YACrD,CAAC;YAED,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,KAAY,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,CAAC;YAE/C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gBACvD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;oBAC7C,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;wBAC9B,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG;4BAC9B,CAAC,CAAC,GAAG,CAAC;gBAER,MAAM,KAAK,GAA2B;oBACpC,OAAO,EAAE,gEAAgE;oBACzE,OAAO,EAAE,kEAAkE;oBAC3E,OAAO,EAAE,kEAAkE;iBAC5E,CAAC;gBAEF,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC;oBAC1B,KAAK,EAAE,GAAG,CAAC,IAAI;oBACf,OAAO,EAAE,WAAW;oBACpB,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS;oBAClC,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,gBAAgB;oBACvB,OAAO,EAAE,kFAAkF;oBAC3F,IAAI,EAAE,+CAA+C;iBACtD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC1C,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IAE3C,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Health check endpoint
3
+ * NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
4
+ * so it's never blocked by rate limiting (Render hits it every ~30s).
5
+ */
6
+ import { Router } from 'express';
7
+ export declare function createHealthRouter(): Router;
8
+ //# sourceMappingURL=health.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../../src/server/routes/health.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAmBpD,wBAAgB,kBAAkB,IAAI,MAAM,CAe3C"}
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Health check endpoint
3
+ * NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
4
+ * so it's never blocked by rate limiting (Render hits it every ~30s).
5
+ */
6
+ import { Router } from 'express';
7
+ import { readFileSync } from 'fs';
8
+ import { join, dirname } from 'path';
9
+ import { fileURLToPath } from 'url';
10
+ const startTime = Date.now();
11
+ // Read version from package.json at startup
12
+ let version = '0.0.0';
13
+ try {
14
+ const __dirname = dirname(fileURLToPath(import.meta.url));
15
+ // Walk up from dist/server/routes/ to project root
16
+ const pkgPath = join(__dirname, '..', '..', '..', 'package.json');
17
+ const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
18
+ version = pkg.version;
19
+ }
20
+ catch {
21
+ // Fallback if package.json can't be read
22
+ }
23
+ export function createHealthRouter() {
24
+ const router = Router();
25
+ router.get('/health', (_req, res) => {
26
+ const uptime = Math.floor((Date.now() - startTime) / 1000);
27
+ res.json({
28
+ status: 'healthy',
29
+ version,
30
+ uptime,
31
+ timestamp: new Date().toISOString(),
32
+ });
33
+ });
34
+ return router;
35
+ }
36
+ //# sourceMappingURL=health.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"health.js","sourceRoot":"","sources":["../../../src/server/routes/health.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAE7B,4CAA4C;AAC5C,IAAI,OAAO,GAAG,OAAO,CAAC;AACtB,IAAI,CAAC;IACH,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,mDAAmD;IACnD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;AACxB,CAAC;AAAC,MAAM,CAAC;IACP,yCAAyC;AAC3C,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC;QAE3D,GAAG,CAAC,IAAI,CAAC;YACP,MAAM,EAAE,SAAS;YACjB,OAAO;YACP,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Async jobs API - crawl endpoints with SSE support
3
+ */
4
+ import { Router } from 'express';
5
+ import type { AuthStore } from '../auth-store.js';
6
+ import type { IJobQueue } from '../job-queue.js';
7
+ export declare function createJobsRouter(jobQueue: IJobQueue, authStore: AuthStore): Router;
8
+ //# sourceMappingURL=jobs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jobs.d.ts","sourceRoot":"","sources":["../../../src/server/routes/jobs.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAKpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAGjD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,GAAG,MAAM,CA4alF"}