webpeel 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -20
- package/dist/cli.js +271 -13
- package/dist/cli.js.map +1 -1
- package/dist/core/fetcher.d.ts +7 -1
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +85 -23
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/markdown.d.ts +5 -0
- package/dist/core/markdown.d.ts.map +1 -1
- package/dist/core/markdown.js +19 -0
- package/dist/core/markdown.js.map +1 -1
- package/dist/core/strategies.d.ts +8 -0
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +14 -4
- package/dist/core/strategies.js.map +1 -1
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +64 -7
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +147 -3
- package/dist/mcp/server.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +29 -7
- package/dist/server/app.d.ts +0 -13
- package/dist/server/app.d.ts.map +0 -1
- package/dist/server/app.js +0 -89
- package/dist/server/app.js.map +0 -1
- package/dist/server/auth-store.d.ts +0 -28
- package/dist/server/auth-store.d.ts.map +0 -1
- package/dist/server/auth-store.js +0 -87
- package/dist/server/auth-store.js.map +0 -1
- package/dist/server/middleware/auth.d.ts +0 -18
- package/dist/server/middleware/auth.d.ts.map +0 -1
- package/dist/server/middleware/auth.js +0 -55
- package/dist/server/middleware/auth.js.map +0 -1
- package/dist/server/middleware/rate-limit.d.ts +0 -23
- package/dist/server/middleware/rate-limit.d.ts.map +0 -1
- package/dist/server/middleware/rate-limit.js +0 -85
- package/dist/server/middleware/rate-limit.js.map +0 -1
- package/dist/server/routes/fetch.d.ts +0 -7
- package/dist/server/routes/fetch.d.ts.map +0 -1
- package/dist/server/routes/fetch.js +0 -127
- package/dist/server/routes/fetch.js.map +0 -1
- package/dist/server/routes/health.d.ts +0 -6
- package/dist/server/routes/health.d.ts.map +0 -1
- package/dist/server/routes/health.js +0 -19
- package/dist/server/routes/health.js.map +0 -1
- package/dist/server/routes/search.d.ts +0 -7
- package/dist/server/routes/search.d.ts.map +0 -1
- package/dist/server/routes/search.js +0 -124
- package/dist/server/routes/search.js.map +0 -1
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Fetch endpoint with caching
|
|
3
|
-
*/
|
|
4
|
-
import { Router } from 'express';
|
|
5
|
-
import { peel } from '../../index.js';
|
|
6
|
-
import { LRUCache } from 'lru-cache';
|
|
7
|
-
export function createFetchRouter(authStore) {
|
|
8
|
-
const router = Router();
|
|
9
|
-
// LRU cache: 5 minute TTL, max 1000 entries, 100MB total size
|
|
10
|
-
const cache = new LRUCache({
|
|
11
|
-
max: 1000,
|
|
12
|
-
ttl: 5 * 60 * 1000, // 5 minutes
|
|
13
|
-
maxSize: 100 * 1024 * 1024, // 100MB
|
|
14
|
-
sizeCalculation: (entry) => {
|
|
15
|
-
return JSON.stringify(entry).length;
|
|
16
|
-
},
|
|
17
|
-
});
|
|
18
|
-
router.get('/v1/fetch', async (req, res) => {
|
|
19
|
-
try {
|
|
20
|
-
const { url, render, wait, format } = req.query;
|
|
21
|
-
// Validate URL parameter
|
|
22
|
-
if (!url || typeof url !== 'string') {
|
|
23
|
-
res.status(400).json({
|
|
24
|
-
error: 'invalid_request',
|
|
25
|
-
message: 'Missing or invalid "url" parameter',
|
|
26
|
-
});
|
|
27
|
-
return;
|
|
28
|
-
}
|
|
29
|
-
// SECURITY: Validate URL format and length
|
|
30
|
-
if (url.length > 2048) {
|
|
31
|
-
res.status(400).json({
|
|
32
|
-
error: 'invalid_url',
|
|
33
|
-
message: 'URL too long (max 2048 characters)',
|
|
34
|
-
});
|
|
35
|
-
return;
|
|
36
|
-
}
|
|
37
|
-
try {
|
|
38
|
-
const parsed = new URL(url);
|
|
39
|
-
// Normalize URL for consistent caching
|
|
40
|
-
const normalizedUrl = parsed.href;
|
|
41
|
-
// Use normalized URL for cache key
|
|
42
|
-
if (normalizedUrl !== url) {
|
|
43
|
-
// URL was normalized, update for caching
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
catch {
|
|
47
|
-
res.status(400).json({
|
|
48
|
-
error: 'invalid_url',
|
|
49
|
-
message: 'Invalid URL format',
|
|
50
|
-
});
|
|
51
|
-
return;
|
|
52
|
-
}
|
|
53
|
-
// Build cache key
|
|
54
|
-
const cacheKey = `fetch:${url}:${render}:${wait}:${format}`;
|
|
55
|
-
// Check cache
|
|
56
|
-
const cached = cache.get(cacheKey);
|
|
57
|
-
if (cached) {
|
|
58
|
-
res.setHeader('X-Cache', 'HIT');
|
|
59
|
-
res.setHeader('X-Cache-Age', Math.floor((Date.now() - cached.timestamp) / 1000).toString());
|
|
60
|
-
res.json(cached.result);
|
|
61
|
-
return;
|
|
62
|
-
}
|
|
63
|
-
// Parse options
|
|
64
|
-
const options = {
|
|
65
|
-
render: render === 'true',
|
|
66
|
-
wait: wait ? parseInt(wait, 10) : undefined,
|
|
67
|
-
format: format || 'markdown',
|
|
68
|
-
};
|
|
69
|
-
// Validate wait parameter
|
|
70
|
-
if (options.wait !== undefined && (isNaN(options.wait) || options.wait < 0 || options.wait > 60000)) {
|
|
71
|
-
res.status(400).json({
|
|
72
|
-
error: 'invalid_request',
|
|
73
|
-
message: 'Invalid "wait" parameter: must be between 0 and 60000ms',
|
|
74
|
-
});
|
|
75
|
-
return;
|
|
76
|
-
}
|
|
77
|
-
// Validate format parameter
|
|
78
|
-
if (!['markdown', 'text', 'html'].includes(options.format || '')) {
|
|
79
|
-
res.status(400).json({
|
|
80
|
-
error: 'invalid_request',
|
|
81
|
-
message: 'Invalid "format" parameter: must be "markdown", "text", or "html"',
|
|
82
|
-
});
|
|
83
|
-
return;
|
|
84
|
-
}
|
|
85
|
-
// Fetch content
|
|
86
|
-
const startTime = Date.now();
|
|
87
|
-
const result = await peel(url, options);
|
|
88
|
-
const elapsed = Date.now() - startTime;
|
|
89
|
-
// Track usage (1 credit per fetch)
|
|
90
|
-
if (req.auth?.keyInfo?.key) {
|
|
91
|
-
await authStore.trackUsage(req.auth.keyInfo.key, 1);
|
|
92
|
-
}
|
|
93
|
-
// Cache result
|
|
94
|
-
cache.set(cacheKey, {
|
|
95
|
-
result,
|
|
96
|
-
timestamp: Date.now(),
|
|
97
|
-
});
|
|
98
|
-
// Add usage headers
|
|
99
|
-
res.setHeader('X-Cache', 'MISS');
|
|
100
|
-
res.setHeader('X-Credits-Used', '1');
|
|
101
|
-
res.setHeader('X-Processing-Time', elapsed.toString());
|
|
102
|
-
res.json(result);
|
|
103
|
-
}
|
|
104
|
-
catch (error) {
|
|
105
|
-
const err = error;
|
|
106
|
-
// SECURITY: Sanitize error messages to prevent information disclosure
|
|
107
|
-
if (err.code) {
|
|
108
|
-
// WebPeelError from core library - safe to expose
|
|
109
|
-
const safeMessage = err.message.replace(/[<>"']/g, ''); // Remove HTML chars
|
|
110
|
-
res.status(500).json({
|
|
111
|
-
error: err.code,
|
|
112
|
-
message: safeMessage,
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
// Unexpected error - generic message only
|
|
117
|
-
console.error('Fetch error:', err); // Log full error server-side
|
|
118
|
-
res.status(500).json({
|
|
119
|
-
error: 'internal_error',
|
|
120
|
-
message: 'An unexpected error occurred while fetching the URL',
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
});
|
|
125
|
-
return router;
|
|
126
|
-
}
|
|
127
|
-
//# sourceMappingURL=fetch.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/server/routes/fetch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,MAAM,UAAU,iBAAiB,CAAC,SAAoB;IACpD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,8DAA8D;IAC9D,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAqB;QAC7C,GAAG,EAAE,IAAI;QACT,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;QAChC,OAAO,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ;QACpC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;YACzB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC5D,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC;YAEhD,yBAAyB;YACzB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,2CAA2C;YAC3C,IAAI,GAAG,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC5B,uCAAuC;gBACvC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC;gBAElC,mCAAmC;gBACnC,IAAI,aAAa,KAAK,GAAG,EAAE,CAAC;oBAC1B,yCAAyC;gBAC3C,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,aAAa;oBACpB,OAAO,EAAE,oBAAoB;iBAC9B,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,kBAAkB;YAClB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,EAAE,CAAC;YAE5D,cAAc;YACd,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAChC,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC5F,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACxB,OAAO;YACT,CAAC;YAED,gBAAgB;YAChB,MAAM,OAAO,GAAgB;gBAC3B,MAAM,EAAE,MAAM,KAAK,MAAM;gBACzB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;gBACrD,MAAM,EAAG,MAAuC,IAAI,UAAU;aAC/D,CAAC;YAEF,0BAA0B;YAC1B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACpG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,yDAAyD;iBACnE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,4BAA4B;YAC5B,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,mEAAmE;iBAC7E,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEvC,mCAAmC;YACnC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;gBAC3B,MAAM,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACtD,CAAC;YAED,eAAe;YACf,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;gBAClB,MAAM;gBACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;YAEH,oBAAoB;YACpB,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEvD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,KAAY,CAAC;YAEzB,sEAAsE;YACtE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,kDAAkD;gBAClD,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,oBAAoB;gBAC5E,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,GAAG,CAAC,IAAI;oBACf,OAAO,EAAE,WAAW;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,0CAA0C;gBAC1C,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;gBACjE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,gBAAgB;oBACvB,OAAO,EAAE,qDAAqD;iBAC/D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../../src/server/routes/health.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAIpD,wBAAgB,kBAAkB,IAAI,MAAM,CAe3C"}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Health check endpoint
|
|
3
|
-
*/
|
|
4
|
-
import { Router } from 'express';
|
|
5
|
-
const startTime = Date.now();
|
|
6
|
-
export function createHealthRouter() {
|
|
7
|
-
const router = Router();
|
|
8
|
-
router.get('/health', (_req, res) => {
|
|
9
|
-
const uptime = Math.floor((Date.now() - startTime) / 1000);
|
|
10
|
-
res.json({
|
|
11
|
-
status: 'healthy',
|
|
12
|
-
version: process.env.npm_package_version || '1.0.0',
|
|
13
|
-
uptime,
|
|
14
|
-
timestamp: new Date().toISOString(),
|
|
15
|
-
});
|
|
16
|
-
});
|
|
17
|
-
return router;
|
|
18
|
-
}
|
|
19
|
-
//# sourceMappingURL=health.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"health.js","sourceRoot":"","sources":["../../../src/server/routes/health.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAEpD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAE7B,MAAM,UAAU,kBAAkB;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,IAAa,EAAE,GAAa,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC;QAE3D,GAAG,CAAC,IAAI,CAAC;YACP,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO;YACnD,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../../src/server/routes/search.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAIpD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAa7C,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAsI/D"}
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Search endpoint with caching
|
|
3
|
-
*/
|
|
4
|
-
import { Router } from 'express';
|
|
5
|
-
import { fetch as undiciFetch } from 'undici';
|
|
6
|
-
import { load } from 'cheerio';
|
|
7
|
-
import { LRUCache } from 'lru-cache';
|
|
8
|
-
export function createSearchRouter(authStore) {
|
|
9
|
-
const router = Router();
|
|
10
|
-
// LRU cache: 15 minute TTL, max 500 entries, 50MB total size
|
|
11
|
-
const cache = new LRUCache({
|
|
12
|
-
max: 500,
|
|
13
|
-
ttl: 15 * 60 * 1000, // 15 minutes
|
|
14
|
-
maxSize: 50 * 1024 * 1024, // 50MB
|
|
15
|
-
sizeCalculation: (entry) => {
|
|
16
|
-
return JSON.stringify(entry).length;
|
|
17
|
-
},
|
|
18
|
-
});
|
|
19
|
-
router.get('/v1/search', async (req, res) => {
|
|
20
|
-
try {
|
|
21
|
-
const { q, count } = req.query;
|
|
22
|
-
// Validate query parameter
|
|
23
|
-
if (!q || typeof q !== 'string') {
|
|
24
|
-
res.status(400).json({
|
|
25
|
-
error: 'invalid_request',
|
|
26
|
-
message: 'Missing or invalid "q" parameter',
|
|
27
|
-
});
|
|
28
|
-
return;
|
|
29
|
-
}
|
|
30
|
-
// Parse and validate count
|
|
31
|
-
const resultCount = count ? parseInt(count, 10) : 5;
|
|
32
|
-
if (isNaN(resultCount) || resultCount < 1 || resultCount > 10) {
|
|
33
|
-
res.status(400).json({
|
|
34
|
-
error: 'invalid_request',
|
|
35
|
-
message: 'Invalid "count" parameter: must be between 1 and 10',
|
|
36
|
-
});
|
|
37
|
-
return;
|
|
38
|
-
}
|
|
39
|
-
// Build cache key
|
|
40
|
-
const cacheKey = `search:${q}:${resultCount}`;
|
|
41
|
-
// Check cache
|
|
42
|
-
const cached = cache.get(cacheKey);
|
|
43
|
-
if (cached) {
|
|
44
|
-
res.setHeader('X-Cache', 'HIT');
|
|
45
|
-
res.setHeader('X-Cache-Age', Math.floor((Date.now() - cached.timestamp) / 1000).toString());
|
|
46
|
-
res.json({
|
|
47
|
-
query: q,
|
|
48
|
-
count: cached.results.length,
|
|
49
|
-
results: cached.results,
|
|
50
|
-
});
|
|
51
|
-
return;
|
|
52
|
-
}
|
|
53
|
-
// Perform search
|
|
54
|
-
const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`;
|
|
55
|
-
const startTime = Date.now();
|
|
56
|
-
const response = await undiciFetch(searchUrl, {
|
|
57
|
-
headers: {
|
|
58
|
-
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
|
59
|
-
},
|
|
60
|
-
});
|
|
61
|
-
if (!response.ok) {
|
|
62
|
-
throw new Error(`Search failed: HTTP ${response.status}`);
|
|
63
|
-
}
|
|
64
|
-
const html = await response.text();
|
|
65
|
-
const $ = load(html);
|
|
66
|
-
const results = [];
|
|
67
|
-
$('.result').each((_i, elem) => {
|
|
68
|
-
if (results.length >= resultCount)
|
|
69
|
-
return;
|
|
70
|
-
const $result = $(elem);
|
|
71
|
-
let title = $result.find('.result__title').text().trim();
|
|
72
|
-
let url = $result.find('.result__url').attr('href') || '';
|
|
73
|
-
let snippet = $result.find('.result__snippet').text().trim();
|
|
74
|
-
// SECURITY: Validate and sanitize results
|
|
75
|
-
if (!title || !url)
|
|
76
|
-
return;
|
|
77
|
-
// Only allow HTTP/HTTPS URLs
|
|
78
|
-
try {
|
|
79
|
-
const parsed = new URL(url);
|
|
80
|
-
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
81
|
-
return;
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
catch {
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
// Limit text lengths to prevent bloat
|
|
88
|
-
title = title.slice(0, 200);
|
|
89
|
-
snippet = snippet.slice(0, 500);
|
|
90
|
-
results.push({ title, url, snippet });
|
|
91
|
-
});
|
|
92
|
-
const elapsed = Date.now() - startTime;
|
|
93
|
-
// Track usage (1 credit per search)
|
|
94
|
-
if (req.auth?.keyInfo?.key) {
|
|
95
|
-
await authStore.trackUsage(req.auth.keyInfo.key, 1);
|
|
96
|
-
}
|
|
97
|
-
// Cache results
|
|
98
|
-
cache.set(cacheKey, {
|
|
99
|
-
results,
|
|
100
|
-
timestamp: Date.now(),
|
|
101
|
-
});
|
|
102
|
-
// Add headers
|
|
103
|
-
res.setHeader('X-Cache', 'MISS');
|
|
104
|
-
res.setHeader('X-Credits-Used', '1');
|
|
105
|
-
res.setHeader('X-Processing-Time', elapsed.toString());
|
|
106
|
-
res.json({
|
|
107
|
-
query: q,
|
|
108
|
-
count: results.length,
|
|
109
|
-
results,
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
catch (error) {
|
|
113
|
-
const err = error;
|
|
114
|
-
// SECURITY: Generic error message to prevent information disclosure
|
|
115
|
-
console.error('Search error:', err); // Log full error server-side
|
|
116
|
-
res.status(500).json({
|
|
117
|
-
error: 'search_failed',
|
|
118
|
-
message: 'Search request failed. Please try again.',
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
});
|
|
122
|
-
return router;
|
|
123
|
-
}
|
|
124
|
-
//# sourceMappingURL=search.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../../src/server/routes/search.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAcrC,MAAM,UAAU,kBAAkB,CAAC,SAAoB;IACrD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,6DAA6D;IAC7D,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAqB;QAC7C,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,aAAa;QAClC,OAAO,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO;QAClC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;YACzB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC7D,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC;YAE/B,2BAA2B;YAC3B,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAChC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,kCAAkC;iBAC5C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,2BAA2B;YAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAe,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;gBAC9D,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,qDAAqD;iBAC/D,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,kBAAkB;YAClB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,WAAW,EAAE,CAAC;YAE9C,cAAc;YACd,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,MAAM,EAAE,CAAC;gBACX,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAChC,GAAG,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC5F,GAAG,CAAC,IAAI,CAAC;oBACP,KAAK,EAAE,CAAC;oBACR,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;oBAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,iBAAiB;YACjB,MAAM,SAAS,GAAG,uCAAuC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;YACjF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE7B,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;gBAC5C,OAAO,EAAE;oBACP,YAAY,EAAE,oEAAoE;iBACnF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAErB,MAAM,OAAO,GAAmB,EAAE,CAAC;YAEnC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;gBAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,WAAW;oBAAE,OAAO;gBAE1C,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxB,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACzD,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC1D,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAE7D,0CAA0C;gBAC1C,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG;oBAAE,OAAO;gBAE3B,6BAA6B;gBAC7B,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACnD,OAAO;oBACT,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;gBAED,sCAAsC;gBACtC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC5B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAEhC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAEvC,oCAAoC;YACpC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;gBAC3B,MAAM,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACtD,CAAC;YAED,gBAAgB;YAChB,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;gBAClB,OAAO;gBACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;YAEH,cAAc;YACd,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;YACrC,GAAG,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEvD,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,CAAC;gBACR,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,oEAAoE;YACpE,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;YAClE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,eAAe;gBACtB,OAAO,EAAE,0CAA0C;aACpD,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|