webpeel 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -9
- package/dist/cli.js +97 -6
- package/dist/cli.js.map +1 -1
- package/dist/core/actions.d.ts +28 -0
- package/dist/core/actions.d.ts.map +1 -1
- package/dist/core/actions.js +60 -0
- package/dist/core/actions.js.map +1 -1
- package/dist/core/bm25-filter.d.ts +10 -0
- package/dist/core/bm25-filter.d.ts.map +1 -1
- package/dist/core/bm25-filter.js +40 -0
- package/dist/core/bm25-filter.js.map +1 -1
- package/dist/core/content-pruner.d.ts +12 -5
- package/dist/core/content-pruner.d.ts.map +1 -1
- package/dist/core/content-pruner.js +247 -190
- package/dist/core/content-pruner.js.map +1 -1
- package/dist/core/research.d.ts +67 -0
- package/dist/core/research.d.ts.map +1 -0
- package/dist/core/research.js +254 -0
- package/dist/core/research.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +37 -3
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +107 -2
- package/dist/mcp/server.js.map +1 -1
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.d.ts.map +1 -0
- package/dist/server/app.js +189 -0
- package/dist/server/app.js.map +1 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.d.ts.map +1 -0
- package/dist/server/auth-store.js +89 -0
- package/dist/server/auth-store.js.map +1 -0
- package/dist/server/job-queue.d.ts +93 -0
- package/dist/server/job-queue.d.ts.map +1 -0
- package/dist/server/job-queue.js +144 -0
- package/dist/server/job-queue.js.map +1 -0
- package/dist/server/middleware/auth.d.ts +28 -0
- package/dist/server/middleware/auth.d.ts.map +1 -0
- package/dist/server/middleware/auth.js +183 -0
- package/dist/server/middleware/auth.js.map +1 -0
- package/dist/server/middleware/rate-limit.d.ts +23 -0
- package/dist/server/middleware/rate-limit.d.ts.map +1 -0
- package/dist/server/middleware/rate-limit.js +126 -0
- package/dist/server/middleware/rate-limit.js.map +1 -0
- package/dist/server/middleware/url-validator.d.ts +16 -0
- package/dist/server/middleware/url-validator.d.ts.map +1 -0
- package/dist/server/middleware/url-validator.js +187 -0
- package/dist/server/middleware/url-validator.js.map +1 -0
- package/dist/server/pg-auth-store.d.ts +129 -0
- package/dist/server/pg-auth-store.d.ts.map +1 -0
- package/dist/server/pg-auth-store.js +457 -0
- package/dist/server/pg-auth-store.js.map +1 -0
- package/dist/server/pg-job-queue.d.ts +60 -0
- package/dist/server/pg-job-queue.d.ts.map +1 -0
- package/dist/server/pg-job-queue.js +365 -0
- package/dist/server/pg-job-queue.js.map +1 -0
- package/dist/server/premium/domain-intel.d.ts +17 -0
- package/dist/server/premium/domain-intel.d.ts.map +1 -0
- package/dist/server/premium/domain-intel.js +134 -0
- package/dist/server/premium/domain-intel.js.map +1 -0
- package/dist/server/premium/index.d.ts +18 -0
- package/dist/server/premium/index.d.ts.map +1 -0
- package/dist/server/premium/index.js +36 -0
- package/dist/server/premium/index.js.map +1 -0
- package/dist/server/premium/swr-cache.d.ts +15 -0
- package/dist/server/premium/swr-cache.d.ts.map +1 -0
- package/dist/server/premium/swr-cache.js +35 -0
- package/dist/server/premium/swr-cache.js.map +1 -0
- package/dist/server/routes/activity.d.ts +7 -0
- package/dist/server/routes/activity.d.ts.map +1 -0
- package/dist/server/routes/activity.js +66 -0
- package/dist/server/routes/activity.js.map +1 -0
- package/dist/server/routes/agent.d.ts +12 -0
- package/dist/server/routes/agent.d.ts.map +1 -0
- package/dist/server/routes/agent.js +356 -0
- package/dist/server/routes/agent.js.map +1 -0
- package/dist/server/routes/answer.d.ts +6 -0
- package/dist/server/routes/answer.d.ts.map +1 -0
- package/dist/server/routes/answer.js +124 -0
- package/dist/server/routes/answer.js.map +1 -0
- package/dist/server/routes/batch.d.ts +7 -0
- package/dist/server/routes/batch.d.ts.map +1 -0
- package/dist/server/routes/batch.js +287 -0
- package/dist/server/routes/batch.js.map +1 -0
- package/dist/server/routes/cli-usage.d.ts +7 -0
- package/dist/server/routes/cli-usage.d.ts.map +1 -0
- package/dist/server/routes/cli-usage.js +121 -0
- package/dist/server/routes/cli-usage.js.map +1 -0
- package/dist/server/routes/compat.d.ts +24 -0
- package/dist/server/routes/compat.d.ts.map +1 -0
- package/dist/server/routes/compat.js +651 -0
- package/dist/server/routes/compat.js.map +1 -0
- package/dist/server/routes/extract.d.ts +9 -0
- package/dist/server/routes/extract.d.ts.map +1 -0
- package/dist/server/routes/extract.js +121 -0
- package/dist/server/routes/extract.js.map +1 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.d.ts.map +1 -0
- package/dist/server/routes/fetch.js +537 -0
- package/dist/server/routes/fetch.js.map +1 -0
- package/dist/server/routes/health.d.ts +8 -0
- package/dist/server/routes/health.d.ts.map +1 -0
- package/dist/server/routes/health.js +36 -0
- package/dist/server/routes/health.js.map +1 -0
- package/dist/server/routes/jobs.d.ts +8 -0
- package/dist/server/routes/jobs.d.ts.map +1 -0
- package/dist/server/routes/jobs.js +374 -0
- package/dist/server/routes/jobs.js.map +1 -0
- package/dist/server/routes/mcp.d.ts +16 -0
- package/dist/server/routes/mcp.d.ts.map +1 -0
- package/dist/server/routes/mcp.js +475 -0
- package/dist/server/routes/mcp.js.map +1 -0
- package/dist/server/routes/oauth.d.ts +10 -0
- package/dist/server/routes/oauth.d.ts.map +1 -0
- package/dist/server/routes/oauth.js +296 -0
- package/dist/server/routes/oauth.js.map +1 -0
- package/dist/server/routes/screenshot.d.ts +10 -0
- package/dist/server/routes/screenshot.d.ts.map +1 -0
- package/dist/server/routes/screenshot.js +217 -0
- package/dist/server/routes/screenshot.js.map +1 -0
- package/dist/server/routes/search.d.ts +7 -0
- package/dist/server/routes/search.d.ts.map +1 -0
- package/dist/server/routes/search.js +287 -0
- package/dist/server/routes/search.js.map +1 -0
- package/dist/server/routes/stats.d.ts +7 -0
- package/dist/server/routes/stats.d.ts.map +1 -0
- package/dist/server/routes/stats.js +65 -0
- package/dist/server/routes/stats.js.map +1 -0
- package/dist/server/routes/stripe.d.ts +9 -0
- package/dist/server/routes/stripe.d.ts.map +1 -0
- package/dist/server/routes/stripe.js +233 -0
- package/dist/server/routes/stripe.js.map +1 -0
- package/dist/server/routes/users.d.ts +9 -0
- package/dist/server/routes/users.d.ts.map +1 -0
- package/dist/server/routes/users.js +954 -0
- package/dist/server/routes/users.js.map +1 -0
- package/dist/server/routes/webhooks.d.ts +15 -0
- package/dist/server/routes/webhooks.d.ts.map +1 -0
- package/dist/server/routes/webhooks.js +73 -0
- package/dist/server/routes/webhooks.js.map +1 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.d.ts.map +1 -0
- package/dist/server/sentry.js +39 -0
- package/dist/server/sentry.js.map +1 -0
- package/dist/types.d.ts +13 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +3 -2
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL validation middleware to prevent SSRF attacks
|
|
3
|
+
* Validates URLs BEFORE any network request is made
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Validate URL to prevent SSRF attacks
|
|
7
|
+
* Blocks localhost, private IPs, link-local addresses, and non-HTTP(S) protocols
|
|
8
|
+
*/
|
|
9
|
+
export function validateUrlForSSRF(urlString) {
|
|
10
|
+
// Parse URL
|
|
11
|
+
let url;
|
|
12
|
+
try {
|
|
13
|
+
url = new URL(urlString);
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
throw new Error('Invalid URL format');
|
|
17
|
+
}
|
|
18
|
+
// Only allow HTTP(S)
|
|
19
|
+
if (!['http:', 'https:'].includes(url.protocol)) {
|
|
20
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
21
|
+
}
|
|
22
|
+
const hostname = url.hostname.toLowerCase();
|
|
23
|
+
// Block localhost patterns
|
|
24
|
+
const localhostPatterns = ['localhost', '0.0.0.0'];
|
|
25
|
+
if (localhostPatterns.some(pattern => hostname === pattern || hostname.endsWith('.' + pattern))) {
|
|
26
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
27
|
+
}
|
|
28
|
+
// Parse and validate IP addresses
|
|
29
|
+
const ipv4Info = parseIPv4(hostname);
|
|
30
|
+
if (ipv4Info) {
|
|
31
|
+
validateIPv4ForSSRF(ipv4Info);
|
|
32
|
+
}
|
|
33
|
+
// Validate IPv6
|
|
34
|
+
if (hostname.includes(':')) {
|
|
35
|
+
validateIPv6ForSSRF(hostname);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* SSRF Error class
|
|
40
|
+
*/
|
|
41
|
+
export class SSRFError extends Error {
|
|
42
|
+
constructor(message) {
|
|
43
|
+
super(message);
|
|
44
|
+
this.name = 'SSRFError';
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Parse IPv4 address in any format (dotted, hex, octal, decimal)
|
|
49
|
+
*/
|
|
50
|
+
function parseIPv4(hostname) {
|
|
51
|
+
const cleaned = hostname.replace(/^\[|\]$/g, '');
|
|
52
|
+
// Standard dotted notation: 192.168.1.1
|
|
53
|
+
const dottedRegex = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
|
|
54
|
+
const dottedMatch = cleaned.match(dottedRegex);
|
|
55
|
+
if (dottedMatch) {
|
|
56
|
+
const octets = dottedMatch.slice(1).map(Number);
|
|
57
|
+
if (octets.every(o => o >= 0 && o <= 255)) {
|
|
58
|
+
return octets;
|
|
59
|
+
}
|
|
60
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
61
|
+
}
|
|
62
|
+
// Hex notation: 0x7f000001
|
|
63
|
+
if (/^0x[0-9a-fA-F]+$/.test(cleaned)) {
|
|
64
|
+
const num = parseInt(cleaned, 16);
|
|
65
|
+
return [
|
|
66
|
+
(num >>> 24) & 0xff,
|
|
67
|
+
(num >>> 16) & 0xff,
|
|
68
|
+
(num >>> 8) & 0xff,
|
|
69
|
+
num & 0xff,
|
|
70
|
+
];
|
|
71
|
+
}
|
|
72
|
+
// Octal notation
|
|
73
|
+
if (/^0[0-7]/.test(cleaned)) {
|
|
74
|
+
if (/^0[0-7]+$/.test(cleaned)) {
|
|
75
|
+
const num = parseInt(cleaned, 8);
|
|
76
|
+
if (num <= 0xffffffff) {
|
|
77
|
+
return [
|
|
78
|
+
(num >>> 24) & 0xff,
|
|
79
|
+
(num >>> 16) & 0xff,
|
|
80
|
+
(num >>> 8) & 0xff,
|
|
81
|
+
num & 0xff,
|
|
82
|
+
];
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const parts = cleaned.split('.');
|
|
86
|
+
if (parts.length === 4) {
|
|
87
|
+
const octets = parts.map(p => parseInt(p, /^0[0-7]/.test(p) ? 8 : 10));
|
|
88
|
+
if (octets.every(o => o >= 0 && o <= 255)) {
|
|
89
|
+
return octets;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// Decimal notation: 2130706433
|
|
94
|
+
if (/^\d+$/.test(cleaned)) {
|
|
95
|
+
const num = parseInt(cleaned, 10);
|
|
96
|
+
if (num <= 0xffffffff) {
|
|
97
|
+
return [
|
|
98
|
+
(num >>> 24) & 0xff,
|
|
99
|
+
(num >>> 16) & 0xff,
|
|
100
|
+
(num >>> 8) & 0xff,
|
|
101
|
+
num & 0xff,
|
|
102
|
+
];
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Validate IPv4 address against private/reserved ranges
|
|
109
|
+
*/
|
|
110
|
+
function validateIPv4ForSSRF(octets) {
|
|
111
|
+
const [a, b, c, d] = octets;
|
|
112
|
+
// Loopback: 127.0.0.0/8
|
|
113
|
+
if (a === 127) {
|
|
114
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
115
|
+
}
|
|
116
|
+
// Private: 10.0.0.0/8
|
|
117
|
+
if (a === 10) {
|
|
118
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
119
|
+
}
|
|
120
|
+
// Private: 172.16.0.0/12
|
|
121
|
+
if (a === 172 && b >= 16 && b <= 31) {
|
|
122
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
123
|
+
}
|
|
124
|
+
// Private: 192.168.0.0/16
|
|
125
|
+
if (a === 192 && b === 168) {
|
|
126
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
127
|
+
}
|
|
128
|
+
// Link-local: 169.254.0.0/16 (includes AWS metadata endpoint)
|
|
129
|
+
if (a === 169 && b === 254) {
|
|
130
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
131
|
+
}
|
|
132
|
+
// Broadcast: 255.255.255.255
|
|
133
|
+
if (a === 255 && b === 255 && c === 255 && d === 255) {
|
|
134
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
135
|
+
}
|
|
136
|
+
// This network: 0.0.0.0/8
|
|
137
|
+
if (a === 0) {
|
|
138
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Validate IPv6 address against private/reserved ranges
|
|
143
|
+
*/
|
|
144
|
+
function validateIPv6ForSSRF(hostname) {
|
|
145
|
+
const addr = hostname.replace(/^\[|\]$/g, '').toLowerCase();
|
|
146
|
+
// Loopback: ::1
|
|
147
|
+
if (addr === '::1' || addr === '0:0:0:0:0:0:0:1') {
|
|
148
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
149
|
+
}
|
|
150
|
+
// IPv6 mapped IPv4: ::ffff:192.168.1.1
|
|
151
|
+
if (addr.startsWith('::ffff:')) {
|
|
152
|
+
const ipv4Part = addr.substring(7);
|
|
153
|
+
if (ipv4Part.includes('.')) {
|
|
154
|
+
const parts = ipv4Part.split('.');
|
|
155
|
+
if (parts.length === 4) {
|
|
156
|
+
const octets = parts.map(p => parseInt(p, 10));
|
|
157
|
+
if (octets.every(o => !isNaN(o) && o >= 0 && o <= 255)) {
|
|
158
|
+
validateIPv4ForSSRF(octets);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
const hexStr = ipv4Part.replace(/:/g, '');
|
|
164
|
+
if (/^[0-9a-f]{1,8}$/.test(hexStr)) {
|
|
165
|
+
const num = parseInt(hexStr, 16);
|
|
166
|
+
const octets = [
|
|
167
|
+
(num >>> 24) & 0xff,
|
|
168
|
+
(num >>> 16) & 0xff,
|
|
169
|
+
(num >>> 8) & 0xff,
|
|
170
|
+
num & 0xff,
|
|
171
|
+
];
|
|
172
|
+
validateIPv4ForSSRF(octets);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
176
|
+
}
|
|
177
|
+
// Unique local addresses: fc00::/7
|
|
178
|
+
if (addr.startsWith('fc') || addr.startsWith('fd')) {
|
|
179
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
180
|
+
}
|
|
181
|
+
// Link-local: fe80::/10
|
|
182
|
+
if (addr.startsWith('fe8') || addr.startsWith('fe9') ||
|
|
183
|
+
addr.startsWith('fea') || addr.startsWith('feb')) {
|
|
184
|
+
throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
//# sourceMappingURL=url-validator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../../src/server/middleware/url-validator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,YAAY;IACZ,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,kCAAkC;IAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,gBAAgB;IAChB,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IAC1B,CAAC;CACF;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,QAAgB;IACjC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,8DAA8D;IAC9D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,uCAAuC;IACvC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,mCAAmC;IACnC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,SAAS,CAAC,4DAA4D,CAAC,CAAC;IACpF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgreSQL-backed auth store for production deployments
|
|
3
|
+
* Uses SHA-256 hashing for API keys and tracks WEEKLY usage with burst limits
|
|
4
|
+
*/
|
|
5
|
+
import { AuthStore, ApiKeyInfo } from './auth-store.js';
|
|
6
|
+
export interface WeeklyUsageInfo {
|
|
7
|
+
week: string;
|
|
8
|
+
basicCount: number;
|
|
9
|
+
stealthCount: number;
|
|
10
|
+
captchaCount: number;
|
|
11
|
+
searchCount: number;
|
|
12
|
+
totalUsed: number;
|
|
13
|
+
weeklyLimit: number;
|
|
14
|
+
rolloverCredits: number;
|
|
15
|
+
totalAvailable: number;
|
|
16
|
+
remaining: number;
|
|
17
|
+
percentUsed: number;
|
|
18
|
+
resetsAt: string;
|
|
19
|
+
}
|
|
20
|
+
export interface BurstInfo {
|
|
21
|
+
hourBucket: string;
|
|
22
|
+
count: number;
|
|
23
|
+
limit: number;
|
|
24
|
+
remaining: number;
|
|
25
|
+
resetsIn: string;
|
|
26
|
+
}
|
|
27
|
+
export interface ExtraUsageInfo {
|
|
28
|
+
enabled: boolean;
|
|
29
|
+
balance: number;
|
|
30
|
+
spent: number;
|
|
31
|
+
spendingLimit: number;
|
|
32
|
+
autoReload: boolean;
|
|
33
|
+
percentUsed: number;
|
|
34
|
+
resetsAt: string;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* PostgreSQL auth store for production
|
|
38
|
+
*/
|
|
39
|
+
export declare class PostgresAuthStore implements AuthStore {
|
|
40
|
+
private pool;
|
|
41
|
+
constructor(connectionString?: string);
|
|
42
|
+
/**
|
|
43
|
+
* Hash API key with SHA-256
|
|
44
|
+
* SECURITY: Never store raw API keys
|
|
45
|
+
*/
|
|
46
|
+
private hashKey;
|
|
47
|
+
/**
|
|
48
|
+
* Get current ISO week in YYYY-WXX format (e.g., "2026-W07")
|
|
49
|
+
*/
|
|
50
|
+
private getCurrentWeek;
|
|
51
|
+
/**
|
|
52
|
+
* Get previous ISO week in YYYY-WXX format
|
|
53
|
+
*/
|
|
54
|
+
private getPreviousWeek;
|
|
55
|
+
/**
|
|
56
|
+
* Get next Monday 00:00 UTC (week reset time)
|
|
57
|
+
*/
|
|
58
|
+
private getWeekResetTime;
|
|
59
|
+
/**
|
|
60
|
+
* Get current hour bucket in YYYY-MM-DDTHH format (UTC)
|
|
61
|
+
*/
|
|
62
|
+
private getCurrentHour;
|
|
63
|
+
/**
|
|
64
|
+
* Get human-readable time until next hour
|
|
65
|
+
*/
|
|
66
|
+
private getTimeUntilNextHour;
|
|
67
|
+
/**
|
|
68
|
+
* Validate API key and return user info
|
|
69
|
+
* SECURITY: Uses SHA-256 hash comparison, updates last_used_at
|
|
70
|
+
*/
|
|
71
|
+
validateKey(key: string): Promise<ApiKeyInfo | null>;
|
|
72
|
+
/**
|
|
73
|
+
* Track weekly usage for an API key
|
|
74
|
+
* SECURITY: Uses UPSERT to prevent race conditions
|
|
75
|
+
*/
|
|
76
|
+
trackUsage(key: string, fetchType: 'basic' | 'stealth' | 'captcha' | 'search'): Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* Track burst usage (hourly limit)
|
|
79
|
+
*/
|
|
80
|
+
trackBurstUsage(key: string): Promise<void>;
|
|
81
|
+
/**
|
|
82
|
+
* Check burst limit (hourly)
|
|
83
|
+
*/
|
|
84
|
+
checkBurstLimit(key: string): Promise<{
|
|
85
|
+
allowed: boolean;
|
|
86
|
+
burst: BurstInfo;
|
|
87
|
+
}>;
|
|
88
|
+
/**
|
|
89
|
+
* Get weekly usage info for an API key with rollover calculation
|
|
90
|
+
*/
|
|
91
|
+
getUsage(key: string): Promise<WeeklyUsageInfo | null>;
|
|
92
|
+
/**
|
|
93
|
+
* Check if API key has exceeded weekly limit
|
|
94
|
+
*/
|
|
95
|
+
checkLimit(key: string): Promise<{
|
|
96
|
+
allowed: boolean;
|
|
97
|
+
usage?: WeeklyUsageInfo;
|
|
98
|
+
}>;
|
|
99
|
+
/**
|
|
100
|
+
* Get extra usage info for a user
|
|
101
|
+
*/
|
|
102
|
+
getExtraUsageInfo(key: string): Promise<ExtraUsageInfo | null>;
|
|
103
|
+
/**
|
|
104
|
+
* Check if extra usage can be used
|
|
105
|
+
*/
|
|
106
|
+
canUseExtraUsage(key: string): Promise<boolean>;
|
|
107
|
+
/**
|
|
108
|
+
* Track extra usage and deduct from balance
|
|
109
|
+
*/
|
|
110
|
+
trackExtraUsage(key: string, fetchType: 'basic' | 'stealth' | 'captcha' | 'search', url?: string, processingTimeMs?: number, statusCode?: number): Promise<{
|
|
111
|
+
success: boolean;
|
|
112
|
+
cost: number;
|
|
113
|
+
newBalance: number;
|
|
114
|
+
}>;
|
|
115
|
+
/**
|
|
116
|
+
* Generate a cryptographically secure API key
|
|
117
|
+
* Format: wp_live_ + 32 random hex chars (total 40 chars)
|
|
118
|
+
*/
|
|
119
|
+
static generateApiKey(): string;
|
|
120
|
+
/**
|
|
121
|
+
* Get key prefix (first 12 characters for display)
|
|
122
|
+
*/
|
|
123
|
+
static getKeyPrefix(key: string): string;
|
|
124
|
+
/**
|
|
125
|
+
* Close the database pool
|
|
126
|
+
*/
|
|
127
|
+
close(): Promise<void>;
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=pg-auth-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pg-auth-store.d.ts","sourceRoot":"","sources":["../../src/server/pg-auth-store.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAIxD,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAUD;;GAEG;AACH,qBAAa,iBAAkB,YAAW,SAAS;IACjD,OAAO,CAAC,IAAI,CAAU;gBAEV,gBAAgB,CAAC,EAAE,MAAM;IAqBrC;;;OAGG;IACH,OAAO,CAAC,OAAO;IAIf;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,eAAe;IASvB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAKtB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAS5B;;;OAGG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAkD1D;;;OAGG;IACG,UAAU,CACd,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GACpD,OAAO,CAAC,IAAI,CAAC;IA2ChB;;OAEG;IACG,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCjD;;OAEG;IACG,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,SAAS,CAAA;KAAE,CAAC;IAyDnF;;OAEG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IA2E5D;;OAEG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,eAAe,CAAA;KAAE,CAAC;IAYrF;;OAEG;IACG,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAoDpE;;OAEG;IACG,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWrD;;OAEG;IACG,eAAe,CACnB,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,EACrD,GAAG,CAAC,EAAE,MAAM,EACZ,gBAAgB,CAAC,EAAE,MAAM,EACzB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAuElE;;;OAGG;IACH,MAAM,CAAC,cAAc,IAAI,MAAM;IAK/B;;OAEG;IACH,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIxC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|