ranvia 0.1.0-alpha.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/crawlers.js +96 -0
- package/index.d.ts +18 -3
- package/index.js +9 -3
- package/package.json +2 -2
- package/wire.js +5 -1
package/README.md
CHANGED
|
@@ -39,10 +39,12 @@ http.createServer(ranvia.wrap(handler)); // plain node:http
|
|
|
39
39
|
|---|---|
|
|
40
40
|
| `key` | Your site key from the Ranvia dashboard. |
|
|
41
41
|
| `network(req)` | The visitor's network, if your edge provides it: an ASN (`as7922`) or a country code. |
|
|
42
|
-
| `signedIn(req)` | Return `true` for a visitor you've verified as signed in; they
|
|
43
|
-
| `verifiedCrawler(req)` |
|
|
42
|
+
| `signedIn(req)` | Return `true` for a visitor you've verified as signed in. They skip the page budget; by default they're asked for proof only past a pace no person browses at (about two pages a second), which you can turn off per site. |
|
|
43
|
+
| `verifiedCrawler(req)` | Search crawlers are checked for you: a request naming Googlebot, Bingbot, Applebot or YandexBot is verified by reverse and forward DNS, and verified crawlers are never counted. Pass your own function to replace the check, or `() => false` to turn it off. |
|
|
44
|
+
| `clientIp(req)` | The visitor's address. Defaults to the connection's address, or the last `X-Forwarded-For` entry when the connection comes from this machine. Behind a CDN, return the address it passes on (`CF-Connecting-IP` for Cloudflare). |
|
|
44
45
|
| `cacheDir` | Where the encrypted engine is kept. Defaults to `~/.cache/ranvia`. |
|
|
45
46
|
| `rulesFile` | A JSON file of your own rule overrides. |
|
|
47
|
+
| `logRefusals` | `true` logs each request Ranvia refuses, checks or rate-limits through `log`: method, path, User-Agent and why. The log stays on your server. |
|
|
46
48
|
| `log(line)` | Receives log lines. |
|
|
47
49
|
|
|
48
50
|
Requires Node.js 20 or later.
|
package/crawlers.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Search engine crawlers fetch pages with no browser headers and no
|
|
3
|
+
// assets, which is what the guard counts. A request whose User-Agent
|
|
4
|
+
// names a search crawler is checked the way the search engines document:
|
|
5
|
+
// the address's reverse DNS name must sit under the engine's domain, and
|
|
6
|
+
// that name must resolve back to the address. A verified crawler is never
|
|
7
|
+
// counted; a request that only claims to be one goes through the guard.
|
|
8
|
+
|
|
9
|
+
const dns = require('node:dns').promises;
|
|
10
|
+
const net = require('node:net');
|
|
11
|
+
|
|
12
|
+
const SEARCH_CRAWLERS = [
|
|
13
|
+
['googlebot', ['.googlebot.com', '.google.com']],
|
|
14
|
+
['google-inspectiontool', ['.googlebot.com', '.google.com']],
|
|
15
|
+
['bingbot', ['.search.msn.com']],
|
|
16
|
+
['applebot', ['.applebot.apple.com']],
|
|
17
|
+
['yandexbot', ['.yandex.ru', '.yandex.net', '.yandex.com']],
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
const CACHE_MAX = 4096;
|
|
21
|
+
const OK_FOR = 24 * 3600e3;
|
|
22
|
+
const FAIL_FOR = 3600e3;
|
|
23
|
+
const LOOKUP_MAX = 4; // concurrent DNS checks; past it a claim waits for the next request
|
|
24
|
+
const TIMEOUT = 2000;
|
|
25
|
+
|
|
26
|
+
function claimedCrawler(ua) {
|
|
27
|
+
if (!ua) return null;
|
|
28
|
+
ua = String(ua).toLowerCase();
|
|
29
|
+
for (const [token, suffixes] of SEARCH_CRAWLERS) if (ua.includes(token)) return suffixes;
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const bare = (a) => (a && a.startsWith('::ffff:') && net.isIPv4(a.slice(7)) ? a.slice(7) : a || '');
|
|
34
|
+
|
|
35
|
+
// remoteIp is the default clientIp: the connection's address, or the last
|
|
36
|
+
// X-Forwarded-For entry when the connection comes from this machine.
|
|
37
|
+
function remoteIp(req) {
|
|
38
|
+
const peer = bare(req.socket && req.socket.remoteAddress);
|
|
39
|
+
const loopback = peer === '::1' || peer.startsWith('127.');
|
|
40
|
+
if (!loopback) return peer;
|
|
41
|
+
const fwd = req.headers['x-forwarded-for'];
|
|
42
|
+
if (!fwd) return peer;
|
|
43
|
+
const parts = String(Array.isArray(fwd) ? fwd[fwd.length - 1] : fwd).split(',');
|
|
44
|
+
const last = parts[parts.length - 1].trim();
|
|
45
|
+
return net.isIP(last) ? last : peer;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const sameIp = (a, b) => net.isIP(a) && net.isIP(b) && bare(a).toLowerCase() === bare(b).toLowerCase();
|
|
49
|
+
|
|
50
|
+
function createCrawlerVerifier(clientIp, resolver = dns) {
|
|
51
|
+
const seen = new Map(); // address -> { ok, at }
|
|
52
|
+
let inFlight = 0;
|
|
53
|
+
|
|
54
|
+
async function check(addr, suffixes) {
|
|
55
|
+
if (!net.isIP(addr)) return false;
|
|
56
|
+
let names;
|
|
57
|
+
try { names = await resolver.reverse(addr); } catch { return false; }
|
|
58
|
+
for (let name of names) {
|
|
59
|
+
name = name.toLowerCase().replace(/\.$/, '');
|
|
60
|
+
if (!suffixes.some((s) => name.endsWith(s))) continue;
|
|
61
|
+
let addrs = [];
|
|
62
|
+
try {
|
|
63
|
+
addrs = (await resolver.lookup(name, { all: true })).map((x) => x.address);
|
|
64
|
+
} catch { continue; }
|
|
65
|
+
if (addrs.some((a) => sameIp(a, addr))) return true;
|
|
66
|
+
}
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return async function verified(req) {
|
|
71
|
+
const suffixes = claimedCrawler(req.headers['user-agent']);
|
|
72
|
+
if (!suffixes) return false;
|
|
73
|
+
const addr = clientIp(req);
|
|
74
|
+
const now = Date.now();
|
|
75
|
+
const got = seen.get(addr);
|
|
76
|
+
if (got && now - got.at < (got.ok ? OK_FOR : FAIL_FOR)) return got.ok;
|
|
77
|
+
if (inFlight >= LOOKUP_MAX) return false;
|
|
78
|
+
inFlight++;
|
|
79
|
+
let ok = false;
|
|
80
|
+
try {
|
|
81
|
+
let timer;
|
|
82
|
+
ok = await Promise.race([
|
|
83
|
+
check(addr, suffixes),
|
|
84
|
+
new Promise((r) => { timer = setTimeout(() => r(false), TIMEOUT); }),
|
|
85
|
+
]);
|
|
86
|
+
clearTimeout(timer);
|
|
87
|
+
} finally {
|
|
88
|
+
inFlight--;
|
|
89
|
+
}
|
|
90
|
+
if (seen.size >= CACHE_MAX) seen.delete(seen.keys().next().value);
|
|
91
|
+
seen.set(addr, { ok, at: now });
|
|
92
|
+
return ok;
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
module.exports = { createCrawlerVerifier, remoteIp, claimedCrawler };
|
package/index.d.ts
CHANGED
|
@@ -9,10 +9,25 @@ export interface RanviaConfig {
|
|
|
9
9
|
cacheDir?: string;
|
|
10
10
|
/** The visitor's network (an ASN like "as7922", or a country code), when your edge tells you. */
|
|
11
11
|
network?: (req: IncomingMessage) => string | undefined;
|
|
12
|
-
/** A visitor you have verified as signed in
|
|
12
|
+
/** A visitor you have verified as signed in: no page budget, but by default a human-pace limit. Runs only for requests with cookies. */
|
|
13
13
|
signedIn?: (req: IncomingMessage) => boolean | Promise<boolean>;
|
|
14
|
-
/**
|
|
15
|
-
|
|
14
|
+
/**
|
|
15
|
+
* A search crawler you have verified; never counted. Defaults to the built-in
|
|
16
|
+
* check: a request naming Googlebot, Bingbot, Applebot or YandexBot is verified
|
|
17
|
+
* by reverse and forward DNS of its address. Return false to turn it off.
|
|
18
|
+
*/
|
|
19
|
+
verifiedCrawler?: (req: IncomingMessage) => boolean | Promise<boolean>;
|
|
20
|
+
/**
|
|
21
|
+
* The visitor's address. Defaults to the connection's address, or the last
|
|
22
|
+
* X-Forwarded-For entry when the connection comes from this machine. Behind a
|
|
23
|
+
* CDN, return the address it passes on (for Cloudflare, CF-Connecting-IP).
|
|
24
|
+
*/
|
|
25
|
+
clientIp?: (req: IncomingMessage) => string;
|
|
26
|
+
/**
|
|
27
|
+
* Log every request Ranvia refuses, checks or rate-limits through `log`:
|
|
28
|
+
* method, path, User-Agent and the reason. The log stays on your server.
|
|
29
|
+
*/
|
|
30
|
+
logRefusals?: boolean;
|
|
16
31
|
/** An optional JSON file of your own rule overrides. */
|
|
17
32
|
rulesFile?: string;
|
|
18
33
|
/** Receives one log line at a time. */
|
package/index.js
CHANGED
|
@@ -22,9 +22,10 @@ const os = require('node:os');
|
|
|
22
22
|
const path = require('node:path');
|
|
23
23
|
const { wasiImports } = require('./wasi');
|
|
24
24
|
const { encodeRequest, decodeDecision } = require('./wire');
|
|
25
|
+
const { createCrawlerVerifier, remoteIp } = require('./crawlers');
|
|
25
26
|
|
|
26
27
|
const DEFAULT_SERVICE_URL = 'https://ranvia.io';
|
|
27
|
-
const LOADER_VERSION = '0.
|
|
28
|
+
const LOADER_VERSION = '0.2.0';
|
|
28
29
|
const RULES_EVERY = 5 * 60e3;
|
|
29
30
|
const USAGE_EVERY = 60e3;
|
|
30
31
|
const ENGINE_EVERY = 6 * 3600e3;
|
|
@@ -113,6 +114,8 @@ class Engine {
|
|
|
113
114
|
function createRanvia(cfg = {}) {
|
|
114
115
|
const key = parseSiteKey(cfg.key);
|
|
115
116
|
const log = cfg.log || (() => {});
|
|
117
|
+
const clientIp = cfg.clientIp || remoteIp;
|
|
118
|
+
const verifiedCrawler = cfg.verifiedCrawler || createCrawlerVerifier(clientIp);
|
|
116
119
|
const base = String(cfg.serviceUrl || DEFAULT_SERVICE_URL).replace(/\/+$/, '');
|
|
117
120
|
const cacheDir = cfg.cacheDir ||
|
|
118
121
|
path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), 'ranvia');
|
|
@@ -287,11 +290,11 @@ function createRanvia(cfg = {}) {
|
|
|
287
290
|
if (!e) return true;
|
|
288
291
|
const r = {
|
|
289
292
|
method: req.method, uri: req.originalUrl || req.url, host: req.headers.host || '',
|
|
290
|
-
header: req.headers,
|
|
293
|
+
header: req.headers, proto: req.httpVersion ? 'HTTP/' + req.httpVersion : '',
|
|
291
294
|
};
|
|
292
295
|
if (req.method === 'POST' && (req.url || '').split('?')[0] === '/guard/verify') r.body = await readBody(req);
|
|
293
296
|
if (cfg.network) r.network = cfg.network(req) || '';
|
|
294
|
-
|
|
297
|
+
r.crawler = !!(await verifiedCrawler(req));
|
|
295
298
|
if (cfg.signedIn && req.headers.cookie) r.signedIn = !!(await cfg.signedIn(req));
|
|
296
299
|
let d;
|
|
297
300
|
try {
|
|
@@ -302,6 +305,9 @@ function createRanvia(cfg = {}) {
|
|
|
302
305
|
}
|
|
303
306
|
for (const [k, v] of d.header) res.appendHeader(k, v);
|
|
304
307
|
if (!d.serve) {
|
|
308
|
+
if (cfg.logRefusals && d.reason) {
|
|
309
|
+
log(`ranvia: ${d.reason} ${req.method} ${(req.originalUrl || req.url || '').split('?')[0]} ${JSON.stringify(req.headers['user-agent'] || '')}`);
|
|
310
|
+
}
|
|
305
311
|
res.statusCode = d.status || 200;
|
|
306
312
|
res.end(d.body);
|
|
307
313
|
return false;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ranvia",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Bot and AI scraper protection that runs on your own server: middleware for Express, Connect and node:http.",
|
|
5
5
|
"keywords": ["bot", "scraper", "ai", "crawler", "protection", "security", "middleware", "express"],
|
|
6
6
|
"homepage": "https://ranvia.io",
|
|
@@ -8,6 +8,6 @@
|
|
|
8
8
|
"author": "Synap6",
|
|
9
9
|
"main": "index.js",
|
|
10
10
|
"types": "index.d.ts",
|
|
11
|
-
"files": ["index.js", "index.d.ts", "wasi.js", "wire.js", "README.md"],
|
|
11
|
+
"files": ["index.js", "index.d.ts", "wasi.js", "wire.js", "crawlers.js", "README.md"],
|
|
12
12
|
"engines": { "node": ">=20" }
|
|
13
13
|
}
|
package/wire.js
CHANGED
|
@@ -12,7 +12,8 @@ function encodeRequest(req) {
|
|
|
12
12
|
const n = Math.min(lines.length, 0xffff);
|
|
13
13
|
const bufs = strs.map((s) => Buffer.from(s, 'utf8'));
|
|
14
14
|
const pairs = lines.slice(0, n).map(([k, v]) => [Buffer.from(k, 'utf8'), Buffer.from(v, 'utf8')]);
|
|
15
|
-
|
|
15
|
+
const proto = Buffer.from(req.proto || '', 'utf8');
|
|
16
|
+
let size = 2 + 2 + 4 + proto.length;
|
|
16
17
|
for (const b of bufs) size += 4 + b.length;
|
|
17
18
|
for (const [k, v] of pairs) size += 8 + k.length + v.length;
|
|
18
19
|
const out = Buffer.allocUnsafe(size);
|
|
@@ -23,6 +24,7 @@ function encodeRequest(req) {
|
|
|
23
24
|
for (const b of bufs) put(b);
|
|
24
25
|
out.writeUInt16LE(n, o); o += 2;
|
|
25
26
|
for (const [k, v] of pairs) { put(k); put(v); }
|
|
27
|
+
put(proto); // added after the first engines, which stop before it
|
|
26
28
|
return out;
|
|
27
29
|
}
|
|
28
30
|
|
|
@@ -41,6 +43,8 @@ function decodeDecision(buf) {
|
|
|
41
43
|
const lines = buf.readUInt16LE(11); o = 13;
|
|
42
44
|
for (let i = 0; i < lines; i++) d.header.push([str().toString('utf8'), str().toString('utf8')]);
|
|
43
45
|
d.body = Buffer.from(str());
|
|
46
|
+
// Engines from before the reason field end here.
|
|
47
|
+
d.reason = o < buf.length ? str().toString('utf8') : '';
|
|
44
48
|
return d;
|
|
45
49
|
}
|
|
46
50
|
|