ranvia 0.1.0-alpha.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,10 +39,12 @@ http.createServer(ranvia.wrap(handler)); // plain node:http
39
39
  |---|---|
40
40
  | `key` | Your site key from the Ranvia dashboard. |
41
41
  | `network(req)` | The visitor's network, if your edge provides it: an ASN (`as7922`) or a country code. |
42
- | `signedIn(req)` | Return `true` for a visitor you've verified as signed in; they are never counted. |
43
- | `verifiedCrawler(req)` | Return `true` for a search crawler you've verified. |
42
+ | `signedIn(req)` | Return `true` for a visitor you've verified as signed in. They skip the page budget; by default they're asked for proof only past a pace no person browses at (about two pages a second), which you can turn off per site. |
43
+ | `verifiedCrawler(req)` | Search crawlers are checked for you: a request naming Googlebot, Bingbot, Applebot or YandexBot is verified by reverse and forward DNS, and verified crawlers are never counted. Pass your own function to replace the check, or `() => false` to turn it off. |
44
+ | `clientIp(req)` | The visitor's address. Defaults to the connection's address, or the last `X-Forwarded-For` entry when the connection comes from this machine. Behind a CDN, return the address it passes on (`CF-Connecting-IP` for Cloudflare). |
44
45
  | `cacheDir` | Where the encrypted engine is kept. Defaults to `~/.cache/ranvia`. |
45
46
  | `rulesFile` | A JSON file of your own rule overrides. |
47
+ | `logRefusals` | `true` logs each request Ranvia refuses, checks or rate-limits through `log`: method, path, User-Agent and why. The log stays on your server. |
46
48
  | `log(line)` | Receives log lines. |
47
49
 
48
50
  Requires Node.js 20 or later.
package/crawlers.js ADDED
@@ -0,0 +1,96 @@
1
+ 'use strict';
2
+ // Search engine crawlers fetch pages with no browser headers and no
3
+ // assets, which is what the guard counts. A request whose User-Agent
4
+ // names a search crawler is checked the way the search engines document:
5
+ // the address's reverse DNS name must sit under the engine's domain, and
6
+ // that name must resolve back to the address. A verified crawler is never
7
+ // counted; a request that only claims to be one goes through the guard.
8
+
9
+ const dns = require('node:dns').promises;
10
+ const net = require('node:net');
11
+
12
+ const SEARCH_CRAWLERS = [
13
+ ['googlebot', ['.googlebot.com', '.google.com']],
14
+ ['google-inspectiontool', ['.googlebot.com', '.google.com']],
15
+ ['bingbot', ['.search.msn.com']],
16
+ ['applebot', ['.applebot.apple.com']],
17
+ ['yandexbot', ['.yandex.ru', '.yandex.net', '.yandex.com']],
18
+ ];
19
+
20
+ const CACHE_MAX = 4096;
21
+ const OK_FOR = 24 * 3600e3;
22
+ const FAIL_FOR = 3600e3;
23
+ const LOOKUP_MAX = 4; // concurrent DNS checks; past it a claim waits for the next request
24
+ const TIMEOUT = 2000;
25
+
26
+ function claimedCrawler(ua) {
27
+ if (!ua) return null;
28
+ ua = String(ua).toLowerCase();
29
+ for (const [token, suffixes] of SEARCH_CRAWLERS) if (ua.includes(token)) return suffixes;
30
+ return null;
31
+ }
32
+
33
+ const bare = (a) => (a && a.startsWith('::ffff:') && net.isIPv4(a.slice(7)) ? a.slice(7) : a || '');
34
+
35
+ // remoteIp is the default clientIp: the connection's address, or the last
36
+ // X-Forwarded-For entry when the connection comes from this machine.
37
+ function remoteIp(req) {
38
+ const peer = bare(req.socket && req.socket.remoteAddress);
39
+ const loopback = peer === '::1' || peer.startsWith('127.');
40
+ if (!loopback) return peer;
41
+ const fwd = req.headers['x-forwarded-for'];
42
+ if (!fwd) return peer;
43
+ const parts = String(Array.isArray(fwd) ? fwd[fwd.length - 1] : fwd).split(',');
44
+ const last = parts[parts.length - 1].trim();
45
+ return net.isIP(last) ? last : peer;
46
+ }
47
+
48
+ const sameIp = (a, b) => net.isIP(a) && net.isIP(b) && bare(a).toLowerCase() === bare(b).toLowerCase();
49
+
50
+ function createCrawlerVerifier(clientIp, resolver = dns) {
51
+ const seen = new Map(); // address -> { ok, at }
52
+ let inFlight = 0;
53
+
54
+ async function check(addr, suffixes) {
55
+ if (!net.isIP(addr)) return false;
56
+ let names;
57
+ try { names = await resolver.reverse(addr); } catch { return false; }
58
+ for (let name of names) {
59
+ name = name.toLowerCase().replace(/\.$/, '');
60
+ if (!suffixes.some((s) => name.endsWith(s))) continue;
61
+ let addrs = [];
62
+ try {
63
+ addrs = (await resolver.lookup(name, { all: true })).map((x) => x.address);
64
+ } catch { continue; }
65
+ if (addrs.some((a) => sameIp(a, addr))) return true;
66
+ }
67
+ return false;
68
+ }
69
+
70
+ return async function verified(req) {
71
+ const suffixes = claimedCrawler(req.headers['user-agent']);
72
+ if (!suffixes) return false;
73
+ const addr = clientIp(req);
74
+ const now = Date.now();
75
+ const got = seen.get(addr);
76
+ if (got && now - got.at < (got.ok ? OK_FOR : FAIL_FOR)) return got.ok;
77
+ if (inFlight >= LOOKUP_MAX) return false;
78
+ inFlight++;
79
+ let ok = false;
80
+ try {
81
+ let timer;
82
+ ok = await Promise.race([
83
+ check(addr, suffixes),
84
+ new Promise((r) => { timer = setTimeout(() => r(false), TIMEOUT); }),
85
+ ]);
86
+ clearTimeout(timer);
87
+ } finally {
88
+ inFlight--;
89
+ }
90
+ if (seen.size >= CACHE_MAX) seen.delete(seen.keys().next().value);
91
+ seen.set(addr, { ok, at: now });
92
+ return ok;
93
+ };
94
+ }
95
+
96
+ module.exports = { createCrawlerVerifier, remoteIp, claimedCrawler };
package/index.d.ts CHANGED
@@ -9,10 +9,25 @@ export interface RanviaConfig {
9
9
  cacheDir?: string;
10
10
  /** The visitor's network (an ASN like "as7922", or a country code), when your edge tells you. */
11
11
  network?: (req: IncomingMessage) => string | undefined;
12
- /** A visitor you have verified as signed in; never counted. Runs only for requests with cookies. */
12
+ /** A visitor you have verified as signed in: no page budget, but by default a human-pace limit. Runs only for requests with cookies. */
13
13
  signedIn?: (req: IncomingMessage) => boolean | Promise<boolean>;
14
- /** A search crawler you have verified; never counted. */
15
- verifiedCrawler?: (req: IncomingMessage) => boolean;
14
+ /**
15
+ * A search crawler you have verified; never counted. Defaults to the built-in
16
+ * check: a request naming Googlebot, Bingbot, Applebot or YandexBot is verified
17
+ * by reverse and forward DNS of its address. Return false to turn it off.
18
+ */
19
+ verifiedCrawler?: (req: IncomingMessage) => boolean | Promise<boolean>;
20
+ /**
21
+ * The visitor's address. Defaults to the connection's address, or the last
22
+ * X-Forwarded-For entry when the connection comes from this machine. Behind a
23
+ * CDN, return the address it passes on (for Cloudflare, CF-Connecting-IP).
24
+ */
25
+ clientIp?: (req: IncomingMessage) => string;
26
+ /**
27
+ * Log every request Ranvia refuses, checks or rate-limits through `log`:
28
+ * method, path, User-Agent and the reason. The log stays on your server.
29
+ */
30
+ logRefusals?: boolean;
16
31
  /** An optional JSON file of your own rule overrides. */
17
32
  rulesFile?: string;
18
33
  /** Receives one log line at a time. */
package/index.js CHANGED
@@ -22,9 +22,10 @@ const os = require('node:os');
22
22
  const path = require('node:path');
23
23
  const { wasiImports } = require('./wasi');
24
24
  const { encodeRequest, decodeDecision } = require('./wire');
25
+ const { createCrawlerVerifier, remoteIp } = require('./crawlers');
25
26
 
26
27
  const DEFAULT_SERVICE_URL = 'https://ranvia.io';
27
- const LOADER_VERSION = '0.1.0';
28
+ const LOADER_VERSION = '0.2.0';
28
29
  const RULES_EVERY = 5 * 60e3;
29
30
  const USAGE_EVERY = 60e3;
30
31
  const ENGINE_EVERY = 6 * 3600e3;
@@ -113,6 +114,8 @@ class Engine {
113
114
  function createRanvia(cfg = {}) {
114
115
  const key = parseSiteKey(cfg.key);
115
116
  const log = cfg.log || (() => {});
117
+ const clientIp = cfg.clientIp || remoteIp;
118
+ const verifiedCrawler = cfg.verifiedCrawler || createCrawlerVerifier(clientIp);
116
119
  const base = String(cfg.serviceUrl || DEFAULT_SERVICE_URL).replace(/\/+$/, '');
117
120
  const cacheDir = cfg.cacheDir ||
118
121
  path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), 'ranvia');
@@ -287,11 +290,11 @@ function createRanvia(cfg = {}) {
287
290
  if (!e) return true;
288
291
  const r = {
289
292
  method: req.method, uri: req.originalUrl || req.url, host: req.headers.host || '',
290
- header: req.headers,
293
+ header: req.headers, proto: req.httpVersion ? 'HTTP/' + req.httpVersion : '',
291
294
  };
292
295
  if (req.method === 'POST' && (req.url || '').split('?')[0] === '/guard/verify') r.body = await readBody(req);
293
296
  if (cfg.network) r.network = cfg.network(req) || '';
294
- if (cfg.verifiedCrawler) r.crawler = !!cfg.verifiedCrawler(req);
297
+ r.crawler = !!(await verifiedCrawler(req));
295
298
  if (cfg.signedIn && req.headers.cookie) r.signedIn = !!(await cfg.signedIn(req));
296
299
  let d;
297
300
  try {
@@ -302,6 +305,9 @@ function createRanvia(cfg = {}) {
302
305
  }
303
306
  for (const [k, v] of d.header) res.appendHeader(k, v);
304
307
  if (!d.serve) {
308
+ if (cfg.logRefusals && d.reason) {
309
+ log(`ranvia: ${d.reason} ${req.method} ${(req.originalUrl || req.url || '').split('?')[0]} ${JSON.stringify(req.headers['user-agent'] || '')}`);
310
+ }
305
311
  res.statusCode = d.status || 200;
306
312
  res.end(d.body);
307
313
  return false;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ranvia",
3
- "version": "0.1.0-alpha.1",
3
+ "version": "0.2.0",
4
4
  "description": "Bot and AI scraper protection that runs on your own server: middleware for Express, Connect and node:http.",
5
5
  "keywords": ["bot", "scraper", "ai", "crawler", "protection", "security", "middleware", "express"],
6
6
  "homepage": "https://ranvia.io",
@@ -8,6 +8,6 @@
8
8
  "author": "Synap6",
9
9
  "main": "index.js",
10
10
  "types": "index.d.ts",
11
- "files": ["index.js", "index.d.ts", "wasi.js", "wire.js", "README.md"],
11
+ "files": ["index.js", "index.d.ts", "wasi.js", "wire.js", "crawlers.js", "README.md"],
12
12
  "engines": { "node": ">=20" }
13
13
  }
package/wire.js CHANGED
@@ -12,7 +12,8 @@ function encodeRequest(req) {
12
12
  const n = Math.min(lines.length, 0xffff);
13
13
  const bufs = strs.map((s) => Buffer.from(s, 'utf8'));
14
14
  const pairs = lines.slice(0, n).map(([k, v]) => [Buffer.from(k, 'utf8'), Buffer.from(v, 'utf8')]);
15
- let size = 2 + 2;
15
+ const proto = Buffer.from(req.proto || '', 'utf8');
16
+ let size = 2 + 2 + 4 + proto.length;
16
17
  for (const b of bufs) size += 4 + b.length;
17
18
  for (const [k, v] of pairs) size += 8 + k.length + v.length;
18
19
  const out = Buffer.allocUnsafe(size);
@@ -23,6 +24,7 @@ function encodeRequest(req) {
23
24
  for (const b of bufs) put(b);
24
25
  out.writeUInt16LE(n, o); o += 2;
25
26
  for (const [k, v] of pairs) { put(k); put(v); }
27
+ put(proto); // added after the first engines, which stop before it
26
28
  return out;
27
29
  }
28
30
 
@@ -41,6 +43,8 @@ function decodeDecision(buf) {
41
43
  const lines = buf.readUInt16LE(11); o = 13;
42
44
  for (let i = 0; i < lines; i++) d.header.push([str().toString('utf8'), str().toString('utf8')]);
43
45
  d.body = Buffer.from(str());
46
+ // Engines from before the reason field end here.
47
+ d.reason = o < buf.length ? str().toString('utf8') : '';
44
48
  return d;
45
49
  }
46
50