ranvia 0.0.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,56 @@
1
- # Ranvia
1
+ # Ranvia for Node.js
2
2
 
3
3
  Bot and AI scraper protection that runs on your own server.
4
+ [ranvia.io](https://ranvia.io)
4
5
 
5
- This package name is reserved for the Ranvia Node.js loader. It is
6
- coming soon; see https://ranvia.io.
6
+ > **Early access.** The Ranvia service is opening soon. Until your site
7
+ > has a key, there is nothing to install.
8
+
9
+ ```js
10
+ const { createRanvia } = require('ranvia');
11
+
12
+ const ranvia = createRanvia({ key: process.env.RANVIA_KEY });
13
+
14
+ app.use(ranvia.middleware); // Express, Connect
15
+ // or
16
+ http.createServer(ranvia.wrap(handler)); // plain node:http
17
+ ```
18
+
19
+ ## How it works
20
+
21
+ - **Your traffic never leaves your server.** Every decision is made
22
+ locally, in about a tenth of a millisecond.
23
+ - **What the service sends and receives:** it sends your site's rules;
24
+ your site sends back counts. Visitor addresses, cookies, URLs and
25
+ page content are never sent.
26
+ - **This package is a loader.** On first start it downloads the Ranvia
27
+ engine for your site key, keeps it on disk encrypted for your site,
28
+ and runs it in memory as WebAssembly.
29
+ - **Ranvia never takes your site down.** If the engine can't run (first
30
+ start with ranvia.io out of reach, a revoked key), every request
31
+ passes and the reason is logged.
32
+ - **What visitors see:** scrapers are refused. Clients that behave like
33
+ scrapers get a short browser check, which a real browser passes in
34
+ about a second with no clicks.
35
+
36
+ ## Options
37
+
38
+ | Option | What it does |
39
+ |---|---|
40
+ | `key` | Your site key from the Ranvia dashboard. |
41
+ | `network(req)` | The visitor's network, if your edge provides it: an ASN (`as7922`) or a country code. |
42
+ | `signedIn(req)` | Return `true` for a visitor you've verified as signed in. They skip the page budget; by default they're asked for proof only past a pace no person browses at (about two pages a second), which you can turn off per site. |
43
+ | `verifiedCrawler(req)` | Search crawlers are checked for you: a request naming Googlebot, Bingbot, Applebot or YandexBot is verified by reverse and forward DNS, and verified crawlers are never counted. Pass your own function to replace the check, or `() => false` to turn it off. |
44
+ | `clientIp(req)` | The visitor's address. Defaults to the connection's address, or the last `X-Forwarded-For` entry when the connection comes from this machine. Behind a CDN, return the address it passes on (`CF-Connecting-IP` for Cloudflare). |
45
+ | `cacheDir` | Where the encrypted engine is kept. Defaults to `~/.cache/ranvia`. |
46
+ | `rulesFile` | A JSON file of your own rule overrides. |
47
+ | `logRefusals` | `true` logs each request Ranvia refuses, checks or rate-limits through `log`: method, path, User-Agent and why. The log stays on your server. |
48
+ | `log(line)` | Receives log lines. |
49
+
50
+ Requires Node.js 20 or later.
51
+
52
+ ## License
53
+
54
+ Use of this package and of the engine it downloads is governed by the
55
+ Ranvia terms of service at https://ranvia.io. Copyright Synap6. All
56
+ rights reserved.
package/crawlers.js ADDED
@@ -0,0 +1,96 @@
1
+ 'use strict';
2
+ // Search engine crawlers fetch pages with no browser headers and no
3
+ // assets, which is what the guard counts. A request whose User-Agent
4
+ // names a search crawler is checked the way the search engines document:
5
+ // the address's reverse DNS name must sit under the engine's domain, and
6
+ // that name must resolve back to the address. A verified crawler is never
7
+ // counted; a request that only claims to be one goes through the guard.
8
+
9
+ const dns = require('node:dns').promises;
10
+ const net = require('node:net');
11
+
12
+ const SEARCH_CRAWLERS = [
13
+ ['googlebot', ['.googlebot.com', '.google.com']],
14
+ ['google-inspectiontool', ['.googlebot.com', '.google.com']],
15
+ ['bingbot', ['.search.msn.com']],
16
+ ['applebot', ['.applebot.apple.com']],
17
+ ['yandexbot', ['.yandex.ru', '.yandex.net', '.yandex.com']],
18
+ ];
19
+
20
+ const CACHE_MAX = 4096;
21
+ const OK_FOR = 24 * 3600e3;
22
+ const FAIL_FOR = 3600e3;
23
+ const LOOKUP_MAX = 4; // concurrent DNS checks; past it a claim waits for the next request
24
+ const TIMEOUT = 2000;
25
+
26
+ function claimedCrawler(ua) {
27
+ if (!ua) return null;
28
+ ua = String(ua).toLowerCase();
29
+ for (const [token, suffixes] of SEARCH_CRAWLERS) if (ua.includes(token)) return suffixes;
30
+ return null;
31
+ }
32
+
33
+ const bare = (a) => (a && a.startsWith('::ffff:') && net.isIPv4(a.slice(7)) ? a.slice(7) : a || '');
34
+
35
+ // remoteIp is the default clientIp: the connection's address, or the last
36
+ // X-Forwarded-For entry when the connection comes from this machine.
37
+ function remoteIp(req) {
38
+ const peer = bare(req.socket && req.socket.remoteAddress);
39
+ const loopback = peer === '::1' || peer.startsWith('127.');
40
+ if (!loopback) return peer;
41
+ const fwd = req.headers['x-forwarded-for'];
42
+ if (!fwd) return peer;
43
+ const parts = String(Array.isArray(fwd) ? fwd[fwd.length - 1] : fwd).split(',');
44
+ const last = parts[parts.length - 1].trim();
45
+ return net.isIP(last) ? last : peer;
46
+ }
47
+
48
+ const sameIp = (a, b) => net.isIP(a) && net.isIP(b) && bare(a).toLowerCase() === bare(b).toLowerCase();
49
+
50
+ function createCrawlerVerifier(clientIp, resolver = dns) {
51
+ const seen = new Map(); // address -> { ok, at }
52
+ let inFlight = 0;
53
+
54
+ async function check(addr, suffixes) {
55
+ if (!net.isIP(addr)) return false;
56
+ let names;
57
+ try { names = await resolver.reverse(addr); } catch { return false; }
58
+ for (let name of names) {
59
+ name = name.toLowerCase().replace(/\.$/, '');
60
+ if (!suffixes.some((s) => name.endsWith(s))) continue;
61
+ let addrs = [];
62
+ try {
63
+ addrs = (await resolver.lookup(name, { all: true })).map((x) => x.address);
64
+ } catch { continue; }
65
+ if (addrs.some((a) => sameIp(a, addr))) return true;
66
+ }
67
+ return false;
68
+ }
69
+
70
+ return async function verified(req) {
71
+ const suffixes = claimedCrawler(req.headers['user-agent']);
72
+ if (!suffixes) return false;
73
+ const addr = clientIp(req);
74
+ const now = Date.now();
75
+ const got = seen.get(addr);
76
+ if (got && now - got.at < (got.ok ? OK_FOR : FAIL_FOR)) return got.ok;
77
+ if (inFlight >= LOOKUP_MAX) return false;
78
+ inFlight++;
79
+ let ok = false;
80
+ try {
81
+ let timer;
82
+ ok = await Promise.race([
83
+ check(addr, suffixes),
84
+ new Promise((r) => { timer = setTimeout(() => r(false), TIMEOUT); }),
85
+ ]);
86
+ clearTimeout(timer);
87
+ } finally {
88
+ inFlight--;
89
+ }
90
+ if (seen.size >= CACHE_MAX) seen.delete(seen.keys().next().value);
91
+ seen.set(addr, { ok, at: now });
92
+ return ok;
93
+ };
94
+ }
95
+
96
+ module.exports = { createCrawlerVerifier, remoteIp, claimedCrawler };
package/index.d.ts ADDED
@@ -0,0 +1,52 @@
1
+ import type { IncomingMessage, ServerResponse } from 'node:http';
2
+
3
+ export interface RanviaConfig {
4
+ /** Your site key from the Ranvia dashboard (rnv_...). */
5
+ key: string;
6
+ /** The Ranvia service; defaults to https://ranvia.io. */
7
+ serviceUrl?: string;
8
+ /** Where the encrypted engine is cached between starts; defaults to ~/.cache/ranvia. */
9
+ cacheDir?: string;
10
+ /** The visitor's network (an ASN like "as7922", or a country code), when your edge tells you. */
11
+ network?: (req: IncomingMessage) => string | undefined;
12
+ /** A visitor you have verified as signed in: no page budget, but by default a human-pace limit. Runs only for requests with cookies. */
13
+ signedIn?: (req: IncomingMessage) => boolean | Promise<boolean>;
14
+ /**
15
+ * A search crawler you have verified; never counted. Defaults to the built-in
16
+ * check: a request naming Googlebot, Bingbot, Applebot or YandexBot is verified
17
+ * by reverse and forward DNS of its address. Return false to turn it off.
18
+ */
19
+ verifiedCrawler?: (req: IncomingMessage) => boolean | Promise<boolean>;
20
+ /**
21
+ * The visitor's address. Defaults to the connection's address, or the last
22
+ * X-Forwarded-For entry when the connection comes from this machine. Behind a
23
+ * CDN, return the address it passes on (for Cloudflare, CF-Connecting-IP).
24
+ */
25
+ clientIp?: (req: IncomingMessage) => string;
26
+ /**
27
+ * Log every request Ranvia refuses, checks or rate-limits through `log`:
28
+ * method, path, User-Agent and the reason. The log stays on your server.
29
+ */
30
+ logRefusals?: boolean;
31
+ /** An optional JSON file of your own rule overrides. */
32
+ rulesFile?: string;
33
+ /** Receives one log line at a time. */
34
+ log?: (line: string) => void;
35
+ /** Public keys for rule bundles, for a service you run yourself. */
36
+ serviceKeys?: Record<string, string>;
37
+ }
38
+
39
+ export interface Ranvia {
40
+ /** Express / Connect middleware. */
41
+ middleware(req: IncomingMessage, res: ServerResponse, next: () => void): void;
42
+ /** Guards a plain node:http request handler. */
43
+ wrap(handler: (req: IncomingMessage, res: ServerResponse) => void): (req: IncomingMessage, res: ServerResponse) => void;
44
+ /** Resolves once the engine is running. Requests pass until then. */
45
+ ready(): Promise<void>;
46
+ isReady(): boolean;
47
+ /** Stops background work and sends a last usage report. */
48
+ close(): Promise<void>;
49
+ }
50
+
51
+ export function createRanvia(config: RanviaConfig): Ranvia;
52
+ export const LOADER_VERSION: string;
package/index.js ADDED
@@ -0,0 +1,347 @@
1
+ 'use strict';
2
+ // Ranvia for Node.js: bot and AI scraper protection that runs on your
3
+ // own server (https://ranvia.io).
4
+ //
5
+ // const { createRanvia } = require('ranvia');
6
+ // const ranvia = createRanvia({ key: process.env.RANVIA_KEY });
7
+ // app.use(ranvia.middleware); // Express, Connect
8
+ // http.createServer(ranvia.wrap(handler)); // plain node:http
9
+ //
10
+ // This package is a loader. The engine that decides is downloaded from
11
+ // ranvia.io for your site key, kept on disk encrypted for your site, and
12
+ // run in memory as WebAssembly. Your traffic never leaves your server:
13
+ // the service only sends rules and receives counts.
14
+ //
15
+ // Until the engine is running (first start with ranvia.io out of reach,
16
+ // a revoked key), every request passes and the reason is logged: Ranvia
17
+ // never takes your site down.
18
+
19
+ const crypto = require('node:crypto');
20
+ const fs = require('node:fs/promises');
21
+ const os = require('node:os');
22
+ const path = require('node:path');
23
+ const { wasiImports } = require('./wasi');
24
+ const { encodeRequest, decodeDecision } = require('./wire');
25
+ const { createCrawlerVerifier, remoteIp } = require('./crawlers');
26
+
27
+ const DEFAULT_SERVICE_URL = 'https://ranvia.io';
28
+ const LOADER_VERSION = '0.2.0';
29
+ const RULES_EVERY = 5 * 60e3;
30
+ const USAGE_EVERY = 60e3;
31
+ const ENGINE_EVERY = 6 * 3600e3;
32
+
33
+ function parseSiteKey(key) {
34
+ const m = /^rnv_([a-z0-9]+)\.([A-Za-z0-9_-]+)$/.exec(String(key || '').trim());
35
+ const secret = m && Buffer.from(m[2], 'base64url');
36
+ if (!m || secret.length < 32) throw new Error('ranvia: a site key looks like rnv_<site>.<secret>');
37
+ return { id: m[1], secret };
38
+ }
39
+
40
+ const jitter = (ms) => ms * (0.8 + Math.random() * 0.4);
41
+
42
+ // Engine is one running copy of the WebAssembly engine.
43
+ class Engine {
44
+ static async start(wasm, config, log) {
45
+ let memory;
46
+ const module = await WebAssembly.compile(wasm);
47
+ const instance = await WebAssembly.instantiate(module, {
48
+ wasi_snapshot_preview1: wasiImports(() => memory, log),
49
+ });
50
+ memory = instance.exports.memory;
51
+ instance.exports._initialize();
52
+ const e = new Engine(instance.exports);
53
+ const err = JSON.parse(e.call('ranvia_init', Buffer.from(JSON.stringify(config))).toString() || '{}').error;
54
+ if (err) throw new Error('ranvia: ' + err);
55
+ return e;
56
+ }
57
+
58
+ constructor(exports) {
59
+ this.x = exports;
60
+ }
61
+
62
+ // call runs an export with input copied into the engine's memory and
63
+ // returns its output, copied out.
64
+ call(name, input, ...extra) {
65
+ const x = this.x;
66
+ let args = extra;
67
+ let ptr = 0;
68
+ if (input) {
69
+ ptr = x.ranvia_alloc(input.length);
70
+ new Uint8Array(x.memory.buffer, ptr, input.length).set(input);
71
+ args = [ptr, input.length, ...extra];
72
+ }
73
+ try {
74
+ const res = BigInt.asUintN(64, BigInt(x[name](...args) ?? 0));
75
+ if (res === 0n) return Buffer.alloc(0);
76
+ const p = Number(res >> 32n);
77
+ const n = Number(res & 0xffffffffn);
78
+ return Buffer.from(new Uint8Array(x.memory.buffer, p, n));
79
+ } finally {
80
+ if (ptr) x.ranvia_free(ptr);
81
+ }
82
+ }
83
+
84
+ decide(req) {
85
+ return decodeDecision(this.call('ranvia_decide', encodeRequest(req)));
86
+ }
87
+
88
+ served(id, html) {
89
+ this.x.ranvia_served(id, html ? 1 : 0);
90
+ }
91
+
92
+ bundle(raw) {
93
+ const err = JSON.parse(this.call('ranvia_bundle', raw).toString() || '{}').error;
94
+ if (err) throw new Error('ranvia: ' + err);
95
+ }
96
+
97
+ rulesVersion() {
98
+ return this.call('ranvia_rules_version').toString();
99
+ }
100
+
101
+ tick() {
102
+ this.x.ranvia_tick();
103
+ }
104
+
105
+ usage() {
106
+ return this.call('ranvia_usage');
107
+ }
108
+
109
+ usageBack(raw) {
110
+ this.call('ranvia_usage_back', raw);
111
+ }
112
+ }
113
+
114
+ function createRanvia(cfg = {}) {
115
+ const key = parseSiteKey(cfg.key);
116
+ const log = cfg.log || (() => {});
117
+ const clientIp = cfg.clientIp || remoteIp;
118
+ const verifiedCrawler = cfg.verifiedCrawler || createCrawlerVerifier(clientIp);
119
+ const base = String(cfg.serviceUrl || DEFAULT_SERVICE_URL).replace(/\/+$/, '');
120
+ const cacheDir = cfg.cacheDir ||
121
+ path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), 'ranvia');
122
+ const cacheFile = path.join(cacheDir, 'engine-' + key.id + '.bin');
123
+ let engine = null;
124
+ let bundle = null;
125
+ let closed = false;
126
+ const timers = [];
127
+
128
+ // sign sets the Ranvia request headers (PROTOCOL.md).
129
+ function signedHeaders(method, urlPath, body) {
130
+ const ts = String(Math.floor(Date.now() / 1000));
131
+ const nonce = crypto.randomBytes(16).toString('hex');
132
+ const sum = crypto.createHash('sha256').update(body || Buffer.alloc(0)).digest('hex');
133
+ const sig = crypto.createHmac('sha256', key.secret)
134
+ .update(method + '\n' + urlPath + '\n' + ts + '\n' + nonce + '\n' + sum).digest('hex');
135
+ return {
136
+ 'Ranvia-Site': key.id, 'Ranvia-Time': ts, 'Ranvia-Nonce': nonce, 'Ranvia-Signature': sig,
137
+ 'User-Agent': 'ranvia-node/' + LOADER_VERSION,
138
+ };
139
+ }
140
+
141
+ async function service(method, urlPath, body, etag) {
142
+ const headers = signedHeaders(method, urlPath, body);
143
+ if (body) headers['Content-Type'] = 'application/json';
144
+ if (etag) headers['If-None-Match'] = etag;
145
+ const res = await fetch(base + urlPath, { method, headers, body, signal: AbortSignal.timeout(30e3) });
146
+ return { status: res.status, body: Buffer.from(await res.arrayBuffer()), etag: res.headers.get('etag') || '' };
147
+ }
148
+
149
+ // open unseals the site's engine copy, in memory only.
150
+ function open(sealed) {
151
+ const sealKey = crypto.createHmac('sha256', key.secret).update('ranvia engine key').digest();
152
+ if (sealed.length < 12 + 16) throw new Error('ranvia: the engine copy is truncated');
153
+ const d = crypto.createDecipheriv('aes-256-gcm', sealKey, sealed.subarray(0, 12));
154
+ d.setAAD(Buffer.from(key.id));
155
+ d.setAuthTag(sealed.subarray(sealed.length - 16));
156
+ try {
157
+ return Buffer.concat([d.update(sealed.subarray(12, sealed.length - 16)), d.final()]);
158
+ } catch {
159
+ throw new Error('ranvia: the engine copy does not open with this site key');
160
+ }
161
+ }
162
+
163
+ async function readCache() {
164
+ try {
165
+ const sealed = await fs.readFile(cacheFile);
166
+ const etag = (await fs.readFile(cacheFile + '.etag', 'utf8').catch(() => '')).trim();
167
+ return { sealed, etag };
168
+ } catch {
169
+ return { sealed: null, etag: '' };
170
+ }
171
+ }
172
+
173
+ async function writeCache(sealed, etag) {
174
+ try {
175
+ await fs.mkdir(cacheDir, { recursive: true, mode: 0o700 });
176
+ await fs.writeFile(cacheFile + '.tmp', sealed, { mode: 0o600 });
177
+ await fs.rename(cacheFile + '.tmp', cacheFile);
178
+ await fs.writeFile(cacheFile + '.etag', etag, { mode: 0o600 });
179
+ } catch (err) {
180
+ log('ranvia: cache: ' + err.message);
181
+ }
182
+ }
183
+
184
+ // fetchEngine returns the sealed engine: the service's copy when it
185
+ // answers, else the cache.
186
+ async function fetchEngine() {
187
+ const cached = await readCache();
188
+ let res;
189
+ try {
190
+ res = await service('GET', '/v1/engine', null, cached.etag);
191
+ } catch (err) {
192
+ if (cached.sealed) {
193
+ log('ranvia: engine check failed (' + err.message + '); using the cached copy');
194
+ return { sealed: cached.sealed, changed: false };
195
+ }
196
+ throw new Error('ranvia: engine download: ' + err.message);
197
+ }
198
+ if (res.status === 200) {
199
+ await writeCache(res.body, res.etag);
200
+ return { sealed: res.body, changed: true };
201
+ }
202
+ if (cached.sealed) {
203
+ if (res.status !== 304) log('ranvia: engine check: service answered ' + res.status + '; using the cached copy');
204
+ return { sealed: cached.sealed, changed: false };
205
+ }
206
+ throw new Error('ranvia: engine download: service answered ' + res.status + ': ' + res.body.toString().trim());
207
+ }
208
+
209
+ async function load(sealed) {
210
+ let rules;
211
+ if (cfg.rulesFile) rules = JSON.parse(await fs.readFile(cfg.rulesFile, 'utf8'));
212
+ const e = await Engine.start(open(sealed), { key: cfg.key, rules, service_keys: cfg.serviceKeys }, log);
213
+ if (bundle) {
214
+ try { e.bundle(bundle); } catch { /* the next refresh brings a fresh one */ }
215
+ }
216
+ return e;
217
+ }
218
+
219
+ async function refreshRules() {
220
+ const v = engine.rulesVersion();
221
+ const res = await service('GET', '/v1/rules', null, v ? '"' + v + '"' : '');
222
+ if (res.status === 304) return;
223
+ if (res.status !== 200) throw new Error('ranvia: rules: service answered ' + res.status);
224
+ engine.bundle(res.body);
225
+ bundle = res.body;
226
+ }
227
+
228
+ async function reportUsage(e = engine) {
229
+ if (!e) return;
230
+ const rep = e.usage();
231
+ if (!rep.length) return;
232
+ try {
233
+ const res = await service('POST', '/v1/usage', rep);
234
+ if (res.status >= 500) e.usageBack(rep);
235
+ } catch {
236
+ e.usageBack(rep);
237
+ }
238
+ }
239
+
240
+ function every(ms, fn) {
241
+ const run = async () => {
242
+ if (closed) return;
243
+ try { await fn(); } catch (err) { log(err.message); }
244
+ if (!closed) timers.push(setTimeout(run, jitter(ms)).unref());
245
+ };
246
+ timers.push(setTimeout(run, jitter(ms)).unref());
247
+ }
248
+
249
+ const started = (async () => {
250
+ let wait = 15e3;
251
+ while (!closed && !engine) {
252
+ try {
253
+ const { sealed } = await fetchEngine();
254
+ engine = await load(sealed);
255
+ log('ranvia: engine running');
256
+ } catch (err) {
257
+ log(err.message + '; passing requests until the engine loads');
258
+ await new Promise((r) => setTimeout(r, jitter(wait)).unref());
259
+ wait = Math.min(wait * 2, 5 * 60e3);
260
+ }
261
+ }
262
+ if (closed) return;
263
+ try { await refreshRules(); } catch (err) { log(err.message); }
264
+ every(RULES_EVERY, refreshRules);
265
+ every(USAGE_EVERY, async () => { engine.tick(); await reportUsage(); });
266
+ every(ENGINE_EVERY, async () => {
267
+ const { sealed, changed } = await fetchEngine();
268
+ if (!changed) return;
269
+ const next = await load(sealed);
270
+ const old = engine;
271
+ engine = next;
272
+ await reportUsage(old);
273
+ log('ranvia: engine updated');
274
+ });
275
+ })();
276
+
277
+ function readBody(req) {
278
+ return new Promise((resolve) => {
279
+ const chunks = [];
280
+ let size = 0;
281
+ req.on('data', (c) => { if (size < 1024) { chunks.push(c); size += c.length; } });
282
+ req.on('end', () => resolve(Buffer.concat(chunks).subarray(0, 1024).toString('utf8')));
283
+ req.on('error', () => resolve(''));
284
+ });
285
+ }
286
+
287
+ // handle decides one request: true when the site should serve it.
288
+ async function handle(req, res) {
289
+ const e = engine;
290
+ if (!e) return true;
291
+ const r = {
292
+ method: req.method, uri: req.originalUrl || req.url, host: req.headers.host || '',
293
+ header: req.headers, proto: req.httpVersion ? 'HTTP/' + req.httpVersion : '',
294
+ };
295
+ if (req.method === 'POST' && (req.url || '').split('?')[0] === '/guard/verify') r.body = await readBody(req);
296
+ if (cfg.network) r.network = cfg.network(req) || '';
297
+ r.crawler = !!(await verifiedCrawler(req));
298
+ if (cfg.signedIn && req.headers.cookie) r.signedIn = !!(await cfg.signedIn(req));
299
+ let d;
300
+ try {
301
+ d = e.decide(r);
302
+ } catch (err) {
303
+ log(err.message + '; passing the request');
304
+ return true;
305
+ }
306
+ for (const [k, v] of d.header) res.appendHeader(k, v);
307
+ if (!d.serve) {
308
+ if (cfg.logRefusals && d.reason) {
309
+ log(`ranvia: ${d.reason} ${req.method} ${(req.originalUrl || req.url || '').split('?')[0]} ${JSON.stringify(req.headers['user-agent'] || '')}`);
310
+ }
311
+ res.statusCode = d.status || 200;
312
+ res.end(d.body);
313
+ return false;
314
+ }
315
+ if (d.sniff) {
316
+ res.once('finish', () => {
317
+ const ct = String(res.getHeader('content-type') || '');
318
+ try { e.served(d.sniff, /^text\/html/i.test(ct)); } catch { /* engine replaced */ }
319
+ });
320
+ }
321
+ return true;
322
+ }
323
+
324
+ return {
325
+ // middleware is Express / Connect middleware.
326
+ middleware(req, res, next) {
327
+ handle(req, res).then((serve) => { if (serve) next(); }, (err) => { log(err.message); next(); });
328
+ },
329
+ // wrap guards a plain node:http request handler.
330
+ wrap(handler) {
331
+ return (req, res) => {
332
+ handle(req, res).then((serve) => { if (serve) handler(req, res); }, (err) => { log(err.message); handler(req, res); });
333
+ };
334
+ },
335
+ // ready resolves once the engine is running (or close is called).
336
+ ready: () => started,
337
+ isReady: () => engine !== null,
338
+ // close stops background work, sending a last usage report.
339
+ async close() {
340
+ closed = true;
341
+ for (const t of timers) clearTimeout(t);
342
+ await reportUsage();
343
+ },
344
+ };
345
+ }
346
+
347
+ module.exports = { createRanvia, LOADER_VERSION };
package/package.json CHANGED
@@ -1,8 +1,13 @@
1
1
  {
2
2
  "name": "ranvia",
3
- "version": "0.0.1",
4
- "description": "Ranvia: bot and AI scraper protection that runs on your own server. Coming soon at https://ranvia.io",
3
+ "version": "0.2.0",
4
+ "description": "Bot and AI scraper protection that runs on your own server: middleware for Express, Connect and node:http.",
5
+ "keywords": ["bot", "scraper", "ai", "crawler", "protection", "security", "middleware", "express"],
5
6
  "homepage": "https://ranvia.io",
6
- "license": "UNLICENSED",
7
- "files": ["README.md"]
7
+ "license": "SEE LICENSE IN README.md",
8
+ "author": "Synap6",
9
+ "main": "index.js",
10
+ "types": "index.d.ts",
11
+ "files": ["index.js", "index.d.ts", "wasi.js", "wire.js", "crawlers.js", "README.md"],
12
+ "engines": { "node": ">=20" }
8
13
  }
package/wasi.js ADDED
@@ -0,0 +1,99 @@
1
+ 'use strict';
2
+ // The few system calls the Ranvia engine (Go, wasip1) makes, done
3
+ // here rather than through node:wasi, which is still experimental in
4
+ // Node. The engine reads no files, arguments or environment; it needs
5
+ // clocks, random bytes and somewhere to write its rare log lines.
6
+
7
+ const { getRandomValues } = require('node:crypto');
8
+
9
+ const SUCCESS = 0;
10
+ const EBADF = 8;
11
+ const ENOSYS = 52;
12
+ const ENOTCAPABLE = 76;
13
+
14
+ function wasiImports(getMemory, log) {
15
+ const view = () => new DataView(getMemory().buffer);
16
+ const bytes = () => new Uint8Array(getMemory().buffer);
17
+ const zeroPair = (a, b) => {
18
+ const v = view();
19
+ v.setUint32(a, 0, true);
20
+ v.setUint32(b, 0, true);
21
+ return SUCCESS;
22
+ };
23
+ return {
24
+ args_sizes_get: zeroPair,
25
+ environ_sizes_get: zeroPair,
26
+ args_get: () => SUCCESS,
27
+ environ_get: () => SUCCESS,
28
+ clock_time_get(id, _precision, out) {
29
+ const ns = id === 0
30
+ ? BigInt(Date.now()) * 1000000n
31
+ : process.hrtime.bigint();
32
+ view().setBigUint64(out, ns, true);
33
+ return SUCCESS;
34
+ },
35
+ random_get(ptr, len) {
36
+ const b = bytes();
37
+ for (let off = 0; off < len; off += 65536) {
38
+ getRandomValues(b.subarray(ptr + off, ptr + Math.min(len, off + 65536)));
39
+ }
40
+ return SUCCESS;
41
+ },
42
+ fd_write(fd, iovs, iovsLen, nwritten) {
43
+ const v = view();
44
+ const b = bytes();
45
+ let total = 0;
46
+ const parts = [];
47
+ for (let i = 0; i < iovsLen; i++) {
48
+ const p = v.getUint32(iovs + i * 8, true);
49
+ const n = v.getUint32(iovs + i * 8 + 4, true);
50
+ parts.push(Buffer.from(b.subarray(p, p + n)));
51
+ total += n;
52
+ }
53
+ if (fd === 1 || fd === 2) {
54
+ const text = Buffer.concat(parts).toString('utf8').trimEnd();
55
+ if (text) log('ranvia engine: ' + text);
56
+ }
57
+ v.setUint32(nwritten, total, true);
58
+ return fd === 1 || fd === 2 ? SUCCESS : EBADF;
59
+ },
60
+ fd_read: () => EBADF,
61
+ fd_close: () => EBADF,
62
+ fd_fdstat_get(fd, out) {
63
+ if (fd > 2) return EBADF;
64
+ const v = view();
65
+ v.setUint8(out, 2); // character device
66
+ v.setUint16(out + 2, 0, true);
67
+ v.setBigUint64(out + 8, 0xffffffffffffffffn, true);
68
+ v.setBigUint64(out + 16, 0xffffffffffffffffn, true);
69
+ return SUCCESS;
70
+ },
71
+ fd_fdstat_set_flags: () => SUCCESS,
72
+ fd_prestat_get: () => EBADF, // no preopened directories
73
+ fd_prestat_dir_name: () => EBADF,
74
+ path_open: () => ENOTCAPABLE,
75
+ path_filestat_get: () => ENOTCAPABLE,
76
+ poll_oneoff(inPtr, outPtr, nsubs, nevents) {
77
+ // The engine never waits on anything real: report every
78
+ // subscription as fired.
79
+ const v = view();
80
+ for (let i = 0; i < nsubs; i++) {
81
+ const sub = inPtr + i * 48;
82
+ const ev = outPtr + i * 32;
83
+ v.setBigUint64(ev, v.getBigUint64(sub, true), true); // userdata
84
+ v.setUint16(ev + 8, SUCCESS, true);
85
+ v.setUint8(ev + 10, v.getUint8(sub + 8)); // event type
86
+ }
87
+ v.setUint32(nevents, nsubs, true);
88
+ return SUCCESS;
89
+ },
90
+ sched_yield: () => SUCCESS,
91
+ proc_exit(code) {
92
+ throw new Error('ranvia: the engine exited (' + code + ')');
93
+ },
94
+ // Anything else the engine might ask for is refused.
95
+ __proto__: new Proxy({}, { get: () => () => ENOSYS }),
96
+ };
97
+ }
98
+
99
+ module.exports = { wasiImports };
package/wire.js ADDED
@@ -0,0 +1,51 @@
1
+ 'use strict';
2
+ // The binary form the engine takes and returns on every request (the
3
+ // engine's wire.go): little-endian, strings as a u32 length and bytes.
4
+
5
+ function encodeRequest(req) {
6
+ const strs = [req.method, req.uri, req.host, req.network || '', req.body || ''];
7
+ const lines = [];
8
+ for (const [k, v] of Object.entries(req.header)) {
9
+ if (v === undefined) continue;
10
+ for (const one of Array.isArray(v) ? v : [String(v)]) lines.push([k, one]);
11
+ }
12
+ const n = Math.min(lines.length, 0xffff);
13
+ const bufs = strs.map((s) => Buffer.from(s, 'utf8'));
14
+ const pairs = lines.slice(0, n).map(([k, v]) => [Buffer.from(k, 'utf8'), Buffer.from(v, 'utf8')]);
15
+ const proto = Buffer.from(req.proto || '', 'utf8');
16
+ let size = 2 + 2 + 4 + proto.length;
17
+ for (const b of bufs) size += 4 + b.length;
18
+ for (const [k, v] of pairs) size += 8 + k.length + v.length;
19
+ const out = Buffer.allocUnsafe(size);
20
+ let o = 0;
21
+ out[o++] = 1;
22
+ out[o++] = (req.signedIn ? 1 : 0) | (req.crawler ? 2 : 0);
23
+ const put = (b) => { out.writeUInt32LE(b.length, o); o += 4; b.copy(out, o); o += b.length; };
24
+ for (const b of bufs) put(b);
25
+ out.writeUInt16LE(n, o); o += 2;
26
+ for (const [k, v] of pairs) { put(k); put(v); }
27
+ put(proto); // added after the first engines, which stop before it
28
+ return out;
29
+ }
30
+
31
+ function decodeDecision(buf) {
32
+ let o = 0;
33
+ const need = (n) => { if (o + n > buf.length) throw new Error('ranvia: malformed answer from the engine'); };
34
+ const str = () => {
35
+ need(4);
36
+ const n = buf.readUInt32LE(o); o += 4;
37
+ need(n);
38
+ const s = buf.subarray(o, o + n); o += n;
39
+ return s;
40
+ };
41
+ need(13);
42
+ const d = { serve: buf[0] === 1, sniff: buf.readBigUInt64LE(1), status: buf.readUInt16LE(9), header: [] };
43
+ const lines = buf.readUInt16LE(11); o = 13;
44
+ for (let i = 0; i < lines; i++) d.header.push([str().toString('utf8'), str().toString('utf8')]);
45
+ d.body = Buffer.from(str());
46
+ // Engines from before the reason field end here.
47
+ d.reason = o < buf.length ? str().toString('utf8') : '';
48
+ return d;
49
+ }
50
+
51
+ module.exports = { encodeRequest, decodeDecision };