ranvia 0.0.0-stage → 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -2
- package/index.d.ts +37 -0
- package/index.js +341 -0
- package/package.json +11 -4
- package/wasi.js +99 -0
- package/wire.js +47 -0
package/README.md
CHANGED
|
@@ -1,3 +1,54 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Ranvia for Node.js
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Bot and AI scraper protection that runs on your own server.
|
|
4
|
+
[ranvia.io](https://ranvia.io)
|
|
5
|
+
|
|
6
|
+
> **Early access.** The Ranvia service is opening soon. Until your site
|
|
7
|
+
> has a key, there is nothing to install.
|
|
8
|
+
|
|
9
|
+
```js
|
|
10
|
+
const { createRanvia } = require('ranvia');
|
|
11
|
+
|
|
12
|
+
const ranvia = createRanvia({ key: process.env.RANVIA_KEY });
|
|
13
|
+
|
|
14
|
+
app.use(ranvia.middleware); // Express, Connect
|
|
15
|
+
// or
|
|
16
|
+
http.createServer(ranvia.wrap(handler)); // plain node:http
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## How it works
|
|
20
|
+
|
|
21
|
+
- **Your traffic never leaves your server.** Every decision is made
|
|
22
|
+
locally, in about a tenth of a millisecond.
|
|
23
|
+
- **What the service sends and receives:** it sends your site's rules;
|
|
24
|
+
your site sends back counts. Visitor addresses, cookies, URLs and
|
|
25
|
+
page content are never sent.
|
|
26
|
+
- **This package is a loader.** On first start it downloads the Ranvia
|
|
27
|
+
engine for your site key, keeps it on disk encrypted for your site,
|
|
28
|
+
and runs it in memory as WebAssembly.
|
|
29
|
+
- **Ranvia never takes your site down.** If the engine can't run (first
|
|
30
|
+
start with ranvia.io out of reach, a revoked key), every request
|
|
31
|
+
passes and the reason is logged.
|
|
32
|
+
- **What visitors see:** scrapers are refused. Clients that behave like
|
|
33
|
+
scrapers get a short browser check, which a real browser passes in
|
|
34
|
+
about a second with no clicks.
|
|
35
|
+
|
|
36
|
+
## Options
|
|
37
|
+
|
|
38
|
+
| Option | What it does |
|
|
39
|
+
|---|---|
|
|
40
|
+
| `key` | Your site key from the Ranvia dashboard. |
|
|
41
|
+
| `network(req)` | The visitor's network, if your edge provides it: an ASN (`as7922`) or a country code. |
|
|
42
|
+
| `signedIn(req)` | Return `true` for a visitor you've verified as signed in; they are never counted. |
|
|
43
|
+
| `verifiedCrawler(req)` | Return `true` for a search crawler you've verified. |
|
|
44
|
+
| `cacheDir` | Where the encrypted engine is kept. Defaults to `~/.cache/ranvia`. |
|
|
45
|
+
| `rulesFile` | A JSON file of your own rule overrides. |
|
|
46
|
+
| `log(line)` | Receives log lines. |
|
|
47
|
+
|
|
48
|
+
Requires Node.js 20 or later.
|
|
49
|
+
|
|
50
|
+
## License
|
|
51
|
+
|
|
52
|
+
Use of this package and of the engine it downloads is governed by the
|
|
53
|
+
Ranvia terms of service at https://ranvia.io. Copyright Synap6. All
|
|
54
|
+
rights reserved.
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
|
+
|
|
3
|
+
export interface RanviaConfig {
|
|
4
|
+
/** Your site key from the Ranvia dashboard (rnv_...). */
|
|
5
|
+
key: string;
|
|
6
|
+
/** The Ranvia service; defaults to https://ranvia.io. */
|
|
7
|
+
serviceUrl?: string;
|
|
8
|
+
/** Where the encrypted engine is cached between starts; defaults to ~/.cache/ranvia. */
|
|
9
|
+
cacheDir?: string;
|
|
10
|
+
/** The visitor's network (an ASN like "as7922", or a country code), when your edge tells you. */
|
|
11
|
+
network?: (req: IncomingMessage) => string | undefined;
|
|
12
|
+
/** A visitor you have verified as signed in; never counted. Runs only for requests with cookies. */
|
|
13
|
+
signedIn?: (req: IncomingMessage) => boolean | Promise<boolean>;
|
|
14
|
+
/** A search crawler you have verified; never counted. */
|
|
15
|
+
verifiedCrawler?: (req: IncomingMessage) => boolean;
|
|
16
|
+
/** An optional JSON file of your own rule overrides. */
|
|
17
|
+
rulesFile?: string;
|
|
18
|
+
/** Receives one log line at a time. */
|
|
19
|
+
log?: (line: string) => void;
|
|
20
|
+
/** Public keys for rule bundles, for a service you run yourself. */
|
|
21
|
+
serviceKeys?: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface Ranvia {
|
|
25
|
+
/** Express / Connect middleware. */
|
|
26
|
+
middleware(req: IncomingMessage, res: ServerResponse, next: () => void): void;
|
|
27
|
+
/** Guards a plain node:http request handler. */
|
|
28
|
+
wrap(handler: (req: IncomingMessage, res: ServerResponse) => void): (req: IncomingMessage, res: ServerResponse) => void;
|
|
29
|
+
/** Resolves once the engine is running. Requests pass until then. */
|
|
30
|
+
ready(): Promise<void>;
|
|
31
|
+
isReady(): boolean;
|
|
32
|
+
/** Stops background work and sends a last usage report. */
|
|
33
|
+
close(): Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function createRanvia(config: RanviaConfig): Ranvia;
|
|
37
|
+
export const LOADER_VERSION: string;
|
package/index.js
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Ranvia for Node.js: bot and AI scraper protection that runs on your
|
|
3
|
+
// own server (https://ranvia.io).
|
|
4
|
+
//
|
|
5
|
+
// const { createRanvia } = require('ranvia');
|
|
6
|
+
// const ranvia = createRanvia({ key: process.env.RANVIA_KEY });
|
|
7
|
+
// app.use(ranvia.middleware); // Express, Connect
|
|
8
|
+
// http.createServer(ranvia.wrap(handler)); // plain node:http
|
|
9
|
+
//
|
|
10
|
+
// This package is a loader. The engine that decides is downloaded from
|
|
11
|
+
// ranvia.io for your site key, kept on disk encrypted for your site, and
|
|
12
|
+
// run in memory as WebAssembly. Your traffic never leaves your server:
|
|
13
|
+
// the service only sends rules and receives counts.
|
|
14
|
+
//
|
|
15
|
+
// Until the engine is running (first start with ranvia.io out of reach,
|
|
16
|
+
// a revoked key), every request passes and the reason is logged: Ranvia
|
|
17
|
+
// never takes your site down.
|
|
18
|
+
|
|
19
|
+
const crypto = require('node:crypto');
|
|
20
|
+
const fs = require('node:fs/promises');
|
|
21
|
+
const os = require('node:os');
|
|
22
|
+
const path = require('node:path');
|
|
23
|
+
const { wasiImports } = require('./wasi');
|
|
24
|
+
const { encodeRequest, decodeDecision } = require('./wire');
|
|
25
|
+
|
|
26
|
+
const DEFAULT_SERVICE_URL = 'https://ranvia.io';
|
|
27
|
+
const LOADER_VERSION = '0.1.0';
|
|
28
|
+
const RULES_EVERY = 5 * 60e3;
|
|
29
|
+
const USAGE_EVERY = 60e3;
|
|
30
|
+
const ENGINE_EVERY = 6 * 3600e3;
|
|
31
|
+
|
|
32
|
+
function parseSiteKey(key) {
|
|
33
|
+
const m = /^rnv_([a-z0-9]+)\.([A-Za-z0-9_-]+)$/.exec(String(key || '').trim());
|
|
34
|
+
const secret = m && Buffer.from(m[2], 'base64url');
|
|
35
|
+
if (!m || secret.length < 32) throw new Error('ranvia: a site key looks like rnv_<site>.<secret>');
|
|
36
|
+
return { id: m[1], secret };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const jitter = (ms) => ms * (0.8 + Math.random() * 0.4);
|
|
40
|
+
|
|
41
|
+
// Engine is one running copy of the WebAssembly engine.
|
|
42
|
+
class Engine {
|
|
43
|
+
static async start(wasm, config, log) {
|
|
44
|
+
let memory;
|
|
45
|
+
const module = await WebAssembly.compile(wasm);
|
|
46
|
+
const instance = await WebAssembly.instantiate(module, {
|
|
47
|
+
wasi_snapshot_preview1: wasiImports(() => memory, log),
|
|
48
|
+
});
|
|
49
|
+
memory = instance.exports.memory;
|
|
50
|
+
instance.exports._initialize();
|
|
51
|
+
const e = new Engine(instance.exports);
|
|
52
|
+
const err = JSON.parse(e.call('ranvia_init', Buffer.from(JSON.stringify(config))).toString() || '{}').error;
|
|
53
|
+
if (err) throw new Error('ranvia: ' + err);
|
|
54
|
+
return e;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
constructor(exports) {
|
|
58
|
+
this.x = exports;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// call runs an export with input copied into the engine's memory and
|
|
62
|
+
// returns its output, copied out.
|
|
63
|
+
call(name, input, ...extra) {
|
|
64
|
+
const x = this.x;
|
|
65
|
+
let args = extra;
|
|
66
|
+
let ptr = 0;
|
|
67
|
+
if (input) {
|
|
68
|
+
ptr = x.ranvia_alloc(input.length);
|
|
69
|
+
new Uint8Array(x.memory.buffer, ptr, input.length).set(input);
|
|
70
|
+
args = [ptr, input.length, ...extra];
|
|
71
|
+
}
|
|
72
|
+
try {
|
|
73
|
+
const res = BigInt.asUintN(64, BigInt(x[name](...args) ?? 0));
|
|
74
|
+
if (res === 0n) return Buffer.alloc(0);
|
|
75
|
+
const p = Number(res >> 32n);
|
|
76
|
+
const n = Number(res & 0xffffffffn);
|
|
77
|
+
return Buffer.from(new Uint8Array(x.memory.buffer, p, n));
|
|
78
|
+
} finally {
|
|
79
|
+
if (ptr) x.ranvia_free(ptr);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
decide(req) {
|
|
84
|
+
return decodeDecision(this.call('ranvia_decide', encodeRequest(req)));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
served(id, html) {
|
|
88
|
+
this.x.ranvia_served(id, html ? 1 : 0);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
bundle(raw) {
|
|
92
|
+
const err = JSON.parse(this.call('ranvia_bundle', raw).toString() || '{}').error;
|
|
93
|
+
if (err) throw new Error('ranvia: ' + err);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
rulesVersion() {
|
|
97
|
+
return this.call('ranvia_rules_version').toString();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
tick() {
|
|
101
|
+
this.x.ranvia_tick();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
usage() {
|
|
105
|
+
return this.call('ranvia_usage');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
usageBack(raw) {
|
|
109
|
+
this.call('ranvia_usage_back', raw);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function createRanvia(cfg = {}) {
|
|
114
|
+
const key = parseSiteKey(cfg.key);
|
|
115
|
+
const log = cfg.log || (() => {});
|
|
116
|
+
const base = String(cfg.serviceUrl || DEFAULT_SERVICE_URL).replace(/\/+$/, '');
|
|
117
|
+
const cacheDir = cfg.cacheDir ||
|
|
118
|
+
path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), 'ranvia');
|
|
119
|
+
const cacheFile = path.join(cacheDir, 'engine-' + key.id + '.bin');
|
|
120
|
+
let engine = null;
|
|
121
|
+
let bundle = null;
|
|
122
|
+
let closed = false;
|
|
123
|
+
const timers = [];
|
|
124
|
+
|
|
125
|
+
// sign sets the Ranvia request headers (PROTOCOL.md).
|
|
126
|
+
function signedHeaders(method, urlPath, body) {
|
|
127
|
+
const ts = String(Math.floor(Date.now() / 1000));
|
|
128
|
+
const nonce = crypto.randomBytes(16).toString('hex');
|
|
129
|
+
const sum = crypto.createHash('sha256').update(body || Buffer.alloc(0)).digest('hex');
|
|
130
|
+
const sig = crypto.createHmac('sha256', key.secret)
|
|
131
|
+
.update(method + '\n' + urlPath + '\n' + ts + '\n' + nonce + '\n' + sum).digest('hex');
|
|
132
|
+
return {
|
|
133
|
+
'Ranvia-Site': key.id, 'Ranvia-Time': ts, 'Ranvia-Nonce': nonce, 'Ranvia-Signature': sig,
|
|
134
|
+
'User-Agent': 'ranvia-node/' + LOADER_VERSION,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function service(method, urlPath, body, etag) {
|
|
139
|
+
const headers = signedHeaders(method, urlPath, body);
|
|
140
|
+
if (body) headers['Content-Type'] = 'application/json';
|
|
141
|
+
if (etag) headers['If-None-Match'] = etag;
|
|
142
|
+
const res = await fetch(base + urlPath, { method, headers, body, signal: AbortSignal.timeout(30e3) });
|
|
143
|
+
return { status: res.status, body: Buffer.from(await res.arrayBuffer()), etag: res.headers.get('etag') || '' };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// open unseals the site's engine copy, in memory only.
|
|
147
|
+
function open(sealed) {
|
|
148
|
+
const sealKey = crypto.createHmac('sha256', key.secret).update('ranvia engine key').digest();
|
|
149
|
+
if (sealed.length < 12 + 16) throw new Error('ranvia: the engine copy is truncated');
|
|
150
|
+
const d = crypto.createDecipheriv('aes-256-gcm', sealKey, sealed.subarray(0, 12));
|
|
151
|
+
d.setAAD(Buffer.from(key.id));
|
|
152
|
+
d.setAuthTag(sealed.subarray(sealed.length - 16));
|
|
153
|
+
try {
|
|
154
|
+
return Buffer.concat([d.update(sealed.subarray(12, sealed.length - 16)), d.final()]);
|
|
155
|
+
} catch {
|
|
156
|
+
throw new Error('ranvia: the engine copy does not open with this site key');
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async function readCache() {
|
|
161
|
+
try {
|
|
162
|
+
const sealed = await fs.readFile(cacheFile);
|
|
163
|
+
const etag = (await fs.readFile(cacheFile + '.etag', 'utf8').catch(() => '')).trim();
|
|
164
|
+
return { sealed, etag };
|
|
165
|
+
} catch {
|
|
166
|
+
return { sealed: null, etag: '' };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function writeCache(sealed, etag) {
|
|
171
|
+
try {
|
|
172
|
+
await fs.mkdir(cacheDir, { recursive: true, mode: 0o700 });
|
|
173
|
+
await fs.writeFile(cacheFile + '.tmp', sealed, { mode: 0o600 });
|
|
174
|
+
await fs.rename(cacheFile + '.tmp', cacheFile);
|
|
175
|
+
await fs.writeFile(cacheFile + '.etag', etag, { mode: 0o600 });
|
|
176
|
+
} catch (err) {
|
|
177
|
+
log('ranvia: cache: ' + err.message);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// fetchEngine returns the sealed engine: the service's copy when it
|
|
182
|
+
// answers, else the cache.
|
|
183
|
+
async function fetchEngine() {
|
|
184
|
+
const cached = await readCache();
|
|
185
|
+
let res;
|
|
186
|
+
try {
|
|
187
|
+
res = await service('GET', '/v1/engine', null, cached.etag);
|
|
188
|
+
} catch (err) {
|
|
189
|
+
if (cached.sealed) {
|
|
190
|
+
log('ranvia: engine check failed (' + err.message + '); using the cached copy');
|
|
191
|
+
return { sealed: cached.sealed, changed: false };
|
|
192
|
+
}
|
|
193
|
+
throw new Error('ranvia: engine download: ' + err.message);
|
|
194
|
+
}
|
|
195
|
+
if (res.status === 200) {
|
|
196
|
+
await writeCache(res.body, res.etag);
|
|
197
|
+
return { sealed: res.body, changed: true };
|
|
198
|
+
}
|
|
199
|
+
if (cached.sealed) {
|
|
200
|
+
if (res.status !== 304) log('ranvia: engine check: service answered ' + res.status + '; using the cached copy');
|
|
201
|
+
return { sealed: cached.sealed, changed: false };
|
|
202
|
+
}
|
|
203
|
+
throw new Error('ranvia: engine download: service answered ' + res.status + ': ' + res.body.toString().trim());
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function load(sealed) {
|
|
207
|
+
let rules;
|
|
208
|
+
if (cfg.rulesFile) rules = JSON.parse(await fs.readFile(cfg.rulesFile, 'utf8'));
|
|
209
|
+
const e = await Engine.start(open(sealed), { key: cfg.key, rules, service_keys: cfg.serviceKeys }, log);
|
|
210
|
+
if (bundle) {
|
|
211
|
+
try { e.bundle(bundle); } catch { /* the next refresh brings a fresh one */ }
|
|
212
|
+
}
|
|
213
|
+
return e;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async function refreshRules() {
|
|
217
|
+
const v = engine.rulesVersion();
|
|
218
|
+
const res = await service('GET', '/v1/rules', null, v ? '"' + v + '"' : '');
|
|
219
|
+
if (res.status === 304) return;
|
|
220
|
+
if (res.status !== 200) throw new Error('ranvia: rules: service answered ' + res.status);
|
|
221
|
+
engine.bundle(res.body);
|
|
222
|
+
bundle = res.body;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function reportUsage(e = engine) {
|
|
226
|
+
if (!e) return;
|
|
227
|
+
const rep = e.usage();
|
|
228
|
+
if (!rep.length) return;
|
|
229
|
+
try {
|
|
230
|
+
const res = await service('POST', '/v1/usage', rep);
|
|
231
|
+
if (res.status >= 500) e.usageBack(rep);
|
|
232
|
+
} catch {
|
|
233
|
+
e.usageBack(rep);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function every(ms, fn) {
|
|
238
|
+
const run = async () => {
|
|
239
|
+
if (closed) return;
|
|
240
|
+
try { await fn(); } catch (err) { log(err.message); }
|
|
241
|
+
if (!closed) timers.push(setTimeout(run, jitter(ms)).unref());
|
|
242
|
+
};
|
|
243
|
+
timers.push(setTimeout(run, jitter(ms)).unref());
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const started = (async () => {
|
|
247
|
+
let wait = 15e3;
|
|
248
|
+
while (!closed && !engine) {
|
|
249
|
+
try {
|
|
250
|
+
const { sealed } = await fetchEngine();
|
|
251
|
+
engine = await load(sealed);
|
|
252
|
+
log('ranvia: engine running');
|
|
253
|
+
} catch (err) {
|
|
254
|
+
log(err.message + '; passing requests until the engine loads');
|
|
255
|
+
await new Promise((r) => setTimeout(r, jitter(wait)).unref());
|
|
256
|
+
wait = Math.min(wait * 2, 5 * 60e3);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
if (closed) return;
|
|
260
|
+
try { await refreshRules(); } catch (err) { log(err.message); }
|
|
261
|
+
every(RULES_EVERY, refreshRules);
|
|
262
|
+
every(USAGE_EVERY, async () => { engine.tick(); await reportUsage(); });
|
|
263
|
+
every(ENGINE_EVERY, async () => {
|
|
264
|
+
const { sealed, changed } = await fetchEngine();
|
|
265
|
+
if (!changed) return;
|
|
266
|
+
const next = await load(sealed);
|
|
267
|
+
const old = engine;
|
|
268
|
+
engine = next;
|
|
269
|
+
await reportUsage(old);
|
|
270
|
+
log('ranvia: engine updated');
|
|
271
|
+
});
|
|
272
|
+
})();
|
|
273
|
+
|
|
274
|
+
function readBody(req) {
|
|
275
|
+
return new Promise((resolve) => {
|
|
276
|
+
const chunks = [];
|
|
277
|
+
let size = 0;
|
|
278
|
+
req.on('data', (c) => { if (size < 1024) { chunks.push(c); size += c.length; } });
|
|
279
|
+
req.on('end', () => resolve(Buffer.concat(chunks).subarray(0, 1024).toString('utf8')));
|
|
280
|
+
req.on('error', () => resolve(''));
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// handle decides one request: true when the site should serve it.
|
|
285
|
+
async function handle(req, res) {
|
|
286
|
+
const e = engine;
|
|
287
|
+
if (!e) return true;
|
|
288
|
+
const r = {
|
|
289
|
+
method: req.method, uri: req.originalUrl || req.url, host: req.headers.host || '',
|
|
290
|
+
header: req.headers,
|
|
291
|
+
};
|
|
292
|
+
if (req.method === 'POST' && (req.url || '').split('?')[0] === '/guard/verify') r.body = await readBody(req);
|
|
293
|
+
if (cfg.network) r.network = cfg.network(req) || '';
|
|
294
|
+
if (cfg.verifiedCrawler) r.crawler = !!cfg.verifiedCrawler(req);
|
|
295
|
+
if (cfg.signedIn && req.headers.cookie) r.signedIn = !!(await cfg.signedIn(req));
|
|
296
|
+
let d;
|
|
297
|
+
try {
|
|
298
|
+
d = e.decide(r);
|
|
299
|
+
} catch (err) {
|
|
300
|
+
log(err.message + '; passing the request');
|
|
301
|
+
return true;
|
|
302
|
+
}
|
|
303
|
+
for (const [k, v] of d.header) res.appendHeader(k, v);
|
|
304
|
+
if (!d.serve) {
|
|
305
|
+
res.statusCode = d.status || 200;
|
|
306
|
+
res.end(d.body);
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
if (d.sniff) {
|
|
310
|
+
res.once('finish', () => {
|
|
311
|
+
const ct = String(res.getHeader('content-type') || '');
|
|
312
|
+
try { e.served(d.sniff, /^text\/html/i.test(ct)); } catch { /* engine replaced */ }
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
return true;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
return {
|
|
319
|
+
// middleware is Express / Connect middleware.
|
|
320
|
+
middleware(req, res, next) {
|
|
321
|
+
handle(req, res).then((serve) => { if (serve) next(); }, (err) => { log(err.message); next(); });
|
|
322
|
+
},
|
|
323
|
+
// wrap guards a plain node:http request handler.
|
|
324
|
+
wrap(handler) {
|
|
325
|
+
return (req, res) => {
|
|
326
|
+
handle(req, res).then((serve) => { if (serve) handler(req, res); }, (err) => { log(err.message); handler(req, res); });
|
|
327
|
+
};
|
|
328
|
+
},
|
|
329
|
+
// ready resolves once the engine is running (or close is called).
|
|
330
|
+
ready: () => started,
|
|
331
|
+
isReady: () => engine !== null,
|
|
332
|
+
// close stops background work, sending a last usage report.
|
|
333
|
+
async close() {
|
|
334
|
+
closed = true;
|
|
335
|
+
for (const t of timers) clearTimeout(t);
|
|
336
|
+
await reportUsage();
|
|
337
|
+
},
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
module.exports = { createRanvia, LOADER_VERSION };
|
package/package.json
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ranvia",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0-alpha.1",
|
|
4
|
+
"description": "Bot and AI scraper protection that runs on your own server: middleware for Express, Connect and node:http.",
|
|
5
|
+
"keywords": ["bot", "scraper", "ai", "crawler", "protection", "security", "middleware", "express"],
|
|
6
|
+
"homepage": "https://ranvia.io",
|
|
7
|
+
"license": "SEE LICENSE IN README.md",
|
|
8
|
+
"author": "Synap6",
|
|
9
|
+
"main": "index.js",
|
|
10
|
+
"types": "index.d.ts",
|
|
11
|
+
"files": ["index.js", "index.d.ts", "wasi.js", "wire.js", "README.md"],
|
|
12
|
+
"engines": { "node": ">=20" }
|
|
13
|
+
}
|
package/wasi.js
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// The few system calls the Ranvia engine (Go, wasip1) makes, done
|
|
3
|
+
// here rather than through node:wasi, which is still experimental in
|
|
4
|
+
// Node. The engine reads no files, arguments or environment; it needs
|
|
5
|
+
// clocks, random bytes and somewhere to write its rare log lines.
|
|
6
|
+
|
|
7
|
+
const { getRandomValues } = require('node:crypto');
|
|
8
|
+
|
|
9
|
+
const SUCCESS = 0;
|
|
10
|
+
const EBADF = 8;
|
|
11
|
+
const ENOSYS = 52;
|
|
12
|
+
const ENOTCAPABLE = 76;
|
|
13
|
+
|
|
14
|
+
function wasiImports(getMemory, log) {
|
|
15
|
+
const view = () => new DataView(getMemory().buffer);
|
|
16
|
+
const bytes = () => new Uint8Array(getMemory().buffer);
|
|
17
|
+
const zeroPair = (a, b) => {
|
|
18
|
+
const v = view();
|
|
19
|
+
v.setUint32(a, 0, true);
|
|
20
|
+
v.setUint32(b, 0, true);
|
|
21
|
+
return SUCCESS;
|
|
22
|
+
};
|
|
23
|
+
return {
|
|
24
|
+
args_sizes_get: zeroPair,
|
|
25
|
+
environ_sizes_get: zeroPair,
|
|
26
|
+
args_get: () => SUCCESS,
|
|
27
|
+
environ_get: () => SUCCESS,
|
|
28
|
+
clock_time_get(id, _precision, out) {
|
|
29
|
+
const ns = id === 0
|
|
30
|
+
? BigInt(Date.now()) * 1000000n
|
|
31
|
+
: process.hrtime.bigint();
|
|
32
|
+
view().setBigUint64(out, ns, true);
|
|
33
|
+
return SUCCESS;
|
|
34
|
+
},
|
|
35
|
+
random_get(ptr, len) {
|
|
36
|
+
const b = bytes();
|
|
37
|
+
for (let off = 0; off < len; off += 65536) {
|
|
38
|
+
getRandomValues(b.subarray(ptr + off, ptr + Math.min(len, off + 65536)));
|
|
39
|
+
}
|
|
40
|
+
return SUCCESS;
|
|
41
|
+
},
|
|
42
|
+
fd_write(fd, iovs, iovsLen, nwritten) {
|
|
43
|
+
const v = view();
|
|
44
|
+
const b = bytes();
|
|
45
|
+
let total = 0;
|
|
46
|
+
const parts = [];
|
|
47
|
+
for (let i = 0; i < iovsLen; i++) {
|
|
48
|
+
const p = v.getUint32(iovs + i * 8, true);
|
|
49
|
+
const n = v.getUint32(iovs + i * 8 + 4, true);
|
|
50
|
+
parts.push(Buffer.from(b.subarray(p, p + n)));
|
|
51
|
+
total += n;
|
|
52
|
+
}
|
|
53
|
+
if (fd === 1 || fd === 2) {
|
|
54
|
+
const text = Buffer.concat(parts).toString('utf8').trimEnd();
|
|
55
|
+
if (text) log('ranvia engine: ' + text);
|
|
56
|
+
}
|
|
57
|
+
v.setUint32(nwritten, total, true);
|
|
58
|
+
return fd === 1 || fd === 2 ? SUCCESS : EBADF;
|
|
59
|
+
},
|
|
60
|
+
fd_read: () => EBADF,
|
|
61
|
+
fd_close: () => EBADF,
|
|
62
|
+
fd_fdstat_get(fd, out) {
|
|
63
|
+
if (fd > 2) return EBADF;
|
|
64
|
+
const v = view();
|
|
65
|
+
v.setUint8(out, 2); // character device
|
|
66
|
+
v.setUint16(out + 2, 0, true);
|
|
67
|
+
v.setBigUint64(out + 8, 0xffffffffffffffffn, true);
|
|
68
|
+
v.setBigUint64(out + 16, 0xffffffffffffffffn, true);
|
|
69
|
+
return SUCCESS;
|
|
70
|
+
},
|
|
71
|
+
fd_fdstat_set_flags: () => SUCCESS,
|
|
72
|
+
fd_prestat_get: () => EBADF, // no preopened directories
|
|
73
|
+
fd_prestat_dir_name: () => EBADF,
|
|
74
|
+
path_open: () => ENOTCAPABLE,
|
|
75
|
+
path_filestat_get: () => ENOTCAPABLE,
|
|
76
|
+
poll_oneoff(inPtr, outPtr, nsubs, nevents) {
|
|
77
|
+
// The engine never waits on anything real: report every
|
|
78
|
+
// subscription as fired.
|
|
79
|
+
const v = view();
|
|
80
|
+
for (let i = 0; i < nsubs; i++) {
|
|
81
|
+
const sub = inPtr + i * 48;
|
|
82
|
+
const ev = outPtr + i * 32;
|
|
83
|
+
v.setBigUint64(ev, v.getBigUint64(sub, true), true); // userdata
|
|
84
|
+
v.setUint16(ev + 8, SUCCESS, true);
|
|
85
|
+
v.setUint8(ev + 10, v.getUint8(sub + 8)); // event type
|
|
86
|
+
}
|
|
87
|
+
v.setUint32(nevents, nsubs, true);
|
|
88
|
+
return SUCCESS;
|
|
89
|
+
},
|
|
90
|
+
sched_yield: () => SUCCESS,
|
|
91
|
+
proc_exit(code) {
|
|
92
|
+
throw new Error('ranvia: the engine exited (' + code + ')');
|
|
93
|
+
},
|
|
94
|
+
// Anything else the engine might ask for is refused.
|
|
95
|
+
__proto__: new Proxy({}, { get: () => () => ENOSYS }),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
module.exports = { wasiImports };
|
package/wire.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// The binary form the engine takes and returns on every request (the
|
|
3
|
+
// engine's wire.go): little-endian, strings as a u32 length and bytes.
|
|
4
|
+
|
|
5
|
+
function encodeRequest(req) {
|
|
6
|
+
const strs = [req.method, req.uri, req.host, req.network || '', req.body || ''];
|
|
7
|
+
const lines = [];
|
|
8
|
+
for (const [k, v] of Object.entries(req.header)) {
|
|
9
|
+
if (v === undefined) continue;
|
|
10
|
+
for (const one of Array.isArray(v) ? v : [String(v)]) lines.push([k, one]);
|
|
11
|
+
}
|
|
12
|
+
const n = Math.min(lines.length, 0xffff);
|
|
13
|
+
const bufs = strs.map((s) => Buffer.from(s, 'utf8'));
|
|
14
|
+
const pairs = lines.slice(0, n).map(([k, v]) => [Buffer.from(k, 'utf8'), Buffer.from(v, 'utf8')]);
|
|
15
|
+
let size = 2 + 2;
|
|
16
|
+
for (const b of bufs) size += 4 + b.length;
|
|
17
|
+
for (const [k, v] of pairs) size += 8 + k.length + v.length;
|
|
18
|
+
const out = Buffer.allocUnsafe(size);
|
|
19
|
+
let o = 0;
|
|
20
|
+
out[o++] = 1;
|
|
21
|
+
out[o++] = (req.signedIn ? 1 : 0) | (req.crawler ? 2 : 0);
|
|
22
|
+
const put = (b) => { out.writeUInt32LE(b.length, o); o += 4; b.copy(out, o); o += b.length; };
|
|
23
|
+
for (const b of bufs) put(b);
|
|
24
|
+
out.writeUInt16LE(n, o); o += 2;
|
|
25
|
+
for (const [k, v] of pairs) { put(k); put(v); }
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function decodeDecision(buf) {
|
|
30
|
+
let o = 0;
|
|
31
|
+
const need = (n) => { if (o + n > buf.length) throw new Error('ranvia: malformed answer from the engine'); };
|
|
32
|
+
const str = () => {
|
|
33
|
+
need(4);
|
|
34
|
+
const n = buf.readUInt32LE(o); o += 4;
|
|
35
|
+
need(n);
|
|
36
|
+
const s = buf.subarray(o, o + n); o += n;
|
|
37
|
+
return s;
|
|
38
|
+
};
|
|
39
|
+
need(13);
|
|
40
|
+
const d = { serve: buf[0] === 1, sniff: buf.readBigUInt64LE(1), status: buf.readUInt16LE(9), header: [] };
|
|
41
|
+
const lines = buf.readUInt16LE(11); o = 13;
|
|
42
|
+
for (let i = 0; i < lines; i++) d.header.push([str().toString('utf8'), str().toString('utf8')]);
|
|
43
|
+
d.body = Buffer.from(str());
|
|
44
|
+
return d;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
module.exports = { encodeRequest, decodeDecision };
|