swarmlite 0.2.0

package/README.md

@@ -0,0 +1,75 @@
+# swarmlite
+
+**Verifiable serverless SQLite on [Ethereum Swarm](https://www.ethswarm.org/).**
+
+Publish an ordinary SQLite file to Swarm and run `SELECT` against it —
+from a browser or Node — fetching only the 4 KB B-tree pages the query
+plan touches. Measured live: a cold point lookup on a published
+41.9 MB database fetched **5 pages / 20 KB**; a whole 4-query session
+23 pages. There is no database server anywhere, and with
+`verify: true` every byte is checked client-side against the 32-byte
+content address — a tampering gateway is caught on the first bad chunk.
+
+Everything is vendored (wa-sqlite, js-sha3, noble-secp256k1): zero
+runtime dependencies, and the whole reader can itself be published on
+Swarm next to your data — one immutable root serves the page, the
+engine, and the database.
+
+## Read (browser or Node)
+
+```js
+import { open, resolveFeed } from 'swarmlite';
+
+const db = await open('http://localhost:1633/bzz/<root>/site.db');
+const rows = await db.query('SELECT title FROM posts WHERE id = ?', [12345]);
+console.log(db.stats()); // { readCount, pagesFetched, bytesFetched, fileSize }
+```
+
+Any URL served with HTTP Range support works. Follow a feed for
+always-latest (resolve once, at load — a database must not move
+mid-session), and pass `{ verify: true }` on untrusted gateways:
+
+```js
+const { reference } = await resolveFeed(api, owner, 'mysite', { verify: true });
+const db = await open(`${api}/bzz/${reference}/site.db`, { verify: true });
+```
+
+In Node, pass the wasm explicitly:
+`open(url, { moduleOptions: { wasmBinary: await readFile(...) } })`.
+
+## Publish (Node)
+
+```js
+import { publish } from 'swarmlite/publisher';
+
+const { root } = await publish(await readFile('site.db'), {
+  feed: 'mysite',            // optional: stable URL + version history
+  signer: process.env.SWARMLITE_SIGNER,
+  log: console.error,
+});
+```
+
+or from the shell (no Python required):
+
+```bash
+npx swarmlite publish site.db --feed mysite --signer <key hex>
+npx swarmlite query "bzz://<root>/site.db" "SELECT count(*) FROM posts" --stats
+```
+
+The publisher runs the same checklist as the Python CLI — page size
+4096 (one page = one Swarm chunk), ANALYZE, VACUUM, integrity check,
+missing-index warnings — entirely in memory via wa-sqlite, then uploads
+and (optionally) signs a feed update. Feed signatures are byte-for-byte
+identical to the Python implementation's (checked in CI against
+fixtures the Python side generates). `--buy` prices and purchases a
+postage batch from the node's wallet, with the cost shown first.
+
+Writes are the publisher's job by design: work on a local SQLite file
+with full SQL, then ship an immutable snapshot. Reader connections are
+strictly read-only — the URL *is* the content hash.
+
+## More
+
+The [project repository](https://github.com/petfold/swarmlite) has the
+full User Guide (stamps, feeds, snapshots, verification, patterns), the
+Python reader/publisher, and a self-contained demo site. BSD-3-Clause.

package/bin/swarmlite.mjs

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+// swarmlite CLI (pure JS — no Python needed):
+//
+//   swarmlite publish <db> [--name F] [--feed TOPIC] [--signer HEX]
+//                          [--stamp ID] [--buy] [--ttl 1d] [--yes]
+//                          [--api-url URL] [--quiet]
+//   swarmlite query <url> <sql> [--verify] [--api-url URL]
+//
+// Same behavior as the Python CLI: bare root on stdout, human lines on
+// stderr, one-line errors, exit 1 on failure. SWARMLITE_DEBUG=1 shows
+// tracebacks; the signer key falls back to $SWARMLITE_SIGNER.
+
+import { readFile } from 'node:fs/promises';
+import { basename } from 'node:path';
+import { parseArgs } from 'node:util';
+import { createInterface } from 'node:readline/promises';
+
+const wasmBinary = await readFile(
+  new URL('../vendor/wa-sqlite/dist/wa-sqlite-async.wasm', import.meta.url));
+const moduleOptions = {
+  wasmBinary,
+  // Emscripten's random-device probe is noisy but harmless in Node
+  printErr: (t) => { if (!/initRandomDevice/.test(String(t))) console.error(t); },
+};
+
+const UNITS = { s: 1, m: 60, h: 3600, d: 86400, w: 7 * 86400 };
+
+function parseTtl(text) {
+  const unit = UNITS[text.slice(-1)];
+  const value = Number(unit ? text.slice(0, -1) : text);
+  if (!Number.isInteger(value) || value <= 0) {
+    throw new Error(`cannot parse TTL '${text}' — use e.g. 36h, 7d, 4w, or seconds`);
+  }
+  return value * (unit ?? 1);
+}
+
+function rejectPlaceholders(values) {
+  for (const [name, value] of Object.entries(values)) {
+    const m = value && /<[^<> ]+>/.exec(value);
+    if (m) {
+      throw new Error(
+        `${name} contains the placeholder ${m[0]} — replace it with the ` +
+        'real value (the root/reference is printed by publish)');
+    }
+  }
+}
+
+async function cmdPublish(args) {
+  const { values, positionals } = parseArgs({
+    args,
+    allowPositionals: true,
+    options: {
+      name: { type: 'string' },
+      feed: { type: 'string' },
+      signer: { type: 'string' },
+      stamp: { type: 'string', default: 'auto' },
+      buy: { type: 'boolean', default: false },
+      ttl: { type: 'string', default: '1d' },
+      yes: { type: 'boolean', default: false },
+      'api-url': { type: 'string' },
+      quiet: { type: 'boolean', default: false },
+    },
+  });
+  const [dbPath] = positionals;
+  if (!dbPath) throw new Error('usage: swarmlite publish <db> [options]');
+  rejectPlaceholders({
+    db_path: dbPath, name: values.name, feed: values.feed,
+    stamp: values.stamp, 'api-url': values['api-url'],
+  });
+  const apiUrl = values['api-url'] ?? process.env.BEE_API_URL ?? 'http://localhost:1633';
+  const log = values.quiet ? () => {} : (l) => console.error(l);
+
+  const bytes = new Uint8Array(await readFile(dbPath).catch(() => {
+    throw new Error(
+      `no such database file: '${dbPath}' — publish uploads an existing ` +
+      'local SQLite file');
+  }));
+
+  let stamp = values.stamp;
+  if (values.buy) {
+    if (stamp !== 'auto') throw new Error('--buy and --stamp are mutually exclusive');
+    const { planBatch, buyBatch } = await import('../src/publisher.js');
+    const plan = await planBatch(apiUrl, bytes.byteLength, parseTtl(values.ttl));
+    console.error(
+      `batch for ${(bytes.byteLength / 2 ** 20).toFixed(1)} MB: depth ${plan.depth}, ` +
+      `amount ${plan.amount}, lasting ~${Math.round(plan.ttlSecs / 3600)} h ` +
+      `-> ${plan.costBzz.toFixed(4)} xBZZ from the node's wallet`);
+    if (!values.yes) {
+      if (!process.stdin.isTTY) {
+        throw new Error('--buy needs a terminal to confirm; pass --yes to skip');
+      }
+      const rl = createInterface({ input: process.stdin, output: process.stderr });
+      const answer = (await rl.question('buy it? [y/N] ')).trim().toLowerCase();
+      rl.close();
+      if (answer !== 'y' && answer !== 'yes') {
+        throw new Error('purchase declined; nothing was bought');
+      }
+    }
+    console.error('buying (waits for on-chain confirmation) ...');
+    stamp = await buyBatch(apiUrl, plan.amount, plan.depth);
+    console.error(`bought batch ${stamp}`);
+  }
+
+  const { publish } = await import('../src/publisher.js');
+  const { root } = await publish(bytes, {
+    apiUrl,
+    name: values.name ?? basename(dbPath),
+    stamp,
+    feed: values.feed ?? null,
+    signer: values.signer ?? process.env.SWARMLITE_SIGNER ?? null,
+    moduleOptions,
+    log,
+  });
+  console.log(root);
+}
+
+async function cmdQuery(args) {
+  const { values, positionals } = parseArgs({
+    args,
+    allowPositionals: true,
+    options: {
+      verify: { type: 'boolean', default: false },
+      stats: { type: 'boolean', default: false },
+      'api-url': { type: 'string' },
+    },
+  });
+  const [url, sql] = positionals;
+  if (!url || !sql) throw new Error('usage: swarmlite query <url> <sql> [--verify]');
+  rejectPlaceholders({ url });
+  // accept bzz://<root>/<path> as shorthand for a gateway URL
+  const apiUrl = values['api-url'] ?? process.env.BEE_API_URL ?? 'http://localhost:1633';
+  const resolved = url.startsWith('bzz://')
+    ? `${apiUrl}/bzz/${url.slice('bzz://'.length)}`
+    : url;
+
+  const { open } = await import('../src/index.js');
+  const db = await open(resolved, { verify: values.verify, moduleOptions });
+  try {
+    for (const row of await db.query(sql)) {
+      console.log(Object.values(row).join('\t'));
+    }
+    if (values.stats) {
+      const s = db.stats();
+      console.error(
+        `fetched ${s.pagesFetched} pages (${(s.bytesFetched / 1024).toFixed(0)} KB) ` +
+        `in ${s.readCount} reads, of a ${(s.fileSize / 2 ** 20).toFixed(1)} MB file` +
+        (s.chunksFetched ? ` — ${s.chunksFetched} verified chunks` : ''));
+    }
+  } finally {
+    await db.close();
+  }
+}
+
+const [command, ...rest] = process.argv.slice(2);
+try {
+  if (command === 'publish') await cmdPublish(rest);
+  else if (command === 'query') await cmdQuery(rest);
+  else {
+    console.error(
+      'usage: swarmlite publish <db> [options] | swarmlite query <url> <sql> [--verify]');
+    process.exit(2);
+  }
+} catch (e) {
+  if (process.env.SWARMLITE_DEBUG) throw e;
+  console.error(`swarmlite: ${e.message}`);
+  process.exit(1);
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "swarmlite",
+  "version": "0.2.0",
+  "description": "Verifiable serverless SQLite on Ethereum Swarm: lazy SELECT over gateway range reads (browser and Node), client-side verification, and a pure-JS publisher",
+  "type": "module",
+  "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./publisher": "./src/publisher.js",
+    "./prepare": "./src/prepare.js",
+    "./verify": "./src/verify.js",
+    "./feeds": "./src/feeds.js"
+  },
+  "bin": {
+    "swarmlite": "bin/swarmlite.mjs"
+  },
+  "files": [
+    "src",
+    "vendor",
+    "bin"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/petfold/swarmlite.git",
+    "directory": "js"
+  },
+  "homepage": "https://github.com/petfold/swarmlite#readme",
+  "keywords": [
+    "swarm",
+    "ethswarm",
+    "sqlite",
+    "wasm",
+    "serverless",
+    "decentralized",
+    "web3",
+    "vfs",
+    "content-addressed"
+  ],
+  "license": "BSD-3-Clause",
+  "author": "Peter Foldiak",
+  "scripts": {
+    "test": "node test/feed.mjs && node test/verify.mjs && node test/publish.mjs",
+    "smoke": "node test/smoke.mjs",
+    "serve": "node test/serve.mjs"
+  }
+}

package/src/SwarmVFS.js

@@ -0,0 +1,274 @@
+// swarmlite browser VFS: lazy, read-only SQLite pages over Swarm gateway
+// range reads. The JS twin of swarmlite/vfs.py — same design, same
+// counters (docs/DESIGN.md section 4).
+//
+// Works with wa-sqlite's Asyncify build: VFS methods may return
+// handleAsync(async () => ...), so page fetches are ordinary async
+// fetch() calls — no SharedArrayBuffer, no COOP/COEP headers.
+//
+// The "filename" SQLite opens is a URL: anything a gateway serves with
+// HTTP Range support, typically
+//     http://localhost:1633/bzz/<root>/site.db        (pin, immutable)
+//     https://gateway.example/bzz/<root>/site.db
+// Feed URLs must be resolved to a root by the caller before opening
+// (a feed can move mid-session; a database must not).
+//
+// Read-only by design: journal/WAL are reported absent, writes fail.
+// Chunk-level BMT verification is a documented follow-up (roadmap):
+// against a local light node, retrieved chunks are verified by the node.
+
+import * as VFS from '../vendor/wa-sqlite/src/VFS.js';
+
+export const PAGE_SIZE = 4096; // SQLite default page == Swarm chunk
+const DEFAULT_CACHE_PAGES = 1024; // ~4 MiB
+const MAX_RUN_PAGES = 16; // cap one Range request at 64 KiB — big ranges
+                          // are the ones gateways abort on fresh content
+const FETCH_TIMEOUT_MS = 30_000;
+
+export class SwarmVFS extends VFS.Base {
+  name = 'swarmlite';
+  mxPathName = 1024; // filenames are URLs
+
+  /** @type {Map<number, object>} sqlite3_file id -> open file state */
+  #files = new Map();
+  /** @type {Map<string, string>} short alias -> URL */
+  #urls = new Map();
+  /** @type {Map<string, object>} short alias -> {size, read} source */
+  #sources = new Map();
+  #cachePages;
+  #fetch;
+
+  constructor({ cachePages = DEFAULT_CACHE_PAGES, fetchFn } = {}) {
+    super();
+    this.#cachePages = Math.max(1, cachePages);
+    this.#fetch = fetchFn ?? ((...args) => globalThis.fetch(...args));
+  }
+
+  /** Map a short alias to a database URL. SQLite opens the alias; the
+   * VFS fetches the URL. Necessary because VFS filename length is
+   * limited (mxPathName can be as small as 64 in some builds), and it
+   * keeps URLs out of SQLite's filename handling entirely. */
+  registerUrl(alias, url) {
+    this.#urls.set(alias, url);
+  }
+
+  /** Map an alias to a custom page source instead of a URL:
+   * `{ size, read(from, toInclusive) -> Promise<Uint8Array> }`. Used
+   * for verified reads, where pages come from a chunk-tree walk rather
+   * than gateway Range requests. */
+  registerSource(alias, source) {
+    this.#sources.set(alias, source);
+  }
+
+  xOpen(name, fileId, flags, pOutFlags) {
+    return this.handleAsync(async () => {
+      if (!(flags & VFS.SQLITE_OPEN_MAIN_DB)) {
+        return VFS.SQLITE_CANTOPEN; // no journals, no temp files
+      }
+      const source = this.#sources.get(name);
+      const url = this.#urls.get(name) ?? name;
+      try {
+        let size;
+        if (source) {
+          size = source.size;
+        } else {
+          // one 1-byte range read: existence check + total size
+          const resp = await this.#fetch(url, {
+            headers: { Range: 'bytes=0-0' },
+          });
+          size = contentRangeTotal(resp);
+          if (size === null) {
+            console.error(
+              `swarmlite: ${url} -> HTTP ${resp.status}, ` +
+              `Content-Range ${resp.headers.get('content-range')} — ` +
+              `need a gateway URL with Range support`,
+            );
+            return VFS.SQLITE_CANTOPEN;
+          }
+        }
+        this.#files.set(fileId, {
+          alias: name,
+          url,
+          source,
+          size,
+          cache: new Map(), // page index -> Uint8Array (Map keeps LRU order)
+          stats: { readCount: 0, pagesFetched: 0, bytesFetched: 0, fileSize: size },
+        });
+        pOutFlags.setInt32(0, VFS.SQLITE_OPEN_READONLY, true);
+        return VFS.SQLITE_OK;
+      } catch (e) {
+        console.error(`swarmlite: open ${url} failed:`, e);
+        return VFS.SQLITE_CANTOPEN;
+      }
+    });
+  }
+
+  xClose(fileId) {
+    this.#files.delete(fileId);
+    return VFS.SQLITE_OK;
+  }
+
+  xRead(fileId, pData, iOffset) {
+    return this.handleAsync(async () => {
+      const file = this.#files.get(fileId);
+      if (!file) return VFS.SQLITE_IOERR;
+      try {
+        const wanted = Math.min(iOffset + pData.byteLength, file.size) - iOffset;
+        if (wanted > 0) {
+          const data = await this.#readRange(file, iOffset, wanted);
+          pData.set(data, 0);
+        }
+        if (wanted < pData.byteLength) {
+          // short read past EOF: zero-fill, per SQLite contract
+          pData.fill(0, Math.max(wanted, 0));
+          return VFS.SQLITE_IOERR_SHORT_READ;
+        }
+        return VFS.SQLITE_OK;
+      } catch (e) {
+        console.error(`swarmlite: read @${iOffset} failed:`, e);
+        return VFS.SQLITE_IOERR;
+      }
+    });
+  }
+
+  xFileSize(fileId, pSize64) {
+    const file = this.#files.get(fileId);
+    if (!file) return VFS.SQLITE_IOERR;
+    pSize64.setBigInt64(0, BigInt(file.size), true);
+    return VFS.SQLITE_OK;
+  }
+
+  xAccess(name, flags, pResOut) {
+    pResOut.setInt32(0, 0, true); // journals/WAL never exist
+    return VFS.SQLITE_OK;
+  }
+
+  xSectorSize(fileId) {
+    return PAGE_SIZE;
+  }
+
+  xDeviceCharacteristics(fileId) {
+    return VFS.SQLITE_IOCAP_IMMUTABLE; // content-addressed: never changes
+  }
+
+  xWrite() {
+    return VFS.SQLITE_READONLY;
+  }
+
+  xTruncate() {
+    return VFS.SQLITE_READONLY;
+  }
+
+  xDelete() {
+    return VFS.SQLITE_READONLY;
+  }
+
+  /** Read counters for an open database URL or alias (last opened wins). */
+  statsFor(urlOrAlias) {
+    for (const file of this.#files.values()) {
+      if (file.url === urlOrAlias || file.alias === urlOrAlias) {
+        return { ...file.stats };
+      }
+    }
+    return null;
+  }
+
+  // ---- transport ----------------------------------------------------------
+
+  /** Ranged fetch with retry: gateways occasionally close a range read
+   * mid-stream on freshly uploaded content (seen live against Bee 2.8.1);
+   * a short backoff and retry recovers. */
+  async #fetchRange(url, from, to, attempts = 5) {
+    let lastErr;
+    for (let attempt = 1; attempt <= attempts; attempt++) {
+      try {
+        const resp = await this.#fetch(url, {
+          headers: { Range: `bytes=${from}-${to}` },
+          signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+        });
+        if (!resp.ok) throw new Error(`HTTP ${resp.status} for bytes=${from}-${to}`);
+        const blob = new Uint8Array(await resp.arrayBuffer());
+        if (blob.byteLength !== to - from + 1) {
+          throw new Error(
+            `truncated range read: got ${blob.byteLength} of ${to - from + 1}`);
+        }
+        return blob;
+      } catch (e) {
+        lastErr = e;
+        if (attempt < attempts) {
+          // seen live: freshly uploaded content can be unretrievable for
+          // seconds while the node syncs — back off up to ~7.5 s total
+          console.warn(
+            `swarmlite: retry ${attempt} for bytes=${from}-${to} (${e})`);
+          await new Promise((r) => setTimeout(r, 500 * 2 ** (attempt - 1)));
+        }
+      }
+    }
+    throw lastErr;
+  }
+
+  // ---- paging ------------------------------------------------------------
+
+  /** Serve [offset, offset+length) from the page cache, fetching each
+   * contiguous run of missing pages with a single Range request. */
+  async #readRange(file, offset, length) {
+    const first = Math.floor(offset / PAGE_SIZE);
+    const last = Math.floor((offset + length - 1) / PAGE_SIZE);
+
+    const pages = new Map();
+    for (let i = first; i <= last; i++) {
+      const hit = file.cache.get(i);
+      if (hit !== undefined) {
+        file.cache.delete(i); // refresh LRU position
+        file.cache.set(i, hit);
+        pages.set(i, hit);
+      }
+    }
+
+    let i = first;
+    while (i <= last) {
+      if (pages.has(i)) { i++; continue; }
+      let j = i;
+      while (j <= last && !pages.has(j) && j - i < MAX_RUN_PAGES) j++;
+      const from = i * PAGE_SIZE;
+      const to = Math.min(j * PAGE_SIZE, file.size) - 1;
+      const blob = file.source
+        ? await file.source.read(from, to)
+        : await this.#fetchRange(file.url, from, to);
+      file.stats.readCount += 1;
+      file.stats.bytesFetched += blob.byteLength;
+      for (let k = i; k < j; k++) {
+        const page = blob.subarray((k - i) * PAGE_SIZE, (k - i + 1) * PAGE_SIZE);
+        pages.set(k, page);
+        file.cache.set(k, page);
+        file.stats.pagesFetched += 1;
+      }
+      i = j;
+    }
+
+    while (file.cache.size > this.#cachePages) {
+      file.cache.delete(file.cache.keys().next().value); // evict LRU
+    }
+
+    const out = new Uint8Array(length);
+    let outPos = 0;
+    let cur = offset;
+    while (outPos < length) {
+      const k = Math.floor(cur / PAGE_SIZE);
+      const page = pages.get(k);
+      const pageStart = cur - k * PAGE_SIZE;
+      const n = Math.min(PAGE_SIZE - pageStart, length - outPos);
+      out.set(page.subarray(pageStart, pageStart + n), outPos);
+      outPos += n;
+      cur += n;
+    }
+    return out;
+  }
+}
+
+/** Parse the total size out of a 206 response's Content-Range header. */
+function contentRangeTotal(resp) {
+  if (resp.status !== 206) return null;
+  const m = /\/(\d+)\s*$/.exec(resp.headers.get('content-range') ?? '');
+  return m ? Number(m[1]) : null;
+}