swarmwire-lib 0.0.0-stage → 0.2.0

package/lib/blake3.js

@@ -0,0 +1,12 @@
+/**
+ * The hashing primitives, imported from one place.
+ *
+ * Everything that needs BLAKE3 or bao goes through here rather than reaching for
+ * `blake3-bao` directly, so the CommonJS detour that keeps us running on Bare — see
+ * `blake3.cjs` — stays in a single file.
+ */
+
+import bridge from './blake3.cjs'
+
+export const blake3 = bridge.blake3
+export const bao = bridge.bao

package/lib/collection.js

@@ -0,0 +1,133 @@
+/**
+ * A collection: a folder reduced to a list of content addresses.
+ *
+ * The manifest is itself content, so it has an address like anything else — sending a folder
+ * is sending one link, and the receiver fetches the manifest first and then the files it
+ * names. Nothing about it is mutable: change a file and you have a different manifest with a
+ * different address. Folders that change belong to SwarmFS, one layer up.
+ *
+ * The encoding is canonical, because the manifest's bytes are its identity: the same folder
+ * must produce the same address on every machine. Entries are sorted by path, there is no
+ * incidental whitespace, and paths are normalised before they are hashed.
+ *
+ *   {"swarmwire":"collection/v0","files":[{"path":"a.txt","root":"<hex>","size":12}]}
+ */
+
+import b4a from 'b4a'
+
+export const MANIFEST_TYPE = 'collection/v0'
+
+/** Enough for a folder of a million files; past that, something has gone wrong. */
+export const MAX_MANIFEST_BYTES = 64 * 1024 * 1024
+
+/**
+ * @param {{path: string, root: string, size: number}[]} files
+ * @returns {Uint8Array} the manifest's bytes, which are what gets addressed and served
+ */
+export function encodeManifest (files) {
+  const entries = files
+    .map((file) => ({
+      path: normalisePath(file.path),
+      root: file.root,
+      size: file.size
+    }))
+    .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
+
+  for (const entry of entries) assertEntry(entry)
+  assertNoDuplicates(entries)
+
+  return b4a.from(JSON.stringify({ swarmwire: MANIFEST_TYPE, files: entries }), 'utf8')
+}
+
+/**
+ * Read a manifest, refusing anything that is not one.
+ *
+ * Everything here arrived from a stranger, so nothing is taken on trust: a path that climbs
+ * out of the destination directory is the obvious attack, and the content addresses are
+ * meaningless if the paths they are written to are not.
+ *
+ * @param {Uint8Array} bytes
+ * @returns {{path: string, root: string, size: number}[]}
+ */
+export function decodeManifest (bytes) {
+  if (bytes.length > MAX_MANIFEST_BYTES) throw new Error('manifest is implausibly large')
+
+  let parsed
+  try {
+    parsed = JSON.parse(b4a.toString(bytes, 'utf8'))
+  } catch {
+    throw new Error('manifest is not valid JSON')
+  }
+
+  if (parsed?.swarmwire !== MANIFEST_TYPE) throw new Error('not a swarmwire collection')
+  if (!Array.isArray(parsed.files)) throw new Error('manifest has no file list')
+
+  // Paths are checked as they arrived, never normalised first. Quietly turning
+  // "/etc/passwd" into "etc/passwd" would write somewhere the sender did not name, and a
+  // manifest whose paths are not already canonical is not one we made.
+  const entries = parsed.files.map((file) => ({
+    path: file?.path,
+    root: file?.root,
+    size: file?.size
+  }))
+
+  for (const entry of entries) assertEntry(entry)
+  assertNoDuplicates(entries)
+
+  return entries
+}
+
+/** The total size of everything a manifest names. */
+export function totalSize (files) {
+  return files.reduce((sum, file) => sum + file.size, 0)
+}
+
+/**
+ * Normalise a path for the manifest: forward slashes, no leading slash, no `.` segments.
+ *
+ * This does not decide whether a path is safe — `assertEntry` does that — it only makes the
+ * same folder produce the same bytes whichever platform packed it.
+ */
+export function normalisePath (path) {
+  return String(path)
+    .replace(/\\/g, '/')
+    .split('/')
+    .filter((segment) => segment !== '' && segment !== '.')
+    .join('/')
+}
+
+/** Is this a path we are willing to write to, under a destination directory? */
+export function isSafePath (path) {
+  if (typeof path !== 'string' || path === '') return false
+  if (path.length > 4096) return false
+  if (path !== normalisePath(path)) return false
+
+  const segments = path.split('/')
+  if (segments.some((segment) => segment === '..')) return false
+
+  // Windows refuses these as filenames, and a path that works on one machine and not another
+  // is a collection that cannot be received everywhere.
+  if (/[<>:"|?*\u0000-\u001f]/.test(path)) return false
+  if (/^[a-zA-Z]:/.test(path)) return false
+
+  return true
+}
+
+function assertEntry (entry) {
+  if (!isSafePath(entry.path)) throw new Error(`unsafe path in manifest: ${JSON.stringify(entry.path)}`)
+  if (typeof entry.root !== 'string' || !/^[0-9a-f]{64}$/.test(entry.root)) {
+    throw new Error(`bad root in manifest for ${entry.path}`)
+  }
+  if (!Number.isInteger(entry.size) || entry.size < 0) {
+    throw new Error(`bad size in manifest for ${entry.path}`)
+  }
+}
+
+function assertNoDuplicates (entries) {
+  const seen = new Set()
+
+  for (const entry of entries) {
+    if (seen.has(entry.path)) throw new Error(`duplicate path in manifest: ${entry.path}`)
+    seen.add(entry.path)
+  }
+}

package/lib/destination.js

@@ -0,0 +1,237 @@
+/**
+ * Where a download lands — and, while it is still going, something other peers can fetch from.
+ *
+ * Three rules, and everything else follows:
+ *
+ *   1. A group is verified against the root *before* it is written. A peer that sends anything
+ *      else is refused and gains nothing: not a byte in the file, not a bit in the bitfield.
+ *   2. Progress survives a crash. Every accepted group appends a 36-byte record to the part
+ *      file beside the destination, so resuming costs nothing and rehashes nothing.
+ *   3. What is held can be proved. Those records carry each group's hash, which is what lets a
+ *      half-finished download serve its groups to somebody else instead of only taking.
+ */
+
+import { bao } from './blake3.js'
+import b4a from 'b4a'
+
+import { GroupTree, groupHash, verifyGroup, groupCount, groupRange, DEFAULT_GROUP_LOG } from './tree.js'
+import { NodeStore } from './nodes.js'
+import { PartFile, SUFFIX } from './partfile.js'
+import * as fsx from './fs.js'
+
+export const SIDECAR_SUFFIX = SUFFIX
+
+export class Destination {
+  /**
+   * @param {string} path - where the content is being written
+   * @param {{root: string, size: number, groupLog?: number}} opts
+   * @returns {Promise<Destination>}
+   */
+  static async open (path, { root, size, groupLog = DEFAULT_GROUP_LOG }) {
+    const count = groupCount(size, groupLog)
+    const { part, nodes } = await PartFile.open(path, { root, size, groupLog })
+
+    try {
+      const file = await fsx.openWrite(path, { size })
+      return new Destination(path, file, part, { root, size, groupLog, count, nodes })
+    } catch (err) {
+      await part.close()
+      throw err
+    }
+  }
+
+  constructor (path, file, part, { root, size, groupLog, count, nodes }) {
+    this.path = path
+    this.root = root
+    this.size = size
+    this.groupLog = groupLog
+    this.groupCount = count
+
+    this._file = file
+    this._part = part
+    this._writing = new Set()
+    this._pending = new Set() // puts still running, which must land before anything closes
+    this._closed = false
+
+    this._bitfield = bao.createBitfield(count)
+    this._nodes = new NodeStore(count)
+    this._have = 0
+
+    for (const { start, end, hash, held } of nodes) {
+      if (start >= count || end > count || end <= start) continue
+
+      this._nodes.set(start, end, hash)
+
+      // Only a group we received is on disk. Knowing a hash from somebody else's proof says
+      // nothing about whether we have the bytes.
+      if (held && !bao.getBit(this._bitfield, start)) {
+        bao.setBit(this._bitfield, start)
+        this._have++
+      }
+    }
+  }
+
+  /** Groups held so far. */
+  get have () {
+    return this._have
+  }
+
+  get complete () {
+    return this._have === this.groupCount
+  }
+
+  has (index) {
+    return bao.getBit(this._bitfield, index)
+  }
+
+  /** @returns {number[]} indices still wanted */
+  missing () {
+    const out = []
+    for (let i = 0; i < this.groupCount; i++) {
+      if (!this.has(i)) out.push(i)
+    }
+    return out
+  }
+
+  /** The bitfield as advertised to peers. */
+  bitfield () {
+    return this._bitfield
+  }
+
+  /**
+   * Offer a group. Returns false if it does not verify — in which case nothing is written and
+   * nothing is recorded, so a lying peer makes no progress by trying again.
+   *
+   * @param {number} index
+   * @param {Uint8Array} data
+   * @param {Uint8Array[]} proof
+   * @returns {Promise<boolean>}
+   */
+  async put (index, data, proof) {
+    if (this._closed) throw new Error('destination is closed')
+
+    const op = this._put(index, data, proof)
+    this._pending.add(op)
+
+    try {
+      return await op
+    } finally {
+      this._pending.delete(op)
+    }
+  }
+
+  async _put (index, data, proof) {
+    if (!Number.isInteger(index) || index < 0 || index >= this.groupCount) return false
+    if (this.has(index)) return true
+
+    const ok = verifyGroup({
+      root: this.root,
+      size: this.size,
+      index,
+      data,
+      proof,
+      groupLog: this.groupLog
+    })
+    if (!ok) return false
+
+    // Two peers can answer the same request at once. Both copies are valid — they verified —
+    // but the group must be counted once, or `have` overshoots and the download calls itself
+    // finished with a hole in it.
+    if (this._writing.has(index)) return true
+    this._writing.add(index)
+
+    try {
+      const { offset } = groupRange(index, this.size, this.groupLog)
+      await this._file.write(offset, data)
+    } finally {
+      this._writing.delete(index)
+    }
+
+    if (this.has(index)) return true
+
+    const hash = groupHash(data, index, {
+      groupLog: this.groupLog,
+      only: this.groupCount === 1
+    })
+
+    // Keep the proof as well as the group: those sibling hashes are exactly what we will need
+    // to prove this group to the next peer that asks for it.
+    const learned = this._nodes.learn(index, hash, proof)
+
+    bao.setBit(this._bitfield, index)
+    this._have++
+
+    // The group itself is marked as held; the proof hashes are knowledge, not data.
+    const records = learned.map((node) => ({ ...node, held: node.start === index && node.end === index + 1 }))
+    if (!records.some((node) => node.held)) {
+      records.push({ start: index, end: index + 1, hash, held: true })
+    }
+
+    await this._part.record(records)
+
+    return true
+  }
+
+  // -- serving what we already have ------------------------------------------
+
+  /** Wait for puts that started before the close. */
+  async _drain () {
+    while (this._pending.size > 0) {
+      await Promise.allSettled([...this._pending])
+    }
+  }
+
+  /** The finished tree. Only meaningful once every group is held. */
+  get tree () {
+    return new GroupTree(this._nodes.leaves(), { size: this.size, groupLog: this.groupLog })
+  }
+
+  /** Can this group be served with a proof right now? */
+  canProve (index) {
+    return this._nodes.canProve(index)
+  }
+
+  /** @returns {Uint8Array[]|null} null when some hash the proof needs was never learned */
+  proof (index) {
+    return this.has(index) ? this._nodes.proof(index) : null
+  }
+
+  /** Read a group back out of the destination file. */
+  async readGroup (index) {
+    if (!this.has(index)) return null
+
+    const { offset, length } = groupRange(index, this.size, this.groupLog)
+    return this._file.read(offset, length)
+  }
+
+  // -- lifecycle -------------------------------------------------------------
+
+  /** Complete the download: flush, drop the resume state, close. */
+  async finish () {
+    if (!this.complete) throw new Error('download is not complete')
+
+    // Closed before awaiting anything, so a group arriving now is refused cleanly rather
+    // than writing into a handle that is about to shut.
+    this._closed = true
+    await this._drain()
+
+    await this._file.sync()
+    await this._file.close()
+    await this._part.close()
+    await PartFile.remove(this.path)
+  }
+
+  /** Stop for now, keeping the resume state. */
+  async close () {
+    if (this._closed) return
+    this._closed = true
+
+    await this._drain()
+
+    try {
+      await this._part.close()
+    } finally {
+      await this._file.close() // never leak the handle, whatever else fails
+    }
+  }
+}

package/lib/fs.js

@@ -0,0 +1,176 @@
+/**
+ * The only module that touches a filesystem.
+ *
+ * Everything else goes through here so that Bare can supply `bare-fs` without the rest of the
+ * library knowing. Keep this thin: open, read at an offset, write at an offset, stat, unlink.
+ */
+
+import fs from 'fs'
+import b4a from 'b4a'
+
+/**
+ * @param {string} path
+ * @returns {Promise<{size: number, mtimeMs: number, directory: boolean}|null>} null if it
+ *   does not exist
+ */
+export async function stat (path) {
+  try {
+    const st = await fs.promises.stat(path)
+    return { size: st.size, mtimeMs: Math.floor(st.mtimeMs), directory: st.isDirectory() }
+  } catch (err) {
+    if (err.code === 'ENOENT') return null
+    throw err
+  }
+}
+
+/** Open a file for reading byte ranges out of it, in place. */
+export async function openRead (path) {
+  const handle = await fs.promises.open(path, 'r')
+
+  return {
+    async read (offset, length) {
+      const buffer = b4a.alloc(length)
+      if (length === 0) return buffer
+
+      const { bytesRead } = await handle.read(buffer, 0, length, offset)
+      return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+    },
+
+    close () {
+      return handle.close()
+    }
+  }
+}
+
+/**
+ * Open a file for writing byte ranges into it, creating it and giving it its final size up
+ * front so ranges can land in any order.
+ */
+export async function openWrite (path, { size }) {
+  const handle = await fs.promises.open(path, (await stat(path)) ? 'r+' : 'w+')
+  await handle.truncate(size)
+
+  return {
+    async write (offset, data) {
+      if (data.length === 0) return
+      await handle.write(data, 0, data.length, offset)
+    },
+
+    /** Reading back matters: a partial download serves the groups it already holds. */
+    async read (offset, length) {
+      const buffer = b4a.alloc(length)
+      if (length === 0) return buffer
+
+      const { bytesRead } = await handle.read(buffer, 0, length, offset)
+      return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+    },
+
+    sync () {
+      return handle.sync()
+    },
+
+    close () {
+      return handle.close()
+    }
+  }
+}
+
+/**
+ * Open a file for appending, readable too. Used for the append-only part file, where every
+ * write goes to the end and reads walk back over what was written.
+ */
+export async function openAppend (path) {
+  // Not 'a+': Windows refuses to truncate a handle opened in append mode, and we need to
+  // truncate when the file turns out to describe different content. Track the end ourselves.
+  const handle = await fs.promises.open(path, (await stat(path)) ? 'r+' : 'w+')
+  const info = await handle.stat()
+
+  // Appends are serialised. Groups are verified concurrently, and two appends that each read
+  // the end of the file before either wrote would land on top of each other.
+  let tail = Promise.resolve()
+
+  return {
+    size: info.size,
+
+    async read (offset, length) {
+      const buffer = b4a.alloc(length)
+      if (length === 0) return buffer
+
+      const { bytesRead } = await handle.read(buffer, 0, length, offset)
+      return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+    },
+
+    append (data) {
+      const write = tail.then(async () => {
+        await handle.write(data, 0, data.length, this.size)
+        this.size += data.length
+      })
+
+      tail = write.catch(() => {}) // a failed append must not wedge the queue
+      return write
+    },
+
+    async truncate (size) {
+      await handle.truncate(size)
+      this.size = size
+    },
+
+    close () {
+      return handle.close()
+    }
+  }
+}
+
+/**
+ * Every file under a directory, as paths relative to it, in a stable order.
+ *
+ * Symbolic links are skipped rather than followed: a link pointing outside the folder would
+ * quietly put somebody else's file into a collection.
+ *
+ * @param {string} dir
+ * @returns {Promise<string[]>}
+ */
+export async function list (dir) {
+  const found = []
+
+  async function walk (current, prefix) {
+    const entries = await fs.promises.readdir(current, { withFileTypes: true })
+
+    for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+      const relative = prefix ? prefix + '/' + entry.name : entry.name
+
+      if (entry.isDirectory()) await walk(current + '/' + entry.name, relative)
+      else if (entry.isFile()) found.push(relative)
+    }
+  }
+
+  await walk(dir, '')
+  return found
+}
+
+/** Create a directory and any parents it needs. */
+export function mkdir (dir) {
+  return fs.promises.mkdir(dir, { recursive: true })
+}
+
+/** Small whole-file helpers, for sidecars only. */
+export async function readFile (path) {
+  try {
+    return await fs.promises.readFile(path)
+  } catch (err) {
+    if (err.code === 'ENOENT') return null
+    throw err
+  }
+}
+
+export function writeFile (path, data) {
+  return fs.promises.writeFile(path, data)
+}
+
+export async function unlink (path) {
+  try {
+    await fs.promises.unlink(path)
+  } catch (err) {
+    if (err.code !== 'ENOENT') throw err
+  }
+}

package/lib/holdings.js

@@ -0,0 +1,125 @@
+/**
+ * What connected peers say they hold, and the counts that follow from it.
+ *
+ * A node only hears about a peer's holdings if it asked: `want` is answered with `have`, and
+ * the peer then pushes `holds` as it gains groups. A node that only serves never asks, so it
+ * keeps a watch set — the roots it serves — and sends `want` for those to every peer. That is
+ * the whole mechanism; there is no message for it.
+ *
+ * Counts are of direct connections. Nothing here knows about the rest of the swarm.
+ */
+
+import b4a from 'b4a'
+
+import { bao } from './blake3.js'
+import { DEFAULT_GROUP_LOG, groupCount } from './tree.js'
+
+export class Holdings {
+  constructor ({ groupLog = DEFAULT_GROUP_LOG } = {}) {
+    this.groupLog = groupLog
+    this.peers = new Map() // peer id → Map(root hex → { size, total, bits, count })
+    this.watched = new Map() // root hex → size: roots we ask every peer about
+  }
+
+  /** A peer described what it holds (`have`). Replaces whatever we knew. */
+  set (peerId, root, { size, groupLog, bitfield }) {
+    if (groupLog !== this.groupLog) return // we cannot read that granularity: unknown, not guessed
+
+    const total = groupCount(size, this.groupLog)
+    const bits = bao.createBitfield(total)
+    bits.set(b4a.from(bitfield ?? []).subarray(0, bits.length))
+
+    this._of(peerId).set(root, { size, total, bits, count: bao.countSetBits(bits, total) })
+  }
+
+  /**
+   * A peer gained one group (`holds`). Without a `have` to build on, a known `size` stands in:
+   * a peer that first said "unavailable" and then began downloading only ever sends this.
+   */
+  grow (peerId, root, index, size) {
+    let held = this.get(peerId, root)
+
+    if (!held) {
+      if (size === undefined) return
+      const total = groupCount(size, this.groupLog)
+      held = { size, total, bits: bao.createBitfield(total), count: 0 }
+      this._of(peerId).set(root, held)
+    }
+
+    if (index < 0 || index >= held.total || bao.getBit(held.bits, index)) return
+
+    bao.setBit(held.bits, index)
+    held.count++
+  }
+
+  /** The peer says it has none of this (`unavailable`). */
+  remove (peerId, root) {
+    this.peers.get(peerId)?.delete(root)
+  }
+
+  dropPeer (peerId) {
+    this.peers.delete(peerId)
+  }
+
+  get (peerId, root) {
+    return this.peers.get(peerId)?.get(root)
+  }
+
+  /**
+   * @param {Iterable<string>} peerIds - the connected peers to count among
+   * @param {string[]} roots - one root for a file; the manifest and every file for a folder
+   * @returns {{peers: number, seeders: number, leechers: number}} a seeder holds every group
+   *   of every root, a leecher holds some but not all, anyone else is in neither
+   */
+  count (peerIds, roots) {
+    const counts = { peers: 0, seeders: 0, leechers: 0 }
+
+    for (const id of peerIds) {
+      counts.peers++
+      if (roots.length === 0) continue
+
+      let complete = true
+      let any = false
+
+      for (const root of roots) {
+        const held = this.get(id, root)
+        if (!held || held.count < held.total) complete = false
+        if (held && held.count > 0) any = true
+      }
+
+      if (complete) counts.seeders++
+      else if (any) counts.leechers++
+    }
+
+    return counts
+  }
+
+  // -- the watch set ---------------------------------------------------------
+
+  /** Ask every peer about a root we hold or follow, now and whenever a new peer connects. */
+  watch (root, size, peers, skip) {
+    this.watched.set(root, size)
+    for (const peer of peers) this._want(peer, root, size, skip)
+  }
+
+  unwatch (root) {
+    this.watched.delete(root)
+  }
+
+  /** A peer just connected: tell it what we are watching. */
+  wantAll (peer, skip) {
+    for (const [root, size] of this.watched) this._want(peer, root, size, skip)
+  }
+
+  _want (peer, root, size, skip) {
+    // shortcut: wants for every file of a served folder; watch lazily if folders grow past ~10k files
+    if (skip(root)) return // a download already asked this peer
+    peer.send.want({ root: b4a.from(root, 'hex'), size })
+  }
+
+  _of (peerId) {
+    let held = this.peers.get(peerId)
+    if (!held) this.peers.set(peerId, held = new Map())
+    return held
+  }
+}