swarmwire-lib 0.0.0-stage → 0.1.0

package/lib/collection.js

@@ -0,0 +1,133 @@
+/**
+ * A collection: a folder reduced to a list of content addresses.
+ *
+ * The manifest is itself content, so it has an address like anything else — sending a folder
+ * is sending one link, and the receiver fetches the manifest first and then the files it
+ * names. Nothing about it is mutable: change a file and you have a different manifest with a
+ * different address. Folders that change belong to SwarmFS, one layer up.
+ *
+ * The encoding is canonical, because the manifest's bytes are its identity: the same folder
+ * must produce the same address on every machine. Entries are sorted by path, there is no
+ * incidental whitespace, and paths are normalised before they are hashed.
+ *
+ *   {"swarmwire":"collection/v0","files":[{"path":"a.txt","root":"<hex>","size":12}]}
+ */
+
+import b4a from 'b4a'
+
+export const MANIFEST_TYPE = 'collection/v0'
+
+/** Enough for a folder of a million files; past that, something has gone wrong. */
+const MAX_MANIFEST_BYTES = 64 * 1024 * 1024
+
+/**
+ * @param {{path: string, root: string, size: number}[]} files
+ * @returns {Uint8Array} the manifest's bytes, which are what gets addressed and served
+ */
+export function encodeManifest (files) {
+  const entries = files
+    .map((file) => ({
+      path: normalisePath(file.path),
+      root: file.root,
+      size: file.size
+    }))
+    .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
+
+  for (const entry of entries) assertEntry(entry)
+  assertNoDuplicates(entries)
+
+  return b4a.from(JSON.stringify({ swarmwire: MANIFEST_TYPE, files: entries }), 'utf8')
+}
+
+/**
+ * Read a manifest, refusing anything that is not one.
+ *
+ * Everything here arrived from a stranger, so nothing is taken on trust: a path that climbs
+ * out of the destination directory is the obvious attack, and the content addresses are
+ * meaningless if the paths they are written to are not.
+ *
+ * @param {Uint8Array} bytes
+ * @returns {{path: string, root: string, size: number}[]}
+ */
+export function decodeManifest (bytes) {
+  if (bytes.length > MAX_MANIFEST_BYTES) throw new Error('manifest is implausibly large')
+
+  let parsed
+  try {
+    parsed = JSON.parse(b4a.toString(bytes, 'utf8'))
+  } catch {
+    throw new Error('manifest is not valid JSON')
+  }
+
+  if (parsed?.swarmwire !== MANIFEST_TYPE) throw new Error('not a swarmwire collection')
+  if (!Array.isArray(parsed.files)) throw new Error('manifest has no file list')
+
+  // Paths are checked as they arrived, never normalised first. Quietly turning
+  // "/etc/passwd" into "etc/passwd" would write somewhere the sender did not name, and a
+  // manifest whose paths are not already canonical is not one we made.
+  const entries = parsed.files.map((file) => ({
+    path: file?.path,
+    root: file?.root,
+    size: file?.size
+  }))
+
+  for (const entry of entries) assertEntry(entry)
+  assertNoDuplicates(entries)
+
+  return entries
+}
+
+/** The total size of everything a manifest names. */
+export function totalSize (files) {
+  return files.reduce((sum, file) => sum + file.size, 0)
+}
+
+/**
+ * Normalise a path for the manifest: forward slashes, no leading slash, no `.` segments.
+ *
+ * This does not decide whether a path is safe — `assertEntry` does that — it only makes the
+ * same folder produce the same bytes whichever platform packed it.
+ */
+export function normalisePath (path) {
+  return String(path)
+    .replace(/\\/g, '/')
+    .split('/')
+    .filter((segment) => segment !== '' && segment !== '.')
+    .join('/')
+}
+
+/** Is this a path we are willing to write to, under a destination directory? */
+export function isSafePath (path) {
+  if (typeof path !== 'string' || path === '') return false
+  if (path.length > 4096) return false
+  if (path !== normalisePath(path)) return false
+
+  const segments = path.split('/')
+  if (segments.some((segment) => segment === '..')) return false
+
+  // Windows refuses these as filenames, and a path that works on one machine and not another
+  // is a collection that cannot be received everywhere.
+  if (/[<>:"|?*\u0000-\u001f]/.test(path)) return false
+  if (/^[a-zA-Z]:/.test(path)) return false
+
+  return true
+}
+
+function assertEntry (entry) {
+  if (!isSafePath(entry.path)) throw new Error(`unsafe path in manifest: ${JSON.stringify(entry.path)}`)
+  if (typeof entry.root !== 'string' || !/^[0-9a-f]{64}$/.test(entry.root)) {
+    throw new Error(`bad root in manifest for ${entry.path}`)
+  }
+  if (!Number.isInteger(entry.size) || entry.size < 0) {
+    throw new Error(`bad size in manifest for ${entry.path}`)
+  }
+}
+
+function assertNoDuplicates (entries) {
+  const seen = new Set()
+
+  for (const entry of entries) {
+    if (seen.has(entry.path)) throw new Error(`duplicate path in manifest: ${entry.path}`)
+    seen.add(entry.path)
+  }
+}

package/lib/destination.js

@@ -0,0 +1,237 @@
+/**
+ * Where a download lands — and, while it is still going, something other peers can fetch from.
+ *
+ * Three rules, and everything else follows:
+ *
+ *   1. A group is verified against the root *before* it is written. A peer that sends anything
+ *      else is refused and gains nothing: not a byte in the file, not a bit in the bitfield.
+ *   2. Progress survives a crash. Every accepted group appends a 36-byte record to the part
+ *      file beside the destination, so resuming costs nothing and rehashes nothing.
+ *   3. What is held can be proved. Those records carry each group's hash, which is what lets a
+ *      half-finished download serve its groups to somebody else instead of only taking.
+ */
+
+import { bao } from './blake3.js'
+import b4a from 'b4a'
+
+import { GroupTree, groupHash, verifyGroup, groupCount, groupRange, DEFAULT_GROUP_LOG } from './tree.js'
+import { NodeStore } from './nodes.js'
+import { PartFile, SUFFIX } from './partfile.js'
+import * as fsx from './fs.js'
+
+export const SIDECAR_SUFFIX = SUFFIX
+
+export class Destination {
+  /**
+   * @param {string} path - where the content is being written
+   * @param {{root: string, size: number, groupLog?: number}} opts
+   * @returns {Promise<Destination>}
+   */
+  static async open (path, { root, size, groupLog = DEFAULT_GROUP_LOG }) {
+    const count = groupCount(size, groupLog)
+    const { part, nodes } = await PartFile.open(path, { root, size, groupLog })
+
+    try {
+      const file = await fsx.openWrite(path, { size })
+      return new Destination(path, file, part, { root, size, groupLog, count, nodes })
+    } catch (err) {
+      await part.close()
+      throw err
+    }
+  }
+
+  constructor (path, file, part, { root, size, groupLog, count, nodes }) {
+    this.path = path
+    this.root = root
+    this.size = size
+    this.groupLog = groupLog
+    this.groupCount = count
+
+    this._file = file
+    this._part = part
+    this._writing = new Set()
+    this._pending = new Set() // puts still running, which must land before anything closes
+    this._closed = false
+
+    this._bitfield = bao.createBitfield(count)
+    this._nodes = new NodeStore(count)
+    this._have = 0
+
+    for (const { start, end, hash, held } of nodes) {
+      if (start >= count || end > count || end <= start) continue
+
+      this._nodes.set(start, end, hash)
+
+      // Only a group we received is on disk. Knowing a hash from somebody else's proof says
+      // nothing about whether we have the bytes.
+      if (held && !bao.getBit(this._bitfield, start)) {
+        bao.setBit(this._bitfield, start)
+        this._have++
+      }
+    }
+  }
+
+  /** Groups held so far. */
+  get have () {
+    return this._have
+  }
+
+  get complete () {
+    return this._have === this.groupCount
+  }
+
+  has (index) {
+    return bao.getBit(this._bitfield, index)
+  }
+
+  /** @returns {number[]} indices still wanted */
+  missing () {
+    const out = []
+    for (let i = 0; i < this.groupCount; i++) {
+      if (!this.has(i)) out.push(i)
+    }
+    return out
+  }
+
+  /** The bitfield as advertised to peers. */
+  bitfield () {
+    return this._bitfield
+  }
+
+  /**
+   * Offer a group. Returns false if it does not verify — in which case nothing is written and
+   * nothing is recorded, so a lying peer makes no progress by trying again.
+   *
+   * @param {number} index
+   * @param {Uint8Array} data
+   * @param {Uint8Array[]} proof
+   * @returns {Promise<boolean>}
+   */
+  async put (index, data, proof) {
+    if (this._closed) throw new Error('destination is closed')
+
+    const op = this._put(index, data, proof)
+    this._pending.add(op)
+
+    try {
+      return await op
+    } finally {
+      this._pending.delete(op)
+    }
+  }
+
+  async _put (index, data, proof) {
+    if (!Number.isInteger(index) || index < 0 || index >= this.groupCount) return false
+    if (this.has(index)) return true
+
+    const ok = verifyGroup({
+      root: this.root,
+      size: this.size,
+      index,
+      data,
+      proof,
+      groupLog: this.groupLog
+    })
+    if (!ok) return false
+
+    // Two peers can answer the same request at once. Both copies are valid — they verified —
+    // but the group must be counted once, or `have` overshoots and the download calls itself
+    // finished with a hole in it.
+    if (this._writing.has(index)) return true
+    this._writing.add(index)
+
+    try {
+      const { offset } = groupRange(index, this.size, this.groupLog)
+      await this._file.write(offset, data)
+    } finally {
+      this._writing.delete(index)
+    }
+
+    if (this.has(index)) return true
+
+    const hash = groupHash(data, index, {
+      groupLog: this.groupLog,
+      only: this.groupCount === 1
+    })
+
+    // Keep the proof as well as the group: those sibling hashes are exactly what we will need
+    // to prove this group to the next peer that asks for it.
+    const learned = this._nodes.learn(index, hash, proof)
+
+    bao.setBit(this._bitfield, index)
+    this._have++
+
+    // The group itself is marked as held; the proof hashes are knowledge, not data.
+    const records = learned.map((node) => ({ ...node, held: node.start === index && node.end === index + 1 }))
+    if (!records.some((node) => node.held)) {
+      records.push({ start: index, end: index + 1, hash, held: true })
+    }
+
+    await this._part.record(records)
+
+    return true
+  }
+
+  // -- serving what we already have ------------------------------------------
+
+  /** Wait for puts that started before the close. */
+  async _drain () {
+    while (this._pending.size > 0) {
+      await Promise.allSettled([...this._pending])
+    }
+  }
+
+  /** The finished tree. Only meaningful once every group is held. */
+  get tree () {
+    return new GroupTree(this._nodes.leaves(), { size: this.size, groupLog: this.groupLog })
+  }
+
+  /** Can this group be served with a proof right now? */
+  canProve (index) {
+    return this._nodes.canProve(index)
+  }
+
+  /** @returns {Uint8Array[]|null} null when some hash the proof needs was never learned */
+  proof (index) {
+    return this.has(index) ? this._nodes.proof(index) : null
+  }
+
+  /** Read a group back out of the destination file. */
+  async readGroup (index) {
+    if (!this.has(index)) return null
+
+    const { offset, length } = groupRange(index, this.size, this.groupLog)
+    return this._file.read(offset, length)
+  }
+
+  // -- lifecycle -------------------------------------------------------------
+
+  /** Complete the download: flush, drop the resume state, close. */
+  async finish () {
+    if (!this.complete) throw new Error('download is not complete')
+
+    // Closed before awaiting anything, so a group arriving now is refused cleanly rather
+    // than writing into a handle that is about to shut.
+    this._closed = true
+    await this._drain()
+
+    await this._file.sync()
+    await this._file.close()
+    await this._part.close()
+    await PartFile.remove(this.path)
+  }
+
+  /** Stop for now, keeping the resume state. */
+  async close () {
+    if (this._closed) return
+    this._closed = true
+
+    await this._drain()
+
+    try {
+      await this._part.close()
+    } finally {
+      await this._file.close() // never leak the handle, whatever else fails
+    }
+  }
+}

package/lib/fs.js

@@ -0,0 +1,176 @@
+/**
+ * The only module that touches a filesystem.
+ *
+ * Everything else goes through here so that Bare can supply `bare-fs` without the rest of the
+ * library knowing. Keep this thin: open, read at an offset, write at an offset, stat, unlink.
+ */
+
+import fs from 'fs'
+import b4a from 'b4a'
+
+/**
+ * @param {string} path
+ * @returns {Promise<{size: number, mtimeMs: number, directory: boolean}|null>} null if it
+ *   does not exist
+ */
+export async function stat (path) {
+  try {
+    const st = await fs.promises.stat(path)
+    return { size: st.size, mtimeMs: Math.floor(st.mtimeMs), directory: st.isDirectory() }
+  } catch (err) {
+    if (err.code === 'ENOENT') return null
+    throw err
+  }
+}
+
+/** Open a file for reading byte ranges out of it, in place. */
+export async function openRead (path) {
+  const handle = await fs.promises.open(path, 'r')
+
+  return {
+    async read (offset, length) {
+      const buffer = b4a.alloc(length)
+      if (length === 0) return buffer
+
+      const { bytesRead } = await handle.read(buffer, 0, length, offset)
+      return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+    },
+
+    close () {
+      return handle.close()
+    }
+  }
+}
+
+/**
+ * Open a file for writing byte ranges into it, creating it and giving it its final size up
+ * front so ranges can land in any order.
+ */
+export async function openWrite (path, { size }) {
+  const handle = await fs.promises.open(path, (await stat(path)) ? 'r+' : 'w+')
+  await handle.truncate(size)
+
+  return {
+    async write (offset, data) {
+      if (data.length === 0) return
+      await handle.write(data, 0, data.length, offset)
+    },
+
+    /** Reading back matters: a partial download serves the groups it already holds. */
+    async read (offset, length) {
+      const buffer = b4a.alloc(length)
+      if (length === 0) return buffer
+
+      const { bytesRead } = await handle.read(buffer, 0, length, offset)
+      return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+    },
+
+    sync () {
+      return handle.sync()
+    },
+
+    close () {
+      return handle.close()
+    }
+  }
+}
+
+/**
+ * Open a file for appending, readable too. Used for the append-only part file, where every
+ * write goes to the end and reads walk back over what was written.
+ */
+export async function openAppend (path) {
+  // Not 'a+': Windows refuses to truncate a handle opened in append mode, and we need to
+  // truncate when the file turns out to describe different content. Track the end ourselves.
+  const handle = await fs.promises.open(path, (await stat(path)) ? 'r+' : 'w+')
+  const info = await handle.stat()
+
+  // Appends are serialised. Groups are verified concurrently, and two appends that each read
+  // the end of the file before either wrote would land on top of each other.
+  let tail = Promise.resolve()
+
+  return {
+    size: info.size,
+
+    async read (offset, length) {
+      const buffer = b4a.alloc(length)
+      if (length === 0) return buffer
+
+      const { bytesRead } = await handle.read(buffer, 0, length, offset)
+      return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+    },
+
+    append (data) {
+      const write = tail.then(async () => {
+        await handle.write(data, 0, data.length, this.size)
+        this.size += data.length
+      })
+
+      tail = write.catch(() => {}) // a failed append must not wedge the queue
+      return write
+    },
+
+    async truncate (size) {
+      await handle.truncate(size)
+      this.size = size
+    },
+
+    close () {
+      return handle.close()
+    }
+  }
+}
+
+/**
+ * Every file under a directory, as paths relative to it, in a stable order.
+ *
+ * Symbolic links are skipped rather than followed: a link pointing outside the folder would
+ * quietly put somebody else's file into a collection.
+ *
+ * @param {string} dir
+ * @returns {Promise<string[]>}
+ */
+export async function list (dir) {
+  const found = []
+
+  async function walk (current, prefix) {
+    const entries = await fs.promises.readdir(current, { withFileTypes: true })
+
+    for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+      const relative = prefix ? prefix + '/' + entry.name : entry.name
+
+      if (entry.isDirectory()) await walk(current + '/' + entry.name, relative)
+      else if (entry.isFile()) found.push(relative)
+    }
+  }
+
+  await walk(dir, '')
+  return found
+}
+
+/** Create a directory and any parents it needs. */
+export function mkdir (dir) {
+  return fs.promises.mkdir(dir, { recursive: true })
+}
+
+/** Small whole-file helpers, for sidecars only. */
+export async function readFile (path) {
+  try {
+    return await fs.promises.readFile(path)
+  } catch (err) {
+    if (err.code === 'ENOENT') return null
+    throw err
+  }
+}
+
+export function writeFile (path, data) {
+  return fs.promises.writeFile(path, data)
+}
+
+export async function unlink (path) {
+  try {
+    await fs.promises.unlink(path)
+  } catch (err) {
+    if (err.code !== 'ENOENT') throw err
+  }
+}

package/lib/nodes.js

@@ -0,0 +1,118 @@
+/**
+ * Every tree node a download has learned.
+ *
+ * Verifying a group teaches you more than the group: the proof that came with it names the
+ * hash of each sibling subtree along the way. Keeping those is what makes a half-finished
+ * download useful to anyone else — it can hand over a group it holds *with the proof it was
+ * given*, instead of being able to prove nothing until it is nearly complete.
+ *
+ * Nodes are keyed by the range of groups they cover, so a hash learned once is shared by every
+ * proof that needs it.
+ */
+
+import { bao } from './blake3.js'
+import b4a from 'b4a'
+
+import { siblingRanges, splitPoint } from './tree.js'
+
+export class NodeStore {
+  /** @param {number} count - total groups in the content */
+  constructor (count) {
+    this.count = count
+    this._nodes = new Map()
+  }
+
+  /**
+   * @param {number} start - first group covered
+   * @param {number} end - one past the last
+   * @param {Uint8Array} hash
+   * @returns {boolean} false when this node was already known
+   */
+  set (start, end, hash) {
+    const key = rangeKey(start, end)
+    if (this._nodes.has(key)) return false
+
+    this._nodes.set(key, b4a.from(hash))
+    return true
+  }
+
+  /** The hash of a subtree, directly known or derived from its halves. Null if unknown. */
+  get (start, end) {
+    const known = this._nodes.get(rangeKey(start, end))
+    if (known) return known
+    if (end - start === 1) return null
+
+    const mid = start + splitPoint(end - start)
+    const left = this.get(start, mid)
+    if (left === null) return null
+
+    const right = this.get(mid, end)
+    if (right === null) return null
+
+    return bao.parentCV(left, right, false)
+  }
+
+  has (index) {
+    return this.get(index, index + 1) !== null
+  }
+
+  /**
+   * A proof that group `index` belongs to the root, built from what is known.
+   *
+   * @param {number} index
+   * @returns {Uint8Array[]|null} null when some sibling has never been learned
+   */
+  proof (index) {
+    const proof = []
+
+    for (const { start, end } of siblingRanges(this.count, index)) {
+      const hash = this.get(start, end)
+      if (hash === null) return null
+
+      proof.push(hash)
+    }
+
+    return proof
+  }
+
+  /** Can this group be handed to another peer with a proof? */
+  canProve (index) {
+    return this.has(index) && this.proof(index) !== null
+  }
+
+  /**
+   * Record everything a verified group taught us: its own hash, and the sibling hashes from
+   * the proof that came with it.
+   *
+   * @param {number} index
+   * @param {Uint8Array} hash - the group's own hash
+   * @param {Uint8Array[]} proof
+   * @returns {{start: number, end: number, hash: Uint8Array}[]} the nodes that were new
+   */
+  learn (index, hash, proof) {
+    const learned = []
+
+    if (this.set(index, index + 1, hash)) {
+      learned.push({ start: index, end: index + 1, hash })
+    }
+
+    const ranges = siblingRanges(this.count, index)
+    for (let i = 0; i < ranges.length && i < proof.length; i++) {
+      const { start, end } = ranges[i]
+      if (this.set(start, end, proof[i])) learned.push({ start, end, hash: proof[i] })
+    }
+
+    return learned
+  }
+
+  /** Leaf hashes, with holes where groups are unknown. For building a finished tree. */
+  leaves () {
+    const out = new Array(this.count).fill(null)
+    for (let i = 0; i < this.count; i++) out[i] = this.get(i, i + 1)
+    return out
+  }
+}
+
+function rangeKey (start, end) {
+  return start * 0x100000000 + end
+}