swarmwire-lib 0.0.0-stage → 0.2.0

package/lib/listing.js

@@ -0,0 +1,158 @@
+/**
+ * Reading a folder's file list: fetch the manifest into memory, validate it, and leave nothing
+ * behind. Nothing here touches disk, and it only ever looks for peers — a node that is merely
+ * reading a list is not a provider of the folder.
+ */
+
+import b4a from 'b4a'
+import { bao } from './blake3.js'
+
+import { decodeManifest, totalSize, MAX_MANIFEST_BYTES } from './collection.js'
+import { discoveryTopic } from './address.js'
+import { GroupTree, groupHash, verifyGroup, groupCount } from './tree.js'
+import { NodeStore } from './nodes.js'
+
+/**
+ * The same surface a `Download` uses on a file destination, kept in memory: verify a group
+ * before storing it, remember the proofs, hand groups back out. A manifest is small and capped,
+ * so there is no resume state to keep.
+ */
+export class MemoryDestination {
+  constructor ({ root, size, groupLog }) {
+    if (size > MAX_MANIFEST_BYTES) throw new Error('manifest is implausibly large')
+
+    this.root = root
+    this.size = size
+    this.groupLog = groupLog
+    this.groupCount = groupCount(size, groupLog)
+
+    this._groups = new Map() // index → bytes
+    this._bitfield = bao.createBitfield(this.groupCount)
+    this._nodes = new NodeStore(this.groupCount)
+  }
+
+  get have () { return this._groups.size }
+  get complete () { return this._groups.size === this.groupCount }
+  has (index) { return bao.getBit(this._bitfield, index) }
+  bitfield () { return this._bitfield }
+
+  missing () {
+    const out = []
+    for (let i = 0; i < this.groupCount; i++) if (!this.has(i)) out.push(i)
+    return out
+  }
+
+  async put (index, data, proof) {
+    if (!Number.isInteger(index) || index < 0 || index >= this.groupCount) return false
+    if (this.has(index)) return true
+
+    const { root, size, groupLog } = this
+    if (!verifyGroup({ root, size, index, data, proof, groupLog })) return false
+
+    const hash = groupHash(data, index, { groupLog, only: this.groupCount === 1 })
+    this._nodes.learn(index, hash, proof)
+    this._groups.set(index, b4a.from(data))
+    bao.setBit(this._bitfield, index)
+
+    return true
+  }
+
+  get tree () {
+    return new GroupTree(this._nodes.leaves(), { size: this.size, groupLog: this.groupLog })
+  }
+
+  proof (index) {
+    return this.has(index) ? this._nodes.proof(index) : null
+  }
+
+  async readGroup (index) {
+    return this._groups.get(index) ?? null
+  }
+
+  /** The whole content, once every group is held. */
+  bytes () {
+    const parts = []
+    for (let i = 0; i < this.groupCount; i++) parts.push(this._groups.get(i))
+    return b4a.concat(parts)
+  }
+
+  async finish () {}
+  async close () {}
+}
+
+/**
+ * A folder's file list being read. Thenable, like a `Download`: `await listing` resolves to
+ * `{ files: [{ path, root, size }], size }`, and `stop()` gives up waiting, which resolves it
+ * to `undefined`.
+ */
+export class Listing {
+  /** A read that failed before it began, such as a link that is not a folder. */
+  static failed (err) {
+    const listing = Object.create(Listing.prototype)
+    listing._promise = Promise.reject(err)
+    listing._promise.catch(() => {}) // the caller sees it; this only keeps it from being unhandled
+    listing.stop = async () => {}
+    return listing
+  }
+
+  /** A read that needs no network: the list is already known. */
+  static known (files) {
+    const listing = Object.create(Listing.prototype)
+    listing._promise = Promise.resolve({ files, size: totalSize(files) })
+    listing.stop = async () => {}
+    return listing
+  }
+
+  constructor (node, parsed) {
+    this._download = null
+    this._session = null
+    this._stopped = false
+
+    this._promise = this._run(node, parsed)
+    this._promise.catch(() => {})
+  }
+
+  async stop () {
+    this._stopped = true
+    await this._download?.stop()
+  }
+
+  then (onfulfilled, onrejected) {
+    return this._promise.then(onfulfilled, onrejected)
+  }
+
+  catch (onrejected) {
+    return this._promise.catch(onrejected)
+  }
+
+  finally (onfinally) {
+    return this._promise.finally(onfinally)
+  }
+
+  async _run (node, parsed) {
+    if (node.downloads.has(parsed.root)) throw new Error('that link is already being downloaded as a file')
+
+    let memory = null
+
+    // Only look: with no server session on the topic this node is never announced for it.
+    this._session = node.swarm.join(discoveryTopic(parsed.root, parsed.secret), { server: false, client: true })
+
+    try {
+      this._download = node.download({ ...parsed, name: null }, null, {
+        announce: false,
+        destination: (options) => (memory = new MemoryDestination(options))
+      })
+      if (this._stopped) await this._download.stop()
+
+      await this._download
+      if (this._stopped) return undefined
+
+      const files = decodeManifest(memory.bytes())
+      node._learnList(parsed.root, files)
+
+      return { files, size: totalSize(files) }
+    } finally {
+      await this._session?.destroy?.()
+    }
+  }
+}

package/lib/nodes.js

@@ -0,0 +1,118 @@
+/**
+ * Every tree node a download has learned.
+ *
+ * Verifying a group teaches you more than the group: the proof that came with it names the
+ * hash of each sibling subtree along the way. Keeping those is what makes a half-finished
+ * download useful to anyone else — it can hand over a group it holds *with the proof it was
+ * given*, instead of being able to prove nothing until it is nearly complete.
+ *
+ * Nodes are keyed by the range of groups they cover, so a hash learned once is shared by every
+ * proof that needs it.
+ */
+
+import { bao } from './blake3.js'
+import b4a from 'b4a'
+
+import { siblingRanges, splitPoint } from './tree.js'
+
+export class NodeStore {
+  /** @param {number} count - total groups in the content */
+  constructor (count) {
+    this.count = count
+    this._nodes = new Map()
+  }
+
+  /**
+   * @param {number} start - first group covered
+   * @param {number} end - one past the last
+   * @param {Uint8Array} hash
+   * @returns {boolean} false when this node was already known
+   */
+  set (start, end, hash) {
+    const key = rangeKey(start, end)
+    if (this._nodes.has(key)) return false
+
+    this._nodes.set(key, b4a.from(hash))
+    return true
+  }
+
+  /** The hash of a subtree, directly known or derived from its halves. Null if unknown. */
+  get (start, end) {
+    const known = this._nodes.get(rangeKey(start, end))
+    if (known) return known
+    if (end - start === 1) return null
+
+    const mid = start + splitPoint(end - start)
+    const left = this.get(start, mid)
+    if (left === null) return null
+
+    const right = this.get(mid, end)
+    if (right === null) return null
+
+    return bao.parentCV(left, right, false)
+  }
+
+  has (index) {
+    return this.get(index, index + 1) !== null
+  }
+
+  /**
+   * A proof that group `index` belongs to the root, built from what is known.
+   *
+   * @param {number} index
+   * @returns {Uint8Array[]|null} null when some sibling has never been learned
+   */
+  proof (index) {
+    const proof = []
+
+    for (const { start, end } of siblingRanges(this.count, index)) {
+      const hash = this.get(start, end)
+      if (hash === null) return null
+
+      proof.push(hash)
+    }
+
+    return proof
+  }
+
+  /** Can this group be handed to another peer with a proof? */
+  canProve (index) {
+    return this.has(index) && this.proof(index) !== null
+  }
+
+  /**
+   * Record everything a verified group taught us: its own hash, and the sibling hashes from
+   * the proof that came with it.
+   *
+   * @param {number} index
+   * @param {Uint8Array} hash - the group's own hash
+   * @param {Uint8Array[]} proof
+   * @returns {{start: number, end: number, hash: Uint8Array}[]} the nodes that were new
+   */
+  learn (index, hash, proof) {
+    const learned = []
+
+    if (this.set(index, index + 1, hash)) {
+      learned.push({ start: index, end: index + 1, hash })
+    }
+
+    const ranges = siblingRanges(this.count, index)
+    for (let i = 0; i < ranges.length && i < proof.length; i++) {
+      const { start, end } = ranges[i]
+      if (this.set(start, end, proof[i])) learned.push({ start, end, hash: proof[i] })
+    }
+
+    return learned
+  }
+
+  /** Leaf hashes, with holes where groups are unknown. For building a finished tree. */
+  leaves () {
+    const out = new Array(this.count).fill(null)
+    for (let i = 0; i < this.count; i++) out[i] = this.get(i, i + 1)
+    return out
+  }
+}
+
+function rangeKey (start, end) {
+  return start * 0x100000000 + end
+}

package/lib/partfile.js

@@ -0,0 +1,169 @@
+/**
+ * The `.swarmwire-part` file: what a half-finished download knows.
+ *
+ * It holds a fixed header describing the content, then one 40-byte record per tree node
+ * learned. Records are appended as groups arrive, so progress survives a crash to the last
+ * group rather than to the last checkpoint, and an append costs 40 bytes rather than
+ * rewriting a bitfield.
+ *
+ * Nodes, not just groups: a record covers a range of groups, so a leaf is the range [i, i+1)
+ * and a proof's sibling hashes are the wider ranges. Keeping those is what lets a resumed,
+ * half-finished download still prove what it holds to other peers rather than only taking.
+ *
+ *   header   "SWPT" | version u8 | groupLog u8 | size u64 | root 32B
+ *   record   start u32 | end u32 | held u8 | hash 32B
+ *
+ * `held` separates the two things a record can mean. A hash learned from somebody's proof
+ * tells you what a group *would* hash to; only a group you actually received is on disk.
+ * Confusing the two would have a resumed download claim data it never had.
+ *
+ * A trailing partial record is ignored: a crash mid-append costs that one node.
+ */
+
+import b4a from 'b4a'
+
+import * as fsx from './fs.js'
+
+export const SUFFIX = '.swarmwire-part'
+
+const MAGIC = 'SWPT'
+const VERSION = 2
+const HEADER_SIZE = 4 + 1 + 1 + 8 + 32
+const RECORD_SIZE = 4 + 4 + 1 + 32
+
+export class PartFile {
+  /**
+   * Open the part file for this download, reusing it only if it describes the same content.
+   *
+   * @param {string} path - the destination path (the suffix is added here)
+   * @param {{root: string, size: number, groupLog: number}} content
+   * @returns {Promise<{part: PartFile, nodes: {start: number, end: number, hash: Uint8Array}[]}>}
+   */
+  static async open (path, { root, size, groupLog }) {
+    const file = await fsx.openAppend(path + SUFFIX)
+    const part = new PartFile(file)
+
+    try {
+      const header = file.size >= HEADER_SIZE ? decodeHeader(await file.read(0, HEADER_SIZE)) : null
+      const matches = header &&
+        header.root === root &&
+        header.size === size &&
+        header.groupLog === groupLog
+
+      if (!matches) {
+        // Someone else's progress, or ours from before the content changed. Never merge it.
+        await file.truncate(0)
+        await file.append(encodeHeader({ root, size, groupLog }))
+        return { part, nodes: [] }
+      }
+
+      return { part, nodes: await readRecords(file) }
+    } catch (err) {
+      await file.close()
+      throw err
+    }
+  }
+
+  constructor (file) {
+    this._file = file
+  }
+
+  /**
+   * Append the nodes learned from one verified group.
+   *
+   * @param {{start: number, end: number, hash: Uint8Array, held?: boolean}[]} nodes
+   */
+  async record (nodes) {
+    if (nodes.length === 0) return
+
+    const batch = b4a.alloc(RECORD_SIZE * nodes.length)
+
+    nodes.forEach(({ start, end, hash, held = false }, i) => {
+      const at = i * RECORD_SIZE
+      writeUint32(batch, at, start)
+      writeUint32(batch, at + 4, end)
+      batch[at + 8] = held ? 1 : 0
+      b4a.copy(b4a.from(hash), batch, at + 9)
+    })
+
+    await this._file.append(batch)
+  }
+
+  close () {
+    return this._file.close()
+  }
+
+  /** Delete the part file for a destination, if there is one. */
+  static remove (path) {
+    return fsx.unlink(path + SUFFIX)
+  }
+}
+
+async function readRecords (file) {
+  const nodes = []
+  const count = Math.floor((file.size - HEADER_SIZE) / RECORD_SIZE)
+  if (count <= 0) return nodes
+
+  // One read: a few megabytes at most, and only once per resume.
+  const body = await file.read(HEADER_SIZE, count * RECORD_SIZE)
+
+  for (let i = 0; i < count; i++) {
+    const at = i * RECORD_SIZE
+    nodes.push({
+      start: readUint32(body, at),
+      end: readUint32(body, at + 4),
+      held: body[at + 8] === 1,
+      hash: b4a.from(body.subarray(at + 9, at + RECORD_SIZE))
+    })
+  }
+
+  return nodes
+}
+
+function encodeHeader ({ root, size, groupLog }) {
+  const header = b4a.alloc(HEADER_SIZE)
+
+  b4a.write(header, MAGIC, 0, 'ascii')
+  header[4] = VERSION
+  header[5] = groupLog
+  writeUint64(header, 6, size)
+  b4a.copy(b4a.from(root, 'hex'), header, 14)
+
+  return header
+}
+
+function decodeHeader (bytes) {
+  if (bytes.length < HEADER_SIZE) return null
+  if (b4a.toString(bytes.subarray(0, 4), 'ascii') !== MAGIC) return null
+  if (bytes[4] !== VERSION) return null
+
+  return {
+    groupLog: bytes[5],
+    size: readUint64(bytes, 6),
+    root: b4a.toString(bytes.subarray(14, 46), 'hex')
+  }
+}
+
+function writeUint32 (buffer, offset, value) {
+  buffer[offset] = value & 0xff
+  buffer[offset + 1] = (value >>> 8) & 0xff
+  buffer[offset + 2] = (value >>> 16) & 0xff
+  buffer[offset + 3] = (value >>> 24) & 0xff
+}
+
+function readUint32 (buffer, offset) {
+  return buffer[offset] |
+    (buffer[offset + 1] << 8) |
+    (buffer[offset + 2] << 16) |
+    (buffer[offset + 3] << 24)
+}
+
+/** Sizes stay well inside 2^53, so splitting into two 32-bit halves is enough. */
+function writeUint64 (buffer, offset, value) {
+  writeUint32(buffer, offset, value >>> 0)
+  writeUint32(buffer, offset + 4, Math.floor(value / 0x100000000))
+}
+
+function readUint64 (buffer, offset) {
+  return (readUint32(buffer, offset) >>> 0) + (readUint32(buffer, offset + 4) >>> 0) * 0x100000000
+}

package/lib/select.js

@@ -0,0 +1,33 @@
+/**
+ * Which group to ask for next.
+ *
+ * Rarest first, as BitTorrent does it: fetch what few peers have before what everyone has.
+ * The reason is not speed on this download but survival of the swarm — a group held by one
+ * peer disappears when that peer leaves, and a swarm where every copy of some group lives on
+ * one machine is one disconnection away from nobody being able to finish.
+ */
+
+/** How many missing groups to consider at once. */
+export const SCAN_LIMIT = 256
+
+/**
+ * Order candidate groups by how few peers hold them, keeping file order as the tie-break so
+ * a transfer still roughly streams.
+ *
+ * Only the first `limit` missing groups are considered: on a large file, sorting every
+ * outstanding group on every request would cost more than it saves, and rarity among the next
+ * few hundred is what actually decides the next request.
+ *
+ * @param {number[]} missing - group indices still wanted, in ascending order
+ * @param {(index: number) => number} holders - how many peers hold this group
+ * @param {{limit?: number}} [opts]
+ * @returns {number[]}
+ */
+export function rarestFirst (missing, holders, { limit = SCAN_LIMIT } = {}) {
+  const candidates = missing.length > limit ? missing.slice(0, limit) : missing.slice()
+
+  return candidates
+    .map((index) => ({ index, holders: holders(index) }))
+    .sort((a, b) => a.holders - b.holders || a.index - b.index)
+    .map((candidate) => candidate.index)
+}

package/lib/source.js

@@ -0,0 +1,171 @@
+/**
+ * Serving content that already exists somewhere on disk.
+ *
+ * There is no import step and no store: a file is addressed where it lies, read where it lies,
+ * and nothing is written next to it. The only state kept is its group hashes, which are derived
+ * on first use and remembered for as long as the file keeps its size and mtime.
+ */
+
+import { GroupTree, GroupTreeBuilder, groupRange, groupLength, DEFAULT_GROUP_LOG } from './tree.js'
+import * as fsx from './fs.js'
+
+/** path+size+mtime → group hashes. Process-local; a persistent cache can come later. */
+const hashes = new Map()
+
+export function clearHashCache () {
+  hashes.clear()
+}
+
+export class ContentSource {
+  /**
+   * @param {string} path
+   * @param {{groupLog?: number}} [opts]
+   * @returns {Promise<ContentSource>}
+   */
+  static async open (path, { groupLog = DEFAULT_GROUP_LOG } = {}) {
+    const info = await fsx.stat(path)
+    if (info === null) throw new Error(`no such file: ${path}`)
+
+    const file = await fsx.openRead(path)
+
+    try {
+      const key = `${path}\u0000${info.size}\u0000${info.mtimeMs}\u0000${groupLog}`
+      const remembered = hashes.get(key)
+
+      const tree = remembered
+        ? GroupTree.fromGroupHashes(remembered, { size: info.size, groupLog })
+        : await hashFile(file, info.size, groupLog)
+
+      if (!remembered) hashes.set(key, tree.groupHashes)
+
+      return new ContentSource(path, file, tree, { cached: !!remembered })
+    } catch (err) {
+      await file.close()
+      throw err
+    }
+  }
+
+  /**
+   * Take over a file whose tree is already known — a download that just finished. Re-hashing
+   * it would be pointless work: every group was hashed on the way in.
+   *
+   * @param {string} path
+   * @param {import('./tree.js').GroupTree} tree
+   * @returns {Promise<ContentSource>}
+   */
+  static async adopt (path, tree) {
+    return new ContentSource(path, await fsx.openRead(path), tree, { cached: true })
+  }
+
+  constructor (path, file, tree, { cached }) {
+    this.path = path
+    this.tree = tree
+    this.cached = cached
+
+    this._file = file
+  }
+
+  /** The content address: the plain BLAKE3 hash, hex encoded. */
+  get root () {
+    return this.tree.root
+  }
+
+  get size () {
+    return this.tree.size
+  }
+
+  get groupLog () {
+    return this.tree.groupLog
+  }
+
+  get groupCount () {
+    return this.tree.groupCount
+  }
+
+  /**
+   * Read one group straight from the file.
+   *
+   * @param {number} index
+   * @returns {Promise<Uint8Array>}
+   */
+  async readGroup (index) {
+    const { offset, length } = groupRange(index, this.size, this.groupLog)
+    return this._file.read(offset, length)
+  }
+
+  /**
+   * The hashes proving a group belongs to this root.
+   *
+   * @param {number} index
+   * @returns {Uint8Array[]}
+   */
+  proof (index) {
+    return this.tree.proof(index)
+  }
+
+  close () {
+    return this._file.close()
+  }
+}
+
+/**
+ * Content held in memory rather than on disk, with the same shape as a ContentSource.
+ *
+ * A collection's manifest is content like any other and has to be served, but writing it
+ * beside the folder it describes would mean the act of sharing a folder changed it.
+ */
+export class MemorySource {
+  /**
+   * @param {Uint8Array} bytes
+   * @param {{groupLog?: number}} [opts]
+   */
+  static from (bytes, { groupLog = DEFAULT_GROUP_LOG } = {}) {
+    return new MemorySource(bytes, GroupTree.fromContent(bytes, { groupLog }))
+  }
+
+  constructor (bytes, tree) {
+    this.path = null
+    this.tree = tree
+    this.cached = true
+
+    this._bytes = bytes
+  }
+
+  get root () {
+    return this.tree.root
+  }
+
+  get size () {
+    return this.tree.size
+  }
+
+  get groupLog () {
+    return this.tree.groupLog
+  }
+
+  get groupCount () {
+    return this.tree.groupCount
+  }
+
+  async readGroup (index) {
+    const { offset, length } = groupRange(index, this.size, this.groupLog)
+    return this._bytes.subarray(offset, offset + length)
+  }
+
+  proof (index) {
+    return this.tree.proof(index)
+  }
+
+  async close () {}
+}
+
+async function hashFile (file, size, groupLog) {
+  const builder = new GroupTreeBuilder({ groupLog })
+  const len = groupLength(groupLog)
+
+  for (let offset = 0; offset < size; offset += len) {
+    builder.push(await file.read(offset, Math.min(len, size - offset)))
+  }
+
+  return builder.finalize()
+}