swarmwire-lib 0.0.0-stage → 0.1.0

package/lib/partfile.js

@@ -0,0 +1,169 @@
+/**
+ * The `.swarmwire-part` file: what a half-finished download knows.
+ *
+ * It holds a fixed header describing the content, then one 40-byte record per tree node
+ * learned. Records are appended as groups arrive, so progress survives a crash to the last
+ * group rather than to the last checkpoint, and an append costs 40 bytes rather than
+ * rewriting a bitfield.
+ *
+ * Nodes, not just groups: a record covers a range of groups, so a leaf is the range [i, i+1)
+ * and a proof's sibling hashes are the wider ranges. Keeping those is what lets a resumed,
+ * half-finished download still prove what it holds to other peers rather than only taking.
+ *
+ *   header   "SWPT" | version u8 | groupLog u8 | size u64 | root 32B
+ *   record   start u32 | end u32 | held u8 | hash 32B
+ *
+ * `held` separates the two things a record can mean. A hash learned from somebody's proof
+ * tells you what a group *would* hash to; only a group you actually received is on disk.
+ * Confusing the two would have a resumed download claim data it never had.
+ *
+ * A trailing partial record is ignored: a crash mid-append costs that one node.
+ */
+
+import b4a from 'b4a'
+
+import * as fsx from './fs.js'
+
+export const SUFFIX = '.swarmwire-part'
+
+const MAGIC = 'SWPT'
+const VERSION = 2
+const HEADER_SIZE = 4 + 1 + 1 + 8 + 32
+const RECORD_SIZE = 4 + 4 + 1 + 32
+
+export class PartFile {
+  /**
+   * Open the part file for this download, reusing it only if it describes the same content.
+   *
+   * @param {string} path - the destination path (the suffix is added here)
+   * @param {{root: string, size: number, groupLog: number}} content
+   * @returns {Promise<{part: PartFile, nodes: {start: number, end: number, hash: Uint8Array}[]}>}
+   */
+  static async open (path, { root, size, groupLog }) {
+    const file = await fsx.openAppend(path + SUFFIX)
+    const part = new PartFile(file)
+
+    try {
+      const header = file.size >= HEADER_SIZE ? decodeHeader(await file.read(0, HEADER_SIZE)) : null
+      const matches = header &&
+        header.root === root &&
+        header.size === size &&
+        header.groupLog === groupLog
+
+      if (!matches) {
+        // Someone else's progress, or ours from before the content changed. Never merge it.
+        await file.truncate(0)
+        await file.append(encodeHeader({ root, size, groupLog }))
+        return { part, nodes: [] }
+      }
+
+      return { part, nodes: await readRecords(file) }
+    } catch (err) {
+      await file.close()
+      throw err
+    }
+  }
+
+  constructor (file) {
+    this._file = file
+  }
+
+  /**
+   * Append the nodes learned from one verified group.
+   *
+   * @param {{start: number, end: number, hash: Uint8Array, held?: boolean}[]} nodes
+   */
+  async record (nodes) {
+    if (nodes.length === 0) return
+
+    const batch = b4a.alloc(RECORD_SIZE * nodes.length)
+
+    nodes.forEach(({ start, end, hash, held = false }, i) => {
+      const at = i * RECORD_SIZE
+      writeUint32(batch, at, start)
+      writeUint32(batch, at + 4, end)
+      batch[at + 8] = held ? 1 : 0
+      b4a.copy(b4a.from(hash), batch, at + 9)
+    })
+
+    await this._file.append(batch)
+  }
+
+  close () {
+    return this._file.close()
+  }
+
+  /** Delete the part file for a destination, if there is one. */
+  static remove (path) {
+    return fsx.unlink(path + SUFFIX)
+  }
+}
+
+async function readRecords (file) {
+  const nodes = []
+  const count = Math.floor((file.size - HEADER_SIZE) / RECORD_SIZE)
+  if (count <= 0) return nodes
+
+  // One read: a few megabytes at most, and only once per resume.
+  const body = await file.read(HEADER_SIZE, count * RECORD_SIZE)
+
+  for (let i = 0; i < count; i++) {
+    const at = i * RECORD_SIZE
+    nodes.push({
+      start: readUint32(body, at),
+      end: readUint32(body, at + 4),
+      held: body[at + 8] === 1,
+      hash: b4a.from(body.subarray(at + 9, at + RECORD_SIZE))
+    })
+  }
+
+  return nodes
+}
+
+function encodeHeader ({ root, size, groupLog }) {
+  const header = b4a.alloc(HEADER_SIZE)
+
+  b4a.write(header, MAGIC, 0, 'ascii')
+  header[4] = VERSION
+  header[5] = groupLog
+  writeUint64(header, 6, size)
+  b4a.copy(b4a.from(root, 'hex'), header, 14)
+
+  return header
+}
+
+function decodeHeader (bytes) {
+  if (bytes.length < HEADER_SIZE) return null
+  if (b4a.toString(bytes.subarray(0, 4), 'ascii') !== MAGIC) return null
+  if (bytes[4] !== VERSION) return null
+
+  return {
+    groupLog: bytes[5],
+    size: readUint64(bytes, 6),
+    root: b4a.toString(bytes.subarray(14, 46), 'hex')
+  }
+}
+
+function writeUint32 (buffer, offset, value) {
+  buffer[offset] = value & 0xff
+  buffer[offset + 1] = (value >>> 8) & 0xff
+  buffer[offset + 2] = (value >>> 16) & 0xff
+  buffer[offset + 3] = (value >>> 24) & 0xff
+}
+
+function readUint32 (buffer, offset) {
+  return buffer[offset] |
+    (buffer[offset + 1] << 8) |
+    (buffer[offset + 2] << 16) |
+    (buffer[offset + 3] << 24)
+}
+
+/** Sizes stay well inside 2^53, so splitting into two 32-bit halves is enough. */
+function writeUint64 (buffer, offset, value) {
+  writeUint32(buffer, offset, value >>> 0)
+  writeUint32(buffer, offset + 4, Math.floor(value / 0x100000000))
+}
+
+function readUint64 (buffer, offset) {
+  return (readUint32(buffer, offset) >>> 0) + (readUint32(buffer, offset + 4) >>> 0) * 0x100000000
+}

package/lib/select.js

@@ -0,0 +1,33 @@
+/**
+ * Which group to ask for next.
+ *
+ * Rarest first, as BitTorrent does it: fetch what few peers have before what everyone has.
+ * The reason is not speed on this download but survival of the swarm — a group held by one
+ * peer disappears when that peer leaves, and a swarm where every copy of some group lives on
+ * one machine is one disconnection away from nobody being able to finish.
+ */
+
+/** How many missing groups to consider at once. */
+export const SCAN_LIMIT = 256
+
+/**
+ * Order candidate groups by how few peers hold them, keeping file order as the tie-break so
+ * a transfer still roughly streams.
+ *
+ * Only the first `limit` missing groups are considered: on a large file, sorting every
+ * outstanding group on every request would cost more than it saves, and rarity among the next
+ * few hundred is what actually decides the next request.
+ *
+ * @param {number[]} missing - group indices still wanted, in ascending order
+ * @param {(index: number) => number} holders - how many peers hold this group
+ * @param {{limit?: number}} [opts]
+ * @returns {number[]}
+ */
+export function rarestFirst (missing, holders, { limit = SCAN_LIMIT } = {}) {
+  const candidates = missing.length > limit ? missing.slice(0, limit) : missing.slice()
+
+  return candidates
+    .map((index) => ({ index, holders: holders(index) }))
+    .sort((a, b) => a.holders - b.holders || a.index - b.index)
+    .map((candidate) => candidate.index)
+}

package/lib/source.js

@@ -0,0 +1,171 @@
+/**
+ * Serving content that already exists somewhere on disk.
+ *
+ * There is no import step and no store: a file is addressed where it lies, read where it lies,
+ * and nothing is written next to it. The only state kept is its group hashes, which are derived
+ * on first use and remembered for as long as the file keeps its size and mtime.
+ */
+
+import { GroupTree, GroupTreeBuilder, groupRange, groupLength, DEFAULT_GROUP_LOG } from './tree.js'
+import * as fsx from './fs.js'
+
+/** path+size+mtime → group hashes. Process-local; a persistent cache can come later. */
+const hashes = new Map()
+
+export function clearHashCache () {
+  hashes.clear()
+}
+
+export class ContentSource {
+  /**
+   * @param {string} path
+   * @param {{groupLog?: number}} [opts]
+   * @returns {Promise<ContentSource>}
+   */
+  static async open (path, { groupLog = DEFAULT_GROUP_LOG } = {}) {
+    const info = await fsx.stat(path)
+    if (info === null) throw new Error(`no such file: ${path}`)
+
+    const file = await fsx.openRead(path)
+
+    try {
+      const key = `${path}\u0000${info.size}\u0000${info.mtimeMs}\u0000${groupLog}`
+      const remembered = hashes.get(key)
+
+      const tree = remembered
+        ? GroupTree.fromGroupHashes(remembered, { size: info.size, groupLog })
+        : await hashFile(file, info.size, groupLog)
+
+      if (!remembered) hashes.set(key, tree.groupHashes)
+
+      return new ContentSource(path, file, tree, { cached: !!remembered })
+    } catch (err) {
+      await file.close()
+      throw err
+    }
+  }
+
+  /**
+   * Take over a file whose tree is already known — a download that just finished. Re-hashing
+   * it would be pointless work: every group was hashed on the way in.
+   *
+   * @param {string} path
+   * @param {import('./tree.js').GroupTree} tree
+   * @returns {Promise<ContentSource>}
+   */
+  static async adopt (path, tree) {
+    return new ContentSource(path, await fsx.openRead(path), tree, { cached: true })
+  }
+
+  constructor (path, file, tree, { cached }) {
+    this.path = path
+    this.tree = tree
+    this.cached = cached
+
+    this._file = file
+  }
+
+  /** The content address: the plain BLAKE3 hash, hex encoded. */
+  get root () {
+    return this.tree.root
+  }
+
+  get size () {
+    return this.tree.size
+  }
+
+  get groupLog () {
+    return this.tree.groupLog
+  }
+
+  get groupCount () {
+    return this.tree.groupCount
+  }
+
+  /**
+   * Read one group straight from the file.
+   *
+   * @param {number} index
+   * @returns {Promise<Uint8Array>}
+   */
+  async readGroup (index) {
+    const { offset, length } = groupRange(index, this.size, this.groupLog)
+    return this._file.read(offset, length)
+  }
+
+  /**
+   * The hashes proving a group belongs to this root.
+   *
+   * @param {number} index
+   * @returns {Uint8Array[]}
+   */
+  proof (index) {
+    return this.tree.proof(index)
+  }
+
+  close () {
+    return this._file.close()
+  }
+}
+
+/**
+ * Content held in memory rather than on disk, with the same shape as a ContentSource.
+ *
+ * A collection's manifest is content like any other and has to be served, but writing it
+ * beside the folder it describes would mean the act of sharing a folder changed it.
+ */
+export class MemorySource {
+  /**
+   * @param {Uint8Array} bytes
+   * @param {{groupLog?: number}} [opts]
+   */
+  static from (bytes, { groupLog = DEFAULT_GROUP_LOG } = {}) {
+    return new MemorySource(bytes, GroupTree.fromContent(bytes, { groupLog }))
+  }
+
+  constructor (bytes, tree) {
+    this.path = null
+    this.tree = tree
+    this.cached = true
+
+    this._bytes = bytes
+  }
+
+  get root () {
+    return this.tree.root
+  }
+
+  get size () {
+    return this.tree.size
+  }
+
+  get groupLog () {
+    return this.tree.groupLog
+  }
+
+  get groupCount () {
+    return this.tree.groupCount
+  }
+
+  async readGroup (index) {
+    const { offset, length } = groupRange(index, this.size, this.groupLog)
+    return this._bytes.subarray(offset, offset + length)
+  }
+
+  proof (index) {
+    return this.tree.proof(index)
+  }
+
+  async close () {}
+}
+
+async function hashFile (file, size, groupLog) {
+  const builder = new GroupTreeBuilder({ groupLog })
+  const len = groupLength(groupLog)
+
+  for (let offset = 0; offset < size; offset += len) {
+    builder.push(await file.read(offset, Math.min(len, size - offset)))
+  }
+
+  return builder.finalize()
+}