swarmwire-lib 0.2.0 → 0.4.0

package/README.md

@@ -55,8 +55,9 @@ The command-line tool built on this library is moving to its own package,
 
 > Status: **pre-alpha.** Transfers work end to end, over a real swarm, with resume,
 > verification, request timeouts, partial seeding, rarest-first selection and an adaptive
-> request window. Behaviour with very many peers is not measured yet. See
-> [docs/roadmap.md](docs/roadmap.md) for what is built and what is next.
+> request window. Behaviour with very many peers is only modelled, not measured. It is enough
+> for a command-line tool; it is not yet the content-addressing layer SwarmFS needs (admission,
+> peer hints, no public discovery). [docs/roadmap.md](docs/roadmap.md) says where it stands.
 
 ## What it is
 
@@ -110,7 +111,9 @@ Anything that needs a name, a directory or a user belongs upstairs. See
 |------------------------------------|-----------------------------------------------------------------------------------------------|
 | [docs/format.md](docs/format.md)   | Addressing, verification, links, topic derivation — the parts that must never change casually |
 | [docs/contract.md](docs/contract.md) | What applications can rely on, supply and must not expect                                  |
-| [docs/roadmap.md](docs/roadmap.md) | Vision and milestones                                                                         |
+| [docs/roadmap.md](docs/roadmap.md) | Vision, milestones, and where things stand                                                    |
+| [docs/scaling.md](docs/scaling.md) | Piece selection in large swarms, simulated                                                    |
+| [docs/research/p2p-landscape.md](docs/research/p2p-landscape.md) | What other P2P systems do, and what is worth taking                  |
 | [AGENTS.md](AGENTS.md)             | How to work in this repo                                                                      |
 
 ## License

package/docs/contract.md

@@ -23,8 +23,9 @@ code (`serveCollection`, `downloadCollection`). See `AGENTS.md` for the invarian
 | Peer identity: the remote public key of the connection | exists | - | `test/swarm/transfer.test.js` |
 | Peer activity events, peer count, seeder and leecher counts | exists | - | `test/swarm/peers.test.js` |
 | Folder file-list reader (look-only, in memory), and downloading one listed file | exists | - | `test/swarm/list.test.js` |
-| Peer hints (preferred peers, priorities) | planned | 003 | - |
-| Admission decisions | planned | 003 | - |
+| Download order spread by random choice (no two newcomers ask for the same thing first) | exists | - | `test/swarm/spread.test.js` |
+| Peer hints (preferred peers, priorities) | planned | 004 | - |
+| Admission decisions | planned | 004 | - |
 
 ## 3. Rules the application can rely on
 
@@ -73,6 +74,13 @@ The application can decide, per peer and per content, whether content is served.
 **Direct connections only**: a snapshot that may lag, not a count of the swarm. Global counts and
 chunk rarity across the network are a tracker's job, not swarmwire's.
 
+### 3.3a Download order
+
+Groups are fetched rarest first, equal rarity broken at random, starting from a random place in
+the file. **A download does not proceed in file order**: a file is complete when every group is
+held, and nothing is promised about reading it while it downloads. How the choice was made, and
+what was measured, is in `docs/scaling.md` in the repository (not part of the package).
+
 ### 3.4 Identity
 
 swarmwire knows the remote public key of each connection and nothing else about a peer. Tying a

package/lib/select.js

@@ -5,29 +5,55 @@
  * The reason is not speed on this download but survival of the swarm — a group held by one
  * peer disappears when that peer leaves, and a swarm where every copy of some group lives on
  * one machine is one disconnection away from nobody being able to finish.
+ *
+ * Two things keep a crowd from asking for the same thing. Equally rare groups are ordered at
+ * random rather than by position, and the scan for candidates starts at a random group rather
+ * than the start of the file — otherwise a thousand fresh downloaders of a large file would
+ * all draw from its first few hundred groups.
  */
 
-/** How many missing groups to consider at once. */
+import { bao } from './blake3.js'
+
+/** How many candidate groups to weigh at once. */
 export const SCAN_LIMIT = 256
 
+/** How many source peers hold each group. Kept up to date by the caller, read in O(1). */
+export class Availability {
+  constructor (groupCount) {
+    this.counts = new Uint16Array(groupCount)
+  }
+
+  at (index) { return this.counts[index] }
+  set (index) { this.counts[index]++ }
+  unset (index) { if (this.counts[index] > 0) this.counts[index]-- }
+
+  add (bitfield) { this._each(bitfield, (i) => this.set(i)) }
+  remove (bitfield) { this._each(bitfield, (i) => this.unset(i)) }
+
+  _each (bitfield, fn) {
+    const n = Math.min(this.counts.length, bitfield.length * 8)
+    for (let i = 0; i < n; i++) if (bao.getBit(bitfield, i)) fn(i)
+  }
+}
+
 /**
- * Order candidate groups by how few peers hold them, keeping file order as the tie-break so
- * a transfer still roughly streams.
- *
- * Only the first `limit` missing groups are considered: on a large file, sorting every
- * outstanding group on every request would cost more than it saves, and rarity among the next
- * few hundred is what actually decides the next request.
- *
- * @param {number[]} missing - group indices still wanted, in ascending order
- * @param {(index: number) => number} holders - how many peers hold this group
- * @param {{limit?: number}} [opts]
- * @returns {number[]}
+ * The groups worth asking this peer for: wanted, offered, and not already being fetched.
+ * Scanning starts at `from` and wraps, so the window is a random stretch of the file.
  */
-export function rarestFirst (missing, holders, { limit = SCAN_LIMIT } = {}) {
-  const candidates = missing.length > limit ? missing.slice(0, limit) : missing.slice()
+export function candidates ({ count, has, peerHas, claimed, from = 0, limit = SCAN_LIMIT }) {
+  const out = []
+
+  for (let k = 0; k < count && out.length < limit; k++) {
+    const i = (from + k) % count
+    if (!has(i) && peerHas(i) && !claimed(i)) out.push(i)
+  }
+  return out
+}
 
-  return candidates
-    .map((index) => ({ index, holders: holders(index) }))
-    .sort((a, b) => a.holders - b.holders || a.index - b.index)
-    .map((candidate) => candidate.index)
+/** Fewest holders first; groups equally rare come in the order `random` gives them. */
+export function rank (pool, availability, random) {
+  return pool
+    .map((index) => ({ index, rarity: availability.at(index), tie: random() }))
+    .sort((a, b) => a.rarity - b.rarity || a.tie - b.tie)
+    .map((entry) => entry.index)
 }

package/lib/swarmwire.js

@@ -18,9 +18,9 @@ import { ContentSource, MemorySource } from './source.js'
 import { encodeManifest, decodeManifest, totalSize } from './collection.js'
 import { Destination } from './destination.js'
 import { discoveryTopic, formatLink, parseLink } from './address.js'
-import { DEFAULT_GROUP_LOG, groupCount } from './tree.js'
+import { DEFAULT_GROUP_LOG, groupCount, groupLength } from './tree.js'
 import { openChannel, VERSION } from './wire.js'
-import { rarestFirst } from './select.js'
+import { Availability, candidates, rank } from './select.js'
 import { Holdings } from './holdings.js'
 import { Listing } from './listing.js'
 import { systemTimers } from './timers.js'
@@ -79,6 +79,7 @@ export class Swarmwire extends EventEmitter {
     this.timers = timers
     this.requestTimeout = requestTimeout
     this.window = window
+    this.random = Math.random // decides between equally rare groups; tests replace it with a seeded one
     this.serving = new Map() // root → ContentSource
     this.downloads = new Map() // root → Download
     this.peers = new Map() // peer id → Peer
@@ -221,10 +222,13 @@ export class Swarmwire extends EventEmitter {
    *
    * @param {string|object} link
    * @param {string} dir
-   * @param {{onfile?: Function}} [opts]
+   * `onfile({ path, size })` runs as each file completes; `onprogress({ bytes, total })` as bytes of
+   * the whole folder arrive, first with `bytes: 0` once the list is known.
+   *
+   * @param {{onfile?: Function, onprogress?: Function}} [opts]
    * @returns {Promise<{files: number, bytes: number}>}
    */
-  async downloadCollection (link, dir, { onfile = null } = {}) {
+  async downloadCollection (link, dir, { onfile = null, onprogress = null } = {}) {
     const parsed = typeof link === 'string' ? parseLink(link) : link
 
     await fsx.mkdir(dir)
@@ -237,14 +241,21 @@ export class Swarmwire extends EventEmitter {
     if (!list) throw new Error('the folder list read was stopped')
 
     let bytes = 0
+    const total = list.size
+    onprogress?.({ bytes, total })
 
     for (const file of list.files) {
       const target = join(dir, file.path)
       await fsx.mkdir(parentOf(target))
 
-      await this.downloadFile(parsed, file, target)
+      const download = this.downloadFile(parsed, file, target)
+      download.on('progress', ({ have }) => {
+        onprogress?.({ bytes: bytes + Math.min(have * groupLength(this.groupLog), file.size), total })
+      })
+      await download
 
       bytes += file.size
+      onprogress?.({ bytes, total })
       if (onfile) onfile({ path: file.path, size: file.size })
     }
 
@@ -553,6 +564,7 @@ export class Download extends EventEmitter {
 
     this._destination = null
     this._bitfields = new Map() // peer id -> what that peer says it holds
+    this._availability = null // how many of those hold each group; built once the size is known
     this._sources = new Set() // peer ids worth asking right now
     this._liars = new Set() // peer ids that sent something that did not verify: never again
     this._asked = new Set() // peer ids already told what we want
@@ -675,6 +687,8 @@ export class Download extends EventEmitter {
     if (this.stopped) return destination.close()
 
     this._destination = destination
+    this._availability = new Availability(destination.groupCount)
+    for (const bitfield of this._bitfields.values()) this._availability.add(bitfield)
     if (this._destination.complete) return this._finish()
 
     for (const peer of this.node.peers.values()) this._askPeer(peer)
@@ -693,7 +707,7 @@ export class Download extends EventEmitter {
 
     if (this._liars.has(peer.id)) return // nothing it says is worth storing
 
-    this._bitfields.set(peer.id, b4a.from(bitfield ?? []))
+    this._setBitfield(peer.id, b4a.from(bitfield ?? []))
     this._sources.add(peer.id)
 
     if (this.size === null) {
@@ -737,7 +751,7 @@ export class Download extends EventEmitter {
       this.emit('rejected', { index, from: peer.id })
       this._liars.add(peer.id) // a peer that sends rubbish is never asked again
       this._sources.delete(peer.id)
-      this._bitfields.delete(peer.id)
+      this._dropBitfield(peer.id)
     }
 
     if (this.complete) return this._finish()
@@ -770,12 +784,20 @@ export class Download extends EventEmitter {
 
     const requests = this._requests(peer)
     const window = this._windowFor(peer)
-    const wanted = rarestFirst(this._destination.missing(), (index) => this._holders(index))
+    const destination = this._destination
+    const random = this.node.random
+
+    const pool = candidates({
+      count: destination.groupCount,
+      has: (index) => destination.has(index),
+      peerHas: (index) => this._peerHas(peer.id, index),
+      claimed: (index) => this._claimed(index),
+      from: Math.floor(random() * destination.groupCount)
+    })
+    const wanted = rank(pool, this._availability, random)
 
     for (const index of wanted) {
       if (requests.size >= window) break
-      if (this._claimed(index)) continue
-      if (!this._peerHas(peer.id, index)) continue // it has not got that one yet
 
       this._claim(peer, index)
       peer.send.request({ root: b4a.from(this.root, 'hex'), index })
@@ -819,13 +841,17 @@ export class Download extends EventEmitter {
     this._windows.set(peer.id, next)
   }
 
-  /** How many connected peers hold this group. Drives rarest-first selection. */
-  _holders (index) {
-    let n = 0
-    for (const peerId of this._sources) {
-      if (this._peerHas(peerId, index)) n++
-    }
-    return n
+  /** Remember what a peer holds, keeping the per-group counts in step. */
+  _setBitfield (peerId, bitfield) {
+    this._dropBitfield(peerId)
+    this._bitfields.set(peerId, bitfield)
+    this._availability?.add(bitfield)
+  }
+
+  _dropBitfield (peerId) {
+    const old = this._bitfields.get(peerId)
+    if (old) this._availability?.remove(old)
+    this._bitfields.delete(peerId)
   }
 
   /** Does this peer hold that group? An empty bitfield means we were told nothing useful. */
@@ -848,7 +874,10 @@ export class Download extends EventEmitter {
     this._narrow(peer)
 
     const bitfield = this._bitfields.get(peer.id)
-    if (bitfield && index < bitfield.length * 8) bao.clearBit(bitfield, index)
+    if (bitfield && index < bitfield.length * 8 && bao.getBit(bitfield, index)) {
+      bao.clearBit(bitfield, index)
+      this._availability?.unset(index)
+    }
 
     this._pumpAll()
   }
@@ -858,7 +887,10 @@ export class Download extends EventEmitter {
     const bitfield = this._bitfields.get(peer.id)
     if (!bitfield || index >= bitfield.length * 8) return
 
-    bao.setBit(bitfield, index)
+    if (!bao.getBit(bitfield, index)) {
+      bao.setBit(bitfield, index)
+      this._availability?.set(index)
+    }
     this._pump(peer)
   }
 
@@ -906,7 +938,7 @@ export class Download extends EventEmitter {
 
   _peerGone (peer) {
     this._sources.delete(peer.id)
-    this._bitfields.delete(peer.id)
+    this._dropBitfield(peer.id)
     this._asked.delete(peer.id)
     this._releaseAll(peer.id) // whatever it owed us is up for grabs again
 

package/lib/tree.js

@@ -129,6 +129,7 @@ export class GroupTree {
     this.groupHashes = groupHashes
     this.size = size
     this.groupLog = groupLog
+    this._known = new Map() // 'start:end' → hash of a larger subtree; a hash once known never changes
   }
 
   /** Hash content held in memory. For files, stream it through GroupTreeBuilder instead. */
@@ -162,7 +163,7 @@ export class GroupTree {
     if (!this.complete) return null
     if (this.groupCount === 1) return b4a.toString(this.groupHashes[0], 'hex')
 
-    return b4a.toString(subtree(this.groupHashes, 0, this.groupCount, true), 'hex')
+    return b4a.toString(subtree(this.groupHashes, 0, this.groupCount, true, this._known), 'hex')
   }
 
   /**
@@ -186,8 +187,8 @@ export class GroupTree {
     while (end - start > 1) {
       const mid = start + splitPoint(end - start)
       const sibling = index < mid
-        ? subtree(this.groupHashes, mid, end, false)
-        : subtree(this.groupHashes, start, mid, false)
+        ? subtree(this.groupHashes, mid, end, false, this._known)
+        : subtree(this.groupHashes, start, mid, false, this._known)
 
       if (sibling === null) return null
       siblings.push(sibling)
@@ -259,18 +260,28 @@ export function splitPoint (count) {
   return p
 }
 
-/** Null when any leaf under this subtree is unknown. */
-function subtree (hashes, start, end, isRoot) {
+// shortcut: only subtrees of at least this many groups are remembered, so the memory is about n/16
+// hashes and a proof recomputes at most ~15 per level; lower it if proofs ever show up in a profile.
+const REMEMBER = 16
+
+/** Null when any leaf under this subtree is unknown. `known` remembers the bigger ones. */
+function subtree (hashes, start, end, isRoot, known) {
   if (end - start === 1) return hashes[start] ?? null
 
+  const key = !isRoot && end - start >= REMEMBER ? start + ':' + end : null
+  const remembered = key && known.get(key)
+  if (remembered) return remembered
+
   const mid = start + splitPoint(end - start)
-  const left = subtree(hashes, start, mid, false)
+  const left = subtree(hashes, start, mid, false, known)
   if (left === null) return null
 
-  const right = subtree(hashes, mid, end, false)
+  const right = subtree(hashes, mid, end, false, known)
   if (right === null) return null
 
-  return bao.parentCV(left, right, isRoot)
+  const hash = bao.parentCV(left, right, isRoot)
+  if (key) known.set(key, hash)
+  return hash
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "swarmwire-lib",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Content-addressed, verified file transfer between filesystems over Hyperswarm. No copies, no cores, no trackers.",
   "type": "module",
   "main": "index.js",
@@ -67,6 +67,7 @@
     "test:bare:net": "bare node_modules/brittle/bin/bare.js test/net/*.test.js",
     "bench": "node bench/swarm.js",
     "bench:net": "node bench/throughput.js",
+    "bench:sim": "node bench/swarm-sim.js",
     "check:pack": "node scripts/check-pack.js"
   },
   "dependencies": {