swarmwire-lib 0.2.0 → 0.3.0

package/docs/contract.md

@@ -23,8 +23,9 @@ code (`serveCollection`, `downloadCollection`). See `AGENTS.md` for the invarian
 | Peer identity: the remote public key of the connection | exists | - | `test/swarm/transfer.test.js` |
 | Peer activity events, peer count, seeder and leecher counts | exists | - | `test/swarm/peers.test.js` |
 | Folder file-list reader (look-only, in memory), and downloading one listed file | exists | - | `test/swarm/list.test.js` |
-| Peer hints (preferred peers, priorities) | planned | 003 | - |
-| Admission decisions | planned | 003 | - |
+| Download order spread by random choice (no two newcomers ask for the same thing first) | exists | - | `test/swarm/spread.test.js` |
+| Peer hints (preferred peers, priorities) | planned | 004 | - |
+| Admission decisions | planned | 004 | - |
 
 ## 3. Rules the application can rely on
 
@@ -73,6 +74,13 @@ The application can decide, per peer and per content, whether content is served.
 **Direct connections only**: a snapshot that may lag, not a count of the swarm. Global counts and
 chunk rarity across the network are a tracker's job, not swarmwire's.
 
+### 3.3a Download order
+
+Groups are fetched rarest first, equal rarity broken at random, starting from a random place in
+the file. **A download does not proceed in file order**: a file is complete when every group is
+held, and nothing is promised about reading it while it downloads. How the choice was made, and
+what was measured, is in `docs/scaling.md` in the repository (not part of the package).
+
 ### 3.4 Identity
 
 swarmwire knows the remote public key of each connection and nothing else about a peer. Tying a

package/lib/select.js

@@ -5,29 +5,55 @@
  * The reason is not speed on this download but survival of the swarm — a group held by one
  * peer disappears when that peer leaves, and a swarm where every copy of some group lives on
  * one machine is one disconnection away from nobody being able to finish.
+ *
+ * Two things keep a crowd from asking for the same thing. Equally rare groups are ordered at
+ * random rather than by position, and the scan for candidates starts at a random group rather
+ * than the start of the file — otherwise a thousand fresh downloaders of a large file would
+ * all draw from its first few hundred groups.
  */
 
-/** How many missing groups to consider at once. */
+import { bao } from './blake3.js'
+
+/** How many candidate groups to weigh at once. */
 export const SCAN_LIMIT = 256
 
+/** How many source peers hold each group. Kept up to date by the caller, read in O(1). */
+export class Availability {
+  constructor (groupCount) {
+    this.counts = new Uint16Array(groupCount)
+  }
+
+  at (index) { return this.counts[index] }
+  set (index) { this.counts[index]++ }
+  unset (index) { if (this.counts[index] > 0) this.counts[index]-- }
+
+  add (bitfield) { this._each(bitfield, (i) => this.set(i)) }
+  remove (bitfield) { this._each(bitfield, (i) => this.unset(i)) }
+
+  _each (bitfield, fn) {
+    const n = Math.min(this.counts.length, bitfield.length * 8)
+    for (let i = 0; i < n; i++) if (bao.getBit(bitfield, i)) fn(i)
+  }
+}
+
 /**
- * Order candidate groups by how few peers hold them, keeping file order as the tie-break so
- * a transfer still roughly streams.
- *
- * Only the first `limit` missing groups are considered: on a large file, sorting every
- * outstanding group on every request would cost more than it saves, and rarity among the next
- * few hundred is what actually decides the next request.
- *
- * @param {number[]} missing - group indices still wanted, in ascending order
- * @param {(index: number) => number} holders - how many peers hold this group
- * @param {{limit?: number}} [opts]
- * @returns {number[]}
+ * The groups worth asking this peer for: wanted, offered, and not already being fetched.
+ * Scanning starts at `from` and wraps, so the window is a random stretch of the file.
  */
-export function rarestFirst (missing, holders, { limit = SCAN_LIMIT } = {}) {
-  const candidates = missing.length > limit ? missing.slice(0, limit) : missing.slice()
+export function candidates ({ count, has, peerHas, claimed, from = 0, limit = SCAN_LIMIT }) {
+  const out = []
+
+  for (let k = 0; k < count && out.length < limit; k++) {
+    const i = (from + k) % count
+    if (!has(i) && peerHas(i) && !claimed(i)) out.push(i)
+  }
+  return out
+}
 
-  return candidates
-    .map((index) => ({ index, holders: holders(index) }))
-    .sort((a, b) => a.holders - b.holders || a.index - b.index)
-    .map((candidate) => candidate.index)
+/** Fewest holders first; groups equally rare come in the order `random` gives them. */
+export function rank (pool, availability, random) {
+  return pool
+    .map((index) => ({ index, rarity: availability.at(index), tie: random() }))
+    .sort((a, b) => a.rarity - b.rarity || a.tie - b.tie)
+    .map((entry) => entry.index)
 }

package/lib/swarmwire.js

@@ -20,7 +20,7 @@ import { Destination } from './destination.js'
 import { discoveryTopic, formatLink, parseLink } from './address.js'
 import { DEFAULT_GROUP_LOG, groupCount } from './tree.js'
 import { openChannel, VERSION } from './wire.js'
-import { rarestFirst } from './select.js'
+import { Availability, candidates, rank } from './select.js'
 import { Holdings } from './holdings.js'
 import { Listing } from './listing.js'
 import { systemTimers } from './timers.js'
@@ -79,6 +79,7 @@ export class Swarmwire extends EventEmitter {
     this.timers = timers
     this.requestTimeout = requestTimeout
     this.window = window
+    this.random = Math.random // decides between equally rare groups; tests replace it with a seeded one
     this.serving = new Map() // root → ContentSource
     this.downloads = new Map() // root → Download
     this.peers = new Map() // peer id → Peer
@@ -553,6 +554,7 @@ export class Download extends EventEmitter {
 
     this._destination = null
     this._bitfields = new Map() // peer id -> what that peer says it holds
+    this._availability = null // how many of those hold each group; built once the size is known
     this._sources = new Set() // peer ids worth asking right now
     this._liars = new Set() // peer ids that sent something that did not verify: never again
     this._asked = new Set() // peer ids already told what we want
@@ -675,6 +677,8 @@ export class Download extends EventEmitter {
     if (this.stopped) return destination.close()
 
     this._destination = destination
+    this._availability = new Availability(destination.groupCount)
+    for (const bitfield of this._bitfields.values()) this._availability.add(bitfield)
     if (this._destination.complete) return this._finish()
 
     for (const peer of this.node.peers.values()) this._askPeer(peer)
@@ -693,7 +697,7 @@ export class Download extends EventEmitter {
 
     if (this._liars.has(peer.id)) return // nothing it says is worth storing
 
-    this._bitfields.set(peer.id, b4a.from(bitfield ?? []))
+    this._setBitfield(peer.id, b4a.from(bitfield ?? []))
     this._sources.add(peer.id)
 
     if (this.size === null) {
@@ -737,7 +741,7 @@ export class Download extends EventEmitter {
       this.emit('rejected', { index, from: peer.id })
       this._liars.add(peer.id) // a peer that sends rubbish is never asked again
       this._sources.delete(peer.id)
-      this._bitfields.delete(peer.id)
+      this._dropBitfield(peer.id)
     }
 
     if (this.complete) return this._finish()
@@ -770,12 +774,20 @@ export class Download extends EventEmitter {
 
     const requests = this._requests(peer)
     const window = this._windowFor(peer)
-    const wanted = rarestFirst(this._destination.missing(), (index) => this._holders(index))
+    const destination = this._destination
+    const random = this.node.random
+
+    const pool = candidates({
+      count: destination.groupCount,
+      has: (index) => destination.has(index),
+      peerHas: (index) => this._peerHas(peer.id, index),
+      claimed: (index) => this._claimed(index),
+      from: Math.floor(random() * destination.groupCount)
+    })
+    const wanted = rank(pool, this._availability, random)
 
     for (const index of wanted) {
       if (requests.size >= window) break
-      if (this._claimed(index)) continue
-      if (!this._peerHas(peer.id, index)) continue // it has not got that one yet
 
       this._claim(peer, index)
       peer.send.request({ root: b4a.from(this.root, 'hex'), index })
@@ -819,13 +831,17 @@ export class Download extends EventEmitter {
     this._windows.set(peer.id, next)
   }
 
-  /** How many connected peers hold this group. Drives rarest-first selection. */
-  _holders (index) {
-    let n = 0
-    for (const peerId of this._sources) {
-      if (this._peerHas(peerId, index)) n++
-    }
-    return n
+  /** Remember what a peer holds, keeping the per-group counts in step. */
+  _setBitfield (peerId, bitfield) {
+    this._dropBitfield(peerId)
+    this._bitfields.set(peerId, bitfield)
+    this._availability?.add(bitfield)
+  }
+
+  _dropBitfield (peerId) {
+    const old = this._bitfields.get(peerId)
+    if (old) this._availability?.remove(old)
+    this._bitfields.delete(peerId)
   }
 
   /** Does this peer hold that group? An empty bitfield means we were told nothing useful. */
@@ -848,7 +864,10 @@ export class Download extends EventEmitter {
     this._narrow(peer)
 
     const bitfield = this._bitfields.get(peer.id)
-    if (bitfield && index < bitfield.length * 8) bao.clearBit(bitfield, index)
+    if (bitfield && index < bitfield.length * 8 && bao.getBit(bitfield, index)) {
+      bao.clearBit(bitfield, index)
+      this._availability?.unset(index)
+    }
 
     this._pumpAll()
   }
@@ -858,7 +877,10 @@ export class Download extends EventEmitter {
     const bitfield = this._bitfields.get(peer.id)
     if (!bitfield || index >= bitfield.length * 8) return
 
-    bao.setBit(bitfield, index)
+    if (!bao.getBit(bitfield, index)) {
+      bao.setBit(bitfield, index)
+      this._availability?.set(index)
+    }
     this._pump(peer)
   }
 
@@ -906,7 +928,7 @@ export class Download extends EventEmitter {
 
   _peerGone (peer) {
     this._sources.delete(peer.id)
-    this._bitfields.delete(peer.id)
+    this._dropBitfield(peer.id)
     this._asked.delete(peer.id)
     this._releaseAll(peer.id) // whatever it owed us is up for grabs again
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "swarmwire-lib",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Content-addressed, verified file transfer between filesystems over Hyperswarm. No copies, no cores, no trackers.",
   "type": "module",
   "main": "index.js",
@@ -67,6 +67,7 @@
     "test:bare:net": "bare node_modules/brittle/bin/bare.js test/net/*.test.js",
     "bench": "node bench/swarm.js",
     "bench:net": "node bench/throughput.js",
+    "bench:sim": "node bench/swarm-sim.js",
     "check:pack": "node scripts/check-pack.js"
   },
   "dependencies": {