swarmwire-lib 0.1.0 → 0.3.0

package/README.md

@@ -9,35 +9,19 @@ Files are served in place, from wherever they already live on disk. Nothing is c
 store, a core, or a cache first.
 
 ```
-npm i swarmwire-lib
+npm i swarmwire-lib hyperswarm
 ```
 
-One side serves a file and prints a link. The other side hands the link to `download`. (This is
-`examples/send-get.js`; in the repository it imports `../index.js` instead of `swarmwire-lib`. It is
-an ES module, so your project needs `"type": "module"` or a `.mjs` file.)
+One side serves a file and prints a link. The other side hands the link to `download`:
 
 ```js
-/**
- * Send a file to someone, or get one.
- *
- *   node examples/send-get.js send ./holidays.mp4
- *   node examples/send-get.js get 'swarmwire://…' ./holidays.mp4
- *
- * `--bootstrap host:port` points both sides at a private DHT; leave it out for the real one.
- */
-
 import process from 'process'
 import Hyperswarm from 'hyperswarm'
-import DHT from 'hyperdht'
-
 import { Swarmwire } from 'swarmwire-lib'
 
-const args = process.argv.slice(2)
-const flag = args.indexOf('--bootstrap')
-const bootstrap = flag === -1 ? null : args.splice(flag, 2)[1]
-const [mode, target, out] = args
+const [mode, target, out] = process.argv.slice(2)
 
-const swarm = bootstrap ? privateSwarm(bootstrap) : new Hyperswarm()
+const swarm = new Hyperswarm()
 const node = new Swarmwire({ swarm })
 
 if (mode === 'send') {
@@ -47,22 +31,16 @@ if (mode === 'send') {
 } else if (mode === 'get') {
   await node.download(target, out)
   console.log(out)
-} else {
-  console.log('usage: send <file> | get <link> <out>   [--bootstrap host:port]')
-  process.exitCode = 1
 }
 
 await node.destroy()
 await swarm.destroy()
-
-// A loopback DHT only works if the node binds there and is not firewalled.
-function privateSwarm (address) {
-  const [host, port] = address.split(':')
-  const dht = new DHT({ bootstrap: [{ host, port: Number(port) }], ephemeral: false, firewalled: false, host: '127.0.0.1' })
-  return new Hyperswarm({ dht })
-}
 ```
 
+This is `examples/send-get.js` without its `--bootstrap` option, which points both sides at a
+private DHT so the test suite needs no internet. It is an ES module, so your project needs
+`"type": "module"` or a `.mjs` file.
+
 ```
 $ node examples/send-get.js send ./holidays.mp4
 swarmwire://10db6398a68dff502d002912f54502fdde9ab55fb30bec32861a2f004d874fc8?s=734003200&n=holidays.mp4
@@ -72,11 +50,13 @@ $ node examples/send-get.js get 'swarmwire://10db63…' ./holidays.mp4
 ```
 
 A folder goes the same way, as one link: `serveCollection` and `downloadCollection`.
-The command-line tool built on this library is moving to its own package, `swarmwire-cli`.
+The command-line tool built on this library is moving to its own package,
+[`swarmwire-cli`](https://github.com/tibocub/swarmwire-cli) ([npm](https://www.npmjs.com/package/swarmwire-cli)).
 
-> Status: **pre-alpha.** Transfers work end to end, over a real swarm, with resume and
-> verification. Multi-peer scheduling is still naive and there are no request timeouts yet —
-> see [docs/roadmap.md](docs/roadmap.md) for what M2 adds.
+> Status: **pre-alpha.** Transfers work end to end, over a real swarm, with resume,
+> verification, request timeouts, partial seeding, rarest-first selection and an adaptive
+> request window. Behaviour with very many peers is not measured yet. See
+> [docs/roadmap.md](docs/roadmap.md) for what is built and what is next.
 
 ## What it is
 

package/docs/contract.md

@@ -19,12 +19,13 @@ code (`serveCollection`, `downloadCollection`). See `AGENTS.md` for the invarian
 | Serve a file in place, download by link, resume, verify every range | exists | - | `test/swarm/transfer.test.js` |
 | Serve and download a folder as one link | exists | - | `test/swarm/collection.test.js` |
 | Partial seeding: a download in progress serves what it holds | exists | - | `test/swarm/partial.test.js` |
-| Application supplies the network connection (any object with `join`, `leave`, a `connection` event) | exists | - | `test/helpers/network.js`, `test/net/real-swarm.test.js` |
+| Application supplies the network connection (any object with `join(topic, opts)` returning a session with `destroy()`, and a `connection` event; Hyperswarm is one) | exists | - | `test/helpers/network.js`, `test/net/real-swarm.test.js` |
 | Peer identity: the remote public key of the connection | exists | - | `test/swarm/transfer.test.js` |
-| Peer hints (preferred peers, priorities) | planned | 002 | - |
-| Admission decisions | planned | 002 | - |
-| Peer activity events and counts | planned | 002 | - |
-| Folder file-list reader, public | planned | 002 | - |
+| Peer activity events, peer count, seeder and leecher counts | exists | - | `test/swarm/peers.test.js` |
+| Folder file-list reader (look-only, in memory), and downloading one listed file | exists | - | `test/swarm/list.test.js` |
+| Download order spread by random choice (no two newcomers ask for the same thing first) | exists | - | `test/swarm/spread.test.js` |
+| Peer hints (preferred peers, priorities) | planned | 004 | - |
+| Admission decisions | planned | 004 | - |
 
 ## 3. Rules the application can rely on
 
@@ -39,8 +40,12 @@ a peer because of a hint. The application dials through the connection it owns.
 The application can decide, per peer and per content, whether content is served.
 
 - Decided **per content root**. Each file of a folder is its own root and is decided separately;
-  the decision is told which folder root the file was requested through, if any.
-- **Deny on failure**: a decision that throws, times out or never answers is a deny.
+  the decision is told which folder root the file was requested through, if any. The folder's own
+  file list is content too, with its own root, and is decided like any other root.
+- **Deny on failure**: a decision that throws, times out or never answers is a deny. How long a
+  decision may take is an option of the admission mechanism, set by the application (its
+  default is chosen when the mechanism is built); the contract promises only that running out
+  of time is a deny.
 - A deny is answered **exactly as content swarmwire does not hold**. A refused peer learns nothing
   about what is held, its size or its name.
 - With no decision installed, content is served as it is today.
@@ -49,10 +54,32 @@ The application can decide, per peer and per content, whether content is served.
 
 ### 3.3 Peer activity
 
-Events when a peer connects and disconnects, and, for one content root, the number of connected
-peers and how many hold all of it. **Direct connections only**: a snapshot that may lag, not a
-count of the swarm. Global counts and chunk rarity across the network are a tracker's job, not
-swarmwire's.
+`Swarmwire` is an event emitter.
+
+- `peer-connected` and `peer-disconnected`, each with `{ id }` (the remote public key, hex). Each
+  fires **once per distinct peer**, and only for a peer that completed the swarmwire handshake: a
+  connection that never speaks the protocol is neither announced nor counted. Two connections
+  from one key are one peer. A peer that vanishes without closing is reported when its connection
+  closes; `peerCount` is already correct when the event fires.
+- `peerCount`: connected peers, never negative.
+- `counts(link)` returns `{ peers, seeders, leechers }` for a file or folder link, synchronously.
+  A **seeder** holds all of it, a **leecher** holds part, and a peer that has said nothing about
+  that content is in `peers` only. For a folder, a seeder holds the file list and every listed
+  file; a peer holding the list alone is a leecher. Content the node knows nothing about, and a
+  folder whose list is not yet known, give zeros, never an error.
+- A node that only serves still learns what its peers hold: it asks each connected peer about
+  the content it serves, with the ordinary `want` message. No message was added. A peer that does
+  not answer is simply uncounted.
+
+**Direct connections only**: a snapshot that may lag, not a count of the swarm. Global counts and
+chunk rarity across the network are a tracker's job, not swarmwire's.
+
+### 3.3a Download order
+
+Groups are fetched rarest first, equal rarity broken at random, starting from a random place in
+the file. **A download does not proceed in file order**: a file is complete when every group is
+held, and nothing is promised about reading it while it downloads. How the choice was made, and
+what was measured, is in `docs/scaling.md` in the repository (not part of the package).
 
 ### 3.4 Identity
 
@@ -61,9 +88,13 @@ key to a user or device is the application's job (Principle II).
 
 ### 3.5 Choosing files of a folder
 
-Read the folder's file list without fetching content, then download the chosen files one by one.
-Which files, and any interface for choosing, belong to the application. *Planned: the list reader
-is not public yet.*
+`readList(link)` reads a folder's file list without fetching any content: it resolves to
+`{ files: [{ path, root, size }], size }`, and `stop()` gives up waiting, like a download. It
+writes nothing to disk and only **looks** for peers: it never announces this node as a provider.
+A link that is not a folder is refused, and so is a list with an unsafe path (the same paths a
+whole-folder download refuses; nothing is repaired). `downloadFile(link, file, outPath)` then
+downloads one listed file, using only the folder's link and the list entry, and does not announce
+it separately. Which files, and any interface for choosing, belong to the application.
 
 ### 3.6 The shared connection
 
@@ -109,9 +140,10 @@ DEFAULT_GROUP_LOG
 ```
 
 **Stable** (changes bump the version and are in the changelog): `Swarmwire`
-(`serve`, `serveCollection`, `download`, `downloadCollection`, `unserve`, `destroy`), `Download`
-(awaitable; events `progress`, `rejected`, `timeout`, `stalled`), `rootOf`, `formatLink`,
-`parseLink`, `SCHEME`.
+(`serve`, `serveCollection`, `download`, `downloadCollection`, `readList`, `downloadFile`,
+`unserve`, `destroy`, `peerCount`, `counts`; events `peer-connected`, `peer-disconnected`),
+`Download` (awaitable; events `progress`, `rejected`, `timeout`, `stalled`), `rootOf`,
+`formatLink`, `parseLink`, `SCHEME`.
 
 **Exported for tests and advanced use, may change in any 0.x**: everything else.
 

package/docs/format.md

@@ -166,8 +166,13 @@ announce.
 size, request block size, wire messages, scheduling, how outboard trees are cached, the sidecar
 format for partial downloads.
 
-Protocol messages carry a version and a capability set in the handshake. Unknown message types
-are ignored rather than treated as errors, so new message types never break old peers.
+The handshake carries a version and the group size (`groupLog`); it carries no capability set.
+Unknown message types are ignored rather than treated as errors, so new message types never
+break old peers.
+
+What a peer holds is learned by asking: `want` is answered with `have` (or `unavailable`) and
+followed by `holds` as the peer gains groups. A node that only serves sends `want` for what it
+serves, which is how it counts its seeders and leechers. No message exists for this.
 
 ## Open questions
 

package/lib/collection.js

@@ -18,7 +18,7 @@ import b4a from 'b4a'
 export const MANIFEST_TYPE = 'collection/v0'
 
 /** Enough for a folder of a million files; past that, something has gone wrong. */
-const MAX_MANIFEST_BYTES = 64 * 1024 * 1024
+export const MAX_MANIFEST_BYTES = 64 * 1024 * 1024
 
 /**
  * @param {{path: string, root: string, size: number}[]} files

package/lib/holdings.js

@@ -0,0 +1,125 @@
+/**
+ * What connected peers say they hold, and the counts that follow from it.
+ *
+ * A node only hears about a peer's holdings if it asked: `want` is answered with `have`, and
+ * the peer then pushes `holds` as it gains groups. A node that only serves never asks, so it
+ * keeps a watch set — the roots it serves — and sends `want` for those to every peer. That is
+ * the whole mechanism; there is no message for it.
+ *
+ * Counts are of direct connections. Nothing here knows about the rest of the swarm.
+ */
+
+import b4a from 'b4a'
+
+import { bao } from './blake3.js'
+import { DEFAULT_GROUP_LOG, groupCount } from './tree.js'
+
+export class Holdings {
+  constructor ({ groupLog = DEFAULT_GROUP_LOG } = {}) {
+    this.groupLog = groupLog
+    this.peers = new Map() // peer id → Map(root hex → { size, total, bits, count })
+    this.watched = new Map() // root hex → size: roots we ask every peer about
+  }
+
+  /** A peer described what it holds (`have`). Replaces whatever we knew. */
+  set (peerId, root, { size, groupLog, bitfield }) {
+    if (groupLog !== this.groupLog) return // we cannot read that granularity: unknown, not guessed
+
+    const total = groupCount(size, this.groupLog)
+    const bits = bao.createBitfield(total)
+    bits.set(b4a.from(bitfield ?? []).subarray(0, bits.length))
+
+    this._of(peerId).set(root, { size, total, bits, count: bao.countSetBits(bits, total) })
+  }
+
+  /**
+   * A peer gained one group (`holds`). Without a `have` to build on, a known `size` stands in:
+   * a peer that first said "unavailable" and then began downloading only ever sends this.
+   */
+  grow (peerId, root, index, size) {
+    let held = this.get(peerId, root)
+
+    if (!held) {
+      if (size === undefined) return
+      const total = groupCount(size, this.groupLog)
+      held = { size, total, bits: bao.createBitfield(total), count: 0 }
+      this._of(peerId).set(root, held)
+    }
+
+    if (index < 0 || index >= held.total || bao.getBit(held.bits, index)) return
+
+    bao.setBit(held.bits, index)
+    held.count++
+  }
+
+  /** The peer says it has none of this (`unavailable`). */
+  remove (peerId, root) {
+    this.peers.get(peerId)?.delete(root)
+  }
+
+  dropPeer (peerId) {
+    this.peers.delete(peerId)
+  }
+
+  get (peerId, root) {
+    return this.peers.get(peerId)?.get(root)
+  }
+
+  /**
+   * @param {Iterable<string>} peerIds - the connected peers to count among
+   * @param {string[]} roots - one root for a file; the manifest and every file for a folder
+   * @returns {{peers: number, seeders: number, leechers: number}} a seeder holds every group
+   *   of every root, a leecher holds some but not all, anyone else is in neither
+   */
+  count (peerIds, roots) {
+    const counts = { peers: 0, seeders: 0, leechers: 0 }
+
+    for (const id of peerIds) {
+      counts.peers++
+      if (roots.length === 0) continue
+
+      let complete = true
+      let any = false
+
+      for (const root of roots) {
+        const held = this.get(id, root)
+        if (!held || held.count < held.total) complete = false
+        if (held && held.count > 0) any = true
+      }
+
+      if (complete) counts.seeders++
+      else if (any) counts.leechers++
+    }
+
+    return counts
+  }
+
+  // -- the watch set ---------------------------------------------------------
+
+  /** Ask every peer about a root we hold or follow, now and whenever a new peer connects. */
+  watch (root, size, peers, skip) {
+    this.watched.set(root, size)
+    for (const peer of peers) this._want(peer, root, size, skip)
+  }
+
+  unwatch (root) {
+    this.watched.delete(root)
+  }
+
+  /** A peer just connected: tell it what we are watching. */
+  wantAll (peer, skip) {
+    for (const [root, size] of this.watched) this._want(peer, root, size, skip)
+  }
+
+  _want (peer, root, size, skip) {
+    // shortcut: wants for every file of a served folder; watch lazily if folders grow past ~10k files
+    if (skip(root)) return // a download already asked this peer
+    peer.send.want({ root: b4a.from(root, 'hex'), size })
+  }
+
+  _of (peerId) {
+    let held = this.peers.get(peerId)
+    if (!held) this.peers.set(peerId, held = new Map())
+    return held
+  }
+}

package/lib/listing.js

@@ -0,0 +1,158 @@
+/**
+ * Reading a folder's file list: fetch the manifest into memory, validate it, and leave nothing
+ * behind. Nothing here touches disk, and it only ever looks for peers — a node that is merely
+ * reading a list is not a provider of the folder.
+ */
+
+import b4a from 'b4a'
+import { bao } from './blake3.js'
+
+import { decodeManifest, totalSize, MAX_MANIFEST_BYTES } from './collection.js'
+import { discoveryTopic } from './address.js'
+import { GroupTree, groupHash, verifyGroup, groupCount } from './tree.js'
+import { NodeStore } from './nodes.js'
+
+/**
+ * The same surface a `Download` uses on a file destination, kept in memory: verify a group
+ * before storing it, remember the proofs, hand groups back out. A manifest is small and capped,
+ * so there is no resume state to keep.
+ */
+export class MemoryDestination {
+  constructor ({ root, size, groupLog }) {
+    if (size > MAX_MANIFEST_BYTES) throw new Error('manifest is implausibly large')
+
+    this.root = root
+    this.size = size
+    this.groupLog = groupLog
+    this.groupCount = groupCount(size, groupLog)
+
+    this._groups = new Map() // index → bytes
+    this._bitfield = bao.createBitfield(this.groupCount)
+    this._nodes = new NodeStore(this.groupCount)
+  }
+
+  get have () { return this._groups.size }
+  get complete () { return this._groups.size === this.groupCount }
+  has (index) { return bao.getBit(this._bitfield, index) }
+  bitfield () { return this._bitfield }
+
+  missing () {
+    const out = []
+    for (let i = 0; i < this.groupCount; i++) if (!this.has(i)) out.push(i)
+    return out
+  }
+
+  async put (index, data, proof) {
+    if (!Number.isInteger(index) || index < 0 || index >= this.groupCount) return false
+    if (this.has(index)) return true
+
+    const { root, size, groupLog } = this
+    if (!verifyGroup({ root, size, index, data, proof, groupLog })) return false
+
+    const hash = groupHash(data, index, { groupLog, only: this.groupCount === 1 })
+    this._nodes.learn(index, hash, proof)
+    this._groups.set(index, b4a.from(data))
+    bao.setBit(this._bitfield, index)
+
+    return true
+  }
+
+  get tree () {
+    return new GroupTree(this._nodes.leaves(), { size: this.size, groupLog: this.groupLog })
+  }
+
+  proof (index) {
+    return this.has(index) ? this._nodes.proof(index) : null
+  }
+
+  async readGroup (index) {
+    return this._groups.get(index) ?? null
+  }
+
+  /** The whole content, once every group is held. */
+  bytes () {
+    const parts = []
+    for (let i = 0; i < this.groupCount; i++) parts.push(this._groups.get(i))
+    return b4a.concat(parts)
+  }
+
+  async finish () {}
+  async close () {}
+}
+
+/**
+ * A folder's file list being read. Thenable, like a `Download`: `await listing` resolves to
+ * `{ files: [{ path, root, size }], size }`, and `stop()` gives up waiting, which resolves it
+ * to `undefined`.
+ */
+export class Listing {
+  /** A read that failed before it began, such as a link that is not a folder. */
+  static failed (err) {
+    const listing = Object.create(Listing.prototype)
+    listing._promise = Promise.reject(err)
+    listing._promise.catch(() => {}) // the caller sees it; this only keeps it from being unhandled
+    listing.stop = async () => {}
+    return listing
+  }
+
+  /** A read that needs no network: the list is already known. */
+  static known (files) {
+    const listing = Object.create(Listing.prototype)
+    listing._promise = Promise.resolve({ files, size: totalSize(files) })
+    listing.stop = async () => {}
+    return listing
+  }
+
+  constructor (node, parsed) {
+    this._download = null
+    this._session = null
+    this._stopped = false
+
+    this._promise = this._run(node, parsed)
+    this._promise.catch(() => {})
+  }
+
+  async stop () {
+    this._stopped = true
+    await this._download?.stop()
+  }
+
+  then (onfulfilled, onrejected) {
+    return this._promise.then(onfulfilled, onrejected)
+  }
+
+  catch (onrejected) {
+    return this._promise.catch(onrejected)
+  }
+
+  finally (onfinally) {
+    return this._promise.finally(onfinally)
+  }
+
+  async _run (node, parsed) {
+    if (node.downloads.has(parsed.root)) throw new Error('that link is already being downloaded as a file')
+
+    let memory = null
+
+    // Only look: with no server session on the topic this node is never announced for it.
+    this._session = node.swarm.join(discoveryTopic(parsed.root, parsed.secret), { server: false, client: true })
+
+    try {
+      this._download = node.download({ ...parsed, name: null }, null, {
+        announce: false,
+        destination: (options) => (memory = new MemoryDestination(options))
+      })
+      if (this._stopped) await this._download.stop()
+
+      await this._download
+      if (this._stopped) return undefined
+
+      const files = decodeManifest(memory.bytes())
+      node._learnList(parsed.root, files)
+
+      return { files, size: totalSize(files) }
+    } finally {
+      await this._session?.destroy?.()
+    }
+  }
+}

package/lib/select.js

@@ -5,29 +5,55 @@
  * The reason is not speed on this download but survival of the swarm — a group held by one
  * peer disappears when that peer leaves, and a swarm where every copy of some group lives on
  * one machine is one disconnection away from nobody being able to finish.
+ *
+ * Two things keep a crowd from asking for the same thing. Equally rare groups are ordered at
+ * random rather than by position, and the scan for candidates starts at a random group rather
+ * than the start of the file — otherwise a thousand fresh downloaders of a large file would
+ * all draw from its first few hundred groups.
  */
 
-/** How many missing groups to consider at once. */
+import { bao } from './blake3.js'
+
+/** How many candidate groups to weigh at once. */
 export const SCAN_LIMIT = 256
 
+/** How many source peers hold each group. Kept up to date by the caller, read in O(1). */
+export class Availability {
+  constructor (groupCount) {
+    this.counts = new Uint16Array(groupCount)
+  }
+
+  at (index) { return this.counts[index] }
+  set (index) { this.counts[index]++ }
+  unset (index) { if (this.counts[index] > 0) this.counts[index]-- }
+
+  add (bitfield) { this._each(bitfield, (i) => this.set(i)) }
+  remove (bitfield) { this._each(bitfield, (i) => this.unset(i)) }
+
+  _each (bitfield, fn) {
+    const n = Math.min(this.counts.length, bitfield.length * 8)
+    for (let i = 0; i < n; i++) if (bao.getBit(bitfield, i)) fn(i)
+  }
+}
+
 /**
- * Order candidate groups by how few peers hold them, keeping file order as the tie-break so
- * a transfer still roughly streams.
- *
- * Only the first `limit` missing groups are considered: on a large file, sorting every
- * outstanding group on every request would cost more than it saves, and rarity among the next
- * few hundred is what actually decides the next request.
- *
- * @param {number[]} missing - group indices still wanted, in ascending order
- * @param {(index: number) => number} holders - how many peers hold this group
- * @param {{limit?: number}} [opts]
- * @returns {number[]}
+ * The groups worth asking this peer for: wanted, offered, and not already being fetched.
+ * Scanning starts at `from` and wraps, so the window is a random stretch of the file.
  */
-export function rarestFirst (missing, holders, { limit = SCAN_LIMIT } = {}) {
-  const candidates = missing.length > limit ? missing.slice(0, limit) : missing.slice()
+export function candidates ({ count, has, peerHas, claimed, from = 0, limit = SCAN_LIMIT }) {
+  const out = []
+
+  for (let k = 0; k < count && out.length < limit; k++) {
+    const i = (from + k) % count
+    if (!has(i) && peerHas(i) && !claimed(i)) out.push(i)
+  }
+  return out
+}
 
-  return candidates
-    .map((index) => ({ index, holders: holders(index) }))
-    .sort((a, b) => a.holders - b.holders || a.index - b.index)
-    .map((candidate) => candidate.index)
+/** Fewest holders first; groups equally rare come in the order `random` gives them. */
+export function rank (pool, availability, random) {
+  return pool
+    .map((index) => ({ index, rarity: availability.at(index), tie: random() }))
+    .sort((a, b) => a.rarity - b.rarity || a.tie - b.tie)
+    .map((entry) => entry.index)
 }

package/lib/swarmwire.js

@@ -1,7 +1,7 @@
 /**
  * A swarmwire node: serve content you already have, download content you want.
  *
- * It takes a swarm — anything with `join(topic)`, `leave(topic)` and a `connection` event —
+ * It takes a swarm — anything with `join(topic, opts)` (returning a session with `destroy()`) and a `connection` event —
  * so the same code runs against Hyperswarm and against the deterministic test network.
  *
  * Scheduling is deliberately plain for now: ask every peer that has the content, keep a small
@@ -20,7 +20,9 @@ import { Destination } from './destination.js'
 import { discoveryTopic, formatLink, parseLink } from './address.js'
 import { DEFAULT_GROUP_LOG, groupCount } from './tree.js'
 import { openChannel, VERSION } from './wire.js'
-import { rarestFirst } from './select.js'
+import { Availability, candidates, rank } from './select.js'
+import { Holdings } from './holdings.js'
+import { Listing } from './listing.js'
 import { systemTimers } from './timers.js'
 import * as fsx from './fs.js'
 
@@ -52,7 +54,14 @@ const REQUEST_TIMEOUT = 15000
 /** Timeouts a peer gets before it stops being asked. */
 const MAX_TIMEOUTS = 3
 
-export class Swarmwire {
+/** Passing options makes `join` return a session that leaves only itself. */
+const ANNOUNCE = { server: true, client: true }
+
+/**
+ * Emits `peer-connected` and `peer-disconnected`, each with `{ id }`, for peers that completed
+ * the swarmwire handshake. `peerCount` and `counts(link)` read the same state.
+ */
+export class Swarmwire extends EventEmitter {
   /**
    * @param {{swarm: object, groupLog?: number, timers?: object, requestTimeout?: number,
    *   window?: number}} opts
@@ -64,14 +73,20 @@ export class Swarmwire {
     requestTimeout = REQUEST_TIMEOUT,
     window = WINDOW
   }) {
+    super()
     this.swarm = swarm
     this.groupLog = groupLog
     this.timers = timers
     this.requestTimeout = requestTimeout
     this.window = window
+    this.random = Math.random // decides between equally rare groups; tests replace it with a seeded one
     this.serving = new Map() // root → ContentSource
     this.downloads = new Map() // root → Download
     this.peers = new Map() // peer id → Peer
+    this.holdings = new Holdings({ groupLog }) // what those peers say they hold
+    this.lists = new Map() // folder root → its file entries, once known
+    this._listings = new Map() // folder root → the list read in progress
+    this._sessions = new Map() // served root → its announce session, ended by unserve
 
     this._onconnection = (stream, info) => this._onPeer(stream, info)
     this.swarm.on('connection', this._onconnection)
@@ -91,10 +106,11 @@ export class Swarmwire {
     const existing = this.serving.get(source.root)
     if (existing && existing !== source) await source.close()
     else this.serving.set(source.root, source)
+    this._watch(source.root, source.size)
 
     // Wait for the announce to actually land. Until it has, a peer looking this content up
     // finds nobody, which is indistinguishable from "that link does not work".
-    const discovery = this.swarm.join(discoveryTopic(source.root, secret))
+    const discovery = this._announce(source.root, discoveryTopic(source.root, secret))
     if (typeof discovery?.flushed === 'function') await discovery.flushed()
 
     return {
@@ -126,12 +142,15 @@ export class Swarmwire {
       else await source.close()
 
       entries.push({ path: relative, root: source.root, size: source.size })
+      this._watch(source.root, source.size)
     }
 
     const manifest = MemorySource.from(encodeManifest(entries), { groupLog: this.groupLog })
     this.serving.set(manifest.root, manifest)
+    this.lists.set(manifest.root, entries)
+    this._watch(manifest.root, manifest.size)
 
-    const discovery = this.swarm.join(discoveryTopic(manifest.root, secret))
+    const discovery = this._announce(manifest.root, discoveryTopic(manifest.root, secret))
     if (typeof discovery?.flushed === 'function') await discovery.flushed()
 
     return {
@@ -143,7 +162,63 @@ export class Swarmwire {
   }
 
   /**
-   * Fetch a collection: the manifest first, then everything it names, into `dir`.
+   * Read a folder's file list without fetching any of it: each file's path, root and size, and
+   * the total. Nothing is written to disk and this node is never announced as a provider.
+   *
+   * @param {string|object} link - a folder link
+   * @returns {Listing} thenable, resolving to `{ files, size }`; `stop()` gives up waiting
+   */
+  readList (link) {
+    const parsed = typeof link === 'string' ? parseLink(link) : link
+    if (parsed.type !== 'collection') return Listing.failed(new Error('not a folder: that link is a single file'))
+
+    return this._list(parsed)
+  }
+
+  /**
+   * Fetch one file of a folder, using only the folder's link and an entry from its list. It is
+   * not announced on its own: the peers holding it are the ones connected through the folder.
+   *
+   * @param {string|object} link - the folder's link
+   * @param {{path: string, root: string, size: number}} file - an entry of `readList`'s result
+   * @param {string} outPath - where to write it; its directory must exist
+   * @returns {Download}
+   */
+  downloadFile (link, file, outPath) {
+    const { secret } = typeof link === 'string' ? parseLink(link) : link
+
+    return this.download(
+      { root: file.root, size: file.size, secret, name: file.path },
+      outPath,
+      { announce: false }
+    )
+  }
+
+  /** One read per folder at a time: asking again while one runs shares it (and its `stop`). */
+  _list (parsed) {
+    // shortcut: a folder's list is kept for the node's lifetime; evict if a node reads thousands of folders
+    const known = this.lists.get(parsed.root)
+    if (known) return Listing.known(known)
+
+    let listing = this._listings.get(parsed.root)
+    if (!listing) {
+      listing = new Listing(this, parsed)
+      this._listings.set(parsed.root, listing)
+      const done = () => this._listings.delete(parsed.root)
+      listing.then(done, done)
+    }
+
+    return listing
+  }
+
+  /** A folder's list became known: remember it, and ask peers what they hold of its files. */
+  _learnList (root, files) {
+    this.lists.set(root, files)
+    for (const file of files) this._watch(file.root, file.size)
+  }
+
+  /**
+   * Fetch a collection: the list first, then every file it names, into `dir`.
    *
    * @param {string|object} link
    * @param {string} dir
@@ -154,43 +229,27 @@ export class Swarmwire {
     const parsed = typeof link === 'string' ? parseLink(link) : link
 
     await fsx.mkdir(dir)
-    const files = await this._fetchManifest(parsed, dir)
+
+    // Reading the list only looks; downloading a folder also announces it, so whoever fetches
+    // it can be fetched from in turn.
+    this.swarm.join(discoveryTopic(parsed.root, parsed.secret), ANNOUNCE)
+
+    const list = await this._list(parsed)
+    if (!list) throw new Error('the folder list read was stopped')
 
     let bytes = 0
 
-    for (const file of files) {
+    for (const file of list.files) {
       const target = join(dir, file.path)
       await fsx.mkdir(parentOf(target))
 
-      // The files are not announced separately: the peers holding them are the ones already
-      // connected through the manifest's topic.
-      await this.download(
-        { root: file.root, size: file.size, secret: parsed.secret, name: file.path },
-        target,
-        { announce: false }
-      )
+      await this.downloadFile(parsed, file, target)
 
       bytes += file.size
       if (onfile) onfile({ path: file.path, size: file.size })
     }
 
-    return { files: files.length, bytes }
-  }
-
-  /** Download the manifest itself, read it, and leave nothing behind. */
-  async _fetchManifest (parsed, dir) {
-    const scratch = join(dir, '.swarmwire-manifest')
-
-    try {
-      await this.download({ ...parsed, name: null }, scratch)
-
-      const bytes = await fsx.readFile(scratch)
-      if (bytes === null) throw new Error('manifest vanished before it could be read')
-
-      return decodeManifest(bytes)
-    } finally {
-      await fsx.unlink(scratch)
-    }
+    return { files: list.files.length, bytes }
   }
 
   /** Stop serving content. The file itself is untouched. */
@@ -199,9 +258,54 @@ export class Swarmwire {
     if (!source) return
 
     this.serving.delete(root)
+    this.holdings.unwatch(root)
+    this.lists.delete(root)
+    await this._sessions.get(root)?.destroy?.()
+    this._sessions.delete(root)
     await source.close()
   }
 
+  _announce (root, topic) {
+    if (!this._sessions.has(root)) this._sessions.set(root, this.swarm.join(topic, ANNOUNCE))
+    return this._sessions.get(root)
+  }
+
+  /** Connected peers that completed the handshake, counting each distinct peer once. */
+  get peerCount () {
+    return this._ready().length
+  }
+
+  /**
+   * How many direct connections hold this content: all of it (seeders) or part of it
+   * (leechers). It describes who is connected to this node, not the swarm, and a peer that has
+   * said nothing about the content is counted in `peers` only. A folder's counts need its file
+   * list, which a node knows once it has served, read or downloaded it; until then they are 0.
+   *
+   * @param {string|object} link
+   * @returns {{peers: number, seeders: number, leechers: number}}
+   */
+  counts (link) {
+    const parsed = typeof link === 'string' ? parseLink(link) : link
+    const list = this.lists.get(parsed.root)
+
+    let roots = [parsed.root]
+    if (list) roots = [parsed.root, ...list.map((file) => file.root)]
+    else if (parsed.type === 'collection') roots = []
+
+    return this.holdings.count(this._ready(), roots)
+  }
+
+  _ready () {
+    const ids = []
+    for (const peer of this.peers.values()) if (peer.ready) ids.push(peer.id)
+    return ids
+  }
+
+  /** Ask every peer about a root we hold, unless a download already did. */
+  _watch (root, size) {
+    this.holdings.watch(root, size, this.peers.values(), (r) => this.downloads.has(r))
+  }
+
   /**
    * Fetch content by link. Returns immediately; await the result for completion, or
    * `await download.ready()` to know the destination has been opened.
@@ -210,19 +314,19 @@ export class Swarmwire {
    * @param {string} outPath
    * @returns {Download}
    */
-  download (link, outPath, { announce = true } = {}) {
+  download (link, outPath, { announce = true, destination = null } = {}) {
     const parsed = typeof link === 'string' ? parseLink(link) : link
 
     const existing = this.downloads.get(parsed.root)
     if (existing) return existing
 
-    const download = new Download(this, parsed, outPath, { announced: announce })
+    const download = new Download(this, parsed, outPath, { announced: announce, destination })
     this.downloads.set(parsed.root, download)
 
     // A file inside a collection is not announced on its own: whoever has it is already
     // connected through the collection's topic, and a folder of a thousand files should not
     // mean a thousand announces.
-    if (announce) this.swarm.join(discoveryTopic(parsed.root, parsed.secret))
+    if (announce) download._session = this.swarm.join(discoveryTopic(parsed.root, parsed.secret), ANNOUNCE)
 
     download._start()
 
@@ -242,8 +346,10 @@ export class Swarmwire {
 
   _onPeer (stream, info) {
     const id = peerId(info)
+    let peer = null
+
     const channel = openChannel(stream, {
-      onhandshake: () => {},
+      onhandshake: () => peer && this._onHandshake(peer),
       onwant: (message) => this._onWant(id, message),
       onhave: (message) => this._onHave(id, message),
       onrequest: (message) => this._onRequest(id, message),
@@ -251,12 +357,13 @@ export class Swarmwire {
       onunavailable: (message) => this._onUnavailable(id, message),
       onholds: (message) => this._onHolds(id, message),
       onreject: (message) => this._onReject(id, message),
-      onclose: () => this._onPeerGone(id)
+      onclose: () => peer && this._onPeerGone(peer)
     })
 
     if (channel === null) return
 
-    const peer = { id, send: channel.send, wants: new Set() }
+    // A second connection from the same peer replaces the first without announcing it again.
+    peer = { id, send: channel.send, wants: new Set(), ready: this.peers.get(id)?.ready === true }
     this.peers.set(id, peer)
 
     peer.send.handshake({ version: VERSION, groupLog: this.groupLog })
@@ -264,14 +371,26 @@ export class Swarmwire {
     // Tell a new peer what we are after. A peer that has it answers; one that does not stays
     // quiet and is simply never asked.
     for (const download of this.downloads.values()) download._askPeer(peer)
+    this.holdings.wantAll(peer, (root) => this.downloads.has(root))
   }
 
-  _onPeerGone (id) {
-    const peer = this.peers.get(id)
-    if (!peer) return
+  /** The remote spoke swarmwire: from here on it is a peer, not just a connection. */
+  _onHandshake (peer) {
+    if (peer.ready) return
+
+    peer.ready = true
+    this.emit('peer-connected', { id: peer.id })
+  }
+
+  _onPeerGone (peer) {
+    // The connection that closed may be one a newer connection from the same peer replaced.
+    if (this.peers.get(peer.id) !== peer) return
 
-    this.peers.delete(id)
+    this.peers.delete(peer.id)
+    this.holdings.dropPeer(peer.id)
     for (const download of this.downloads.values()) download._peerGone(peer)
+
+    if (peer.ready) this.emit('peer-disconnected', { id: peer.id })
   }
 
   _onWant (id, { root }) {
@@ -348,7 +467,9 @@ export class Swarmwire {
     if (this.serving.has(root)) return
 
     try {
-      this.serving.set(root, await ContentSource.adopt(path, tree))
+      const source = await ContentSource.adopt(path, tree)
+      this.serving.set(root, source)
+      this._watch(root, source.size) // peers that connect from now on are asked what they hold
     } catch {
       // The file was moved or deleted between finishing and adopting it. Nothing to serve.
     }
@@ -364,9 +485,14 @@ export class Swarmwire {
   }
 
   _onHolds (id, message) {
-    const download = this.downloads.get(hex(message.root))
+    const rootHex = hex(message.root)
+    const download = this.downloads.get(rootHex)
     const peer = this.peers.get(id)
-    if (download && peer) download._onHolds(peer, message)
+    if (!peer) return
+
+    const size = this.serving.get(rootHex)?.size ?? download?.size ?? this.holdings.watched.get(rootHex)
+    this.holdings.grow(id, rootHex, message.index, size ?? undefined)
+    download?._onHolds(peer, message)
   }
 
   _onReject (id, message) {
@@ -376,9 +502,13 @@ export class Swarmwire {
   }
 
   _onHave (id, message) {
-    const download = this.downloads.get(hex(message.root))
+    const rootHex = hex(message.root)
+    const download = this.downloads.get(rootHex)
     const peer = this.peers.get(id)
-    if (download && peer) download._onHave(peer, message)
+    if (!peer) return
+
+    this.holdings.set(id, rootHex, message)
+    download?._onHave(peer, message)
   }
 
   _onData (id, message) {
@@ -388,9 +518,13 @@ export class Swarmwire {
   }
 
   _onUnavailable (id, message) {
-    const download = this.downloads.get(hex(message.root))
+    const rootHex = hex(message.root)
+    const download = this.downloads.get(rootHex)
     const peer = this.peers.get(id)
-    if (download && peer) download._sources.delete(peer.id)
+    if (!peer) return
+
+    this.holdings.remove(id, rootHex)
+    download?._sources.delete(peer.id)
   }
 }
 
@@ -398,10 +532,12 @@ export class Swarmwire {
  * One transfer in progress. Thenable, so `await download` waits for it to finish.
  */
 export class Download extends EventEmitter {
-  constructor (node, { root, size, secret }, outPath, { announced = true } = {}) {
+  constructor (node, { root, size, secret }, outPath, { announced = true, destination = null } = {}) {
     super()
 
     this.announced = announced
+    this._session = null // the announce session, when there is one
+    this._destinationFactory = destination // where the content lands, when not a file at `outPath`
 
     this.node = node
     this.root = root
@@ -418,6 +554,7 @@ export class Download extends EventEmitter {
 
     this._destination = null
     this._bitfields = new Map() // peer id -> what that peer says it holds
+    this._availability = null // how many of those hold each group; built once the size is known
     this._sources = new Set() // peer ids worth asking right now
     this._liars = new Set() // peer ids that sent something that did not verify: never again
     this._asked = new Set() // peer ids already told what we want
@@ -497,7 +634,7 @@ export class Download extends EventEmitter {
 
     this._cancelEverything()
     this.node.downloads.delete(this.root)
-    if (this.announced) this.node.swarm.leave?.(discoveryTopic(this.root, this.secret))
+    await this._session?.destroy?.() // only ours: the same topic may be served or read elsewhere
 
     if (this._destination) await this._destination.close()
 
@@ -530,17 +667,18 @@ export class Download extends EventEmitter {
   async _open () {
     if (this._destination) return
 
-    const destination = await Destination.open(this.path, {
-      root: this.root,
-      size: this.size,
-      groupLog: this.node.groupLog
-    })
+    const options = { root: this.root, size: this.size, groupLog: this.node.groupLog }
+    const destination = this._destinationFactory
+      ? await this._destinationFactory(options)
+      : await Destination.open(this.path, options)
 
     // The download can be stopped while the file is still opening. Nobody would ever close
     // this handle otherwise.
     if (this.stopped) return destination.close()
 
     this._destination = destination
+    this._availability = new Availability(destination.groupCount)
+    for (const bitfield of this._bitfields.values()) this._availability.add(bitfield)
     if (this._destination.complete) return this._finish()
 
     for (const peer of this.node.peers.values()) this._askPeer(peer)
@@ -559,7 +697,7 @@ export class Download extends EventEmitter {
 
     if (this._liars.has(peer.id)) return // nothing it says is worth storing
 
-    this._bitfields.set(peer.id, b4a.from(bitfield ?? []))
+    this._setBitfield(peer.id, b4a.from(bitfield ?? []))
     this._sources.add(peer.id)
 
     if (this.size === null) {
@@ -603,7 +741,7 @@ export class Download extends EventEmitter {
       this.emit('rejected', { index, from: peer.id })
       this._liars.add(peer.id) // a peer that sends rubbish is never asked again
       this._sources.delete(peer.id)
-      this._bitfields.delete(peer.id)
+      this._dropBitfield(peer.id)
     }
 
     if (this.complete) return this._finish()
@@ -636,12 +774,20 @@ export class Download extends EventEmitter {
 
     const requests = this._requests(peer)
     const window = this._windowFor(peer)
-    const wanted = rarestFirst(this._destination.missing(), (index) => this._holders(index))
+    const destination = this._destination
+    const random = this.node.random
+
+    const pool = candidates({
+      count: destination.groupCount,
+      has: (index) => destination.has(index),
+      peerHas: (index) => this._peerHas(peer.id, index),
+      claimed: (index) => this._claimed(index),
+      from: Math.floor(random() * destination.groupCount)
+    })
+    const wanted = rank(pool, this._availability, random)
 
     for (const index of wanted) {
       if (requests.size >= window) break
-      if (this._claimed(index)) continue
-      if (!this._peerHas(peer.id, index)) continue // it has not got that one yet
 
       this._claim(peer, index)
       peer.send.request({ root: b4a.from(this.root, 'hex'), index })
@@ -685,13 +831,17 @@ export class Download extends EventEmitter {
     this._windows.set(peer.id, next)
   }
 
-  /** How many connected peers hold this group. Drives rarest-first selection. */
-  _holders (index) {
-    let n = 0
-    for (const peerId of this._sources) {
-      if (this._peerHas(peerId, index)) n++
-    }
-    return n
+  /** Remember what a peer holds, keeping the per-group counts in step. */
+  _setBitfield (peerId, bitfield) {
+    this._dropBitfield(peerId)
+    this._bitfields.set(peerId, bitfield)
+    this._availability?.add(bitfield)
+  }
+
+  _dropBitfield (peerId) {
+    const old = this._bitfields.get(peerId)
+    if (old) this._availability?.remove(old)
+    this._bitfields.delete(peerId)
   }
 
   /** Does this peer hold that group? An empty bitfield means we were told nothing useful. */
@@ -714,7 +864,10 @@ export class Download extends EventEmitter {
     this._narrow(peer)
 
     const bitfield = this._bitfields.get(peer.id)
-    if (bitfield && index < bitfield.length * 8) bao.clearBit(bitfield, index)
+    if (bitfield && index < bitfield.length * 8 && bao.getBit(bitfield, index)) {
+      bao.clearBit(bitfield, index)
+      this._availability?.unset(index)
+    }
 
     this._pumpAll()
   }
@@ -724,7 +877,10 @@ export class Download extends EventEmitter {
     const bitfield = this._bitfields.get(peer.id)
     if (!bitfield || index >= bitfield.length * 8) return
 
-    bao.setBit(bitfield, index)
+    if (!bao.getBit(bitfield, index)) {
+      bao.setBit(bitfield, index)
+      this._availability?.set(index)
+    }
     this._pump(peer)
   }
 
@@ -772,7 +928,7 @@ export class Download extends EventEmitter {
 
   _peerGone (peer) {
     this._sources.delete(peer.id)
-    this._bitfields.delete(peer.id)
+    this._dropBitfield(peer.id)
     this._asked.delete(peer.id)
     this._releaseAll(peer.id) // whatever it owed us is up for grabs again
 
@@ -792,7 +948,7 @@ export class Download extends EventEmitter {
     const tree = this._destination.tree
 
     await this._destination.finish()
-    await this.node._adopt(this.root, this.path, tree)
+    if (!this._destinationFactory) await this.node._adopt(this.root, this.path, tree)
 
     this.emit('progress', { have: this.have, total: this.total, from: null })
     this._resolve()

package/lib/wire.js

@@ -1,7 +1,7 @@
 /**
  * The wire: what peers say to each other, and nothing more.
  *
- * Six messages. A peer asks who has some content, holders say so, the asker requests groups
+ * Eight messages. A peer asks who has some content, holders say so, the asker requests groups
  * one at a time, and each answer carries the proof that makes it checkable on arrival.
  * Scheduling, windows and peer choice live in swarmwire.js — none of that is on the wire.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "swarmwire-lib",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Content-addressed, verified file transfer between filesystems over Hyperswarm. No copies, no cores, no trackers.",
   "type": "module",
   "main": "index.js",
@@ -67,6 +67,7 @@
     "test:bare:net": "bare node_modules/brittle/bin/bare.js test/net/*.test.js",
     "bench": "node bench/swarm.js",
     "bench:net": "node bench/throughput.js",
+    "bench:sim": "node bench/swarm-sim.js",
     "check:pack": "node scripts/check-pack.js"
   },
   "dependencies": {
@@ -87,7 +88,8 @@
     "bare-process": "^4.2.4",
     "bare-subprocess": "^5.0.0",
     "bare-url": "^2.1.7",
-    "brittle": "^3.7.0"
+    "brittle": "^3.7.0",
+    "hyperdht": "^6.34.1"
   },
   "engines": {
     "bare": ">=1.16",