swarmwire-lib 0.3.0 → 0.4.1

package/README.md

@@ -55,8 +55,9 @@ The command-line tool built on this library is moving to its own package,
 
 > Status: **pre-alpha.** Transfers work end to end, over a real swarm, with resume,
 > verification, request timeouts, partial seeding, rarest-first selection and an adaptive
-> request window. Behaviour with very many peers is not measured yet. See
-> [docs/roadmap.md](docs/roadmap.md) for what is built and what is next.
+> request window. Behaviour with very many peers is only modelled, not measured. It is enough
+> for a command-line tool; it is not yet the content-addressing layer SwarmFS needs (admission,
+> peer hints, no public discovery). [docs/roadmap.md](docs/roadmap.md) says where it stands.
 
 ## What it is
 
@@ -110,7 +111,9 @@ Anything that needs a name, a directory or a user belongs upstairs. See
 |------------------------------------|-----------------------------------------------------------------------------------------------|
 | [docs/format.md](docs/format.md)   | Addressing, verification, links, topic derivation — the parts that must never change casually |
 | [docs/contract.md](docs/contract.md) | What applications can rely on, supply and must not expect                                  |
-| [docs/roadmap.md](docs/roadmap.md) | Vision and milestones                                                                         |
+| [docs/roadmap.md](docs/roadmap.md) | Vision, milestones, and where things stand                                                    |
+| [docs/scaling.md](docs/scaling.md) | Piece selection in large swarms, simulated                                                    |
+| [docs/research/p2p-landscape.md](docs/research/p2p-landscape.md) | What other P2P systems do, and what is worth taking                  |
 | [AGENTS.md](AGENTS.md)             | How to work in this repo                                                                      |
 
 ## License

package/lib/fs.js

@@ -8,12 +8,25 @@
 import fs from 'fs'
 import b4a from 'b4a'
 
+/**
+ * `fd:N` names an already-open descriptor instead of a path (Android hands out descriptors
+ * for picked files, with no path). Read-only, files only; the caller keeps it open.
+ * shortcut: sync fstat/read on the fd, upgrade if the blocking reads ever show up in profiles.
+ */
+const fdOf = (path) => (/^fd:\d+$/.test(path) ? Number(path.slice(3)) : null)
+
 /**
  * @param {string} path
  * @returns {Promise<{size: number, mtimeMs: number, directory: boolean}|null>} null if it
  *   does not exist
  */
 export async function stat (path) {
+  const fd = fdOf(path)
+  if (fd !== null) {
+    const st = fs.fstatSync(fd)
+    return { size: st.size, mtimeMs: Math.floor(st.mtimeMs), directory: false }
+  }
+
   try {
     const st = await fs.promises.stat(path)
     return { size: st.size, mtimeMs: Math.floor(st.mtimeMs), directory: st.isDirectory() }
@@ -25,6 +38,19 @@ export async function stat (path) {
 
 /** Open a file for reading byte ranges out of it, in place. */
 export async function openRead (path) {
+  const fd = fdOf(path)
+  if (fd !== null) {
+    return {
+      async read (offset, length) {
+        const buffer = b4a.alloc(length)
+        const bytesRead = length === 0 ? 0 : fs.readSync(fd, buffer, 0, length, offset)
+        return bytesRead === length ? buffer : buffer.subarray(0, bytesRead)
+      },
+
+      close () {} // not ours to close
+    }
+  }
+
   const handle = await fs.promises.open(path, 'r')
 
   return {

package/lib/swarmwire.js

@@ -18,7 +18,7 @@ import { ContentSource, MemorySource } from './source.js'
 import { encodeManifest, decodeManifest, totalSize } from './collection.js'
 import { Destination } from './destination.js'
 import { discoveryTopic, formatLink, parseLink } from './address.js'
-import { DEFAULT_GROUP_LOG, groupCount } from './tree.js'
+import { DEFAULT_GROUP_LOG, groupCount, groupLength } from './tree.js'
 import { openChannel, VERSION } from './wire.js'
 import { Availability, candidates, rank } from './select.js'
 import { Holdings } from './holdings.js'
@@ -222,10 +222,13 @@ export class Swarmwire extends EventEmitter {
    *
    * @param {string|object} link
    * @param {string} dir
-   * @param {{onfile?: Function}} [opts]
+   * `onfile({ path, size })` runs as each file completes; `onprogress({ bytes, total })` as bytes of
+   * the whole folder arrive, first with `bytes: 0` once the list is known.
+   *
+   * @param {{onfile?: Function, onprogress?: Function}} [opts]
    * @returns {Promise<{files: number, bytes: number}>}
    */
-  async downloadCollection (link, dir, { onfile = null } = {}) {
+  async downloadCollection (link, dir, { onfile = null, onprogress = null } = {}) {
     const parsed = typeof link === 'string' ? parseLink(link) : link
 
     await fsx.mkdir(dir)
@@ -238,14 +241,21 @@ export class Swarmwire extends EventEmitter {
     if (!list) throw new Error('the folder list read was stopped')
 
     let bytes = 0
+    const total = list.size
+    onprogress?.({ bytes, total })
 
     for (const file of list.files) {
       const target = join(dir, file.path)
       await fsx.mkdir(parentOf(target))
 
-      await this.downloadFile(parsed, file, target)
+      const download = this.downloadFile(parsed, file, target)
+      download.on('progress', ({ have }) => {
+        onprogress?.({ bytes: bytes + Math.min(have * groupLength(this.groupLog), file.size), total })
+      })
+      await download
 
       bytes += file.size
+      onprogress?.({ bytes, total })
       if (onfile) onfile({ path: file.path, size: file.size })
     }
 

package/lib/tree.js

@@ -129,6 +129,7 @@ export class GroupTree {
     this.groupHashes = groupHashes
     this.size = size
     this.groupLog = groupLog
+    this._known = new Map() // 'start:end' → hash of a larger subtree; a hash once known never changes
   }
 
   /** Hash content held in memory. For files, stream it through GroupTreeBuilder instead. */
@@ -162,7 +163,7 @@ export class GroupTree {
     if (!this.complete) return null
     if (this.groupCount === 1) return b4a.toString(this.groupHashes[0], 'hex')
 
-    return b4a.toString(subtree(this.groupHashes, 0, this.groupCount, true), 'hex')
+    return b4a.toString(subtree(this.groupHashes, 0, this.groupCount, true, this._known), 'hex')
   }
 
   /**
@@ -186,8 +187,8 @@ export class GroupTree {
     while (end - start > 1) {
       const mid = start + splitPoint(end - start)
       const sibling = index < mid
-        ? subtree(this.groupHashes, mid, end, false)
-        : subtree(this.groupHashes, start, mid, false)
+        ? subtree(this.groupHashes, mid, end, false, this._known)
+        : subtree(this.groupHashes, start, mid, false, this._known)
 
       if (sibling === null) return null
       siblings.push(sibling)
@@ -259,18 +260,28 @@ export function splitPoint (count) {
   return p
 }
 
-/** Null when any leaf under this subtree is unknown. */
-function subtree (hashes, start, end, isRoot) {
+// shortcut: only subtrees of at least this many groups are remembered, so the memory is about n/16
+// hashes and a proof recomputes at most ~15 per level; lower it if proofs ever show up in a profile.
+const REMEMBER = 16
+
+/** Null when any leaf under this subtree is unknown. `known` remembers the bigger ones. */
+function subtree (hashes, start, end, isRoot, known) {
   if (end - start === 1) return hashes[start] ?? null
 
+  const key = !isRoot && end - start >= REMEMBER ? start + ':' + end : null
+  const remembered = key && known.get(key)
+  if (remembered) return remembered
+
   const mid = start + splitPoint(end - start)
-  const left = subtree(hashes, start, mid, false)
+  const left = subtree(hashes, start, mid, false, known)
   if (left === null) return null
 
-  const right = subtree(hashes, mid, end, false)
+  const right = subtree(hashes, mid, end, false, known)
   if (right === null) return null
 
-  return bao.parentCV(left, right, isRoot)
+  const hash = bao.parentCV(left, right, isRoot)
+  if (key) known.set(key, hash)
+  return hash
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "swarmwire-lib",
-  "version": "0.3.0",
+  "version": "0.4.1",
   "description": "Content-addressed, verified file transfer between filesystems over Hyperswarm. No copies, no cores, no trackers.",
   "type": "module",
   "main": "index.js",