swarmwire-lib 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +137 -2
- package/docs/contract.md +121 -0
- package/docs/format.md +180 -0
- package/index.js +12 -0
- package/lib/address.js +166 -0
- package/lib/blake3.cjs +15 -0
- package/lib/blake3.js +12 -0
- package/lib/collection.js +133 -0
- package/lib/destination.js +237 -0
- package/lib/fs.js +176 -0
- package/lib/nodes.js +118 -0
- package/lib/partfile.js +169 -0
- package/lib/select.js +33 -0
- package/lib/source.js +171 -0
- package/lib/swarmwire.js +841 -0
- package/lib/timers.js +21 -0
- package/lib/tree.js +326 -0
- package/lib/wire.js +95 -0
- package/package.json +94 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tibotibotibotibotibotibotibotibotibotibotibo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,138 @@
|
|
|
1
|
-
#
|
|
1
|
+
# swarmwire
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**Content-addressed file transfer over Hyperswarm.**
|
|
4
|
+
|
|
5
|
+
> Transfers directly from one's filesystem to another, no need to duplicate the data in a hyperblob
|
|
6
|
+
|
|
7
|
+
Give someone a link, they get the bytes — from you, or from anyone else who already has them.
|
|
8
|
+
Files are served in place, from wherever they already live on disk. Nothing is copied into a
|
|
9
|
+
store, a core, or a cache first.
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
npm i swarmwire-lib
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
One side serves a file and prints a link. The other side hands the link to `download`. (This is
|
|
16
|
+
`examples/send-get.js`; in the repository it imports `../index.js` instead of `swarmwire-lib`. It is
|
|
17
|
+
an ES module, so your project needs `"type": "module"` or a `.mjs` file.)
|
|
18
|
+
|
|
19
|
+
```js
|
|
20
|
+
/**
|
|
21
|
+
* Send a file to someone, or get one.
|
|
22
|
+
*
|
|
23
|
+
* node examples/send-get.js send ./holidays.mp4
|
|
24
|
+
* node examples/send-get.js get 'swarmwire://…' ./holidays.mp4
|
|
25
|
+
*
|
|
26
|
+
* `--bootstrap host:port` points both sides at a private DHT; leave it out for the real one.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import process from 'process'
|
|
30
|
+
import Hyperswarm from 'hyperswarm'
|
|
31
|
+
import DHT from 'hyperdht'
|
|
32
|
+
|
|
33
|
+
import { Swarmwire } from 'swarmwire-lib'
|
|
34
|
+
|
|
35
|
+
const args = process.argv.slice(2)
|
|
36
|
+
const flag = args.indexOf('--bootstrap')
|
|
37
|
+
const bootstrap = flag === -1 ? null : args.splice(flag, 2)[1]
|
|
38
|
+
const [mode, target, out] = args
|
|
39
|
+
|
|
40
|
+
const swarm = bootstrap ? privateSwarm(bootstrap) : new Hyperswarm()
|
|
41
|
+
const node = new Swarmwire({ swarm })
|
|
42
|
+
|
|
43
|
+
if (mode === 'send') {
|
|
44
|
+
const { link } = await node.serve(target)
|
|
45
|
+
console.log(link) // hand this to the other side; keep running while they fetch
|
|
46
|
+
await new Promise((resolve) => process.once('SIGINT', resolve))
|
|
47
|
+
} else if (mode === 'get') {
|
|
48
|
+
await node.download(target, out)
|
|
49
|
+
console.log(out)
|
|
50
|
+
} else {
|
|
51
|
+
console.log('usage: send <file> | get <link> <out> [--bootstrap host:port]')
|
|
52
|
+
process.exitCode = 1
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
await node.destroy()
|
|
56
|
+
await swarm.destroy()
|
|
57
|
+
|
|
58
|
+
// A loopback DHT only works if the node binds there and is not firewalled.
|
|
59
|
+
function privateSwarm (address) {
|
|
60
|
+
const [host, port] = address.split(':')
|
|
61
|
+
const dht = new DHT({ bootstrap: [{ host, port: Number(port) }], ephemeral: false, firewalled: false, host: '127.0.0.1' })
|
|
62
|
+
return new Hyperswarm({ dht })
|
|
63
|
+
}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
$ node examples/send-get.js send ./holidays.mp4
|
|
68
|
+
swarmwire://10db6398a68dff502d002912f54502fdde9ab55fb30bec32861a2f004d874fc8?s=734003200&n=holidays.mp4
|
|
69
|
+
|
|
70
|
+
$ node examples/send-get.js get 'swarmwire://10db63…' ./holidays.mp4
|
|
71
|
+
./holidays.mp4
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
A folder goes the same way, as one link: `serveCollection` and `downloadCollection`.
|
|
75
|
+
The command-line tool built on this library is moving to its own package, `swarmwire-cli`.
|
|
76
|
+
|
|
77
|
+
> Status: **pre-alpha.** Transfers work end to end, over a real swarm, with resume and
|
|
78
|
+
> verification. Multi-peer scheduling is still naive and there are no request timeouts yet —
|
|
79
|
+
> see [docs/roadmap.md](docs/roadmap.md) for what M2 adds.
|
|
80
|
+
|
|
81
|
+
## What it is
|
|
82
|
+
|
|
83
|
+
A library that moves bytes between peers:
|
|
84
|
+
|
|
85
|
+
- **Content-addressed.** A file's address is its BLAKE3 hash — the same value `b3sum` prints.
|
|
86
|
+
You ask for content, never for a peer, so any peer holding it can answer.
|
|
87
|
+
- **Verified in transit.** Every received range is checked against the root hash before it
|
|
88
|
+
touches the destination file, using [bao](https://github.com/oconnor663/bao) verified
|
|
89
|
+
streaming. A lying peer is caught on its first bad range, not at the end of the download.
|
|
90
|
+
- **Served in place.** A shared file is read from its existing path. There is no import step,
|
|
91
|
+
no second copy, no storage layer that owns your data.
|
|
92
|
+
- **Swarm-shaped.** Many peers per transfer, partial seeders included, like BitTorrent —
|
|
93
|
+
but with Hyperswarm's DHT and holepunching instead of trackers.
|
|
94
|
+
|
|
95
|
+
## What it is not
|
|
96
|
+
|
|
97
|
+
- **Not a filesystem.** Names, directories, mutation, sharing and permissions live one layer
|
|
98
|
+
up, in SwarmFS. swarmwire only knows roots and bytes.
|
|
99
|
+
- **Not a hypercore.** It never writes your content into a core, a blob store or a database.
|
|
100
|
+
(Hypercore/hypergraph are excellent at distributing the *references* — that is SwarmFS's job,
|
|
101
|
+
not this one.)
|
|
102
|
+
- **Not BitTorrent-compatible.** It borrows BitTorrent's swarm algorithms, not its wire format.
|
|
103
|
+
- **Not a live-stream transport.** Addresses name fixed content. Open-ended streams need an
|
|
104
|
+
append-only log; that is what hypercore is for.
|
|
105
|
+
|
|
106
|
+
## Runtimes
|
|
107
|
+
|
|
108
|
+
Node and Bare, with the same test suite passing on both — Bare because that is what mobile
|
|
109
|
+
builds run on. See [AGENTS.md](AGENTS.md) for what that costs in practice.
|
|
110
|
+
|
|
111
|
+
## Layering
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
SwarmFS virtual filesystem: names, directories, multi-writer, permissions
|
|
115
|
+
│ (hypergraph/hypercore hold the references)
|
|
116
|
+
▼
|
|
117
|
+
swarmwire roots → bytes: discovery, verified transfer, in-place serving
|
|
118
|
+
│
|
|
119
|
+
▼
|
|
120
|
+
Hyperswarm DHT discovery, holepunching, encrypted streams
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
The dependency runs one way. swarmwire must never import SwarmFS, hypergraph or hypercore.
|
|
124
|
+
Anything that needs a name, a directory or a user belongs upstairs. See
|
|
125
|
+
[`ECOSYSTEM.md`](../hypergraph/ECOSYSTEM.md) for how the wider family fits together.
|
|
126
|
+
|
|
127
|
+
## Documentation
|
|
128
|
+
|
|
129
|
+
| Document | What's in it |
|
|
130
|
+
|------------------------------------|-----------------------------------------------------------------------------------------------|
|
|
131
|
+
| [docs/format.md](docs/format.md) | Addressing, verification, links, topic derivation — the parts that must never change casually |
|
|
132
|
+
| [docs/contract.md](docs/contract.md) | What applications can rely on, supply and must not expect |
|
|
133
|
+
| [docs/roadmap.md](docs/roadmap.md) | Vision and milestones |
|
|
134
|
+
| [AGENTS.md](AGENTS.md) | How to work in this repo |
|
|
135
|
+
|
|
136
|
+
## License
|
|
137
|
+
|
|
138
|
+
MIT
|
package/docs/contract.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# swarmwire contract
|
|
2
|
+
|
|
3
|
+
What an application built on `swarmwire-lib` (SwarmFS, `swarmwire-cli`, anything else) can rely on,
|
|
4
|
+
what it can supply, and what swarmwire will never do. Only what is listed here is stable.
|
|
5
|
+
Version 0.x: a minor release may change anything marked *exists* if this document and the
|
|
6
|
+
changelog say so; anything marked *planned* is a direction, not a promise.
|
|
7
|
+
|
|
8
|
+
## 1. Summary
|
|
9
|
+
|
|
10
|
+
swarmwire takes a content root (a plain BLAKE3 hash) and returns verified bytes on disk, from
|
|
11
|
+
whichever connected peers hold them. The application owns the network connection, keys, names,
|
|
12
|
+
directories, users and permissions. A "folder" in this document is called a *collection* in the
|
|
13
|
+
code (`serveCollection`, `downloadCollection`). See `AGENTS.md` for the invariants this contract rests on.
|
|
14
|
+
|
|
15
|
+
## 2. Capabilities
|
|
16
|
+
|
|
17
|
+
| Capability | Status | Delivered by | Proof |
|
|
18
|
+
|---|---|---|---|
|
|
19
|
+
| Serve a file in place, download by link, resume, verify every range | exists | - | `test/swarm/transfer.test.js` |
|
|
20
|
+
| Serve and download a folder as one link | exists | - | `test/swarm/collection.test.js` |
|
|
21
|
+
| Partial seeding: a download in progress serves what it holds | exists | - | `test/swarm/partial.test.js` |
|
|
22
|
+
| Application supplies the network connection (any object with `join`, `leave`, a `connection` event) | exists | - | `test/helpers/network.js`, `test/net/real-swarm.test.js` |
|
|
23
|
+
| Peer identity: the remote public key of the connection | exists | - | `test/swarm/transfer.test.js` |
|
|
24
|
+
| Peer hints (preferred peers, priorities) | planned | 002 | - |
|
|
25
|
+
| Admission decisions | planned | 002 | - |
|
|
26
|
+
| Peer activity events and counts | planned | 002 | - |
|
|
27
|
+
| Folder file-list reader, public | planned | 002 | - |
|
|
28
|
+
|
|
29
|
+
## 3. Rules the application can rely on
|
|
30
|
+
|
|
31
|
+
### 3.1 Hints add options, never correctness
|
|
32
|
+
|
|
33
|
+
A transfer completes correctly when every hint is wrong or stale. A hint cannot stall a transfer.
|
|
34
|
+
A **peer hint** only changes which already-connected peers are asked first; swarmwire never dials
|
|
35
|
+
a peer because of a hint. The application dials through the connection it owns.
|
|
36
|
+
|
|
37
|
+
### 3.2 Admission
|
|
38
|
+
|
|
39
|
+
The application can decide, per peer and per content, whether content is served.
|
|
40
|
+
|
|
41
|
+
- Decided **per content root**. Each file of a folder is its own root and is decided separately;
|
|
42
|
+
the decision is told which folder root the file was requested through, if any.
|
|
43
|
+
- **Deny on failure**: a decision that throws, times out or never answers is a deny.
|
|
44
|
+
- A deny is answered **exactly as content swarmwire does not hold**. A refused peer learns nothing
|
|
45
|
+
about what is held, its size or its name.
|
|
46
|
+
- With no decision installed, content is served as it is today.
|
|
47
|
+
- A folder where some files are allowed and some denied transfers the allowed ones; the denied
|
|
48
|
+
ones read as absent, and the folder download reports which were unavailable.
|
|
49
|
+
|
|
50
|
+
### 3.3 Peer activity
|
|
51
|
+
|
|
52
|
+
Events when a peer connects and disconnects, and, for one content root, the number of connected
|
|
53
|
+
peers and how many hold all of it. **Direct connections only**: a snapshot that may lag, not a
|
|
54
|
+
count of the swarm. Global counts and chunk rarity across the network are a tracker's job, not
|
|
55
|
+
swarmwire's.
|
|
56
|
+
|
|
57
|
+
### 3.4 Identity
|
|
58
|
+
|
|
59
|
+
swarmwire knows the remote public key of each connection and nothing else about a peer. Tying a
|
|
60
|
+
key to a user or device is the application's job (Principle II).
|
|
61
|
+
|
|
62
|
+
### 3.5 Choosing files of a folder
|
|
63
|
+
|
|
64
|
+
Read the folder's file list without fetching content, then download the chosen files one by one.
|
|
65
|
+
Which files, and any interface for choosing, belong to the application. *Planned: the list reader
|
|
66
|
+
is not public yet.*
|
|
67
|
+
|
|
68
|
+
### 3.6 The shared connection
|
|
69
|
+
|
|
70
|
+
The application creates and destroys the swarm, owns its keypair, and may use the same connections
|
|
71
|
+
for other protocols. swarmwire attaches to the `connection` event and leaves the connection alone
|
|
72
|
+
when it is destroyed.
|
|
73
|
+
|
|
74
|
+
## 4. What swarmwire will not do
|
|
75
|
+
|
|
76
|
+
| Not in swarmwire | Where it lives |
|
|
77
|
+
|---|---|
|
|
78
|
+
| Names, directories, mutable references | SwarmFS |
|
|
79
|
+
| Users, devices, permissions (it only asks, via admission) | Hypergraph, SwarmFS |
|
|
80
|
+
| A tracker, global peer counts, global chunk rarity | SwarmFS (coarse, per file), not per chunk |
|
|
81
|
+
| Choosing which files of a folder to download | SwarmFS or the CLI |
|
|
82
|
+
| Storing content in a core, blob store or database | never |
|
|
83
|
+
|
|
84
|
+
## 5. Exports
|
|
85
|
+
|
|
86
|
+
The importable surface of `swarmwire-lib`. A test checks every name below exists.
|
|
87
|
+
|
|
88
|
+
```exports
|
|
89
|
+
Swarmwire
|
|
90
|
+
Download
|
|
91
|
+
rootOf
|
|
92
|
+
formatLink
|
|
93
|
+
parseLink
|
|
94
|
+
discoveryTopic
|
|
95
|
+
swarmTopic
|
|
96
|
+
SCHEME
|
|
97
|
+
ContentSource
|
|
98
|
+
MemorySource
|
|
99
|
+
encodeManifest
|
|
100
|
+
decodeManifest
|
|
101
|
+
isSafePath
|
|
102
|
+
MANIFEST_TYPE
|
|
103
|
+
Destination
|
|
104
|
+
verifyGroup
|
|
105
|
+
groupCount
|
|
106
|
+
groupRange
|
|
107
|
+
groupLength
|
|
108
|
+
DEFAULT_GROUP_LOG
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**Stable** (changes bump the version and are in the changelog): `Swarmwire`
|
|
112
|
+
(`serve`, `serveCollection`, `download`, `downloadCollection`, `unserve`, `destroy`), `Download`
|
|
113
|
+
(awaitable; events `progress`, `rejected`, `timeout`, `stalled`), `rootOf`, `formatLink`,
|
|
114
|
+
`parseLink`, `SCHEME`.
|
|
115
|
+
|
|
116
|
+
**Exported for tests and advanced use, may change in any 0.x**: everything else.
|
|
117
|
+
|
|
118
|
+
## 6. Wire format
|
|
119
|
+
|
|
120
|
+
Described in `docs/format.md`. Changes only deliberately; unknown messages are ignored so old
|
|
121
|
+
peers survive new ones.
|
package/docs/format.md
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# The swarmwire format — v0
|
|
2
|
+
|
|
3
|
+
Everything here ends up inside links that people share and store. Links must keep working after
|
|
4
|
+
the engine has been rewritten three times, so this document changes slowly and deliberately,
|
|
5
|
+
and everything in it is pinned by tests in `test/unit/`.
|
|
6
|
+
|
|
7
|
+
## 1. Addressing
|
|
8
|
+
|
|
9
|
+
**A content address is the 32-byte BLAKE3 hash of the content.** Nothing custom, nothing
|
|
10
|
+
swarmwire-specific: it is the value `b3sum` prints, and the value any other BLAKE3 tool agrees
|
|
11
|
+
on.
|
|
12
|
+
|
|
13
|
+
Two consequences worth stating plainly:
|
|
14
|
+
|
|
15
|
+
- **The same bytes always have the same address**, whoever hashed them, with whatever tool, at
|
|
16
|
+
whatever chunk size. Deduplication across peers and across apps is free.
|
|
17
|
+
- **The address commits to the content's length.** BLAKE3's tree shape depends on the length, so
|
|
18
|
+
truncation and extension attacks are not possible. (SwarmFS's hand-rolled tree duplicated the
|
|
19
|
+
last node of an odd level, which made `[a,b,c]` and `[a,b,c,c]` collide. Dropping the custom
|
|
20
|
+
tree drops the bug.)
|
|
21
|
+
|
|
22
|
+
Size and filename are **hints**, never part of the address. A peer may lie about either; the
|
|
23
|
+
hash is what is enforced.
|
|
24
|
+
|
|
25
|
+
## 2. Verification
|
|
26
|
+
|
|
27
|
+
Ranges are verified with [bao](https://github.com/oconnor663/bao) verified streaming, as
|
|
28
|
+
implemented by the pure-JavaScript `blake3-bao`.
|
|
29
|
+
|
|
30
|
+
- A sender keeps an **outboard** tree for the content it serves — hashes only, no data copy.
|
|
31
|
+
- For a requested byte range, the sender produces a **slice**: the range plus the minimal set of
|
|
32
|
+
tree nodes proving it belongs to the root.
|
|
33
|
+
- The receiver verifies each slice against the root **before writing it to disk**. A peer that
|
|
34
|
+
sends one bad byte is caught immediately, not at the end of the transfer.
|
|
35
|
+
|
|
36
|
+
**Chunk groups.** Standard bao verifies at 1 KiB granularity, which makes the outboard tree
|
|
37
|
+
large. Grouping chunks shrinks it while keeping the same root hash:
|
|
38
|
+
|
|
39
|
+
| Group size | Outboard for 1 GiB | Smallest verifiable unit |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| 1 KiB (plain bao) | ~64 MiB | 1 KiB |
|
|
42
|
+
| **16 KiB (default)** | ~4 MiB | 16 KiB |
|
|
43
|
+
| 256 KiB | ~256 KiB | 256 KiB |
|
|
44
|
+
|
|
45
|
+
**Defaults: 16 KiB verification groups, 64 KiB request blocks.** 16 KiB is what the rest of the
|
|
46
|
+
world converged on — BitTorrent v1's request block, BitTorrent v2's leaf block size, and
|
|
47
|
+
bao-tree's default — so it is where we start, not where we must stay. Both numbers are
|
|
48
|
+
negotiated per transfer and must be re-judged against real measurements once M2's benchmark
|
|
49
|
+
exists; very large files may want bigger groups to keep outboards small.
|
|
50
|
+
|
|
51
|
+
`blake3-bao` calls grouped encoding "Iroh chunk groups" because iroh popularised the technique.
|
|
52
|
+
**swarmwire does not depend on iroh, Rust, or any native module.** Keeping the whole stack pure
|
|
53
|
+
JavaScript is deliberate: it has to run on Bare for mobile.
|
|
54
|
+
|
|
55
|
+
## 3. Links
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
swarmwire://<root-hex>[?s=<size>][&n=<name>][&k=<secret-hex>]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
| Part | Meaning |
|
|
62
|
+
|---|---|
|
|
63
|
+
| `root-hex` | 64 lowercase hex characters: the BLAKE3 address |
|
|
64
|
+
| `s` | Size hint in bytes. Lets the receiver preallocate and show progress before any peer answers. Unverified until the content is complete. |
|
|
65
|
+
| `n` | Filename hint, percent-encoded. Display only — never used as a path without sanitising. |
|
|
66
|
+
| `k` | 32-byte swarm key, hex. See §4. |
|
|
67
|
+
| `t` | What the content is: `c` for a collection (§6). Absent means a plain file. |
|
|
68
|
+
|
|
69
|
+
A link with no parameters is valid and sufficient: everything else is an optimisation or a
|
|
70
|
+
privacy choice.
|
|
71
|
+
|
|
72
|
+
**Why hex and not z32.** The Holepunch convention (`hypercore-id-encoding`) renders *Hypercore
|
|
73
|
+
keys* as z-base-32 for users — 52 characters instead of 64, with visually ambiguous characters
|
|
74
|
+
removed. A swarmwire root is not a key, it is a BLAKE3 hash, and the whole world prints those as
|
|
75
|
+
hex: a link should be directly comparable to `b3sum <file>` output. Parsers may later accept
|
|
76
|
+
z32 on input; swarmwire only ever emits hex.
|
|
77
|
+
|
|
78
|
+
Unknown query parameters are ignored, so later versions can add hints without breaking today's
|
|
79
|
+
parsers.
|
|
80
|
+
|
|
81
|
+
## 4. Discovery
|
|
82
|
+
|
|
83
|
+
Content is found through Hyperswarm topics derived by BLAKE3 with domain separation. There are
|
|
84
|
+
two modes, and a peer may announce on both at once.
|
|
85
|
+
|
|
86
|
+
### Content topics — "here is a link"
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
public: topic = BLAKE3("swarmwire/v0/discovery" || root)
|
|
90
|
+
private: topic = BLAKE3("swarmwire/v0/discovery" || root || swarmKey)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Holding the link is enough to find peers: no topic to join, no tracker.
|
|
94
|
+
|
|
95
|
+
**Public** behaves like a magnet link: every peer holding that content, anywhere, helps — and
|
|
96
|
+
anyone who knows the address can discover that you have it. **Private** mixes in a 32-byte
|
|
97
|
+
swarm key, so only people holding the link can derive the topic. The cost is that unrelated
|
|
98
|
+
holders of the same content cannot help, so a private swarm is only as resilient as the people
|
|
99
|
+
in it.
|
|
100
|
+
|
|
101
|
+
`send` defaults to **private**. Sharing a file should not announce your holdings to anyone who
|
|
102
|
+
can guess the hash; `--public` opts into the wider swarm.
|
|
103
|
+
|
|
104
|
+
### Swarm topics — "here is a group of people"
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
topic = BLAKE3("swarmwire/v0/swarm" || swarmKey)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A swarm key identifies a *set of peers*, not a piece of content. Members announce on one topic,
|
|
111
|
+
stay connected, and then ask each other for roots over the existing connection.
|
|
112
|
+
|
|
113
|
+
This exists because the per-content topic does not scale to a shared workspace: a directory of
|
|
114
|
+
a thousand files must not mean a thousand DHT announces. Dropping a file into a shared workspace
|
|
115
|
+
should make it reachable by that workspace's members immediately, with no per-file setup and no
|
|
116
|
+
authorisation step — which is exactly what SwarmFS needs for multi-writer virtual directories.
|
|
117
|
+
|
|
118
|
+
swarmwire treats a swarm key as an opaque 32 bytes. Where it comes from — a link, a workspace,
|
|
119
|
+
a hypergraph context — is SwarmFS's business, not this layer's.
|
|
120
|
+
|
|
121
|
+
### Passphrases
|
|
122
|
+
|
|
123
|
+
A swarm key is 32 bytes. When a link cannot be pasted — read aloud, typed from another screen —
|
|
124
|
+
it may be *displayed* as a word sequence from a fixed wordlist, hyperbeam-style. That is a
|
|
125
|
+
display encoding of the same bytes, never a second kind of secret: words in, bytes out, same
|
|
126
|
+
topic. Not in v0.
|
|
127
|
+
|
|
128
|
+
## 6. Collections
|
|
129
|
+
|
|
130
|
+
A folder is shared as a **manifest**: a list of the files in it, each named by its address.
|
|
131
|
+
The manifest is itself content, so it has an address like anything else — sharing a folder is
|
|
132
|
+
sharing one link, and the receiver fetches the manifest first and then the files it names.
|
|
133
|
+
|
|
134
|
+
```json
|
|
135
|
+
{"swarmwire":"collection/v0","files":[{"path":"photos/one.jpg","root":"<hex>","size":70000}]}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The encoding is canonical, because the manifest's bytes are its identity and the same folder
|
|
139
|
+
must produce the same address on every machine: entries sorted by path, no incidental
|
|
140
|
+
whitespace, paths with forward slashes and no `.` segments. A folder packed on Windows and the
|
|
141
|
+
same folder packed on Linux have the same address.
|
|
142
|
+
|
|
143
|
+
**Paths are validated, never repaired.** A manifest arrives from a stranger, and a path is
|
|
144
|
+
refused — rather than normalised — if it is absolute, contains `..`, is empty, carries a drive
|
|
145
|
+
letter or control characters, or is simply not already canonical. Quietly turning
|
|
146
|
+
`/etc/passwd` into `etc/passwd` would write somewhere the sender did not name. Two entries for
|
|
147
|
+
the same path are refused too, since which one won would otherwise depend on arrival order.
|
|
148
|
+
|
|
149
|
+
Collections are immutable, like everything else here: change a file and you have a different
|
|
150
|
+
manifest with a different address. Folders that change are SwarmFS's problem, one layer up.
|
|
151
|
+
|
|
152
|
+
**Discovery:** only the manifest's root is announced. The files are fetched over the
|
|
153
|
+
connections that topic already produced, so a folder of a thousand files still costs one
|
|
154
|
+
announce.
|
|
155
|
+
|
|
156
|
+
## 5. Stability rules
|
|
157
|
+
|
|
158
|
+
**Fixed for v0 — changing any of these invalidates existing links:**
|
|
159
|
+
|
|
160
|
+
- The address is the plain BLAKE3 hash of the content
|
|
161
|
+
- The link scheme, the hex encoding of the root, and the meaning of `s`, `n`, `k`, `t`
|
|
162
|
+
- The collection manifest's encoding, including that entries are sorted by path
|
|
163
|
+
- The domain-separation strings `swarmwire/v0/discovery` and `swarmwire/v0/swarm`
|
|
164
|
+
|
|
165
|
+
**Free to change at any time** (negotiated per connection, never stored in a link): chunk group
|
|
166
|
+
size, request block size, wire messages, scheduling, how outboard trees are cached, the sidecar
|
|
167
|
+
format for partial downloads.
|
|
168
|
+
|
|
169
|
+
Protocol messages carry a version and a capability set in the handshake. Unknown message types
|
|
170
|
+
are ignored rather than treated as errors, so new message types never break old peers.
|
|
171
|
+
|
|
172
|
+
## Open questions
|
|
173
|
+
|
|
174
|
+
1. **Which wordlist** for passphrases — BIP39 (2048 words, widely implemented, 24 words for 32
|
|
175
|
+
bytes) or a shorter custom list. Deferred until the feature is actually built.
|
|
176
|
+
2. **Should a peer announce partial content on the content topic?** Announcing while still
|
|
177
|
+
downloading helps the swarm, but tells the topic you are interested before you have anything.
|
|
178
|
+
Measure in M2.
|
|
179
|
+
3. **Per-size default group size.** A single 16 KiB default is one less rule to remember; a
|
|
180
|
+
size-dependent default keeps outboards small for very large files. Decide with the benchmark.
|
package/index.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* swarmwire - content-addressed file transfer between filesystems, over Hyperswarm.
|
|
3
|
+
*
|
|
4
|
+
* Runs on Node and on Bare. See docs/roadmap.md for what exists and what is next.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export { Swarmwire, Download } from './lib/swarmwire.js'
|
|
8
|
+
export { rootOf, formatLink, parseLink, discoveryTopic, swarmTopic, SCHEME } from './lib/address.js'
|
|
9
|
+
export { ContentSource, MemorySource } from './lib/source.js'
|
|
10
|
+
export { encodeManifest, decodeManifest, isSafePath, MANIFEST_TYPE } from './lib/collection.js'
|
|
11
|
+
export { Destination } from './lib/destination.js'
|
|
12
|
+
export { verifyGroup, groupCount, groupRange, groupLength, DEFAULT_GROUP_LOG } from './lib/tree.js'
|
package/lib/address.js
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Addressing: content roots, links, and the topics content is announced on.
|
|
3
|
+
*
|
|
4
|
+
* This is the part of swarmwire that ends up inside links other people keep, so it changes
|
|
5
|
+
* slowly and deliberately. See docs/format.md; every rule here is pinned by test/unit.
|
|
6
|
+
*
|
|
7
|
+
* Runs on Node and on Bare: no Buffer, no node builtins.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { blake3 } from './blake3.js'
|
|
11
|
+
import b4a from 'b4a'
|
|
12
|
+
|
|
13
|
+
export const SCHEME = 'swarmwire'
|
|
14
|
+
|
|
15
|
+
/** Domain separation for topic derivation. Changing these strings splits the network. */
|
|
16
|
+
const CONTENT_PREFIX = 'swarmwire/v0/discovery'
|
|
17
|
+
const SWARM_PREFIX = 'swarmwire/v0/swarm'
|
|
18
|
+
|
|
19
|
+
const HEX32 = /^[0-9a-f]{64}$/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The content address of a buffer: the plain BLAKE3 hash, as `b3sum` prints it.
|
|
23
|
+
*
|
|
24
|
+
* @param {Uint8Array} content
|
|
25
|
+
* @returns {string} 64 lowercase hex characters
|
|
26
|
+
*/
|
|
27
|
+
export function rootOf (content) {
|
|
28
|
+
if (!(content instanceof Uint8Array)) throw new TypeError('content must be a Uint8Array')
|
|
29
|
+
return blake3.hashHex(content)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Build a link.
|
|
34
|
+
*
|
|
35
|
+
* @param {object} opts
|
|
36
|
+
* @param {string} opts.root - 64 hex characters
|
|
37
|
+
* @param {number} [opts.size] - size hint in bytes
|
|
38
|
+
* @param {string} [opts.name] - filename hint, for display only
|
|
39
|
+
* @param {string} [opts.secret] - 64 hex characters: the swarm key for a private swarm
|
|
40
|
+
* @param {'collection'|null} [opts.type] - what the content is, when it is not a plain file
|
|
41
|
+
* @returns {string}
|
|
42
|
+
*/
|
|
43
|
+
export function formatLink ({ root, size = null, name = null, secret = null, type = null } = {}) {
|
|
44
|
+
assertHex32(root, 'root')
|
|
45
|
+
if (secret !== null) assertHex32(secret, 'secret')
|
|
46
|
+
if (size !== null && !isSize(size)) throw new Error('size must be a non-negative integer')
|
|
47
|
+
if (type !== null && type !== 'collection') throw new Error('unknown link type')
|
|
48
|
+
|
|
49
|
+
const params = []
|
|
50
|
+
if (size !== null) params.push('s=' + size)
|
|
51
|
+
if (name !== null) params.push('n=' + encodeURIComponent(name))
|
|
52
|
+
if (secret !== null) params.push('k=' + secret)
|
|
53
|
+
if (type === 'collection') params.push('t=c')
|
|
54
|
+
|
|
55
|
+
return `${SCHEME}://${root}` + (params.length ? '?' + params.join('&') : '')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Parse a link. Hints are returned as given: validating a name against a destination path is
|
|
60
|
+
* the caller's job, and this must not silently "fix" a hostile one.
|
|
61
|
+
*
|
|
62
|
+
* @param {string} link
|
|
63
|
+
* @returns {{root: string, size: number|null, name: string|null, secret: string|null,
|
|
64
|
+
* type: 'collection'|null}}
|
|
65
|
+
*/
|
|
66
|
+
export function parseLink (link) {
|
|
67
|
+
if (typeof link !== 'string') throw new TypeError('link must be a string')
|
|
68
|
+
|
|
69
|
+
const prefix = `${SCHEME}://`
|
|
70
|
+
if (!link.startsWith(prefix)) throw new Error(`link must use the ${SCHEME}:// scheme`)
|
|
71
|
+
|
|
72
|
+
const rest = link.slice(prefix.length)
|
|
73
|
+
const q = rest.indexOf('?')
|
|
74
|
+
const root = q === -1 ? rest : rest.slice(0, q)
|
|
75
|
+
assertHex32(root, 'root')
|
|
76
|
+
|
|
77
|
+
const out = { root, size: null, name: null, secret: null, type: null }
|
|
78
|
+
if (q === -1) return out
|
|
79
|
+
|
|
80
|
+
for (const pair of rest.slice(q + 1).split('&')) {
|
|
81
|
+
if (pair === '') continue
|
|
82
|
+
const eq = pair.indexOf('=')
|
|
83
|
+
const key = eq === -1 ? pair : pair.slice(0, eq)
|
|
84
|
+
const value = eq === -1 ? '' : pair.slice(eq + 1)
|
|
85
|
+
|
|
86
|
+
switch (key) {
|
|
87
|
+
case 's': {
|
|
88
|
+
const size = Number(value)
|
|
89
|
+
if (!isSize(size)) throw new Error('size hint must be a non-negative integer')
|
|
90
|
+
out.size = size
|
|
91
|
+
break
|
|
92
|
+
}
|
|
93
|
+
case 'n':
|
|
94
|
+
out.name = decodeURIComponent(value)
|
|
95
|
+
break
|
|
96
|
+
case 'k':
|
|
97
|
+
assertHex32(value, 'secret')
|
|
98
|
+
out.secret = value
|
|
99
|
+
break
|
|
100
|
+
case 't':
|
|
101
|
+
// A hint, like the others: the content is checked when it arrives, and a link that
|
|
102
|
+
// lies about this produces a manifest that does not parse, not a wrong folder.
|
|
103
|
+
out.type = value === 'c' ? 'collection' : null
|
|
104
|
+
break
|
|
105
|
+
default:
|
|
106
|
+
// Unknown parameters are ignored, so later versions can add hints without
|
|
107
|
+
// breaking today's parsers.
|
|
108
|
+
break
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return out
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The topic a single piece of content is announced on — what a link resolves through.
|
|
117
|
+
*
|
|
118
|
+
* With a swarm key, only holders of the link can derive it; without one, anyone who knows the
|
|
119
|
+
* content's address can.
|
|
120
|
+
*
|
|
121
|
+
* @param {string} root - 64 hex characters
|
|
122
|
+
* @param {string|null} [swarmKey] - 64 hex characters
|
|
123
|
+
* @returns {Uint8Array} 32 bytes
|
|
124
|
+
*/
|
|
125
|
+
export function discoveryTopic (root, swarmKey = null) {
|
|
126
|
+
assertHex32(root, 'root')
|
|
127
|
+
if (swarmKey !== null) assertHex32(swarmKey, 'swarmKey')
|
|
128
|
+
|
|
129
|
+
const parts = [b4a.from(CONTENT_PREFIX, 'ascii'), b4a.from(root, 'hex')]
|
|
130
|
+
if (swarmKey !== null) parts.push(b4a.from(swarmKey, 'hex'))
|
|
131
|
+
|
|
132
|
+
return hash(parts)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* The topic a *group of peers* meets on, independent of what they hold.
|
|
137
|
+
*
|
|
138
|
+
* Members announce once and then ask each other for roots over the connection, so a shared
|
|
139
|
+
* workspace of a thousand files costs one announce rather than a thousand. Where the swarm key
|
|
140
|
+
* comes from — a link, a workspace, a hypergraph context — is the caller's business.
|
|
141
|
+
*
|
|
142
|
+
* @param {string} swarmKey - 64 hex characters
|
|
143
|
+
* @returns {Uint8Array} 32 bytes
|
|
144
|
+
*/
|
|
145
|
+
export function swarmTopic (swarmKey) {
|
|
146
|
+
assertHex32(swarmKey, 'swarmKey')
|
|
147
|
+
return hash([b4a.from(SWARM_PREFIX, 'ascii'), b4a.from(swarmKey, 'hex')])
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function hash (parts) {
|
|
151
|
+
return b4a.from(blake3.hashHex(toU8(b4a.concat(parts))), 'hex')
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function toU8 (buf) {
|
|
155
|
+
return buf instanceof Uint8Array ? buf : new Uint8Array(buf)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function assertHex32 (value, what) {
|
|
159
|
+
if (typeof value !== 'string' || !HEX32.test(value)) {
|
|
160
|
+
throw new Error(`${what} must be 64 lowercase hex characters`)
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function isSize (n) {
|
|
165
|
+
return Number.isInteger(n) && n >= 0
|
|
166
|
+
}
|
package/lib/blake3.cjs
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLAKE3 and bao, reached through CommonJS.
|
|
3
|
+
*
|
|
4
|
+
* `blake3-bao`'s ESM entry points are thin wrappers that call `createRequire` from Node's
|
|
5
|
+
* `module` builtin, which does not exist on Bare — so importing them directly makes the whole
|
|
6
|
+
* library Node-only. The CommonJS files underneath are plain JavaScript and load identically
|
|
7
|
+
* on both runtimes, so this is the one place that reaches for them.
|
|
8
|
+
*
|
|
9
|
+
* Verified: the same input hashes to the same value under Node and Bare.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const blake3 = require('blake3-bao/blake3')
|
|
13
|
+
const bao = require('blake3-bao/bao')
|
|
14
|
+
|
|
15
|
+
module.exports = { blake3, bao }
|
package/lib/blake3.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The hashing primitives, imported from one place.
|
|
3
|
+
*
|
|
4
|
+
* Everything that needs BLAKE3 or bao goes through here rather than reaching for
|
|
5
|
+
* `blake3-bao` directly, so the CommonJS detour that keeps us running on Bare — see
|
|
6
|
+
* `blake3.cjs` — stays in a single file.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import bridge from './blake3.cjs'
|
|
10
|
+
|
|
11
|
+
export const blake3 = bridge.blake3
|
|
12
|
+
export const bao = bridge.bao
|