peerpigeon 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -1365
- package/cli.js +1382 -0
- package/dist/peerpigeon-browser.js +12290 -0
- package/dist/peerpigeon-browser.js.map +7 -0
- package/docs/API_DOCUMENTATION.md +1945 -0
- package/docs/BOOTSTRAP_HUBS.md +319 -0
- package/docs/CLI_README.md +371 -0
- package/docs/DOCUMENTATION_REORGANIZATION.md +172 -0
- package/docs/HUB_QUICK_REF.md +107 -0
- package/docs/HUB_SCRIPTS.md +331 -0
- package/docs/HUB_SYSTEM.md +552 -0
- package/docs/MIGRATION_COMPLETE.md +93 -0
- package/docs/NETWORK_NAMESPACES.md +212 -0
- package/docs/README.md +107 -0
- package/docs/SELECTIVE_STREAMING_GUIDE.md +427 -0
- package/docs/TEST_CLEANUP_COMPLETE.md +80 -0
- package/examples/bootstrap-hub-example.js +292 -0
- package/examples/browser/README.md +278 -0
- package/examples/browser/app.js +2426 -49
- package/examples/browser/index.html +482 -543
- package/examples/browser/minimal/README.md +55 -0
- package/examples/browser/minimal/index.html +75 -0
- package/examples/browser/minimal/minimal.js +234 -0
- package/examples/browser/styles.css +687 -1425
- package/examples/browser/test-bundle.html +35 -0
- package/examples/browser-2/index.html +30 -0
- package/examples/browser-2/package.json +21 -0
- package/examples/browser-2/src/App-simple.vue +21 -0
- package/examples/browser-2/src/App.vue +420 -0
- package/examples/browser-2/src/components/CryptoView.vue +1661 -0
- package/examples/browser-2/src/components/DHTView.vue +1281 -0
- package/examples/browser-2/src/components/DebugView.vue +1773 -0
- package/examples/browser-2/src/components/MediaView.vue +1048 -0
- package/examples/browser-2/src/components/MessagingView.vue +775 -0
- package/examples/browser-2/src/components/NetworkView.vue +930 -0
- package/examples/browser-2/src/components/StorageView.vue +1376 -0
- package/examples/browser-2/src/main.js +5 -0
- package/examples/browser-2/src/stores/peerpigeon.js +895 -0
- package/examples/browser-2/vite.config.js +6 -0
- package/examples/hub-example.js +186 -0
- package/examples/multi-hub-test.js +186 -0
- package/examples/node/README.md +2 -2
- package/examples/node/storage-test.js +141 -6
- package/examples/selective-streaming-demo.html +698 -0
- package/examples/standard-events-demo.js +126 -0
- package/package.json +33 -16
- package/server/index.js +1032 -98
- package/server/start.js +49 -0
- package/src/ConnectionManager.js +239 -225
- package/src/CryptoManager.js +147 -18
- package/src/DebugLogger.js +0 -7
- package/src/DistributedStorageManager.js +245 -205
- package/src/EnvironmentDetector.js +109 -2
- package/src/EventEmitter.js +79 -1
- package/src/EvictionManager.js +10 -9
- package/src/GossipManager.js +163 -11
- package/src/LexicalStorageInterface.js +332 -0
- package/src/PeerConnection.js +333 -310
- package/src/PeerPigeonMesh.js +577 -135
- package/src/SignalingClient.js +14 -243
- package/src/SignalingHandler.js +31 -10
- package/src/StorageManager.js +0 -39
- package/src/WebDHT-Simple.js +389 -0
- package/src/WebDHT.js +363 -1165
- package/src/browser-entry.js +40 -0
- package/examples/browser/assets/images/favicon.png +0 -0
- package/examples/browser/assets/images/pigeon.png +0 -0
- package/examples/browser/crypto-test.html +0 -427
- package/examples/browser/ui-old.js +0 -1926
- package/examples/browser/ui.js +0 -3010
- package/websocket-server/README.md +0 -247
- package/websocket-server/server.js +0 -391
package/src/WebDHT.js
CHANGED
|
@@ -1,1276 +1,474 @@
|
|
|
1
1
|
import { EventEmitter } from './EventEmitter.js';
|
|
2
|
-
import { safeSetInterval, safeClearInterval, safeSetTimeout, safeClearTimeout } from './TimerUtils.js';
|
|
3
2
|
import DebugLogger from './DebugLogger.js';
|
|
4
3
|
|
|
5
4
|
/**
|
|
6
|
-
*
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
*
|
|
5
|
+
* SimpleWebDHT - Efficient Distributed Hash Table for millions of WebR const targetPeers = this.findClosestPeers(keyHash, replicationFactor);
|
|
6
|
+
|
|
7
|
+
const spaceInfo = options.space ? ` (${options.space} space, RF=${replicationFactor})` : '';
|
|
8
|
+
this.debug.log(`GET: Querying ${targetPeers.length} peers for ${key} in network ${this.mesh.networkName}${spaceInfo}`);
|
|
9
|
+
|
|
10
|
+
// Query peers in parallelers
|
|
11
|
+
*
|
|
12
|
+
* Design principles:
|
|
13
|
+
* 1. SIMPLE: Minimal complexity, maximum reliability
|
|
14
|
+
* 2. EFFICIENT: Optimized for millions of peers with O(log N) operations
|
|
15
|
+
* 3. SCALABLE: Consistent hashing with automatic load balancing
|
|
16
|
+
* 4. RELIABLE: Gossip-based replication with eventual consistency
|
|
17
|
+
*
|
|
18
|
+
* Key features:
|
|
19
|
+
* - Consistent hashing for efficient key distribution
|
|
20
|
+
* - Minimal routing table (only closest peers)
|
|
21
|
+
* - Gossip-based replication (fire-and-forget)
|
|
22
|
+
* - Automatic peer discovery and failure handling
|
|
23
|
+
* - No complex Kademlia routing - just simple consistent hashing
|
|
13
24
|
*/
|
|
14
25
|
export class WebDHT extends EventEmitter {
|
|
15
26
|
constructor(mesh) {
|
|
16
27
|
super();
|
|
17
|
-
this.debug = DebugLogger.create('
|
|
28
|
+
this.debug = DebugLogger.create('SimpleWebDHT');
|
|
18
29
|
this.mesh = mesh;
|
|
19
30
|
this.peerId = mesh.peerId;
|
|
20
31
|
|
|
21
|
-
//
|
|
22
|
-
this.
|
|
23
|
-
|
|
24
|
-
//
|
|
25
|
-
this.
|
|
26
|
-
|
|
27
|
-
//
|
|
28
|
-
this.
|
|
29
|
-
|
|
30
|
-
//
|
|
31
|
-
this.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
// Configuration
|
|
37
|
-
this.config = {
|
|
38
|
-
k: 20, // Bucket size (number of peers per bucket)
|
|
39
|
-
alpha: 3, // Parallelism factor for lookups
|
|
40
|
-
replicationFactor: 3, // Number of peers to store each key
|
|
41
|
-
ttl: null, // Default TTL: null = no expiration (records persist indefinitely)
|
|
42
|
-
refreshInterval: 300000 // Refresh interval (5 minutes)
|
|
43
|
-
};
|
|
44
|
-
|
|
45
|
-
// Setup message handlers
|
|
46
|
-
this.setupMessageHandlers();
|
|
47
|
-
|
|
48
|
-
// Start periodic maintenance
|
|
32
|
+
// Simple local storage
|
|
33
|
+
this.storage = new Map();
|
|
34
|
+
|
|
35
|
+
// Minimal routing: just track closest peers for efficient forwarding
|
|
36
|
+
this.closestPeers = new Set();
|
|
37
|
+
|
|
38
|
+
// Simple replication factor
|
|
39
|
+
this.replicationFactor = 3;
|
|
40
|
+
|
|
41
|
+
// Hash ring position for this peer
|
|
42
|
+
this.hashPosition = this.hashPeerId(this.peerId);
|
|
43
|
+
|
|
44
|
+
this.debug.log(`SimpleWebDHT initialized for peer ${this.peerId.substring(0, 8)} at position ${this.hashPosition.toString(16).substring(0, 8)}`);
|
|
45
|
+
|
|
46
|
+
this.setupMessageHandling();
|
|
49
47
|
this.startMaintenance();
|
|
50
|
-
|
|
51
|
-
this.debug.log(`WebDHT initialized for peer ${this.peerId.substring(0, 8)}...`);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* Setup message handlers for DHT operations
|
|
56
|
-
*/
|
|
57
|
-
setupMessageHandlers() {
|
|
58
|
-
this.debug.log(`🔥 DHT Setting up message handlers for peer ${this.peerId.substring(0, 8)}`);
|
|
59
|
-
|
|
60
|
-
// DHT messages now come through ConnectionManager's handleIncomingMessage
|
|
61
|
-
// which routes 'dht' type messages to this.mesh.webDHT.handleMessage
|
|
62
|
-
// So we don't need to listen to gossip manager events anymore
|
|
63
|
-
|
|
64
|
-
this.debug.log(`🔥 DHT Message handlers setup complete for peer ${this.peerId.substring(0, 8)}`);
|
|
65
48
|
}
|
|
66
49
|
|
|
67
50
|
/**
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
break;
|
|
114
|
-
case 'store_response':
|
|
115
|
-
case 'find_value_response':
|
|
116
|
-
case 'find_node_response':
|
|
117
|
-
case 'update_value_response':
|
|
118
|
-
this.debug.log(`🔥 DHT Handling response (${type}) from ${fromPeerId.substring(0, 8)}`);
|
|
119
|
-
this.handleResponse(fromPeerId, data);
|
|
120
|
-
break;
|
|
121
|
-
default:
|
|
122
|
-
this.debug.warn(`Unknown DHT message type: ${type}`);
|
|
123
|
-
}
|
|
124
|
-
} catch (error) {
|
|
125
|
-
this.debug.error(`🔥 DHT Error handling message type ${type} from ${fromPeerId.substring(0, 8)}:`, error);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/**
|
|
130
|
-
* Store a key-value pair in the DHT
|
|
131
|
-
*/
|
|
132
|
-
async put(key, value, options = {}) {
|
|
133
|
-
const keyId = await this.generateKeyId(key);
|
|
134
|
-
const ttl = options.ttl || this.config.ttl;
|
|
135
|
-
|
|
136
|
-
this.debug.log(`DHT PUT: ${key} -> ${keyId.substring(0, 8)}...`);
|
|
137
|
-
|
|
138
|
-
const storeData = {
|
|
139
|
-
key,
|
|
140
|
-
keyId,
|
|
141
|
-
value,
|
|
142
|
-
timestamp: Date.now(),
|
|
143
|
-
ttl,
|
|
144
|
-
publisher: this.peerId
|
|
145
|
-
};
|
|
146
|
-
|
|
147
|
-
// Check if this is a new key (for pending subscription activation)
|
|
148
|
-
const isNewKey = !this.localStorage.has(keyId);
|
|
149
|
-
|
|
150
|
-
// CRITICAL FIX: Always store locally first (the originator always keeps a copy)
|
|
151
|
-
this.storeLocally(keyId, storeData);
|
|
152
|
-
this.debug.log(`DHT PUT: Stored locally: ${key}`);
|
|
153
|
-
|
|
154
|
-
// If this is a new key, activate any pending subscriptions
|
|
155
|
-
if (isNewKey) {
|
|
156
|
-
const activatedSubscribers = this.activatePendingSubscriptions(keyId);
|
|
157
|
-
|
|
158
|
-
// Notify locally activated subscribers about the new key
|
|
159
|
-
if (activatedSubscribers.length > 0) {
|
|
160
|
-
this.debug.log(`DHT PUT: Notifying ${activatedSubscribers.length} locally pending subscribers about new key ${key}`);
|
|
161
|
-
|
|
162
|
-
const notificationPromises = activatedSubscribers.map(async (subscriberId) => {
|
|
163
|
-
if (subscriberId !== this.peerId) {
|
|
164
|
-
try {
|
|
165
|
-
return await this.sendDHTMessage(subscriberId, 'value_changed', {
|
|
166
|
-
key,
|
|
167
|
-
keyId,
|
|
168
|
-
newValue: value,
|
|
169
|
-
timestamp: storeData.timestamp,
|
|
170
|
-
isNewKey: true
|
|
171
|
-
});
|
|
172
|
-
} catch (error) {
|
|
173
|
-
this.debug.warn(`Failed to notify subscriber ${subscriberId.substring(0, 8)} about new key:`, error.message);
|
|
174
|
-
return null;
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
}).filter(Boolean);
|
|
178
|
-
|
|
179
|
-
await Promise.allSettled(notificationPromises);
|
|
180
|
-
}
|
|
51
|
+
* Calculate replication factor based on storage space and network size
|
|
52
|
+
* @param {string} space - Storage space type (private, public, frozen)
|
|
53
|
+
* @param {number} totalPeers - Total number of connected peers
|
|
54
|
+
* @returns {number} Appropriate replication factor
|
|
55
|
+
*/
|
|
56
|
+
getReplicationFactor(space, totalPeers = null) {
|
|
57
|
+
const peerCount = totalPeers || Array.from(this.mesh.connectionManager.peers.keys())
|
|
58
|
+
.filter(peerId => this.mesh.connectionManager.peers.get(peerId).getStatus() === 'connected').length;
|
|
59
|
+
|
|
60
|
+
if (peerCount === 0) return 0;
|
|
61
|
+
|
|
62
|
+
switch(space) {
|
|
63
|
+
case 'private':
|
|
64
|
+
// Private data: minimal replication (only owner can decrypt)
|
|
65
|
+
return Math.min(3, peerCount);
|
|
66
|
+
|
|
67
|
+
case 'public':
|
|
68
|
+
// Public data: moderate replication for good availability
|
|
69
|
+
// 30% of peers, minimum 3, maximum 7
|
|
70
|
+
return Math.max(3, Math.min(Math.ceil(peerCount * 0.3), 7));
|
|
71
|
+
|
|
72
|
+
case 'frozen':
|
|
73
|
+
// Frozen data: high replication for immutability and permanence
|
|
74
|
+
// 50% of peers, minimum 5, maximum 10
|
|
75
|
+
return Math.max(5, Math.min(Math.ceil(peerCount * 0.5), 10));
|
|
76
|
+
|
|
77
|
+
default:
|
|
78
|
+
// Default: use base replication factor
|
|
79
|
+
return Math.min(this.replicationFactor, peerCount);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Simple hash function for consistent hashing
|
|
85
|
+
*/
|
|
86
|
+
async hash(data) {
|
|
87
|
+
const encoder = new TextEncoder();
|
|
88
|
+
const dataBuffer = encoder.encode(typeof data === 'string' ? data : JSON.stringify(data));
|
|
89
|
+
const hashBuffer = await crypto.subtle.digest('SHA-256', dataBuffer);
|
|
90
|
+
const hashArray = new Uint8Array(hashBuffer);
|
|
91
|
+
|
|
92
|
+
// Convert to number for position on hash ring
|
|
93
|
+
let hash = 0;
|
|
94
|
+
for (let i = 0; i < 4; i++) {
|
|
95
|
+
hash = (hash * 256 + hashArray[i]) >>> 0;
|
|
181
96
|
}
|
|
182
|
-
|
|
183
|
-
// AUTO-SUBSCRIBE: When putting a value, automatically subscribe to future changes
|
|
184
|
-
this.mySubscriptions.add(keyId);
|
|
185
|
-
this.debug.log(`DHT PUT: Auto-subscribed to ${key} for future updates`);
|
|
186
|
-
|
|
187
|
-
// Find the closest peers to store this key for replication
|
|
188
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.k); // Get more peers initially
|
|
189
|
-
|
|
190
|
-
// Implement iterative storage for better replication success
|
|
191
|
-
return await this.performIterativePut(keyId, storeData, closestPeers);
|
|
97
|
+
return hash;
|
|
192
98
|
}
|
|
193
99
|
|
|
194
100
|
/**
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
this.debug.log(`DHT GET: ${key} -> ${keyId.substring(0, 8)}... (auto-subscribing: ${subscribe}, forceRefresh: ${forceRefresh})`);
|
|
204
|
-
|
|
205
|
-
// Check local storage FIRST (unless force refresh is requested)
|
|
206
|
-
if (!forceRefresh && this.localStorage.has(keyId)) {
|
|
207
|
-
const data = this.localStorage.get(keyId);
|
|
208
|
-
if (!this.isExpired(data)) {
|
|
209
|
-
this.debug.log(`DHT GET: Found locally: ${key} (using cached data)`);
|
|
210
|
-
// Add subscription even for local data (unless skipSubscribe is true)
|
|
211
|
-
if (subscribe) {
|
|
212
|
-
this.mySubscriptions.add(keyId);
|
|
213
|
-
}
|
|
214
|
-
return data.value;
|
|
215
|
-
} else {
|
|
216
|
-
// Remove expired data
|
|
217
|
-
this.localStorage.delete(keyId);
|
|
218
|
-
this.debug.log(`DHT GET: Removed expired local data for: ${key}`);
|
|
219
|
-
}
|
|
101
|
+
* Hash peer ID to position on ring
|
|
102
|
+
*/
|
|
103
|
+
hashPeerId(peerId) {
|
|
104
|
+
// Simple deterministic hash of peer ID
|
|
105
|
+
let hash = 0;
|
|
106
|
+
for (let i = 0; i < peerId.length; i++) {
|
|
107
|
+
hash = ((hash << 5) - hash + peerId.charCodeAt(i)) >>> 0;
|
|
220
108
|
}
|
|
221
|
-
|
|
222
|
-
if (forceRefresh) {
|
|
223
|
-
this.debug.log('DHT GET: Force refresh requested - bypassing local cache, fetching from original storing peers');
|
|
224
|
-
} else {
|
|
225
|
-
this.debug.log(`DHT GET: Not found locally, routing to network for key: ${key}`);
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
// Implement iterative Kademlia lookup with automatic subscription
|
|
229
|
-
return await this.performIterativeGet(keyId, key, subscribe);
|
|
109
|
+
return hash;
|
|
230
110
|
}
|
|
231
111
|
|
|
232
112
|
/**
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
const
|
|
238
|
-
|
|
239
|
-
this.debug.log(`DHT EXPLICIT SUBSCRIBE: ${key} -> ${keyId.substring(0, 8)}...`);
|
|
240
|
-
|
|
241
|
-
// Add to our subscriptions immediately
|
|
242
|
-
this.mySubscriptions.add(keyId);
|
|
243
|
-
|
|
244
|
-
// Find the peers responsible for storing this key
|
|
245
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.replicationFactor);
|
|
246
|
-
|
|
247
|
-
// Send subscription requests to all potential storage peers
|
|
248
|
-
const subscribePromises = closestPeers.map(async (peerId) => {
|
|
249
|
-
if (peerId !== this.peerId) {
|
|
250
|
-
try {
|
|
251
|
-
return await this.sendDHTMessage(peerId, 'subscribe', {
|
|
252
|
-
keyId,
|
|
253
|
-
key,
|
|
254
|
-
subscriber: this.peerId
|
|
255
|
-
});
|
|
256
|
-
} catch (error) {
|
|
257
|
-
this.debug.warn(`DHT SUBSCRIBE: Failed to subscribe to peer ${peerId.substring(0, 8)}:`, error.message);
|
|
258
|
-
return null;
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
}).filter(Boolean);
|
|
262
|
-
|
|
263
|
-
await Promise.allSettled(subscribePromises);
|
|
264
|
-
|
|
265
|
-
// Also add ourselves to pending subscriptions if we're among the closest
|
|
266
|
-
if (this.isAmongClosest(keyId, closestPeers)) {
|
|
267
|
-
this.addPendingSubscription(keyId, this.peerId);
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
// Try to get the current value (if it exists)
|
|
271
|
-
let currentValue = null;
|
|
272
|
-
try {
|
|
273
|
-
currentValue = await this.get(key, { skipSubscribe: true }); // Skip auto-subscribe since we're already subscribed
|
|
274
|
-
} catch (error) {
|
|
275
|
-
this.debug.log(`DHT SUBSCRIBE: Key ${key} doesn't exist yet, but subscription is active for when it's created`);
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
this.debug.log(`DHT SUBSCRIBE: Subscribed to ${key}, current value:`, currentValue);
|
|
279
|
-
return currentValue;
|
|
113
|
+
* Calculate distance between two positions on hash ring
|
|
114
|
+
*/
|
|
115
|
+
ringDistance(pos1, pos2) {
|
|
116
|
+
const diff = Math.abs(pos1 - pos2);
|
|
117
|
+
const maxUint32 = 0xFFFFFFFF;
|
|
118
|
+
return Math.min(diff, maxUint32 - diff);
|
|
280
119
|
}
|
|
281
120
|
|
|
282
121
|
/**
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
// Find the peers responsible for this key and unsubscribe
|
|
293
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.replicationFactor);
|
|
294
|
-
|
|
295
|
-
const unsubscribePromises = closestPeers.map(async (peerId) => {
|
|
296
|
-
if (peerId !== this.peerId) {
|
|
297
|
-
return this.sendDHTMessage(peerId, 'unsubscribe', {
|
|
298
|
-
keyId,
|
|
299
|
-
key,
|
|
300
|
-
subscriber: this.peerId
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
}).filter(Boolean);
|
|
304
|
-
|
|
305
|
-
await Promise.allSettled(unsubscribePromises);
|
|
306
|
-
this.mySubscriptions.delete(keyId);
|
|
307
|
-
|
|
308
|
-
// Remove local subscription if we're storing this key
|
|
309
|
-
if (this.subscriptions.has(keyId)) {
|
|
310
|
-
this.subscriptions.get(keyId).delete(this.peerId);
|
|
311
|
-
if (this.subscriptions.get(keyId).size === 0) {
|
|
312
|
-
this.subscriptions.delete(keyId);
|
|
313
|
-
}
|
|
122
|
+
* Find closest peers to a hash position
|
|
123
|
+
*/
|
|
124
|
+
findClosestPeers(targetHash, count = this.replicationFactor) {
|
|
125
|
+
const connectedPeers = Array.from(this.mesh.connectionManager.peers.keys())
|
|
126
|
+
.filter(peerId => this.mesh.connectionManager.peers.get(peerId).getStatus() === 'connected');
|
|
127
|
+
|
|
128
|
+
if (connectedPeers.length === 0) {
|
|
129
|
+
return [];
|
|
314
130
|
}
|
|
315
131
|
|
|
316
|
-
//
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
}
|
|
322
|
-
}
|
|
132
|
+
// Calculate distances and sort
|
|
133
|
+
const peersWithDistance = connectedPeers.map(peerId => ({
|
|
134
|
+
peerId,
|
|
135
|
+
distance: this.ringDistance(targetHash, this.hashPeerId(peerId))
|
|
136
|
+
}));
|
|
323
137
|
|
|
324
|
-
|
|
138
|
+
peersWithDistance.sort((a, b) => a.distance - b.distance);
|
|
139
|
+
|
|
140
|
+
return peersWithDistance.slice(0, count).map(p => p.peerId);
|
|
325
141
|
}
|
|
326
142
|
|
|
327
143
|
/**
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
//
|
|
336
|
-
this.
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
key,
|
|
345
|
-
keyId,
|
|
346
|
-
value: newValue,
|
|
144
|
+
* Store key-value pair with network namespace support
|
|
145
|
+
* @param {string} key - Storage key
|
|
146
|
+
* @param {*} value - Value to store
|
|
147
|
+
* @param {Object} options - Storage options
|
|
148
|
+
* @param {string} options.space - Storage space (private, public, frozen) for replication strategy
|
|
149
|
+
*/
|
|
150
|
+
async put(key, value, options = {}) {
|
|
151
|
+
// Add network namespace to the key to ensure network isolation
|
|
152
|
+
const namespacedKey = `${this.mesh.networkName}:${key}`;
|
|
153
|
+
const keyHash = await this.hash(namespacedKey);
|
|
154
|
+
|
|
155
|
+
const storeData = {
|
|
156
|
+
key: namespacedKey,
|
|
157
|
+
originalKey: key, // Store original key for retrieval
|
|
158
|
+
value,
|
|
159
|
+
networkName: this.mesh.networkName,
|
|
347
160
|
timestamp: Date.now(),
|
|
348
|
-
|
|
349
|
-
|
|
161
|
+
publisher: this.peerId,
|
|
162
|
+
space: options.space // Track space for replication info
|
|
350
163
|
};
|
|
351
164
|
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
this.
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
165
|
+
// Always store locally first
|
|
166
|
+
this.storage.set(namespacedKey, storeData);
|
|
167
|
+
this.debug.log(`PUT: Stored ${key} locally in network ${this.mesh.networkName}`);
|
|
168
|
+
|
|
169
|
+
// Determine replication factor based on storage space
|
|
170
|
+
const replicationFactor = options.space
|
|
171
|
+
? this.getReplicationFactor(options.space)
|
|
172
|
+
: this.replicationFactor;
|
|
173
|
+
|
|
174
|
+
// Find closest peers for replication
|
|
175
|
+
const targetPeers = this.findClosestPeers(keyHash, replicationFactor);
|
|
176
|
+
|
|
177
|
+
this.debug.log(`PUT: Replicating ${key} to ${targetPeers.length} peers`);
|
|
178
|
+
|
|
179
|
+
// Simple fire-and-forget replication
|
|
180
|
+
const replicationPromises = targetPeers.map(async (peerId) => {
|
|
364
181
|
if (peerId !== this.peerId) {
|
|
365
182
|
try {
|
|
366
|
-
this.
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
return { peerId, success: true };
|
|
371
|
-
} else {
|
|
372
|
-
this.debug.warn(`DHT UPDATE: Replica peer ${peerId.substring(0, 8)} rejected update:`, response);
|
|
373
|
-
return { peerId, success: false, response };
|
|
374
|
-
}
|
|
183
|
+
this.sendMessage(peerId, 'dht_store', storeData);
|
|
184
|
+
// Add small delay to ensure message is sent
|
|
185
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
186
|
+
return { peerId, success: true };
|
|
375
187
|
} catch (error) {
|
|
376
|
-
this.debug.warn(`
|
|
188
|
+
this.debug.warn(`Replication to ${peerId.substring(0, 8)} failed:`, error.message);
|
|
377
189
|
return { peerId, success: false, error: error.message };
|
|
378
190
|
}
|
|
379
191
|
}
|
|
380
|
-
return
|
|
381
|
-
})
|
|
382
|
-
|
|
383
|
-
const results = await Promise.allSettled(updatePromises);
|
|
384
|
-
const updateResults = results
|
|
385
|
-
.filter(r => r.status === 'fulfilled' && r.value)
|
|
386
|
-
.map(r => r.value);
|
|
387
|
-
|
|
388
|
-
const successfulUpdates = updateResults.filter(r => r.success).length;
|
|
389
|
-
const failedUpdates = updateResults.filter(r => !r.success);
|
|
390
|
-
|
|
391
|
-
this.debug.log(`DHT UPDATE: Replica peer update results for ${key}:`);
|
|
392
|
-
this.debug.log(` - Successful: ${successfulUpdates}`);
|
|
393
|
-
this.debug.log(` - Failed: ${failedUpdates.length}`);
|
|
394
|
-
this.debug.log(` - Local: ${localUpdateSuccess ? 'success' : 'not applicable'}`);
|
|
395
|
-
|
|
396
|
-
// ENHANCED: Log failed updates for debugging
|
|
397
|
-
if (failedUpdates.length > 0) {
|
|
398
|
-
this.debug.warn('DHT UPDATE: Failed to update these replica peers:',
|
|
399
|
-
failedUpdates.map(f => ({ peer: f.peerId.substring(0, 8), reason: f.error || f.response })));
|
|
400
|
-
this.debug.warn('DHT UPDATE: This may lead to consistency issues. Consider implementing retry logic.');
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
// CRITICAL: Broadcast value change to ALL peers (including non-replicas)
|
|
404
|
-
// This ensures any peer with cached values gets the update immediately
|
|
405
|
-
await this.broadcastValueChange(key, keyId, newValue, updateData.timestamp);
|
|
406
|
-
|
|
407
|
-
// SUCCESS CRITERIA: ALL replica peers must be updated for consistency
|
|
408
|
-
// This prevents the case where a later GET operation hits a replica peer with stale data
|
|
409
|
-
const totalSuccessful = (localUpdateSuccess ? 1 : 0) + successfulUpdates;
|
|
410
|
-
const totalReplicas = closestPeers.length;
|
|
411
|
-
const wasSuccessful = totalSuccessful >= totalReplicas;
|
|
412
|
-
|
|
413
|
-
if (!wasSuccessful) {
|
|
414
|
-
this.debug.warn(`DHT UPDATE: ${key} update FAILED - only ${totalSuccessful}/${totalReplicas} replicas updated successfully`);
|
|
415
|
-
this.debug.warn('DHT UPDATE: This may cause consistency issues where GET operations return stale data');
|
|
416
|
-
} else {
|
|
417
|
-
this.debug.log(`DHT UPDATE: ${key} update SUCCESSFUL - all ${totalSuccessful}/${totalReplicas} replicas updated`);
|
|
418
|
-
}
|
|
192
|
+
return { peerId, success: true };
|
|
193
|
+
});
|
|
419
194
|
|
|
420
|
-
|
|
195
|
+
// Wait for all sends to complete
|
|
196
|
+
await Promise.allSettled(replicationPromises);
|
|
197
|
+
|
|
198
|
+
const spaceInfo = options.space ? ` (${options.space} space, RF=${replicationFactor})` : '';
|
|
199
|
+
this.debug.log(`PUT: ${key} replicated to ${targetPeers.length} peers in network ${this.mesh.networkName}${spaceInfo}`);
|
|
200
|
+
|
|
201
|
+
return true;
|
|
421
202
|
}
|
|
422
203
|
|
|
423
204
|
/**
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
205
|
+
* Retrieve value by key with network namespace support
|
|
206
|
+
* @param {string} key - Storage key
|
|
207
|
+
* @param {Object} options - Retrieval options
|
|
208
|
+
* @param {boolean} options.forceRefresh - Force refresh from network
|
|
209
|
+
* @param {string} options.space - Storage space for space-aware replication
|
|
210
|
+
*/
|
|
211
|
+
async get(key, options = {}) {
|
|
212
|
+
const forceRefresh = options.forceRefresh || false;
|
|
213
|
+
const namespacedKey = `${this.mesh.networkName}:${key}`;
|
|
214
|
+
|
|
215
|
+
// Check local storage first (unless force refresh)
|
|
216
|
+
if (!forceRefresh && this.storage.has(namespacedKey)) {
|
|
217
|
+
const data = this.storage.get(namespacedKey);
|
|
218
|
+
this.debug.log(`GET: Found ${key} locally in network ${this.mesh.networkName}`);
|
|
219
|
+
return data.value;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// If not found locally or force refresh, query the network
|
|
223
|
+
const keyHash = await this.hash(namespacedKey);
|
|
224
|
+
|
|
225
|
+
// Use space-aware replication factor for querying
|
|
226
|
+
const replicationFactor = options.space
|
|
227
|
+
? this.getReplicationFactor(options.space)
|
|
228
|
+
: this.replicationFactor;
|
|
229
|
+
|
|
230
|
+
const targetPeers = this.findClosestPeers(keyHash, replicationFactor);
|
|
231
|
+
|
|
232
|
+
this.debug.log(`GET: Querying ${targetPeers.length} peers for ${key} in network ${this.mesh.networkName}`);
|
|
233
|
+
|
|
234
|
+
// Query peers in parallel
|
|
235
|
+
const queryPromises = targetPeers.map(async (peerId) => {
|
|
236
|
+
if (peerId === this.peerId) return null;
|
|
237
|
+
|
|
434
238
|
try {
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
newValue,
|
|
440
|
-
timestamp
|
|
441
|
-
});
|
|
239
|
+
this.debug.log(`GET: Querying peer ${peerId.substring(0, 8)} for ${key}`);
|
|
240
|
+
const result = await this.queryPeer(peerId, namespacedKey);
|
|
241
|
+
this.debug.log(`GET: Peer ${peerId.substring(0, 8)} response for ${key}:`, result ? 'found' : 'not found');
|
|
242
|
+
return result;
|
|
442
243
|
} catch (error) {
|
|
443
|
-
this.debug.warn(`
|
|
244
|
+
this.debug.warn(`Query to ${peerId.substring(0, 8)} failed:`, error.message);
|
|
444
245
|
return null;
|
|
445
246
|
}
|
|
446
247
|
});
|
|
447
248
|
|
|
448
|
-
const results = await Promise.allSettled(
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
* Handle store request from another peer
|
|
461
|
-
*/
|
|
462
|
-
async handleStoreRequest(fromPeerId, data) {
|
|
463
|
-
const { keyId, key, value, timestamp, ttl, publisher, messageId } = data;
|
|
464
|
-
|
|
465
|
-
this.debug.log(`🔥 DHT STORE REQUEST: ${key} from ${fromPeerId.substring(0, 8)}, messageId: ${messageId}`);
|
|
466
|
-
|
|
467
|
-
// Check if we should store this key (are we close enough?)
|
|
468
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.replicationFactor);
|
|
469
|
-
|
|
470
|
-
if (this.isAmongClosest(keyId, closestPeers)) {
|
|
471
|
-
// Check if this is a new key (for pending subscription activation)
|
|
472
|
-
const isNewKey = !this.localStorage.has(keyId);
|
|
473
|
-
|
|
474
|
-
this.storeLocally(keyId, { key, value, timestamp, ttl, publisher });
|
|
475
|
-
|
|
476
|
-
// If this is a new key, activate any pending subscriptions
|
|
477
|
-
if (isNewKey) {
|
|
478
|
-
const activatedSubscribers = this.activatePendingSubscriptions(keyId);
|
|
479
|
-
|
|
480
|
-
// Notify newly activated subscribers about the new key
|
|
481
|
-
if (activatedSubscribers.length > 0) {
|
|
482
|
-
this.debug.log(`DHT STORE: Notifying ${activatedSubscribers.length} subscribers about new key ${key}`);
|
|
483
|
-
|
|
484
|
-
const notificationPromises = activatedSubscribers.map(async (subscriberId) => {
|
|
485
|
-
if (subscriberId !== this.peerId && subscriberId !== fromPeerId) {
|
|
486
|
-
try {
|
|
487
|
-
return await this.sendDHTMessage(subscriberId, 'value_changed', {
|
|
488
|
-
key,
|
|
489
|
-
keyId,
|
|
490
|
-
newValue: value,
|
|
491
|
-
timestamp,
|
|
492
|
-
isNewKey: true
|
|
493
|
-
});
|
|
494
|
-
} catch (error) {
|
|
495
|
-
this.debug.warn(`Failed to notify subscriber ${subscriberId.substring(0, 8)} about new key:`, error.message);
|
|
496
|
-
return null;
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
}).filter(Boolean);
|
|
500
|
-
|
|
501
|
-
await Promise.allSettled(notificationPromises);
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
this.debug.log(`🔥 DHT STORE: Accepting and storing ${key}, sending success response`);
|
|
506
|
-
|
|
507
|
-
// Send response using the same direct messaging structure
|
|
508
|
-
this.sendDirectToPeer(fromPeerId, {
|
|
509
|
-
type: 'dht',
|
|
510
|
-
messageType: 'store_response',
|
|
511
|
-
data: {
|
|
512
|
-
keyId,
|
|
513
|
-
success: true,
|
|
514
|
-
storedBy: this.peerId,
|
|
515
|
-
messageId // Echo back the messageId for response matching
|
|
516
|
-
}
|
|
517
|
-
});
|
|
518
|
-
|
|
519
|
-
this.debug.log(`DHT STORE: accepted ${key} from ${fromPeerId.substring(0, 8)}...`);
|
|
520
|
-
} else {
|
|
521
|
-
this.debug.log(`🔥 DHT STORE: Rejecting ${key} (not among closest), sending failure response`);
|
|
522
|
-
|
|
523
|
-
this.sendDirectToPeer(fromPeerId, {
|
|
524
|
-
type: 'dht',
|
|
525
|
-
messageType: 'store_response',
|
|
526
|
-
data: {
|
|
527
|
-
keyId,
|
|
528
|
-
success: false,
|
|
529
|
-
reason: 'not_closest',
|
|
530
|
-
closestPeers: closestPeers.slice(0, this.config.replicationFactor),
|
|
531
|
-
messageId // Echo back the messageId for response matching
|
|
532
|
-
}
|
|
533
|
-
});
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
/**
|
|
538
|
-
* Handle find value request
|
|
539
|
-
*/
|
|
540
|
-
async handleFindValueRequest(fromPeerId, data) {
|
|
541
|
-
const { keyId, key, subscribe, requester, messageId } = data;
|
|
542
|
-
|
|
543
|
-
this.debug.log(`🔥 DHT FIND_VALUE REQUEST: ${key} from ${fromPeerId.substring(0, 8)}, messageId: ${messageId}`);
|
|
544
|
-
this.debug.log('🔥 DHT FIND_VALUE REQUEST: Full data received:', data);
|
|
545
|
-
|
|
546
|
-
if (this.localStorage.has(keyId)) {
|
|
547
|
-
const storedData = this.localStorage.get(keyId);
|
|
548
|
-
|
|
549
|
-
if (!this.isExpired(storedData)) {
|
|
550
|
-
this.debug.log(`🔥 DHT FIND_VALUE: Found ${key} locally, sending success response with messageId: ${messageId}`);
|
|
551
|
-
|
|
552
|
-
// Add subscription if requested
|
|
553
|
-
if (subscribe) {
|
|
554
|
-
this.addSubscription(keyId, requester);
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
// CRITICAL FIX: Use the request's messageId, not generate a new one
|
|
558
|
-
const responseData = {
|
|
559
|
-
keyId,
|
|
560
|
-
found: true,
|
|
561
|
-
messageId, // Echo back the EXACT messageId from request
|
|
562
|
-
key: storedData.key,
|
|
563
|
-
value: storedData.value,
|
|
564
|
-
timestamp: storedData.timestamp,
|
|
565
|
-
ttl: storedData.ttl,
|
|
566
|
-
publisher: storedData.publisher
|
|
567
|
-
};
|
|
568
|
-
|
|
569
|
-
this.debug.log(`🔥 DHT FIND_VALUE: Sending response data with messageId ${messageId}:`, responseData);
|
|
570
|
-
|
|
571
|
-
this.sendDirectToPeer(fromPeerId, {
|
|
572
|
-
type: 'dht',
|
|
573
|
-
messageType: 'find_value_response',
|
|
574
|
-
data: responseData
|
|
575
|
-
});
|
|
576
|
-
|
|
577
|
-
this.debug.log(`DHT FIND_VALUE: found ${key} for ${fromPeerId.substring(0, 8)}...`);
|
|
578
|
-
return;
|
|
579
|
-
} else {
|
|
580
|
-
this.localStorage.delete(keyId);
|
|
581
|
-
this.debug.log(`🔥 DHT FIND_VALUE: Found ${key} but expired, removed from storage`);
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
this.debug.log(`🔥 DHT FIND_VALUE: ${key} not found locally, sending closest peers response with messageId: ${messageId}`);
|
|
586
|
-
|
|
587
|
-
// If not found, return closest peers
|
|
588
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.k);
|
|
589
|
-
|
|
590
|
-
const responseData = {
|
|
591
|
-
keyId,
|
|
592
|
-
found: false,
|
|
593
|
-
messageId, // Echo back the EXACT messageId from request
|
|
594
|
-
closestPeers
|
|
595
|
-
};
|
|
596
|
-
|
|
597
|
-
this.debug.log(`🔥 DHT FIND_VALUE: Sending not-found response data with messageId ${messageId}:`, responseData);
|
|
598
|
-
|
|
599
|
-
this.sendDirectToPeer(fromPeerId, {
|
|
600
|
-
type: 'dht',
|
|
601
|
-
messageType: 'find_value_response',
|
|
602
|
-
data: responseData
|
|
603
|
-
});
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
/**
|
|
607
|
-
* Handle subscribe request
|
|
608
|
-
*/
|
|
609
|
-
async handleSubscribeRequest(fromPeerId, data) {
|
|
610
|
-
const { keyId, key, subscriber } = data;
|
|
611
|
-
|
|
612
|
-
this.debug.log(`DHT SUBSCRIBE REQUEST: ${key} from ${fromPeerId.substring(0, 8)} for subscriber ${subscriber.substring(0, 8)}`);
|
|
613
|
-
|
|
614
|
-
if (this.localStorage.has(keyId)) {
|
|
615
|
-
// Key exists - add to active subscriptions
|
|
616
|
-
this.addSubscription(keyId, subscriber);
|
|
617
|
-
this.debug.log(`DHT SUBSCRIBE: added ${subscriber.substring(0, 8)}... to ${key} notifications (key exists)`);
|
|
618
|
-
} else {
|
|
619
|
-
// Key doesn't exist yet - add to pending subscriptions
|
|
620
|
-
this.addPendingSubscription(keyId, subscriber);
|
|
621
|
-
this.debug.log(`DHT SUBSCRIBE: added ${subscriber.substring(0, 8)}... to ${key} pending notifications (key doesn't exist yet)`);
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
/**
|
|
626
|
-
* Handle unsubscribe request
|
|
627
|
-
*/
|
|
628
|
-
async handleUnsubscribeRequest(fromPeerId, data) {
|
|
629
|
-
const { keyId, key, subscriber } = data;
|
|
630
|
-
|
|
631
|
-
// Remove from active subscriptions
|
|
632
|
-
if (this.subscriptions.has(keyId)) {
|
|
633
|
-
this.subscriptions.get(keyId).delete(subscriber);
|
|
634
|
-
if (this.subscriptions.get(keyId).size === 0) {
|
|
635
|
-
this.subscriptions.delete(keyId);
|
|
636
|
-
}
|
|
637
|
-
this.debug.log(`DHT UNSUBSCRIBE: removed ${subscriber.substring(0, 8)}... from ${key} active notifications`);
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
// Remove from pending subscriptions
|
|
641
|
-
if (this.pendingSubscriptions.has(keyId)) {
|
|
642
|
-
this.pendingSubscriptions.get(keyId).delete(subscriber);
|
|
643
|
-
if (this.pendingSubscriptions.get(keyId).size === 0) {
|
|
644
|
-
this.pendingSubscriptions.delete(keyId);
|
|
645
|
-
}
|
|
646
|
-
this.debug.log(`DHT UNSUBSCRIBE: removed ${subscriber.substring(0, 8)}... from ${key} pending notifications`);
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
|
|
650
|
-
/**
|
|
651
|
-
* Handle value changed notification
|
|
652
|
-
*/
|
|
653
|
-
async handleValueChangedNotification(fromPeerId, data) {
|
|
654
|
-
const { key, keyId, newValue, timestamp, isNewKey } = data;
|
|
655
|
-
|
|
656
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED notification: ${key} = ${newValue} from ${fromPeerId.substring(0, 8)}${isNewKey ? ' (NEW KEY)' : ''}`);
|
|
657
|
-
|
|
658
|
-
// CRITICAL FIX: Always update local cache if we have it, regardless of subscriptions
|
|
659
|
-
// This prevents stale cached values from being returned by get() operations
|
|
660
|
-
if (this.localStorage.has(keyId)) {
|
|
661
|
-
const storedData = this.localStorage.get(keyId);
|
|
662
|
-
// Only update if the new timestamp is newer (prevent old notifications from overwriting newer data)
|
|
663
|
-
if (!storedData.timestamp || timestamp >= storedData.timestamp) {
|
|
664
|
-
storedData.value = newValue;
|
|
665
|
-
storedData.timestamp = timestamp;
|
|
666
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: Updated local cache for ${key} (timestamp: ${timestamp})`);
|
|
667
|
-
} else {
|
|
668
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: Ignoring older notification for ${key} (current: ${storedData.timestamp}, notification: ${timestamp})`);
|
|
669
|
-
}
|
|
670
|
-
} else if (isNewKey) {
|
|
671
|
-
// For new keys, store the data locally even if we don't normally store this key
|
|
672
|
-
// This helps with caching and consistency
|
|
673
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: Caching new key ${key} locally`);
|
|
674
|
-
this.storeLocally(keyId, {
|
|
675
|
-
key,
|
|
676
|
-
value: newValue,
|
|
677
|
-
timestamp,
|
|
678
|
-
ttl: null, // Use default TTL
|
|
679
|
-
publisher: fromPeerId
|
|
680
|
-
}, false); // Not a primary replica
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
// Check if we have subscribers for this key
|
|
684
|
-
const hasSubscribers = this.mySubscriptions.has(keyId);
|
|
685
|
-
const hasLocalSubscribers = this.subscriptions.has(keyId) && this.subscriptions.get(keyId).size > 0;
|
|
686
|
-
|
|
687
|
-
if (hasSubscribers || hasLocalSubscribers) {
|
|
688
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: Processing notification for ${key} (mySubscriptions: ${hasSubscribers}, localSubscribers: ${hasLocalSubscribers})`);
|
|
689
|
-
|
|
690
|
-
// Emit change event if this peer is subscribed
|
|
691
|
-
if (hasSubscribers) {
|
|
692
|
-
this.emit('valueChanged', { key, keyId, newValue, timestamp, isNewKey });
|
|
693
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: Emitted local event for ${key}${isNewKey ? ' (NEW KEY)' : ''}`);
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
// Forward to local subscribers (other peers that subscribed through this peer)
|
|
697
|
-
if (hasLocalSubscribers) {
|
|
698
|
-
const localSubscribers = Array.from(this.subscriptions.get(keyId));
|
|
699
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: Forwarding to ${localSubscribers.length} local subscribers for ${key}`);
|
|
700
|
-
|
|
701
|
-
const forwardPromises = localSubscribers.map(async (subscriberId) => {
|
|
702
|
-
if (subscriberId !== this.peerId && subscriberId !== fromPeerId) {
|
|
703
|
-
try {
|
|
704
|
-
return await this.sendDHTMessage(subscriberId, 'value_changed', {
|
|
705
|
-
key,
|
|
706
|
-
keyId,
|
|
707
|
-
newValue,
|
|
708
|
-
timestamp,
|
|
709
|
-
isNewKey
|
|
710
|
-
});
|
|
711
|
-
} catch (error) {
|
|
712
|
-
this.debug.warn(`Failed to forward notification to ${subscriberId.substring(0, 8)}:`, error.message);
|
|
713
|
-
return null;
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
}).filter(Boolean);
|
|
717
|
-
|
|
718
|
-
await Promise.allSettled(forwardPromises);
|
|
249
|
+
const results = await Promise.allSettled(queryPromises);
|
|
250
|
+
|
|
251
|
+
// Find the first successful result
|
|
252
|
+
for (const result of results) {
|
|
253
|
+
if (result.status === 'fulfilled' && result.value) {
|
|
254
|
+
const data = result.value;
|
|
255
|
+
|
|
256
|
+
// Cache locally for future use
|
|
257
|
+
this.storage.set(namespacedKey, data);
|
|
258
|
+
|
|
259
|
+
this.debug.log(`GET: Found ${key} from network ${this.mesh.networkName}`);
|
|
260
|
+
return data.value;
|
|
719
261
|
}
|
|
720
|
-
} else {
|
|
721
|
-
this.debug.log(`🔥 DHT VALUE_CHANGED: No subscriptions for ${key}, but updated cache anyway`);
|
|
722
262
|
}
|
|
723
|
-
}
|
|
724
263
|
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
*/
|
|
728
|
-
async handleUpdateValueRequest(fromPeerId, data) {
|
|
729
|
-
const { keyId, key, value, timestamp, ttl, publisher, messageId } = data;
|
|
730
|
-
|
|
731
|
-
this.debug.log(`🔥 DHT UPDATE_VALUE REQUEST: ${key} from ${fromPeerId.substring(0, 8)}`);
|
|
732
|
-
|
|
733
|
-
// Check if we should store this key (are we among the closest?)
|
|
734
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.replicationFactor);
|
|
735
|
-
|
|
736
|
-
if (this.isAmongClosest(keyId, closestPeers)) {
|
|
737
|
-
// Update our local copy
|
|
738
|
-
this.storeLocally(keyId, { key, value, timestamp, ttl, publisher });
|
|
739
|
-
this.debug.log(`DHT UPDATE_VALUE: Updated ${key} locally`);
|
|
740
|
-
|
|
741
|
-
// Notify any local subscribers
|
|
742
|
-
if (this.subscriptions.has(keyId)) {
|
|
743
|
-
const subscribers = Array.from(this.subscriptions.get(keyId));
|
|
744
|
-
this.debug.log(`DHT UPDATE_VALUE: Notifying ${subscribers.length} local subscribers for ${key}`);
|
|
745
|
-
|
|
746
|
-
const notificationPromises = subscribers.map(async (subscriberId) => {
|
|
747
|
-
if (subscriberId !== this.peerId) {
|
|
748
|
-
try {
|
|
749
|
-
return await this.sendDHTMessage(subscriberId, 'value_changed', {
|
|
750
|
-
key,
|
|
751
|
-
keyId,
|
|
752
|
-
newValue: value,
|
|
753
|
-
timestamp
|
|
754
|
-
});
|
|
755
|
-
} catch (error) {
|
|
756
|
-
this.debug.warn(`Failed to notify subscriber ${subscriberId.substring(0, 8)}:`, error.message);
|
|
757
|
-
return null;
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
}).filter(Boolean);
|
|
761
|
-
|
|
762
|
-
await Promise.allSettled(notificationPromises);
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
// Send success response
|
|
766
|
-
this.sendDirectToPeer(fromPeerId, {
|
|
767
|
-
type: 'dht',
|
|
768
|
-
messageType: 'update_value_response',
|
|
769
|
-
data: {
|
|
770
|
-
keyId,
|
|
771
|
-
success: true,
|
|
772
|
-
updatedBy: this.peerId,
|
|
773
|
-
messageId
|
|
774
|
-
}
|
|
775
|
-
});
|
|
776
|
-
} else {
|
|
777
|
-
this.debug.log(`DHT UPDATE_VALUE: Not responsible for ${key}, rejecting update`);
|
|
778
|
-
|
|
779
|
-
// Send failure response
|
|
780
|
-
this.sendDirectToPeer(fromPeerId, {
|
|
781
|
-
type: 'dht',
|
|
782
|
-
messageType: 'update_value_response',
|
|
783
|
-
data: {
|
|
784
|
-
keyId,
|
|
785
|
-
success: false,
|
|
786
|
-
reason: 'not_closest',
|
|
787
|
-
messageId
|
|
788
|
-
}
|
|
789
|
-
});
|
|
790
|
-
}
|
|
264
|
+
this.debug.log(`GET: ${key} not found after querying ${targetPeers.length} peers`);
|
|
265
|
+
return null;
|
|
791
266
|
}
|
|
792
267
|
|
|
793
268
|
/**
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
async
|
|
269
|
+
* Query a specific peer for a key
|
|
270
|
+
*/
|
|
271
|
+
async queryPeer(peerId, key) {
|
|
797
272
|
return new Promise((resolve, reject) => {
|
|
798
|
-
const
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
const timeout = safeSetTimeout(() => {
|
|
805
|
-
this.debug.log(`🔥 DHT Timeout for messageId ${messageId} to peer ${targetPeerId.substring(0, 8)}`);
|
|
806
|
-
this.removeResponseHandler(messageId);
|
|
807
|
-
reject(new Error('DHT message timeout'));
|
|
273
|
+
const requestId = Math.random().toString(36).substring(7);
|
|
274
|
+
|
|
275
|
+
// Set timeout for query
|
|
276
|
+
const timeout = setTimeout(() => {
|
|
277
|
+
this.responseHandlers.delete(requestId);
|
|
278
|
+
reject(new Error('Query timeout'));
|
|
808
279
|
}, 5000);
|
|
809
280
|
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
281
|
+
// Store response handler
|
|
282
|
+
this.responseHandlers.set(requestId, (response) => {
|
|
283
|
+
clearTimeout(timeout);
|
|
284
|
+
this.responseHandlers.delete(requestId);
|
|
285
|
+
|
|
286
|
+
if (response.found) {
|
|
287
|
+
resolve(response.data);
|
|
288
|
+
} else {
|
|
289
|
+
resolve(null);
|
|
290
|
+
}
|
|
814
291
|
});
|
|
815
292
|
|
|
816
|
-
// Send
|
|
817
|
-
|
|
818
|
-
type: 'dht', // This tells ConnectionManager to route to DHT
|
|
819
|
-
data: { ...data, messageId },
|
|
820
|
-
messageType: type // The actual DHT operation type
|
|
821
|
-
};
|
|
822
|
-
|
|
823
|
-
this.debug.log(`🔥 DHT Full message being sent to ${targetPeerId.substring(0, 8)}:`, messageToSend);
|
|
824
|
-
|
|
825
|
-
const success = this.sendDirectToPeer(targetPeerId, messageToSend);
|
|
826
|
-
|
|
827
|
-
if (!success) {
|
|
828
|
-
this.debug.log(`🔥 DHT Failed to send to ${targetPeerId.substring(0, 8)}, cleaning up messageId ${messageId}`);
|
|
829
|
-
safeClearTimeout(timeout);
|
|
830
|
-
this.removeResponseHandler(messageId);
|
|
831
|
-
reject(new Error('Failed to send DHT message - no direct connection'));
|
|
832
|
-
}
|
|
293
|
+
// Send query
|
|
294
|
+
this.sendMessage(peerId, 'dht_query', { key, requestId });
|
|
833
295
|
});
|
|
834
296
|
}
|
|
835
297
|
|
|
836
298
|
/**
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
299
|
+
* Send message to peer through connection manager
|
|
300
|
+
*/
|
|
301
|
+
sendMessage(peerId, type, data) {
|
|
302
|
+
const message = {
|
|
303
|
+
type: 'dht',
|
|
304
|
+
messageType: type,
|
|
305
|
+
data,
|
|
306
|
+
from: this.peerId,
|
|
307
|
+
timestamp: Date.now()
|
|
308
|
+
};
|
|
845
309
|
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
310
|
+
const peer = this.mesh.connectionManager.peers.get(peerId);
|
|
311
|
+
if (peer && peer.getStatus() === 'connected') {
|
|
312
|
+
peer.sendMessage(message);
|
|
849
313
|
} else {
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
// Use gossip/routing mechanism to reach the target peer
|
|
853
|
-
// Create a special DHT routing message that will be forwarded through the mesh
|
|
854
|
-
const routingMessage = {
|
|
855
|
-
id: this.generateMessageId(),
|
|
856
|
-
from: this.peerId,
|
|
857
|
-
to: targetPeerId, // Target peer ID for routing
|
|
858
|
-
subtype: 'dht-routing', // Special subtype for DHT routing
|
|
859
|
-
content: message, // The actual DHT message to deliver
|
|
860
|
-
timestamp: Date.now(),
|
|
861
|
-
ttl: 5, // Allow up to 5 hops to reach the target
|
|
862
|
-
path: [this.peerId] // Track routing path
|
|
863
|
-
};
|
|
864
|
-
|
|
865
|
-
this.debug.log(`DHT: Routing message to ${targetPeerId.substring(0, 8)} via gossip network`);
|
|
866
|
-
this.mesh.gossipManager.propagateMessage(routingMessage);
|
|
867
|
-
return true;
|
|
314
|
+
throw new Error(`Peer ${peerId} not connected`);
|
|
868
315
|
}
|
|
869
316
|
}
|
|
870
317
|
|
|
871
318
|
/**
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
this.debug.log(`DHT findClosestPeers: Looking for ${count} closest peers to key ${keyId.substring(0, 8)}... from ${allPeers.length} total peers`);
|
|
878
|
-
|
|
879
|
-
// Sort by XOR distance to the key
|
|
880
|
-
allPeers.sort((a, b) => {
|
|
881
|
-
const distA = this.xorDistance(keyId, a);
|
|
882
|
-
const distB = this.xorDistance(keyId, b);
|
|
883
|
-
return distA.localeCompare(distB);
|
|
884
|
-
});
|
|
885
|
-
|
|
886
|
-
const result = allPeers.slice(0, count);
|
|
887
|
-
|
|
888
|
-
this.debug.log('DHT findClosestPeers result:', result.map(peerId => ({
|
|
889
|
-
peerId: peerId.substring(0, 8) + '...',
|
|
890
|
-
distance: this.xorDistance(keyId, peerId).substring(0, 8) + '...',
|
|
891
|
-
isSelf: peerId === this.peerId
|
|
892
|
-
})));
|
|
893
|
-
|
|
894
|
-
return result;
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
/**
|
|
898
|
-
* Calculate XOR distance between two IDs
|
|
899
|
-
*/
|
|
900
|
-
xorDistance(id1, id2) {
|
|
901
|
-
// Convert hex strings to numbers and XOR them
|
|
902
|
-
let distance = '';
|
|
903
|
-
for (let i = 0; i < Math.min(id1.length, id2.length); i++) {
|
|
904
|
-
const xor = parseInt(id1[i], 16) ^ parseInt(id2[i], 16);
|
|
905
|
-
distance += xor.toString(16);
|
|
906
|
-
}
|
|
907
|
-
return distance;
|
|
319
|
+
* Setup message handling
|
|
320
|
+
*/
|
|
321
|
+
setupMessageHandling() {
|
|
322
|
+
this.responseHandlers = new Map();
|
|
908
323
|
}
|
|
909
324
|
|
|
910
325
|
/**
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
async
|
|
914
|
-
|
|
915
|
-
const encoder = new TextEncoder();
|
|
916
|
-
const data = encoder.encode(key);
|
|
917
|
-
const hashBuffer = await crypto.subtle.digest('SHA-1', data);
|
|
918
|
-
const hashArray = new Uint8Array(hashBuffer);
|
|
919
|
-
return Array.from(hashArray, byte => byte.toString(16).padStart(2, '0')).join('');
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
/**
|
|
923
|
-
* Generate a unique message ID
|
|
924
|
-
*/
|
|
925
|
-
generateMessageId() {
|
|
926
|
-
return Date.now().toString(36) + Math.random().toString(36).substr(2);
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
/**
|
|
930
|
-
* Store data locally
|
|
931
|
-
*/
|
|
932
|
-
storeLocally(keyId, data, isPrimary = true) {
|
|
933
|
-
this.localStorage.set(keyId, {
|
|
934
|
-
...data,
|
|
935
|
-
storedAt: Date.now(),
|
|
936
|
-
isPrimary
|
|
937
|
-
});
|
|
938
|
-
}
|
|
326
|
+
* Handle incoming DHT message
|
|
327
|
+
*/
|
|
328
|
+
async handleMessage(message, fromPeerId) {
|
|
329
|
+
const { messageType, data } = message;
|
|
939
330
|
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
331
|
+
switch (messageType) {
|
|
332
|
+
case 'dht_store':
|
|
333
|
+
this.handleStore(data, fromPeerId);
|
|
334
|
+
break;
|
|
335
|
+
|
|
336
|
+
case 'dht_query':
|
|
337
|
+
await this.handleQuery(data, fromPeerId);
|
|
338
|
+
break;
|
|
339
|
+
|
|
340
|
+
case 'dht_query_response':
|
|
341
|
+
this.handleQueryResponse(data);
|
|
342
|
+
break;
|
|
343
|
+
|
|
344
|
+
default:
|
|
345
|
+
this.debug.warn(`Unknown DHT message type: ${messageType}`);
|
|
946
346
|
}
|
|
947
|
-
this.subscriptions.get(keyId).add(subscriberId);
|
|
948
347
|
}
|
|
949
348
|
|
|
950
349
|
/**
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
350
|
+
* Handle store request from peer with network filtering
|
|
351
|
+
*/
|
|
352
|
+
handleStore(data, fromPeerId) {
|
|
353
|
+
const { key, value, timestamp, publisher, networkName, space } = data;
|
|
354
|
+
|
|
355
|
+
// Filter by network namespace
|
|
356
|
+
const messageNetwork = networkName || 'global';
|
|
357
|
+
if (messageNetwork !== this.mesh.networkName) {
|
|
358
|
+
this.debug.log(`Filtering DHT store from different network: ${messageNetwork} (current: ${this.mesh.networkName})`);
|
|
359
|
+
return;
|
|
956
360
|
}
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
const pendingSubscribers = this.pendingSubscriptions.get(keyId);
|
|
966
|
-
|
|
967
|
-
// Move all pending subscribers to active subscriptions
|
|
968
|
-
if (!this.subscriptions.has(keyId)) {
|
|
969
|
-
this.subscriptions.set(keyId, new Set());
|
|
361
|
+
|
|
362
|
+
try {
|
|
363
|
+
// Simple conflict resolution: latest timestamp wins
|
|
364
|
+
if (this.storage.has(key)) {
|
|
365
|
+
const existing = this.storage.get(key);
|
|
366
|
+
if (existing.timestamp >= timestamp) {
|
|
367
|
+
return; // Ignore older data
|
|
368
|
+
}
|
|
970
369
|
}
|
|
971
370
|
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
});
|
|
975
|
-
|
|
976
|
-
this.debug.
|
|
977
|
-
|
|
978
|
-
// Clear pending subscriptions for this key
|
|
979
|
-
this.pendingSubscriptions.delete(keyId);
|
|
980
|
-
|
|
981
|
-
return Array.from(pendingSubscribers);
|
|
371
|
+
// Store the complete data with all fields including space
|
|
372
|
+
this.storage.set(key, { key, value, timestamp, publisher, space, networkName });
|
|
373
|
+
this.debug.log(`STORE: Received ${key} from ${fromPeerId.substring(0, 8)}`);
|
|
374
|
+
} catch (error) {
|
|
375
|
+
this.debug.warn(`Store failed for ${key}:`, error.message);
|
|
982
376
|
}
|
|
983
|
-
return [];
|
|
984
377
|
}
|
|
985
378
|
|
|
986
379
|
/**
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
380
|
+
* Handle query request from peer
|
|
381
|
+
*/
|
|
382
|
+
async handleQuery(data, fromPeerId) {
|
|
383
|
+
const { key, requestId } = data;
|
|
384
|
+
|
|
385
|
+
const response = {
|
|
386
|
+
requestId,
|
|
387
|
+
found: false,
|
|
388
|
+
data: null
|
|
389
|
+
};
|
|
992
390
|
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
isExpired(data) {
|
|
997
|
-
// If TTL is null/undefined, record never expires
|
|
998
|
-
if (!data.ttl) {
|
|
999
|
-
return false;
|
|
391
|
+
if (this.storage.has(key)) {
|
|
392
|
+
response.found = true;
|
|
393
|
+
response.data = this.storage.get(key);
|
|
1000
394
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
395
|
+
|
|
396
|
+
// Send response
|
|
397
|
+
this.sendMessage(fromPeerId, 'dht_query_response', response);
|
|
1003
398
|
}
|
|
1004
399
|
|
|
1005
400
|
/**
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
401
|
+
* Handle query response
|
|
402
|
+
*/
|
|
403
|
+
handleQueryResponse(data) {
|
|
404
|
+
const { requestId } = data;
|
|
405
|
+
const handler = this.responseHandlers.get(requestId);
|
|
406
|
+
|
|
407
|
+
if (handler) {
|
|
408
|
+
handler(data);
|
|
1011
409
|
}
|
|
1012
|
-
this.responseHandlers.set(messageId, handler);
|
|
1013
410
|
}
|
|
1014
411
|
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
412
|
+
/**
|
|
413
|
+
* Update closest peers for efficient routing
|
|
414
|
+
*/
|
|
415
|
+
updateClosestPeers() {
|
|
416
|
+
const connectedPeers = Array.from(this.mesh.connectionManager.peers.keys())
|
|
417
|
+
.filter(peerId => this.mesh.connectionManager.peers.get(peerId).getStatus() === 'connected');
|
|
1020
418
|
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
419
|
+
// Keep track of closest peers for efficient routing
|
|
420
|
+
const peersWithDistance = connectedPeers.map(peerId => ({
|
|
421
|
+
peerId,
|
|
422
|
+
distance: this.ringDistance(this.hashPosition, this.hashPeerId(peerId))
|
|
423
|
+
}));
|
|
1025
424
|
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
} else {
|
|
1033
|
-
this.debug.warn(`🔥 DHT Response: No handler found for messageId ${data.messageId}`);
|
|
1034
|
-
this.debug.log('🔥 DHT Response: Available handlers:', Array.from(this.responseHandlers.keys()));
|
|
1035
|
-
}
|
|
1036
|
-
} else {
|
|
1037
|
-
this.debug.warn('🔥 DHT Response: Missing responseHandlers or messageId', {
|
|
1038
|
-
hasHandlers: !!this.responseHandlers,
|
|
1039
|
-
messageId: data.messageId
|
|
1040
|
-
});
|
|
1041
|
-
}
|
|
425
|
+
peersWithDistance.sort((a, b) => a.distance - b.distance);
|
|
426
|
+
|
|
427
|
+
// Keep top 10 closest peers for efficient routing
|
|
428
|
+
this.closestPeers = new Set(
|
|
429
|
+
peersWithDistance.slice(0, 10).map(p => p.peerId)
|
|
430
|
+
);
|
|
1042
431
|
}
|
|
1043
432
|
|
|
1044
433
|
/**
|
|
1045
|
-
|
|
1046
|
-
|
|
434
|
+
* Periodic maintenance
|
|
435
|
+
*/
|
|
1047
436
|
startMaintenance() {
|
|
1048
|
-
//
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
437
|
+
// Update routing table every 30 seconds
|
|
438
|
+
setInterval(() => {
|
|
439
|
+
this.updateClosestPeers();
|
|
440
|
+
}, 30000);
|
|
441
|
+
|
|
442
|
+
// Clean up old data every 5 minutes
|
|
443
|
+
setInterval(() => {
|
|
444
|
+
this.cleanupOldData();
|
|
445
|
+
}, 300000);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* Clean up old data (optional TTL support)
|
|
450
|
+
*/
|
|
451
|
+
cleanupOldData() {
|
|
452
|
+
const now = Date.now();
|
|
453
|
+
const maxAge = 24 * 60 * 60 * 1000; // 24 hours
|
|
454
|
+
|
|
455
|
+
for (const [key, data] of this.storage) {
|
|
456
|
+
if (now - data.timestamp > maxAge) {
|
|
457
|
+
this.storage.delete(key);
|
|
458
|
+
this.debug.log(`Cleaned up old data: ${key}`);
|
|
1065
459
|
}
|
|
1066
460
|
}
|
|
1067
|
-
|
|
1068
|
-
// TODO: Republish keys if needed
|
|
1069
|
-
// TODO: Update routing table
|
|
1070
|
-
|
|
1071
|
-
this.debug.log(`DHT MAINTENANCE: ${this.localStorage.size} keys stored, ${this.subscriptions.size} subscriptions active`);
|
|
1072
461
|
}
|
|
1073
462
|
|
|
1074
463
|
/**
|
|
1075
|
-
|
|
1076
|
-
|
|
464
|
+
* Get DHT statistics
|
|
465
|
+
*/
|
|
1077
466
|
getStats() {
|
|
1078
467
|
return {
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
connectedPeers: this.mesh.connectionManager.getConnectedPeerCount()
|
|
468
|
+
localKeys: this.storage.size,
|
|
469
|
+
connectedPeers: this.mesh.connectionManager.getConnectedPeerCount(),
|
|
470
|
+
closestPeers: this.closestPeers.size,
|
|
471
|
+
hashPosition: this.hashPosition.toString(16).substring(0, 8)
|
|
1084
472
|
};
|
|
1085
473
|
}
|
|
1086
|
-
|
|
1087
|
-
/**
|
|
1088
|
-
* Cleanup DHT resources
|
|
1089
|
-
*/
|
|
1090
|
-
cleanup() {
|
|
1091
|
-
if (this.maintenanceInterval) {
|
|
1092
|
-
safeClearInterval(this.maintenanceInterval);
|
|
1093
|
-
this.maintenanceInterval = null;
|
|
1094
|
-
}
|
|
1095
|
-
|
|
1096
|
-
this.localStorage.clear();
|
|
1097
|
-
this.subscriptions.clear();
|
|
1098
|
-
this.pendingSubscriptions.clear();
|
|
1099
|
-
this.mySubscriptions.clear();
|
|
1100
|
-
|
|
1101
|
-
if (this.responseHandlers) {
|
|
1102
|
-
this.responseHandlers.clear();
|
|
1103
|
-
}
|
|
1104
|
-
|
|
1105
|
-
this.debug.log('WebDHT cleaned up');
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
/**
|
|
1109
|
-
* Perform iterative Kademlia lookup for GET operations
|
|
1110
|
-
* This improves success rate by querying multiple rounds of peers
|
|
1111
|
-
*/
|
|
1112
|
-
async performIterativeGet(keyId, key, subscribe = false) {
|
|
1113
|
-
const queriedPeers = new Set([this.peerId]);
|
|
1114
|
-
const closestPeers = await this.findClosestPeers(keyId, this.config.k);
|
|
1115
|
-
|
|
1116
|
-
// Remove self from the list
|
|
1117
|
-
const availablePeers = closestPeers.filter(peerId => peerId !== this.peerId);
|
|
1118
|
-
|
|
1119
|
-
this.debug.log(`DHT Iterative GET: Found ${availablePeers.length} potential peers for key ${key}`);
|
|
1120
|
-
|
|
1121
|
-
// Try querying peers in rounds, increasing the number each round
|
|
1122
|
-
const maxRounds = 3;
|
|
1123
|
-
let currentRound = 0;
|
|
1124
|
-
let peersPerRound = this.config.alpha; // Start with alpha peers
|
|
1125
|
-
|
|
1126
|
-
while (currentRound < maxRounds && queriedPeers.size < availablePeers.length + 1) {
|
|
1127
|
-
currentRound++;
|
|
1128
|
-
|
|
1129
|
-
// Get the next batch of peers to query
|
|
1130
|
-
const peersToQuery = availablePeers
|
|
1131
|
-
.filter(peerId => !queriedPeers.has(peerId))
|
|
1132
|
-
.slice(0, peersPerRound);
|
|
1133
|
-
|
|
1134
|
-
if (peersToQuery.length === 0) {
|
|
1135
|
-
this.debug.log(`DHT Iterative GET Round ${currentRound}: No more peers to query`);
|
|
1136
|
-
break;
|
|
1137
|
-
}
|
|
1138
|
-
|
|
1139
|
-
this.debug.log(`DHT Iterative GET Round ${currentRound}: Querying ${peersToQuery.length} peers`);
|
|
1140
|
-
|
|
1141
|
-
// Mark these peers as queried
|
|
1142
|
-
peersToQuery.forEach(peerId => queriedPeers.add(peerId));
|
|
1143
|
-
|
|
1144
|
-
// Query this batch of peers in parallel
|
|
1145
|
-
const queryPromises = peersToQuery.map(async (peerId) => {
|
|
1146
|
-
try {
|
|
1147
|
-
return await this.sendDHTMessage(peerId, 'find_value', {
|
|
1148
|
-
keyId,
|
|
1149
|
-
key,
|
|
1150
|
-
subscribe,
|
|
1151
|
-
requester: this.peerId
|
|
1152
|
-
});
|
|
1153
|
-
} catch (error) {
|
|
1154
|
-
this.debug.warn(`DHT GET: Failed to query peer ${peerId.substring(0, 8)}:`, error.message);
|
|
1155
|
-
return null;
|
|
1156
|
-
}
|
|
1157
|
-
});
|
|
1158
|
-
|
|
1159
|
-
const results = await Promise.allSettled(queryPromises);
|
|
1160
|
-
|
|
1161
|
-
// Collect all valid responses from this round
|
|
1162
|
-
const validResponses = [];
|
|
1163
|
-
for (let i = 0; i < results.length; i++) {
|
|
1164
|
-
const result = results[i];
|
|
1165
|
-
if (result.status === 'fulfilled' && result.value) {
|
|
1166
|
-
const data = result.value;
|
|
1167
|
-
if (data.found && !this.isExpired(data)) {
|
|
1168
|
-
validResponses.push({
|
|
1169
|
-
peerId: peersToQuery[i],
|
|
1170
|
-
data,
|
|
1171
|
-
timestamp: data.timestamp || 0
|
|
1172
|
-
});
|
|
1173
|
-
this.debug.log(`DHT Iterative GET: Found value on peer ${peersToQuery[i].substring(0, 8)} with timestamp ${data.timestamp}, value: ${JSON.stringify(data.value).substring(0, 50)}...`);
|
|
1174
|
-
}
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
// If we found any valid responses, find the most recent one
|
|
1179
|
-
if (validResponses.length > 0) {
|
|
1180
|
-
// Sort by timestamp descending to get the most recent value
|
|
1181
|
-
validResponses.sort((a, b) => b.timestamp - a.timestamp);
|
|
1182
|
-
const mostRecent = validResponses[0];
|
|
1183
|
-
|
|
1184
|
-
this.debug.log(`DHT Iterative GET: Selected most recent value from peer ${mostRecent.peerId.substring(0, 8)} (timestamp: ${mostRecent.timestamp}) in round ${currentRound}`);
|
|
1185
|
-
|
|
1186
|
-
if (validResponses.length > 1) {
|
|
1187
|
-
this.debug.log(`DHT Iterative GET: Found ${validResponses.length} versions, timestamps: ${validResponses.map(r => r.timestamp).join(', ')}`);
|
|
1188
|
-
// Log inconsistency warning if timestamps are very different
|
|
1189
|
-
const timeDiffs = validResponses.map(r => Math.abs(mostRecent.timestamp - r.timestamp));
|
|
1190
|
-
const maxTimeDiff = Math.max(...timeDiffs);
|
|
1191
|
-
if (maxTimeDiff > 10000) { // More than 10 seconds difference
|
|
1192
|
-
this.debug.warn(`DHT CONSISTENCY WARNING: Large timestamp differences found (max: ${maxTimeDiff}ms). This suggests replica peers have inconsistent data!`);
|
|
1193
|
-
this.debug.warn('DHT CONSISTENCY: Detailed responses:', validResponses.map(r => ({
|
|
1194
|
-
peer: r.peerId.substring(0, 8),
|
|
1195
|
-
timestamp: r.timestamp,
|
|
1196
|
-
value: JSON.stringify(r.data.value).substring(0, 30) + '...'
|
|
1197
|
-
})));
|
|
1198
|
-
}
|
|
1199
|
-
}
|
|
1200
|
-
|
|
1201
|
-
// Cache the most recent value locally
|
|
1202
|
-
this.storeLocally(keyId, mostRecent.data, false);
|
|
1203
|
-
|
|
1204
|
-
// ALWAYS add subscription when we successfully get a value
|
|
1205
|
-
if (subscribe) {
|
|
1206
|
-
this.mySubscriptions.add(keyId);
|
|
1207
|
-
this.debug.log(`DHT GET: Auto-subscribed to ${key}`);
|
|
1208
|
-
}
|
|
1209
|
-
|
|
1210
|
-
return mostRecent.data.value;
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
// Increase peers per round for next iteration (more aggressive search)
|
|
1214
|
-
peersPerRound = Math.min(peersPerRound * 2, Math.max(1, Math.floor(availablePeers.length / maxRounds)));
|
|
1215
|
-
}
|
|
1216
|
-
|
|
1217
|
-
this.debug.log(`DHT Iterative GET failed: key ${key} not found after querying ${queriedPeers.size - 1} peers`);
|
|
1218
|
-
return null;
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
/**
|
|
1222
|
-
* Perform iterative storage for PUT operations with better replication
|
|
1223
|
-
*/
|
|
1224
|
-
async performIterativePut(keyId, storeData, closestPeers) {
|
|
1225
|
-
const targetReplicas = this.config.replicationFactor;
|
|
1226
|
-
const availablePeers = closestPeers.filter(peerId => peerId !== this.peerId);
|
|
1227
|
-
|
|
1228
|
-
this.debug.log(`DHT Iterative PUT: Attempting to store ${storeData.key} on ${targetReplicas} of ${availablePeers.length} available peers`);
|
|
1229
|
-
|
|
1230
|
-
let successful = 0;
|
|
1231
|
-
let attempted = 0;
|
|
1232
|
-
const batchSize = Math.min(this.config.alpha, availablePeers.length);
|
|
1233
|
-
|
|
1234
|
-
// Try to store on peers in batches until we reach target replication
|
|
1235
|
-
for (let i = 0; i < availablePeers.length && successful < targetReplicas; i += batchSize) {
|
|
1236
|
-
const batch = availablePeers.slice(i, i + batchSize);
|
|
1237
|
-
attempted += batch.length;
|
|
1238
|
-
|
|
1239
|
-
this.debug.log(`DHT PUT Batch ${Math.floor(i / batchSize) + 1}: Storing on ${batch.length} peers`);
|
|
1240
|
-
|
|
1241
|
-
const storePromises = batch.map(async (peerId) => {
|
|
1242
|
-
try {
|
|
1243
|
-
const result = await this.sendDHTMessage(peerId, 'store', storeData);
|
|
1244
|
-
return { peerId, success: true, result };
|
|
1245
|
-
} catch (error) {
|
|
1246
|
-
this.debug.warn(`DHT PUT: Failed to store on peer ${peerId.substring(0, 8)}:`, error.message);
|
|
1247
|
-
return { peerId, success: false, error };
|
|
1248
|
-
}
|
|
1249
|
-
});
|
|
1250
|
-
|
|
1251
|
-
const results = await Promise.allSettled(storePromises);
|
|
1252
|
-
|
|
1253
|
-
// Count successful stores in this batch
|
|
1254
|
-
const batchSuccessful = results.filter(r =>
|
|
1255
|
-
r.status === 'fulfilled' && r.value && r.value.success
|
|
1256
|
-
).length;
|
|
1257
|
-
|
|
1258
|
-
successful += batchSuccessful;
|
|
1259
|
-
|
|
1260
|
-
this.debug.log(`DHT PUT Batch completed: ${batchSuccessful}/${batch.length} successful (total: ${successful}/${targetReplicas})`);
|
|
1261
|
-
|
|
1262
|
-
// If we've reached our target, we can stop
|
|
1263
|
-
if (successful >= targetReplicas) {
|
|
1264
|
-
break;
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
// Small delay between batches to avoid overwhelming the network
|
|
1268
|
-
if (i + batchSize < availablePeers.length) {
|
|
1269
|
-
await new Promise(resolve => safeSetTimeout(resolve, 100));
|
|
1270
|
-
}
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
this.debug.log(`DHT PUT completed: ${successful}/${targetReplicas} target replicas stored (${attempted} peers attempted)`);
|
|
1274
|
-
return successful > 0;
|
|
1275
|
-
}
|
|
1276
474
|
}
|