peerpigeon 0.1.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +141 -1365
  2. package/cli.js +1382 -0
  3. package/dist/peerpigeon-browser.js +12450 -0
  4. package/dist/peerpigeon-browser.js.map +7 -0
  5. package/docs/API_DOCUMENTATION.md +1976 -0
  6. package/docs/BOOTSTRAP_HUBS.md +319 -0
  7. package/docs/CLI_README.md +371 -0
  8. package/docs/CUSTOM_HUB_NAMESPACES.md +182 -0
  9. package/docs/CUSTOM_NAMESPACE_IMPLEMENTATION.md +187 -0
  10. package/docs/DOCUMENTATION_REORGANIZATION.md +172 -0
  11. package/docs/HUB_QUICK_REF.md +108 -0
  12. package/docs/HUB_SCRIPTS.md +341 -0
  13. package/docs/HUB_SYSTEM.md +599 -0
  14. package/docs/MIGRATION_COMPLETE.md +93 -0
  15. package/docs/NETWORK_NAMESPACES.md +212 -0
  16. package/docs/PIGEONRTC_MIGRATION.md +280 -0
  17. package/docs/README.md +108 -0
  18. package/docs/SELECTIVE_STREAMING_GUIDE.md +427 -0
  19. package/docs/TEST_CLEANUP_COMPLETE.md +80 -0
  20. package/examples/bootstrap-hub-example.js +292 -0
  21. package/examples/browser/README.md +278 -0
  22. package/examples/browser/app.js +2426 -49
  23. package/examples/browser/index.html +481 -630
  24. package/examples/browser/minimal/README.md +55 -0
  25. package/examples/browser/minimal/index.html +75 -0
  26. package/examples/browser/minimal/minimal.js +234 -0
  27. package/examples/browser/styles.css +687 -1425
  28. package/examples/browser/test-bundle.html +35 -0
  29. package/examples/browser-2/index.html +30 -0
  30. package/examples/browser-2/package.json +21 -0
  31. package/examples/browser-2/src/App-simple.vue +21 -0
  32. package/examples/browser-2/src/App.vue +420 -0
  33. package/examples/browser-2/src/components/CryptoView.vue +1661 -0
  34. package/examples/browser-2/src/components/DHTView.vue +1281 -0
  35. package/examples/browser-2/src/components/DebugView.vue +1773 -0
  36. package/examples/browser-2/src/components/MediaView.vue +1048 -0
  37. package/examples/browser-2/src/components/MessagingView.vue +775 -0
  38. package/examples/browser-2/src/components/NetworkView.vue +930 -0
  39. package/examples/browser-2/src/components/StorageView.vue +1376 -0
  40. package/examples/browser-2/src/main.js +5 -0
  41. package/examples/browser-2/src/stores/peerpigeon.js +895 -0
  42. package/examples/browser-2/vite.config.js +6 -0
  43. package/examples/hub-example.js +189 -0
  44. package/examples/multi-hub-test.js +186 -0
  45. package/examples/node/README.md +2 -2
  46. package/examples/programmatic-namespace-example.js +108 -0
  47. package/examples/selective-streaming-demo.html +698 -0
  48. package/examples/standard-events-demo.js +126 -0
  49. package/index.js +7 -1
  50. package/package.json +34 -16
  51. package/server/index.js +1058 -98
  52. package/server/start.js +52 -0
  53. package/src/ConnectionManager.js +239 -225
  54. package/src/CryptoManager.js +147 -18
  55. package/src/DebugLogger.js +0 -7
  56. package/src/DistributedStorageManager.js +224 -205
  57. package/src/EnvironmentDetector.js +88 -4
  58. package/src/EventEmitter.js +79 -1
  59. package/src/EvictionManager.js +10 -9
  60. package/src/GossipManager.js +163 -11
  61. package/src/MediaManager.js +12 -2
  62. package/src/PeerConnection.js +343 -311
  63. package/src/PeerPigeonMesh.js +576 -135
  64. package/src/PigeonRTC-browser.js +247 -0
  65. package/src/SignalingClient.js +14 -243
  66. package/src/SignalingHandler.js +31 -10
  67. package/src/StorageManager.js +0 -39
  68. package/src/WebDHT-Simple.js +389 -0
  69. package/src/WebDHT.js +363 -1165
  70. package/src/browser-entry.js +58 -0
  71. package/examples/browser/assets/images/favicon.png +0 -0
  72. package/examples/browser/assets/images/pigeon.png +0 -0
  73. package/examples/browser/crypto-test.html +0 -427
  74. package/examples/browser/ui-old.js +0 -1926
  75. package/examples/browser/ui.js +0 -3391
  76. package/websocket-server/README.md +0 -247
  77. package/websocket-server/server.js +0 -391
@@ -9,16 +9,29 @@ async function initializeUnsea() {
9
9
  if (unsea) return unsea;
10
10
 
11
11
  try {
12
+ // Check if UnSEA is bundled (browser bundle case)
13
+ if (typeof window !== 'undefined' && window.__PEERPIGEON_UNSEA__) {
14
+ unsea = window.__PEERPIGEON_UNSEA__;
15
+ console.log('✅ Using bundled UnSEA crypto for storage');
16
+ return unsea;
17
+ }
18
+
19
+ // Check global scope for bundled version
20
+ if (typeof globalThis !== 'undefined' && globalThis.__PEERPIGEON_UNSEA__) {
21
+ unsea = globalThis.__PEERPIGEON_UNSEA__;
22
+ console.log('✅ Using bundled UnSEA crypto for storage');
23
+ return unsea;
24
+ }
25
+
12
26
  // Detect environment - prioritize Node.js detection for tests
13
27
  const isNode = typeof process !== 'undefined' && process.versions && process.versions.node;
14
- const isBrowser = !isNode && typeof window !== 'undefined' && typeof document !== 'undefined';
15
28
 
16
29
  if (isNode) {
17
30
  // For Node.js environments (including tests), use npm package
18
31
  unsea = await import('unsea');
19
32
  console.log('✅ Loaded unsea from npm package (Node.js) for storage');
20
- } else if (isBrowser) {
21
- // For browser environments, try CDN sources
33
+ } else {
34
+ // For browser environments without bundle, try CDN sources as fallback
22
35
  try {
23
36
  unsea = await import('https://cdn.jsdelivr.net/npm/unsea@latest/+esm');
24
37
  console.log('✅ Loaded unsea from jsDelivr CDN for storage');
@@ -33,8 +46,6 @@ async function initializeUnsea() {
33
46
  console.log('✅ Loaded unsea from Skypack CDN for storage');
34
47
  }
35
48
  }
36
- } else {
37
- throw new Error('Unknown environment - cannot load unsea');
38
49
  }
39
50
 
40
51
  if (!unsea) {
@@ -49,22 +60,33 @@ async function initializeUnsea() {
49
60
  }
50
61
 
51
62
  /**
52
- * DistributedStorageManager - Optional storage layer for PeerPigeon mesh
63
+ * DistributedStorageManager - High-level distributed storage layer for PeerPigeon mesh
53
64
  *
54
65
  * Features:
55
66
  * - Encrypted storage using unsea directly
56
- * - Optional public/private data visibility
67
+ * - Optional public/private data visibility with access control
57
68
  * - Optional immutability with CRDT support for collaborative editing
58
69
  * - Mutable layer for data originators
59
- * - Integration with WebDHT for distributed storage
70
+ * - Uses WebDHT as low-level storage backend (but is separate conceptually)
60
71
  * - Space separation for preventing data overwrites (Private, Public, Frozen)
72
+ *
73
+ * IMPORTANT: This is a HIGH-LEVEL storage system that provides encryption,
74
+ * access control, and advanced features. It uses WebDHT as its storage backend
75
+ * but they are separate concepts:
76
+ * - WebDHT: Low-level distributed hash table for raw key-value storage
77
+ * - DistributedStorageManager: High-level storage with encryption/access control
61
78
  */
62
79
  export class DistributedStorageManager extends EventEmitter {
63
80
  constructor(mesh) {
64
81
  super();
65
82
  this.debug = DebugLogger.create('DistributedStorageManager');
66
83
  this.mesh = mesh;
67
- this.webDHT = mesh.webDHT;
84
+
85
+ // IMPORTANT: DistributedStorageManager uses WebDHT as its storage backend
86
+ // but they are conceptually separate systems:
87
+ // - WebDHT: Low-level DHT for raw key-value storage across the mesh
88
+ // - DistributedStorageManager: High-level storage with encryption, access control, spaces
89
+ this.webDHT = mesh.webDHT; // Uses WebDHT as backend storage layer
68
90
  this.cryptoManager = mesh.cryptoManager;
69
91
 
70
92
  // Initialize unsea for encryption
@@ -97,7 +119,7 @@ export class DistributedStorageManager extends EventEmitter {
97
119
  // Track data ownership by space
98
120
  this.ownedKeys = new Set(); // Keys owned by this peer
99
121
  this.spaceOwnership = new Map(); // space:key -> owner mapping
100
- this.keyToSpaceMapping = new Map(); // baseKey -> {space, fullKey} mapping for transparent access
122
+ this.keyToSpaceMapping = new Map(); // baseKey -> {space, storageKey} mapping for transparent access
101
123
 
102
124
  // Track enabled state
103
125
  this.enabled = true; // Enable by default
@@ -149,102 +171,50 @@ export class DistributedStorageManager extends EventEmitter {
149
171
 
150
172
  /**
151
173
  * Resolve a key to its actual storage location across spaces
152
- * @param {string} key - The storage key (can be plain or space-prefixed)
153
- * @returns {Object} - {space, baseKey, fullKey, keyId}
174
+ * @param {string} key - The storage key (base key only)
175
+ * @returns {Object} - {space, baseKey, keyId}
154
176
  */
155
177
  async resolveKey(key) {
156
- // If key already has space prefix, use it as-is
157
- const keyInfo = this.parseStorageKey(key);
158
-
159
- // If it's not a bare key (has space prefix), use it directly
160
- if (key.includes(':')) {
161
- const keyId = await this.webDHT.generateKeyId(keyInfo.fullKey);
162
- return {
163
- space: keyInfo.space,
164
- baseKey: keyInfo.baseKey,
165
- fullKey: keyInfo.fullKey,
166
- keyId
167
- };
168
- }
178
+ // Use the key directly - NO PREFIX PARSING
179
+ const baseKey = key;
180
+
181
+ // Generate keyId from baseKey (no prefixes)
182
+ const keyId = await this.webDHT.hash(baseKey);
169
183
 
170
184
  // For bare keys, check if we have a mapping
171
- const mapping = this.keyToSpaceMapping.get(keyInfo.baseKey);
185
+ const mapping = this.keyToSpaceMapping.get(baseKey);
172
186
  if (mapping) {
173
- const keyId = await this.webDHT.generateKeyId(mapping.fullKey);
174
187
  return {
175
188
  space: mapping.space,
176
- baseKey: keyInfo.baseKey,
177
- fullKey: mapping.fullKey,
189
+ baseKey,
178
190
  keyId
179
191
  };
180
192
  }
181
193
 
182
- // If no mapping exists, try to find it by checking each space
183
- for (const space of Object.values(this.spaces)) {
184
- const testFullKey = this.createSpacedKey(space, keyInfo.baseKey);
185
- const testKeyId = await this.webDHT.generateKeyId(testFullKey);
186
-
187
- // Check if we have metadata for this key in this space
188
- if (this.storageMetadata.has(testKeyId) || this.spaceOwnership.has(testFullKey)) {
189
- // Update the mapping for future use
190
- this.keyToSpaceMapping.set(keyInfo.baseKey, { space, fullKey: testFullKey });
191
- return {
192
- space,
193
- baseKey: keyInfo.baseKey,
194
- fullKey: testFullKey,
195
- keyId: testKeyId
196
- };
197
- }
194
+ // If no mapping exists, check if we have metadata for this key
195
+ if (this.storageMetadata.has(keyId)) {
196
+ const metadata = this.storageMetadata.get(keyId);
197
+ const space = metadata.space || this.spaces.PRIVATE;
198
+ // Update the mapping for future use
199
+ this.keyToSpaceMapping.set(baseKey, { space, storageKey: baseKey });
200
+ return {
201
+ space,
202
+ baseKey,
203
+ keyId
204
+ };
198
205
  }
199
206
 
200
- // If still not found, default to the parsed key (likely private space)
201
- const keyId = await this.webDHT.generateKeyId(keyInfo.fullKey);
207
+ // Default to private space if no other information is available
208
+ const defaultSpace = this.spaces.PRIVATE;
209
+ this.keyToSpaceMapping.set(baseKey, { space: defaultSpace, storageKey: baseKey });
202
210
  return {
203
- space: keyInfo.space,
204
- baseKey: keyInfo.baseKey,
205
- fullKey: keyInfo.fullKey,
211
+ space: defaultSpace,
212
+ baseKey,
206
213
  keyId
207
214
  };
208
215
  }
209
216
 
210
- /**
211
- * Parse a storage key to extract space and base key
212
- * @param {string} key - The storage key (can be plain or space-prefixed)
213
- * @returns {Object} - {space, baseKey, fullKey}
214
- */
215
- parseStorageKey(key) {
216
- // Check if key has space prefix (e.g., "private:user:123" or "public:config")
217
- const spacePrefixes = Object.values(this.spaces);
218
- for (const space of spacePrefixes) {
219
- if (key.startsWith(`${space}:`)) {
220
- return {
221
- space,
222
- baseKey: key.substring(space.length + 1),
223
- fullKey: key
224
- };
225
- }
226
- }
227
-
228
- // Default to private space if no prefix specified
229
- return {
230
- space: this.spaces.PRIVATE,
231
- baseKey: key,
232
- fullKey: `${this.spaces.PRIVATE}:${key}`
233
- };
234
- }
235
217
 
236
- /**
237
- * Create a space-prefixed key
238
- * @param {string} space - The storage space
239
- * @param {string} baseKey - The base key
240
- * @returns {string} - The full key with space prefix
241
- */
242
- createSpacedKey(space, baseKey) {
243
- if (!Object.values(this.spaces).includes(space)) {
244
- throw new Error(`Invalid storage space: ${space}. Must be one of: ${Object.values(this.spaces).join(', ')}`);
245
- }
246
- return `${space}:${baseKey}`;
247
- }
248
218
 
249
219
  /**
250
220
  * Check if a peer can access a key in a specific space
@@ -332,14 +302,21 @@ export class DistributedStorageManager extends EventEmitter {
332
302
  * @returns {string|null} - The space where the key exists, or null if not found
333
303
  */
334
304
  findKeyInDifferentSpace(baseKey, excludeSpace) {
335
- for (const space of Object.values(this.spaces)) {
336
- if (space !== excludeSpace) {
337
- const spacedKey = this.createSpacedKey(space, baseKey);
338
- if (this.spaceOwnership.has(spacedKey)) {
339
- return space;
340
- }
305
+ // Check our metadata to see if this baseKey exists in a different space
306
+ for (const metadata of this.storageMetadata.values()) {
307
+ if (metadata.baseKey === baseKey && metadata.space !== excludeSpace) {
308
+ return metadata.space;
309
+ }
310
+ }
311
+
312
+ // Also check space ownership using baseKey directly
313
+ if (this.spaceOwnership.has(baseKey)) {
314
+ const mapping = this.keyToSpaceMapping.get(baseKey);
315
+ if (mapping && mapping.space !== excludeSpace) {
316
+ return mapping.space;
341
317
  }
342
318
  }
319
+
343
320
  return null;
344
321
  }
345
322
 
@@ -417,31 +394,24 @@ export class DistributedStorageManager extends EventEmitter {
417
394
  await this.waitForCrypto();
418
395
  }
419
396
 
420
- // Parse key and determine storage space
421
- const keyInfo = this.parseStorageKey(key);
397
+ // Use the key directly - NO PREFIX PARSING
422
398
  const spaceConfig = this.determineSpaceFromOptions(options);
423
399
 
424
- // Use explicit space from options if provided, otherwise use parsed space
400
+ // Use the base key directly - NO SPACE PREFIXES!
401
+ const baseKey = key;
425
402
  const space = options.space || spaceConfig.space;
426
- const fullKey = keyInfo.space === space ? keyInfo.fullKey : this.createSpacedKey(space, keyInfo.baseKey);
427
- const baseKey = keyInfo.baseKey;
403
+
404
+ // The stored key is just the base key - space info goes in metadata
405
+ const storageKey = baseKey;
428
406
 
429
407
  // Check space access permissions
430
408
  if (!this.canAccessSpace(space, baseKey, this.mesh.peerId, 'write')) {
431
409
  throw new Error(`Write access denied for space "${space}" and key "${baseKey}"`);
432
410
  }
433
411
 
434
- // Check if key already exists in a different space
435
- if (this.config.spaceEnforcement) {
436
- const existingSpaceKey = this.findKeyInDifferentSpace(baseKey, space);
437
- if (existingSpaceKey) {
438
- throw new Error(`Key "${baseKey}" already exists in space "${existingSpaceKey}" - cannot store in "${space}" space`);
439
- }
440
- }
441
-
442
- const keyId = await this.webDHT.generateKeyId(fullKey);
412
+ const keyId = await this.webDHT.hash(storageKey);
443
413
  const timestamp = Date.now();
444
- this.debug.log(`📦 Storing key: ${fullKey}, keyId: ${keyId.substring(0, 8)}...`);
414
+ this.debug.log(`📦 Storing key: ${storageKey} in ${space} space, keyId: ${keyId.toString(16).substring(0, 8)}...`);
445
415
 
446
416
  // Validate value size
447
417
  const serializedValue = JSON.stringify(value);
@@ -449,9 +419,9 @@ export class DistributedStorageManager extends EventEmitter {
449
419
  throw new Error(`Value size exceeds maximum allowed size of ${this.config.maxValueSize} bytes`);
450
420
  }
451
421
 
452
- // Create storage metadata with space information
422
+ // Create storage metadata with type and space information
453
423
  const metadata = {
454
- key: fullKey,
424
+ key: baseKey, // Just the base key, no prefixes - space is its own attribute
455
425
  baseKey,
456
426
  space,
457
427
  keyId,
@@ -463,18 +433,20 @@ export class DistributedStorageManager extends EventEmitter {
463
433
  createdAt: timestamp,
464
434
  updatedAt: timestamp,
465
435
  version: 1,
466
- ttl: options.ttl || this.config.defaultTTL
436
+ ttl: options.ttl || this.config.defaultTTL,
437
+ type: 'storage', // Mark this as storage data type
438
+ dataType: 'distributed-storage' // Specific storage system identifier
467
439
  };
468
440
 
469
441
  // Store metadata locally
470
442
  this.storageMetadata.set(keyId, metadata);
471
443
  this.ownedKeys.add(keyId);
472
444
 
473
- // Track space ownership
474
- this.spaceOwnership.set(fullKey, this.mesh.peerId);
445
+ // Track space ownership using just the base key
446
+ this.spaceOwnership.set(baseKey, this.mesh.peerId);
475
447
 
476
448
  // Track base key to space mapping for transparent access
477
- this.keyToSpaceMapping.set(baseKey, { space, fullKey });
449
+ this.keyToSpaceMapping.set(baseKey, { space, storageKey });
478
450
 
479
451
  // Set up access control
480
452
  this.accessControl.set(keyId, {
@@ -512,33 +484,47 @@ export class DistributedStorageManager extends EventEmitter {
512
484
  encrypted: true,
513
485
  encryptedBy: this.mesh.peerId // Track who encrypted it
514
486
  };
515
- this.debug.log(`📦 Encrypted private space storage data for key: ${fullKey} (owner-only access)`);
487
+ this.debug.log(`📦 Encrypted private space storage data for key: ${storageKey} (owner-only access)`);
516
488
  } catch (error) {
517
- this.debug.warn(`Failed to encrypt private space storage data for key ${fullKey}:`, error);
489
+ this.debug.warn(`Failed to encrypt private space storage data for key ${storageKey}:`, error);
518
490
  // Fall back to unencrypted storage if encryption fails
519
491
  }
520
492
  } else {
521
- this.debug.log(`📦 Storing ${space} space data for key: ${fullKey}`);
522
- }
493
+ this.debug.log(`📦 Storing ${space} space data for key: ${storageKey}`);
494
+ }
495
+
496
+ // CRITICAL DEBUG: Log exactly what we're storing
497
+ this.debug.log(`📦 STORING PAYLOAD STRUCTURE for key ${storageKey}:`, {
498
+ hasValue: !!storagePayload.value,
499
+ hasMetadata: !!storagePayload.metadata,
500
+ valueType: typeof storagePayload.value,
501
+ metadataType: typeof storagePayload.metadata,
502
+ metadataKeys: storagePayload.metadata ? Object.keys(storagePayload.metadata) : 'none',
503
+ encrypted: storagePayload.encrypted,
504
+ payloadKeys: Object.keys(storagePayload)
505
+ });
523
506
 
524
- // Store in WebDHT with space prefix
507
+ // Store in WebDHT with storage namespace prefix to separate from raw DHT data
508
+ // This maintains separation between low-level DHT operations and high-level storage
525
509
  try {
526
- this.debug.log(`📦 Storing payload for key ${fullKey}:`, {
510
+ this.debug.log(`📦 Storing payload for key ${storageKey}:`, {
527
511
  hasValue: !!storagePayload.value,
528
512
  hasMetadata: !!storagePayload.metadata,
529
513
  encrypted: storagePayload.encrypted,
530
514
  space
531
515
  });
532
516
 
533
- await this.webDHT.put(`storage:${fullKey}`, storagePayload, {
534
- ttl: metadata.ttl
517
+ // Store directly with clean key - NO PREFIXES, type information is in metadata
518
+ await this.webDHT.put(storageKey, storagePayload, {
519
+ ttl: metadata.ttl,
520
+ space: space // Pass space for space-aware replication
535
521
  });
536
522
 
537
- this.debug.log(`📦 Stored ${space} space data for key: ${fullKey}`);
523
+ this.debug.log(`📦 Stored ${space} space data for key: ${storageKey}`);
538
524
 
539
525
  // Emit storage event
540
526
  this.emit('dataStored', {
541
- key: fullKey,
527
+ key: storageKey,
542
528
  baseKey,
543
529
  space,
544
530
  keyId,
@@ -549,13 +535,13 @@ export class DistributedStorageManager extends EventEmitter {
549
535
 
550
536
  return true;
551
537
  } catch (error) {
552
- this.debug.error(`Failed to store data for key ${fullKey}:`, error);
538
+ this.debug.error(`Failed to store data for key ${storageKey}:`, error);
553
539
  // Clean up local metadata on failure
554
540
  this.storageMetadata.delete(keyId);
555
541
  this.ownedKeys.delete(keyId);
556
542
  this.accessControl.delete(keyId);
557
543
  this.crdtStates.delete(keyId);
558
- this.spaceOwnership.delete(fullKey);
544
+ this.spaceOwnership.delete(baseKey);
559
545
  this.keyToSpaceMapping.delete(baseKey);
560
546
  throw error;
561
547
  }
@@ -563,8 +549,9 @@ export class DistributedStorageManager extends EventEmitter {
563
549
 
564
550
  /**
565
551
  * Retrieve data with access control and decryption
566
- * @param {string} key - The storage key
552
+ * @param {string} key - The storage key (base key only, no prefixes)
567
553
  * @param {Object} options - Retrieval options
554
+ * @param {string} options.space - Specific space to look in (optional)
568
555
  * @param {boolean} options.forceRefresh - Force refresh from network
569
556
  * @returns {Promise<any>} The stored value or null if not accessible
570
557
  */
@@ -578,70 +565,102 @@ export class DistributedStorageManager extends EventEmitter {
578
565
  await this.waitForCrypto();
579
566
  }
580
567
 
581
- // Resolve the key to its actual storage location
582
- const resolved = await this.resolveKey(key);
583
- const { space, baseKey, fullKey, keyId } = resolved;
568
+ // Use the key directly - NO PREFIX STRIPPING
569
+ const baseKey = key;
570
+
571
+ this.debug.log(`📦 Retrieving data for base key: ${baseKey}`);
584
572
 
585
- // Check space access permissions
586
- if (!this.canAccessSpace(space, baseKey, this.mesh.peerId, 'read')) {
587
- this.debug.warn(`📦 Access denied for space "${space}" and key "${baseKey}"`);
573
+ try {
574
+ // Get data from WebDHT using just the clean base key - NO PREFIXES
575
+ const webDHTPayload = await this.webDHT.get(baseKey, {
576
+ forceRefresh: options.forceRefresh,
577
+ space: options.space // Pass space for space-aware replication
578
+ });
579
+
580
+ this.debug.log(`📦 Retrieved WebDHT payload for key ${baseKey}:`, {
581
+ payloadExists: !!webDHTPayload,
582
+ payloadType: typeof webDHTPayload,
583
+ keys: webDHTPayload ? Object.keys(webDHTPayload) : 'none'
584
+ });
585
+
586
+ if (!webDHTPayload || typeof webDHTPayload !== 'object') {
587
+ this.debug.log(`📦 No data found for key: ${baseKey}`);
588
+ return null;
589
+ }
590
+
591
+ // Process the data using the metadata to determine space and access control
592
+ return await this.processRetrievedData(baseKey, webDHTPayload);
593
+ } catch (error) {
594
+ this.debug.error(`Failed to retrieve data for key ${baseKey}:`, error);
588
595
  return null;
589
596
  }
597
+ }
590
598
 
591
- this.debug.log(`📦 Retrieving key: ${fullKey}, keyId: ${keyId.substring(0, 8)}...`);
592
-
599
+ /**
600
+ * Process retrieved data payload with decryption and access control
601
+ * @private
602
+ */
603
+ async processRetrievedData(baseKey, webDHTPayload) {
593
604
  try {
594
- // Get data from WebDHT
595
- this.debug.log(`📦 Attempting to retrieve data for key: ${fullKey}`);
596
- const storagePayload = await this.webDHT.get(`storage:${fullKey}`, {
597
- forceRefresh: options.forceRefresh
598
- });
599
-
600
- this.debug.log(`📦 Retrieved payload for key ${fullKey}:`, {
601
- payloadExists: !!storagePayload,
602
- payloadType: typeof storagePayload,
603
- space,
604
- keys: storagePayload ? Object.keys(storagePayload) : 'none'
605
+ // Generate keyId for this baseKey
606
+ const keyId = await this.webDHT.hash(baseKey);
607
+
608
+ // WebDHT returns the storage payload directly as {value, metadata, encrypted}
609
+ const storagePayload = webDHTPayload;
610
+
611
+ console.log(`📦 CRITICAL DEBUG: WebDHT payload structure for ${baseKey}:`, {
612
+ webDHTKeys: Object.keys(webDHTPayload),
613
+ payloadType: typeof webDHTPayload,
614
+ directValue: !!webDHTPayload.value,
615
+ directMetadata: !!webDHTPayload.metadata,
616
+ directEncrypted: webDHTPayload.encrypted
605
617
  });
606
618
 
607
619
  if (!storagePayload || typeof storagePayload !== 'object') {
608
- this.debug.log(`📦 No data found for key: ${fullKey}`);
620
+ console.log(`📦 CRITICAL: Invalid storage payload for key: ${baseKey}`);
621
+ this.debug.log(`📦 Invalid storage payload for key: ${baseKey}`);
609
622
  return null;
610
623
  }
611
624
 
612
- // Debug: Log the raw payload structure
613
- this.debug.log('📦 Raw payload structure:', JSON.stringify(storagePayload, null, 2));
614
-
615
625
  // Safely destructure the payload with detailed logging
616
626
  const value = storagePayload.value;
617
627
  const metadata = storagePayload.metadata;
618
628
  const encrypted = storagePayload.encrypted || false;
619
629
  const encryptedBy = storagePayload.encryptedBy;
620
630
 
631
+ if (!metadata) {
632
+ this.debug.warn(`📦 Invalid storage payload for key ${baseKey}: missing metadata`);
633
+ return null;
634
+ }
635
+
636
+ // Validate this is actually storage data by checking metadata type
637
+ if (!metadata.type || metadata.type !== 'storage') {
638
+ this.debug.log(`📦 Data for key ${baseKey} is not storage data (type: ${metadata.type || 'unknown'}) - ignoring`);
639
+ return null;
640
+ }
641
+
642
+ // Get space from metadata - this is where space info is stored!
643
+ const space = metadata.space;
644
+
621
645
  this.debug.log('📦 Payload components:', {
622
646
  hasValue: value !== undefined,
623
647
  valueType: typeof value,
624
648
  hasMetadata: metadata !== undefined,
625
649
  metadataType: typeof metadata,
626
650
  encrypted,
627
- space: metadata?.space || 'unknown',
651
+ space: space || 'unknown',
628
652
  encryptedBy: encryptedBy?.substring(0, 8) + '...' || 'unknown'
629
653
  });
630
654
 
631
- if (!metadata) {
632
- this.debug.warn(`📦 Invalid storage payload for key ${fullKey}: missing metadata`);
633
- return null;
634
- }
635
-
636
- // Verify space consistency
637
- if (metadata.space && metadata.space !== space) {
638
- this.debug.warn(`📦 Space mismatch for key ${fullKey}: expected ${space}, got ${metadata.space}`);
655
+ // Check space access permissions using space from metadata
656
+ if (!this.canAccessSpace(space, baseKey, this.mesh.peerId, 'read')) {
657
+ this.debug.warn(`📦 Access denied for key: ${baseKey} in space: ${space}`);
639
658
  return null;
640
659
  }
641
660
 
642
661
  // Check access permissions using space-aware logic
643
662
  if (!this.hasReadAccessWithSpace(keyId, metadata, space)) {
644
- this.debug.warn(`📦 Access denied for key: ${fullKey} in space: ${space}`);
663
+ this.debug.warn(`📦 Access denied for key: ${baseKey} in space: ${space}`);
645
664
  return null;
646
665
  }
647
666
 
@@ -650,10 +669,10 @@ export class DistributedStorageManager extends EventEmitter {
650
669
  if (encrypted && this.unsea && this.storageKeypair) {
651
670
  // Only allow decryption if this peer is the owner (encrypted the data)
652
671
  if (encryptedBy && encryptedBy !== this.mesh.peerId) {
653
- this.debug.warn(`📦 Cannot decrypt data for key ${key}: encrypted by different peer (${encryptedBy.substring(0, 8)}...), current peer: ${this.mesh.peerId.substring(0, 8)}...`);
672
+ this.debug.warn(`📦 Cannot decrypt data for key ${baseKey}: encrypted by different peer (${encryptedBy.substring(0, 8)}...), current peer: ${this.mesh.peerId.substring(0, 8)}...`);
654
673
  // For private data encrypted by another peer, deny access
655
674
  if (!metadata.isPublic) {
656
- this.debug.warn(`Access denied for key: ${key} - private data encrypted by different peer`);
675
+ this.debug.warn(`Access denied for key: ${baseKey} - private data encrypted by different peer`);
657
676
  return null;
658
677
  }
659
678
  }
@@ -661,15 +680,15 @@ export class DistributedStorageManager extends EventEmitter {
661
680
  try {
662
681
  const decryptedValue = await this.unsea.decryptMessageWithMeta(value, this.storageKeypair.epriv);
663
682
  finalValue = JSON.parse(decryptedValue);
664
- this.debug.log(`🔓 Decrypted storage data for key: ${key}`);
683
+ this.debug.log(`🔓 Decrypted storage data for key: ${baseKey}`);
665
684
  } catch (error) {
666
- this.debug.error(`Failed to decrypt data for key ${key}:`, error);
685
+ this.debug.error(`Failed to decrypt data for key ${baseKey}:`, error);
667
686
  // If this is private data and decryption fails, deny access
668
687
  if (!metadata.isPublic) {
669
688
  return null;
670
689
  }
671
690
  // For public data, if decryption fails, try to use the raw value
672
- this.debug.warn(`📦 Using raw value for public key ${key} due to decryption failure`);
691
+ this.debug.warn(`📦 Using raw value for public key ${baseKey} due to decryption failure`);
673
692
  finalValue = value;
674
693
  }
675
694
  }
@@ -685,22 +704,20 @@ export class DistributedStorageManager extends EventEmitter {
685
704
  space: metadata.space || space
686
705
  });
687
706
 
688
- // Track space ownership from retrieved metadata
689
- if (metadata.owner && !this.spaceOwnership.has(fullKey)) {
690
- this.spaceOwnership.set(fullKey, metadata.owner);
707
+ // Track space ownership from retrieved metadata using baseKey
708
+ if (metadata.owner && !this.spaceOwnership.has(baseKey)) {
709
+ this.spaceOwnership.set(baseKey, metadata.owner);
691
710
  }
692
711
 
693
712
  // Update key mapping for future transparent access
694
- if (!key.includes(':')) {
695
- this.keyToSpaceMapping.set(baseKey, { space, fullKey });
696
- }
713
+ this.keyToSpaceMapping.set(baseKey, { space, storageKey: baseKey });
697
714
  }
698
715
 
699
- this.debug.log(`📦 Retrieved ${space} space data for key: ${fullKey}`);
716
+ this.debug.log(`📦 Retrieved ${space} space data for key: ${baseKey}`);
700
717
 
701
718
  // Emit retrieval event
702
719
  this.emit('dataRetrieved', {
703
- key: fullKey,
720
+ key: baseKey,
704
721
  baseKey,
705
722
  space,
706
723
  keyId,
@@ -710,7 +727,7 @@ export class DistributedStorageManager extends EventEmitter {
710
727
 
711
728
  return finalValue;
712
729
  } catch (error) {
713
- this.debug.error(`Failed to retrieve data for key ${key}:`, error);
730
+ this.debug.error(`Failed to retrieve data for key ${baseKey}:`, error);
714
731
  return null;
715
732
  }
716
733
  }
@@ -730,7 +747,7 @@ export class DistributedStorageManager extends EventEmitter {
730
747
 
731
748
  // Resolve the key to its actual storage location
732
749
  const resolved = await this.resolveKey(key);
733
- const { fullKey, keyId } = resolved;
750
+ const { baseKey, keyId } = resolved;
734
751
  const timestamp = Date.now();
735
752
 
736
753
  // Check if we have access to update
@@ -739,7 +756,7 @@ export class DistributedStorageManager extends EventEmitter {
739
756
 
740
757
  if (!accessControl || !metadata) {
741
758
  // Try to retrieve metadata first
742
- const existingData = await this.retrieve(fullKey);
759
+ const existingData = await this.retrieve(baseKey);
743
760
  if (!existingData) {
744
761
  throw new Error(`Key ${key} does not exist or is not accessible`);
745
762
  }
@@ -798,19 +815,19 @@ export class DistributedStorageManager extends EventEmitter {
798
815
  }
799
816
 
800
817
  try {
801
- // Use WebDHT update to notify all replicas and subscribers
802
- await this.webDHT.update(`storage:${fullKey}`, storagePayload, {
818
+ // Use WebDHT update to notify all replicas and subscribers - NO PREFIXES
819
+ await this.webDHT.update(baseKey, storagePayload, {
803
820
  ttl: metadata.ttl
804
821
  });
805
822
 
806
823
  // Update local metadata
807
824
  this.storageMetadata.set(keyId, metadata);
808
825
 
809
- this.debug.log(`📦 Updated ${metadata.isPublic ? 'public' : 'private'} data for key: ${fullKey}`);
826
+ this.debug.log(`📦 Updated ${metadata.isPublic ? 'public' : 'private'} data for key: ${baseKey}`);
810
827
 
811
828
  // Emit update event
812
829
  this.emit('dataUpdated', {
813
- key: fullKey,
830
+ key: baseKey,
814
831
  keyId,
815
832
  isPublic: metadata.isPublic,
816
833
  version: metadata.version,
@@ -819,7 +836,7 @@ export class DistributedStorageManager extends EventEmitter {
819
836
 
820
837
  return true;
821
838
  } catch (error) {
822
- this.debug.error(`Failed to update data for key ${fullKey}:`, error);
839
+ this.debug.error(`Failed to update data for key ${baseKey}:`, error);
823
840
  throw error;
824
841
  }
825
842
  }
@@ -900,7 +917,7 @@ export class DistributedStorageManager extends EventEmitter {
900
917
 
901
918
  // Resolve the key to its actual storage location
902
919
  const resolved = await this.resolveKey(key);
903
- const { baseKey, fullKey, keyId } = resolved;
920
+ const { baseKey, keyId } = resolved;
904
921
 
905
922
  const accessControl = this.accessControl.get(keyId);
906
923
 
@@ -916,7 +933,8 @@ export class DistributedStorageManager extends EventEmitter {
916
933
  deletedBy: this.mesh.peerId
917
934
  };
918
935
 
919
- await this.webDHT.update(`storage:${fullKey}`, tombstone);
936
+ // Mark as deleted in WebDHT - NO PREFIXES
937
+ await this.webDHT.update(baseKey, tombstone);
920
938
 
921
939
  // Clean up local state
922
940
  this.storageMetadata.delete(keyId);
@@ -925,17 +943,17 @@ export class DistributedStorageManager extends EventEmitter {
925
943
  this.crdtStates.delete(keyId);
926
944
 
927
945
  // Clean up space ownership tracking and key mapping
928
- this.spaceOwnership.delete(fullKey);
946
+ this.spaceOwnership.delete(baseKey);
929
947
  this.keyToSpaceMapping.delete(baseKey);
930
948
 
931
- this.debug.log(`📦 Deleted data for key: ${fullKey}`);
949
+ this.debug.log(`📦 Deleted data for key: ${baseKey}`);
932
950
 
933
951
  // Emit deletion event
934
- this.emit('dataDeleted', { key: fullKey, keyId });
952
+ this.emit('dataDeleted', { key: baseKey, keyId });
935
953
 
936
954
  return true;
937
955
  } catch (error) {
938
- this.debug.error(`Failed to delete data for key ${fullKey}:`, error);
956
+ this.debug.error(`Failed to delete data for key ${baseKey}:`, error);
939
957
  throw error;
940
958
  }
941
959
  }
@@ -950,8 +968,8 @@ export class DistributedStorageManager extends EventEmitter {
950
968
  throw new Error('WebDHT not available - ensure it is enabled in mesh configuration');
951
969
  }
952
970
 
953
- // Subscribe to the WebDHT key
954
- const currentValue = await this.webDHT.subscribe(`storage:${key}`);
971
+ // Subscribe to the WebDHT key - NO PREFIXES
972
+ const currentValue = await this.webDHT.subscribe(key);
955
973
 
956
974
  this.debug.log(`📦 Subscribed to storage key: ${key}`);
957
975
 
@@ -967,7 +985,8 @@ export class DistributedStorageManager extends EventEmitter {
967
985
  return;
968
986
  }
969
987
 
970
- await this.webDHT.unsubscribe(`storage:${key}`);
988
+ // Unsubscribe from the WebDHT key - NO PREFIXES
989
+ await this.webDHT.unsubscribe(key);
971
990
  this.debug.log(`📦 Unsubscribed from storage key: ${key}`);
972
991
  }
973
992
 
@@ -1016,7 +1035,7 @@ export class DistributedStorageManager extends EventEmitter {
1016
1035
  async grantAccess(key, peerId) {
1017
1036
  // Resolve the key to its actual storage location
1018
1037
  const resolved = await this.resolveKey(key);
1019
- const { fullKey, keyId } = resolved;
1038
+ const { baseKey, keyId } = resolved;
1020
1039
 
1021
1040
  const accessControl = this.accessControl.get(keyId);
1022
1041
  const metadata = this.storageMetadata.get(keyId);
@@ -1036,16 +1055,16 @@ export class DistributedStorageManager extends EventEmitter {
1036
1055
 
1037
1056
  // Update the stored metadata
1038
1057
  try {
1039
- const currentPayload = await this.webDHT.get(`storage:${fullKey}`);
1058
+ const currentPayload = await this.webDHT.get(baseKey);
1040
1059
  if (currentPayload) {
1041
1060
  currentPayload.metadata = metadata;
1042
- await this.webDHT.update(`storage:${fullKey}`, currentPayload);
1061
+ await this.webDHT.update(baseKey, currentPayload);
1043
1062
  }
1044
1063
 
1045
- this.debug.log(`📦 Granted access to peer ${peerId.substring(0, 8)}... for key: ${fullKey}`);
1064
+ this.debug.log(`📦 Granted access to peer ${peerId.substring(0, 8)}... for key: ${baseKey}`);
1046
1065
 
1047
1066
  // Emit access granted event
1048
- this.emit('accessGranted', { key: fullKey, keyId, peerId });
1067
+ this.emit('accessGranted', { key: baseKey, keyId, peerId });
1049
1068
 
1050
1069
  return true;
1051
1070
  } catch (error) {
@@ -1065,7 +1084,7 @@ export class DistributedStorageManager extends EventEmitter {
1065
1084
  async revokeAccess(key, peerId) {
1066
1085
  // Resolve the key to its actual storage location
1067
1086
  const resolved = await this.resolveKey(key);
1068
- const { fullKey, keyId } = resolved;
1087
+ const { baseKey, keyId } = resolved;
1069
1088
 
1070
1089
  const accessControl = this.accessControl.get(keyId);
1071
1090
  const metadata = this.storageMetadata.get(keyId);
@@ -1081,16 +1100,16 @@ export class DistributedStorageManager extends EventEmitter {
1081
1100
 
1082
1101
  // Update the stored metadata
1083
1102
  try {
1084
- const currentPayload = await this.webDHT.get(`storage:${fullKey}`);
1103
+ const currentPayload = await this.webDHT.get(baseKey);
1085
1104
  if (currentPayload) {
1086
1105
  currentPayload.metadata = metadata;
1087
- await this.webDHT.update(`storage:${fullKey}`, currentPayload);
1106
+ await this.webDHT.update(baseKey, currentPayload);
1088
1107
  }
1089
1108
 
1090
- this.debug.log(`📦 Revoked access from peer ${peerId.substring(0, 8)}... for key: ${fullKey}`);
1109
+ this.debug.log(`📦 Revoked access from peer ${peerId.substring(0, 8)}... for key: ${baseKey}`);
1091
1110
 
1092
1111
  // Emit access revoked event
1093
- this.emit('accessRevoked', { key: fullKey, keyId, peerId });
1112
+ this.emit('accessRevoked', { key: baseKey, keyId, peerId });
1094
1113
 
1095
1114
  return true;
1096
1115
  } catch (error) {
@@ -1494,14 +1513,14 @@ export class DistributedStorageManager extends EventEmitter {
1494
1513
  async getKeyInfo(key) {
1495
1514
  // Resolve the key to its actual storage location
1496
1515
  const resolved = await this.resolveKey(key);
1497
- const { fullKey, keyId } = resolved;
1516
+ const { baseKey, keyId } = resolved;
1498
1517
 
1499
1518
  let metadata = this.storageMetadata.get(keyId);
1500
1519
 
1501
1520
  // If metadata is not found locally, try to retrieve it from the network
1502
1521
  if (!metadata) {
1503
1522
  try {
1504
- const existingData = await this.retrieve(fullKey);
1523
+ const existingData = await this.retrieve(baseKey);
1505
1524
  if (existingData !== null) {
1506
1525
  // Metadata should now be available after retrieve
1507
1526
  metadata = this.storageMetadata.get(keyId);
@@ -1606,11 +1625,11 @@ export class DistributedStorageManager extends EventEmitter {
1606
1625
  for (const keyId of this.ownedKeys) {
1607
1626
  const metadata = this.storageMetadata.get(keyId);
1608
1627
  if (metadata) {
1609
- // Check both the full key and the base key for prefix match
1610
- const fullKey = metadata.key;
1628
+ // Check both the stored key and the base key for prefix match
1629
+ const storedKey = metadata.key;
1611
1630
  const baseKey = metadata.baseKey;
1612
1631
 
1613
- if (fullKey.startsWith(prefix) || baseKey.startsWith(prefix)) {
1632
+ if (storedKey.startsWith(prefix) || baseKey.startsWith(prefix)) {
1614
1633
  // Return the base key for user convenience (since they stored with base key)
1615
1634
  keys.push(baseKey);
1616
1635
  }