peerpigeon 0.1.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +141 -1365
  2. package/cli.js +1382 -0
  3. package/dist/peerpigeon-browser.js +12290 -0
  4. package/dist/peerpigeon-browser.js.map +7 -0
  5. package/docs/API_DOCUMENTATION.md +1945 -0
  6. package/docs/BOOTSTRAP_HUBS.md +319 -0
  7. package/docs/CLI_README.md +371 -0
  8. package/docs/DOCUMENTATION_REORGANIZATION.md +172 -0
  9. package/docs/HUB_QUICK_REF.md +107 -0
  10. package/docs/HUB_SCRIPTS.md +331 -0
  11. package/docs/HUB_SYSTEM.md +552 -0
  12. package/docs/MIGRATION_COMPLETE.md +93 -0
  13. package/docs/NETWORK_NAMESPACES.md +212 -0
  14. package/docs/README.md +107 -0
  15. package/docs/SELECTIVE_STREAMING_GUIDE.md +427 -0
  16. package/docs/TEST_CLEANUP_COMPLETE.md +80 -0
  17. package/examples/bootstrap-hub-example.js +292 -0
  18. package/examples/browser/README.md +278 -0
  19. package/examples/browser/app.js +2426 -49
  20. package/examples/browser/index.html +482 -543
  21. package/examples/browser/minimal/README.md +55 -0
  22. package/examples/browser/minimal/index.html +75 -0
  23. package/examples/browser/minimal/minimal.js +234 -0
  24. package/examples/browser/styles.css +687 -1425
  25. package/examples/browser/test-bundle.html +35 -0
  26. package/examples/browser-2/index.html +30 -0
  27. package/examples/browser-2/package.json +21 -0
  28. package/examples/browser-2/src/App-simple.vue +21 -0
  29. package/examples/browser-2/src/App.vue +420 -0
  30. package/examples/browser-2/src/components/CryptoView.vue +1661 -0
  31. package/examples/browser-2/src/components/DHTView.vue +1281 -0
  32. package/examples/browser-2/src/components/DebugView.vue +1773 -0
  33. package/examples/browser-2/src/components/MediaView.vue +1048 -0
  34. package/examples/browser-2/src/components/MessagingView.vue +775 -0
  35. package/examples/browser-2/src/components/NetworkView.vue +930 -0
  36. package/examples/browser-2/src/components/StorageView.vue +1376 -0
  37. package/examples/browser-2/src/main.js +5 -0
  38. package/examples/browser-2/src/stores/peerpigeon.js +895 -0
  39. package/examples/browser-2/vite.config.js +6 -0
  40. package/examples/hub-example.js +186 -0
  41. package/examples/multi-hub-test.js +186 -0
  42. package/examples/node/README.md +2 -2
  43. package/examples/node/storage-test.js +141 -6
  44. package/examples/selective-streaming-demo.html +698 -0
  45. package/examples/standard-events-demo.js +126 -0
  46. package/package.json +33 -16
  47. package/server/index.js +1032 -98
  48. package/server/start.js +49 -0
  49. package/src/ConnectionManager.js +239 -225
  50. package/src/CryptoManager.js +147 -18
  51. package/src/DebugLogger.js +0 -7
  52. package/src/DistributedStorageManager.js +245 -205
  53. package/src/EnvironmentDetector.js +109 -2
  54. package/src/EventEmitter.js +79 -1
  55. package/src/EvictionManager.js +10 -9
  56. package/src/GossipManager.js +163 -11
  57. package/src/LexicalStorageInterface.js +332 -0
  58. package/src/PeerConnection.js +333 -310
  59. package/src/PeerPigeonMesh.js +577 -135
  60. package/src/SignalingClient.js +14 -243
  61. package/src/SignalingHandler.js +31 -10
  62. package/src/StorageManager.js +0 -39
  63. package/src/WebDHT-Simple.js +389 -0
  64. package/src/WebDHT.js +363 -1165
  65. package/src/browser-entry.js +40 -0
  66. package/examples/browser/assets/images/favicon.png +0 -0
  67. package/examples/browser/assets/images/pigeon.png +0 -0
  68. package/examples/browser/crypto-test.html +0 -427
  69. package/examples/browser/ui-old.js +0 -1926
  70. package/examples/browser/ui.js +0 -3010
  71. package/websocket-server/README.md +0 -247
  72. package/websocket-server/server.js +0 -391
@@ -1,5 +1,6 @@
1
1
  import { EventEmitter } from './EventEmitter.js';
2
2
  import DebugLogger from './DebugLogger.js';
3
+ import { createLexicalInterface } from './LexicalStorageInterface.js';
3
4
 
4
5
  // Dynamic import for unsea to handle both Node.js and browser environments
5
6
  let unsea = null;
@@ -8,16 +9,29 @@ async function initializeUnsea() {
8
9
  if (unsea) return unsea;
9
10
 
10
11
  try {
12
+ // Check if UnSEA is bundled (browser bundle case)
13
+ if (typeof window !== 'undefined' && window.__PEERPIGEON_UNSEA__) {
14
+ unsea = window.__PEERPIGEON_UNSEA__;
15
+ console.log('✅ Using bundled UnSEA crypto for storage');
16
+ return unsea;
17
+ }
18
+
19
+ // Check global scope for bundled version
20
+ if (typeof globalThis !== 'undefined' && globalThis.__PEERPIGEON_UNSEA__) {
21
+ unsea = globalThis.__PEERPIGEON_UNSEA__;
22
+ console.log('✅ Using bundled UnSEA crypto for storage');
23
+ return unsea;
24
+ }
25
+
11
26
  // Detect environment - prioritize Node.js detection for tests
12
27
  const isNode = typeof process !== 'undefined' && process.versions && process.versions.node;
13
- const isBrowser = !isNode && typeof window !== 'undefined' && typeof document !== 'undefined';
14
28
 
15
29
  if (isNode) {
16
30
  // For Node.js environments (including tests), use npm package
17
31
  unsea = await import('unsea');
18
32
  console.log('✅ Loaded unsea from npm package (Node.js) for storage');
19
- } else if (isBrowser) {
20
- // For browser environments, try CDN sources
33
+ } else {
34
+ // For browser environments without bundle, try CDN sources as fallback
21
35
  try {
22
36
  unsea = await import('https://cdn.jsdelivr.net/npm/unsea@latest/+esm');
23
37
  console.log('✅ Loaded unsea from jsDelivr CDN for storage');
@@ -32,8 +46,6 @@ async function initializeUnsea() {
32
46
  console.log('✅ Loaded unsea from Skypack CDN for storage');
33
47
  }
34
48
  }
35
- } else {
36
- throw new Error('Unknown environment - cannot load unsea');
37
49
  }
38
50
 
39
51
  if (!unsea) {
@@ -48,22 +60,33 @@ async function initializeUnsea() {
48
60
  }
49
61
 
50
62
  /**
51
- * DistributedStorageManager - Optional storage layer for PeerPigeon mesh
63
+ * DistributedStorageManager - High-level distributed storage layer for PeerPigeon mesh
52
64
  *
53
65
  * Features:
54
66
  * - Encrypted storage using unsea directly
55
- * - Optional public/private data visibility
67
+ * - Optional public/private data visibility with access control
56
68
  * - Optional immutability with CRDT support for collaborative editing
57
69
  * - Mutable layer for data originators
58
- * - Integration with WebDHT for distributed storage
70
+ * - Uses WebDHT as low-level storage backend (but is separate conceptually)
59
71
  * - Space separation for preventing data overwrites (Private, Public, Frozen)
72
+ *
73
+ * IMPORTANT: This is a HIGH-LEVEL storage system that provides encryption,
74
+ * access control, and advanced features. It uses WebDHT as its storage backend
75
+ * but they are separate concepts:
76
+ * - WebDHT: Low-level distributed hash table for raw key-value storage
77
+ * - DistributedStorageManager: High-level storage with encryption/access control
60
78
  */
61
79
  export class DistributedStorageManager extends EventEmitter {
62
80
  constructor(mesh) {
63
81
  super();
64
82
  this.debug = DebugLogger.create('DistributedStorageManager');
65
83
  this.mesh = mesh;
66
- this.webDHT = mesh.webDHT;
84
+
85
+ // IMPORTANT: DistributedStorageManager uses WebDHT as its storage backend
86
+ // but they are conceptually separate systems:
87
+ // - WebDHT: Low-level DHT for raw key-value storage across the mesh
88
+ // - DistributedStorageManager: High-level storage with encryption, access control, spaces
89
+ this.webDHT = mesh.webDHT; // Uses WebDHT as backend storage layer
67
90
  this.cryptoManager = mesh.cryptoManager;
68
91
 
69
92
  // Initialize unsea for encryption
@@ -96,7 +119,7 @@ export class DistributedStorageManager extends EventEmitter {
96
119
  // Track data ownership by space
97
120
  this.ownedKeys = new Set(); // Keys owned by this peer
98
121
  this.spaceOwnership = new Map(); // space:key -> owner mapping
99
- this.keyToSpaceMapping = new Map(); // baseKey -> {space, fullKey} mapping for transparent access
122
+ this.keyToSpaceMapping = new Map(); // baseKey -> {space, storageKey} mapping for transparent access
100
123
 
101
124
  // Track enabled state
102
125
  this.enabled = true; // Enable by default
@@ -148,102 +171,50 @@ export class DistributedStorageManager extends EventEmitter {
148
171
 
149
172
  /**
150
173
  * Resolve a key to its actual storage location across spaces
151
- * @param {string} key - The storage key (can be plain or space-prefixed)
152
- * @returns {Object} - {space, baseKey, fullKey, keyId}
174
+ * @param {string} key - The storage key (base key only)
175
+ * @returns {Object} - {space, baseKey, keyId}
153
176
  */
154
177
  async resolveKey(key) {
155
- // If key already has space prefix, use it as-is
156
- const keyInfo = this.parseStorageKey(key);
157
-
158
- // If it's not a bare key (has space prefix), use it directly
159
- if (key.includes(':')) {
160
- const keyId = await this.webDHT.generateKeyId(keyInfo.fullKey);
161
- return {
162
- space: keyInfo.space,
163
- baseKey: keyInfo.baseKey,
164
- fullKey: keyInfo.fullKey,
165
- keyId
166
- };
167
- }
178
+ // Use the key directly - NO PREFIX PARSING
179
+ const baseKey = key;
180
+
181
+ // Generate keyId from baseKey (no prefixes)
182
+ const keyId = await this.webDHT.hash(baseKey);
168
183
 
169
184
  // For bare keys, check if we have a mapping
170
- const mapping = this.keyToSpaceMapping.get(keyInfo.baseKey);
185
+ const mapping = this.keyToSpaceMapping.get(baseKey);
171
186
  if (mapping) {
172
- const keyId = await this.webDHT.generateKeyId(mapping.fullKey);
173
187
  return {
174
188
  space: mapping.space,
175
- baseKey: keyInfo.baseKey,
176
- fullKey: mapping.fullKey,
189
+ baseKey,
177
190
  keyId
178
191
  };
179
192
  }
180
193
 
181
- // If no mapping exists, try to find it by checking each space
182
- for (const space of Object.values(this.spaces)) {
183
- const testFullKey = this.createSpacedKey(space, keyInfo.baseKey);
184
- const testKeyId = await this.webDHT.generateKeyId(testFullKey);
185
-
186
- // Check if we have metadata for this key in this space
187
- if (this.storageMetadata.has(testKeyId) || this.spaceOwnership.has(testFullKey)) {
188
- // Update the mapping for future use
189
- this.keyToSpaceMapping.set(keyInfo.baseKey, { space, fullKey: testFullKey });
190
- return {
191
- space,
192
- baseKey: keyInfo.baseKey,
193
- fullKey: testFullKey,
194
- keyId: testKeyId
195
- };
196
- }
194
+ // If no mapping exists, check if we have metadata for this key
195
+ if (this.storageMetadata.has(keyId)) {
196
+ const metadata = this.storageMetadata.get(keyId);
197
+ const space = metadata.space || this.spaces.PRIVATE;
198
+ // Update the mapping for future use
199
+ this.keyToSpaceMapping.set(baseKey, { space, storageKey: baseKey });
200
+ return {
201
+ space,
202
+ baseKey,
203
+ keyId
204
+ };
197
205
  }
198
206
 
199
- // If still not found, default to the parsed key (likely private space)
200
- const keyId = await this.webDHT.generateKeyId(keyInfo.fullKey);
207
+ // Default to private space if no other information is available
208
+ const defaultSpace = this.spaces.PRIVATE;
209
+ this.keyToSpaceMapping.set(baseKey, { space: defaultSpace, storageKey: baseKey });
201
210
  return {
202
- space: keyInfo.space,
203
- baseKey: keyInfo.baseKey,
204
- fullKey: keyInfo.fullKey,
211
+ space: defaultSpace,
212
+ baseKey,
205
213
  keyId
206
214
  };
207
215
  }
208
216
 
209
- /**
210
- * Parse a storage key to extract space and base key
211
- * @param {string} key - The storage key (can be plain or space-prefixed)
212
- * @returns {Object} - {space, baseKey, fullKey}
213
- */
214
- parseStorageKey(key) {
215
- // Check if key has space prefix (e.g., "private:user:123" or "public:config")
216
- const spacePrefixes = Object.values(this.spaces);
217
- for (const space of spacePrefixes) {
218
- if (key.startsWith(`${space}:`)) {
219
- return {
220
- space,
221
- baseKey: key.substring(space.length + 1),
222
- fullKey: key
223
- };
224
- }
225
- }
226
217
 
227
- // Default to private space if no prefix specified
228
- return {
229
- space: this.spaces.PRIVATE,
230
- baseKey: key,
231
- fullKey: `${this.spaces.PRIVATE}:${key}`
232
- };
233
- }
234
-
235
- /**
236
- * Create a space-prefixed key
237
- * @param {string} space - The storage space
238
- * @param {string} baseKey - The base key
239
- * @returns {string} - The full key with space prefix
240
- */
241
- createSpacedKey(space, baseKey) {
242
- if (!Object.values(this.spaces).includes(space)) {
243
- throw new Error(`Invalid storage space: ${space}. Must be one of: ${Object.values(this.spaces).join(', ')}`);
244
- }
245
- return `${space}:${baseKey}`;
246
- }
247
218
 
248
219
  /**
249
220
  * Check if a peer can access a key in a specific space
@@ -331,14 +302,21 @@ export class DistributedStorageManager extends EventEmitter {
331
302
  * @returns {string|null} - The space where the key exists, or null if not found
332
303
  */
333
304
  findKeyInDifferentSpace(baseKey, excludeSpace) {
334
- for (const space of Object.values(this.spaces)) {
335
- if (space !== excludeSpace) {
336
- const spacedKey = this.createSpacedKey(space, baseKey);
337
- if (this.spaceOwnership.has(spacedKey)) {
338
- return space;
339
- }
305
+ // Check our metadata to see if this baseKey exists in a different space
306
+ for (const metadata of this.storageMetadata.values()) {
307
+ if (metadata.baseKey === baseKey && metadata.space !== excludeSpace) {
308
+ return metadata.space;
309
+ }
310
+ }
311
+
312
+ // Also check space ownership using baseKey directly
313
+ if (this.spaceOwnership.has(baseKey)) {
314
+ const mapping = this.keyToSpaceMapping.get(baseKey);
315
+ if (mapping && mapping.space !== excludeSpace) {
316
+ return mapping.space;
340
317
  }
341
318
  }
319
+
342
320
  return null;
343
321
  }
344
322
 
@@ -403,6 +381,10 @@ export class DistributedStorageManager extends EventEmitter {
403
381
  * @returns {Promise<boolean>} Success status
404
382
  */
405
383
  async store(key, value, options = {}) {
384
+ if (!this.enabled) {
385
+ return false;
386
+ }
387
+
406
388
  if (!this.webDHT) {
407
389
  throw new Error('WebDHT not available - ensure it is enabled in mesh configuration');
408
390
  }
@@ -412,31 +394,24 @@ export class DistributedStorageManager extends EventEmitter {
412
394
  await this.waitForCrypto();
413
395
  }
414
396
 
415
- // Parse key and determine storage space
416
- const keyInfo = this.parseStorageKey(key);
397
+ // Use the key directly - NO PREFIX PARSING
417
398
  const spaceConfig = this.determineSpaceFromOptions(options);
418
399
 
419
- // Use explicit space from options if provided, otherwise use parsed space
400
+ // Use the base key directly - NO SPACE PREFIXES!
401
+ const baseKey = key;
420
402
  const space = options.space || spaceConfig.space;
421
- const fullKey = keyInfo.space === space ? keyInfo.fullKey : this.createSpacedKey(space, keyInfo.baseKey);
422
- const baseKey = keyInfo.baseKey;
403
+
404
+ // The stored key is just the base key - space info goes in metadata
405
+ const storageKey = baseKey;
423
406
 
424
407
  // Check space access permissions
425
408
  if (!this.canAccessSpace(space, baseKey, this.mesh.peerId, 'write')) {
426
409
  throw new Error(`Write access denied for space "${space}" and key "${baseKey}"`);
427
410
  }
428
411
 
429
- // Check if key already exists in a different space
430
- if (this.config.spaceEnforcement) {
431
- const existingSpaceKey = this.findKeyInDifferentSpace(baseKey, space);
432
- if (existingSpaceKey) {
433
- throw new Error(`Key "${baseKey}" already exists in space "${existingSpaceKey}" - cannot store in "${space}" space`);
434
- }
435
- }
436
-
437
- const keyId = await this.webDHT.generateKeyId(fullKey);
412
+ const keyId = await this.webDHT.hash(storageKey);
438
413
  const timestamp = Date.now();
439
- this.debug.log(`📦 Storing key: ${fullKey}, keyId: ${keyId.substring(0, 8)}...`);
414
+ this.debug.log(`📦 Storing key: ${storageKey} in ${space} space, keyId: ${keyId.toString(16).substring(0, 8)}...`);
440
415
 
441
416
  // Validate value size
442
417
  const serializedValue = JSON.stringify(value);
@@ -444,9 +419,9 @@ export class DistributedStorageManager extends EventEmitter {
444
419
  throw new Error(`Value size exceeds maximum allowed size of ${this.config.maxValueSize} bytes`);
445
420
  }
446
421
 
447
- // Create storage metadata with space information
422
+ // Create storage metadata with type and space information
448
423
  const metadata = {
449
- key: fullKey,
424
+ key: baseKey, // Just the base key, no prefixes - space is its own attribute
450
425
  baseKey,
451
426
  space,
452
427
  keyId,
@@ -458,18 +433,20 @@ export class DistributedStorageManager extends EventEmitter {
458
433
  createdAt: timestamp,
459
434
  updatedAt: timestamp,
460
435
  version: 1,
461
- ttl: options.ttl || this.config.defaultTTL
436
+ ttl: options.ttl || this.config.defaultTTL,
437
+ type: 'storage', // Mark this as storage data type
438
+ dataType: 'distributed-storage' // Specific storage system identifier
462
439
  };
463
440
 
464
441
  // Store metadata locally
465
442
  this.storageMetadata.set(keyId, metadata);
466
443
  this.ownedKeys.add(keyId);
467
444
 
468
- // Track space ownership
469
- this.spaceOwnership.set(fullKey, this.mesh.peerId);
445
+ // Track space ownership using just the base key
446
+ this.spaceOwnership.set(baseKey, this.mesh.peerId);
470
447
 
471
448
  // Track base key to space mapping for transparent access
472
- this.keyToSpaceMapping.set(baseKey, { space, fullKey });
449
+ this.keyToSpaceMapping.set(baseKey, { space, storageKey });
473
450
 
474
451
  // Set up access control
475
452
  this.accessControl.set(keyId, {
@@ -507,33 +484,47 @@ export class DistributedStorageManager extends EventEmitter {
507
484
  encrypted: true,
508
485
  encryptedBy: this.mesh.peerId // Track who encrypted it
509
486
  };
510
- this.debug.log(`📦 Encrypted private space storage data for key: ${fullKey} (owner-only access)`);
487
+ this.debug.log(`📦 Encrypted private space storage data for key: ${storageKey} (owner-only access)`);
511
488
  } catch (error) {
512
- this.debug.warn(`Failed to encrypt private space storage data for key ${fullKey}:`, error);
489
+ this.debug.warn(`Failed to encrypt private space storage data for key ${storageKey}:`, error);
513
490
  // Fall back to unencrypted storage if encryption fails
514
491
  }
515
492
  } else {
516
- this.debug.log(`📦 Storing ${space} space data for key: ${fullKey}`);
517
- }
493
+ this.debug.log(`📦 Storing ${space} space data for key: ${storageKey}`);
494
+ }
495
+
496
+ // CRITICAL DEBUG: Log exactly what we're storing
497
+ this.debug.log(`📦 STORING PAYLOAD STRUCTURE for key ${storageKey}:`, {
498
+ hasValue: !!storagePayload.value,
499
+ hasMetadata: !!storagePayload.metadata,
500
+ valueType: typeof storagePayload.value,
501
+ metadataType: typeof storagePayload.metadata,
502
+ metadataKeys: storagePayload.metadata ? Object.keys(storagePayload.metadata) : 'none',
503
+ encrypted: storagePayload.encrypted,
504
+ payloadKeys: Object.keys(storagePayload)
505
+ });
518
506
 
519
- // Store in WebDHT with space prefix
507
+ // Store in WebDHT with storage namespace prefix to separate from raw DHT data
508
+ // This maintains separation between low-level DHT operations and high-level storage
520
509
  try {
521
- this.debug.log(`📦 Storing payload for key ${fullKey}:`, {
510
+ this.debug.log(`📦 Storing payload for key ${storageKey}:`, {
522
511
  hasValue: !!storagePayload.value,
523
512
  hasMetadata: !!storagePayload.metadata,
524
513
  encrypted: storagePayload.encrypted,
525
514
  space
526
515
  });
527
516
 
528
- await this.webDHT.put(`storage:${fullKey}`, storagePayload, {
529
- ttl: metadata.ttl
517
+ // Store directly with clean key - NO PREFIXES, type information is in metadata
518
+ await this.webDHT.put(storageKey, storagePayload, {
519
+ ttl: metadata.ttl,
520
+ space: space // Pass space for space-aware replication
530
521
  });
531
522
 
532
- this.debug.log(`📦 Stored ${space} space data for key: ${fullKey}`);
523
+ this.debug.log(`📦 Stored ${space} space data for key: ${storageKey}`);
533
524
 
534
525
  // Emit storage event
535
526
  this.emit('dataStored', {
536
- key: fullKey,
527
+ key: storageKey,
537
528
  baseKey,
538
529
  space,
539
530
  keyId,
@@ -544,13 +535,13 @@ export class DistributedStorageManager extends EventEmitter {
544
535
 
545
536
  return true;
546
537
  } catch (error) {
547
- this.debug.error(`Failed to store data for key ${fullKey}:`, error);
538
+ this.debug.error(`Failed to store data for key ${storageKey}:`, error);
548
539
  // Clean up local metadata on failure
549
540
  this.storageMetadata.delete(keyId);
550
541
  this.ownedKeys.delete(keyId);
551
542
  this.accessControl.delete(keyId);
552
543
  this.crdtStates.delete(keyId);
553
- this.spaceOwnership.delete(fullKey);
544
+ this.spaceOwnership.delete(baseKey);
554
545
  this.keyToSpaceMapping.delete(baseKey);
555
546
  throw error;
556
547
  }
@@ -558,8 +549,9 @@ export class DistributedStorageManager extends EventEmitter {
558
549
 
559
550
  /**
560
551
  * Retrieve data with access control and decryption
561
- * @param {string} key - The storage key
552
+ * @param {string} key - The storage key (base key only, no prefixes)
562
553
  * @param {Object} options - Retrieval options
554
+ * @param {string} options.space - Specific space to look in (optional)
563
555
  * @param {boolean} options.forceRefresh - Force refresh from network
564
556
  * @returns {Promise<any>} The stored value or null if not accessible
565
557
  */
@@ -573,70 +565,102 @@ export class DistributedStorageManager extends EventEmitter {
573
565
  await this.waitForCrypto();
574
566
  }
575
567
 
576
- // Resolve the key to its actual storage location
577
- const resolved = await this.resolveKey(key);
578
- const { space, baseKey, fullKey, keyId } = resolved;
568
+ // Use the key directly - NO PREFIX STRIPPING
569
+ const baseKey = key;
570
+
571
+ this.debug.log(`📦 Retrieving data for base key: ${baseKey}`);
579
572
 
580
- // Check space access permissions
581
- if (!this.canAccessSpace(space, baseKey, this.mesh.peerId, 'read')) {
582
- this.debug.warn(`📦 Access denied for space "${space}" and key "${baseKey}"`);
573
+ try {
574
+ // Get data from WebDHT using just the clean base key - NO PREFIXES
575
+ const webDHTPayload = await this.webDHT.get(baseKey, {
576
+ forceRefresh: options.forceRefresh,
577
+ space: options.space // Pass space for space-aware replication
578
+ });
579
+
580
+ this.debug.log(`📦 Retrieved WebDHT payload for key ${baseKey}:`, {
581
+ payloadExists: !!webDHTPayload,
582
+ payloadType: typeof webDHTPayload,
583
+ keys: webDHTPayload ? Object.keys(webDHTPayload) : 'none'
584
+ });
585
+
586
+ if (!webDHTPayload || typeof webDHTPayload !== 'object') {
587
+ this.debug.log(`📦 No data found for key: ${baseKey}`);
588
+ return null;
589
+ }
590
+
591
+ // Process the data using the metadata to determine space and access control
592
+ return await this.processRetrievedData(baseKey, webDHTPayload);
593
+ } catch (error) {
594
+ this.debug.error(`Failed to retrieve data for key ${baseKey}:`, error);
583
595
  return null;
584
596
  }
597
+ }
585
598
 
586
- this.debug.log(`📦 Retrieving key: ${fullKey}, keyId: ${keyId.substring(0, 8)}...`);
587
-
599
+ /**
600
+ * Process retrieved data payload with decryption and access control
601
+ * @private
602
+ */
603
+ async processRetrievedData(baseKey, webDHTPayload) {
588
604
  try {
589
- // Get data from WebDHT
590
- this.debug.log(`📦 Attempting to retrieve data for key: ${fullKey}`);
591
- const storagePayload = await this.webDHT.get(`storage:${fullKey}`, {
592
- forceRefresh: options.forceRefresh
593
- });
594
-
595
- this.debug.log(`📦 Retrieved payload for key ${fullKey}:`, {
596
- payloadExists: !!storagePayload,
597
- payloadType: typeof storagePayload,
598
- space,
599
- keys: storagePayload ? Object.keys(storagePayload) : 'none'
605
+ // Generate keyId for this baseKey
606
+ const keyId = await this.webDHT.hash(baseKey);
607
+
608
+ // WebDHT returns the storage payload directly as {value, metadata, encrypted}
609
+ const storagePayload = webDHTPayload;
610
+
611
+ console.log(`📦 CRITICAL DEBUG: WebDHT payload structure for ${baseKey}:`, {
612
+ webDHTKeys: Object.keys(webDHTPayload),
613
+ payloadType: typeof webDHTPayload,
614
+ directValue: !!webDHTPayload.value,
615
+ directMetadata: !!webDHTPayload.metadata,
616
+ directEncrypted: webDHTPayload.encrypted
600
617
  });
601
618
 
602
619
  if (!storagePayload || typeof storagePayload !== 'object') {
603
- this.debug.log(`📦 No data found for key: ${fullKey}`);
620
+ console.log(`📦 CRITICAL: Invalid storage payload for key: ${baseKey}`);
621
+ this.debug.log(`📦 Invalid storage payload for key: ${baseKey}`);
604
622
  return null;
605
623
  }
606
624
 
607
- // Debug: Log the raw payload structure
608
- this.debug.log('📦 Raw payload structure:', JSON.stringify(storagePayload, null, 2));
609
-
610
625
  // Safely destructure the payload with detailed logging
611
626
  const value = storagePayload.value;
612
627
  const metadata = storagePayload.metadata;
613
628
  const encrypted = storagePayload.encrypted || false;
614
629
  const encryptedBy = storagePayload.encryptedBy;
615
630
 
631
+ if (!metadata) {
632
+ this.debug.warn(`📦 Invalid storage payload for key ${baseKey}: missing metadata`);
633
+ return null;
634
+ }
635
+
636
+ // Validate this is actually storage data by checking metadata type
637
+ if (!metadata.type || metadata.type !== 'storage') {
638
+ this.debug.log(`📦 Data for key ${baseKey} is not storage data (type: ${metadata.type || 'unknown'}) - ignoring`);
639
+ return null;
640
+ }
641
+
642
+ // Get space from metadata - this is where space info is stored!
643
+ const space = metadata.space;
644
+
616
645
  this.debug.log('📦 Payload components:', {
617
646
  hasValue: value !== undefined,
618
647
  valueType: typeof value,
619
648
  hasMetadata: metadata !== undefined,
620
649
  metadataType: typeof metadata,
621
650
  encrypted,
622
- space: metadata?.space || 'unknown',
651
+ space: space || 'unknown',
623
652
  encryptedBy: encryptedBy?.substring(0, 8) + '...' || 'unknown'
624
653
  });
625
654
 
626
- if (!metadata) {
627
- this.debug.warn(`📦 Invalid storage payload for key ${fullKey}: missing metadata`);
628
- return null;
629
- }
630
-
631
- // Verify space consistency
632
- if (metadata.space && metadata.space !== space) {
633
- this.debug.warn(`📦 Space mismatch for key ${fullKey}: expected ${space}, got ${metadata.space}`);
655
+ // Check space access permissions using space from metadata
656
+ if (!this.canAccessSpace(space, baseKey, this.mesh.peerId, 'read')) {
657
+ this.debug.warn(`📦 Access denied for key: ${baseKey} in space: ${space}`);
634
658
  return null;
635
659
  }
636
660
 
637
661
  // Check access permissions using space-aware logic
638
662
  if (!this.hasReadAccessWithSpace(keyId, metadata, space)) {
639
- this.debug.warn(`📦 Access denied for key: ${fullKey} in space: ${space}`);
663
+ this.debug.warn(`📦 Access denied for key: ${baseKey} in space: ${space}`);
640
664
  return null;
641
665
  }
642
666
 
@@ -645,10 +669,10 @@ export class DistributedStorageManager extends EventEmitter {
645
669
  if (encrypted && this.unsea && this.storageKeypair) {
646
670
  // Only allow decryption if this peer is the owner (encrypted the data)
647
671
  if (encryptedBy && encryptedBy !== this.mesh.peerId) {
648
- this.debug.warn(`📦 Cannot decrypt data for key ${key}: encrypted by different peer (${encryptedBy.substring(0, 8)}...), current peer: ${this.mesh.peerId.substring(0, 8)}...`);
672
+ this.debug.warn(`📦 Cannot decrypt data for key ${baseKey}: encrypted by different peer (${encryptedBy.substring(0, 8)}...), current peer: ${this.mesh.peerId.substring(0, 8)}...`);
649
673
  // For private data encrypted by another peer, deny access
650
674
  if (!metadata.isPublic) {
651
- this.debug.warn(`Access denied for key: ${key} - private data encrypted by different peer`);
675
+ this.debug.warn(`Access denied for key: ${baseKey} - private data encrypted by different peer`);
652
676
  return null;
653
677
  }
654
678
  }
@@ -656,15 +680,15 @@ export class DistributedStorageManager extends EventEmitter {
656
680
  try {
657
681
  const decryptedValue = await this.unsea.decryptMessageWithMeta(value, this.storageKeypair.epriv);
658
682
  finalValue = JSON.parse(decryptedValue);
659
- this.debug.log(`🔓 Decrypted storage data for key: ${key}`);
683
+ this.debug.log(`🔓 Decrypted storage data for key: ${baseKey}`);
660
684
  } catch (error) {
661
- this.debug.error(`Failed to decrypt data for key ${key}:`, error);
685
+ this.debug.error(`Failed to decrypt data for key ${baseKey}:`, error);
662
686
  // If this is private data and decryption fails, deny access
663
687
  if (!metadata.isPublic) {
664
688
  return null;
665
689
  }
666
690
  // For public data, if decryption fails, try to use the raw value
667
- this.debug.warn(`📦 Using raw value for public key ${key} due to decryption failure`);
691
+ this.debug.warn(`📦 Using raw value for public key ${baseKey} due to decryption failure`);
668
692
  finalValue = value;
669
693
  }
670
694
  }
@@ -680,22 +704,20 @@ export class DistributedStorageManager extends EventEmitter {
680
704
  space: metadata.space || space
681
705
  });
682
706
 
683
- // Track space ownership from retrieved metadata
684
- if (metadata.owner && !this.spaceOwnership.has(fullKey)) {
685
- this.spaceOwnership.set(fullKey, metadata.owner);
707
+ // Track space ownership from retrieved metadata using baseKey
708
+ if (metadata.owner && !this.spaceOwnership.has(baseKey)) {
709
+ this.spaceOwnership.set(baseKey, metadata.owner);
686
710
  }
687
711
 
688
712
  // Update key mapping for future transparent access
689
- if (!key.includes(':')) {
690
- this.keyToSpaceMapping.set(baseKey, { space, fullKey });
691
- }
713
+ this.keyToSpaceMapping.set(baseKey, { space, storageKey: baseKey });
692
714
  }
693
715
 
694
- this.debug.log(`📦 Retrieved ${space} space data for key: ${fullKey}`);
716
+ this.debug.log(`📦 Retrieved ${space} space data for key: ${baseKey}`);
695
717
 
696
718
  // Emit retrieval event
697
719
  this.emit('dataRetrieved', {
698
- key: fullKey,
720
+ key: baseKey,
699
721
  baseKey,
700
722
  space,
701
723
  keyId,
@@ -705,7 +727,7 @@ export class DistributedStorageManager extends EventEmitter {
705
727
 
706
728
  return finalValue;
707
729
  } catch (error) {
708
- this.debug.error(`Failed to retrieve data for key ${key}:`, error);
730
+ this.debug.error(`Failed to retrieve data for key ${baseKey}:`, error);
709
731
  return null;
710
732
  }
711
733
  }
@@ -725,7 +747,7 @@ export class DistributedStorageManager extends EventEmitter {
725
747
 
726
748
  // Resolve the key to its actual storage location
727
749
  const resolved = await this.resolveKey(key);
728
- const { fullKey, keyId } = resolved;
750
+ const { baseKey, keyId } = resolved;
729
751
  const timestamp = Date.now();
730
752
 
731
753
  // Check if we have access to update
@@ -734,7 +756,7 @@ export class DistributedStorageManager extends EventEmitter {
734
756
 
735
757
  if (!accessControl || !metadata) {
736
758
  // Try to retrieve metadata first
737
- const existingData = await this.retrieve(fullKey);
759
+ const existingData = await this.retrieve(baseKey);
738
760
  if (!existingData) {
739
761
  throw new Error(`Key ${key} does not exist or is not accessible`);
740
762
  }
@@ -793,19 +815,19 @@ export class DistributedStorageManager extends EventEmitter {
793
815
  }
794
816
 
795
817
  try {
796
- // Use WebDHT update to notify all replicas and subscribers
797
- await this.webDHT.update(`storage:${fullKey}`, storagePayload, {
818
+ // Use WebDHT update to notify all replicas and subscribers - NO PREFIXES
819
+ await this.webDHT.update(baseKey, storagePayload, {
798
820
  ttl: metadata.ttl
799
821
  });
800
822
 
801
823
  // Update local metadata
802
824
  this.storageMetadata.set(keyId, metadata);
803
825
 
804
- this.debug.log(`📦 Updated ${metadata.isPublic ? 'public' : 'private'} data for key: ${fullKey}`);
826
+ this.debug.log(`📦 Updated ${metadata.isPublic ? 'public' : 'private'} data for key: ${baseKey}`);
805
827
 
806
828
  // Emit update event
807
829
  this.emit('dataUpdated', {
808
- key: fullKey,
830
+ key: baseKey,
809
831
  keyId,
810
832
  isPublic: metadata.isPublic,
811
833
  version: metadata.version,
@@ -814,7 +836,7 @@ export class DistributedStorageManager extends EventEmitter {
814
836
 
815
837
  return true;
816
838
  } catch (error) {
817
- this.debug.error(`Failed to update data for key ${fullKey}:`, error);
839
+ this.debug.error(`Failed to update data for key ${baseKey}:`, error);
818
840
  throw error;
819
841
  }
820
842
  }
@@ -895,7 +917,7 @@ export class DistributedStorageManager extends EventEmitter {
895
917
 
896
918
  // Resolve the key to its actual storage location
897
919
  const resolved = await this.resolveKey(key);
898
- const { baseKey, fullKey, keyId } = resolved;
920
+ const { baseKey, keyId } = resolved;
899
921
 
900
922
  const accessControl = this.accessControl.get(keyId);
901
923
 
@@ -911,7 +933,8 @@ export class DistributedStorageManager extends EventEmitter {
911
933
  deletedBy: this.mesh.peerId
912
934
  };
913
935
 
914
- await this.webDHT.update(`storage:${fullKey}`, tombstone);
936
+ // Mark as deleted in WebDHT - NO PREFIXES
937
+ await this.webDHT.update(baseKey, tombstone);
915
938
 
916
939
  // Clean up local state
917
940
  this.storageMetadata.delete(keyId);
@@ -920,17 +943,17 @@ export class DistributedStorageManager extends EventEmitter {
920
943
  this.crdtStates.delete(keyId);
921
944
 
922
945
  // Clean up space ownership tracking and key mapping
923
- this.spaceOwnership.delete(fullKey);
946
+ this.spaceOwnership.delete(baseKey);
924
947
  this.keyToSpaceMapping.delete(baseKey);
925
948
 
926
- this.debug.log(`📦 Deleted data for key: ${fullKey}`);
949
+ this.debug.log(`📦 Deleted data for key: ${baseKey}`);
927
950
 
928
951
  // Emit deletion event
929
- this.emit('dataDeleted', { key: fullKey, keyId });
952
+ this.emit('dataDeleted', { key: baseKey, keyId });
930
953
 
931
954
  return true;
932
955
  } catch (error) {
933
- this.debug.error(`Failed to delete data for key ${fullKey}:`, error);
956
+ this.debug.error(`Failed to delete data for key ${baseKey}:`, error);
934
957
  throw error;
935
958
  }
936
959
  }
@@ -945,8 +968,8 @@ export class DistributedStorageManager extends EventEmitter {
945
968
  throw new Error('WebDHT not available - ensure it is enabled in mesh configuration');
946
969
  }
947
970
 
948
- // Subscribe to the WebDHT key
949
- const currentValue = await this.webDHT.subscribe(`storage:${key}`);
971
+ // Subscribe to the WebDHT key - NO PREFIXES
972
+ const currentValue = await this.webDHT.subscribe(key);
950
973
 
951
974
  this.debug.log(`📦 Subscribed to storage key: ${key}`);
952
975
 
@@ -962,7 +985,8 @@ export class DistributedStorageManager extends EventEmitter {
962
985
  return;
963
986
  }
964
987
 
965
- await this.webDHT.unsubscribe(`storage:${key}`);
988
+ // Unsubscribe from the WebDHT key - NO PREFIXES
989
+ await this.webDHT.unsubscribe(key);
966
990
  this.debug.log(`📦 Unsubscribed from storage key: ${key}`);
967
991
  }
968
992
 
@@ -1011,7 +1035,7 @@ export class DistributedStorageManager extends EventEmitter {
1011
1035
  async grantAccess(key, peerId) {
1012
1036
  // Resolve the key to its actual storage location
1013
1037
  const resolved = await this.resolveKey(key);
1014
- const { fullKey, keyId } = resolved;
1038
+ const { baseKey, keyId } = resolved;
1015
1039
 
1016
1040
  const accessControl = this.accessControl.get(keyId);
1017
1041
  const metadata = this.storageMetadata.get(keyId);
@@ -1031,16 +1055,16 @@ export class DistributedStorageManager extends EventEmitter {
1031
1055
 
1032
1056
  // Update the stored metadata
1033
1057
  try {
1034
- const currentPayload = await this.webDHT.get(`storage:${fullKey}`);
1058
+ const currentPayload = await this.webDHT.get(baseKey);
1035
1059
  if (currentPayload) {
1036
1060
  currentPayload.metadata = metadata;
1037
- await this.webDHT.update(`storage:${fullKey}`, currentPayload);
1061
+ await this.webDHT.update(baseKey, currentPayload);
1038
1062
  }
1039
1063
 
1040
- this.debug.log(`📦 Granted access to peer ${peerId.substring(0, 8)}... for key: ${fullKey}`);
1064
+ this.debug.log(`📦 Granted access to peer ${peerId.substring(0, 8)}... for key: ${baseKey}`);
1041
1065
 
1042
1066
  // Emit access granted event
1043
- this.emit('accessGranted', { key: fullKey, keyId, peerId });
1067
+ this.emit('accessGranted', { key: baseKey, keyId, peerId });
1044
1068
 
1045
1069
  return true;
1046
1070
  } catch (error) {
@@ -1060,7 +1084,7 @@ export class DistributedStorageManager extends EventEmitter {
1060
1084
  async revokeAccess(key, peerId) {
1061
1085
  // Resolve the key to its actual storage location
1062
1086
  const resolved = await this.resolveKey(key);
1063
- const { fullKey, keyId } = resolved;
1087
+ const { baseKey, keyId } = resolved;
1064
1088
 
1065
1089
  const accessControl = this.accessControl.get(keyId);
1066
1090
  const metadata = this.storageMetadata.get(keyId);
@@ -1076,16 +1100,16 @@ export class DistributedStorageManager extends EventEmitter {
1076
1100
 
1077
1101
  // Update the stored metadata
1078
1102
  try {
1079
- const currentPayload = await this.webDHT.get(`storage:${fullKey}`);
1103
+ const currentPayload = await this.webDHT.get(baseKey);
1080
1104
  if (currentPayload) {
1081
1105
  currentPayload.metadata = metadata;
1082
- await this.webDHT.update(`storage:${fullKey}`, currentPayload);
1106
+ await this.webDHT.update(baseKey, currentPayload);
1083
1107
  }
1084
1108
 
1085
- this.debug.log(`📦 Revoked access from peer ${peerId.substring(0, 8)}... for key: ${fullKey}`);
1109
+ this.debug.log(`📦 Revoked access from peer ${peerId.substring(0, 8)}... for key: ${baseKey}`);
1086
1110
 
1087
1111
  // Emit access revoked event
1088
- this.emit('accessRevoked', { key: fullKey, keyId, peerId });
1112
+ this.emit('accessRevoked', { key: baseKey, keyId, peerId });
1089
1113
 
1090
1114
  return true;
1091
1115
  } catch (error) {
@@ -1489,14 +1513,14 @@ export class DistributedStorageManager extends EventEmitter {
1489
1513
  async getKeyInfo(key) {
1490
1514
  // Resolve the key to its actual storage location
1491
1515
  const resolved = await this.resolveKey(key);
1492
- const { fullKey, keyId } = resolved;
1516
+ const { baseKey, keyId } = resolved;
1493
1517
 
1494
1518
  let metadata = this.storageMetadata.get(keyId);
1495
1519
 
1496
1520
  // If metadata is not found locally, try to retrieve it from the network
1497
1521
  if (!metadata) {
1498
1522
  try {
1499
- const existingData = await this.retrieve(fullKey);
1523
+ const existingData = await this.retrieve(baseKey);
1500
1524
  if (existingData !== null) {
1501
1525
  // Metadata should now be available after retrieve
1502
1526
  metadata = this.storageMetadata.get(keyId);
@@ -1601,11 +1625,11 @@ export class DistributedStorageManager extends EventEmitter {
1601
1625
  for (const keyId of this.ownedKeys) {
1602
1626
  const metadata = this.storageMetadata.get(keyId);
1603
1627
  if (metadata) {
1604
- // Check both the full key and the base key for prefix match
1605
- const fullKey = metadata.key;
1628
+ // Check both the stored key and the base key for prefix match
1629
+ const storedKey = metadata.key;
1606
1630
  const baseKey = metadata.baseKey;
1607
1631
 
1608
- if (fullKey.startsWith(prefix) || baseKey.startsWith(prefix)) {
1632
+ if (storedKey.startsWith(prefix) || baseKey.startsWith(prefix)) {
1609
1633
  // Return the base key for user convenience (since they stored with base key)
1610
1634
  keys.push(baseKey);
1611
1635
  }
@@ -1698,4 +1722,20 @@ export class DistributedStorageManager extends EventEmitter {
1698
1722
 
1699
1723
  return results;
1700
1724
  }
1725
+
1726
+ /**
1727
+ * Get a lexical interface for GUN-like chaining
1728
+ * @returns {Proxy} Lexical interface
1729
+ */
1730
+ lexical() {
1731
+ return createLexicalInterface(this);
1732
+ }
1733
+
1734
+ /**
1735
+ * Alias for lexical() - more GUN-like naming
1736
+ * @returns {Proxy} Lexical interface
1737
+ */
1738
+ gun() {
1739
+ return this.lexical();
1740
+ }
1701
1741
  }