sliftutils 0.78.0 → 0.80.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.78.0",
+    "version": "0.80.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/StorageObservable.ts

@@ -12,12 +12,13 @@ export class StorageSync<T> implements IStorageSync<T> {
     }, undefined, { deep: false });
 
     constructor(public storage: IStorage<T>) {
-        storage.watchResync?.(() => {
-            this.cached.clear();
-            this.infoCached.clear();
-            this.keys.clear();
+        storage.watchResync?.(async () => {
+            // NOTE: If there's multiple tabs open, this'll trigger a lot, so we can't just clear all the values, as that'll cause a render where nothing's loaded. 
             this.loadedKeys = false;
-            this.synced.keySeqNum++;
+            let keys = await this.getKeysPromise();
+            for (let key of keys) {
+                this.get(key);
+            }
         });
     }
 

package/storage/TransactionStorage.ts

@@ -55,6 +55,7 @@ const WRITE_DELAY = 500;
 interface TransactionEntry {
     key: string;
     value: Buffer | undefined;
+    // NOTE: We disabled per-entry zipping. Either use a raw collection so you don't have to load all of them into memory or shard your collection. Zipping is only a band-aid solution for memory issues. ALSO, it makes loading slower, which makes all other applications that don't need zipping worse.
     isZipped: boolean;
     time: number;
 }
@@ -295,16 +296,17 @@ export class TransactionStorage implements IStorage<Buffer> {
             this.diskFiles.set(file, info?.lastModified ?? Date.now());
         }
 
+        let size = { value: 0 };
         let entryList: TransactionEntry[][] = [];
         for (let file of transactionFiles) {
-            entryList.push(await this.parseTransactionFile(file));
+            entryList.push(await this.parseTransactionFile(file, size));
         }
         let entries = entryList.flat();
         this.applyTransactionEntries(entries, initialLoad);
 
         time = Date.now() - time;
         if (time > 50) {
-            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, entries ${formatNumber(entries.length)}B`, transactionFiles);
+            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, entries ${formatNumber(entries.length)}, total size ${formatNumber(size.value)}B`, transactionFiles);
         }
 
         this.init = undefined;
@@ -312,13 +314,14 @@ export class TransactionStorage implements IStorage<Buffer> {
     }
 
     // ONLY call this inside of loadAllTransactions
-    private async parseTransactionFile(filename: string): Promise<TransactionEntry[]> {
+    private async parseTransactionFile(filename: string, size: { value: number }): Promise<TransactionEntry[]> {
         const fullFile = await this.rawStorage.get(filename);
         if (!fullFile) return [];
         if (fullFile.length < 4) {
             //console.error(`Transaction in ${this.debugName} file ${filename} is too small, skipping`);
             return [];
         }
+        size.value += fullFile.length;
         let headerSize = fullFile.readUInt32LE(0);
         let headerBuffer = fullFile.slice(4, 4 + headerSize);
         let header: TransactionHeader;