sliftutils 0.59.0 → 0.60.0

package/index.d.ts

@@ -786,6 +786,7 @@ declare module "sliftutils/storage/IStorage" {
             lastModified: number;
         }>;
         reset(): Promise<void>;
+        watchResync?: (callback: () => void) => void;
     };
     export type IStorageRaw = {
         get(key: string): Promise<Buffer | undefined>;
@@ -904,6 +905,9 @@ declare module "sliftutils/storage/StorageObservable" {
         synced: {
             keySeqNum: number;
         };
+        resynced: {
+            seqNum: number;
+        };
         constructor(storage: IStorage<T>);
         get(key: string): T | undefined;
         set(key: string, value: T): void;
@@ -942,11 +946,14 @@ declare module "sliftutils/storage/TransactionStorage" {
         private debugName;
         private writeDelay;
         cache: Map<string, TransactionEntry>;
+        private diskFiles;
         private currentChunk;
         private entryCount;
         private static allStorage;
         constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
         static compressAll(): Promise<void>;
+        private resyncCallbacks;
+        watchResync(callback: () => void): void;
         private init;
         private getCurrentChunk;
         private onAddToChunk;
@@ -963,6 +970,7 @@ declare module "sliftutils/storage/TransactionStorage" {
         pushAppend(entry: TransactionEntry): Promise<void>;
         private updatePendingAppends;
         getKeys(): Promise<string[]>;
+        checkDisk(): Promise<void>;
         private loadAllTransactions;
         private parseTransactionFile;
         private applyTransactionEntries;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.59.0",
+    "version": "0.60.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/IStorage.d.ts

@@ -23,6 +23,7 @@ export type IStorage<T> = {
         lastModified: number;
     }>;
     reset(): Promise<void>;
+    watchResync?: (callback: () => void) => void;
 };
 export type IStorageRaw = {
     get(key: string): Promise<Buffer | undefined>;

package/storage/IStorage.ts

@@ -22,6 +22,8 @@ export type IStorage<T> = {
         lastModified: number;
     }>;
     reset(): Promise<void>;
+    // Allows watching for when the storage detects and underlying changes, and resyncs all of it's data (which might 
+    watchResync?: (callback: () => void) => void;
 };
 // NOTE: In the file system some characters are disallowed, and some characters do special things
 //  (/ makes a folder). And there are even more rules, such as lengths per folder, etc, etc.

package/storage/StorageObservable.d.ts

@@ -10,6 +10,9 @@ export declare class StorageSync<T> implements IStorageSync<T> {
     synced: {
         keySeqNum: number;
     };
+    resynced: {
+        seqNum: number;
+    };
     constructor(storage: IStorage<T>);
     get(key: string): T | undefined;
     set(key: string, value: T): void;

package/storage/StorageObservable.ts

@@ -11,9 +11,16 @@ export class StorageSync<T> implements IStorageSync<T> {
         keySeqNum: 0,
     }, undefined, { deep: false });
 
-    constructor(public storage: IStorage<T>) { }
+    resynced = observable({ seqNum: 0 });
+
+    constructor(public storage: IStorage<T>) {
+        storage.watchResync?.(() => {
+            this.resynced.seqNum++;
+        });
+    }
 
     public get(key: string): T | undefined {
+        this.resynced.seqNum;
         if (!this.cached.has(key)) {
             this.cached.set(key, undefined);
             void this.getPromise(key);
@@ -41,12 +48,14 @@ export class StorageSync<T> implements IStorageSync<T> {
     }
     private loadedKeys = false;
     public getKeys(): string[] {
+        this.resynced.seqNum;
         void this.getKeysPromise();
         this.synced.keySeqNum;
         return Array.from(this.keys);
     }
 
     public getInfo(key: string): { size: number; lastModified: number } | undefined {
+        this.resynced.seqNum;
         if (!this.infoCached.has(key)) {
             this.infoCached.set(key, { size: 0, lastModified: 0 });
             void this.storage.getInfo(key).then(info => {

package/storage/TransactionStorage.d.ts

@@ -12,11 +12,14 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     private debugName;
     private writeDelay;
     cache: Map<string, TransactionEntry>;
+    private diskFiles;
     private currentChunk;
     private entryCount;
     private static allStorage;
     constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
     static compressAll(): Promise<void>;
+    private resyncCallbacks;
+    watchResync(callback: () => void): void;
     private init;
     private getCurrentChunk;
     private onAddToChunk;
@@ -33,6 +36,7 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     pushAppend(entry: TransactionEntry): Promise<void>;
     private updatePendingAppends;
     getKeys(): Promise<string[]>;
+    checkDisk(): Promise<void>;
     private loadAllTransactions;
     private parseTransactionFile;
     private applyTransactionEntries;

package/storage/TransactionStorage.ts

@@ -6,6 +6,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 import { setPending } from "./PendingManager";
 import { isInBuild } from "../misc/environment";
 import { isNode } from "typesafecss";
+import { runInfinitePoll } from "socket-function/src/batching";
 
 /*
 // Spec:
@@ -39,6 +40,7 @@ UPDATE now we use chunks, because append is too slow.
 IMPORTANT! If there are multiple writers, we clobber writes from other writers when we compress
 */
 
+const DISK_CHECK_INTERVAL = timeInMinute * 5;
 
 const FILE_CHUNK_SIZE = 1024 * 1024;
 const FILE_MAX_LIFETIME = timeInMinute * 30;
@@ -76,6 +78,7 @@ function getNextChunkPath(): string {
 
 export class TransactionStorage implements IStorage<Buffer> {
     public cache: Map<string, TransactionEntry> = new Map();
+    private diskFiles: Set<string> = new Set();
     private currentChunk: {
         path: string;
         size: number;
@@ -91,6 +94,8 @@ export class TransactionStorage implements IStorage<Buffer> {
         private writeDelay = WRITE_DELAY
     ) {
         TransactionStorage.allStorage.push(this);
+        // VERY useful for debugging.
+        (globalThis as any)[`transactionStorage-${this.debugName}`] = this;
     }
     // Helps get rid of parse errors which constantly log. Also, uses less space
     public static async compressAll() {
@@ -101,6 +106,11 @@ export class TransactionStorage implements IStorage<Buffer> {
         });
     }
 
+    private resyncCallbacks: (() => void)[] = [];
+    public watchResync(callback: () => void): void {
+        this.resyncCallbacks.push(callback);
+    }
+
     private init: Promise<unknown> | undefined = this.loadAllTransactions();
 
     private getCurrentChunk(): string {
@@ -113,6 +123,7 @@ export class TransactionStorage implements IStorage<Buffer> {
                 size: 0,
                 timeCreated: Date.now()
             };
+            this.diskFiles.add(this.currentChunk.path);
         }
         return this.currentChunk.path;
     }
@@ -239,14 +250,35 @@ export class TransactionStorage implements IStorage<Buffer> {
         return Array.from(this.cache.keys());
     }
 
+    public async checkDisk(): Promise<void> {
+        if (this.init) await this.init;
+        const anyChanges = async () => {
+            let keys = await this.rawStorage.getKeys();
+            let diskFiles = keys.filter(key => key.endsWith(CHUNK_EXT));
+            let hasNew = diskFiles.some(file => !this.diskFiles.has(file));
+            return hasNew;
+        };
+        if (!await anyChanges()) return;
+
+        await fileLockSection(async () => {
+            if (!await anyChanges()) return;
+            await this.loadAllTransactions();
+        });
+    }
+
 
     // NOTE: This is either called in init (which blocks all other calls), or inside of the global file lock, so it is safe to load.
     private async loadAllTransactions(): Promise<string[]> {
         if (isInBuild()) return [];
 
+        if (this.init) {
+            runInfinitePoll(DISK_CHECK_INTERVAL, () => this.checkDisk());
+        }
+
         let time = Date.now();
         const keys = await this.rawStorage.getKeys();
         const transactionFiles = keys.filter(key => key.endsWith(CHUNK_EXT));
+        this.diskFiles = new Set(transactionFiles);
 
         let entryList: TransactionEntry[][] = [];
         for (let file of transactionFiles) {
@@ -330,25 +362,62 @@ export class TransactionStorage implements IStorage<Buffer> {
             }
             pendingWriteTimes.set(entry.key, entry.time);
         }
-
-        sort(entries, x => x.time);
-        for (let entry of entries) {
-            let time = entry.time;
-            let prevTime = pendingWriteTimes.get(entry.key);
-            if (prevTime && prevTime > time) {
+        let latest = new Map<string, TransactionEntry>();
+        for (const entry of entries) {
+            let pendingTime = pendingWriteTimes.get(entry.key);
+            if (pendingTime && pendingTime > entry.time) {
+                continue;
+            }
+            let prev = latest.get(entry.key);
+            if (prev && prev.time > entry.time) {
                 continue;
             }
+            latest.set(entry.key, entry);
+        }
 
+        let anyChanged = false;
+        for (let entry of latest.values()) {
             if (entry.value === undefined) {
+                if (!this.cache.has(entry.key)) continue;
+                anyChanged = true;
                 this.cache.delete(entry.key);
             } else {
-                let prev = this.cache.get(entry.key);
-                if (prev && (prev.time > entry.time)) {
-                    continue;
+                if (!anyChanged) {
+                    let prev = this.cache.get(entry.key);
+                    if (!prev || prev.isZipped !== entry.isZipped) {
+                        anyChanged = true;
+                    } else {
+                        if (!prev.value) {
+                            if (entry.value) {
+                                anyChanged = true;
+                            }
+                        } else {
+                            if (!entry.value) {
+                                anyChanged = true;
+                            } else {
+                                // Both values, so... it might not have changed
+                                if (!prev.value.equals(entry.value)) {
+                                    anyChanged = true;
+                                }
+                            }
+                        }
+                    }
                 }
                 this.cache.set(entry.key, entry);
             }
         }
+
+        if (anyChanged) {
+            for (const callback of this.resyncCallbacks) {
+                try {
+                    callback();
+                } catch (e) {
+                    setImmediate(() => {
+                        throw e;
+                    });
+                }
+            }
+        }
     }
 
     private readTransactionEntry(buffer: Buffer, offset: number): {
@@ -511,6 +580,7 @@ export class TransactionStorage implements IStorage<Buffer> {
             //      the other generations, which is annoying).
             for (const file of existingDiskEntries) {
                 await this.rawStorage.remove(file);
+                this.diskFiles.delete(file);
             }
         } finally {
             this.compressing = false;
@@ -528,6 +598,7 @@ export class TransactionStorage implements IStorage<Buffer> {
 
             this.pendingAppends = [];
             this.cache.clear();
+            this.diskFiles.clear();
             this.currentChunk = undefined;
             this.entryCount = 0;
         });