sliftutils 1.1.4 → 1.2.0

package/index.d.ts

@@ -296,6 +296,7 @@ declare module "sliftutils/misc/zip" {
     import { MaybePromise } from "socket-function/src/types";
     export declare class Zip {
         static gzip(buffer: Buffer, level?: number): Promise<Buffer>;
+        static gzipSync(buffer: Buffer, level?: number): Buffer;
         static gunzip(buffer: Buffer): MaybePromise<Buffer>;
         static gunzipAsyncBase(buffer: Buffer): Promise<Buffer>;
         static gunzipUntracked(buffer: Buffer): Promise<Buffer>;
@@ -1121,6 +1122,9 @@ declare module "sliftutils/storage/PrivateFileSystemStorage" {
 
 declare module "sliftutils/storage/StorageObservable" {
     import { IStorage, IStorageSync } from "./IStorage";
+    export declare const storagePendingAccesses: {
+        value: number;
+    };
     export declare class StorageSync<T> implements IStorageSync<T> {
         storage: IStorage<T>;
         private config?;
@@ -1160,6 +1164,13 @@ declare module "sliftutils/storage/StorageObservable" {
         reloadKey(key: string): void;
         reset(): Promise<void>;
     }
+    export declare function waitUntilNextLoad(): Promise<void>;
+
+}
+
+declare module "sliftutils/storage/StorageObservableAsync" {
+    /** Reruns the code until all StorageSyncs accessed have loaded their values. Not efficient,although will usually be O(values accessed), just due to how loading works (it won't be quadratic). */
+    export declare function rerunCodeUntilAllLoaded<T>(code: () => T): Promise<T>;
 
 }
 

package/misc/zip.d.ts

@@ -3,6 +3,7 @@
 import { MaybePromise } from "socket-function/src/types";
 export declare class Zip {
     static gzip(buffer: Buffer, level?: number): Promise<Buffer>;
+    static gzipSync(buffer: Buffer, level?: number): Buffer;
     static gunzip(buffer: Buffer): MaybePromise<Buffer>;
     static gunzipAsyncBase(buffer: Buffer): Promise<Buffer>;
     static gunzipUntracked(buffer: Buffer): Promise<Buffer>;

package/misc/zip.ts

@@ -24,6 +24,13 @@ export class Zip {
             return await doStream(new CompressionStream("gzip"), buffer);
         }
     }
+    @measureFnc
+    public static gzipSync(buffer: Buffer, level?: number): Buffer {
+        if (isNode()) {
+            return Buffer.from(zlib.gzipSync(buffer, { level }));
+        }
+        return Buffer.from(pako.deflate(buffer));
+    }
     public static gunzip(buffer: Buffer): MaybePromise<Buffer> {
         // Switch to the synchronous version if the buffer is small. This is a lot faster in Node.js and clientside.
         //  - On tests of random small amounts of data, this seems to be up to 7X faster (on node). However, on non-random data, on the actual data we're using, it seems to be almost 50 times faster. So... definitely worth it...

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.1.4",
+    "version": "1.2.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/StorageObservable.d.ts

@@ -1,4 +1,7 @@
 import { IStorage, IStorageSync } from "./IStorage";
+export declare const storagePendingAccesses: {
+    value: number;
+};
 export declare class StorageSync<T> implements IStorageSync<T> {
     storage: IStorage<T>;
     private config?;
@@ -38,3 +41,4 @@ export declare class StorageSync<T> implements IStorageSync<T> {
     reloadKey(key: string): void;
     reset(): Promise<void>;
 }
+export declare function waitUntilNextLoad(): Promise<void>;

package/storage/StorageObservable.ts

@@ -1,8 +1,11 @@
 import { observable } from "mobx";
 import { deepFreezeObject, freezeObject, isDefined } from "../misc/types";
 import { IStorage, IStorageSync } from "./IStorage";
+import { PromiseObj } from "socket-function/src/misc";
 
-// NOTE: At around 500K values (depending on their size to some degree), this will take about 2 minutes to load. But once it does it will be fast. So... keep that in mind. I recommend not exceeding 100K
+export const storagePendingAccesses = { value: 0 };
+
+// NOTE: At around 500K values (depending on their size to some degree), this will take about 2 minutes to load. But once it does it will be fast. So... keep that in mind. I recommend not exceeding 100K.
 export class StorageSync<T> implements IStorageSync<T> {
     cached = observable.map<string, T | undefined>(undefined, { deep: false });
     infoCached = observable.map<string, { size: number; lastModified: number } | undefined>(undefined, { deep: false });
@@ -85,6 +88,7 @@ export class StorageSync<T> implements IStorageSync<T> {
     public async getPromise(key: string): Promise<T | undefined> {
         let value = this.cached.get(key);
         if (value === undefined) {
+            storagePendingAccesses.value++;
             value = await this.storage.get(key);
             if (value !== undefined && this.cached.get(key) === undefined) {
                 if (this.config?.freeze === "shallow") {
@@ -94,15 +98,19 @@ export class StorageSync<T> implements IStorageSync<T> {
                 }
                 this.cached.set(key, value);
             }
+            triggerLoad();
         }
         return value;
     }
     private pendingGetKeys: Promise<string[]> | undefined;
     public async getKeysPromise(): Promise<string[]> {
+        if (this.loadedKeys) {
+            return Array.from(this.keys);
+        }
+        storagePendingAccesses.value++;
         if (this.pendingGetKeys) {
             return this.pendingGetKeys;
         }
-        if (this.loadedKeys) return Array.from(this.keys);
         this.loadedKeys = true;
         this.pendingGetKeys = this.storage.getKeys();
         void this.pendingGetKeys.finally(() => {
@@ -113,6 +121,7 @@ export class StorageSync<T> implements IStorageSync<T> {
             this.keys = new Set(keys);
             this.synced.keySeqNum++;
         }
+        triggerLoad();
         return Array.from(this.keys);
     }
 
@@ -140,4 +149,20 @@ export class StorageSync<T> implements IStorageSync<T> {
         this.synced.keySeqNum++;
         await this.storage.reset();
     }
+}
+
+let waitUntilNextLoadWatchers: PromiseObj<void>[] = [];
+export function waitUntilNextLoad(): Promise<void> {
+    let promise = new PromiseObj<void>();
+    waitUntilNextLoadWatchers.push(promise);
+    return promise.promise;
+}
+function triggerLoad() {
+    let watchers = waitUntilNextLoadWatchers;
+    waitUntilNextLoadWatchers = [];
+    void Promise.resolve().finally(() => {
+        for (let watcher of watchers) {
+            watcher.resolve();
+        }
+    });
 }

package/storage/StorageObservableAsync.d.ts

@@ -0,0 +1,2 @@
+/** Reruns the code until all StorageSyncs accessed have loaded their values. Not efficient,although will usually be O(values accessed), just due to how loading works (it won't be quadratic). */
+export declare function rerunCodeUntilAllLoaded<T>(code: () => T): Promise<T>;

package/storage/StorageObservableAsync.ts

@@ -0,0 +1,20 @@
+import { storagePendingAccesses, waitUntilNextLoad } from "./StorageObservable";
+
+/** Reruns the code until all StorageSyncs accessed have loaded their values. Not efficient,although will usually be O(values accessed), just due to how loading works (it won't be quadratic). */
+export async function rerunCodeUntilAllLoaded<T>(code: () => T): Promise<T> {
+    while (true) {
+        let beforeAccesses = storagePendingAccesses.value;
+        try {
+            let result = await code();
+            if (storagePendingAccesses.value === beforeAccesses) {
+                return result;
+            }
+        } catch (error) {
+            if (storagePendingAccesses.value === beforeAccesses) {
+                throw error;
+            }
+        }
+        console.log(`Rerunning synchronous check as loading starting while evaluating code. Function name`, code);
+        await waitUntilNextLoad();
+    }
+}