sliftutils 0.77.0 → 0.79.0

package/index.d.ts

@@ -145,9 +145,14 @@ declare module "sliftutils/misc/yamlBase" {
 declare module "sliftutils/misc/zip" {
     /// <reference types="node" />
     /// <reference types="node" />
+    import { MaybePromise } from "socket-function/src/types";
     export declare class Zip {
         static gzip(buffer: Buffer, level?: number): Promise<Buffer>;
-        static gunzip(buffer: Buffer): Promise<Buffer>;
+        static gunzip(buffer: Buffer): MaybePromise<Buffer>;
+        static gunzipAsyncBase(buffer: Buffer): Promise<Buffer>;
+        static gunzipUntracked(buffer: Buffer): Promise<Buffer>;
+        static gunzipSync(buffer: Buffer): Buffer;
+        private static gunzipUntrackedSync;
         static gunzipBatch(buffers: Buffer[]): Promise<Buffer[]>;
     }
 

package/misc/zip.d.ts

@@ -1,7 +1,12 @@
 /// <reference types="node" />
 /// <reference types="node" />
+import { MaybePromise } from "socket-function/src/types";
 export declare class Zip {
     static gzip(buffer: Buffer, level?: number): Promise<Buffer>;
-    static gunzip(buffer: Buffer): Promise<Buffer>;
+    static gunzip(buffer: Buffer): MaybePromise<Buffer>;
+    static gunzipAsyncBase(buffer: Buffer): Promise<Buffer>;
+    static gunzipUntracked(buffer: Buffer): Promise<Buffer>;
+    static gunzipSync(buffer: Buffer): Buffer;
+    private static gunzipUntrackedSync;
     static gunzipBatch(buffers: Buffer[]): Promise<Buffer[]>;
 }

package/misc/zip.ts

@@ -1,18 +1,86 @@
+import { isNode } from "socket-function/src/misc";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import zlib from "zlib";
+import * as pako from "pako";
 
+import { setFlag } from "socket-function/require/compileFlags";
+import { MaybePromise } from "socket-function/src/types";
+setFlag(require, "pako", "allowclient", true);
+
+const SYNC_THRESHOLD_BYTES = 100_000_000;
 
 export class Zip {
+    @measureFnc
     public static async gzip(buffer: Buffer, level?: number): Promise<Buffer> {
-        return await doStream(new CompressionStream("gzip"), buffer);
+        if (isNode()) {
+            return new Promise((resolve, reject) => {
+                zlib.gzip(buffer, { level }, (err: any, result: Buffer) => {
+                    if (err) reject(err);
+                    else resolve(result);
+                });
+            });
+        } else {
+            // @ts-ignore
+            return await doStream(new CompressionStream("gzip"), buffer);
+        }
+    }
+    public static gunzip(buffer: Buffer): MaybePromise<Buffer> {
+        // Switch to the synchronous version if the buffer is small. This is a lot faster in Node.js and clientside.
+        //  - On tests of random small amounts of data, this seems to be up to 7X faster (on node). However, on non-random data, on the actual data we're using, it seems to be almost 50 times faster. So... definitely worth it...
+        if (buffer.length < SYNC_THRESHOLD_BYTES) {
+            let time = Date.now();
+            let result = Zip.gunzipSync(buffer);
+            let duration = Date.now() - time;
+            if (duration > 50) {
+                // Wait, so we don't lock up the main thread. And if we already wait it 50ms, then waiting for one frame is marginal, even client-side. 
+                return ((async () => {
+                    await new Promise(resolve => setTimeout(resolve, 0));
+                    return result;
+                }))();
+            }
+            return result;
+        }
+        return Zip.gunzipAsyncBase(buffer);
+    }
+    @measureFnc
+    public static async gunzipAsyncBase(buffer: Buffer): Promise<Buffer> {
+        return Zip.gunzipUntracked(buffer);
     }
-    public static async gunzip(buffer: Buffer): Promise<Buffer> {
+    // A base function, so we can avoid instrumentation for batch calls
+    public static async gunzipUntracked(buffer: Buffer): Promise<Buffer> {
+        if (isNode()) {
+            return await new Promise<Buffer>((resolve, reject) => {
+                zlib.gunzip(buffer, (err: any, result: Buffer) => {
+                    if (err) reject(err);
+                    else resolve(result);
+                });
+            });
+        }
         return await doStream(new DecompressionStream("gzip"), buffer);
     }
 
+    @measureFnc
+    public static gunzipSync(buffer: Buffer): Buffer {
+        return this.gunzipUntrackedSync(buffer);
+    }
+    private static gunzipUntrackedSync(buffer: Buffer): Buffer {
+        if (isNode()) {
+            return Buffer.from(zlib.gunzipSync(buffer));
+        }
+        return Buffer.from(pako.inflate(buffer));
+    }
+
+    @measureFnc
     public static async gunzipBatch(buffers: Buffer[]): Promise<Buffer[]> {
         let time = Date.now();
-        buffers = await Promise.all(buffers.map(Zip.gunzip));
+        buffers = await Promise.all(buffers.map(x => {
+            if (x.length < SYNC_THRESHOLD_BYTES) {
+                return this.gunzipUntrackedSync(x);
+            }
+            return this.gunzipUntracked(x);
+        }));
         time = Date.now() - time;
-        //let totalSize = buffers.reduce((acc, buffer) => acc + buffer.length, 0);
+        let totalSize = buffers.reduce((acc, buffer) => acc + buffer.length, 0);
         //console.log(`Gunzip ${formatNumber(totalSize)}B at ${formatNumber(totalSize / time * 1000)}B/s`);
         return buffers;
     }
@@ -34,4 +102,5 @@ async function doStream(stream: GenericTransformStream, buffer: Buffer): Promise
         }
         outputBuffers.push(Buffer.from(value));
     }
+
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.77.0",
+    "version": "0.79.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/TransactionStorage.ts

@@ -47,7 +47,6 @@ const FILE_MAX_LIFETIME = timeInMinute * 30;
 
 const FILE_ZIP_THRESHOLD = 16 * 1024 * 1024;
 
-const ZIP_THRESHOLD = 4096;
 const START_BYTES = Buffer.from([236, 49, 112, 121, 27, 127, 227, 63]);
 const END_BYTES = Buffer.from([220, 111, 243, 202, 200, 79, 213, 63]);
 // Delay writes, so we batch better, and thrash the disk less
@@ -56,6 +55,7 @@ const WRITE_DELAY = 500;
 interface TransactionEntry {
     key: string;
     value: Buffer | undefined;
+    // NOTE: We disabled per-entry zipping. Either use a raw collection so you don't have to load all of them into memory or shard your collection. Zipping is only a band-aid solution for memory issues. ALSO, it makes loading slower, which makes all other applications that don't need zipping worse.
     isZipped: boolean;
     time: number;
 }
@@ -143,9 +143,8 @@ export class TransactionStorage implements IStorage<Buffer> {
     public async get(key: string): Promise<Buffer | undefined> {
         await this.init;
         const value = this.cache.get(key);
-        if (value && value.isZipped && value.value) {
-            value.value = await Zip.gunzip(value.value);
-            value.isZipped = false;
+        if (value && value.isZipped) {
+            throw new Error(`Transactions should be unzipped on file ready, so this state should be impossible. Key: ${key}`);
         }
         return value?.value;
     }
@@ -157,11 +156,6 @@ export class TransactionStorage implements IStorage<Buffer> {
         let entry: TransactionEntry = { key, value, isZipped: false, time: 0 };
         this.cache.set(key, entry);
 
-        if (value.length >= ZIP_THRESHOLD) {
-            value = await Zip.gzip(value);
-            entry.value = value;
-            entry.isZipped = true;
-        }
         await this.pushAppend(entry);
     }
 
@@ -302,16 +296,17 @@ export class TransactionStorage implements IStorage<Buffer> {
             this.diskFiles.set(file, info?.lastModified ?? Date.now());
         }
 
+        let size = { value: 0 };
         let entryList: TransactionEntry[][] = [];
         for (let file of transactionFiles) {
-            entryList.push(await this.parseTransactionFile(file));
+            entryList.push(await this.parseTransactionFile(file, size));
         }
         let entries = entryList.flat();
-        this.applyTransactionEntries(entries);
+        this.applyTransactionEntries(entries, initialLoad);
 
         time = Date.now() - time;
         if (time > 50) {
-            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, entries ${formatNumber(entries.length)}B`, transactionFiles);
+            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, entries ${formatNumber(entries.length)}, total size ${formatNumber(size.value)}B`, transactionFiles);
         }
 
         this.init = undefined;
@@ -319,13 +314,14 @@ export class TransactionStorage implements IStorage<Buffer> {
     }
 
     // ONLY call this inside of loadAllTransactions
-    private async parseTransactionFile(filename: string): Promise<TransactionEntry[]> {
+    private async parseTransactionFile(filename: string, size: { value: number }): Promise<TransactionEntry[]> {
         const fullFile = await this.rawStorage.get(filename);
         if (!fullFile) return [];
         if (fullFile.length < 4) {
             //console.error(`Transaction in ${this.debugName} file ${filename} is too small, skipping`);
             return [];
         }
+        size.value += fullFile.length;
         let headerSize = fullFile.readUInt32LE(0);
         let headerBuffer = fullFile.slice(4, 4 + headerSize);
         let header: TransactionHeader;
@@ -375,7 +371,7 @@ export class TransactionStorage implements IStorage<Buffer> {
         }
         return entries;
     }
-    private applyTransactionEntries(entries: TransactionEntry[]): void {
+    private applyTransactionEntries(entries: TransactionEntry[], initialLoad?: boolean): void {
         let pendingWriteTimes = new Map<string, number>();
         for (const entry of this.pendingAppends) {
             let prevTime = pendingWriteTimes.get(entry.key);
@@ -404,9 +400,9 @@ export class TransactionStorage implements IStorage<Buffer> {
                 anyChanged = true;
                 this.cache.delete(entry.key);
             } else {
-                if (!anyChanged) {
+                if (!anyChanged && !initialLoad) {
                     let prev = this.cache.get(entry.key);
-                    if (!prev || prev.isZipped !== entry.isZipped) {
+                    if (!prev) {
                         anyChanged = true;
                     } else {
                         if (!prev.value) {
@@ -429,7 +425,8 @@ export class TransactionStorage implements IStorage<Buffer> {
             }
         }
 
-        if (anyChanged) {
+        if (anyChanged && !initialLoad) {
+            console.warn(`Transaction storage ${this.debugName} changed on disk, triggering full download from disk`);
             for (const callback of this.resyncCallbacks) {
                 try {
                     callback();
@@ -467,6 +464,11 @@ export class TransactionStorage implements IStorage<Buffer> {
         let isZipped = (flags & 1) === 1;
         let isDelete = (flags & 2) === 2;
 
+        if (isZipped) {
+            value = Zip.gunzipSync(value);
+            isZipped = false;
+        }
+
         let entry: TransactionEntry = { key, value, isZipped, time };
         if (isDelete) {
             entry.value = undefined;

package/web/Page.tsx

@@ -18,7 +18,6 @@ export class Page extends preact.Component {
         );
         if (ignore) return;
 
-        console.log("Checking hotkey", e.key, e);
         let key = e.key;
         if (e.ctrlKey) key = "Ctrl+" + key;
         if (e.shiftKey) key = "Shift+" + key;