sliftutils 0.50.0 → 0.51.0

package/bundler/bundleRequire.ts

@@ -252,7 +252,7 @@ export function bundleRequire(config: BundleRequireConfig) {
                             if (property === "default") return newModule.exports;
 
                             console.warn(
-                                `Module ${newModule.filename} is not available (are you missing "module.allowclient = true"?). Tried to access ${String(property)}.`
+                                `Module ${newModule.filename} is not available. It might have not been imported in Node.js due to a flag which is checking the browser or an environment variable. It might also be that the entry point is weirdly configured and could not be detected.`
                             );
                             return undefined;
                         },

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.50.0",
+    "version": "0.51.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/TransactionStorage.ts

@@ -101,9 +101,11 @@ export class TransactionStorage implements IStorage<Buffer> {
     }
     // Helps get rid of parse errors which constantly log. Also, uses less space
     public static async compressAll() {
-        for (let storage of TransactionStorage.allStorage) {
-            await storage.compressTransactionLog(true);
-        }
+        await fileLockSection(async () => {
+            for (let storage of TransactionStorage.allStorage) {
+                await storage.compressTransactionLog(true);
+            }
+        });
     }
 
     private init: Promise<void> | undefined = this.loadAllTransactions();
@@ -287,6 +289,15 @@ export class TransactionStorage implements IStorage<Buffer> {
             content = await Zip.gunzip(content);
         }
 
+        let pendingWriteTimes = new Map<string, number>();
+        for (const entry of this.pendingAppends) {
+            let prevTime = pendingWriteTimes.get(entry.key);
+            if (prevTime && prevTime > entry.time) {
+                continue;
+            }
+            pendingWriteTimes.set(entry.key, entry.time);
+        }
+
         let offset = 0;
         let entries: TransactionEntry[] = [];
         while (offset < content.length) {
@@ -314,11 +325,18 @@ export class TransactionStorage implements IStorage<Buffer> {
                 offset++;
                 continue;
             }
+
             this.entryCount++;
             let { entry } = entryObj;
             offset = entryObj.offset;
             entries.push(entry);
 
+            let time = entry.time;
+            let prevTime = pendingWriteTimes.get(entry.key);
+            if (prevTime && prevTime > time) {
+                continue;
+            }
+
             if (entry.value === undefined) {
                 this.cache.delete(entry.key);
             } else {
@@ -432,79 +450,73 @@ export class TransactionStorage implements IStorage<Buffer> {
     private compressing = false;
     private async compressTransactionLog(force?: boolean): Promise<void> {
         if (this.compressing) return;
-        this.compressing = true;
-
-        let existingDiskEntries = await this.rawStorage.getKeys();
-        existingDiskEntries = existingDiskEntries.filter(x => x.endsWith(CHUNK_EXT));
-        let compressNow = force || (
-            this.entryCount > 100 && this.entryCount > this.cache.size * 3
-            // NOTE: This compress check breaks down if we only have very large values, but... those
-            //  don't work ANYWAYS (it is better to use one file per value instead).
-            //  - Maybe we should throw, or at least warn, on sets of value > 1MB,
-            //      at which point they should just use a file per value
-            || existingDiskEntries.length > Math.max(10, Math.ceil(this.entryCount / 1000))
-            || existingDiskEntries.length > 1000 * 10
-        );
-        if (!compressNow) return;
-        console.log(`Compressing ${this.debugName} transaction log, ${this.entryCount} entries, ${this.cache.size} keys`);
-
-        // Load off disk, in case there are other writes. We still race with them, but at least
-        //  this reduces the race condition considerably
+        try {
+            this.compressing = true;
 
-        sortChunks(existingDiskEntries);
-        for (let entry of existingDiskEntries) {
-            await this.loadTransactionFile(entry);
-        }
+            let existingDiskEntries = await this.rawStorage.getKeys();
+            existingDiskEntries = existingDiskEntries.filter(x => x.endsWith(CHUNK_EXT));
+            let compressNow = force || (
+                this.entryCount > 100 && this.entryCount > this.cache.size * 3
+                // NOTE: This compress check breaks down if we only have very large values, but... those
+                //  don't work ANYWAYS (it is better to use one file per value instead).
+                //  - Maybe we should throw, or at least warn, on sets of value > 1MB,
+                //      at which point they should just use a file per value
+                || existingDiskEntries.length > Math.max(10, Math.ceil(this.entryCount / 1000))
+                || existingDiskEntries.length > 1000 * 10
+            );
+            if (!compressNow) return;
+            console.log(`Compressing ${this.debugName} transaction log, ${this.entryCount} entries, ${this.cache.size} keys`);
 
-        this.entryCount = this.cache.size;
+            // Load off disk, in case there are other writes. We still race with them, but at least
+            //  this reduces the race condition considerably
 
+            sortChunks(existingDiskEntries);
+            for (let entry of existingDiskEntries) {
+                await this.loadTransactionFile(entry);
+            }
 
-        let buffers: Buffer[] = [];
-        for (const entry of this.cache.values()) {
-            let buffer = this.serializeTransactionEntry(entry);
-            buffers.push(buffer);
-        }
+            this.entryCount = this.cache.size;
 
-        let newChunks = this.chunkBuffers(buffers);
 
-        let newFiles: string[] = [];
-        for (let chunk of newChunks) {
-            let file = this.getCurrentChunk();
-            newFiles.push(file);
-            this.currentChunk = undefined;
-            let content = chunk.buffer;
-            let { header, headerBuffer } = this.getHeader(
-                // AND, never compress the last one, otherwise we can't append to it!
-                content.length >= FILE_ZIP_THRESHOLD && chunk !== newChunks[newChunks.length - 1]
-            );
-            if (header.zipped) {
-                content = await Zip.gzip(content);
+            let buffers: Buffer[] = [];
+            for (const entry of this.cache.values()) {
+                let buffer = this.serializeTransactionEntry(entry);
+                buffers.push(buffer);
             }
-            let buffer = Buffer.concat([headerBuffer, content]);
-            await this.rawStorage.set(file, buffer);
-            let verified = await this.rawStorage.get(file);
-            if (!verified?.equals(buffer)) {
-                console.error(`Failed to verify transaction file ${file} in ${this.debugName}`);
-                throw new Error(`Failed to verify transaction file ${file} in ${this.debugName}`);
+
+            let newChunks = this.chunkBuffers(buffers);
+
+            let newFiles: string[] = [];
+            for (let chunk of newChunks) {
+                // Don't use the previous current chunk for the new file. Also clear it afterwards so it's not reused for any future rights. 
+                this.currentChunk = undefined;
+                let file = this.getCurrentChunk();
+                this.currentChunk = undefined;
+                newFiles.push(file);
+                let content = chunk.buffer;
+                let { header, headerBuffer } = this.getHeader(
+                    content.length >= FILE_ZIP_THRESHOLD
+                );
+                if (header.zipped) {
+                    content = await Zip.gzip(content);
+                }
+                let buffer = Buffer.concat([headerBuffer, content]);
+                await this.rawStorage.set(file, buffer);
+                let verified = await this.rawStorage.get(file);
+                if (!verified?.equals(buffer)) {
+                    console.error(`Failed to verify transaction file ${file} in ${this.debugName}`);
+                    throw new Error(`Failed to verify transaction file ${file} in ${this.debugName}`);
+                }
             }
-        }
-        // Delay, because we want to be absolutely certain that all the writes have finished before deleting the old files.
-        await delay(5000);
-        if (!isNode()) {
-            localStorage.setItem(`${this.debugName}-last-compress`, JSON.stringify({
-                time: Date.now(),
-                entryCount: this.entryCount,
-                cacheSize: this.cache.size,
-                newFiles,
-                existingDiskEntries,
-            }));
-        }
 
-        // This is the ONLY time we can delete old files, as we know for sure the new file has all of our data.
-        //  Any future readers won't know this, unless they write it themselves (or unless they audit it against
-        //      the other generations, which is annoying).
-        for (const file of existingDiskEntries) {
-            await this.rawStorage.remove(file);
+            // This is the ONLY time we can delete old files, as we know for sure the new file has all of our data.
+            //  Any future readers won't know this, unless they write it themselves (or unless they audit it against
+            //      the other generations, which is annoying).
+            for (const file of existingDiskEntries) {
+                await this.rawStorage.remove(file);
+            }
+        } finally {
+            this.compressing = false;
         }
     }