sliftutils 0.56.0 → 0.57.0

package/index.d.ts

@@ -950,7 +950,8 @@ declare module "sliftutils/storage/TransactionStorage" {
         private updatePendingAppends;
         getKeys(): Promise<string[]>;
         private loadAllTransactions;
-        private loadTransactionFile;
+        private parseTransactionFile;
+        private applyTransactionEntries;
         private readTransactionEntry;
         private serializeTransactionEntry;
         private getHeader;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.56.0",
+    "version": "0.57.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/storage/TransactionStorage.d.ts

@@ -34,7 +34,8 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     private updatePendingAppends;
     getKeys(): Promise<string[]>;
     private loadAllTransactions;
-    private loadTransactionFile;
+    private parseTransactionFile;
+    private applyTransactionEntries;
     private readTransactionEntry;
     private serializeTransactionEntry;
     private getHeader;

package/storage/TransactionStorage.ts

@@ -73,13 +73,6 @@ let seqNum = 0;
 function getNextChunkPath(): string {
     return `${Date.now()}_${seqNum++}_${ourId}.chunk`;
 }
-function sortChunks(chunks: string[]): string[] {
-    function getChunkParts(chunk: string): unknown[] {
-        const parts = chunk.split("_");
-        return parts.map(part => +part);
-    }
-    return chunks.sort((a, b) => compareArray(getChunkParts(a), getChunkParts(b)));
-}
 
 export class TransactionStorage implements IStorage<Buffer> {
     public cache: Map<string, TransactionEntry> = new Map();
@@ -108,7 +101,7 @@ export class TransactionStorage implements IStorage<Buffer> {
         });
     }
 
-    private init: Promise<void> | undefined = this.loadAllTransactions();
+    private init: Promise<unknown> | undefined = this.loadAllTransactions();
 
     private getCurrentChunk(): string {
         if (this.currentChunk && this.currentChunk.timeCreated < Date.now() - FILE_MAX_LIFETIME) {
@@ -247,33 +240,37 @@ export class TransactionStorage implements IStorage<Buffer> {
     }
 
 
-    private async loadAllTransactions(): Promise<void> {
-        if (isInBuild()) return;
+    // NOTE: This is either called in init (which blocks all other calls), or inside of the global file lock, so it is safe to load.
+    private async loadAllTransactions(): Promise<string[]> {
+        if (isInBuild()) return [];
+
         let time = Date.now();
         const keys = await this.rawStorage.getKeys();
         const transactionFiles = keys.filter(key => key.endsWith(CHUNK_EXT));
 
-        sortChunks(transactionFiles);
-
-        let size = 0;
-        for (const file of transactionFiles) {
-            let curSize = await this.loadTransactionFile(file);
-            size += curSize;
+        let entryList: TransactionEntry[][] = [];
+        for (let file of transactionFiles) {
+            entryList.push(await this.parseTransactionFile(file));
         }
+        let entries = entryList.flat();
+        this.applyTransactionEntries(entries);
+
         time = Date.now() - time;
         if (time > 50) {
-            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, ${formatNumber(size)}B`, transactionFiles);
+            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, entries ${formatNumber(entries.length)}B`, transactionFiles);
         }
 
         this.init = undefined;
+        return transactionFiles;
     }
 
-    private async loadTransactionFile(filename: string): Promise<number> {
+    // ONLY call this inside of loadAllTransactions
+    private async parseTransactionFile(filename: string): Promise<TransactionEntry[]> {
         const fullFile = await this.rawStorage.get(filename);
-        if (!fullFile) return 0;
+        if (!fullFile) return [];
         if (fullFile.length < 4) {
             //console.error(`Transaction in ${this.debugName} file ${filename} is too small, skipping`);
-            return 0;
+            return [];
         }
         let headerSize = fullFile.readUInt32LE(0);
         let headerBuffer = fullFile.slice(4, 4 + headerSize);
@@ -282,22 +279,13 @@ export class TransactionStorage implements IStorage<Buffer> {
             header = JSON.parse(headerBuffer.toString());
         } catch (e) {
             console.error(`Failed to parse header of transaction file in ${this.debugName}, ${filename}`);
-            return 0;
+            return [];
         }
         let content = fullFile.slice(4 + headerSize);
         if (header.zipped) {
             content = await Zip.gunzip(content);
         }
 
-        let pendingWriteTimes = new Map<string, number>();
-        for (const entry of this.pendingAppends) {
-            let prevTime = pendingWriteTimes.get(entry.key);
-            if (prevTime && prevTime > entry.time) {
-                continue;
-            }
-            pendingWriteTimes.set(entry.key, entry.time);
-        }
-
         let offset = 0;
         let entries: TransactionEntry[] = [];
         while (offset < content.length) {
@@ -330,7 +318,21 @@ export class TransactionStorage implements IStorage<Buffer> {
             let { entry } = entryObj;
             offset = entryObj.offset;
             entries.push(entry);
+        }
+        return entries;
+    }
+    private applyTransactionEntries(entries: TransactionEntry[]): void {
+        let pendingWriteTimes = new Map<string, number>();
+        for (const entry of this.pendingAppends) {
+            let prevTime = pendingWriteTimes.get(entry.key);
+            if (prevTime && prevTime > entry.time) {
+                continue;
+            }
+            pendingWriteTimes.set(entry.key, entry.time);
+        }
 
+        sort(entries, x => x.time);
+        for (let entry of entries) {
             let time = entry.time;
             let prevTime = pendingWriteTimes.get(entry.key);
             if (prevTime && prevTime > time) {
@@ -347,7 +349,6 @@ export class TransactionStorage implements IStorage<Buffer> {
                 this.cache.set(entry.key, entry);
             }
         }
-        return fullFile.length;
     }
 
     private readTransactionEntry(buffer: Buffer, offset: number): {
@@ -469,11 +470,7 @@ export class TransactionStorage implements IStorage<Buffer> {
 
             // Load off disk, in case there are other writes. We still race with them, but at least
             //  this reduces the race condition considerably
-
-            sortChunks(existingDiskEntries);
-            for (let entry of existingDiskEntries) {
-                await this.loadTransactionFile(entry);
-            }
+            existingDiskEntries = await this.loadAllTransactions();
 
             this.entryCount = this.cache.size;