sliftutils 0.34.0 → 0.36.0

package/builders/webBuild.ts

@@ -38,8 +38,8 @@ async function main() {
     // Collect all files to copy
     let filesToCopy: string[] = [];
 
+    filesToCopy.push(yargObj.indexHtml);
     if (hasIndexHtml) {
-        filesToCopy.push(yargObj.indexHtml);
     }
 
     // Add assets folder files if it exists
@@ -59,6 +59,8 @@ async function main() {
         }
     }
 
+    let indexHtmlOutput = "";
+
     let filesCopied = 0;
     let root = path.resolve(".");
     for (const file of filesToCopy) {
@@ -70,6 +72,10 @@ async function main() {
         let relativePath = path.relative(root, sourcePath);
         let destPath = path.join(yargObj.outputFolder, relativePath);
 
+        if (file === yargObj.indexHtml) {
+            indexHtmlOutput = destPath;
+        }
+
         let sourceTimestamp = await getTimestamp(sourcePath);
         let destTimestamp = await getTimestamp(destPath);
         if (sourceTimestamp > destTimestamp) {
@@ -84,6 +90,7 @@ async function main() {
 
     let duration = Date.now() - time;
     console.log(`Web build completed in ${formatTime(duration)}`);
+    console.log(`file://${path.resolve(indexHtmlOutput).replaceAll("\\", "/")}`);
 }
 main().catch(console.error).finally(() => process.exit());
 

package/index.d.ts

@@ -803,13 +803,13 @@ declare module "sliftutils/storage/TransactionStorage" {
         private writeDelay;
         cache: Map<string, TransactionEntry>;
         private currentChunk;
-        private currentChunkSize;
         private entryCount;
         private static allStorage;
         constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
         static compressAll(): Promise<void>;
         private init;
-        private getChunk;
+        private getCurrentChunk;
+        private onAddToChunk;
         get(key: string): Promise<Buffer | undefined>;
         set(key: string, value: Buffer): Promise<void>;
         remove(key: string): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.34.0",
+    "version": "0.36.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/spec.txt

@@ -1,3 +1,6 @@
+TODO: Take the half finished apiServer code in vidfrontend, finish the todos, and move it here.
+    - Having an alternate version, where we run a server that does watching is nice, it's heavier and does require setting up certificates. but it makes hot reloading faster. And... It can be used to run an actual site that's public.  
+
 ALSO, go fix our typings in socket-function as well
 
 TODO:

package/storage/DiskCollection.ts

@@ -1,7 +1,7 @@
 import { isNode } from "typesafecss";
 import { DelayedStorage } from "./DelayedStorage";
 import { FileStorage, getFileStorage, getFileStorageNested } from "./FileFolderAPI";
-import { IStorage, IStorageSync } from "./IStorage";
+import { IStorage, IStorageRaw, IStorageSync } from "./IStorage";
 import { JSONStorage } from "./JSONStorage";
 import { StorageSync } from "./StorageObservable";
 import { TransactionStorage } from "./TransactionStorage";
@@ -90,8 +90,14 @@ export class DiskCollectionBrowser<T> implements IStorageSync<T> {
     }
     public transactionStorage: TransactionStorage | undefined;
     async initStorage(): Promise<IStorage<T>> {
-        if (isNode()) return undefined as any;
-        let curCollection = await new PrivateFileSystemStorage(`collections/${this.collectionName}`);
+        let curCollection: IStorageRaw | undefined;
+        if (isNode()) {
+            let fileStorage = await getFileStorage();
+            let collections = await fileStorage.folder.getStorage("collections");
+            curCollection = await collections.folder.getStorage(this.collectionName);
+        } else {
+            curCollection = await new PrivateFileSystemStorage(`collections/${this.collectionName}`);
+        }
         let baseStorage = new TransactionStorage(curCollection, this.collectionName);
         return new JSONStorage<T>(baseStorage);
     }

package/storage/TransactionStorage.d.ts

@@ -13,13 +13,13 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     private writeDelay;
     cache: Map<string, TransactionEntry>;
     private currentChunk;
-    private currentChunkSize;
     private entryCount;
     private static allStorage;
     constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
     static compressAll(): Promise<void>;
     private init;
-    private getChunk;
+    private getCurrentChunk;
+    private onAddToChunk;
     get(key: string): Promise<Buffer | undefined>;
     set(key: string, value: Buffer): Promise<void>;
     remove(key: string): Promise<void>;

package/storage/TransactionStorage.ts

@@ -1,4 +1,4 @@
-import { PromiseObj, sort, throttleFunction } from "socket-function/src/misc";
+import { PromiseObj, compareArray, sort, throttleFunction, timeInMinute } from "socket-function/src/misc";
 import { IStorage, IStorageRaw } from "./IStorage";
 import { Zip } from "../misc/zip";
 import { runInSerial } from "socket-function/src/batching";
@@ -40,6 +40,7 @@ IMPORTANT! If there are multiple writers, we clobber writes from other writers w
 
 
 const FILE_CHUNK_SIZE = 1024 * 1024;
+const FILE_MAX_LIFETIME = timeInMinute * 30;
 
 const FILE_ZIP_THRESHOLD = 16 * 1024 * 1024;
 
@@ -66,11 +67,26 @@ const fileLockSection = runInSerial(async (fnc: () => Promise<void>) => {
 
 const CHUNK_EXT = ".chunk";
 const ourId = Date.now() + Math.random();
+let seqNum = 0;
+
+function getNextChunkPath(): string {
+    return `${Date.now()}_${seqNum++}_${ourId}.chunk`;
+}
+function sortChunks(chunks: string[]): string[] {
+    function getChunkParts(chunk: string): unknown[] {
+        const parts = chunk.split("_");
+        return parts.map(part => +part);
+    }
+    return chunks.sort((a, b) => compareArray(getChunkParts(a), getChunkParts(b)));
+}
 
 export class TransactionStorage implements IStorage<Buffer> {
     public cache: Map<string, TransactionEntry> = new Map();
-    private currentChunk = 0;
-    private currentChunkSize = 0;
+    private currentChunk: {
+        path: string;
+        size: number;
+        timeCreated: number;
+    } | undefined = undefined;
     private entryCount = 0;
 
     private static allStorage: TransactionStorage[] = [];
@@ -91,7 +107,26 @@ export class TransactionStorage implements IStorage<Buffer> {
 
     private init: Promise<void> | undefined = this.loadAllTransactions();
 
-    private getChunk(chunk: number) { return `${chunk}_${ourId}${CHUNK_EXT}`; }
+    private getCurrentChunk(): string {
+        if (this.currentChunk && this.currentChunk.timeCreated < Date.now() - FILE_MAX_LIFETIME) {
+            this.currentChunk = undefined;
+        }
+        if (!this.currentChunk) {
+            this.currentChunk = {
+                path: getNextChunkPath(),
+                size: 0,
+                timeCreated: Date.now()
+            };
+        }
+        return this.currentChunk.path;
+    }
+    private onAddToChunk(size: number): void {
+        if (!this.currentChunk) return;
+        this.currentChunk.size += size;
+        if (this.currentChunk.size >= FILE_CHUNK_SIZE) {
+            this.currentChunk = undefined;
+        }
+    }
 
     public async get(key: string): Promise<Buffer | undefined> {
         await this.init;
@@ -173,18 +208,14 @@ export class TransactionStorage implements IStorage<Buffer> {
 
                 let newChunks = this.chunkBuffers(buffers);
                 for (let chunk of newChunks) {
-                    let file = this.getChunk(this.currentChunk);
+                    let file = this.getCurrentChunk();
                     if (!await this.rawStorage.get(file)) {
                         let { header, headerBuffer } = this.getHeader(false);
                         await this.rawStorage.set(file, headerBuffer);
                     }
                     let content = chunk.buffer;
                     await this.rawStorage.append(file, content);
-                    this.currentChunkSize += content.length;
-                    if (this.currentChunkSize >= FILE_CHUNK_SIZE) {
-                        this.currentChunk++;
-                        this.currentChunkSize = 0;
-                    }
+                    this.onAddToChunk(content.length);
                 }
 
                 await this.compressTransactionLog();
@@ -219,21 +250,16 @@ export class TransactionStorage implements IStorage<Buffer> {
         const keys = await this.rawStorage.getKeys();
         const transactionFiles = keys.filter(key => key.endsWith(CHUNK_EXT));
 
-        sort(transactionFiles, x => parseInt(x));
+        sortChunks(transactionFiles);
 
         let size = 0;
         for (const file of transactionFiles) {
-            let chunk = parseInt(file);
             let curSize = await this.loadTransactionFile(file);
-            if (chunk >= this.currentChunk) {
-                this.currentChunk = chunk;
-                this.currentChunkSize = curSize;
-            }
             size += curSize;
         }
         time = Date.now() - time;
         if (time > 50) {
-            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, ${formatNumber(size)}B`);
+            console.log(`Loaded ${this.debugName} in ${formatTime(time)}, ${formatNumber(this.cache.size)} keys, from ${formatNumber(transactionFiles.length)} files, ${formatNumber(size)}B`, transactionFiles);
         }
 
         this.init = undefined;
@@ -424,14 +450,13 @@ export class TransactionStorage implements IStorage<Buffer> {
         // Load off disk, in case there are other writes. We still race with them, but at least
         //  this reduces the race condition considerably
 
-        sort(existingDiskEntries, x => parseInt(x));
+        sortChunks(existingDiskEntries);
         for (let entry of existingDiskEntries) {
             await this.loadTransactionFile(entry);
         }
 
         this.entryCount = this.cache.size;
 
-        let nextStart = Math.max(...existingDiskEntries.map(x => parseInt(x))) + 1;
 
         let buffers: Buffer[] = [];
         for (const entry of this.cache.values()) {
@@ -441,9 +466,9 @@ export class TransactionStorage implements IStorage<Buffer> {
 
         let newChunks = this.chunkBuffers(buffers);
 
-        let curChunk = nextStart;
         for (let chunk of newChunks) {
-            let file = this.getChunk(curChunk++);
+            let file = this.getCurrentChunk();
+            this.currentChunk = undefined;
             let content = chunk.buffer;
             let { header, headerBuffer } = this.getHeader(
                 // AND, never compress the last one, otherwise we can't append to it!
@@ -462,8 +487,6 @@ export class TransactionStorage implements IStorage<Buffer> {
         for (const file of existingDiskEntries) {
             await this.rawStorage.remove(file);
         }
-
-        this.currentChunk = curChunk;
     }
 
     public async reset() {
@@ -477,8 +500,7 @@ export class TransactionStorage implements IStorage<Buffer> {
 
             this.pendingAppends = [];
             this.cache.clear();
-            this.currentChunk = 0;
-            this.currentChunkSize = 0;
+            this.currentChunk = undefined;
             this.entryCount = 0;
         });
     }