sliftutils 0.58.0 → 0.60.0

package/index.d.ts

@@ -140,6 +140,20 @@ declare module "sliftutils/misc/zip" {
 
 }
 
+declare module "sliftutils/render-utils/Anchor" {
+    import preact from "preact";
+    import { URLParam } from "./URLParam";
+    export declare class Anchor extends preact.Component<{
+        className?: string;
+        params: ([URLParam, unknown] | [string, string])[];
+        button?: boolean;
+    } & Omit<preact.JSX.HTMLAttributes<HTMLAnchorElement>, "href">> {
+        render(): preact.JSX.Element;
+    }
+    export declare function createLinkRaw(params: ([URLParam, unknown])[]): string;
+
+}
+
 declare module "sliftutils/render-utils/DropdownCustom" {
     import preact from "preact";
     import { LengthOrPercentage } from "typesafecss/cssTypes";
@@ -772,6 +786,7 @@ declare module "sliftutils/storage/IStorage" {
             lastModified: number;
         }>;
         reset(): Promise<void>;
+        watchResync?: (callback: () => void) => void;
     };
     export type IStorageRaw = {
         get(key: string): Promise<Buffer | undefined>;
@@ -890,6 +905,9 @@ declare module "sliftutils/storage/StorageObservable" {
         synced: {
             keySeqNum: number;
         };
+        resynced: {
+            seqNum: number;
+        };
         constructor(storage: IStorage<T>);
         get(key: string): T | undefined;
         set(key: string, value: T): void;
@@ -928,11 +946,14 @@ declare module "sliftutils/storage/TransactionStorage" {
         private debugName;
         private writeDelay;
         cache: Map<string, TransactionEntry>;
+        private diskFiles;
         private currentChunk;
         private entryCount;
         private static allStorage;
         constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
         static compressAll(): Promise<void>;
+        private resyncCallbacks;
+        watchResync(callback: () => void): void;
         private init;
         private getCurrentChunk;
         private onAddToChunk;
@@ -949,6 +970,7 @@ declare module "sliftutils/storage/TransactionStorage" {
         pushAppend(entry: TransactionEntry): Promise<void>;
         private updatePendingAppends;
         getKeys(): Promise<string[]>;
+        checkDisk(): Promise<void>;
         private loadAllTransactions;
         private parseTransactionFile;
         private applyTransactionEntries;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "0.58.0",
+    "version": "0.60.0",
     "main": "index.js",
     "license": "MIT",
     "files": [

package/render-utils/Anchor.d.ts

@@ -0,0 +1,10 @@
+import preact from "preact";
+import { URLParam } from "./URLParam";
+export declare class Anchor extends preact.Component<{
+    className?: string;
+    params: ([URLParam, unknown] | [string, string])[];
+    button?: boolean;
+} & Omit<preact.JSX.HTMLAttributes<HTMLAnchorElement>, "href">> {
+    render(): preact.JSX.Element;
+}
+export declare function createLinkRaw(params: ([URLParam, unknown])[]): string;

package/render-utils/Anchor.tsx

@@ -1,10 +1,6 @@
 import preact from "preact";
 import { URLParam, batchURLParamUpdate, getResolvedParam } from "./URLParam";
-import { css, isNode } from "typesafecss";
-
-export const AnchorClass = (
-    css.textDecoration("none").color("hsl(210, 75%, 65%)").opacity(0.8, "hover")
-);
+import { CSSType, css, isNode } from "typesafecss";
 
 export class Anchor extends preact.Component<{
     className?: string;

package/render-utils/colors.tsx

@@ -11,4 +11,9 @@ export const errorMessage = css.hsl(0, 75, 50).color("white")
 export const warnMessage = css.hsl(50, 75, 50).color("hsl(0, 0%, 7%)", "important", "soft")
     .padding("4px 6px", "soft")
     .whiteSpace("pre-wrap").display("inline-block", "soft")
-    ;
+    ;
+
+
+export const AnchorClass = (
+    css.textDecoration("none").color("hsl(210, 75%, 65%)").opacity(0.8, "hover")
+);

package/storage/IStorage.d.ts

@@ -23,6 +23,7 @@ export type IStorage<T> = {
         lastModified: number;
     }>;
     reset(): Promise<void>;
+    watchResync?: (callback: () => void) => void;
 };
 export type IStorageRaw = {
     get(key: string): Promise<Buffer | undefined>;

package/storage/IStorage.ts

@@ -22,6 +22,8 @@ export type IStorage<T> = {
         lastModified: number;
     }>;
     reset(): Promise<void>;
+    // Allows watching for when the storage detects and underlying changes, and resyncs all of it's data (which might 
+    watchResync?: (callback: () => void) => void;
 };
 // NOTE: In the file system some characters are disallowed, and some characters do special things
 //  (/ makes a folder). And there are even more rules, such as lengths per folder, etc, etc.

package/storage/StorageObservable.d.ts

@@ -10,6 +10,9 @@ export declare class StorageSync<T> implements IStorageSync<T> {
     synced: {
         keySeqNum: number;
     };
+    resynced: {
+        seqNum: number;
+    };
     constructor(storage: IStorage<T>);
     get(key: string): T | undefined;
     set(key: string, value: T): void;

package/storage/StorageObservable.ts

@@ -11,9 +11,16 @@ export class StorageSync<T> implements IStorageSync<T> {
         keySeqNum: 0,
     }, undefined, { deep: false });
 
-    constructor(public storage: IStorage<T>) { }
+    resynced = observable({ seqNum: 0 });
+
+    constructor(public storage: IStorage<T>) {
+        storage.watchResync?.(() => {
+            this.resynced.seqNum++;
+        });
+    }
 
     public get(key: string): T | undefined {
+        this.resynced.seqNum;
         if (!this.cached.has(key)) {
             this.cached.set(key, undefined);
             void this.getPromise(key);
@@ -41,12 +48,14 @@ export class StorageSync<T> implements IStorageSync<T> {
     }
     private loadedKeys = false;
     public getKeys(): string[] {
+        this.resynced.seqNum;
         void this.getKeysPromise();
         this.synced.keySeqNum;
         return Array.from(this.keys);
     }
 
     public getInfo(key: string): { size: number; lastModified: number } | undefined {
+        this.resynced.seqNum;
         if (!this.infoCached.has(key)) {
             this.infoCached.set(key, { size: 0, lastModified: 0 });
             void this.storage.getInfo(key).then(info => {

package/storage/TransactionStorage.d.ts

@@ -12,11 +12,14 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     private debugName;
     private writeDelay;
     cache: Map<string, TransactionEntry>;
+    private diskFiles;
     private currentChunk;
     private entryCount;
     private static allStorage;
     constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
     static compressAll(): Promise<void>;
+    private resyncCallbacks;
+    watchResync(callback: () => void): void;
     private init;
     private getCurrentChunk;
     private onAddToChunk;
@@ -33,6 +36,7 @@ export declare class TransactionStorage implements IStorage<Buffer> {
     pushAppend(entry: TransactionEntry): Promise<void>;
     private updatePendingAppends;
     getKeys(): Promise<string[]>;
+    checkDisk(): Promise<void>;
     private loadAllTransactions;
     private parseTransactionFile;
     private applyTransactionEntries;

package/storage/TransactionStorage.ts

@@ -6,6 +6,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 import { setPending } from "./PendingManager";
 import { isInBuild } from "../misc/environment";
 import { isNode } from "typesafecss";
+import { runInfinitePoll } from "socket-function/src/batching";
 
 /*
 // Spec:
@@ -39,6 +40,7 @@ UPDATE now we use chunks, because append is too slow.
 IMPORTANT! If there are multiple writers, we clobber writes from other writers when we compress
 */
 
+const DISK_CHECK_INTERVAL = timeInMinute * 5;
 
 const FILE_CHUNK_SIZE = 1024 * 1024;
 const FILE_MAX_LIFETIME = timeInMinute * 30;
@@ -76,6 +78,7 @@ function getNextChunkPath(): string {
 
 export class TransactionStorage implements IStorage<Buffer> {
     public cache: Map<string, TransactionEntry> = new Map();
+    private diskFiles: Set<string> = new Set();
     private currentChunk: {
         path: string;
         size: number;
@@ -91,6 +94,8 @@ export class TransactionStorage implements IStorage<Buffer> {
         private writeDelay = WRITE_DELAY
     ) {
         TransactionStorage.allStorage.push(this);
+        // VERY useful for debugging.
+        (globalThis as any)[`transactionStorage-${this.debugName}`] = this;
     }
     // Helps get rid of parse errors which constantly log. Also, uses less space
     public static async compressAll() {
@@ -101,6 +106,11 @@ export class TransactionStorage implements IStorage<Buffer> {
         });
     }
 
+    private resyncCallbacks: (() => void)[] = [];
+    public watchResync(callback: () => void): void {
+        this.resyncCallbacks.push(callback);
+    }
+
     private init: Promise<unknown> | undefined = this.loadAllTransactions();
 
     private getCurrentChunk(): string {
@@ -113,6 +123,7 @@ export class TransactionStorage implements IStorage<Buffer> {
                 size: 0,
                 timeCreated: Date.now()
             };
+            this.diskFiles.add(this.currentChunk.path);
         }
         return this.currentChunk.path;
     }
@@ -239,14 +250,35 @@ export class TransactionStorage implements IStorage<Buffer> {
         return Array.from(this.cache.keys());
     }
 
+    public async checkDisk(): Promise<void> {
+        if (this.init) await this.init;
+        const anyChanges = async () => {
+            let keys = await this.rawStorage.getKeys();
+            let diskFiles = keys.filter(key => key.endsWith(CHUNK_EXT));
+            let hasNew = diskFiles.some(file => !this.diskFiles.has(file));
+            return hasNew;
+        };
+        if (!await anyChanges()) return;
+
+        await fileLockSection(async () => {
+            if (!await anyChanges()) return;
+            await this.loadAllTransactions();
+        });
+    }
+
 
     // NOTE: This is either called in init (which blocks all other calls), or inside of the global file lock, so it is safe to load.
     private async loadAllTransactions(): Promise<string[]> {
         if (isInBuild()) return [];
 
+        if (this.init) {
+            runInfinitePoll(DISK_CHECK_INTERVAL, () => this.checkDisk());
+        }
+
         let time = Date.now();
         const keys = await this.rawStorage.getKeys();
         const transactionFiles = keys.filter(key => key.endsWith(CHUNK_EXT));
+        this.diskFiles = new Set(transactionFiles);
 
         let entryList: TransactionEntry[][] = [];
         for (let file of transactionFiles) {
@@ -330,25 +362,62 @@ export class TransactionStorage implements IStorage<Buffer> {
             }
             pendingWriteTimes.set(entry.key, entry.time);
         }
-
-        sort(entries, x => x.time);
-        for (let entry of entries) {
-            let time = entry.time;
-            let prevTime = pendingWriteTimes.get(entry.key);
-            if (prevTime && prevTime > time) {
+        let latest = new Map<string, TransactionEntry>();
+        for (const entry of entries) {
+            let pendingTime = pendingWriteTimes.get(entry.key);
+            if (pendingTime && pendingTime > entry.time) {
+                continue;
+            }
+            let prev = latest.get(entry.key);
+            if (prev && prev.time > entry.time) {
                 continue;
             }
+            latest.set(entry.key, entry);
+        }
 
+        let anyChanged = false;
+        for (let entry of latest.values()) {
             if (entry.value === undefined) {
+                if (!this.cache.has(entry.key)) continue;
+                anyChanged = true;
                 this.cache.delete(entry.key);
             } else {
-                let prev = this.cache.get(entry.key);
-                if (prev && (prev.time > entry.time)) {
-                    continue;
+                if (!anyChanged) {
+                    let prev = this.cache.get(entry.key);
+                    if (!prev || prev.isZipped !== entry.isZipped) {
+                        anyChanged = true;
+                    } else {
+                        if (!prev.value) {
+                            if (entry.value) {
+                                anyChanged = true;
+                            }
+                        } else {
+                            if (!entry.value) {
+                                anyChanged = true;
+                            } else {
+                                // Both values, so... it might not have changed
+                                if (!prev.value.equals(entry.value)) {
+                                    anyChanged = true;
+                                }
+                            }
+                        }
+                    }
                 }
                 this.cache.set(entry.key, entry);
             }
         }
+
+        if (anyChanged) {
+            for (const callback of this.resyncCallbacks) {
+                try {
+                    callback();
+                } catch (e) {
+                    setImmediate(() => {
+                        throw e;
+                    });
+                }
+            }
+        }
     }
 
     private readTransactionEntry(buffer: Buffer, offset: number): {
@@ -511,6 +580,7 @@ export class TransactionStorage implements IStorage<Buffer> {
             //      the other generations, which is annoying).
             for (const file of existingDiskEntries) {
                 await this.rawStorage.remove(file);
+                this.diskFiles.delete(file);
             }
         } finally {
             this.compressing = false;
@@ -528,6 +598,7 @@ export class TransactionStorage implements IStorage<Buffer> {
 
             this.pendingAppends = [];
             this.cache.clear();
+            this.diskFiles.clear();
             this.currentChunk = undefined;
             this.entryCount = 0;
         });

package/tsconfig.declarations.json

@@ -3,8 +3,8 @@
     "compilerOptions": {
         "declaration": true,
         "emitDeclarationOnly": true,
-        "outDir": "./render-utils",
-        "rootDir": "./render-utils"
+        "outDir": "./",
+        "rootDir": "./"
     },
     "include": [
         "render-utils/**/*.ts",