sliftutils 1.4.2 → 1.4.4

package/bin/filehoster.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+"use strict";
+require("typenode");
+require("../storage/remoteFileServer").runFileHoster();

package/index.d.ts

@@ -849,8 +849,29 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
         getColumnInfo(): Promise<BulkColumnInfo[]>;
         /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
         getReaderInfo(): Promise<BulkReaderInfo>;
+        /**
+         * Per-file breakdown of the on-disk files, read fresh from disk each call (latest sizes, including
+         * stream files still being appended). `bytes` is the actual on-disk size. Good for showing collection
+         * size/fragmentation and deciding whether to call tryMergeNow()/compact().
+         */
+        getFileInfo(): Promise<{
+            files: {
+                name: string;
+                type: "bulk" | "stream";
+                bytes: number;
+            }[];
+            count: number;
+            totalBytes: number;
+        }>;
         /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
         compact(): Promise<void>;
+        /**
+         * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+         * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+         * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+         * also run automatically on tab hide/close and before every merge.
+         */
+        flush(): Promise<void>;
         /**
          * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
          * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
@@ -878,8 +899,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
     export declare const bulkDatabase2Timing: {
         streamSealAgeMs: number;
         mergeCheckIntervalMs: number;
+        mergeSpacingMs: number;
         firstMergeTriggerFiles: number;
         firstMergeTriggerRangeMs: number;
+        writeFlushMaxDelayMs: number;
     };
     export interface ReactiveDeps {
         observe(signal: string): void;
@@ -895,6 +918,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         protected deps: ReactiveDeps;
         private storageFactory;
         constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+        private pendingAppends;
+        private flushTimer;
+        private flushChain;
+        private currentFlushDelay;
+        private lastWriteTime;
         static clearCache(): void;
         storage: {
             (): Promise<FileStorage>;
@@ -919,6 +947,10 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         writeBatch(entries: T[]): Promise<void>;
         delete(key: string): Promise<void>;
         deleteBatch(keys: string[]): Promise<void>;
+        private streamAppend;
+        flush(): Promise<void>;
+        private flushPending;
+        private doFlush;
         update(entry: Partial<T> & {
             key: string;
         }): Promise<void>;
@@ -944,7 +976,9 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
         private resolveReaders;
         private mergeFileSet;
         private canDeleteStream;
+        private mergeSpacingDelay;
         private testMerge;
+        private findDuplicateGroups;
         private formatInfo;
         private patchColumn;
         getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
@@ -990,6 +1024,15 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
                 byteSize: number;
             }[];
         }>;
+        getFileInfo(): Promise<{
+            files: {
+                name: string;
+                type: "bulk" | "stream";
+                bytes: number;
+            }[];
+            count: number;
+            totalBytes: number;
+        }>;
     }
 
 }
@@ -1797,6 +1840,68 @@ declare module "sliftutils/storage/fileSystemPointer" {
 
 }
 
+declare module "sliftutils/storage/remoteFileServer" {
+    export declare function generatePassword(wordCount: number): string;
+    export type RemoteFileServerOptions = {
+        root: string;
+        port?: number;
+        host?: string;
+        password?: string;
+        logAccess?: boolean;
+    };
+    export type RemoteFileServerHandle = {
+        port: number;
+        password: string;
+        url: string;
+        close: () => Promise<void>;
+    };
+    export declare function startRemoteFileServer(options: RemoteFileServerOptions): Promise<RemoteFileServerHandle>;
+    export declare function runFileHoster(): Promise<void>;
+
+}
+
+declare module "sliftutils/storage/remoteFileStorage" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import https from "https";
+    import type { FileStorage } from "./FileFolderAPI";
+    export type RemoteFileStorageOptions = {
+        chunkBytes?: number;
+        cacheBytes?: number;
+        latencyMs?: number;
+    };
+    type Connection = {
+        url: string;
+        password: string;
+        latencyMs: number;
+        agent: https.Agent | undefined;
+        cache: RangeCache;
+        stats: {
+            requestCount: number;
+            bytesFetched: number;
+        };
+    };
+    declare class RangeCache {
+        private chunkBytes;
+        private budget;
+        private chunks;
+        private bytes;
+        constructor(chunkBytes: number, budget: number);
+        private key;
+        private peek;
+        private store;
+        invalidate(path: string): void;
+        getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined>;
+    }
+    export type RemoteStorageFactory = ((path: string) => Promise<FileStorage>) & {
+        stats: Connection["stats"];
+    };
+    export declare function getRemoteFileStorage(url: string, password: string, options?: RemoteFileStorageOptions): RemoteStorageFactory;
+    export {};
+
+}
+
 declare module "sliftutils/storage/storage" {
 
 

package/package.json

@@ -1,70 +1,72 @@
-{
-    "name": "sliftutils",
-    "version": "1.4.2",
-    "main": "index.js",
-    "license": "MIT",
-    "files": [
-        "**/*",
-        ".gitignore"
-    ],
-    "scripts": {
-        "type": "yarn tsc --noEmit",
-        "emit-dts": "yarn tsc --project tsconfig.declarations.json || true",
-        "generate-index-dts": "typenode ./builders/generateIndexDts.ts",
-        "update-types": "yarn emit-dts && yarn generate-index-dts",
-        "prepublishOnly": "yarn update-types",
-        "run-nodejs": "node ./build-nodejs/server.js",
-        "run-nodejs-dev": "typenode ./nodejs/server.ts --npminstall",
-        "run-web": "node ./builders/webRun.js",
-        "run-electron": "node ./node_modules/electron/cli.js ./build-electron/electronMain.js",
-        "build-nodejs": "node ./builders/nodeJSBuildRun.js",
-        "build-web": "node ./builders/webBuildRun.js",
-        "build-extension": "node ./builders/extensionBuildRun.js",
-        "build-electron": "node ./builders/electronBuildRun.js",
-        "watch-nodejs": "node ./builders/watchRun.js --port 9876 \"nodejs/*.ts\" \"nodejs/*.tsx\" \"yarn build-nodejs\"",
-        "watch-web": "node ./builders/watchRun.js --port 9877 \"web/*.ts\" \"web/*.tsx\" \"yarn build-web\"",
-        "watch-extension": "node ./builders/watchRun.js --port 9878 \"extension/*.ts\" \"extension/*.tsx\" \"yarn build-extension\"",
-        "watch-electron": "node ./builders/watchRun.js --port 9879 \"electron/*.ts\" \"electron/*.tsx\" \"yarn build-electron\"",
-        "notes": "mobx, preact, socket-function, typenode SHOULD be peerDependencies. But we want to use yarn (better dependency deduplication), so we can't use peerDependencies (as they aren't installed by default, which makes them a nightmare to use). If you want to override the versions, feel free to use overrides/resolutions.",
-        "test": "typenode ./test.ts"
-    },
-    "bin": {
-        "build-nodejs": "./builders/nodeJSBuildRun.js",
-        "buildnodejs": "./builders/nodeJSBuildRun.js",
-        "build-extension": "./builders/extensionBuildRun.js",
-        "buildextension": "./builders/extensionBuildRun.js",
-        "build-web": "./builders/webBuildRun.js",
-        "buildweb": "./builders/webBuildRun.js",
-        "build-electron": "./builders/electronBuildRun.js",
-        "buildelectron": "./builders/electronBuildRun.js",
-        "slift-watch": "./builders/watchRun.js",
-        "sliftwatch": "./builders/watchRun.js",
-        "slift-setup": "./builders/setupRun.js",
-        "sliftsetup": "./builders/setupRun.js"
-    },
-    "dependencies": {
-        "@types/chrome": "^0.0.237",
-        "@types/node-forge": "^1.3.11",
-        "@types/shell-quote": "^1.7.5",
-        "acme-client": "^5.0.0",
-        "js-sha256": "^0.11.1",
-        "mobx": "^6.13.3",
-        "preact-old-types": "^10.28.1",
-        "shell-quote": "^1.8.3",
-        "socket-function": "^1.1.42",
-        "typenode": "*",
-        "typesafecss": "*",
-        "ws": "^8.18.3",
-        "yargs": "15.4.1"
-    },
-    "resolutions": {
-        "preact": "npm:preact-old-types@*"
-    },
-    "devDependencies": {
-        "@types/yargs": "15.0.19",
-        "debugbreak": "^0.9.9",
-        "electron": "^33.2.1",
-        "open": "^8.4.0",
-        "typedev": "^0.1.1"
-    }
-}
+{
+    "name": "sliftutils",
+    "version": "1.4.4",
+    "main": "index.js",
+    "license": "MIT",
+    "files": [
+        "**/*",
+        ".gitignore"
+    ],
+    "scripts": {
+        "type": "yarn tsc --noEmit",
+        "emit-dts": "yarn tsc --project tsconfig.declarations.json || true",
+        "generate-index-dts": "typenode ./builders/generateIndexDts.ts",
+        "update-types": "yarn emit-dts && yarn generate-index-dts",
+        "prepublishOnly": "yarn update-types",
+        "run-nodejs": "node ./build-nodejs/server.js",
+        "run-nodejs-dev": "typenode ./nodejs/server.ts --npminstall",
+        "run-web": "node ./builders/webRun.js",
+        "run-electron": "node ./node_modules/electron/cli.js ./build-electron/electronMain.js",
+        "build-nodejs": "node ./builders/nodeJSBuildRun.js",
+        "build-web": "node ./builders/webBuildRun.js",
+        "build-extension": "node ./builders/extensionBuildRun.js",
+        "build-electron": "node ./builders/electronBuildRun.js",
+        "watch-nodejs": "node ./builders/watchRun.js --port 9876 \"nodejs/*.ts\" \"nodejs/*.tsx\" \"yarn build-nodejs\"",
+        "watch-web": "node ./builders/watchRun.js --port 9877 \"web/*.ts\" \"web/*.tsx\" \"yarn build-web\"",
+        "watch-extension": "node ./builders/watchRun.js --port 9878 \"extension/*.ts\" \"extension/*.tsx\" \"yarn build-extension\"",
+        "watch-electron": "node ./builders/watchRun.js --port 9879 \"electron/*.ts\" \"electron/*.tsx\" \"yarn build-electron\"",
+        "notes": "mobx, preact, socket-function, typenode SHOULD be peerDependencies. But we want to use yarn (better dependency deduplication), so we can't use peerDependencies (as they aren't installed by default, which makes them a nightmare to use). If you want to override the versions, feel free to use overrides/resolutions.",
+        "test": "typenode ./test.ts",
+        "filehoster": "node ./bin/filehoster.js"
+    },
+    "bin": {
+        "filehoster": "./bin/filehoster.js",
+        "build-nodejs": "./builders/nodeJSBuildRun.js",
+        "buildnodejs": "./builders/nodeJSBuildRun.js",
+        "build-extension": "./builders/extensionBuildRun.js",
+        "buildextension": "./builders/extensionBuildRun.js",
+        "build-web": "./builders/webBuildRun.js",
+        "buildweb": "./builders/webBuildRun.js",
+        "build-electron": "./builders/electronBuildRun.js",
+        "buildelectron": "./builders/electronBuildRun.js",
+        "slift-watch": "./builders/watchRun.js",
+        "sliftwatch": "./builders/watchRun.js",
+        "slift-setup": "./builders/setupRun.js",
+        "sliftsetup": "./builders/setupRun.js"
+    },
+    "dependencies": {
+        "@types/chrome": "^0.0.237",
+        "@types/node-forge": "^1.3.11",
+        "@types/shell-quote": "^1.7.5",
+        "acme-client": "^5.0.0",
+        "js-sha256": "^0.11.1",
+        "mobx": "^6.13.3",
+        "preact-old-types": "^10.28.1",
+        "shell-quote": "^1.8.3",
+        "socket-function": "^1.1.42",
+        "typenode": "*",
+        "typesafecss": "*",
+        "ws": "^8.18.3",
+        "yargs": "15.4.1"
+    },
+    "resolutions": {
+        "preact": "npm:preact-old-types@*"
+    },
+    "devDependencies": {
+        "@types/yargs": "15.0.19",
+        "debugbreak": "^0.9.9",
+        "electron": "^33.2.1",
+        "open": "^8.4.0",
+        "typedev": "^0.1.1"
+    }
+}

package/render-utils/FullscreenModal.tsx

@@ -48,7 +48,11 @@ export class FullscreenModal extends preact.Component<{
                         cursor: "pointer",
                         ...this.props.outerStyle,
                     }}
-                    onClick={e => {
+                    // Close on mouse DOWN on the backdrop, not click — otherwise a drag that starts
+                    // inside the modal (e.g. selecting text) and releases out on the backdrop fires a
+                    // click on the backdrop and wrongly closes it. currentTarget === target means the
+                    // press began on the backdrop itself, not bubbled up from the modal content.
+                    onMouseDown={e => {
                         if (e.currentTarget === e.target) {
                             if (parentState) parentState.open = false;
                             this.props.onCancel?.();

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -84,8 +84,29 @@ export interface IBulkDatabase2<T extends {
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
     getReaderInfo(): Promise<BulkReaderInfo>;
+    /**
+     * Per-file breakdown of the on-disk files, read fresh from disk each call (latest sizes, including
+     * stream files still being appended). `bytes` is the actual on-disk size. Good for showing collection
+     * size/fragmentation and deciding whether to call tryMergeNow()/compact().
+     */
+    getFileInfo(): Promise<{
+        files: {
+            name: string;
+            type: "bulk" | "stream";
+            bytes: number;
+        }[];
+        count: number;
+        totalBytes: number;
+    }>;
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
+    /**
+     * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+     * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+     * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+     * also run automatically on tab hide/close and before every merge.
+     */
+    flush(): Promise<void>;
     /**
      * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
      * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -69,10 +69,24 @@ export interface IBulkDatabase2<T extends { key: string }> {
     getColumnInfo(): Promise<BulkColumnInfo[]>;
     /** A cheap snapshot of the collection's shape (row/key counts, total bytes, columns) — no row data. */
     getReaderInfo(): Promise<BulkReaderInfo>;
+    /**
+     * Per-file breakdown of the on-disk files, read fresh from disk each call (latest sizes, including
+     * stream files still being appended). `bytes` is the actual on-disk size. Good for showing collection
+     * size/fragmentation and deciding whether to call tryMergeNow()/compact().
+     */
+    getFileInfo(): Promise<{ files: { name: string; type: "bulk" | "stream"; bytes: number }[]; count: number; totalBytes: number }>;
 
     /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
     compact(): Promise<void>;
 
+    /**
+     * Flush buffered stream writes to disk now. Writes are coalesced and flushed on a ramping delay (to
+     * avoid the browser rewriting the whole stream file per write), so a write's promise resolving means
+     * "accepted" (in memory + cross-tab), not necessarily "on disk". Call this to force durability — it's
+     * also run automatically on tab hide/close and before every merge.
+     */
+    flush(): Promise<void>;
+
     /**
      * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
      * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -2,8 +2,10 @@ import type { FileStorage } from "../FileFolderAPI";
 export declare const bulkDatabase2Timing: {
     streamSealAgeMs: number;
     mergeCheckIntervalMs: number;
+    mergeSpacingMs: number;
     firstMergeTriggerFiles: number;
     firstMergeTriggerRangeMs: number;
+    writeFlushMaxDelayMs: number;
 };
 export interface ReactiveDeps {
     observe(signal: string): void;
@@ -19,6 +21,11 @@ export declare class BulkDatabaseBase<T extends {
     protected deps: ReactiveDeps;
     private storageFactory;
     constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+    private pendingAppends;
+    private flushTimer;
+    private flushChain;
+    private currentFlushDelay;
+    private lastWriteTime;
     static clearCache(): void;
     storage: {
         (): Promise<FileStorage>;
@@ -43,6 +50,10 @@ export declare class BulkDatabaseBase<T extends {
     writeBatch(entries: T[]): Promise<void>;
     delete(key: string): Promise<void>;
     deleteBatch(keys: string[]): Promise<void>;
+    private streamAppend;
+    flush(): Promise<void>;
+    private flushPending;
+    private doFlush;
     update(entry: Partial<T> & {
         key: string;
     }): Promise<void>;
@@ -68,7 +79,9 @@ export declare class BulkDatabaseBase<T extends {
     private resolveReaders;
     private mergeFileSet;
     private canDeleteStream;
+    private mergeSpacingDelay;
     private testMerge;
+    private findDuplicateGroups;
     private formatInfo;
     private patchColumn;
     getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
@@ -114,4 +127,13 @@ export declare class BulkDatabaseBase<T extends {
             byteSize: number;
         }[];
     }>;
+    getFileInfo(): Promise<{
+        files: {
+            name: string;
+            type: "bulk" | "stream";
+            bytes: number;
+        }[];
+        count: number;
+        totalBytes: number;
+    }>;
 }