sliftutils 1.2.38 → 1.3.0

package/.claude/settings.local.json

@@ -1,7 +1,11 @@
 {
   "permissions": {
     "allow": [
-      "WebFetch(domain:openrouter.ai)"
+      "WebFetch(domain:openrouter.ai)",
+      "Bash(git pull *)",
+      "Bash(echo \"---EXIT:$?---\")",
+      "Bash(yarn publish *)",
+      "Bash(git push *)"
     ]
   }
 }

package/index.d.ts

@@ -55,6 +55,7 @@ declare module "sliftutils/misc/getSecret" {
         getAllKeys(): string[];
         get(key: string): Promise<string> | undefined;
     };
+    export declare function setSecret(key: string, value: string): void;
 
 }
 
@@ -156,6 +157,11 @@ declare module "sliftutils/misc/https/dns" {
 }
 
 declare module "sliftutils/misc/https/httpsCerts" {
+    /// <reference path="node-forge-ed25519.d.ts" />
+    /// <reference path="../../storage/storage.d.ts" />
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import * as forge from "node-forge";
     /** NOTE: We also generate the domain *.domain */
     export declare const getHTTPSCert: {
         (key: string): Promise<{
@@ -174,6 +180,18 @@ declare module "sliftutils/misc/https/httpsCerts" {
             cert: string;
         }> | undefined;
     };
+    export declare const getAccountKey: (domain: string) => Promise<string>;
+    export declare function parseCert(PEMorDER: string | Buffer): forge.pki.Certificate;
+    export declare function normalizeCertToPEM(PEMorDER: string | Buffer): string;
+    export declare function generateCert(config: {
+        accountKey: string;
+        domain: string;
+        altDomains?: string[];
+    }): Promise<{
+        domains: string[];
+        key: string;
+        cert: string;
+    }>;
 
 }
 
@@ -697,6 +715,7 @@ declare module "sliftutils/render-utils/colors" {
 }
 
 declare module "sliftutils/render-utils/mobxTyped" {
+    export { observable, runInAction, computed, autorun } from "mobx";
     export declare function configureMobxNextFrameScheduler(): void;
 
 }
@@ -743,6 +762,197 @@ declare module "sliftutils/render-utils/observer" {
 
 }
 
+declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
+    import { BulkDatabaseBase } from "./BulkDatabaseBase";
+    export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+    export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+    export declare class BulkDatabase2<T extends {
+        key: string;
+    }> extends BulkDatabaseBase<T> {
+        constructor(name: string);
+    }
+
+}
+
+declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
+    import type { FileStorage } from "../FileFolderAPI";
+    export interface ReactiveDeps {
+        observe(signal: string): void;
+        invalidate(signal: string): void;
+        batch(fn: () => void): void;
+    }
+    export declare const noopReactiveDeps: ReactiveDeps;
+    export type StorageFactory = (path: string) => Promise<FileStorage>;
+    export declare class BulkDatabaseBase<T extends {
+        key: string;
+    }> {
+        readonly name: string;
+        protected deps: ReactiveDeps;
+        private storageFactory;
+        constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+        static clearCache(): void;
+        storage: {
+            (): Promise<FileStorage>;
+            reset(): void;
+            set(newValue: Promise<FileStorage>): void;
+        };
+        private overlay;
+        private streamTimes;
+        private streamFileName;
+        private streamRowsWritten;
+        private getStreamFileName;
+        private setOverlay;
+        private reader;
+        private syncSetup;
+        private localTime;
+        private applyRemote;
+        private resetReader;
+        write(entry: T): Promise<void>;
+        writeBatch(entries: T[]): Promise<void>;
+        delete(key: string): Promise<void>;
+        deleteBatch(keys: string[]): Promise<void>;
+        private writeBulkFile;
+        private listStreamFiles;
+        private loadStreamEntries;
+        private orderStreamEntries;
+        private maybeRolloverStream;
+        private rolloverStream;
+        compact(): Promise<void>;
+        private listFiles;
+        private makeRawGetRange;
+        private loadFileReader;
+        private fileLogicalSize;
+        private handleUnreadableFile;
+        private mergeFilesBase;
+        private mergeFiles;
+        private formatInfo;
+        private patchColumn;
+        getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+        getColumn<Column extends keyof T>(column: Column): Promise<{
+            key: string;
+            value: T[Column];
+        }[]>;
+        getKeys(): Promise<string[]>;
+        private baseColumns;
+        private baseColumnsLoading;
+        private baseFields;
+        private baseFieldsLoading;
+        private ensureBaseColumn;
+        private ensureBaseField;
+        getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+        getColumnSync<Column extends keyof T>(column: Column): {
+            key: string;
+            value: T[Column];
+        }[] | undefined;
+        getColumnInfo(): Promise<{
+            column: string;
+            byteSize: number;
+        }[]>;
+        getReaderInfo(): Promise<{
+            rowCount: number;
+            totalBytes: number;
+            keyCount: number;
+            sampleKey: string | undefined;
+            columns: {
+                column: string;
+                byteSize: number;
+            }[];
+        }>;
+    }
+
+}
+
+declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseFormat" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    export declare const KEY_COLUMN = "key";
+    export declare const EMPTY_BUFFER: Buffer;
+    export declare function buildFileBuffer(rows: Record<string, unknown>[]): Buffer[];
+    export type BaseBulkDatabaseReader = {
+        rowCount: number;
+        totalBytes: number;
+        keys: string[];
+        columns: {
+            column: string;
+            byteSize: number;
+        }[];
+        deletedKeys?: Set<string>;
+        getColumn: (column: string) => Promise<{
+            key: string;
+            value: unknown;
+        }[]>;
+        getSingleField: (key: string, column: string) => Promise<unknown | undefined>;
+    };
+    export declare function loadBulkDatabase(config: {
+        totalBytes: number;
+        getRange: (start: number, end: number) => Promise<Buffer>;
+    }): Promise<BaseBulkDatabaseReader>;
+
+}
+
+declare module "sliftutils/storage/BulkDatabase2/blockCache" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    export type GetRange = (start: number, end: number) => Promise<Buffer>;
+    export declare function encodeCompressedBlocks(data: Buffer): Buffer;
+    export declare class BlockCache {
+        private blocks;
+        private indexes;
+        clear(): void;
+        private touch;
+        private readIndex;
+        open(fileId: string, fileSize: number, rawGetRange: GetRange): Promise<{
+            uncompressedSize: number;
+            getRange: GetRange;
+        }>;
+        private makeGetRange;
+    }
+    export declare const blockCache: BlockCache;
+
+}
+
+declare module "sliftutils/storage/BulkDatabase2/streamLog" {
+    /// <reference types="node" />
+    /// <reference types="node" />
+    import { BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
+    export declare const STREAM_EXTENSION = ".stream";
+    export type StreamEntry = {
+        time: number;
+        row?: Record<string, unknown>;
+        deletedKey?: string;
+    };
+    export declare function frameRows(entries: {
+        time: number;
+        row: Record<string, unknown>;
+    }[]): Buffer;
+    export declare function frameDeletes(entries: {
+        time: number;
+        key: string;
+    }[]): Buffer;
+    export declare function parseStream(buffer: Buffer): {
+        entries: StreamEntry[];
+        badBytes: number;
+    };
+    export declare function streamReaderFromEntries(entries: StreamEntry[], totalBytes: number): {
+        reader: BaseBulkDatabaseReader;
+        times: Map<string, number>;
+    };
+
+}
+
+declare module "sliftutils/storage/BulkDatabase2/syncClient" {
+    export type RemoteWrite = {
+        key: string;
+        time: number;
+        deleted?: boolean;
+        value?: unknown;
+    };
+    export declare function isSyncSupported(): boolean;
+    export declare function connect(collection: string, onWrite: (write: RemoteWrite) => void): Promise<RemoteWrite[]>;
+    export declare function broadcast(collection: string, write: RemoteWrite): void;
+
+}
+
 declare module "sliftutils/storage/CBORStorage" {
     /// <reference types="node" />
     /// <reference types="node" />
@@ -798,6 +1008,8 @@ declare module "sliftutils/storage/DiskCollection" {
     export declare class DiskCollection<T> implements IStorageSync<T> {
         private collectionName;
         private config?;
+        static getForceNoPrompt(): boolean;
+        static setForceNoPrompt(forceNoPrompt: boolean): void;
         constructor(collectionName: string, config?: {
             writeDelay?: number | undefined;
             cbor?: boolean | undefined;
@@ -967,6 +1179,52 @@ declare module "sliftutils/storage/FileFolderAPI" {
         ]>;
     };
     export declare function setFileAPIKey(key: string): void;
+    export declare class NodeJSFileHandleWrapper implements FileWrapper {
+        private filePath;
+        constructor(filePath: string);
+        getFile(): Promise<{
+            size: number;
+            lastModified: number;
+            arrayBuffer: () => Promise<ArrayBuffer>;
+            slice: (start: number, end: number) => {
+                arrayBuffer: () => Promise<ArrayBuffer>;
+            };
+        }>;
+        createWritable(config?: {
+            keepExistingData?: boolean;
+        }): Promise<{
+            seek: (offset: number) => Promise<void>;
+            write: (value: Buffer) => Promise<void>;
+            close: () => Promise<void>;
+        }>;
+    }
+    export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
+        private rootPath;
+        constructor(rootPath: string);
+        removeEntry(key: string, options?: {
+            recursive?: boolean;
+        }): Promise<void>;
+        getFileHandle(key: string, options?: {
+            create?: boolean;
+        }): Promise<FileWrapper>;
+        getDirectoryHandle(key: string, options?: {
+            create?: boolean;
+        }): Promise<DirectoryWrapper>;
+        [Symbol.asyncIterator](): AsyncIterableIterator<[
+            string,
+            {
+                kind: "file";
+                name: string;
+                getFile(): Promise<FileWrapper>;
+            } | {
+                kind: "directory";
+                name: string;
+                getDirectoryHandle(key: string, options?: {
+                    create?: boolean;
+                }): Promise<DirectoryWrapper>;
+            }
+        ]>;
+    }
     export declare const getDirectoryHandle: {
         (): Promise<DirectoryWrapper>;
         reset(): void;
@@ -980,6 +1238,14 @@ declare module "sliftutils/storage/FileFolderAPI" {
         getAllKeys(): string[];
         get(key: string): Promise<FileStorage> | undefined;
     };
+    export declare const getFileStorageNested2: {
+        (key: string): Promise<FileStorage>;
+        clear(key: string): void;
+        clearAll(): void;
+        forceSet(key: string, value: Promise<FileStorage>): void;
+        getAllKeys(): string[];
+        get(key: string): Promise<FileStorage> | undefined;
+    };
     export declare const getFileStorage: {
         (): Promise<FileStorage>;
         reset(): void;
@@ -995,6 +1261,8 @@ declare module "sliftutils/storage/FileFolderAPI" {
     export type FileStorage = IStorageRaw & {
         folder: NestedFileStorage;
     };
+    export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
+    export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
     export {};
 
 }

package/misc/getSecret.d.ts

@@ -7,3 +7,4 @@ export declare const getSecret: {
     getAllKeys(): string[];
     get(key: string): Promise<string> | undefined;
 };
+export declare function setSecret(key: string, value: string): void;

package/misc/getSecret.ts

@@ -96,3 +96,10 @@ export const getSecret = cache(async function getSecret(key: string): Promise<st
         });
     });
 });
+
+export function setSecret(key: string, value: string) {
+    if (isNode()) {
+        throw new Error("setSecret is only supported in the browser. We don't want to break the user's file system with setSecret in NodeJS.");
+    }
+    localStorage.setItem(getStorageKey(key), value);
+}

package/misc/https/httpsCerts.d.ts

@@ -1,3 +1,8 @@
+/// <reference path="node-forge-ed25519.d.ts" />
+/// <reference path="../../storage/storage.d.ts" />
+/// <reference types="node" />
+/// <reference types="node" />
+import * as forge from "node-forge";
 /** NOTE: We also generate the domain *.domain */
 export declare const getHTTPSCert: {
     (key: string): Promise<{
@@ -16,3 +21,15 @@ export declare const getHTTPSCert: {
         cert: string;
     }> | undefined;
 };
+export declare const getAccountKey: (domain: string) => Promise<string>;
+export declare function parseCert(PEMorDER: string | Buffer): forge.pki.Certificate;
+export declare function normalizeCertToPEM(PEMorDER: string | Buffer): string;
+export declare function generateCert(config: {
+    accountKey: string;
+    domain: string;
+    altDomains?: string[];
+}): Promise<{
+    domains: string[];
+    key: string;
+    cert: string;
+}>;

package/misc/https/httpsCerts.ts

@@ -91,7 +91,7 @@ export const getHTTPSCert = cache(async (domain: string): Promise<{ key: string;
 });
 
 
-const getAccountKey = async function getAccountKey(domain: string) {
+export const getAccountKey = async function getAccountKey(domain: string) {
     let accountKey = getKeyStore<string>(domain, "letsEncryptAccountKey");
     let secret = await accountKey.get();
     if (!secret) {
@@ -105,11 +105,11 @@ const getAccountKey = async function getAccountKey(domain: string) {
 };
 
 
-function parseCert(PEMorDER: string | Buffer) {
+export function parseCert(PEMorDER: string | Buffer) {
     return forge.pki.certificateFromPem(normalizeCertToPEM(PEMorDER));
 }
 
-function normalizeCertToPEM(PEMorDER: string | Buffer): string {
+export function normalizeCertToPEM(PEMorDER: string | Buffer): string {
     if (PEMorDER.toString().startsWith("-----BEGIN CERTIFICATE-----")) {
         return PEMorDER.toString();
     }
@@ -118,7 +118,7 @@ function normalizeCertToPEM(PEMorDER: string | Buffer): string {
 }
 
 
-async function generateCert(config: {
+export async function generateCert(config: {
     accountKey: string;
     domain: string;
     altDomains?: string[];

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "sliftutils",
-    "version": "1.2.38",
+    "version": "1.3.0",
     "main": "index.js",
     "license": "MIT",
     "files": [
@@ -51,7 +51,7 @@
         "mobx": "^6.13.3",
         "preact-old-types": "^10.28.1",
         "shell-quote": "^1.8.3",
-        "socket-function": "^1.1.30",
+        "socket-function": "^1.1.42",
         "typenode": "*",
         "typesafecss": "*",
         "ws": "^8.18.3",

package/render-utils/mobxTyped.d.ts

@@ -1 +1,2 @@
+export { observable, runInAction, computed, autorun } from "mobx";
 export declare function configureMobxNextFrameScheduler(): void;

package/render-utils/mobxTyped.ts

@@ -1,5 +1,9 @@
 import * as mobx from "mobx";
 import { batchFunction } from "socket-function/src/batching";
+
+// Re-export the core mobx primitives so downstream packages (which may have their own duplicate mobx
+// in node_modules) can share THIS mobx instance, otherwise reactivity doesn't cross package boundaries.
+export { observable, runInAction, computed, autorun } from "mobx";
 export function configureMobxNextFrameScheduler() {
     // NOTE: This makes a big difference if we do await calls in a loop which mutates observable state. BUT... we should probably just do those await calls before the loop?
     let batchReactionScheduler = batchFunction({

package/storage/BulkDatabase2/BulkDatabase2.d.ts

@@ -0,0 +1,8 @@
+import { BulkDatabaseBase } from "./BulkDatabaseBase";
+export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+export declare class BulkDatabase2<T extends {
+    key: string;
+}> extends BulkDatabaseBase<T> {
+    constructor(name: string);
+}

package/storage/BulkDatabase2/BulkDatabase2.ts

@@ -0,0 +1,41 @@
+import { getFileStorageNested2 } from "../FileFolderAPI";
+import { observable, runInAction } from "../../render-utils/mobxTyped";
+import { BulkDatabaseBase, ReactiveDeps } from "./BulkDatabaseBase";
+
+export { BulkDatabaseBase, noopReactiveDeps } from "./BulkDatabaseBase";
+export type { ReactiveDeps, StorageFactory } from "./BulkDatabaseBase";
+
+// mobx-backed reactivity: each signal string gets its own observable box. observe() reads it (so an
+// observer/autorun that calls it tracks that box) and invalidate() bumps it (so those reactions re-run).
+// This reproduces the fine-grained per-key + load-version reactivity the class had when it used
+// observable.map/observable.box directly, while keeping all that logic in the mobx-free base.
+class MobxReactiveDeps implements ReactiveDeps {
+    private boxes = new Map<string, { get(): number; set(value: number): void }>();
+    private box(signal: string) {
+        let box = this.boxes.get(signal);
+        if (!box) {
+            box = observable.box(0);
+            this.boxes.set(signal, box);
+        }
+        return box;
+    }
+    observe(signal: string) {
+        this.box(signal).get();
+    }
+    invalidate(signal: string) {
+        let box = this.box(signal);
+        box.set(box.get() + 1);
+    }
+    batch(fn: () => void) {
+        runInAction(fn);
+    }
+}
+
+// Backwards-compatible BulkDatabase2: the mobx-reactive flavor. All behavior lives in BulkDatabaseBase;
+// this just supplies mobx reactivity and the default (getFileStorageNested2) storage backend, so the
+// sync reads (getSingleFieldSync / getColumnSync) stay observable for mobx components.
+export class BulkDatabase2<T extends { key: string }> extends BulkDatabaseBase<T> {
+    constructor(name: string) {
+        super(name, new MobxReactiveDeps(), getFileStorageNested2);
+    }
+}

package/storage/BulkDatabase2/BulkDatabaseBase.d.ts

@@ -0,0 +1,84 @@
+import type { FileStorage } from "../FileFolderAPI";
+export interface ReactiveDeps {
+    observe(signal: string): void;
+    invalidate(signal: string): void;
+    batch(fn: () => void): void;
+}
+export declare const noopReactiveDeps: ReactiveDeps;
+export type StorageFactory = (path: string) => Promise<FileStorage>;
+export declare class BulkDatabaseBase<T extends {
+    key: string;
+}> {
+    readonly name: string;
+    protected deps: ReactiveDeps;
+    private storageFactory;
+    constructor(name: string, deps: ReactiveDeps, storageFactory: StorageFactory);
+    static clearCache(): void;
+    storage: {
+        (): Promise<FileStorage>;
+        reset(): void;
+        set(newValue: Promise<FileStorage>): void;
+    };
+    private overlay;
+    private streamTimes;
+    private streamFileName;
+    private streamRowsWritten;
+    private getStreamFileName;
+    private setOverlay;
+    private reader;
+    private syncSetup;
+    private localTime;
+    private applyRemote;
+    private resetReader;
+    write(entry: T): Promise<void>;
+    writeBatch(entries: T[]): Promise<void>;
+    delete(key: string): Promise<void>;
+    deleteBatch(keys: string[]): Promise<void>;
+    private writeBulkFile;
+    private listStreamFiles;
+    private loadStreamEntries;
+    private orderStreamEntries;
+    private maybeRolloverStream;
+    private rolloverStream;
+    compact(): Promise<void>;
+    private listFiles;
+    private makeRawGetRange;
+    private loadFileReader;
+    private fileLogicalSize;
+    private handleUnreadableFile;
+    private mergeFilesBase;
+    private mergeFiles;
+    private formatInfo;
+    private patchColumn;
+    getSingleField<Column extends keyof T>(key: string, column: Column): Promise<T[Column] | undefined>;
+    getColumn<Column extends keyof T>(column: Column): Promise<{
+        key: string;
+        value: T[Column];
+    }[]>;
+    getKeys(): Promise<string[]>;
+    private baseColumns;
+    private baseColumnsLoading;
+    private baseFields;
+    private baseFieldsLoading;
+    private ensureBaseColumn;
+    private ensureBaseField;
+    getSingleFieldSync<Column extends keyof T>(key: string, column: Column): T[Column] | undefined;
+    getColumnSync<Column extends keyof T>(column: Column): {
+        key: string;
+        value: T[Column];
+    }[] | undefined;
+    getColumnInfo(): Promise<{
+        column: string;
+        byteSize: number;
+    }[]>;
+    getReaderInfo(): Promise<{
+        rowCount: number;
+        totalBytes: number;
+        keyCount: number;
+        sampleKey: string | undefined;
+        columns: {
+            column: string;
+            byteSize: number;
+        }[];
+    }>;
+}