sliftutils 1.7.4 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +281 -2
- package/misc/https/persistentLocalStorage.ts +2 -0
- package/package.json +1 -1
- package/render-utils/Input.tsx +2 -0
- package/render-utils/InputLabel.tsx +2 -0
- package/render-utils/colors.tsx +2 -0
- package/storage/BulkDatabase2/syncClient.ts +10 -7
- package/storage/DiskCollection.d.ts +1 -0
- package/storage/DiskCollection.ts +3 -1
- package/storage/FileFolderAPI.d.ts +1 -0
- package/storage/FileFolderAPI.tsx +18 -0
- package/storage/IArchives.d.ts +37 -0
- package/storage/IArchives.ts +20 -0
- package/storage/TransactionStorage.d.ts +2 -1
- package/storage/TransactionStorage.ts +6 -2
- package/storage/backblaze.d.ts +2 -1
- package/storage/backblaze.ts +2 -1
- package/storage/remoteStorage/ArchivesRemote.d.ts +67 -0
- package/storage/remoteStorage/ArchivesRemote.ts +195 -0
- package/storage/remoteStorage/accessPage.d.ts +1 -0
- package/storage/remoteStorage/accessPage.tsx +133 -0
- package/storage/remoteStorage/blobStore.d.ts +56 -0
- package/storage/remoteStorage/blobStore.ts +403 -0
- package/storage/remoteStorage/storageController.d.ts +89 -0
- package/storage/remoteStorage/storageController.ts +398 -0
- package/storage/remoteStorage/storageServer.d.ts +1 -0
- package/storage/remoteStorage/storageServer.ts +132 -0
- package/teststorage/browser.tsx +148 -0
- package/teststorage/server.ts +43 -0
- package/yarn.lock +3 -3
package/index.d.ts
CHANGED
|
@@ -1717,6 +1717,7 @@ declare module "sliftutils/storage/DiskCollection" {
|
|
|
1717
1717
|
writeDelay?: number | undefined;
|
|
1718
1718
|
cbor?: boolean | undefined;
|
|
1719
1719
|
noPrompt?: boolean | undefined;
|
|
1720
|
+
resyncFromDisk?: boolean | undefined;
|
|
1720
1721
|
freeze?: "deep" | "shallow" | undefined;
|
|
1721
1722
|
beforeWrite?: ((update: {
|
|
1722
1723
|
newValue: T;
|
|
@@ -1878,6 +1879,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1878
1879
|
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileWrapper | DirectoryWrapper]>;
|
|
1879
1880
|
};
|
|
1880
1881
|
export declare function setFileAPIKey(key: string): void;
|
|
1882
|
+
export declare function setStorageRootOverride(handle: FileSystemDirectoryHandle | undefined): void;
|
|
1881
1883
|
export declare class NodeJSFileHandleWrapper implements FileWrapper {
|
|
1882
1884
|
private filePath;
|
|
1883
1885
|
constructor(filePath: string);
|
|
@@ -1967,6 +1969,47 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1967
1969
|
|
|
1968
1970
|
}
|
|
1969
1971
|
|
|
1972
|
+
declare module "sliftutils/storage/IArchives" {
|
|
1973
|
+
/// <reference types="node" />
|
|
1974
|
+
/// <reference types="node" />
|
|
1975
|
+
export type ArchiveFileInfo = {
|
|
1976
|
+
path: string;
|
|
1977
|
+
createTime: number;
|
|
1978
|
+
size: number;
|
|
1979
|
+
};
|
|
1980
|
+
export interface IArchives {
|
|
1981
|
+
getDebugName(): string;
|
|
1982
|
+
get(fileName: string, config?: {
|
|
1983
|
+
range?: {
|
|
1984
|
+
start: number;
|
|
1985
|
+
end: number;
|
|
1986
|
+
};
|
|
1987
|
+
}): Promise<Buffer | undefined>;
|
|
1988
|
+
set(fileName: string, data: Buffer): Promise<void>;
|
|
1989
|
+
del(fileName: string): Promise<void>;
|
|
1990
|
+
/** Streams a file too large to hold in memory. getNextData returns undefined when done. */
|
|
1991
|
+
setLargeFile(config: {
|
|
1992
|
+
path: string;
|
|
1993
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
1994
|
+
}): Promise<void>;
|
|
1995
|
+
getInfo(fileName: string): Promise<{
|
|
1996
|
+
writeTime: number;
|
|
1997
|
+
size: number;
|
|
1998
|
+
} | undefined>;
|
|
1999
|
+
find(prefix: string, config?: {
|
|
2000
|
+
shallow?: boolean;
|
|
2001
|
+
type: "files" | "folders";
|
|
2002
|
+
}): Promise<string[]>;
|
|
2003
|
+
findInfo(prefix: string, config?: {
|
|
2004
|
+
shallow?: boolean;
|
|
2005
|
+
type: "files" | "folders";
|
|
2006
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
2007
|
+
/** Only works for public buckets (private buckets are API-access only). */
|
|
2008
|
+
getURL(path: string): Promise<string>;
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
}
|
|
2012
|
+
|
|
1970
2013
|
declare module "sliftutils/storage/IStorage" {
|
|
1971
2014
|
/// <reference types="node" />
|
|
1972
2015
|
/// <reference types="node" />
|
|
@@ -2177,12 +2220,13 @@ declare module "sliftutils/storage/TransactionStorage" {
|
|
|
2177
2220
|
private rawStorage;
|
|
2178
2221
|
private debugName;
|
|
2179
2222
|
private writeDelay;
|
|
2223
|
+
private resyncFromDisk;
|
|
2180
2224
|
cache: Map<string, TransactionEntry>;
|
|
2181
2225
|
private diskFiles;
|
|
2182
2226
|
private currentChunk;
|
|
2183
2227
|
private entryCount;
|
|
2184
2228
|
private static allStorage;
|
|
2185
|
-
constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
|
|
2229
|
+
constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number, resyncFromDisk?: boolean);
|
|
2186
2230
|
static compressAll(): Promise<void>;
|
|
2187
2231
|
private resyncCallbacks;
|
|
2188
2232
|
watchResync(callback: () => void): void;
|
|
@@ -2222,7 +2266,8 @@ declare module "sliftutils/storage/TransactionStorage" {
|
|
|
2222
2266
|
declare module "sliftutils/storage/backblaze" {
|
|
2223
2267
|
/// <reference types="node" />
|
|
2224
2268
|
/// <reference types="node" />
|
|
2225
|
-
|
|
2269
|
+
import { IArchives } from "./IArchives";
|
|
2270
|
+
export declare class ArchivesBackblaze implements IArchives {
|
|
2226
2271
|
private config;
|
|
2227
2272
|
constructor(config: {
|
|
2228
2273
|
bucketName: string;
|
|
@@ -2525,6 +2570,240 @@ declare module "sliftutils/storage/remoteFileStorage" {
|
|
|
2525
2570
|
|
|
2526
2571
|
}
|
|
2527
2572
|
|
|
2573
|
+
declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
2574
|
+
/// <reference types="node" />
|
|
2575
|
+
/// <reference types="node" />
|
|
2576
|
+
import { IArchives, ArchiveFileInfo } from "../IArchives";
|
|
2577
|
+
export type ArchivesRemoteConfig = {
|
|
2578
|
+
address: string;
|
|
2579
|
+
port: number;
|
|
2580
|
+
account: string;
|
|
2581
|
+
bucketName: string;
|
|
2582
|
+
public?: boolean;
|
|
2583
|
+
fast?: boolean;
|
|
2584
|
+
writeDelay?: number;
|
|
2585
|
+
};
|
|
2586
|
+
export declare function buildPublicFileURL(config: {
|
|
2587
|
+
address: string;
|
|
2588
|
+
port: number;
|
|
2589
|
+
account: string;
|
|
2590
|
+
bucketName: string;
|
|
2591
|
+
path: string;
|
|
2592
|
+
}): string;
|
|
2593
|
+
export declare function authenticateStorage(config: {
|
|
2594
|
+
address: string;
|
|
2595
|
+
port: number;
|
|
2596
|
+
nodeId: string;
|
|
2597
|
+
}): Promise<{
|
|
2598
|
+
machineId: string;
|
|
2599
|
+
ip: string;
|
|
2600
|
+
}>;
|
|
2601
|
+
export declare class ArchivesRemote implements IArchives {
|
|
2602
|
+
private config;
|
|
2603
|
+
constructor(config: ArchivesRemoteConfig);
|
|
2604
|
+
private nodeId;
|
|
2605
|
+
private controller;
|
|
2606
|
+
private setupDone;
|
|
2607
|
+
private lastDeniedLog;
|
|
2608
|
+
getDebugName(): string;
|
|
2609
|
+
private authenticate;
|
|
2610
|
+
private callAuthed;
|
|
2611
|
+
waitingForAccess(): Promise<string | undefined>;
|
|
2612
|
+
private onAccessDenied;
|
|
2613
|
+
private ensureSetup;
|
|
2614
|
+
private call;
|
|
2615
|
+
get(fileName: string, config?: {
|
|
2616
|
+
range?: {
|
|
2617
|
+
start: number;
|
|
2618
|
+
end: number;
|
|
2619
|
+
};
|
|
2620
|
+
}): Promise<Buffer | undefined>;
|
|
2621
|
+
set(fileName: string, data: Buffer): Promise<void>;
|
|
2622
|
+
del(fileName: string): Promise<void>;
|
|
2623
|
+
getInfo(fileName: string): Promise<{
|
|
2624
|
+
writeTime: number;
|
|
2625
|
+
size: number;
|
|
2626
|
+
} | undefined>;
|
|
2627
|
+
findInfo(prefix: string, config?: {
|
|
2628
|
+
shallow?: boolean;
|
|
2629
|
+
type: "files" | "folders";
|
|
2630
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
2631
|
+
find(prefix: string, config?: {
|
|
2632
|
+
shallow?: boolean;
|
|
2633
|
+
type: "files" | "folders";
|
|
2634
|
+
}): Promise<string[]>;
|
|
2635
|
+
setLargeFile(config: {
|
|
2636
|
+
path: string;
|
|
2637
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
2638
|
+
}): Promise<void>;
|
|
2639
|
+
getURL(path: string): Promise<string>;
|
|
2640
|
+
}
|
|
2641
|
+
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2644
|
+
declare module "sliftutils/storage/remoteStorage/accessPage" {
|
|
2645
|
+
export {};
|
|
2646
|
+
|
|
2647
|
+
}
|
|
2648
|
+
|
|
2649
|
+
declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
2650
|
+
/// <reference types="node" />
|
|
2651
|
+
/// <reference types="node" />
|
|
2652
|
+
import { ArchiveFileInfo } from "../IArchives";
|
|
2653
|
+
export declare const DEFAULT_FAST_WRITE_DELAY: number;
|
|
2654
|
+
export type WriteConfig = {
|
|
2655
|
+
fast?: boolean;
|
|
2656
|
+
writeDelay?: number;
|
|
2657
|
+
};
|
|
2658
|
+
export declare class BlobStore {
|
|
2659
|
+
private folder;
|
|
2660
|
+
constructor(folder: string);
|
|
2661
|
+
private memCache;
|
|
2662
|
+
private overlay;
|
|
2663
|
+
private writeQueue;
|
|
2664
|
+
private openBlobs;
|
|
2665
|
+
private largeUploads;
|
|
2666
|
+
private nextLargeUploadId;
|
|
2667
|
+
private currentBlobNumber;
|
|
2668
|
+
private currentBlobOffset;
|
|
2669
|
+
private currentBlobFd;
|
|
2670
|
+
private index;
|
|
2671
|
+
private deadBytes;
|
|
2672
|
+
private blobsDir;
|
|
2673
|
+
init: {
|
|
2674
|
+
(): Promise<void>;
|
|
2675
|
+
reset(): void;
|
|
2676
|
+
set(newValue: Promise<void>): void;
|
|
2677
|
+
};
|
|
2678
|
+
private blobName;
|
|
2679
|
+
private blobPath;
|
|
2680
|
+
private getBlobHandle;
|
|
2681
|
+
private closeBlobHandle;
|
|
2682
|
+
private addDeadBytes;
|
|
2683
|
+
private appendData;
|
|
2684
|
+
private setIndexEntry;
|
|
2685
|
+
set(key: string, data: Buffer, config?: WriteConfig): Promise<void>;
|
|
2686
|
+
del(key: string, config?: WriteConfig): Promise<void>;
|
|
2687
|
+
get(key: string, range?: {
|
|
2688
|
+
start: number;
|
|
2689
|
+
end: number;
|
|
2690
|
+
}): Promise<Buffer | undefined>;
|
|
2691
|
+
getInfo(key: string): Promise<{
|
|
2692
|
+
writeTime: number;
|
|
2693
|
+
size: number;
|
|
2694
|
+
} | undefined>;
|
|
2695
|
+
findInfo(prefix: string, config?: {
|
|
2696
|
+
shallow?: boolean;
|
|
2697
|
+
type?: "files" | "folders";
|
|
2698
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
2699
|
+
startLargeUpload(): Promise<string>;
|
|
2700
|
+
appendLargeUpload(id: string, data: Buffer): Promise<void>;
|
|
2701
|
+
finishLargeUpload(id: string, key: string): Promise<void>;
|
|
2702
|
+
cancelLargeUpload(id: string): Promise<void>;
|
|
2703
|
+
private flushOverlay;
|
|
2704
|
+
private compact;
|
|
2705
|
+
}
|
|
2706
|
+
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
declare module "sliftutils/storage/remoteStorage/storageController" {
|
|
2710
|
+
/// <reference types="node" />
|
|
2711
|
+
/// <reference types="node" />
|
|
2712
|
+
import { ArchiveFileInfo } from "../IArchives";
|
|
2713
|
+
import type { BlobStore } from "./blobStore";
|
|
2714
|
+
import type { IStorage } from "../IStorage";
|
|
2715
|
+
export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
|
|
2716
|
+
export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
|
|
2717
|
+
export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
|
|
2718
|
+
export declare const STORAGE_ACCESS_DENIED = "REMOTE_STORAGE_ACCESS_DENIED_9d81a4c0";
|
|
2719
|
+
export type AuthToken = {
|
|
2720
|
+
certPem: string;
|
|
2721
|
+
time: number;
|
|
2722
|
+
signature: string;
|
|
2723
|
+
};
|
|
2724
|
+
export type AccessRequest = {
|
|
2725
|
+
requestId: string;
|
|
2726
|
+
account: string;
|
|
2727
|
+
machineId: string;
|
|
2728
|
+
ip: string;
|
|
2729
|
+
time: number;
|
|
2730
|
+
};
|
|
2731
|
+
export type TrustRecord = {
|
|
2732
|
+
account: string;
|
|
2733
|
+
machineId: string;
|
|
2734
|
+
ip: string;
|
|
2735
|
+
time: number;
|
|
2736
|
+
};
|
|
2737
|
+
export type BucketConfig = {
|
|
2738
|
+
public?: boolean;
|
|
2739
|
+
fast?: boolean;
|
|
2740
|
+
writeDelay?: number;
|
|
2741
|
+
};
|
|
2742
|
+
export type AccessState = {
|
|
2743
|
+
machineId: string;
|
|
2744
|
+
ip: string;
|
|
2745
|
+
hasAccess: boolean;
|
|
2746
|
+
listAccessCommand: string;
|
|
2747
|
+
grantAccessCommand?: string;
|
|
2748
|
+
machines?: (AccessRequest & {
|
|
2749
|
+
trusted: boolean;
|
|
2750
|
+
})[];
|
|
2751
|
+
};
|
|
2752
|
+
export type StorageServerState = {
|
|
2753
|
+
domain: string;
|
|
2754
|
+
port: number;
|
|
2755
|
+
rootDomain: string;
|
|
2756
|
+
sshTarget: string;
|
|
2757
|
+
serverCommand: string;
|
|
2758
|
+
blobStore: BlobStore;
|
|
2759
|
+
trust: IStorage<TrustRecord>;
|
|
2760
|
+
requests: IStorage<AccessRequest[]>;
|
|
2761
|
+
buckets: IStorage<BucketConfig>;
|
|
2762
|
+
};
|
|
2763
|
+
export declare function setStorageServerState(state: StorageServerState): void;
|
|
2764
|
+
export declare const RemoteStorageController: import("socket-function/SocketFunctionTypes").SocketRegistered<{
|
|
2765
|
+
authenticate: (token: AuthToken) => Promise<{
|
|
2766
|
+
machineId: string;
|
|
2767
|
+
ip: string;
|
|
2768
|
+
}>;
|
|
2769
|
+
requestAccess: (account: string) => Promise<{
|
|
2770
|
+
machineId: string;
|
|
2771
|
+
ip: string;
|
|
2772
|
+
requestId: string;
|
|
2773
|
+
grantAccessCommand: string;
|
|
2774
|
+
}>;
|
|
2775
|
+
getAccessState: (account: string) => Promise<AccessState>;
|
|
2776
|
+
adminListRequests: (ip: string) => Promise<AccessRequest[]>;
|
|
2777
|
+
adminGrantAccess: (requestId: string) => Promise<TrustRecord>;
|
|
2778
|
+
ensureBucket: (account: string, bucketName: string, config: BucketConfig) => Promise<void>;
|
|
2779
|
+
get: (account: string, bucketName: string, path: string, range?: {
|
|
2780
|
+
start: number;
|
|
2781
|
+
end: number;
|
|
2782
|
+
}) => Promise<Buffer | undefined>;
|
|
2783
|
+
set: (account: string, bucketName: string, path: string, data: Buffer) => Promise<void>;
|
|
2784
|
+
del: (account: string, bucketName: string, path: string) => Promise<void>;
|
|
2785
|
+
getInfo: (account: string, bucketName: string, path: string) => Promise<{
|
|
2786
|
+
writeTime: number;
|
|
2787
|
+
size: number;
|
|
2788
|
+
} | undefined>;
|
|
2789
|
+
findInfo: (account: string, bucketName: string, prefix: string, config?: {
|
|
2790
|
+
shallow?: boolean;
|
|
2791
|
+
type?: "files" | "folders";
|
|
2792
|
+
}) => Promise<ArchiveFileInfo[]>;
|
|
2793
|
+
startLargeFile: (account: string, bucketName: string, path: string) => Promise<string>;
|
|
2794
|
+
uploadPart: (uploadId: string, data: Buffer) => Promise<void>;
|
|
2795
|
+
finishLargeFile: (uploadId: string) => Promise<void>;
|
|
2796
|
+
cancelLargeFile: (uploadId: string) => Promise<void>;
|
|
2797
|
+
getPublicFile: (account: string, bucketName: string, path: string) => Promise<Buffer>;
|
|
2798
|
+
}>;
|
|
2799
|
+
|
|
2800
|
+
}
|
|
2801
|
+
|
|
2802
|
+
declare module "sliftutils/storage/remoteStorage/storageServer" {
|
|
2803
|
+
import "./accessPage";
|
|
2804
|
+
|
|
2805
|
+
}
|
|
2806
|
+
|
|
2528
2807
|
declare module "sliftutils/storage/storage" {
|
|
2529
2808
|
|
|
2530
2809
|
|
package/package.json
CHANGED
package/render-utils/Input.tsx
CHANGED
package/render-utils/colors.tsx
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { isNode } from "typesafecss";
|
|
2
|
-
|
|
3
1
|
// Browser-side cross-tab write sync for BulkDatabase2, over BroadcastChannel (one channel per
|
|
4
|
-
// collection, same-origin tabs). It does NOT persist anything — each
|
|
5
|
-
// just relays live writes to other open tabs and, when
|
|
6
|
-
// made recently that may not be on disk yet. No-op in Node / where
|
|
7
|
-
// BulkDatabase2 can call these unconditionally. This is an
|
|
2
|
+
// collection, same-origin tabs and workers). It does NOT persist anything — each side writes to disk
|
|
3
|
+
// itself; this just relays live writes to other open tabs/workers and, when one starts up, asks peers
|
|
4
|
+
// for writes they've made recently that may not be on disk yet. No-op in Node / where
|
|
5
|
+
// BroadcastChannel is unavailable, so BulkDatabase2 can call these unconditionally. This is an
|
|
6
|
+
// optional feature, so there is no fallback.
|
|
8
7
|
|
|
9
8
|
export type RemoteWrite = { key: string; time: number; deleted?: boolean; value?: unknown };
|
|
10
9
|
|
|
@@ -24,7 +23,11 @@ type Channel = {
|
|
|
24
23
|
const channels = new Map<string, Channel>();
|
|
25
24
|
|
|
26
25
|
export function isSyncSupported(): boolean {
|
|
27
|
-
|
|
26
|
+
// window (main-thread browser) or WorkerGlobalScope (Web/Service/Shared worker) — but not Node
|
|
27
|
+
// (typesafecss's isNode is `typeof window === "undefined"`, which misclassifies workers)
|
|
28
|
+
let isBrowserOrWorker = typeof window !== "undefined"
|
|
29
|
+
|| "WorkerGlobalScope" in globalThis;
|
|
30
|
+
return isBrowserOrWorker && typeof BroadcastChannel !== "undefined";
|
|
28
31
|
}
|
|
29
32
|
|
|
30
33
|
function pruneRecent(channel: Channel) {
|
|
@@ -12,6 +12,7 @@ export declare class DiskCollection<T> implements IStorageSync<T> {
|
|
|
12
12
|
writeDelay?: number | undefined;
|
|
13
13
|
cbor?: boolean | undefined;
|
|
14
14
|
noPrompt?: boolean | undefined;
|
|
15
|
+
resyncFromDisk?: boolean | undefined;
|
|
15
16
|
freeze?: "deep" | "shallow" | undefined;
|
|
16
17
|
beforeWrite?: ((update: {
|
|
17
18
|
newValue: T;
|
|
@@ -34,6 +34,8 @@ export class DiskCollection<T> implements IStorageSync<T> {
|
|
|
34
34
|
writeDelay?: number;
|
|
35
35
|
cbor?: boolean;
|
|
36
36
|
noPrompt?: boolean;
|
|
37
|
+
// Poll disk and reload when another process/tab writes to this collection. Off by default.
|
|
38
|
+
resyncFromDisk?: boolean;
|
|
37
39
|
freeze?: "shallow" | "deep";
|
|
38
40
|
// May mutate newValue in order to change what will be written
|
|
39
41
|
beforeWrite?: (update: { newValue: T; key: string; collection: DiskCollection<T> }) => void;
|
|
@@ -52,7 +54,7 @@ export class DiskCollection<T> implements IStorageSync<T> {
|
|
|
52
54
|
let collections = await fileStorage.folder.getStorage("collections");
|
|
53
55
|
curCollection = await collections.folder.getStorage(this.collectionName);
|
|
54
56
|
}
|
|
55
|
-
let baseStorage = new TransactionStorage(curCollection, this.collectionName, this.config?.writeDelay);
|
|
57
|
+
let baseStorage = new TransactionStorage(curCollection, this.collectionName, this.config?.writeDelay, this.config?.resyncFromDisk);
|
|
56
58
|
this.transactionStorage = baseStorage;
|
|
57
59
|
return this.config?.cbor ? new CBORStorage<T>(baseStorage) : new JSONStorage<T>(baseStorage);
|
|
58
60
|
}
|
|
@@ -66,6 +66,7 @@ export type DirectoryWrapper = {
|
|
|
66
66
|
[Symbol.asyncIterator](): AsyncIterableIterator<[string, FileWrapper | DirectoryWrapper]>;
|
|
67
67
|
};
|
|
68
68
|
export declare function setFileAPIKey(key: string): void;
|
|
69
|
+
export declare function setStorageRootOverride(handle: FileSystemDirectoryHandle | undefined): void;
|
|
69
70
|
export declare class NodeJSFileHandleWrapper implements FileWrapper {
|
|
70
71
|
private filePath;
|
|
71
72
|
constructor(filePath: string);
|
|
@@ -91,6 +91,18 @@ export function setFileAPIKey(key: string) {
|
|
|
91
91
|
fileAPIKey = key;
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
// A directory handle to serve as the storage root instead of resolving one via
|
|
95
|
+
// the picker / stored-pointer / remote flow. Set this in a context that can't run
|
|
96
|
+
// the interactive resolution — most importantly a Web Worker, which has no DOM to
|
|
97
|
+
// prompt with and no localStorage pointer. The owning window resolves the handle
|
|
98
|
+
// (with its permission grant) and postMessages it in; the worker calls this before
|
|
99
|
+
// touching any storage. When set, getDirectoryHandle returns it directly and skips
|
|
100
|
+
// every DOM / localStorage branch below.
|
|
101
|
+
let storageRootOverride: FileSystemDirectoryHandle | undefined;
|
|
102
|
+
export function setStorageRootOverride(handle: FileSystemDirectoryHandle | undefined): void {
|
|
103
|
+
storageRootOverride = handle;
|
|
104
|
+
}
|
|
105
|
+
|
|
94
106
|
// ---- remote (server) storage config ----
|
|
95
107
|
// Instead of a local folder, the user can point at a remoteFileServer.js instance (URL + password).
|
|
96
108
|
// When configured, getFileStorageNested2 serves everything from that server. Persisted in localStorage.
|
|
@@ -442,6 +454,12 @@ export function pickPrivateFolder(name: string): void {
|
|
|
442
454
|
// (we actually connect) before use; if it no longer works we re-prompt, just like a local folder whose
|
|
443
455
|
// permission was lost. Blocks until ready (or the user dismisses, which rejects).
|
|
444
456
|
export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
|
|
457
|
+
// An injected root (e.g. a Web Worker handed the handle by its owning window)
|
|
458
|
+
// short-circuits the whole interactive resolution — no picker, no DOM, no
|
|
459
|
+
// localStorage. Must come first so a worker never reaches the branches below.
|
|
460
|
+
if (storageRootOverride) {
|
|
461
|
+
return storageRootOverride as unknown as DirectoryWrapper;
|
|
462
|
+
}
|
|
445
463
|
if (isNode()) {
|
|
446
464
|
return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
|
|
447
465
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
export type ArchiveFileInfo = {
|
|
4
|
+
path: string;
|
|
5
|
+
createTime: number;
|
|
6
|
+
size: number;
|
|
7
|
+
};
|
|
8
|
+
export interface IArchives {
|
|
9
|
+
getDebugName(): string;
|
|
10
|
+
get(fileName: string, config?: {
|
|
11
|
+
range?: {
|
|
12
|
+
start: number;
|
|
13
|
+
end: number;
|
|
14
|
+
};
|
|
15
|
+
}): Promise<Buffer | undefined>;
|
|
16
|
+
set(fileName: string, data: Buffer): Promise<void>;
|
|
17
|
+
del(fileName: string): Promise<void>;
|
|
18
|
+
/** Streams a file too large to hold in memory. getNextData returns undefined when done. */
|
|
19
|
+
setLargeFile(config: {
|
|
20
|
+
path: string;
|
|
21
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
22
|
+
}): Promise<void>;
|
|
23
|
+
getInfo(fileName: string): Promise<{
|
|
24
|
+
writeTime: number;
|
|
25
|
+
size: number;
|
|
26
|
+
} | undefined>;
|
|
27
|
+
find(prefix: string, config?: {
|
|
28
|
+
shallow?: boolean;
|
|
29
|
+
type: "files" | "folders";
|
|
30
|
+
}): Promise<string[]>;
|
|
31
|
+
findInfo(prefix: string, config?: {
|
|
32
|
+
shallow?: boolean;
|
|
33
|
+
type: "files" | "folders";
|
|
34
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
35
|
+
/** Only works for public buckets (private buckets are API-access only). */
|
|
36
|
+
getURL(path: string): Promise<string>;
|
|
37
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
module.allowclient = true;
|
|
2
|
+
|
|
3
|
+
// The important operations of an archive bucket (extracted from ArchivesBackblaze), so other
|
|
4
|
+
// backends (e.g. our own remote storage server) can be used interchangeably.
|
|
5
|
+
|
|
6
|
+
export type ArchiveFileInfo = { path: string; createTime: number; size: number };
|
|
7
|
+
|
|
8
|
+
export interface IArchives {
|
|
9
|
+
getDebugName(): string;
|
|
10
|
+
get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined>;
|
|
11
|
+
set(fileName: string, data: Buffer): Promise<void>;
|
|
12
|
+
del(fileName: string): Promise<void>;
|
|
13
|
+
/** Streams a file too large to hold in memory. getNextData returns undefined when done. */
|
|
14
|
+
setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void>;
|
|
15
|
+
getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined>;
|
|
16
|
+
find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]>;
|
|
17
|
+
findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]>;
|
|
18
|
+
/** Only works for public buckets (private buckets are API-access only). */
|
|
19
|
+
getURL(path: string): Promise<string>;
|
|
20
|
+
}
|
|
@@ -11,12 +11,13 @@ export declare class TransactionStorage implements IStorage<Buffer> {
|
|
|
11
11
|
private rawStorage;
|
|
12
12
|
private debugName;
|
|
13
13
|
private writeDelay;
|
|
14
|
+
private resyncFromDisk;
|
|
14
15
|
cache: Map<string, TransactionEntry>;
|
|
15
16
|
private diskFiles;
|
|
16
17
|
private currentChunk;
|
|
17
18
|
private entryCount;
|
|
18
19
|
private static allStorage;
|
|
19
|
-
constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number);
|
|
20
|
+
constructor(rawStorage: IStorageRaw, debugName: string, writeDelay?: number, resyncFromDisk?: boolean);
|
|
20
21
|
static compressAll(): Promise<void>;
|
|
21
22
|
private resyncCallbacks;
|
|
22
23
|
watchResync(callback: () => void): void;
|
|
@@ -92,7 +92,11 @@ export class TransactionStorage implements IStorage<Buffer> {
|
|
|
92
92
|
constructor(
|
|
93
93
|
private rawStorage: IStorageRaw,
|
|
94
94
|
private debugName: string,
|
|
95
|
-
private writeDelay = WRITE_DELAY
|
|
95
|
+
private writeDelay = WRITE_DELAY,
|
|
96
|
+
// When set, we poll the underlying storage and reload if another process/tab writes new
|
|
97
|
+
// chunk files. Off by default, as it costs a periodic disk scan and most collections are
|
|
98
|
+
// only written by a single process.
|
|
99
|
+
private resyncFromDisk = false
|
|
96
100
|
) {
|
|
97
101
|
TransactionStorage.allStorage.push(this);
|
|
98
102
|
// VERY useful for debugging.
|
|
@@ -282,7 +286,7 @@ export class TransactionStorage implements IStorage<Buffer> {
|
|
|
282
286
|
private async loadAllTransactions(initialLoad?: boolean): Promise<string[]> {
|
|
283
287
|
if (isInBuild()) return [];
|
|
284
288
|
|
|
285
|
-
if (initialLoad) {
|
|
289
|
+
if (initialLoad && this.resyncFromDisk) {
|
|
286
290
|
runInfinitePoll(DISK_CHECK_INTERVAL, () => this.checkDisk());
|
|
287
291
|
}
|
|
288
292
|
|
package/storage/backblaze.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
|
|
3
|
+
import { IArchives } from "./IArchives";
|
|
4
|
+
export declare class ArchivesBackblaze implements IArchives {
|
|
4
5
|
private config;
|
|
5
6
|
constructor(config: {
|
|
6
7
|
bucketName: string;
|
package/storage/backblaze.ts
CHANGED
|
@@ -9,6 +9,7 @@ import debugbreak from "debugbreak";
|
|
|
9
9
|
import dns from "dns";
|
|
10
10
|
import { getSecret } from "../misc/getSecret";
|
|
11
11
|
import { httpsRequest } from "socket-function/src/https";
|
|
12
|
+
import { IArchives } from "./IArchives";
|
|
12
13
|
|
|
13
14
|
type BackblazeCreds = {
|
|
14
15
|
applicationKeyId: string;
|
|
@@ -372,7 +373,7 @@ const getAPI = lazy(async () => {
|
|
|
372
373
|
type B2Api = (typeof getAPI) extends () => Promise<infer T> ? T : never;
|
|
373
374
|
|
|
374
375
|
|
|
375
|
-
export class ArchivesBackblaze {
|
|
376
|
+
export class ArchivesBackblaze implements IArchives {
|
|
376
377
|
public constructor(private config: {
|
|
377
378
|
bucketName: string;
|
|
378
379
|
public?: boolean;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import { IArchives, ArchiveFileInfo } from "../IArchives";
|
|
4
|
+
export type ArchivesRemoteConfig = {
|
|
5
|
+
address: string;
|
|
6
|
+
port: number;
|
|
7
|
+
account: string;
|
|
8
|
+
bucketName: string;
|
|
9
|
+
public?: boolean;
|
|
10
|
+
fast?: boolean;
|
|
11
|
+
writeDelay?: number;
|
|
12
|
+
};
|
|
13
|
+
export declare function buildPublicFileURL(config: {
|
|
14
|
+
address: string;
|
|
15
|
+
port: number;
|
|
16
|
+
account: string;
|
|
17
|
+
bucketName: string;
|
|
18
|
+
path: string;
|
|
19
|
+
}): string;
|
|
20
|
+
export declare function authenticateStorage(config: {
|
|
21
|
+
address: string;
|
|
22
|
+
port: number;
|
|
23
|
+
nodeId: string;
|
|
24
|
+
}): Promise<{
|
|
25
|
+
machineId: string;
|
|
26
|
+
ip: string;
|
|
27
|
+
}>;
|
|
28
|
+
export declare class ArchivesRemote implements IArchives {
|
|
29
|
+
private config;
|
|
30
|
+
constructor(config: ArchivesRemoteConfig);
|
|
31
|
+
private nodeId;
|
|
32
|
+
private controller;
|
|
33
|
+
private setupDone;
|
|
34
|
+
private lastDeniedLog;
|
|
35
|
+
getDebugName(): string;
|
|
36
|
+
private authenticate;
|
|
37
|
+
private callAuthed;
|
|
38
|
+
waitingForAccess(): Promise<string | undefined>;
|
|
39
|
+
private onAccessDenied;
|
|
40
|
+
private ensureSetup;
|
|
41
|
+
private call;
|
|
42
|
+
get(fileName: string, config?: {
|
|
43
|
+
range?: {
|
|
44
|
+
start: number;
|
|
45
|
+
end: number;
|
|
46
|
+
};
|
|
47
|
+
}): Promise<Buffer | undefined>;
|
|
48
|
+
set(fileName: string, data: Buffer): Promise<void>;
|
|
49
|
+
del(fileName: string): Promise<void>;
|
|
50
|
+
getInfo(fileName: string): Promise<{
|
|
51
|
+
writeTime: number;
|
|
52
|
+
size: number;
|
|
53
|
+
} | undefined>;
|
|
54
|
+
findInfo(prefix: string, config?: {
|
|
55
|
+
shallow?: boolean;
|
|
56
|
+
type: "files" | "folders";
|
|
57
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
58
|
+
find(prefix: string, config?: {
|
|
59
|
+
shallow?: boolean;
|
|
60
|
+
type: "files" | "folders";
|
|
61
|
+
}): Promise<string[]>;
|
|
62
|
+
setLargeFile(config: {
|
|
63
|
+
path: string;
|
|
64
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
65
|
+
}): Promise<void>;
|
|
66
|
+
getURL(path: string): Promise<string>;
|
|
67
|
+
}
|