sliftutils 1.4.10 → 1.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +10 -52
- package/package.json +1 -1
- package/storage/FileFolderAPI.d.ts +4 -19
- package/storage/FileFolderAPI.tsx +69 -57
- package/storage/remoteFileServer.ts +21 -17
- package/storage/remoteFileStorage.d.ts +6 -33
- package/storage/remoteFileStorage.ts +189 -175
package/index.d.ts
CHANGED
|
@@ -1315,6 +1315,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1315
1315
|
/// <reference types="node" />
|
|
1316
1316
|
/// <reference types="node" />
|
|
1317
1317
|
import { IStorageRaw } from "./IStorage";
|
|
1318
|
+
import { RemoteOptions } from "./remoteFileStorage";
|
|
1318
1319
|
declare global {
|
|
1319
1320
|
interface Window {
|
|
1320
1321
|
showSaveFilePicker(config?: {
|
|
@@ -1341,7 +1342,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1341
1342
|
}): Promise<PermissionState>;
|
|
1342
1343
|
}
|
|
1343
1344
|
}
|
|
1344
|
-
type FileWrapper = {
|
|
1345
|
+
export type FileWrapper = {
|
|
1345
1346
|
getFile(): Promise<{
|
|
1346
1347
|
size: number;
|
|
1347
1348
|
lastModified: number;
|
|
@@ -1358,7 +1359,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1358
1359
|
close(): Promise<void>;
|
|
1359
1360
|
}>;
|
|
1360
1361
|
};
|
|
1361
|
-
type DirectoryWrapper = {
|
|
1362
|
+
export type DirectoryWrapper = {
|
|
1362
1363
|
removeEntry(key: string, options?: {
|
|
1363
1364
|
recursive?: boolean;
|
|
1364
1365
|
}): Promise<void>;
|
|
@@ -1384,10 +1385,6 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1384
1385
|
]>;
|
|
1385
1386
|
};
|
|
1386
1387
|
export declare function setFileAPIKey(key: string): void;
|
|
1387
|
-
type RemoteConfig = {
|
|
1388
|
-
url: string;
|
|
1389
|
-
password: string;
|
|
1390
|
-
};
|
|
1391
1388
|
export declare class NodeJSFileHandleWrapper implements FileWrapper {
|
|
1392
1389
|
private filePath;
|
|
1393
1390
|
constructor(filePath: string);
|
|
@@ -1434,18 +1431,6 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1434
1431
|
}
|
|
1435
1432
|
]>;
|
|
1436
1433
|
}
|
|
1437
|
-
type StorageChoice = {
|
|
1438
|
-
type: "local";
|
|
1439
|
-
handle: DirectoryWrapper;
|
|
1440
|
-
} | {
|
|
1441
|
-
type: "remote";
|
|
1442
|
-
config: RemoteConfig;
|
|
1443
|
-
};
|
|
1444
|
-
export declare const getStorageChoice: {
|
|
1445
|
-
(): Promise<StorageChoice>;
|
|
1446
|
-
reset(): void;
|
|
1447
|
-
set(newValue: Promise<StorageChoice>): void;
|
|
1448
|
-
};
|
|
1449
1434
|
export declare const getDirectoryHandle: {
|
|
1450
1435
|
(): Promise<DirectoryWrapper>;
|
|
1451
1436
|
reset(): void;
|
|
@@ -1483,8 +1468,8 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1483
1468
|
folder: NestedFileStorage;
|
|
1484
1469
|
};
|
|
1485
1470
|
export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
|
|
1471
|
+
export declare function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage>;
|
|
1486
1472
|
export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
|
|
1487
|
-
export {};
|
|
1488
1473
|
|
|
1489
1474
|
}
|
|
1490
1475
|
|
|
@@ -1877,44 +1862,18 @@ declare module "sliftutils/storage/remoteFileServer" {
|
|
|
1877
1862
|
}
|
|
1878
1863
|
|
|
1879
1864
|
declare module "sliftutils/storage/remoteFileStorage" {
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
/// <reference types="node" />
|
|
1883
|
-
import https from "https";
|
|
1884
|
-
import type { FileStorage } from "./FileFolderAPI";
|
|
1885
|
-
export type RemoteFileStorageOptions = {
|
|
1865
|
+
import type { DirectoryWrapper } from "./FileFolderAPI";
|
|
1866
|
+
export type RemoteOptions = {
|
|
1886
1867
|
chunkBytes?: number;
|
|
1887
1868
|
cacheBytes?: number;
|
|
1888
1869
|
latencyMs?: number;
|
|
1889
|
-
|
|
1890
|
-
type Connection = {
|
|
1891
|
-
url: string;
|
|
1892
|
-
password: string;
|
|
1893
|
-
latencyMs: number;
|
|
1894
|
-
agent: https.Agent | undefined;
|
|
1895
|
-
cache: RangeCache;
|
|
1896
|
-
stats: {
|
|
1870
|
+
stats?: {
|
|
1897
1871
|
requestCount: number;
|
|
1898
1872
|
bytesFetched: number;
|
|
1899
1873
|
};
|
|
1900
1874
|
};
|
|
1901
|
-
declare
|
|
1902
|
-
|
|
1903
|
-
private budget;
|
|
1904
|
-
private chunks;
|
|
1905
|
-
private bytes;
|
|
1906
|
-
constructor(chunkBytes: number, budget: number);
|
|
1907
|
-
private key;
|
|
1908
|
-
private peek;
|
|
1909
|
-
private store;
|
|
1910
|
-
invalidate(path: string): void;
|
|
1911
|
-
getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined>;
|
|
1912
|
-
}
|
|
1913
|
-
export type RemoteStorageFactory = ((path: string) => Promise<FileStorage>) & {
|
|
1914
|
-
stats: Connection["stats"];
|
|
1915
|
-
};
|
|
1916
|
-
export declare function getRemoteFileStorage(url: string, password: string, options?: RemoteFileStorageOptions): RemoteStorageFactory;
|
|
1917
|
-
export type RemoteProbeResult = {
|
|
1875
|
+
export declare function getRemoteDirectoryHandle(url: string, password: string, options?: RemoteOptions): DirectoryWrapper;
|
|
1876
|
+
export type RemoteConnectResult = {
|
|
1918
1877
|
status: "ok";
|
|
1919
1878
|
} | {
|
|
1920
1879
|
status: "unauthorized";
|
|
@@ -1922,8 +1881,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
|
|
|
1922
1881
|
status: "unreachable";
|
|
1923
1882
|
error: string;
|
|
1924
1883
|
};
|
|
1925
|
-
export declare function
|
|
1926
|
-
export {};
|
|
1884
|
+
export declare function testRemoteConnection(url: string, password: string, options?: RemoteOptions): Promise<RemoteConnectResult>;
|
|
1927
1885
|
|
|
1928
1886
|
}
|
|
1929
1887
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
3
|
import { IStorageRaw } from "./IStorage";
|
|
4
|
+
import { RemoteOptions } from "./remoteFileStorage";
|
|
4
5
|
declare global {
|
|
5
6
|
interface Window {
|
|
6
7
|
showSaveFilePicker(config?: {
|
|
@@ -27,7 +28,7 @@ declare global {
|
|
|
27
28
|
}): Promise<PermissionState>;
|
|
28
29
|
}
|
|
29
30
|
}
|
|
30
|
-
type FileWrapper = {
|
|
31
|
+
export type FileWrapper = {
|
|
31
32
|
getFile(): Promise<{
|
|
32
33
|
size: number;
|
|
33
34
|
lastModified: number;
|
|
@@ -44,7 +45,7 @@ type FileWrapper = {
|
|
|
44
45
|
close(): Promise<void>;
|
|
45
46
|
}>;
|
|
46
47
|
};
|
|
47
|
-
type DirectoryWrapper = {
|
|
48
|
+
export type DirectoryWrapper = {
|
|
48
49
|
removeEntry(key: string, options?: {
|
|
49
50
|
recursive?: boolean;
|
|
50
51
|
}): Promise<void>;
|
|
@@ -70,10 +71,6 @@ type DirectoryWrapper = {
|
|
|
70
71
|
]>;
|
|
71
72
|
};
|
|
72
73
|
export declare function setFileAPIKey(key: string): void;
|
|
73
|
-
type RemoteConfig = {
|
|
74
|
-
url: string;
|
|
75
|
-
password: string;
|
|
76
|
-
};
|
|
77
74
|
export declare class NodeJSFileHandleWrapper implements FileWrapper {
|
|
78
75
|
private filePath;
|
|
79
76
|
constructor(filePath: string);
|
|
@@ -120,18 +117,6 @@ export declare class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
|
|
|
120
117
|
}
|
|
121
118
|
]>;
|
|
122
119
|
}
|
|
123
|
-
type StorageChoice = {
|
|
124
|
-
type: "local";
|
|
125
|
-
handle: DirectoryWrapper;
|
|
126
|
-
} | {
|
|
127
|
-
type: "remote";
|
|
128
|
-
config: RemoteConfig;
|
|
129
|
-
};
|
|
130
|
-
export declare const getStorageChoice: {
|
|
131
|
-
(): Promise<StorageChoice>;
|
|
132
|
-
reset(): void;
|
|
133
|
-
set(newValue: Promise<StorageChoice>): void;
|
|
134
|
-
};
|
|
135
120
|
export declare const getDirectoryHandle: {
|
|
136
121
|
(): Promise<DirectoryWrapper>;
|
|
137
122
|
reset(): void;
|
|
@@ -169,5 +154,5 @@ export type FileStorage = IStorageRaw & {
|
|
|
169
154
|
folder: NestedFileStorage;
|
|
170
155
|
};
|
|
171
156
|
export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
|
|
157
|
+
export declare function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage>;
|
|
172
158
|
export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
|
|
173
|
-
export {};
|
|
@@ -7,7 +7,7 @@ import { css, isNode } from "typesafecss";
|
|
|
7
7
|
import { IStorageRaw } from "./IStorage";
|
|
8
8
|
import { runInSerial } from "socket-function/src/batching";
|
|
9
9
|
import { getFileStorageIndexDB } from "./IndexedDBFileFolderAPI";
|
|
10
|
-
import {
|
|
10
|
+
import { getRemoteDirectoryHandle, testRemoteConnection, RemoteOptions } from "./remoteFileStorage";
|
|
11
11
|
import fs from "fs";
|
|
12
12
|
import path from "path";
|
|
13
13
|
|
|
@@ -36,7 +36,7 @@ declare global {
|
|
|
36
36
|
// DO NOT enable this is isNode
|
|
37
37
|
const USE_INDEXED_DB = false;
|
|
38
38
|
|
|
39
|
-
type FileWrapper = {
|
|
39
|
+
export type FileWrapper = {
|
|
40
40
|
getFile(): Promise<{
|
|
41
41
|
size: number;
|
|
42
42
|
lastModified: number;
|
|
@@ -51,7 +51,7 @@ type FileWrapper = {
|
|
|
51
51
|
close(): Promise<void>;
|
|
52
52
|
}>;
|
|
53
53
|
};
|
|
54
|
-
type DirectoryWrapper = {
|
|
54
|
+
export type DirectoryWrapper = {
|
|
55
55
|
removeEntry(key: string, options?: { recursive?: boolean }): Promise<void>;
|
|
56
56
|
getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper>;
|
|
57
57
|
getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
|
|
@@ -105,14 +105,14 @@ function normalizeServerUrl(raw: string): string {
|
|
|
105
105
|
return "https://" + s;
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
-
// One shared
|
|
109
|
-
let
|
|
110
|
-
function
|
|
108
|
+
// One shared remote DirectoryWrapper (and therefore one shared range cache) per remote config.
|
|
109
|
+
let remoteHandle: { key: string; handle: DirectoryWrapper } | undefined;
|
|
110
|
+
function getRemoteHandle(remote: RemoteConfig): DirectoryWrapper {
|
|
111
111
|
const key = remote.url + "\0" + remote.password;
|
|
112
|
-
if (!
|
|
113
|
-
|
|
112
|
+
if (!remoteHandle || remoteHandle.key !== key) {
|
|
113
|
+
remoteHandle = { key, handle: getRemoteDirectoryHandle(remote.url, remote.password) };
|
|
114
114
|
}
|
|
115
|
-
return
|
|
115
|
+
return remoteHandle.handle;
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
@observer
|
|
@@ -134,12 +134,18 @@ class DirectoryPrompter extends preact.Component {
|
|
|
134
134
|
}
|
|
135
135
|
|
|
136
136
|
// "Connect to a server" option for the directory prompt: collapses to a button, expands to address +
|
|
137
|
-
// password fields. On connect it
|
|
138
|
-
//
|
|
139
|
-
//
|
|
137
|
+
// password fields. On connect it ACTUALLY connects (testRemoteConnection); only on success does it
|
|
138
|
+
// persist the config and call onConnected, so the caller (getDirectoryHandle) resolves with a working
|
|
139
|
+
// remote handle. Failures are shown to the user (and logged, without the password), never swallowed.
|
|
140
|
+
// `initial` pre-fills + expands the form (used to retry a remembered server that stopped working).
|
|
140
141
|
@observer
|
|
141
|
-
class ServerConnectForm extends preact.Component<{ onConnected: (config: RemoteConfig) => void }> {
|
|
142
|
-
private obs = observable({
|
|
142
|
+
class ServerConnectForm extends preact.Component<{ onConnected: (config: RemoteConfig) => void; initial?: RemoteConfig }> {
|
|
143
|
+
private obs = observable({
|
|
144
|
+
expanded: !!this.props.initial,
|
|
145
|
+
url: this.props.initial?.url || "",
|
|
146
|
+
password: this.props.initial?.password || "",
|
|
147
|
+
error: "", connecting: false, needsCert: false, showPassword: false,
|
|
148
|
+
});
|
|
143
149
|
private cleanUrl() { return normalizeServerUrl(this.obs.url); }
|
|
144
150
|
private connect = async () => {
|
|
145
151
|
const s = this.obs;
|
|
@@ -149,11 +155,11 @@ class ServerConnectForm extends preact.Component<{ onConnected: (config: RemoteC
|
|
|
149
155
|
try {
|
|
150
156
|
const url = this.cleanUrl();
|
|
151
157
|
if (!url) { s.error = "Enter a server address."; return; }
|
|
152
|
-
const result = await
|
|
158
|
+
const result = await testRemoteConnection(url, s.password.trim());
|
|
153
159
|
if (result.status === "ok") {
|
|
154
160
|
const config: RemoteConfig = { url, password: s.password.trim() };
|
|
155
161
|
saveRemoteConfig(config); // remember for next session
|
|
156
|
-
this.props.onConnected(config); //
|
|
162
|
+
this.props.onConnected(config); // hand the verified server back to getDirectoryHandle
|
|
157
163
|
return;
|
|
158
164
|
}
|
|
159
165
|
if (result.status === "unauthorized") {
|
|
@@ -167,7 +173,6 @@ class ServerConnectForm extends preact.Component<{ onConnected: (config: RemoteC
|
|
|
167
173
|
console.error("Remote connect: unreachable", url, "-", result.error);
|
|
168
174
|
}
|
|
169
175
|
} catch (e) {
|
|
170
|
-
// Should be rare (probeRemoteConnection handles its own errors), but never swallow it.
|
|
171
176
|
s.error = "Connection error: " + ((e as Error).message || String(e));
|
|
172
177
|
console.error("Remote connect threw:", e);
|
|
173
178
|
} finally {
|
|
@@ -363,27 +368,30 @@ export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
|
|
|
363
368
|
}
|
|
364
369
|
|
|
365
370
|
|
|
366
|
-
//
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
//
|
|
370
|
-
|
|
371
|
-
// actually connects to the server and resolves with its config — so when this returns, the storage is
|
|
372
|
-
// ready. No page reload. Blocks until the user chooses (or dismisses, which rejects).
|
|
373
|
-
export const getStorageChoice = lazy(async function getStorageChoice(): Promise<StorageChoice> {
|
|
371
|
+
// Returns the directory handle to use — local (Node / picked folder) OR a remote server, both as the
|
|
372
|
+
// same DirectoryWrapper, so callers don't know or care which it is. A remembered server is VERIFIED
|
|
373
|
+
// (we actually connect) before use; if it no longer works we re-prompt, just like a local folder whose
|
|
374
|
+
// permission was lost. Blocks until ready (or the user dismisses, which rejects).
|
|
375
|
+
export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
|
|
374
376
|
if (isNode()) {
|
|
375
|
-
return
|
|
377
|
+
return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
|
|
376
378
|
}
|
|
379
|
+
|
|
380
|
+
// A server connected in a previous session: only reuse it if it still actually works.
|
|
377
381
|
const savedRemote = loadRemoteConfig();
|
|
378
|
-
if (savedRemote)
|
|
382
|
+
if (savedRemote) {
|
|
383
|
+
const result = await testRemoteConnection(savedRemote.url, savedRemote.password);
|
|
384
|
+
if (result.status === "ok") return getRemoteHandle(savedRemote);
|
|
385
|
+
console.warn(`Saved remote server didn't connect (${result.status})${result.status === "unreachable" ? ": " + result.error : ""} — re-prompting.`);
|
|
386
|
+
}
|
|
379
387
|
|
|
380
388
|
let root = document.createElement("div");
|
|
381
389
|
document.body.appendChild(root);
|
|
382
390
|
preact.render(<DirectoryPrompter />, root);
|
|
383
391
|
try {
|
|
384
|
-
let
|
|
385
|
-
let
|
|
386
|
-
const
|
|
392
|
+
let resolveHandle!: (h: DirectoryWrapper) => void;
|
|
393
|
+
let rejectHandle!: (e: Error) => void;
|
|
394
|
+
const promise = new Promise<DirectoryWrapper>((res, rej) => { resolveHandle = res; rejectHandle = rej; });
|
|
387
395
|
|
|
388
396
|
const pickLocal = async () => {
|
|
389
397
|
try {
|
|
@@ -391,30 +399,33 @@ export const getStorageChoice = lazy(async function getStorageChoice(): Promise<
|
|
|
391
399
|
await handle.requestPermission({ mode: "readwrite" });
|
|
392
400
|
const storedId = await storeFileSystemPointer({ mode: "readwrite", handle });
|
|
393
401
|
localStorage.setItem(getFileAPIKey(), storedId);
|
|
394
|
-
|
|
402
|
+
resolveHandle(handle as any);
|
|
395
403
|
} catch (e) {
|
|
396
404
|
console.error("Directory pick failed/cancelled:", e); // stay on the prompt
|
|
397
405
|
}
|
|
398
406
|
};
|
|
399
|
-
|
|
407
|
+
const onConnected = (config: RemoteConfig) => { saveRemoteConfig(config); resolveHandle(getRemoteHandle(config)); };
|
|
408
|
+
// The three options, rendered fresh each time. If a saved server just failed, pre-fill + expand
|
|
409
|
+
// the connect form so the user can retry or fix it.
|
|
400
410
|
const renderOptions = () => (
|
|
401
411
|
<>
|
|
402
412
|
<button className={css.fontSize(40).pad2(80, 40)} onClick={pickLocal}>Pick Data Directory</button>
|
|
403
|
-
<ServerConnectForm onConnected={
|
|
413
|
+
<ServerConnectForm onConnected={onConnected} initial={savedRemote} />
|
|
404
414
|
<button className={css.fontSize(40).pad2(80, 40)}
|
|
405
|
-
onClick={() =>
|
|
415
|
+
onClick={() => rejectHandle(new Error("User dismissed file system access prompt"))}>Dismiss</button>
|
|
406
416
|
</>
|
|
407
417
|
);
|
|
408
418
|
|
|
409
|
-
// A previously-picked local folder: try to restore
|
|
410
|
-
|
|
419
|
+
// A previously-picked local folder: try to restore it (may need a click). Skipped when a saved
|
|
420
|
+
// server failed — that user wants the server, so go straight to the (pre-filled) prompt.
|
|
421
|
+
const storedId = !savedRemote && localStorage.getItem(getFileAPIKey());
|
|
411
422
|
if (storedId) {
|
|
412
423
|
let doneLoad = false;
|
|
413
424
|
setTimeout(() => { if (!doneLoad) displayData.ui = "Click anywhere to allow file system access"; }, 500);
|
|
414
425
|
try {
|
|
415
426
|
const handle = await tryToLoadPointer(storedId);
|
|
416
427
|
doneLoad = true;
|
|
417
|
-
if (handle) return
|
|
428
|
+
if (handle) return handle;
|
|
418
429
|
} catch (e) {
|
|
419
430
|
doneLoad = true;
|
|
420
431
|
console.error(e);
|
|
@@ -426,37 +437,27 @@ export const getStorageChoice = lazy(async function getStorageChoice(): Promise<
|
|
|
426
437
|
displayData.ui = "Loading...";
|
|
427
438
|
try {
|
|
428
439
|
const h = await tryToLoadPointer(storedId);
|
|
429
|
-
if (h) {
|
|
440
|
+
if (h) { resolveHandle(h); return; }
|
|
430
441
|
} catch (retryError) { console.error("Retry failed:", retryError); }
|
|
431
442
|
displayData.ui = <div className={css.vbox(20).center}>{renderOptions()}</div>;
|
|
432
443
|
}}>Click to restore file system access</button>
|
|
433
444
|
{renderOptions()}
|
|
434
445
|
</div>
|
|
435
446
|
);
|
|
436
|
-
return await
|
|
447
|
+
return await promise;
|
|
437
448
|
}
|
|
438
449
|
}
|
|
439
450
|
}
|
|
440
451
|
displayData.ui = <div className={css.vbox(20).center}>{renderOptions()}</div>;
|
|
441
|
-
return await
|
|
452
|
+
return await promise;
|
|
442
453
|
} finally {
|
|
443
454
|
preact.render(null, root);
|
|
444
455
|
root.remove();
|
|
445
456
|
}
|
|
446
457
|
});
|
|
447
458
|
|
|
448
|
-
// Local-only directory handle, for callers that don't support remote storage. Throws if the user has
|
|
449
|
-
// configured a remote server (rather than silently doing the wrong thing).
|
|
450
|
-
export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
|
|
451
|
-
const choice = await getStorageChoice();
|
|
452
|
-
if (choice.type === "remote") throw new Error("Storage is configured to use a remote server; a local directory handle isn't available here.");
|
|
453
|
-
return choice.handle;
|
|
454
|
-
});
|
|
455
|
-
|
|
456
459
|
export const getFileStorageNested = cache(async function getFileStorage(path: string): Promise<FileStorage> {
|
|
457
|
-
|
|
458
|
-
if (choice.type === "remote") return getRemoteFactory(choice.config)(path);
|
|
459
|
-
let base = choice.handle;
|
|
460
|
+
let base = await getDirectoryHandle();
|
|
460
461
|
for (let part of path.split("/")) {
|
|
461
462
|
if (!part) continue;
|
|
462
463
|
base = await base.getDirectoryHandle(part, { create: true });
|
|
@@ -473,9 +474,7 @@ export const getFileStorageNested2 = cache(async function getFileStorage(pathStr
|
|
|
473
474
|
}
|
|
474
475
|
base = new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
|
|
475
476
|
} else {
|
|
476
|
-
|
|
477
|
-
if (choice.type === "remote") return getRemoteFactory(choice.config)(pathStr);
|
|
478
|
-
base = choice.handle;
|
|
477
|
+
base = await getDirectoryHandle();
|
|
479
478
|
let dirs: string[] = [];
|
|
480
479
|
let alls: string[] = [];
|
|
481
480
|
for await (const [name, entry] of base) {
|
|
@@ -499,9 +498,8 @@ export const getFileStorage = lazy(async function getFileStorage(): Promise<File
|
|
|
499
498
|
if (USE_INDEXED_DB) {
|
|
500
499
|
return await getFileStorageIndexDB();
|
|
501
500
|
}
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
return wrapHandle(choice.handle);
|
|
501
|
+
let handle = await getDirectoryHandle();
|
|
502
|
+
return wrapHandle(handle);
|
|
505
503
|
});
|
|
506
504
|
export function resetStorageLocation() {
|
|
507
505
|
localStorage.removeItem(getFileAPIKey());
|
|
@@ -715,6 +713,20 @@ export function wrapHandle(handle: DirectoryWrapper): FileStorage {
|
|
|
715
713
|
};
|
|
716
714
|
}
|
|
717
715
|
|
|
716
|
+
// A StorageFactory backed by a remote server (path -> FileStorage), for code that injects its own
|
|
717
|
+
// storage into BulkDatabase2 rather than going through the directory prompt.
|
|
718
|
+
export function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage> {
|
|
719
|
+
const root = getRemoteDirectoryHandle(url, password, options);
|
|
720
|
+
return async (pathStr: string) => {
|
|
721
|
+
let base: DirectoryWrapper = root;
|
|
722
|
+
for (const part of pathStr.replaceAll("\\", "/").split("/")) {
|
|
723
|
+
if (!part) continue;
|
|
724
|
+
base = await base.getDirectoryHandle(part, { create: true });
|
|
725
|
+
}
|
|
726
|
+
return wrapHandle(base);
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
718
730
|
export async function tryToLoadPointer(pointer: string) {
|
|
719
731
|
let result = await getFileSystemPointer({ pointer });
|
|
720
732
|
if (!result) return;
|
|
@@ -203,6 +203,22 @@ function normalizePassword(p: string): string {
|
|
|
203
203
|
return String(p || "").toLowerCase().replace(/[^a-z]/g, "");
|
|
204
204
|
}
|
|
205
205
|
|
|
206
|
+
// The password is generated once and saved (next to the cert, outside the served folder), then reused
|
|
207
|
+
// across restarts — so connected clients keep working after the server bounces, instead of getting a
|
|
208
|
+
// new password every time. Override with the PASSWORD env var.
|
|
209
|
+
function getOrCreatePassword(): string {
|
|
210
|
+
const dir = path.join(os.homedir(), ".sliftutils-remote");
|
|
211
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
212
|
+
const file = path.join(dir, "password.txt");
|
|
213
|
+
try {
|
|
214
|
+
const existing = fs.readFileSync(file, "utf8").trim();
|
|
215
|
+
if (existing) return existing;
|
|
216
|
+
} catch { /* not created yet */ }
|
|
217
|
+
const pw = generatePassword(6);
|
|
218
|
+
fs.writeFileSync(file, pw + "\n", { mode: 0o600 });
|
|
219
|
+
return pw;
|
|
220
|
+
}
|
|
221
|
+
|
|
206
222
|
// Generate (once) and cache a self-signed key + cert via openssl, outside the served folder so its
|
|
207
223
|
// private key is never reachable through the API.
|
|
208
224
|
function getCert(): { key: Buffer; cert: Buffer } {
|
|
@@ -278,7 +294,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
278
294
|
if (!fs.existsSync(root)) fs.mkdirSync(root, { recursive: true });
|
|
279
295
|
const port = options.port ?? 8787;
|
|
280
296
|
const host = options.host || "0.0.0.0";
|
|
281
|
-
const password = options.password ||
|
|
297
|
+
const password = options.password || getOrCreatePassword();
|
|
282
298
|
const normPassword = normalizePassword(password);
|
|
283
299
|
const { key, cert } = getCert();
|
|
284
300
|
|
|
@@ -340,21 +356,17 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
340
356
|
const full = safeResolve(root, relPath);
|
|
341
357
|
if (full === undefined) return sendJson(403, { error: "path escapes root" });
|
|
342
358
|
|
|
359
|
+
// One listing returns every entry with its type, so a directory read is a single round trip.
|
|
343
360
|
if (req.method === "GET" && op === "/list") {
|
|
344
|
-
const wantFolders = url.searchParams.get("folders") === "1";
|
|
345
361
|
let entries: fs.Dirent[];
|
|
346
362
|
try { entries = fs.readdirSync(full, { withFileTypes: true }); }
|
|
347
363
|
catch (e) { return (e as NodeJS.ErrnoException).code === "ENOENT" ? sendJson(200, []) : sendJson(500, { error: (e as Error).message }); }
|
|
348
|
-
return sendJson(200, entries.filter(d =>
|
|
364
|
+
return sendJson(200, entries.filter(d => d.isFile() || d.isDirectory()).map(d => ({ name: d.name, dir: d.isDirectory() })));
|
|
349
365
|
}
|
|
350
366
|
if (req.method === "GET" && op === "/info") {
|
|
351
367
|
let st: fs.Stats;
|
|
352
368
|
try { st = fs.statSync(full); } catch { return sendJson(404, { error: "not found" }); }
|
|
353
|
-
return sendJson(200, { size: st.size, lastModified: st.mtimeMs });
|
|
354
|
-
}
|
|
355
|
-
if (req.method === "GET" && op === "/hasDir") {
|
|
356
|
-
let st: fs.Stats; try { st = fs.statSync(full); } catch { return sendJson(200, { exists: false }); }
|
|
357
|
-
return sendJson(200, { exists: st.isDirectory() });
|
|
369
|
+
return sendJson(200, { size: st.size, lastModified: st.mtimeMs, dir: st.isDirectory() });
|
|
358
370
|
}
|
|
359
371
|
if (req.method === "GET" && op === "/read") {
|
|
360
372
|
let st: fs.Stats;
|
|
@@ -377,19 +389,11 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
377
389
|
recordAccess(clientIp(req), "write", relPath, body.length);
|
|
378
390
|
return sendJson(200, { ok: true });
|
|
379
391
|
}
|
|
392
|
+
// Removes a file or a directory (recursively); missing is fine.
|
|
380
393
|
if (req.method === "DELETE" && op === "/remove") {
|
|
381
|
-
try { fs.unlinkSync(full); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") return sendJson(500, { error: (e as Error).message }); }
|
|
382
|
-
return sendJson(200, { ok: true });
|
|
383
|
-
}
|
|
384
|
-
if (req.method === "DELETE" && op === "/removeDir") {
|
|
385
394
|
try { fs.rmSync(full, { recursive: true, force: true }); } catch (e) { return sendJson(500, { error: (e as Error).message }); }
|
|
386
395
|
return sendJson(200, { ok: true });
|
|
387
396
|
}
|
|
388
|
-
if (req.method === "POST" && op === "/reset") {
|
|
389
|
-
try { for (const name of fs.readdirSync(full)) fs.rmSync(path.join(full, name), { recursive: true, force: true }); }
|
|
390
|
-
catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") return sendJson(500, { error: (e as Error).message }); }
|
|
391
|
-
return sendJson(200, { ok: true });
|
|
392
|
-
}
|
|
393
397
|
return sendJson(404, { error: "unknown endpoint" });
|
|
394
398
|
} catch (e) {
|
|
395
399
|
try { sendJson(500, { error: String((e as Error)?.message || e) }); } catch { /* response already sent */ }
|
|
@@ -1,41 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
/// <reference types="node" />
|
|
4
|
-
import https from "https";
|
|
5
|
-
import type { FileStorage } from "./FileFolderAPI";
|
|
6
|
-
export type RemoteFileStorageOptions = {
|
|
1
|
+
import type { DirectoryWrapper } from "./FileFolderAPI";
|
|
2
|
+
export type RemoteOptions = {
|
|
7
3
|
chunkBytes?: number;
|
|
8
4
|
cacheBytes?: number;
|
|
9
5
|
latencyMs?: number;
|
|
10
|
-
|
|
11
|
-
type Connection = {
|
|
12
|
-
url: string;
|
|
13
|
-
password: string;
|
|
14
|
-
latencyMs: number;
|
|
15
|
-
agent: https.Agent | undefined;
|
|
16
|
-
cache: RangeCache;
|
|
17
|
-
stats: {
|
|
6
|
+
stats?: {
|
|
18
7
|
requestCount: number;
|
|
19
8
|
bytesFetched: number;
|
|
20
9
|
};
|
|
21
10
|
};
|
|
22
|
-
declare
|
|
23
|
-
|
|
24
|
-
private budget;
|
|
25
|
-
private chunks;
|
|
26
|
-
private bytes;
|
|
27
|
-
constructor(chunkBytes: number, budget: number);
|
|
28
|
-
private key;
|
|
29
|
-
private peek;
|
|
30
|
-
private store;
|
|
31
|
-
invalidate(path: string): void;
|
|
32
|
-
getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined>;
|
|
33
|
-
}
|
|
34
|
-
export type RemoteStorageFactory = ((path: string) => Promise<FileStorage>) & {
|
|
35
|
-
stats: Connection["stats"];
|
|
36
|
-
};
|
|
37
|
-
export declare function getRemoteFileStorage(url: string, password: string, options?: RemoteFileStorageOptions): RemoteStorageFactory;
|
|
38
|
-
export type RemoteProbeResult = {
|
|
11
|
+
export declare function getRemoteDirectoryHandle(url: string, password: string, options?: RemoteOptions): DirectoryWrapper;
|
|
12
|
+
export type RemoteConnectResult = {
|
|
39
13
|
status: "ok";
|
|
40
14
|
} | {
|
|
41
15
|
status: "unauthorized";
|
|
@@ -43,5 +17,4 @@ export type RemoteProbeResult = {
|
|
|
43
17
|
status: "unreachable";
|
|
44
18
|
error: string;
|
|
45
19
|
};
|
|
46
|
-
export declare function
|
|
47
|
-
export {};
|
|
20
|
+
export declare function testRemoteConnection(url: string, password: string, options?: RemoteOptions): Promise<RemoteConnectResult>;
|
|
@@ -1,99 +1,140 @@
|
|
|
1
1
|
import { isNode } from "typesafecss";
|
|
2
2
|
import https from "https";
|
|
3
|
-
import type {
|
|
3
|
+
import type { DirectoryWrapper, FileWrapper } from "./FileFolderAPI";
|
|
4
4
|
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
5
|
+
// A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the
|
|
6
|
+
// Node.js and File-System-Access handles. So getDirectoryHandle can return one of these and everything
|
|
7
|
+
// downstream (wrapHandle, navigation, BulkDatabase2) works unchanged; nothing else knows it's remote.
|
|
8
|
+
//
|
|
9
|
+
// Files here are immutable (written once) or append-only, so the range cache (raw compressed bytes, in
|
|
10
|
+
// ~1MB aligned chunks) is always valid — over the network latency dominates, so reading 1MB costs about
|
|
11
|
+
// the same as 64KB and far fewer round trips wins.
|
|
9
12
|
|
|
10
13
|
const EMPTY = Buffer.alloc(0);
|
|
11
|
-
|
|
12
|
-
// Over the network, fetch reads in chunks this big and cache them. Big because each request pays a
|
|
13
|
-
// full round trip; over-reading a bit is far cheaper than another round trip.
|
|
14
14
|
const DEFAULT_CHUNK_BYTES = 1024 * 1024;
|
|
15
15
|
const DEFAULT_CACHE_BYTES = 128 * 1024 * 1024;
|
|
16
16
|
|
|
17
|
-
export type
|
|
18
|
-
// Bytes per cached chunk (aligned). Reads coalesce/serve from these.
|
|
17
|
+
export type RemoteOptions = {
|
|
19
18
|
chunkBytes?: number;
|
|
20
|
-
// Max total bytes held in the range cache (LRU eviction).
|
|
21
19
|
cacheBytes?: number;
|
|
22
|
-
//
|
|
23
|
-
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
type Connection = {
|
|
27
|
-
url: string;
|
|
28
|
-
password: string;
|
|
29
|
-
latencyMs: number;
|
|
30
|
-
agent: https.Agent | undefined;
|
|
31
|
-
cache: RangeCache;
|
|
32
|
-
// Observable stats (handy for tests / diagnostics).
|
|
33
|
-
stats: { requestCount: number; bytesFetched: number };
|
|
20
|
+
latencyMs?: number; // artificial per-request delay, for simulating network latency in tests
|
|
21
|
+
stats?: { requestCount: number; bytesFetched: number };
|
|
34
22
|
};
|
|
35
23
|
|
|
24
|
+
function enoent(p: string): Error {
|
|
25
|
+
return Object.assign(new Error(`File not found: ${p}`), { code: "ENOENT", name: "NotFoundError" });
|
|
26
|
+
}
|
|
27
|
+
function toArrayBuffer(b: Buffer): ArrayBuffer {
|
|
28
|
+
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength) as ArrayBuffer;
|
|
29
|
+
}
|
|
36
30
|
const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
|
|
37
31
|
|
|
38
|
-
|
|
39
|
-
// uses fetch (the user accepts the self-signed cert once). Returns the raw status + body bytes.
|
|
40
|
-
async function httpRequest(conn: Connection, method: string, op: string, params: Record<string, string>, body?: Buffer): Promise<{ status: number; body: Buffer }> {
|
|
41
|
-
if (conn.latencyMs > 0) await sleep(conn.latencyMs);
|
|
42
|
-
conn.stats.requestCount++;
|
|
43
|
-
const qs = new URLSearchParams(params).toString();
|
|
44
|
-
const fullUrl = conn.url.replace(/\/+$/, "") + op + (qs ? "?" + qs : "");
|
|
45
|
-
const headers: Record<string, string> = { authorization: "Bearer " + conn.password };
|
|
46
|
-
if (body) headers["content-type"] = "application/octet-stream";
|
|
32
|
+
type Stat = { size: number; lastModified: number; dir: boolean };
|
|
47
33
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
34
|
+
// A stat is cached this long, so the burst of size lookups during a single read pass is one request per
|
|
35
|
+
// file, not one per block. Short enough that an append by another writer shows up promptly; our own
|
|
36
|
+
// writes invalidate it immediately.
|
|
37
|
+
const INFO_TTL_MS = 2000;
|
|
38
|
+
|
|
39
|
+
class Connection {
|
|
40
|
+
private agent = isNode() ? new https.Agent({ rejectUnauthorized: false }) : undefined;
|
|
41
|
+
private cache: RangeCache;
|
|
42
|
+
private infoCache = new Map<string, { stat: Stat | undefined; at: number }>();
|
|
43
|
+
constructor(public url: string, public password: string, private opts: RemoteOptions) {
|
|
44
|
+
this.cache = new RangeCache(opts.chunkBytes || DEFAULT_CHUNK_BYTES, opts.cacheBytes || DEFAULT_CACHE_BYTES);
|
|
45
|
+
this.url = url.replace(/\/+$/, "");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// One HTTP request. Node uses the https module (self-signed cert → verification disabled); the
|
|
49
|
+
// browser uses fetch (the user has accepted the cert). Returns the raw status + body bytes.
|
|
50
|
+
async request(method: string, op: string, params: Record<string, string>, body?: Buffer): Promise<{ status: number; body: Buffer }> {
|
|
51
|
+
if (this.opts.latencyMs) await sleep(this.opts.latencyMs);
|
|
52
|
+
if (this.opts.stats) this.opts.stats.requestCount++;
|
|
53
|
+
const qs = new URLSearchParams(params).toString();
|
|
54
|
+
const fullUrl = this.url + op + (qs ? "?" + qs : "");
|
|
55
|
+
const headers: Record<string, string> = { authorization: "Bearer " + this.password };
|
|
56
|
+
if (body) headers["content-type"] = "application/octet-stream";
|
|
57
|
+
if (isNode()) {
|
|
58
|
+
return await new Promise((resolve, reject) => {
|
|
59
|
+
const u = new URL(fullUrl);
|
|
60
|
+
const req = https.request({ hostname: u.hostname, port: u.port, path: u.pathname + u.search, method, headers, agent: this.agent }, res => {
|
|
61
|
+
const chunks: Buffer[] = [];
|
|
62
|
+
res.on("data", d => chunks.push(d as Buffer));
|
|
63
|
+
res.on("end", () => resolve({ status: res.statusCode || 0, body: Buffer.concat(chunks) }));
|
|
64
|
+
});
|
|
65
|
+
req.on("error", reject);
|
|
66
|
+
if (body) req.write(body);
|
|
67
|
+
req.end();
|
|
57
68
|
});
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
});
|
|
69
|
+
}
|
|
70
|
+
const res = await fetch(fullUrl, { method, headers, body: body ? new Uint8Array(body) : undefined });
|
|
71
|
+
return { status: res.status, body: Buffer.from(await res.arrayBuffer()) };
|
|
62
72
|
}
|
|
63
|
-
const res = await fetch(fullUrl, { method, headers, body: body ? new Uint8Array(body) : undefined });
|
|
64
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
65
|
-
return { status: res.status, body: buf };
|
|
66
|
-
}
|
|
67
73
|
|
|
68
|
-
async
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
async stat(path: string): Promise<Stat | undefined> {
|
|
75
|
+
const cached = this.infoCache.get(path);
|
|
76
|
+
if (cached && Date.now() - cached.at < INFO_TTL_MS) return cached.stat;
|
|
77
|
+
const r = await this.request("GET", "/info", { path });
|
|
78
|
+
let stat: Stat | undefined;
|
|
79
|
+
if (r.status === 404) stat = undefined;
|
|
80
|
+
else if (r.status !== 200) throw new Error(`remote info failed (${r.status})`);
|
|
81
|
+
else stat = JSON.parse(r.body.toString("utf8")) as Stat;
|
|
82
|
+
this.infoCache.set(path, { stat, at: Date.now() });
|
|
83
|
+
return stat;
|
|
84
|
+
}
|
|
85
|
+
async list(path: string): Promise<{ name: string; dir: boolean }[]> {
|
|
86
|
+
const r = await this.request("GET", "/list", { path });
|
|
87
|
+
if (r.status !== 200) throw new Error(`remote list failed (${r.status})`);
|
|
88
|
+
return JSON.parse(r.body.toString("utf8"));
|
|
89
|
+
}
|
|
90
|
+
async read(path: string, start: number, end: number): Promise<Buffer> {
|
|
91
|
+
return (await this.cache.read(this, path, start, end)) ?? EMPTY;
|
|
92
|
+
}
|
|
93
|
+
async readServer(path: string, start: number, end: number): Promise<Buffer | undefined> {
|
|
94
|
+
const r = await this.request("GET", "/read", { path, start: String(start), end: String(end) });
|
|
95
|
+
if (r.status === 404) return undefined;
|
|
96
|
+
if (r.status !== 200) throw new Error(`remote read failed (${r.status})`);
|
|
97
|
+
if (this.opts.stats) this.opts.stats.bytesFetched += r.body.length;
|
|
98
|
+
return r.body;
|
|
99
|
+
}
|
|
100
|
+
async append(path: string, body: Buffer): Promise<void> {
|
|
101
|
+
const r = await this.request("PUT", "/append", { path }, body);
|
|
102
|
+
if (r.status !== 200) throw new Error(`remote append failed (${r.status})`);
|
|
103
|
+
// Append-only keeps existing bytes, so cached chunks stay valid; just the size changed.
|
|
104
|
+
this.infoCache.delete(path);
|
|
105
|
+
}
|
|
106
|
+
async set(path: string, body: Buffer): Promise<void> {
|
|
107
|
+
const r = await this.request("PUT", "/set", { path }, body);
|
|
108
|
+
if (r.status !== 200) throw new Error(`remote set failed (${r.status})`);
|
|
109
|
+
this.cache.invalidate(path);
|
|
110
|
+
this.infoCache.delete(path);
|
|
111
|
+
}
|
|
112
|
+
async remove(path: string): Promise<void> {
|
|
113
|
+
const r = await this.request("DELETE", "/remove", { path });
|
|
114
|
+
if (r.status !== 200) throw new Error(`remote remove failed (${r.status})`);
|
|
115
|
+
this.cache.invalidate(path);
|
|
116
|
+
this.infoCache.delete(path);
|
|
117
|
+
}
|
|
74
118
|
}
|
|
75
119
|
|
|
76
|
-
// Caches the longest-known prefix of each aligned chunk per path.
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
// that needs MORE than we have refetches (and picks up appended bytes). It never distinguishes file
|
|
80
|
-
// types — it only does range reads.
|
|
120
|
+
// Caches the longest-known prefix of each aligned chunk per path. Valid because files are immutable or
|
|
121
|
+
// append-only (the bytes at an offset never change). It only does range reads — it doesn't care what
|
|
122
|
+
// the files are.
|
|
81
123
|
class RangeCache {
|
|
82
|
-
private chunks = new Map<string, Buffer>(); //
|
|
124
|
+
private chunks = new Map<string, Buffer>(); // path + "" + chunkIndex, insertion-ordered (LRU)
|
|
83
125
|
private bytes = 0;
|
|
84
126
|
constructor(private chunkBytes: number, private budget: number) { }
|
|
85
|
-
|
|
86
127
|
private key(path: string, c: number) { return path + "" + c; }
|
|
87
128
|
private peek(path: string, c: number): Buffer | undefined {
|
|
88
129
|
const k = this.key(path, c);
|
|
89
130
|
const v = this.chunks.get(k);
|
|
90
|
-
if (v) { this.chunks.delete(k); this.chunks.set(k, v); }
|
|
131
|
+
if (v) { this.chunks.delete(k); this.chunks.set(k, v); }
|
|
91
132
|
return v;
|
|
92
133
|
}
|
|
93
134
|
private store(path: string, c: number, buf: Buffer) {
|
|
94
135
|
const k = this.key(path, c);
|
|
95
136
|
const ex = this.chunks.get(k);
|
|
96
|
-
if (ex && ex.length >= buf.length) { this.chunks.delete(k); this.chunks.set(k, ex); return; }
|
|
137
|
+
if (ex && ex.length >= buf.length) { this.chunks.delete(k); this.chunks.set(k, ex); return; }
|
|
97
138
|
if (ex) this.bytes -= ex.length;
|
|
98
139
|
this.chunks.delete(k);
|
|
99
140
|
this.chunks.set(k, buf);
|
|
@@ -106,17 +147,13 @@ class RangeCache {
|
|
|
106
147
|
}
|
|
107
148
|
invalidate(path: string) {
|
|
108
149
|
const prefix = path + "";
|
|
109
|
-
for (const k of [...this.chunks.keys()]) {
|
|
110
|
-
if (k.startsWith(prefix)) { this.bytes -= (this.chunks.get(k) as Buffer).length; this.chunks.delete(k); }
|
|
111
|
-
}
|
|
150
|
+
for (const k of [...this.chunks.keys()]) if (k.startsWith(prefix)) { this.bytes -= (this.chunks.get(k) as Buffer).length; this.chunks.delete(k); }
|
|
112
151
|
}
|
|
113
|
-
|
|
114
|
-
async getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined> {
|
|
152
|
+
async read(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined> {
|
|
115
153
|
if (end <= start) return EMPTY;
|
|
116
154
|
const CHUNK = this.chunkBytes;
|
|
117
155
|
const firstChunk = Math.floor(start / CHUNK);
|
|
118
156
|
const lastChunk = Math.floor((end - 1) / CHUNK);
|
|
119
|
-
// Find the contiguous span of chunks we don't have enough of, and fetch it in one request.
|
|
120
157
|
let fetchFrom = -1, fetchTo = -1;
|
|
121
158
|
for (let c = firstChunk; c <= lastChunk; c++) {
|
|
122
159
|
const cStart = c * CHUNK;
|
|
@@ -125,11 +162,11 @@ class RangeCache {
|
|
|
125
162
|
if (!have || have.length < needEnd) { if (fetchFrom < 0) fetchFrom = c; fetchTo = c; }
|
|
126
163
|
}
|
|
127
164
|
if (fetchFrom >= 0) {
|
|
128
|
-
const bytes = await
|
|
129
|
-
if (bytes === undefined) return undefined;
|
|
165
|
+
const bytes = await conn.readServer(path, fetchFrom * CHUNK, (fetchTo + 1) * CHUNK);
|
|
166
|
+
if (bytes === undefined) return undefined;
|
|
130
167
|
for (let c = fetchFrom; c <= fetchTo; c++) {
|
|
131
168
|
const off = (c - fetchFrom) * CHUNK;
|
|
132
|
-
if (off >= bytes.length) break;
|
|
169
|
+
if (off >= bytes.length) break;
|
|
133
170
|
this.store(path, c, bytes.subarray(off, Math.min(off + CHUNK, bytes.length)));
|
|
134
171
|
}
|
|
135
172
|
}
|
|
@@ -139,122 +176,99 @@ class RangeCache {
|
|
|
139
176
|
const from = Math.max(start, cStart) - cStart;
|
|
140
177
|
const to = Math.min(end, cStart + CHUNK) - cStart;
|
|
141
178
|
const chunk = this.peek(path, c);
|
|
142
|
-
if (!chunk || chunk.length < to) {
|
|
143
|
-
// File ended before the requested end — return what's available (callers read within EOF).
|
|
144
|
-
if (chunk && chunk.length > from) parts.push(chunk.subarray(from, chunk.length));
|
|
145
|
-
break;
|
|
146
|
-
}
|
|
179
|
+
if (!chunk || chunk.length < to) { if (chunk && chunk.length > from) parts.push(chunk.subarray(from, chunk.length)); break; }
|
|
147
180
|
parts.push(chunk.subarray(from, to));
|
|
148
181
|
}
|
|
149
182
|
return parts.length === 1 ? parts[0] : Buffer.concat(parts);
|
|
150
183
|
}
|
|
151
184
|
}
|
|
152
185
|
|
|
153
|
-
|
|
154
|
-
const rel = (key: string) => basePath ? basePath + "/" + key : key;
|
|
186
|
+
const joinPath = (base: string, key: string) => (base ? base + "/" + key : key);
|
|
155
187
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
},
|
|
188
|
-
async getRange(key: string, config: { start: number; end: number }) {
|
|
189
|
-
return conn.cache.getRange(conn, rel(key), config.start, config.end);
|
|
190
|
-
},
|
|
191
|
-
async append(key: string, value: Buffer) {
|
|
192
|
-
const r = await httpRequest(conn, "PUT", "/append", { path: rel(key) }, value);
|
|
193
|
-
if (r.status !== 200) throw new Error(`remote append failed (${r.status})`);
|
|
194
|
-
// No invalidation needed: append-only files keep their prefix; a read past it refetches.
|
|
195
|
-
},
|
|
196
|
-
async set(key: string, value: Buffer) {
|
|
197
|
-
const r = await httpRequest(conn, "PUT", "/set", { path: rel(key) }, value);
|
|
198
|
-
if (r.status !== 200) throw new Error(`remote set failed (${r.status})`);
|
|
199
|
-
conn.cache.invalidate(rel(key));
|
|
200
|
-
},
|
|
201
|
-
async remove(key: string) {
|
|
202
|
-
await httpRequest(conn, "DELETE", "/remove", { path: rel(key) });
|
|
203
|
-
conn.cache.invalidate(rel(key));
|
|
204
|
-
},
|
|
205
|
-
async getKeys(includeFolders: boolean = false) {
|
|
206
|
-
const r = await httpRequest(conn, "GET", "/list", { path: basePath, folders: includeFolders ? "1" : "0" });
|
|
207
|
-
if (r.status !== 200) throw new Error(`remote list failed (${r.status})`);
|
|
208
|
-
return JSON.parse(r.body.toString("utf8"));
|
|
209
|
-
},
|
|
210
|
-
async reset() {
|
|
211
|
-
await httpRequest(conn, "POST", "/reset", { path: basePath });
|
|
212
|
-
conn.cache.invalidate(basePath);
|
|
213
|
-
},
|
|
214
|
-
folder,
|
|
215
|
-
};
|
|
188
|
+
class RemoteFileWrapper implements FileWrapper {
|
|
189
|
+
// `stat` is supplied when the parent already statted us (avoids a second /info). `createIntent` means
|
|
190
|
+
// this was opened with create:true, so a missing file reads as empty (it'll be created on write).
|
|
191
|
+
constructor(private conn: Connection, private filePath: string, private stat?: Stat, private createIntent = false) { }
|
|
192
|
+
async getFile() {
|
|
193
|
+
let stat = this.stat ?? await this.conn.stat(this.filePath);
|
|
194
|
+
if (!stat) {
|
|
195
|
+
if (!this.createIntent) throw enoent(this.filePath);
|
|
196
|
+
stat = { size: 0, lastModified: Date.now(), dir: false };
|
|
197
|
+
}
|
|
198
|
+
const conn = this.conn, filePath = this.filePath, size = stat.size;
|
|
199
|
+
return {
|
|
200
|
+
size, lastModified: stat.lastModified,
|
|
201
|
+
async arrayBuffer() { return toArrayBuffer(await conn.read(filePath, 0, size)); },
|
|
202
|
+
slice(start: number, end: number) {
|
|
203
|
+
return { async arrayBuffer() { return toArrayBuffer(await conn.read(filePath, start, end)); } };
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
async createWritable(config?: { keepExistingData?: boolean }) {
|
|
208
|
+
const conn = this.conn, filePath = this.filePath, append = !!config?.keepExistingData;
|
|
209
|
+
const chunks: Buffer[] = [];
|
|
210
|
+
return {
|
|
211
|
+
async seek() { /* the server appends to the end / overwrites the whole file; offset unused */ },
|
|
212
|
+
async write(value: Buffer) { chunks.push(value); },
|
|
213
|
+
async close() {
|
|
214
|
+
const body = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
|
|
215
|
+
if (append) await conn.append(filePath, body); else await conn.set(filePath, body);
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
216
219
|
}
|
|
217
220
|
|
|
218
|
-
|
|
221
|
+
class RemoteDirectoryWrapper implements DirectoryWrapper {
|
|
222
|
+
constructor(private conn: Connection, private dirPath: string) { }
|
|
223
|
+
async removeEntry(key: string): Promise<void> {
|
|
224
|
+
await this.conn.remove(joinPath(this.dirPath, key));
|
|
225
|
+
}
|
|
226
|
+
async getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper> {
|
|
227
|
+
const p = joinPath(this.dirPath, key);
|
|
228
|
+
if (options?.create) return new RemoteFileWrapper(this.conn, p, undefined, true);
|
|
229
|
+
const stat = await this.conn.stat(p); // matches the File API: throw if missing
|
|
230
|
+
if (!stat || stat.dir) throw enoent(p);
|
|
231
|
+
return new RemoteFileWrapper(this.conn, p, stat, false);
|
|
232
|
+
}
|
|
233
|
+
async getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper> {
|
|
234
|
+
const p = joinPath(this.dirPath, key);
|
|
235
|
+
if (!options?.create) {
|
|
236
|
+
const stat = await this.conn.stat(p);
|
|
237
|
+
if (!stat || !stat.dir) throw enoent(p);
|
|
238
|
+
}
|
|
239
|
+
return new RemoteDirectoryWrapper(this.conn, p); // dirs are created lazily on first write
|
|
240
|
+
}
|
|
241
|
+
async *[Symbol.asyncIterator](): AsyncIterableIterator<[string, { kind: "file"; name: string; getFile(): Promise<FileWrapper> } | { kind: "directory"; name: string; getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper> }]> {
|
|
242
|
+
const entries = await this.conn.list(this.dirPath);
|
|
243
|
+
for (const e of entries) {
|
|
244
|
+
const childPath = joinPath(this.dirPath, e.name);
|
|
245
|
+
if (e.dir) {
|
|
246
|
+
yield [e.name, { kind: "directory", name: e.name, getDirectoryHandle: (k, o) => new RemoteDirectoryWrapper(this.conn, childPath).getDirectoryHandle(k, o) }];
|
|
247
|
+
} else {
|
|
248
|
+
yield [e.name, { kind: "file", name: e.name, getFile: async () => new RemoteFileWrapper(this.conn, childPath) }];
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
219
253
|
|
|
220
|
-
//
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const conn: Connection = {
|
|
224
|
-
url,
|
|
225
|
-
password,
|
|
226
|
-
latencyMs: options.latencyMs || 0,
|
|
227
|
-
// Self-signed cert: skip verification in Node. (The browser path uses fetch, where the user has
|
|
228
|
-
// already accepted the cert.) The password is what authorizes access.
|
|
229
|
-
agent: isNode() ? new https.Agent({ rejectUnauthorized: false }) : undefined,
|
|
230
|
-
cache: new RangeCache(options.chunkBytes || DEFAULT_CHUNK_BYTES, options.cacheBytes || DEFAULT_CACHE_BYTES),
|
|
231
|
-
stats: { requestCount: 0, bytesFetched: 0 },
|
|
232
|
-
};
|
|
233
|
-
const factory = ((path: string) => Promise.resolve(makeStorage(conn, path.replace(/^\/+|\/+$/g, "")))) as RemoteStorageFactory;
|
|
234
|
-
factory.stats = conn.stats;
|
|
235
|
-
return factory;
|
|
254
|
+
// A DirectoryWrapper rooted at a remote server. Drop-in for the Node / File-API handles.
|
|
255
|
+
export function getRemoteDirectoryHandle(url: string, password: string, options: RemoteOptions = {}): DirectoryWrapper {
|
|
256
|
+
return new RemoteDirectoryWrapper(new Connection(url, password, options), "");
|
|
236
257
|
}
|
|
237
258
|
|
|
238
|
-
export type
|
|
239
|
-
| { status: "ok" }
|
|
240
|
-
| { status: "unauthorized" }
|
|
241
|
-
// Couldn't reach the server at all — in the browser this is usually the self-signed certificate not
|
|
242
|
-
// being trusted yet (fetch throws with no detail), but also covers a wrong URL / server down / CORS.
|
|
243
|
-
| { status: "unreachable"; error: string };
|
|
259
|
+
export type RemoteConnectResult = { status: "ok" } | { status: "unauthorized" } | { status: "unreachable"; error: string };
|
|
244
260
|
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
export async function
|
|
261
|
+
// Verifies a server is reachable and the password works — by actually listing the root. Distinguishes
|
|
262
|
+
// "connected" / "wrong password" / "couldn't reach it" (the last usually meaning the self-signed cert
|
|
263
|
+
// isn't trusted yet in the browser, since fetch failures carry no detail).
|
|
264
|
+
export async function testRemoteConnection(url: string, password: string, options: RemoteOptions = {}): Promise<RemoteConnectResult> {
|
|
265
|
+
const conn = new Connection(url, password, options);
|
|
249
266
|
try {
|
|
250
|
-
|
|
251
|
-
return { status: "ok" };
|
|
267
|
+
const r = await conn.request("GET", "/list", { path: "" });
|
|
268
|
+
if (r.status === 200) return { status: "ok" };
|
|
269
|
+
if (r.status === 401) return { status: "unauthorized" };
|
|
270
|
+
return { status: "unreachable", error: `server returned ${r.status}` };
|
|
252
271
|
} catch (e) {
|
|
253
|
-
|
|
254
|
-
// Our client throws "remote ... failed (NNN)" when it actually got an HTTP response (so the cert
|
|
255
|
-
// is trusted); a thrown fetch with no status means we never connected.
|
|
256
|
-
const m = msg.match(/\((\d{3})\)/);
|
|
257
|
-
if (m) return m[1] === "401" ? { status: "unauthorized" } : { status: "unreachable", error: msg };
|
|
258
|
-
return { status: "unreachable", error: msg };
|
|
272
|
+
return { status: "unreachable", error: (e as Error)?.message || String(e) };
|
|
259
273
|
}
|
|
260
274
|
}
|