sliftutils 1.4.9 → 1.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +10 -36
- package/package.json +1 -1
- package/storage/FileFolderAPI.d.ts +4 -3
- package/storage/FileFolderAPI.tsx +109 -139
- package/storage/remoteFileServer.ts +21 -17
- package/storage/remoteFileStorage.d.ts +6 -33
- package/storage/remoteFileStorage.ts +189 -175
package/index.d.ts
CHANGED
|
@@ -1315,6 +1315,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1315
1315
|
/// <reference types="node" />
|
|
1316
1316
|
/// <reference types="node" />
|
|
1317
1317
|
import { IStorageRaw } from "./IStorage";
|
|
1318
|
+
import { RemoteOptions } from "./remoteFileStorage";
|
|
1318
1319
|
declare global {
|
|
1319
1320
|
interface Window {
|
|
1320
1321
|
showSaveFilePicker(config?: {
|
|
@@ -1341,7 +1342,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1341
1342
|
}): Promise<PermissionState>;
|
|
1342
1343
|
}
|
|
1343
1344
|
}
|
|
1344
|
-
type FileWrapper = {
|
|
1345
|
+
export type FileWrapper = {
|
|
1345
1346
|
getFile(): Promise<{
|
|
1346
1347
|
size: number;
|
|
1347
1348
|
lastModified: number;
|
|
@@ -1358,7 +1359,7 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1358
1359
|
close(): Promise<void>;
|
|
1359
1360
|
}>;
|
|
1360
1361
|
};
|
|
1361
|
-
type DirectoryWrapper = {
|
|
1362
|
+
export type DirectoryWrapper = {
|
|
1362
1363
|
removeEntry(key: string, options?: {
|
|
1363
1364
|
recursive?: boolean;
|
|
1364
1365
|
}): Promise<void>;
|
|
@@ -1467,8 +1468,8 @@ declare module "sliftutils/storage/FileFolderAPI" {
|
|
|
1467
1468
|
folder: NestedFileStorage;
|
|
1468
1469
|
};
|
|
1469
1470
|
export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
|
|
1471
|
+
export declare function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage>;
|
|
1470
1472
|
export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
|
|
1471
|
-
export {};
|
|
1472
1473
|
|
|
1473
1474
|
}
|
|
1474
1475
|
|
|
@@ -1861,44 +1862,18 @@ declare module "sliftutils/storage/remoteFileServer" {
|
|
|
1861
1862
|
}
|
|
1862
1863
|
|
|
1863
1864
|
declare module "sliftutils/storage/remoteFileStorage" {
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
/// <reference types="node" />
|
|
1867
|
-
import https from "https";
|
|
1868
|
-
import type { FileStorage } from "./FileFolderAPI";
|
|
1869
|
-
export type RemoteFileStorageOptions = {
|
|
1865
|
+
import type { DirectoryWrapper } from "./FileFolderAPI";
|
|
1866
|
+
export type RemoteOptions = {
|
|
1870
1867
|
chunkBytes?: number;
|
|
1871
1868
|
cacheBytes?: number;
|
|
1872
1869
|
latencyMs?: number;
|
|
1873
|
-
|
|
1874
|
-
type Connection = {
|
|
1875
|
-
url: string;
|
|
1876
|
-
password: string;
|
|
1877
|
-
latencyMs: number;
|
|
1878
|
-
agent: https.Agent | undefined;
|
|
1879
|
-
cache: RangeCache;
|
|
1880
|
-
stats: {
|
|
1870
|
+
stats?: {
|
|
1881
1871
|
requestCount: number;
|
|
1882
1872
|
bytesFetched: number;
|
|
1883
1873
|
};
|
|
1884
1874
|
};
|
|
1885
|
-
declare
|
|
1886
|
-
|
|
1887
|
-
private budget;
|
|
1888
|
-
private chunks;
|
|
1889
|
-
private bytes;
|
|
1890
|
-
constructor(chunkBytes: number, budget: number);
|
|
1891
|
-
private key;
|
|
1892
|
-
private peek;
|
|
1893
|
-
private store;
|
|
1894
|
-
invalidate(path: string): void;
|
|
1895
|
-
getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined>;
|
|
1896
|
-
}
|
|
1897
|
-
export type RemoteStorageFactory = ((path: string) => Promise<FileStorage>) & {
|
|
1898
|
-
stats: Connection["stats"];
|
|
1899
|
-
};
|
|
1900
|
-
export declare function getRemoteFileStorage(url: string, password: string, options?: RemoteFileStorageOptions): RemoteStorageFactory;
|
|
1901
|
-
export type RemoteProbeResult = {
|
|
1875
|
+
export declare function getRemoteDirectoryHandle(url: string, password: string, options?: RemoteOptions): DirectoryWrapper;
|
|
1876
|
+
export type RemoteConnectResult = {
|
|
1902
1877
|
status: "ok";
|
|
1903
1878
|
} | {
|
|
1904
1879
|
status: "unauthorized";
|
|
@@ -1906,8 +1881,7 @@ declare module "sliftutils/storage/remoteFileStorage" {
|
|
|
1906
1881
|
status: "unreachable";
|
|
1907
1882
|
error: string;
|
|
1908
1883
|
};
|
|
1909
|
-
export declare function
|
|
1910
|
-
export {};
|
|
1884
|
+
export declare function testRemoteConnection(url: string, password: string, options?: RemoteOptions): Promise<RemoteConnectResult>;
|
|
1911
1885
|
|
|
1912
1886
|
}
|
|
1913
1887
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
3
|
import { IStorageRaw } from "./IStorage";
|
|
4
|
+
import { RemoteOptions } from "./remoteFileStorage";
|
|
4
5
|
declare global {
|
|
5
6
|
interface Window {
|
|
6
7
|
showSaveFilePicker(config?: {
|
|
@@ -27,7 +28,7 @@ declare global {
|
|
|
27
28
|
}): Promise<PermissionState>;
|
|
28
29
|
}
|
|
29
30
|
}
|
|
30
|
-
type FileWrapper = {
|
|
31
|
+
export type FileWrapper = {
|
|
31
32
|
getFile(): Promise<{
|
|
32
33
|
size: number;
|
|
33
34
|
lastModified: number;
|
|
@@ -44,7 +45,7 @@ type FileWrapper = {
|
|
|
44
45
|
close(): Promise<void>;
|
|
45
46
|
}>;
|
|
46
47
|
};
|
|
47
|
-
type DirectoryWrapper = {
|
|
48
|
+
export type DirectoryWrapper = {
|
|
48
49
|
removeEntry(key: string, options?: {
|
|
49
50
|
recursive?: boolean;
|
|
50
51
|
}): Promise<void>;
|
|
@@ -153,5 +154,5 @@ export type FileStorage = IStorageRaw & {
|
|
|
153
154
|
folder: NestedFileStorage;
|
|
154
155
|
};
|
|
155
156
|
export declare function wrapHandle(handle: DirectoryWrapper): FileStorage;
|
|
157
|
+
export declare function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage>;
|
|
156
158
|
export declare function tryToLoadPointer(pointer: string): Promise<DirectoryWrapper | undefined>;
|
|
157
|
-
export {};
|
|
@@ -7,7 +7,7 @@ import { css, isNode } from "typesafecss";
|
|
|
7
7
|
import { IStorageRaw } from "./IStorage";
|
|
8
8
|
import { runInSerial } from "socket-function/src/batching";
|
|
9
9
|
import { getFileStorageIndexDB } from "./IndexedDBFileFolderAPI";
|
|
10
|
-
import {
|
|
10
|
+
import { getRemoteDirectoryHandle, testRemoteConnection, RemoteOptions } from "./remoteFileStorage";
|
|
11
11
|
import fs from "fs";
|
|
12
12
|
import path from "path";
|
|
13
13
|
|
|
@@ -36,7 +36,7 @@ declare global {
|
|
|
36
36
|
// DO NOT enable this is isNode
|
|
37
37
|
const USE_INDEXED_DB = false;
|
|
38
38
|
|
|
39
|
-
type FileWrapper = {
|
|
39
|
+
export type FileWrapper = {
|
|
40
40
|
getFile(): Promise<{
|
|
41
41
|
size: number;
|
|
42
42
|
lastModified: number;
|
|
@@ -51,7 +51,7 @@ type FileWrapper = {
|
|
|
51
51
|
close(): Promise<void>;
|
|
52
52
|
}>;
|
|
53
53
|
};
|
|
54
|
-
type DirectoryWrapper = {
|
|
54
|
+
export type DirectoryWrapper = {
|
|
55
55
|
removeEntry(key: string, options?: { recursive?: boolean }): Promise<void>;
|
|
56
56
|
getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper>;
|
|
57
57
|
getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper>;
|
|
@@ -86,20 +86,14 @@ type RemoteConfig = { url: string; password: string };
|
|
|
86
86
|
function remoteConfigKey() { return getFileAPIKey() + ":remote"; }
|
|
87
87
|
function loadRemoteConfig(): RemoteConfig | undefined {
|
|
88
88
|
try {
|
|
89
|
-
const
|
|
90
|
-
const s = localStorage.getItem(key);
|
|
91
|
-
console.log("[remotecfg] load", JSON.stringify(key), "->", s);
|
|
89
|
+
const s = localStorage.getItem(remoteConfigKey());
|
|
92
90
|
if (!s) return undefined;
|
|
93
91
|
const c = JSON.parse(s);
|
|
94
92
|
if (c && typeof c.url === "string" && typeof c.password === "string") return c;
|
|
95
|
-
} catch
|
|
93
|
+
} catch { /* ignore */ }
|
|
96
94
|
return undefined;
|
|
97
95
|
}
|
|
98
|
-
function saveRemoteConfig(c: RemoteConfig) {
|
|
99
|
-
const key = remoteConfigKey();
|
|
100
|
-
localStorage.setItem(key, JSON.stringify(c));
|
|
101
|
-
console.log("[remotecfg] saved", JSON.stringify(key), "readback ->", localStorage.getItem(key));
|
|
102
|
-
}
|
|
96
|
+
function saveRemoteConfig(c: RemoteConfig) { localStorage.setItem(remoteConfigKey(), JSON.stringify(c)); }
|
|
103
97
|
|
|
104
98
|
// The server is always HTTPS on the filehoster's default port, so the user only needs to type the host
|
|
105
99
|
// (e.g. "65.109.93.113"). We strip any scheme/path they include and default the port if omitted.
|
|
@@ -111,14 +105,14 @@ function normalizeServerUrl(raw: string): string {
|
|
|
111
105
|
return "https://" + s;
|
|
112
106
|
}
|
|
113
107
|
|
|
114
|
-
// One shared
|
|
115
|
-
let
|
|
116
|
-
function
|
|
108
|
+
// One shared remote DirectoryWrapper (and therefore one shared range cache) per remote config.
|
|
109
|
+
let remoteHandle: { key: string; handle: DirectoryWrapper } | undefined;
|
|
110
|
+
function getRemoteHandle(remote: RemoteConfig): DirectoryWrapper {
|
|
117
111
|
const key = remote.url + "\0" + remote.password;
|
|
118
|
-
if (!
|
|
119
|
-
|
|
112
|
+
if (!remoteHandle || remoteHandle.key !== key) {
|
|
113
|
+
remoteHandle = { key, handle: getRemoteDirectoryHandle(remote.url, remote.password) };
|
|
120
114
|
}
|
|
121
|
-
return
|
|
115
|
+
return remoteHandle.handle;
|
|
122
116
|
}
|
|
123
117
|
|
|
124
118
|
@observer
|
|
@@ -139,12 +133,19 @@ class DirectoryPrompter extends preact.Component {
|
|
|
139
133
|
}
|
|
140
134
|
}
|
|
141
135
|
|
|
142
|
-
// "Connect to a server" option for the directory prompt: collapses to a button, expands to
|
|
143
|
-
// password fields. On connect it
|
|
144
|
-
//
|
|
136
|
+
// "Connect to a server" option for the directory prompt: collapses to a button, expands to address +
|
|
137
|
+
// password fields. On connect it ACTUALLY connects (testRemoteConnection); only on success does it
|
|
138
|
+
// persist the config and call onConnected, so the caller (getDirectoryHandle) resolves with a working
|
|
139
|
+
// remote handle. Failures are shown to the user (and logged, without the password), never swallowed.
|
|
140
|
+
// `initial` pre-fills + expands the form (used to retry a remembered server that stopped working).
|
|
145
141
|
@observer
|
|
146
|
-
class ServerConnectForm extends preact.Component {
|
|
147
|
-
private obs = observable({
|
|
142
|
+
class ServerConnectForm extends preact.Component<{ onConnected: (config: RemoteConfig) => void; initial?: RemoteConfig }> {
|
|
143
|
+
private obs = observable({
|
|
144
|
+
expanded: !!this.props.initial,
|
|
145
|
+
url: this.props.initial?.url || "",
|
|
146
|
+
password: this.props.initial?.password || "",
|
|
147
|
+
error: "", connecting: false, needsCert: false, showPassword: false,
|
|
148
|
+
});
|
|
148
149
|
private cleanUrl() { return normalizeServerUrl(this.obs.url); }
|
|
149
150
|
private connect = async () => {
|
|
150
151
|
const s = this.obs;
|
|
@@ -153,22 +154,27 @@ class ServerConnectForm extends preact.Component {
|
|
|
153
154
|
s.connecting = true;
|
|
154
155
|
try {
|
|
155
156
|
const url = this.cleanUrl();
|
|
156
|
-
if (!url) { s.error = "Enter a server
|
|
157
|
-
const result = await
|
|
157
|
+
if (!url) { s.error = "Enter a server address."; return; }
|
|
158
|
+
const result = await testRemoteConnection(url, s.password.trim());
|
|
158
159
|
if (result.status === "ok") {
|
|
159
|
-
|
|
160
|
-
|
|
160
|
+
const config: RemoteConfig = { url, password: s.password.trim() };
|
|
161
|
+
saveRemoteConfig(config); // remember for next session
|
|
162
|
+
this.props.onConnected(config); // hand the verified server back to getDirectoryHandle
|
|
161
163
|
return;
|
|
162
164
|
}
|
|
163
165
|
if (result.status === "unauthorized") {
|
|
164
166
|
s.error = "The server rejected that password.";
|
|
167
|
+
console.error("Remote connect: unauthorized for", url);
|
|
165
168
|
} else {
|
|
166
|
-
//
|
|
169
|
+
// Got nothing back — usually the self-signed certificate isn't trusted yet, but show the
|
|
170
|
+
// actual error too so a wrong address / down server / CORS issue is diagnosable.
|
|
167
171
|
s.needsCert = true;
|
|
168
|
-
s.error = "Couldn't reach the server.
|
|
172
|
+
s.error = "Couldn't reach the server: " + result.error;
|
|
173
|
+
console.error("Remote connect: unreachable", url, "-", result.error);
|
|
169
174
|
}
|
|
170
175
|
} catch (e) {
|
|
171
|
-
s.error = "
|
|
176
|
+
s.error = "Connection error: " + ((e as Error).message || String(e));
|
|
177
|
+
console.error("Remote connect threw:", e);
|
|
172
178
|
} finally {
|
|
173
179
|
s.connecting = false;
|
|
174
180
|
}
|
|
@@ -362,132 +368,87 @@ export class NodeJSDirectoryHandleWrapper implements DirectoryWrapper {
|
|
|
362
368
|
}
|
|
363
369
|
|
|
364
370
|
|
|
365
|
-
//
|
|
366
|
-
//
|
|
371
|
+
// Returns the directory handle to use — local (Node / picked folder) OR a remote server, both as the
|
|
372
|
+
// same DirectoryWrapper, so callers don't know or care which it is. A remembered server is VERIFIED
|
|
373
|
+
// (we actually connect) before use; if it no longer works we re-prompt, just like a local folder whose
|
|
374
|
+
// permission was lost. Blocks until ready (or the user dismisses, which rejects).
|
|
367
375
|
export const getDirectoryHandle = lazy(async function getDirectoryHandle(): Promise<DirectoryWrapper> {
|
|
368
376
|
if (isNode()) {
|
|
369
377
|
return new NodeJSDirectoryHandleWrapper(path.resolve("./data/"));
|
|
370
378
|
}
|
|
379
|
+
|
|
380
|
+
// A server connected in a previous session: only reuse it if it still actually works.
|
|
381
|
+
const savedRemote = loadRemoteConfig();
|
|
382
|
+
if (savedRemote) {
|
|
383
|
+
const result = await testRemoteConnection(savedRemote.url, savedRemote.password);
|
|
384
|
+
if (result.status === "ok") return getRemoteHandle(savedRemote);
|
|
385
|
+
console.warn(`Saved remote server didn't connect (${result.status})${result.status === "unreachable" ? ": " + result.error : ""} — re-prompting.`);
|
|
386
|
+
}
|
|
387
|
+
|
|
371
388
|
let root = document.createElement("div");
|
|
372
389
|
document.body.appendChild(root);
|
|
373
390
|
preact.render(<DirectoryPrompter />, root);
|
|
374
391
|
try {
|
|
392
|
+
let resolveHandle!: (h: DirectoryWrapper) => void;
|
|
393
|
+
let rejectHandle!: (e: Error) => void;
|
|
394
|
+
const promise = new Promise<DirectoryWrapper>((res, rej) => { resolveHandle = res; rejectHandle = rej; });
|
|
375
395
|
|
|
376
|
-
|
|
396
|
+
const pickLocal = async () => {
|
|
397
|
+
try {
|
|
398
|
+
const handle = await window.showDirectoryPicker();
|
|
399
|
+
await handle.requestPermission({ mode: "readwrite" });
|
|
400
|
+
const storedId = await storeFileSystemPointer({ mode: "readwrite", handle });
|
|
401
|
+
localStorage.setItem(getFileAPIKey(), storedId);
|
|
402
|
+
resolveHandle(handle as any);
|
|
403
|
+
} catch (e) {
|
|
404
|
+
console.error("Directory pick failed/cancelled:", e); // stay on the prompt
|
|
405
|
+
}
|
|
406
|
+
};
|
|
407
|
+
const onConnected = (config: RemoteConfig) => { saveRemoteConfig(config); resolveHandle(getRemoteHandle(config)); };
|
|
408
|
+
// The three options, rendered fresh each time. If a saved server just failed, pre-fill + expand
|
|
409
|
+
// the connect form so the user can retry or fix it.
|
|
410
|
+
const renderOptions = () => (
|
|
411
|
+
<>
|
|
412
|
+
<button className={css.fontSize(40).pad2(80, 40)} onClick={pickLocal}>Pick Data Directory</button>
|
|
413
|
+
<ServerConnectForm onConnected={onConnected} initial={savedRemote} />
|
|
414
|
+
<button className={css.fontSize(40).pad2(80, 40)}
|
|
415
|
+
onClick={() => rejectHandle(new Error("User dismissed file system access prompt"))}>Dismiss</button>
|
|
416
|
+
</>
|
|
417
|
+
);
|
|
377
418
|
|
|
378
|
-
|
|
419
|
+
// A previously-picked local folder: try to restore it (may need a click). Skipped when a saved
|
|
420
|
+
// server failed — that user wants the server, so go straight to the (pre-filled) prompt.
|
|
421
|
+
const storedId = !savedRemote && localStorage.getItem(getFileAPIKey());
|
|
379
422
|
if (storedId) {
|
|
380
423
|
let doneLoad = false;
|
|
381
|
-
setTimeout(() => {
|
|
382
|
-
if (doneLoad) return;
|
|
383
|
-
console.log("Waiting for user to click");
|
|
384
|
-
displayData.ui = "Click anywhere to allow file system access";
|
|
385
|
-
}, 500);
|
|
424
|
+
setTimeout(() => { if (!doneLoad) displayData.ui = "Click anywhere to allow file system access"; }, 500);
|
|
386
425
|
try {
|
|
387
|
-
handle = await tryToLoadPointer(storedId);
|
|
426
|
+
const handle = await tryToLoadPointer(storedId);
|
|
427
|
+
doneLoad = true;
|
|
428
|
+
if (handle) return handle;
|
|
388
429
|
} catch (e) {
|
|
430
|
+
doneLoad = true;
|
|
389
431
|
console.error(e);
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
if (errorMessage.includes("user activation") || errorMessage.includes("User activation")) {
|
|
393
|
-
doneLoad = true;
|
|
394
|
-
// Show UI to get user to click and retry
|
|
395
|
-
let retryCallback: (success: boolean) => void;
|
|
396
|
-
let retryReject: (err: Error) => void;
|
|
397
|
-
let retryPromise = new Promise<boolean>((resolve, reject) => {
|
|
398
|
-
retryCallback = resolve;
|
|
399
|
-
retryReject = reject;
|
|
400
|
-
});
|
|
432
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
433
|
+
if (msg.includes("user activation") || msg.includes("User activation")) {
|
|
401
434
|
displayData.ui = (
|
|
402
435
|
<div className={css.vbox(20).center}>
|
|
403
|
-
<button
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
} else {
|
|
413
|
-
retryCallback(false);
|
|
414
|
-
}
|
|
415
|
-
} catch (retryError) {
|
|
416
|
-
console.error("Retry failed:", retryError);
|
|
417
|
-
retryCallback(false);
|
|
418
|
-
}
|
|
419
|
-
}}
|
|
420
|
-
>
|
|
421
|
-
Click to restore file system access
|
|
422
|
-
</button>
|
|
423
|
-
<button
|
|
424
|
-
className={css.fontSize(40).pad2(80, 40)}
|
|
425
|
-
onClick={async () => {
|
|
426
|
-
console.log("Waiting for user to give permission");
|
|
427
|
-
const pickedHandle = await window.showDirectoryPicker();
|
|
428
|
-
await pickedHandle.requestPermission({ mode: "readwrite" });
|
|
429
|
-
let newStoredId = await storeFileSystemPointer({ mode: "readwrite", handle: pickedHandle });
|
|
430
|
-
localStorage.setItem(getFileAPIKey(), newStoredId);
|
|
431
|
-
handle = pickedHandle as any;
|
|
432
|
-
retryCallback(true);
|
|
433
|
-
}}
|
|
434
|
-
>
|
|
435
|
-
Pick Data Directory
|
|
436
|
-
</button>
|
|
437
|
-
<ServerConnectForm />
|
|
438
|
-
<button
|
|
439
|
-
className={css.fontSize(40).pad2(80, 40)}
|
|
440
|
-
onClick={() => {
|
|
441
|
-
retryReject(new Error("User dismissed file system access prompt"));
|
|
442
|
-
}}
|
|
443
|
-
>
|
|
444
|
-
Dismiss
|
|
445
|
-
</button>
|
|
436
|
+
<button className={css.fontSize(40).pad2(80, 40)} onClick={async () => {
|
|
437
|
+
displayData.ui = "Loading...";
|
|
438
|
+
try {
|
|
439
|
+
const h = await tryToLoadPointer(storedId);
|
|
440
|
+
if (h) { resolveHandle(h); return; }
|
|
441
|
+
} catch (retryError) { console.error("Retry failed:", retryError); }
|
|
442
|
+
displayData.ui = <div className={css.vbox(20).center}>{renderOptions()}</div>;
|
|
443
|
+
}}>Click to restore file system access</button>
|
|
444
|
+
{renderOptions()}
|
|
446
445
|
</div>
|
|
447
446
|
);
|
|
448
|
-
|
|
449
|
-
if (handle) {
|
|
450
|
-
return handle;
|
|
451
|
-
}
|
|
447
|
+
return await promise;
|
|
452
448
|
}
|
|
453
449
|
}
|
|
454
|
-
doneLoad = true;
|
|
455
|
-
if (handle) {
|
|
456
|
-
return handle;
|
|
457
|
-
}
|
|
458
450
|
}
|
|
459
|
-
|
|
460
|
-
let fileReject: (err: Error) => void;
|
|
461
|
-
let promise = new Promise<DirectoryWrapper>((resolve, reject) => {
|
|
462
|
-
fileCallback = resolve;
|
|
463
|
-
fileReject = reject;
|
|
464
|
-
});
|
|
465
|
-
displayData.ui = (
|
|
466
|
-
<div className={css.vbox(20).center}>
|
|
467
|
-
<button
|
|
468
|
-
className={css.fontSize(40).pad2(80, 40)}
|
|
469
|
-
onClick={async () => {
|
|
470
|
-
console.log("Waiting for user to give permission");
|
|
471
|
-
const handle = await window.showDirectoryPicker();
|
|
472
|
-
await handle.requestPermission({ mode: "readwrite" });
|
|
473
|
-
let storedId = await storeFileSystemPointer({ mode: "readwrite", handle });
|
|
474
|
-
localStorage.setItem(getFileAPIKey(), storedId);
|
|
475
|
-
fileCallback(handle as any);
|
|
476
|
-
}}
|
|
477
|
-
>
|
|
478
|
-
Pick Data Directory
|
|
479
|
-
</button>
|
|
480
|
-
<ServerConnectForm />
|
|
481
|
-
<button
|
|
482
|
-
className={css.fontSize(40).pad2(80, 40)}
|
|
483
|
-
onClick={() => {
|
|
484
|
-
fileReject(new Error("User dismissed file system access prompt"));
|
|
485
|
-
}}
|
|
486
|
-
>
|
|
487
|
-
Dismiss
|
|
488
|
-
</button>
|
|
489
|
-
</div>
|
|
490
|
-
);
|
|
451
|
+
displayData.ui = <div className={css.vbox(20).center}>{renderOptions()}</div>;
|
|
491
452
|
return await promise;
|
|
492
453
|
} finally {
|
|
493
454
|
preact.render(null, root);
|
|
@@ -507,10 +468,6 @@ export const getFileStorageNested = cache(async function getFileStorage(path: st
|
|
|
507
468
|
export const getFileStorageNested2 = cache(async function getFileStorage(pathStr: string): Promise<FileStorage> {
|
|
508
469
|
let base: DirectoryWrapper;
|
|
509
470
|
pathStr = pathStr.replaceAll("\\", "/");
|
|
510
|
-
if (!isNode()) {
|
|
511
|
-
const remote = loadRemoteConfig();
|
|
512
|
-
if (remote) return getRemoteFactory(remote)(pathStr);
|
|
513
|
-
}
|
|
514
471
|
if (isNode()) {
|
|
515
472
|
if (path.isAbsolute(pathStr)) {
|
|
516
473
|
return wrapHandle(new NodeJSDirectoryHandleWrapper(pathStr));
|
|
@@ -541,7 +498,6 @@ export const getFileStorage = lazy(async function getFileStorage(): Promise<File
|
|
|
541
498
|
if (USE_INDEXED_DB) {
|
|
542
499
|
return await getFileStorageIndexDB();
|
|
543
500
|
}
|
|
544
|
-
|
|
545
501
|
let handle = await getDirectoryHandle();
|
|
546
502
|
return wrapHandle(handle);
|
|
547
503
|
});
|
|
@@ -757,6 +713,20 @@ export function wrapHandle(handle: DirectoryWrapper): FileStorage {
|
|
|
757
713
|
};
|
|
758
714
|
}
|
|
759
715
|
|
|
716
|
+
// A StorageFactory backed by a remote server (path -> FileStorage), for code that injects its own
|
|
717
|
+
// storage into BulkDatabase2 rather than going through the directory prompt.
|
|
718
|
+
export function getRemoteFileStorageFactory(url: string, password: string, options?: RemoteOptions): (pathStr: string) => Promise<FileStorage> {
|
|
719
|
+
const root = getRemoteDirectoryHandle(url, password, options);
|
|
720
|
+
return async (pathStr: string) => {
|
|
721
|
+
let base: DirectoryWrapper = root;
|
|
722
|
+
for (const part of pathStr.replaceAll("\\", "/").split("/")) {
|
|
723
|
+
if (!part) continue;
|
|
724
|
+
base = await base.getDirectoryHandle(part, { create: true });
|
|
725
|
+
}
|
|
726
|
+
return wrapHandle(base);
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
760
730
|
export async function tryToLoadPointer(pointer: string) {
|
|
761
731
|
let result = await getFileSystemPointer({ pointer });
|
|
762
732
|
if (!result) return;
|
|
@@ -203,6 +203,22 @@ function normalizePassword(p: string): string {
|
|
|
203
203
|
return String(p || "").toLowerCase().replace(/[^a-z]/g, "");
|
|
204
204
|
}
|
|
205
205
|
|
|
206
|
+
// The password is generated once and saved (next to the cert, outside the served folder), then reused
|
|
207
|
+
// across restarts — so connected clients keep working after the server bounces, instead of getting a
|
|
208
|
+
// new password every time. Override with the PASSWORD env var.
|
|
209
|
+
function getOrCreatePassword(): string {
|
|
210
|
+
const dir = path.join(os.homedir(), ".sliftutils-remote");
|
|
211
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
212
|
+
const file = path.join(dir, "password.txt");
|
|
213
|
+
try {
|
|
214
|
+
const existing = fs.readFileSync(file, "utf8").trim();
|
|
215
|
+
if (existing) return existing;
|
|
216
|
+
} catch { /* not created yet */ }
|
|
217
|
+
const pw = generatePassword(6);
|
|
218
|
+
fs.writeFileSync(file, pw + "\n", { mode: 0o600 });
|
|
219
|
+
return pw;
|
|
220
|
+
}
|
|
221
|
+
|
|
206
222
|
// Generate (once) and cache a self-signed key + cert via openssl, outside the served folder so its
|
|
207
223
|
// private key is never reachable through the API.
|
|
208
224
|
function getCert(): { key: Buffer; cert: Buffer } {
|
|
@@ -278,7 +294,7 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
278
294
|
if (!fs.existsSync(root)) fs.mkdirSync(root, { recursive: true });
|
|
279
295
|
const port = options.port ?? 8787;
|
|
280
296
|
const host = options.host || "0.0.0.0";
|
|
281
|
-
const password = options.password ||
|
|
297
|
+
const password = options.password || getOrCreatePassword();
|
|
282
298
|
const normPassword = normalizePassword(password);
|
|
283
299
|
const { key, cert } = getCert();
|
|
284
300
|
|
|
@@ -340,21 +356,17 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
340
356
|
const full = safeResolve(root, relPath);
|
|
341
357
|
if (full === undefined) return sendJson(403, { error: "path escapes root" });
|
|
342
358
|
|
|
359
|
+
// One listing returns every entry with its type, so a directory read is a single round trip.
|
|
343
360
|
if (req.method === "GET" && op === "/list") {
|
|
344
|
-
const wantFolders = url.searchParams.get("folders") === "1";
|
|
345
361
|
let entries: fs.Dirent[];
|
|
346
362
|
try { entries = fs.readdirSync(full, { withFileTypes: true }); }
|
|
347
363
|
catch (e) { return (e as NodeJS.ErrnoException).code === "ENOENT" ? sendJson(200, []) : sendJson(500, { error: (e as Error).message }); }
|
|
348
|
-
return sendJson(200, entries.filter(d =>
|
|
364
|
+
return sendJson(200, entries.filter(d => d.isFile() || d.isDirectory()).map(d => ({ name: d.name, dir: d.isDirectory() })));
|
|
349
365
|
}
|
|
350
366
|
if (req.method === "GET" && op === "/info") {
|
|
351
367
|
let st: fs.Stats;
|
|
352
368
|
try { st = fs.statSync(full); } catch { return sendJson(404, { error: "not found" }); }
|
|
353
|
-
return sendJson(200, { size: st.size, lastModified: st.mtimeMs });
|
|
354
|
-
}
|
|
355
|
-
if (req.method === "GET" && op === "/hasDir") {
|
|
356
|
-
let st: fs.Stats; try { st = fs.statSync(full); } catch { return sendJson(200, { exists: false }); }
|
|
357
|
-
return sendJson(200, { exists: st.isDirectory() });
|
|
369
|
+
return sendJson(200, { size: st.size, lastModified: st.mtimeMs, dir: st.isDirectory() });
|
|
358
370
|
}
|
|
359
371
|
if (req.method === "GET" && op === "/read") {
|
|
360
372
|
let st: fs.Stats;
|
|
@@ -377,19 +389,11 @@ export function startRemoteFileServer(options: RemoteFileServerOptions): Promise
|
|
|
377
389
|
recordAccess(clientIp(req), "write", relPath, body.length);
|
|
378
390
|
return sendJson(200, { ok: true });
|
|
379
391
|
}
|
|
392
|
+
// Removes a file or a directory (recursively); missing is fine.
|
|
380
393
|
if (req.method === "DELETE" && op === "/remove") {
|
|
381
|
-
try { fs.unlinkSync(full); } catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") return sendJson(500, { error: (e as Error).message }); }
|
|
382
|
-
return sendJson(200, { ok: true });
|
|
383
|
-
}
|
|
384
|
-
if (req.method === "DELETE" && op === "/removeDir") {
|
|
385
394
|
try { fs.rmSync(full, { recursive: true, force: true }); } catch (e) { return sendJson(500, { error: (e as Error).message }); }
|
|
386
395
|
return sendJson(200, { ok: true });
|
|
387
396
|
}
|
|
388
|
-
if (req.method === "POST" && op === "/reset") {
|
|
389
|
-
try { for (const name of fs.readdirSync(full)) fs.rmSync(path.join(full, name), { recursive: true, force: true }); }
|
|
390
|
-
catch (e) { if ((e as NodeJS.ErrnoException).code !== "ENOENT") return sendJson(500, { error: (e as Error).message }); }
|
|
391
|
-
return sendJson(200, { ok: true });
|
|
392
|
-
}
|
|
393
397
|
return sendJson(404, { error: "unknown endpoint" });
|
|
394
398
|
} catch (e) {
|
|
395
399
|
try { sendJson(500, { error: String((e as Error)?.message || e) }); } catch { /* response already sent */ }
|
|
@@ -1,41 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
/// <reference types="node" />
|
|
4
|
-
import https from "https";
|
|
5
|
-
import type { FileStorage } from "./FileFolderAPI";
|
|
6
|
-
export type RemoteFileStorageOptions = {
|
|
1
|
+
import type { DirectoryWrapper } from "./FileFolderAPI";
|
|
2
|
+
export type RemoteOptions = {
|
|
7
3
|
chunkBytes?: number;
|
|
8
4
|
cacheBytes?: number;
|
|
9
5
|
latencyMs?: number;
|
|
10
|
-
|
|
11
|
-
type Connection = {
|
|
12
|
-
url: string;
|
|
13
|
-
password: string;
|
|
14
|
-
latencyMs: number;
|
|
15
|
-
agent: https.Agent | undefined;
|
|
16
|
-
cache: RangeCache;
|
|
17
|
-
stats: {
|
|
6
|
+
stats?: {
|
|
18
7
|
requestCount: number;
|
|
19
8
|
bytesFetched: number;
|
|
20
9
|
};
|
|
21
10
|
};
|
|
22
|
-
declare
|
|
23
|
-
|
|
24
|
-
private budget;
|
|
25
|
-
private chunks;
|
|
26
|
-
private bytes;
|
|
27
|
-
constructor(chunkBytes: number, budget: number);
|
|
28
|
-
private key;
|
|
29
|
-
private peek;
|
|
30
|
-
private store;
|
|
31
|
-
invalidate(path: string): void;
|
|
32
|
-
getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined>;
|
|
33
|
-
}
|
|
34
|
-
export type RemoteStorageFactory = ((path: string) => Promise<FileStorage>) & {
|
|
35
|
-
stats: Connection["stats"];
|
|
36
|
-
};
|
|
37
|
-
export declare function getRemoteFileStorage(url: string, password: string, options?: RemoteFileStorageOptions): RemoteStorageFactory;
|
|
38
|
-
export type RemoteProbeResult = {
|
|
11
|
+
export declare function getRemoteDirectoryHandle(url: string, password: string, options?: RemoteOptions): DirectoryWrapper;
|
|
12
|
+
export type RemoteConnectResult = {
|
|
39
13
|
status: "ok";
|
|
40
14
|
} | {
|
|
41
15
|
status: "unauthorized";
|
|
@@ -43,5 +17,4 @@ export type RemoteProbeResult = {
|
|
|
43
17
|
status: "unreachable";
|
|
44
18
|
error: string;
|
|
45
19
|
};
|
|
46
|
-
export declare function
|
|
47
|
-
export {};
|
|
20
|
+
export declare function testRemoteConnection(url: string, password: string, options?: RemoteOptions): Promise<RemoteConnectResult>;
|
|
@@ -1,99 +1,140 @@
|
|
|
1
1
|
import { isNode } from "typesafecss";
|
|
2
2
|
import https from "https";
|
|
3
|
-
import type {
|
|
3
|
+
import type { DirectoryWrapper, FileWrapper } from "./FileFolderAPI";
|
|
4
4
|
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
5
|
+
// A remote server (remoteFileServer.js) exposed as a DirectoryWrapper — the SAME interface as the
|
|
6
|
+
// Node.js and File-System-Access handles. So getDirectoryHandle can return one of these and everything
|
|
7
|
+
// downstream (wrapHandle, navigation, BulkDatabase2) works unchanged; nothing else knows it's remote.
|
|
8
|
+
//
|
|
9
|
+
// Files here are immutable (written once) or append-only, so the range cache (raw compressed bytes, in
|
|
10
|
+
// ~1MB aligned chunks) is always valid — over the network latency dominates, so reading 1MB costs about
|
|
11
|
+
// the same as 64KB and far fewer round trips wins.
|
|
9
12
|
|
|
10
13
|
const EMPTY = Buffer.alloc(0);
|
|
11
|
-
|
|
12
|
-
// Over the network, fetch reads in chunks this big and cache them. Big because each request pays a
|
|
13
|
-
// full round trip; over-reading a bit is far cheaper than another round trip.
|
|
14
14
|
const DEFAULT_CHUNK_BYTES = 1024 * 1024;
|
|
15
15
|
const DEFAULT_CACHE_BYTES = 128 * 1024 * 1024;
|
|
16
16
|
|
|
17
|
-
export type
|
|
18
|
-
// Bytes per cached chunk (aligned). Reads coalesce/serve from these.
|
|
17
|
+
export type RemoteOptions = {
|
|
19
18
|
chunkBytes?: number;
|
|
20
|
-
// Max total bytes held in the range cache (LRU eviction).
|
|
21
19
|
cacheBytes?: number;
|
|
22
|
-
//
|
|
23
|
-
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
type Connection = {
|
|
27
|
-
url: string;
|
|
28
|
-
password: string;
|
|
29
|
-
latencyMs: number;
|
|
30
|
-
agent: https.Agent | undefined;
|
|
31
|
-
cache: RangeCache;
|
|
32
|
-
// Observable stats (handy for tests / diagnostics).
|
|
33
|
-
stats: { requestCount: number; bytesFetched: number };
|
|
20
|
+
latencyMs?: number; // artificial per-request delay, for simulating network latency in tests
|
|
21
|
+
stats?: { requestCount: number; bytesFetched: number };
|
|
34
22
|
};
|
|
35
23
|
|
|
24
|
+
function enoent(p: string): Error {
|
|
25
|
+
return Object.assign(new Error(`File not found: ${p}`), { code: "ENOENT", name: "NotFoundError" });
|
|
26
|
+
}
|
|
27
|
+
function toArrayBuffer(b: Buffer): ArrayBuffer {
|
|
28
|
+
return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength) as ArrayBuffer;
|
|
29
|
+
}
|
|
36
30
|
const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));
|
|
37
31
|
|
|
38
|
-
|
|
39
|
-
// uses fetch (the user accepts the self-signed cert once). Returns the raw status + body bytes.
|
|
40
|
-
async function httpRequest(conn: Connection, method: string, op: string, params: Record<string, string>, body?: Buffer): Promise<{ status: number; body: Buffer }> {
|
|
41
|
-
if (conn.latencyMs > 0) await sleep(conn.latencyMs);
|
|
42
|
-
conn.stats.requestCount++;
|
|
43
|
-
const qs = new URLSearchParams(params).toString();
|
|
44
|
-
const fullUrl = conn.url.replace(/\/+$/, "") + op + (qs ? "?" + qs : "");
|
|
45
|
-
const headers: Record<string, string> = { authorization: "Bearer " + conn.password };
|
|
46
|
-
if (body) headers["content-type"] = "application/octet-stream";
|
|
32
|
+
type Stat = { size: number; lastModified: number; dir: boolean };
|
|
47
33
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
34
|
+
// A stat is cached this long, so the burst of size lookups during a single read pass is one request per
|
|
35
|
+
// file, not one per block. Short enough that an append by another writer shows up promptly; our own
|
|
36
|
+
// writes invalidate it immediately.
|
|
37
|
+
const INFO_TTL_MS = 2000;
|
|
38
|
+
|
|
39
|
+
class Connection {
|
|
40
|
+
private agent = isNode() ? new https.Agent({ rejectUnauthorized: false }) : undefined;
|
|
41
|
+
private cache: RangeCache;
|
|
42
|
+
private infoCache = new Map<string, { stat: Stat | undefined; at: number }>();
|
|
43
|
+
constructor(public url: string, public password: string, private opts: RemoteOptions) {
|
|
44
|
+
this.cache = new RangeCache(opts.chunkBytes || DEFAULT_CHUNK_BYTES, opts.cacheBytes || DEFAULT_CACHE_BYTES);
|
|
45
|
+
this.url = url.replace(/\/+$/, "");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// One HTTP request. Node uses the https module (self-signed cert → verification disabled); the
|
|
49
|
+
// browser uses fetch (the user has accepted the cert). Returns the raw status + body bytes.
|
|
50
|
+
async request(method: string, op: string, params: Record<string, string>, body?: Buffer): Promise<{ status: number; body: Buffer }> {
|
|
51
|
+
if (this.opts.latencyMs) await sleep(this.opts.latencyMs);
|
|
52
|
+
if (this.opts.stats) this.opts.stats.requestCount++;
|
|
53
|
+
const qs = new URLSearchParams(params).toString();
|
|
54
|
+
const fullUrl = this.url + op + (qs ? "?" + qs : "");
|
|
55
|
+
const headers: Record<string, string> = { authorization: "Bearer " + this.password };
|
|
56
|
+
if (body) headers["content-type"] = "application/octet-stream";
|
|
57
|
+
if (isNode()) {
|
|
58
|
+
return await new Promise((resolve, reject) => {
|
|
59
|
+
const u = new URL(fullUrl);
|
|
60
|
+
const req = https.request({ hostname: u.hostname, port: u.port, path: u.pathname + u.search, method, headers, agent: this.agent }, res => {
|
|
61
|
+
const chunks: Buffer[] = [];
|
|
62
|
+
res.on("data", d => chunks.push(d as Buffer));
|
|
63
|
+
res.on("end", () => resolve({ status: res.statusCode || 0, body: Buffer.concat(chunks) }));
|
|
64
|
+
});
|
|
65
|
+
req.on("error", reject);
|
|
66
|
+
if (body) req.write(body);
|
|
67
|
+
req.end();
|
|
57
68
|
});
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
});
|
|
69
|
+
}
|
|
70
|
+
const res = await fetch(fullUrl, { method, headers, body: body ? new Uint8Array(body) : undefined });
|
|
71
|
+
return { status: res.status, body: Buffer.from(await res.arrayBuffer()) };
|
|
62
72
|
}
|
|
63
|
-
const res = await fetch(fullUrl, { method, headers, body: body ? new Uint8Array(body) : undefined });
|
|
64
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
65
|
-
return { status: res.status, body: buf };
|
|
66
|
-
}
|
|
67
73
|
|
|
68
|
-
async
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
async stat(path: string): Promise<Stat | undefined> {
|
|
75
|
+
const cached = this.infoCache.get(path);
|
|
76
|
+
if (cached && Date.now() - cached.at < INFO_TTL_MS) return cached.stat;
|
|
77
|
+
const r = await this.request("GET", "/info", { path });
|
|
78
|
+
let stat: Stat | undefined;
|
|
79
|
+
if (r.status === 404) stat = undefined;
|
|
80
|
+
else if (r.status !== 200) throw new Error(`remote info failed (${r.status})`);
|
|
81
|
+
else stat = JSON.parse(r.body.toString("utf8")) as Stat;
|
|
82
|
+
this.infoCache.set(path, { stat, at: Date.now() });
|
|
83
|
+
return stat;
|
|
84
|
+
}
|
|
85
|
+
async list(path: string): Promise<{ name: string; dir: boolean }[]> {
|
|
86
|
+
const r = await this.request("GET", "/list", { path });
|
|
87
|
+
if (r.status !== 200) throw new Error(`remote list failed (${r.status})`);
|
|
88
|
+
return JSON.parse(r.body.toString("utf8"));
|
|
89
|
+
}
|
|
90
|
+
async read(path: string, start: number, end: number): Promise<Buffer> {
|
|
91
|
+
return (await this.cache.read(this, path, start, end)) ?? EMPTY;
|
|
92
|
+
}
|
|
93
|
+
async readServer(path: string, start: number, end: number): Promise<Buffer | undefined> {
|
|
94
|
+
const r = await this.request("GET", "/read", { path, start: String(start), end: String(end) });
|
|
95
|
+
if (r.status === 404) return undefined;
|
|
96
|
+
if (r.status !== 200) throw new Error(`remote read failed (${r.status})`);
|
|
97
|
+
if (this.opts.stats) this.opts.stats.bytesFetched += r.body.length;
|
|
98
|
+
return r.body;
|
|
99
|
+
}
|
|
100
|
+
async append(path: string, body: Buffer): Promise<void> {
|
|
101
|
+
const r = await this.request("PUT", "/append", { path }, body);
|
|
102
|
+
if (r.status !== 200) throw new Error(`remote append failed (${r.status})`);
|
|
103
|
+
// Append-only keeps existing bytes, so cached chunks stay valid; just the size changed.
|
|
104
|
+
this.infoCache.delete(path);
|
|
105
|
+
}
|
|
106
|
+
async set(path: string, body: Buffer): Promise<void> {
|
|
107
|
+
const r = await this.request("PUT", "/set", { path }, body);
|
|
108
|
+
if (r.status !== 200) throw new Error(`remote set failed (${r.status})`);
|
|
109
|
+
this.cache.invalidate(path);
|
|
110
|
+
this.infoCache.delete(path);
|
|
111
|
+
}
|
|
112
|
+
async remove(path: string): Promise<void> {
|
|
113
|
+
const r = await this.request("DELETE", "/remove", { path });
|
|
114
|
+
if (r.status !== 200) throw new Error(`remote remove failed (${r.status})`);
|
|
115
|
+
this.cache.invalidate(path);
|
|
116
|
+
this.infoCache.delete(path);
|
|
117
|
+
}
|
|
74
118
|
}
|
|
75
119
|
|
|
76
|
-
// Caches the longest-known prefix of each aligned chunk per path.
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
// that needs MORE than we have refetches (and picks up appended bytes). It never distinguishes file
|
|
80
|
-
// types — it only does range reads.
|
|
120
|
+
// Caches the longest-known prefix of each aligned chunk per path. Valid because files are immutable or
|
|
121
|
+
// append-only (the bytes at an offset never change). It only does range reads — it doesn't care what
|
|
122
|
+
// the files are.
|
|
81
123
|
class RangeCache {
|
|
82
|
-
private chunks = new Map<string, Buffer>(); //
|
|
124
|
+
private chunks = new Map<string, Buffer>(); // path + "" + chunkIndex, insertion-ordered (LRU)
|
|
83
125
|
private bytes = 0;
|
|
84
126
|
constructor(private chunkBytes: number, private budget: number) { }
|
|
85
|
-
|
|
86
127
|
private key(path: string, c: number) { return path + "" + c; }
|
|
87
128
|
private peek(path: string, c: number): Buffer | undefined {
|
|
88
129
|
const k = this.key(path, c);
|
|
89
130
|
const v = this.chunks.get(k);
|
|
90
|
-
if (v) { this.chunks.delete(k); this.chunks.set(k, v); }
|
|
131
|
+
if (v) { this.chunks.delete(k); this.chunks.set(k, v); }
|
|
91
132
|
return v;
|
|
92
133
|
}
|
|
93
134
|
private store(path: string, c: number, buf: Buffer) {
|
|
94
135
|
const k = this.key(path, c);
|
|
95
136
|
const ex = this.chunks.get(k);
|
|
96
|
-
if (ex && ex.length >= buf.length) { this.chunks.delete(k); this.chunks.set(k, ex); return; }
|
|
137
|
+
if (ex && ex.length >= buf.length) { this.chunks.delete(k); this.chunks.set(k, ex); return; }
|
|
97
138
|
if (ex) this.bytes -= ex.length;
|
|
98
139
|
this.chunks.delete(k);
|
|
99
140
|
this.chunks.set(k, buf);
|
|
@@ -106,17 +147,13 @@ class RangeCache {
|
|
|
106
147
|
}
|
|
107
148
|
invalidate(path: string) {
|
|
108
149
|
const prefix = path + "";
|
|
109
|
-
for (const k of [...this.chunks.keys()]) {
|
|
110
|
-
if (k.startsWith(prefix)) { this.bytes -= (this.chunks.get(k) as Buffer).length; this.chunks.delete(k); }
|
|
111
|
-
}
|
|
150
|
+
for (const k of [...this.chunks.keys()]) if (k.startsWith(prefix)) { this.bytes -= (this.chunks.get(k) as Buffer).length; this.chunks.delete(k); }
|
|
112
151
|
}
|
|
113
|
-
|
|
114
|
-
async getRange(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined> {
|
|
152
|
+
async read(conn: Connection, path: string, start: number, end: number): Promise<Buffer | undefined> {
|
|
115
153
|
if (end <= start) return EMPTY;
|
|
116
154
|
const CHUNK = this.chunkBytes;
|
|
117
155
|
const firstChunk = Math.floor(start / CHUNK);
|
|
118
156
|
const lastChunk = Math.floor((end - 1) / CHUNK);
|
|
119
|
-
// Find the contiguous span of chunks we don't have enough of, and fetch it in one request.
|
|
120
157
|
let fetchFrom = -1, fetchTo = -1;
|
|
121
158
|
for (let c = firstChunk; c <= lastChunk; c++) {
|
|
122
159
|
const cStart = c * CHUNK;
|
|
@@ -125,11 +162,11 @@ class RangeCache {
|
|
|
125
162
|
if (!have || have.length < needEnd) { if (fetchFrom < 0) fetchFrom = c; fetchTo = c; }
|
|
126
163
|
}
|
|
127
164
|
if (fetchFrom >= 0) {
|
|
128
|
-
const bytes = await
|
|
129
|
-
if (bytes === undefined) return undefined;
|
|
165
|
+
const bytes = await conn.readServer(path, fetchFrom * CHUNK, (fetchTo + 1) * CHUNK);
|
|
166
|
+
if (bytes === undefined) return undefined;
|
|
130
167
|
for (let c = fetchFrom; c <= fetchTo; c++) {
|
|
131
168
|
const off = (c - fetchFrom) * CHUNK;
|
|
132
|
-
if (off >= bytes.length) break;
|
|
169
|
+
if (off >= bytes.length) break;
|
|
133
170
|
this.store(path, c, bytes.subarray(off, Math.min(off + CHUNK, bytes.length)));
|
|
134
171
|
}
|
|
135
172
|
}
|
|
@@ -139,122 +176,99 @@ class RangeCache {
|
|
|
139
176
|
const from = Math.max(start, cStart) - cStart;
|
|
140
177
|
const to = Math.min(end, cStart + CHUNK) - cStart;
|
|
141
178
|
const chunk = this.peek(path, c);
|
|
142
|
-
if (!chunk || chunk.length < to) {
|
|
143
|
-
// File ended before the requested end — return what's available (callers read within EOF).
|
|
144
|
-
if (chunk && chunk.length > from) parts.push(chunk.subarray(from, chunk.length));
|
|
145
|
-
break;
|
|
146
|
-
}
|
|
179
|
+
if (!chunk || chunk.length < to) { if (chunk && chunk.length > from) parts.push(chunk.subarray(from, chunk.length)); break; }
|
|
147
180
|
parts.push(chunk.subarray(from, to));
|
|
148
181
|
}
|
|
149
182
|
return parts.length === 1 ? parts[0] : Buffer.concat(parts);
|
|
150
183
|
}
|
|
151
184
|
}
|
|
152
185
|
|
|
153
|
-
|
|
154
|
-
const rel = (key: string) => basePath ? basePath + "/" + key : key;
|
|
186
|
+
const joinPath = (base: string, key: string) => (base ? base + "/" + key : key);
|
|
155
187
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
},
|
|
188
|
-
async getRange(key: string, config: { start: number; end: number }) {
|
|
189
|
-
return conn.cache.getRange(conn, rel(key), config.start, config.end);
|
|
190
|
-
},
|
|
191
|
-
async append(key: string, value: Buffer) {
|
|
192
|
-
const r = await httpRequest(conn, "PUT", "/append", { path: rel(key) }, value);
|
|
193
|
-
if (r.status !== 200) throw new Error(`remote append failed (${r.status})`);
|
|
194
|
-
// No invalidation needed: append-only files keep their prefix; a read past it refetches.
|
|
195
|
-
},
|
|
196
|
-
async set(key: string, value: Buffer) {
|
|
197
|
-
const r = await httpRequest(conn, "PUT", "/set", { path: rel(key) }, value);
|
|
198
|
-
if (r.status !== 200) throw new Error(`remote set failed (${r.status})`);
|
|
199
|
-
conn.cache.invalidate(rel(key));
|
|
200
|
-
},
|
|
201
|
-
async remove(key: string) {
|
|
202
|
-
await httpRequest(conn, "DELETE", "/remove", { path: rel(key) });
|
|
203
|
-
conn.cache.invalidate(rel(key));
|
|
204
|
-
},
|
|
205
|
-
async getKeys(includeFolders: boolean = false) {
|
|
206
|
-
const r = await httpRequest(conn, "GET", "/list", { path: basePath, folders: includeFolders ? "1" : "0" });
|
|
207
|
-
if (r.status !== 200) throw new Error(`remote list failed (${r.status})`);
|
|
208
|
-
return JSON.parse(r.body.toString("utf8"));
|
|
209
|
-
},
|
|
210
|
-
async reset() {
|
|
211
|
-
await httpRequest(conn, "POST", "/reset", { path: basePath });
|
|
212
|
-
conn.cache.invalidate(basePath);
|
|
213
|
-
},
|
|
214
|
-
folder,
|
|
215
|
-
};
|
|
188
|
+
class RemoteFileWrapper implements FileWrapper {
|
|
189
|
+
// `stat` is supplied when the parent already statted us (avoids a second /info). `createIntent` means
|
|
190
|
+
// this was opened with create:true, so a missing file reads as empty (it'll be created on write).
|
|
191
|
+
constructor(private conn: Connection, private filePath: string, private stat?: Stat, private createIntent = false) { }
|
|
192
|
+
async getFile() {
|
|
193
|
+
let stat = this.stat ?? await this.conn.stat(this.filePath);
|
|
194
|
+
if (!stat) {
|
|
195
|
+
if (!this.createIntent) throw enoent(this.filePath);
|
|
196
|
+
stat = { size: 0, lastModified: Date.now(), dir: false };
|
|
197
|
+
}
|
|
198
|
+
const conn = this.conn, filePath = this.filePath, size = stat.size;
|
|
199
|
+
return {
|
|
200
|
+
size, lastModified: stat.lastModified,
|
|
201
|
+
async arrayBuffer() { return toArrayBuffer(await conn.read(filePath, 0, size)); },
|
|
202
|
+
slice(start: number, end: number) {
|
|
203
|
+
return { async arrayBuffer() { return toArrayBuffer(await conn.read(filePath, start, end)); } };
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
async createWritable(config?: { keepExistingData?: boolean }) {
|
|
208
|
+
const conn = this.conn, filePath = this.filePath, append = !!config?.keepExistingData;
|
|
209
|
+
const chunks: Buffer[] = [];
|
|
210
|
+
return {
|
|
211
|
+
async seek() { /* the server appends to the end / overwrites the whole file; offset unused */ },
|
|
212
|
+
async write(value: Buffer) { chunks.push(value); },
|
|
213
|
+
async close() {
|
|
214
|
+
const body = chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
|
|
215
|
+
if (append) await conn.append(filePath, body); else await conn.set(filePath, body);
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
216
219
|
}
|
|
217
220
|
|
|
218
|
-
|
|
221
|
+
class RemoteDirectoryWrapper implements DirectoryWrapper {
|
|
222
|
+
constructor(private conn: Connection, private dirPath: string) { }
|
|
223
|
+
async removeEntry(key: string): Promise<void> {
|
|
224
|
+
await this.conn.remove(joinPath(this.dirPath, key));
|
|
225
|
+
}
|
|
226
|
+
async getFileHandle(key: string, options?: { create?: boolean }): Promise<FileWrapper> {
|
|
227
|
+
const p = joinPath(this.dirPath, key);
|
|
228
|
+
if (options?.create) return new RemoteFileWrapper(this.conn, p, undefined, true);
|
|
229
|
+
const stat = await this.conn.stat(p); // matches the File API: throw if missing
|
|
230
|
+
if (!stat || stat.dir) throw enoent(p);
|
|
231
|
+
return new RemoteFileWrapper(this.conn, p, stat, false);
|
|
232
|
+
}
|
|
233
|
+
async getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper> {
|
|
234
|
+
const p = joinPath(this.dirPath, key);
|
|
235
|
+
if (!options?.create) {
|
|
236
|
+
const stat = await this.conn.stat(p);
|
|
237
|
+
if (!stat || !stat.dir) throw enoent(p);
|
|
238
|
+
}
|
|
239
|
+
return new RemoteDirectoryWrapper(this.conn, p); // dirs are created lazily on first write
|
|
240
|
+
}
|
|
241
|
+
async *[Symbol.asyncIterator](): AsyncIterableIterator<[string, { kind: "file"; name: string; getFile(): Promise<FileWrapper> } | { kind: "directory"; name: string; getDirectoryHandle(key: string, options?: { create?: boolean }): Promise<DirectoryWrapper> }]> {
|
|
242
|
+
const entries = await this.conn.list(this.dirPath);
|
|
243
|
+
for (const e of entries) {
|
|
244
|
+
const childPath = joinPath(this.dirPath, e.name);
|
|
245
|
+
if (e.dir) {
|
|
246
|
+
yield [e.name, { kind: "directory", name: e.name, getDirectoryHandle: (k, o) => new RemoteDirectoryWrapper(this.conn, childPath).getDirectoryHandle(k, o) }];
|
|
247
|
+
} else {
|
|
248
|
+
yield [e.name, { kind: "file", name: e.name, getFile: async () => new RemoteFileWrapper(this.conn, childPath) }];
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
219
253
|
|
|
220
|
-
//
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const conn: Connection = {
|
|
224
|
-
url,
|
|
225
|
-
password,
|
|
226
|
-
latencyMs: options.latencyMs || 0,
|
|
227
|
-
// Self-signed cert: skip verification in Node. (The browser path uses fetch, where the user has
|
|
228
|
-
// already accepted the cert.) The password is what authorizes access.
|
|
229
|
-
agent: isNode() ? new https.Agent({ rejectUnauthorized: false }) : undefined,
|
|
230
|
-
cache: new RangeCache(options.chunkBytes || DEFAULT_CHUNK_BYTES, options.cacheBytes || DEFAULT_CACHE_BYTES),
|
|
231
|
-
stats: { requestCount: 0, bytesFetched: 0 },
|
|
232
|
-
};
|
|
233
|
-
const factory = ((path: string) => Promise.resolve(makeStorage(conn, path.replace(/^\/+|\/+$/g, "")))) as RemoteStorageFactory;
|
|
234
|
-
factory.stats = conn.stats;
|
|
235
|
-
return factory;
|
|
254
|
+
// A DirectoryWrapper rooted at a remote server. Drop-in for the Node / File-API handles.
|
|
255
|
+
export function getRemoteDirectoryHandle(url: string, password: string, options: RemoteOptions = {}): DirectoryWrapper {
|
|
256
|
+
return new RemoteDirectoryWrapper(new Connection(url, password, options), "");
|
|
236
257
|
}
|
|
237
258
|
|
|
238
|
-
export type
|
|
239
|
-
| { status: "ok" }
|
|
240
|
-
| { status: "unauthorized" }
|
|
241
|
-
// Couldn't reach the server at all — in the browser this is usually the self-signed certificate not
|
|
242
|
-
// being trusted yet (fetch throws with no detail), but also covers a wrong URL / server down / CORS.
|
|
243
|
-
| { status: "unreachable"; error: string };
|
|
259
|
+
export type RemoteConnectResult = { status: "ok" } | { status: "unauthorized" } | { status: "unreachable"; error: string };
|
|
244
260
|
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
export async function
|
|
261
|
+
// Verifies a server is reachable and the password works — by actually listing the root. Distinguishes
|
|
262
|
+
// "connected" / "wrong password" / "couldn't reach it" (the last usually meaning the self-signed cert
|
|
263
|
+
// isn't trusted yet in the browser, since fetch failures carry no detail).
|
|
264
|
+
export async function testRemoteConnection(url: string, password: string, options: RemoteOptions = {}): Promise<RemoteConnectResult> {
|
|
265
|
+
const conn = new Connection(url, password, options);
|
|
249
266
|
try {
|
|
250
|
-
|
|
251
|
-
return { status: "ok" };
|
|
267
|
+
const r = await conn.request("GET", "/list", { path: "" });
|
|
268
|
+
if (r.status === 200) return { status: "ok" };
|
|
269
|
+
if (r.status === 401) return { status: "unauthorized" };
|
|
270
|
+
return { status: "unreachable", error: `server returned ${r.status}` };
|
|
252
271
|
} catch (e) {
|
|
253
|
-
|
|
254
|
-
// Our client throws "remote ... failed (NNN)" when it actually got an HTTP response (so the cert
|
|
255
|
-
// is trusted); a thrown fetch with no status means we never connected.
|
|
256
|
-
const m = msg.match(/\((\d{3})\)/);
|
|
257
|
-
if (m) return m[1] === "401" ? { status: "unauthorized" } : { status: "unreachable", error: msg };
|
|
258
|
-
return { status: "unreachable", error: msg };
|
|
272
|
+
return { status: "unreachable", error: (e as Error)?.message || String(e) };
|
|
259
273
|
}
|
|
260
274
|
}
|