sliftutils 1.7.9 → 1.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/storageserver.js +4 -0
- package/{teststorage → examplestorage}/browser.tsx +18 -22
- package/{teststorage/server.ts → examplestorage/exampleserver.ts} +4 -6
- package/index.d.ts +770 -84
- package/misc/dist/environment.ts.cache +3 -3
- package/misc/dist/getSecret.ts.cache +139 -0
- package/misc/dist/strings.ts.cache +13 -0
- package/misc/dist/zip.ts.cache +9 -0
- package/misc/getSecret.ts +38 -16
- package/misc/https/cloudflareHelpers.d.ts +19 -0
- package/misc/https/cloudflareHelpers.ts +69 -0
- package/misc/https/dist/certs.ts.cache +115 -179
- package/misc/https/dist/cloudflareHelpers.ts.cache +68 -0
- package/misc/https/dist/dns.ts.cache +112 -100
- package/misc/https/dist/hostServer.ts.cache +130 -0
- package/misc/https/dist/httpsCerts.ts.cache +24 -5
- package/misc/https/dist/persistentLocalStorage.ts.cache +3 -2
- package/misc/https/dns.d.ts +41 -8
- package/misc/https/dns.ts +104 -106
- package/misc/https/hostServer.d.ts +0 -3
- package/misc/https/hostServer.ts +12 -8
- package/package.json +3 -2
- package/render-utils/dist/observer.tsx.cache +3 -2
- package/spec.txt +0 -55
- package/storage/ArchivesDisk.d.ts +67 -0
- package/storage/ArchivesDisk.ts +369 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +1242 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +513 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +352 -0
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +306 -0
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +435 -0
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +58 -0
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +179 -0
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +149 -0
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +101 -0
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +227 -0
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +104 -0
- package/storage/IArchives.d.ts +127 -1
- package/storage/IArchives.ts +201 -1
- package/storage/backblaze.d.ts +29 -2
- package/storage/backblaze.ts +214 -78
- package/storage/dist/ArchivesDisk.ts.cache +373 -0
- package/storage/dist/FileFolderAPI.tsx.cache +508 -65
- package/storage/dist/IArchives.ts.cache +44 -0
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +8 -2
- package/storage/dist/JSONStorage.ts.cache +6 -2
- package/storage/dist/PendingManager.tsx.cache +1 -1
- package/storage/dist/TransactionStorage.ts.cache +231 -92
- package/storage/dist/backblaze.ts.cache +797 -0
- package/storage/dist/fileSystemPointer.ts.cache +36 -3
- package/storage/dist/remoteFileStorage.ts.cache +495 -0
- package/storage/remoteStorage/ArchivesRemote.d.ts +34 -18
- package/storage/remoteStorage/ArchivesRemote.ts +85 -65
- package/storage/remoteStorage/ArchivesUrl.d.ts +48 -0
- package/storage/remoteStorage/ArchivesUrl.ts +86 -0
- package/storage/remoteStorage/accessPage.tsx +111 -28
- package/storage/remoteStorage/blobStore.d.ts +128 -25
- package/storage/remoteStorage/blobStore.ts +757 -290
- package/storage/remoteStorage/cliArgs.d.ts +1 -0
- package/storage/remoteStorage/cliArgs.ts +9 -0
- package/storage/remoteStorage/createArchives.d.ts +85 -0
- package/storage/remoteStorage/createArchives.ts +612 -0
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +197 -0
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +85 -0
- package/storage/remoteStorage/dist/accessPage.tsx.cache +222 -0
- package/storage/remoteStorage/dist/blobStore.ts.cache +837 -0
- package/storage/remoteStorage/dist/cliArgs.ts.cache +16 -0
- package/storage/remoteStorage/dist/createArchives.ts.cache +610 -0
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +206 -0
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +223 -0
- package/storage/remoteStorage/dist/storageController.ts.cache +500 -0
- package/storage/remoteStorage/dist/storageServer.ts.cache +106 -0
- package/storage/remoteStorage/dist/storageServerCli.ts.cache +39 -0
- package/storage/remoteStorage/dist/storageServerState.ts.cache +518 -0
- package/storage/remoteStorage/grantAccess.js +4 -0
- package/storage/remoteStorage/grantAccessCli.d.ts +1 -0
- package/storage/remoteStorage/grantAccessCli.ts +27 -0
- package/storage/remoteStorage/remoteConfig.d.ts +29 -0
- package/storage/remoteStorage/remoteConfig.ts +194 -0
- package/storage/remoteStorage/sourceWrapper.d.ts +41 -0
- package/storage/remoteStorage/sourceWrapper.ts +223 -0
- package/storage/remoteStorage/spec.md +31 -0
- package/storage/remoteStorage/storageController.d.ts +30 -27
- package/storage/remoteStorage/storageController.ts +297 -159
- package/storage/remoteStorage/storageServer.d.ts +6 -0
- package/storage/remoteStorage/storageServer.ts +88 -94
- package/storage/remoteStorage/storageServerCli.d.ts +1 -0
- package/storage/remoteStorage/storageServerCli.ts +38 -0
- package/storage/remoteStorage/storageServerState.d.ts +43 -0
- package/storage/remoteStorage/storageServerState.ts +540 -0
- package/testsite/server.ts +0 -2
- package/yarn.lock +11 -6
|
@@ -1,22 +1,15 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { IArchives, ArchiveFileInfo } from "../IArchives";
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
|
|
4
4
|
export type ArchivesRemoteConfig = {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
bucketName: string;
|
|
9
|
-
public?: boolean;
|
|
10
|
-
fast?: boolean;
|
|
11
|
-
writeDelay?: number;
|
|
5
|
+
url: string;
|
|
6
|
+
accountName?: string;
|
|
7
|
+
waitForAccess?: boolean;
|
|
12
8
|
};
|
|
13
|
-
export declare function
|
|
9
|
+
export declare function parseStorageUrl(url: string): {
|
|
14
10
|
address: string;
|
|
15
11
|
port: number;
|
|
16
|
-
|
|
17
|
-
bucketName: string;
|
|
18
|
-
path: string;
|
|
19
|
-
}): string;
|
|
12
|
+
};
|
|
20
13
|
export declare function authenticateStorage(config: {
|
|
21
14
|
address: string;
|
|
22
15
|
port: number;
|
|
@@ -28,16 +21,24 @@ export declare function authenticateStorage(config: {
|
|
|
28
21
|
export declare class ArchivesRemote implements IArchives {
|
|
29
22
|
private config;
|
|
30
23
|
constructor(config: ArchivesRemoteConfig);
|
|
24
|
+
private parsed;
|
|
25
|
+
private account;
|
|
26
|
+
private bucketName;
|
|
31
27
|
private nodeId;
|
|
32
28
|
private controller;
|
|
33
|
-
private setupDone;
|
|
34
29
|
private lastDeniedLog;
|
|
35
30
|
getDebugName(): string;
|
|
31
|
+
isConnected(): boolean;
|
|
32
|
+
ping(): Promise<void>;
|
|
36
33
|
private authenticate;
|
|
37
34
|
private callAuthed;
|
|
38
|
-
waitingForAccess(): Promise<
|
|
39
|
-
|
|
40
|
-
|
|
35
|
+
waitingForAccess(): Promise<{
|
|
36
|
+
link: string;
|
|
37
|
+
machineId: string;
|
|
38
|
+
ip: string;
|
|
39
|
+
} | undefined>;
|
|
40
|
+
hasWriteAccess(): Promise<boolean>;
|
|
41
|
+
private registerAccessRequest;
|
|
41
42
|
private call;
|
|
42
43
|
get(fileName: string, config?: {
|
|
43
44
|
range?: {
|
|
@@ -45,7 +46,19 @@ export declare class ArchivesRemote implements IArchives {
|
|
|
45
46
|
end: number;
|
|
46
47
|
};
|
|
47
48
|
}): Promise<Buffer | undefined>;
|
|
48
|
-
|
|
49
|
+
get2(fileName: string, config?: {
|
|
50
|
+
range?: {
|
|
51
|
+
start: number;
|
|
52
|
+
end: number;
|
|
53
|
+
};
|
|
54
|
+
}): Promise<{
|
|
55
|
+
data: Buffer;
|
|
56
|
+
writeTime: number;
|
|
57
|
+
size: number;
|
|
58
|
+
} | undefined>;
|
|
59
|
+
set(fileName: string, data: Buffer, config?: {
|
|
60
|
+
lastModified?: number;
|
|
61
|
+
}): Promise<void>;
|
|
49
62
|
del(fileName: string): Promise<void>;
|
|
50
63
|
getInfo(fileName: string): Promise<{
|
|
51
64
|
writeTime: number;
|
|
@@ -59,6 +72,9 @@ export declare class ArchivesRemote implements IArchives {
|
|
|
59
72
|
shallow?: boolean;
|
|
60
73
|
type: "files" | "folders";
|
|
61
74
|
}): Promise<string[]>;
|
|
75
|
+
getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
|
|
76
|
+
getConfig(): Promise<ArchivesConfig>;
|
|
77
|
+
getSyncStatus(): Promise<ArchivesSyncStatus>;
|
|
62
78
|
setLargeFile(config: {
|
|
63
79
|
path: string;
|
|
64
80
|
getNextData(): Promise<Buffer | undefined>;
|
|
@@ -4,37 +4,38 @@ import { SocketFunction } from "socket-function/SocketFunction";
|
|
|
4
4
|
import { timeInMinute } from "socket-function/src/misc";
|
|
5
5
|
import { delay } from "socket-function/src/batching";
|
|
6
6
|
import { getIdentityCA, loadIdentityCA, sign } from "../../misc/https/certs";
|
|
7
|
-
import { IArchives, ArchiveFileInfo } from "../IArchives";
|
|
7
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
|
|
8
|
+
import { parseHostedUrl, getBucketBaseUrl, buildFileUrl } from "./remoteConfig";
|
|
8
9
|
import {
|
|
9
|
-
RemoteStorageController,
|
|
10
|
+
RemoteStorageController, STORAGE_AUTH_PURPOSE,
|
|
10
11
|
STORAGE_NOT_AUTHENTICATED, STORAGE_ACCESS_DENIED,
|
|
11
12
|
} from "./storageController";
|
|
12
13
|
|
|
13
14
|
// A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in
|
|
14
15
|
// Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account
|
|
15
|
-
// hasn't trusted this machine yet it requests access, waits,
|
|
16
|
-
// (calls block until access is granted).
|
|
16
|
+
// hasn't trusted this machine yet it requests access, and by default waits, logging instructions
|
|
17
|
+
// every minute (calls block until access is granted).
|
|
17
18
|
|
|
18
19
|
const ACCESS_RETRY_DELAY = 1000 * 30;
|
|
19
20
|
const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
|
|
20
21
|
|
|
21
22
|
export type ArchivesRemoteConfig = {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
//
|
|
29
|
-
|
|
30
|
-
// writes that haven't flushed yet.
|
|
31
|
-
fast?: boolean;
|
|
32
|
-
writeDelay?: number;
|
|
23
|
+
// The bucket's routing URL, which addresses the server, account, and bucket in one:
|
|
24
|
+
// https://storage.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json
|
|
25
|
+
url: string;
|
|
26
|
+
// Access is checked against this account (defaults to the account in the url)
|
|
27
|
+
accountName?: string;
|
|
28
|
+
// false: access-denied calls throw immediately (the error includes the access page link)
|
|
29
|
+
// instead of requesting access and blocking until it is granted (the default).
|
|
30
|
+
waitForAccess?: boolean;
|
|
33
31
|
};
|
|
34
32
|
|
|
35
|
-
export function
|
|
36
|
-
let
|
|
37
|
-
|
|
33
|
+
export function parseStorageUrl(url: string): { address: string; port: number } {
|
|
34
|
+
let u = new URL(url);
|
|
35
|
+
if (u.protocol !== "https:") {
|
|
36
|
+
throw new Error(`Storage URL must use https, got ${JSON.stringify(u.protocol)} in ${JSON.stringify(url)}`);
|
|
37
|
+
}
|
|
38
|
+
return { address: u.hostname, port: +u.port || 443 };
|
|
38
39
|
}
|
|
39
40
|
|
|
40
41
|
// Authenticates a connection to a storage server with this machine's certs.ts identity
|
|
@@ -61,17 +62,27 @@ export class ArchivesRemote implements IArchives {
|
|
|
61
62
|
SocketFunction.ENABLE_CLIENT_MODE = true;
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
private
|
|
65
|
+
private parsed = parseHostedUrl(this.config.url);
|
|
66
|
+
private account = this.config.accountName || this.parsed.account;
|
|
67
|
+
private bucketName = this.parsed.bucketName;
|
|
68
|
+
private nodeId = SocketFunction.connect({ address: this.parsed.address, port: this.parsed.port });
|
|
65
69
|
private controller = RemoteStorageController.nodes[this.nodeId];
|
|
66
|
-
private setupDone = false;
|
|
67
70
|
private lastDeniedLog = 0;
|
|
68
71
|
|
|
69
72
|
public getDebugName() {
|
|
70
|
-
return `remoteStorage/${this.
|
|
73
|
+
return `remoteStorage/${this.parsed.address}:${this.parsed.port}/${this.account}/${this.bucketName}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
public isConnected(): boolean {
|
|
77
|
+
return SocketFunction.isNodeConnected(this.nodeId);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
public async ping(): Promise<void> {
|
|
81
|
+
await this.controller.ping();
|
|
71
82
|
}
|
|
72
83
|
|
|
73
84
|
private async authenticate(): Promise<void> {
|
|
74
|
-
await authenticateStorage({ address: this.
|
|
85
|
+
await authenticateStorage({ address: this.parsed.address, port: this.parsed.port, nodeId: this.nodeId });
|
|
75
86
|
}
|
|
76
87
|
|
|
77
88
|
// Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike
|
|
@@ -87,50 +98,54 @@ export class ArchivesRemote implements IArchives {
|
|
|
87
98
|
}
|
|
88
99
|
|
|
89
100
|
// Returns undefined if this machine has access to the account. Otherwise puts in an access
|
|
90
|
-
// request and returns
|
|
91
|
-
|
|
92
|
-
|
|
101
|
+
// request and returns our machineId + ip (so the caller can display them alongside the link,
|
|
102
|
+
// for the approver to match the incoming request) and the link to the grant page.
|
|
103
|
+
public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
|
|
104
|
+
let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
|
|
93
105
|
if (state.hasAccess) return undefined;
|
|
94
|
-
await this.callAuthed(() => this.controller.requestAccess(this.
|
|
95
|
-
return
|
|
106
|
+
let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
|
|
107
|
+
return {
|
|
108
|
+
link: `https://${this.parsed.address}:${this.parsed.port}/${this.account}`,
|
|
109
|
+
machineId: requested.machineId,
|
|
110
|
+
ip: requested.ip,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
public async hasWriteAccess(): Promise<boolean> {
|
|
115
|
+
let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
|
|
116
|
+
return !!state.hasAccess;
|
|
96
117
|
}
|
|
97
118
|
|
|
98
|
-
|
|
99
|
-
|
|
119
|
+
// Registers our access request server-side (so an admin has a requestId to grant) and logs the
|
|
120
|
+
// grant instructions, at most once a minute
|
|
121
|
+
private async registerAccessRequest(): Promise<void> {
|
|
122
|
+
let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
|
|
100
123
|
if (Date.now() - this.lastDeniedLog > timeInMinute) {
|
|
101
124
|
this.lastDeniedLog = Date.now();
|
|
102
|
-
console.log(`No access to storage account ${JSON.stringify(this.
|
|
125
|
+
console.log(`No access to storage account ${JSON.stringify(this.account)} on ${this.parsed.address}:${this.parsed.port} (our machine ${requested.machineId}, ip ${requested.ip}). See https://${this.parsed.address}:${this.parsed.port}/${this.account} - or grant it with: ${requested.grantAccessCommand}`);
|
|
103
126
|
}
|
|
104
|
-
await delay(ACCESS_RETRY_DELAY);
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
private async ensureSetup(): Promise<void> {
|
|
108
|
-
if (this.setupDone) return;
|
|
109
|
-
await this.controller.ensureBucket(this.config.account, this.config.bucketName, {
|
|
110
|
-
public: this.config.public,
|
|
111
|
-
fast: this.config.fast,
|
|
112
|
-
writeDelay: this.config.writeDelay,
|
|
113
|
-
});
|
|
114
|
-
this.setupDone = true;
|
|
115
127
|
}
|
|
116
128
|
|
|
117
129
|
// Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
|
|
118
|
-
// access as needed.
|
|
130
|
+
// access as needed. With waitForAccess false, denied calls throw immediately instead - but the
|
|
131
|
+
// access request is still registered (in the background), so the denial is grantable.
|
|
119
132
|
private async call<T>(fnc: () => Promise<T>): Promise<T> {
|
|
120
133
|
while (true) {
|
|
121
134
|
try {
|
|
122
|
-
await this.ensureSetup();
|
|
123
135
|
return await fnc();
|
|
124
136
|
} catch (e: any) {
|
|
125
137
|
let message = String(e.stack || e);
|
|
126
138
|
if (message.includes(STORAGE_NOT_AUTHENTICATED)) {
|
|
127
|
-
this.setupDone = false;
|
|
128
139
|
await this.authenticate();
|
|
129
140
|
continue;
|
|
130
141
|
}
|
|
131
142
|
if (message.includes(STORAGE_ACCESS_DENIED)) {
|
|
132
|
-
this.
|
|
133
|
-
|
|
143
|
+
if (this.config.waitForAccess === false) {
|
|
144
|
+
void this.registerAccessRequest().catch(() => { });
|
|
145
|
+
throw e;
|
|
146
|
+
}
|
|
147
|
+
await this.registerAccessRequest();
|
|
148
|
+
await delay(ACCESS_RETRY_DELAY);
|
|
134
149
|
continue;
|
|
135
150
|
}
|
|
136
151
|
throw e;
|
|
@@ -139,30 +154,43 @@ export class ArchivesRemote implements IArchives {
|
|
|
139
154
|
}
|
|
140
155
|
|
|
141
156
|
public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
|
|
142
|
-
let result = await this.
|
|
143
|
-
return result &&
|
|
157
|
+
let result = await this.get2(fileName, config);
|
|
158
|
+
return result && result.data || undefined;
|
|
144
159
|
}
|
|
145
|
-
public async
|
|
146
|
-
await this.call(() => this.controller.
|
|
160
|
+
public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
|
|
161
|
+
let result = await this.call(() => this.controller.get2(this.account, this.bucketName, fileName, config?.range));
|
|
162
|
+
return result && { data: Buffer.from(result.data), writeTime: result.writeTime, size: result.size } || undefined;
|
|
163
|
+
}
|
|
164
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
165
|
+
await this.call(() => this.controller.set(this.account, this.bucketName, fileName, data, config?.lastModified));
|
|
147
166
|
}
|
|
148
167
|
public async del(fileName: string): Promise<void> {
|
|
149
|
-
await this.call(() => this.controller.del(this.
|
|
168
|
+
await this.call(() => this.controller.del(this.account, this.bucketName, fileName));
|
|
150
169
|
}
|
|
151
170
|
public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
|
|
152
|
-
return await this.call(() => this.controller.getInfo(this.
|
|
171
|
+
return await this.call(() => this.controller.getInfo(this.account, this.bucketName, fileName));
|
|
153
172
|
}
|
|
154
173
|
public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
|
|
155
|
-
return await this.call(() => this.controller.findInfo(this.
|
|
174
|
+
return await this.call(() => this.controller.findInfo(this.account, this.bucketName, prefix, config));
|
|
156
175
|
}
|
|
157
176
|
public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
|
|
158
177
|
return (await this.findInfo(prefix, config)).map(x => x.path);
|
|
159
178
|
}
|
|
179
|
+
public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
|
|
180
|
+
return await this.call(() => this.controller.getChangesAfter(this.account, this.bucketName, time));
|
|
181
|
+
}
|
|
182
|
+
public async getConfig(): Promise<ArchivesConfig> {
|
|
183
|
+
return await this.call(() => this.controller.getArchivesConfig(this.account, this.bucketName));
|
|
184
|
+
}
|
|
185
|
+
public async getSyncStatus(): Promise<ArchivesSyncStatus> {
|
|
186
|
+
return await this.call(() => this.controller.getSyncStatus(this.account, this.bucketName));
|
|
187
|
+
}
|
|
160
188
|
|
|
161
189
|
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
|
|
162
190
|
// Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be
|
|
163
191
|
// rewound, so we can't use the retry loop around the actual upload)
|
|
164
|
-
await this.call(() => this.controller.getInfo(this.
|
|
165
|
-
let uploadId = await this.controller.startLargeFile(this.
|
|
192
|
+
await this.call(() => this.controller.getInfo(this.account, this.bucketName, config.path));
|
|
193
|
+
let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path);
|
|
166
194
|
try {
|
|
167
195
|
while (true) {
|
|
168
196
|
let data = await config.getNextData();
|
|
@@ -181,15 +209,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
181
209
|
}
|
|
182
210
|
|
|
183
211
|
public async getURL(path: string): Promise<string> {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
}
|
|
187
|
-
return buildPublicFileURL({
|
|
188
|
-
address: this.config.address,
|
|
189
|
-
port: this.config.port,
|
|
190
|
-
account: this.config.account,
|
|
191
|
-
bucketName: this.config.bucketName,
|
|
192
|
-
path,
|
|
193
|
-
});
|
|
212
|
+
// Only actually loads for public buckets (the server rejects plain URL reads otherwise)
|
|
213
|
+
return buildFileUrl(getBucketBaseUrl(this.config.url), path);
|
|
194
214
|
}
|
|
195
215
|
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
|
|
4
|
+
export declare class ArchivesUrl implements IArchives {
|
|
5
|
+
private base;
|
|
6
|
+
constructor(base: string);
|
|
7
|
+
getDebugName(): string;
|
|
8
|
+
private readOnlyError;
|
|
9
|
+
get(fileName: string, config?: {
|
|
10
|
+
range?: {
|
|
11
|
+
start: number;
|
|
12
|
+
end: number;
|
|
13
|
+
};
|
|
14
|
+
}): Promise<Buffer | undefined>;
|
|
15
|
+
get2(fileName: string, config?: {
|
|
16
|
+
range?: {
|
|
17
|
+
start: number;
|
|
18
|
+
end: number;
|
|
19
|
+
};
|
|
20
|
+
}): Promise<{
|
|
21
|
+
data: Buffer;
|
|
22
|
+
writeTime: number;
|
|
23
|
+
size: number;
|
|
24
|
+
} | undefined>;
|
|
25
|
+
getInfo(fileName: string): Promise<{
|
|
26
|
+
writeTime: number;
|
|
27
|
+
size: number;
|
|
28
|
+
} | undefined>;
|
|
29
|
+
set(fileName: string, data: Buffer, config?: {
|
|
30
|
+
lastModified?: number;
|
|
31
|
+
}): Promise<void>;
|
|
32
|
+
del(fileName: string): Promise<void>;
|
|
33
|
+
setLargeFile(config: {
|
|
34
|
+
path: string;
|
|
35
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
36
|
+
}): Promise<void>;
|
|
37
|
+
find(prefix: string, config?: {
|
|
38
|
+
shallow?: boolean;
|
|
39
|
+
type: "files" | "folders";
|
|
40
|
+
}): Promise<string[]>;
|
|
41
|
+
findInfo(prefix: string, config?: {
|
|
42
|
+
shallow?: boolean;
|
|
43
|
+
type: "files" | "folders";
|
|
44
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
45
|
+
getURL(path: string): Promise<string>;
|
|
46
|
+
getConfig(): Promise<ArchivesConfig>;
|
|
47
|
+
hasWriteAccess(): Promise<boolean>;
|
|
48
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
module.allowclient = true;
|
|
2
|
+
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
|
|
4
|
+
import { buildFileUrl } from "./remoteConfig";
|
|
5
|
+
|
|
6
|
+
// Read-only IArchives over a public bucket's plain-URL form (our storage server's
|
|
7
|
+
// /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API
|
|
8
|
+
// access to a source. Only single-file reads work: there is no listing, and writes always throw.
|
|
9
|
+
|
|
10
|
+
export class ArchivesUrl implements IArchives {
|
|
11
|
+
// base is the bucket's public base URL, e.g. https://host:port/file/<account>/<bucketName>
|
|
12
|
+
constructor(private base: string) { }
|
|
13
|
+
|
|
14
|
+
public getDebugName() {
|
|
15
|
+
return `url/${this.base}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
private readOnlyError(operation: string): Error {
|
|
19
|
+
return new Error(`${operation} is not supported over URL-form access (no API access to this source, only public URL reads). Source: ${this.base}`);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
|
|
23
|
+
let result = await this.get2(fileName, config);
|
|
24
|
+
return result && result.data || undefined;
|
|
25
|
+
}
|
|
26
|
+
public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
|
|
27
|
+
let url = buildFileUrl(this.base, fileName);
|
|
28
|
+
let headers: Record<string, string> = {};
|
|
29
|
+
let range = config?.range;
|
|
30
|
+
if (range) {
|
|
31
|
+
headers["Range"] = `bytes=${range.start}-${range.end - 1}`;
|
|
32
|
+
}
|
|
33
|
+
let response = await fetch(url, { headers });
|
|
34
|
+
if (response.status === 404) return undefined;
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
throw new Error(`Read of ${url} failed: ${response.status} ${response.statusText}`);
|
|
37
|
+
}
|
|
38
|
+
let data = Buffer.from(await response.arrayBuffer());
|
|
39
|
+
// A real 206 only returns the requested slice, so the full size lives in Content-Range's total.
|
|
40
|
+
let size = data.length;
|
|
41
|
+
let contentRange = response.headers.get("content-range");
|
|
42
|
+
let total = contentRange && Number(contentRange.split("/")[1]);
|
|
43
|
+
if (total && Number.isFinite(total)) {
|
|
44
|
+
size = total;
|
|
45
|
+
}
|
|
46
|
+
// Servers that don't support ranges return the full file with a 200 (ours serves real
|
|
47
|
+
// 206s, but backblaze friendly URLs and proxies may not)
|
|
48
|
+
if (range && response.status === 200) {
|
|
49
|
+
data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
|
|
50
|
+
}
|
|
51
|
+
let lastModified = response.headers.get("last-modified");
|
|
52
|
+
let writeTime = lastModified && new Date(lastModified).getTime() || 0;
|
|
53
|
+
return { data, writeTime, size };
|
|
54
|
+
}
|
|
55
|
+
public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
|
|
56
|
+
let result = await this.get2(fileName);
|
|
57
|
+
return result && { writeTime: result.writeTime, size: result.size } || undefined;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
61
|
+
throw this.readOnlyError("set");
|
|
62
|
+
}
|
|
63
|
+
public async del(fileName: string): Promise<void> {
|
|
64
|
+
throw this.readOnlyError("del");
|
|
65
|
+
}
|
|
66
|
+
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
|
|
67
|
+
throw this.readOnlyError("setLargeFile");
|
|
68
|
+
}
|
|
69
|
+
public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
|
|
70
|
+
throw this.readOnlyError("find (listing)");
|
|
71
|
+
}
|
|
72
|
+
public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
|
|
73
|
+
throw this.readOnlyError("findInfo (listing)");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
public async getURL(path: string): Promise<string> {
|
|
77
|
+
return buildFileUrl(this.base, path);
|
|
78
|
+
}
|
|
79
|
+
public async getConfig(): Promise<ArchivesConfig> {
|
|
80
|
+
return {};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
public async hasWriteAccess(): Promise<boolean> {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -6,12 +6,14 @@ import { observer } from "../../render-utils/observer";
|
|
|
6
6
|
import { isNode } from "socket-function/src/misc";
|
|
7
7
|
import { css } from "typesafecss";
|
|
8
8
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
9
|
-
import { RemoteStorageController, AccessState } from "./storageController";
|
|
9
|
+
import { RemoteStorageController, AccessState, AccessRequest } from "./storageController";
|
|
10
10
|
import { authenticateStorage } from "./ArchivesRemote";
|
|
11
11
|
|
|
12
12
|
// The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request
|
|
13
|
-
// access to that account for this browser's machine identity.
|
|
14
|
-
//
|
|
13
|
+
// access to that account for this browser's machine identity. Once granted, lists the machines that
|
|
14
|
+
// have access. To approve someone else's request, the user must type in the requester's IP —
|
|
15
|
+
// pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a
|
|
16
|
+
// random machine's request).
|
|
15
17
|
|
|
16
18
|
const REFRESH_INTERVAL = 1000 * 15;
|
|
17
19
|
const COPIED_RESET_DELAY = 2000;
|
|
@@ -47,6 +49,11 @@ class AccessPage extends preact.Component {
|
|
|
47
49
|
account: "",
|
|
48
50
|
state: undefined as AccessState | undefined,
|
|
49
51
|
error: "",
|
|
52
|
+
lookupIp: "",
|
|
53
|
+
lookupResults: undefined as { ip: string; requests: AccessRequest[] } | undefined,
|
|
54
|
+
lookupError: "",
|
|
55
|
+
looking: false,
|
|
56
|
+
granting: "",
|
|
50
57
|
});
|
|
51
58
|
|
|
52
59
|
componentDidMount() {
|
|
@@ -67,22 +74,58 @@ class AccessPage extends preact.Component {
|
|
|
67
74
|
}
|
|
68
75
|
|
|
69
76
|
private authenticated = false;
|
|
70
|
-
private async
|
|
77
|
+
private async controller() {
|
|
71
78
|
let address = location.hostname;
|
|
72
79
|
let port = +location.port || 443;
|
|
73
80
|
let nodeId = SocketFunction.connect({ address, port });
|
|
74
|
-
let controller = RemoteStorageController.nodes[nodeId];
|
|
75
81
|
if (!this.authenticated) {
|
|
76
82
|
await authenticateStorage({ address, port, nodeId });
|
|
77
83
|
this.authenticated = true;
|
|
78
84
|
}
|
|
85
|
+
return RemoteStorageController.nodes[nodeId];
|
|
86
|
+
}
|
|
87
|
+
private async refresh(account: string) {
|
|
88
|
+
let controller = await this.controller();
|
|
79
89
|
await controller.requestAccess(account);
|
|
80
90
|
this.synced.state = await controller.getAccessState(account);
|
|
81
91
|
this.synced.error = "";
|
|
82
92
|
}
|
|
83
93
|
|
|
94
|
+
private lookupIP = async () => {
|
|
95
|
+
let ip = this.synced.lookupIp.trim();
|
|
96
|
+
if (!ip) return;
|
|
97
|
+
this.synced.looking = true;
|
|
98
|
+
this.synced.lookupError = "";
|
|
99
|
+
try {
|
|
100
|
+
let controller = await this.controller();
|
|
101
|
+
let requests = await controller.listRequestsForIP(this.synced.account, ip);
|
|
102
|
+
this.synced.lookupResults = { ip, requests };
|
|
103
|
+
} catch (e) {
|
|
104
|
+
this.synced.lookupError = String(e);
|
|
105
|
+
} finally {
|
|
106
|
+
this.synced.looking = false;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
private approve = async (request: AccessRequest) => {
|
|
111
|
+
this.synced.granting = request.requestId;
|
|
112
|
+
try {
|
|
113
|
+
let controller = await this.controller();
|
|
114
|
+
await controller.grantAccess(request.requestId);
|
|
115
|
+
if (this.synced.lookupResults) {
|
|
116
|
+
this.synced.lookupResults.requests = this.synced.lookupResults.requests.filter(r => r.requestId !== request.requestId);
|
|
117
|
+
}
|
|
118
|
+
await this.refresh(this.synced.account);
|
|
119
|
+
} catch (e) {
|
|
120
|
+
this.synced.lookupError = String(e);
|
|
121
|
+
} finally {
|
|
122
|
+
this.synced.granting = "";
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
|
|
84
126
|
render() {
|
|
85
|
-
let
|
|
127
|
+
let synced = this.synced;
|
|
128
|
+
let { account, state, error } = synced;
|
|
86
129
|
if (!account) {
|
|
87
130
|
return <div className={css.vbox(8).pad2(16)}>
|
|
88
131
|
<div>Remote storage server.</div>
|
|
@@ -95,31 +138,71 @@ class AccessPage extends preact.Component {
|
|
|
95
138
|
{!state && !error && <div>Requesting access...</div>}
|
|
96
139
|
{state && !state.hasAccess && <div className={css.vbox(8)}>
|
|
97
140
|
<div>This machine ({state.machineId}, ip {state.ip}) does NOT have access yet.</div>
|
|
98
|
-
<div>An access request has been made. To grant it, run this
|
|
99
|
-
<CopyableCommand command={state.grantAccessCommand
|
|
141
|
+
<div>An access request has been made. To grant it, run this command:</div>
|
|
142
|
+
{state.grantAccessCommand && <CopyableCommand command={state.grantAccessCommand} />}
|
|
100
143
|
<div>This page rechecks every {REFRESH_INTERVAL / 1000} seconds.</div>
|
|
101
144
|
</div>}
|
|
102
|
-
{state && state.hasAccess && <div className={css.vbox(
|
|
145
|
+
{state && state.hasAccess && <div className={css.vbox(16)}>
|
|
103
146
|
<div>This machine ({state.machineId}, ip {state.ip}) has access.</div>
|
|
104
|
-
<div
|
|
105
|
-
|
|
106
|
-
<
|
|
107
|
-
<
|
|
108
|
-
<
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
</
|
|
121
|
-
</
|
|
122
|
-
</
|
|
147
|
+
<div className={css.vbox(6)}>
|
|
148
|
+
<div>Machines with access:</div>
|
|
149
|
+
<table>
|
|
150
|
+
<thead>
|
|
151
|
+
<tr>
|
|
152
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Machine</th>
|
|
153
|
+
<th className={css.pad2(8, 2).textAlign("left")}>IP</th>
|
|
154
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Granted</th>
|
|
155
|
+
</tr>
|
|
156
|
+
</thead>
|
|
157
|
+
<tbody>
|
|
158
|
+
{(state.trustedMachines || []).map(m => <tr key={m.machineId}>
|
|
159
|
+
<td className={css.pad2(8, 2)}>{m.machineId}</td>
|
|
160
|
+
<td className={css.pad2(8, 2)}>{m.ip}</td>
|
|
161
|
+
<td className={css.pad2(8, 2)}>{new Date(m.time).toLocaleString()}</td>
|
|
162
|
+
</tr>)}
|
|
163
|
+
</tbody>
|
|
164
|
+
</table>
|
|
165
|
+
</div>
|
|
166
|
+
<div className={css.vbox(6)}>
|
|
167
|
+
<div>Approve request</div>
|
|
168
|
+
<div className={css.hbox(8).alignItems("center")}>
|
|
169
|
+
<input
|
|
170
|
+
placeholder="ip"
|
|
171
|
+
value={synced.lookupIp}
|
|
172
|
+
onInput={e => { synced.lookupIp = (e.currentTarget as HTMLInputElement).value; }}
|
|
173
|
+
onKeyDown={e => { if (e.key === "Enter") void this.lookupIP(); }}
|
|
174
|
+
/>
|
|
175
|
+
<button disabled={synced.looking || !synced.lookupIp.trim()} onClick={this.lookupIP}>
|
|
176
|
+
search
|
|
177
|
+
</button>
|
|
178
|
+
</div>
|
|
179
|
+
{synced.lookupError && <div>Error: {synced.lookupError}</div>}
|
|
180
|
+
{synced.lookupResults && synced.lookupResults.requests.length === 0 && <div>
|
|
181
|
+
No pending requests for {synced.lookupResults.ip} on this account.
|
|
182
|
+
</div>}
|
|
183
|
+
{synced.lookupResults && synced.lookupResults.requests.length > 0 && <table>
|
|
184
|
+
<thead>
|
|
185
|
+
<tr>
|
|
186
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Machine</th>
|
|
187
|
+
<th className={css.pad2(8, 2).textAlign("left")}>IP</th>
|
|
188
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Requested</th>
|
|
189
|
+
<th className={css.pad2(8, 2).textAlign("left")}></th>
|
|
190
|
+
</tr>
|
|
191
|
+
</thead>
|
|
192
|
+
<tbody>
|
|
193
|
+
{synced.lookupResults.requests.map(r => <tr key={r.requestId}>
|
|
194
|
+
<td className={css.pad2(8, 2)}>{r.machineId}</td>
|
|
195
|
+
<td className={css.pad2(8, 2)}>{r.ip}</td>
|
|
196
|
+
<td className={css.pad2(8, 2)}>{new Date(r.time).toLocaleString()}</td>
|
|
197
|
+
<td className={css.pad2(8, 2)}>
|
|
198
|
+
<button disabled={!!synced.granting} onClick={async () => this.approve(r)}>
|
|
199
|
+
{synced.granting === r.requestId && "Approving..." || "Approve"}
|
|
200
|
+
</button>
|
|
201
|
+
</td>
|
|
202
|
+
</tr>)}
|
|
203
|
+
</tbody>
|
|
204
|
+
</table>}
|
|
205
|
+
</div>
|
|
123
206
|
</div>}
|
|
124
207
|
</div>;
|
|
125
208
|
}
|