sliftutils 1.7.9 → 1.7.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/storageserver.js +4 -0
- package/{teststorage → examplestorage}/browser.tsx +18 -22
- package/{teststorage/server.ts → examplestorage/exampleserver.ts} +5 -5
- package/index.d.ts +528 -78
- package/misc/https/dns.d.ts +7 -3
- package/misc/https/dns.ts +4 -9
- package/misc/https/hostServer.d.ts +6 -3
- package/misc/https/hostServer.ts +6 -6
- package/package.json +2 -1
- package/spec.txt +77 -42
- package/storage/ArchivesDisk.d.ts +65 -0
- package/storage/ArchivesDisk.ts +365 -0
- package/storage/IArchives.d.ts +95 -1
- package/storage/IArchives.ts +146 -1
- package/storage/backblaze.d.ts +14 -2
- package/storage/backblaze.ts +18 -2
- package/storage/remoteStorage/ArchivesRemote.d.ts +29 -17
- package/storage/remoteStorage/ArchivesRemote.ts +63 -63
- package/storage/remoteStorage/ArchivesUrl.d.ts +46 -0
- package/storage/remoteStorage/ArchivesUrl.ts +74 -0
- package/storage/remoteStorage/accessPage.tsx +111 -28
- package/storage/remoteStorage/blobStore.d.ts +79 -25
- package/storage/remoteStorage/blobStore.ts +441 -287
- package/storage/remoteStorage/cliArgs.d.ts +1 -0
- package/storage/remoteStorage/cliArgs.ts +9 -0
- package/storage/remoteStorage/createArchives.d.ts +64 -0
- package/storage/remoteStorage/createArchives.ts +395 -0
- package/storage/remoteStorage/grantAccess.js +4 -0
- package/storage/remoteStorage/grantAccessCli.d.ts +1 -0
- package/storage/remoteStorage/grantAccessCli.ts +27 -0
- package/storage/remoteStorage/remoteConfig.d.ts +21 -0
- package/storage/remoteStorage/remoteConfig.ts +94 -0
- package/storage/remoteStorage/storageController.d.ts +19 -27
- package/storage/remoteStorage/storageController.ts +205 -136
- package/storage/remoteStorage/storageServer.d.ts +11 -0
- package/storage/remoteStorage/storageServer.ts +80 -93
- package/storage/remoteStorage/storageServerCli.d.ts +1 -0
- package/storage/remoteStorage/storageServerCli.ts +41 -0
- package/storage/remoteStorage/storageServerState.d.ts +37 -0
- package/storage/remoteStorage/storageServerState.ts +358 -0
- package/testsite/server.ts +1 -1
package/storage/backblaze.ts
CHANGED
|
@@ -9,7 +9,7 @@ import debugbreak from "debugbreak";
|
|
|
9
9
|
import dns from "dns";
|
|
10
10
|
import { getSecret } from "../misc/getSecret";
|
|
11
11
|
import { httpsRequest } from "socket-function/src/https";
|
|
12
|
-
import { IArchives } from "./IArchives";
|
|
12
|
+
import { IArchives, ArchivesConfig, assertValidLastModified } from "./IArchives";
|
|
13
13
|
|
|
14
14
|
type BackblazeCreds = {
|
|
15
15
|
applicationKeyId: string;
|
|
@@ -594,7 +594,23 @@ export class ArchivesBackblaze implements IArchives {
|
|
|
594
594
|
downloading = false;
|
|
595
595
|
}
|
|
596
596
|
}
|
|
597
|
-
public async
|
|
597
|
+
public async get2(fileName: string, config?: { range?: { start: number; end: number; } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
|
|
598
|
+
// B2 downloads don't return the upload time, so this takes a second API call
|
|
599
|
+
let [data, info] = await Promise.all([this.get(fileName, config), this.getInfo(fileName)]);
|
|
600
|
+
if (!data || !info) return undefined;
|
|
601
|
+
return { data, writeTime: info.writeTime };
|
|
602
|
+
}
|
|
603
|
+
public async getConfig(): Promise<ArchivesConfig> {
|
|
604
|
+
return {};
|
|
605
|
+
}
|
|
606
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
607
|
+
if (config?.lastModified) {
|
|
608
|
+
assertValidLastModified(config.lastModified);
|
|
609
|
+
let existing = await this.getInfo(fileName);
|
|
610
|
+
// An older write never overwrites a newer one (see IArchives.set). B2 stamps its own
|
|
611
|
+
// upload time, so the exact lastModified is not preserved on the stored file.
|
|
612
|
+
if (existing && config.lastModified < existing.writeTime) return;
|
|
613
|
+
}
|
|
598
614
|
this.log(`backblaze upload (${formatNumber(data.length)}B) ${fileName}`);
|
|
599
615
|
let f = fileName;
|
|
600
616
|
await this.apiRetryLogic(async (api) => {
|
|
@@ -1,22 +1,15 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { IArchives, ArchiveFileInfo } from "../IArchives";
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
|
|
4
4
|
export type ArchivesRemoteConfig = {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
bucketName: string;
|
|
9
|
-
public?: boolean;
|
|
10
|
-
fast?: boolean;
|
|
11
|
-
writeDelay?: number;
|
|
5
|
+
url: string;
|
|
6
|
+
accountName?: string;
|
|
7
|
+
waitForAccess?: boolean;
|
|
12
8
|
};
|
|
13
|
-
export declare function
|
|
9
|
+
export declare function parseStorageUrl(url: string): {
|
|
14
10
|
address: string;
|
|
15
11
|
port: number;
|
|
16
|
-
|
|
17
|
-
bucketName: string;
|
|
18
|
-
path: string;
|
|
19
|
-
}): string;
|
|
12
|
+
};
|
|
20
13
|
export declare function authenticateStorage(config: {
|
|
21
14
|
address: string;
|
|
22
15
|
port: number;
|
|
@@ -28,16 +21,21 @@ export declare function authenticateStorage(config: {
|
|
|
28
21
|
export declare class ArchivesRemote implements IArchives {
|
|
29
22
|
private config;
|
|
30
23
|
constructor(config: ArchivesRemoteConfig);
|
|
24
|
+
private parsed;
|
|
25
|
+
private account;
|
|
26
|
+
private bucketName;
|
|
31
27
|
private nodeId;
|
|
32
28
|
private controller;
|
|
33
|
-
private setupDone;
|
|
34
29
|
private lastDeniedLog;
|
|
35
30
|
getDebugName(): string;
|
|
36
31
|
private authenticate;
|
|
37
32
|
private callAuthed;
|
|
38
|
-
waitingForAccess(): Promise<
|
|
33
|
+
waitingForAccess(): Promise<{
|
|
34
|
+
link: string;
|
|
35
|
+
machineId: string;
|
|
36
|
+
ip: string;
|
|
37
|
+
} | undefined>;
|
|
39
38
|
private onAccessDenied;
|
|
40
|
-
private ensureSetup;
|
|
41
39
|
private call;
|
|
42
40
|
get(fileName: string, config?: {
|
|
43
41
|
range?: {
|
|
@@ -45,7 +43,18 @@ export declare class ArchivesRemote implements IArchives {
|
|
|
45
43
|
end: number;
|
|
46
44
|
};
|
|
47
45
|
}): Promise<Buffer | undefined>;
|
|
48
|
-
|
|
46
|
+
get2(fileName: string, config?: {
|
|
47
|
+
range?: {
|
|
48
|
+
start: number;
|
|
49
|
+
end: number;
|
|
50
|
+
};
|
|
51
|
+
}): Promise<{
|
|
52
|
+
data: Buffer;
|
|
53
|
+
writeTime: number;
|
|
54
|
+
} | undefined>;
|
|
55
|
+
set(fileName: string, data: Buffer, config?: {
|
|
56
|
+
lastModified?: number;
|
|
57
|
+
}): Promise<void>;
|
|
49
58
|
del(fileName: string): Promise<void>;
|
|
50
59
|
getInfo(fileName: string): Promise<{
|
|
51
60
|
writeTime: number;
|
|
@@ -59,6 +68,9 @@ export declare class ArchivesRemote implements IArchives {
|
|
|
59
68
|
shallow?: boolean;
|
|
60
69
|
type: "files" | "folders";
|
|
61
70
|
}): Promise<string[]>;
|
|
71
|
+
getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
|
|
72
|
+
getConfig(): Promise<ArchivesConfig>;
|
|
73
|
+
getSyncStatus(): Promise<ArchivesSyncStatus>;
|
|
62
74
|
setLargeFile(config: {
|
|
63
75
|
path: string;
|
|
64
76
|
getNextData(): Promise<Buffer | undefined>;
|
|
@@ -4,37 +4,38 @@ import { SocketFunction } from "socket-function/SocketFunction";
|
|
|
4
4
|
import { timeInMinute } from "socket-function/src/misc";
|
|
5
5
|
import { delay } from "socket-function/src/batching";
|
|
6
6
|
import { getIdentityCA, loadIdentityCA, sign } from "../../misc/https/certs";
|
|
7
|
-
import { IArchives, ArchiveFileInfo } from "../IArchives";
|
|
7
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
|
|
8
|
+
import { parseHostedUrl, getBucketBaseUrl, buildFileUrl } from "./remoteConfig";
|
|
8
9
|
import {
|
|
9
|
-
RemoteStorageController,
|
|
10
|
+
RemoteStorageController, STORAGE_AUTH_PURPOSE,
|
|
10
11
|
STORAGE_NOT_AUTHENTICATED, STORAGE_ACCESS_DENIED,
|
|
11
12
|
} from "./storageController";
|
|
12
13
|
|
|
13
14
|
// A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in
|
|
14
15
|
// Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account
|
|
15
|
-
// hasn't trusted this machine yet it requests access, waits,
|
|
16
|
-
// (calls block until access is granted).
|
|
16
|
+
// hasn't trusted this machine yet it requests access, and by default waits, logging instructions
|
|
17
|
+
// every minute (calls block until access is granted).
|
|
17
18
|
|
|
18
19
|
const ACCESS_RETRY_DELAY = 1000 * 30;
|
|
19
20
|
const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
|
|
20
21
|
|
|
21
22
|
export type ArchivesRemoteConfig = {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
//
|
|
29
|
-
|
|
30
|
-
// writes that haven't flushed yet.
|
|
31
|
-
fast?: boolean;
|
|
32
|
-
writeDelay?: number;
|
|
23
|
+
// The bucket's routing URL, which addresses the server, account, and bucket in one:
|
|
24
|
+
// https://storage.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json
|
|
25
|
+
url: string;
|
|
26
|
+
// Access is checked against this account (defaults to the account in the url)
|
|
27
|
+
accountName?: string;
|
|
28
|
+
// false: access-denied calls throw immediately (the error includes the access page link)
|
|
29
|
+
// instead of requesting access and blocking until it is granted (the default).
|
|
30
|
+
waitForAccess?: boolean;
|
|
33
31
|
};
|
|
34
32
|
|
|
35
|
-
export function
|
|
36
|
-
let
|
|
37
|
-
|
|
33
|
+
export function parseStorageUrl(url: string): { address: string; port: number } {
|
|
34
|
+
let u = new URL(url);
|
|
35
|
+
if (u.protocol !== "https:") {
|
|
36
|
+
throw new Error(`Storage URL must use https, got ${JSON.stringify(u.protocol)} in ${JSON.stringify(url)}`);
|
|
37
|
+
}
|
|
38
|
+
return { address: u.hostname, port: +u.port || 443 };
|
|
38
39
|
}
|
|
39
40
|
|
|
40
41
|
// Authenticates a connection to a storage server with this machine's certs.ts identity
|
|
@@ -61,17 +62,19 @@ export class ArchivesRemote implements IArchives {
|
|
|
61
62
|
SocketFunction.ENABLE_CLIENT_MODE = true;
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
private
|
|
65
|
+
private parsed = parseHostedUrl(this.config.url);
|
|
66
|
+
private account = this.config.accountName || this.parsed.account;
|
|
67
|
+
private bucketName = this.parsed.bucketName;
|
|
68
|
+
private nodeId = SocketFunction.connect({ address: this.parsed.address, port: this.parsed.port });
|
|
65
69
|
private controller = RemoteStorageController.nodes[this.nodeId];
|
|
66
|
-
private setupDone = false;
|
|
67
70
|
private lastDeniedLog = 0;
|
|
68
71
|
|
|
69
72
|
public getDebugName() {
|
|
70
|
-
return `remoteStorage/${this.
|
|
73
|
+
return `remoteStorage/${this.parsed.address}:${this.parsed.port}/${this.account}/${this.bucketName}`;
|
|
71
74
|
}
|
|
72
75
|
|
|
73
76
|
private async authenticate(): Promise<void> {
|
|
74
|
-
await authenticateStorage({ address: this.
|
|
77
|
+
await authenticateStorage({ address: this.parsed.address, port: this.parsed.port, nodeId: this.nodeId });
|
|
75
78
|
}
|
|
76
79
|
|
|
77
80
|
// Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike
|
|
@@ -87,49 +90,41 @@ export class ArchivesRemote implements IArchives {
|
|
|
87
90
|
}
|
|
88
91
|
|
|
89
92
|
// Returns undefined if this machine has access to the account. Otherwise puts in an access
|
|
90
|
-
// request and returns
|
|
91
|
-
|
|
92
|
-
|
|
93
|
+
// request and returns our machineId + ip (so the caller can display them alongside the link,
|
|
94
|
+
// for the approver to match the incoming request) and the link to the grant page.
|
|
95
|
+
public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
|
|
96
|
+
let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
|
|
93
97
|
if (state.hasAccess) return undefined;
|
|
94
|
-
await this.callAuthed(() => this.controller.requestAccess(this.
|
|
95
|
-
return
|
|
98
|
+
let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
|
|
99
|
+
return {
|
|
100
|
+
link: `https://${this.parsed.address}:${this.parsed.port}/${this.account}`,
|
|
101
|
+
machineId: requested.machineId,
|
|
102
|
+
ip: requested.ip,
|
|
103
|
+
};
|
|
96
104
|
}
|
|
97
105
|
|
|
98
106
|
private async onAccessDenied(): Promise<void> {
|
|
99
|
-
let requested = await this.callAuthed(() => this.controller.requestAccess(this.
|
|
107
|
+
let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
|
|
100
108
|
if (Date.now() - this.lastDeniedLog > timeInMinute) {
|
|
101
109
|
this.lastDeniedLog = Date.now();
|
|
102
|
-
console.log(`No access to storage account ${JSON.stringify(this.
|
|
110
|
+
console.log(`No access to storage account ${JSON.stringify(this.account)} on ${this.parsed.address}:${this.parsed.port} (our machine ${requested.machineId}, ip ${requested.ip}). Waiting for access to be granted. See https://${this.parsed.address}:${this.parsed.port}/${this.account} - or grant it with: ${requested.grantAccessCommand}`);
|
|
103
111
|
}
|
|
104
112
|
await delay(ACCESS_RETRY_DELAY);
|
|
105
113
|
}
|
|
106
114
|
|
|
107
|
-
private async ensureSetup(): Promise<void> {
|
|
108
|
-
if (this.setupDone) return;
|
|
109
|
-
await this.controller.ensureBucket(this.config.account, this.config.bucketName, {
|
|
110
|
-
public: this.config.public,
|
|
111
|
-
fast: this.config.fast,
|
|
112
|
-
writeDelay: this.config.writeDelay,
|
|
113
|
-
});
|
|
114
|
-
this.setupDone = true;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
115
|
// Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
|
|
118
|
-
// access as needed.
|
|
116
|
+
// access as needed (unless waitForAccess is false, in which case denied calls throw).
|
|
119
117
|
private async call<T>(fnc: () => Promise<T>): Promise<T> {
|
|
120
118
|
while (true) {
|
|
121
119
|
try {
|
|
122
|
-
await this.ensureSetup();
|
|
123
120
|
return await fnc();
|
|
124
121
|
} catch (e: any) {
|
|
125
122
|
let message = String(e.stack || e);
|
|
126
123
|
if (message.includes(STORAGE_NOT_AUTHENTICATED)) {
|
|
127
|
-
this.setupDone = false;
|
|
128
124
|
await this.authenticate();
|
|
129
125
|
continue;
|
|
130
126
|
}
|
|
131
|
-
if (message.includes(STORAGE_ACCESS_DENIED)) {
|
|
132
|
-
this.setupDone = false;
|
|
127
|
+
if (message.includes(STORAGE_ACCESS_DENIED) && this.config.waitForAccess !== false) {
|
|
133
128
|
await this.onAccessDenied();
|
|
134
129
|
continue;
|
|
135
130
|
}
|
|
@@ -139,30 +134,43 @@ export class ArchivesRemote implements IArchives {
|
|
|
139
134
|
}
|
|
140
135
|
|
|
141
136
|
public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
|
|
142
|
-
let result = await this.
|
|
143
|
-
return result &&
|
|
137
|
+
let result = await this.get2(fileName, config);
|
|
138
|
+
return result && result.data || undefined;
|
|
139
|
+
}
|
|
140
|
+
public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
|
|
141
|
+
let result = await this.call(() => this.controller.get2(this.account, this.bucketName, fileName, config?.range));
|
|
142
|
+
return result && { data: Buffer.from(result.data), writeTime: result.writeTime } || undefined;
|
|
144
143
|
}
|
|
145
|
-
public async set(fileName: string, data: Buffer): Promise<void> {
|
|
146
|
-
await this.call(() => this.controller.set(this.
|
|
144
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
145
|
+
await this.call(() => this.controller.set(this.account, this.bucketName, fileName, data, config?.lastModified));
|
|
147
146
|
}
|
|
148
147
|
public async del(fileName: string): Promise<void> {
|
|
149
|
-
await this.call(() => this.controller.del(this.
|
|
148
|
+
await this.call(() => this.controller.del(this.account, this.bucketName, fileName));
|
|
150
149
|
}
|
|
151
150
|
public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
|
|
152
|
-
return await this.call(() => this.controller.getInfo(this.
|
|
151
|
+
return await this.call(() => this.controller.getInfo(this.account, this.bucketName, fileName));
|
|
153
152
|
}
|
|
154
153
|
public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
|
|
155
|
-
return await this.call(() => this.controller.findInfo(this.
|
|
154
|
+
return await this.call(() => this.controller.findInfo(this.account, this.bucketName, prefix, config));
|
|
156
155
|
}
|
|
157
156
|
public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
|
|
158
157
|
return (await this.findInfo(prefix, config)).map(x => x.path);
|
|
159
158
|
}
|
|
159
|
+
public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
|
|
160
|
+
return await this.call(() => this.controller.getChangesAfter(this.account, this.bucketName, time));
|
|
161
|
+
}
|
|
162
|
+
public async getConfig(): Promise<ArchivesConfig> {
|
|
163
|
+
return await this.call(() => this.controller.getArchivesConfig(this.account, this.bucketName));
|
|
164
|
+
}
|
|
165
|
+
public async getSyncStatus(): Promise<ArchivesSyncStatus> {
|
|
166
|
+
return await this.call(() => this.controller.getSyncStatus(this.account, this.bucketName));
|
|
167
|
+
}
|
|
160
168
|
|
|
161
169
|
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
|
|
162
170
|
// Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be
|
|
163
171
|
// rewound, so we can't use the retry loop around the actual upload)
|
|
164
|
-
await this.call(() => this.controller.getInfo(this.
|
|
165
|
-
let uploadId = await this.controller.startLargeFile(this.
|
|
172
|
+
await this.call(() => this.controller.getInfo(this.account, this.bucketName, config.path));
|
|
173
|
+
let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path);
|
|
166
174
|
try {
|
|
167
175
|
while (true) {
|
|
168
176
|
let data = await config.getNextData();
|
|
@@ -181,15 +189,7 @@ export class ArchivesRemote implements IArchives {
|
|
|
181
189
|
}
|
|
182
190
|
|
|
183
191
|
public async getURL(path: string): Promise<string> {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
}
|
|
187
|
-
return buildPublicFileURL({
|
|
188
|
-
address: this.config.address,
|
|
189
|
-
port: this.config.port,
|
|
190
|
-
account: this.config.account,
|
|
191
|
-
bucketName: this.config.bucketName,
|
|
192
|
-
path,
|
|
193
|
-
});
|
|
192
|
+
// Only actually loads for public buckets (the server rejects plain URL reads otherwise)
|
|
193
|
+
return buildFileUrl(getBucketBaseUrl(this.config.url), path);
|
|
194
194
|
}
|
|
195
195
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
|
|
4
|
+
export declare class ArchivesUrl implements IArchives {
|
|
5
|
+
private base;
|
|
6
|
+
constructor(base: string);
|
|
7
|
+
getDebugName(): string;
|
|
8
|
+
private readOnlyError;
|
|
9
|
+
get(fileName: string, config?: {
|
|
10
|
+
range?: {
|
|
11
|
+
start: number;
|
|
12
|
+
end: number;
|
|
13
|
+
};
|
|
14
|
+
}): Promise<Buffer | undefined>;
|
|
15
|
+
get2(fileName: string, config?: {
|
|
16
|
+
range?: {
|
|
17
|
+
start: number;
|
|
18
|
+
end: number;
|
|
19
|
+
};
|
|
20
|
+
}): Promise<{
|
|
21
|
+
data: Buffer;
|
|
22
|
+
writeTime: number;
|
|
23
|
+
} | undefined>;
|
|
24
|
+
getInfo(fileName: string): Promise<{
|
|
25
|
+
writeTime: number;
|
|
26
|
+
size: number;
|
|
27
|
+
} | undefined>;
|
|
28
|
+
set(fileName: string, data: Buffer, config?: {
|
|
29
|
+
lastModified?: number;
|
|
30
|
+
}): Promise<void>;
|
|
31
|
+
del(fileName: string): Promise<void>;
|
|
32
|
+
setLargeFile(config: {
|
|
33
|
+
path: string;
|
|
34
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
35
|
+
}): Promise<void>;
|
|
36
|
+
find(prefix: string, config?: {
|
|
37
|
+
shallow?: boolean;
|
|
38
|
+
type: "files" | "folders";
|
|
39
|
+
}): Promise<string[]>;
|
|
40
|
+
findInfo(prefix: string, config?: {
|
|
41
|
+
shallow?: boolean;
|
|
42
|
+
type: "files" | "folders";
|
|
43
|
+
}): Promise<ArchiveFileInfo[]>;
|
|
44
|
+
getURL(path: string): Promise<string>;
|
|
45
|
+
getConfig(): Promise<ArchivesConfig>;
|
|
46
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
module.allowclient = true;
|
|
2
|
+
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
|
|
4
|
+
import { buildFileUrl } from "./remoteConfig";
|
|
5
|
+
|
|
6
|
+
// Read-only IArchives over a public bucket's plain-URL form (our storage server's
|
|
7
|
+
// /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API
|
|
8
|
+
// access to a source. Only single-file reads work: there is no listing, and writes always throw.
|
|
9
|
+
|
|
10
|
+
export class ArchivesUrl implements IArchives {
|
|
11
|
+
// base is the bucket's public base URL, e.g. https://host:port/file/<account>/<bucketName>
|
|
12
|
+
constructor(private base: string) { }
|
|
13
|
+
|
|
14
|
+
public getDebugName() {
|
|
15
|
+
return `url/${this.base}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
private readOnlyError(operation: string): Error {
|
|
19
|
+
return new Error(`${operation} is not supported over URL-form access (no API access to this source, only public URL reads). Source: ${this.base}`);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
|
|
23
|
+
let result = await this.get2(fileName, config);
|
|
24
|
+
return result && result.data || undefined;
|
|
25
|
+
}
|
|
26
|
+
public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number } | undefined> {
|
|
27
|
+
let url = buildFileUrl(this.base, fileName);
|
|
28
|
+
let headers: Record<string, string> = {};
|
|
29
|
+
let range = config?.range;
|
|
30
|
+
if (range) {
|
|
31
|
+
headers["Range"] = `bytes=${range.start}-${range.end - 1}`;
|
|
32
|
+
}
|
|
33
|
+
let response = await fetch(url, { headers });
|
|
34
|
+
if (response.status === 404) return undefined;
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
throw new Error(`Read of ${url} failed: ${response.status} ${response.statusText}`);
|
|
37
|
+
}
|
|
38
|
+
let data = Buffer.from(await response.arrayBuffer());
|
|
39
|
+
// Servers that don't support ranges (ours doesn't) return the full file with a 200
|
|
40
|
+
if (range && response.status === 200) {
|
|
41
|
+
data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
|
|
42
|
+
}
|
|
43
|
+
let lastModified = response.headers.get("last-modified");
|
|
44
|
+
let writeTime = lastModified && new Date(lastModified).getTime() || 0;
|
|
45
|
+
return { data, writeTime };
|
|
46
|
+
}
|
|
47
|
+
public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
|
|
48
|
+
let result = await this.get2(fileName);
|
|
49
|
+
return result && { writeTime: result.writeTime, size: result.data.length } || undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
|
|
53
|
+
throw this.readOnlyError("set");
|
|
54
|
+
}
|
|
55
|
+
public async del(fileName: string): Promise<void> {
|
|
56
|
+
throw this.readOnlyError("del");
|
|
57
|
+
}
|
|
58
|
+
public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
|
|
59
|
+
throw this.readOnlyError("setLargeFile");
|
|
60
|
+
}
|
|
61
|
+
public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
|
|
62
|
+
throw this.readOnlyError("find (listing)");
|
|
63
|
+
}
|
|
64
|
+
public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
|
|
65
|
+
throw this.readOnlyError("findInfo (listing)");
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
public async getURL(path: string): Promise<string> {
|
|
69
|
+
return buildFileUrl(this.base, path);
|
|
70
|
+
}
|
|
71
|
+
public async getConfig(): Promise<ArchivesConfig> {
|
|
72
|
+
return {};
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -6,12 +6,14 @@ import { observer } from "../../render-utils/observer";
|
|
|
6
6
|
import { isNode } from "socket-function/src/misc";
|
|
7
7
|
import { css } from "typesafecss";
|
|
8
8
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
9
|
-
import { RemoteStorageController, AccessState } from "./storageController";
|
|
9
|
+
import { RemoteStorageController, AccessState, AccessRequest } from "./storageController";
|
|
10
10
|
import { authenticateStorage } from "./ArchivesRemote";
|
|
11
11
|
|
|
12
12
|
// The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request
|
|
13
|
-
// access to that account for this browser's machine identity.
|
|
14
|
-
//
|
|
13
|
+
// access to that account for this browser's machine identity. Once granted, lists the machines that
|
|
14
|
+
// have access. To approve someone else's request, the user must type in the requester's IP —
|
|
15
|
+
// pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a
|
|
16
|
+
// random machine's request).
|
|
15
17
|
|
|
16
18
|
const REFRESH_INTERVAL = 1000 * 15;
|
|
17
19
|
const COPIED_RESET_DELAY = 2000;
|
|
@@ -47,6 +49,11 @@ class AccessPage extends preact.Component {
|
|
|
47
49
|
account: "",
|
|
48
50
|
state: undefined as AccessState | undefined,
|
|
49
51
|
error: "",
|
|
52
|
+
lookupIp: "",
|
|
53
|
+
lookupResults: undefined as { ip: string; requests: AccessRequest[] } | undefined,
|
|
54
|
+
lookupError: "",
|
|
55
|
+
looking: false,
|
|
56
|
+
granting: "",
|
|
50
57
|
});
|
|
51
58
|
|
|
52
59
|
componentDidMount() {
|
|
@@ -67,22 +74,58 @@ class AccessPage extends preact.Component {
|
|
|
67
74
|
}
|
|
68
75
|
|
|
69
76
|
private authenticated = false;
|
|
70
|
-
private async
|
|
77
|
+
private async controller() {
|
|
71
78
|
let address = location.hostname;
|
|
72
79
|
let port = +location.port || 443;
|
|
73
80
|
let nodeId = SocketFunction.connect({ address, port });
|
|
74
|
-
let controller = RemoteStorageController.nodes[nodeId];
|
|
75
81
|
if (!this.authenticated) {
|
|
76
82
|
await authenticateStorage({ address, port, nodeId });
|
|
77
83
|
this.authenticated = true;
|
|
78
84
|
}
|
|
85
|
+
return RemoteStorageController.nodes[nodeId];
|
|
86
|
+
}
|
|
87
|
+
private async refresh(account: string) {
|
|
88
|
+
let controller = await this.controller();
|
|
79
89
|
await controller.requestAccess(account);
|
|
80
90
|
this.synced.state = await controller.getAccessState(account);
|
|
81
91
|
this.synced.error = "";
|
|
82
92
|
}
|
|
83
93
|
|
|
94
|
+
private lookupIP = async () => {
|
|
95
|
+
let ip = this.synced.lookupIp.trim();
|
|
96
|
+
if (!ip) return;
|
|
97
|
+
this.synced.looking = true;
|
|
98
|
+
this.synced.lookupError = "";
|
|
99
|
+
try {
|
|
100
|
+
let controller = await this.controller();
|
|
101
|
+
let requests = await controller.listRequestsForIP(this.synced.account, ip);
|
|
102
|
+
this.synced.lookupResults = { ip, requests };
|
|
103
|
+
} catch (e) {
|
|
104
|
+
this.synced.lookupError = String(e);
|
|
105
|
+
} finally {
|
|
106
|
+
this.synced.looking = false;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
private approve = async (request: AccessRequest) => {
|
|
111
|
+
this.synced.granting = request.requestId;
|
|
112
|
+
try {
|
|
113
|
+
let controller = await this.controller();
|
|
114
|
+
await controller.grantAccess(request.requestId);
|
|
115
|
+
if (this.synced.lookupResults) {
|
|
116
|
+
this.synced.lookupResults.requests = this.synced.lookupResults.requests.filter(r => r.requestId !== request.requestId);
|
|
117
|
+
}
|
|
118
|
+
await this.refresh(this.synced.account);
|
|
119
|
+
} catch (e) {
|
|
120
|
+
this.synced.lookupError = String(e);
|
|
121
|
+
} finally {
|
|
122
|
+
this.synced.granting = "";
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
|
|
84
126
|
render() {
|
|
85
|
-
let
|
|
127
|
+
let synced = this.synced;
|
|
128
|
+
let { account, state, error } = synced;
|
|
86
129
|
if (!account) {
|
|
87
130
|
return <div className={css.vbox(8).pad2(16)}>
|
|
88
131
|
<div>Remote storage server.</div>
|
|
@@ -95,31 +138,71 @@ class AccessPage extends preact.Component {
|
|
|
95
138
|
{!state && !error && <div>Requesting access...</div>}
|
|
96
139
|
{state && !state.hasAccess && <div className={css.vbox(8)}>
|
|
97
140
|
<div>This machine ({state.machineId}, ip {state.ip}) does NOT have access yet.</div>
|
|
98
|
-
<div>An access request has been made. To grant it, run this
|
|
99
|
-
<CopyableCommand command={state.grantAccessCommand
|
|
141
|
+
<div>An access request has been made. To grant it, run this command:</div>
|
|
142
|
+
{state.grantAccessCommand && <CopyableCommand command={state.grantAccessCommand} />}
|
|
100
143
|
<div>This page rechecks every {REFRESH_INTERVAL / 1000} seconds.</div>
|
|
101
144
|
</div>}
|
|
102
|
-
{state && state.hasAccess && <div className={css.vbox(
|
|
145
|
+
{state && state.hasAccess && <div className={css.vbox(16)}>
|
|
103
146
|
<div>This machine ({state.machineId}, ip {state.ip}) has access.</div>
|
|
104
|
-
<div
|
|
105
|
-
|
|
106
|
-
<
|
|
107
|
-
<
|
|
108
|
-
<
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
</
|
|
121
|
-
</
|
|
122
|
-
</
|
|
147
|
+
<div className={css.vbox(6)}>
|
|
148
|
+
<div>Machines with access:</div>
|
|
149
|
+
<table>
|
|
150
|
+
<thead>
|
|
151
|
+
<tr>
|
|
152
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Machine</th>
|
|
153
|
+
<th className={css.pad2(8, 2).textAlign("left")}>IP</th>
|
|
154
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Granted</th>
|
|
155
|
+
</tr>
|
|
156
|
+
</thead>
|
|
157
|
+
<tbody>
|
|
158
|
+
{(state.trustedMachines || []).map(m => <tr key={m.machineId}>
|
|
159
|
+
<td className={css.pad2(8, 2)}>{m.machineId}</td>
|
|
160
|
+
<td className={css.pad2(8, 2)}>{m.ip}</td>
|
|
161
|
+
<td className={css.pad2(8, 2)}>{new Date(m.time).toLocaleString()}</td>
|
|
162
|
+
</tr>)}
|
|
163
|
+
</tbody>
|
|
164
|
+
</table>
|
|
165
|
+
</div>
|
|
166
|
+
<div className={css.vbox(6)}>
|
|
167
|
+
<div>Approve request</div>
|
|
168
|
+
<div className={css.hbox(8).alignItems("center")}>
|
|
169
|
+
<input
|
|
170
|
+
placeholder="ip"
|
|
171
|
+
value={synced.lookupIp}
|
|
172
|
+
onInput={e => { synced.lookupIp = (e.currentTarget as HTMLInputElement).value; }}
|
|
173
|
+
onKeyDown={e => { if (e.key === "Enter") void this.lookupIP(); }}
|
|
174
|
+
/>
|
|
175
|
+
<button disabled={synced.looking || !synced.lookupIp.trim()} onClick={this.lookupIP}>
|
|
176
|
+
search
|
|
177
|
+
</button>
|
|
178
|
+
</div>
|
|
179
|
+
{synced.lookupError && <div>Error: {synced.lookupError}</div>}
|
|
180
|
+
{synced.lookupResults && synced.lookupResults.requests.length === 0 && <div>
|
|
181
|
+
No pending requests for {synced.lookupResults.ip} on this account.
|
|
182
|
+
</div>}
|
|
183
|
+
{synced.lookupResults && synced.lookupResults.requests.length > 0 && <table>
|
|
184
|
+
<thead>
|
|
185
|
+
<tr>
|
|
186
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Machine</th>
|
|
187
|
+
<th className={css.pad2(8, 2).textAlign("left")}>IP</th>
|
|
188
|
+
<th className={css.pad2(8, 2).textAlign("left")}>Requested</th>
|
|
189
|
+
<th className={css.pad2(8, 2).textAlign("left")}></th>
|
|
190
|
+
</tr>
|
|
191
|
+
</thead>
|
|
192
|
+
<tbody>
|
|
193
|
+
{synced.lookupResults.requests.map(r => <tr key={r.requestId}>
|
|
194
|
+
<td className={css.pad2(8, 2)}>{r.machineId}</td>
|
|
195
|
+
<td className={css.pad2(8, 2)}>{r.ip}</td>
|
|
196
|
+
<td className={css.pad2(8, 2)}>{new Date(r.time).toLocaleString()}</td>
|
|
197
|
+
<td className={css.pad2(8, 2)}>
|
|
198
|
+
<button disabled={!!synced.granting} onClick={async () => this.approve(r)}>
|
|
199
|
+
{synced.granting === r.requestId && "Approving..." || "Approve"}
|
|
200
|
+
</button>
|
|
201
|
+
</td>
|
|
202
|
+
</tr>)}
|
|
203
|
+
</tbody>
|
|
204
|
+
</table>}
|
|
205
|
+
</div>
|
|
123
206
|
</div>}
|
|
124
207
|
</div>;
|
|
125
208
|
}
|