sliftutils 1.7.4 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +281 -2
- package/misc/https/persistentLocalStorage.ts +2 -0
- package/package.json +1 -1
- package/render-utils/Input.tsx +2 -0
- package/render-utils/InputLabel.tsx +2 -0
- package/render-utils/colors.tsx +2 -0
- package/storage/BulkDatabase2/syncClient.ts +10 -7
- package/storage/DiskCollection.d.ts +1 -0
- package/storage/DiskCollection.ts +3 -1
- package/storage/FileFolderAPI.d.ts +1 -0
- package/storage/FileFolderAPI.tsx +18 -0
- package/storage/IArchives.d.ts +37 -0
- package/storage/IArchives.ts +20 -0
- package/storage/TransactionStorage.d.ts +2 -1
- package/storage/TransactionStorage.ts +6 -2
- package/storage/backblaze.d.ts +2 -1
- package/storage/backblaze.ts +2 -1
- package/storage/remoteStorage/ArchivesRemote.d.ts +67 -0
- package/storage/remoteStorage/ArchivesRemote.ts +195 -0
- package/storage/remoteStorage/accessPage.d.ts +1 -0
- package/storage/remoteStorage/accessPage.tsx +133 -0
- package/storage/remoteStorage/blobStore.d.ts +56 -0
- package/storage/remoteStorage/blobStore.ts +403 -0
- package/storage/remoteStorage/storageController.d.ts +89 -0
- package/storage/remoteStorage/storageController.ts +398 -0
- package/storage/remoteStorage/storageServer.d.ts +1 -0
- package/storage/remoteStorage/storageServer.ts +132 -0
- package/teststorage/browser.tsx +148 -0
- package/teststorage/server.ts +43 -0
- package/yarn.lock +3 -3
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
module.allowclient = true;
|
|
2
|
+
|
|
3
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
4
|
+
import { getNodeIdIP } from "socket-function/src/nodeCache";
|
|
5
|
+
import { setHTTPResultHeaders } from "socket-function/src/callHTTPHandler";
|
|
6
|
+
import { timeInMinute } from "socket-function/src/misc";
|
|
7
|
+
import { getCommonName, getPublicIdentifier, getOwnMachineId, verify, verifyMachineIdForPublicKey } from "../../misc/https/certs";
|
|
8
|
+
import { ArchiveFileInfo } from "../IArchives";
|
|
9
|
+
import type { BlobStore, WriteConfig } from "./blobStore";
|
|
10
|
+
import type { IStorage } from "../IStorage";
|
|
11
|
+
|
|
12
|
+
// The remote storage server's API. Authentication uses certs.ts machine identities: a client
|
|
13
|
+
// proves it owns its machine key by signing a timestamped token (bound to this server, so tokens
|
|
14
|
+
// can't be replayed elsewhere), and the server then trusts that connection as that machineId.
|
|
15
|
+
// Access to an account is granted to specific machineIds, via a command line command run on the
|
|
16
|
+
// storage machine (see storageServer.ts).
|
|
17
|
+
|
|
18
|
+
export const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
|
|
19
|
+
export const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
|
|
20
|
+
// Error markers, so clients can identify these failures inside error messages
|
|
21
|
+
export const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
|
|
22
|
+
export const STORAGE_ACCESS_DENIED = "REMOTE_STORAGE_ACCESS_DENIED_9d81a4c0";
|
|
23
|
+
|
|
24
|
+
const AUTH_TIME_WINDOW = timeInMinute * 10;
|
|
25
|
+
const MAX_SESSIONS = 100 * 1000;
|
|
26
|
+
const MAX_REQUESTS_PER_IP = 50;
|
|
27
|
+
|
|
28
|
+
export type AuthToken = {
|
|
29
|
+
certPem: string;
|
|
30
|
+
time: number;
|
|
31
|
+
signature: string;
|
|
32
|
+
};
|
|
33
|
+
export type AccessRequest = {
|
|
34
|
+
requestId: string;
|
|
35
|
+
account: string;
|
|
36
|
+
machineId: string;
|
|
37
|
+
ip: string;
|
|
38
|
+
time: number;
|
|
39
|
+
};
|
|
40
|
+
export type TrustRecord = {
|
|
41
|
+
account: string;
|
|
42
|
+
machineId: string;
|
|
43
|
+
ip: string;
|
|
44
|
+
time: number;
|
|
45
|
+
};
|
|
46
|
+
export type BucketConfig = {
|
|
47
|
+
public?: boolean;
|
|
48
|
+
fast?: boolean;
|
|
49
|
+
writeDelay?: number;
|
|
50
|
+
};
|
|
51
|
+
export type AccessState = {
|
|
52
|
+
machineId: string;
|
|
53
|
+
ip: string;
|
|
54
|
+
hasAccess: boolean;
|
|
55
|
+
// Single ssh commands, runnable from anywhere, that run the admin CLI on the storage machine
|
|
56
|
+
// (shown when the caller has no access). grantAccessCommand grants the caller's own request.
|
|
57
|
+
listAccessCommand: string;
|
|
58
|
+
grantAccessCommand?: string;
|
|
59
|
+
// Only provided when the caller has access
|
|
60
|
+
machines?: (AccessRequest & { trusted: boolean })[];
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export type StorageServerState = {
|
|
64
|
+
domain: string;
|
|
65
|
+
port: number;
|
|
66
|
+
rootDomain: string;
|
|
67
|
+
// user@externalIp of the storage machine, for generated ssh commands
|
|
68
|
+
sshTarget: string;
|
|
69
|
+
// Absolute command that runs storageServer.ts on the storage machine (admin args appended)
|
|
70
|
+
serverCommand: string;
|
|
71
|
+
blobStore: BlobStore;
|
|
72
|
+
trust: IStorage<TrustRecord>;
|
|
73
|
+
requests: IStorage<AccessRequest[]>;
|
|
74
|
+
buckets: IStorage<BucketConfig>;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
let serverState: StorageServerState | undefined;
|
|
78
|
+
export function setStorageServerState(state: StorageServerState) {
|
|
79
|
+
serverState = state;
|
|
80
|
+
}
|
|
81
|
+
function getState(): StorageServerState {
|
|
82
|
+
let state = serverState;
|
|
83
|
+
if (!state) throw new Error(`Storage server is not initialized (this API only works on the storage server)`);
|
|
84
|
+
return state;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// callerNodeId -> authenticated machineId. Connections are long-lived websockets, so a session
|
|
88
|
+
// lasts until the connection drops (clients re-authenticate on reconnect).
|
|
89
|
+
const sessions = new Map<string, string>();
|
|
90
|
+
|
|
91
|
+
const CONTENT_TYPES: { [ext: string]: string } = {
|
|
92
|
+
html: "text/html", js: "text/javascript", css: "text/css", json: "application/json",
|
|
93
|
+
txt: "text/plain", png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif",
|
|
94
|
+
svg: "image/svg+xml", webp: "image/webp", mp4: "video/mp4", webm: "video/webm",
|
|
95
|
+
mp3: "audio/mpeg", wav: "audio/wav", pdf: "application/pdf",
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
function assertValidName(value: string, kind: string) {
|
|
99
|
+
if (!/^[\w-]{1,64}$/.test(value)) {
|
|
100
|
+
throw new Error(`Invalid ${kind} ${JSON.stringify(value)}, expected 1-64 characters of letters/numbers/underscore/dash`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function assertValidPath(path: string) {
|
|
104
|
+
if (Buffer.from(path, "utf8").length > 1000) {
|
|
105
|
+
throw new Error(`Path too long: ${path.length} characters > 1000. Path: ${path.slice(0, 200)}`);
|
|
106
|
+
}
|
|
107
|
+
if (!path || path.startsWith("/") || path.endsWith("/") || path.includes("//") || path.includes("\\") || path.includes("\x00")) {
|
|
108
|
+
throw new Error(`Invalid path ${JSON.stringify(path.slice(0, 200))}, paths cannot be empty, start or end with /, or contain //, backslashes, or null characters`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function getCallerMachineId(): string {
|
|
113
|
+
let caller = SocketFunction.getCaller();
|
|
114
|
+
let machineId = sessions.get(caller.nodeId);
|
|
115
|
+
if (!machineId) {
|
|
116
|
+
throw new Error(`${STORAGE_NOT_AUTHENTICATED} Call authenticate first (connection ${caller.nodeId})`);
|
|
117
|
+
}
|
|
118
|
+
return machineId;
|
|
119
|
+
}
|
|
120
|
+
function getCallerIP(): string {
|
|
121
|
+
return getNodeIdIP(SocketFunction.getCaller().nodeId);
|
|
122
|
+
}
|
|
123
|
+
function isAdmin(machineId: string): boolean {
|
|
124
|
+
let state = getState();
|
|
125
|
+
return machineId === getOwnMachineId(state.rootDomain);
|
|
126
|
+
}
|
|
127
|
+
function requireAdmin(): string {
|
|
128
|
+
let machineId = getCallerMachineId();
|
|
129
|
+
if (!isAdmin(machineId)) {
|
|
130
|
+
throw new Error(`${STORAGE_ACCESS_DENIED} Admin commands must be run from the storage machine itself (caller machine ${machineId})`);
|
|
131
|
+
}
|
|
132
|
+
return machineId;
|
|
133
|
+
}
|
|
134
|
+
async function requireAccess(account: string): Promise<string> {
|
|
135
|
+
assertValidName(account, "account");
|
|
136
|
+
let state = getState();
|
|
137
|
+
let machineId = getCallerMachineId();
|
|
138
|
+
if (isAdmin(machineId)) return machineId;
|
|
139
|
+
let trusted = await state.trust.get(`${account}|${machineId}`);
|
|
140
|
+
if (!trusted) {
|
|
141
|
+
throw new Error(`${STORAGE_ACCESS_DENIED} Machine ${machineId} has no access to account ${JSON.stringify(account)}. Visit https://${state.domain}:${state.port}/${account} for access instructions.`);
|
|
142
|
+
}
|
|
143
|
+
return machineId;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// A single command, runnable from anywhere, that sshes into the storage machine and runs the
|
|
147
|
+
// admin CLI there
|
|
148
|
+
function getAdminCommand(args: string): string {
|
|
149
|
+
let state = getState();
|
|
150
|
+
return `ssh ${state.sshTarget} '${state.serverCommand} ${args}'`;
|
|
151
|
+
}
|
|
152
|
+
function getListAccessCommand(ip: string): string {
|
|
153
|
+
return getAdminCommand(`--listAccess ${ip}`);
|
|
154
|
+
}
|
|
155
|
+
function getGrantAccessCommand(requestId: string): string {
|
|
156
|
+
return getAdminCommand(`--grantAccess ${requestId}`);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function getBucketWriteConfig(account: string, bucketName: string): Promise<WriteConfig> {
|
|
160
|
+
let state = getState();
|
|
161
|
+
let config = await state.buckets.get(`${account}/${bucketName}`);
|
|
162
|
+
return { fast: config?.fast, writeDelay: config?.writeDelay };
|
|
163
|
+
}
|
|
164
|
+
function fileKey(account: string, bucketName: string, path: string): string {
|
|
165
|
+
assertValidName(account, "account");
|
|
166
|
+
assertValidName(bucketName, "bucket name");
|
|
167
|
+
assertValidPath(path);
|
|
168
|
+
return `${account}/${bucketName}/${path}`;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
class RemoteStorageControllerBase {
|
|
172
|
+
// Proves the caller owns the machine key for the machineId in its certificate. The signature
|
|
173
|
+
// must be fresh and bound to this server, so it cannot be replayed to (or from) other servers.
|
|
174
|
+
async authenticate(token: AuthToken): Promise<{ machineId: string; ip: string }> {
|
|
175
|
+
let state = getState();
|
|
176
|
+
let caller = SocketFunction.getCaller();
|
|
177
|
+
if (Math.abs(Date.now() - token.time) > AUTH_TIME_WINDOW) {
|
|
178
|
+
throw new Error(`Auth token time is too far from the server time (token ${token.time}, server ${Date.now()}, allowed drift ${AUTH_TIME_WINDOW}ms)`);
|
|
179
|
+
}
|
|
180
|
+
verify(token.certPem, token.signature, {
|
|
181
|
+
purpose: STORAGE_AUTH_PURPOSE,
|
|
182
|
+
time: token.time,
|
|
183
|
+
server: `${state.domain}:${state.port}`,
|
|
184
|
+
});
|
|
185
|
+
let machineId = getCommonName(token.certPem).split(".")[0];
|
|
186
|
+
if (!verifyMachineIdForPublicKey({ machineId, publicKey: getPublicIdentifier(token.certPem) })) {
|
|
187
|
+
throw new Error(`Certificate common name ${JSON.stringify(getCommonName(token.certPem))} does not match its public key`);
|
|
188
|
+
}
|
|
189
|
+
sessions.set(caller.nodeId, machineId);
|
|
190
|
+
while (sessions.size > MAX_SESSIONS) {
|
|
191
|
+
let oldest = sessions.keys().next().value;
|
|
192
|
+
if (oldest === undefined) break;
|
|
193
|
+
sessions.delete(oldest);
|
|
194
|
+
}
|
|
195
|
+
return { machineId, ip: getCallerIP() };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Records that the calling machine wants access to an account. Requests are kept per requesting
|
|
199
|
+
// IP, so the storage machine's admin can list them with --listAccess <ip> and grant one.
|
|
200
|
+
async requestAccess(account: string): Promise<{ machineId: string; ip: string; requestId: string; grantAccessCommand: string }> {
|
|
201
|
+
assertValidName(account, "account");
|
|
202
|
+
let state = getState();
|
|
203
|
+
let machineId = getCallerMachineId();
|
|
204
|
+
let ip = getCallerIP();
|
|
205
|
+
let requests = await state.requests.get(ip) || [];
|
|
206
|
+
let existing = requests.find(x => x.account === account && x.machineId === machineId);
|
|
207
|
+
if (existing) {
|
|
208
|
+
existing.time = Date.now();
|
|
209
|
+
} else {
|
|
210
|
+
existing = {
|
|
211
|
+
requestId: Math.random().toString(36).slice(2, 10),
|
|
212
|
+
account,
|
|
213
|
+
machineId,
|
|
214
|
+
ip,
|
|
215
|
+
time: Date.now(),
|
|
216
|
+
};
|
|
217
|
+
requests.push(existing);
|
|
218
|
+
}
|
|
219
|
+
while (requests.length > MAX_REQUESTS_PER_IP) requests.shift();
|
|
220
|
+
await state.requests.set(ip, requests);
|
|
221
|
+
return { machineId, ip, requestId: existing.requestId, grantAccessCommand: getGrantAccessCommand(existing.requestId) };
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async getAccessState(account: string): Promise<AccessState> {
|
|
225
|
+
assertValidName(account, "account");
|
|
226
|
+
let state = getState();
|
|
227
|
+
let machineId = getCallerMachineId();
|
|
228
|
+
let ip = getCallerIP();
|
|
229
|
+
let hasAccess = isAdmin(machineId) || !!await state.trust.get(`${account}|${machineId}`);
|
|
230
|
+
let result: AccessState = { machineId, ip, hasAccess, listAccessCommand: getListAccessCommand(ip) };
|
|
231
|
+
if (!hasAccess) {
|
|
232
|
+
let ownRequest = (await state.requests.get(ip) || []).find(x => x.account === account && x.machineId === machineId);
|
|
233
|
+
if (ownRequest) {
|
|
234
|
+
result.grantAccessCommand = getGrantAccessCommand(ownRequest.requestId);
|
|
235
|
+
}
|
|
236
|
+
return result;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
let machines = new Map<string, AccessRequest & { trusted: boolean }>();
|
|
240
|
+
for (let requestIp of await state.requests.getKeys()) {
|
|
241
|
+
for (let request of await state.requests.get(requestIp) || []) {
|
|
242
|
+
if (request.account !== account) continue;
|
|
243
|
+
machines.set(request.machineId, { ...request, trusted: false });
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
for (let key of await state.trust.getKeys()) {
|
|
247
|
+
if (!key.startsWith(`${account}|`)) continue;
|
|
248
|
+
let record = await state.trust.get(key);
|
|
249
|
+
if (!record) continue;
|
|
250
|
+
let existing = machines.get(record.machineId);
|
|
251
|
+
machines.set(record.machineId, {
|
|
252
|
+
requestId: existing?.requestId || "",
|
|
253
|
+
account,
|
|
254
|
+
machineId: record.machineId,
|
|
255
|
+
ip: record.ip,
|
|
256
|
+
time: record.time,
|
|
257
|
+
trusted: true,
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
result.machines = Array.from(machines.values());
|
|
261
|
+
return result;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Admin (must be run from the storage machine itself, which shares the server's machineId).
|
|
265
|
+
// Only returns requests for the given IP, so you cannot accidentally grant a request from an
|
|
266
|
+
// IP you didn't explicitly type in.
|
|
267
|
+
async adminListRequests(ip: string): Promise<AccessRequest[]> {
|
|
268
|
+
requireAdmin();
|
|
269
|
+
let state = getState();
|
|
270
|
+
return await state.requests.get(ip) || [];
|
|
271
|
+
}
|
|
272
|
+
async adminGrantAccess(requestId: string): Promise<TrustRecord> {
|
|
273
|
+
requireAdmin();
|
|
274
|
+
let state = getState();
|
|
275
|
+
for (let ip of await state.requests.getKeys()) {
|
|
276
|
+
for (let request of await state.requests.get(ip) || []) {
|
|
277
|
+
if (request.requestId !== requestId) continue;
|
|
278
|
+
let record: TrustRecord = {
|
|
279
|
+
account: request.account,
|
|
280
|
+
machineId: request.machineId,
|
|
281
|
+
ip: request.ip,
|
|
282
|
+
time: Date.now(),
|
|
283
|
+
};
|
|
284
|
+
await state.trust.set(`${request.account}|${request.machineId}`, record);
|
|
285
|
+
return record;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
throw new Error(`No access request found with id ${JSON.stringify(requestId)}. Run --listAccess <ip> to see request ids.`);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async ensureBucket(account: string, bucketName: string, config: BucketConfig): Promise<void> {
|
|
292
|
+
await requireAccess(account);
|
|
293
|
+
assertValidName(bucketName, "bucket name");
|
|
294
|
+
let state = getState();
|
|
295
|
+
let key = `${account}/${bucketName}`;
|
|
296
|
+
let existing = await state.buckets.get(key);
|
|
297
|
+
if (existing && JSON.stringify(existing) === JSON.stringify(config)) return;
|
|
298
|
+
await state.buckets.set(key, config);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async get(account: string, bucketName: string, path: string, range?: { start: number; end: number }): Promise<Buffer | undefined> {
|
|
302
|
+
await requireAccess(account);
|
|
303
|
+
return await getState().blobStore.get(fileKey(account, bucketName, path), range);
|
|
304
|
+
}
|
|
305
|
+
async set(account: string, bucketName: string, path: string, data: Buffer): Promise<void> {
|
|
306
|
+
await requireAccess(account);
|
|
307
|
+
let writeConfig = await getBucketWriteConfig(account, bucketName);
|
|
308
|
+
await getState().blobStore.set(fileKey(account, bucketName, path), Buffer.from(data), writeConfig);
|
|
309
|
+
}
|
|
310
|
+
async del(account: string, bucketName: string, path: string): Promise<void> {
|
|
311
|
+
await requireAccess(account);
|
|
312
|
+
let writeConfig = await getBucketWriteConfig(account, bucketName);
|
|
313
|
+
await getState().blobStore.del(fileKey(account, bucketName, path), writeConfig);
|
|
314
|
+
}
|
|
315
|
+
async getInfo(account: string, bucketName: string, path: string): Promise<{ writeTime: number; size: number } | undefined> {
|
|
316
|
+
await requireAccess(account);
|
|
317
|
+
return await getState().blobStore.getInfo(fileKey(account, bucketName, path));
|
|
318
|
+
}
|
|
319
|
+
async findInfo(account: string, bucketName: string, prefix: string, config?: { shallow?: boolean; type?: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
|
|
320
|
+
await requireAccess(account);
|
|
321
|
+
assertValidName(bucketName, "bucket name");
|
|
322
|
+
let bucketRoot = `${account}/${bucketName}/`;
|
|
323
|
+
let infos = await getState().blobStore.findInfo(bucketRoot + prefix, config);
|
|
324
|
+
return infos.map(info => ({ ...info, path: info.path.slice(bucketRoot.length) }));
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async startLargeFile(account: string, bucketName: string, path: string): Promise<string> {
|
|
328
|
+
await requireAccess(account);
|
|
329
|
+
// Validates now, so the upload doesn't fail at the end
|
|
330
|
+
fileKey(account, bucketName, path);
|
|
331
|
+
let id = await getState().blobStore.startLargeUpload();
|
|
332
|
+
largeUploadInfo.set(id, { account, bucketName, path });
|
|
333
|
+
return id;
|
|
334
|
+
}
|
|
335
|
+
async uploadPart(uploadId: string, data: Buffer): Promise<void> {
|
|
336
|
+
let info = largeUploadInfo.get(uploadId);
|
|
337
|
+
if (!info) throw new Error(`Unknown large upload ${uploadId}`);
|
|
338
|
+
await requireAccess(info.account);
|
|
339
|
+
await getState().blobStore.appendLargeUpload(uploadId, Buffer.from(data));
|
|
340
|
+
}
|
|
341
|
+
async finishLargeFile(uploadId: string): Promise<void> {
|
|
342
|
+
let info = largeUploadInfo.get(uploadId);
|
|
343
|
+
if (!info) throw new Error(`Unknown large upload ${uploadId}`);
|
|
344
|
+
await requireAccess(info.account);
|
|
345
|
+
largeUploadInfo.delete(uploadId);
|
|
346
|
+
await getState().blobStore.finishLargeUpload(uploadId, fileKey(info.account, info.bucketName, info.path));
|
|
347
|
+
}
|
|
348
|
+
async cancelLargeFile(uploadId: string): Promise<void> {
|
|
349
|
+
let info = largeUploadInfo.get(uploadId);
|
|
350
|
+
if (!info) return;
|
|
351
|
+
await requireAccess(info.account);
|
|
352
|
+
largeUploadInfo.delete(uploadId);
|
|
353
|
+
await getState().blobStore.cancelLargeUpload(uploadId);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Serves files from public buckets over plain HTTP GET (see IArchives getURL). No
|
|
357
|
+
// authentication, which is what public means (private buckets are API-access only).
|
|
358
|
+
async getPublicFile(account: string, bucketName: string, path: string): Promise<Buffer> {
|
|
359
|
+
let state = getState();
|
|
360
|
+
let bucket = await state.buckets.get(`${account}/${bucketName}`);
|
|
361
|
+
if (!bucket?.public) {
|
|
362
|
+
throw new Error(`Bucket ${account}/${bucketName} is not public`);
|
|
363
|
+
}
|
|
364
|
+
let data = await state.blobStore.get(fileKey(account, bucketName, path));
|
|
365
|
+
if (!data) {
|
|
366
|
+
throw new Error(`File not found: ${path} in ${account}/${bucketName}`);
|
|
367
|
+
}
|
|
368
|
+
let ext = path.split(".").pop() || "";
|
|
369
|
+
return setHTTPResultHeaders(data, {
|
|
370
|
+
"Content-Type": CONTENT_TYPES[ext.toLowerCase()] || "application/octet-stream",
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
const largeUploadInfo = new Map<string, { account: string; bucketName: string; path: string }>();
|
|
376
|
+
|
|
377
|
+
export const RemoteStorageController = SocketFunction.register(
|
|
378
|
+
REMOTE_STORAGE_CLASS_GUID,
|
|
379
|
+
new RemoteStorageControllerBase(),
|
|
380
|
+
() => ({
|
|
381
|
+
authenticate: {},
|
|
382
|
+
requestAccess: {},
|
|
383
|
+
getAccessState: {},
|
|
384
|
+
adminListRequests: {},
|
|
385
|
+
adminGrantAccess: {},
|
|
386
|
+
ensureBucket: {},
|
|
387
|
+
get: {},
|
|
388
|
+
set: {},
|
|
389
|
+
del: {},
|
|
390
|
+
getInfo: {},
|
|
391
|
+
findInfo: {},
|
|
392
|
+
startLargeFile: {},
|
|
393
|
+
uploadPart: {},
|
|
394
|
+
finishLargeFile: {},
|
|
395
|
+
cancelLargeFile: {},
|
|
396
|
+
getPublicFile: {},
|
|
397
|
+
})
|
|
398
|
+
);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import "./accessPage";
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
process.env.NODE_ENV = "production";
|
|
2
|
+
import os from "os";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
5
|
+
import { getExternalIP } from "socket-function/src/networking";
|
|
6
|
+
import { RequireController } from "socket-function/require/RequireController";
|
|
7
|
+
import { hostServer } from "../../misc/https/hostServer";
|
|
8
|
+
import { getFileStorageNested2 } from "../FileFolderAPI";
|
|
9
|
+
import { TransactionStorage } from "../TransactionStorage";
|
|
10
|
+
import { JSONStorage } from "../JSONStorage";
|
|
11
|
+
import { BlobStore } from "./blobStore";
|
|
12
|
+
import {
|
|
13
|
+
RemoteStorageController, setStorageServerState,
|
|
14
|
+
AccessRequest, TrustRecord, BucketConfig,
|
|
15
|
+
} from "./storageController";
|
|
16
|
+
import { authenticateStorage } from "./ArchivesRemote";
|
|
17
|
+
// Import browser code, so it is allowed to be required by the client
|
|
18
|
+
import "./accessPage";
|
|
19
|
+
|
|
20
|
+
// The remote storage server. Run modes:
|
|
21
|
+
// Host the server:
|
|
22
|
+
// typenode storage/remoteStorage/storageServer.ts --domain storage.example.com --port 4444
|
|
23
|
+
// --folder /storage/data --cloudflareApiTokenPath ~/example.com.key
|
|
24
|
+
// Admin commands (run on the storage machine itself, against the running server):
|
|
25
|
+
// --listAccess <ip> lists pending access requests from that IP (with requestIds)
|
|
26
|
+
// --grantAccess <requestId> trusts the machine from that request for the requested account
|
|
27
|
+
|
|
28
|
+
process.on("unhandledRejection", (error) => {
|
|
29
|
+
console.error("Unhandled promise rejection:", error);
|
|
30
|
+
});
|
|
31
|
+
process.on("uncaughtException", (error) => {
|
|
32
|
+
console.error("Uncaught exception:", error);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// The absolute command that runs this script (with the given args) on this machine, usable from
|
|
36
|
+
// any working directory (ex: over ssh)
|
|
37
|
+
function getServerCommand(args: string): string {
|
|
38
|
+
return `${process.execPath} ${require.resolve("typenode/bootstrap.js")} ${__filename} ${args}`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function getArg(name: string): string | undefined {
|
|
42
|
+
let index = process.argv.indexOf(`--${name}`);
|
|
43
|
+
if (index < 0) return undefined;
|
|
44
|
+
let value = process.argv[index + 1];
|
|
45
|
+
if (!value || value.startsWith("--")) {
|
|
46
|
+
throw new Error(`Missing value for --${name}`);
|
|
47
|
+
}
|
|
48
|
+
return value;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function runAdminCommand(config: { domain: string; port: number; listAccess?: string; grantAccess?: string }) {
|
|
52
|
+
let nodeId = SocketFunction.connect({ address: config.domain, port: config.port });
|
|
53
|
+
let controller = RemoteStorageController.nodes[nodeId];
|
|
54
|
+
await authenticateStorage({ address: config.domain, port: config.port, nodeId });
|
|
55
|
+
if (config.listAccess) {
|
|
56
|
+
let requests = await controller.adminListRequests(config.listAccess);
|
|
57
|
+
if (!requests.length) {
|
|
58
|
+
console.log(`No access requests from ${config.listAccess}`);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
console.log(`Access requests from ${config.listAccess}:`);
|
|
62
|
+
for (let request of requests) {
|
|
63
|
+
console.log(` --grantAccess ${request.requestId} (account ${request.account}, machine ${request.machineId}, requested ${new Date(request.time).toISOString()})`);
|
|
64
|
+
}
|
|
65
|
+
console.log(`Grant one with: ${getServerCommand(`--domain ${config.domain} --port ${config.port} --grantAccess <requestId>`)}`);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (config.grantAccess) {
|
|
69
|
+
let record = await controller.adminGrantAccess(config.grantAccess);
|
|
70
|
+
console.log(`Granted machine ${record.machineId} access to account ${JSON.stringify(record.account)}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function main() {
|
|
75
|
+
let domain = getArg("domain");
|
|
76
|
+
if (!domain) throw new Error(`--domain is required (ex: --domain storage.example.com)`);
|
|
77
|
+
let port = +(getArg("port") || 443);
|
|
78
|
+
|
|
79
|
+
let listAccess = getArg("listAccess");
|
|
80
|
+
let grantAccess = getArg("grantAccess");
|
|
81
|
+
if (listAccess || grantAccess) {
|
|
82
|
+
await runAdminCommand({ domain, port, listAccess, grantAccess });
|
|
83
|
+
process.exit(0);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
let folder = getArg("folder");
|
|
87
|
+
if (!folder) throw new Error(`--folder is required (where the storage server keeps its data)`);
|
|
88
|
+
let cloudflareApiTokenPath = getArg("cloudflareApiTokenPath");
|
|
89
|
+
if (!cloudflareApiTokenPath) throw new Error(`--cloudflareApiTokenPath is required (path to a Cloudflare API token file, for HTTPS certs)`);
|
|
90
|
+
|
|
91
|
+
let root = await getFileStorageNested2(path.resolve(folder));
|
|
92
|
+
let system = await root.folder.getStorage("system");
|
|
93
|
+
let trust = new JSONStorage<TrustRecord>(new TransactionStorage(await system.folder.getStorage("trust"), "storageTrust"));
|
|
94
|
+
let requests = new JSONStorage<AccessRequest[]>(new TransactionStorage(await system.folder.getStorage("requests"), "storageRequests"));
|
|
95
|
+
let buckets = new JSONStorage<BucketConfig>(new TransactionStorage(await system.folder.getStorage("buckets"), "storageBuckets"));
|
|
96
|
+
|
|
97
|
+
setStorageServerState({
|
|
98
|
+
domain,
|
|
99
|
+
port,
|
|
100
|
+
rootDomain: domain.split(".").slice(-2).join("."),
|
|
101
|
+
sshTarget: `${os.userInfo().username}@${await getExternalIP()}`,
|
|
102
|
+
serverCommand: getServerCommand(`--domain ${domain} --port ${port}`),
|
|
103
|
+
blobStore: new BlobStore(path.resolve(folder)),
|
|
104
|
+
trust,
|
|
105
|
+
requests,
|
|
106
|
+
buckets,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
RequireController.allowAllNodeModules();
|
|
110
|
+
SocketFunction.expose(RequireController);
|
|
111
|
+
SocketFunction.expose(RemoteStorageController);
|
|
112
|
+
// No static roots, so the access page HTML is served at every path (the path is the account
|
|
113
|
+
// name, see accessPage.tsx)
|
|
114
|
+
// A full URL, so the page resolves modules from the origin root even when served at
|
|
115
|
+
// /accountName (a relative require would resolve inside the account path)
|
|
116
|
+
SocketFunction.setDefaultHTTPCall(RequireController, "requireHTML", {
|
|
117
|
+
requireCalls: [`https://${domain}:${port}/./storage/remoteStorage/accessPage.tsx`],
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
await hostServer({
|
|
121
|
+
domain,
|
|
122
|
+
port,
|
|
123
|
+
cloudflareApiTokenPath,
|
|
124
|
+
setDNSRecord: true,
|
|
125
|
+
});
|
|
126
|
+
console.log(`Storage server running at https://${domain}:${port}`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
main().catch(e => {
|
|
130
|
+
console.error(e);
|
|
131
|
+
process.exit(1);
|
|
132
|
+
});
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
module.allowclient = true;
|
|
2
|
+
|
|
3
|
+
import preact from "preact";
|
|
4
|
+
import { observable } from "mobx";
|
|
5
|
+
import { observer } from "../render-utils/observer";
|
|
6
|
+
import { isNode } from "socket-function/src/misc";
|
|
7
|
+
import { delay } from "socket-function/src/batching";
|
|
8
|
+
import { formatNumber, formatDateTime } from "socket-function/src/formatting/format";
|
|
9
|
+
import { css } from "typesafecss";
|
|
10
|
+
import { InputLabel } from "../render-utils/InputLabel";
|
|
11
|
+
import { redButton, greenButton, errorMessage } from "../render-utils/colors";
|
|
12
|
+
import { ArchivesRemote } from "../storage/remoteStorage/ArchivesRemote";
|
|
13
|
+
import { ArchiveFileInfo } from "../storage/IArchives";
|
|
14
|
+
|
|
15
|
+
const STORAGE_ADDRESS = "storage.vidgridweb.com";
|
|
16
|
+
const STORAGE_PORT = 4444;
|
|
17
|
+
const ACCOUNT = "test";
|
|
18
|
+
const BUCKET = "testfiles";
|
|
19
|
+
const ACCESS_CHECK_INTERVAL = 1000 * 15;
|
|
20
|
+
|
|
21
|
+
const archives = new ArchivesRemote({
|
|
22
|
+
address: STORAGE_ADDRESS,
|
|
23
|
+
port: STORAGE_PORT,
|
|
24
|
+
account: ACCOUNT,
|
|
25
|
+
bucketName: BUCKET,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
@observer
|
|
29
|
+
class TestStoragePage extends preact.Component {
|
|
30
|
+
synced = observable({
|
|
31
|
+
files: [] as ArchiveFileInfo[],
|
|
32
|
+
loaded: false,
|
|
33
|
+
selectedPath: "",
|
|
34
|
+
content: "",
|
|
35
|
+
contentLoaded: false,
|
|
36
|
+
newFileName: "",
|
|
37
|
+
accessLink: "",
|
|
38
|
+
error: "",
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
componentDidMount() {
|
|
42
|
+
void (async () => {
|
|
43
|
+
while (true) {
|
|
44
|
+
let link = await archives.waitingForAccess();
|
|
45
|
+
this.synced.accessLink = link || "";
|
|
46
|
+
if (!link) break;
|
|
47
|
+
await delay(ACCESS_CHECK_INTERVAL);
|
|
48
|
+
}
|
|
49
|
+
await this.refresh();
|
|
50
|
+
})();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
private async refresh() {
|
|
54
|
+
try {
|
|
55
|
+
this.synced.files = await archives.findInfo("", { type: "files" });
|
|
56
|
+
this.synced.loaded = true;
|
|
57
|
+
this.synced.error = "";
|
|
58
|
+
} catch (e) {
|
|
59
|
+
this.synced.error = String(e);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private async openFile(path: string) {
|
|
64
|
+
this.synced.selectedPath = path;
|
|
65
|
+
this.synced.contentLoaded = false;
|
|
66
|
+
let data = await archives.get(path);
|
|
67
|
+
this.synced.content = data && data.toString() || "";
|
|
68
|
+
this.synced.contentLoaded = true;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
private async createFile() {
|
|
72
|
+
let name = this.synced.newFileName.trim();
|
|
73
|
+
if (!name) return;
|
|
74
|
+
await archives.set(name, Buffer.from(""));
|
|
75
|
+
this.synced.newFileName = "";
|
|
76
|
+
await this.refresh();
|
|
77
|
+
await this.openFile(name);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
private async deleteFile(path: string) {
|
|
81
|
+
await archives.del(path);
|
|
82
|
+
if (this.synced.selectedPath === path) {
|
|
83
|
+
this.synced.selectedPath = "";
|
|
84
|
+
this.synced.content = "";
|
|
85
|
+
this.synced.contentLoaded = false;
|
|
86
|
+
}
|
|
87
|
+
await this.refresh();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
render() {
|
|
91
|
+
let synced = this.synced;
|
|
92
|
+
return <div className={css.vbox(12).pad2(16)}>
|
|
93
|
+
<div>Storage test site. Account {ACCOUNT}, bucket {BUCKET} on {STORAGE_ADDRESS}:{STORAGE_PORT}</div>
|
|
94
|
+
{synced.error && <div className={errorMessage}>{synced.error}</div>}
|
|
95
|
+
{!synced.loaded && !synced.accessLink && <div>Loading files...</div>}
|
|
96
|
+
{!synced.loaded && synced.accessLink && <div className={css.vbox(8)}>
|
|
97
|
+
<div>This machine does not have access to the account yet. Grant it here:</div>
|
|
98
|
+
<a href={synced.accessLink} target="_blank">{synced.accessLink}</a>
|
|
99
|
+
</div>}
|
|
100
|
+
{synced.loaded && <div className={css.hbox(24).alignItems("flex-start")}>
|
|
101
|
+
<div className={css.vbox(8)}>
|
|
102
|
+
{synced.files.length === 0 && <div>No files yet</div>}
|
|
103
|
+
{synced.files.map(file => <div key={file.path} className={css.hbox(8).alignItems("center")}>
|
|
104
|
+
<button onClick={async () => this.openFile(file.path)}>
|
|
105
|
+
{file.path}
|
|
106
|
+
</button>
|
|
107
|
+
<div>{formatNumber(file.size)}B, {formatDateTime(file.createTime)}</div>
|
|
108
|
+
<button className={redButton} onClick={async () => this.deleteFile(file.path)}>
|
|
109
|
+
Delete
|
|
110
|
+
</button>
|
|
111
|
+
</div>)}
|
|
112
|
+
<div className={css.hbox(8).alignItems("center")}>
|
|
113
|
+
<InputLabel
|
|
114
|
+
label="New file"
|
|
115
|
+
hot
|
|
116
|
+
value={synced.newFileName}
|
|
117
|
+
onChangeValue={value => { synced.newFileName = value; }}
|
|
118
|
+
/>
|
|
119
|
+
<button className={greenButton} onClick={async () => this.createFile()}>
|
|
120
|
+
Create
|
|
121
|
+
</button>
|
|
122
|
+
</div>
|
|
123
|
+
</div>
|
|
124
|
+
{synced.selectedPath && <div className={css.vbox(8).flexGrow(1)}>
|
|
125
|
+
<div>Editing {synced.selectedPath}</div>
|
|
126
|
+
{!synced.contentLoaded && <div>Loading content...</div>}
|
|
127
|
+
{synced.contentLoaded && <InputLabel
|
|
128
|
+
textarea
|
|
129
|
+
fillWidth
|
|
130
|
+
value={synced.content}
|
|
131
|
+
onChangeValue={async value => {
|
|
132
|
+
await archives.set(synced.selectedPath, Buffer.from(value));
|
|
133
|
+
synced.content = value;
|
|
134
|
+
await this.refresh();
|
|
135
|
+
}}
|
|
136
|
+
/>}
|
|
137
|
+
</div>}
|
|
138
|
+
</div>}
|
|
139
|
+
</div>;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function main() {
|
|
144
|
+
if (isNode()) return;
|
|
145
|
+
preact.render(<TestStoragePage />, document.body);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
main().catch(console.error);
|