sliftutils 1.7.9 → 1.7.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/bin/storageserver.js +4 -0
  2. package/{teststorage → examplestorage}/browser.tsx +18 -22
  3. package/{teststorage/server.ts → examplestorage/exampleserver.ts} +4 -6
  4. package/index.d.ts +770 -84
  5. package/misc/dist/environment.ts.cache +3 -3
  6. package/misc/dist/getSecret.ts.cache +139 -0
  7. package/misc/dist/strings.ts.cache +13 -0
  8. package/misc/dist/zip.ts.cache +9 -0
  9. package/misc/getSecret.ts +38 -16
  10. package/misc/https/cloudflareHelpers.d.ts +19 -0
  11. package/misc/https/cloudflareHelpers.ts +69 -0
  12. package/misc/https/dist/certs.ts.cache +115 -179
  13. package/misc/https/dist/cloudflareHelpers.ts.cache +68 -0
  14. package/misc/https/dist/dns.ts.cache +112 -100
  15. package/misc/https/dist/hostServer.ts.cache +130 -0
  16. package/misc/https/dist/httpsCerts.ts.cache +24 -5
  17. package/misc/https/dist/persistentLocalStorage.ts.cache +3 -2
  18. package/misc/https/dns.d.ts +41 -8
  19. package/misc/https/dns.ts +104 -106
  20. package/misc/https/hostServer.d.ts +0 -3
  21. package/misc/https/hostServer.ts +12 -8
  22. package/package.json +3 -2
  23. package/render-utils/dist/observer.tsx.cache +3 -2
  24. package/spec.txt +0 -55
  25. package/storage/ArchivesDisk.d.ts +67 -0
  26. package/storage/ArchivesDisk.ts +369 -0
  27. package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +1242 -0
  28. package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +513 -0
  29. package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +352 -0
  30. package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +306 -0
  31. package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +435 -0
  32. package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +58 -0
  33. package/storage/BulkDatabase2/dist/blockCache.ts.cache +179 -0
  34. package/storage/BulkDatabase2/dist/mergeLock.ts.cache +149 -0
  35. package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +101 -0
  36. package/storage/BulkDatabase2/dist/streamLog.ts.cache +227 -0
  37. package/storage/BulkDatabase2/dist/syncClient.ts.cache +104 -0
  38. package/storage/IArchives.d.ts +127 -1
  39. package/storage/IArchives.ts +201 -1
  40. package/storage/backblaze.d.ts +29 -2
  41. package/storage/backblaze.ts +214 -78
  42. package/storage/dist/ArchivesDisk.ts.cache +373 -0
  43. package/storage/dist/FileFolderAPI.tsx.cache +508 -65
  44. package/storage/dist/IArchives.ts.cache +44 -0
  45. package/storage/dist/IndexedDBFileFolderAPI.ts.cache +8 -2
  46. package/storage/dist/JSONStorage.ts.cache +6 -2
  47. package/storage/dist/PendingManager.tsx.cache +1 -1
  48. package/storage/dist/TransactionStorage.ts.cache +231 -92
  49. package/storage/dist/backblaze.ts.cache +797 -0
  50. package/storage/dist/fileSystemPointer.ts.cache +36 -3
  51. package/storage/dist/remoteFileStorage.ts.cache +495 -0
  52. package/storage/remoteStorage/ArchivesRemote.d.ts +34 -18
  53. package/storage/remoteStorage/ArchivesRemote.ts +85 -65
  54. package/storage/remoteStorage/ArchivesUrl.d.ts +48 -0
  55. package/storage/remoteStorage/ArchivesUrl.ts +86 -0
  56. package/storage/remoteStorage/accessPage.tsx +111 -28
  57. package/storage/remoteStorage/blobStore.d.ts +128 -25
  58. package/storage/remoteStorage/blobStore.ts +757 -290
  59. package/storage/remoteStorage/cliArgs.d.ts +1 -0
  60. package/storage/remoteStorage/cliArgs.ts +9 -0
  61. package/storage/remoteStorage/createArchives.d.ts +85 -0
  62. package/storage/remoteStorage/createArchives.ts +612 -0
  63. package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +197 -0
  64. package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +85 -0
  65. package/storage/remoteStorage/dist/accessPage.tsx.cache +222 -0
  66. package/storage/remoteStorage/dist/blobStore.ts.cache +837 -0
  67. package/storage/remoteStorage/dist/cliArgs.ts.cache +16 -0
  68. package/storage/remoteStorage/dist/createArchives.ts.cache +610 -0
  69. package/storage/remoteStorage/dist/remoteConfig.ts.cache +206 -0
  70. package/storage/remoteStorage/dist/sourceWrapper.ts.cache +223 -0
  71. package/storage/remoteStorage/dist/storageController.ts.cache +500 -0
  72. package/storage/remoteStorage/dist/storageServer.ts.cache +106 -0
  73. package/storage/remoteStorage/dist/storageServerCli.ts.cache +39 -0
  74. package/storage/remoteStorage/dist/storageServerState.ts.cache +518 -0
  75. package/storage/remoteStorage/grantAccess.js +4 -0
  76. package/storage/remoteStorage/grantAccessCli.d.ts +1 -0
  77. package/storage/remoteStorage/grantAccessCli.ts +27 -0
  78. package/storage/remoteStorage/remoteConfig.d.ts +29 -0
  79. package/storage/remoteStorage/remoteConfig.ts +194 -0
  80. package/storage/remoteStorage/sourceWrapper.d.ts +41 -0
  81. package/storage/remoteStorage/sourceWrapper.ts +223 -0
  82. package/storage/remoteStorage/spec.md +31 -0
  83. package/storage/remoteStorage/storageController.d.ts +30 -27
  84. package/storage/remoteStorage/storageController.ts +297 -159
  85. package/storage/remoteStorage/storageServer.d.ts +6 -0
  86. package/storage/remoteStorage/storageServer.ts +88 -94
  87. package/storage/remoteStorage/storageServerCli.d.ts +1 -0
  88. package/storage/remoteStorage/storageServerCli.ts +38 -0
  89. package/storage/remoteStorage/storageServerState.d.ts +43 -0
  90. package/storage/remoteStorage/storageServerState.ts +540 -0
  91. package/testsite/server.ts +0 -2
  92. package/yarn.lock +11 -6
@@ -1,22 +1,15 @@
1
1
  /// <reference types="node" />
2
2
  /// <reference types="node" />
3
- import { IArchives, ArchiveFileInfo } from "../IArchives";
3
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
4
4
  export type ArchivesRemoteConfig = {
5
- address: string;
6
- port: number;
7
- account: string;
8
- bucketName: string;
9
- public?: boolean;
10
- fast?: boolean;
11
- writeDelay?: number;
5
+ url: string;
6
+ accountName?: string;
7
+ waitForAccess?: boolean;
12
8
  };
13
- export declare function buildPublicFileURL(config: {
9
+ export declare function parseStorageUrl(url: string): {
14
10
  address: string;
15
11
  port: number;
16
- account: string;
17
- bucketName: string;
18
- path: string;
19
- }): string;
12
+ };
20
13
  export declare function authenticateStorage(config: {
21
14
  address: string;
22
15
  port: number;
@@ -28,16 +21,24 @@ export declare function authenticateStorage(config: {
28
21
  export declare class ArchivesRemote implements IArchives {
29
22
  private config;
30
23
  constructor(config: ArchivesRemoteConfig);
24
+ private parsed;
25
+ private account;
26
+ private bucketName;
31
27
  private nodeId;
32
28
  private controller;
33
- private setupDone;
34
29
  private lastDeniedLog;
35
30
  getDebugName(): string;
31
+ isConnected(): boolean;
32
+ ping(): Promise<void>;
36
33
  private authenticate;
37
34
  private callAuthed;
38
- waitingForAccess(): Promise<string | undefined>;
39
- private onAccessDenied;
40
- private ensureSetup;
35
+ waitingForAccess(): Promise<{
36
+ link: string;
37
+ machineId: string;
38
+ ip: string;
39
+ } | undefined>;
40
+ hasWriteAccess(): Promise<boolean>;
41
+ private registerAccessRequest;
41
42
  private call;
42
43
  get(fileName: string, config?: {
43
44
  range?: {
@@ -45,7 +46,19 @@ export declare class ArchivesRemote implements IArchives {
45
46
  end: number;
46
47
  };
47
48
  }): Promise<Buffer | undefined>;
48
- set(fileName: string, data: Buffer): Promise<void>;
49
+ get2(fileName: string, config?: {
50
+ range?: {
51
+ start: number;
52
+ end: number;
53
+ };
54
+ }): Promise<{
55
+ data: Buffer;
56
+ writeTime: number;
57
+ size: number;
58
+ } | undefined>;
59
+ set(fileName: string, data: Buffer, config?: {
60
+ lastModified?: number;
61
+ }): Promise<void>;
49
62
  del(fileName: string): Promise<void>;
50
63
  getInfo(fileName: string): Promise<{
51
64
  writeTime: number;
@@ -59,6 +72,9 @@ export declare class ArchivesRemote implements IArchives {
59
72
  shallow?: boolean;
60
73
  type: "files" | "folders";
61
74
  }): Promise<string[]>;
75
+ getChangesAfter(time: number): Promise<ArchiveFileInfo[]>;
76
+ getConfig(): Promise<ArchivesConfig>;
77
+ getSyncStatus(): Promise<ArchivesSyncStatus>;
62
78
  setLargeFile(config: {
63
79
  path: string;
64
80
  getNextData(): Promise<Buffer | undefined>;
@@ -4,37 +4,38 @@ import { SocketFunction } from "socket-function/SocketFunction";
4
4
  import { timeInMinute } from "socket-function/src/misc";
5
5
  import { delay } from "socket-function/src/batching";
6
6
  import { getIdentityCA, loadIdentityCA, sign } from "../../misc/https/certs";
7
- import { IArchives, ArchiveFileInfo } from "../IArchives";
7
+ import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
8
+ import { parseHostedUrl, getBucketBaseUrl, buildFileUrl } from "./remoteConfig";
8
9
  import {
9
- RemoteStorageController, REMOTE_STORAGE_CLASS_GUID, STORAGE_AUTH_PURPOSE,
10
+ RemoteStorageController, STORAGE_AUTH_PURPOSE,
10
11
  STORAGE_NOT_AUTHENTICATED, STORAGE_ACCESS_DENIED,
11
12
  } from "./storageController";
12
13
 
13
14
  // A bucket on our remote storage server (storageServer.ts), used like ArchivesBackblaze. Works in
14
15
  // Node.js and the browser. Authenticates with this machine's certs.ts identity; if the account
15
- // hasn't trusted this machine yet it requests access, waits, and logs instructions every minute
16
- // (calls block until access is granted).
16
+ // hasn't trusted this machine yet it requests access, and by default waits, logging instructions
17
+ // every minute (calls block until access is granted).
17
18
 
18
19
  const ACCESS_RETRY_DELAY = 1000 * 30;
19
20
  const LARGE_FILE_PART_SIZE = 8 * 1024 * 1024;
20
21
 
21
22
  export type ArchivesRemoteConfig = {
22
- address: string;
23
- port: number;
24
- account: string;
25
- bucketName: string;
26
- // Public buckets are served over plain HTTPS GETs (getURL). Private buckets are API-access only.
27
- public?: boolean;
28
- // Fast mode: the server acknowledges writes once they are in memory, flushing to disk after
29
- // writeDelay (default 5 minutes) and coalescing writes to the same file. A server crash loses
30
- // writes that haven't flushed yet.
31
- fast?: boolean;
32
- writeDelay?: number;
23
+ // The bucket's routing URL, which addresses the server, account, and bucket in one:
24
+ // https://storage.example.com:4444/file/<account>/<bucketName>/storage/storagerouting.json
25
+ url: string;
26
+ // Access is checked against this account (defaults to the account in the url)
27
+ accountName?: string;
28
+ // false: access-denied calls throw immediately (the error includes the access page link)
29
+ // instead of requesting access and blocking until it is granted (the default).
30
+ waitForAccess?: boolean;
33
31
  };
34
32
 
35
- export function buildPublicFileURL(config: { address: string; port: number; account: string; bucketName: string; path: string }): string {
36
- let args = encodeURIComponent(JSON.stringify([config.account, config.bucketName, config.path]));
37
- return `https://${config.address}:${config.port}/?classGuid=${REMOTE_STORAGE_CLASS_GUID}&functionName=getPublicFile&args=${args}`;
33
+ export function parseStorageUrl(url: string): { address: string; port: number } {
34
+ let u = new URL(url);
35
+ if (u.protocol !== "https:") {
36
+ throw new Error(`Storage URL must use https, got ${JSON.stringify(u.protocol)} in ${JSON.stringify(url)}`);
37
+ }
38
+ return { address: u.hostname, port: +u.port || 443 };
38
39
  }
39
40
 
40
41
  // Authenticates a connection to a storage server with this machine's certs.ts identity
@@ -61,17 +62,27 @@ export class ArchivesRemote implements IArchives {
61
62
  SocketFunction.ENABLE_CLIENT_MODE = true;
62
63
  }
63
64
 
64
- private nodeId = SocketFunction.connect({ address: this.config.address, port: this.config.port });
65
+ private parsed = parseHostedUrl(this.config.url);
66
+ private account = this.config.accountName || this.parsed.account;
67
+ private bucketName = this.parsed.bucketName;
68
+ private nodeId = SocketFunction.connect({ address: this.parsed.address, port: this.parsed.port });
65
69
  private controller = RemoteStorageController.nodes[this.nodeId];
66
- private setupDone = false;
67
70
  private lastDeniedLog = 0;
68
71
 
69
72
  public getDebugName() {
70
- return `remoteStorage/${this.config.address}:${this.config.port}/${this.config.account}/${this.config.bucketName}`;
73
+ return `remoteStorage/${this.parsed.address}:${this.parsed.port}/${this.account}/${this.bucketName}`;
74
+ }
75
+
76
+ public isConnected(): boolean {
77
+ return SocketFunction.isNodeConnected(this.nodeId);
78
+ }
79
+
80
+ public async ping(): Promise<void> {
81
+ await this.controller.ping();
71
82
  }
72
83
 
73
84
  private async authenticate(): Promise<void> {
74
- await authenticateStorage({ address: this.config.address, port: this.config.port, nodeId: this.nodeId });
85
+ await authenticateStorage({ address: this.parsed.address, port: this.parsed.port, nodeId: this.nodeId });
75
86
  }
76
87
 
77
88
  // Runs a call, authenticating (and re-authenticating after reconnects) as needed. Unlike
@@ -87,50 +98,54 @@ export class ArchivesRemote implements IArchives {
87
98
  }
88
99
 
89
100
  // Returns undefined if this machine has access to the account. Otherwise puts in an access
90
- // request and returns the link to the page where access can be granted.
91
- public async waitingForAccess(): Promise<string | undefined> {
92
- let state = await this.callAuthed(() => this.controller.getAccessState(this.config.account));
101
+ // request and returns our machineId + ip (so the caller can display them alongside the link,
102
+ // for the approver to match the incoming request) and the link to the grant page.
103
+ public async waitingForAccess(): Promise<{ link: string; machineId: string; ip: string } | undefined> {
104
+ let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
93
105
  if (state.hasAccess) return undefined;
94
- await this.callAuthed(() => this.controller.requestAccess(this.config.account));
95
- return `https://${this.config.address}:${this.config.port}/${this.config.account}`;
106
+ let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
107
+ return {
108
+ link: `https://${this.parsed.address}:${this.parsed.port}/${this.account}`,
109
+ machineId: requested.machineId,
110
+ ip: requested.ip,
111
+ };
112
+ }
113
+
114
+ public async hasWriteAccess(): Promise<boolean> {
115
+ let state = await this.callAuthed(() => this.controller.getAccessState(this.account));
116
+ return !!state.hasAccess;
96
117
  }
97
118
 
98
- private async onAccessDenied(): Promise<void> {
99
- let requested = await this.callAuthed(() => this.controller.requestAccess(this.config.account));
119
+ // Registers our access request server-side (so an admin has a requestId to grant) and logs the
120
+ // grant instructions, at most once a minute
121
+ private async registerAccessRequest(): Promise<void> {
122
+ let requested = await this.callAuthed(() => this.controller.requestAccess(this.account));
100
123
  if (Date.now() - this.lastDeniedLog > timeInMinute) {
101
124
  this.lastDeniedLog = Date.now();
102
- console.log(`No access to storage account ${JSON.stringify(this.config.account)} on ${this.config.address}:${this.config.port} (our machine ${requested.machineId}, ip ${requested.ip}). Waiting for access to be granted. See https://${this.config.address}:${this.config.port}/${this.config.account} - or grant it with: ${requested.grantAccessCommand}`);
125
+ console.log(`No access to storage account ${JSON.stringify(this.account)} on ${this.parsed.address}:${this.parsed.port} (our machine ${requested.machineId}, ip ${requested.ip}). See https://${this.parsed.address}:${this.parsed.port}/${this.account} - or grant it with: ${requested.grantAccessCommand}`);
103
126
  }
104
- await delay(ACCESS_RETRY_DELAY);
105
- }
106
-
107
- private async ensureSetup(): Promise<void> {
108
- if (this.setupDone) return;
109
- await this.controller.ensureBucket(this.config.account, this.config.bucketName, {
110
- public: this.config.public,
111
- fast: this.config.fast,
112
- writeDelay: this.config.writeDelay,
113
- });
114
- this.setupDone = true;
115
127
  }
116
128
 
117
129
  // Runs a call, authenticating (and re-authenticating after reconnects) and waiting for account
118
- // access as needed.
130
+ // access as needed. With waitForAccess false, denied calls throw immediately instead - but the
131
+ // access request is still registered (in the background), so the denial is grantable.
119
132
  private async call<T>(fnc: () => Promise<T>): Promise<T> {
120
133
  while (true) {
121
134
  try {
122
- await this.ensureSetup();
123
135
  return await fnc();
124
136
  } catch (e: any) {
125
137
  let message = String(e.stack || e);
126
138
  if (message.includes(STORAGE_NOT_AUTHENTICATED)) {
127
- this.setupDone = false;
128
139
  await this.authenticate();
129
140
  continue;
130
141
  }
131
142
  if (message.includes(STORAGE_ACCESS_DENIED)) {
132
- this.setupDone = false;
133
- await this.onAccessDenied();
143
+ if (this.config.waitForAccess === false) {
144
+ void this.registerAccessRequest().catch(() => { });
145
+ throw e;
146
+ }
147
+ await this.registerAccessRequest();
148
+ await delay(ACCESS_RETRY_DELAY);
134
149
  continue;
135
150
  }
136
151
  throw e;
@@ -139,30 +154,43 @@ export class ArchivesRemote implements IArchives {
139
154
  }
140
155
 
141
156
  public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
142
- let result = await this.call(() => this.controller.get(this.config.account, this.config.bucketName, fileName, config?.range));
143
- return result && Buffer.from(result) || undefined;
157
+ let result = await this.get2(fileName, config);
158
+ return result && result.data || undefined;
144
159
  }
145
- public async set(fileName: string, data: Buffer): Promise<void> {
146
- await this.call(() => this.controller.set(this.config.account, this.config.bucketName, fileName, data));
160
+ public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
161
+ let result = await this.call(() => this.controller.get2(this.account, this.bucketName, fileName, config?.range));
162
+ return result && { data: Buffer.from(result.data), writeTime: result.writeTime, size: result.size } || undefined;
163
+ }
164
+ public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
165
+ await this.call(() => this.controller.set(this.account, this.bucketName, fileName, data, config?.lastModified));
147
166
  }
148
167
  public async del(fileName: string): Promise<void> {
149
- await this.call(() => this.controller.del(this.config.account, this.config.bucketName, fileName));
168
+ await this.call(() => this.controller.del(this.account, this.bucketName, fileName));
150
169
  }
151
170
  public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
152
- return await this.call(() => this.controller.getInfo(this.config.account, this.config.bucketName, fileName));
171
+ return await this.call(() => this.controller.getInfo(this.account, this.bucketName, fileName));
153
172
  }
154
173
  public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
155
- return await this.call(() => this.controller.findInfo(this.config.account, this.config.bucketName, prefix, config));
174
+ return await this.call(() => this.controller.findInfo(this.account, this.bucketName, prefix, config));
156
175
  }
157
176
  public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
158
177
  return (await this.findInfo(prefix, config)).map(x => x.path);
159
178
  }
179
+ public async getChangesAfter(time: number): Promise<ArchiveFileInfo[]> {
180
+ return await this.call(() => this.controller.getChangesAfter(this.account, this.bucketName, time));
181
+ }
182
+ public async getConfig(): Promise<ArchivesConfig> {
183
+ return await this.call(() => this.controller.getArchivesConfig(this.account, this.bucketName));
184
+ }
185
+ public async getSyncStatus(): Promise<ArchivesSyncStatus> {
186
+ return await this.call(() => this.controller.getSyncStatus(this.account, this.bucketName));
187
+ }
160
188
 
161
189
  public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
162
190
  // Ensure we're authenticated with access BEFORE consuming any data (the stream cannot be
163
191
  // rewound, so we can't use the retry loop around the actual upload)
164
- await this.call(() => this.controller.getInfo(this.config.account, this.config.bucketName, config.path));
165
- let uploadId = await this.controller.startLargeFile(this.config.account, this.config.bucketName, config.path);
192
+ await this.call(() => this.controller.getInfo(this.account, this.bucketName, config.path));
193
+ let uploadId = await this.controller.startLargeFile(this.account, this.bucketName, config.path);
166
194
  try {
167
195
  while (true) {
168
196
  let data = await config.getNextData();
@@ -181,15 +209,7 @@ export class ArchivesRemote implements IArchives {
181
209
  }
182
210
 
183
211
  public async getURL(path: string): Promise<string> {
184
- if (!this.config.public) {
185
- throw new Error(`getURL only works on public buckets (private buckets are API-access only). Bucket: ${this.getDebugName()}`);
186
- }
187
- return buildPublicFileURL({
188
- address: this.config.address,
189
- port: this.config.port,
190
- account: this.config.account,
191
- bucketName: this.config.bucketName,
192
- path,
193
- });
212
+ // Only actually loads for public buckets (the server rejects plain URL reads otherwise)
213
+ return buildFileUrl(getBucketBaseUrl(this.config.url), path);
194
214
  }
195
215
  }
@@ -0,0 +1,48 @@
1
+ /// <reference types="node" />
2
+ /// <reference types="node" />
3
+ import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
4
+ export declare class ArchivesUrl implements IArchives {
5
+ private base;
6
+ constructor(base: string);
7
+ getDebugName(): string;
8
+ private readOnlyError;
9
+ get(fileName: string, config?: {
10
+ range?: {
11
+ start: number;
12
+ end: number;
13
+ };
14
+ }): Promise<Buffer | undefined>;
15
+ get2(fileName: string, config?: {
16
+ range?: {
17
+ start: number;
18
+ end: number;
19
+ };
20
+ }): Promise<{
21
+ data: Buffer;
22
+ writeTime: number;
23
+ size: number;
24
+ } | undefined>;
25
+ getInfo(fileName: string): Promise<{
26
+ writeTime: number;
27
+ size: number;
28
+ } | undefined>;
29
+ set(fileName: string, data: Buffer, config?: {
30
+ lastModified?: number;
31
+ }): Promise<void>;
32
+ del(fileName: string): Promise<void>;
33
+ setLargeFile(config: {
34
+ path: string;
35
+ getNextData(): Promise<Buffer | undefined>;
36
+ }): Promise<void>;
37
+ find(prefix: string, config?: {
38
+ shallow?: boolean;
39
+ type: "files" | "folders";
40
+ }): Promise<string[]>;
41
+ findInfo(prefix: string, config?: {
42
+ shallow?: boolean;
43
+ type: "files" | "folders";
44
+ }): Promise<ArchiveFileInfo[]>;
45
+ getURL(path: string): Promise<string>;
46
+ getConfig(): Promise<ArchivesConfig>;
47
+ hasWriteAccess(): Promise<boolean>;
48
+ }
@@ -0,0 +1,86 @@
1
+ module.allowclient = true;
2
+
3
+ import { IArchives, ArchiveFileInfo, ArchivesConfig } from "../IArchives";
4
+ import { buildFileUrl } from "./remoteConfig";
5
+
6
+ // Read-only IArchives over a public bucket's plain-URL form (our storage server's
7
+ // /file/<account>/<bucketName>/... route, or a backblaze friendly URL). Used when we have no API
8
+ // access to a source. Only single-file reads work: there is no listing, and writes always throw.
9
+
10
+ export class ArchivesUrl implements IArchives {
11
+ // base is the bucket's public base URL, e.g. https://host:port/file/<account>/<bucketName>
12
+ constructor(private base: string) { }
13
+
14
+ public getDebugName() {
15
+ return `url/${this.base}`;
16
+ }
17
+
18
+ private readOnlyError(operation: string): Error {
19
+ return new Error(`${operation} is not supported over URL-form access (no API access to this source, only public URL reads). Source: ${this.base}`);
20
+ }
21
+
22
+ public async get(fileName: string, config?: { range?: { start: number; end: number } }): Promise<Buffer | undefined> {
23
+ let result = await this.get2(fileName, config);
24
+ return result && result.data || undefined;
25
+ }
26
+ public async get2(fileName: string, config?: { range?: { start: number; end: number } }): Promise<{ data: Buffer; writeTime: number; size: number } | undefined> {
27
+ let url = buildFileUrl(this.base, fileName);
28
+ let headers: Record<string, string> = {};
29
+ let range = config?.range;
30
+ if (range) {
31
+ headers["Range"] = `bytes=${range.start}-${range.end - 1}`;
32
+ }
33
+ let response = await fetch(url, { headers });
34
+ if (response.status === 404) return undefined;
35
+ if (!response.ok) {
36
+ throw new Error(`Read of ${url} failed: ${response.status} ${response.statusText}`);
37
+ }
38
+ let data = Buffer.from(await response.arrayBuffer());
39
+ // A real 206 only returns the requested slice, so the full size lives in Content-Range's total.
40
+ let size = data.length;
41
+ let contentRange = response.headers.get("content-range");
42
+ let total = contentRange && Number(contentRange.split("/")[1]);
43
+ if (total && Number.isFinite(total)) {
44
+ size = total;
45
+ }
46
+ // Servers that don't support ranges return the full file with a 200 (ours serves real
47
+ // 206s, but backblaze friendly URLs and proxies may not)
48
+ if (range && response.status === 200) {
49
+ data = data.subarray(Math.min(range.start, data.length), Math.min(range.end, data.length));
50
+ }
51
+ let lastModified = response.headers.get("last-modified");
52
+ let writeTime = lastModified && new Date(lastModified).getTime() || 0;
53
+ return { data, writeTime, size };
54
+ }
55
+ public async getInfo(fileName: string): Promise<{ writeTime: number; size: number } | undefined> {
56
+ let result = await this.get2(fileName);
57
+ return result && { writeTime: result.writeTime, size: result.size } || undefined;
58
+ }
59
+
60
+ public async set(fileName: string, data: Buffer, config?: { lastModified?: number }): Promise<void> {
61
+ throw this.readOnlyError("set");
62
+ }
63
+ public async del(fileName: string): Promise<void> {
64
+ throw this.readOnlyError("del");
65
+ }
66
+ public async setLargeFile(config: { path: string; getNextData(): Promise<Buffer | undefined> }): Promise<void> {
67
+ throw this.readOnlyError("setLargeFile");
68
+ }
69
+ public async find(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<string[]> {
70
+ throw this.readOnlyError("find (listing)");
71
+ }
72
+ public async findInfo(prefix: string, config?: { shallow?: boolean; type: "files" | "folders" }): Promise<ArchiveFileInfo[]> {
73
+ throw this.readOnlyError("findInfo (listing)");
74
+ }
75
+
76
+ public async getURL(path: string): Promise<string> {
77
+ return buildFileUrl(this.base, path);
78
+ }
79
+ public async getConfig(): Promise<ArchivesConfig> {
80
+ return {};
81
+ }
82
+
83
+ public async hasWriteAccess(): Promise<boolean> {
84
+ return false;
85
+ }
86
+ }
@@ -6,12 +6,14 @@ import { observer } from "../../render-utils/observer";
6
6
  import { isNode } from "socket-function/src/misc";
7
7
  import { css } from "typesafecss";
8
8
  import { SocketFunction } from "socket-function/SocketFunction";
9
- import { RemoteStorageController, AccessState } from "./storageController";
9
+ import { RemoteStorageController, AccessState, AccessRequest } from "./storageController";
10
10
  import { authenticateStorage } from "./ArchivesRemote";
11
11
 
12
12
  // The storage server's access page. Visit https://<storageDomain>:<port>/<accountName> to request
13
- // access to that account for this browser's machine identity. Shows the command to run on the
14
- // storage machine to grant it, and once granted, the machines with (or requesting) access.
13
+ // access to that account for this browser's machine identity. Once granted, lists the machines that
14
+ // have access. To approve someone else's request, the user must type in the requester's IP —
15
+ // pending requests are NEVER shown unsolicited (so a trusted user can't accidentally approve a
16
+ // random machine's request).
15
17
 
16
18
  const REFRESH_INTERVAL = 1000 * 15;
17
19
  const COPIED_RESET_DELAY = 2000;
@@ -47,6 +49,11 @@ class AccessPage extends preact.Component {
47
49
  account: "",
48
50
  state: undefined as AccessState | undefined,
49
51
  error: "",
52
+ lookupIp: "",
53
+ lookupResults: undefined as { ip: string; requests: AccessRequest[] } | undefined,
54
+ lookupError: "",
55
+ looking: false,
56
+ granting: "",
50
57
  });
51
58
 
52
59
  componentDidMount() {
@@ -67,22 +74,58 @@ class AccessPage extends preact.Component {
67
74
  }
68
75
 
69
76
  private authenticated = false;
70
- private async refresh(account: string) {
77
+ private async controller() {
71
78
  let address = location.hostname;
72
79
  let port = +location.port || 443;
73
80
  let nodeId = SocketFunction.connect({ address, port });
74
- let controller = RemoteStorageController.nodes[nodeId];
75
81
  if (!this.authenticated) {
76
82
  await authenticateStorage({ address, port, nodeId });
77
83
  this.authenticated = true;
78
84
  }
85
+ return RemoteStorageController.nodes[nodeId];
86
+ }
87
+ private async refresh(account: string) {
88
+ let controller = await this.controller();
79
89
  await controller.requestAccess(account);
80
90
  this.synced.state = await controller.getAccessState(account);
81
91
  this.synced.error = "";
82
92
  }
83
93
 
94
+ private lookupIP = async () => {
95
+ let ip = this.synced.lookupIp.trim();
96
+ if (!ip) return;
97
+ this.synced.looking = true;
98
+ this.synced.lookupError = "";
99
+ try {
100
+ let controller = await this.controller();
101
+ let requests = await controller.listRequestsForIP(this.synced.account, ip);
102
+ this.synced.lookupResults = { ip, requests };
103
+ } catch (e) {
104
+ this.synced.lookupError = String(e);
105
+ } finally {
106
+ this.synced.looking = false;
107
+ }
108
+ };
109
+
110
+ private approve = async (request: AccessRequest) => {
111
+ this.synced.granting = request.requestId;
112
+ try {
113
+ let controller = await this.controller();
114
+ await controller.grantAccess(request.requestId);
115
+ if (this.synced.lookupResults) {
116
+ this.synced.lookupResults.requests = this.synced.lookupResults.requests.filter(r => r.requestId !== request.requestId);
117
+ }
118
+ await this.refresh(this.synced.account);
119
+ } catch (e) {
120
+ this.synced.lookupError = String(e);
121
+ } finally {
122
+ this.synced.granting = "";
123
+ }
124
+ };
125
+
84
126
  render() {
85
- let { account, state, error } = this.synced;
127
+ let synced = this.synced;
128
+ let { account, state, error } = synced;
86
129
  if (!account) {
87
130
  return <div className={css.vbox(8).pad2(16)}>
88
131
  <div>Remote storage server.</div>
@@ -95,31 +138,71 @@ class AccessPage extends preact.Component {
95
138
  {!state && !error && <div>Requesting access...</div>}
96
139
  {state && !state.hasAccess && <div className={css.vbox(8)}>
97
140
  <div>This machine ({state.machineId}, ip {state.ip}) does NOT have access yet.</div>
98
- <div>An access request has been made. To grant it, run this single command (it sshes into the storage machine):</div>
99
- <CopyableCommand command={state.grantAccessCommand || state.listAccessCommand} />
141
+ <div>An access request has been made. To grant it, run this command:</div>
142
+ {state.grantAccessCommand && <CopyableCommand command={state.grantAccessCommand} />}
100
143
  <div>This page rechecks every {REFRESH_INTERVAL / 1000} seconds.</div>
101
144
  </div>}
102
- {state && state.hasAccess && <div className={css.vbox(8)}>
145
+ {state && state.hasAccess && <div className={css.vbox(16)}>
103
146
  <div>This machine ({state.machineId}, ip {state.ip}) has access.</div>
104
- <div>Machines with or requesting access:</div>
105
- <table>
106
- <thead>
107
- <tr>
108
- <th>Machine</th>
109
- <th>IP</th>
110
- <th>Time</th>
111
- <th>Status</th>
112
- </tr>
113
- </thead>
114
- <tbody>
115
- {(state.machines || []).map(machine => <tr key={machine.machineId}>
116
- <td className={css.pad2(8, 2)}>{machine.machineId}</td>
117
- <td className={css.pad2(8, 2)}>{machine.ip}</td>
118
- <td className={css.pad2(8, 2)}>{new Date(machine.time).toLocaleString()}</td>
119
- <td className={css.pad2(8, 2)}>{machine.trusted && "has access" || "requesting access"}</td>
120
- </tr>)}
121
- </tbody>
122
- </table>
147
+ <div className={css.vbox(6)}>
148
+ <div>Machines with access:</div>
149
+ <table>
150
+ <thead>
151
+ <tr>
152
+ <th className={css.pad2(8, 2).textAlign("left")}>Machine</th>
153
+ <th className={css.pad2(8, 2).textAlign("left")}>IP</th>
154
+ <th className={css.pad2(8, 2).textAlign("left")}>Granted</th>
155
+ </tr>
156
+ </thead>
157
+ <tbody>
158
+ {(state.trustedMachines || []).map(m => <tr key={m.machineId}>
159
+ <td className={css.pad2(8, 2)}>{m.machineId}</td>
160
+ <td className={css.pad2(8, 2)}>{m.ip}</td>
161
+ <td className={css.pad2(8, 2)}>{new Date(m.time).toLocaleString()}</td>
162
+ </tr>)}
163
+ </tbody>
164
+ </table>
165
+ </div>
166
+ <div className={css.vbox(6)}>
167
+ <div>Approve request</div>
168
+ <div className={css.hbox(8).alignItems("center")}>
169
+ <input
170
+ placeholder="ip"
171
+ value={synced.lookupIp}
172
+ onInput={e => { synced.lookupIp = (e.currentTarget as HTMLInputElement).value; }}
173
+ onKeyDown={e => { if (e.key === "Enter") void this.lookupIP(); }}
174
+ />
175
+ <button disabled={synced.looking || !synced.lookupIp.trim()} onClick={this.lookupIP}>
176
+ search
177
+ </button>
178
+ </div>
179
+ {synced.lookupError && <div>Error: {synced.lookupError}</div>}
180
+ {synced.lookupResults && synced.lookupResults.requests.length === 0 && <div>
181
+ No pending requests for {synced.lookupResults.ip} on this account.
182
+ </div>}
183
+ {synced.lookupResults && synced.lookupResults.requests.length > 0 && <table>
184
+ <thead>
185
+ <tr>
186
+ <th className={css.pad2(8, 2).textAlign("left")}>Machine</th>
187
+ <th className={css.pad2(8, 2).textAlign("left")}>IP</th>
188
+ <th className={css.pad2(8, 2).textAlign("left")}>Requested</th>
189
+ <th className={css.pad2(8, 2).textAlign("left")}></th>
190
+ </tr>
191
+ </thead>
192
+ <tbody>
193
+ {synced.lookupResults.requests.map(r => <tr key={r.requestId}>
194
+ <td className={css.pad2(8, 2)}>{r.machineId}</td>
195
+ <td className={css.pad2(8, 2)}>{r.ip}</td>
196
+ <td className={css.pad2(8, 2)}>{new Date(r.time).toLocaleString()}</td>
197
+ <td className={css.pad2(8, 2)}>
198
+ <button disabled={!!synced.granting} onClick={async () => this.approve(r)}>
199
+ {synced.granting === r.requestId && "Approving..." || "Approve"}
200
+ </button>
201
+ </td>
202
+ </tr>)}
203
+ </tbody>
204
+ </table>}
205
+ </div>
123
206
  </div>}
124
207
  </div>;
125
208
  }