sliftutils 1.7.87 → 1.7.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +3 -1
- package/dist/treeSummary.ts.cache +2 -2
- package/index.d.ts +675 -296
- package/misc/dist/environment.ts.cache +2 -2
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/dist/zip.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/cloudflareHelpers.ts.cache +2 -2
- package/misc/https/dist/dns.ts.cache +2 -2
- package/misc/https/dist/hostServer.ts.cache +49 -21
- package/misc/https/dist/hostsFile.ts.cache +82 -0
- package/misc/https/dist/httpsCerts.ts.cache +2 -2
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/misc/https/hostServer.d.ts +6 -2
- package/misc/https/hostServer.ts +52 -19
- package/misc/https/hostsFile.d.ts +7 -0
- package/misc/https/hostsFile.ts +66 -0
- package/package.json +1 -1
- package/render-utils/dist/observer.tsx.cache +2 -2
- package/storage/ArchivesDisk.d.ts +3 -6
- package/storage/ArchivesDisk.ts +26 -2
- package/storage/BulkDatabase2/BulkDatabaseReader.ts +1 -1
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +3 -3
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +2 -2
- package/storage/IArchives.d.ts +37 -24
- package/storage/IArchives.ts +47 -62
- package/storage/LogMap.d.ts +56 -0
- package/storage/LogMap.ts +220 -0
- package/storage/archiveHelpers.d.ts +36 -0
- package/storage/archiveHelpers.ts +103 -0
- package/storage/backblaze.d.ts +7 -12
- package/storage/backblaze.ts +53 -48
- package/storage/dist/ArchivesDisk.ts.cache +28 -2
- package/storage/dist/FileFolderAPI.tsx.cache +2 -2
- package/storage/dist/IArchives.ts.cache +16 -52
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
- package/storage/dist/JSONStorage.ts.cache +2 -2
- package/storage/dist/LogMap.ts.cache +212 -0
- package/storage/dist/PendingManager.tsx.cache +2 -2
- package/storage/dist/TransactionStorage.ts.cache +2 -2
- package/storage/dist/backblaze.ts.cache +58 -44
- package/storage/dist/fileSystemPointer.ts.cache +2 -2
- package/storage/dist/remoteFileStorage.ts.cache +2 -2
- package/storage/remoteStorage/ArchivesDelayed.d.ts +66 -0
- package/storage/remoteStorage/ArchivesDelayed.ts +237 -0
- package/storage/remoteStorage/ArchivesRemote.d.ts +5 -6
- package/storage/remoteStorage/ArchivesRemote.ts +26 -5
- package/storage/remoteStorage/ArchivesUrl.d.ts +2 -5
- package/storage/remoteStorage/ArchivesUrl.ts +2 -2
- package/storage/remoteStorage/accessPage.tsx +12 -0
- package/storage/remoteStorage/accessStats.d.ts +1 -1
- package/storage/remoteStorage/accessStats.ts +3 -3
- package/storage/remoteStorage/blobStore.d.ts +162 -189
- package/storage/remoteStorage/blobStore.ts +495 -952
- package/storage/remoteStorage/bucketDisk.d.ts +24 -3
- package/storage/remoteStorage/bucketDisk.ts +56 -22
- package/storage/remoteStorage/certTrustModal.d.ts +1 -0
- package/storage/remoteStorage/certTrustModal.tsx +48 -0
- package/storage/remoteStorage/cliArgs.d.ts +2 -0
- package/storage/remoteStorage/cliArgs.ts +5 -0
- package/storage/remoteStorage/createArchives.d.ts +8 -7
- package/storage/remoteStorage/createArchives.ts +65 -26
- package/storage/remoteStorage/deployTakeover.ts +1 -1
- package/storage/remoteStorage/dist/ArchivesDelayed.ts.cache +220 -0
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +25 -5
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +3 -3
- package/storage/remoteStorage/dist/accessPage.tsx.cache +13 -3
- package/storage/remoteStorage/dist/accessStats.ts.cache +6 -6
- package/storage/remoteStorage/dist/blobStore.ts.cache +483 -965
- package/storage/remoteStorage/dist/bucketDisk.ts.cache +60 -29
- package/storage/remoteStorage/dist/certTrustModal.tsx.cache +55 -0
- package/storage/remoteStorage/dist/cliArgs.ts.cache +9 -4
- package/storage/remoteStorage/dist/createArchives.ts.cache +63 -26
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +5 -5
- package/storage/remoteStorage/dist/intermediateManagement.ts.cache +286 -0
- package/storage/remoteStorage/dist/intermediateSources.ts.cache +2 -2
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +122 -56
- package/storage/remoteStorage/dist/serverConfig.ts.cache +4 -4
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +7 -11
- package/storage/remoteStorage/dist/sourcesList.ts.cache +3 -6
- package/storage/remoteStorage/dist/storageClientController.ts.cache +2 -2
- package/storage/remoteStorage/dist/storageController.ts.cache +27 -12
- package/storage/remoteStorage/dist/storageServer.ts.cache +27 -7
- package/storage/remoteStorage/dist/storageServerCli.ts.cache +7 -3
- package/storage/remoteStorage/dist/storageServerState.ts.cache +133 -696
- package/storage/remoteStorage/dist/storeConfig.ts.cache +52 -0
- package/storage/remoteStorage/dist/storePlan.ts.cache +110 -81
- package/storage/remoteStorage/dist/storeSources.ts.cache +51 -0
- package/storage/remoteStorage/dist/storeSync.ts.cache +650 -0
- package/storage/remoteStorage/dist/validation.ts.cache +14 -4
- package/storage/remoteStorage/intermediateManagement.d.ts +12 -0
- package/storage/remoteStorage/intermediateManagement.ts +264 -0
- package/storage/remoteStorage/remoteConfig.d.ts +32 -0
- package/storage/remoteStorage/remoteConfig.ts +115 -50
- package/storage/remoteStorage/serverConfig.d.ts +1 -1
- package/storage/remoteStorage/serverConfig.ts +1 -1
- package/storage/remoteStorage/sourceWrapper.ts +4 -8
- package/storage/remoteStorage/sourcesList.d.ts +0 -2
- package/storage/remoteStorage/sourcesList.ts +0 -3
- package/storage/remoteStorage/spec.md +18 -4
- package/storage/remoteStorage/storageController.d.ts +10 -0
- package/storage/remoteStorage/storageController.ts +29 -16
- package/storage/remoteStorage/storageServer.d.ts +2 -0
- package/storage/remoteStorage/storageServer.ts +28 -4
- package/storage/remoteStorage/storageServerCli.ts +5 -1
- package/storage/remoteStorage/storageServerState.d.ts +34 -21
- package/storage/remoteStorage/storageServerState.ts +147 -662
- package/storage/remoteStorage/storeConfig.d.ts +30 -0
- package/storage/remoteStorage/storeConfig.ts +62 -0
- package/storage/remoteStorage/storePlan.d.ts +29 -17
- package/storage/remoteStorage/storePlan.ts +112 -85
- package/storage/remoteStorage/storeSources.d.ts +11 -0
- package/storage/remoteStorage/storeSources.ts +47 -0
- package/storage/remoteStorage/storeSync.d.ts +53 -0
- package/storage/remoteStorage/storeSync.ts +632 -0
- package/storage/remoteStorage/validation.d.ts +2 -0
- package/storage/remoteStorage/validation.ts +10 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
|
|
2
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, FindConfig, GetConfig, GetInfoConfig } from "../IArchives";
|
|
2
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, FindConfig, GetConfig, GetInfoConfig, SetLargeFileConfig } from "../IArchives";
|
|
3
3
|
import { buildFileUrl } from "./remoteConfig";
|
|
4
4
|
|
|
5
5
|
// Read-only IArchives over a public bucket's plain-URL form (our storage server's
|
|
@@ -74,7 +74,7 @@ export class ArchivesUrl implements IArchives {
|
|
|
74
74
|
public async del(fileName: string): Promise<void> {
|
|
75
75
|
throw this.readOnlyError("del");
|
|
76
76
|
}
|
|
77
|
-
public async setLargeFile(config:
|
|
77
|
+
public async setLargeFile(config: SetLargeFileConfig): Promise<void> {
|
|
78
78
|
throw this.readOnlyError("setLargeFile");
|
|
79
79
|
}
|
|
80
80
|
public async find(prefix: string, config?: FindConfig): Promise<string[]> {
|
|
@@ -204,8 +204,20 @@ class AccessPage extends preact.Component {
|
|
|
204
204
|
}
|
|
205
205
|
}
|
|
206
206
|
|
|
207
|
+
// Reached via the cert-trust flow (showCertTrustModal links here as ?trustCert=1). The browser only loads this page if it has ACCEPTED the certificate - so the mere fact that it rendered is the confirmation. Once accepted, the app's background connection to this same server works, so the user just returns.
|
|
208
|
+
function CertTrustedPage() {
|
|
209
|
+
return <div className={css.vbox(8).pad2(16)}>
|
|
210
|
+
<div>This server's certificate is now trusted by this browser.</div>
|
|
211
|
+
<div>Close this tab and return to the page you came from - it will connect to the storage server automatically.</div>
|
|
212
|
+
</div>;
|
|
213
|
+
}
|
|
214
|
+
|
|
207
215
|
async function main() {
|
|
208
216
|
if (isNode()) return;
|
|
217
|
+
if (new URLSearchParams(location.search).has("trustCert")) {
|
|
218
|
+
preact.render(<CertTrustedPage />, document.body);
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
209
221
|
preact.render(<AccessPage />, document.body);
|
|
210
222
|
}
|
|
211
223
|
|
|
@@ -15,7 +15,7 @@ export declare function trackAccess(config: {
|
|
|
15
15
|
path: string;
|
|
16
16
|
size?: number;
|
|
17
17
|
}): void;
|
|
18
|
-
/** Method decorator factory, for API methods whose single config-object argument has account and bucketName: tracks the access (as `bucketName/path`) after the method succeeds. Sizes come from the config's data (writes) or the result's data (reads); operations without either are count-only. Array results (listings - findInfo, getChangesAfter) are tracked as two
|
|
18
|
+
/** Method decorator factory, for API methods whose single config-object argument has account and bucketName: tracks the access (as `bucketName/path`) after the method succeeds. Sizes come from the config's data (writes) or the result's data (reads); operations without either are count-only. Array results (listings - findInfo, getChangesAfter) are tracked as two breakdowns: "<op> queries" - one access per CALL, at the query prefix, sized by the number of results (so the tree shows which QUERY returns the most) - and "<op> results" - one count-only access per returned path (so the tree shows which PATHS come back most). */
|
|
19
19
|
export declare function trackAccessCall(operation: string): (target: unknown, key: string, descriptor: PropertyDescriptor) => void;
|
|
20
20
|
export declare function getAccessTotals(account: string): AccessTotals;
|
|
21
21
|
export declare function readAccessSummaries(config: {
|
|
@@ -57,7 +57,7 @@ export function trackAccess(config: { account: string; operation: string; path:
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
/** Method decorator factory, for API methods whose single config-object argument has account and bucketName: tracks the access (as `bucketName/path`) after the method succeeds. Sizes come from the config's data (writes) or the result's data (reads); operations without either are count-only. Array results (listings - findInfo, getChangesAfter) are tracked as two
|
|
60
|
+
/** Method decorator factory, for API methods whose single config-object argument has account and bucketName: tracks the access (as `bucketName/path`) after the method succeeds. Sizes come from the config's data (writes) or the result's data (reads); operations without either are count-only. Array results (listings - findInfo, getChangesAfter) are tracked as two breakdowns: "<op> queries" - one access per CALL, at the query prefix, sized by the number of results (so the tree shows which QUERY returns the most) - and "<op> results" - one count-only access per returned path (so the tree shows which PATHS come back most). */
|
|
61
61
|
export function trackAccessCall(operation: string) {
|
|
62
62
|
return function (target: unknown, key: string, descriptor: PropertyDescriptor): void {
|
|
63
63
|
let original = descriptor.value as (...args: unknown[]) => Promise<unknown>;
|
|
@@ -66,8 +66,8 @@ export function trackAccessCall(operation: string) {
|
|
|
66
66
|
let result = await original.apply(this, args);
|
|
67
67
|
let base = `${config.bucketName}/`;
|
|
68
68
|
if (Array.isArray(result)) {
|
|
69
|
-
//
|
|
70
|
-
trackAccess({ account: config.account, operation: `${operation} queries`, path: base + (config.prefix || config.path || "") });
|
|
69
|
+
// The query, sized by how many results it returned (which query is heaviest); and each returned path, count-only (which paths come back most)
|
|
70
|
+
trackAccess({ account: config.account, operation: `${operation} queries`, path: base + (config.prefix || config.path || ""), size: result.length });
|
|
71
71
|
for (let entry of result as { path: string }[]) {
|
|
72
72
|
trackAccess({ account: config.account, operation: `${operation} results`, path: base + entry.path });
|
|
73
73
|
}
|
|
@@ -1,143 +1,19 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig,
|
|
4
|
-
import {
|
|
5
|
-
|
|
3
|
+
import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig, SourceConfig, SyncActivity } from "../IArchives";
|
|
4
|
+
import { StoreSync } from "./storeSync";
|
|
5
|
+
import { StoreConfig } from "./storeConfig";
|
|
6
6
|
export declare const WINDOW_END_FLUSH_MARGIN: number;
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
includeTombstones?: boolean;
|
|
17
|
-
}): Promise<{
|
|
18
|
-
data: Buffer;
|
|
19
|
-
writeTime: number;
|
|
20
|
-
size: number;
|
|
21
|
-
} | undefined>;
|
|
22
|
-
/** internal (store-to-store) writes go to the local disk + index with no fan-out; see SetConfig.internal */
|
|
23
|
-
set(config: {
|
|
24
|
-
path: string;
|
|
25
|
-
data: Buffer;
|
|
26
|
-
lastModified?: number;
|
|
27
|
-
forceSetImmutable?: boolean;
|
|
28
|
-
internal?: boolean;
|
|
29
|
-
}): Promise<void>;
|
|
30
|
-
del(config: {
|
|
31
|
-
path: string;
|
|
32
|
-
lastModified?: number;
|
|
33
|
-
internal?: boolean;
|
|
34
|
-
}): Promise<void>;
|
|
35
|
-
getInfo(config: {
|
|
36
|
-
path: string;
|
|
37
|
-
includeTombstones?: boolean;
|
|
38
|
-
}): Promise<{
|
|
39
|
-
writeTime: number;
|
|
40
|
-
size: number;
|
|
41
|
-
} | undefined>;
|
|
42
|
-
findInfo(config: FindConfig & {
|
|
43
|
-
prefix: string;
|
|
44
|
-
}): Promise<ArchiveFileInfo[]>;
|
|
45
|
-
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
46
|
-
getSyncStatus?(): Promise<ArchivesSyncStatus>;
|
|
47
|
-
getSyncProgress?(): {
|
|
48
|
-
index: {
|
|
49
|
-
fileCount: number;
|
|
50
|
-
byteCount: number;
|
|
51
|
-
};
|
|
52
|
-
sources: {
|
|
53
|
-
debugName: string;
|
|
54
|
-
fileCount: number;
|
|
55
|
-
byteCount: number;
|
|
56
|
-
}[];
|
|
57
|
-
readerDiskLimit?: number;
|
|
58
|
-
syncing: SyncActivity[];
|
|
59
|
-
};
|
|
60
|
-
computeIndexTotals?(): Promise<{
|
|
61
|
-
fileCount: number;
|
|
62
|
-
byteCount: number;
|
|
63
|
-
sources: {
|
|
64
|
-
debugName: string;
|
|
65
|
-
fileCount: number;
|
|
66
|
-
byteCount: number;
|
|
67
|
-
}[];
|
|
68
|
-
}>;
|
|
69
|
-
/** path/lastModified let the store reject an upload into an immutable bucket before any bytes move */
|
|
70
|
-
startLargeUpload(config?: {
|
|
71
|
-
path?: string;
|
|
72
|
-
lastModified?: number;
|
|
73
|
-
}): Promise<string>;
|
|
74
|
-
appendLargeUpload(config: {
|
|
75
|
-
id: string;
|
|
76
|
-
data: Buffer;
|
|
77
|
-
}): Promise<void>;
|
|
78
|
-
finishLargeUpload(config: {
|
|
79
|
-
id: string;
|
|
80
|
-
path: string;
|
|
81
|
-
lastModified?: number;
|
|
82
|
-
}): Promise<void>;
|
|
83
|
-
cancelLargeUpload(config: {
|
|
84
|
-
id: string;
|
|
85
|
-
}): Promise<void>;
|
|
7
|
+
/** What we store about a file. Its times are not in here: the index keeps those for every key, deleted ones included (see LogMap). */
|
|
8
|
+
type IndexValue = {
|
|
9
|
+
size: number;
|
|
10
|
+
sourcesListIndex: number;
|
|
11
|
+
};
|
|
12
|
+
/** One file we hold, as everything outside the index sees it. */
|
|
13
|
+
export type IndexEntry = IndexValue & {
|
|
14
|
+
writeTime: number;
|
|
15
|
+
changedAt: number;
|
|
86
16
|
};
|
|
87
|
-
/** rawDisk buckets: the disk IS the store. No index, no synchronization, no window/route/immutability validation. */
|
|
88
|
-
export declare class RawDiskStore implements IBucketStore {
|
|
89
|
-
private disk;
|
|
90
|
-
constructor(disk: ArchivesDisk);
|
|
91
|
-
get2(config: {
|
|
92
|
-
path: string;
|
|
93
|
-
range?: {
|
|
94
|
-
start: number;
|
|
95
|
-
end: number;
|
|
96
|
-
};
|
|
97
|
-
internal?: boolean;
|
|
98
|
-
includeTombstones?: boolean;
|
|
99
|
-
}): Promise<{
|
|
100
|
-
data: Buffer;
|
|
101
|
-
writeTime: number;
|
|
102
|
-
size: number;
|
|
103
|
-
} | undefined>;
|
|
104
|
-
set(config: {
|
|
105
|
-
path: string;
|
|
106
|
-
data: Buffer;
|
|
107
|
-
lastModified?: number;
|
|
108
|
-
forceSetImmutable?: boolean;
|
|
109
|
-
internal?: boolean;
|
|
110
|
-
}): Promise<void>;
|
|
111
|
-
del(config: {
|
|
112
|
-
path: string;
|
|
113
|
-
lastModified?: number;
|
|
114
|
-
internal?: boolean;
|
|
115
|
-
}): Promise<void>;
|
|
116
|
-
getInfo(config: {
|
|
117
|
-
path: string;
|
|
118
|
-
includeTombstones?: boolean;
|
|
119
|
-
}): Promise<{
|
|
120
|
-
writeTime: number;
|
|
121
|
-
size: number;
|
|
122
|
-
} | undefined>;
|
|
123
|
-
findInfo(config: FindConfig & {
|
|
124
|
-
prefix: string;
|
|
125
|
-
}): Promise<ArchiveFileInfo[]>;
|
|
126
|
-
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
127
|
-
startLargeUpload(): Promise<string>;
|
|
128
|
-
appendLargeUpload(config: {
|
|
129
|
-
id: string;
|
|
130
|
-
data: Buffer;
|
|
131
|
-
}): Promise<void>;
|
|
132
|
-
finishLargeUpload(config: {
|
|
133
|
-
id: string;
|
|
134
|
-
path: string;
|
|
135
|
-
lastModified?: number;
|
|
136
|
-
}): Promise<void>;
|
|
137
|
-
cancelLargeUpload(config: {
|
|
138
|
-
id: string;
|
|
139
|
-
}): Promise<void>;
|
|
140
|
-
}
|
|
141
17
|
export type BlobSourceSpec = {
|
|
142
18
|
identity: string;
|
|
143
19
|
url: string;
|
|
@@ -147,23 +23,69 @@ export type BlobSourceSpec = {
|
|
|
147
23
|
intermediate?: string;
|
|
148
24
|
sourceConfig?: SourceConfig;
|
|
149
25
|
create: () => IArchives;
|
|
26
|
+
applyConfig?: (source: IArchives) => void;
|
|
150
27
|
};
|
|
151
|
-
export declare class BlobStore
|
|
152
|
-
|
|
153
|
-
|
|
28
|
+
export declare class BlobStore {
|
|
29
|
+
folder: string;
|
|
30
|
+
/** The name this store answers to (see CommonConfig.name) - the entries of the routing config that carry it are the ones that configure it, and the rest are its peers. */
|
|
31
|
+
storeName: string;
|
|
154
32
|
private config?;
|
|
155
|
-
|
|
33
|
+
stopped: {
|
|
34
|
+
stop: boolean;
|
|
35
|
+
};
|
|
36
|
+
syncStarted: boolean;
|
|
37
|
+
/** Its sources, in config order: slot 0 is always its own disk folder, the rest are the peers it synchronizes with. Filled by updateSources, which is also how they change. The store OWNS them - writes pick among them, reads resolve holders through them - and StoreSync only scans whatever is in here at the time. */
|
|
38
|
+
sources: ArchivesSource[];
|
|
39
|
+
private discardedUploads;
|
|
40
|
+
private nextDiscardedUpload;
|
|
41
|
+
private sourcesList;
|
|
42
|
+
private slotSourcesListIndexes;
|
|
43
|
+
private slotRegistrations;
|
|
44
|
+
private index;
|
|
45
|
+
/** Keeping the index in agreement with the sources: scanning, pulling, pushing, and the maintenance that follows from holding an index (disk-limit eviction, tombstone expiry). It reads and writes this store's index and sources - it does not own them. */
|
|
46
|
+
sync: StoreSync;
|
|
47
|
+
constructor(folder: string,
|
|
48
|
+
/** The name this store answers to (see CommonConfig.name) - the entries of the routing config that carry it are the ones that configure it, and the rest are its peers. */
|
|
49
|
+
storeName: string, config?: {
|
|
50
|
+
/** Whether a config entry is THIS SERVER (same account, same bucket, our own address). Injected because a store knows nothing about servers - it only needs to tell its own entries apart from its peers'. Absent means nothing is us, which is what a bare store (no server around it) wants. */
|
|
51
|
+
isSelf?: ((source: SourceConfig) => boolean) | undefined;
|
|
52
|
+
/** Builds one of its sources. Injected for the same reason, and because the delay it is created with is this store's policy. */
|
|
53
|
+
createSource?: ((config: {
|
|
54
|
+
sourceConfig?: SourceConfig;
|
|
55
|
+
writeDelay: number;
|
|
56
|
+
}) => IArchives) | undefined;
|
|
57
|
+
/** Hands a running source a changed config, so an endpoint we already talk to is never rebuilt just because a flag moved. */
|
|
58
|
+
applySource?: ((source: IArchives, sourceConfig: SourceConfig | undefined, writeDelay: number) => void) | undefined;
|
|
156
59
|
onIndexChanged?: ((key: string) => void) | undefined;
|
|
157
|
-
readerDiskLimit?: number | undefined;
|
|
158
60
|
onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
|
|
159
61
|
resolveSourceUrl?: ((url: string) => IArchives) | undefined;
|
|
160
|
-
entries?: HostedConfig[] | undefined;
|
|
161
62
|
} | undefined);
|
|
63
|
+
/** This store's folder, unwrapped: the same bytes slot 0 serves, but reached without its write delay. Used for the two things that cannot go through a buffered source - reading our own routing config before we have any sources, and streaming a large upload that must not sit in memory. */
|
|
64
|
+
private ownDisk;
|
|
65
|
+
/** What this store is configured to be. It owns this: the routing config is a file IN the store, so the store reads it, applies it to itself, and re-applies it whenever the file changes - by our own write, or by a peer's copy arriving through synchronization. */
|
|
66
|
+
storeConfig: StoreConfig;
|
|
67
|
+
private appliedRoutingVersion;
|
|
162
68
|
init: {
|
|
163
69
|
(): Promise<void>;
|
|
164
70
|
reset(): void;
|
|
165
71
|
set(newValue: Promise<void>): void;
|
|
166
72
|
};
|
|
73
|
+
/**
|
|
74
|
+
* Re-reads the routing config out of this store and applies it to itself. Called at startup and
|
|
75
|
+
* whenever that file changes here - which is the ONE mechanism: a config written by an operator
|
|
76
|
+
* and a config pulled off a peer are the same event, a write of that path into this store.
|
|
77
|
+
*
|
|
78
|
+
* A store with no routing config configures itself as its own disk, valid always, for the whole
|
|
79
|
+
* key space. That is a complete, working store - it just has nobody to synchronize with - and it
|
|
80
|
+
* is what lets a store exist before it has ever heard of a configuration.
|
|
81
|
+
*/
|
|
82
|
+
applyRoutingConfig(): Promise<void>;
|
|
83
|
+
private readRoutingConfig;
|
|
84
|
+
/** The version of the routing config this store is running, so a copy found on a peer is only taken when it is genuinely newer. -1 means it has none. */
|
|
85
|
+
routingVersion(): number;
|
|
86
|
+
private routingApplies;
|
|
87
|
+
reapplyRoutingConfig(): void;
|
|
88
|
+
private planSources;
|
|
167
89
|
dispose(): Promise<void>;
|
|
168
90
|
get2(config: {
|
|
169
91
|
path: string;
|
|
@@ -190,6 +112,11 @@ export declare class BlobStore implements IBucketStore {
|
|
|
190
112
|
lastModified?: number;
|
|
191
113
|
internal?: boolean;
|
|
192
114
|
}): Promise<void>;
|
|
115
|
+
/** A node-side move: the bytes never travel through the client. Deliberately just get2 + set + del rather than a disk rename, so the destination write passes EVERY rule a set passes (windows, routes, immutability, only-take-latest, index, fan-out to peers) and the deletion propagates as a normal tombstone - a rename would bypass all of it. The set stamps fresh, so the moved file beats any tombstone at its new path. */
|
|
116
|
+
move(config: {
|
|
117
|
+
fromPath: string;
|
|
118
|
+
toPath: string;
|
|
119
|
+
}): Promise<void>;
|
|
193
120
|
getInfo(config: {
|
|
194
121
|
path: string;
|
|
195
122
|
includeTombstones?: boolean;
|
|
@@ -202,7 +129,7 @@ export declare class BlobStore implements IBucketStore {
|
|
|
202
129
|
}): Promise<ArchiveFileInfo[]>;
|
|
203
130
|
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
204
131
|
getSyncStatus(): Promise<ArchivesSyncStatus>;
|
|
205
|
-
/** The
|
|
132
|
+
/** The index's totals plus any in-progress background synchronization. */
|
|
206
133
|
getSyncProgress(): {
|
|
207
134
|
index: {
|
|
208
135
|
fileCount: number;
|
|
@@ -216,7 +143,7 @@ export declare class BlobStore implements IBucketStore {
|
|
|
216
143
|
readerDiskLimit?: number;
|
|
217
144
|
syncing: SyncActivity[];
|
|
218
145
|
};
|
|
219
|
-
/**
|
|
146
|
+
/** getSyncProgress's totals, but loading the index first, so they are never the zeroes of a store nothing has touched yet. */
|
|
220
147
|
computeIndexTotals(): Promise<{
|
|
221
148
|
fileCount: number;
|
|
222
149
|
byteCount: number;
|
|
@@ -226,11 +153,22 @@ export declare class BlobStore implements IBucketStore {
|
|
|
226
153
|
byteCount: number;
|
|
227
154
|
}[];
|
|
228
155
|
}>;
|
|
229
|
-
|
|
230
|
-
|
|
156
|
+
private namedIndexTotals;
|
|
157
|
+
/**
|
|
158
|
+
* The store's sources, as the current routing config says they should be. This is the ONLY way
|
|
159
|
+
* they are ever set: the first call populates an empty store, every later one applies a change to
|
|
160
|
+
* the running one. Windows, routes and flags move in place, genuinely new endpoints are added and
|
|
161
|
+
* start scanning, and endpoints that are gone go dead (their scans stop, their index entries
|
|
162
|
+
* drop).
|
|
163
|
+
*
|
|
164
|
+
* A store is never rebuilt for a config change. Its name decides its folder and its identity, and
|
|
165
|
+
* a config change cannot change either - so there is nothing a change can do to a store except
|
|
166
|
+
* this.
|
|
167
|
+
*/
|
|
168
|
+
updateSources(specs: BlobSourceSpec[]): void;
|
|
231
169
|
/** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
|
|
232
170
|
rescanBase(): Promise<void>;
|
|
233
|
-
/** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes)
|
|
171
|
+
/** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes). */
|
|
234
172
|
boundaryScanRemote(source: IArchives, config: {
|
|
235
173
|
since: number;
|
|
236
174
|
route?: [number, number];
|
|
@@ -238,6 +176,9 @@ export declare class BlobStore implements IBucketStore {
|
|
|
238
176
|
startLargeUpload(config?: {
|
|
239
177
|
path?: string;
|
|
240
178
|
lastModified?: number;
|
|
179
|
+
forceSetImmutable?: boolean;
|
|
180
|
+
noChecks?: boolean;
|
|
181
|
+
internal?: boolean;
|
|
241
182
|
}): Promise<string>;
|
|
242
183
|
appendLargeUpload(config: {
|
|
243
184
|
id: string;
|
|
@@ -247,62 +188,94 @@ export declare class BlobStore implements IBucketStore {
|
|
|
247
188
|
id: string;
|
|
248
189
|
path: string;
|
|
249
190
|
lastModified?: number;
|
|
191
|
+
forceSetImmutable?: boolean;
|
|
192
|
+
noChecks?: boolean;
|
|
193
|
+
internal?: boolean;
|
|
250
194
|
}): Promise<void>;
|
|
251
195
|
cancelLargeUpload(config: {
|
|
252
196
|
id: string;
|
|
253
197
|
}): Promise<void>;
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
private indexByteCount;
|
|
259
|
-
private sourceFileCounts;
|
|
260
|
-
private sourceByteCounts;
|
|
261
|
-
private syncActivities;
|
|
262
|
-
private dirty;
|
|
263
|
-
private overlay;
|
|
264
|
-
private sourceStates;
|
|
265
|
-
private syncStarted;
|
|
266
|
-
private entries;
|
|
267
|
-
private sourcesList;
|
|
268
|
-
private slotSourcesListIndexes;
|
|
269
|
-
private slotRegistrations;
|
|
198
|
+
/** Bytes of read cache the disk may hold; see CommonConfig.readerDiskLimit (StoreSync enforces it). Read from the config in effect, so raising or removing the limit takes effect on the next eviction pass. */
|
|
199
|
+
get readerDiskLimit(): number | undefined;
|
|
200
|
+
/** The write time a new write has to beat, or 0 when we have never heard of the key. Counts DELETIONS too: a write older than the deletion that removed it must not bring it back. The index is authoritative even for a write still buffered in a delayed source, since the entry is recorded when the write is accepted rather than when it reaches storage. */
|
|
201
|
+
currentWriteTime(key: string): number;
|
|
270
202
|
private isLive;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
203
|
+
registerSlot(slot: number): Promise<void>;
|
|
204
|
+
/** The persistent sourcesListIndex of a slot, or undefined when the slot never got that far (a source removed before its registration resolved). */
|
|
205
|
+
slotSourcesListIndex(slot: number): number | undefined;
|
|
206
|
+
sourcesListIndexOfSlot(slot: number): number;
|
|
207
|
+
slotForSourcesListIndex(sourcesListIndex: number): number | undefined;
|
|
208
|
+
getEntryHolder(entry: IndexEntry): Promise<IArchives | undefined>;
|
|
275
209
|
private loadIndex;
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
210
|
+
/** A file we hold. A deleted one is not one: it is a tombstone, and only getDeletedEntry knows about it. */
|
|
211
|
+
getIndexEntry(key: string): IndexEntry | undefined;
|
|
212
|
+
/** When a key was deleted, if it was. A deletion is an absence with a time attached - that time is what makes it propagate and what expires it. */
|
|
213
|
+
getDeletedEntry(key: string): {
|
|
214
|
+
writeTime: number;
|
|
215
|
+
changedAt: number;
|
|
216
|
+
} | undefined;
|
|
217
|
+
/** Every file we hold, for the passes that walk them all (listings, scans, reconciliation, eviction). Deletions are not in here - see deletedEntries. Live: deleting entries while iterating is expected here, and safe. */
|
|
218
|
+
indexEntries(): IterableIterator<[string, IndexEntry]>;
|
|
219
|
+
/** Every deletion we know of. A much smaller walk than the files, which is what makes expiring them cheap. */
|
|
220
|
+
deletedEntries(): IterableIterator<[string, {
|
|
221
|
+
writeTime: number;
|
|
222
|
+
changedAt: number;
|
|
223
|
+
}]>;
|
|
224
|
+
/** How many files we hold, deletions excluded. */
|
|
225
|
+
indexSize(): number;
|
|
226
|
+
/** Totals over the files we hold, broken down by the slot holding each (entries can name a source that is no longer configured, which counts towards the total but no slot). */
|
|
227
|
+
indexTotals(): {
|
|
228
|
+
fileCount: number;
|
|
229
|
+
byteCount: number;
|
|
230
|
+
slots: {
|
|
231
|
+
fileCount: number;
|
|
232
|
+
byteCount: number;
|
|
233
|
+
}[];
|
|
234
|
+
};
|
|
235
|
+
/** Records a file, as of its write time. Returns false, having changed nothing, when we already know something at least as new - the index cannot be made to go backwards, whichever path the write came in by. */
|
|
236
|
+
setIndexEntry(key: string, entry: {
|
|
237
|
+
writeTime: number;
|
|
238
|
+
size: number;
|
|
239
|
+
sourcesListIndex: number;
|
|
240
|
+
}): boolean;
|
|
241
|
+
/** Records a DELETION, as of its time: the key stops existing here, and the tombstone is what makes that fact propagate and reconcile like any other write. Same ordering rule as setIndexEntry. */
|
|
242
|
+
setIndexDeleted(key: string, writeTime: number): boolean;
|
|
243
|
+
/** Forgets a key entirely, tombstone included. NOT a deletion: it says nothing happened to the file, only that we no longer know anything about it - for an entry whose holder turned out not to have it, and for a tombstone old enough that everyone has heard. */
|
|
244
|
+
purgeIndexEntry(key: string): void;
|
|
245
|
+
/**
|
|
246
|
+
* Every write, however it is stamped, has to be one we are actually meant to hold - because the
|
|
247
|
+
* alternative is not a smaller problem, it is a silent one. A write that lands on a store that
|
|
248
|
+
* does not serve its route (or on a server that is not in the bucket's config at all) goes into a
|
|
249
|
+
* folder nothing scans and no peer reconciles: it succeeds, and then it is gone. The markers make
|
|
250
|
+
* the client re-read the routing config and retry, which is exactly the right outcome when the
|
|
251
|
+
* reason it aimed here is that its config was stale.
|
|
252
|
+
*/
|
|
253
|
+
private assertWriteTarget;
|
|
254
|
+
/**
|
|
255
|
+
* Whether a routing config may be written here. Two rules, and this is the one place either is
|
|
256
|
+
* applied - a config only ever enters the system through a write, so a config that got in is a
|
|
257
|
+
* config that passed, and reading one back never judges it again.
|
|
258
|
+
*
|
|
259
|
+
* The config has to be valid as a whole (see assertValidRemoteConfig), and it has to outrank what
|
|
260
|
+
* we are running: the same version means the same config, so re-writing it is harmless, but a
|
|
261
|
+
* lower one is an older config arriving late and must never undo a newer one.
|
|
262
|
+
*/
|
|
263
|
+
private assertRoutingConfigWritable;
|
|
264
|
+
private assertFreshWriteTarget;
|
|
281
265
|
private assertMutable;
|
|
282
266
|
private assertInternalWriteAccepted;
|
|
283
|
-
|
|
284
|
-
private isDeadIntermediate;
|
|
285
|
-
private scanSource;
|
|
286
|
-
private reconcileSource;
|
|
287
|
-
private updateScanIndex;
|
|
288
|
-
private pollChanges;
|
|
289
|
-
private copySourceFiles;
|
|
290
|
-
private waitForRequiredScans;
|
|
291
|
-
private checkMissingKey;
|
|
292
|
-
private getIndexEntry;
|
|
293
|
-
/** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note fast writes still sitting in the overlay are invisible here; the caller re-finds them after our flush. */
|
|
267
|
+
/** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note this reads the disk past any write delay, so a fast write still buffered in memory is invisible here; the caller re-finds it once it flushes. */
|
|
294
268
|
private getInternal2;
|
|
295
269
|
/** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Only-take-latest still applies here. */
|
|
296
270
|
private setInternal;
|
|
297
271
|
private cacheRead;
|
|
298
272
|
private setOrDelete;
|
|
299
|
-
|
|
273
|
+
/** The instant every delayed write must be on its source: the end of our own write window that contains now, minus the flush margin (so the next window's source finds the data on handoff). The LATEST end among covering windows - overlapping windows hand off at the last one. No window contains now (an inert store, or a moment between our windows) -> 0, i.e. nothing may be delayed at all. */
|
|
274
|
+
writeFlushDeadline(): number;
|
|
300
275
|
private getWritableSources;
|
|
301
276
|
private writeToSources;
|
|
302
277
|
private getDiskSource;
|
|
303
|
-
|
|
304
|
-
private
|
|
305
|
-
private evicting;
|
|
306
|
-
private enforceDiskLimit;
|
|
307
|
-
private cleanupTombstones;
|
|
278
|
+
/** Writes everything still held by a delayed source (see ArchivesDelayed). force also writes what isn't due yet - shutdown cannot leave writes in memory. */
|
|
279
|
+
private flushDelayedWrites;
|
|
308
280
|
}
|
|
281
|
+
export {};
|