sliftutils 1.7.87 → 1.7.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +3 -1
- package/dist/treeSummary.ts.cache +2 -2
- package/index.d.ts +675 -296
- package/misc/dist/environment.ts.cache +2 -2
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/dist/zip.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/cloudflareHelpers.ts.cache +2 -2
- package/misc/https/dist/dns.ts.cache +2 -2
- package/misc/https/dist/hostServer.ts.cache +49 -21
- package/misc/https/dist/hostsFile.ts.cache +82 -0
- package/misc/https/dist/httpsCerts.ts.cache +2 -2
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/misc/https/hostServer.d.ts +6 -2
- package/misc/https/hostServer.ts +52 -19
- package/misc/https/hostsFile.d.ts +7 -0
- package/misc/https/hostsFile.ts +66 -0
- package/package.json +1 -1
- package/render-utils/dist/observer.tsx.cache +2 -2
- package/storage/ArchivesDisk.d.ts +3 -6
- package/storage/ArchivesDisk.ts +26 -2
- package/storage/BulkDatabase2/BulkDatabaseReader.ts +1 -1
- package/storage/BulkDatabase2/dist/BulkDatabaseBase.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseFormat.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseMerge.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/BulkDatabaseReader.ts.cache +3 -3
- package/storage/BulkDatabase2/dist/LoadedIndex.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/WriteOverlay.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/blockCache.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/mergeLock.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/mergeMarkers.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/streamLog.ts.cache +2 -2
- package/storage/BulkDatabase2/dist/syncClient.ts.cache +2 -2
- package/storage/IArchives.d.ts +37 -24
- package/storage/IArchives.ts +47 -62
- package/storage/LogMap.d.ts +56 -0
- package/storage/LogMap.ts +220 -0
- package/storage/archiveHelpers.d.ts +36 -0
- package/storage/archiveHelpers.ts +103 -0
- package/storage/backblaze.d.ts +7 -12
- package/storage/backblaze.ts +53 -48
- package/storage/dist/ArchivesDisk.ts.cache +28 -2
- package/storage/dist/FileFolderAPI.tsx.cache +2 -2
- package/storage/dist/IArchives.ts.cache +16 -52
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
- package/storage/dist/JSONStorage.ts.cache +2 -2
- package/storage/dist/LogMap.ts.cache +212 -0
- package/storage/dist/PendingManager.tsx.cache +2 -2
- package/storage/dist/TransactionStorage.ts.cache +2 -2
- package/storage/dist/backblaze.ts.cache +58 -44
- package/storage/dist/fileSystemPointer.ts.cache +2 -2
- package/storage/dist/remoteFileStorage.ts.cache +2 -2
- package/storage/remoteStorage/ArchivesDelayed.d.ts +66 -0
- package/storage/remoteStorage/ArchivesDelayed.ts +237 -0
- package/storage/remoteStorage/ArchivesRemote.d.ts +5 -6
- package/storage/remoteStorage/ArchivesRemote.ts +26 -5
- package/storage/remoteStorage/ArchivesUrl.d.ts +2 -5
- package/storage/remoteStorage/ArchivesUrl.ts +2 -2
- package/storage/remoteStorage/accessPage.tsx +12 -0
- package/storage/remoteStorage/accessStats.d.ts +1 -1
- package/storage/remoteStorage/accessStats.ts +3 -3
- package/storage/remoteStorage/blobStore.d.ts +162 -189
- package/storage/remoteStorage/blobStore.ts +495 -952
- package/storage/remoteStorage/bucketDisk.d.ts +24 -3
- package/storage/remoteStorage/bucketDisk.ts +56 -22
- package/storage/remoteStorage/certTrustModal.d.ts +1 -0
- package/storage/remoteStorage/certTrustModal.tsx +48 -0
- package/storage/remoteStorage/cliArgs.d.ts +2 -0
- package/storage/remoteStorage/cliArgs.ts +5 -0
- package/storage/remoteStorage/createArchives.d.ts +8 -7
- package/storage/remoteStorage/createArchives.ts +65 -26
- package/storage/remoteStorage/deployTakeover.ts +1 -1
- package/storage/remoteStorage/dist/ArchivesDelayed.ts.cache +220 -0
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +25 -5
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +3 -3
- package/storage/remoteStorage/dist/accessPage.tsx.cache +13 -3
- package/storage/remoteStorage/dist/accessStats.ts.cache +6 -6
- package/storage/remoteStorage/dist/blobStore.ts.cache +483 -965
- package/storage/remoteStorage/dist/bucketDisk.ts.cache +60 -29
- package/storage/remoteStorage/dist/certTrustModal.tsx.cache +55 -0
- package/storage/remoteStorage/dist/cliArgs.ts.cache +9 -4
- package/storage/remoteStorage/dist/createArchives.ts.cache +63 -26
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +5 -5
- package/storage/remoteStorage/dist/intermediateManagement.ts.cache +286 -0
- package/storage/remoteStorage/dist/intermediateSources.ts.cache +2 -2
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +122 -56
- package/storage/remoteStorage/dist/serverConfig.ts.cache +4 -4
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +7 -11
- package/storage/remoteStorage/dist/sourcesList.ts.cache +3 -6
- package/storage/remoteStorage/dist/storageClientController.ts.cache +2 -2
- package/storage/remoteStorage/dist/storageController.ts.cache +27 -12
- package/storage/remoteStorage/dist/storageServer.ts.cache +27 -7
- package/storage/remoteStorage/dist/storageServerCli.ts.cache +7 -3
- package/storage/remoteStorage/dist/storageServerState.ts.cache +133 -696
- package/storage/remoteStorage/dist/storeConfig.ts.cache +52 -0
- package/storage/remoteStorage/dist/storePlan.ts.cache +110 -81
- package/storage/remoteStorage/dist/storeSources.ts.cache +51 -0
- package/storage/remoteStorage/dist/storeSync.ts.cache +650 -0
- package/storage/remoteStorage/dist/validation.ts.cache +14 -4
- package/storage/remoteStorage/intermediateManagement.d.ts +12 -0
- package/storage/remoteStorage/intermediateManagement.ts +264 -0
- package/storage/remoteStorage/remoteConfig.d.ts +32 -0
- package/storage/remoteStorage/remoteConfig.ts +115 -50
- package/storage/remoteStorage/serverConfig.d.ts +1 -1
- package/storage/remoteStorage/serverConfig.ts +1 -1
- package/storage/remoteStorage/sourceWrapper.ts +4 -8
- package/storage/remoteStorage/sourcesList.d.ts +0 -2
- package/storage/remoteStorage/sourcesList.ts +0 -3
- package/storage/remoteStorage/spec.md +18 -4
- package/storage/remoteStorage/storageController.d.ts +10 -0
- package/storage/remoteStorage/storageController.ts +29 -16
- package/storage/remoteStorage/storageServer.d.ts +2 -0
- package/storage/remoteStorage/storageServer.ts +28 -4
- package/storage/remoteStorage/storageServerCli.ts +5 -1
- package/storage/remoteStorage/storageServerState.d.ts +34 -21
- package/storage/remoteStorage/storageServerState.ts +147 -662
- package/storage/remoteStorage/storeConfig.d.ts +30 -0
- package/storage/remoteStorage/storeConfig.ts +62 -0
- package/storage/remoteStorage/storePlan.d.ts +29 -17
- package/storage/remoteStorage/storePlan.ts +112 -85
- package/storage/remoteStorage/storeSources.d.ts +11 -0
- package/storage/remoteStorage/storeSources.ts +47 -0
- package/storage/remoteStorage/storeSync.d.ts +53 -0
- package/storage/remoteStorage/storeSync.ts +632 -0
- package/storage/remoteStorage/validation.d.ts +2 -0
- package/storage/remoteStorage/validation.ts +10 -0
package/index.d.ts
CHANGED
|
@@ -220,6 +220,10 @@ declare module "sliftutils/misc/https/hostServer" {
|
|
|
220
220
|
setDNSRecord?: boolean;
|
|
221
221
|
publicIp?: string;
|
|
222
222
|
allowHostnames?: string[];
|
|
223
|
+
/** LAN-only: do NOT forward the port (no UPnP/NAT mapping), for a server reachable only on the local network. */
|
|
224
|
+
internal?: boolean;
|
|
225
|
+
/** Serve a TLS cert signed by this machine's CA instead of obtaining a real (ACME) one. See getFreshHTTPSCert. */
|
|
226
|
+
selfSigned?: boolean;
|
|
223
227
|
/** When the port is busy (e.g. the previous deploy still holds it), mount on an alternate port instead (the socket server's built-in free-port scan), and keep trying to take the real port - once it frees, a raw TCP relay on the real port forwards to our listener (SocketFunction can only mount once per process). */
|
|
224
228
|
portFallback?: {
|
|
225
229
|
/** Delay until the next main-port acquisition attempt (tightened around the predecessor's scheduled death) */
|
|
@@ -232,14 +236,25 @@ declare module "sliftutils/misc/https/hostServer" {
|
|
|
232
236
|
};
|
|
233
237
|
/** Hosts a SocketFunction server on a real domain, with an automatically created and renewed Let's Encrypt HTTPS certificate (cached in the home folder, shared between processes on the machine). Expose your controllers (and any RequireController setup) before calling this. Returns the mounted nodeId. */
|
|
234
238
|
export declare function hostServer(config: HostServerConfig): Promise<string>;
|
|
235
|
-
/** Returns the cached HTTPS cert for the domain, creating/renewing it first if it is past this process's renewal threshold. Reads the disk cache on every call, so a renewal done by a parallel process is picked up instead of renewing again. */
|
|
236
|
-
export declare function getFreshHTTPSCert(domain: string): Promise<{
|
|
239
|
+
/** Returns the cached HTTPS cert for the domain, creating/renewing it first if it is past this process's renewal threshold. Reads the disk cache on every call, so a renewal done by a parallel process is picked up instead of renewing again. selfSigned: sign it with this machine's CA instead of getting a real ACME cert. */
|
|
240
|
+
export declare function getFreshHTTPSCert(domain: string, selfSigned?: boolean): Promise<{
|
|
237
241
|
key: string;
|
|
238
242
|
cert: string;
|
|
239
243
|
}>;
|
|
240
244
|
|
|
241
245
|
}
|
|
242
246
|
|
|
247
|
+
declare module "sliftutils/misc/https/hostsFile" {
|
|
248
|
+
/** Ensures the hosts file maps `hostname` to `ip` (adding our tagged line, or updating it/an existing line for the same hostname). Idempotent. Returns false, with a warning telling the user the line to add by hand, if the file can't be written (writing the hosts file needs admin/root). */
|
|
249
|
+
export declare function setHostsEntry(config: {
|
|
250
|
+
ip: string;
|
|
251
|
+
hostname: string;
|
|
252
|
+
}): boolean;
|
|
253
|
+
/** Removes our managed entry for `hostname` (only lines we added, tagged with the marker). No-op if absent or the file can't be written. */
|
|
254
|
+
export declare function removeHostsEntry(hostname: string): void;
|
|
255
|
+
|
|
256
|
+
}
|
|
257
|
+
|
|
243
258
|
declare module "sliftutils/misc/https/httpsCerts" {
|
|
244
259
|
/// <reference path="node-forge-ed25519.d.ts" />
|
|
245
260
|
/// <reference path="../../storage/storage.d.ts" />
|
|
@@ -858,7 +873,7 @@ declare module "sliftutils/render-utils/observer" {
|
|
|
858
873
|
declare module "sliftutils/storage/ArchivesDisk" {
|
|
859
874
|
/// <reference types="node" />
|
|
860
875
|
/// <reference types="node" />
|
|
861
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
|
|
876
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig } from "./IArchives";
|
|
862
877
|
export declare class ArchivesDisk implements IArchives {
|
|
863
878
|
private folder;
|
|
864
879
|
constructor(folder: string);
|
|
@@ -879,6 +894,7 @@ declare module "sliftutils/storage/ArchivesDisk" {
|
|
|
879
894
|
private filePath;
|
|
880
895
|
set(key: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
881
896
|
del(key: string, config?: DelConfig): Promise<void>;
|
|
897
|
+
move(config: MoveFileConfig): Promise<void>;
|
|
882
898
|
get(key: string, config?: GetConfig): Promise<Buffer | undefined>;
|
|
883
899
|
get2(key: string, config?: GetConfig): Promise<{
|
|
884
900
|
data: Buffer;
|
|
@@ -892,11 +908,7 @@ declare module "sliftutils/storage/ArchivesDisk" {
|
|
|
892
908
|
find(prefix: string, config?: FindConfig): Promise<string[]>;
|
|
893
909
|
findInfo(prefix: string, config?: FindConfig): Promise<ArchiveFileInfo[]>;
|
|
894
910
|
private collectFiles;
|
|
895
|
-
setLargeFile(config:
|
|
896
|
-
path: string;
|
|
897
|
-
lastModified?: number;
|
|
898
|
-
getNextData(): Promise<Buffer | undefined>;
|
|
899
|
-
}): Promise<void>;
|
|
911
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
900
912
|
startLargeUpload(): Promise<string>;
|
|
901
913
|
appendLargeUpload(id: string, data: Buffer): Promise<void>;
|
|
902
914
|
finishLargeUpload(id: string, key: string, lastModified?: number): Promise<void>;
|
|
@@ -2104,6 +2116,21 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2104
2116
|
/** One configured source in a routing config: a hosted (our storage server) or backblaze entry. Requests carry the exact SourceConfig they selected, and the server matches it against its own entries to pick the backing store. */
|
|
2105
2117
|
export type SourceConfig = HostedConfig | BackblazeConfig;
|
|
2106
2118
|
export type CommonConfig = {
|
|
2119
|
+
/**
|
|
2120
|
+
* The storage this entry names, as opposed to the rules for using it. Every entry with the same
|
|
2121
|
+
* name (for the same account and bucket) IS the same storage: one folder on the server, one
|
|
2122
|
+
* store, one index - however many entries there are and whatever their windows and routes say.
|
|
2123
|
+
* Everything about WHEN and WHICH KEYS (validWindow, route) is policy layered on top of it, and
|
|
2124
|
+
* changing that policy never moves data.
|
|
2125
|
+
*
|
|
2126
|
+
* Letters, numbers, underscore, dash and periods, up to 64 characters - so a host or a version
|
|
2127
|
+
* can be used as-is. It is the folder name, so it must stay unique and must never be reused for
|
|
2128
|
+
* different storage:
|
|
2129
|
+
* pointing two unrelated entries at one name merges their data, and re-using a retired name
|
|
2130
|
+
* hands the new entry the retired one's files. Deciding that is the developer's job - the server
|
|
2131
|
+
* only ever does what the name says.
|
|
2132
|
+
*/
|
|
2133
|
+
name: string;
|
|
2107
2134
|
/** By default a server hosting this bucket eagerly copies this source's full contents onto its own disk (on top of the lazy read-through caching). Set this to be a front end for a very large database without copying the full database - reads still down-cache individual files on demand. */
|
|
2108
2135
|
noFullSync?: boolean;
|
|
2109
2136
|
/** Bytes of read-cache this server's disk may hold; least-recently-used files are deleted from disk to stay under it (only ever when another source verifiably holds the file - the only copy is never deleted). Requires noFullSync (a full copy can't be bounded). */
|
|
@@ -2121,7 +2148,6 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2121
2148
|
public?: boolean;
|
|
2122
2149
|
fast?: boolean;
|
|
2123
2150
|
writeDelay?: number;
|
|
2124
|
-
rawDisk?: boolean;
|
|
2125
2151
|
immutable?: boolean;
|
|
2126
2152
|
};
|
|
2127
2153
|
export type BackblazeConfig = CommonConfig & {
|
|
@@ -2182,6 +2208,13 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2182
2208
|
/** Writes normally go ONLY to the write node (the first current-window source covering the key), retrying it even while it is down - consistent, but unavailable when that node is. With fallbacks, the write node is still tried first, but on failure the write lands on the next current-window source covering the key (synchronization moves it to the write node later) - availability at the cost of reads possibly missing the write until it propagates. Single-source archives ignore the flag. */
|
|
2183
2209
|
fallbacks?: boolean;
|
|
2184
2210
|
};
|
|
2211
|
+
/** setLargeFile's config: a SetConfig (it IS a set - the same immutability, ordering, internal, and fallbacks rules apply) plus the stream carrying the bytes. */
|
|
2212
|
+
export type SetLargeFileConfig = SetConfig & {
|
|
2213
|
+
path: string;
|
|
2214
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
2215
|
+
/** Rewinds the stream to its first byte. Without it the write gets exactly ONE attempt: a retry (a fallback source, or the write node coming back) would upload whatever is left of an already-consumed stream as if it were the whole file. Callers holding the data (a buffer, or a source they can re-read) always pass it - a large set with fallbacks is only as available as this. */
|
|
2216
|
+
restartStream?(): Promise<void> | void;
|
|
2217
|
+
};
|
|
2185
2218
|
export type ArchiveFileInfo = {
|
|
2186
2219
|
path: string;
|
|
2187
2220
|
createTime: number;
|
|
@@ -2229,23 +2262,17 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2229
2262
|
export declare function windowAcceptsWrites(validWindow: [number, number] | undefined): boolean;
|
|
2230
2263
|
export declare function windowsAcceptWrites(validWindows: [number, number][]): boolean;
|
|
2231
2264
|
export declare const LARGE_SET_THRESHOLD: number;
|
|
2232
|
-
/**
|
|
2233
|
-
export declare function bufferChunkStream(data: Buffer):
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
internal?: boolean;
|
|
2244
|
-
noFallbacks?: boolean;
|
|
2245
|
-
}): Promise<{
|
|
2246
|
-
writeTime: number;
|
|
2247
|
-
size: number;
|
|
2248
|
-
} | undefined>;
|
|
2265
|
+
/** The setLargeFile stream over an in-memory buffer, in LARGE_SET_THRESHOLD slices - how set transparently becomes setLargeFile for large buffers. Spread into the config: it provides both getNextData and restartStream (the buffer is still held, so a retry costs nothing). */
|
|
2266
|
+
export declare function bufferChunkStream(data: Buffer): {
|
|
2267
|
+
getNextData(): Promise<Buffer | undefined>;
|
|
2268
|
+
restartStream(): void;
|
|
2269
|
+
};
|
|
2270
|
+
export { copyArchiveFile } from "./archiveHelpers";
|
|
2271
|
+
/** move's config. There is deliberately no lastModified: the destination is ALWAYS stamped fresh (see IArchives.move) - a move is a new write at the new path, and a preserved old stamp is how a moved file loses to a stale tombstone there and vanishes. */
|
|
2272
|
+
export type MoveFileConfig = {
|
|
2273
|
+
fromPath: string;
|
|
2274
|
+
toPath: string;
|
|
2275
|
+
};
|
|
2249
2276
|
export type ArchivesSyncSourceStatus = {
|
|
2250
2277
|
debugName: string;
|
|
2251
2278
|
validWindows: [number, number][];
|
|
@@ -2300,12 +2327,10 @@ declare module "sliftutils/storage/IArchives" {
|
|
|
2300
2327
|
*/
|
|
2301
2328
|
set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
2302
2329
|
del(fileName: string, config?: DelConfig): Promise<void>;
|
|
2303
|
-
/**
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
getNextData(): Promise<Buffer | undefined>;
|
|
2308
|
-
}): Promise<void>;
|
|
2330
|
+
/** Moves a file to a new path within THIS archives, backend-side where the backend can (backblaze copies server-side, disk renames, the storage server relocates node-side) - the bytes never travel through the caller. The destination is stamped with a FRESH write time, even when the underlying operation (a rename) would preserve the old one, so the moved file cannot immediately lose to something newer sitting at its new path (e.g. the tombstone of an earlier deletion there); the source is then deleted, exactly like del. THROWS when the source file does not exist. Optional - callers go through moveArchiveFile (archiveHelpers.ts), which falls back to copy + confirm + delete. */
|
|
2331
|
+
move?(config: MoveFileConfig): Promise<void>;
|
|
2332
|
+
/** Streams a file too large to hold in memory. getNextData returns undefined when done. This only needs to be called when you CANNOT materialize the entire file in memory - if you can, just call set: above LARGE_SET_THRESHOLD it streams through setLargeFile internally, keeping the client responsive and not overwhelming the server. The rest of the config is a plain SetConfig and means exactly what it means on set (that is what makes a large set behave like a small one instead of quietly losing immutability, ordering, internal, or fallbacks semantics as the file crosses the threshold); backends that stamp their own times (backblaze) accept and ignore lastModified. THROWS when the stream produces no data at all - same rule as set: an empty file IS a deletion and would read back as missing. */
|
|
2333
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
2309
2334
|
/** writeTime is the last-write time — see ArchiveFileInfo.createTime, which is the same value. url as in get2. Size-0 entries (tombstones) report undefined unless config.includeTombstones. */
|
|
2310
2335
|
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2311
2336
|
writeTime: number;
|
|
@@ -2418,6 +2443,66 @@ declare module "sliftutils/storage/JSONStorage" {
|
|
|
2418
2443
|
|
|
2419
2444
|
}
|
|
2420
2445
|
|
|
2446
|
+
declare module "sliftutils/storage/LogMap" {
|
|
2447
|
+
/** A live value: what was stored, when it was written (the caller's ordering), and when we last changed it (ours). */
|
|
2448
|
+
export type LogEntry<T> = {
|
|
2449
|
+
value: T;
|
|
2450
|
+
time: number;
|
|
2451
|
+
changedAt: number;
|
|
2452
|
+
};
|
|
2453
|
+
/** A deleted key: when it was deleted, and when we learned. The value is gone; the time is the point of a tombstone. */
|
|
2454
|
+
export type LogTombstone = {
|
|
2455
|
+
time: number;
|
|
2456
|
+
changedAt: number;
|
|
2457
|
+
};
|
|
2458
|
+
export declare class LogMap<T> {
|
|
2459
|
+
private filePath;
|
|
2460
|
+
constructor(filePath: string);
|
|
2461
|
+
private values;
|
|
2462
|
+
private deleted;
|
|
2463
|
+
private logRecords;
|
|
2464
|
+
private pending;
|
|
2465
|
+
private flushTimer;
|
|
2466
|
+
private writeChain;
|
|
2467
|
+
/** Reads the log and replays it into memory. Every other method assumes this has finished. */
|
|
2468
|
+
load: {
|
|
2469
|
+
(): Promise<void>;
|
|
2470
|
+
reset(): void;
|
|
2471
|
+
set(newValue: Promise<void>): void;
|
|
2472
|
+
};
|
|
2473
|
+
/** The live value, or undefined when the key does not exist here (deleted included - a deletion is an absence, see getDeleted for its time). */
|
|
2474
|
+
get(key: string): LogEntry<T> | undefined;
|
|
2475
|
+
/** When the key was deleted, if it was. Absent both here and in get means we have never heard of it. */
|
|
2476
|
+
getDeleted(key: string): LogTombstone | undefined;
|
|
2477
|
+
/** The time the key last changed either way, or 0 if we have never heard of it - what a new write has to beat. */
|
|
2478
|
+
timeOf(key: string): number;
|
|
2479
|
+
/** O(1), and counts only what exists. */
|
|
2480
|
+
get size(): number;
|
|
2481
|
+
get deletedSize(): number;
|
|
2482
|
+
/** Live values only. Live, in insertion order - deleting during iteration is safe (JS skips entries removed before they are reached), which is what the passes that walk everything and prune as they go rely on. */
|
|
2483
|
+
entries(): IterableIterator<[string, LogEntry<T>]>;
|
|
2484
|
+
/** The tombstones, which is a much smaller walk than the values - so expiring them, or listing what was deleted since some time, costs what it should. */
|
|
2485
|
+
deletedEntries(): IterableIterator<[string, LogTombstone]>;
|
|
2486
|
+
/** Stores a value as of `time`. Returns false when something at least as new is already here, in which case nothing changed - an out-of-order write is not an error, it is just late. */
|
|
2487
|
+
set(key: string, value: T, time: number): boolean;
|
|
2488
|
+
/** Deletes as of `time`, keeping the tombstone. Returns false when something at least as new is already here. */
|
|
2489
|
+
delete(key: string, time: number): boolean;
|
|
2490
|
+
/** Forgets the key entirely, tombstone included - for a tombstone old enough that nobody needs to hear about the deletion any more, and for an entry that turned out never to have existed. Not a deletion: it leaves nothing behind to propagate. */
|
|
2491
|
+
purge(key: string): void;
|
|
2492
|
+
private applySet;
|
|
2493
|
+
private applyDelete;
|
|
2494
|
+
private append;
|
|
2495
|
+
private scheduleFlush;
|
|
2496
|
+
/** Writes everything pending (rewriting the log first if it has grown too far past what it describes). */
|
|
2497
|
+
flush(): Promise<void>;
|
|
2498
|
+
private directory;
|
|
2499
|
+
private write;
|
|
2500
|
+
private appendPending;
|
|
2501
|
+
private compact;
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2421
2506
|
declare module "sliftutils/storage/PendingManager" {
|
|
2422
2507
|
import preact from "preact";
|
|
2423
2508
|
export declare function setPending(group: string, message: string): void;
|
|
@@ -2593,10 +2678,50 @@ declare module "sliftutils/storage/TransactionStorage" {
|
|
|
2593
2678
|
|
|
2594
2679
|
}
|
|
2595
2680
|
|
|
2681
|
+
declare module "sliftutils/storage/archiveHelpers" {
|
|
2682
|
+
import type { IArchives } from "./IArchives";
|
|
2683
|
+
/** Copies one file between two archives. Small files go as a single get2+set; past LARGE_COPY_THRESHOLD the copy streams through setLargeFile in LARGE_COPY_CHUNK ranged reads, so the whole file is never in memory. Returns the copied file's info, or undefined when the source doesn't have the file. */
|
|
2684
|
+
export declare function copyArchiveFile(config: {
|
|
2685
|
+
from: IArchives;
|
|
2686
|
+
to: IArchives;
|
|
2687
|
+
path: string;
|
|
2688
|
+
/** The path at the destination - defaults to path (the common case: the same key moving between two archives). */
|
|
2689
|
+
toPath?: string;
|
|
2690
|
+
size?: number;
|
|
2691
|
+
/** Stamps the destination write with this time INSTEAD of now - which makes the copy lose (silently) to anything newer at the destination, tombstones included. That is exactly right for synchronization, which copies the same key between replicas and must preserve its ordering, and exactly wrong for anything else - so omit it unless the destination is another copy of the same key. */
|
|
2692
|
+
writeTime?: number;
|
|
2693
|
+
forceSetImmutable?: boolean;
|
|
2694
|
+
noChecks?: boolean;
|
|
2695
|
+
internal?: boolean;
|
|
2696
|
+
noFallbacks?: boolean;
|
|
2697
|
+
}): Promise<{
|
|
2698
|
+
writeTime: number;
|
|
2699
|
+
size: number;
|
|
2700
|
+
} | undefined>;
|
|
2701
|
+
/**
|
|
2702
|
+
* Moves one file - between two archives, or between two paths of one. When from and to are the SAME
|
|
2703
|
+
* archives instance and it implements move, the backend moves the file itself (backblaze copies
|
|
2704
|
+
* server-side, disk renames, the storage server relocates it node-side) and the bytes never travel
|
|
2705
|
+
* through us. Everything else is the safe fallback: copy, then CONFIRM the destination actually
|
|
2706
|
+
* reports the file (getInfo), and only then delete the source - the one order in which no failure
|
|
2707
|
+
* can lose the file, only at worst leave it in both places. Throws when the source doesn't have the
|
|
2708
|
+
* file, or when the copy cannot be confirmed (the source is then left untouched).
|
|
2709
|
+
*/
|
|
2710
|
+
export declare function moveArchiveFile(config: {
|
|
2711
|
+
from: IArchives;
|
|
2712
|
+
to: IArchives;
|
|
2713
|
+
path: string;
|
|
2714
|
+
/** The path at the destination - defaults to path (moving between two archives). Required in practice when from and to are the same archives, where the same path would be a no-op. */
|
|
2715
|
+
toPath?: string;
|
|
2716
|
+
noFallbacks?: boolean;
|
|
2717
|
+
}): Promise<void>;
|
|
2718
|
+
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2596
2721
|
declare module "sliftutils/storage/backblaze" {
|
|
2597
2722
|
/// <reference types="node" />
|
|
2598
2723
|
/// <reference types="node" />
|
|
2599
|
-
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig } from "./IArchives";
|
|
2724
|
+
import { IArchives, ArchivesConfig, ChangesAfterConfig, ArchiveFileInfo, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig, SourceConfig } from "./IArchives";
|
|
2600
2725
|
export declare class ArchivesBackblaze implements IArchives {
|
|
2601
2726
|
private config;
|
|
2602
2727
|
constructor(config: {
|
|
@@ -2612,6 +2737,8 @@ declare module "sliftutils/storage/backblaze" {
|
|
|
2612
2737
|
enableLogging(): void;
|
|
2613
2738
|
private log;
|
|
2614
2739
|
getDebugName(): string;
|
|
2740
|
+
/** Policy flags changed in the routing config. The bucket-level settings getBucketAPI applied (CORS, cache time) are NOT re-applied - those are one-time bucket setup, and re-running them on every config edit would rewrite the bucket's settings from whichever process noticed first. */
|
|
2741
|
+
updateSourceConfig(sourceConfig: SourceConfig): void;
|
|
2615
2742
|
private getBucketAPI;
|
|
2616
2743
|
private currentReset;
|
|
2617
2744
|
private last503Reset;
|
|
@@ -2624,13 +2751,11 @@ declare module "sliftutils/storage/backblaze" {
|
|
|
2624
2751
|
} | undefined>;
|
|
2625
2752
|
getConfig(): Promise<ArchivesConfig>;
|
|
2626
2753
|
hasWriteAccess(): Promise<boolean>;
|
|
2754
|
+
/** Whether an existing file already beats this write, so it must not be sent at all. Shared by set and setLargeFile - the size a value happens to have must never change which ordering rules apply to it. */
|
|
2755
|
+
private writeIsSuperseded;
|
|
2627
2756
|
set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
2628
2757
|
del(fileName: string, config?: DelConfig): Promise<void>;
|
|
2629
|
-
setLargeFile(config:
|
|
2630
|
-
path: string;
|
|
2631
|
-
lastModified?: number;
|
|
2632
|
-
getNextData(): Promise<Buffer | undefined>;
|
|
2633
|
-
}): Promise<void>;
|
|
2758
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
2634
2759
|
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2635
2760
|
writeTime: number;
|
|
2636
2761
|
size: number;
|
|
@@ -2643,12 +2768,7 @@ declare module "sliftutils/storage/backblaze" {
|
|
|
2643
2768
|
size: number;
|
|
2644
2769
|
}[]>;
|
|
2645
2770
|
assertPathValid(path: string): Promise<void>;
|
|
2646
|
-
move(config:
|
|
2647
|
-
path: string;
|
|
2648
|
-
target: IArchives;
|
|
2649
|
-
targetPath: string;
|
|
2650
|
-
copyInstead?: boolean;
|
|
2651
|
-
}): Promise<void>;
|
|
2771
|
+
move(config: MoveFileConfig): Promise<void>;
|
|
2652
2772
|
copy(config: {
|
|
2653
2773
|
path: string;
|
|
2654
2774
|
target: IArchives;
|
|
@@ -2891,10 +3011,80 @@ declare module "sliftutils/storage/remoteFileStorage" {
|
|
|
2891
3011
|
|
|
2892
3012
|
}
|
|
2893
3013
|
|
|
3014
|
+
declare module "sliftutils/storage/remoteStorage/ArchivesDelayed" {
|
|
3015
|
+
/// <reference types="node" />
|
|
3016
|
+
/// <reference types="node" />
|
|
3017
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, SetConfig, SetLargeFileConfig, SourceConfig } from "../IArchives";
|
|
3018
|
+
export declare const DEFAULT_FAST_WRITE_DELAY: number;
|
|
3019
|
+
export declare const MAX_REMOTE_FAST_BUFFER: number;
|
|
3020
|
+
export declare class ArchivesDelayed implements IArchives {
|
|
3021
|
+
inner: IArchives;
|
|
3022
|
+
private delay;
|
|
3023
|
+
private pending;
|
|
3024
|
+
private stopped;
|
|
3025
|
+
/** The instant every pending write must be on the source no matter its delay - our own valid window's end, minus the flush margin (the next window's source has to find the data on handoff). The store binds it; without it there is no deadline, only the delay. */
|
|
3026
|
+
private flushBefore?;
|
|
3027
|
+
constructor(inner: IArchives, delay: number);
|
|
3028
|
+
/** Called by the store that owns this source: fast writes are never delayed past the store's own write window. */
|
|
3029
|
+
bindFlushDeadline(flushBefore: () => number): void;
|
|
3030
|
+
/** The config changed the delay (fast writes turned on or off, or a different writeDelay). Anything already pending keeps the due time it was accepted with - shortening the delay must not strand it, and lengthening it must not hold it longer than promised. */
|
|
3031
|
+
setDelay(delay: number): void;
|
|
3032
|
+
getDebugName(): string;
|
|
3033
|
+
hasWriteAccess(): Promise<boolean>;
|
|
3034
|
+
set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
3035
|
+
del(fileName: string, config?: DelConfig): Promise<void>;
|
|
3036
|
+
private buffer;
|
|
3037
|
+
private deadlinePassed;
|
|
3038
|
+
private write;
|
|
3039
|
+
/** Writes everything due (its delay elapsed, or the store's window deadline reached). force writes everything, however recent - shutdown and window handoffs cannot leave writes in memory. */
|
|
3040
|
+
flush(force?: boolean): Promise<void>;
|
|
3041
|
+
private flushDue;
|
|
3042
|
+
/** Stops the flush loop, after writing everything still pending. */
|
|
3043
|
+
dispose(): Promise<void>;
|
|
3044
|
+
get(fileName: string, config?: GetConfig): Promise<Buffer | undefined>;
|
|
3045
|
+
get2(fileName: string, config?: GetConfig): Promise<{
|
|
3046
|
+
data: Buffer;
|
|
3047
|
+
writeTime: number;
|
|
3048
|
+
size: number;
|
|
3049
|
+
url?: string;
|
|
3050
|
+
} | {
|
|
3051
|
+
data?: undefined;
|
|
3052
|
+
writeTime?: undefined;
|
|
3053
|
+
size?: undefined;
|
|
3054
|
+
url: string;
|
|
3055
|
+
} | undefined>;
|
|
3056
|
+
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
3057
|
+
writeTime: number;
|
|
3058
|
+
size: number;
|
|
3059
|
+
url?: string;
|
|
3060
|
+
} | undefined>;
|
|
3061
|
+
find(prefix: string, config?: FindConfig): Promise<string[]>;
|
|
3062
|
+
/** Pending writes are part of the listing: a scan of this source that couldn't see them would conclude the files had vanished from it, and drop them from the scanner's index. */
|
|
3063
|
+
findInfo(prefix: string, config?: FindConfig): Promise<ArchiveFileInfo[]>;
|
|
3064
|
+
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
3065
|
+
/** Never buffered: a file too large to hold in memory is exactly the file that must not sit in memory. Any pending write for the path goes first, so the two land in order. */
|
|
3066
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
3067
|
+
getURL(path: string): Promise<string>;
|
|
3068
|
+
getConfig(): Promise<ArchivesConfig>;
|
|
3069
|
+
getSyncStatus(): Promise<ArchivesSyncStatus>;
|
|
3070
|
+
}
|
|
3071
|
+
/** How long a source may hold a write. Our own disk and our own storage servers take the SHORT delay however large writeDelay is: they are what cross-node reads and redundancy depend on, and making those wait minutes is not a trade anyone asked for. Everything else (backblaze) takes the full delay, which is where coalescing actually saves money. Not fast -> no delay at all, and no wrapper. */
|
|
3072
|
+
export declare function sourceWriteDelay(config: {
|
|
3073
|
+
sourceConfig?: SourceConfig;
|
|
3074
|
+
fast?: boolean;
|
|
3075
|
+
writeDelay?: number;
|
|
3076
|
+
}): number;
|
|
3077
|
+
/** The delayed wrapper around a source, when it has one - how the store reaches past the delay (to the real disk for large uploads) and flushes on shutdown. */
|
|
3078
|
+
export declare function asDelayed(source: IArchives): ArchivesDelayed | undefined;
|
|
3079
|
+
/** The source itself, past any write delay. */
|
|
3080
|
+
export declare function unwrapDelayed(source: IArchives): IArchives;
|
|
3081
|
+
|
|
3082
|
+
}
|
|
3083
|
+
|
|
2894
3084
|
declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
2895
3085
|
/// <reference types="node" />
|
|
2896
3086
|
/// <reference types="node" />
|
|
2897
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, SourceConfig, SetConfig } from "../IArchives";
|
|
3087
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SourceConfig, SetConfig, SetLargeFileConfig } from "../IArchives";
|
|
2898
3088
|
export type ArchivesRemoteConfig = {
|
|
2899
3089
|
url: string;
|
|
2900
3090
|
waitForAccess?: boolean;
|
|
@@ -2923,6 +3113,8 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
|
2923
3113
|
private controller;
|
|
2924
3114
|
private lastDeniedLog;
|
|
2925
3115
|
getDebugName(): string;
|
|
3116
|
+
/** The config travels with every request (the server matches it against its own entries to pick the store), so a config change has to land here - otherwise we keep asking for a source description the server no longer recognizes. Only ever called with a config for the SAME endpoint (see sourceIdentity), so the connection, account, and bucket cannot change under us. */
|
|
3117
|
+
updateSourceConfig(sourceConfig: SourceConfig): void;
|
|
2926
3118
|
isConnected(): boolean;
|
|
2927
3119
|
ping(): Promise<{}>;
|
|
2928
3120
|
private authenticate;
|
|
@@ -2943,6 +3135,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
|
2943
3135
|
} | undefined>;
|
|
2944
3136
|
set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
2945
3137
|
del(fileName: string, config?: DelConfig): Promise<void>;
|
|
3138
|
+
move(config: MoveFileConfig): Promise<void>;
|
|
2946
3139
|
getInfo(fileName: string, config?: GetInfoConfig): Promise<{
|
|
2947
3140
|
writeTime: number;
|
|
2948
3141
|
size: number;
|
|
@@ -2952,11 +3145,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
|
2952
3145
|
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
2953
3146
|
getConfig(): Promise<ArchivesConfig>;
|
|
2954
3147
|
getSyncStatus(): Promise<ArchivesSyncStatus>;
|
|
2955
|
-
setLargeFile(config:
|
|
2956
|
-
path: string;
|
|
2957
|
-
lastModified?: number;
|
|
2958
|
-
getNextData(): Promise<Buffer | undefined>;
|
|
2959
|
-
}): Promise<void>;
|
|
3148
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
2960
3149
|
getURL(path: string): Promise<string>;
|
|
2961
3150
|
}
|
|
2962
3151
|
|
|
@@ -2965,7 +3154,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesRemote" {
|
|
|
2965
3154
|
declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
|
|
2966
3155
|
/// <reference types="node" />
|
|
2967
3156
|
/// <reference types="node" />
|
|
2968
|
-
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, FindConfig, GetConfig, GetInfoConfig } from "../IArchives";
|
|
3157
|
+
import { IArchives, ArchiveFileInfo, ArchivesConfig, ChangesAfterConfig, FindConfig, GetConfig, GetInfoConfig, SetLargeFileConfig } from "../IArchives";
|
|
2969
3158
|
export declare class ArchivesUrl implements IArchives {
|
|
2970
3159
|
private base;
|
|
2971
3160
|
constructor(base: string);
|
|
@@ -2985,10 +3174,7 @@ declare module "sliftutils/storage/remoteStorage/ArchivesUrl" {
|
|
|
2985
3174
|
lastModified?: number;
|
|
2986
3175
|
}): Promise<string>;
|
|
2987
3176
|
del(fileName: string): Promise<void>;
|
|
2988
|
-
setLargeFile(config:
|
|
2989
|
-
path: string;
|
|
2990
|
-
getNextData(): Promise<Buffer | undefined>;
|
|
2991
|
-
}): Promise<void>;
|
|
3177
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
2992
3178
|
find(prefix: string, config?: FindConfig): Promise<string[]>;
|
|
2993
3179
|
findInfo(prefix: string, config?: FindConfig): Promise<ArchiveFileInfo[]>;
|
|
2994
3180
|
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
@@ -3022,7 +3208,7 @@ declare module "sliftutils/storage/remoteStorage/accessStats" {
|
|
|
3022
3208
|
path: string;
|
|
3023
3209
|
size?: number;
|
|
3024
3210
|
}): void;
|
|
3025
|
-
/** Method decorator factory, for API methods whose single config-object argument has account and bucketName: tracks the access (as `bucketName/path`) after the method succeeds. Sizes come from the config's data (writes) or the result's data (reads); operations without either are count-only. Array results (listings - findInfo, getChangesAfter) are tracked as two
|
|
3211
|
+
/** Method decorator factory, for API methods whose single config-object argument has account and bucketName: tracks the access (as `bucketName/path`) after the method succeeds. Sizes come from the config's data (writes) or the result's data (reads); operations without either are count-only. Array results (listings - findInfo, getChangesAfter) are tracked as two breakdowns: "<op> queries" - one access per CALL, at the query prefix, sized by the number of results (so the tree shows which QUERY returns the most) - and "<op> results" - one count-only access per returned path (so the tree shows which PATHS come back most). */
|
|
3026
3212
|
export declare function trackAccessCall(operation: string): (target: unknown, key: string, descriptor: PropertyDescriptor) => void;
|
|
3027
3213
|
export declare function getAccessTotals(account: string): AccessTotals;
|
|
3028
3214
|
export declare function readAccessSummaries(config: {
|
|
@@ -3038,144 +3224,20 @@ declare module "sliftutils/storage/remoteStorage/accessStats" {
|
|
|
3038
3224
|
declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
3039
3225
|
/// <reference types="node" />
|
|
3040
3226
|
/// <reference types="node" />
|
|
3041
|
-
import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig,
|
|
3042
|
-
import {
|
|
3043
|
-
|
|
3227
|
+
import { IArchives, ArchiveFileInfo, ArchivesSource, ArchivesSyncStatus, ChangesAfterConfig, FindConfig, SourceConfig, SyncActivity } from "../IArchives";
|
|
3228
|
+
import { StoreSync } from "./storeSync";
|
|
3229
|
+
import { StoreConfig } from "./storeConfig";
|
|
3044
3230
|
export declare const WINDOW_END_FLUSH_MARGIN: number;
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
includeTombstones?: boolean;
|
|
3055
|
-
}): Promise<{
|
|
3056
|
-
data: Buffer;
|
|
3057
|
-
writeTime: number;
|
|
3058
|
-
size: number;
|
|
3059
|
-
} | undefined>;
|
|
3060
|
-
/** internal (store-to-store) writes go to the local disk + index with no fan-out; see SetConfig.internal */
|
|
3061
|
-
set(config: {
|
|
3062
|
-
path: string;
|
|
3063
|
-
data: Buffer;
|
|
3064
|
-
lastModified?: number;
|
|
3065
|
-
forceSetImmutable?: boolean;
|
|
3066
|
-
internal?: boolean;
|
|
3067
|
-
}): Promise<void>;
|
|
3068
|
-
del(config: {
|
|
3069
|
-
path: string;
|
|
3070
|
-
lastModified?: number;
|
|
3071
|
-
internal?: boolean;
|
|
3072
|
-
}): Promise<void>;
|
|
3073
|
-
getInfo(config: {
|
|
3074
|
-
path: string;
|
|
3075
|
-
includeTombstones?: boolean;
|
|
3076
|
-
}): Promise<{
|
|
3077
|
-
writeTime: number;
|
|
3078
|
-
size: number;
|
|
3079
|
-
} | undefined>;
|
|
3080
|
-
findInfo(config: FindConfig & {
|
|
3081
|
-
prefix: string;
|
|
3082
|
-
}): Promise<ArchiveFileInfo[]>;
|
|
3083
|
-
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
3084
|
-
getSyncStatus?(): Promise<ArchivesSyncStatus>;
|
|
3085
|
-
getSyncProgress?(): {
|
|
3086
|
-
index: {
|
|
3087
|
-
fileCount: number;
|
|
3088
|
-
byteCount: number;
|
|
3089
|
-
};
|
|
3090
|
-
sources: {
|
|
3091
|
-
debugName: string;
|
|
3092
|
-
fileCount: number;
|
|
3093
|
-
byteCount: number;
|
|
3094
|
-
}[];
|
|
3095
|
-
readerDiskLimit?: number;
|
|
3096
|
-
syncing: SyncActivity[];
|
|
3097
|
-
};
|
|
3098
|
-
computeIndexTotals?(): Promise<{
|
|
3099
|
-
fileCount: number;
|
|
3100
|
-
byteCount: number;
|
|
3101
|
-
sources: {
|
|
3102
|
-
debugName: string;
|
|
3103
|
-
fileCount: number;
|
|
3104
|
-
byteCount: number;
|
|
3105
|
-
}[];
|
|
3106
|
-
}>;
|
|
3107
|
-
/** path/lastModified let the store reject an upload into an immutable bucket before any bytes move */
|
|
3108
|
-
startLargeUpload(config?: {
|
|
3109
|
-
path?: string;
|
|
3110
|
-
lastModified?: number;
|
|
3111
|
-
}): Promise<string>;
|
|
3112
|
-
appendLargeUpload(config: {
|
|
3113
|
-
id: string;
|
|
3114
|
-
data: Buffer;
|
|
3115
|
-
}): Promise<void>;
|
|
3116
|
-
finishLargeUpload(config: {
|
|
3117
|
-
id: string;
|
|
3118
|
-
path: string;
|
|
3119
|
-
lastModified?: number;
|
|
3120
|
-
}): Promise<void>;
|
|
3121
|
-
cancelLargeUpload(config: {
|
|
3122
|
-
id: string;
|
|
3123
|
-
}): Promise<void>;
|
|
3231
|
+
/** What we store about a file. Its times are not in here: the index keeps those for every key, deleted ones included (see LogMap). */
|
|
3232
|
+
type IndexValue = {
|
|
3233
|
+
size: number;
|
|
3234
|
+
sourcesListIndex: number;
|
|
3235
|
+
};
|
|
3236
|
+
/** One file we hold, as everything outside the index sees it. */
|
|
3237
|
+
export type IndexEntry = IndexValue & {
|
|
3238
|
+
writeTime: number;
|
|
3239
|
+
changedAt: number;
|
|
3124
3240
|
};
|
|
3125
|
-
/** rawDisk buckets: the disk IS the store. No index, no synchronization, no window/route/immutability validation. */
|
|
3126
|
-
export declare class RawDiskStore implements IBucketStore {
|
|
3127
|
-
private disk;
|
|
3128
|
-
constructor(disk: ArchivesDisk);
|
|
3129
|
-
get2(config: {
|
|
3130
|
-
path: string;
|
|
3131
|
-
range?: {
|
|
3132
|
-
start: number;
|
|
3133
|
-
end: number;
|
|
3134
|
-
};
|
|
3135
|
-
internal?: boolean;
|
|
3136
|
-
includeTombstones?: boolean;
|
|
3137
|
-
}): Promise<{
|
|
3138
|
-
data: Buffer;
|
|
3139
|
-
writeTime: number;
|
|
3140
|
-
size: number;
|
|
3141
|
-
} | undefined>;
|
|
3142
|
-
set(config: {
|
|
3143
|
-
path: string;
|
|
3144
|
-
data: Buffer;
|
|
3145
|
-
lastModified?: number;
|
|
3146
|
-
forceSetImmutable?: boolean;
|
|
3147
|
-
internal?: boolean;
|
|
3148
|
-
}): Promise<void>;
|
|
3149
|
-
del(config: {
|
|
3150
|
-
path: string;
|
|
3151
|
-
lastModified?: number;
|
|
3152
|
-
internal?: boolean;
|
|
3153
|
-
}): Promise<void>;
|
|
3154
|
-
getInfo(config: {
|
|
3155
|
-
path: string;
|
|
3156
|
-
includeTombstones?: boolean;
|
|
3157
|
-
}): Promise<{
|
|
3158
|
-
writeTime: number;
|
|
3159
|
-
size: number;
|
|
3160
|
-
} | undefined>;
|
|
3161
|
-
findInfo(config: FindConfig & {
|
|
3162
|
-
prefix: string;
|
|
3163
|
-
}): Promise<ArchiveFileInfo[]>;
|
|
3164
|
-
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
3165
|
-
startLargeUpload(): Promise<string>;
|
|
3166
|
-
appendLargeUpload(config: {
|
|
3167
|
-
id: string;
|
|
3168
|
-
data: Buffer;
|
|
3169
|
-
}): Promise<void>;
|
|
3170
|
-
finishLargeUpload(config: {
|
|
3171
|
-
id: string;
|
|
3172
|
-
path: string;
|
|
3173
|
-
lastModified?: number;
|
|
3174
|
-
}): Promise<void>;
|
|
3175
|
-
cancelLargeUpload(config: {
|
|
3176
|
-
id: string;
|
|
3177
|
-
}): Promise<void>;
|
|
3178
|
-
}
|
|
3179
3241
|
export type BlobSourceSpec = {
|
|
3180
3242
|
identity: string;
|
|
3181
3243
|
url: string;
|
|
@@ -3185,23 +3247,69 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3185
3247
|
intermediate?: string;
|
|
3186
3248
|
sourceConfig?: SourceConfig;
|
|
3187
3249
|
create: () => IArchives;
|
|
3250
|
+
applyConfig?: (source: IArchives) => void;
|
|
3188
3251
|
};
|
|
3189
|
-
export declare class BlobStore
|
|
3190
|
-
|
|
3191
|
-
|
|
3252
|
+
export declare class BlobStore {
|
|
3253
|
+
folder: string;
|
|
3254
|
+
/** The name this store answers to (see CommonConfig.name) - the entries of the routing config that carry it are the ones that configure it, and the rest are its peers. */
|
|
3255
|
+
storeName: string;
|
|
3192
3256
|
private config?;
|
|
3193
|
-
|
|
3257
|
+
stopped: {
|
|
3258
|
+
stop: boolean;
|
|
3259
|
+
};
|
|
3260
|
+
syncStarted: boolean;
|
|
3261
|
+
/** Its sources, in config order: slot 0 is always its own disk folder, the rest are the peers it synchronizes with. Filled by updateSources, which is also how they change. The store OWNS them - writes pick among them, reads resolve holders through them - and StoreSync only scans whatever is in here at the time. */
|
|
3262
|
+
sources: ArchivesSource[];
|
|
3263
|
+
private discardedUploads;
|
|
3264
|
+
private nextDiscardedUpload;
|
|
3265
|
+
private sourcesList;
|
|
3266
|
+
private slotSourcesListIndexes;
|
|
3267
|
+
private slotRegistrations;
|
|
3268
|
+
private index;
|
|
3269
|
+
/** Keeping the index in agreement with the sources: scanning, pulling, pushing, and the maintenance that follows from holding an index (disk-limit eviction, tombstone expiry). It reads and writes this store's index and sources - it does not own them. */
|
|
3270
|
+
sync: StoreSync;
|
|
3271
|
+
constructor(folder: string,
|
|
3272
|
+
/** The name this store answers to (see CommonConfig.name) - the entries of the routing config that carry it are the ones that configure it, and the rest are its peers. */
|
|
3273
|
+
storeName: string, config?: {
|
|
3274
|
+
/** Whether a config entry is THIS SERVER (same account, same bucket, our own address). Injected because a store knows nothing about servers - it only needs to tell its own entries apart from its peers'. Absent means nothing is us, which is what a bare store (no server around it) wants. */
|
|
3275
|
+
isSelf?: ((source: SourceConfig) => boolean) | undefined;
|
|
3276
|
+
/** Builds one of its sources. Injected for the same reason, and because the delay it is created with is this store's policy. */
|
|
3277
|
+
createSource?: ((config: {
|
|
3278
|
+
sourceConfig?: SourceConfig;
|
|
3279
|
+
writeDelay: number;
|
|
3280
|
+
}) => IArchives) | undefined;
|
|
3281
|
+
/** Hands a running source a changed config, so an endpoint we already talk to is never rebuilt just because a flag moved. */
|
|
3282
|
+
applySource?: ((source: IArchives, sourceConfig: SourceConfig | undefined, writeDelay: number) => void) | undefined;
|
|
3194
3283
|
onIndexChanged?: ((key: string) => void) | undefined;
|
|
3195
|
-
readerDiskLimit?: number | undefined;
|
|
3196
3284
|
onWriteCounted?: ((kind: "original" | "flushed", bytes: number) => void) | undefined;
|
|
3197
3285
|
resolveSourceUrl?: ((url: string) => IArchives) | undefined;
|
|
3198
|
-
entries?: HostedConfig[] | undefined;
|
|
3199
3286
|
} | undefined);
|
|
3287
|
+
/** This store's folder, unwrapped: the same bytes slot 0 serves, but reached without its write delay. Used for the two things that cannot go through a buffered source - reading our own routing config before we have any sources, and streaming a large upload that must not sit in memory. */
|
|
3288
|
+
private ownDisk;
|
|
3289
|
+
/** What this store is configured to be. It owns this: the routing config is a file IN the store, so the store reads it, applies it to itself, and re-applies it whenever the file changes - by our own write, or by a peer's copy arriving through synchronization. */
|
|
3290
|
+
storeConfig: StoreConfig;
|
|
3291
|
+
private appliedRoutingVersion;
|
|
3200
3292
|
init: {
|
|
3201
3293
|
(): Promise<void>;
|
|
3202
3294
|
reset(): void;
|
|
3203
3295
|
set(newValue: Promise<void>): void;
|
|
3204
3296
|
};
|
|
3297
|
+
/**
|
|
3298
|
+
* Re-reads the routing config out of this store and applies it to itself. Called at startup and
|
|
3299
|
+
* whenever that file changes here - which is the ONE mechanism: a config written by an operator
|
|
3300
|
+
* and a config pulled off a peer are the same event, a write of that path into this store.
|
|
3301
|
+
*
|
|
3302
|
+
* A store with no routing config configures itself as its own disk, valid always, for the whole
|
|
3303
|
+
* key space. That is a complete, working store - it just has nobody to synchronize with - and it
|
|
3304
|
+
* is what lets a store exist before it has ever heard of a configuration.
|
|
3305
|
+
*/
|
|
3306
|
+
applyRoutingConfig(): Promise<void>;
|
|
3307
|
+
private readRoutingConfig;
|
|
3308
|
+
/** The version of the routing config this store is running, so a copy found on a peer is only taken when it is genuinely newer. -1 means it has none. */
|
|
3309
|
+
routingVersion(): number;
|
|
3310
|
+
private routingApplies;
|
|
3311
|
+
reapplyRoutingConfig(): void;
|
|
3312
|
+
private planSources;
|
|
3205
3313
|
dispose(): Promise<void>;
|
|
3206
3314
|
get2(config: {
|
|
3207
3315
|
path: string;
|
|
@@ -3228,6 +3336,11 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3228
3336
|
lastModified?: number;
|
|
3229
3337
|
internal?: boolean;
|
|
3230
3338
|
}): Promise<void>;
|
|
3339
|
+
/** A node-side move: the bytes never travel through the client. Deliberately just get2 + set + del rather than a disk rename, so the destination write passes EVERY rule a set passes (windows, routes, immutability, only-take-latest, index, fan-out to peers) and the deletion propagates as a normal tombstone - a rename would bypass all of it. The set stamps fresh, so the moved file beats any tombstone at its new path. */
|
|
3340
|
+
move(config: {
|
|
3341
|
+
fromPath: string;
|
|
3342
|
+
toPath: string;
|
|
3343
|
+
}): Promise<void>;
|
|
3231
3344
|
getInfo(config: {
|
|
3232
3345
|
path: string;
|
|
3233
3346
|
includeTombstones?: boolean;
|
|
@@ -3240,7 +3353,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3240
3353
|
}): Promise<ArchiveFileInfo[]>;
|
|
3241
3354
|
getChangesAfter2(config: ChangesAfterConfig): Promise<ArchiveFileInfo[]>;
|
|
3242
3355
|
getSyncStatus(): Promise<ArchivesSyncStatus>;
|
|
3243
|
-
/** The
|
|
3356
|
+
/** The index's totals plus any in-progress background synchronization. */
|
|
3244
3357
|
getSyncProgress(): {
|
|
3245
3358
|
index: {
|
|
3246
3359
|
fileCount: number;
|
|
@@ -3254,7 +3367,7 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3254
3367
|
readerDiskLimit?: number;
|
|
3255
3368
|
syncing: SyncActivity[];
|
|
3256
3369
|
};
|
|
3257
|
-
/**
|
|
3370
|
+
/** getSyncProgress's totals, but loading the index first, so they are never the zeroes of a store nothing has touched yet. */
|
|
3258
3371
|
computeIndexTotals(): Promise<{
|
|
3259
3372
|
fileCount: number;
|
|
3260
3373
|
byteCount: number;
|
|
@@ -3264,11 +3377,22 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3264
3377
|
byteCount: number;
|
|
3265
3378
|
}[];
|
|
3266
3379
|
}>;
|
|
3267
|
-
|
|
3268
|
-
|
|
3380
|
+
private namedIndexTotals;
|
|
3381
|
+
/**
|
|
3382
|
+
* The store's sources, as the current routing config says they should be. This is the ONLY way
|
|
3383
|
+
* they are ever set: the first call populates an empty store, every later one applies a change to
|
|
3384
|
+
* the running one. Windows, routes and flags move in place, genuinely new endpoints are added and
|
|
3385
|
+
* start scanning, and endpoints that are gone go dead (their scans stop, their index entries
|
|
3386
|
+
* drop).
|
|
3387
|
+
*
|
|
3388
|
+
* A store is never rebuilt for a config change. Its name decides its folder and its identity, and
|
|
3389
|
+
* a config change cannot change either - so there is nothing a change can do to a store except
|
|
3390
|
+
* this.
|
|
3391
|
+
*/
|
|
3392
|
+
updateSources(specs: BlobSourceSpec[]): void;
|
|
3269
3393
|
/** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
|
|
3270
3394
|
rescanBase(): Promise<void>;
|
|
3271
|
-
/** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes)
|
|
3395
|
+
/** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes). */
|
|
3272
3396
|
boundaryScanRemote(source: IArchives, config: {
|
|
3273
3397
|
since: number;
|
|
3274
3398
|
route?: [number, number];
|
|
@@ -3276,6 +3400,9 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3276
3400
|
startLargeUpload(config?: {
|
|
3277
3401
|
path?: string;
|
|
3278
3402
|
lastModified?: number;
|
|
3403
|
+
forceSetImmutable?: boolean;
|
|
3404
|
+
noChecks?: boolean;
|
|
3405
|
+
internal?: boolean;
|
|
3279
3406
|
}): Promise<string>;
|
|
3280
3407
|
appendLargeUpload(config: {
|
|
3281
3408
|
id: string;
|
|
@@ -3285,65 +3412,97 @@ declare module "sliftutils/storage/remoteStorage/blobStore" {
|
|
|
3285
3412
|
id: string;
|
|
3286
3413
|
path: string;
|
|
3287
3414
|
lastModified?: number;
|
|
3415
|
+
forceSetImmutable?: boolean;
|
|
3416
|
+
noChecks?: boolean;
|
|
3417
|
+
internal?: boolean;
|
|
3288
3418
|
}): Promise<void>;
|
|
3289
3419
|
cancelLargeUpload(config: {
|
|
3290
3420
|
id: string;
|
|
3291
3421
|
}): Promise<void>;
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
private indexByteCount;
|
|
3297
|
-
private sourceFileCounts;
|
|
3298
|
-
private sourceByteCounts;
|
|
3299
|
-
private syncActivities;
|
|
3300
|
-
private dirty;
|
|
3301
|
-
private overlay;
|
|
3302
|
-
private sourceStates;
|
|
3303
|
-
private syncStarted;
|
|
3304
|
-
private entries;
|
|
3305
|
-
private sourcesList;
|
|
3306
|
-
private slotSourcesListIndexes;
|
|
3307
|
-
private slotRegistrations;
|
|
3422
|
+
/** Bytes of read cache the disk may hold; see CommonConfig.readerDiskLimit (StoreSync enforces it). Read from the config in effect, so raising or removing the limit takes effect on the next eviction pass. */
|
|
3423
|
+
get readerDiskLimit(): number | undefined;
|
|
3424
|
+
/** The write time a new write has to beat, or 0 when we have never heard of the key. Counts DELETIONS too: a write older than the deletion that removed it must not bring it back. The index is authoritative even for a write still buffered in a delayed source, since the entry is recorded when the write is accepted rather than when it reaches storage. */
|
|
3425
|
+
currentWriteTime(key: string): number;
|
|
3308
3426
|
private isLive;
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3427
|
+
registerSlot(slot: number): Promise<void>;
|
|
3428
|
+
/** The persistent sourcesListIndex of a slot, or undefined when the slot never got that far (a source removed before its registration resolved). */
|
|
3429
|
+
slotSourcesListIndex(slot: number): number | undefined;
|
|
3430
|
+
sourcesListIndexOfSlot(slot: number): number;
|
|
3431
|
+
slotForSourcesListIndex(sourcesListIndex: number): number | undefined;
|
|
3432
|
+
getEntryHolder(entry: IndexEntry): Promise<IArchives | undefined>;
|
|
3313
3433
|
private loadIndex;
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3434
|
+
/** A file we hold. A deleted one is not one: it is a tombstone, and only getDeletedEntry knows about it. */
|
|
3435
|
+
getIndexEntry(key: string): IndexEntry | undefined;
|
|
3436
|
+
/** When a key was deleted, if it was. A deletion is an absence with a time attached - that time is what makes it propagate and what expires it. */
|
|
3437
|
+
getDeletedEntry(key: string): {
|
|
3438
|
+
writeTime: number;
|
|
3439
|
+
changedAt: number;
|
|
3440
|
+
} | undefined;
|
|
3441
|
+
/** Every file we hold, for the passes that walk them all (listings, scans, reconciliation, eviction). Deletions are not in here - see deletedEntries. Live: deleting entries while iterating is expected here, and safe. */
|
|
3442
|
+
indexEntries(): IterableIterator<[string, IndexEntry]>;
|
|
3443
|
+
/** Every deletion we know of. A much smaller walk than the files, which is what makes expiring them cheap. */
|
|
3444
|
+
deletedEntries(): IterableIterator<[string, {
|
|
3445
|
+
writeTime: number;
|
|
3446
|
+
changedAt: number;
|
|
3447
|
+
}]>;
|
|
3448
|
+
/** How many files we hold, deletions excluded. */
|
|
3449
|
+
indexSize(): number;
|
|
3450
|
+
/** Totals over the files we hold, broken down by the slot holding each (entries can name a source that is no longer configured, which counts towards the total but no slot). */
|
|
3451
|
+
indexTotals(): {
|
|
3452
|
+
fileCount: number;
|
|
3453
|
+
byteCount: number;
|
|
3454
|
+
slots: {
|
|
3455
|
+
fileCount: number;
|
|
3456
|
+
byteCount: number;
|
|
3457
|
+
}[];
|
|
3458
|
+
};
|
|
3459
|
+
/** Records a file, as of its write time. Returns false, having changed nothing, when we already know something at least as new - the index cannot be made to go backwards, whichever path the write came in by. */
|
|
3460
|
+
setIndexEntry(key: string, entry: {
|
|
3461
|
+
writeTime: number;
|
|
3462
|
+
size: number;
|
|
3463
|
+
sourcesListIndex: number;
|
|
3464
|
+
}): boolean;
|
|
3465
|
+
/** Records a DELETION, as of its time: the key stops existing here, and the tombstone is what makes that fact propagate and reconcile like any other write. Same ordering rule as setIndexEntry. */
|
|
3466
|
+
setIndexDeleted(key: string, writeTime: number): boolean;
|
|
3467
|
+
/** Forgets a key entirely, tombstone included. NOT a deletion: it says nothing happened to the file, only that we no longer know anything about it - for an entry whose holder turned out not to have it, and for a tombstone old enough that everyone has heard. */
|
|
3468
|
+
purgeIndexEntry(key: string): void;
|
|
3469
|
+
/**
|
|
3470
|
+
* Every write, however it is stamped, has to be one we are actually meant to hold - because the
|
|
3471
|
+
* alternative is not a smaller problem, it is a silent one. A write that lands on a store that
|
|
3472
|
+
* does not serve its route (or on a server that is not in the bucket's config at all) goes into a
|
|
3473
|
+
* folder nothing scans and no peer reconciles: it succeeds, and then it is gone. The markers make
|
|
3474
|
+
* the client re-read the routing config and retry, which is exactly the right outcome when the
|
|
3475
|
+
* reason it aimed here is that its config was stale.
|
|
3476
|
+
*/
|
|
3477
|
+
private assertWriteTarget;
|
|
3478
|
+
/**
|
|
3479
|
+
* Whether a routing config may be written here. Two rules, and this is the one place either is
|
|
3480
|
+
* applied - a config only ever enters the system through a write, so a config that got in is a
|
|
3481
|
+
* config that passed, and reading one back never judges it again.
|
|
3482
|
+
*
|
|
3483
|
+
* The config has to be valid as a whole (see assertValidRemoteConfig), and it has to outrank what
|
|
3484
|
+
* we are running: the same version means the same config, so re-writing it is harmless, but a
|
|
3485
|
+
* lower one is an older config arriving late and must never undo a newer one.
|
|
3486
|
+
*/
|
|
3487
|
+
private assertRoutingConfigWritable;
|
|
3488
|
+
private assertFreshWriteTarget;
|
|
3319
3489
|
private assertMutable;
|
|
3320
3490
|
private assertInternalWriteAccepted;
|
|
3321
|
-
|
|
3322
|
-
private isDeadIntermediate;
|
|
3323
|
-
private scanSource;
|
|
3324
|
-
private reconcileSource;
|
|
3325
|
-
private updateScanIndex;
|
|
3326
|
-
private pollChanges;
|
|
3327
|
-
private copySourceFiles;
|
|
3328
|
-
private waitForRequiredScans;
|
|
3329
|
-
private checkMissingKey;
|
|
3330
|
-
private getIndexEntry;
|
|
3331
|
-
/** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note fast writes still sitting in the overlay are invisible here; the caller re-finds them after our flush. */
|
|
3491
|
+
/** Internal (store-to-store) read: purely the local disk, completely short-circuiting the index and holder resolution - the caller is another store, and chasing OUR remote holders while answering it is how infinite get loops between stores form. No window or route checks: if the bytes are on our disk, the caller may have them. Note this reads the disk past any write delay, so a fast write still buffered in memory is invisible here; the caller re-finds it once it flushes. */
|
|
3332
3492
|
private getInternal2;
|
|
3333
3493
|
/** Internal (store-to-store) write: the local disk plus our index, with NO downstream fan-out - the pushing store owns propagation, and fanning its pushes back out is how write loops between stores form. Only-take-latest still applies here. */
|
|
3334
3494
|
private setInternal;
|
|
3335
3495
|
private cacheRead;
|
|
3336
3496
|
private setOrDelete;
|
|
3337
|
-
|
|
3497
|
+
/** The instant every delayed write must be on its source: the end of our own write window that contains now, minus the flush margin (so the next window's source finds the data on handoff). The LATEST end among covering windows - overlapping windows hand off at the last one. No window contains now (an inert store, or a moment between our windows) -> 0, i.e. nothing may be delayed at all. */
|
|
3498
|
+
writeFlushDeadline(): number;
|
|
3338
3499
|
private getWritableSources;
|
|
3339
3500
|
private writeToSources;
|
|
3340
3501
|
private getDiskSource;
|
|
3341
|
-
|
|
3342
|
-
private
|
|
3343
|
-
private evicting;
|
|
3344
|
-
private enforceDiskLimit;
|
|
3345
|
-
private cleanupTombstones;
|
|
3502
|
+
/** Writes everything still held by a delayed source (see ArchivesDelayed). force also writes what isn't due yet - shutdown cannot leave writes in memory. */
|
|
3503
|
+
private flushDelayedWrites;
|
|
3346
3504
|
}
|
|
3505
|
+
export {};
|
|
3347
3506
|
|
|
3348
3507
|
}
|
|
3349
3508
|
|
|
@@ -3351,10 +3510,31 @@ declare module "sliftutils/storage/remoteStorage/bucketDisk" {
|
|
|
3351
3510
|
/// <reference types="node" />
|
|
3352
3511
|
/// <reference types="node" />
|
|
3353
3512
|
import { RemoteConfig } from "../IArchives";
|
|
3354
|
-
|
|
3355
|
-
export declare function
|
|
3513
|
+
/** A store's folder, from the only three things that identify it. The name is the config entry's name and nothing else: the same name is the same storage, whatever its window or route say, and a different name is different storage even for the same URL. Nothing about a folder changes when the routing does. */
|
|
3514
|
+
export declare function getBucketFolder(name: string, account: string, bucketName: string): string;
|
|
3515
|
+
export type StoreFolder = {
|
|
3516
|
+
account: string;
|
|
3517
|
+
name: string;
|
|
3518
|
+
bucketName: string;
|
|
3519
|
+
folder: string;
|
|
3520
|
+
};
|
|
3521
|
+
/** Every store this server holds for an account, found by walking the disk. */
|
|
3522
|
+
export declare function listAccountStoreFolders(account: string): Promise<StoreFolder[]>;
|
|
3523
|
+
/** Every store this server holds for ONE bucket - one per name it has ever been given. */
|
|
3524
|
+
export declare function listBucketStoreFolders(account: string, bucketName: string): Promise<StoreFolder[]>;
|
|
3525
|
+
export declare function readRoutingFile(folder: string): Promise<{
|
|
3526
|
+
data: Buffer;
|
|
3527
|
+
writeTime: number;
|
|
3528
|
+
} | undefined>;
|
|
3529
|
+
/** The bucket's routing config as this server holds it: the newest copy among its stores. Each store keeps its own, and they converge - so when they disagree, the one written most recently is the one that has heard the most. */
|
|
3530
|
+
export declare function readNewestRoutingFile(account: string, bucketName: string): Promise<{
|
|
3531
|
+
data: Buffer;
|
|
3532
|
+
writeTime: number;
|
|
3533
|
+
size: number;
|
|
3534
|
+
name: string;
|
|
3535
|
+
} | undefined>;
|
|
3356
3536
|
export declare function readRoutingFromDisk(account: string, bucketName: string): Promise<RemoteConfig | undefined>;
|
|
3357
|
-
/**
|
|
3537
|
+
/** What an anonymous URL read of the routing file gets: the same newest copy. */
|
|
3358
3538
|
export declare function getRoutingFileResult(account: string, bucketName: string): Promise<{
|
|
3359
3539
|
data: Buffer;
|
|
3360
3540
|
writeTime: number;
|
|
@@ -3369,19 +3549,27 @@ declare module "sliftutils/storage/remoteStorage/bucketDisk" {
|
|
|
3369
3549
|
|
|
3370
3550
|
}
|
|
3371
3551
|
|
|
3552
|
+
declare module "sliftutils/storage/remoteStorage/certTrustModal" {
|
|
3553
|
+
export declare function showCertTrustModal(serverUrl: string): void;
|
|
3554
|
+
|
|
3555
|
+
}
|
|
3556
|
+
|
|
3372
3557
|
declare module "sliftutils/storage/remoteStorage/cliArgs" {
|
|
3373
3558
|
export declare function getArg(name: string): string | undefined;
|
|
3559
|
+
/** A valueless boolean flag: true when --name is present (with nothing, or a following flag). */
|
|
3560
|
+
export declare function getFlag(name: string): boolean;
|
|
3374
3561
|
|
|
3375
3562
|
}
|
|
3376
3563
|
|
|
3377
3564
|
declare module "sliftutils/storage/remoteStorage/createArchives" {
|
|
3378
3565
|
/// <reference types="node" />
|
|
3379
3566
|
/// <reference types="node" />
|
|
3380
|
-
import { IArchives, RemoteConfig, RemoteConfigBase,
|
|
3567
|
+
import { IArchives, RemoteConfig, RemoteConfigBase, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus, ChangesAfterConfig, DelConfig, FindConfig, GetConfig, GetInfoConfig, MoveFileConfig, SetConfig, SetLargeFileConfig } from "../IArchives";
|
|
3381
3568
|
import { ServerBucketInfo, ActiveBucketInfo } from "./storageServerState";
|
|
3382
3569
|
/** The address, port, account, and bucket name a bucket routing URL addresses. Throws when the URL isn't a hosted bucket routing URL (https://host:port/file/<account>/<bucketName>/storage/storagerouting.json). */
|
|
3383
3570
|
export { parseHostedUrl, parseBackblazeUrl, getBucketBaseUrl } from "./remoteConfig";
|
|
3384
|
-
|
|
3571
|
+
/** A client for ONE source - see storeSources.ts. Re-exported here because a chain is built out of them. */
|
|
3572
|
+
export { createApiArchives } from "./storeSources";
|
|
3385
3573
|
export type ArchivesChainOptions = {
|
|
3386
3574
|
/** Outside of node we default to read-only downloads over the public URLs (no API connection) when the config has public sources. Set this to connect to the API anyway - needed for writing, listing, and any other operation the plain URL form cannot serve. */
|
|
3387
3575
|
directConnect?: boolean;
|
|
@@ -3468,15 +3656,15 @@ declare module "sliftutils/storage/remoteStorage/createArchives" {
|
|
|
3468
3656
|
set(fileName: string, data: Buffer, config?: SetConfig): Promise<string>;
|
|
3469
3657
|
private setRoutingConfig;
|
|
3470
3658
|
del(fileName: string, config?: DelConfig): Promise<void>;
|
|
3659
|
+
/** See IArchives.move. When one node is the write target for BOTH paths, that node moves the file itself - the bytes never come through us - with the same wrong-window/route re-resolution as any write. When the paths route to different shards no single node holds both, so the move degrades to a copy through us plus a delete, CONFIRMED at the destination before the source is touched. No smart timeout on the node-side move: it can be a big file's worth of node-side work, which the upload-sized deadlines would misjudge. */
|
|
3660
|
+
move(config: MoveFileConfig): Promise<void>;
|
|
3471
3661
|
private getVariableShardTargets;
|
|
3472
3662
|
/** The key setVariableShard would materialize for this VARIABLE_SHARD key (a value in the preferred shard's route range), without writing anything. */
|
|
3473
3663
|
getShardKey(key: string): Promise<string>;
|
|
3474
3664
|
private setVariableShard;
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
|
|
3478
|
-
getNextData(): Promise<Buffer | undefined>;
|
|
3479
|
-
}): Promise<void>;
|
|
3665
|
+
/** A large file is written exactly like a small one - same write node, same wrong-window/route re-resolution, same fallbacks - so a value's SIZE never decides its write semantics (set streams through here past LARGE_SET_THRESHOLD, and a file that grew past it must not suddenly lose the availability its caller asked for). The one difference: every attempt after the first has to rewind the stream, so a config without restartStream gets a single attempt. */
|
|
3666
|
+
setLargeFile(config: SetLargeFileConfig): Promise<void>;
|
|
3667
|
+
private setLargeFileOnce;
|
|
3480
3668
|
getURL(path: string): Promise<string>;
|
|
3481
3669
|
/** Every URL that could serve this path: public sources matching both the path's route and the current valid window. The first is the write node's (first matching source in config order, see runPrimary - the one guaranteed current); the rest are ranked fastest-first by measured latency. Empty when none qualify. */
|
|
3482
3670
|
getURLs(path: string): Promise<string[]>;
|
|
@@ -3553,6 +3741,22 @@ declare module "sliftutils/storage/remoteStorage/grantAccessCli" {
|
|
|
3553
3741
|
|
|
3554
3742
|
}
|
|
3555
3743
|
|
|
3744
|
+
declare module "sliftutils/storage/remoteStorage/intermediateManagement" {
|
|
3745
|
+
import { RemoteConfig } from "../IArchives";
|
|
3746
|
+
import { BucketState } from "./storageServerState";
|
|
3747
|
+
/** Called whenever a bucket's state is (re)built: arms the scans its upcoming window boundaries need. Each scan is scheduled once - the key includes the boundary it is for - so re-arming on every config change is harmless. */
|
|
3748
|
+
export declare function scheduleBoundaryWork(loaded: BucketState): void;
|
|
3749
|
+
/** An operator's config knows nothing about a switchover that is in flight right now, so writing it as-is would cancel one mid-handover. The in-flight windows are put back into it first. */
|
|
3750
|
+
export declare function reinjectIntermediates(current: RemoteConfig | undefined, incoming: RemoteConfig): RemoteConfig;
|
|
3751
|
+
/** Started by deployTakeover once we are actually a deploy successor listening on an alternate port. Until then there are no switchover windows to write or expire, so nothing polls. */
|
|
3752
|
+
export declare const startIntermediateMaintenance: {
|
|
3753
|
+
(): void;
|
|
3754
|
+
reset(): void;
|
|
3755
|
+
set(newValue: void): void;
|
|
3756
|
+
};
|
|
3757
|
+
|
|
3758
|
+
}
|
|
3759
|
+
|
|
3556
3760
|
declare module "sliftutils/storage/remoteStorage/intermediateSources" {
|
|
3557
3761
|
import { RemoteConfig, SourceConfig } from "../IArchives";
|
|
3558
3762
|
export declare const INTERMEDIATE_EXPIRE_GRACE: number;
|
|
@@ -3607,8 +3811,40 @@ declare module "sliftutils/storage/remoteStorage/remoteConfig" {
|
|
|
3607
3811
|
bucketName: string;
|
|
3608
3812
|
};
|
|
3609
3813
|
export declare function replaceHostedUrlPort(url: string, port: number): string;
|
|
3814
|
+
/**
|
|
3815
|
+
* Puts a source into the shape the code expects. It does NOT judge it: this runs every time a config
|
|
3816
|
+
* is READ, and a config that is already on disk has to keep working - a server that cannot parse its
|
|
3817
|
+
* own routing file is a server that cannot serve, and it would stay that way forever. Anything
|
|
3818
|
+
* missing or unusable is filled in with the safest equivalent instead, loudly where it matters.
|
|
3819
|
+
*
|
|
3820
|
+
* Judging happens on the way IN, in assertValidRemoteConfig.
|
|
3821
|
+
*/
|
|
3610
3822
|
export declare function normalizeSource(source: RemoteConfigBase): SourceConfig;
|
|
3823
|
+
/** Puts a whole config into the shape the code expects, without judging it - see normalizeSource, and see assertValidRemoteConfig for the judging. */
|
|
3611
3824
|
export declare function normalizeRemoteConfig(config: RemoteConfig | RemoteConfigBase): RemoteConfig;
|
|
3825
|
+
/**
|
|
3826
|
+
* Whether a config may be WRITTEN. Everything here is a rule about the config as a whole, which is
|
|
3827
|
+
* exactly why it cannot run on read: a config that is already stored somewhere has to keep being
|
|
3828
|
+
* readable, or a server that once accepted a bad one could never start again. Rejecting it at the
|
|
3829
|
+
* point it is introduced is what keeps a bad one from ever being stored in the first place.
|
|
3830
|
+
*/
|
|
3831
|
+
export declare function assertValidRemoteConfig(config: RemoteConfig): void;
|
|
3832
|
+
/**
|
|
3833
|
+
* The identity of one of a store's SOURCE SLOTS - which is the endpoint it talks to, and not the same
|
|
3834
|
+
* question as which store this is (that is CommonConfig.name). A switchover's alternate port is a
|
|
3835
|
+
* distinct slot even though it names the same storage, because a slot holds a connection to a port.
|
|
3836
|
+
*
|
|
3837
|
+
* ONLY the type, the url, and the intermediate's alternate port are part of it: everything else is
|
|
3838
|
+
* policy about how we USE the endpoint, and changing policy must never make a store believe it is
|
|
3839
|
+
* looking at a NEW source - that would drop every index entry the old one held and rescan it from
|
|
3840
|
+
* scratch, so the files it holds go missing from listings until the rescan finishes, for a flag flip.
|
|
3841
|
+
*
|
|
3842
|
+
* Built by hand rather than by serializing the config, so it cannot change just because the routing
|
|
3843
|
+
* file was written with its keys in a different order.
|
|
3844
|
+
*/
|
|
3845
|
+
export declare function sourceIdentity(sourceConfig: SourceConfig | undefined): string;
|
|
3846
|
+
/** What an index entry records as the holder of its bytes (see ArchivesSource.url), so it must name the endpoint FOREVER. An intermediate is a switchover's temporary alternate port onto another source, and that port is gone for good once its window passes - so it is recorded as the source it was split out of, which holds the same bucket and outlives it. */
|
|
3847
|
+
export declare function sourcePersistentUrl(sourceConfig: SourceConfig | undefined, folder: string): string;
|
|
3612
3848
|
export declare function parseRoutingData(data: Buffer): RemoteConfig;
|
|
3613
3849
|
export declare function serializeRemoteConfig(config: RemoteConfig): Buffer;
|
|
3614
3850
|
|
|
@@ -3640,7 +3876,7 @@ declare module "sliftutils/storage/remoteStorage/serverConfig" {
|
|
|
3640
3876
|
}): Promise<void>;
|
|
3641
3877
|
export declare function addExtraListenPort(port: number): void;
|
|
3642
3878
|
export declare function removeExtraListenPort(port: number): void;
|
|
3643
|
-
/** Whether address:port is this server process
|
|
3879
|
+
/** Whether address:port is this server process, including its extra listen ports (a deploy switchover's alternate port is still us). Used to tell which config entries are OUR copy of a bucket - the stores we run - as opposed to peers we synchronize with. Talking to ourselves is not one of the things it prevents: a source that happens to be us is reached over the API like any other. */
|
|
3644
3880
|
export declare function isOwnAddress(address: string, port: number): boolean;
|
|
3645
3881
|
|
|
3646
3882
|
}
|
|
@@ -3709,9 +3945,7 @@ declare module "sliftutils/storage/remoteStorage/sourcesList" {
|
|
|
3709
3945
|
private appendQueue;
|
|
3710
3946
|
private load;
|
|
3711
3947
|
getUrl(sourcesListIndex: number): string | undefined;
|
|
3712
|
-
/** For a sourcesListIndex beyond our in-memory list (another process appended since we read): re-reads the file, at most once per RELOAD_THROTTLE. Returning undefined can therefore mean "throttled", not "definitely absent" - never treat it as proof the index is bogus. */
|
|
3713
3948
|
getUrlReloading(sourcesListIndex: number): Promise<string | undefined>;
|
|
3714
|
-
/** The sourcesListIndex of the url, appending it if it is new. Appends are serialized within this process; before each append the file is re-read, so appends by other processes are picked up instead of duplicated. */
|
|
3715
3949
|
ensure(url: string): Promise<number>;
|
|
3716
3950
|
}
|
|
3717
3951
|
|
|
@@ -3836,6 +4070,13 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
|
|
|
3836
4070
|
lastModified?: number;
|
|
3837
4071
|
internal?: boolean;
|
|
3838
4072
|
}) => Promise<void>;
|
|
4073
|
+
move: (config: {
|
|
4074
|
+
account: string;
|
|
4075
|
+
bucketName: string;
|
|
4076
|
+
fromPath: string;
|
|
4077
|
+
toPath: string;
|
|
4078
|
+
sourceConfig: SourceConfig;
|
|
4079
|
+
}) => Promise<void>;
|
|
3839
4080
|
getInfo: (config: {
|
|
3840
4081
|
account: string;
|
|
3841
4082
|
bucketName: string;
|
|
@@ -3910,6 +4151,9 @@ declare module "sliftutils/storage/remoteStorage/storageController" {
|
|
|
3910
4151
|
path: string;
|
|
3911
4152
|
sourceConfig: SourceConfig;
|
|
3912
4153
|
lastModified?: number;
|
|
4154
|
+
forceSetImmutable?: boolean;
|
|
4155
|
+
noChecks?: boolean;
|
|
4156
|
+
internal?: boolean;
|
|
3913
4157
|
}) => Promise<string>;
|
|
3914
4158
|
uploadPart: (config: {
|
|
3915
4159
|
uploadId: string;
|
|
@@ -3935,6 +4179,8 @@ declare module "sliftutils/storage/remoteStorage/storageServer" {
|
|
|
3935
4179
|
url: string;
|
|
3936
4180
|
folder: string;
|
|
3937
4181
|
lowSpaceThresholdBytes?: number;
|
|
4182
|
+
internal?: boolean;
|
|
4183
|
+
selfSigned?: boolean;
|
|
3938
4184
|
};
|
|
3939
4185
|
export declare function hostStorageServer(config: HostStorageServerConfig): Promise<void>;
|
|
3940
4186
|
|
|
@@ -3948,30 +4194,38 @@ declare module "sliftutils/storage/remoteStorage/storageServerCli" {
|
|
|
3948
4194
|
declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
3949
4195
|
/// <reference types="node" />
|
|
3950
4196
|
/// <reference types="node" />
|
|
3951
|
-
import {
|
|
4197
|
+
import { BlobStore } from "./blobStore";
|
|
3952
4198
|
import { RemoteConfig, HostedConfig, SourceConfig, IArchives, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
|
|
3953
4199
|
import { BucketDiskInfo } from "./bucketDisk";
|
|
3954
4200
|
import { SelfSummary } from "./storePlan";
|
|
4201
|
+
/**
|
|
4202
|
+
* What this server HAS, as opposed to what any of it does. Four things, and nothing else:
|
|
4203
|
+
*
|
|
4204
|
+
* - the stores, by folder (getStore) - made once, and self-configuring from there
|
|
4205
|
+
* - which of them a request means (findBucketStore), which is only ever a name
|
|
4206
|
+
* - what exists on disk, for listings and for the pages that report it
|
|
4207
|
+
*
|
|
4208
|
+
* It decides nothing about routes, windows or peers - a store reads its own config and works that
|
|
4209
|
+
* out itself (BlobStore.applyRoutingConfig) - and it schedules nothing: switchovers and the handovers
|
|
4210
|
+
* they cause live in intermediateManagement.
|
|
4211
|
+
*/
|
|
3955
4212
|
export type LoadedStore = {
|
|
3956
|
-
|
|
3957
|
-
route?: [number, number];
|
|
4213
|
+
name: string;
|
|
3958
4214
|
entries: HostedConfig[];
|
|
3959
4215
|
folder: string;
|
|
3960
|
-
store:
|
|
4216
|
+
store: BlobStore;
|
|
3961
4217
|
};
|
|
3962
4218
|
export type BucketState = {
|
|
3963
4219
|
account: string;
|
|
3964
4220
|
bucketName: string;
|
|
3965
|
-
routing
|
|
3966
|
-
routingJSON: string;
|
|
4221
|
+
routing?: RemoteConfig;
|
|
3967
4222
|
selfEntries: HostedConfig[];
|
|
3968
4223
|
self: SelfSummary | undefined;
|
|
3969
4224
|
stores: LoadedStore[];
|
|
3970
|
-
structureKey: string;
|
|
3971
4225
|
};
|
|
3972
4226
|
/** The loaded bucket, loading it (which instantiates its stores and starts their synchronization) if needed. A bucket that does not exist on this server throws - callers never see undefined buckets. */
|
|
3973
4227
|
export declare function requireBucket(account: string, bucketName: string): Promise<BucketState>;
|
|
3974
|
-
/** The store serving a request: the
|
|
4228
|
+
/** The store serving a request: the one the client's selected entry NAMES. Nothing else is compared - not the window, not the route, not the flags - because the name is what says which storage this is, and everything else is policy the store itself applies to the request. A client a config version behind on some flag therefore still reaches the right store, which is the whole point of naming it. */
|
|
3975
4229
|
export declare function findBucketStore(account: string, bucketName: string, sourceConfig: SourceConfig | undefined): Promise<LoadedStore>;
|
|
3976
4230
|
/** Internal (store-to-store) reads skip store selection entirely: the caller is another store whose index says this MACHINE holds the bytes - the persisted holder identity is just a URL, which cannot name a store. Whichever store's folder has the newest copy answers. */
|
|
3977
4231
|
export declare function readBucketInternal(account: string, bucketName: string, config: {
|
|
@@ -3996,12 +4250,25 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
|
3996
4250
|
fileCount: number;
|
|
3997
4251
|
byteCount: number;
|
|
3998
4252
|
}[];
|
|
3999
|
-
}
|
|
4253
|
+
}>;
|
|
4254
|
+
/** The bucket states currently in memory - what the switchover machinery walks. Only what is loaded: loading a bucket starts its synchronization, and a bucket nothing has touched has no in-flight writes to hand over anyway. */
|
|
4255
|
+
export declare function getBucketStates(): BucketState[];
|
|
4000
4256
|
/** A cached IArchives for a persisted source identity: a routing URL (hosted/backblaze) or a disk folder path - the form BlobStore's sources list stores. Configuration (valid windows, routes) decides WHEN a source should be used; for reading bytes the index says a source holds, the URL alone is enough - even for sources no longer in any config. */
|
|
4001
4257
|
export declare function resolveSourceArchives(url: string): IArchives;
|
|
4258
|
+
export declare function getStore(account: string, bucketName: string, name: string): BlobStore;
|
|
4002
4259
|
export declare function getLoadedBucket(account: string, bucketName: string): Promise<BucketState | undefined>;
|
|
4003
|
-
/**
|
|
4004
|
-
export declare function
|
|
4260
|
+
/** Re-reads the bucket off the disk, replacing the cached snapshot: what a routing-config write calls, because that write can CREATE a store (a new name gets a new folder) and the snapshot would keep answering with the stores it saw at load time. Existing stores are reused (getStore caches by folder), so this only picks up folder changes - it never restarts a store. */
|
|
4261
|
+
export declare function reloadBucket(account: string, bucketName: string): Promise<BucketState | undefined>;
|
|
4262
|
+
/**
|
|
4263
|
+
* Writing the routing config is a write like any other: it goes into a store, and the store applies
|
|
4264
|
+
* it to itself and lets its peers pull it. The only thing that happens here is picking WHICH store,
|
|
4265
|
+
* because the writer names a source and a source names a store.
|
|
4266
|
+
*
|
|
4267
|
+
* In-flight switchover windows are re-injected first (see intermediateManagement): an operator's
|
|
4268
|
+
* config knows nothing about a switchover that is happening right now, and writing it as-is would
|
|
4269
|
+
* cancel it mid-flight.
|
|
4270
|
+
*/
|
|
4271
|
+
export declare function writeRoutingConfig(account: string, bucketName: string, name: string, data: Buffer, config?: {
|
|
4005
4272
|
lastModified?: number;
|
|
4006
4273
|
}): Promise<void>;
|
|
4007
4274
|
/** Which buckets this process currently has loaded - what a deploy successor asks its predecessor for, so it activates exactly the buckets that are actually in use. */
|
|
@@ -4009,13 +4276,6 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
|
4009
4276
|
account: string;
|
|
4010
4277
|
bucketName: string;
|
|
4011
4278
|
}[];
|
|
4012
|
-
export declare function rebuildAllLoadedBuckets(): Promise<void>;
|
|
4013
|
-
/** Started by deployTakeover once we are actually a deploy successor listening on an alternate port. Until then there are no switchover windows to write or expire, so nothing polls. */
|
|
4014
|
-
export declare const startIntermediateMaintenance: {
|
|
4015
|
-
(): void;
|
|
4016
|
-
reset(): void;
|
|
4017
|
-
set(newValue: void): void;
|
|
4018
|
-
};
|
|
4019
4279
|
export type ServerBucketInfo = {
|
|
4020
4280
|
bucketName: string;
|
|
4021
4281
|
active: boolean;
|
|
@@ -4030,8 +4290,8 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
|
4030
4290
|
};
|
|
4031
4291
|
export type ActiveBucketInfo = {
|
|
4032
4292
|
folder: string;
|
|
4033
|
-
/** The routing config the bucket is RUNNING on, straight from memory - including switchover windows written since it loaded */
|
|
4034
|
-
routing
|
|
4293
|
+
/** The routing config the bucket is RUNNING on, straight from memory - including switchover windows written since it loaded. Absent when none of its stores has one yet. */
|
|
4294
|
+
routing?: RemoteConfig;
|
|
4035
4295
|
/** Our own entries in that config, and their summarized current role (routes union + flags) */
|
|
4036
4296
|
selfEntries: HostedConfig[];
|
|
4037
4297
|
self?: SelfSummary;
|
|
@@ -4052,14 +4312,70 @@ declare module "sliftutils/storage/remoteStorage/storageServerState" {
|
|
|
4052
4312
|
};
|
|
4053
4313
|
/** Zeroes the write statistics of every bucket in the account. */
|
|
4054
4314
|
export declare function clearAccountWriteStats(account: string): number;
|
|
4055
|
-
|
|
4315
|
+
|
|
4316
|
+
}
|
|
4317
|
+
|
|
4318
|
+
declare module "sliftutils/storage/remoteStorage/storeConfig" {
|
|
4319
|
+
import { HostedConfig } from "../IArchives";
|
|
4320
|
+
export declare class StoreConfig {
|
|
4321
|
+
readonly name: string;
|
|
4322
|
+
constructor(name: string, entries: HostedConfig[]);
|
|
4323
|
+
private entries;
|
|
4324
|
+
/** The routing config changed. Same store either way: its name did not change, so neither did its folder, its index, or the data in it. */
|
|
4325
|
+
/** No entries is a real state, not an error: a store exists as soon as its folder does, and it may never have heard of a configuration (see StorePolicy). */
|
|
4326
|
+
update(entries: HostedConfig[]): void;
|
|
4327
|
+
all(): HostedConfig[];
|
|
4328
|
+
/**
|
|
4329
|
+
* What this store is configured to be right now: the entry whose window contains this moment;
|
|
4330
|
+
* failing that the next one due to start; failing that the last one to have ended.
|
|
4331
|
+
*
|
|
4332
|
+
* There is always an answer, and it matters that there is: a store between windows still holds
|
|
4333
|
+
* data and still has to answer for it, so it needs a route and flags even when nothing is
|
|
4334
|
+
* currently pointing writes at it. Which of the three cases produced the answer is deliberately
|
|
4335
|
+
* not exposed - the valid window itself is what says whether writes belong here, and every write
|
|
4336
|
+
* is checked against it separately.
|
|
4337
|
+
*/
|
|
4338
|
+
current(): StorePolicy;
|
|
4339
|
+
}
|
|
4340
|
+
/** The parts of a config entry that describe what a store IS - as opposed to which entry said so. Kept separate because a store with no entries still has all of them. */
|
|
4341
|
+
export type StorePolicy = {
|
|
4342
|
+
validWindow: [number, number];
|
|
4343
|
+
route?: [number, number];
|
|
4344
|
+
fast?: boolean;
|
|
4345
|
+
writeDelay?: number;
|
|
4346
|
+
noFullSync?: boolean;
|
|
4347
|
+
readerDiskLimit?: number;
|
|
4348
|
+
};
|
|
4056
4349
|
|
|
4057
4350
|
}
|
|
4058
4351
|
|
|
4059
4352
|
declare module "sliftutils/storage/remoteStorage/storePlan" {
|
|
4060
4353
|
import { RemoteConfig, HostedConfig, SourceConfig } from "../IArchives";
|
|
4354
|
+
/** Whether a config entry is THIS server's copy of this bucket - the same account and bucket, at an address this process answers on. */
|
|
4355
|
+
export declare function isSelfSource(source: SourceConfig, account: string, bucketName: string): boolean;
|
|
4061
4356
|
export declare function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[];
|
|
4062
4357
|
export declare function selectEntryAt(entries: HostedConfig[], time: number, route?: number): HostedConfig | undefined;
|
|
4358
|
+
/** What one of our stores has to pull in at a valid-window boundary, so the writes that landed just before the handover are not missed. */
|
|
4359
|
+
export type BoundaryHandover = {
|
|
4360
|
+
name: string;
|
|
4361
|
+
route: [number, number];
|
|
4362
|
+
scanOwnDisk: boolean;
|
|
4363
|
+
remotes: Map<number, [number, number]>;
|
|
4364
|
+
};
|
|
4365
|
+
/**
|
|
4366
|
+
* Who held each slice of our route in the window before windowStart, for every self entry whose
|
|
4367
|
+
* window starts exactly then. This is the whole of "who do we take over from": a store taking over a
|
|
4368
|
+
* route may be taking it from several previous owners at once (their shards need not line up with
|
|
4369
|
+
* ours), and from itself for the parts it already held.
|
|
4370
|
+
*
|
|
4371
|
+
* A self entry is skipped when an EARLIER entry valid at the boundary already covers its whole route:
|
|
4372
|
+
* config order is priority, so that entry is the write target and we are not the one taking over.
|
|
4373
|
+
* Owners are then resolved in config order too, each claiming the part of our route still unclaimed -
|
|
4374
|
+
* the same first-match-wins rule that picks a write target at any other moment.
|
|
4375
|
+
*
|
|
4376
|
+
* Pure: config in, plan out. Nothing here reads a store, a clock, or the network.
|
|
4377
|
+
*/
|
|
4378
|
+
export declare function previousWindowOwners(config: RemoteConfig, windowStart: number, selfIndexes: number[]): BoundaryHandover[];
|
|
4063
4379
|
/** Our role in a bucket's routing config, summarized across ALL currently-valid self entries. Stored instead of a single representative HostedConfig, so nothing can accidentally use one entry's route or flags where the union is required - the standard config has the same URL twice: a routed write-shard entry plus an unrouted read-everything entry. */
|
|
4064
4380
|
export type SelfSummary = {
|
|
4065
4381
|
/** The union of the current entries' routes, with overlapping/adjacent ranges combined - which commonly collapses to a single full range, making matching trivial. */
|
|
@@ -4067,35 +4383,98 @@ declare module "sliftutils/storage/remoteStorage/storePlan" {
|
|
|
4067
4383
|
public: boolean;
|
|
4068
4384
|
immutable: boolean;
|
|
4069
4385
|
noFullSync: boolean;
|
|
4070
|
-
rawDisk: boolean;
|
|
4071
4386
|
readerDiskLimit?: number;
|
|
4072
4387
|
};
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
route?: [number, number];
|
|
4077
|
-
noFullSync?: boolean;
|
|
4078
|
-
};
|
|
4079
|
-
export type StorePlanStore = {
|
|
4080
|
-
routeKey: string;
|
|
4081
|
-
route?: [number, number];
|
|
4388
|
+
/** One store this server runs for a bucket: the name that IS the store, and the entries that name it. What it synchronizes with is NOT decided here - the store reads its own copy of the config and works that out itself (see BlobStore.applyRoutingConfig). */
|
|
4389
|
+
export type SelfStore = {
|
|
4390
|
+
name: string;
|
|
4082
4391
|
entries: HostedConfig[];
|
|
4083
|
-
rawDisk: boolean;
|
|
4084
|
-
readerDiskLimit?: number;
|
|
4085
|
-
sourceSpecs: StoreSourceSpec[];
|
|
4086
4392
|
};
|
|
4087
|
-
|
|
4393
|
+
/** Which stores this server runs for a bucket, by name. Nothing more: a store is found by name, and configures itself. */
|
|
4394
|
+
export declare function planSelfStores(account: string, bucketName: string, routing: RemoteConfig): {
|
|
4088
4395
|
selfEntries: HostedConfig[];
|
|
4089
4396
|
self: SelfSummary | undefined;
|
|
4090
|
-
stores:
|
|
4091
|
-
structureKey: string;
|
|
4397
|
+
stores: SelfStore[];
|
|
4092
4398
|
};
|
|
4093
|
-
|
|
4399
|
+
|
|
4400
|
+
}
|
|
4401
|
+
|
|
4402
|
+
declare module "sliftutils/storage/remoteStorage/storeSources" {
|
|
4403
|
+
import { IArchives, SourceConfig } from "../IArchives";
|
|
4404
|
+
/** The client for one configured source: backblaze, or a storage server - including this one. */
|
|
4405
|
+
export declare function createApiArchives(source: SourceConfig): IArchives;
|
|
4406
|
+
/** The ONE place a store's source is built. Every source a store synchronizes with is one of exactly two things: a configured peer, or the store's own disk folder (no sourceConfig). writeDelay wraps it so its writes are buffered in memory for that long (see ArchivesDelayed) - the whole of "fast writes", per source, decided here. */
|
|
4407
|
+
export declare function createStoreSource(config: {
|
|
4408
|
+
sourceConfig?: SourceConfig;
|
|
4409
|
+
folder: string;
|
|
4410
|
+
writeDelay?: number;
|
|
4411
|
+
}): IArchives;
|
|
4412
|
+
/** Applies a changed config to an ALREADY RUNNING source (same endpoint, see sourceIdentity - only policy moved). Sources that carry their config into every request MUST be updated in place, or they keep sending the old one: the server matches the config it is handed against its own entries, so a source left holding a stale config eventually stops resolving to a store at all. The write delay is policy too, so it moves here as well. */
|
|
4413
|
+
export declare function applySourceConfig(source: IArchives, sourceConfig: SourceConfig | undefined, writeDelay?: number): void;
|
|
4414
|
+
|
|
4415
|
+
}
|
|
4416
|
+
|
|
4417
|
+
declare module "sliftutils/storage/remoteStorage/storeSync" {
|
|
4418
|
+
import { IArchives, ArchivesSyncStatus, SyncActivity } from "../IArchives";
|
|
4419
|
+
import type { BlobStore } from "./blobStore";
|
|
4420
|
+
export declare class StoreSync {
|
|
4421
|
+
private store;
|
|
4422
|
+
private states;
|
|
4423
|
+
private activities;
|
|
4424
|
+
private evicting;
|
|
4425
|
+
private lastAccess;
|
|
4426
|
+
constructor(store: BlobStore);
|
|
4427
|
+
/** Starts every live source's synchronization, plus the maintenance loops. Called once, by the store's init - the store's index must already be loaded, since scans write straight into it. */
|
|
4428
|
+
start(): void;
|
|
4429
|
+
/**
|
|
4430
|
+
* Asks every peer for the routing config, and takes it if it is newer than ours. This is the whole
|
|
4431
|
+
* of configuration propagation: it rides beside the scans rather than being part of them, because
|
|
4432
|
+
* one small read is worth doing every few minutes while a full listing is not, and because a
|
|
4433
|
+
* config we are missing is what stops everything else from being right.
|
|
4434
|
+
*
|
|
4435
|
+
* The copy is stored the same way a scan stores anything it pulls - an internal write into our own
|
|
4436
|
+
* store - and the store notices that file landing and re-configures itself. Nothing here knows what
|
|
4437
|
+
* a config means; it only knows this one file is worth asking for often.
|
|
4438
|
+
*/
|
|
4439
|
+
private pollRoutingConfig;
|
|
4440
|
+
/** Stops every source's loops (the store's own stop token stops the maintenance loops). */
|
|
4441
|
+
stop(): void;
|
|
4442
|
+
/** A slot the store just appended to its sources array: it starts from nothing, so it gets a full scan. */
|
|
4443
|
+
addSource(slot: number): void;
|
|
4444
|
+
/** The slot stays in the store's arrays forever (running loops hold slot numbers); it just goes dead - loops stop, and its index entries drop (other sources' scans re-find any copy that's still reachable through the new config). */
|
|
4445
|
+
removeSource(slot: number): Promise<void>;
|
|
4446
|
+
/** Whether a slot is still configured. Dead slots are never scanned, written, or read. */
|
|
4447
|
+
isLive(slot: number): boolean;
|
|
4448
|
+
getActivities(): SyncActivity[];
|
|
4449
|
+
/** A key was just served, so it goes to the back of the eviction queue. */
|
|
4450
|
+
noteAccess(key: string): void;
|
|
4451
|
+
private entryUnchanged;
|
|
4452
|
+
getStatus(): ArchivesSyncStatus;
|
|
4453
|
+
/** Listings come straight from the index, so they must wait for our own base source's initial scan (which might lag minutes) before they are trustworthy. The base (local disk) is implicitly required - remote sources are not, they come and go. */
|
|
4454
|
+
waitForRequiredScans(): Promise<void>;
|
|
4455
|
+
/** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
|
|
4456
|
+
rescanBase(): Promise<void>;
|
|
4457
|
+
/** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
|
|
4458
|
+
boundaryScanRemote(source: IArchives, config: {
|
|
4459
|
+
since: number;
|
|
4460
|
+
route?: [number, number];
|
|
4461
|
+
}): Promise<void>;
|
|
4462
|
+
private runSourceSync;
|
|
4463
|
+
private scanSource;
|
|
4464
|
+
private reconcileSource;
|
|
4465
|
+
private updateScanIndex;
|
|
4466
|
+
private pollChanges;
|
|
4467
|
+
private copySourceFiles;
|
|
4468
|
+
private enforceDiskLimit;
|
|
4469
|
+
private cleanupTombstones;
|
|
4470
|
+
}
|
|
4094
4471
|
|
|
4095
4472
|
}
|
|
4096
4473
|
|
|
4097
4474
|
declare module "sliftutils/storage/remoteStorage/validation" {
|
|
4098
4475
|
export declare function assertValidName(value: string, kind: string): void;
|
|
4476
|
+
/** A store's name (see CommonConfig.name), which also allows dots - a name is often a host or a version, and both read wrong without them. It is one path segment of the store's folder, so the two names that would mean a different folder entirely are rejected: everything else containing dots is just a name. */
|
|
4477
|
+
export declare function assertValidSourceName(value: string): void;
|
|
4099
4478
|
export declare function assertValidPath(path: string): void;
|
|
4100
4479
|
/** Method decorator: validates the well-known fields of the method's single config-object argument - account/bucketName as names, path as a path - before the method runs. Fields the config doesn't have are skipped, so it applies to every API method uniformly. prefix is deliberately NOT validated: prefixes may be empty or end with "/", both invalid for paths. */
|
|
4101
4480
|
export declare function assertValidArgs(target: unknown, key: string, descriptor: PropertyDescriptor): void;
|