sliftutils 1.7.112 → 1.7.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/treeSummary.ts.cache +2 -2
- package/index.d.ts +2 -1
- package/misc/dist/environment.ts.cache +2 -2
- package/misc/dist/getSecret.ts.cache +2 -2
- package/misc/dist/strings.ts.cache +2 -2
- package/misc/dist/zip.ts.cache +2 -2
- package/misc/https/dist/certs.ts.cache +2 -2
- package/misc/https/dist/persistentLocalStorage.ts.cache +2 -2
- package/package.json +1 -1
- package/render-utils/dist/observer.tsx.cache +2 -2
- package/storage/ArchivesDisk.ts +10 -8
- package/storage/IArchives.d.ts +1 -0
- package/storage/IArchives.ts +2 -1
- package/storage/TransactionFile.d.ts +1 -1
- package/storage/TransactionFile.ts +13 -2
- package/storage/archiveHelpers.ts +11 -9
- package/storage/dist/ArchivesDisk.ts.cache +12 -10
- package/storage/dist/FileFolderAPI.tsx.cache +2 -2
- package/storage/dist/IArchives.ts.cache +3 -3
- package/storage/dist/IndexedDBFileFolderAPI.ts.cache +2 -2
- package/storage/dist/JSONStorage.ts.cache +2 -2
- package/storage/dist/PendingManager.tsx.cache +2 -2
- package/storage/dist/StreamingLogs.ts.cache +2 -2
- package/storage/dist/TransactionFile.ts.cache +16 -5
- package/storage/dist/TransactionStorage.ts.cache +2 -2
- package/storage/dist/archiveHelpers.ts.cache +13 -11
- package/storage/dist/backblaze.ts.cache +2 -2
- package/storage/dist/fileSystemPointer.ts.cache +2 -2
- package/storage/dist/remoteFileStorage.ts.cache +2 -2
- package/storage/remoteStorage/ArchivesDelayed.ts +3 -3
- package/storage/remoteStorage/ArchivesRemote.ts +2 -2
- package/storage/remoteStorage/blobStore.ts +18 -17
- package/storage/remoteStorage/bucketDisk.ts +2 -1
- package/storage/remoteStorage/chainStartup.ts +5 -4
- package/storage/remoteStorage/createArchives.ts +16 -16
- package/storage/remoteStorage/deployTakeover.ts +3 -2
- package/storage/remoteStorage/dist/ArchivesDelayed.ts.cache +6 -6
- package/storage/remoteStorage/dist/ArchivesRemote.ts.cache +5 -5
- package/storage/remoteStorage/dist/ArchivesUrl.ts.cache +2 -2
- package/storage/remoteStorage/dist/accessStats.ts.cache +2 -2
- package/storage/remoteStorage/dist/blobStore.ts.cache +20 -19
- package/storage/remoteStorage/dist/bucketDisk.ts.cache +5 -4
- package/storage/remoteStorage/dist/certTrustModal.tsx.cache +2 -2
- package/storage/remoteStorage/dist/chainStartup.ts.cache +8 -7
- package/storage/remoteStorage/dist/cliArgs.ts.cache +2 -2
- package/storage/remoteStorage/dist/createArchives.ts.cache +17 -17
- package/storage/remoteStorage/dist/deployTakeover.ts.cache +6 -5
- package/storage/remoteStorage/dist/intermediateManagement.ts.cache +6 -5
- package/storage/remoteStorage/dist/intermediateSources.ts.cache +2 -2
- package/storage/remoteStorage/dist/remoteConfig.ts.cache +4 -4
- package/storage/remoteStorage/dist/serverConfig.ts.cache +2 -2
- package/storage/remoteStorage/dist/sourceWrapper.ts.cache +10 -9
- package/storage/remoteStorage/dist/sourcesList.ts.cache +2 -2
- package/storage/remoteStorage/dist/storageClientController.ts.cache +2 -2
- package/storage/remoteStorage/dist/storageController.ts.cache +7 -7
- package/storage/remoteStorage/dist/storageServerState.ts.cache +4 -4
- package/storage/remoteStorage/dist/storeConfig.ts.cache +2 -2
- package/storage/remoteStorage/dist/storePlan.ts.cache +2 -2
- package/storage/remoteStorage/dist/storeSources.ts.cache +2 -2
- package/storage/remoteStorage/dist/storeSync.ts.cache +9 -9
- package/storage/remoteStorage/dist/validation.ts.cache +2 -2
- package/storage/remoteStorage/intermediateManagement.ts +3 -2
- package/storage/remoteStorage/remoteConfig.ts +1 -1
- package/storage/remoteStorage/sourceWrapper.ts +7 -6
- package/storage/remoteStorage/storageController.ts +4 -4
- package/storage/remoteStorage/storageServer.ts +2 -1
- package/storage/remoteStorage/storageServerState.ts +1 -1
- package/storage/remoteStorage/storeSync.ts +8 -8
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"use strict";
|
|
1
|
+
"use strict"; var console = (function shimConsole() { let baseConsole = globalThis.console; if (typeof window !== "undefined") return baseConsole; let name = __filename.replaceAll("\\", "/").split("/").pop(); let dirName = __dirname.replaceAll("\\", "/").split("/").pop(); let context = { __FILE__: __filename, __DIR__: __dirname, __NAME__: name, __DIRNAME__: dirName }; let newConsole = { ...baseConsole }; for (let shim of ["log", "info", "warn", "error"]) { newConsole[shim] = (...args) => { args.push(context); baseConsole[shim](...args); }; } newConsole.__context = context; return newConsole; })(); "use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true , configurable: true});
|
|
3
3
|
//exports.assertValidArgs = exports.assertValidPath = exports.assertValidSourceName = exports.assertValidName = void 0;
|
|
4
4
|
function assertValidName(value, kind) {
|
|
@@ -47,4 +47,4 @@ function assertValidArgs(target, key, descriptor) {
|
|
|
47
47
|
}
|
|
48
48
|
exports.assertValidArgs = assertValidArgs;
|
|
49
49
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidmFsaWRhdGlvbi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInZhbGlkYXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUEsU0FBZ0IsZUFBZSxDQUFDLEtBQWEsRUFBRSxJQUFZO0lBQ3ZELElBQUksQ0FBQyxlQUFlLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDL0IsTUFBTSxJQUFJLEtBQUssQ0FBQyxXQUFXLElBQUksSUFBSSxJQUFJLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQywrREFBK0QsQ0FBQyxDQUFDO0lBQzdILENBQUM7QUFDTCxDQUFDO0FBSkQsMENBSUM7QUFFRCx1VEFBdVQ7QUFDdlQsU0FBZ0IscUJBQXFCLENBQUMsS0FBYTtJQUMvQyxJQUFJLENBQUMsZ0JBQWdCLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDaEMsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxLQUFLLENBQUMsbUVBQW1FLENBQUMsQ0FBQztJQUNySSxDQUFDO0lBQ0QsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDdEIsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxLQUFLLENBQUMsb0VBQW9FLENBQUMsQ0FBQztJQUN0SSxDQUFDO0FBQ0wsQ0FBQztBQVBELHNEQU9DO0FBRUQsU0FBZ0IsZUFBZSxDQUFDLElBQVk7SUFDeEMsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsQ0FBQyxNQUFNLEdBQUcsSUFBSSxFQUFFLENBQUM7UUFDMUMsTUFBTSxJQUFJLEtBQUssQ0FBQyxrQkFBa0IsSUFBSSxDQUFDLE1BQU0sNkJBQTZCLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUNwRyxDQUFDO0lBQ0QsSUFBSSxDQUFDLElBQUksSUFBSSxJQUFJLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLElBQUksSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQztRQUM3SCxNQUFNLElBQUksS0FBSyxDQUFDLGdCQUFnQixJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDLDhGQUE4RixDQUFDLENBQUM7SUFDdEssQ0FBQztJQUNELElBQUksSUFBSSxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxJQUFJLEtBQUssR0FBRyxJQUFJLElBQUksS0FBSyxJQUFJLENBQUMsRUFBRSxDQUFDO1FBQzlELE1BQU0sSUFBSSxLQUFLLENBQUMsZ0JBQWdCLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLENBQUMseUNBQXlDLENBQUMsQ0FBQztJQUNqSCxDQUFDO0FBQ0wsQ0FBQztBQVZELDBDQVVDO0FBRUQsNFdBQTRXO0FBQzVXLFNBQWdCLGVBQWUsQ0FBQyxNQUFlLEVBQUUsR0FBVyxFQUFFLFVBQThCO0lBQ3hGLElBQUksUUFBUSxHQUFHLFVBQVUsQ0FBQyxLQUF3QyxDQUFDO0lBQ25FLFVBQVUsQ0FBQyxLQUFLLEdBQUcsVUFBVSxHQUFHLElBQWU7UUFDM0MsSUFBSSxNQUFNLEdBQUcsSUFBSSxDQUFDLENBQUMsQ0FBeUUsQ0FBQztRQUM3RixJQUFJLE1BQU0sSUFBSSxPQUFPLE1BQU0sS0FBSyxRQUFRLEVBQUUsQ0FBQztZQUN2QyxJQUFJLFNBQVMsSUFBSSxNQUFNO2dCQUFFLGVBQWUsQ0FBQyxNQUFNLENBQUMsT0FBaUIsRUFBRSxTQUFTLENBQUMsQ0FBQztZQUM5RSxJQUFJLFlBQVksSUFBSSxNQUFNO2dCQUFFLGVBQWUsQ0FBQyxNQUFNLENBQUMsVUFBb0IsRUFBRSxhQUFhLENBQUMsQ0FBQztZQUN4RixJQUFJLE1BQU0sSUFBSSxNQUFNO2dCQUFFLGVBQWUsQ0FBQyxNQUFNLENBQUMsSUFBYyxDQUFDLENBQUM7UUFDakUsQ0FBQztRQUNELE9BQU8sUUFBUSxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsSUFBSSxDQUFDLENBQUM7SUFDdEMsQ0FBQyxDQUFDO0FBQ04sQ0FBQztBQVhELDBDQVdDIiwic291cmNlc0NvbnRlbnQiOlsiZXhwb3J0IGZ1bmN0aW9uIGFzc2VydFZhbGlkTmFtZSh2YWx1ZTogc3RyaW5nLCBraW5kOiBzdHJpbmcpOiB2b2lkIHtcbiAgICBpZiAoIS9eW1xcdy1dezEsNjR9JC8udGVzdCh2YWx1ZSkpIHtcbiAgICAgICAgdGhyb3cgbmV3IEVycm9yKGBJbnZhbGlkICR7a2luZH0gJHtKU09OLnN0cmluZ2lmeSh2YWx1ZSl9LCBleHBlY3RlZCAxLTY0IGNoYXJhY3RlcnMgb2YgbGV0dGVycy9udW1iZXJzL3VuZGVyc2NvcmUvZGFzaGApO1xuICAgIH1cbn1cblxuLyoqIEEgc3RvcmUncyBuYW1lIChzZWUgQ29tbW9uQ29uZmlnLm5hbWUpLCB3aGljaCBhbHNvIGFsbG93cyBkb3RzIC0gYSBuYW1lIGlzIG9mdGVuIGEgaG9zdCBvciBhIHZlcnNpb24sIGFuZCBib3RoIHJlYWQgd3Jvbmcgd2l0aG91dCB0aGVtLiBJdCBpcyBvbmUgcGF0aCBzZWdtZW50IG9mIHRoZSBzdG9yZSdzIGZvbGRlciwgc28gdGhlIHR3byBuYW1lcyB0aGF0IHdvdWxkIG1lYW4gYSBkaWZmZXJlbnQgZm9sZGVyIGVudGlyZWx5IGFyZSByZWplY3RlZDogZXZlcnl0aGluZyBlbHNlIGNvbnRhaW5pbmcgZG90cyBpcyBqdXN0IGEgbmFtZS4gKi9cbmV4cG9ydCBmdW5jdGlvbiBhc3NlcnRWYWxpZFNvdXJjZU5hbWUodmFsdWU6IHN0cmluZyk6IHZvaWQge1xuICAgIGlmICghL15bXFx3Li1dezEsNjR9JC8udGVzdCh2YWx1ZSkpIHtcbiAgICAgICAgdGhyb3cgbmV3IEVycm9yKGBJbnZhbGlkIHNvdXJjZSBuYW1lICR7SlNPTi5zdHJpbmdpZnkodmFsdWUpfSwgZXhwZWN0ZWQgMS02NCBjaGFyYWN0ZXJzIG9mIGxldHRlcnMvbnVtYmVycy91bmRlcnNjb3JlL2Rhc2gvZG90YCk7XG4gICAgfVxuICAgIGlmICgvXlxcLiskLy50ZXN0KHZhbHVlKSkge1xuICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYEludmFsaWQgc291cmNlIG5hbWUgJHtKU09OLnN0cmluZ2lmeSh2YWx1ZSl9OiBhIG5hbWUgb2Ygb25seSBkb3RzIHJlZmVycyB0byBhIGRpcmVjdG9yeSByYXRoZXIgdGhhbiBuYW1pbmcgb25lYCk7XG4gICAgfVxufVxuXG5leHBvcnQgZnVuY3Rpb24gYXNzZXJ0VmFsaWRQYXRoKHBhdGg6IHN0cmluZyk6IHZvaWQge1xuICAgIGlmIChCdWZmZXIuZnJvbShwYXRoLCBcInV0ZjhcIikubGVuZ3RoID4gMTAwMCkge1xuICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYFBhdGggdG9vIGxvbmc6ICR7cGF0aC5sZW5ndGh9IGNoYXJhY3RlcnMgPiAxMDAwLiBQYXRoOiAke3BhdGguc2xpY2UoMCwgMjAwKX1gKTtcbiAgICB9XG4gICAgaWYgKCFwYXRoIHx8IHBhdGguc3RhcnRzV2l0aChcIi9cIikgfHwgcGF0aC5lbmRzV2l0aChcIi9cIikgfHwgcGF0aC5pbmNsdWRlcyhcIi8vXCIpIHx8IHBhdGguaW5jbHVkZXMoXCJcXFxcXCIpIHx8IHBhdGguaW5jbHVkZXMoXCJcXHgwMFwiKSkge1xuICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYEludmFsaWQgcGF0aCAke0pTT04uc3RyaW5naWZ5KHBhdGguc2xpY2UoMCwgMjAwKSl9LCBwYXRocyBjYW5ub3QgYmUgZW1wdHksIHN0YXJ0IG9yIGVuZCB3aXRoIC8sIG9yIGNvbnRhaW4gLy8sIGJhY2tzbGFzaGVzLCBvciBudWxsIGNoYXJhY3RlcnNgKTtcbiAgICB9XG4gICAgaWYgKHBhdGguc3BsaXQoXCIvXCIpLnNvbWUocGFydCA9PiBwYXJ0ID09PSBcIi5cIiB8fCBwYXJ0ID09PSBcIi4uXCIpKSB7XG4gICAgICAgIHRocm93IG5ldyBFcnJvcihgSW52YWxpZCBwYXRoICR7SlNPTi5zdHJpbmdpZnkocGF0aC5zbGljZSgwLCAyMDApKX0sIHBhdGhzIGNhbm5vdCBjb250YWluIC4gb3IgLi4gc2VnbWVudHNgKTtcbiAgICB9XG59XG5cbi8qKiBNZXRob2QgZGVjb3JhdG9yOiB2YWxpZGF0ZXMgdGhlIHdlbGwta25vd24gZmllbGRzIG9mIHRoZSBtZXRob2QncyBzaW5nbGUgY29uZmlnLW9iamVjdCBhcmd1bWVudCAtIGFjY291bnQvYnVja2V0TmFtZSBhcyBuYW1lcywgcGF0aCBhcyBhIHBhdGggLSBiZWZvcmUgdGhlIG1ldGhvZCBydW5zLiBGaWVsZHMgdGhlIGNvbmZpZyBkb2Vzbid0IGhhdmUgYXJlIHNraXBwZWQsIHNvIGl0IGFwcGxpZXMgdG8gZXZlcnkgQVBJIG1ldGhvZCB1bmlmb3JtbHkuIHByZWZpeCBpcyBkZWxpYmVyYXRlbHkgTk9UIHZhbGlkYXRlZDogcHJlZml4ZXMgbWF5IGJlIGVtcHR5IG9yIGVuZCB3aXRoIFwiL1wiLCBib3RoIGludmFsaWQgZm9yIHBhdGhzLiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGFzc2VydFZhbGlkQXJncyh0YXJnZXQ6IHVua25vd24sIGtleTogc3RyaW5nLCBkZXNjcmlwdG9yOiBQcm9wZXJ0eURlc2NyaXB0b3IpOiB2b2lkIHtcbiAgICBsZXQgb3JpZ2luYWwgPSBkZXNjcmlwdG9yLnZhbHVlIGFzICguLi5hcmdzOiB1bmtub3duW10pID0+IHVua25vd247XG4gICAgZGVzY3JpcHRvci52YWx1ZSA9IGZ1bmN0aW9uICguLi5hcmdzOiB1bmtub3duW10pOiB1bmtub3duIHtcbiAgICAgICAgbGV0IGNvbmZpZyA9IGFyZ3NbMF0gYXMgeyBhY2NvdW50Pzogc3RyaW5nOyBidWNrZXROYW1lPzogc3RyaW5nOyBwYXRoPzogc3RyaW5nIH0gfCB1bmRlZmluZWQ7XG4gICAgICAgIGlmIChjb25maWcgJiYgdHlwZW9mIGNvbmZpZyA9PT0gXCJvYmplY3RcIikge1xuICAgICAgICAgICAgaWYgKFwiYWNjb3VudFwiIGluIGNvbmZpZykgYXNzZXJ0VmFsaWROYW1lKGNvbmZpZy5hY2NvdW50IGFzIHN0cmluZywgXCJhY2NvdW50XCIpO1xuICAgICAgICAgICAgaWYgKFwiYnVja2V0TmFtZVwiIGluIGNvbmZpZykgYXNzZXJ0VmFsaWROYW1lKGNvbmZpZy5idWNrZXROYW1lIGFzIHN0cmluZywgXCJidWNrZXQgbmFtZVwiKTtcbiAgICAgICAgICAgIGlmIChcInBhdGhcIiBpbiBjb25maWcpIGFzc2VydFZhbGlkUGF0aChjb25maWcucGF0aCBhcyBzdHJpbmcpO1xuICAgICAgICB9XG4gICAgICAgIHJldHVybiBvcmlnaW5hbC5hcHBseSh0aGlzLCBhcmdzKTtcbiAgICB9O1xufVxuIl19
|
|
50
|
-
/* _JS_SOURCE_HASH = "
|
|
50
|
+
/* _JS_SOURCE_HASH = "3c47af61d817d76651ac4b91cc2d83a30a45619b8072517ce152b34b6075b1d8"; */
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { lazy } from "socket-function/src/caching";
|
|
2
|
+
import { formatDateTimeDetailed } from "socket-function/src/formatting/format";
|
|
2
3
|
import { runInfinitePoll } from "socket-function/src/batching";
|
|
3
4
|
import { RemoteConfig, SourceConfig, HostedConfig } from "../IArchives";
|
|
4
5
|
import { ROUTING_FILE, getConfigVersion, serializeRemoteConfig, parseRoutingData, parseHostedUrl, replaceHostedUrlPort } from "./remoteConfig";
|
|
@@ -78,7 +79,7 @@ export function scheduleBoundaryWork(account: string, bucketName: string, routin
|
|
|
78
79
|
/** Collects the writes that landed on the previous window's owners just before a boundary, into the store that has just taken over. Who owned what is worked out purely from the config (see previousWindowOwners); this only does the pulling. The config is read off the disk at FIRE time, not captured when the scan was armed - it may have changed in between, and the newest copy is the one the handover actually happened under. */
|
|
79
80
|
async function runBoundaryScan(account: string, bucketName: string, windowStart: number, offset: number, offsetLabel?: string): Promise<void> {
|
|
80
81
|
let key = `${account}/${bucketName}`;
|
|
81
|
-
let label = `bucket ${key}, window starting ${
|
|
82
|
+
let label = `bucket ${key}, window starting ${formatDateTimeDetailed(windowStart)}, offset ${offsetLabel || `${offset / 1000}s`}`;
|
|
82
83
|
if (boundaryScansRunning.has(key)) {
|
|
83
84
|
console.log(`Skipping boundary scan (${label}): the previous boundary scan is still running`);
|
|
84
85
|
return;
|
|
@@ -220,7 +221,7 @@ async function maintainIntermediates(): Promise<void> {
|
|
|
220
221
|
alreadyCorrect.push(key);
|
|
221
222
|
continue;
|
|
222
223
|
}
|
|
223
|
-
await writeOwnRoutingConfig({ account, bucketName, storeName: newest.name, current: routing, updated: injected, reason: `we are a deploy successor: writes route to our alternate port ${takeover.altPort} from ${
|
|
224
|
+
await writeOwnRoutingConfig({ account, bucketName, storeName: newest.name, current: routing, updated: injected, reason: `we are a deploy successor: writes route to our alternate port ${takeover.altPort} from ${formatDateTimeDetailed(takeover.start)} until our predecessor is killed at ${formatDateTimeDetailed(takeover.end)}` });
|
|
224
225
|
written.push(key);
|
|
225
226
|
}
|
|
226
227
|
if (!takeover) return;
|
|
@@ -253,7 +253,7 @@ export function assertValidRemoteConfig(config: RemoteConfig): void {
|
|
|
253
253
|
let b = sources[j];
|
|
254
254
|
if (storeIdentity(a) !== storeIdentity(b)) continue;
|
|
255
255
|
if (a.validWindow[0] < b.validWindow[1] && b.validWindow[0] < a.validWindow[1]) {
|
|
256
|
-
throw new Error(`The name ${JSON.stringify(a.name)} is
|
|
256
|
+
throw new Error(`Store name reused with conflicting configurations. The name ${JSON.stringify(a.name)} is used by multiple sources at the same time:\n ${JSON.stringify(a)}\n ${JSON.stringify(b)}\nA name is one store - one folder, one index - so it can only be configured one way at a time. Give them non-overlapping valid windows, or different names if they are meant to be different storage.`);
|
|
257
257
|
}
|
|
258
258
|
}
|
|
259
259
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { isNode, sort } from "socket-function/src/misc";
|
|
2
|
+
import { formatDateTimeDetailed } from "socket-function/src/formatting/format";
|
|
2
3
|
import { delay } from "socket-function/src/batching";
|
|
3
4
|
import { getSecret } from "../../misc/getSecret";
|
|
4
5
|
import { IArchives, SourceConfig, RemoteConfig } from "../IArchives";
|
|
@@ -92,8 +93,8 @@ export class SourceWrapper {
|
|
|
92
93
|
}
|
|
93
94
|
let [start, end] = this.config.validWindow;
|
|
94
95
|
if (start !== 0 || end !== Number.MAX_SAFE_INTEGER) {
|
|
95
|
-
let startText = start === 0 && "0" ||
|
|
96
|
-
let endText = end === Number.MAX_SAFE_INTEGER && "forever" ||
|
|
96
|
+
let startText = start === 0 && "0" || formatDateTimeDetailed(start);
|
|
97
|
+
let endText = end === Number.MAX_SAFE_INTEGER && "forever" || formatDateTimeDetailed(end);
|
|
97
98
|
parts.push(`validWindow [${startText}, ${endText}]`);
|
|
98
99
|
}
|
|
99
100
|
return `source ${this.config.url}${parts.length && ` (${parts.join(", ")})` || ""}`;
|
|
@@ -121,7 +122,7 @@ export class SourceWrapper {
|
|
|
121
122
|
if (this.disposed || this.isConnected()) return;
|
|
122
123
|
let now = Date.now();
|
|
123
124
|
if (now >= this.cooldownUntil && this.isConnectionProblemWorthReporting()) {
|
|
124
|
-
console.error(`Cannot connect to storage ${this.getDebugName()}; skipping it until ${
|
|
125
|
+
console.error(`Cannot connect to storage ${this.getDebugName()}; skipping it until ${formatDateTimeDetailed(now + SOURCE_FAILURE_COOLDOWN)} (now ${formatDateTimeDetailed(now)}) unless no other source can serve the request`);
|
|
125
126
|
// In a browser this is often an untrusted (self-signed) certificate the user hasn't accepted yet - offer them the trust flow (a no-op in Node, and shown at most once per server)
|
|
126
127
|
showCertTrustModal(this.config.url);
|
|
127
128
|
}
|
|
@@ -186,7 +187,7 @@ export class SourceWrapper {
|
|
|
186
187
|
// No public URL form - let the API call throw its own error (which includes the access instructions for hosted sources)
|
|
187
188
|
return await run(this.api);
|
|
188
189
|
}
|
|
189
|
-
throw new Error(`Source
|
|
190
|
+
throw new Error(`Source has no API access and no public URL form (public: false): ${this.config.url}`);
|
|
190
191
|
}
|
|
191
192
|
|
|
192
193
|
public async readRoutingConfig(): Promise<RemoteConfig | undefined> {
|
|
@@ -271,11 +272,11 @@ export class SourceWrapper {
|
|
|
271
272
|
/** Writes always go through the API, so a permission error throws to the caller on every write (and access granted in the meantime is picked up automatically). */
|
|
272
273
|
public async write<T>(run: (archives: IArchives) => Promise<T>): Promise<T> {
|
|
273
274
|
if (this.writeBlocked) {
|
|
274
|
-
throw new Error(this.writeBlocked);
|
|
275
|
+
throw new Error(`Writes are blocked on this source: ${this.writeBlocked}`);
|
|
275
276
|
}
|
|
276
277
|
let api = this.api;
|
|
277
278
|
if (!api) {
|
|
278
|
-
throw new Error(`Source
|
|
279
|
+
throw new Error(`Source has no API access, so it cannot accept writes: ${this.config.url}`);
|
|
279
280
|
}
|
|
280
281
|
let result = await run(api);
|
|
281
282
|
this.accessCache = { hasAccess: true, time: Date.now() };
|
|
@@ -122,7 +122,7 @@ async function requireAccess(account: string): Promise<string> {
|
|
|
122
122
|
let trusted = await trust.get(`${account}|${machineId}`);
|
|
123
123
|
if (!trusted) {
|
|
124
124
|
let { domain, port } = getStorageServerConfig();
|
|
125
|
-
throw new Error(`${STORAGE_ACCESS_DENIED} Machine
|
|
125
|
+
throw new Error(`${STORAGE_ACCESS_DENIED} Machine has no access to this account. Machine ${machineId}, account ${JSON.stringify(account)}. Visit https://${domain}:${port}/${account} for access instructions.`);
|
|
126
126
|
}
|
|
127
127
|
return machineId;
|
|
128
128
|
}
|
|
@@ -155,7 +155,7 @@ class RemoteStorageControllerBase {
|
|
|
155
155
|
}
|
|
156
156
|
let machineId = getCommonName(token.certPem).split(".")[0];
|
|
157
157
|
if (!verifyMachineIdForPublicKey({ machineId, publicKey: getPublicIdentifier(token.certPem) })) {
|
|
158
|
-
throw new Error(`Certificate common name ${JSON.stringify(getCommonName(token.certPem))}
|
|
158
|
+
throw new Error(`Certificate common name does not match its public key: ${JSON.stringify(getCommonName(token.certPem))}`);
|
|
159
159
|
}
|
|
160
160
|
sessions.set(caller.nodeId, machineId);
|
|
161
161
|
while (sessions.size > MAX_SESSIONS) {
|
|
@@ -232,7 +232,7 @@ class RemoteStorageControllerBase {
|
|
|
232
232
|
for (let request of await requests.get(ip) || []) {
|
|
233
233
|
if (request.requestId !== requestId) continue;
|
|
234
234
|
if (!isAdmin(callerMachineId) && !await trust.get(`${request.account}|${callerMachineId}`)) {
|
|
235
|
-
throw new Error(`${STORAGE_ACCESS_DENIED} Machine
|
|
235
|
+
throw new Error(`${STORAGE_ACCESS_DENIED} Machine has no access to this account. Machine ${callerMachineId}, account ${JSON.stringify(request.account)}`);
|
|
236
236
|
}
|
|
237
237
|
let record: TrustRecord = {
|
|
238
238
|
account: request.account,
|
|
@@ -468,7 +468,7 @@ class RemoteStorageControllerBase {
|
|
|
468
468
|
await Promise.all(bucketStores.map(x => x.store.init()));
|
|
469
469
|
let policies = bucketStores.map(x => ({ store: x.store, policy: x.store.storeConfig.current() }));
|
|
470
470
|
if (!policies.some(x => x.policy.public)) {
|
|
471
|
-
throw new Error(`Bucket
|
|
471
|
+
throw new Error(`Bucket is not public, so its files cannot be read over plain URLs: ${account}/${bucketName}`);
|
|
472
472
|
}
|
|
473
473
|
// Anonymous URL reads carry no source selection - the file's route picks the store, and a route no store here covers is simply not found (the routing file itself lives outside every store)
|
|
474
474
|
let httpStore: BlobStore | undefined;
|
|
@@ -5,6 +5,7 @@ import path from "path";
|
|
|
5
5
|
import fsp from "fs/promises";
|
|
6
6
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
7
7
|
import { getExternalIP } from "socket-function/src/networking";
|
|
8
|
+
import { formatDateTimeDetailed } from "socket-function/src/formatting/format";
|
|
8
9
|
import { RequireController } from "socket-function/require/RequireController";
|
|
9
10
|
import { hostServer } from "../../misc/https/hostServer";
|
|
10
11
|
import { RemoteStorageController } from "./storageController";
|
|
@@ -144,7 +145,7 @@ export async function hostStorageServer(config: HostStorageServerConfig): Promis
|
|
|
144
145
|
setAltPort(listeningPort);
|
|
145
146
|
let listenEnd = getAltPortListenEnd();
|
|
146
147
|
let timer = setTimeout(() => {
|
|
147
|
-
console.log(`Alternate port ${listeningPort} reached its end of life at ${
|
|
148
|
+
console.log(`Alternate port ${listeningPort} reached its end of life at ${formatDateTimeDetailed(listenEnd)}; it no longer counts as one of our addresses (the socket itself stays bound, as the main-port relay forwards into it)`);
|
|
148
149
|
removeExtraListenPort(listeningPort);
|
|
149
150
|
}, Math.max(0, listenEnd - Date.now()));
|
|
150
151
|
(timer as { unref?: () => void }).unref?.();
|
|
@@ -90,7 +90,7 @@ export async function getBucketStores(account: string, bucketName: string): Prom
|
|
|
90
90
|
async function requireBucketStores(account: string, bucketName: string): Promise<{ name: string; store: BlobStore }[]> {
|
|
91
91
|
let list = await getBucketStores(account, bucketName);
|
|
92
92
|
if (!list.length) {
|
|
93
|
-
throw new Error(`Bucket
|
|
93
|
+
throw new Error(`Bucket does not exist on this server: ${account}/${bucketName}. Write its routing config to ${JSON.stringify(ROUTING_FILE)} to create it.`);
|
|
94
94
|
}
|
|
95
95
|
return list;
|
|
96
96
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { runInfinitePoll, delay, runInfinitePollCallAtStart, runInSerial } from "socket-function/src/batching";
|
|
2
2
|
import { cache } from "socket-function/src/caching";
|
|
3
3
|
import { timeInMinute, sort, promiseObj } from "socket-function/src/misc";
|
|
4
|
-
import { formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
4
|
+
import { formatNumber, formatTime, formatDateTimeDetailed } from "socket-function/src/formatting/format";
|
|
5
5
|
import {
|
|
6
6
|
IArchives, ArchiveFileInfo, ArchivesSyncStatus, SyncActivity, FULL_ROUTE,
|
|
7
7
|
windowsAcceptWrites,
|
|
@@ -263,12 +263,12 @@ export class StoreSync {
|
|
|
263
263
|
// Undefined is two cases (see copyArchiveFile): our own disk already holding something newer is the boundary scan working as intended (our writes since the boundary outrank the neighbor's), only a genuinely unreadable file is worth a warning
|
|
264
264
|
let local = await this.store.sources[0].source.getInfo(file.path);
|
|
265
265
|
if (local && local.writeTime > file.createTime) {
|
|
266
|
-
logSyncEvent({ event: "boundaryScanLocalNewer", store: this.store.folder, source: source.getDebugName(), path: file.path, theirWriteTime:
|
|
266
|
+
logSyncEvent({ event: "boundaryScanLocalNewer", store: this.store.folder, source: source.getDebugName(), path: file.path, theirWriteTime: formatDateTimeDetailed(file.createTime), theirSize: file.size, localWriteTime: formatDateTimeDetailed(local.writeTime), localSize: local.size });
|
|
267
267
|
this.store.setIndexEntry(file.path, { writeTime: local.writeTime, size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
|
|
268
268
|
tally.unchanged++;
|
|
269
269
|
continue;
|
|
270
270
|
}
|
|
271
|
-
console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${
|
|
271
|
+
console.warn(`Boundary scan could not copy ${file.path} from ${source.getDebugName()} (store ${this.store.folder}): its change feed listed it (${file.size} bytes, writeTime ${formatDateTimeDetailed(file.createTime)}) but the read found nothing`);
|
|
272
272
|
continue;
|
|
273
273
|
}
|
|
274
274
|
this.store.noteSyncTransfer("sync get", file.path, copied.size);
|
|
@@ -321,7 +321,7 @@ export class StoreSync {
|
|
|
321
321
|
state.initialScan.resolve(undefined);
|
|
322
322
|
if (!loggedStale) {
|
|
323
323
|
loggedStale = true;
|
|
324
|
-
logSyncEvent({ event: "scanSkippedStaleWindows", store: this.store.folder, source: source.getDebugName(), validWindows: this.store.sources[sourceIndex].validWindows.map(w => [
|
|
324
|
+
logSyncEvent({ event: "scanSkippedStaleWindows", store: this.store.folder, source: source.getDebugName(), validWindows: this.store.sources[sourceIndex].validWindows.map(w => [formatDateTimeDetailed(w[0]), formatDateTimeDetailed(w[1])]), pastReadDurationMs: PAST_READ_DURATION });
|
|
325
325
|
}
|
|
326
326
|
return true;
|
|
327
327
|
};
|
|
@@ -500,10 +500,10 @@ export class StoreSync {
|
|
|
500
500
|
let theirs = await source.getInfo(key);
|
|
501
501
|
if (theirs && theirs.writeTime > writeTime) {
|
|
502
502
|
this.store.setIndexEntry(key, { writeTime: theirs.writeTime, size: theirs.size, sourcesListIndex: targetSourcesListIndex });
|
|
503
|
-
logSyncEvent({ event: "pushFoundNewerOnSource", store: this.store.folder, source: source.getDebugName(), path: key, ourWriteTime:
|
|
503
|
+
logSyncEvent({ event: "pushFoundNewerOnSource", store: this.store.folder, source: source.getDebugName(), path: key, ourWriteTime: formatDateTimeDetailed(writeTime), theirWriteTime: formatDateTimeDetailed(theirs.writeTime), ourSize: entry.size, theirSize: theirs.size });
|
|
504
504
|
continue;
|
|
505
505
|
}
|
|
506
|
-
logSyncEvent({ event: "pushCopyUnavailable", store: this.store.folder, source: source.getDebugName(), path: key, holder: holder.getDebugName(), expectedSize: entry.size, expectedWriteTime:
|
|
506
|
+
logSyncEvent({ event: "pushCopyUnavailable", store: this.store.folder, source: source.getDebugName(), path: key, holder: holder.getDebugName(), expectedSize: entry.size, expectedWriteTime: formatDateTimeDetailed(writeTime), sourceHas: theirs && `${theirs.size} bytes at ${formatDateTimeDetailed(theirs.writeTime)}` || "nothing" });
|
|
507
507
|
continue;
|
|
508
508
|
}
|
|
509
509
|
this.store.noteSyncTransfer("sync set", key, copied.size);
|
|
@@ -611,7 +611,7 @@ export class StoreSync {
|
|
|
611
611
|
let etaText = "unknown (no bytes transferred yet)";
|
|
612
612
|
if (bytesPerSecond > 0) {
|
|
613
613
|
let remainingMs = remainingBytes / bytesPerSecond * 1000;
|
|
614
|
-
etaText = `${formatTime(remainingMs)} remaining, completing around ${
|
|
614
|
+
etaText = `${formatTime(remainingMs)} remaining, completing around ${formatDateTimeDetailed(Date.now() + remainingMs)}`;
|
|
615
615
|
}
|
|
616
616
|
console.warn(`Delta sync from ${source.getDebugName()} (store ${this.store.folder}) has been running for ${formatTime(elapsed)}: ${doneFiles}/${pending.length} files (${(doneFiles / pending.length * 100).toFixed(1)}%), ${formatNumber(doneBytes)}B/${formatNumber(totalBytes)}B (${(totalBytes && doneBytes / totalBytes * 100 || 100).toFixed(1)}%), ${formatNumber(bytesPerSecond)}B/s. Estimated ${etaText}.`);
|
|
617
617
|
}, FULL_SYNC_SLOW_ERROR_INTERVAL);
|
|
@@ -628,7 +628,7 @@ export class StoreSync {
|
|
|
628
628
|
if (local) {
|
|
629
629
|
// The local copy can be NEWER than the entry (which is exactly why copyArchiveFile refuses to overwrite it - see its destination check), so the repoint keeps the newer of the two times rather than rolling the index back
|
|
630
630
|
this.store.setIndexEntry(key, { writeTime: Math.max(entry.writeTime, local.writeTime), size: local.size, sourcesListIndex: this.store.sourcesListIndexOfSlot(0) });
|
|
631
|
-
logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size, localWriteTime:
|
|
631
|
+
logSyncEvent({ ...base, resolution: "repointed to our local copy", localSize: local.size, localWriteTime: formatDateTimeDetailed(local.writeTime) });
|
|
632
632
|
return;
|
|
633
633
|
}
|
|
634
634
|
if (this.store.currentWriteTime(key) <= entry.writeTime) {
|