sliftutils 1.7.11 → 1.7.13

package/storage/remoteStorage/createArchives.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { IArchives, RemoteConfig, RemoteConfigBase, HostedConfig, BackblazeConfig, ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { ServerBucketInfo } from "./storageServerState";
 export declare function createApiArchives(source: HostedConfig | BackblazeConfig): IArchives;
 export declare class ArchivesChain implements IArchives {
     private configured;
@@ -83,3 +84,18 @@ export declare class ArchivesChain implements IArchives {
     dispose(): void;
 }
 export declare function createArchives(config: RemoteConfig | RemoteConfigBase): ArchivesChain;
+/** Every bucket an account has on one storage server - active and inactive - with each bucket's
+ *  configuration. One authenticated call (the normal trust system applies): no ArchivesChain, no
+ *  synchronization, and inactive buckets on the server stay inactive. Any URL addressing the
+ *  server works (a bucket routing URL, or just https://host:port). */
+export declare function listServerBuckets(config: {
+    url: string;
+    account: string;
+}): Promise<ServerBucketInfo[]>;
+/** Live info for one bucket given its routing URL (getConfig: routing config, index totals, disk
+ *  limit, in-progress synchronization). One authenticated call to that server - a light, safe
+ *  alternative to instantiating an ArchivesChain, which would start synchronization machinery. */
+export declare function getBucketInfo(config: {
+    url: string;
+    accountName?: string;
+}): Promise<ArchivesConfig>;

package/storage/remoteStorage/createArchives.ts

@@ -11,9 +11,11 @@ import {
     normalizeRemoteConfig, normalizeSource, parseRoutingData, serializeRemoteConfig,
     getRoute, routeContains, parseVariableRoute,
 } from "./remoteConfig";
-import { ArchivesRemote } from "./ArchivesRemote";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { ArchivesRemote, parseStorageUrl, authenticateStorage } from "./ArchivesRemote";
 import { ArchivesBackblaze } from "../backblaze";
-import { getStorageServerConfigOptional, getLocalArchives } from "./storageServerState";
+import { getStorageServerConfigOptional, getLocalArchives, ServerBucketInfo } from "./storageServerState";
+import { RemoteStorageController, STORAGE_NOT_AUTHENTICATED } from "./storageController";
 import { SourceWrapper, RETRY_START_DELAY, RETRY_MAX_DELAY, RETRY_GROWTH } from "./sourceWrapper";
 
 // Turns a RemoteConfig into a usable IArchives (createArchives). Initialization is lazy: on the
@@ -610,3 +612,29 @@ export class ArchivesChain implements IArchives {
 export function createArchives(config: RemoteConfig | RemoteConfigBase): ArchivesChain {
     return new ArchivesChain(config);
 }
+
+/** Every bucket an account has on one storage server - active and inactive - with each bucket's
+ *  configuration. One authenticated call (the normal trust system applies): no ArchivesChain, no
+ *  synchronization, and inactive buckets on the server stay inactive. Any URL addressing the
+ *  server works (a bucket routing URL, or just https://host:port). */
+export async function listServerBuckets(config: { url: string; account: string }): Promise<ServerBucketInfo[]> {
+    SocketFunction.ENABLE_CLIENT_MODE = true;
+    let parsed = parseStorageUrl(config.url);
+    let nodeId = SocketFunction.connect({ address: parsed.address, port: parsed.port });
+    let controller = RemoteStorageController.nodes[nodeId];
+    try {
+        return await controller.listBuckets(config.account);
+    } catch (e) {
+        if (!String((e as Error).stack ?? e).includes(STORAGE_NOT_AUTHENTICATED)) throw e;
+        await authenticateStorage({ address: parsed.address, port: parsed.port, nodeId });
+        return await controller.listBuckets(config.account);
+    }
+}
+
+/** Live info for one bucket given its routing URL (getConfig: routing config, index totals, disk
+ *  limit, in-progress synchronization). One authenticated call to that server - a light, safe
+ *  alternative to instantiating an ArchivesChain, which would start synchronization machinery. */
+export async function getBucketInfo(config: { url: string; accountName?: string }): Promise<ArchivesConfig> {
+    let remote = new ArchivesRemote({ url: config.url, accountName: config.accountName, waitForAccess: false });
+    return await remote.getConfig();
+}

package/storage/remoteStorage/sourceWrapper.ts

@@ -165,7 +165,7 @@ export class SourceWrapper {
     /** Starts measuring this source's latency (for variable-shard target preference). Only hosted
      *  remotes are pinged; our own local server counts as 0, everything else as Infinity. */
     public startPinging(): void {
-        let remote = this.remote;
+        const remote = this.remote;
         if (!remote || this.pingTimer || this.disposed) return;
         let measure = async () => {
             let start = Date.now();

package/storage/remoteStorage/storageController.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { ArchiveFileInfo, ArchivesConfig, ArchivesSyncStatus } from "../IArchives";
+import { ServerBucketInfo } from "./storageServerState";
 export declare const REMOTE_STORAGE_CLASS_GUID = "RemoteStorageController-b7e42a91";
 export declare const STORAGE_AUTH_PURPOSE = "remoteStorage-auth-1";
 export declare const STORAGE_NOT_AUTHENTICATED = "REMOTE_STORAGE_NOT_AUTHENTICATED_cf2f7b1e";
@@ -71,6 +72,7 @@ export declare const RemoteStorageController: import("socket-function/SocketFunc
     }) => Promise<ArchiveFileInfo[]>;
     getChangesAfter: (account: string, bucketName: string, time: number) => Promise<ArchiveFileInfo[]>;
     getArchivesConfig: (account: string, bucketName: string) => Promise<ArchivesConfig>;
+    listBuckets: (account: string) => Promise<ServerBucketInfo[]>;
     getIndexInfo: (account: string, bucketName: string) => Promise<{
         fileCount: number;
         byteCount: number;

package/storage/remoteStorage/storageController.ts

@@ -13,6 +13,7 @@ import { ROUTING_FILE } from "./remoteConfig";
 import {
     getStorageServerConfig, getTrust, getRequests, getLoadedBucket, writeBucketFile,
     deleteBucketFile, assertWritesAllowed, assertMutable, LoadedBucket,
+    getBucketConfig, listAccountBuckets, ServerBucketInfo,
 } from "./storageServerState";
 
 // The remote storage server's API. Authentication uses certs.ts machine identities: a client
@@ -332,15 +333,13 @@ class RemoteStorageControllerBase {
     async getArchivesConfig(account: string, bucketName: string): Promise<ArchivesConfig> {
         let bucket = await getBucket(account, bucketName);
         // Missing buckets say true, matching what they become once created (the default store type)
-        let progress = bucket?.store.getSyncProgress?.();
-        return {
-            supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter,
-            remoteConfig: bucket?.routing,
-            index: progress?.index,
-            indexSources: progress?.sources,
-            readerDiskLimit: progress?.readerDiskLimit,
-            syncing: progress?.syncing,
-        };
+        if (!bucket) return { supportsChangesAfter: true };
+        return getBucketConfig(bucket);
+    }
+    /** Every bucket the account has on this host - active and inactive - with each bucket's
+     *  configuration. Inactive buckets stay inactive (their routing is read straight off disk). */
+    async listBuckets(account: string): Promise<ServerBucketInfo[]> {
+        return await listAccountBuckets(account);
     }
     /** Walks the whole index for exact totals (overall and per holding source) - more expensive
      *  than the maintained counters that getArchivesConfig returns, but immune to counter drift. */
@@ -522,6 +521,7 @@ export const RemoteStorageController = SocketFunction.register(
         getChangesAfter: { hooks: [accountAccess] },
         getArchivesConfig: { hooks: [accountAccess] },
         getIndexInfo: { hooks: [accountAccess] },
+        listBuckets: { hooks: [accountAccess] },
         getSyncStatus: { hooks: [accountAccess] },
         startLargeFile: { hooks: [accountAccess] },
         uploadPart: { hooks: [uploadAccess] },

package/storage/remoteStorage/storageServerState.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { IBucketStore } from "./blobStore";
-import { RemoteConfig, HostedConfig, IArchives } from "../IArchives";
+import { RemoteConfig, HostedConfig, IArchives, ArchivesConfig } from "../IArchives";
 import type { IStorage } from "../IStorage";
 import type { AccessRequest, TrustRecord } from "./storageController";
 export type StorageServerConfig = {
@@ -39,5 +39,17 @@ export declare function assertMutable(bucket: LoadedBucket, filePath: string, wr
 export declare function writeBucketFile(account: string, bucketName: string, filePath: string, data: Buffer, config?: {
     lastModified?: number;
 }): Promise<void>;
+export declare function getBucketConfig(bucket: LoadedBucket): ArchivesConfig;
+export type ServerBucketInfo = {
+    bucketName: string;
+    active: boolean;
+    config?: ArchivesConfig;
+    error?: string;
+};
+/** Every bucket the account has on this server, active or not, each with its configuration.
+ *  Inactive buckets are inspected straight from disk WITHOUT loading them - loading would start
+ *  their synchronization, and old invalid buckets must stay inert (their parse error is reported
+ *  instead). */
+export declare function listAccountBuckets(account: string): Promise<ServerBucketInfo[]>;
 export declare function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void>;
 export declare function getLocalArchives(account: string, bucketName: string): IArchives;

package/storage/remoteStorage/storageServerState.ts

@@ -1,4 +1,5 @@
 import path from "path";
+import fs from "fs";
 import { getFileStorageNested2 } from "../FileFolderAPI";
 import { TransactionStorage } from "../TransactionStorage";
 import { JSONStorage } from "../JSONStorage";
@@ -416,6 +417,67 @@ export async function writeBucketFile(account: string, bucketName: string, fileP
     await loaded.store.set(filePath, data, { ...getWriteConfig(loaded, writeTime, route), lastModified: writeTime });
 }
 
+export function getBucketConfig(bucket: LoadedBucket): ArchivesConfig {
+    let progress = bucket.store.getSyncProgress?.();
+    return {
+        supportsChangesAfter: !!bucket.store.getChangesAfter,
+        remoteConfig: bucket.routing,
+        index: progress?.index,
+        indexSources: progress?.sources,
+        readerDiskLimit: progress?.readerDiskLimit,
+        syncing: progress?.syncing,
+    };
+}
+
+export type ServerBucketInfo = {
+    bucketName: string;
+    // Loaded in memory (created or accessed since startup), so its synchronization is running
+    active: boolean;
+    config?: ArchivesConfig;
+    error?: string;
+};
+
+/** Every bucket the account has on this server, active or not, each with its configuration.
+ *  Inactive buckets are inspected straight from disk WITHOUT loading them - loading would start
+ *  their synchronization, and old invalid buckets must stay inert (their parse error is reported
+ *  instead). */
+export async function listAccountBuckets(account: string): Promise<ServerBucketInfo[]> {
+    let accountFolder = path.join(getStorageServerConfig().folder, "buckets2", account);
+    let names: string[];
+    try {
+        names = await fs.promises.readdir(accountFolder);
+    } catch {
+        return [];
+    }
+    let results: ServerBucketInfo[] = [];
+    for (let bucketName of names) {
+        let loadedPromise = buckets.get(`${account}/${bucketName}`);
+        if (loadedPromise) {
+            try {
+                let loaded = await loadedPromise;
+                if (loaded) {
+                    results.push({ bucketName, active: true, config: getBucketConfig(loaded) });
+                    continue;
+                }
+            } catch (e) {
+                results.push({ bucketName, active: true, error: String((e as Error).stack ?? e).slice(0, 500) });
+                continue;
+            }
+        }
+        try {
+            let data = await new ArchivesDisk(getBucketFolder(account, bucketName)).get(ROUTING_FILE);
+            if (!data) {
+                results.push({ bucketName, active: false, error: `No routing file (${ROUTING_FILE})` });
+                continue;
+            }
+            results.push({ bucketName, active: false, config: { remoteConfig: parseRoutingData(data) } });
+        } catch (e) {
+            results.push({ bucketName, active: false, error: String((e as Error).stack ?? e).slice(0, 500) });
+        }
+    }
+    return results;
+}
+
 export async function deleteBucketFile(account: string, bucketName: string, filePath: string): Promise<void> {
     if (filePath === ROUTING_FILE) {
         throw new Error(`The routing config ${JSON.stringify(ROUTING_FILE)} cannot be deleted (overwrite it to change the bucket's configuration)`);
@@ -498,15 +560,8 @@ class ArchivesLocalBucket implements IArchives {
     public async getConfig(): Promise<ArchivesConfig> {
         let bucket = await this.getBucket();
         // Missing buckets say true, matching what they become once created (the default store type)
-        let progress = bucket?.store.getSyncProgress?.();
-        return {
-            supportsChangesAfter: !bucket || !!bucket.store.getChangesAfter,
-            remoteConfig: bucket?.routing,
-            index: progress?.index,
-            indexSources: progress?.sources,
-            readerDiskLimit: progress?.readerDiskLimit,
-            syncing: progress?.syncing,
-        };
+        if (!bucket) return { supportsChangesAfter: true };
+        return getBucketConfig(bucket);
     }
     public async hasWriteAccess(): Promise<boolean> {
         return true;

package/yarn.lock

@@ -1944,10 +1944,10 @@ slash@^3.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
   integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
 
-socket-function@^1.2.16:
-  version "1.2.16"
-  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.16.tgz#6ad7fc1971202d9768582b0212f086675fba61b5"
-  integrity sha512-o3GATr5Fa+496StsPWJIb0yytY/XRMZjJQ4cgb1LOQIfFv7uJUJ3NZtcldMeN/kOmqI0hBaMvhRLC3VHke9awA==
+socket-function@^1.2.17:
+  version "1.2.17"
+  resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.2.17.tgz#d6f28be00253d4a0a73dab6f0981c9af8e750a49"
+  integrity sha512-HGQ8YmGwS5xLG03v9koa8l4PIPVIDmi177a/sdfa6KSGRkaxTWwinfakun4rddZNnlxb7Iu6NFSUl0TfTyv7/w==
   dependencies:
     "@types/pako" "^2.0.3"
     "@types/ws" "^8.5.3"