sliftutils 1.7.89 → 1.7.90

package/storage/remoteStorage/storageServerState.d.ts

@@ -31,8 +31,8 @@ export type BucketState = {
 };
 /** The loaded bucket, loading it (which instantiates its stores and starts their synchronization) if needed. A bucket that does not exist on this server throws - callers never see undefined buckets. */
 export declare function requireBucket(account: string, bucketName: string): Promise<BucketState>;
-/** The store serving a request: the one the client's selected entry NAMES. Nothing else is compared - not the window, not the route, not the flags - because the name is what says which storage this is, and everything else is policy the store itself applies to the request. A client a config version behind on some flag therefore still reaches the right store, which is the whole point of naming it. */
-export declare function findBucketStore(account: string, bucketName: string, sourceConfig: SourceConfig | undefined): Promise<LoadedStore>;
+/** The store serving a request: the one the client's selected entry NAMES. Account, name, and bucket ARE the folder, so this is a direct lookup, never a search of what exists - and a name this server has never seen is CREATED, never rejected, because asking for a name is the instruction to have that store (one name, one folder, one index; it configures itself once the routing config lands in it). Nothing else about the request is compared - not the window, not the route, not the flags - which is the whole point of naming it: a client a config version behind on some flag still reaches the right store. */
+export declare function findBucketStore(account: string, bucketName: string, sourceConfig: SourceConfig | undefined): BlobStore;
 /** Internal (store-to-store) reads skip store selection entirely: the caller is another store whose index says this MACHINE holds the bytes - the persisted holder identity is just a URL, which cannot name a store. Whichever store's folder has the newest copy answers. */
 export declare function readBucketInternal(account: string, bucketName: string, config: {
     path: string;
@@ -48,7 +48,7 @@ export declare function readBucketInternal(account: string, bucketName: string,
 } | undefined>;
 export declare function getBucketConfig(bucket: BucketState): ArchivesConfig;
 export declare function bucketSyncStatus(bucket: BucketState): Promise<ArchivesSyncStatus>;
-export declare function bucketIndexTotals(bucket: BucketState): Promise<{
+export declare function debugBucketIndexTotals(bucket: BucketState): Promise<{
     fileCount: number;
     byteCount: number;
     sources: {
@@ -104,10 +104,10 @@ export type ActiveBucketInfo = {
     config: ArchivesConfig;
 };
 /** The live in-memory state of ONE bucket, answered without touching the disk (no routing file read, no statfs, no stored write stats). Returns an error string when the bucket is not loaded here, which is the normal state for a bucket nothing has accessed since startup. */
-export declare function getActiveBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string>;
+export declare function debugGetActiveBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string>;
 /** Loads a bucket that exists on this server's disk into memory, which starts its synchronization and window timers, and returns its live state. Nothing is written and no other server is contacted - unlike building an ArchivesChain for it, which would probe every source and could write the routing config. Already-loaded buckets just return their state. */
 export declare function activateBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string>;
-export declare function listAccountBuckets(account: string): Promise<ServerBucketInfo[]>;
+export declare function debugListAccountBuckets(account: string): Promise<ServerBucketInfo[]>;
 export type BucketWriteStats = {
     /** Every set call the bucket accepted */
     originalWrites: number;
@@ -117,4 +117,4 @@ export type BucketWriteStats = {
     flushedBytes: number;
 };
 /** Zeroes the write statistics of every bucket in the account. */
-export declare function clearAccountWriteStats(account: string): number;
+export declare function debugClearAccountWriteStats(account: string): number;

package/storage/remoteStorage/storageServerState.ts

@@ -8,6 +8,7 @@ import { ROUTING_FILE, parseRoutingData, serializeRemoteConfig, normalizeSource
 import { createStoreSource, applySourceConfig } from "./storeSources";
 import { broadcastRoutingChanged } from "./storageController";
 import { getStorageServerConfig, assertWritesAllowed } from "./serverConfig";
+import { assertValidSourceName } from "./validation";
 import { getBucketFolder, listAccountStoreFolders, listBucketStoreFolders, readRoutingFromDisk, getDiskInfo, BucketDiskInfo } from "./bucketDisk";
 import { scheduleBoundaryWork, reinjectIntermediates } from "./intermediateManagement";
 import { planSelfStores, isSelfSource, SelfSummary } from "./storePlan";
@@ -53,23 +54,14 @@ export async function requireBucket(account: string, bucketName: string): Promis
     return bucket;
 }
 
-/** The store serving a request: the one the client's selected entry NAMES. Nothing else is compared - not the window, not the route, not the flags - because the name is what says which storage this is, and everything else is policy the store itself applies to the request. A client a config version behind on some flag therefore still reaches the right store, which is the whole point of naming it. */
-export async function findBucketStore(account: string, bucketName: string, sourceConfig: SourceConfig | undefined): Promise<LoadedStore> {
-    let bucket = await requireBucket(account, bucketName);
+/** The store serving a request: the one the client's selected entry NAMES. Account, name, and bucket ARE the folder, so this is a direct lookup, never a search of what exists - and a name this server has never seen is CREATED, never rejected, because asking for a name is the instruction to have that store (one name, one folder, one index; it configures itself once the routing config lands in it). Nothing else about the request is compared - not the window, not the route, not the flags - which is the whole point of naming it: a client a config version behind on some flag still reaches the right store. */
+export function findBucketStore(account: string, bucketName: string, sourceConfig: SourceConfig | undefined): BlobStore {
     if (!sourceConfig) {
-        throw new Error(`No remote source configuration was provided for bucket ${account}/${bucketName}: every request must say which configured source it selected. This server has: ${JSON.stringify(bucket.stores.map(x => x.name))}`);
-    }
-    let store = bucket.stores.find(x => x.name === sourceConfig.name);
-    if (store) return store;
-    // The loaded bucket is a snapshot of the disk from when it loaded; a store created since (a new name in a rewritten routing config) is on the disk but not in the snapshot, so a miss re-reads the disk once before it is believed
-    let previousNames = bucket.stores.map(x => x.name);
-    let reloaded = await reloadBucket(account, bucketName);
-    store = reloaded?.stores.find(x => x.name === sourceConfig.name);
-    if (store) {
-        console.log(`Reloaded bucket ${account}/${bucketName} to find store ${JSON.stringify(sourceConfig.name)}: its stores changed from ${JSON.stringify(previousNames)} to ${JSON.stringify(reloaded?.stores.map(x => x.name))}`);
-        return store;
+        throw new Error(`No remote source configuration was provided for bucket ${account}/${bucketName}: every request must say which configured source it selected`);
     }
-    throw new Error(`This server has no store named ${JSON.stringify(sourceConfig.name)} for bucket ${account}/${bucketName} (it has: ${JSON.stringify((reloaded || bucket).stores.map(x => x.name))}). Requested: ${JSON.stringify(sourceConfig)}`);
+    // Not part of resolving the request (the name already says everything): loading the bucket is what schedules its switchover work and marks it active for deploy handoffs, and a bucket serving requests must be loaded
+    void getLoadedBucket(account, bucketName).catch((e: Error) => console.error(`Loading bucket ${account}/${bucketName} failed: ${e.stack ?? e}`));
+    return getStore(account, bucketName, sourceConfig.name);
 }
 
 /** Internal (store-to-store) reads skip store selection entirely: the caller is another store whose index says this MACHINE holds the bytes - the persisted holder identity is just a URL, which cannot name a store. Whichever store's folder has the newest copy answers. */
@@ -118,7 +110,7 @@ export async function bucketSyncStatus(bucket: BucketState): Promise<ArchivesSyn
     };
 }
 
-export async function bucketIndexTotals(bucket: BucketState): Promise<{ fileCount: number; byteCount: number; sources: { debugName: string; fileCount: number; byteCount: number }[] }> {
+export async function debugBucketIndexTotals(bucket: BucketState): Promise<{ fileCount: number; byteCount: number; sources: { debugName: string; fileCount: number; byteCount: number }[] }> {
     let totals = await Promise.all(bucket.stores.map(s => s.store.computeIndexTotals()));
     return {
         fileCount: totals.reduce((sum, x) => sum + x.fileCount, 0),
@@ -166,9 +158,12 @@ function sourcePersistentUrl(sourceConfig: SourceConfig | undefined, folder: str
 const stores = new Map<string, BlobStore>();
 
 export function getStore(account: string, bucketName: string, name: string): BlobStore {
+    // The one place a name becomes a folder, so the one place it is validated (it can come straight off the wire)
+    assertValidSourceName(name);
     let folder = getBucketFolder(name, account, bucketName);
     let existing = stores.get(folder);
     if (existing) return existing;
+    console.log(`Opening store ${JSON.stringify(name)} of bucket ${account}/${bucketName} (folder ${folder}) - created on this first use if it never existed`);
     let store = new BlobStore(folder, name, {
         // Which entries in ITS routing config are this server's copy of it
         isSelf: source => isSelfSource(source, account, bucketName),
@@ -297,7 +292,7 @@ function toActiveBucketInfo(loaded: BucketState): ActiveBucketInfo {
 }
 
 /** The live in-memory state of ONE bucket, answered without touching the disk (no routing file read, no statfs, no stored write stats). Returns an error string when the bucket is not loaded here, which is the normal state for a bucket nothing has accessed since startup. */
-export async function getActiveBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string> {
+export async function debugGetActiveBucket(account: string, bucketName: string): Promise<ActiveBucketInfo | string> {
     let key = `${account}/${bucketName}`;
     let loadedPromise = buckets.get(key);
     if (!loadedPromise) {
@@ -338,7 +333,7 @@ export async function activateBucket(account: string, bucketName: string): Promi
     return toActiveBucketInfo(loaded);
 }
 
-export async function listAccountBuckets(account: string): Promise<ServerBucketInfo[]> {
+export async function debugListAccountBuckets(account: string): Promise<ServerBucketInfo[]> {
     let start = Date.now();
     let timings = new Map<string, number>();
     async function timed<T>(name: string, run: () => Promise<T>): Promise<T> {
@@ -391,7 +386,7 @@ export async function listAccountBuckets(account: string): Promise<ServerBucketI
     } finally {
         // The parts run concurrently, so these are summed wall times per part, not a breakdown of the total
         let parts = [...timings].map(([name, ms]) => `${name} ${ms}ms`).join(", ");
-        console.log(`listAccountBuckets(${account}) took ${Date.now() - start}ms for ${names.length} buckets: ${parts}`);
+        console.log(`debugListAccountBuckets(${account}) took ${Date.now() - start}ms for ${names.length} buckets: ${parts}`);
     }
 }
 
@@ -430,7 +425,7 @@ function getBucketWriteStats(key: string): BucketWriteStats {
 }
 
 /** Zeroes the write statistics of every bucket in the account. */
-export function clearAccountWriteStats(account: string): number {
+export function debugClearAccountWriteStats(account: string): number {
     let prefix = `${account}/`;
     let cleared = 0;
     for (let key of [...writeStats.keys()]) {