pyric-admin 0.1.0-alpha.30.2 → 0.1.0-alpha.32

package/src/storage/index.ts

@@ -93,10 +93,10 @@ import {
 import { assertAdminAppActive } from '../app/lifecycle.js';
 import { defineDeferredMembers } from 'pyric/app/internal';
 import { BUCKET_DEFERRED_MEMBERS, FILE_DEFERRED_MEMBERS } from './deferred-members.js';
-import { deleteListedFiles } from './delete-files.js';
+import { deletedNamesFound, deleteListedFiles } from './delete-files.js';
 import { assertQuery, type DeleteFilesOptions, type GetFilesOptions } from './file-query.js';
-import { getFilesResponse, type GetFilesResponse } from './get-files.js';
-import { namesUnderPrefix, type FolderListing } from './remote-names.js';
+import { getFilesResponse, pageNamesFound, type GetFilesResponse } from './get-files.js';
+import { namesForQuery, type FolderListing } from './remote-names.js';
 
 export type { DeleteFilesOptions, GetFilesApiResponse, GetFilesOptions } from './file-query.js';
 export type { GetFilesResponse } from './get-files.js';
@@ -793,22 +793,24 @@ class RemoteBucket implements Bucket {
 
   async getFiles(query: GetFilesOptions = {}): Promise<GetFilesResponse<File>> {
     assertQuery(query, 'getFiles');
-    return getFilesResponse(await this.namesUnder(query.prefix ?? ''), query, name => this.file(name));
+    const names = await this.namesFor(query, pageNamesFound(query));
+    return getFilesResponse(names, query, name => this.file(name));
   }
 
   async deleteFiles(query: DeleteFilesOptions = {}): Promise<void> {
     assertQuery(query, 'deleteFiles');
-    await deleteListedFiles(await this.namesUnder(query.prefix ?? ''), query, name => this.file(name).delete());
+    const names = await this.namesFor(query, deletedNamesFound(query));
+    await deleteListedFiles(names, query, name => this.file(name).delete());
   }
 
-  /** The host's object names under `prefix`, walked folder by folder with `storage.listAll`. */
-  private namesUnder(prefix: string): Promise<string[]> {
-    return namesUnderPrefix(prefix, async path => (await this.channel.op({
+  /** The host's object names `query` reads, walked folder by folder with `storage.listAll` until `enough`. */
+  private namesFor(query: GetFilesOptions, enough: (name: string) => boolean): Promise<string[]> {
+    return namesForQuery(query, async path => (await this.channel.op({
       method: 'storage.listAll',
       ...this.wire,
       path,
       actAs: STORAGE_REMOTE_ADMIN_LENS,
-    })) as FolderListing);
+    })) as FolderListing, enough);
   }
 }
 

package/src/storage/remote-names.ts

@@ -1,10 +1,25 @@
 /**
- * The object names under a prefix on a remote sandbox's host, read with the
- * host's `storage.listAll` op. That op lists one folder: its files and its
+ * The object names a `getFiles` query reads on a remote sandbox's host, with
+ * the host's `storage.listAll` op. That op lists one folder: its files and its
  * subfolders. A `getFiles` prefix is not a folder (`users/12` matches
  * `users/123/a.png`), so the walk starts at the folder holding the prefix and
  * enters each subfolder whose path can hold a matching name.
+ *
+ * The walk goes in name order, depth first, with a few folder listings in
+ * flight at once (#1183): a client may have 256 pending operations, and a
+ * store with a folder per object has more folders than that. Because names
+ * come in order, a query that needs one page stops once it has that page.
+ *
+ *   - Name order: every name in folder `f` starts with `f/`, and no file in
+ *     the parent sorts between two such names, so visiting a folder's files
+ *     and subfolders sorted by `name` and `path + '/'` yields names in order.
+ *   - A subfolder whose names all roll up to one prefix at the delimiter is
+ *     not entered: the walk yields that prefix in place of its names, which
+ *     the listing reports as a prefix, never as a file.
+ *   - A subfolder whose names all sort at or before the `pageToken` is not
+ *     entered: every name in `f` sorts before `f0`.
  */
+import { compareKeys, entryOf, resumesAfter, type GetFilesOptions } from './file-query.js';
 
 /** A folder's listing, as the host's `storage.listAll` answers it. */
 export interface FolderListing {
@@ -12,22 +27,138 @@ export interface FolderListing {
   prefixes: Array<{ fullPath: string }>;
 }
 
-/** Every object name starting with `prefix`, listing folders with `listFolder`. */
-export async function namesUnderPrefix(
-  prefix: string,
-  listFolder: (path: string) => Promise<FolderListing>,
+export type ListFolder = (path: string) => Promise<FolderListing>;
+
+/** How many folder listings the walk keeps in flight, well under the 256 a client may have pending. */
+export const WALK_CONCURRENCY = 8;
+
+/**
+ * The names under `query`'s prefix, in name order, listing folders with
+ * `listFolder`. `enough` sees each name as it is found and stops the walk
+ * once it answers true.
+ */
+export async function namesForQuery(
+  query: GetFilesOptions,
+  listFolder: ListFolder,
+  enough: (name: string) => boolean = () => false,
 ): Promise<string[]> {
   const names: string[] = [];
-  const walk = async (folder: string): Promise<void> => {
-    const { items, prefixes } = await listFolder(folder);
-    for (const item of items) if (item.fullPath.startsWith(prefix)) names.push(item.fullPath);
-    const entered = prefixes.map(sub => sub.fullPath).filter(path => canHoldMatch(`${path}/`, prefix));
-    await Promise.all(entered.map(walk));
-  };
-  await walk(prefix.slice(0, Math.max(prefix.lastIndexOf('/'), 0)));
+  for await (const name of walkNames(query, listFolder)) {
+    names.push(name);
+    if (enough(name)) break;
+  }
   return names;
 }
 
+interface FolderNode {
+  kind: 'folder';
+  path: string;
+  listing?: Promise<FolderListing>;
+}
+
+type WalkNode = { kind: 'names'; names: string[] } | FolderNode;
+
+/** Yield the names under `query`'s prefix in name order, `concurrency` listings at a time at most. */
+export async function* walkNames(
+  query: GetFilesOptions,
+  listFolder: ListFolder,
+  concurrency = WALK_CONCURRENCY,
+): AsyncGenerator<string> {
+  const prefix = query.prefix ?? '';
+  const start = prefix.slice(0, Math.max(prefix.lastIndexOf('/'), 0));
+  let inFlight = 0;
+  let onSettle: Array<() => void> = [];
+  const aSlotFrees = (): Promise<void> => new Promise(resolve => onSettle.push(resolve));
+  const startListing = (node: FolderNode): Promise<FolderListing> => {
+    inFlight += 1;
+    const listing = listFolder(node.path);
+    // Count the listing until it settles. Handling both outcomes here also
+    // marks a rejection handled, so a walk that stops before reaching a
+    // looked-ahead folder leaves no unhandled rejection.
+    const settled = (): void => {
+      inFlight -= 1;
+      const waiting = onSettle;
+      onSettle = [];
+      for (const resolve of waiting) resolve();
+    };
+    listing.then(settled, settled);
+    node.listing = listing;
+    return listing;
+  };
+  // The next node is at the end; nodes below it come later in name order.
+  const stack: WalkNode[] = [{ kind: 'folder', path: start }];
+  for (let node = stack.pop(); node !== undefined; node = stack.pop()) {
+    if (node.kind === 'names') {
+      yield* node.names;
+      continue;
+    }
+    // Lookahead may hold every slot; this folder's listing waits for one.
+    while (node.listing === undefined && inFlight >= concurrency) await aSlotFrees();
+    const listing = node.listing ?? startListing(node);
+    listAhead(stack, concurrency, () => inFlight < concurrency, startListing);
+    stack.push(...childrenOf(await listing, query).reverse());
+  }
+}
+
+/**
+ * Start the listings of the next folders in name order while `hasRoom`,
+ * looking at most `window` folders ahead. A listing started here is awaited
+ * when its folder comes up.
+ */
+function listAhead(
+  stack: WalkNode[],
+  window: number,
+  hasRoom: () => boolean,
+  startListing: (node: FolderNode) => Promise<FolderListing>,
+): void {
+  let seen = 0;
+  for (let index = stack.length - 1; index >= 0 && seen < window && hasRoom(); index -= 1) {
+    const node = stack[index]!;
+    if (node.kind !== 'folder') continue;
+    seen += 1;
+    if (node.listing === undefined) void startListing(node);
+  }
+}
+
+/** A folder's files and the subfolders worth entering, in name order, with runs of names grouped. */
+function childrenOf(listing: FolderListing, query: GetFilesOptions): WalkNode[] {
+  const prefix = query.prefix ?? '';
+  const after = resumesAfter(query);
+  const children: Array<{ key: string; node: WalkNode }> = [];
+  for (const { fullPath } of listing.items) {
+    if (fullPath.startsWith(prefix)) children.push({ key: fullPath, node: { kind: 'names', names: [fullPath] } });
+  }
+  for (const { fullPath } of listing.prefixes) {
+    const key = `${fullPath}/`;
+    const holdsMatch = canHoldMatch(key, prefix);
+    const passedByToken = after !== undefined && `${fullPath}0` <= after;
+    if (!holdsMatch || passedByToken) continue;
+    const rolledUp = rollUpOf(key, query);
+    const node: WalkNode = rolledUp === undefined ? { kind: 'folder', path: fullPath } : { kind: 'names', names: [rolledUp] };
+    children.push({ key, node });
+  }
+  children.sort((a, b) => compareKeys(a.key, b.key));
+  return groupNames(children.map(child => child.node));
+}
+
+/** The prefix every name in the folder `folderWithSlash` rolls up to, if they all roll up to one. */
+function rollUpOf(folderWithSlash: string, query: GetFilesOptions): string | undefined {
+  const entry = entryOf(folderWithSlash, query);
+  return entry?.rolledUp === true ? entry.key : undefined;
+}
+
+/** Merge each run of adjacent name nodes into one, so looking ahead skips a run in one step. */
+function groupNames(nodes: WalkNode[]): WalkNode[] {
+  const grouped: WalkNode[] = [];
+  for (const node of nodes) {
+    const last = grouped[grouped.length - 1];
+    const joinsRun = node.kind === 'names' && last?.kind === 'names';
+    if (joinsRun) last.names.push(...node.names);
+    else grouped.push(node);
+  }
+  return grouped;
+}
+
 /** Can a name in the folder `folderWithSlash` start with `prefix`? */
 function canHoldMatch(folderWithSlash: string, prefix: string): boolean {
   return folderWithSlash.startsWith(prefix) || prefix.startsWith(folderWithSlash);