sliftutils 1.7.88 → 1.7.90

package/storage/remoteStorage/storeConfig.d.ts

@@ -0,0 +1,30 @@
+import { HostedConfig } from "../IArchives";
+export declare class StoreConfig {
+    readonly name: string;
+    constructor(name: string, entries: HostedConfig[]);
+    private entries;
+    /** The routing config changed. Same store either way: its name did not change, so neither did its folder, its index, or the data in it. */
+    /** No entries is a real state, not an error: a store exists as soon as its folder does, and it may never have heard of a configuration (see StorePolicy). */
+    update(entries: HostedConfig[]): void;
+    all(): HostedConfig[];
+    /**
+     * What this store is configured to be right now: the entry whose window contains this moment;
+     * failing that the next one due to start; failing that the last one to have ended.
+     *
+     * There is always an answer, and it matters that there is: a store between windows still holds
+     * data and still has to answer for it, so it needs a route and flags even when nothing is
+     * currently pointing writes at it. Which of the three cases produced the answer is deliberately
+     * not exposed - the valid window itself is what says whether writes belong here, and every write
+     * is checked against it separately.
+     */
+    current(): StorePolicy;
+}
+/** The parts of a config entry that describe what a store IS - as opposed to which entry said so. Kept separate because a store with no entries still has all of them. */
+export type StorePolicy = {
+    validWindow: [number, number];
+    route?: [number, number];
+    fast?: boolean;
+    writeDelay?: number;
+    noFullSync?: boolean;
+    readerDiskLimit?: number;
+};

package/storage/remoteStorage/storeConfig.ts

@@ -0,0 +1,62 @@
+import { sort } from "socket-function/src/misc";
+import { HostedConfig, FULL_VALID_WINDOW } from "../IArchives";
+
+// The configuration of ONE store, kept current. A store is named by its entries (see CommonConfig.name), and at any instant exactly one of them is in effect - the config write path enforces that - so "the configuration of this store right now" is a single entry, and everything the store does with routes, flags and limits reads it from here rather than being handed a snapshot that goes stale.
+
+export class StoreConfig {
+    constructor(
+        public readonly name: string,
+        entries: HostedConfig[],
+    ) {
+        this.update(entries);
+    }
+
+    // Every entry naming this store, oldest window first - past ones included, because a stamped write or a scan can legitimately concern a window that has already ended
+    private entries: HostedConfig[] = [];
+
+    /** The routing config changed. Same store either way: its name did not change, so neither did its folder, its index, or the data in it. */
+    /** No entries is a real state, not an error: a store exists as soon as its folder does, and it may never have heard of a configuration (see StorePolicy). */
+    public update(entries: HostedConfig[]): void {
+        let ordered = [...entries];
+        sort(ordered, x => x.validWindow[0]);
+        this.entries = ordered;
+    }
+
+    public all(): HostedConfig[] {
+        return this.entries;
+    }
+
+    /**
+     * What this store is configured to be right now: the entry whose window contains this moment;
+     * failing that the next one due to start; failing that the last one to have ended.
+     *
+     * There is always an answer, and it matters that there is: a store between windows still holds
+     * data and still has to answer for it, so it needs a route and flags even when nothing is
+     * currently pointing writes at it. Which of the three cases produced the answer is deliberately
+     * not exposed - the valid window itself is what says whether writes belong here, and every write
+     * is checked against it separately.
+     */
+    public current(): StorePolicy {
+        let now = Date.now();
+        let containing = this.entries.find(x => x.validWindow[0] <= now && now < x.validWindow[1]);
+        if (containing) return containing;
+        let upcoming = this.entries.find(x => x.validWindow[0] > now);
+        if (upcoming) return upcoming;
+        let last = this.entries[this.entries.length - 1];
+        if (last) return last;
+        return UNCONFIGURED;
+    }
+}
+
+/** The parts of a config entry that describe what a store IS - as opposed to which entry said so. Kept separate because a store with no entries still has all of them. */
+export type StorePolicy = {
+    validWindow: [number, number];
+    route?: [number, number];
+    fast?: boolean;
+    writeDelay?: number;
+    noFullSync?: boolean;
+    readerDiskLimit?: number;
+};
+
+// A store nobody has configured: always valid, the whole key space, no delays and no limits. Every one of these is the answer that does the least - it holds its own data and nothing else happens to it - which is what makes a store usable before it has ever seen a config.
+const UNCONFIGURED: StorePolicy = { validWindow: FULL_VALID_WINDOW };

package/storage/remoteStorage/storePlan.d.ts

@@ -1,6 +1,29 @@
 import { RemoteConfig, HostedConfig, SourceConfig } from "../IArchives";
+/** Whether a config entry is THIS server's copy of this bucket - the same account and bucket, at an address this process answers on. */
+export declare function isSelfSource(source: SourceConfig, account: string, bucketName: string): boolean;
 export declare function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[];
 export declare function selectEntryAt(entries: HostedConfig[], time: number, route?: number): HostedConfig | undefined;
+/** What one of our stores has to pull in at a valid-window boundary, so the writes that landed just before the handover are not missed. */
+export type BoundaryHandover = {
+    name: string;
+    route: [number, number];
+    scanOwnDisk: boolean;
+    remotes: Map<number, [number, number]>;
+};
+/**
+ * Who held each slice of our route in the window before windowStart, for every self entry whose
+ * window starts exactly then. This is the whole of "who do we take over from": a store taking over a
+ * route may be taking it from several previous owners at once (their shards need not line up with
+ * ours), and from itself for the parts it already held.
+ *
+ * A self entry is skipped when an EARLIER entry valid at the boundary already covers its whole route:
+ * config order is priority, so that entry is the write target and we are not the one taking over.
+ * Owners are then resolved in config order too, each claiming the part of our route still unclaimed -
+ * the same first-match-wins rule that picks a write target at any other moment.
+ *
+ * Pure: config in, plan out. Nothing here reads a store, a clock, or the network.
+ */
+export declare function previousWindowOwners(config: RemoteConfig, windowStart: number, selfIndexes: number[]): BoundaryHandover[];
 /** Our role in a bucket's routing config, summarized across ALL currently-valid self entries. Stored instead of a single representative HostedConfig, so nothing can accidentally use one entry's route or flags where the union is required - the standard config has the same URL twice: a routed write-shard entry plus an unrouted read-everything entry. */
 export type SelfSummary = {
     /** The union of the current entries' routes, with overlapping/adjacent ranges combined - which commonly collapses to a single full range, making matching trivial. */
@@ -8,27 +31,16 @@ export type SelfSummary = {
     public: boolean;
     immutable: boolean;
     noFullSync: boolean;
-    rawDisk: boolean;
     readerDiskLimit?: number;
 };
-export type StoreSourceSpec = {
-    sourceConfig?: SourceConfig;
-    validWindows: [number, number][];
-    route?: [number, number];
-    noFullSync?: boolean;
-};
-export type StorePlanStore = {
-    routeKey: string;
-    route?: [number, number];
+/** One store this server runs for a bucket: the name that IS the store, and the entries that name it. What it synchronizes with is NOT decided here - the store reads its own copy of the config and works that out itself (see BlobStore.applyRoutingConfig). */
+export type SelfStore = {
+    name: string;
     entries: HostedConfig[];
-    rawDisk: boolean;
-    readerDiskLimit?: number;
-    sourceSpecs: StoreSourceSpec[];
 };
-export type StorePlan = {
+/** Which stores this server runs for a bucket, by name. Nothing more: a store is found by name, and configures itself. */
+export declare function planSelfStores(account: string, bucketName: string, routing: RemoteConfig): {
     selfEntries: HostedConfig[];
     self: SelfSummary | undefined;
-    stores: StorePlanStore[];
-    structureKey: string;
+    stores: SelfStore[];
 };
-export declare function computeStorePlan(account: string, bucketName: string, routing: RemoteConfig): StorePlan;

package/storage/remoteStorage/storePlan.ts

@@ -5,14 +5,20 @@ import { isOwnAddress } from "./serverConfig";
 
 // Turns a bucket's routing config into this server's store plan: which of the entries are us, what stores (one per route) we run, and which peers each store synchronizes from.
 
+/** Whether a config entry is THIS server's copy of this bucket - the same account and bucket, at an address this process answers on. */
+export function isSelfSource(source: SourceConfig, account: string, bucketName: string): boolean {
+    if (source.type !== "remote") return false;
+    let parsed = parseHostedUrl(source.url);
+    if (parsed.account !== account || parsed.bucketName !== bucketName) return false;
+    return isOwnAddress(parsed.address, parsed.port);
+}
+
 export function findSelfIndexes(routing: RemoteConfig, account: string, bucketName: string): number[] {
     let indexes: number[] = [];
     for (let i = 0; i < routing.sources.length; i++) {
         let source = routing.sources[i];
-        if (typeof source === "string" || source.type !== "remote") continue;
-        let parsed = parseHostedUrl(source.url);
-        if (parsed.account !== account || parsed.bucketName !== bucketName) continue;
-        if (isOwnAddress(parsed.address, parsed.port)) {
+        if (typeof source === "string") continue;
+        if (isSelfSource(source, account, bucketName)) {
             indexes.push(i);
         }
     }
@@ -40,6 +46,93 @@ export function selectEntryAt(entries: HostedConfig[], time: number, route?: num
     return best;
 }
 
+/** The parts of `ranges` that `cut` does not cover, plus the single range spanning what it did. Routes are half-open [start, end), so subtracting one from another leaves at most a piece on each side. */
+function subtractRoute(ranges: [number, number][], cut: [number, number]): { remaining: [number, number][]; claimed?: [number, number] } {
+    let remaining: [number, number][] = [];
+    let claimed: [number, number] | undefined;
+    for (let range of ranges) {
+        let overlap = routeIntersection(range, cut);
+        if (!overlap) {
+            remaining.push(range);
+            continue;
+        }
+        claimed = claimed && [Math.min(claimed[0], overlap[0]), Math.max(claimed[1], overlap[1])] as [number, number] || overlap;
+        if (range[0] < overlap[0]) remaining.push([range[0], overlap[0]]);
+        if (overlap[1] < range[1]) remaining.push([overlap[1], range[1]]);
+    }
+    return { remaining, claimed };
+}
+
+/** What one of our stores has to pull in at a valid-window boundary, so the writes that landed just before the handover are not missed. */
+export type BoundaryHandover = {
+    // The store that needs the data: the name of the self entry taking over at the boundary
+    name: string;
+    // The route it is taking over, which is the slice of the key space the pulls below are limited to
+    route: [number, number];
+    // We held part of this route in the previous window too, so those writes are already in our own folder and a disk rescan finds them
+    scanOwnDisk: boolean;
+    // Per source index in the config, the slice of our route THAT source held in the previous window - a boundary scan pulls its recent changes
+    remotes: Map<number, [number, number]>;
+};
+
+/**
+ * Who held each slice of our route in the window before windowStart, for every self entry whose
+ * window starts exactly then. This is the whole of "who do we take over from": a store taking over a
+ * route may be taking it from several previous owners at once (their shards need not line up with
+ * ours), and from itself for the parts it already held.
+ *
+ * A self entry is skipped when an EARLIER entry valid at the boundary already covers its whole route:
+ * config order is priority, so that entry is the write target and we are not the one taking over.
+ * Owners are then resolved in config order too, each claiming the part of our route still unclaimed -
+ * the same first-match-wins rule that picks a write target at any other moment.
+ *
+ * Pure: config in, plan out. Nothing here reads a store, a clock, or the network.
+ */
+export function previousWindowOwners(config: RemoteConfig, windowStart: number, selfIndexes: number[]): BoundaryHandover[] {
+    let selfIndexSet = new Set(selfIndexes);
+    let previousTime = windowStart - 1;
+    let validAt = (source: SourceConfig, time: number) => source.validWindow[0] <= time && time < source.validWindow[1];
+    let byRoute = new Map<string, BoundaryHandover>();
+    for (let selfIndex of selfIndexes) {
+        let self = config.sources[selfIndex];
+        if (typeof self === "string" || self.validWindow[0] !== windowStart) continue;
+        let selfRoute = self.route || FULL_ROUTE;
+        let shadowed = false;
+        for (let i = 0; i < selfIndex; i++) {
+            let other = config.sources[i];
+            if (typeof other === "string" || !validAt(other, windowStart)) continue;
+            let route = other.route || FULL_ROUTE;
+            if (route[0] <= selfRoute[0] && selfRoute[1] <= route[1]) {
+                shadowed = true;
+                break;
+            }
+        }
+        if (shadowed) continue;
+        // Keyed by both: one name can take over several routes at the same boundary, and each is pulled separately
+        let handoverKey = `${self.name}|${JSON.stringify(selfRoute)}`;
+        let handover = byRoute.get(handoverKey);
+        if (!handover) {
+            handover = { name: self.name, route: selfRoute, scanOwnDisk: false, remotes: new Map() };
+            byRoute.set(handoverKey, handover);
+        }
+        let unclaimed: [number, number][] = [[selfRoute[0], selfRoute[1]]];
+        for (let i = 0; i < config.sources.length && unclaimed.length; i++) {
+            let other = config.sources[i];
+            if (typeof other === "string" || !validAt(other, previousTime)) continue;
+            let { remaining, claimed } = subtractRoute(unclaimed, other.route || FULL_ROUTE);
+            unclaimed = remaining;
+            if (!claimed) continue;
+            if (selfIndexSet.has(i)) {
+                handover.scanOwnDisk = true;
+                continue;
+            }
+            let existing = handover.remotes.get(i);
+            handover.remotes.set(i, existing && [Math.min(existing[0], claimed[0]), Math.max(existing[1], claimed[1])] as [number, number] || claimed);
+        }
+    }
+    return [...byRoute.values()].filter(x => x.scanOwnDisk || x.remotes.size);
+}
+
 /** Our role in a bucket's routing config, summarized across ALL currently-valid self entries. Stored instead of a single representative HostedConfig, so nothing can accidentally use one entry's route or flags where the union is required - the standard config has the same URL twice: a routed write-shard entry plus an unrouted read-everything entry. */
 export type SelfSummary = {
     /** The union of the current entries' routes, with overlapping/adjacent ranges combined - which commonly collapses to a single full range, making matching trivial. */
@@ -47,7 +140,6 @@ export type SelfSummary = {
     public: boolean;
     immutable: boolean;
     noFullSync: boolean;
-    rawDisk: boolean;
     readerDiskLimit?: number;
 };
 
@@ -80,96 +172,31 @@ function summarizeSelf(selfEntries: HostedConfig[], now: number): SelfSummary |
         public: current.some(x => x.public),
         immutable: current.some(x => x.immutable),
         noFullSync: current.some(x => x.noFullSync),
-        rawDisk: current.some(x => x.rawDisk),
         readerDiskLimit: current.find(x => x.readerDiskLimit !== undefined)?.readerDiskLimit,
     };
 }
 
-export type StoreSourceSpec = { sourceConfig?: SourceConfig; validWindows: [number, number][]; route?: [number, number]; noFullSync?: boolean };
-export type StorePlanStore = {
-    routeKey: string;
-    route?: [number, number];
+/** One store this server runs for a bucket: the name that IS the store, and the entries that name it. What it synchronizes with is NOT decided here - the store reads its own copy of the config and works that out itself (see BlobStore.applyRoutingConfig). */
+export type SelfStore = {
+    name: string;
     entries: HostedConfig[];
-    rawDisk: boolean;
-    readerDiskLimit?: number;
-    sourceSpecs: StoreSourceSpec[];
-};
-export type StorePlan = {
-    selfEntries: HostedConfig[];
-    self: SelfSummary | undefined;
-    stores: StorePlanStore[];
-    structureKey: string;
 };
 
-export function computeStorePlan(account: string, bucketName: string, routing: RemoteConfig): StorePlan {
+/** Which stores this server runs for a bucket, by name. Nothing more: a store is found by name, and configures itself. */
+export function planSelfStores(account: string, bucketName: string, routing: RemoteConfig): { selfEntries: HostedConfig[]; self: SelfSummary | undefined; stores: SelfStore[] } {
     let selfIndexes = findSelfIndexes(routing, account, bucketName);
     let selfEntries = selfIndexes.map(i => routing.sources[i] as HostedConfig);
     let self = summarizeSelf(selfEntries, Date.now());
-    let ownIndexes = new Set(selfIndexes);
-    // One store per distinct route among ALL our entries - past and future windows included, so historical route folders keep serving their data and upcoming routes sync ahead of their window
-    let groups = new Map<string, { route?: [number, number]; entries: HostedConfig[]; firstIndex: number }>();
-    for (let i of selfIndexes) {
-        let entry = routing.sources[i] as HostedConfig;
-        let routeKey = JSON.stringify(entry.route || FULL_ROUTE);
-        let group = groups.get(routeKey);
-        if (!group) {
-            group = { route: entry.route, entries: [], firstIndex: i };
-            groups.set(routeKey, group);
+    // One store per NAME among ALL our entries - past and future windows included, so a name we no longer write to keeps serving its data and a name we are about to use exists ahead of its window
+    let byName = new Map<string, HostedConfig[]>();
+    for (let entry of selfEntries) {
+        let entries = byName.get(entry.name);
+        if (!entries) {
+            entries = [];
+            byName.set(entry.name, entries);
         }
-        group.entries.push(entry);
-    }
-    let stores: StorePlanStore[] = [];
-    for (let [routeKey, group] of groups) {
-        let anchor = selectEntryAt(group.entries, Date.now());
-        let diskWindow: [number, number] = [0, 0];
-        if (anchor) {
-            let [start, end] = anchor.validWindow;
-            let merged = true;
-            while (merged) {
-                merged = false;
-                for (let entry of group.entries) {
-                    let [entryStart, entryEnd] = entry.validWindow;
-                    if (entryStart > end || entryEnd < start) continue;
-                    if (entryStart < start || entryEnd > end) {
-                        start = Math.min(start, entryStart);
-                        end = Math.max(end, entryEnd);
-                        merged = true;
-                    }
-                }
-            }
-            diskWindow = [start, end];
-        }
-        let sourceSpecs: StoreSourceSpec[] = [{ validWindows: [diskWindow] }];
-        let noFullSync = group.entries.some(x => x.noFullSync);
-        // The same peer endpoint (url+route+flags) can appear under several windows at once (a switchover splits one window around an intermediate). It is ONE sync source that carries ALL those windows - never several sources, which would double-scan it and, worse, make matching a request depend on which window the caller happened to hold.
-        let peerByKey = new Map<string, StoreSourceSpec>();
-        for (let i = group.firstIndex + 1; i < routing.sources.length; i++) {
-            let source = routing.sources[i];
-            if (typeof source === "string" || ownIndexes.has(i)) continue;
-            let sharedRoute = routeIntersection(group.route, source.route);
-            if (!sharedRoute) continue;
-            let key = JSON.stringify({ ...source, validWindow: undefined });
-            let spec = peerByKey.get(key);
-            if (!spec) {
-                spec = { sourceConfig: source, validWindows: [], route: sharedRoute, noFullSync: source.noFullSync || noFullSync };
-                peerByKey.set(key, spec);
-                sourceSpecs.push(spec);
-            }
-            spec.validWindows.push(source.validWindow);
-        }
-        stores.push({
-            routeKey,
-            route: group.route,
-            entries: group.entries,
-            rawDisk: group.entries.some(x => x.rawDisk),
-            readerDiskLimit: group.entries.find(x => x.readerDiskLimit !== undefined)?.readerDiskLimit,
-            sourceSpecs,
-        });
-    }
-    if (!stores.length) {
-        // Not in the config at all: still serve whatever the plain folder holds, through one inert full-route store
-        stores.push({ routeKey: JSON.stringify(FULL_ROUTE), route: undefined, entries: [], rawDisk: false, readerDiskLimit: undefined, sourceSpecs: [{ validWindows: [[0, 0]] }] });
+        entries.push(entry);
     }
-    let structureKey = JSON.stringify(stores.map(s => ({ routeKey: s.routeKey, rawDisk: s.rawDisk, readerDiskLimit: s.readerDiskLimit })));
-    return { selfEntries, self, stores, structureKey };
+    // Not in the config at all: no name means no store here
+    return { selfEntries, self, stores: [...byName].map(([name, entries]) => ({ name, entries })) };
 }

package/storage/remoteStorage/storeSources.d.ts

@@ -0,0 +1,11 @@
+import { IArchives, SourceConfig } from "../IArchives";
+/** The client for one configured source: backblaze, or a storage server - including this one. */
+export declare function createApiArchives(source: SourceConfig): IArchives;
+/** The ONE place a store's source is built. Every source a store synchronizes with is one of exactly two things: a configured peer, or the store's own disk folder (no sourceConfig). writeDelay wraps it so its writes are buffered in memory for that long (see ArchivesDelayed) - the whole of "fast writes", per source, decided here. */
+export declare function createStoreSource(config: {
+    sourceConfig?: SourceConfig;
+    folder: string;
+    writeDelay?: number;
+}): IArchives;
+/** Applies a changed config to an ALREADY RUNNING source (same endpoint, see sourceIdentity - only policy moved). Sources that carry their config into every request MUST be updated in place, or they keep sending the old one: the server matches the config it is handed against its own entries, so a source left holding a stale config eventually stops resolving to a store at all. The write delay is policy too, so it moves here as well. */
+export declare function applySourceConfig(source: IArchives, sourceConfig: SourceConfig | undefined, writeDelay?: number): void;

package/storage/remoteStorage/storeSources.ts

@@ -0,0 +1,47 @@
+import { IArchives, SourceConfig } from "../IArchives";
+import { ArchivesDisk } from "../ArchivesDisk";
+import { ArchivesBackblaze } from "../backblaze";
+import { ArchivesRemote } from "./ArchivesRemote";
+import { parseBackblazeUrl } from "./remoteConfig";
+import { ArchivesDelayed, asDelayed, unwrapDelayed } from "./ArchivesDelayed";
+
+// ONE source, talked to directly - the counterpart of createArchives, which resolves a whole routing config into a chain that falls back and re-resolves across many sources. Nothing here picks a source or retries against another one: the caller (a chain, or a store's synchronization) already decided which endpoint it wants, and gets a client bound to exactly that endpoint. Source identity and config updates live here too, because they are the same question - which endpoint is this, and what policy are we applying to it.
+
+/** The client for one configured source: backblaze, or a storage server - including this one. */
+export function createApiArchives(source: SourceConfig): IArchives {
+    if (source.type === "backblaze") {
+        return new ArchivesBackblaze({ bucketName: parseBackblazeUrl(source.url).bucketName, public: source.public, immutable: source.immutable, allowedOrigins: source.allowedOrigins });
+    }
+    // Including when that server is us: a call to ourselves is a WebSocket to our own port, which authenticates as this machine (so it is always allowed) and costs a round trip on loopback. Not worth a second implementation of every operation to avoid.
+    return new ArchivesRemote({ url: source.url, waitForAccess: false, sourceConfig: source });
+}
+
+/** The ONE place a store's source is built. Every source a store synchronizes with is one of exactly two things: a configured peer, or the store's own disk folder (no sourceConfig). writeDelay wraps it so its writes are buffered in memory for that long (see ArchivesDelayed) - the whole of "fast writes", per source, decided here. */
+export function createStoreSource(config: { sourceConfig?: SourceConfig; folder: string; writeDelay?: number }): IArchives {
+    let source: IArchives;
+    if (!config.sourceConfig) {
+        source = new ArchivesDisk(config.folder);
+    } else {
+        source = createApiArchives(config.sourceConfig);
+    }
+    if (config.writeDelay) {
+        source = new ArchivesDelayed(source, config.writeDelay);
+    }
+    return source;
+}
+
+/** A source that holds its own SourceConfig (to send with every request, or to read policy flags off). Duck-typed rather than a base class: the three implementations have nothing else in common, and a source that ignores its config simply doesn't have the method. */
+type ConfigurableSource = { updateSourceConfig(sourceConfig: SourceConfig): void };
+
+/** Applies a changed config to an ALREADY RUNNING source (same endpoint, see sourceIdentity - only policy moved). Sources that carry their config into every request MUST be updated in place, or they keep sending the old one: the server matches the config it is handed against its own entries, so a source left holding a stale config eventually stops resolving to a store at all. The write delay is policy too, so it moves here as well. */
+export function applySourceConfig(source: IArchives, sourceConfig: SourceConfig | undefined, writeDelay?: number): void {
+    if (writeDelay !== undefined) {
+        asDelayed(source)?.setDelay(writeDelay);
+    }
+    // The store's own disk has no config to apply - only a delay
+    if (!sourceConfig) return;
+    // The config belongs to the source itself, not to the delay wrapped around it
+    let configurable = unwrapDelayed(source) as Partial<ConfigurableSource>;
+    if (typeof configurable.updateSourceConfig !== "function") return;
+    configurable.updateSourceConfig(sourceConfig);
+}

package/storage/remoteStorage/storeSync.d.ts

@@ -0,0 +1,53 @@
+import { IArchives, ArchivesSyncStatus, SyncActivity } from "../IArchives";
+import type { BlobStore } from "./blobStore";
+export declare class StoreSync {
+    private store;
+    private states;
+    private activities;
+    private evicting;
+    private lastAccess;
+    constructor(store: BlobStore);
+    /** Starts every live source's synchronization, plus the maintenance loops. Called once, by the store's init - the store's index must already be loaded, since scans write straight into it. */
+    start(): void;
+    /**
+     * Asks every peer for the routing config, and takes it if it is newer than ours. This is the whole
+     * of configuration propagation: it rides beside the scans rather than being part of them, because
+     * one small read is worth doing every few minutes while a full listing is not, and because a
+     * config we are missing is what stops everything else from being right.
+     *
+     * The copy is stored the same way a scan stores anything it pulls - an internal write into our own
+     * store - and the store notices that file landing and re-configures itself. Nothing here knows what
+     * a config means; it only knows this one file is worth asking for often.
+     */
+    private pollRoutingConfig;
+    /** Stops every source's loops (the store's own stop token stops the maintenance loops). */
+    stop(): void;
+    /** A slot the store just appended to its sources array: it starts from nothing, so it gets a full scan. */
+    addSource(slot: number): void;
+    /** The slot stays in the store's arrays forever (running loops hold slot numbers); it just goes dead - loops stop, and its index entries drop (other sources' scans re-find any copy that's still reachable through the new config). */
+    removeSource(slot: number): Promise<void>;
+    /** Whether a slot is still configured. Dead slots are never scanned, written, or read. */
+    isLive(slot: number): boolean;
+    getActivities(): SyncActivity[];
+    /** A key was just served, so it goes to the back of the eviction queue. */
+    noteAccess(key: string): void;
+    private entryUnchanged;
+    getStatus(): ArchivesSyncStatus;
+    /** Listings come straight from the index, so they must wait for our own base source's initial scan (which might lag minutes) before they are trustworthy. The base (local disk) is implicitly required - remote sources are not, they come and go. */
+    waitForRequiredScans(): Promise<void>;
+    /** Rescans our own disk's metadata into the index - used around valid window handoffs, where another process wrote files to the shared folder that our index hasn't seen. */
+    rescanBase(): Promise<void>;
+    /** A boundary scan of the node that owned (part of) our route in the valid window before ours, when that node is different storage (a disk rescan can't see its writes): just its changes since the boundary neighborhood, with matching values pulled onto our own disk. */
+    boundaryScanRemote(source: IArchives, config: {
+        since: number;
+        route?: [number, number];
+    }): Promise<void>;
+    private runSourceSync;
+    private scanSource;
+    private reconcileSource;
+    private updateScanIndex;
+    private pollChanges;
+    private copySourceFiles;
+    private enforceDiskLimit;
+    private cleanupTombstones;
+}