vlist 1.9.1 → 2.0.0

package/dist/plugins/async/manager.js

@@ -0,0 +1,568 @@
+/**
+ * vlist - Data Management
+ * Handles data with sparse storage for million+ item support
+ */
+import { createSparseStorage, } from "./sparse";
+import { createPlaceholderManager, isPlaceholderItem, } from "./placeholder";
+// =============================================================================
+// Constants
+// =============================================================================
+const LOAD_SIZE = 50;
+// =============================================================================
+// Data Manager Implementation
+// =============================================================================
+/**
+ * Create a data manager with sparse storage support
+ */
+export const createDataManager = (config = {}) => {
+    const { adapter, initialItems, initialTotal, storage: storageConfig, placeholder: placeholderConfig, pageSize = LOAD_SIZE, maxConcurrent = 0, onStateChange, onDataChange, onItemsLoaded, onItemsEvicted, } = config;
+    // Create sparse storage
+    const storage = createSparseStorage({
+        ...storageConfig,
+        onEvict: (count, _ranges) => {
+            onItemsEvicted?.(count);
+            notifyDataChange();
+            notifyStateChange();
+        },
+    });
+    // Lazy-init placeholder manager — only created when first needed
+    // Static lists (no adapter) never request unloaded items, so this avoids
+    // instantiating ~300 lines of placeholder logic for the common case.
+    let placeholders = null;
+    const getOrCreatePlaceholders = () => {
+        if (!placeholders) {
+            placeholders = createPlaceholderManager(placeholderConfig);
+        }
+        return placeholders;
+    };
+    // ID to index mapping (only for loaded items)
+    const idToIndex = new Map();
+    // State
+    let isLoading = false;
+    let error;
+    let hasMore = true;
+    let cursor;
+    // The offset for which the stored cursor is valid. A server cursor returned
+    // for offset N is only meaningful as input for the request at offset N+limit.
+    // Passing it to any other offset would return wrong data.
+    let cursorValidForOffset;
+    let pendingRanges = [];
+    // Track the furthest loaded offset so that concurrent chunk responses
+    // don't race: an earlier chunk's hasMore:true can't override a later
+    // chunk's hasMore:false.
+    let hasMoreHighWater = 0;
+    // Track active chunk requests to dedupe and abort stale work.
+    const activeLoads = new Map();
+    /** Abort all in-flight requests and clear both tracking maps. */
+    const abortAndClearLoads = () => {
+        for (const load of activeLoads.values())
+            load[1].abort();
+        activeLoads.clear();
+    };
+    // ==========================================================================
+    // Internal Helpers
+    // ==========================================================================
+    const notifyStateChange = () => {
+        onStateChange?.(getState());
+    };
+    const notifyDataChange = () => {
+        onDataChange?.();
+    };
+    /**
+     * Rebuild ID index for loaded items
+     */
+    const rebuildIdIndex = () => {
+        idToIndex.clear();
+        const loadedRanges = storage.getLoadedRanges();
+        for (const range of loadedRanges) {
+            for (let i = range.start; i <= range.end; i++) {
+                const item = storage.get(i);
+                if (item && !isPlaceholderItem(item)) {
+                    idToIndex.set(item.id, i);
+                }
+            }
+        }
+    };
+    /**
+     * Update ID index for a specific item
+     */
+    const updateIdIndex = (index, item) => {
+        if (!isPlaceholderItem(item)) {
+            idToIndex.set(item.id, index);
+        }
+    };
+    /**
+     * Remove from ID index
+     */
+    const removeFromIdIndex = (id) => {
+        idToIndex.delete(id);
+    };
+    /**
+     * Get range key for deduplication
+     */
+    const getRangeKey = (start, end) => {
+        return `${start}-${end}`;
+    };
+    // ==========================================================================
+    // State
+    // ==========================================================================
+    // Direct getters for hot-path access (avoid object allocation)
+    const getTotal = () => storage.getTotal();
+    const getCached = () => storage.getCachedCount();
+    const getIsLoading = () => isLoading;
+    const getHasMore = () => hasMore;
+    const getState = () => ({
+        total: storage.getTotal(),
+        cached: storage.getCachedCount(),
+        isLoading,
+        pendingRanges,
+        error,
+        hasMore,
+        cursor,
+    });
+    const getStorage = () => storage;
+    const getPlaceholders = () => getOrCreatePlaceholders();
+    // ==========================================================================
+    // Item Access
+    // ==========================================================================
+    const getItem = (index) => {
+        const item = storage.get(index);
+        // Return loaded item
+        if (item !== undefined) {
+            return item;
+        }
+        // Return placeholder for unloaded within total
+        if (index >= 0 && index < storage.getTotal()) {
+            return getOrCreatePlaceholders().generate(index);
+        }
+        return undefined;
+    };
+    const getItemById = (id) => {
+        const index = idToIndex.get(id);
+        if (index === undefined) {
+            return undefined;
+        }
+        return storage.get(index);
+    };
+    const getIndexById = (id) => {
+        return idToIndex.get(id) ?? -1;
+    };
+    const isItemLoaded = (index) => {
+        const item = storage.get(index);
+        return item !== undefined && !isPlaceholderItem(item);
+    };
+    const getItemsInRange = (start, end) => {
+        const items = [];
+        const total = storage.getTotal();
+        // S2: Batch LRU timestamp update — single Date.now() for all chunks
+        // instead of per-item in storage.get()
+        storage.touchChunksForRange(start, Math.min(end, total - 1));
+        for (let i = start; i <= end && i < total; i++) {
+            const item = storage.get(i);
+            if (item !== undefined) {
+                items.push(item);
+            }
+            else {
+                // Generate placeholder for unloaded
+                items.push(getOrCreatePlaceholders().generate(i));
+            }
+        }
+        return items;
+    };
+    // ==========================================================================
+    // Data Operations
+    // ==========================================================================
+    const setTotal = (total) => {
+        storage.setTotal(total);
+        hasMore = storage.getCachedCount() < total;
+        notifyDataChange();
+        notifyStateChange();
+    };
+    const setItems = (items, offset = 0, total) => {
+        // Analyze structure for placeholders from first batch.
+        // Use getOrCreatePlaceholders() so the profile is captured eagerly —
+        // before the user scrolls fast into unloaded territory. Without this,
+        // the lazy-init guard (`placeholders` is null) causes the first batch
+        // to be missed, and placeholders fall back to a generic single-field
+        // skeleton until a later setItems finally triggers analysis.
+        // Only do this when an adapter exists (async loading); static lists
+        // never generate placeholders so we keep the lazy path for them.
+        if (adapter && items.length > 0) {
+            const pm = getOrCreatePlaceholders();
+            if (!pm.hasAnalyzedStructure()) {
+                pm.analyzeStructure(items);
+            }
+        }
+        // Store items
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            if (item !== undefined) {
+                const index = offset + i;
+                storage.set(index, item);
+                updateIdIndex(index, item);
+            }
+        }
+        // Update total
+        if (total !== undefined) {
+            storage.setTotal(total);
+        }
+        else if (offset + items.length > storage.getTotal()) {
+            storage.setTotal(offset + items.length);
+        }
+        // Only allow hasMore to decrease here, never increase. With sparse
+        // loading, getCachedCount() is almost always less than total (we never
+        // cache all 1M items), so setting hasMore=true would trigger endless
+        // loadMore calls at the bottom. But when all items ARE cached (e.g.
+        // direct setItems use without an adapter), we must turn it off.
+        if (storage.getCachedCount() >= storage.getTotal() &&
+            storage.getTotal() > 0) {
+            hasMore = false;
+        }
+        // Data change MUST fire before onItemsLoaded so that the size cache
+        // is rebuilt (via onDataChange → sizeCache.rebuild) before
+        // onItemsLoaded triggers a forced render. Without this order the
+        // render sees sizeCache.totalSize=0 and only shows a few rows.
+        notifyDataChange();
+        notifyStateChange();
+        onItemsLoaded?.(items, offset, storage.getTotal());
+    };
+    const updateItem = (index, updates) => {
+        const existing = storage.get(index);
+        if (!existing) {
+            return false;
+        }
+        const oldId = existing.id;
+        const updated = { ...existing, ...updates };
+        storage.set(index, updated);
+        // Update ID index if ID changed
+        if (updates.id !== undefined && updates.id !== oldId) {
+            removeFromIdIndex(oldId);
+        }
+        updateIdIndex(index, updated);
+        notifyDataChange();
+        notifyStateChange();
+        return true;
+    };
+    const insertItem = (item, index) => {
+        storage.insert(index, item);
+        rebuildIdIndex();
+        abortAndClearLoads();
+        notifyDataChange();
+        notifyStateChange();
+    };
+    const removeItem = (id) => {
+        const index = idToIndex.get(id);
+        if (index === undefined)
+            return false;
+        // storage.delete now shifts all items after `index` down by 1
+        // and decrements totalItems — no manual setTotal needed.
+        const deleted = storage.delete(index);
+        if (!deleted)
+            return false;
+        rebuildIdIndex();
+        abortAndClearLoads();
+        notifyDataChange();
+        notifyStateChange();
+        return true;
+    };
+    // ==========================================================================
+    // Loading
+    // ==========================================================================
+    const loadRange = async (start, end) => {
+        if (!adapter) {
+            return;
+        }
+        // Find missing chunks — O(range/chunkSize) scan, not O(all-cached-items)
+        const chunkSize = storage.chunkSize;
+        const firstChunk = Math.floor(start / chunkSize);
+        const lastChunk = Math.floor(end / chunkSize);
+        // Abort in-flight loads that belong to a previous scroll position.
+        // Keep a buffer of 2 chunks around the new range — anything further is
+        // stale (the user has scrolled away) and should be cancelled immediately
+        // to free browser connections and server resources.
+        // 2 chunks keeps at most 3 concurrent requests (current + 1 on each side),
+        // well under the browser's 6-connection HTTP/1.1 limit.
+        const keepBuffer = chunkSize * 2;
+        for (const [loadKey, load] of activeLoads) {
+            const dash = loadKey.indexOf("-");
+            const loadStart = parseInt(loadKey.slice(0, dash), 10);
+            if (Math.abs(loadStart - start) > keepBuffer) {
+                load[1].abort();
+                activeLoads.delete(loadKey);
+            }
+        }
+        const chunksToLoad = [];
+        const loadPromises = [];
+        for (let chunkIdx = firstChunk; chunkIdx <= lastChunk; chunkIdx++) {
+            const chunkStart = chunkIdx * chunkSize;
+            const chunkEnd = chunkStart + chunkSize - 1;
+            const key = getRangeKey(chunkStart, chunkEnd);
+            const chunkFull = storage.isChunkFullyLoaded(chunkIdx);
+            if (chunkFull) {
+                continue;
+            }
+            const active = activeLoads.get(key);
+            if (active) {
+                loadPromises.push(active[0]);
+            }
+            else {
+                chunksToLoad.push({ start: chunkStart, end: chunkEnd });
+            }
+        }
+        if (chunksToLoad.length === 0) {
+            if (loadPromises.length > 0) {
+                await Promise.all(loadPromises);
+            }
+            return;
+        }
+        // Enforce max concurrent request limit (zero-allocation path)
+        if (maxConcurrent > 0) {
+            const totalAfterNew = activeLoads.size + chunksToLoad.length;
+            if (totalAfterNew > maxConcurrent) {
+                const center = (start + end) / 2;
+                // Abort furthest existing loads first — new loads are closer to
+                // the current viewport and therefore higher priority.
+                // Single-pass find-max loop instead of spread+map+sort to avoid
+                // heap allocations on the hot path.
+                let toAbort = Math.min(totalAfterNew - maxConcurrent, activeLoads.size);
+                while (toAbort > 0) {
+                    let furthestKey = null;
+                    let furthestDist = -1;
+                    let furthestCtrl = null;
+                    for (const [key, load] of activeLoads) {
+                        const dash = key.indexOf("-");
+                        const loadStart = parseInt(key.slice(0, dash), 10);
+                        const loadEnd = parseInt(key.slice(dash + 1), 10);
+                        const dist = Math.abs((loadStart + loadEnd) / 2 - center);
+                        if (dist > furthestDist) {
+                            furthestDist = dist;
+                            furthestKey = key;
+                            furthestCtrl = load[1];
+                        }
+                    }
+                    if (furthestKey) {
+                        furthestCtrl.abort();
+                        activeLoads.delete(furthestKey);
+                    }
+                    toAbort--;
+                }
+                // If still over limit after aborting existing, trim new chunks.
+                // In-place sort on the already-allocated chunksToLoad array.
+                const remaining = maxConcurrent - activeLoads.size;
+                if (chunksToLoad.length > remaining) {
+                    chunksToLoad.sort((a, b) => {
+                        const distA = Math.abs((a.start + a.end) / 2 - center);
+                        const distB = Math.abs((b.start + b.end) / 2 - center);
+                        return distA - distB;
+                    });
+                    chunksToLoad.length = Math.max(0, remaining);
+                }
+            }
+        }
+        if (chunksToLoad.length === 0) {
+            if (loadPromises.length > 0) {
+                await Promise.all(loadPromises);
+            }
+            return;
+        }
+        // Load chunks that aren't already loading
+        for (const chunk of chunksToLoad) {
+            const key = getRangeKey(chunk.start, chunk.end);
+            // Create the load promise for this chunk
+            const controller = new AbortController();
+            const loadPromise = (async () => {
+                pendingRanges.push(chunk);
+                isLoading = true;
+                error = undefined;
+                notifyStateChange();
+                try {
+                    const limit = chunk.end - chunk.start + 1;
+                    const params = {
+                        offset: chunk.start,
+                        limit,
+                        cursor: chunk.start === cursorValidForOffset ? cursor : undefined,
+                        signal: controller.signal,
+                    };
+                    const response = await adapter.read(params);
+                    // Store items
+                    setItems(response.items, chunk.start, response.total);
+                    // Update cursor — record the offset at which this cursor is valid
+                    // (the next sequential offset after this chunk).
+                    if (response.cursor) {
+                        cursor = response.cursor;
+                        cursorValidForOffset = chunk.start + response.items.length;
+                    }
+                    // Update hasMore — but only if this chunk covers a range at or
+                    // beyond anything we've seen before. This prevents an earlier
+                    // concurrent chunk (hasMore:true) from overriding a later chunk
+                    // (hasMore:false) that already reached the end of the dataset.
+                    const chunkEndOffset = chunk.start + response.items.length;
+                    if (chunkEndOffset >= hasMoreHighWater) {
+                        hasMoreHighWater = chunkEndOffset;
+                        if (response.hasMore !== undefined) {
+                            hasMore = response.hasMore;
+                        }
+                        else if (response.total !== undefined) {
+                            hasMore = chunkEndOffset < response.total;
+                        }
+                    }
+                }
+                catch (err) {
+                    // AbortError is intentional cancellation — don't surface as an error
+                    if (err?.name === "AbortError")
+                        return;
+                    error = err instanceof Error ? err : new Error(String(err));
+                }
+                finally {
+                    activeLoads.delete(key);
+                    const prIdx = pendingRanges.findIndex((r) => r.start === chunk.start && r.end === chunk.end);
+                    if (prIdx >= 0)
+                        pendingRanges.splice(prIdx, 1);
+                    isLoading = activeLoads.size > 0;
+                    notifyStateChange();
+                }
+            })();
+            activeLoads.set(key, [loadPromise, controller]);
+            loadPromises.push(loadPromise);
+        }
+        // Wait for all loads to complete
+        await Promise.all(loadPromises);
+    };
+    const ensureRange = async (start, end) => {
+        // Check if range is already fully loaded
+        const rangeLoaded = storage.isRangeLoaded(start, end);
+        if (rangeLoaded) {
+            return;
+        }
+        await loadRange(start, end);
+    };
+    const loadInitial = async () => {
+        if (!adapter) {
+            return;
+        }
+        await loadRange(0, pageSize - 1);
+    };
+    const loadMore = async () => {
+        if (!adapter || isLoading || !hasMore) {
+            return false;
+        }
+        const currentCached = storage.getCachedCount();
+        const total = storage.getTotal();
+        // Calculate next range to load
+        const start = currentCached;
+        const end = Math.min(start + pageSize - 1, total > 0 ? total - 1 : start + pageSize - 1);
+        if (start >= total && total > 0) {
+            hasMore = false;
+            return false;
+        }
+        await loadRange(start, end);
+        return storage.getCachedCount() > currentCached;
+    };
+    const reload = async () => {
+        // Clear everything — but do NOT load initial data.
+        // The caller (e.g. withAsync feature) decides what range to load,
+        // which avoids a wasted request to offset=0 when the viewport
+        // is scrolled elsewhere.
+        storage.clear();
+        storage.setTotal(0); // Reset total to 0 so scrollbar updates correctly
+        idToIndex.clear();
+        if (placeholders)
+            placeholders.clear();
+        abortAndClearLoads();
+        pendingRanges = [];
+        isLoading = false;
+        cursor = undefined;
+        cursorValidForOffset = undefined;
+        hasMore = true;
+        hasMoreHighWater = 0;
+        error = undefined;
+        notifyDataChange();
+        notifyStateChange();
+    };
+    // ==========================================================================
+    // Memory Management
+    // ==========================================================================
+    const evictDistant = (visibleStart, visibleEnd) => {
+        const evicted = storage.evictDistant(visibleStart, visibleEnd);
+        if (evicted > 0) {
+            // Rebuild ID index after eviction
+            rebuildIdIndex();
+        }
+    };
+    // ==========================================================================
+    // Lifecycle
+    // ==========================================================================
+    const clear = () => {
+        storage.clear();
+        idToIndex.clear();
+        cursor = undefined;
+        cursorValidForOffset = undefined;
+        error = undefined;
+        pendingRanges = [];
+        isLoading = false;
+        notifyDataChange();
+        notifyStateChange();
+    };
+    const reset = () => {
+        storage.reset();
+        idToIndex.clear();
+        if (placeholders)
+            placeholders.clear();
+        cursor = undefined;
+        cursorValidForOffset = undefined;
+        hasMore = true;
+        hasMoreHighWater = 0;
+        error = undefined;
+        pendingRanges = [];
+        isLoading = false;
+        notifyDataChange();
+        notifyStateChange();
+    };
+    // ==========================================================================
+    // Initialization
+    // ==========================================================================
+    // Initialize with provided items
+    if (initialItems && initialItems.length > 0) {
+        setItems(initialItems, 0, initialTotal ?? initialItems.length);
+    }
+    else if (initialTotal !== undefined) {
+        storage.setTotal(initialTotal);
+        notifyDataChange();
+        notifyStateChange();
+    }
+    // ==========================================================================
+    // Return Public API
+    // ==========================================================================
+    return {
+        getState,
+        getTotal,
+        getCached,
+        getIsLoading,
+        getHasMore,
+        getStorage,
+        getPlaceholders,
+        getItem,
+        getItemById,
+        getIndexById,
+        isItemLoaded,
+        getItemsInRange,
+        setTotal,
+        setItems,
+        updateItem,
+        insertItem,
+        removeItem,
+        loadRange,
+        ensureRange,
+        loadInitial,
+        loadMore,
+        reload,
+        evictDistant,
+        clear,
+        reset,
+    };
+};
+// =============================================================================
+// Utility Functions (Re-export from sparse for convenience)
+// =============================================================================
+export { mergeRanges, calculateMissingRanges } from "./sparse";
+export { isPlaceholderItem, filterPlaceholders, countRealItems, } from "./placeholder";