scrapely 1.2.3 → 1.2.5

package/dist/storage/rq-list-requests.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.upperBoundSorted = upperBoundSorted;
+exports.listRequestsPage = listRequestsPage;
+/**
+ * Index of the first element strictly greater than `target` in a
+ * lexicographically sorted array (binary search).
+ *
+ * Unlike indexOf(), it returns the correct position even when `target` itself
+ * is absent, so a cursor item deleted between pages advances pagination past
+ * its old position instead of restarting from page 1 (which used to loop
+ * pagination clients back to the first page forever). This also matches the
+ * documented exclusiveStartId semantics: "all requests up to this one
+ * (including) are skipped from the result".
+ */
+function upperBoundSorted(sortedIds, target) {
+    let lo = 0;
+    let hi = sortedIds.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >>> 1;
+        if (sortedIds[mid] <= target) {
+            lo = mid + 1;
+        }
+        else {
+            hi = mid;
+        }
+    }
+    return lo;
+}
+/** Map a stored request JSON blob to the API response shape. */
+function parseStoredRequest(raw) {
+    const req = JSON.parse(raw);
+    return {
+        id: req.id,
+        retryCount: req.retryCount ?? 0,
+        uniqueKey: req.uniqueKey,
+        url: req.url,
+        method: req.method ?? 'GET',
+        loadedUrl: req.loadedUrl ?? null,
+        payload: req.payload ?? null,
+        noRetry: req.noRetry ?? false,
+        errorMessages: req.errorMessages ?? null,
+        headers: req.headers ?? null,
+        userData: req.userData ?? null,
+        handledAt: req.handledAt ?? null,
+    };
+}
+/**
+ * One page of request-queue listing: filtering + cursor pagination in two
+ * round-trips, shared by the storage client and the API server route.
+ *
+ * Round-trip 1 — a single pipeline:
+ *   - HKEYS: all request IDs (field names only, not the bodies);
+ *   - ZRANGEBYSCORE locked (now +inf — only when filtering by 'locked'. The
+ *     locked ZSET is scored by lockUntil ms, so one exclusive-bound range read
+ *     exactly replaces the old per-id ZSCORE loop (score > now);
+ *   - ZRANGE pending 0 -1 — only when filtering by 'pending'.
+ * Round-trip 2 — one HMGET for the page's request bodies.
+ */
+async function listRequestsPage(redis, keys, params) {
+    const hasLockedFilter = params.filter?.includes('locked') ?? false;
+    const hasPendingFilter = params.filter?.includes('pending') ?? false;
+    const readPipeline = redis.pipeline();
+    readPipeline.hkeys(keys.requestsKey);
+    if (hasLockedFilter)
+        readPipeline.zrangebyscore(keys.lockedKey, `(${params.now}`, '+inf');
+    // ioredis 6 types: stop must be string | Buffer (not number)
+    if (hasPendingFilter)
+        readPipeline.zrange(keys.pendingKey, 0, '-1');
+    const results = await readPipeline.exec();
+    const at = (i) => {
+        const value = results?.[i]?.[1];
+        return Array.isArray(value) ? value : [];
+    };
+    const allRequestIds = at(0);
+    // Sort IDs for consistent lexicographic ordering (API contract)
+    allRequestIds.sort();
+    const activeLockedIds = hasLockedFilter ? new Set(at(1)) : new Set();
+    const pendingIds = hasPendingFilter
+        ? new Set(at(hasLockedFilter ? 2 : 1))
+        : new Set();
+    // Filter requests based on filter parameter
+    let filteredIds = allRequestIds;
+    if (hasLockedFilter && hasPendingFilter) {
+        // Both filters: return locked OR pending (not handled-only requests)
+        filteredIds = allRequestIds.filter(id => activeLockedIds.has(id) || pendingIds.has(id));
+    }
+    else if (hasLockedFilter) {
+        // Only locked: requests with non-expired locks
+        filteredIds = allRequestIds.filter(id => activeLockedIds.has(id));
+    }
+    else if (hasPendingFilter) {
+        // Only pending: requests in pending set
+        filteredIds = allRequestIds.filter(id => pendingIds.has(id));
+    }
+    // If no filter, return all requests
+    // Advance past the cursor (first ID strictly greater than startId —
+    // see upperBoundSorted for why this must not be indexOf()).
+    const startIndex = params.startId ? upperBoundSorted(filteredIds, params.startId) : 0;
+    // Apply pagination
+    const paginatedIds = filteredIds.slice(startIndex, startIndex + params.limit);
+    const hasMore = startIndex + params.limit < filteredIds.length;
+    // Fetch full request data for the page in one round-trip
+    const items = [];
+    if (paginatedIds.length > 0) {
+        const raws = await redis.hmget(keys.requestsKey, ...paginatedIds);
+        for (const raw of raws) {
+            if (!raw)
+                continue;
+            items.push(parseStoredRequest(raw));
+        }
+    }
+    return { items, hasMore };
+}

package/dist/storage/rq-lock-script.d.ts

@@ -0,0 +1,56 @@
+import type Redis from 'ioredis';
+/**
+ * Atomic list-and-lock. Returns a two-element Lua table:
+ *   { lockedIds, ZCARD(lockedKey) }
+ * which ioredis delivers as [string[], number]. The ZCARD is taken after the
+ * script's expired-lock purge and after this call's own locks are added, so it
+ * equals the number of active locks held by ANY client — exactly what the
+ * Apify `queueHasLockedRequests` spec describes ("locked by any client, either
+ * the one calling the endpoint or a different one"). Callers get the flag from
+ * this same atomic reply instead of re-reading the whole locked + pending sets.
+ */
+export declare const LIST_AND_LOCK_HEAD_LUA = "\nlocal pendingKey = KEYS[1]\nlocal lockedKey = KEYS[2]\nlocal lockOwnersKey = KEYS[3]\nlocal requestsKey = KEYS[4]\nlocal now = tonumber(ARGV[1])\nlocal lockUntil = tonumber(ARGV[2])\nlocal limit = tonumber(ARGV[3])\nlocal ownerValue = ARGV[4]\nlocal scanLimit = tonumber(ARGV[5])\n\nredis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)\n\nlocal pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)\nlocal lockedIds = {}\n\nfor _, id in ipairs(pendingIds) do\n  if #lockedIds >= limit then break end\n  local lockScore = redis.call('ZSCORE', lockedKey, id)\n  if not lockScore or tonumber(lockScore) <= now then\n    if redis.call('HEXISTS', requestsKey, id) == 1 then\n      redis.call('ZADD', lockedKey, lockUntil, id)\n      redis.call('HSET', lockOwnersKey, id, ownerValue)\n      table.insert(lockedIds, id)\n    end\n  end\nend\n\n-- Everything left in lockedKey is an active lock (score > now): the purge\n-- above removed the expired ones and we just added ours. An O(1) ZCARD here\n-- replaces the callers' old tail of ZRANGEBYSCORE(locked) + full\n-- ZRANGE(pending, 0, -1) scans on every head fetch.\nreturn { lockedIds, redis.call('ZCARD', lockedKey) }\n";
+export interface ListAndLockHeadKeys {
+    pendingKey: string;
+    lockedKey: string;
+    lockOwnersKey: string;
+    requestsKey: string;
+}
+export interface ListAndLockHeadParams {
+    now: number;
+    lockUntil: number;
+    limit: number;
+    ownerValue: string;
+    /** How many pending IDs to scan (should be >= limit). */
+    scanLimit?: number;
+}
+export interface ListAndLockHeadAtomicResult {
+    /** IDs locked by this call, in queue-head order. */
+    lockedIds: string[];
+    /**
+     * Active locks (lockUntil > now) immediately after this atomic operation,
+     * including the locks this call just created (any client's locks count).
+     */
+    activeLockCount: number;
+}
+export declare function listAndLockHeadAtomic(redis: Redis, keys: ListAndLockHeadKeys, params: ListAndLockHeadParams): Promise<ListAndLockHeadAtomicResult>;
+/** Returns true if lockOwners entry matches the caller's clientKey (and optional runId). */
+export declare function matchesLockOwner(owner: string, clientKey: string, runId?: string | null): boolean;
+/**
+ * Atomic unlock-by-owner: scans lockOwners, unlocks every request whose owner
+ * matches the clientKey (and optional runId) using the same rules as the old
+ * per-request loop, optionally re-queueing unlocked requests at the forefront.
+ * Returns the number of unlocked requests. One round-trip, no race window
+ * between ZREM and HDEL, and the whole owners hash never leaves the server.
+ */
+export declare const UNLOCK_REQUESTS_LUA = "\nlocal lockedKey = KEYS[1]\nlocal lockOwnersKey = KEYS[2]\nlocal pendingKey = KEYS[3]\nlocal clientKey = ARGV[1]\nlocal runId = ARGV[2]\nlocal forefront = ARGV[3]\n\nlocal expectedOwner = clientKey\nif runId ~= '' then\n  expectedOwner = clientKey .. ':' .. runId\nend\n\nlocal clientKeyPrefix = clientKey .. ':'\n\nlocal flat = redis.call('HGETALL', lockOwnersKey)\nlocal unlockedCount = 0\n\nfor i = 1, #flat, 2 do\n  local requestId = flat[i]\n  local owner = flat[i + 1]\n  -- Per Apify spec: in an Actor run, unlock both (1) requests locked by this\n  -- specific run (clientKey:runId) and (2) requests locked by the same\n  -- clientKey without run context (clientKey), but NOT requests locked by the\n  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).\n  if owner == expectedOwner or owner == clientKey\n      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then\n    redis.call('ZREM', lockedKey, requestId)\n    redis.call('HDEL', lockOwnersKey, requestId)\n    if forefront == '1' then\n      redis.call('ZADD', pendingKey, 0, requestId)\n    end\n    unlockedCount = unlockedCount + 1\n  end\nend\n\nreturn unlockedCount\n";
+export interface UnlockRequestsKeys {
+    lockedKey: string;
+    lockOwnersKey: string;
+    pendingKey: string;
+}
+export declare function unlockRequestsAtomic(redis: Redis, keys: UnlockRequestsKeys, params: {
+    clientKey: string;
+    forefront?: boolean;
+    runId?: string;
+}): Promise<number>;

package/dist/storage/rq-lock-script.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UNLOCK_REQUESTS_LUA = exports.LIST_AND_LOCK_HEAD_LUA = void 0;
+exports.listAndLockHeadAtomic = listAndLockHeadAtomic;
+exports.matchesLockOwner = matchesLockOwner;
+exports.unlockRequestsAtomic = unlockRequestsAtomic;
+/**
+ * Atomic list-and-lock. Returns a two-element Lua table:
+ *   { lockedIds, ZCARD(lockedKey) }
+ * which ioredis delivers as [string[], number]. The ZCARD is taken after the
+ * script's expired-lock purge and after this call's own locks are added, so it
+ * equals the number of active locks held by ANY client — exactly what the
+ * Apify `queueHasLockedRequests` spec describes ("locked by any client, either
+ * the one calling the endpoint or a different one"). Callers get the flag from
+ * this same atomic reply instead of re-reading the whole locked + pending sets.
+ */
+exports.LIST_AND_LOCK_HEAD_LUA = `
+local pendingKey = KEYS[1]
+local lockedKey = KEYS[2]
+local lockOwnersKey = KEYS[3]
+local requestsKey = KEYS[4]
+local now = tonumber(ARGV[1])
+local lockUntil = tonumber(ARGV[2])
+local limit = tonumber(ARGV[3])
+local ownerValue = ARGV[4]
+local scanLimit = tonumber(ARGV[5])
+
+redis.call('ZREMRANGEBYSCORE', lockedKey, '-inf', now)
+
+local pendingIds = redis.call('ZRANGE', pendingKey, 0, scanLimit - 1)
+local lockedIds = {}
+
+for _, id in ipairs(pendingIds) do
+  if #lockedIds >= limit then break end
+  local lockScore = redis.call('ZSCORE', lockedKey, id)
+  if not lockScore or tonumber(lockScore) <= now then
+    if redis.call('HEXISTS', requestsKey, id) == 1 then
+      redis.call('ZADD', lockedKey, lockUntil, id)
+      redis.call('HSET', lockOwnersKey, id, ownerValue)
+      table.insert(lockedIds, id)
+    end
+  end
+end
+
+-- Everything left in lockedKey is an active lock (score > now): the purge
+-- above removed the expired ones and we just added ours. An O(1) ZCARD here
+-- replaces the callers' old tail of ZRANGEBYSCORE(locked) + full
+-- ZRANGE(pending, 0, -1) scans on every head fetch.
+return { lockedIds, redis.call('ZCARD', lockedKey) }
+`;
+async function listAndLockHeadAtomic(redis, keys, params) {
+    const scanLimit = params.scanLimit ?? Math.max(params.limit * 4, 100);
+    const result = await redis.eval(exports.LIST_AND_LOCK_HEAD_LUA, 4, keys.pendingKey, keys.lockedKey, keys.lockOwnersKey, keys.requestsKey, String(params.now), String(params.lockUntil), String(params.limit), params.ownerValue, String(scanLimit));
+    // Expected reply shape: [string[], number]. Tolerate degraded replies the
+    // same way the old string[] version did (null / JSON-stringified table).
+    if (!Array.isArray(result) || result.length < 2) {
+        return { lockedIds: [], activeLockCount: 0 };
+    }
+    let lockedIds = result[0];
+    if (typeof lockedIds === 'string') {
+        try {
+            lockedIds = JSON.parse(lockedIds);
+        }
+        catch {
+            lockedIds = [];
+        }
+    }
+    if (!Array.isArray(lockedIds))
+        lockedIds = [];
+    const count = result[1];
+    const activeLockCount = typeof count === 'number' ? count : parseInt(String(count), 10) || 0;
+    return { lockedIds, activeLockCount };
+}
+/** Returns true if lockOwners entry matches the caller's clientKey (and optional runId). */
+function matchesLockOwner(owner, clientKey, runId) {
+    const expectedOwner = runId ? `${clientKey}:${runId}` : clientKey;
+    return owner === expectedOwner || owner === clientKey;
+}
+/**
+ * Atomic unlock-by-owner: scans lockOwners, unlocks every request whose owner
+ * matches the clientKey (and optional runId) using the same rules as the old
+ * per-request loop, optionally re-queueing unlocked requests at the forefront.
+ * Returns the number of unlocked requests. One round-trip, no race window
+ * between ZREM and HDEL, and the whole owners hash never leaves the server.
+ */
+exports.UNLOCK_REQUESTS_LUA = `
+local lockedKey = KEYS[1]
+local lockOwnersKey = KEYS[2]
+local pendingKey = KEYS[3]
+local clientKey = ARGV[1]
+local runId = ARGV[2]
+local forefront = ARGV[3]
+
+local expectedOwner = clientKey
+if runId ~= '' then
+  expectedOwner = clientKey .. ':' .. runId
+end
+
+local clientKeyPrefix = clientKey .. ':'
+
+local flat = redis.call('HGETALL', lockOwnersKey)
+local unlockedCount = 0
+
+for i = 1, #flat, 2 do
+  local requestId = flat[i]
+  local owner = flat[i + 1]
+  -- Per Apify spec: in an Actor run, unlock both (1) requests locked by this
+  -- specific run (clientKey:runId) and (2) requests locked by the same
+  -- clientKey without run context (clientKey), but NOT requests locked by the
+  -- same clientKey in DIFFERENT runs (clientKey:otherRunId).
+  if owner == expectedOwner or owner == clientKey
+      or (runId == '' and string.sub(owner, 1, #clientKeyPrefix) == clientKeyPrefix) then
+    redis.call('ZREM', lockedKey, requestId)
+    redis.call('HDEL', lockOwnersKey, requestId)
+    if forefront == '1' then
+      redis.call('ZADD', pendingKey, 0, requestId)
+    end
+    unlockedCount = unlockedCount + 1
+  end
+end
+
+return unlockedCount
+`;
+async function unlockRequestsAtomic(redis, keys, params) {
+    const result = await redis.eval(exports.UNLOCK_REQUESTS_LUA, 3, keys.lockedKey, keys.lockOwnersKey, keys.pendingKey, params.clientKey, params.runId ?? '', params.forefront ? '1' : '0');
+    return typeof result === 'number' ? result : parseInt(String(result), 10) || 0;
+}

package/dist/storage/run-metadata.d.ts

@@ -0,0 +1,98 @@
+import type { DragonflyClient } from './dragonfly-client.js';
+import type { UserPlan } from './ttl-manager.js';
+/**
+ * Metadata for a single actor run
+ */
+export interface RunMetadata {
+    /** Unique run ID */
+    runId: string;
+    /** User ID who initiated the run */
+    userId: string;
+    /** User's plan (affects TTL) */
+    plan: UserPlan;
+    /** When the run was created */
+    createdAt: Date;
+    /** When the run finished (null if still running) */
+    finishedAt: Date | null;
+    /** Storage IDs created for this run */
+    storageIds: {
+        dataset: string;
+        kvStore: string;
+        requestQueue: string;
+    };
+    /** Current run status */
+    status: RunStatus;
+    /** Total storage bytes used (for billing) */
+    storageBytes: number;
+    /** Container ID if running in Docker */
+    containerId?: string;
+    /** Actor name/version */
+    actorName?: string;
+    actorVersion?: string;
+}
+export type RunStatus = 'running' | 'success' | 'failed' | 'timed_out' | 'killed';
+/**
+ * Manages run metadata in Dragonfly.
+ * Keys are stored with TTL matching the storage TTL so metadata is cleaned up
+ * when the associated storage expires.
+ */
+export declare class RunMetadataManager {
+    private client;
+    constructor(client: DragonflyClient);
+    /**
+     * Get the Redis sorted-set key that indexes all run IDs for a user.
+     * Sorted by createdAt timestamp (score) so ZREVRANGE gives most-recent-first.
+     */
+    private getUserRunsKey;
+    /**
+     * Create a new run metadata entry
+     */
+    create(data: Omit<RunMetadata, 'finishedAt' | 'storageBytes'> & {
+        storageBytes?: number;
+    }): Promise<void>;
+    /**
+     * Get run metadata by ID
+     */
+    get(runId: string): Promise<RunMetadata | null>;
+    /**
+     * Update run metadata.
+     * Silently no-ops if the run does not exist (prevents ghost hash creation).
+     */
+    update(runId: string, updates: Partial<Pick<RunMetadata, 'status' | 'storageBytes' | 'finishedAt' | 'containerId'>>): Promise<void>;
+    /**
+     * Mark a run as finished with a specific status
+     */
+    finish(runId: string, status: RunStatus, storageBytes?: number): Promise<void>;
+    /**
+     * Delete run metadata and remove from the user's sorted-set index.
+     */
+    delete(runId: string): Promise<void>;
+    /**
+     * List runs by user ID, most-recent first.
+     *
+     * Uses a per-user sorted set (ZADD on create, ZREM on delete) instead of a
+     * full KEYS scan so lookups are O(log N + M) rather than O(N-total-runs).
+     * Stale entries left behind by TTL expiry are detected and pruned lazily.
+     */
+    listByUser(userId: string): Promise<RunMetadata[]>;
+    /**
+     * Enforce the top-N indefinite retention policy for a user.
+     *
+     * - The KEEP_N_RUNS most-recent runs get PERSIST (no TTL → kept forever).
+     * - Runs beyond that window that currently have no TTL (were previously in the
+     *   top N) get the plan-based TTL re-applied so they will eventually expire.
+     * - The per-user sorted set is PERSISTed while protected runs exist, and gets
+     *   a plan-based TTL when no protected runs remain (prevents a memory leak for
+     *   inactive users whose sorted set would otherwise never expire).
+     *
+     * Called after every create() and delete() so the window stays accurate.
+     *
+     * Performance: previously used N sequential awaits per phase; now each phase is
+     * a single pipelined round-trip regardless of the number of runs.
+     */
+    private refreshTopNProtection;
+    /**
+     * Get the Redis key for a run
+     */
+    private getRunKey;
+}

package/dist/storage/run-metadata.js

@@ -0,0 +1,256 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunMetadataManager = void 0;
+const billing_tracker_js_1 = require("./billing-tracker.js");
+/**
+ * Number of most-recent runs per user that are retained indefinitely (no TTL).
+ * Mirrors Apify's model: the 10 latest runs are always accessible regardless of plan.
+ * Older runs beyond this window fall back to the standard plan-based TTL.
+ */
+const KEEP_N_RUNS = 10;
+/**
+ * Manages run metadata in Dragonfly.
+ * Keys are stored with TTL matching the storage TTL so metadata is cleaned up
+ * when the associated storage expires.
+ */
+class RunMetadataManager {
+    client;
+    constructor(client) {
+        this.client = client;
+    }
+    /**
+     * Get the Redis sorted-set key that indexes all run IDs for a user.
+     * Sorted by createdAt timestamp (score) so ZREVRANGE gives most-recent-first.
+     */
+    getUserRunsKey(userId) {
+        return this.client.key('users', userId, 'runs');
+    }
+    /**
+     * Create a new run metadata entry
+     */
+    async create(data) {
+        const key = this.getRunKey(data.runId);
+        const metadata = {
+            ...data,
+            finishedAt: null,
+            storageBytes: data.storageBytes ?? 0,
+        };
+        await this.client.redis.hset(key, {
+            runId: metadata.runId,
+            userId: metadata.userId,
+            plan: metadata.plan,
+            createdAt: metadata.createdAt.toISOString(),
+            finishedAt: '',
+            storageIds: JSON.stringify(metadata.storageIds),
+            status: metadata.status,
+            storageBytes: String(metadata.storageBytes),
+            containerId: metadata.containerId ?? '',
+            actorName: metadata.actorName ?? '',
+            actorVersion: metadata.actorVersion ?? '',
+        });
+        // Register in the per-user sorted set (score = ms timestamp → most-recent-first via ZREVRANGE)
+        const userRunsKey = this.getUserRunsKey(metadata.userId);
+        await this.client.redis.zadd(userRunsKey, metadata.createdAt.getTime(), metadata.runId);
+        // Set TTL on run metadata - use dataset TTL as reference (longest retention)
+        const ttl = this.client.ttlManager.getTtl('dataset', metadata.plan, false);
+        if (ttl) {
+            await this.client.redis.expire(key, ttl);
+        }
+        // Enforce top-N indefinite retention — will PERSIST this run if it's in the
+        // most-recent KEEP_N_RUNS and may re-apply TTL to any run that was just demoted.
+        await this.refreshTopNProtection(metadata.userId);
+    }
+    /**
+     * Get run metadata by ID
+     */
+    async get(runId) {
+        const key = this.getRunKey(runId);
+        const data = await this.client.redis.hgetall(key);
+        if (!data || !data.runId)
+            return null;
+        return {
+            runId: data.runId,
+            userId: data.userId,
+            plan: data.plan,
+            createdAt: new Date(data.createdAt),
+            finishedAt: data.finishedAt ? new Date(data.finishedAt) : null,
+            storageIds: JSON.parse(data.storageIds),
+            status: data.status,
+            storageBytes: parseInt(data.storageBytes || '0', 10),
+            containerId: data.containerId || undefined,
+            actorName: data.actorName || undefined,
+            actorVersion: data.actorVersion || undefined,
+        };
+    }
+    /**
+     * Update run metadata.
+     * Silently no-ops if the run does not exist (prevents ghost hash creation).
+     */
+    async update(runId, updates) {
+        const key = this.getRunKey(runId);
+        // Guard: only update if the hash already exists
+        const exists = await this.client.redis.exists(key);
+        if (!exists)
+            return;
+        const updateData = {};
+        if (updates.status !== undefined) {
+            updateData.status = updates.status;
+        }
+        if (updates.storageBytes !== undefined) {
+            updateData.storageBytes = String(updates.storageBytes);
+        }
+        if (updates.finishedAt !== undefined && updates.finishedAt !== null) {
+            updateData.finishedAt = updates.finishedAt.toISOString();
+        }
+        if (updates.containerId !== undefined) {
+            updateData.containerId = updates.containerId;
+        }
+        if (Object.keys(updateData).length > 0) {
+            await this.client.redis.hset(key, updateData);
+        }
+    }
+    /**
+     * Mark a run as finished with a specific status
+     */
+    async finish(runId, status, storageBytes) {
+        await this.update(runId, {
+            status,
+            finishedAt: new Date(),
+            ...(storageBytes !== undefined && { storageBytes }),
+        });
+    }
+    /**
+     * Delete run metadata and remove from the user's sorted-set index.
+     */
+    async delete(runId) {
+        // Fetch metadata first so we know the userId for the sorted-set cleanup
+        const metadata = await this.get(runId);
+        const key = this.getRunKey(runId);
+        await this.client.redis.del(key);
+        if (metadata) {
+            await new billing_tracker_js_1.BillingTracker(this.client).releaseStorageBytes(runId, metadata.userId);
+            const userRunsKey = this.getUserRunsKey(metadata.userId);
+            await this.client.redis.zrem(userRunsKey, runId);
+            // Re-evaluate top-N so the run that was sitting at position N+1 (if any)
+            // gets promoted to PERSIST now that there's a slot open.
+            await this.refreshTopNProtection(metadata.userId);
+        }
+    }
+    /**
+     * List runs by user ID, most-recent first.
+     *
+     * Uses a per-user sorted set (ZADD on create, ZREM on delete) instead of a
+     * full KEYS scan so lookups are O(log N + M) rather than O(N-total-runs).
+     * Stale entries left behind by TTL expiry are detected and pruned lazily.
+     */
+    async listByUser(userId) {
+        const userRunsKey = this.getUserRunsKey(userId);
+        // ZREVRANGE returns members sorted by score descending (most-recent first)
+        const runIds = await this.client.redis.zrevrange(userRunsKey, 0, -1);
+        const runs = [];
+        const staleIds = [];
+        for (const runId of runIds) {
+            const metadata = await this.get(runId);
+            if (!metadata) {
+                // Run hash expired via TTL or was deleted without going through delete()
+                staleIds.push(runId);
+                continue;
+            }
+            runs.push(metadata);
+        }
+        // Lazily prune stale entries from the sorted set
+        if (staleIds.length > 0) {
+            await this.client.redis.zrem(userRunsKey, ...staleIds);
+        }
+        return runs; // Already in descending createdAt order from ZREVRANGE
+    }
+    /**
+     * Enforce the top-N indefinite retention policy for a user.
+     *
+     * - The KEEP_N_RUNS most-recent runs get PERSIST (no TTL → kept forever).
+     * - Runs beyond that window that currently have no TTL (were previously in the
+     *   top N) get the plan-based TTL re-applied so they will eventually expire.
+     * - The per-user sorted set is PERSISTed while protected runs exist, and gets
+     *   a plan-based TTL when no protected runs remain (prevents a memory leak for
+     *   inactive users whose sorted set would otherwise never expire).
+     *
+     * Called after every create() and delete() so the window stays accurate.
+     *
+     * Performance: previously used N sequential awaits per phase; now each phase is
+     * a single pipelined round-trip regardless of the number of runs.
+     */
+    async refreshTopNProtection(userId) {
+        const userRunsKey = this.getUserRunsKey(userId);
+        const allRunIds = await this.client.redis.zrevrange(userRunsKey, 0, -1);
+        // All runs have been deleted — clean up the now-empty sorted set immediately
+        // rather than leaving it to drift in memory indefinitely.
+        if (allRunIds.length === 0) {
+            await this.client.redis.del(userRunsKey);
+            return;
+        }
+        const topIds = allRunIds.slice(0, KEEP_N_RUNS);
+        const olderIds = allRunIds.slice(KEEP_N_RUNS);
+        // ── Phase 1: PERSIST all top-N run keys and the sorted set itself ─────────
+        // The sorted set must outlive the runs it indexes, so it gets PERSIST too.
+        if (topIds.length > 0) {
+            const persistPipeline = this.client.redis.pipeline();
+            for (const runId of topIds) {
+                persistPipeline.persist(this.getRunKey(runId));
+            }
+            persistPipeline.persist(userRunsKey);
+            await persistPipeline.exec();
+        }
+        // ── Phase 2: Batch-read TTLs for all older-than-top-N run keys ───────────
+        if (olderIds.length === 0)
+            return;
+        const ttlPipeline = this.client.redis.pipeline();
+        for (const runId of olderIds) {
+            ttlPipeline.ttl(this.getRunKey(runId));
+        }
+        const ttlResults = await ttlPipeline.exec();
+        // Collect only the runs that were demoted (exist but have no expiry, TTL = -1)
+        const demotedIds = olderIds.filter((_, i) => ttlResults?.[i]?.[1] === -1);
+        if (demotedIds.length === 0) {
+            // No demoted runs, but if top-N is also empty we still need to set the
+            // sorted-set TTL so inactive users' index keys don't live forever.
+            if (topIds.length === 0) {
+                const sortedSetTtl = this.client.ttlManager.getTtl('dataset', 'free', false);
+                if (sortedSetTtl)
+                    await this.client.redis.expire(userRunsKey, sortedSetTtl, 'NX');
+            }
+            return;
+        }
+        // ── Phase 3: Batch-read the plan field for each demoted run ──────────────
+        const planPipeline = this.client.redis.pipeline();
+        for (const runId of demotedIds) {
+            planPipeline.hget(this.getRunKey(runId), 'plan');
+        }
+        const planResults = await planPipeline.exec();
+        // ── Phase 4: Apply plan-based TTLs in a single pipeline ──────────────────
+        const expirePipeline = this.client.redis.pipeline();
+        for (let i = 0; i < demotedIds.length; i++) {
+            const runId = demotedIds[i];
+            const plan = (planResults?.[i]?.[1] ?? 'free');
+            const expireSecs = this.client.ttlManager.getTtl('dataset', plan, false);
+            if (expireSecs) {
+                expirePipeline.expire(this.getRunKey(runId), expireSecs);
+            }
+        }
+        // If there are no top-N protected runs, also set the sorted-set TTL so it
+        // doesn't persist forever once all remaining runs age out of protection.
+        // NX ensures we only apply this once; subsequent calls won't shorten it.
+        if (topIds.length === 0) {
+            const sortedSetTtl = this.client.ttlManager.getTtl('dataset', 'free', false);
+            if (sortedSetTtl)
+                expirePipeline.expire(userRunsKey, sortedSetTtl, 'NX');
+        }
+        await expirePipeline.exec();
+    }
+    /**
+     * Get the Redis key for a run
+     */
+    getRunKey(runId) {
+        return this.client.key('run', runId);
+    }
+}
+exports.RunMetadataManager = RunMetadataManager;