sidekick-shared 0.26.4 → 0.26.6

package/dist/providers/codex.js

@@ -54,6 +54,7 @@ const codexParser_1 = require("../parsers/codexParser");
 const codexDatabase_1 = require("./codexDatabase");
 const codexProfiles_1 = require("../codexProfiles");
 const rolloutWalker_1 = require("./rolloutWalker");
+const directoryListingCache_1 = require("./directoryListingCache");
 const diagnostics_1 = require("./diagnostics");
 const modelContext_1 = require("../modelContext");
 const observedContextWindows_1 = require("../observedContextWindows");
@@ -193,7 +194,13 @@ function readSessionMeta(rolloutPath) {
     }
     return null;
 }
-async function readSessionMetaAsync(rolloutPath) {
+/**
+ * First line of a rollout, chunk-read under the same 256 KiB cap as
+ * readFirstLines(). `complete` is false while the line has no newline yet
+ * (a rollout that was just created) or the file could not be read, so
+ * callers know not to cache the answer.
+ */
+async function readFirstLineAsync(rolloutPath) {
     // Mirrors readSessionMeta()/readFirstLines(): chunk-read until the first
     // line is complete under the same 256 KiB cap — session_meta carries
     // `instructions` and can far exceed a single chunk.
@@ -216,18 +223,26 @@ async function readSessionMetaAsync(rolloutPath) {
             newlineFound = chunk.includes(0x0a);
         }
         const firstLine = Buffer.concat(chunks).toString('utf8').split('\n', 1)[0]?.trim();
-        if (!firstLine)
-            return null;
-        const parsed = JSON.parse(firstLine);
-        return parsed.type === 'session_meta' ? parsed.payload : null;
+        return { line: firstLine || null, complete: newlineFound || totalRead >= maxBytes };
     }
     catch {
-        return null;
+        return { line: null, complete: false };
     }
     finally {
         await handle?.close().catch(() => undefined);
     }
 }
+function parseSessionMetaLine(line) {
+    if (!line)
+        return null;
+    try {
+        const parsed = JSON.parse(line);
+        return parsed.type === 'session_meta' ? parsed.payload : null;
+    }
+    catch {
+        return null;
+    }
+}
 /**
  * Read the CWD from a rollout file's session_meta. Cached per path, bounded
  * so a long-lived host watching a large history cannot grow it without limit
@@ -257,11 +272,23 @@ async function readSessionCwdAsync(rolloutPath) {
     const cached = cwdCache.get(rolloutPath);
     if (cached !== undefined)
         return cached;
-    const meta = await readSessionMetaAsync(rolloutPath);
-    const cwd = meta?.cwd || null;
-    rememberCwd(rolloutPath, cwd);
+    const { line, complete } = await readFirstLineAsync(rolloutPath);
+    const cwd = parseSessionMetaLine(line)?.cwd || null;
+    // A rollout observed before its session_meta line is flushed has no cwd
+    // yet; caching that null would hide the session from its workspace until
+    // the entry aged out, so only conclusive answers are remembered.
+    if (cwd !== null || complete)
+        rememberCwd(rolloutPath, cwd);
     return cwd;
 }
+/** Built-in enumerators report explicit zeros rather than leaving a counter unset. */
+function initializeCounters(stats) {
+    if (!stats)
+        return;
+    stats.directoriesListed ??= 0;
+    stats.directoriesStatted ??= 0;
+    stats.filesStatted ??= 0;
+}
 /** Check if a session's CWD matches a workspace path. */
 function cwdMatches(sessionCwd, workspacePath) {
     const normalizedSession = normalizePath(sessionCwd);
@@ -646,6 +673,8 @@ class CodexProvider {
     dynamicContextWindowLimit = null;
     lastRateLimitsData = null;
     lastTokenUsageData = null;
+    /** Dated rollout-directory listings reused across async enumerations. */
+    directoryCache = new directoryListingCache_1.DirectoryListingCache();
     constructor(options = {}) {
         this.diagnostics = new diagnostics_1.ProviderDiagnosticTracker(this.id, options);
     }
@@ -802,10 +831,22 @@ class CodexProvider {
         this.recordDatabaseFallback('listAllSessionFiles', true);
         return files;
     }
-    async listSessionFilesAsync(workspacePath) {
+    async listSessionFilesAsync(workspacePath, options = {}) {
         const results = [];
-        // One capped walk across the homes (deduplicated, newest first).
-        for (const file of await (0, rolloutWalker_1.walkRolloutFilesAsync)(getSessionsDirs())) {
+        initializeCounters(options.stats);
+        // One capped walk across the homes (deduplicated, newest first) through
+        // the directory cache. With a workspace filter the limit applies after
+        // filtering, so the walk itself is bounded only by its file cap.
+        const files = await (0, rolloutWalker_1.walkRolloutFilesAsync)(getSessionsDirs(), {
+            limit: workspacePath ? undefined : options.limit,
+            since: options.since,
+            cache: this.directoryCache,
+            revalidateRecent: options.revalidateRecent,
+            stats: options.stats,
+        });
+        for (const file of files) {
+            if (options.limit !== undefined && results.length >= options.limit)
+                break;
             const info = { ...file };
             if (workspacePath) {
                 const cwd = await readSessionCwdAsync(file.path);
@@ -821,6 +862,47 @@ class CodexProvider {
         this.recordDatabaseFallback('listSessionFilesAsync', true);
         return results;
     }
+    /**
+     * Resolve a recursive-watch event under a sessions root with one stat.
+     * Rollouts are matched by name at any depth the walker would visit; a
+     * dated directory appearing or vanishing needs a full walk.
+     */
+    async statWatchedSessionFile(root, relativePath, workspacePath) {
+        const segments = (0, directoryListingCache_1.splitWatchedPath)(relativePath);
+        if (!segments)
+            return { status: 'unknown' };
+        const fullPath = path.join(root, ...segments);
+        const name = segments[segments.length - 1];
+        if (segments.length - 1 > rolloutWalker_1.DEFAULT_ROLLOUT_WALK_MAX_DEPTH)
+            return { status: 'ignored' };
+        if (!(0, rolloutWalker_1.isRolloutFile)(name)) {
+            this.directoryCache.invalidate(fullPath);
+            try {
+                const stat = await fs.promises.stat(fullPath);
+                return stat.isDirectory() ? { status: 'unknown' } : { status: 'ignored' };
+            }
+            catch {
+                return { status: 'unknown' };
+            }
+        }
+        const sessionId = (0, rolloutWalker_1.extractRolloutSessionId)(name);
+        const stat = await this.directoryCache.refreshFile(fullPath);
+        if (!stat || stat.sizeBytes <= 0)
+            return { status: 'missing', path: fullPath, sessionId };
+        const file = {
+            path: fullPath,
+            mtime: new Date(stat.mtimeMs),
+            sizeBytes: stat.sizeBytes,
+            sessionId,
+        };
+        if (workspacePath) {
+            const cwd = await readSessionCwdAsync(fullPath);
+            if (!cwd || !cwdMatches(cwd, workspacePath))
+                return { status: 'ignored' };
+            file.workspacePath = cwd;
+        }
+        return { status: 'present', file };
+    }
     getAllProjectFolders(workspacePath) {
         const folders = [];
         const seenCwds = new Map();
@@ -1153,6 +1235,7 @@ class CodexProvider {
         this.dynamicContextWindowLimit = null;
         this.lastRateLimitsData = null;
         this.lastTokenUsageData = null;
+        this.directoryCache.invalidate();
     }
     recordDatabaseFallback(operation, usable) {
         if (!this.dbInitialized) {

package/dist/providers/directoryListingCache.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Directory listings keyed by path and validated by the directory's own
+ * mtime and inode.
+ *
+ * A create, delete, or rename inside a directory bumps that directory's
+ * mtime; an append to an existing file does not. So a listing can be reused
+ * for as long as the directory's mtime and inode are unchanged, and the per
+ * file size/mtime it carries is only refreshed by `refreshFile()` (the watch
+ * event path) or by `revalidateRecent` on catch-up passes. Full discovery is
+ * then one stat per directory instead of one readdir per directory plus one
+ * stat per file.
+ *
+ * @module providers/directoryListingCache
+ */
+import * as fs from 'fs';
+export interface CachedFileStat {
+    sizeBytes: number;
+    mtimeMs: number;
+}
+export interface CachedDirectoryListing {
+    /** Directory mtime when the listing was taken. */
+    mtimeMs: number;
+    ino: number;
+    /** Wall-clock time the listing was taken (coarse-mtime guard). */
+    listedAtMs: number;
+    /** Files accepted by the caller's filter, with the stat seen last. */
+    files: Map<string, CachedFileStat>;
+    /** Subdirectory names, unsorted. */
+    subdirectories: string[];
+}
+/** Payload-free counters a caller can pass to measure what a pass cost. */
+export interface DirectoryListingCounters {
+    directoriesListed?: number;
+    directoriesStatted?: number;
+    filesStatted?: number;
+}
+export interface DirectoryListingCacheOptions {
+    /** Bound on cached directories, least-recently-used first (default 10 000). */
+    maxDirectories?: number;
+    /**
+     * Files whose cached mtime is within this window are re-stat'ed when a
+     * caller asks for `revalidateRecent` (default 30 minutes).
+     */
+    recentWindowMs?: number;
+}
+export interface ListDirectoryOptions {
+    /** Re-stat files modified within the recent window even if the directory is unchanged. */
+    revalidateRecent?: boolean;
+    /** A stat of the directory the caller already took; saves one syscall. */
+    dirStat?: fs.Stats;
+    stats?: DirectoryListingCounters;
+}
+export declare const DEFAULT_DIRECTORY_CACHE_MAX_DIRECTORIES = 10000;
+export declare const DEFAULT_DIRECTORY_CACHE_RECENT_WINDOW_MS: number;
+export declare class DirectoryListingCache {
+    private readonly listings;
+    private readonly maxDirectories;
+    private readonly recentWindowMs;
+    constructor(options?: DirectoryListingCacheOptions);
+    get size(): number;
+    /** The cached listing without touching the filesystem, if any. */
+    peek(dir: string): CachedDirectoryListing | undefined;
+    /**
+     * List `dir`, reusing the cached listing when the directory is unchanged.
+     * Returns null when `dir` is missing, unreadable, or not a directory.
+     */
+    listDirectory(dir: string, accept: (name: string) => boolean, options?: ListDirectoryOptions): Promise<CachedDirectoryListing | null>;
+    /**
+     * Stat one file and reflect the result in its parent's cached listing when
+     * that listing exists. Returns null when the path is missing or not a file.
+     * Never creates a listing: a partial listing must not pass for a full one.
+     */
+    refreshFile(fullPath: string, stats?: DirectoryListingCounters): Promise<CachedFileStat | null>;
+    /** Drop one directory's listing, or every listing. */
+    invalidate(dir?: string): void;
+    private revalidateRecent;
+    private isRecent;
+    private touch;
+}
+/**
+ * Split a watcher-reported root-relative path into segments. Both separators
+ * are accepted (Windows watchers report backslashes). Returns null for an
+ * empty path or one that escapes the root.
+ */
+export declare function splitWatchedPath(relativePath: string): string[] | null;

package/dist/providers/directoryListingCache.js

@@ -0,0 +1,244 @@
+"use strict";
+/**
+ * Directory listings keyed by path and validated by the directory's own
+ * mtime and inode.
+ *
+ * A create, delete, or rename inside a directory bumps that directory's
+ * mtime; an append to an existing file does not. So a listing can be reused
+ * for as long as the directory's mtime and inode are unchanged, and the per
+ * file size/mtime it carries is only refreshed by `refreshFile()` (the watch
+ * event path) or by `revalidateRecent` on catch-up passes. Full discovery is
+ * then one stat per directory instead of one readdir per directory plus one
+ * stat per file.
+ *
+ * @module providers/directoryListingCache
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DirectoryListingCache = exports.DEFAULT_DIRECTORY_CACHE_RECENT_WINDOW_MS = exports.DEFAULT_DIRECTORY_CACHE_MAX_DIRECTORIES = void 0;
+exports.splitWatchedPath = splitWatchedPath;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+exports.DEFAULT_DIRECTORY_CACHE_MAX_DIRECTORIES = 10_000;
+exports.DEFAULT_DIRECTORY_CACHE_RECENT_WINDOW_MS = 30 * 60_000;
+/**
+ * A directory whose mtime is within this many milliseconds of the moment it
+ * was listed may still receive entries that land on the same coarse mtime
+ * (HFS+, exFAT, SMB), so it is re-listed rather than trusted.
+ */
+const COARSE_MTIME_GUARD_MS = 2_000;
+class DirectoryListingCache {
+    listings = new Map();
+    maxDirectories;
+    recentWindowMs;
+    constructor(options = {}) {
+        this.maxDirectories = Math.max(1, Math.floor(options.maxDirectories ?? exports.DEFAULT_DIRECTORY_CACHE_MAX_DIRECTORIES));
+        this.recentWindowMs = Math.max(0, options.recentWindowMs ?? exports.DEFAULT_DIRECTORY_CACHE_RECENT_WINDOW_MS);
+    }
+    get size() {
+        return this.listings.size;
+    }
+    /** The cached listing without touching the filesystem, if any. */
+    peek(dir) {
+        return this.listings.get(dir);
+    }
+    /**
+     * List `dir`, reusing the cached listing when the directory is unchanged.
+     * Returns null when `dir` is missing, unreadable, or not a directory.
+     */
+    async listDirectory(dir, accept, options = {}) {
+        let dirStat = options.dirStat;
+        if (!dirStat) {
+            try {
+                dirStat = await fs.promises.stat(dir);
+                count(options.stats, 'directoriesStatted');
+            }
+            catch {
+                this.listings.delete(dir);
+                return null;
+            }
+        }
+        if (!dirStat.isDirectory()) {
+            this.listings.delete(dir);
+            return null;
+        }
+        const cached = this.listings.get(dir);
+        const unchanged = cached !== undefined &&
+            cached.mtimeMs === dirStat.mtimeMs &&
+            cached.ino === dirStat.ino &&
+            dirStat.mtimeMs < cached.listedAtMs - COARSE_MTIME_GUARD_MS;
+        if (cached && unchanged) {
+            this.touch(dir, cached);
+            if (options.revalidateRecent)
+                await this.revalidateRecent(dir, cached, options.stats);
+            return cached;
+        }
+        let entries;
+        try {
+            entries = await fs.promises.readdir(dir, { withFileTypes: true });
+            count(options.stats, 'directoriesListed');
+        }
+        catch {
+            this.listings.delete(dir);
+            return null;
+        }
+        const listedAtMs = Date.now();
+        const files = new Map();
+        const subdirectories = [];
+        for (const entry of entries) {
+            if (entry.isDirectory()) {
+                subdirectories.push(entry.name);
+                continue;
+            }
+            const isCandidateFile = entry.isFile() && accept(entry.name);
+            if (!isCandidateFile && !entry.isSymbolicLink())
+                continue;
+            const fullPath = path.join(dir, entry.name);
+            // A file already in the listing keeps its cached stat unless it is
+            // recent enough to be live; a directory change is a create, delete, or
+            // rename, and only the new names need a fresh stat.
+            const previous = cached?.files.get(entry.name);
+            if (isCandidateFile && previous && !this.isRecent(previous, listedAtMs)) {
+                files.set(entry.name, previous);
+                continue;
+            }
+            if (entry.isSymbolicLink() && !accept(entry.name)) {
+                // Symlinked directories are followed so linked project folders count.
+                try {
+                    const target = await fs.promises.stat(fullPath);
+                    count(options.stats, 'directoriesStatted');
+                    if (target.isDirectory())
+                        subdirectories.push(entry.name);
+                }
+                catch {
+                    // Dangling link.
+                }
+                continue;
+            }
+            try {
+                const stat = await fs.promises.stat(fullPath);
+                count(options.stats, 'filesStatted');
+                if (stat.isFile())
+                    files.set(entry.name, { sizeBytes: stat.size, mtimeMs: stat.mtimeMs });
+                else if (stat.isDirectory())
+                    subdirectories.push(entry.name);
+            }
+            catch {
+                // Vanished between readdir and stat.
+            }
+        }
+        const listing = {
+            mtimeMs: dirStat.mtimeMs,
+            ino: dirStat.ino,
+            listedAtMs,
+            files,
+            subdirectories,
+        };
+        this.touch(dir, listing);
+        return listing;
+    }
+    /**
+     * Stat one file and reflect the result in its parent's cached listing when
+     * that listing exists. Returns null when the path is missing or not a file.
+     * Never creates a listing: a partial listing must not pass for a full one.
+     */
+    async refreshFile(fullPath, stats) {
+        const dir = path.dirname(fullPath);
+        const name = path.basename(fullPath);
+        const listing = this.listings.get(dir);
+        try {
+            const stat = await fs.promises.stat(fullPath);
+            count(stats, 'filesStatted');
+            if (!stat.isFile()) {
+                listing?.files.delete(name);
+                return null;
+            }
+            const fileStat = { sizeBytes: stat.size, mtimeMs: stat.mtimeMs };
+            listing?.files.set(name, fileStat);
+            return fileStat;
+        }
+        catch {
+            listing?.files.delete(name);
+            return null;
+        }
+    }
+    /** Drop one directory's listing, or every listing. */
+    invalidate(dir) {
+        if (dir === undefined)
+            this.listings.clear();
+        else
+            this.listings.delete(dir);
+    }
+    async revalidateRecent(dir, listing, stats) {
+        const now = Date.now();
+        for (const [name, cached] of [...listing.files]) {
+            if (!this.isRecent(cached, now))
+                continue;
+            await this.refreshFile(path.join(dir, name), stats);
+        }
+    }
+    isRecent(stat, now) {
+        return now - stat.mtimeMs <= this.recentWindowMs;
+    }
+    touch(dir, listing) {
+        this.listings.delete(dir);
+        this.listings.set(dir, listing);
+        while (this.listings.size > this.maxDirectories) {
+            const oldest = this.listings.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.listings.delete(oldest);
+        }
+    }
+}
+exports.DirectoryListingCache = DirectoryListingCache;
+/**
+ * Split a watcher-reported root-relative path into segments. Both separators
+ * are accepted (Windows watchers report backslashes). Returns null for an
+ * empty path or one that escapes the root.
+ */
+function splitWatchedPath(relativePath) {
+    const segments = relativePath.split(/[\\/]+/).filter((segment) => segment.length > 0);
+    if (segments.length === 0)
+        return null;
+    if (segments.some((segment) => segment === '.' || segment === '..'))
+        return null;
+    return segments;
+}
+function count(stats, field) {
+    if (!stats)
+        return;
+    stats[field] = (stats[field] ?? 0) + 1;
+}

package/dist/providers/rolloutWalker.d.ts

@@ -8,14 +8,20 @@
  * `limit` can stop early, stats each file exactly once, and returns results
  * newest first with the size and session id already attached.
  *
- * With `limit`, the early exit is by directory date: a rollout that was
- * created days ago but is still being written lands in its creation day's
- * directory, so a very small limit can miss it in favour of newer-dated,
- * older-modified files. Callers that need exact mtime order over the whole
- * history omit `limit` (the `maxFiles` cap still applies).
+ * With `limit` or `since`, the early exit is by directory date: a rollout
+ * that was created days ago but is still being written lands in its creation
+ * day's directory, so a very small limit (or a recent `since`) can miss it in
+ * favour of newer-dated, older-modified files. Callers that need exact mtime
+ * order over the whole history omit both (the `maxFiles` cap still applies);
+ * a live session missed this way is picked up by its next watch event.
+ *
+ * The async walker accepts a {@link DirectoryListingCache}: unchanged
+ * directories are then one stat each instead of a readdir plus a stat per
+ * file, which is what makes repeated discovery over a large history cheap.
  *
  * @module providers/rolloutWalker
  */
+import type { DirectoryListingCache, DirectoryListingCounters } from './directoryListingCache';
 export interface RolloutFileInfo {
     path: string;
     mtime: Date;
@@ -30,10 +36,21 @@ export interface WalkRolloutFilesOptions {
     maxFiles?: number;
     /** Stop once this many files are collected, visiting newest-dated directories first. */
     limit?: number;
+    /**
+     * Skip files modified before this epoch-ms, and skip dated directories
+     * whose whole day, month, or year ended more than a day before it.
+     */
+    since?: number;
     /** Only files whose name carries this session id (case-insensitive). */
     sessionId?: string;
     /** Include zero-byte files (default false). */
     includeEmpty?: boolean;
+    /** Reuse directory listings across walks (async walker only). */
+    cache?: DirectoryListingCache;
+    /** With `cache`: re-stat recently modified files even in unchanged directories. */
+    revalidateRecent?: boolean;
+    /** Payload-free counters of what the walk cost. */
+    stats?: DirectoryListingCounters;
 }
 export declare const DEFAULT_ROLLOUT_WALK_MAX_DEPTH = 6;
 export declare const DEFAULT_ROLLOUT_WALK_MAX_FILES = 20000;
@@ -44,6 +61,12 @@ export declare function isRolloutFile(filename: string): boolean;
  * Format: rollout-<timestamp>-<uuid>.jsonl -> <uuid>
  */
 export declare function extractRolloutSessionId(filename: string): string;
+/**
+ * True when `segments` (the directory names below the root, e.g. `['2026',
+ * '09', '04']`) are all numeric and describe a year, month, or day that ended
+ * more than a day before `since`. Non-numeric names are never pruned.
+ */
+export declare function isDatedDirectoryBefore(segments: readonly string[], since: number): boolean;
 /**
  * Walk one or more `sessions` roots for rollout files, newest first.
  * Missing or unreadable directories are skipped; paths are deduplicated.