ragcode-context-engine 0.1.0

package/dist/src/watch/event-journal.js

@@ -0,0 +1,81 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { normalizeRepoPath, normalizeUserPath } from "../utils/path.js";
+export class FileEventJournal {
+    journalPath;
+    repoRoot;
+    constructor(journalPath, repoRoot) {
+        this.journalPath = journalPath;
+        this.repoRoot = repoRoot;
+    }
+    static forRepo(repoRoot, fileName = "watch-events.jsonl") {
+        return new FileEventJournal(path.join(repoRoot, ".ragcode", fileName), path.resolve(repoRoot));
+    }
+    async append(entry) {
+        await this.appendBatch([entry]);
+    }
+    async appendBatch(entries) {
+        if (entries.length === 0)
+            return;
+        await fs.mkdir(path.dirname(this.journalPath), { recursive: true });
+        await fs.appendFile(this.journalPath, `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`, "utf8");
+    }
+    async replay() {
+        const content = await fs.readFile(this.journalPath, "utf8").catch((error) => {
+            if (isNotFound(error))
+                return "";
+            throw error;
+        });
+        const entries = [];
+        for (const line of content.split(/\r?\n/)) {
+            if (!line.trim())
+                continue;
+            const parsed = parseEntry(line);
+            if (parsed)
+                entries.push(parsed);
+        }
+        return entries;
+    }
+    async replayPaths() {
+        const paths = new Set();
+        for (const entry of await this.replay()) {
+            paths.add(this.normalizeJournalPath(entry.filePath));
+        }
+        return [...paths].sort();
+    }
+    async truncate() {
+        await fs.mkdir(path.dirname(this.journalPath), { recursive: true });
+        await fs.writeFile(this.journalPath, "", "utf8");
+    }
+    normalizeJournalPath(filePath) {
+        if (this.repoRoot && path.isAbsolute(filePath))
+            return normalizeRepoPath(this.repoRoot, path.resolve(filePath));
+        return normalizeUserPath(filePath);
+    }
+}
+function parseEntry(line) {
+    try {
+        const parsed = JSON.parse(line);
+        if (!parsed || typeof parsed !== "object")
+            return undefined;
+        const candidate = parsed;
+        if (!isWatchEvent(candidate.event))
+            return undefined;
+        if (typeof candidate.filePath !== "string" || !candidate.filePath.trim())
+            return undefined;
+        return {
+            event: candidate.event,
+            filePath: normalizeUserPath(candidate.filePath),
+            observedAtMs: typeof candidate.observedAtMs === "number" ? candidate.observedAtMs : Date.now()
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function isWatchEvent(value) {
+    return value === "add" || value === "change" || value === "unlink" || value === "addDir" || value === "unlinkDir";
+}
+function isNotFound(error) {
+    return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
+}

package/dist/src/watch/file-event-coalescer.d.ts

@@ -0,0 +1,9 @@
+import type { WatcherEventOptions } from "../core/types.js";
+export interface CoalescedFileEvents {
+    dirtyFiles: string[];
+    burstMode: boolean;
+    droppedEvents: number;
+    eventCountByFile: Map<string, number>;
+    lastEventAtMs: number;
+}
+export declare function coalesceFileEvents(repoRoot: string, filePaths: string[], options?: WatcherEventOptions): CoalescedFileEvents;

package/dist/src/watch/file-event-coalescer.js

@@ -0,0 +1,39 @@
+import path from "node:path";
+import { normalizeRepoPath, normalizeUserPath } from "../utils/path.js";
+const DEFAULT_BURST_THRESHOLD = 100;
+const DEFAULT_MAX_DIRTY_FILES = 1_000;
+export function coalesceFileEvents(repoRoot, filePaths, options = {}) {
+    const eventCountByFile = new Map();
+    for (const filePath of filePaths) {
+        const normalized = normalizeEventPath(repoRoot, filePath);
+        if (!normalized)
+            continue;
+        eventCountByFile.set(normalized, (eventCountByFile.get(normalized) ?? 0) + 1);
+    }
+    const maxDirtyFiles = options.maxDirtyFiles ?? DEFAULT_MAX_DIRTY_FILES;
+    const burstThreshold = options.burstThreshold ?? DEFAULT_BURST_THRESHOLD;
+    const allDirtyFiles = [...eventCountByFile.keys()].sort();
+    const dirtyFiles = allDirtyFiles.slice(0, maxDirtyFiles);
+    const droppedEvents = Math.max(0, allDirtyFiles.length - dirtyFiles.length);
+    const burstMode = allDirtyFiles.length >= burstThreshold || droppedEvents > 0;
+    const keptCounts = new Map(dirtyFiles.map((filePath) => [filePath, eventCountByFile.get(filePath) ?? 1]));
+    return {
+        dirtyFiles,
+        burstMode,
+        droppedEvents,
+        eventCountByFile: keptCounts,
+        lastEventAtMs: Date.now()
+    };
+}
+function normalizeEventPath(repoRoot, filePath) {
+    const trimmed = filePath.trim();
+    if (!trimmed)
+        return undefined;
+    if (path.isAbsolute(trimmed)) {
+        const relative = normalizeRepoPath(repoRoot, path.resolve(trimmed));
+        if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative))
+            return undefined;
+        return relative;
+    }
+    return normalizeUserPath(trimmed);
+}

package/dist/src/watch/index-scheduler.d.ts

@@ -0,0 +1,52 @@
+import type { ContextEngine } from "../core/contracts.js";
+import type { RepoIndex, WatcherEventOptions } from "../core/types.js";
+export interface WatchIndexSchedulerOptions extends WatcherEventOptions {
+    batchDelayMs?: number;
+    minQuietMs?: number;
+    maxBatchFiles?: number;
+    maxRetryAttempts?: number;
+    maxRetryDelayMs?: number;
+    autoIndex?: boolean;
+    /**
+     * Idle re-scan interval. Even with no in-process chokidar events, the scheduler periodically
+     * re-reads dirty state from the shared store so that dirty files written by *another* process
+     * (an MCP client calling record_file_events, a manual `ragcode record-events`) get picked up by
+     * the live daemon. This is what makes the SQLite dirty table a cross-process work queue without
+     * any IPC. Set 0 to disable. Defaults to 3000ms.
+     */
+    idlePollMs?: number;
+    onStatus?: (status: WatchIndexSchedulerStatus) => void;
+}
+export interface WatchIndexSchedulerStatus {
+    repoRoot: string;
+    running: boolean;
+    scheduled: boolean;
+    indexing: boolean;
+    pendingFiles: number;
+    indexingFiles: number;
+    lastIndexedAtMs?: number;
+    lastError?: string;
+}
+export declare class WatchIndexScheduler {
+    private readonly engine;
+    private readonly repoRoot;
+    private readonly options;
+    private timer;
+    private idleTimer;
+    private running;
+    private indexing;
+    private lastIndexedAtMs;
+    private lastError;
+    private readonly failureAttemptsByFile;
+    constructor(engine: ContextEngine, repoRoot: string, options?: WatchIndexSchedulerOptions);
+    start(): void;
+    stop(): Promise<void>;
+    private startIdlePoll;
+    schedule(delayMs?: number): void;
+    flush(): Promise<RepoIndex | undefined>;
+    status(): Promise<WatchIndexSchedulerStatus>;
+    private requeueIndexingFiles;
+    private retryDelayMs;
+    private emitStatus;
+    private statusFromIndex;
+}

package/dist/src/watch/index-scheduler.js

@@ -0,0 +1,190 @@
+const DEFAULT_BATCH_DELAY_MS = 750;
+const DEFAULT_MIN_QUIET_MS = 250;
+const DEFAULT_MAX_BATCH_FILES = 1_000;
+const DEFAULT_MAX_RETRY_ATTEMPTS = 5;
+const DEFAULT_MAX_RETRY_DELAY_MS = 30_000;
+const DEFAULT_IDLE_POLL_MS = 3_000;
+export class WatchIndexScheduler {
+    engine;
+    repoRoot;
+    options;
+    timer;
+    idleTimer;
+    running = false;
+    indexing = false;
+    lastIndexedAtMs;
+    lastError;
+    failureAttemptsByFile = new Map();
+    constructor(engine, repoRoot, options = {}) {
+        this.engine = engine;
+        this.repoRoot = repoRoot;
+        this.options = options;
+    }
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        this.startIdlePoll();
+        this.schedule();
+    }
+    async stop() {
+        this.running = false;
+        if (this.timer)
+            clearTimeout(this.timer);
+        if (this.idleTimer)
+            clearInterval(this.idleTimer);
+        this.timer = undefined;
+        this.idleTimer = undefined;
+        while (this.indexing) {
+            await sleep(25);
+        }
+        await this.emitStatus();
+    }
+    // Low-frequency safety net: when no in-process event timer is armed, re-check the shared dirty
+    // store and arm a flush if another process queued work. Cheap (one indexStatus read) and skipped
+    // entirely while a flush is already scheduled or running, so it never competes with event flow.
+    startIdlePoll() {
+        const interval = this.options.idlePollMs ?? DEFAULT_IDLE_POLL_MS;
+        if (interval <= 0 || this.options.autoIndex === false)
+            return;
+        this.idleTimer = setInterval(() => {
+            if (!this.running || this.indexing || this.timer)
+                return;
+            void this.engine
+                .indexStatus(this.repoRoot)
+                .then((status) => {
+                if (this.running && !this.indexing && !this.timer && (status.pendingFileCount > 0 || status.indexingFileCount > 0)) {
+                    this.schedule(0);
+                }
+            })
+                .catch(() => undefined);
+        }, interval);
+        // Don't keep the event loop alive solely for the idle poll; the watcher itself holds the process.
+        this.idleTimer.unref?.();
+    }
+    schedule(delayMs = this.options.batchDelayMs ?? DEFAULT_BATCH_DELAY_MS) {
+        if (!this.running)
+            return;
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = setTimeout(() => {
+            this.timer = undefined;
+            void this.flush();
+        }, delayMs);
+        void this.emitStatus(true);
+    }
+    async flush() {
+        if (!this.running || this.indexing || this.options.autoIndex === false)
+            return undefined;
+        this.indexing = true;
+        try {
+            const status = await this.engine.indexStatus(this.repoRoot);
+            const dirtyFiles = dirtyFilesForBatch(status, this.options.maxBatchFiles ?? DEFAULT_MAX_BATCH_FILES);
+            if (dirtyFiles.length === 0) {
+                this.lastError = undefined;
+                return undefined;
+            }
+            const quietMs = Date.now() - latestDirtySeenAtMs(status);
+            if (quietMs < (this.options.minQuietMs ?? DEFAULT_MIN_QUIET_MS)) {
+                this.schedule((this.options.minQuietMs ?? DEFAULT_MIN_QUIET_MS) - quietMs);
+                return undefined;
+            }
+            await this.engine.markDirtyFilesIndexing(this.repoRoot, dirtyFiles);
+            const index = await this.engine.refreshIndex(this.repoRoot, { affectedFiles: dirtyFiles });
+            this.lastIndexedAtMs = index.indexedAtMs;
+            this.lastError = undefined;
+            for (const filePath of dirtyFiles)
+                this.failureAttemptsByFile.delete(filePath);
+            return index;
+        }
+        catch (error) {
+            this.lastError = error instanceof Error ? error.message : String(error);
+            await this.requeueIndexingFiles(this.lastError);
+            return undefined;
+        }
+        finally {
+            this.indexing = false;
+            await this.emitStatus();
+            if (this.running) {
+                const status = await this.engine.indexStatus(this.repoRoot).catch(() => undefined);
+                if (status && (status.pendingFileCount > 0 || status.indexingFileCount > 0))
+                    this.schedule(this.retryDelayMs(status));
+            }
+        }
+    }
+    async status() {
+        return this.statusFromIndex(await this.engine.indexStatus(this.repoRoot).catch(() => undefined));
+    }
+    async requeueIndexingFiles(reason) {
+        const status = await this.engine.indexStatus(this.repoRoot).catch(() => undefined);
+        if (!status?.freshness.indexingFiles.length)
+            return;
+        const maxAttempts = this.options.maxRetryAttempts ?? DEFAULT_MAX_RETRY_ATTEMPTS;
+        const retryable = [];
+        const deadLetter = [];
+        for (const filePath of status.freshness.indexingFiles) {
+            const attempts = (this.failureAttemptsByFile.get(filePath) ?? 0) + 1;
+            this.failureAttemptsByFile.set(filePath, attempts);
+            if (attempts >= maxAttempts)
+                deadLetter.push(filePath);
+            else
+                retryable.push(filePath);
+        }
+        if (deadLetter.length > 0) {
+            await this.engine.markDirtyFilesDeadLetter(this.repoRoot, deadLetter, `background indexing failed ${maxAttempts} times: ${reason}`);
+            // Drop dead-lettered files from the attempt counter so it can't grow unbounded over a long watch.
+            for (const filePath of deadLetter)
+                this.failureAttemptsByFile.delete(filePath);
+        }
+        if (retryable.length > 0)
+            await this.engine.recordFileEvents(this.repoRoot, retryable, this.options);
+    }
+    retryDelayMs(status) {
+        const attempts = maxRetryAttemptsForFiles(this.failureAttemptsByFile, [...status.freshness.pendingFiles, ...status.freshness.indexingFiles]);
+        if (attempts === 0)
+            return this.options.batchDelayMs ?? DEFAULT_BATCH_DELAY_MS;
+        const baseDelay = this.options.batchDelayMs ?? DEFAULT_BATCH_DELAY_MS;
+        const delay = baseDelay * 2 ** Math.max(0, attempts - 1);
+        return Math.min(delay, this.options.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS);
+    }
+    async emitStatus(scheduled = Boolean(this.timer)) {
+        if (!this.options.onStatus)
+            return;
+        this.options.onStatus(await this.statusFromIndex(await this.engine.indexStatus(this.repoRoot).catch(() => undefined), scheduled));
+    }
+    statusFromIndex(status, scheduled = Boolean(this.timer)) {
+        return {
+            repoRoot: this.repoRoot,
+            running: this.running,
+            scheduled,
+            indexing: this.indexing,
+            pendingFiles: status?.pendingFileCount ?? 0,
+            indexingFiles: status?.indexingFileCount ?? 0,
+            lastIndexedAtMs: this.lastIndexedAtMs,
+            lastError: this.lastError
+        };
+    }
+}
+function dirtyFilesForBatch(status, maxBatchFiles) {
+    return [...new Set([...status.freshness.pendingFiles, ...status.freshness.indexingFiles])].sort().slice(0, maxBatchFiles);
+}
+function latestDirtySeenAtMs(status) {
+    let latest = 0;
+    for (const file of status.freshness.dirtyFiles) {
+        if (file.lastSeenAtMs > latest)
+            latest = file.lastSeenAtMs;
+    }
+    return latest;
+}
+function maxRetryAttemptsForFiles(attemptsByFile, filePaths) {
+    let maxAttempts = 0;
+    for (const filePath of filePaths) {
+        const attempts = attemptsByFile.get(filePath) ?? 0;
+        if (attempts > maxAttempts)
+            maxAttempts = attempts;
+    }
+    return maxAttempts;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/src/watch/watch-daemon.d.ts

@@ -0,0 +1,73 @@
+import type { ContextEngine } from "../core/contracts.js";
+import { FileEventJournal, type WatchEventJournalEntry } from "./event-journal.js";
+import { type WatchIndexSchedulerOptions, type WatchIndexSchedulerStatus } from "./index-scheduler.js";
+export interface FileWatchDaemonOptions extends Omit<WatchIndexSchedulerOptions, "onStatus"> {
+    awaitWriteFinishMs?: number;
+    pollIntervalMs?: number;
+    usePolling?: boolean;
+    flushEventsMs?: number;
+    maxBufferedEvents?: number;
+    maxFlushWaitMs?: number;
+    flushRetryMaxDelayMs?: number;
+    maxFileBytes?: number;
+    indexOnStart?: boolean;
+    /** Interval for refreshing the on-disk heartbeat even when idle. Defaults to 10s. */
+    heartbeatIntervalMs?: number;
+    /**
+     * Acquire the per-repo watcher lock on start (refusing if another live watcher holds it) and
+     * publish a heartbeat file. Defaults to true. Set false for embedded/in-process usage where
+     * no cross-process coordination is needed (e.g. the dashboard's observation daemon).
+     */
+    manageLifecycleFiles?: boolean;
+    journal?: FileEventJournal;
+    onEvent?: (event: WatchEventJournalEntry) => void;
+    onStatus?: (status: WatchDaemonStatus) => void;
+}
+export interface WatchDaemonStatus {
+    repoRoot: string;
+    running: boolean;
+    ready: boolean;
+    bufferedEvents: number;
+    scheduler: WatchIndexSchedulerStatus;
+}
+export declare class FileWatchDaemon {
+    private readonly engine;
+    private readonly options;
+    private watcher;
+    private readonly journal;
+    private readonly scheduler;
+    private readonly repoRoot;
+    private readonly bufferedPaths;
+    private flushTimer;
+    private maxFlushTimer;
+    private flushRetryTimer;
+    private journalQueue;
+    private pendingJournalEntries;
+    private firstBufferedAtMs;
+    private flushFailureCount;
+    private lastError;
+    private running;
+    private ready;
+    private lockHandle;
+    private heartbeatTimer;
+    private lastIndexedAtMs;
+    constructor(engine: ContextEngine, repoRoot: string, options?: FileWatchDaemonOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    status(): Promise<WatchDaemonStatus>;
+    private startHeartbeat;
+    private publishHeartbeat;
+    private ensureIndexed;
+    private replayJournal;
+    private handleWatchEvent;
+    private recordEvent;
+    private scheduleEventFlush;
+    private flushBufferedEvents;
+    private flushBufferedEventsLocked;
+    private flushJournalEntriesLocked;
+    private scheduleFlushRetry;
+    private isIgnored;
+    private emitStatus;
+    private enqueueJournalOperation;
+    private setLastError;
+}