ragcode-context-engine 0.1.0

package/dist/src/watch/watch-daemon.js

@@ -0,0 +1,368 @@
+import path from "node:path";
+import chokidar from "chokidar";
+import { shouldIgnoreDirectory, shouldIgnoreFile } from "../indexing/ignore-policy.js";
+import { normalizeRepoPath, normalizeUserPath } from "../utils/path.js";
+import { FileEventJournal } from "./event-journal.js";
+import { WatchIndexScheduler } from "./index-scheduler.js";
+import { acquireWatcherLock, clearHeartbeat, writeHeartbeat } from "./watcher-liveness.js";
+const DEFAULT_EVENT_FLUSH_MS = 250;
+const DEFAULT_AWAIT_WRITE_FINISH_MS = 500;
+const DEFAULT_MAX_FILE_BYTES = 1_000_000;
+const DEFAULT_MAX_BUFFERED_EVENTS = 1_000;
+const DEFAULT_MAX_FLUSH_WAIT_MS = 5_000;
+const DEFAULT_FLUSH_RETRY_MAX_DELAY_MS = 30_000;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000;
+export class FileWatchDaemon {
+    engine;
+    options;
+    watcher;
+    journal;
+    scheduler;
+    repoRoot;
+    bufferedPaths = new Set();
+    flushTimer;
+    maxFlushTimer;
+    flushRetryTimer;
+    journalQueue = Promise.resolve();
+    pendingJournalEntries = [];
+    firstBufferedAtMs;
+    flushFailureCount = 0;
+    lastError;
+    running = false;
+    ready = false;
+    lockHandle;
+    heartbeatTimer;
+    lastIndexedAtMs;
+    constructor(engine, repoRoot, options = {}) {
+        this.engine = engine;
+        this.options = options;
+        this.repoRoot = path.resolve(repoRoot);
+        this.journal = options.journal ?? FileEventJournal.forRepo(this.repoRoot);
+        this.scheduler = new WatchIndexScheduler(engine, this.repoRoot, {
+            ...options,
+            onStatus: (status) => {
+                if (status.lastIndexedAtMs)
+                    this.lastIndexedAtMs = status.lastIndexedAtMs;
+                options.onStatus?.({
+                    repoRoot: this.repoRoot,
+                    running: this.running,
+                    ready: this.ready,
+                    bufferedEvents: this.bufferedPaths.size,
+                    scheduler: status
+                });
+                // Refresh the heartbeat on every scheduler tick so an actively-indexing daemon publishes
+                // freshness faster than the idle interval, and so pending/indexing counts stay current.
+                if (this.running && this.options.manageLifecycleFiles !== false)
+                    void this.publishHeartbeat();
+            }
+        });
+    }
+    async start() {
+        if (this.running)
+            return;
+        // Acquire the per-repo lock first, before any indexing work, so a second watcher on the same
+        // repo fails fast (throwing WatcherLockError) instead of racing the live one as a second writer.
+        if (this.options.manageLifecycleFiles !== false) {
+            this.lockHandle = acquireWatcherLock(this.repoRoot);
+        }
+        this.running = true;
+        await this.ensureIndexed();
+        await this.replayJournal();
+        this.scheduler.start();
+        this.startHeartbeat();
+        this.watcher = chokidar.watch(this.repoRoot, {
+            cwd: this.repoRoot,
+            persistent: true,
+            ignoreInitial: true,
+            followSymlinks: false,
+            usePolling: this.options.usePolling,
+            interval: this.options.pollIntervalMs,
+            atomic: true,
+            awaitWriteFinish: {
+                stabilityThreshold: this.options.awaitWriteFinishMs ?? DEFAULT_AWAIT_WRITE_FINISH_MS,
+                pollInterval: Math.min(100, this.options.awaitWriteFinishMs ?? DEFAULT_AWAIT_WRITE_FINISH_MS)
+            },
+            ignored: (candidate, stats) => this.isIgnored(candidate, stats)
+        });
+        this.watcher
+            .on("add", (filePath) => this.handleWatchEvent("add", filePath))
+            .on("change", (filePath) => this.handleWatchEvent("change", filePath))
+            .on("unlink", (filePath) => this.handleWatchEvent("unlink", filePath))
+            .on("addDir", (filePath) => this.handleWatchEvent("addDir", filePath))
+            .on("unlinkDir", (filePath) => this.handleWatchEvent("unlinkDir", filePath))
+            .on("ready", () => {
+            this.ready = true;
+            void this.emitStatus();
+        })
+            .on("error", (error) => {
+            this.options.onStatus?.({
+                repoRoot: this.repoRoot,
+                running: this.running,
+                ready: this.ready,
+                bufferedEvents: this.bufferedPaths.size,
+                scheduler: {
+                    repoRoot: this.repoRoot,
+                    running: true,
+                    scheduled: false,
+                    indexing: false,
+                    pendingFiles: 0,
+                    indexingFiles: 0,
+                    lastError: this.setLastError(error)
+                }
+            });
+        });
+    }
+    async stop() {
+        this.running = false;
+        if (this.flushTimer)
+            clearTimeout(this.flushTimer);
+        if (this.maxFlushTimer)
+            clearTimeout(this.maxFlushTimer);
+        if (this.flushRetryTimer)
+            clearTimeout(this.flushRetryTimer);
+        if (this.heartbeatTimer)
+            clearInterval(this.heartbeatTimer);
+        this.flushTimer = undefined;
+        this.maxFlushTimer = undefined;
+        this.flushRetryTimer = undefined;
+        this.heartbeatTimer = undefined;
+        await this.watcher?.close();
+        this.watcher = undefined;
+        try {
+            await this.flushBufferedEvents();
+            await this.journalQueue;
+        }
+        finally {
+            await this.scheduler.stop();
+            this.ready = false;
+            await this.emitStatus();
+            // Tear down lifecycle files last, after the scheduler has drained, so a reader never sees
+            // a cleared heartbeat while the daemon is still writing the index. Clear the heartbeat
+            // before releasing the lock so the window where the lock exists without a heartbeat
+            // (which a reader would classify as "dead") is as small as possible.
+            //
+            // Only touch lifecycle files if WE own the lock. A second watcher that failed to acquire the
+            // lock (start() threw) reaches stop() via the CLI's shutdown path with lockHandle undefined —
+            // it must NOT clear the live watcher's heartbeat or release its lock.
+            if (this.options.manageLifecycleFiles !== false && this.lockHandle) {
+                await clearHeartbeat(this.repoRoot).catch(() => undefined);
+                this.lockHandle.release();
+                this.lockHandle = undefined;
+            }
+        }
+    }
+    async status() {
+        return {
+            repoRoot: this.repoRoot,
+            running: this.running,
+            ready: this.ready,
+            bufferedEvents: this.bufferedPaths.size,
+            scheduler: await this.scheduler.status()
+        };
+    }
+    // Publish a fresh heartbeat immediately, then on a fixed interval so doctor/dashboard/MCP can
+    // distinguish a live-but-idle daemon from a dead one even when no file events are flowing. The
+    // scheduler's own onStatus callback also refreshes the heartbeat on every tick (see constructor),
+    // so an actively-indexing daemon heartbeats more often than the interval.
+    startHeartbeat() {
+        if (this.options.manageLifecycleFiles === false)
+            return;
+        void this.publishHeartbeat();
+        const interval = this.options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
+        this.heartbeatTimer = setInterval(() => {
+            void this.publishHeartbeat();
+        }, interval);
+        this.heartbeatTimer.unref?.();
+    }
+    async publishHeartbeat() {
+        if (this.options.manageLifecycleFiles === false || !this.lockHandle)
+            return;
+        const scheduler = await this.scheduler.status().catch(() => undefined);
+        if (scheduler?.lastIndexedAtMs)
+            this.lastIndexedAtMs = scheduler.lastIndexedAtMs;
+        const heartbeat = {
+            pid: this.lockHandle.info.pid,
+            hostname: this.lockHandle.info.hostname,
+            repoRoot: this.repoRoot,
+            startedAtMs: this.lockHandle.info.startedAtMs,
+            lastHeartbeatMs: Date.now(),
+            lastIndexedAtMs: this.lastIndexedAtMs,
+            pendingFiles: scheduler?.pendingFiles ?? 0,
+            indexingFiles: scheduler?.indexingFiles ?? 0,
+            ready: this.ready,
+            lastError: this.lastError ?? scheduler?.lastError
+        };
+        await writeHeartbeat(this.repoRoot, heartbeat).catch((error) => {
+            // A heartbeat write failure shouldn't take down the watcher; surface it as lastError.
+            this.setLastError(error);
+        });
+    }
+    async ensureIndexed() {
+        if (this.options.indexOnStart === false) {
+            await this.engine.indexStatus(this.repoRoot);
+            return;
+        }
+        try {
+            await this.engine.indexStatus(this.repoRoot);
+        }
+        catch {
+            await this.engine.indexRepo(this.repoRoot);
+        }
+    }
+    async replayJournal() {
+        const replayed = await this.journal.replayPaths();
+        if (replayed.length === 0)
+            return;
+        await this.engine.recordFileEvents(this.repoRoot, replayed, this.options);
+        await this.journal.truncate();
+        this.scheduler.schedule(0);
+    }
+    handleWatchEvent(event, rawPath) {
+        void this.recordEvent(event, rawPath).catch((error) => {
+            this.setLastError(error);
+            void this.emitStatus();
+        });
+    }
+    async recordEvent(event, rawPath) {
+        const filePath = normalizeWatchPath(this.repoRoot, rawPath);
+        if (!filePath)
+            return;
+        const entry = { event, filePath, observedAtMs: Date.now() };
+        await this.enqueueJournalOperation(async () => {
+            this.pendingJournalEntries.push(entry);
+            await this.flushJournalEntriesLocked();
+            this.options.onEvent?.(entry);
+            this.bufferedPaths.add(filePath);
+            this.firstBufferedAtMs ??= entry.observedAtMs;
+            this.scheduleEventFlush();
+        });
+    }
+    scheduleEventFlush() {
+        if (this.bufferedPaths.size >= (this.options.maxBufferedEvents ?? DEFAULT_MAX_BUFFERED_EVENTS)) {
+            if (this.flushTimer)
+                clearTimeout(this.flushTimer);
+            this.flushTimer = undefined;
+            void this.flushBufferedEvents().catch((error) => {
+                this.setLastError(error);
+                this.scheduleFlushRetry();
+                void this.emitStatus();
+            });
+            return;
+        }
+        if (this.flushTimer)
+            clearTimeout(this.flushTimer);
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = undefined;
+            void this.flushBufferedEvents().catch((error) => {
+                this.setLastError(error);
+                this.scheduleFlushRetry();
+                void this.emitStatus();
+            });
+        }, this.options.flushEventsMs ?? DEFAULT_EVENT_FLUSH_MS);
+        if (!this.maxFlushTimer) {
+            const elapsed = this.firstBufferedAtMs ? Date.now() - this.firstBufferedAtMs : 0;
+            this.maxFlushTimer = setTimeout(() => {
+                this.maxFlushTimer = undefined;
+                if (this.flushTimer)
+                    clearTimeout(this.flushTimer);
+                this.flushTimer = undefined;
+                void this.flushBufferedEvents().catch((error) => {
+                    this.setLastError(error);
+                    this.scheduleFlushRetry();
+                    void this.emitStatus();
+                });
+            }, Math.max(0, (this.options.maxFlushWaitMs ?? DEFAULT_MAX_FLUSH_WAIT_MS) - elapsed));
+        }
+    }
+    async flushBufferedEvents() {
+        await this.enqueueJournalOperation(() => this.flushBufferedEventsLocked());
+    }
+    async flushBufferedEventsLocked() {
+        await this.flushJournalEntriesLocked();
+        if (this.bufferedPaths.size === 0)
+            return;
+        const paths = [...this.bufferedPaths].sort();
+        await this.engine.recordFileEvents(this.repoRoot, paths, this.options);
+        this.bufferedPaths.clear();
+        this.firstBufferedAtMs = undefined;
+        this.flushFailureCount = 0;
+        if (this.maxFlushTimer)
+            clearTimeout(this.maxFlushTimer);
+        if (this.flushRetryTimer)
+            clearTimeout(this.flushRetryTimer);
+        this.maxFlushTimer = undefined;
+        this.flushRetryTimer = undefined;
+        await this.journal.truncate();
+        this.lastError = undefined;
+        this.scheduler.schedule();
+        await this.emitStatus();
+    }
+    async flushJournalEntriesLocked() {
+        if (this.pendingJournalEntries.length === 0)
+            return;
+        const entries = this.pendingJournalEntries.splice(0);
+        try {
+            await this.journal.appendBatch(entries);
+        }
+        catch (error) {
+            this.pendingJournalEntries.unshift(...entries);
+            throw error;
+        }
+    }
+    scheduleFlushRetry() {
+        if (!this.running || this.flushRetryTimer)
+            return;
+        this.flushFailureCount += 1;
+        const baseDelay = this.options.flushEventsMs ?? DEFAULT_EVENT_FLUSH_MS;
+        const delay = Math.min(baseDelay * 2 ** Math.max(0, this.flushFailureCount - 1), this.options.flushRetryMaxDelayMs ?? DEFAULT_FLUSH_RETRY_MAX_DELAY_MS);
+        this.flushRetryTimer = setTimeout(() => {
+            this.flushRetryTimer = undefined;
+            void this.flushBufferedEvents().catch((error) => {
+                this.setLastError(error);
+                this.scheduleFlushRetry();
+                void this.emitStatus();
+            });
+        }, delay);
+    }
+    isIgnored(candidate, stats) {
+        const relative = normalizeWatchPath(this.repoRoot, candidate);
+        if (!relative)
+            return false;
+        const parts = relative.split("/");
+        if (parts.some((part) => shouldIgnoreDirectory(part).ignored))
+            return true;
+        if (stats?.isDirectory())
+            return false;
+        if (stats?.isFile())
+            return shouldIgnoreFile(relative, this.options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES, stats.size).ignored;
+        return false;
+    }
+    async emitStatus() {
+        if (!this.options.onStatus)
+            return;
+        const status = await this.status();
+        if (this.lastError)
+            status.scheduler.lastError = this.lastError;
+        this.options.onStatus(status);
+    }
+    enqueueJournalOperation(operation) {
+        const next = this.journalQueue.then(operation, operation);
+        this.journalQueue = next.catch(() => undefined);
+        return next;
+    }
+    setLastError(error) {
+        this.lastError = error instanceof Error ? error.message : String(error);
+        return this.lastError;
+    }
+}
+function normalizeWatchPath(repoRoot, rawPath) {
+    const candidate = rawPath.trim();
+    if (!candidate || candidate === ".")
+        return undefined;
+    const relative = path.isAbsolute(candidate)
+        ? normalizeRepoPath(repoRoot, path.resolve(candidate))
+        : normalizeUserPath(candidate);
+    if (!relative || relative === "." || relative.startsWith("..") || path.isAbsolute(relative))
+        return undefined;
+    return relative;
+}

package/dist/src/watch/watcher-liveness.d.ts

@@ -0,0 +1,47 @@
+export declare const WATCHER_LOCK_FILE = "watcher.lock";
+export declare const WATCHER_HEARTBEAT_FILE = "watcher-heartbeat.json";
+export declare const HEARTBEAT_STALE_MS = 30000;
+export interface WatcherLockInfo {
+    pid: number;
+    hostname: string;
+    repoRoot: string;
+    startedAtMs: number;
+}
+export interface WatcherHeartbeat {
+    pid: number;
+    hostname: string;
+    repoRoot: string;
+    startedAtMs: number;
+    lastHeartbeatMs: number;
+    lastIndexedAtMs?: number;
+    pendingFiles: number;
+    indexingFiles: number;
+    ready: boolean;
+    lastError?: string;
+}
+export type WatcherLivenessState = "running" | "stale" | "dead" | "not_running";
+export interface WatcherLiveness {
+    state: WatcherLivenessState;
+    /** True only when a lock holder process is actually alive on this host. */
+    processAlive: boolean;
+    /** True when a heartbeat exists and is within HEARTBEAT_STALE_MS. */
+    heartbeatFresh: boolean;
+    lock?: WatcherLockInfo;
+    heartbeat?: WatcherHeartbeat;
+    heartbeatAgeMs?: number;
+}
+export declare function watcherLockPath(repoRoot: string): string;
+export declare function watcherHeartbeatPath(repoRoot: string): string;
+export declare function isProcessAlive(pid: number): boolean;
+export declare class WatcherLockError extends Error {
+    readonly existing: WatcherLockInfo;
+    constructor(message: string, existing: WatcherLockInfo);
+}
+export interface WatcherLockHandle {
+    readonly info: WatcherLockInfo;
+    release(): void;
+}
+export declare function acquireWatcherLock(repoRoot: string): WatcherLockHandle;
+export declare function writeHeartbeat(repoRoot: string, heartbeat: WatcherHeartbeat): Promise<void>;
+export declare function clearHeartbeat(repoRoot: string): Promise<void>;
+export declare function readWatcherLiveness(repoRoot: string, nowMs?: number): Promise<WatcherLiveness>;

package/dist/src/watch/watcher-liveness.js

@@ -0,0 +1,168 @@
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+// Cross-process liveness for the watch daemon. Two files under <repo>/.ragcode/:
+//
+//   watcher.lock      — pid-stamped advisory lock. Acquired on `ragcode watch` start;
+//                       a second watcher on the same repo refuses rather than double-indexing.
+//   watcher-heartbeat.json — periodically rewritten with last-heartbeat / last-index /
+//                       backlog so doctor, dashboard, and MCP can answer
+//                       "is a watcher alive and current?" with only a file read (no IPC).
+//
+// Both live inside .ragcode/, which the watcher's ignore policy already excludes, so writing
+// them never feeds back as a file event.
+export const WATCHER_LOCK_FILE = "watcher.lock";
+export const WATCHER_HEARTBEAT_FILE = "watcher-heartbeat.json";
+// A heartbeat older than this is treated as a dead/hung daemon. The daemon writes a heartbeat
+// on every scheduler tick and on a fixed liveness interval, both well under this bound.
+export const HEARTBEAT_STALE_MS = 30_000;
+function ragcodeDir(repoRoot) {
+    return path.join(path.resolve(repoRoot), ".ragcode");
+}
+export function watcherLockPath(repoRoot) {
+    return path.join(ragcodeDir(repoRoot), WATCHER_LOCK_FILE);
+}
+export function watcherHeartbeatPath(repoRoot) {
+    return path.join(ragcodeDir(repoRoot), WATCHER_HEARTBEAT_FILE);
+}
+// `process.kill(pid, 0)` sends no signal; it only probes whether the pid exists and is
+// signalable by us. ESRCH = gone, EPERM = alive but owned by another user (still "alive").
+export function isProcessAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+function readLockSync(repoRoot) {
+    try {
+        const raw = fs.readFileSync(watcherLockPath(repoRoot), "utf8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.pid !== "number")
+            return undefined;
+        return {
+            pid: parsed.pid,
+            hostname: typeof parsed.hostname === "string" ? parsed.hostname : "",
+            repoRoot: typeof parsed.repoRoot === "string" ? parsed.repoRoot : path.resolve(repoRoot),
+            startedAtMs: typeof parsed.startedAtMs === "number" ? parsed.startedAtMs : 0
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+export class WatcherLockError extends Error {
+    existing;
+    constructor(message, existing) {
+        super(message);
+        this.existing = existing;
+        this.name = "WatcherLockError";
+    }
+}
+// Acquire the per-repo watcher lock. Throws WatcherLockError if a live watcher already holds it.
+// A lock left behind by a crashed process (pid no longer alive, or a different host) is treated
+// as stale and reclaimed. Acquisition is atomic via O_EXCL with a takeover fallback.
+export function acquireWatcherLock(repoRoot) {
+    const resolvedRoot = path.resolve(repoRoot);
+    const lockPath = watcherLockPath(resolvedRoot);
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    const info = {
+        pid: process.pid,
+        hostname: os.hostname(),
+        repoRoot: resolvedRoot,
+        startedAtMs: Date.now()
+    };
+    const payload = `${JSON.stringify(info, null, 2)}\n`;
+    const writeExclusive = () => {
+        try {
+            fs.writeFileSync(lockPath, payload, { flag: "wx" });
+            return true;
+        }
+        catch (error) {
+            if (error.code === "EEXIST")
+                return false;
+            throw error;
+        }
+    };
+    if (!writeExclusive()) {
+        const existing = readLockSync(resolvedRoot);
+        const heldByLiveProcess = existing !== undefined && existing.hostname === info.hostname && isProcessAlive(existing.pid);
+        if (heldByLiveProcess) {
+            throw new WatcherLockError(`A watcher is already running for ${resolvedRoot} (pid ${existing.pid}).`, existing);
+        }
+        // Stale lock (crashed pid, or a lock from another host that can't be verified here):
+        // reclaim it. The takeover write is not O_EXCL because we've decided the holder is gone.
+        fs.writeFileSync(lockPath, payload, { flag: "w" });
+    }
+    let released = false;
+    return {
+        info,
+        release() {
+            if (released)
+                return;
+            released = true;
+            // Only remove the lock if we still own it — guards against deleting a lock a newer
+            // watcher reclaimed after we were declared stale.
+            const current = readLockSync(resolvedRoot);
+            if (current && current.pid === info.pid && current.startedAtMs === info.startedAtMs) {
+                try {
+                    fs.rmSync(lockPath, { force: true });
+                }
+                catch {
+                    // Best-effort: a missing lock on release is fine.
+                }
+            }
+        }
+    };
+}
+export async function writeHeartbeat(repoRoot, heartbeat) {
+    const heartbeatPath = watcherHeartbeatPath(repoRoot);
+    await fsp.mkdir(path.dirname(heartbeatPath), { recursive: true });
+    await fsp.writeFile(heartbeatPath, `${JSON.stringify(heartbeat, null, 2)}\n`, "utf8");
+}
+export async function clearHeartbeat(repoRoot) {
+    await fsp.rm(watcherHeartbeatPath(repoRoot), { force: true }).catch(() => undefined);
+}
+async function readHeartbeat(repoRoot) {
+    try {
+        const raw = await fsp.readFile(watcherHeartbeatPath(repoRoot), "utf8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.pid !== "number" || typeof parsed.lastHeartbeatMs !== "number")
+            return undefined;
+        return parsed;
+    }
+    catch {
+        return undefined;
+    }
+}
+// Read-only liveness probe used by doctor / dashboard / MCP. Combines the lock (does a holder
+// process exist?) with the heartbeat (is it current?) into one verdict. Pure reads — never
+// mutates lock or heartbeat, so it is safe to call from any process.
+export async function readWatcherLiveness(repoRoot, nowMs = Date.now()) {
+    const lock = readLockSync(repoRoot);
+    const heartbeat = await readHeartbeat(repoRoot);
+    const sameHost = lock ? lock.hostname === os.hostname() : false;
+    // Off-host locks can't be probed with process.kill; trust a fresh heartbeat instead.
+    const processAlive = lock ? (sameHost ? isProcessAlive(lock.pid) : Boolean(heartbeat)) : false;
+    const heartbeatAgeMs = heartbeat ? Math.max(0, nowMs - heartbeat.lastHeartbeatMs) : undefined;
+    const heartbeatFresh = heartbeatAgeMs !== undefined && heartbeatAgeMs <= HEARTBEAT_STALE_MS;
+    let state;
+    if (!lock && !heartbeat) {
+        state = "not_running";
+    }
+    else if (processAlive && heartbeatFresh) {
+        state = "running";
+    }
+    else if (processAlive && !heartbeatFresh) {
+        state = "stale";
+    }
+    else {
+        state = "dead";
+    }
+    return { state, processAlive, heartbeatFresh, lock, heartbeat, heartbeatAgeMs };
+}

package/dist/src/web/server.d.ts

@@ -0,0 +1 @@
+export {};