vellum 0.2.0 → 0.2.2

package/src/workspace/git-service.ts

@@ -3,6 +3,7 @@ import { join } from 'node:path';
 import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
 import { getLogger } from '../util/logger.js';
+import { getConfig } from '../config/loader.js';
 
 const execFileAsync = promisify(execFile);
 const log = getLogger('workspace-git');
@@ -51,15 +52,6 @@ const WORKSPACE_GITIGNORE_RULES = [
   'http-token',
 ];
 
-/**
- * Rules that were used in older versions but have been superseded.
- * These are removed from existing .gitignore files during normalization
- * to prevent them from overriding the newer, more selective rules.
- */
-const DEPRECATED_GITIGNORE_RULES = [
-  'data/',
-];
-
 /** Properties added by Node's child_process errors. */
 interface ExecError extends Error {
   killed?: boolean;
@@ -142,12 +134,88 @@ export class WorkspaceGitService {
   private readonly mutex: Mutex;
   private initialized = false;
   private initPromise: Promise<void> | null = null;
+  private consecutiveFailures = 0;
+  private nextAllowedAttemptMs = 0;
+  private initConsecutiveFailures = 0;
+  private initNextAllowedAttemptMs = 0;
 
   constructor(workspaceDir: string) {
     this.workspaceDir = workspaceDir;
     this.mutex = new Mutex();
   }
 
+  /**
+   * Check if the circuit breaker is open (too many recent failures).
+   * When open, commit attempts are skipped until the backoff window expires.
+   */
+  private isBreakerOpen(): boolean {
+    if (this.consecutiveFailures === 0) return false;
+    return Date.now() < this.nextAllowedAttemptMs;
+  }
+
+  private recordSuccess(): void {
+    if (this.consecutiveFailures > 0) {
+      log.info(
+        { workspaceDir: this.workspaceDir, previousFailures: this.consecutiveFailures },
+        'Circuit breaker closed: commit succeeded after failures',
+      );
+    }
+    this.consecutiveFailures = 0;
+    this.nextAllowedAttemptMs = 0;
+  }
+
+  private recordFailure(): void {
+    const config = getConfig();
+    const failureBackoffBaseMs = config.workspaceGit?.failureBackoffBaseMs ?? 2000;
+    const failureBackoffMaxMs = config.workspaceGit?.failureBackoffMaxMs ?? 60000;
+    this.consecutiveFailures++;
+    const delay = Math.min(
+      failureBackoffBaseMs * Math.pow(2, this.consecutiveFailures - 1),
+      failureBackoffMaxMs,
+    );
+    this.nextAllowedAttemptMs = Date.now() + delay;
+    log.warn(
+      { workspaceDir: this.workspaceDir, consecutiveFailures: this.consecutiveFailures, backoffMs: delay },
+      'Circuit breaker opened: commit failed, backing off',
+    );
+  }
+
+  /**
+   * Check if the init circuit breaker is open (too many recent init failures).
+   * When open, init attempts are skipped until the backoff window expires.
+   */
+  private isInitBreakerOpen(): boolean {
+    if (this.initConsecutiveFailures < 2) return false;
+    return Date.now() < this.initNextAllowedAttemptMs;
+  }
+
+  private recordInitSuccess(): void {
+    if (this.initConsecutiveFailures > 0) {
+      log.info(
+        { workspaceDir: this.workspaceDir, previousFailures: this.initConsecutiveFailures },
+        'Init circuit breaker closed: initialization succeeded after failures',
+      );
+    }
+    this.initConsecutiveFailures = 0;
+    this.initNextAllowedAttemptMs = 0;
+  }
+
+  private recordInitFailure(): void {
+    const config = getConfig();
+    const failureBackoffBaseMs = config.workspaceGit?.failureBackoffBaseMs ?? 2000;
+    const failureBackoffMaxMs = config.workspaceGit?.failureBackoffMaxMs ?? 60000;
+    this.initConsecutiveFailures++;
+    const delay = Math.min(
+      failureBackoffBaseMs * Math.pow(2, this.initConsecutiveFailures - 1),
+      failureBackoffMaxMs,
+    );
+    this.initNextAllowedAttemptMs = Date.now() + delay;
+    log.warn(
+      { workspaceDir: this.workspaceDir, consecutiveFailures: this.initConsecutiveFailures, backoffMs: delay },
+      'Init circuit breaker opened: initialization failed, backing off',
+    );
+  }
+
   /**
    * Ensure the git repository is initialized.
    * Idempotent: safe to call multiple times.
@@ -172,6 +240,14 @@ export class WorkspaceGitService {
       return this.initPromise;
     }
 
+    // Circuit breaker: skip if multiple recent init attempts have been failing.
+    // Checked AFTER initPromise so callers waiting on in-progress init aren't
+    // blocked, and only activates after 2+ consecutive failures so that a
+    // single transient failure allows immediate retry.
+    if (this.isInitBreakerOpen()) {
+      throw new Error('Init circuit breaker open: backing off after repeated failures');
+    }
+
     // Start initialization
     this.initPromise = this.mutex.withLock(async () => {
       // Double-check after acquiring lock
@@ -264,6 +340,7 @@ export class WorkspaceGitService {
             await this.ensureCommitIdentityLocked();
             await this.ensureOnMainLocked();
             this.initialized = true;
+            this.recordInitSuccess();
             return;
           }
         }
@@ -298,12 +375,14 @@ export class WorkspaceGitService {
       await this.execGit(['commit', '-m', message, '--allow-empty']);
 
       this.initialized = true;
+      this.recordInitSuccess();
     });
 
     // If initialization fails, clear the cached promise so subsequent
     // calls can retry instead of permanently returning the rejected promise.
     this.initPromise.catch(() => {
       this.initPromise = null;
+      this.recordInitFailure();
     });
 
     return this.initPromise;
@@ -344,47 +423,119 @@ export class WorkspaceGitService {
    *
    * @param decide - Called with the current status. Return an object with `message`
    *   (and optional `metadata`) to commit, or `null` to skip.
+   * @param options.bypassBreaker - Skip circuit breaker checks (used for shutdown commits).
+   * @param options.deadlineMs - Absolute timestamp (Date.now()) after which the commit
+   *   should be skipped. Checked before lock acquisition, after lock acquisition, and
+   *   before git add/commit to prevent stale queued attempts from doing expensive work.
    * @returns Whether a commit was created and the status at check time.
    */
   async commitIfDirty(
     decide: (status: GitStatus) => { message: string; metadata?: GitCommitMetadata } | null,
+    options?: { bypassBreaker?: boolean; deadlineMs?: number },
   ): Promise<{ committed: boolean; status: GitStatus }> {
+    const emptyStatus: GitStatus = { staged: [], modified: [], untracked: [], clean: false };
+
+    // Circuit breaker: skip expensive git work if recent attempts have been failing.
+    // Shutdown commits bypass the breaker because the process is about to exit and
+    // this is the last chance to persist workspace state.
+    if (!options?.bypassBreaker && this.isBreakerOpen()) {
+      log.debug(
+        { workspaceDir: this.workspaceDir, consecutiveFailures: this.consecutiveFailures },
+        'Circuit breaker open, skipping commit attempt',
+      );
+      return { committed: false, status: emptyStatus };
+    }
+
+    // Deadline fast-path: bail before acquiring the lock if already past deadline.
+    if (isDeadlineExpired(options?.deadlineMs)) {
+      log.debug(
+        { workspaceDir: this.workspaceDir },
+        'Deadline expired before lock acquisition, skipping commit',
+      );
+      return { committed: false, status: emptyStatus };
+    }
+
     await this.ensureInitialized();
 
-    return this.mutex.withLock(async () => {
-      const status = await this.getStatusInternal();
-      if (status.clean) {
-        return { committed: false, status };
-      }
+    try {
+      const result = await this.mutex.withLock(async () => {
+        // Re-check breaker under lock: a queued call that started before the
+        // breaker opened should not proceed with expensive git work now that
+        // the breaker is open.
+        if (!options?.bypassBreaker && this.isBreakerOpen()) {
+          log.debug(
+            { workspaceDir: this.workspaceDir, consecutiveFailures: this.consecutiveFailures },
+            'Circuit breaker open after lock acquisition, skipping commit',
+          );
+          return { committed: false, status: emptyStatus, didRunGit: false as const };
+        }
 
-      const decision = decide(status);
-      if (!decision) {
-        return { committed: false, status };
-      }
+        // Re-check deadline after lock acquisition: the call may have waited
+        // in the mutex queue past its deadline.
+        if (isDeadlineExpired(options?.deadlineMs)) {
+          log.debug(
+            { workspaceDir: this.workspaceDir },
+            'Deadline expired after lock acquisition, skipping commit',
+          );
+          return { committed: false, status: emptyStatus, didRunGit: false as const };
+        }
 
-      await this.execGit(['add', '-A']);
+        const status = await this.getStatusInternal();
+        if (status.clean) {
+          return { committed: false, status, didRunGit: true as const };
+        }
 
-      // Verify something was actually staged. Another service instance
-      // (or external process) could have committed between our status
-      // check and the add, leaving the index clean.
-      try {
-        await this.execGit(['diff', '--cached', '--quiet']);
-        // Exit code 0 means nothing staged — nothing to commit
-        return { committed: false, status };
-      } catch {
-        // Exit code 1 means there ARE staged changes — proceed
-      }
+        const decision = decide(status);
+        if (!decision) {
+          return { committed: false, status, didRunGit: true as const };
+        }
 
-      let fullMessage = decision.message;
-      if (decision.metadata && Object.keys(decision.metadata).length > 0) {
-        fullMessage += '\n\n' + Object.entries(decision.metadata)
-          .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
-          .join('\n');
-      }
+        // Check deadline before expensive git add/commit operations.
+        if (isDeadlineExpired(options?.deadlineMs)) {
+          log.debug(
+            { workspaceDir: this.workspaceDir },
+            'Deadline expired before git add/commit, skipping commit',
+          );
+          return { committed: false, status, didRunGit: true as const };
+        }
 
-      await this.execGit(['commit', '-m', fullMessage]);
-      return { committed: true, status };
-    });
+        await this.execGit(['add', '-A']);
+
+        // Verify something was actually staged. Another service instance
+        // (or external process) could have committed between our status
+        // check and the add, leaving the index clean.
+        try {
+          await this.execGit(['diff', '--cached', '--quiet']);
+          // Exit code 0 means nothing staged — nothing to commit
+          return { committed: false, status, didRunGit: true as const };
+        } catch (err) {
+          // git diff --cached --quiet exits with code 1 when there are staged changes.
+          // Any other error (timeout, permission, etc.) should be treated as a failure.
+          const execErr = err as ExecError;
+          if (execErr.code !== 1) {
+            throw err;
+          }
+          // Exit code 1 = staged changes exist — proceed with commit
+        }
+
+        let fullMessage = decision.message;
+        if (decision.metadata && Object.keys(decision.metadata).length > 0) {
+          fullMessage += '\n\n' + Object.entries(decision.metadata)
+            .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
+            .join('\n');
+        }
+
+        await this.execGit(['commit', '-m', fullMessage]);
+        return { committed: true, status, didRunGit: true as const };
+      });
+      if (result.didRunGit) {
+        this.recordSuccess();
+      }
+      return { committed: result.committed, status: result.status };
+    } catch (err) {
+      this.recordFailure();
+      throw err;
+    }
   }
 
   /**
@@ -439,7 +590,6 @@ export class WorkspaceGitService {
   /**
    * Ensure .gitignore contains all required workspace exclusion rules.
    * Idempotent: checks for missing rules and only appends what's needed.
-   * Also removes deprecated rules that have been superseded by newer ones.
    * Must be called with the mutex lock held.
    */
   private ensureGitignoreRulesLocked(): void {
@@ -447,18 +597,28 @@ export class WorkspaceGitService {
     if (existsSync(gitignorePath)) {
       let content = readFileSync(gitignorePath, 'utf-8');
 
-      // Remove deprecated rules (e.g. broad 'data/' replaced by selective 'data/db/' etc.)
-      const deprecatedPresent = DEPRECATED_GITIGNORE_RULES.filter(rule => content.includes(rule));
-      if (deprecatedPresent.length > 0) {
-        const lines = content.split('\n');
-        content = lines.filter(line => !DEPRECATED_GITIGNORE_RULES.includes(line.trim())).join('\n');
+      // Migrate legacy broad ignore rule to selective data subdirectory rules.
+      // This keeps user-tracked files under data/ visible to git.
+      const lines = content.split('\n');
+      const hadLegacyDataRule = lines.some(line => line.trim() === 'data/');
+      if (hadLegacyDataRule) {
+        content = lines
+          .filter(line => line.trim() !== 'data/')
+          .join('\n');
+        if (!content.endsWith('\n')) {
+          content += '\n';
+        }
       }
 
       const missingRules = WORKSPACE_GITIGNORE_RULES.filter(rule => !content.includes(rule));
-      if (missingRules.length > 0 || deprecatedPresent.length > 0) {
-        const updated = missingRules.length > 0
-          ? content + '\n# Vellum runtime state (auto-added)\n' + missingRules.join('\n') + '\n'
-          : content;
+      if (hadLegacyDataRule || missingRules.length > 0) {
+        let updated = content;
+        if (missingRules.length > 0) {
+          if (!updated.endsWith('\n')) {
+            updated += '\n';
+          }
+          updated += '# Vellum runtime state (auto-added)\n' + missingRules.join('\n') + '\n';
+        }
         writeFileSync(gitignorePath, updated, 'utf-8');
       }
     } else {
@@ -532,16 +692,21 @@ export class WorkspaceGitService {
 
   /**
    * Execute a git command in the workspace directory.
-   * Includes a 30-second timeout to prevent hung operations
-   * (e.g. stale git lock files).
+   * Uses the configurable interactiveGitTimeoutMs (default 10 000 ms) to
+   * prevent hung operations (e.g. stale git lock files). The timeout is
+   * intentionally short for interactive workspace operations — background
+   * enrichment jobs use their own dedicated timeout.
    */
-  private async execGit(args: string[]): Promise<{ stdout: string; stderr: string }> {
+  private async execGit(args: string[], options?: { signal?: AbortSignal }): Promise<{ stdout: string; stderr: string }> {
+    const config = getConfig();
+    const timeoutMs = config.workspaceGit?.interactiveGitTimeoutMs ?? 10_000;
     try {
       const { stdout, stderr } = await execFileAsync('git', args, {
         cwd: this.workspaceDir,
         encoding: 'utf-8',
-        timeout: 30_000,
+        timeout: timeoutMs,
         env: cleanGitEnv(this.workspaceDir),
+        signal: options?.signal,
       });
       return { stdout, stderr };
     } catch (err) {
@@ -567,6 +732,23 @@ export class WorkspaceGitService {
     }
   }
 
+  /**
+   * Get the commit hash of the current HEAD.
+   * This is a lightweight read-only operation that does not require the mutex.
+   */
+  async getHeadHash(): Promise<string> {
+    const { stdout } = await this.execGit(['rev-parse', 'HEAD']);
+    return stdout.trim();
+  }
+
+  /**
+   * Write a git note to a specific commit.
+   * Uses the 'vellum' notes ref to avoid conflicts with default notes.
+   */
+  async writeNote(commitHash: string, noteContent: string, signal?: AbortSignal): Promise<void> {
+    await this.execGit(['notes', '--ref=vellum', 'add', '-f', '-m', noteContent, commitHash], { signal });
+  }
+
   /**
    * Check if the workspace has a git repository initialized.
    * This is a non-blocking check that doesn't trigger initialization.
@@ -583,6 +765,14 @@ export class WorkspaceGitService {
   }
 }
 
+/**
+ * Check whether a deadline has expired.
+ * Returns true when `deadlineMs` is provided and `Date.now()` has reached or passed it.
+ */
+export function isDeadlineExpired(deadlineMs?: number): boolean {
+  return deadlineMs !== undefined && Date.now() >= deadlineMs;
+}
+
 /**
  * Singleton registry for workspace git services.
  * Ensures one service instance per workspace directory.
@@ -618,3 +808,33 @@ export function getAllWorkspaceGitServices(): ReadonlyMap<string, WorkspaceGitSe
 export function _resetGitServiceRegistry(): void {
   serviceRegistry.clear();
 }
+
+/**
+ * @internal Test-only: reset circuit breaker state for a service instance
+ */
+export function _resetBreaker(service: WorkspaceGitService): void {
+  (service as unknown as { consecutiveFailures: number }).consecutiveFailures = 0;
+  (service as unknown as { nextAllowedAttemptMs: number }).nextAllowedAttemptMs = 0;
+}
+
+/**
+ * @internal Test-only: get consecutive failure count
+ */
+export function _getConsecutiveFailures(service: WorkspaceGitService): number {
+  return (service as unknown as { consecutiveFailures: number }).consecutiveFailures;
+}
+
+/**
+ * @internal Test-only: reset init circuit breaker state for a service instance
+ */
+export function _resetInitBreaker(service: WorkspaceGitService): void {
+  (service as unknown as { initConsecutiveFailures: number }).initConsecutiveFailures = 0;
+  (service as unknown as { initNextAllowedAttemptMs: number }).initNextAllowedAttemptMs = 0;
+}
+
+/**
+ * @internal Test-only: get init consecutive failure count
+ */
+export function _getInitConsecutiveFailures(service: WorkspaceGitService): number {
+  return (service as unknown as { initConsecutiveFailures: number }).initConsecutiveFailures;
+}

package/src/workspace/heartbeat-service.ts

@@ -1,5 +1,11 @@
 import { getLogger } from '../util/logger.js';
 import { getAllWorkspaceGitServices, type WorkspaceGitService } from './git-service.js';
+import {
+  DefaultCommitMessageProvider,
+  type CommitContext,
+  type CommitMessageProvider,
+} from './commit-message-provider.js';
+import { getEnrichmentService } from './commit-message-enrichment-service.js';
 
 const log = getLogger('heartbeat');
 
@@ -23,6 +29,8 @@ export interface HeartbeatServiceOptions {
   getServices?: () => ReadonlyMap<string, WorkspaceGitService>;
   /** Override for getting the current timestamp (for testing). */
   now?: () => number;
+  /** Custom commit message provider. */
+  commitMessageProvider?: CommitMessageProvider;
 }
 
 /**
@@ -61,6 +69,7 @@ export class HeartbeatService {
   private readonly intervalMs: number;
   private readonly getServices: () => ReadonlyMap<string, WorkspaceGitService>;
   private readonly now: () => number;
+  private readonly commitMessageProvider: CommitMessageProvider;
   private timer: ReturnType<typeof setInterval> | null = null;
   /** Tracks the currently in-flight check to prevent overlapping runs and allow clean shutdown. */
   private activeCheck: Promise<HeartbeatCheckResult> | null = null;
@@ -71,6 +80,7 @@ export class HeartbeatService {
     this.intervalMs = options?.intervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS;
     this.getServices = options?.getServices ?? getAllWorkspaceGitServices;
     this.now = options?.now ?? Date.now;
+    this.commitMessageProvider = options?.commitMessageProvider ?? new DefaultCommitMessageProvider();
   }
 
   /**
@@ -139,7 +149,7 @@ export class HeartbeatService {
         result.checked++;
 
         try {
-          const committed = await this.checkWorkspace(workspaceDir, service, 'heartbeat');
+          const committed = await this.checkWorkspace(workspaceDir, service);
           if (committed) {
             result.committed++;
           } else {
@@ -189,18 +199,39 @@ export class HeartbeatService {
 
       try {
         const now = this.now();
-        const { committed } = await service.commitIfDirty((status) => {
-          const totalChanges = new Set([...status.staged, ...status.modified, ...status.untracked]).size;
-          log.info({ workspaceDir, totalChanges }, 'Committing pending changes on shutdown');
-          return {
-            message: `auto-commit: shutdown safety net (${totalChanges} files)`,
-            metadata: { trigger: 'shutdown', timestamp: now },
+        let shutdownFiles: string[] = [];
+        const { committed } = await service.commitIfDirty((st) => {
+          const uniqueFiles = [...new Set([...st.staged, ...st.modified, ...st.untracked])];
+          shutdownFiles = uniqueFiles;
+          log.info({ workspaceDir, totalChanges: uniqueFiles.length }, 'Committing pending changes on shutdown');
+
+          const ctx: CommitContext = {
+            workspaceDir,
+            trigger: 'shutdown',
+            changedFiles: uniqueFiles,
+            timestampMs: now,
           };
-        });
+
+          return this.commitMessageProvider.buildImmediateMessage(ctx);
+        }, { bypassBreaker: true });
 
         if (committed) {
           firstSeenDirty.delete(workspaceDir);
           result.committed++;
+
+          // Fire-and-forget enrichment
+          try {
+            const commitHash = await service.getHeadHash();
+            const shutdownCtx: CommitContext = {
+              workspaceDir,
+              trigger: 'shutdown',
+              changedFiles: shutdownFiles,
+              timestampMs: this.now(),
+            };
+            getEnrichmentService().enqueue({ workspaceDir, commitHash, context: shutdownCtx, gitService: service });
+          } catch (enrichErr) {
+            log.debug({ enrichErr }, 'Failed to enqueue shutdown enrichment (non-fatal)');
+          }
         } else {
           result.skipped++;
         }
@@ -225,13 +256,15 @@ export class HeartbeatService {
   private async checkWorkspace(
     workspaceDir: string,
     service: WorkspaceGitService,
-    trigger: string,
   ): Promise<boolean> {
     const now = this.now();
+    let heartbeatFiles: string[] = [];
+    let heartbeatReason: string | undefined;
 
     // Atomic status check + conditional commit within a single mutex lock.
     const { committed, status } = await service.commitIfDirty((st) => {
-      const totalChanges = new Set([...st.staged, ...st.modified, ...st.untracked]).size;
+      const uniqueFiles = [...new Set([...st.staged, ...st.modified, ...st.untracked])];
+      const totalChanges = uniqueFiles.length;
 
       // Track when we first saw this workspace as dirty
       if (!firstSeenDirty.has(workspaceDir)) {
@@ -256,19 +289,43 @@ export class HeartbeatService {
         ? `changes older than ${Math.round(dirtyAge / 1000)}s`
         : `${totalChanges} files changed (threshold: ${this.fileThreshold})`;
 
+      heartbeatFiles = uniqueFiles;
+      heartbeatReason = reason;
+
       log.info(
         { workspaceDir, totalChanges, dirtyAgeMs: dirtyAge, reason },
         'Heartbeat auto-committing workspace changes',
       );
 
-      return {
-        message: `auto-commit: ${trigger} safety net (${totalChanges} files, ${reason})`,
-        metadata: { trigger, timestamp: now },
+      const ctx: CommitContext = {
+        workspaceDir,
+        trigger: 'heartbeat',
+        changedFiles: uniqueFiles,
+        timestampMs: now,
+        reason,
       };
+
+      return this.commitMessageProvider.buildImmediateMessage(ctx);
     });
 
     if (committed) {
       firstSeenDirty.delete(workspaceDir);
+
+      // Fire-and-forget enrichment
+      try {
+        const commitHash = await service.getHeadHash();
+        const hbCtx: CommitContext = {
+          workspaceDir,
+          trigger: 'heartbeat',
+          changedFiles: heartbeatFiles,
+          timestampMs: now,
+          reason: heartbeatReason,
+        };
+        getEnrichmentService().enqueue({ workspaceDir, commitHash, context: hbCtx, gitService: service });
+      } catch (enrichErr) {
+        log.debug({ enrichErr }, 'Failed to enqueue heartbeat enrichment (non-fatal)');
+      }
+
       return true;
     }