tlc-claude-code 1.4.9 → 1.5.2

package/server/lib/quality-presets.test.js

@@ -0,0 +1,269 @@
+/**
+ * Quality Presets Tests
+ *
+ * Tests for pre-configured quality levels for common use cases
+ */
+
+const { describe, it, beforeEach } = require('node:test');
+const assert = require('node:assert');
+
+const {
+  getPreset,
+  createCustomPreset,
+  recommendPreset,
+  applyPreset,
+  listPresets,
+  PRESET_FAST,
+  PRESET_BALANCED,
+  PRESET_THOROUGH,
+  PRESET_CRITICAL,
+} = require('./quality-presets.js');
+
+describe('Quality Presets', () => {
+  describe('getPreset', () => {
+    it('returns preset configuration', () => {
+      const preset = getPreset('fast');
+      assert.ok(preset);
+      assert.ok(preset.thresholds);
+    });
+
+    it('returns null for unknown preset', () => {
+      const preset = getPreset('unknown');
+      assert.strictEqual(preset, null);
+    });
+
+    it('returns preset by constant', () => {
+      const preset = getPreset(PRESET_FAST);
+      assert.ok(preset);
+      assert.strictEqual(preset.name, 'fast');
+    });
+  });
+
+  describe('preset fast', () => {
+    it('has low thresholds', () => {
+      const preset = getPreset('fast');
+      assert.ok(preset.thresholds.default <= 60);
+    });
+
+    it('allows cheaper models', () => {
+      const preset = getPreset('fast');
+      assert.ok(
+        preset.modelTier === 'basic' ||
+        (preset.allowedModels && preset.allowedModels.some((m) => m.includes('3.5')))
+      );
+    });
+
+    it('has minimal retry configuration', () => {
+      const preset = getPreset('fast');
+      assert.ok(preset.maxRetries <= 1);
+    });
+
+    it('skips some dimensions', () => {
+      const preset = getPreset('fast');
+      assert.ok(preset.skipDimensions && preset.skipDimensions.length > 0);
+    });
+  });
+
+  describe('preset balanced', () => {
+    it('has moderate thresholds', () => {
+      const preset = getPreset('balanced');
+      assert.ok(preset.thresholds.default >= 65 && preset.thresholds.default <= 80);
+    });
+
+    it('allows mid-tier models', () => {
+      const preset = getPreset('balanced');
+      assert.ok(preset.modelTier === 'standard' || !preset.modelTier);
+    });
+
+    it('has reasonable retry count', () => {
+      const preset = getPreset('balanced');
+      assert.ok(preset.maxRetries >= 1 && preset.maxRetries <= 3);
+    });
+  });
+
+  describe('preset thorough', () => {
+    it('has high thresholds', () => {
+      const preset = getPreset('thorough');
+      assert.ok(preset.thresholds.default >= 80);
+    });
+
+    it('requires better models', () => {
+      const preset = getPreset('thorough');
+      assert.ok(
+        preset.modelTier === 'premium' ||
+        preset.minModel ||
+        (preset.allowedModels && preset.allowedModels.some((m) => m.includes('4')))
+      );
+    });
+
+    it('enables all dimensions', () => {
+      const preset = getPreset('thorough');
+      assert.ok(!preset.skipDimensions || preset.skipDimensions.length === 0);
+    });
+
+    it('has higher retry budget', () => {
+      const preset = getPreset('thorough');
+      assert.ok(preset.maxRetries >= 2);
+    });
+  });
+
+  describe('preset critical', () => {
+    it('has highest thresholds', () => {
+      const preset = getPreset('critical');
+      assert.ok(preset.thresholds.default >= 90);
+    });
+
+    it('requires premium models', () => {
+      const preset = getPreset('critical');
+      assert.ok(preset.modelTier === 'premium' || preset.minModel);
+    });
+
+    it('has strictest correctness threshold', () => {
+      const preset = getPreset('critical');
+      assert.ok(
+        preset.thresholds.dimensions?.correctness >= 95 ||
+        preset.thresholds.default >= 90
+      );
+    });
+
+    it('enables all quality checks', () => {
+      const preset = getPreset('critical');
+      assert.ok(!preset.skipDimensions || preset.skipDimensions.length === 0);
+    });
+  });
+
+  describe('createCustomPreset', () => {
+    it('creates preset with custom thresholds', () => {
+      const custom = createCustomPreset('my-preset', {
+        thresholds: { default: 75 },
+      });
+      assert.strictEqual(custom.name, 'my-preset');
+      assert.strictEqual(custom.thresholds.default, 75);
+    });
+
+    it('validates threshold values', () => {
+      assert.throws(() => {
+        createCustomPreset('invalid', { thresholds: { default: 150 } });
+      });
+    });
+
+    it('allows extending existing preset', () => {
+      const custom = createCustomPreset('my-fast', {
+        extends: 'fast',
+        thresholds: { default: 55 },
+      });
+      assert.ok(custom.skipDimensions);
+      assert.strictEqual(custom.thresholds.default, 55);
+    });
+
+    it('saves to preset registry', () => {
+      createCustomPreset('saved-preset', { thresholds: { default: 70 } });
+      const retrieved = getPreset('saved-preset');
+      assert.ok(retrieved);
+      assert.strictEqual(retrieved.thresholds.default, 70);
+    });
+  });
+
+  describe('recommendPreset', () => {
+    it('suggests preset by task type', () => {
+      const recommendation = recommendPreset({ task: 'quick-fix' });
+      assert.strictEqual(recommendation, 'fast');
+    });
+
+    it('suggests balanced for normal work', () => {
+      const recommendation = recommendPreset({ task: 'feature' });
+      assert.strictEqual(recommendation, 'balanced');
+    });
+
+    it('suggests thorough for production', () => {
+      const recommendation = recommendPreset({ task: 'release' });
+      assert.ok(recommendation === 'thorough' || recommendation === 'critical');
+    });
+
+    it('suggests critical for security code', () => {
+      const recommendation = recommendPreset({ task: 'security' });
+      assert.strictEqual(recommendation, 'critical');
+    });
+
+    it('considers time constraints', () => {
+      const rushed = recommendPreset({ task: 'feature', timeConstrained: true });
+      const normal = recommendPreset({ task: 'feature', timeConstrained: false });
+      const rushedPreset = getPreset(rushed);
+      const normalPreset = getPreset(normal);
+      assert.ok(rushedPreset.thresholds.default <= normalPreset.thresholds.default);
+    });
+
+    it('considers importance level', () => {
+      const lowImportance = recommendPreset({ importance: 'low' });
+      const highImportance = recommendPreset({ importance: 'high' });
+      const lowPreset = getPreset(lowImportance);
+      const highPreset = getPreset(highImportance);
+      assert.ok(lowPreset.thresholds.default < highPreset.thresholds.default);
+    });
+  });
+
+  describe('applyPreset', () => {
+    it('updates current config with preset values', () => {
+      const current = { thresholds: { default: 50 } };
+      const updated = applyPreset(current, 'balanced');
+      const balanced = getPreset('balanced');
+      assert.strictEqual(updated.thresholds.default, balanced.thresholds.default);
+    });
+
+    it('preserves non-preset config', () => {
+      const current = {
+        thresholds: { default: 50 },
+        customField: 'preserved',
+      };
+      const updated = applyPreset(current, 'fast');
+      assert.strictEqual(updated.customField, 'preserved');
+    });
+
+    it('records applied preset name', () => {
+      const current = {};
+      const updated = applyPreset(current, 'thorough');
+      assert.strictEqual(updated.appliedPreset, 'thorough');
+    });
+
+    it('returns new config without mutating original', () => {
+      const current = { thresholds: { default: 50 } };
+      const updated = applyPreset(current, 'balanced');
+      assert.strictEqual(current.thresholds.default, 50);
+      assert.notStrictEqual(updated.thresholds.default, 50);
+    });
+  });
+
+  describe('listPresets', () => {
+    it('returns all available presets', () => {
+      const presets = listPresets();
+      assert.ok(Array.isArray(presets));
+      assert.ok(presets.length >= 4);
+    });
+
+    it('includes built-in presets', () => {
+      const presets = listPresets();
+      const names = presets.map((p) => p.name);
+      assert.ok(names.includes('fast'));
+      assert.ok(names.includes('balanced'));
+      assert.ok(names.includes('thorough'));
+      assert.ok(names.includes('critical'));
+    });
+
+    it('includes custom presets', () => {
+      createCustomPreset('my-custom', { thresholds: { default: 77 } });
+      const presets = listPresets();
+      const names = presets.map((p) => p.name);
+      assert.ok(names.includes('my-custom'));
+    });
+
+    it('returns preset descriptions', () => {
+      const presets = listPresets();
+      assert.ok(presets.every((p) => p.description));
+    });
+
+    it('returns preset threshold summaries', () => {
+      const presets = listPresets({ summary: true });
+      assert.ok(presets.every((p) => p.thresholdSummary !== undefined));
+    });
+  });
+});

package/server/lib/quality-retry.js

@@ -0,0 +1,323 @@
+/**
+ * Quality Retry Module
+ *
+ * Auto-retry logic with better models on quality failure
+ */
+
+/**
+ * Create a retry manager
+ * @param {Object} options - Manager options
+ * @returns {Object} Retry manager
+ */
+function createRetryManager(options = {}) {
+  const budgetLimit = options.budgetLimit ?? Infinity;
+  return {
+    options: {
+      maxRetries: options.maxRetries ?? 3,
+      budgetLimit,
+      ...options,
+    },
+    history: [],
+    totalCost: 0,
+    budgetLimit,
+    costPerAttempt: {},
+    modelPerAttempt: {},
+  };
+}
+
+/**
+ * Determine if retry should be attempted
+ * @param {Object} evaluation - Evaluation result
+ * @param {Object} options - Retry options
+ * @param {Object} resultOptions - Result options
+ * @returns {boolean|Object} Whether to retry
+ */
+function shouldRetry(evaluation, options = {}, resultOptions = {}) {
+  const {
+    maxRetries = 3,
+    currentRetry = 0,
+    budgetLimit = Infinity,
+    spentBudget = 0,
+    retryOnDimensions = null,
+  } = options;
+
+  // Don't retry if passed
+  if (evaluation.pass) {
+    if (resultOptions.reason) {
+      return { retry: false, reason: 'evaluation passed' };
+    }
+    return false;
+  }
+
+  // Check max retries
+  if (currentRetry >= maxRetries) {
+    if (resultOptions.reason) {
+      return { retry: false, reason: 'max retries reached' };
+    }
+    return false;
+  }
+
+  // Check budget
+  if (spentBudget >= budgetLimit) {
+    if (resultOptions.reason) {
+      return { retry: false, reason: 'budget exceeded' };
+    }
+    return false;
+  }
+
+  // Check specific dimension filtering
+  if (retryOnDimensions && evaluation.failed) {
+    const shouldRetryDimension = evaluation.failed.some((d) =>
+      retryOnDimensions.includes(d)
+    );
+    if (!shouldRetryDimension) {
+      if (resultOptions.reason) {
+        return { retry: false, reason: 'no matching dimensions to retry' };
+      }
+      return false;
+    }
+  }
+
+  if (resultOptions.reason) {
+    return { retry: true, reason: 'quality failure' };
+  }
+  return true;
+}
+
+/**
+ * Select a better model for retry
+ * @param {string} currentModel - Current model
+ * @param {string[]} models - Available models (ordered by quality)
+ * @param {Object} options - Selection options
+ * @returns {string|null|Object} Better model or null
+ */
+function selectBetterModel(currentModel, models, options = {}) {
+  const {
+    remainingBudget = Infinity,
+    costs = {},
+    requiredCapability = null,
+    capabilities = {},
+    details = false,
+  } = options;
+
+  const currentIndex = models.indexOf(currentModel);
+
+  // Find next better model
+  for (let i = currentIndex + 1; i < models.length; i++) {
+    const candidate = models[i];
+
+    // Check budget
+    const cost = costs[candidate] || 0;
+    if (cost > remainingBudget) {
+      continue;
+    }
+
+    // Check capabilities
+    if (requiredCapability && capabilities[candidate]) {
+      if (!capabilities[candidate].includes(requiredCapability)) {
+        continue;
+      }
+    }
+
+    if (details) {
+      return {
+        model: candidate,
+        tier: i,
+        cost: costs[candidate],
+      };
+    }
+    return candidate;
+  }
+
+  if (details) {
+    return { model: null, tier: null, reason: 'no better model available' };
+  }
+  return null;
+}
+
+/**
+ * Build retry prompt with failure context
+ * @param {string} originalPrompt - Original prompt
+ * @param {Object} context - Failure context
+ * @returns {string} Enhanced retry prompt
+ */
+function buildRetryPrompt(originalPrompt, context) {
+  const { failedDimensions = [], failures = {}, suggestions = [] } = context;
+
+  let prompt = originalPrompt;
+
+  if (failedDimensions.length > 0) {
+    prompt += '\n\n---\nPrevious attempt failed quality checks. Please improve the following:';
+
+    for (const dim of failedDimensions) {
+      const failure = failures[dim];
+      if (failure) {
+        prompt += `\n- ${dim}: score ${failure.score}/${failure.threshold}`;
+        if (failure.reason) {
+          prompt += ` (${failure.reason})`;
+        }
+      } else {
+        prompt += `\n- ${dim}`;
+      }
+    }
+  }
+
+  if (suggestions.length > 0) {
+    prompt += '\n\nSuggestions to fix:';
+    for (const suggestion of suggestions) {
+      prompt += `\n- ${suggestion}`;
+    }
+  }
+
+  return prompt;
+}
+
+/**
+ * Track cost of a retry attempt
+ * @param {Object} manager - Retry manager
+ * @param {number} cost - Cost of attempt
+ * @param {Object} options - Tracking options
+ */
+function trackRetryCost(manager, cost, options = {}) {
+  const { attempt, model } = options;
+
+  manager.totalCost = (manager.totalCost || 0) + cost;
+
+  if (attempt !== undefined) {
+    if (!manager.costPerAttempt) {
+      manager.costPerAttempt = {};
+    }
+    manager.costPerAttempt[attempt] = cost;
+  }
+
+  if (model && attempt !== undefined) {
+    if (!manager.modelPerAttempt) {
+      manager.modelPerAttempt = {};
+    }
+    manager.modelPerAttempt[attempt] = model;
+  }
+}
+
+/**
+ * Get retry history
+ * @param {Object} manager - Retry manager
+ * @param {Object} options - History options
+ * @returns {Array|Object} History records
+ */
+function getRetryHistory(manager, options = {}) {
+  const { escalationPath = false, improvements = false } = options;
+
+  const history = manager.history || [];
+
+  if (!escalationPath && !improvements) {
+    return history;
+  }
+
+  const result = {
+    attempts: history,
+  };
+
+  if (escalationPath) {
+    result.escalationPath = history.map((h) => h.model).filter(Boolean);
+  }
+
+  if (improvements) {
+    result.improvements = {};
+    for (let i = 1; i < history.length; i++) {
+      const prev = history[i - 1].composite || 0;
+      const curr = history[i].composite || 0;
+      result.improvements[i] = curr - prev;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Execute retry loop with feedback
+ * @param {string} prompt - Original prompt
+ * @param {Object} options - Execution options
+ * @returns {Promise<Object>} Final result
+ */
+async function retryWithFeedback(prompt, options = {}) {
+  const {
+    execute,
+    evaluate,
+    maxRetries = 3,
+    models = ['basic', 'advanced', 'premium'],
+    initialModel = 'basic',
+    budgetLimit = Infinity,
+  } = options;
+
+  const history = [];
+  let currentModel = initialModel;
+  let currentPrompt = prompt;
+  let totalCost = 0;
+  let attempt = 0;
+
+  while (attempt < maxRetries) {
+    attempt++;
+
+    // Execute with current model and prompt
+    const output = await execute(currentPrompt, currentModel);
+
+    // Evaluate the output
+    const evaluation = await evaluate(output);
+
+    // Record in history
+    history.push({
+      attempt,
+      model: currentModel,
+      ...evaluation,
+    });
+
+    // Check if passed
+    if (evaluation.pass) {
+      return {
+        pass: true,
+        output,
+        ...evaluation,
+        history,
+        attempts: attempt,
+      };
+    }
+
+    // Check if should retry
+    if (attempt >= maxRetries) {
+      break;
+    }
+
+    // Try to escalate model
+    const betterModel = selectBetterModel(currentModel, models, {
+      remainingBudget: budgetLimit - totalCost,
+    });
+
+    if (betterModel) {
+      currentModel = betterModel;
+    }
+
+    // Build retry prompt with feedback
+    currentPrompt = buildRetryPrompt(prompt, {
+      failedDimensions: evaluation.failed || [],
+    });
+  }
+
+  // Return final result (failed)
+  const lastEval = history[history.length - 1];
+  return {
+    pass: false,
+    ...lastEval,
+    history,
+    attempts: attempt,
+  };
+}
+
+module.exports = {
+  createRetryManager,
+  shouldRetry,
+  selectBetterModel,
+  buildRetryPrompt,
+  trackRetryCost,
+  getRetryHistory,
+  retryWithFeedback,
+};