tuneprompt 1.0.0

package/dist/engine/optimizer.js

@@ -0,0 +1,246 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PromptOptimizer = void 0;
+const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const openai_1 = __importDefault(require("openai"));
+const metaPrompt_1 = require("./metaPrompt");
+const constraintExtractor_1 = require("./constraintExtractor");
+class PromptOptimizer {
+    anthropic;
+    openai;
+    constructor() {
+        const anthropicKey = process.env.ANTHROPIC_API_KEY;
+        if (anthropicKey && !anthropicKey.startsWith('api_key') && anthropicKey !== 'phc_xxxxx') {
+            this.anthropic = new sdk_1.default({
+                apiKey: anthropicKey
+            });
+        }
+        const openaiKey = process.env.OPENAI_API_KEY;
+        if (openaiKey && !openaiKey.startsWith('api_key')) {
+            this.openai = new openai_1.default({
+                apiKey: openaiKey
+            });
+        }
+    }
+    /**
+     * Main optimization method
+     */
+    async optimize(failedTest) {
+        console.log(`\n🧠 Analyzing failure: "${failedTest.description}"`);
+        // Step 1: Extract constraints and build context
+        const errorContext = (0, constraintExtractor_1.generateErrorContext)(failedTest);
+        // Step 2: Choose the right meta-prompt based on error type
+        const metaPrompt = this.selectMetaPrompt(failedTest, errorContext);
+        // Step 3: Generate fix candidates using Claude
+        console.log('⚡ Generating optimized prompt candidates...');
+        const candidates = await this.generateCandidates(metaPrompt, failedTest);
+        // Step 4: Shadow test each candidate
+        console.log('🧪 Shadow testing candidates...');
+        const bestCandidate = await this.selectBestCandidate(candidates, failedTest);
+        return {
+            originalPrompt: failedTest.prompt,
+            optimizedPrompt: bestCandidate.prompt,
+            reasoning: bestCandidate.reasoning,
+            confidence: bestCandidate.score,
+            testResults: {
+                score: bestCandidate.score,
+                passed: bestCandidate.score >= failedTest.threshold,
+                output: '' // Will be filled by shadow tester
+            }
+        };
+    }
+    /**
+     * Select appropriate meta-prompt based on error type
+     */
+    selectMetaPrompt(test, errorContext) {
+        const input = {
+            originalPrompt: test.prompt,
+            testInput: test.input,
+            expectedOutput: test.expectedOutput,
+            actualOutput: test.actualOutput,
+            errorType: test.errorType,
+            errorMessage: errorContext
+        };
+        switch (test.errorType) {
+            case 'json':
+                return (0, metaPrompt_1.generateJSONFixPrompt)(input);
+            case 'semantic':
+                return (0, metaPrompt_1.generateSemanticFixPrompt)(input);
+            default:
+                return (0, metaPrompt_1.generateOptimizationPrompt)(input);
+        }
+    }
+    /**
+     * Generate multiple fix candidates using available LLMs with fallback
+     */
+    async generateCandidates(metaPrompt, failedTest) {
+        // Define provider priority order for candidate generation
+        const providers = ['anthropic', 'openai', 'openrouter'];
+        for (const provider of providers) {
+            try {
+                // Check if we have the required client for this provider
+                if (provider === 'anthropic' && this.anthropic) {
+                    console.log(`⚡ Using Anthropic for candidate generation...`);
+                    const response = await this.anthropic.messages.create({
+                        model: 'claude-sonnet-4-20250514',
+                        max_tokens: 4000,
+                        temperature: 0.7, // Some creativity for prompt rewriting
+                        messages: [{
+                                role: 'user',
+                                content: metaPrompt
+                            }]
+                    });
+                    const content = response.content[0];
+                    if (content.type !== 'text') {
+                        throw new Error('Unexpected response type from Claude');
+                    }
+                    // Parse the JSON response
+                    const parsed = JSON.parse(content.text);
+                    return [
+                        {
+                            prompt: parsed.candidateA.prompt,
+                            reasoning: parsed.candidateA.reasoning,
+                            score: 0 // Will be filled by shadow testing
+                        },
+                        {
+                            prompt: parsed.candidateB.prompt,
+                            reasoning: parsed.candidateB.reasoning,
+                            score: 0
+                        }
+                    ];
+                }
+                else if (provider === 'openai' && this.openai) {
+                    console.log(`⚡ Using OpenAI for candidate generation...`);
+                    const response = await this.openai.chat.completions.create({
+                        model: 'gpt-4o',
+                        messages: [{
+                                role: 'user',
+                                content: metaPrompt
+                            }],
+                        response_format: { type: 'json_object' }
+                    });
+                    const content = response.choices[0]?.message?.content;
+                    if (!content) {
+                        throw new Error('No content returned from OpenAI');
+                    }
+                    // Parse the JSON response
+                    const parsed = JSON.parse(content);
+                    return [
+                        {
+                            prompt: parsed.candidateA.prompt,
+                            reasoning: parsed.candidateA.reasoning,
+                            score: 0 // Will be filled by shadow testing
+                        },
+                        {
+                            prompt: parsed.candidateB.prompt,
+                            reasoning: parsed.candidateB.reasoning,
+                            score: 0
+                        }
+                    ];
+                }
+                else if (provider === 'openrouter') {
+                    // For OpenRouter, we'll use the shadowTester to get a response
+                    console.log(`⚡ Using OpenRouter for candidate generation...`);
+                    // Since OpenRouter is used in shadow testing, we'll use a different approach
+                    // For now, we'll return a basic fallback since OpenRouter doesn't support structured outputs as well
+                    return [{
+                            prompt: this.createFallbackPrompt(failedTest),
+                            reasoning: 'Generated using fallback method',
+                            score: 0
+                        }];
+                }
+            }
+            catch (error) {
+                console.log(`⚠️  ${provider} provider failed for candidate generation: ${error.message}`);
+                continue; // Try next provider
+            }
+        }
+        // All providers failed
+        console.error('All providers failed for candidate generation');
+        return [{
+                prompt: this.createFallbackPrompt(failedTest),
+                reasoning: 'Fallback prompt with basic improvements',
+                score: 0
+            }];
+    }
+    /**
+     * Shadow test each candidate and return the best one
+     */
+    async selectBestCandidate(candidates, originalTest) {
+        const { runShadowTest } = await Promise.resolve().then(() => __importStar(require('./shadowTester')));
+        const testedCandidates = await Promise.all(candidates.map(async (candidate) => {
+            const result = await runShadowTest(candidate.prompt, originalTest);
+            return {
+                ...candidate,
+                score: result.score
+            };
+        }));
+        // Sort by score (highest first)
+        testedCandidates.sort((a, b) => b.score - a.score);
+        return testedCandidates[0];
+    }
+    /**
+     * Fallback prompt improvement - generates a clean rewritten prompt
+     */
+    createFallbackPrompt(test) {
+        // Extract the core intent from the original prompt
+        // Remove any existing "You must provide..." prefixes to avoid duplication
+        let corePrompt = test.prompt
+            .replace(/You must provide a response that includes the following key information:\n[^\n]*\n\n/g, '')
+            .trim();
+        // For JSON errors, create a structured prompt
+        if (test.errorType === 'json') {
+            return `${corePrompt}
+
+IMPORTANT: You must respond with valid JSON only. No explanations, no markdown, just the raw JSON object.`;
+        }
+        // For semantic errors, be more specific about expected output
+        if (test.errorType === 'semantic') {
+            return `${corePrompt}
+
+Your response must match this exactly: "${test.expectedOutput}"
+Do not add any extra text, greetings, or explanations. Output only what is requested.`;
+        }
+        // Default: add clarity
+        return `${corePrompt}
+
+Be concise and match the expected output format exactly.`;
+    }
+}
+exports.PromptOptimizer = PromptOptimizer;

package/dist/engine/reporter.d.ts

@@ -0,0 +1,7 @@
+import { TestRun } from '../types';
+export declare class TestReporter {
+    printResults(run: TestRun, format?: 'json' | 'table' | 'both'): void;
+    private printJSON;
+    private printTable;
+    private printSummary;
+}

package/dist/engine/reporter.js

@@ -0,0 +1,58 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestReporter = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const cli_table3_1 = __importDefault(require("cli-table3"));
+class TestReporter {
+    printResults(run, format = 'both') {
+        if (format === 'json' || format === 'both') {
+            this.printJSON(run);
+        }
+        if (format === 'table' || format === 'both') {
+            this.printTable(run);
+        }
+        this.printSummary(run);
+    }
+    printJSON(run) {
+        console.log(JSON.stringify(run, null, 2));
+    }
+    printTable(run) {
+        const table = new cli_table3_1.default({
+            head: ['Status', 'Test', 'Score', 'Method', 'Duration'],
+            colWidths: [10, 40, 10, 15, 12]
+        });
+        for (const result of run.results) {
+            const statusIcon = result.status === 'pass'
+                ? chalk_1.default.green('✓ PASS')
+                : result.status === 'fail'
+                    ? chalk_1.default.red('✗ FAIL')
+                    : chalk_1.default.yellow('⚠ ERROR');
+            const scoreColor = result.score >= 0.8
+                ? chalk_1.default.green
+                : result.score >= 0.5
+                    ? chalk_1.default.yellow
+                    : chalk_1.default.red;
+            table.push([
+                statusIcon,
+                result.testCase.description,
+                scoreColor(result.score.toFixed(2)),
+                result.testCase.config?.method || 'semantic',
+                `${result.metadata.duration}ms`
+            ]);
+        }
+        console.log(table.toString());
+    }
+    printSummary(run) {
+        console.log('\n' + chalk_1.default.bold('Summary:'));
+        console.log(`  Total: ${run.totalTests}`);
+        console.log(chalk_1.default.green(`  Passed: ${run.passed}`));
+        console.log(chalk_1.default.red(`  Failed: ${run.failed}`));
+        console.log(`  Duration: ${run.duration}ms`);
+        const totalCost = run.results.reduce((sum, r) => sum + (r.metadata.cost || 0), 0);
+        console.log(`  Cost: $${totalCost.toFixed(4)}`);
+    }
+}
+exports.TestReporter = TestReporter;

package/dist/engine/runner.d.ts

@@ -0,0 +1,9 @@
+import { TestCase, TestRun, TunePromptConfig } from "../types";
+export declare class TestRunner {
+    private config;
+    private providers;
+    constructor(config: TunePromptConfig);
+    private initializeProviders;
+    runTests(testCases: TestCase[]): Promise<TestRun>;
+    private runSingleTest;
+}

package/dist/engine/runner.js

@@ -0,0 +1,169 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestRunner = void 0;
+const uuid_1 = require("uuid");
+const openai_1 = require("../providers/openai");
+const anthropic_1 = require("../providers/anthropic");
+const openrouter_1 = require("../providers/openrouter");
+const exact_match_1 = require("../scoring/exact-match");
+const json_validator_1 = require("../scoring/json-validator");
+const semantic_1 = require("../scoring/semantic");
+class TestRunner {
+    config;
+    providers = new Map();
+    constructor(config) {
+        this.config = config;
+        this.initializeProviders();
+    }
+    initializeProviders() {
+        if (this.config.providers.openai) {
+            const provider = new openai_1.OpenAIProvider(this.config.providers.openai);
+            this.providers.set("openai", provider);
+        }
+        if (this.config.providers.anthropic) {
+            this.providers.set("anthropic", new anthropic_1.AnthropicProvider(this.config.providers.anthropic));
+        }
+        if (this.config.providers.openrouter) {
+            const provider = new openrouter_1.OpenRouterProvider(this.config.providers.openrouter);
+            this.providers.set("openrouter", provider);
+        }
+    }
+    async runTests(testCases) {
+        const runId = (0, uuid_1.v4)();
+        const startTime = Date.now();
+        const results = [];
+        for (const testCase of testCases) {
+            const result = await this.runSingleTest(testCase);
+            results.push(result);
+        }
+        const duration = Date.now() - startTime;
+        const passed = results.filter((r) => r.status === "pass").length;
+        const failed = results.filter((r) => r.status === "fail").length;
+        return {
+            id: runId,
+            timestamp: new Date(),
+            totalTests: testCases.length,
+            passed,
+            failed,
+            duration,
+            results,
+        };
+    }
+    async runSingleTest(testCase) {
+        const testId = (0, uuid_1.v4)();
+        const startTime = Date.now();
+        // Define fallback order: Primary -> Fallbacks
+        const fallbackChain = ["openai", "anthropic", "openrouter"];
+        // Determine starting provider
+        const initialProvider = testCase.config?.provider || "openai";
+        // Build the sequence of providers to try
+        let providersToTry;
+        if (testCase.config?.provider) {
+            // If provider is explicitly set, only try that one
+            providersToTry = [testCase.config.provider];
+        }
+        else {
+            providersToTry = [
+                initialProvider,
+                ...fallbackChain.filter((p) => p !== initialProvider),
+            ];
+        }
+        let lastError;
+        let errors = [];
+        for (const providerName of providersToTry) {
+            const provider = this.providers.get(providerName);
+            if (!provider)
+                continue;
+            try {
+                // Execute prompt
+                const response = await provider.complete(testCase.prompt);
+                // Score result
+                const scoringMethod = testCase.config?.method || "semantic";
+                const threshold = testCase.config?.threshold || this.config.threshold || 0.8;
+                let score;
+                let error;
+                if (scoringMethod === "exact") {
+                    score = (0, exact_match_1.exactMatch)(String(testCase.expect), response.content);
+                }
+                else if (scoringMethod === "json") {
+                    const result = (0, json_validator_1.validateJSON)(testCase.expect, response.content);
+                    score = result.score;
+                    error = result.error;
+                }
+                else if (scoringMethod === "semantic") {
+                    let calculatedScore;
+                    let lastScoringError;
+                    // potential embedding providers
+                    const embeddingCapable = ["openai", "openrouter"];
+                    // Order: Current provider (if capable) -> OpenAI -> OpenRouter -> others
+                    const scoringProvidersToTry = [
+                        ...(embeddingCapable.includes(providerName) ? [providerName] : []),
+                        ...embeddingCapable.filter((p) => p !== providerName),
+                    ].filter((p) => this.providers.has(p));
+                    if (scoringProvidersToTry.length === 0) {
+                        throw new Error("No embedding-capable providers available for semantic scoring");
+                    }
+                    for (const scoreProviderName of scoringProvidersToTry) {
+                        try {
+                            const scoreProvider = this.providers.get(scoreProviderName);
+                            if (!scoreProvider)
+                                continue;
+                            const scorer = new semantic_1.SemanticScorer(scoreProvider);
+                            calculatedScore = await scorer.score(String(testCase.expect), response.content);
+                            break;
+                        }
+                        catch (err) {
+                            lastScoringError = err;
+                            continue;
+                        }
+                    }
+                    if (calculatedScore === undefined) {
+                        throw new Error(`Semantic scoring failed. Last error: ${lastScoringError?.message || "Unknown error"}`);
+                    }
+                    score = calculatedScore;
+                }
+                else {
+                    throw new Error(`Unknown scoring method: ${scoringMethod}`);
+                }
+                const status = score >= threshold ? "pass" : "fail";
+                const duration = Date.now() - startTime;
+                return {
+                    id: testId,
+                    testCase,
+                    status,
+                    score,
+                    actualOutput: response.content,
+                    expectedOutput: String(testCase.expect),
+                    error,
+                    metadata: {
+                        duration,
+                        timestamp: new Date(),
+                        tokens: response.tokens,
+                        cost: response.cost,
+                        provider: providerName,
+                    },
+                };
+            }
+            catch (error) {
+                lastError = error;
+                errors.push(`${providerName.toUpperCase()}: ${error.message}`);
+                continue;
+            }
+        }
+        // If all attempts failed
+        return {
+            id: testId,
+            testCase,
+            status: "error",
+            score: 0,
+            actualOutput: "",
+            expectedOutput: String(testCase.expect),
+            error: errors.join(" | ") || lastError?.message || "All providers failed",
+            metadata: {
+                duration: Date.now() - startTime,
+                timestamp: new Date(),
+            },
+        };
+    }
+}
+exports.TestRunner = TestRunner;

package/dist/engine/shadowTester.d.ts

@@ -0,0 +1,11 @@
+import { FailedTest } from '../types/fix';
+export interface ShadowTestResult {
+    score: number;
+    output: string;
+    passed: boolean;
+}
+/**
+ * Test a candidate prompt against the original test case
+ * Tries providers in sequence until one succeeds
+ */
+export declare function runShadowTest(candidatePrompt: string, originalTest: FailedTest): Promise<ShadowTestResult>;

package/dist/engine/shadowTester.js

@@ -0,0 +1,156 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runShadowTest = runShadowTest;
+const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const openai_1 = __importDefault(require("openai"));
+const semantic_1 = require("../scoring/semantic"); // From Phase 1
+/**
+ * Test a candidate prompt against the original test case
+ * Tries providers in sequence until one succeeds
+ */
+async function runShadowTest(candidatePrompt, originalTest) {
+    // Define provider priority order
+    const providers = ['anthropic', 'openai', 'openrouter'];
+    // Try each provider in order
+    for (const provider of providers) {
+        try {
+            // Check if API key exists for this provider
+            const apiKey = getApiKeyForProvider(provider);
+            if (!apiKey || apiKey.startsWith('api_key') || apiKey === 'phc_xxxxx') {
+                // Silently skip placeholders or missing keys
+                continue;
+            }
+            let output;
+            if (provider === 'anthropic') {
+                output = await runAnthropicTest(candidatePrompt, originalTest.input);
+            }
+            else if (provider === 'openai') {
+                output = await runOpenAITest(candidatePrompt, originalTest.input);
+            }
+            else if (provider === 'openrouter') {
+                output = await runOpenRouterTest(candidatePrompt, originalTest.input);
+            }
+            else {
+                continue; // Unsupported provider
+            }
+            // Score the output using the same method as Phase 1
+            const score = await scoreOutput(output, originalTest.expectedOutput, originalTest.errorType);
+            return {
+                score,
+                output,
+                passed: score >= originalTest.threshold
+            };
+        }
+        catch (error) {
+            console.log(`⚠️  ${provider} provider failed: ${error.message}`);
+            continue; // Try next provider
+        }
+    }
+    // All providers failed
+    console.error('All providers failed for shadow test');
+    return {
+        score: 0,
+        output: '',
+        passed: false
+    };
+}
+function getApiKeyForProvider(provider) {
+    switch (provider) {
+        case 'anthropic':
+            return process.env.ANTHROPIC_API_KEY;
+        case 'openai':
+            return process.env.OPENAI_API_KEY;
+        case 'openrouter':
+            return process.env.OPENROUTER_API_KEY;
+        default:
+            return undefined;
+    }
+}
+async function runAnthropicTest(prompt, input) {
+    const anthropic = new sdk_1.default({
+        apiKey: process.env.ANTHROPIC_API_KEY
+    });
+    // Interpolate variables if present
+    const finalPrompt = interpolateVariables(prompt, input);
+    const response = await anthropic.messages.create({
+        model: 'claude-sonnet-4-20250514',
+        max_tokens: 2000,
+        messages: [{
+                role: 'user',
+                content: finalPrompt
+            }]
+    });
+    const content = response.content[0];
+    return content.type === 'text' ? content.text : '';
+}
+async function runOpenAITest(prompt, input) {
+    const openai = new openai_1.default({
+        apiKey: process.env.OPENAI_API_KEY
+    });
+    const finalPrompt = interpolateVariables(prompt, input);
+    const response = await openai.chat.completions.create({
+        model: 'gpt-4o',
+        messages: [{
+                role: 'user',
+                content: finalPrompt
+            }]
+    });
+    return response.choices[0]?.message?.content || '';
+}
+async function runOpenRouterTest(prompt, input) {
+    const key = process.env.OPENROUTER_API_KEY;
+    // Save original key and temporarily remove it to prevent OpenAI client confusion
+    const originalOpenAIKey = process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_API_KEY;
+    try {
+        const openai = new openai_1.default({
+            baseURL: 'https://openrouter.ai/api/v1',
+            apiKey: key
+        });
+        const finalPrompt = interpolateVariables(prompt, input);
+        const response = await openai.chat.completions.create({
+            model: 'nvidia/nemotron-3-nano-30b-a3b:free',
+            messages: [{
+                    role: 'user',
+                    content: finalPrompt
+                }]
+        });
+        return response.choices[0]?.message?.content || '';
+    }
+    finally {
+        // Restore original key
+        if (originalOpenAIKey) {
+            process.env.OPENAI_API_KEY = originalOpenAIKey;
+        }
+    }
+}
+function interpolateVariables(prompt, variables) {
+    if (!variables)
+        return prompt;
+    let result = prompt;
+    for (const [key, value] of Object.entries(variables)) {
+        result = result.replace(new RegExp(`{{${key}}}`, 'g'), String(value));
+    }
+    return result;
+}
+async function scoreOutput(actual, expected, method) {
+    switch (method) {
+        case 'semantic':
+            return await (0, semantic_1.calculateSemanticSimilarity)(actual, expected);
+        case 'exact':
+            return actual.trim() === expected.trim() ? 1.0 : 0.0;
+        case 'json':
+            try {
+                JSON.parse(actual);
+                return 1.0;
+            }
+            catch {
+                return 0.0;
+            }
+        default:
+            return 0.5;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export * from './engine/runner';
+export * from './engine/loader';
+export * from './engine/optimizer';
+export * from './services/cloud.service';
+export * from './types';
+export { runTests } from './commands/run';
+export { loadConfig } from './utils/config';