tuneprompt 1.0.7 → 1.1.2

package/dist/engine/shadowTester.js

@@ -1,156 +1,114 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runShadowTest = runShadowTest;
-const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
-const openai_1 = __importDefault(require("openai"));
-const semantic_1 = require("../scoring/semantic"); // From Phase 1
+exports.runSuiteShadowTest = runSuiteShadowTest;
+const semantic_1 = require("../scoring/semantic");
+const factory_1 = require("../providers/factory");
+const interpolation_1 = require("../utils/interpolation");
 /**
  * Test a candidate prompt against the original test case
- * Tries providers in sequence until one succeeds
+ * Uses specified provider/model or falls back to priority sequence
  */
-async function runShadowTest(candidatePrompt, originalTest) {
-    // Define provider priority order
-    const providers = ['anthropic', 'openai', 'openrouter'];
-    // Try each provider in order
-    for (const provider of providers) {
+async function runShadowTest(candidatePrompt, test) {
+    // For integration tests: bypass real API calls if mock mode is on
+    if (process.env.TUNEPROMPT_MOCK_OPTIMIZER === 'true') {
+        return {
+            score: 0.95,
+            output: 'Mock satisfied output',
+            passed: true
+        };
+    }
+    const providerName = test.config?.provider;
+    const model = test.config?.model;
+    // Determine providers to try
+    let providersToTry = [];
+    if (providerName && model) {
+        providersToTry.push({ name: providerName, model });
+    }
+    // Fallback queue
+    const fallbackQueue = [
+        { name: 'anthropic', model: 'claude-3-5-sonnet-latest' },
+        { name: 'openai', model: 'gpt-4o' },
+        { name: 'gemini', model: 'gemini-2.0-flash' },
+        { name: 'openrouter', model: 'nvidia/nemotron-3-nano-30b-a3b:free' }
+    ];
+    for (const entry of fallbackQueue) {
+        if (entry.name !== providerName) {
+            providersToTry.push(entry);
+        }
+    }
+    let errors = [];
+    for (const target of providersToTry) {
         try {
-            // Check if API key exists for this provider
-            const apiKey = getApiKeyForProvider(provider);
-            if (!apiKey || apiKey.startsWith('api_key') || apiKey === 'phc_xxxxx') {
-                // Silently skip placeholders or missing keys
+            const apiKey = factory_1.ProviderFactory.getApiKey(target.name);
+            if (!apiKey)
                 continue;
-            }
-            let output;
-            if (provider === 'anthropic') {
-                output = await runAnthropicTest(candidatePrompt, originalTest.input);
-            }
-            else if (provider === 'openai') {
-                output = await runOpenAITest(candidatePrompt, originalTest.input);
-            }
-            else if (provider === 'openrouter') {
-                output = await runOpenRouterTest(candidatePrompt, originalTest.input);
-            }
-            else {
-                continue; // Unsupported provider
-            }
-            // Score the output using the same method as Phase 1
-            const score = await scoreOutput(output, originalTest.expectedOutput, originalTest.errorType);
+            const provider = factory_1.ProviderFactory.create(target.name, {
+                apiKey,
+                model: target.model || 'latest',
+                maxTokens: 2000
+            });
+            const finalPrompt = (0, interpolation_1.interpolateVariables)(candidatePrompt, test.input);
+            const response = await provider.complete(finalPrompt);
+            const output = response.content;
+            const { score, failureReason } = await scoreOutput(output, test.expectedOutput, test.errorType);
             return {
                 score,
                 output,
-                passed: score >= originalTest.threshold
+                passed: score >= test.threshold,
+                failureReason
             };
         }
         catch (error) {
-            console.log(`⚠️  ${provider} provider failed: ${error.message}`);
-            continue; // Try next provider
+            errors.push(`${target.name}: ${error.message}`);
+            continue;
         }
     }
-    // All providers failed
-    console.error('All providers failed for shadow test');
+    throw new Error(`Shadow test failed for all providers: ${errors.join(' | ')}`);
+}
+/**
+ * Run a candidate prompt against multiple tests and return aggregate results
+ */
+async function runSuiteShadowTest(candidatePrompt, tests) {
+    const results = await Promise.all(tests.map(async (test) => {
+        const result = await runShadowTest(candidatePrompt, test);
+        return {
+            testId: test.id,
+            score: result.score,
+            passed: result.passed,
+            output: result.output,
+            failureReason: result.failureReason
+        };
+    }));
+    const aggregateScore = results.reduce((sum, r) => sum + r.score, 0) / results.length;
     return {
-        score: 0,
-        output: '',
-        passed: false
+        aggregateScore,
+        results
     };
 }
-function getApiKeyForProvider(provider) {
-    switch (provider) {
-        case 'anthropic':
-            return process.env.ANTHROPIC_API_KEY;
-        case 'openai':
-            return process.env.OPENAI_API_KEY;
-        case 'openrouter':
-            return process.env.OPENROUTER_API_KEY;
-        default:
-            return undefined;
-    }
-}
-async function runAnthropicTest(prompt, input) {
-    const anthropic = new sdk_1.default({
-        apiKey: process.env.ANTHROPIC_API_KEY
-    });
-    // Interpolate variables if present
-    const finalPrompt = interpolateVariables(prompt, input);
-    const response = await anthropic.messages.create({
-        model: 'claude-sonnet-4-20250514',
-        max_tokens: 2000,
-        messages: [{
-                role: 'user',
-                content: finalPrompt
-            }]
-    });
-    const content = response.content[0];
-    return content.type === 'text' ? content.text : '';
-}
-async function runOpenAITest(prompt, input) {
-    const openai = new openai_1.default({
-        apiKey: process.env.OPENAI_API_KEY
-    });
-    const finalPrompt = interpolateVariables(prompt, input);
-    const response = await openai.chat.completions.create({
-        model: 'gpt-4o',
-        messages: [{
-                role: 'user',
-                content: finalPrompt
-            }]
-    });
-    return response.choices[0]?.message?.content || '';
-}
-async function runOpenRouterTest(prompt, input) {
-    const key = process.env.OPENROUTER_API_KEY;
-    // Save original key and temporarily remove it to prevent OpenAI client confusion
-    const originalOpenAIKey = process.env.OPENAI_API_KEY;
-    delete process.env.OPENAI_API_KEY;
-    try {
-        const openai = new openai_1.default({
-            baseURL: 'https://openrouter.ai/api/v1',
-            apiKey: key
-        });
-        const finalPrompt = interpolateVariables(prompt, input);
-        const response = await openai.chat.completions.create({
-            model: 'nvidia/nemotron-3-nano-30b-a3b:free',
-            messages: [{
-                    role: 'user',
-                    content: finalPrompt
-                }]
-        });
-        return response.choices[0]?.message?.content || '';
-    }
-    finally {
-        // Restore original key
-        if (originalOpenAIKey) {
-            process.env.OPENAI_API_KEY = originalOpenAIKey;
-        }
-    }
-}
-function interpolateVariables(prompt, variables) {
-    if (!variables)
-        return prompt;
-    let result = prompt;
-    for (const [key, value] of Object.entries(variables)) {
-        result = result.replace(new RegExp(`{{${key}}}`, 'g'), String(value));
-    }
-    return result;
-}
 async function scoreOutput(actual, expected, method) {
     switch (method) {
-        case 'semantic':
-            return await (0, semantic_1.calculateSemanticSimilarity)(actual, expected);
-        case 'exact':
-            return actual.trim() === expected.trim() ? 1.0 : 0.0;
-        case 'json':
+        case 'semantic': {
+            const score = await (0, semantic_1.calculateSemanticSimilarity)(actual, expected);
+            return { score, failureReason: score < 0.9 ? `Semantic similarity (${score.toFixed(2)}) is low. Output did not capture expected meaning.` : undefined };
+        }
+        case 'exact': {
+            const exactMatch = actual.trim() === expected.trim();
+            return {
+                score: exactMatch ? 1.0 : 0.0,
+                failureReason: exactMatch ? undefined : `Expected exact match but output differed.`
+            };
+        }
+        case 'json': {
             try {
                 JSON.parse(actual);
-                return 1.0;
+                return { score: 1.0 };
             }
-            catch {
-                return 0.0;
+            catch (e) {
+                return { score: 0.0, failureReason: `Did not output valid JSON. Parse error: ${e.message}` };
             }
+        }
         default:
-            return 0.5;
+            return { score: 0.5, failureReason: `Unknown scoring method: ${method}` };
     }
 }

package/dist/providers/__tests__/custom.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/providers/__tests__/custom.test.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const custom_1 = require("../custom");
+describe('CustomProvider', () => {
+    it('should instantiate custom provider with arbitrary endpoint', () => {
+        const provider = new custom_1.CustomProvider({ endpoint: 'http://localhost:11434/api/generate' });
+        expect(provider.endpoint).toBe('http://localhost:11434/api/generate');
+    });
+});

package/dist/providers/custom.d.ts

@@ -0,0 +1,6 @@
+export declare class CustomProvider {
+    endpoint: string;
+    constructor(config: {
+        endpoint: string;
+    });
+}

package/dist/providers/custom.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CustomProvider = void 0;
+class CustomProvider {
+    endpoint;
+    constructor(config) {
+        this.endpoint = config.endpoint;
+    }
+}
+exports.CustomProvider = CustomProvider;

package/dist/providers/factory.d.ts

@@ -0,0 +1,6 @@
+import { ProviderConfig } from '../types';
+import { BaseProvider } from './base';
+export declare class ProviderFactory {
+    static create(provider: string, config: ProviderConfig): BaseProvider;
+    static getApiKey(provider: string): string | undefined;
+}

package/dist/providers/factory.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProviderFactory = void 0;
+const openai_1 = require("./openai");
+const anthropic_1 = require("./anthropic");
+const gemini_1 = require("./gemini");
+const openrouter_1 = require("./openrouter");
+class ProviderFactory {
+    static create(provider, config) {
+        switch (provider.toLowerCase()) {
+            case 'openai':
+                return new openai_1.OpenAIProvider(config);
+            case 'anthropic':
+                return new anthropic_1.AnthropicProvider(config);
+            case 'gemini':
+                return new gemini_1.GeminiProvider(config);
+            case 'openrouter':
+                return new openrouter_1.OpenRouterProvider(config);
+            default:
+                throw new Error(`Unsupported provider: ${provider}`);
+        }
+    }
+    static getApiKey(provider) {
+        switch (provider.toLowerCase()) {
+            case 'openai':
+                return process.env.OPENAI_API_KEY;
+            case 'anthropic':
+                return process.env.ANTHROPIC_API_KEY;
+            case 'gemini':
+                return process.env.GEMINI_API_KEY;
+            case 'openrouter':
+                return process.env.OPENROUTER_API_KEY;
+            default:
+                return undefined;
+        }
+    }
+}
+exports.ProviderFactory = ProviderFactory;

package/dist/providers/gemini.d.ts

@@ -0,0 +1,11 @@
+import { BaseProvider, ProviderResponse } from './base';
+import { ProviderConfig } from '../types';
+export declare class GeminiProvider extends BaseProvider {
+    private ai;
+    constructor(config: ProviderConfig);
+    complete(prompt: string | {
+        system?: string;
+        user: string;
+    }): Promise<ProviderResponse>;
+    getEmbedding(text: string): Promise<number[]>;
+}

package/dist/providers/gemini.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GeminiProvider = void 0;
+const genai_1 = require("@google/genai");
+const base_1 = require("./base");
+class GeminiProvider extends base_1.BaseProvider {
+    ai;
+    constructor(config) {
+        super(config);
+        this.ai = new genai_1.GoogleGenAI({ apiKey: config.apiKey });
+    }
+    async complete(prompt) {
+        let textContent = '';
+        let systemInstruction = undefined;
+        if (typeof prompt === 'string') {
+            textContent = prompt;
+        }
+        else {
+            textContent = prompt.user;
+            systemInstruction = prompt.system;
+        }
+        const response = await this.ai.models.generateContent({
+            model: this.config.model,
+            contents: textContent,
+            config: {
+                systemInstruction: systemInstruction,
+                maxOutputTokens: this.config.maxTokens,
+                temperature: this.config.temperature
+            }
+        });
+        const content = response.text || '';
+        return {
+            content,
+            tokens: 0,
+            cost: 0
+        };
+    }
+    async getEmbedding(text) {
+        const response = await this.ai.models.embedContent({
+            model: 'text-embedding-004',
+            contents: text
+        });
+        return response.embeddings?.[0]?.values || [];
+    }
+}
+exports.GeminiProvider = GeminiProvider;

package/dist/scoring/__tests__/rag.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/scoring/__tests__/rag.test.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const rag_1 = require("../rag");
+describe('RAGScorer', () => {
+    it('should instantiate and evaluate', async () => {
+        const scorer = new rag_1.RAGScorer();
+        const result = await scorer.score({ expected: '', actual: 'Test' });
+        expect(result.score).toBeDefined();
+    });
+});

package/dist/scoring/rag.d.ts

@@ -0,0 +1,9 @@
+export declare class RAGScorer {
+    score(params: {
+        expected: string;
+        actual: string;
+    }): Promise<{
+        score: number;
+        reasoning: string;
+    }>;
+}

package/dist/scoring/rag.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RAGScorer = void 0;
+class RAGScorer {
+    async score(params) {
+        return { score: 1.0, reasoning: 'Simulated RAG pass' };
+    }
+}
+exports.RAGScorer = RAGScorer;

package/dist/services/cloud.service.js

@@ -10,7 +10,7 @@ class CloudService {
     backendUrl;
     subscriptionId;
     constructor() {
-        this.backendUrl = process.env.TUNEPROMPT_API_URL || process.env.BACKEND_URL || 'https://i8e3mu8jlk.execute-api.ap-south-1.amazonaws.com/dev';
+        this.backendUrl = process.env.TUNEPROMPT_API_URL || 'https://api.tuneprompt.com';
     }
     async init() {
         // Load subscription ID from local storage (Phase 2 activation)

package/dist/storage/database.js

@@ -118,7 +118,7 @@ class TestDatabase {
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
         for (const result of run.results) {
-            insertResult.run(result.id, run.id, result.testCase.description, typeof result.testCase.prompt === 'string' ? result.testCase.prompt : JSON.stringify(result.testCase.prompt), result.testCase.variables ? JSON.stringify(result.testCase.variables) : null, typeof result.testCase.expect === 'string' ? result.testCase.expect : JSON.stringify(result.testCase.expect), result.testCase.config ? JSON.stringify(result.testCase.config) : null, result.testCase.filePath || null, result.status, result.score, result.actualOutput, result.expectedOutput, result.error || null, result.metadata.duration, result.metadata.tokens || null, result.metadata.cost || null, result.metadata.provider || null);
+            insertResult.run(result.id, run.id, result.testCase.description || 'No description', typeof result.testCase.prompt === 'string' ? result.testCase.prompt : JSON.stringify(result.testCase.prompt), result.testCase.variables ? JSON.stringify(result.testCase.variables) : null, typeof result.testCase.expect === 'string' ? result.testCase.expect : JSON.stringify(result.testCase.expect), result.testCase.config ? JSON.stringify(result.testCase.config) : null, result.testCase.filePath || null, result.status, result.score, result.actualOutput, result.expectedOutput, result.error || null, result.metadata.duration, result.metadata.tokens || null, result.metadata.cost || null, result.metadata.provider || null);
         }
     }
     getRecentRuns(limit = 10) {

package/dist/types/fix.d.ts

@@ -9,6 +9,10 @@ export interface FailedTest {
     threshold: number;
     errorType: 'semantic' | 'json' | 'exact' | 'length';
     errorMessage: string;
+    config?: {
+        provider?: string;
+        model?: string;
+    };
 }
 export interface OptimizationResult {
     originalPrompt: string;
@@ -19,10 +23,17 @@ export interface OptimizationResult {
         score: number;
         passed: boolean;
         output: string;
+        aggregateScore?: number;
     };
+    iterations?: number;
 }
 export interface FixCandidate {
     prompt: string;
     score: number;
     reasoning: string;
+    testResults?: {
+        testId: string;
+        score: number;
+        passed: boolean;
+    }[];
 }

package/dist/types/index.d.ts

@@ -10,7 +10,7 @@ export interface TestCase {
         threshold?: number;
         method?: 'exact' | 'semantic' | 'json' | 'llm-judge';
         model?: string;
-        provider?: 'openai' | 'anthropic' | 'openrouter';
+        provider?: 'openai' | 'anthropic' | 'gemini' | 'openrouter';
     };
     filePath?: string;
 }
@@ -50,6 +50,7 @@ export interface TunePromptConfig {
     providers: {
         openai?: ProviderConfig;
         anthropic?: ProviderConfig;
+        gemini?: ProviderConfig;
         openrouter?: ProviderConfig;
     };
     threshold?: number;

package/dist/types/test.d.ts

@@ -0,0 +1,8 @@
+export type Step = {
+    prompt: string;
+    expect: string;
+};
+export type TestConfig = {
+    prompt?: string;
+    steps?: Step[];
+};

package/dist/types/test.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/config.js

@@ -21,11 +21,11 @@ function validateConfig(config) {
     if (!config.providers || Object.keys(config.providers).length === 0) {
         throw new Error('At least one provider must be configured');
     }
-    // Validate API keys
-    for (const [provider, cfg] of Object.entries(config.providers)) {
-        if (!cfg.apiKey) {
-            throw new Error(`API key missing for provider: ${provider}`);
-        }
+    // Validate API keys - ensure at least one provider is valid
+    const validProviders = Object.entries(config.providers)
+        .filter(([_, cfg]) => !!cfg.apiKey);
+    if (validProviders.length === 0) {
+        throw new Error('No valid API keys found. Please provide at least one API key in your .env file.');
     }
     return {
         threshold: config.threshold || 0.8,
@@ -49,6 +49,12 @@ function getDefaultConfigTemplate() {
       maxTokens: 1000,
       temperature: 0.7
     },
+    gemini: {
+      apiKey: process.env.GEMINI_API_KEY,
+      model: 'gemini-2.5-flash',
+      maxTokens: 1000,
+      temperature: 0.7
+    },
     openrouter: {
       apiKey: process.env.OPENROUTER_API_KEY,
       model: 'nvidia/nemotron-3-nano-30b-a3b:free',

package/dist/utils/interpolation.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Interpolate variables in a prompt string using {{variableName}} syntax
+ */
+export declare function interpolateVariables(prompt: string, variables?: Record<string, any>): string;

package/dist/utils/interpolation.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.interpolateVariables = interpolateVariables;
+/**
+ * Interpolate variables in a prompt string using {{variableName}} syntax
+ */
+function interpolateVariables(prompt, variables) {
+    if (!variables)
+        return prompt;
+    let result = prompt;
+    for (const [key, value] of Object.entries(variables)) {
+        // Use a global regex to replace all occurrences
+        result = result.replace(new RegExp(`{{${key}}}`, 'g'), String(value));
+    }
+    return result;
+}

package/dist/utils/storage.d.ts

@@ -1,2 +1,6 @@
 import { FailedTest } from '../types/fix';
 export declare function getFailedTests(): Promise<FailedTest[]>;
+/**
+ * Get the full suite of tests (passing and failing) for a specific prompt file
+ */
+export declare function getSuiteTests(filePath: string): Promise<FailedTest[]>;

package/dist/utils/storage.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getFailedTests = getFailedTests;
+exports.getSuiteTests = getSuiteTests;
 const database_1 = require("../storage/database");
 async function getFailedTests() {
     const db = new database_1.TestDatabase();
@@ -10,16 +11,36 @@ async function getFailedTests() {
     }
     const latestRun = recentRuns[0];
     const failures = latestRun.results.filter(r => r.status === 'fail' || r.status === 'error');
-    return failures.map(r => ({
-        id: r.testCase.filePath || r.id, // Prefer filePath for targeting the correct file
+    return failures.map(mapResultToFailedTest);
+}
+/**
+ * Get the full suite of tests (passing and failing) for a specific prompt file
+ */
+async function getSuiteTests(filePath) {
+    const db = new database_1.TestDatabase();
+    const recentRuns = db.getRecentRuns(1);
+    if (recentRuns.length === 0) {
+        return [];
+    }
+    const latestRun = recentRuns[0];
+    const suite = latestRun.results.filter(r => r.testCase.filePath === filePath);
+    return suite.map(mapResultToFailedTest);
+}
+function mapResultToFailedTest(r) {
+    return {
+        id: r.testCase.filePath || r.id,
         description: r.testCase.description,
         prompt: !r.testCase.prompt ? '' : (typeof r.testCase.prompt === 'string' ? r.testCase.prompt : r.testCase.prompt.user),
         input: r.testCase.variables,
         expectedOutput: typeof r.testCase.expect === 'string' ? r.testCase.expect : JSON.stringify(r.testCase.expect),
-        actualOutput: r.actualOutput,
+        actualOutput: r.actualOutput || '',
         score: r.score,
         threshold: r.testCase.config?.threshold || 0.8,
         errorType: r.testCase.config?.method || 'semantic',
-        errorMessage: r.error || ''
-    }));
+        errorMessage: r.error || '',
+        config: {
+            provider: r.metadata?.provider,
+            model: r.testCase.config?.model || r.testCase.config?.modelId
+        }
+    };
 }

package/dist/utils/validator.d.ts

@@ -0,0 +1,2 @@
+import { TestConfig } from '../types/test';
+export declare function validateTestFile(configs: TestConfig[]): void;

package/dist/utils/validator.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateTestFile = validateTestFile;
+function validateTestFile(configs) {
+    for (const config of configs) {
+        if (!config.prompt && (!config.steps || config.steps.length === 0)) {
+            throw new Error("Invalid configuration: missing required 'prompt' field");
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuneprompt",
-  "version": "1.0.7",
+  "version": "1.1.2",
   "description": "Industrial-grade testing framework for LLM prompts",
   "repository": {
     "type": "git",
@@ -13,7 +13,7 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "bin": {
-    "tuneprompt": "./dist/cli.js"
+    "tuneprompt": "dist/cli.js"
   },
   "files": [
     "dist",
@@ -58,6 +58,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.71.2",
+    "@google/genai": "^1.44.0",
     "@types/chokidar": "^1.7.5",
     "axios": "^1.13.2",
     "better-sqlite3": "^12.5.0",