tuneprompt 1.1.1 → 1.1.2

package/README.md

@@ -10,17 +10,26 @@ Industrial-grade testing framework for LLM prompts
 
 TunePrompt is a comprehensive testing framework designed specifically for Large Language Model (LLM) prompts. It helps developers validate, test, and optimize their prompts with industrial-grade reliability and accuracy.
 
+## 🚀 What's New in v1.1.1
+
+The first production-ready release of **TunePrompt**, the industrial-grade testing framework for the modern LLM stack.
+
+- **Multi-Provider Support**: Seamlessly test across **OpenAI**, **Anthropic**, **Gemini**, and **OpenRouter**.
+- **Semantic Evaluation**: Advanced vector-based scoring to detect logic drift and nuance shifts.
+- **Auto-Fix Engine (Premium)**: AI-powered prompt optimization for failing tests.
+- **Cloud Orchestration**: Unified synchronization with the [TunePrompt Dashboard](https://www.tuneprompt.xyz).
+- **Industrial CLI**: Built-in watch mode, CI/CD integration, and historical analytics.
+
 ## Features
 
-- **Multi-provider Support**: Test prompts across OpenAI, Anthropic, OpenRouter, and other LLM providers
-- **Semantic Testing**: Compare outputs using semantic similarity rather than exact matches
-- **JSON Validation**: Validate structured JSON outputs
-- **LLM-based Judging**: Use advanced LLMs to evaluate prompt quality
-- **Watch Mode**: Automatically re-run tests when files change
-- **CI/CD Integration**: Seamlessly integrate with your CI/CD pipeline
-- **Cloud Sync**: Upload results to the TunePrompt Cloud dashboard
-- **Auto-fix Engine**: Premium feature to automatically fix failing prompts using AI
-- **Detailed Reporting**: Comprehensive test reports with scores, methods, and durations
+- **Multi-provider Support**: Native integration with Google Gemini, OpenAI, Anthropic, and OpenRouter.
+- **Semantic Testing**: Compare outputs using high-precision embedding similarity.
+- **JSON Validation**: Validate structured outputs with schema-aware checks.
+- **LLM-based Judging**: Utilize advanced providers as evaluators for qualitative metrics.
+- **Watch Mode**: Immediate feedback loop with automatic re-runs on file changes.
+- **CI/CD Ready**: Native integration patterns for industrial deployment pipelines.
+- **Cloud Sync**: Global telemetry and result storage via the dashboard.
+- **Auto-fix Engine**: Iterative refinement loop for intelligent prompt repair.
 
 ## Installation
 

package/dist/engine/loader.js

@@ -44,12 +44,16 @@ class TestLoader {
         if (ext === '.json') {
             const data = JSON.parse(content);
             const tests = Array.isArray(data) ? data : [data];
-            return tests.map(t => ({ ...t, filePath: path.resolve(filePath) }));
+            return tests
+                .filter((t) => t && typeof t === 'object' && t.prompt)
+                .map(t => ({ ...t, filePath: path.resolve(filePath) }));
         }
         else if (ext === '.yaml' || ext === '.yml') {
             const data = yaml.load(content);
             const tests = Array.isArray(data) ? data : [data];
-            return tests.map(t => ({ ...t, filePath: path.resolve(filePath) }));
+            return tests
+                .filter((t) => t && typeof t === 'object' && t.prompt)
+                .map(t => ({ ...t, filePath: path.resolve(filePath) }));
         }
         else {
             throw new Error(`Unsupported file format: ${ext}`);

package/dist/engine/optimizer.js

@@ -130,7 +130,7 @@ class PromptOptimizer {
                 // Pick a strong model for optimization if not defined
                 const model = providerName === 'anthropic' ? 'claude-3-5-sonnet-latest' :
                     providerName === 'openai' ? 'gpt-4o' :
-                        providerName === 'gemini' ? 'gemini-1.5-pro' : undefined;
+                        providerName === 'gemini' ? 'gemini-2.0-flash' : undefined;
                 if (!model)
                     continue;
                 const provider = factory_1.ProviderFactory.create(providerName, {

package/dist/engine/runner.js

@@ -18,7 +18,7 @@ class TestRunner {
         const providerNames = ["openai", "anthropic", "openrouter", "gemini"];
         for (const name of providerNames) {
             const providerConfig = this.config.providers[name];
-            if (providerConfig) {
+            if (providerConfig && providerConfig.apiKey) {
                 this.providers.set(name, factory_1.ProviderFactory.create(name, providerConfig));
             }
         }

package/dist/engine/shadowTester.js

@@ -20,16 +20,32 @@ async function runShadowTest(candidatePrompt, test) {
     }
     const providerName = test.config?.provider;
     const model = test.config?.model;
-    // If specific provider/model is requested, use it directly (Strict Mode)
+    // Determine providers to try
+    let providersToTry = [];
     if (providerName && model) {
+        providersToTry.push({ name: providerName, model });
+    }
+    // Fallback queue
+    const fallbackQueue = [
+        { name: 'anthropic', model: 'claude-3-5-sonnet-latest' },
+        { name: 'openai', model: 'gpt-4o' },
+        { name: 'gemini', model: 'gemini-2.0-flash' },
+        { name: 'openrouter', model: 'nvidia/nemotron-3-nano-30b-a3b:free' }
+    ];
+    for (const entry of fallbackQueue) {
+        if (entry.name !== providerName) {
+            providersToTry.push(entry);
+        }
+    }
+    let errors = [];
+    for (const target of providersToTry) {
         try {
-            const apiKey = factory_1.ProviderFactory.getApiKey(providerName);
-            if (!apiKey) {
-                throw new Error(`No API key found for provider: ${providerName}`);
-            }
-            const provider = factory_1.ProviderFactory.create(providerName, {
+            const apiKey = factory_1.ProviderFactory.getApiKey(target.name);
+            if (!apiKey)
+                continue;
+            const provider = factory_1.ProviderFactory.create(target.name, {
                 apiKey,
-                model,
+                model: target.model || 'latest',
                 maxTokens: 2000
             });
             const finalPrompt = (0, interpolation_1.interpolateVariables)(candidatePrompt, test.input);
@@ -44,12 +60,11 @@ async function runShadowTest(candidatePrompt, test) {
             };
         }
         catch (error) {
-            console.log(`⚠️ Specified provider ${providerName} failed: ${error.message}`);
-            throw new Error(`Failed to validate on target model: ${error.message}`);
+            errors.push(`${target.name}: ${error.message}`);
+            continue;
         }
     }
-    // Phase 2 Decision: Fail fast if no provider/model is defined (Strict Awareness)
-    throw new Error(`Test "${test.description}" lacks provider/model configuration. Validation aborted.`);
+    throw new Error(`Shadow test failed for all providers: ${errors.join(' | ')}`);
 }
 /**
  * Run a candidate prompt against multiple tests and return aggregate results

package/dist/storage/database.js

@@ -118,7 +118,7 @@ class TestDatabase {
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
         for (const result of run.results) {
-            insertResult.run(result.id, run.id, result.testCase.description, typeof result.testCase.prompt === 'string' ? result.testCase.prompt : JSON.stringify(result.testCase.prompt), result.testCase.variables ? JSON.stringify(result.testCase.variables) : null, typeof result.testCase.expect === 'string' ? result.testCase.expect : JSON.stringify(result.testCase.expect), result.testCase.config ? JSON.stringify(result.testCase.config) : null, result.testCase.filePath || null, result.status, result.score, result.actualOutput, result.expectedOutput, result.error || null, result.metadata.duration, result.metadata.tokens || null, result.metadata.cost || null, result.metadata.provider || null);
+            insertResult.run(result.id, run.id, result.testCase.description || 'No description', typeof result.testCase.prompt === 'string' ? result.testCase.prompt : JSON.stringify(result.testCase.prompt), result.testCase.variables ? JSON.stringify(result.testCase.variables) : null, typeof result.testCase.expect === 'string' ? result.testCase.expect : JSON.stringify(result.testCase.expect), result.testCase.config ? JSON.stringify(result.testCase.config) : null, result.testCase.filePath || null, result.status, result.score, result.actualOutput, result.expectedOutput, result.error || null, result.metadata.duration, result.metadata.tokens || null, result.metadata.cost || null, result.metadata.provider || null);
         }
     }
     getRecentRuns(limit = 10) {

package/dist/utils/config.js

@@ -21,11 +21,11 @@ function validateConfig(config) {
     if (!config.providers || Object.keys(config.providers).length === 0) {
         throw new Error('At least one provider must be configured');
     }
-    // Validate API keys
-    for (const [provider, cfg] of Object.entries(config.providers)) {
-        if (!cfg.apiKey) {
-            throw new Error(`API key missing for provider: ${provider}`);
-        }
+    // Validate API keys - ensure at least one provider is valid
+    const validProviders = Object.entries(config.providers)
+        .filter(([_, cfg]) => !!cfg.apiKey);
+    if (validProviders.length === 0) {
+        throw new Error('No valid API keys found. Please provide at least one API key in your .env file.');
     }
     return {
         threshold: config.threshold || 0.8,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tuneprompt",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "Industrial-grade testing framework for LLM prompts",
   "repository": {
     "type": "git",