tuneprompt 1.0.0

package/dist/index.js

@@ -0,0 +1,26 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadConfig = exports.runTests = void 0;
+__exportStar(require("./engine/runner"), exports);
+__exportStar(require("./engine/loader"), exports);
+__exportStar(require("./engine/optimizer"), exports);
+__exportStar(require("./services/cloud.service"), exports);
+__exportStar(require("./types"), exports);
+var run_1 = require("./commands/run");
+Object.defineProperty(exports, "runTests", { enumerable: true, get: function () { return run_1.runTests; } });
+var config_1 = require("./utils/config");
+Object.defineProperty(exports, "loadConfig", { enumerable: true, get: function () { return config_1.loadConfig; } });

package/dist/providers/anthropic.d.ts

@@ -0,0 +1,12 @@
+import { BaseProvider, ProviderResponse } from './base';
+import { ProviderConfig } from '../types';
+export declare class AnthropicProvider extends BaseProvider {
+    private client;
+    constructor(config: ProviderConfig);
+    complete(prompt: string | {
+        system?: string;
+        user: string;
+    }): Promise<ProviderResponse>;
+    getEmbedding(text: string): Promise<number[]>;
+    private calculateCost;
+}

package/dist/providers/anthropic.js

@@ -0,0 +1,51 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AnthropicProvider = void 0;
+const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const base_1 = require("./base");
+class AnthropicProvider extends base_1.BaseProvider {
+    client;
+    constructor(config) {
+        super(config);
+        this.client = new sdk_1.default({
+            apiKey: config.apiKey
+        });
+    }
+    async complete(prompt) {
+        const systemPrompt = typeof prompt === 'string' ? undefined : prompt.system;
+        const userPrompt = typeof prompt === 'string' ? prompt : prompt.user;
+        const response = await this.client.messages.create({
+            model: this.config.model,
+            max_tokens: this.config.maxTokens || 1000,
+            temperature: this.config.temperature,
+            system: systemPrompt,
+            messages: [
+                { role: 'user', content: userPrompt }
+            ]
+        });
+        const content = response.content[0].type === 'text'
+            ? response.content[0].text
+            : '';
+        return {
+            content,
+            tokens: response.usage.input_tokens + response.usage.output_tokens,
+            cost: this.calculateCost(response.usage.input_tokens, response.usage.output_tokens)
+        };
+    }
+    async getEmbedding(text) {
+        // Anthropic doesn't provide embeddings, use OpenAI as fallback
+        // or implement a separate embedding service
+        throw new Error('Anthropic does not support embeddings. Use OpenAI provider.');
+    }
+    calculateCost(inputTokens, outputTokens) {
+        // Claude Sonnet 4 pricing (example)
+        const inputCostPer1M = 3.00;
+        const outputCostPer1M = 15.00;
+        return ((inputTokens / 1_000_000) * inputCostPer1M +
+            (outputTokens / 1_000_000) * outputCostPer1M);
+    }
+}
+exports.AnthropicProvider = AnthropicProvider;

package/dist/providers/base.d.ts

@@ -0,0 +1,15 @@
+import { ProviderConfig } from '../types';
+export interface ProviderResponse {
+    content: string;
+    tokens?: number;
+    cost?: number;
+}
+export declare abstract class BaseProvider {
+    protected config: ProviderConfig;
+    constructor(config: ProviderConfig);
+    abstract complete(prompt: string | {
+        system?: string;
+        user: string;
+    }): Promise<ProviderResponse>;
+    abstract getEmbedding(text: string): Promise<number[]>;
+}

package/dist/providers/base.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseProvider = void 0;
+class BaseProvider {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+}
+exports.BaseProvider = BaseProvider;

package/dist/providers/openai.d.ts

@@ -0,0 +1,12 @@
+import { BaseProvider, ProviderResponse } from './base';
+import { ProviderConfig } from '../types';
+export declare class OpenAIProvider extends BaseProvider {
+    private client;
+    constructor(config: ProviderConfig);
+    complete(prompt: string | {
+        system?: string;
+        user: string;
+    }): Promise<ProviderResponse>;
+    getEmbedding(text: string): Promise<number[]>;
+    private calculateCost;
+}

package/dist/providers/openai.js

@@ -0,0 +1,58 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIProvider = void 0;
+const openai_1 = __importDefault(require("openai"));
+const base_1 = require("./base");
+class OpenAIProvider extends base_1.BaseProvider {
+    client;
+    constructor(config) {
+        super(config);
+        this.client = new openai_1.default({
+            apiKey: config.apiKey,
+            baseURL: config.baseURL
+        });
+    }
+    async complete(prompt) {
+        const messages = [];
+        if (typeof prompt === 'string') {
+            messages.push({ role: 'user', content: prompt });
+        }
+        else {
+            if (prompt.system) {
+                messages.push({ role: 'system', content: prompt.system });
+            }
+            messages.push({ role: 'user', content: prompt.user });
+        }
+        const response = await this.client.chat.completions.create({
+            model: this.config.model,
+            messages,
+            max_tokens: this.config.maxTokens,
+            temperature: this.config.temperature
+        });
+        const content = response.choices[0]?.message?.content || '';
+        const tokens = response.usage?.total_tokens;
+        return {
+            content,
+            tokens,
+            cost: this.calculateCost(tokens || 0)
+        };
+    }
+    async getEmbedding(text) {
+        const response = await this.client.embeddings.create({
+            model: 'text-embedding-3-small',
+            input: text
+        });
+        return response.data[0].embedding;
+    }
+    calculateCost(tokens) {
+        // Pricing for GPT-4o (example rates)
+        const inputCostPer1k = 0.005;
+        const outputCostPer1k = 0.015;
+        // Simplified: assume 50/50 input/output split
+        return ((tokens / 1000) * (inputCostPer1k + outputCostPer1k)) / 2;
+    }
+}
+exports.OpenAIProvider = OpenAIProvider;

package/dist/providers/openrouter.d.ts

@@ -0,0 +1,11 @@
+import { BaseProvider, ProviderResponse } from './base';
+import { ProviderConfig } from '../types';
+export declare class OpenRouterProvider extends BaseProvider {
+    private baseURL;
+    constructor(config: ProviderConfig);
+    complete(prompt: string | {
+        system?: string;
+        user: string;
+    }): Promise<ProviderResponse>;
+    getEmbedding(text: string): Promise<number[]>;
+}

package/dist/providers/openrouter.js

@@ -0,0 +1,83 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenRouterProvider = void 0;
+const axios_1 = __importDefault(require("axios"));
+const base_1 = require("./base");
+class OpenRouterProvider extends base_1.BaseProvider {
+    baseURL;
+    constructor(config) {
+        super(config);
+        this.baseURL = config.baseURL || 'https://openrouter.ai/api/v1';
+    }
+    async complete(prompt) {
+        const messages = [];
+        if (typeof prompt === 'string') {
+            messages.push({ role: 'user', content: prompt });
+        }
+        else {
+            if (prompt.system) {
+                messages.push({ role: 'system', content: prompt.system });
+            }
+            messages.push({ role: 'user', content: prompt.user });
+        }
+        try {
+            const response = await axios_1.default.post(`${this.baseURL}/chat/completions`, {
+                model: this.config.model,
+                messages,
+                max_tokens: this.config.maxTokens,
+                temperature: this.config.temperature
+            }, {
+                headers: {
+                    'Authorization': `Bearer ${this.config.apiKey}`,
+                    'HTTP-Referer': 'https://github.com/tuneprompt/tuneprompt',
+                    'X-Title': 'TunePrompt',
+                    'Content-Type': 'application/json'
+                }
+            });
+            const data = response.data;
+            const content = data.choices?.[0]?.message?.content || '';
+            const tokens = data.usage?.total_tokens || 0;
+            return {
+                content,
+                tokens,
+                cost: 0
+            };
+        }
+        catch (error) {
+            if (error.response) {
+                const errorMsg = error.response.data?.error?.message || JSON.stringify(error.response.data);
+                throw new Error(`OpenRouter API Error (${error.response.status}): ${errorMsg}`);
+            }
+            throw new Error(`OpenRouter network error: ${error.message}`);
+        }
+    }
+    async getEmbedding(text) {
+        try {
+            // Attempt to use OpenRouter's embedding endpoint (which proxies to OpenAI or others)
+            // Note: User must be entitled to use the requested embedding model
+            const response = await axios_1.default.post(`${this.baseURL}/embeddings`, {
+                model: 'text-embedding-3-small', // Default fallback, customizable in future
+                input: text
+            }, {
+                headers: {
+                    'Authorization': `Bearer ${this.config.apiKey}`,
+                    'HTTP-Referer': 'https://github.com/tuneprompt/tuneprompt',
+                    'X-Title': 'TunePrompt',
+                    'Content-Type': 'application/json'
+                }
+            });
+            return response.data.data[0].embedding;
+        }
+        catch (error) {
+            if (error.response) {
+                const errorMsg = error.response.data?.error?.message || JSON.stringify(error.response.data);
+                throw new Error(`OpenRouter Embedding Error (${error.response.status}): ${errorMsg}`);
+            }
+            throw new Error(`OpenRouter embedding network error: ${error.message}`);
+        }
+    }
+}
+exports.OpenRouterProvider = OpenRouterProvider;

package/dist/scoring/exact-match.d.ts

@@ -0,0 +1 @@
+export declare function exactMatch(expected: string, actual: string): number;

package/dist/scoring/exact-match.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.exactMatch = exactMatch;
+function exactMatch(expected, actual) {
+    const normalizedExpected = expected.trim().toLowerCase();
+    const normalizedActual = actual.trim().toLowerCase();
+    return normalizedExpected === normalizedActual ? 1.0 : 0.0;
+}

package/dist/scoring/json-validator.d.ts

@@ -0,0 +1,4 @@
+export declare function validateJSON(expected: any, actual: string): {
+    score: number;
+    error?: string;
+};

package/dist/scoring/json-validator.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateJSON = validateJSON;
+function validateJSON(expected, actual) {
+    try {
+        const parsed = JSON.parse(actual);
+        // Check if structure matches
+        const matches = deepCompare(expected, parsed);
+        return { score: matches ? 1.0 : 0.0 };
+    }
+    catch (error) {
+        return {
+            score: 0.0,
+            error: `Invalid JSON: ${error.message}`
+        };
+    }
+}
+function deepCompare(expected, actual) {
+    if (typeof expected !== typeof actual)
+        return false;
+    if (typeof expected === 'object' && expected !== null) {
+        const expectedKeys = Object.keys(expected);
+        const actualKeys = Object.keys(actual);
+        if (expectedKeys.length !== actualKeys.length)
+            return false;
+        return expectedKeys.every(key => actualKeys.includes(key) && deepCompare(expected[key], actual[key]));
+    }
+    return expected === actual;
+}

package/dist/scoring/semantic.d.ts

@@ -0,0 +1,8 @@
+import { BaseProvider } from '../providers/base';
+export declare class SemanticScorer {
+    private embeddingProvider;
+    constructor(provider: BaseProvider);
+    score(expected: string, actual: string): Promise<number>;
+    private cosineSimilarity;
+}
+export declare function calculateSemanticSimilarity(actual: string, expected: string): Promise<number>;

package/dist/scoring/semantic.js

@@ -0,0 +1,107 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SemanticScorer = void 0;
+exports.calculateSemanticSimilarity = calculateSemanticSimilarity;
+class SemanticScorer {
+    embeddingProvider;
+    constructor(provider) {
+        this.embeddingProvider = provider;
+    }
+    async score(expected, actual) {
+        const [expectedEmb, actualEmb] = await Promise.all([
+            this.embeddingProvider.getEmbedding(expected),
+            this.embeddingProvider.getEmbedding(actual)
+        ]);
+        return this.cosineSimilarity(expectedEmb, actualEmb);
+    }
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error('Vectors must have same length');
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+}
+exports.SemanticScorer = SemanticScorer;
+async function calculateSemanticSimilarity(actual, expected) {
+    const providersToTry = [];
+    // Prioritize OpenAI if key looks potentially valid
+    if (process.env.OPENAI_API_KEY && !process.env.OPENAI_API_KEY.startsWith('api_key')) {
+        providersToTry.push('openai');
+    }
+    // Always try OpenRouter if key is present
+    if (process.env.OPENROUTER_API_KEY) {
+        providersToTry.push('openrouter');
+    }
+    // Fallback to OpenAI if nothing else was added
+    if (providersToTry.length === 0) {
+        providersToTry.push('openai');
+    }
+    let lastError;
+    for (const providerType of providersToTry) {
+        try {
+            let provider;
+            if (providerType === 'openai') {
+                const { OpenAIProvider } = await Promise.resolve().then(() => __importStar(require('../providers/openai')));
+                provider = new OpenAIProvider({
+                    apiKey: process.env.OPENAI_API_KEY || '',
+                    model: 'text-embedding-3-small'
+                });
+            }
+            else {
+                const { OpenRouterProvider } = await Promise.resolve().then(() => __importStar(require('../providers/openrouter')));
+                provider = new OpenRouterProvider({
+                    apiKey: process.env.OPENROUTER_API_KEY || '',
+                    model: 'text-embedding-3-small'
+                });
+            }
+            const scorer = new SemanticScorer(provider);
+            return await scorer.score(expected, actual);
+        }
+        catch (error) {
+            lastError = error;
+            continue;
+        }
+    }
+    throw lastError || new Error('No embedding provider available');
+}

package/dist/services/cloud.service.d.ts

@@ -0,0 +1,49 @@
+export interface RunData {
+    project_id: string;
+    commit_hash?: string;
+    branch_name?: string;
+    commit_message?: string;
+    environment: 'local' | 'ci' | 'production';
+    ci_provider?: string;
+    total_tests: number;
+    passed_tests: number;
+    failed_tests: number;
+    duration_ms: number;
+    cost_usd: number;
+    started_at: string;
+    completed_at: string;
+    test_results: CloudTestResult[];
+}
+export interface CloudTestResult {
+    test_name: string;
+    test_description?: string;
+    prompt: string;
+    input_data?: any;
+    expected_output: string;
+    actual_output: string;
+    score?: number;
+    method: string;
+    status: 'pass' | 'fail' | 'error';
+    model: string;
+    tokens_used?: number;
+    latency_ms?: number;
+    cost_usd?: number;
+    error_message?: string;
+    error_type?: string;
+}
+export declare class CloudService {
+    private backendUrl;
+    private subscriptionId?;
+    constructor();
+    init(): Promise<void>;
+    private getSubscriptionId;
+    uploadRun(data: RunData): Promise<{
+        success: boolean;
+        run_id?: string;
+        url?: string;
+        error?: string;
+    }>;
+    createProject(name: string, description?: string): Promise<any>;
+    getProjects(): Promise<any[]>;
+    isAuthenticated(): Promise<boolean>;
+}

package/dist/services/cloud.service.js

@@ -0,0 +1,82 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CloudService = void 0;
+const axios_1 = __importDefault(require("axios"));
+const license_1 = require("../utils/license");
+class CloudService {
+    backendUrl;
+    subscriptionId;
+    constructor() {
+        this.backendUrl = process.env.TUNEPROMPT_API_URL || process.env.BACKEND_URL || 'https://api.tuneprompt.com';
+    }
+    async init() {
+        // Load subscription ID from local storage (Phase 2 activation)
+        this.subscriptionId = await this.getSubscriptionId();
+    }
+    async getSubscriptionId() {
+        const license = (0, license_1.loadLicense)();
+        return license?.subscriptionId;
+    }
+    async uploadRun(data) {
+        if (!this.subscriptionId) {
+            return { success: false, error: 'Not activated. Run `tuneprompt activate` first.' };
+        }
+        try {
+            const response = await axios_1.default.post(`${this.backendUrl}/api/cloud/ingest-run`, data, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-subscription-id': this.subscriptionId,
+                },
+                timeout: 30000,
+            });
+            return response.data;
+        }
+        catch (error) {
+            console.error('Failed to upload run:', error.message);
+            return {
+                success: false,
+                error: error.response?.data?.error || error.message
+            };
+        }
+    }
+    async createProject(name, description) {
+        if (!this.subscriptionId) {
+            throw new Error('Not activated');
+        }
+        try {
+            const response = await axios_1.default.post(`${this.backendUrl}/api/cloud/projects`, { name, description }, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-subscription-id': this.subscriptionId,
+                },
+            });
+            return response.data.project;
+        }
+        catch (error) {
+            throw new Error(error.response?.data?.error || error.message);
+        }
+    }
+    async getProjects() {
+        if (!this.subscriptionId) {
+            return [];
+        }
+        try {
+            const response = await axios_1.default.get(`${this.backendUrl}/api/cloud/projects`, {
+                headers: {
+                    'x-subscription-id': this.subscriptionId,
+                },
+            });
+            return response.data.projects || [];
+        }
+        catch (error) {
+            return [];
+        }
+    }
+    async isAuthenticated() {
+        return !!this.subscriptionId;
+    }
+}
+exports.CloudService = CloudService;

package/dist/storage/database.d.ts

@@ -0,0 +1,10 @@
+import { TestRun } from '../types';
+export declare class TestDatabase {
+    private db;
+    constructor(dbPath?: string);
+    private migrate;
+    saveRun(run: TestRun): void;
+    getRecentRuns(limit?: number): TestRun[];
+    private getRunResults;
+    close(): void;
+}