qualifire 1.2.2 → 1.3.0

package/lib/frameworks/canonical.d.ts

@@ -0,0 +1,4 @@
+import { EvaluationProxyAPIRequest } from '../types';
+export interface CanonicalEvaluationStrategy<RequestType, ResponseType> {
+    convertToQualifireEvaluationRequest(request: RequestType, response: ResponseType): Promise<EvaluationProxyAPIRequest>;
+}

package/lib/frameworks/canonical.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/frameworks/claude/claude-converter.d.ts

@@ -0,0 +1,15 @@
+import { Message, type MessageCreateParams, type MessageStreamParams } from '@anthropic-ai/sdk/resources';
+import { RawMessageStreamEvent } from '@anthropic-ai/sdk/resources/messages';
+import { EvaluationProxyAPIRequest } from '../../types';
+import { CanonicalEvaluationStrategy } from '../canonical';
+type AnthropicCreateAPIResponsesType = Message | RawMessageStreamEvent;
+type AnthropicAPIRequestsType = MessageCreateParams;
+type AnthropicAPIResponsesType = AnthropicCreateAPIResponsesType | MessageStreamParams;
+export declare class ClaudeCanonicalEvaluationStrategy implements CanonicalEvaluationStrategy<AnthropicAPIRequestsType, AnthropicAPIResponsesType> {
+    convertToQualifireEvaluationRequest(request: AnthropicAPIRequestsType, response: AnthropicAPIResponsesType): Promise<EvaluationProxyAPIRequest>;
+    convertRequest(request: any): EvaluationProxyAPIRequest;
+    private handleStreaming;
+    private handleNonStreamingResponse;
+    private convertClaudeMessagesToLLMMessages;
+}
+export {};

package/lib/frameworks/claude/claude-converter.js

@@ -0,0 +1,229 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClaudeCanonicalEvaluationStrategy = void 0;
+class ClaudeCanonicalEvaluationStrategy {
+    async convertToQualifireEvaluationRequest(request, response) {
+        const { messages: requestMessages, available_tools: requestAvailableTools, } = this.convertRequest(request);
+        const messages = requestMessages || [];
+        const availableTools = requestAvailableTools || [];
+        // Avoid undefined response
+        if (!response) {
+            return {
+                messages,
+                available_tools: availableTools,
+            };
+        }
+        // Check if response is streaming or non-streaming
+        if (Array.isArray(response)) {
+            const streamingResultMessages = await this.handleStreaming(response);
+            messages.push(...streamingResultMessages);
+        }
+        else {
+            const nonStreamingResultMessages = await this.handleNonStreamingResponse(response);
+            messages.push(...nonStreamingResultMessages);
+        }
+        return {
+            messages,
+            available_tools: availableTools,
+        };
+    }
+    convertRequest(request) {
+        const messages = [];
+        const availableTools = [];
+        // Handle Claude system message first (if present)
+        if (request?.system) {
+            messages.push({
+                role: 'system',
+                content: request.system,
+            });
+        }
+        // Handle Claude request messages
+        if (request?.messages) {
+            messages.push(...this.convertClaudeMessagesToLLMMessages(request.messages));
+        }
+        // Handle tools
+        if (request?.tools) {
+            for (const tool of request.tools) {
+                availableTools.push({
+                    name: tool.name,
+                    description: tool.description,
+                    parameters: tool.input_schema?.properties || {},
+                });
+            }
+        }
+        return {
+            messages,
+            available_tools: availableTools,
+        };
+    }
+    async handleStreaming(response) {
+        const messages = [];
+        let role;
+        let accumulatedContent = [];
+        let accumulatedToolName;
+        let accumulatedToolId;
+        let accumulatedToolInput = [];
+        for (const responseEvent of response) {
+            switch (responseEvent.type) {
+                case 'message_start':
+                    const rawMessageStartEvent = responseEvent;
+                    role = rawMessageStartEvent.message.role;
+                    accumulatedContent = [];
+                    accumulatedToolName = undefined;
+                    accumulatedToolId = undefined;
+                    accumulatedToolInput = [];
+                    break;
+                case 'content_block_start':
+                    const rawContentBlockStartEvent = responseEvent;
+                    switch (rawContentBlockStartEvent.content_block.type) {
+                        case 'text':
+                            const textBlock = rawContentBlockStartEvent.content_block;
+                            accumulatedContent.push(textBlock.text);
+                            break;
+                        case 'tool_use':
+                            const toolUseBlock = rawContentBlockStartEvent.content_block;
+                            accumulatedToolId = toolUseBlock.id;
+                            accumulatedToolName = toolUseBlock.name;
+                            accumulatedToolInput = [];
+                            break;
+                        case 'thinking':
+                            const thinkingBlock = rawContentBlockStartEvent.content_block;
+                            accumulatedContent.push(thinkingBlock.thinking);
+                            break;
+                        default:
+                            console.debug(`Invalid content block type: ${responseEvent}`);
+                    }
+                    break;
+                case 'content_block_delta':
+                    const rawContentBlockDeltaEvent = responseEvent;
+                    switch (rawContentBlockDeltaEvent.delta.type) {
+                        case 'text_delta':
+                            const textDelta = rawContentBlockDeltaEvent.delta;
+                            accumulatedContent.push(textDelta.text);
+                            break;
+                        case 'input_json_delta':
+                            const inputJsonDelta = rawContentBlockDeltaEvent.delta;
+                            accumulatedToolInput.push(inputJsonDelta.partial_json);
+                            break;
+                        default:
+                            console.debug(`Invalid delta type: ${rawContentBlockDeltaEvent}`);
+                    }
+                    break;
+                case 'message_stop':
+                    let finalContent;
+                    if (accumulatedContent.length > 0) {
+                        finalContent = accumulatedContent.join('').trim();
+                    }
+                    let finalTool;
+                    if (accumulatedToolName) {
+                        finalTool = {
+                            id: accumulatedToolId,
+                            name: accumulatedToolName,
+                            arguments: JSON.parse(accumulatedToolInput.join('')),
+                        };
+                    }
+                    ;
+                    if (!role) {
+                        console.debug('role was not set');
+                        continue;
+                    }
+                    messages.push({
+                        role: role == 'model' ? 'assistant' : role,
+                        content: finalContent ?? undefined,
+                        tool_calls: finalTool ? [finalTool] : undefined,
+                    });
+                    role = undefined;
+                    accumulatedContent = [];
+                    accumulatedToolName = undefined;
+                    accumulatedToolId = undefined;
+                    accumulatedToolInput = [];
+                    break;
+                case 'content_block_stop':
+                case 'message_delta':
+                    break;
+                default:
+                    console.debug(`Invalid event: ${responseEvent}`);
+            }
+        }
+        return messages;
+    }
+    async handleNonStreamingResponse(response) {
+        const messages = [];
+        if (response.role !== 'assistant') {
+            throw new Error(`Response role must be 'assistant'. Make sure to use response 
+        from anthropic.messages.create() when not using streaming.`);
+        }
+        messages.push(...this.convertClaudeMessagesToLLMMessages([response]));
+        return messages;
+    }
+    // Claude-specific function to convert Response API messages to LLM messages
+    convertClaudeMessagesToLLMMessages(messages) {
+        const extractedMessages = [];
+        for (const message of messages) {
+            if (typeof message.content === 'string') {
+                const llmMessage = {
+                    role: message.role,
+                    content: message.content,
+                };
+                extractedMessages.push(llmMessage);
+                continue;
+            }
+            const aggregatedContent = [];
+            const aggregatedToolCalls = [];
+            let role = message.role;
+            if (!message.content) {
+                continue;
+            }
+            for (const part of message.content) {
+                switch (part.type) {
+                    case 'tool_use':
+                        const toolUseBlock = part;
+                        aggregatedToolCalls.push({
+                            name: toolUseBlock.name,
+                            arguments: toolUseBlock.input,
+                            id: toolUseBlock.id,
+                        });
+                        break;
+                    case 'tool_result':
+                        role = 'tool'; // Claude expects 'user' role for tool results. But Qualifire treats tool as results as it is sent from 'tool'
+                        const toolResultBlock = part;
+                        if (typeof toolResultBlock.content === 'string') {
+                            aggregatedContent.push(toolResultBlock.content);
+                        }
+                        else {
+                            toolResultBlock.content.filter(part => part.type === 'text').forEach(part => {
+                                const textPart = part;
+                                aggregatedContent.push(textPart.text);
+                            });
+                        }
+                        break;
+                    case 'text':
+                        const textBlock = part;
+                        aggregatedContent.push(textBlock.text);
+                        break;
+                    default:
+                        console.debug('Invalid Claude output: message - ' +
+                            JSON.stringify(message) +
+                            ' part - ' +
+                            JSON.stringify(part));
+                }
+            }
+            // If we accumulated aggregatedContent or aggregatedToolCalls, add the message
+            if (aggregatedContent.length > 0 || aggregatedToolCalls.length > 0) {
+                const accumulatedMessage = {
+                    role,
+                };
+                if (aggregatedContent.length > 0) {
+                    accumulatedMessage.content = aggregatedContent.join('');
+                }
+                // Only add aggregatedToolCalls property for assistant messages
+                if (aggregatedToolCalls.length > 0) {
+                    accumulatedMessage.tool_calls = aggregatedToolCalls;
+                }
+                extractedMessages.push(accumulatedMessage);
+            }
+        }
+        return extractedMessages;
+    }
+}
+exports.ClaudeCanonicalEvaluationStrategy = ClaudeCanonicalEvaluationStrategy;

package/lib/frameworks/gemini/gemini-converter.d.ts

@@ -0,0 +1,11 @@
+import { EvaluationProxyAPIRequest } from '../../types';
+import { CanonicalEvaluationStrategy } from '../canonical';
+type GeminiAICanonicalEvaluationStrategyResponse = any;
+type GeminiAICanonicalEvaluationStrategyRequest = any;
+export declare class GeminiAICanonicalEvaluationStrategy implements CanonicalEvaluationStrategy<GeminiAICanonicalEvaluationStrategyRequest, GeminiAICanonicalEvaluationStrategyResponse> {
+    convertToQualifireEvaluationRequest(request: GeminiAICanonicalEvaluationStrategyRequest, response: GeminiAICanonicalEvaluationStrategyResponse): Promise<EvaluationProxyAPIRequest>;
+    convertRequest(request: GeminiAICanonicalEvaluationStrategyRequest): Promise<EvaluationProxyAPIRequest>;
+    private handleNonStreamingResponse;
+    private handleStreaming;
+}
+export {};

package/lib/frameworks/gemini/gemini-converter.js

@@ -0,0 +1,241 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GeminiAICanonicalEvaluationStrategy = void 0;
+class GeminiAICanonicalEvaluationStrategy {
+    async convertToQualifireEvaluationRequest(request, response) {
+        const { messages: requestMessages, available_tools: requestAvailableTools, } = await this.convertRequest(request);
+        const messages = requestMessages || [];
+        const available_tools = requestAvailableTools || [];
+        // Avoid undefined response
+        if (!response) {
+            return {
+                messages,
+                available_tools,
+            };
+        }
+        if (Array.isArray(response)) {
+            const streamingResultMessages = await this.handleStreaming(response);
+            messages.push(...streamingResultMessages);
+        }
+        else {
+            const nonStreamingResultMessages = await this.handleNonStreamingResponse(response);
+            messages.push(...nonStreamingResultMessages);
+        }
+        return {
+            messages,
+            available_tools,
+        };
+    }
+    async convertRequest(request) {
+        const messages = [];
+        const available_tools = [];
+        // Handle available tools
+        if (request?.config?.tools && request.config.tools.length > 0) {
+            for (const tool of request.config.tools) {
+                if (tool?.functionDeclarations?.length > 0) {
+                    for (const functionDeclaration of tool.functionDeclarations) {
+                        available_tools.push({
+                            name: functionDeclaration.name,
+                            description: functionDeclaration.description,
+                            parameters: functionDeclaration.parameters,
+                        });
+                    }
+                }
+            }
+        }
+        const contents = [];
+        if (request?.config?.systemInstruction) {
+            /*
+              Gemini request.contents is an object called contentListUnions which can be
+              list of parts or content objects:
+              https://github.com/googleapis/js-genai/blob/b5d77e1bfea5c6b4903bc7ade986e91d6e146835/src/types.ts#L1937
+            */
+            const convertedSystemContent = convertContentListUnionsToContentList(request.config.systemInstruction, 'system');
+            if (convertedSystemContent.length != 1) {
+                throw new Error(`Invalid system instruction given. Gemini Does not support multiple system instructions: ${JSON.stringify(request.contents)}`);
+            }
+            contents.push(convertedSystemContent[0]);
+        }
+        // Handle request contents
+        if (request?.contents) {
+            const convertedContents = convertContentListUnionsToContentList(request.contents, 'user');
+            contents.push(...convertedContents);
+        }
+        for (const content of contents) {
+            const message = convertContentToLLMMessage(content);
+            if (message) {
+                messages.push(message);
+            }
+        }
+        return {
+            messages,
+            available_tools,
+        };
+    }
+    async handleNonStreamingResponse(response) {
+        const messages = [];
+        // Handle response candidates
+        if (response?.candidates && response.candidates.length > 0) {
+            if (response.candidates.length > 1) {
+                console.debug('Multiple candidates found in the response. Only first candidate is supported.');
+            }
+            const firstCandidate = response.candidates[0];
+            if (firstCandidate.content?.role &&
+                firstCandidate.content?.parts &&
+                firstCandidate.content.parts.length > 0) {
+                const message = convertContentToLLMMessage(firstCandidate.content);
+                if (message) {
+                    messages.push(message);
+                }
+            }
+        }
+        return messages;
+    }
+    async handleStreaming(response) {
+        const messages = [];
+        const accumulatedContentParts = [];
+        let currentRole = 'assistant';
+        const toolCalls = [];
+        for (const chunk of response) {
+            if (chunk?.candidates && chunk.candidates.length > 0) {
+                if (chunk.candidates.length > 1) {
+                    console.debug('Multiple candidates found in the response. Only first candidate is supported.');
+                }
+                const firstCandidate = chunk.candidates[0]; // we currently only support one response message
+                if (firstCandidate.content?.role) {
+                    currentRole = firstCandidate.content.role;
+                    if (currentRole === 'model') {
+                        currentRole = 'assistant'; // Answers returned from the model are always from the assistant role
+                    }
+                }
+                if (!firstCandidate.content) {
+                    console.debug('Content is missing required fields. Skipping message.');
+                    continue;
+                }
+                const message = convertContentToLLMMessage(firstCandidate.content);
+                if (message?.content) {
+                    accumulatedContentParts.push(message.content);
+                }
+                if (message?.tool_calls) {
+                    toolCalls.push(...message.tool_calls);
+                }
+            }
+        }
+        const accumulatedContent = accumulatedContentParts.length > 0
+            ? accumulatedContentParts.join('').trim()
+            : undefined;
+        if (accumulatedContent || toolCalls.length > 0) {
+            messages.push({
+                role: currentRole,
+                content: accumulatedContent,
+                tool_calls: toolCalls.length > 0 ? toolCalls : undefined,
+            });
+        }
+        return messages;
+    }
+}
+exports.GeminiAICanonicalEvaluationStrategy = GeminiAICanonicalEvaluationStrategy;
+// Helper function to convert a Gemini content object (with multiple parts) to LLMMessage
+function convertContentToLLMMessage(content) {
+    if (!content.parts || content.parts.length === 0) {
+        return null;
+    }
+    // In Gemini role is optional, but by default the api is changing it to 'user' when no role is provided
+    let role = content.role || 'user';
+    const textContent = [];
+    const tool_calls = [];
+    // Process all parts and aggregate them
+    for (const part of content.parts) {
+        if (typeof part === 'string') {
+            textContent.push(part);
+        }
+        else if (part.text) {
+            textContent.push(part.text);
+        }
+        else if (part.functionCall) {
+            role = 'assistant'; // Function calls are always from assistant
+            tool_calls.push({
+                name: part.functionCall.name,
+                arguments: part.functionCall.args,
+            });
+        }
+        else if (part.functionResponse?.response?.result) {
+            role = 'tool';
+            textContent.push(JSON.stringify(part.functionResponse.response?.result));
+        }
+    }
+    // Determine final role
+    const finalRole = role === 'model' ? 'assistant' : role;
+    // Aggregate content based on message type
+    let finalContent;
+    if (textContent.length > 0) {
+        finalContent = textContent.join(' ');
+    }
+    return {
+        role: finalRole,
+        content: finalContent,
+        tool_calls: tool_calls.length > 0 ? tool_calls : undefined,
+    };
+}
+function isPartOrString(obj) {
+    if (typeof obj === 'string') {
+        return true;
+    }
+    if (typeof obj === 'object' && obj !== null) {
+        return ('fileData' in obj ||
+            'text' in obj ||
+            'functionCall' in obj ||
+            'functionResponse' in obj ||
+            'inlineData' in obj ||
+            'videoMetadata' in obj ||
+            'codeExecutionResult' in obj ||
+            'executableCode' in obj);
+    }
+    return false;
+}
+function isContent(obj) {
+    if (typeof obj === 'object' && obj !== null) {
+        return 'parts' in obj || 'role' in obj;
+    }
+    return false;
+}
+function convertContentListUnionsToContentList(input, defaultRole) {
+    /*
+        Gemini request.contents is an object called contentListUnions which can be
+        list of parts or content objects:
+        https://github.com/googleapis/js-genai/blob/b5d77e1bfea5c6b4903bc7ade986e91d6e146835/src/types.ts#L1937
+      */
+    let inputs;
+    if (!Array.isArray(input)) {
+        inputs = [input];
+    }
+    else {
+        inputs = input;
+    }
+    if (inputs.length === 0) {
+        return [];
+    }
+    let convertedContents = [];
+    if (inputs.every(isContent)) {
+        convertedContents = inputs;
+    }
+    else if (inputs.every(isPartOrString)) {
+        const partInputs = [];
+        for (const partOrString of inputs) {
+            if (typeof partOrString === 'string') {
+                partInputs.push({ text: partOrString });
+            }
+            else {
+                partInputs.push(partOrString);
+            }
+        }
+        convertedContents.push({
+            role: defaultRole,
+            parts: partInputs,
+        });
+    }
+    else {
+        throw new Error(`Invalid contents given. Gemini Does not support mixing parts and contents: ${JSON.stringify(inputs)}`);
+    }
+    return convertedContents;
+}

package/lib/frameworks/openai/openai-converter.d.ts

@@ -0,0 +1,28 @@
+import { EvaluationProxyAPIRequest, LLMToolDefinition } from '../../types';
+import { CanonicalEvaluationStrategy } from '../canonical';
+import { ChatCompletion, ChatCompletionChunk } from 'openai/resources/chat/completions';
+import { ChatCompletionCreateParamsBase, ChatCompletionCreateParamsNonStreaming, ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions/completions';
+import { Response, ResponseCreateParamsBase, ResponseCreateParamsNonStreaming, ResponseCreateParamsStreaming, ResponseStreamEvent } from 'openai/resources/responses/responses';
+type OpenAIResponseCreateRequest = ResponseCreateParamsNonStreaming | ResponseCreateParamsStreaming | ResponseCreateParamsBase;
+type OpenAIResponseCreateRequestResponse = Response | Array<ResponseStreamEvent>;
+type OpenAIChatCompletionsCreateRequest = ChatCompletionCreateParamsNonStreaming | ChatCompletionCreateParamsStreaming | ChatCompletionCreateParamsBase;
+type OpenAIChatCompletionsCreateResponse = ChatCompletion | Array<ChatCompletionChunk>;
+type OpenAICanonicalEvaluationStrategyRequest = OpenAIResponseCreateRequest | OpenAIChatCompletionsCreateRequest;
+type OpenAICanonicalEvaluationStrategyResponse = OpenAIResponseCreateRequestResponse | OpenAIChatCompletionsCreateResponse;
+export declare class OpenAICanonicalEvaluationStrategy implements CanonicalEvaluationStrategy<OpenAICanonicalEvaluationStrategyRequest, OpenAICanonicalEvaluationStrategyResponse> {
+    convertToQualifireEvaluationRequest(request: OpenAICanonicalEvaluationStrategyRequest, response: OpenAICanonicalEvaluationStrategyResponse): Promise<EvaluationProxyAPIRequest>;
+    convertRequest(request: any): Promise<EvaluationProxyAPIRequest>;
+    convertRequestForChatCompletions(request: OpenAIChatCompletionsCreateRequest): Promise<EvaluationProxyAPIRequest>;
+    private convertRequestMessagesForChatCompletions;
+    convertRequestForResponsesAPI(request: OpenAIResponseCreateRequest): Promise<EvaluationProxyAPIRequest>;
+    private handleStreaming;
+    private handleChatCompletionsStreaming;
+    private handleResponsesApiStreaming;
+    private processChatCompletionsChunk;
+    private processResponsesApiChunkMessage;
+    private processResponsesApiChunkToolCall;
+    private handleChatCompletionsNonStreaming;
+    private handleResponsesApiNonStreaming;
+}
+export declare function convertToolsToLLMDefinitions(tools: unknown[]): LLMToolDefinition[];
+export {};