promptfoo 0.119.13 → 0.119.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +28 -26
- package/dist/src/app/assets/index-eJ2lMe94.js +51 -0
- package/dist/src/app/assets/{source-map-support-Bnh0UQ2S.js → source-map-support-1v4oeb7P.js} +1 -1
- package/dist/src/app/assets/sync-CtLQRuC1.js +1 -0
- package/dist/src/app/assets/{vendor-charts-T60Uk0Z3.js → vendor-charts-DnVv66VV.js} +1 -1
- package/dist/src/app/assets/{vendor-markdown-DLig-KJh.js → vendor-markdown-DCpQIyMA.js} +1 -1
- package/dist/src/app/assets/{vendor-mui-core-5BLaiG3c.js → vendor-mui-core-Boqnpf9f.js} +1 -1
- package/dist/src/app/assets/{vendor-mui-icons-fn39Fu2e.js → vendor-mui-icons-B8MqoVbj.js} +1 -1
- package/dist/src/app/assets/vendor-mui-x-CGSS6QHF.js +45 -0
- package/dist/src/app/assets/{vendor-utils-DYBMEuwX.js → vendor-utils-DdfHIEy8.js} +1 -1
- package/dist/src/app/index.html +7 -7
- package/dist/src/assertions/guardrails.d.ts +1 -1
- package/dist/src/assertions/guardrails.js +18 -9
- package/dist/src/assertions/index.d.ts +1 -1
- package/dist/src/assertions/index.js +9 -3
- package/dist/src/assertions/searchRubric.d.ts +3 -0
- package/dist/src/assertions/searchRubric.js +18 -0
- package/dist/src/commands/eval.js +1 -1
- package/dist/src/commands/modelScan.d.ts +7 -1
- package/dist/src/commands/modelScan.js +121 -59
- package/dist/src/database/index.d.ts +6 -0
- package/dist/src/database/index.js +11 -0
- package/dist/src/database/tables.d.ts +46 -24
- package/dist/src/envars.d.ts +17 -0
- package/dist/src/generated/constants.js +1 -1
- package/dist/src/logger.d.ts +5 -0
- package/dist/src/logger.js +28 -0
- package/dist/src/main.js +17 -6
- package/dist/src/matchers.d.ts +1 -0
- package/dist/src/matchers.js +80 -0
- package/dist/src/models/eval.d.ts +2 -1
- package/dist/src/models/eval.js +44 -2
- package/dist/src/prompts/grading.d.ts +1 -0
- package/dist/src/prompts/grading.js +26 -1
- package/dist/src/prompts/index.d.ts +1 -0
- package/dist/src/prompts/index.js +4 -1
- package/dist/src/providers/adaline.gateway.js +2 -2
- package/dist/src/providers/anthropic/defaults.d.ts +1 -1
- package/dist/src/providers/anthropic/defaults.js +15 -0
- package/dist/src/providers/azure/chat.d.ts +3 -1
- package/dist/src/providers/azure/chat.js +16 -3
- package/dist/src/providers/azure/defaults.js +660 -141
- package/dist/src/providers/azure/responses.d.ts +5 -0
- package/dist/src/providers/azure/responses.js +33 -4
- package/dist/src/providers/azure/types.d.ts +4 -0
- package/dist/src/providers/bedrock/agents.d.ts +1 -1
- package/dist/src/providers/bedrock/agents.js +2 -2
- package/dist/src/providers/bedrock/base.d.ts +40 -0
- package/dist/src/providers/bedrock/base.js +171 -0
- package/dist/src/providers/bedrock/converse.d.ts +146 -0
- package/dist/src/providers/bedrock/converse.js +1044 -0
- package/dist/src/providers/bedrock/index.d.ts +1 -34
- package/dist/src/providers/bedrock/index.js +4 -159
- package/dist/src/providers/bedrock/knowledgeBase.d.ts +1 -1
- package/dist/src/providers/bedrock/knowledgeBase.js +2 -2
- package/dist/src/providers/bedrock/nova-sonic.d.ts +2 -1
- package/dist/src/providers/bedrock/nova-sonic.js +2 -2
- package/dist/src/providers/claude-agent-sdk.d.ts +58 -1
- package/dist/src/providers/claude-agent-sdk.js +22 -1
- package/dist/src/providers/defaults.js +4 -0
- package/dist/src/providers/github/defaults.js +6 -6
- package/dist/src/providers/google/types.d.ts +25 -0
- package/dist/src/providers/google/util.d.ts +2 -0
- package/dist/src/providers/google/vertex.js +78 -22
- package/dist/src/providers/{groq.d.ts → groq/chat.d.ts} +26 -20
- package/dist/src/providers/groq/chat.js +79 -0
- package/dist/src/providers/groq/index.d.ts +5 -0
- package/dist/src/providers/groq/index.js +24 -0
- package/dist/src/providers/groq/responses.d.ts +106 -0
- package/dist/src/providers/groq/responses.js +64 -0
- package/dist/src/providers/groq/types.d.ts +44 -0
- package/dist/src/providers/groq/types.js +3 -0
- package/dist/src/providers/groq/util.d.ts +15 -0
- package/dist/src/providers/groq/util.js +28 -0
- package/dist/src/providers/mcp/client.d.ts +8 -0
- package/dist/src/providers/mcp/client.js +60 -10
- package/dist/src/providers/mcp/types.d.ts +21 -0
- package/dist/src/providers/openai/chatkit-pool.d.ts +114 -0
- package/dist/src/providers/openai/chatkit-pool.js +548 -0
- package/dist/src/providers/openai/chatkit-types.d.ts +73 -0
- package/dist/src/providers/openai/chatkit-types.js +3 -0
- package/dist/src/providers/openai/chatkit.d.ts +76 -0
- package/dist/src/providers/openai/chatkit.js +879 -0
- package/dist/src/providers/openai/codex-sdk.d.ts +109 -0
- package/dist/src/providers/openai/codex-sdk.js +346 -0
- package/dist/src/providers/openai/defaults.d.ts +2 -0
- package/dist/src/providers/openai/defaults.js +10 -4
- package/dist/src/providers/registry.js +48 -9
- package/dist/src/providers/responses/types.d.ts +1 -1
- package/dist/src/providers/sagemaker.d.ts +2 -2
- package/dist/src/providers/webSearchUtils.d.ts +17 -0
- package/dist/src/providers/webSearchUtils.js +169 -0
- package/dist/src/providers/xai/chat.d.ts +61 -0
- package/dist/src/providers/xai/chat.js +68 -3
- package/dist/src/providers/xai/responses.d.ts +189 -0
- package/dist/src/providers/xai/responses.js +268 -0
- package/dist/src/redteam/constants/plugins.d.ts +1 -1
- package/dist/src/redteam/constants/plugins.js +1 -1
- package/dist/src/redteam/constants/strategies.d.ts +1 -1
- package/dist/src/redteam/constants/strategies.js +1 -0
- package/dist/src/redteam/plugins/vlguard.d.ts +53 -4
- package/dist/src/redteam/plugins/vlguard.js +362 -46
- package/dist/src/redteam/providers/constants.d.ts +2 -2
- package/dist/src/redteam/providers/constants.js +2 -2
- package/dist/src/redteam/providers/crescendo/index.d.ts +1 -1
- package/dist/src/redteam/providers/crescendo/index.js +5 -3
- package/dist/src/redteam/providers/hydra/index.js +1 -1
- package/dist/src/server/routes/modelAudit.js +4 -4
- package/dist/src/share.js +4 -2
- package/dist/src/telemetry.js +44 -8
- package/dist/src/types/env.d.ts +3 -0
- package/dist/src/types/env.js +1 -0
- package/dist/src/types/index.d.ts +896 -615
- package/dist/src/types/index.js +1 -0
- package/dist/src/types/providers.d.ts +1 -0
- package/dist/src/types/tracing.d.ts +3 -0
- package/dist/src/util/database.d.ts +6 -4
- package/dist/src/util/file.js +6 -4
- package/dist/src/util/modelAuditCliParser.d.ts +4 -4
- package/dist/src/util/xlsx.js +52 -26
- package/dist/src/validators/providers.d.ts +142 -122
- package/dist/src/validators/providers.js +4 -6
- package/dist/src/validators/redteam.d.ts +36 -28
- package/dist/src/validators/redteam.js +9 -3
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +28 -26
- package/dist/drizzle/CLAUDE.md +0 -65
- package/dist/src/app/assets/index-DifT6VGT.js +0 -51
- package/dist/src/app/assets/sync-Oo-W_Rbj.js +0 -1
- package/dist/src/app/assets/vendor-mui-x-C2xF-yiO.js +0 -45
- package/dist/src/providers/groq.js +0 -48
|
@@ -1,19 +1,26 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import type {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
1
|
+
import type { CallApiContextParams, CallApiOptionsParams } from '../../types/index';
|
|
2
|
+
import { OpenAiChatCompletionProvider } from '../openai/chat';
|
|
3
|
+
import type { GroqProviderOptions } from './types';
|
|
4
|
+
/**
|
|
5
|
+
* Groq Chat Completions API Provider
|
|
6
|
+
*
|
|
7
|
+
* Extends OpenAI Chat Completions provider with Groq-specific configuration.
|
|
8
|
+
* Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* groq:llama-3.3-70b-versatile
|
|
12
|
+
* groq:openai/gpt-oss-120b
|
|
13
|
+
* groq:qwen/qwen3-32b
|
|
14
|
+
*/
|
|
12
15
|
export declare class GroqProvider extends OpenAiChatCompletionProvider {
|
|
13
16
|
protected get apiKey(): string | undefined;
|
|
14
17
|
protected isReasoningModel(): boolean;
|
|
15
18
|
protected supportsTemperature(): boolean;
|
|
16
19
|
constructor(modelName: string, providerOptions: GroqProviderOptions);
|
|
20
|
+
getOpenAiBody(prompt: string, context?: CallApiContextParams, callApiOptions?: CallApiOptionsParams): Promise<{
|
|
21
|
+
body: any;
|
|
22
|
+
config: any;
|
|
23
|
+
}>;
|
|
17
24
|
id(): string;
|
|
18
25
|
toString(): string;
|
|
19
26
|
toJSON(): {
|
|
@@ -38,11 +45,11 @@ export declare class GroqProvider extends OpenAiChatCompletionProvider {
|
|
|
38
45
|
frequency_penalty?: number;
|
|
39
46
|
presence_penalty?: number;
|
|
40
47
|
best_of?: number;
|
|
41
|
-
functions?: import("
|
|
48
|
+
functions?: import("../openai/util").OpenAiFunction[];
|
|
42
49
|
function_call?: "none" | "auto" | {
|
|
43
50
|
name: string;
|
|
44
51
|
};
|
|
45
|
-
tools?: (import("
|
|
52
|
+
tools?: (import("../openai/util").OpenAiTool | import("../openai/types").OpenAiMCPTool | import("../openai/types").OpenAiWebSearchTool | import("../openai/types").OpenAiCodeInterpreterTool)[];
|
|
46
53
|
tool_choice?: "none" | "auto" | "required" | {
|
|
47
54
|
type: "function";
|
|
48
55
|
function?: {
|
|
@@ -69,8 +76,8 @@ export declare class GroqProvider extends OpenAiChatCompletionProvider {
|
|
|
69
76
|
stop?: string[];
|
|
70
77
|
seed?: number;
|
|
71
78
|
passthrough?: object;
|
|
72
|
-
reasoning_effort?: import("
|
|
73
|
-
reasoning?: import("
|
|
79
|
+
reasoning_effort?: import("../openai/types").ReasoningEffort;
|
|
80
|
+
reasoning?: import("../openai/types").Reasoning | import("../openai/types").GPT5Reasoning;
|
|
74
81
|
service_tier?: ("auto" | "default" | "premium") | null;
|
|
75
82
|
modalities?: string[];
|
|
76
83
|
audio?: {
|
|
@@ -91,11 +98,10 @@ export declare class GroqProvider extends OpenAiChatCompletionProvider {
|
|
|
91
98
|
user?: string;
|
|
92
99
|
background?: boolean;
|
|
93
100
|
webhook_url?: string;
|
|
94
|
-
functionToolCallbacks?: Record<import("openai").default.FunctionDefinition["name"], import("
|
|
95
|
-
mcp?: import("
|
|
96
|
-
verbosity?: import("
|
|
101
|
+
functionToolCallbacks?: Record<import("openai").default.FunctionDefinition["name"], import("../openai/types").AssistantFunctionCallback | string>;
|
|
102
|
+
mcp?: import("../mcp/types").MCPConfig;
|
|
103
|
+
verbosity?: import("../openai/types").GPT5Verbosity;
|
|
97
104
|
};
|
|
98
105
|
};
|
|
99
106
|
}
|
|
100
|
-
|
|
101
|
-
//# sourceMappingURL=groq.d.ts.map
|
|
107
|
+
//# sourceMappingURL=chat.d.ts.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.GroqProvider = void 0;
|
|
4
|
+
const chat_1 = require("../openai/chat");
|
|
5
|
+
const util_1 = require("./util");
|
|
6
|
+
const GROQ_API_BASE_URL = 'https://api.groq.com/openai/v1';
|
|
7
|
+
/**
|
|
8
|
+
* Groq Chat Completions API Provider
|
|
9
|
+
*
|
|
10
|
+
* Extends OpenAI Chat Completions provider with Groq-specific configuration.
|
|
11
|
+
* Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
|
|
12
|
+
*
|
|
13
|
+
* Usage:
|
|
14
|
+
* groq:llama-3.3-70b-versatile
|
|
15
|
+
* groq:openai/gpt-oss-120b
|
|
16
|
+
* groq:qwen/qwen3-32b
|
|
17
|
+
*/
|
|
18
|
+
class GroqProvider extends chat_1.OpenAiChatCompletionProvider {
|
|
19
|
+
get apiKey() {
|
|
20
|
+
return this.config?.apiKey;
|
|
21
|
+
}
|
|
22
|
+
isReasoningModel() {
|
|
23
|
+
return (0, util_1.isGroqReasoningModel)(this.modelName) || super.isReasoningModel();
|
|
24
|
+
}
|
|
25
|
+
supportsTemperature() {
|
|
26
|
+
// Groq's reasoning models support temperature, unlike OpenAI's o1 models
|
|
27
|
+
if ((0, util_1.groqSupportsTemperature)(this.modelName)) {
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
return super.supportsTemperature();
|
|
31
|
+
}
|
|
32
|
+
constructor(modelName, providerOptions) {
|
|
33
|
+
super(modelName, {
|
|
34
|
+
...providerOptions,
|
|
35
|
+
config: {
|
|
36
|
+
...providerOptions.config,
|
|
37
|
+
apiKeyEnvar: 'GROQ_API_KEY',
|
|
38
|
+
apiBaseUrl: GROQ_API_BASE_URL,
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
async getOpenAiBody(prompt, context, callApiOptions) {
|
|
43
|
+
const { body, config } = await super.getOpenAiBody(prompt, context, callApiOptions);
|
|
44
|
+
const groqConfig = this.config;
|
|
45
|
+
// Add Groq-specific reasoning parameters
|
|
46
|
+
if (groqConfig.reasoning_format !== undefined) {
|
|
47
|
+
body.reasoning_format = groqConfig.reasoning_format;
|
|
48
|
+
}
|
|
49
|
+
if (groqConfig.include_reasoning !== undefined) {
|
|
50
|
+
body.include_reasoning = groqConfig.include_reasoning;
|
|
51
|
+
}
|
|
52
|
+
// Add Compound model parameters
|
|
53
|
+
if (groqConfig.compound_custom) {
|
|
54
|
+
body.compound_custom = groqConfig.compound_custom;
|
|
55
|
+
}
|
|
56
|
+
if (groqConfig.search_settings) {
|
|
57
|
+
body.search_settings = groqConfig.search_settings;
|
|
58
|
+
}
|
|
59
|
+
return { body, config };
|
|
60
|
+
}
|
|
61
|
+
id() {
|
|
62
|
+
return `groq:${this.modelName}`;
|
|
63
|
+
}
|
|
64
|
+
toString() {
|
|
65
|
+
return `[Groq Provider ${this.modelName}]`;
|
|
66
|
+
}
|
|
67
|
+
toJSON() {
|
|
68
|
+
return {
|
|
69
|
+
provider: 'groq',
|
|
70
|
+
model: this.modelName,
|
|
71
|
+
config: {
|
|
72
|
+
...this.config,
|
|
73
|
+
...(this.apiKey && { apiKey: undefined }),
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
exports.GroqProvider = GroqProvider;
|
|
79
|
+
//# sourceMappingURL=chat.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.GroqResponsesProvider = exports.GroqProvider = void 0;
|
|
18
|
+
var chat_1 = require("./chat");
|
|
19
|
+
Object.defineProperty(exports, "GroqProvider", { enumerable: true, get: function () { return chat_1.GroqProvider; } });
|
|
20
|
+
var responses_1 = require("./responses");
|
|
21
|
+
Object.defineProperty(exports, "GroqResponsesProvider", { enumerable: true, get: function () { return responses_1.GroqResponsesProvider; } });
|
|
22
|
+
__exportStar(require("./types"), exports);
|
|
23
|
+
__exportStar(require("./util"), exports);
|
|
24
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { OpenAiResponsesProvider } from '../openai/responses';
|
|
2
|
+
import type { GroqResponsesProviderOptions } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Groq Responses API Provider
|
|
5
|
+
*
|
|
6
|
+
* Extends OpenAI Responses API provider with Groq-specific configuration.
|
|
7
|
+
* Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
|
|
8
|
+
*
|
|
9
|
+
* Note: Unlike the Chat Completions API, the Responses API does NOT support
|
|
10
|
+
* `reasoning_format` or `include_reasoning` parameters. Reasoning is controlled
|
|
11
|
+
* via the `reasoning.effort` parameter inherited from OpenAiCompletionOptions.
|
|
12
|
+
*
|
|
13
|
+
* Usage:
|
|
14
|
+
* groq:responses:llama-3.3-70b-versatile
|
|
15
|
+
* groq:responses:openai/gpt-oss-120b
|
|
16
|
+
* groq:responses:qwen/qwen3-32b
|
|
17
|
+
*/
|
|
18
|
+
export declare class GroqResponsesProvider extends OpenAiResponsesProvider {
|
|
19
|
+
protected get apiKey(): string | undefined;
|
|
20
|
+
protected isReasoningModel(): boolean;
|
|
21
|
+
protected supportsTemperature(): boolean;
|
|
22
|
+
constructor(modelName: string, providerOptions: GroqResponsesProviderOptions);
|
|
23
|
+
id(): string;
|
|
24
|
+
toString(): string;
|
|
25
|
+
toJSON(): {
|
|
26
|
+
provider: string;
|
|
27
|
+
model: string;
|
|
28
|
+
config: {
|
|
29
|
+
apiKey?: string;
|
|
30
|
+
apiKeyEnvar?: string;
|
|
31
|
+
apiKeyRequired?: boolean;
|
|
32
|
+
apiHost?: string;
|
|
33
|
+
apiBaseUrl?: string;
|
|
34
|
+
organization?: string;
|
|
35
|
+
cost?: number;
|
|
36
|
+
headers?: {
|
|
37
|
+
[key: string]: string;
|
|
38
|
+
};
|
|
39
|
+
maxRetries?: number;
|
|
40
|
+
temperature?: number;
|
|
41
|
+
max_completion_tokens?: number;
|
|
42
|
+
max_tokens?: number;
|
|
43
|
+
top_p?: number;
|
|
44
|
+
frequency_penalty?: number;
|
|
45
|
+
presence_penalty?: number;
|
|
46
|
+
best_of?: number;
|
|
47
|
+
functions?: import("../openai/util").OpenAiFunction[];
|
|
48
|
+
function_call?: "none" | "auto" | {
|
|
49
|
+
name: string;
|
|
50
|
+
};
|
|
51
|
+
tools?: (import("../openai/util").OpenAiTool | import("../openai/types").OpenAiMCPTool | import("../openai/types").OpenAiWebSearchTool | import("../openai/types").OpenAiCodeInterpreterTool)[];
|
|
52
|
+
tool_choice?: "none" | "auto" | "required" | {
|
|
53
|
+
type: "function";
|
|
54
|
+
function?: {
|
|
55
|
+
name: string;
|
|
56
|
+
};
|
|
57
|
+
};
|
|
58
|
+
tool_resources?: Record<string, any>;
|
|
59
|
+
showThinking?: boolean;
|
|
60
|
+
response_format?: {
|
|
61
|
+
type: "json_object";
|
|
62
|
+
} | {
|
|
63
|
+
type: "json_schema";
|
|
64
|
+
json_schema: {
|
|
65
|
+
name: string;
|
|
66
|
+
strict: boolean;
|
|
67
|
+
schema: {
|
|
68
|
+
type: "object";
|
|
69
|
+
properties: Record<string, any>;
|
|
70
|
+
required?: string[];
|
|
71
|
+
additionalProperties: false;
|
|
72
|
+
};
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
stop?: string[];
|
|
76
|
+
seed?: number;
|
|
77
|
+
passthrough?: object;
|
|
78
|
+
reasoning_effort?: import("../openai/types").ReasoningEffort;
|
|
79
|
+
reasoning?: import("../openai/types").Reasoning | import("../openai/types").GPT5Reasoning;
|
|
80
|
+
service_tier?: ("auto" | "default" | "premium") | null;
|
|
81
|
+
modalities?: string[];
|
|
82
|
+
audio?: {
|
|
83
|
+
bitrate?: string;
|
|
84
|
+
format?: string | "wav" | "mp3" | "flac" | "opus" | "pcm16" | "aac";
|
|
85
|
+
speed?: number;
|
|
86
|
+
voice?: string;
|
|
87
|
+
};
|
|
88
|
+
instructions?: string;
|
|
89
|
+
max_output_tokens?: number;
|
|
90
|
+
max_tool_calls?: number;
|
|
91
|
+
metadata?: Record<string, string>;
|
|
92
|
+
parallel_tool_calls?: boolean;
|
|
93
|
+
previous_response_id?: string;
|
|
94
|
+
store?: boolean;
|
|
95
|
+
stream?: boolean;
|
|
96
|
+
truncation?: "auto" | "disabled";
|
|
97
|
+
user?: string;
|
|
98
|
+
background?: boolean;
|
|
99
|
+
webhook_url?: string;
|
|
100
|
+
functionToolCallbacks?: Record<import("openai").default.FunctionDefinition["name"], import("../openai/types").AssistantFunctionCallback | string>;
|
|
101
|
+
mcp?: import("../mcp/types").MCPConfig;
|
|
102
|
+
verbosity?: import("../openai/types").GPT5Verbosity;
|
|
103
|
+
};
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=responses.d.ts.map
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.GroqResponsesProvider = void 0;
|
|
4
|
+
const responses_1 = require("../openai/responses");
|
|
5
|
+
const util_1 = require("./util");
|
|
6
|
+
const GROQ_API_BASE_URL = 'https://api.groq.com/openai/v1';
|
|
7
|
+
/**
|
|
8
|
+
* Groq Responses API Provider
|
|
9
|
+
*
|
|
10
|
+
* Extends OpenAI Responses API provider with Groq-specific configuration.
|
|
11
|
+
* Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
|
|
12
|
+
*
|
|
13
|
+
* Note: Unlike the Chat Completions API, the Responses API does NOT support
|
|
14
|
+
* `reasoning_format` or `include_reasoning` parameters. Reasoning is controlled
|
|
15
|
+
* via the `reasoning.effort` parameter inherited from OpenAiCompletionOptions.
|
|
16
|
+
*
|
|
17
|
+
* Usage:
|
|
18
|
+
* groq:responses:llama-3.3-70b-versatile
|
|
19
|
+
* groq:responses:openai/gpt-oss-120b
|
|
20
|
+
* groq:responses:qwen/qwen3-32b
|
|
21
|
+
*/
|
|
22
|
+
class GroqResponsesProvider extends responses_1.OpenAiResponsesProvider {
|
|
23
|
+
get apiKey() {
|
|
24
|
+
return this.config?.apiKey;
|
|
25
|
+
}
|
|
26
|
+
isReasoningModel() {
|
|
27
|
+
return (0, util_1.isGroqReasoningModel)(this.modelName) || super.isReasoningModel();
|
|
28
|
+
}
|
|
29
|
+
supportsTemperature() {
|
|
30
|
+
// Groq's reasoning models support temperature, unlike OpenAI's o1 models
|
|
31
|
+
if ((0, util_1.groqSupportsTemperature)(this.modelName)) {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
return super.supportsTemperature();
|
|
35
|
+
}
|
|
36
|
+
constructor(modelName, providerOptions) {
|
|
37
|
+
super(modelName, {
|
|
38
|
+
...providerOptions,
|
|
39
|
+
config: {
|
|
40
|
+
...providerOptions.config,
|
|
41
|
+
apiKeyEnvar: 'GROQ_API_KEY',
|
|
42
|
+
apiBaseUrl: GROQ_API_BASE_URL,
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
id() {
|
|
47
|
+
return `groq:responses:${this.modelName}`;
|
|
48
|
+
}
|
|
49
|
+
toString() {
|
|
50
|
+
return `[Groq Responses Provider ${this.modelName}]`;
|
|
51
|
+
}
|
|
52
|
+
toJSON() {
|
|
53
|
+
return {
|
|
54
|
+
provider: 'groq:responses',
|
|
55
|
+
model: this.modelName,
|
|
56
|
+
config: {
|
|
57
|
+
...this.config,
|
|
58
|
+
...(this.apiKey && { apiKey: undefined }),
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
exports.GroqResponsesProvider = GroqResponsesProvider;
|
|
64
|
+
//# sourceMappingURL=responses.js.map
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ProviderOptions } from '../../types/providers';
|
|
2
|
+
import type { OpenAiCompletionOptions } from '../openai/types';
|
|
3
|
+
/**
|
|
4
|
+
* Groq-specific completion options for Chat Completions API.
|
|
5
|
+
*/
|
|
6
|
+
export type GroqCompletionOptions = OpenAiCompletionOptions & {
|
|
7
|
+
systemPrompt?: string;
|
|
8
|
+
parallel_tool_calls?: boolean | null;
|
|
9
|
+
/** Controls how reasoning is presented. Options: 'parsed', 'raw', 'hidden'. */
|
|
10
|
+
reasoning_format?: 'parsed' | 'raw' | 'hidden' | null;
|
|
11
|
+
/** For GPT-OSS models, set to false to hide reasoning output. */
|
|
12
|
+
include_reasoning?: boolean;
|
|
13
|
+
/** Compound model tool configuration. */
|
|
14
|
+
compound_custom?: {
|
|
15
|
+
tools?: {
|
|
16
|
+
enabled_tools?: string[];
|
|
17
|
+
wolfram_settings?: {
|
|
18
|
+
authorization?: string;
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
/** Web search customization settings. */
|
|
23
|
+
search_settings?: {
|
|
24
|
+
exclude_domains?: string[];
|
|
25
|
+
include_domains?: string[];
|
|
26
|
+
country?: string;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Groq Responses API options.
|
|
31
|
+
* Note: Unlike Chat Completions API, does NOT support reasoning_format or include_reasoning.
|
|
32
|
+
* Reasoning is controlled via reasoning.effort inherited from OpenAiCompletionOptions.
|
|
33
|
+
*/
|
|
34
|
+
export type GroqResponsesOptions = OpenAiCompletionOptions & {
|
|
35
|
+
systemPrompt?: string;
|
|
36
|
+
parallel_tool_calls?: boolean | null;
|
|
37
|
+
};
|
|
38
|
+
export type GroqProviderOptions = ProviderOptions & {
|
|
39
|
+
config?: GroqCompletionOptions;
|
|
40
|
+
};
|
|
41
|
+
export type GroqResponsesProviderOptions = ProviderOptions & {
|
|
42
|
+
config?: GroqResponsesOptions;
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Groq-specific utility functions shared between Chat and Responses providers.
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Check if a model name corresponds to a Groq reasoning model.
|
|
6
|
+
* Groq's reasoning models include DeepSeek R1, GPT-OSS, and Qwen variants.
|
|
7
|
+
*/
|
|
8
|
+
export declare function isGroqReasoningModel(modelName: string): boolean;
|
|
9
|
+
/**
|
|
10
|
+
* Check if a Groq model supports temperature configuration.
|
|
11
|
+
* Unlike some reasoning models (e.g., o1), Groq's reasoning models
|
|
12
|
+
* (DeepSeek R1, GPT-OSS, Qwen) support temperature settings.
|
|
13
|
+
*/
|
|
14
|
+
export declare function groqSupportsTemperature(modelName: string): boolean;
|
|
15
|
+
//# sourceMappingURL=util.d.ts.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Groq-specific utility functions shared between Chat and Responses providers.
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.isGroqReasoningModel = isGroqReasoningModel;
|
|
7
|
+
exports.groqSupportsTemperature = groqSupportsTemperature;
|
|
8
|
+
/**
|
|
9
|
+
* Groq reasoning models that support extended thinking capabilities.
|
|
10
|
+
* These models include DeepSeek R1 variants, OpenAI GPT-OSS, and Qwen models.
|
|
11
|
+
*/
|
|
12
|
+
const GROQ_REASONING_MODEL_PATTERNS = ['deepseek-r1', 'gpt-oss', 'qwen'];
|
|
13
|
+
/**
|
|
14
|
+
* Check if a model name corresponds to a Groq reasoning model.
|
|
15
|
+
* Groq's reasoning models include DeepSeek R1, GPT-OSS, and Qwen variants.
|
|
16
|
+
*/
|
|
17
|
+
function isGroqReasoningModel(modelName) {
|
|
18
|
+
return GROQ_REASONING_MODEL_PATTERNS.some((pattern) => modelName.includes(pattern));
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Check if a Groq model supports temperature configuration.
|
|
22
|
+
* Unlike some reasoning models (e.g., o1), Groq's reasoning models
|
|
23
|
+
* (DeepSeek R1, GPT-OSS, Qwen) support temperature settings.
|
|
24
|
+
*/
|
|
25
|
+
function groqSupportsTemperature(modelName) {
|
|
26
|
+
return isGroqReasoningModel(modelName);
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=util.js.map
|
|
@@ -6,6 +6,14 @@ export declare class MCPClient {
|
|
|
6
6
|
private transports;
|
|
7
7
|
get hasInitialized(): boolean;
|
|
8
8
|
get connectedServers(): string[];
|
|
9
|
+
/**
|
|
10
|
+
* Check if debug mode is enabled (config takes priority over env var)
|
|
11
|
+
*/
|
|
12
|
+
private get isDebugEnabled();
|
|
13
|
+
/**
|
|
14
|
+
* Check if verbose mode is enabled (config takes priority over env var)
|
|
15
|
+
*/
|
|
16
|
+
private get isVerboseEnabled();
|
|
9
17
|
constructor(config: MCPConfig);
|
|
10
18
|
initialize(): Promise<void>;
|
|
11
19
|
private connectToServer;
|
|
@@ -38,8 +38,31 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.MCPClient = void 0;
|
|
40
40
|
const index_js_1 = require("@modelcontextprotocol/sdk/client/index.js");
|
|
41
|
+
const envars_1 = require("../../envars");
|
|
41
42
|
const logger_1 = __importDefault(require("../../logger"));
|
|
42
43
|
const util_1 = require("./util");
|
|
44
|
+
/**
|
|
45
|
+
* Get the effective request options for MCP requests.
|
|
46
|
+
* Priority: config values > MCP_REQUEST_TIMEOUT_MS env var > undefined (SDK default of 60s)
|
|
47
|
+
*/
|
|
48
|
+
function getEffectiveRequestOptions(config) {
|
|
49
|
+
const timeout = config.timeout ?? (0, envars_1.getEnvInt)('MCP_REQUEST_TIMEOUT_MS');
|
|
50
|
+
// If no timeout options are set, return undefined to use SDK defaults
|
|
51
|
+
if (!timeout && !config.resetTimeoutOnProgress && !config.maxTotalTimeout) {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
const options = {};
|
|
55
|
+
if (timeout) {
|
|
56
|
+
options.timeout = timeout;
|
|
57
|
+
}
|
|
58
|
+
if (config.resetTimeoutOnProgress) {
|
|
59
|
+
options.resetTimeoutOnProgress = config.resetTimeoutOnProgress;
|
|
60
|
+
}
|
|
61
|
+
if (config.maxTotalTimeout) {
|
|
62
|
+
options.maxTotalTimeout = config.maxTotalTimeout;
|
|
63
|
+
}
|
|
64
|
+
return options;
|
|
65
|
+
}
|
|
43
66
|
class MCPClient {
|
|
44
67
|
get hasInitialized() {
|
|
45
68
|
return this.clients.size > 0;
|
|
@@ -47,6 +70,18 @@ class MCPClient {
|
|
|
47
70
|
get connectedServers() {
|
|
48
71
|
return Array.from(this.clients.keys());
|
|
49
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Check if debug mode is enabled (config takes priority over env var)
|
|
75
|
+
*/
|
|
76
|
+
get isDebugEnabled() {
|
|
77
|
+
return this.config.debug ?? (0, envars_1.getEnvBool)('MCP_DEBUG') ?? false;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Check if verbose mode is enabled (config takes priority over env var)
|
|
81
|
+
*/
|
|
82
|
+
get isVerboseEnabled() {
|
|
83
|
+
return this.config.verbose ?? (0, envars_1.getEnvBool)('MCP_VERBOSE') ?? false;
|
|
84
|
+
}
|
|
50
85
|
constructor(config) {
|
|
51
86
|
this.clients = new Map();
|
|
52
87
|
this.tools = new Map();
|
|
@@ -69,6 +104,7 @@ class MCPClient {
|
|
|
69
104
|
const client = new index_js_1.Client({ name: 'promptfoo-MCP', version: '1.0.0' });
|
|
70
105
|
let transport;
|
|
71
106
|
try {
|
|
107
|
+
const requestOptions = getEffectiveRequestOptions(this.config);
|
|
72
108
|
if (server.command && server.args) {
|
|
73
109
|
const { StdioClientTransport } = await Promise.resolve().then(() => __importStar(require('@modelcontextprotocol/sdk/client/stdio.js')));
|
|
74
110
|
// NPM package or other command execution
|
|
@@ -77,7 +113,7 @@ class MCPClient {
|
|
|
77
113
|
args: server.args,
|
|
78
114
|
env: process.env,
|
|
79
115
|
});
|
|
80
|
-
await client.connect(transport);
|
|
116
|
+
await client.connect(transport, requestOptions);
|
|
81
117
|
}
|
|
82
118
|
else if (server.path) {
|
|
83
119
|
// Local server file
|
|
@@ -97,7 +133,7 @@ class MCPClient {
|
|
|
97
133
|
args: [server.path],
|
|
98
134
|
env: process.env,
|
|
99
135
|
});
|
|
100
|
-
await client.connect(transport);
|
|
136
|
+
await client.connect(transport, requestOptions);
|
|
101
137
|
}
|
|
102
138
|
else if (server.url) {
|
|
103
139
|
// Get auth headers and combine with custom headers
|
|
@@ -111,22 +147,34 @@ class MCPClient {
|
|
|
111
147
|
try {
|
|
112
148
|
const { StreamableHTTPClientTransport } = await Promise.resolve().then(() => __importStar(require('@modelcontextprotocol/sdk/client/streamableHttp.js')));
|
|
113
149
|
transport = new StreamableHTTPClientTransport(new URL(server.url), options);
|
|
114
|
-
await client.connect(transport);
|
|
150
|
+
await client.connect(transport, requestOptions);
|
|
115
151
|
logger_1.default.debug('Connected using Streamable HTTP transport');
|
|
116
152
|
}
|
|
117
153
|
catch (error) {
|
|
118
154
|
logger_1.default.debug(`Failed to connect to MCP server with Streamable HTTP transport ${serverKey}: ${error}`);
|
|
119
155
|
const { SSEClientTransport } = await Promise.resolve().then(() => __importStar(require('@modelcontextprotocol/sdk/client/sse.js')));
|
|
120
156
|
transport = new SSEClientTransport(new URL(server.url), options);
|
|
121
|
-
await client.connect(transport);
|
|
157
|
+
await client.connect(transport, requestOptions);
|
|
122
158
|
logger_1.default.debug('Connected using SSE transport');
|
|
123
159
|
}
|
|
124
160
|
}
|
|
125
161
|
else {
|
|
126
162
|
throw new Error('Either command+args or path or url must be specified for MCP server');
|
|
127
163
|
}
|
|
164
|
+
// Ping server to verify connection if configured
|
|
165
|
+
if (this.config.pingOnConnect) {
|
|
166
|
+
try {
|
|
167
|
+
await client.ping(requestOptions);
|
|
168
|
+
logger_1.default.debug(`MCP server ${serverKey} ping successful`);
|
|
169
|
+
}
|
|
170
|
+
catch (pingError) {
|
|
171
|
+
const pingErrorMessage = pingError instanceof Error ? pingError.message : String(pingError);
|
|
172
|
+
throw new Error(`MCP server ${serverKey} ping failed: ${pingErrorMessage}`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
128
175
|
// List available tools
|
|
129
|
-
const toolsResult = await client.listTools(
|
|
176
|
+
const toolsResult = await client.listTools(undefined, // no pagination params
|
|
177
|
+
requestOptions);
|
|
130
178
|
const serverTools = toolsResult?.tools?.map((tool) => ({
|
|
131
179
|
name: tool.name,
|
|
132
180
|
description: tool.description || '',
|
|
@@ -143,13 +191,13 @@ class MCPClient {
|
|
|
143
191
|
this.transports.set(serverKey, transport);
|
|
144
192
|
this.clients.set(serverKey, client);
|
|
145
193
|
this.tools.set(serverKey, filteredTools);
|
|
146
|
-
if (this.
|
|
194
|
+
if (this.isVerboseEnabled) {
|
|
147
195
|
console.log(`Connected to MCP server ${serverKey} with tools:`, filteredTools.map((tool) => tool.name));
|
|
148
196
|
}
|
|
149
197
|
}
|
|
150
198
|
catch (error) {
|
|
151
199
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
152
|
-
if (this.
|
|
200
|
+
if (this.isDebugEnabled) {
|
|
153
201
|
logger_1.default.error(`Failed to connect to MCP server ${serverKey}: ${errorMessage}`);
|
|
154
202
|
}
|
|
155
203
|
throw new Error(`Failed to connect to MCP server ${serverKey}: ${errorMessage}`);
|
|
@@ -159,12 +207,14 @@ class MCPClient {
|
|
|
159
207
|
return Array.from(this.tools.values()).flat();
|
|
160
208
|
}
|
|
161
209
|
async callTool(name, args) {
|
|
210
|
+
const requestOptions = getEffectiveRequestOptions(this.config);
|
|
162
211
|
// Find which server has this tool
|
|
163
212
|
for (const [serverKey, client] of this.clients.entries()) {
|
|
164
213
|
const serverTools = this.tools.get(serverKey) || [];
|
|
165
214
|
if (serverTools.some((tool) => tool.name === name)) {
|
|
166
215
|
try {
|
|
167
|
-
const result = await client.callTool({ name, arguments: args }
|
|
216
|
+
const result = await client.callTool({ name, arguments: args }, undefined, // use default result schema
|
|
217
|
+
requestOptions);
|
|
168
218
|
// Handle different content types appropriately
|
|
169
219
|
let content = '';
|
|
170
220
|
if (result?.content) {
|
|
@@ -191,7 +241,7 @@ class MCPClient {
|
|
|
191
241
|
}
|
|
192
242
|
catch (error) {
|
|
193
243
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
194
|
-
if (this.
|
|
244
|
+
if (this.isDebugEnabled) {
|
|
195
245
|
logger_1.default.error(`Error calling tool ${name}: ${errorMessage}`);
|
|
196
246
|
}
|
|
197
247
|
return {
|
|
@@ -213,7 +263,7 @@ class MCPClient {
|
|
|
213
263
|
await client.close();
|
|
214
264
|
}
|
|
215
265
|
catch (error) {
|
|
216
|
-
if (this.
|
|
266
|
+
if (this.isDebugEnabled) {
|
|
217
267
|
logger_1.default.error(`Error during cleanup: ${error instanceof Error ? error.message : String(error)}`);
|
|
218
268
|
}
|
|
219
269
|
}
|
|
@@ -19,7 +19,28 @@ export interface MCPConfig {
|
|
|
19
19
|
enabled: boolean;
|
|
20
20
|
server?: MCPServerConfig;
|
|
21
21
|
servers?: MCPServerConfig[];
|
|
22
|
+
/**
|
|
23
|
+
* Request timeout in milliseconds for MCP operations.
|
|
24
|
+
* Defaults to 60000 (60 seconds) if not specified.
|
|
25
|
+
*/
|
|
22
26
|
timeout?: number;
|
|
27
|
+
/**
|
|
28
|
+
* If true, receiving progress notifications will reset the request timeout.
|
|
29
|
+
* Useful for long-running operations that send periodic progress updates.
|
|
30
|
+
* Default: false
|
|
31
|
+
*/
|
|
32
|
+
resetTimeoutOnProgress?: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Maximum total time in milliseconds to wait for a response, regardless of progress.
|
|
35
|
+
* If exceeded, the request will timeout even if progress notifications are received.
|
|
36
|
+
* If not specified, there is no maximum total timeout.
|
|
37
|
+
*/
|
|
38
|
+
maxTotalTimeout?: number;
|
|
39
|
+
/**
|
|
40
|
+
* If true, ping the MCP server after connecting to verify it's responsive.
|
|
41
|
+
* Default: false
|
|
42
|
+
*/
|
|
43
|
+
pingOnConnect?: boolean;
|
|
23
44
|
tools?: string[];
|
|
24
45
|
exclude_tools?: string[];
|
|
25
46
|
debug?: boolean;
|