promptfoo 0.119.13 → 0.119.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/package.json +28 -26
  2. package/dist/src/app/assets/index-eJ2lMe94.js +51 -0
  3. package/dist/src/app/assets/{source-map-support-Bnh0UQ2S.js → source-map-support-1v4oeb7P.js} +1 -1
  4. package/dist/src/app/assets/sync-CtLQRuC1.js +1 -0
  5. package/dist/src/app/assets/{vendor-charts-T60Uk0Z3.js → vendor-charts-DnVv66VV.js} +1 -1
  6. package/dist/src/app/assets/{vendor-markdown-DLig-KJh.js → vendor-markdown-DCpQIyMA.js} +1 -1
  7. package/dist/src/app/assets/{vendor-mui-core-5BLaiG3c.js → vendor-mui-core-Boqnpf9f.js} +1 -1
  8. package/dist/src/app/assets/{vendor-mui-icons-fn39Fu2e.js → vendor-mui-icons-B8MqoVbj.js} +1 -1
  9. package/dist/src/app/assets/vendor-mui-x-CGSS6QHF.js +45 -0
  10. package/dist/src/app/assets/{vendor-utils-DYBMEuwX.js → vendor-utils-DdfHIEy8.js} +1 -1
  11. package/dist/src/app/index.html +7 -7
  12. package/dist/src/assertions/guardrails.d.ts +1 -1
  13. package/dist/src/assertions/guardrails.js +18 -9
  14. package/dist/src/assertions/index.d.ts +1 -1
  15. package/dist/src/assertions/index.js +9 -3
  16. package/dist/src/assertions/searchRubric.d.ts +3 -0
  17. package/dist/src/assertions/searchRubric.js +18 -0
  18. package/dist/src/commands/eval.js +1 -1
  19. package/dist/src/commands/modelScan.d.ts +7 -1
  20. package/dist/src/commands/modelScan.js +121 -59
  21. package/dist/src/database/index.d.ts +6 -0
  22. package/dist/src/database/index.js +11 -0
  23. package/dist/src/database/tables.d.ts +46 -24
  24. package/dist/src/envars.d.ts +17 -0
  25. package/dist/src/generated/constants.js +1 -1
  26. package/dist/src/logger.d.ts +5 -0
  27. package/dist/src/logger.js +28 -0
  28. package/dist/src/main.js +17 -6
  29. package/dist/src/matchers.d.ts +1 -0
  30. package/dist/src/matchers.js +80 -0
  31. package/dist/src/models/eval.d.ts +2 -1
  32. package/dist/src/models/eval.js +44 -2
  33. package/dist/src/prompts/grading.d.ts +1 -0
  34. package/dist/src/prompts/grading.js +26 -1
  35. package/dist/src/prompts/index.d.ts +1 -0
  36. package/dist/src/prompts/index.js +4 -1
  37. package/dist/src/providers/adaline.gateway.js +2 -2
  38. package/dist/src/providers/anthropic/defaults.d.ts +1 -1
  39. package/dist/src/providers/anthropic/defaults.js +15 -0
  40. package/dist/src/providers/azure/chat.d.ts +3 -1
  41. package/dist/src/providers/azure/chat.js +16 -3
  42. package/dist/src/providers/azure/defaults.js +660 -141
  43. package/dist/src/providers/azure/responses.d.ts +5 -0
  44. package/dist/src/providers/azure/responses.js +33 -4
  45. package/dist/src/providers/azure/types.d.ts +4 -0
  46. package/dist/src/providers/bedrock/agents.d.ts +1 -1
  47. package/dist/src/providers/bedrock/agents.js +2 -2
  48. package/dist/src/providers/bedrock/base.d.ts +40 -0
  49. package/dist/src/providers/bedrock/base.js +171 -0
  50. package/dist/src/providers/bedrock/converse.d.ts +146 -0
  51. package/dist/src/providers/bedrock/converse.js +1044 -0
  52. package/dist/src/providers/bedrock/index.d.ts +1 -34
  53. package/dist/src/providers/bedrock/index.js +4 -159
  54. package/dist/src/providers/bedrock/knowledgeBase.d.ts +1 -1
  55. package/dist/src/providers/bedrock/knowledgeBase.js +2 -2
  56. package/dist/src/providers/bedrock/nova-sonic.d.ts +2 -1
  57. package/dist/src/providers/bedrock/nova-sonic.js +2 -2
  58. package/dist/src/providers/claude-agent-sdk.d.ts +58 -1
  59. package/dist/src/providers/claude-agent-sdk.js +22 -1
  60. package/dist/src/providers/defaults.js +4 -0
  61. package/dist/src/providers/github/defaults.js +6 -6
  62. package/dist/src/providers/google/types.d.ts +25 -0
  63. package/dist/src/providers/google/util.d.ts +2 -0
  64. package/dist/src/providers/google/vertex.js +78 -22
  65. package/dist/src/providers/{groq.d.ts → groq/chat.d.ts} +26 -20
  66. package/dist/src/providers/groq/chat.js +79 -0
  67. package/dist/src/providers/groq/index.d.ts +5 -0
  68. package/dist/src/providers/groq/index.js +24 -0
  69. package/dist/src/providers/groq/responses.d.ts +106 -0
  70. package/dist/src/providers/groq/responses.js +64 -0
  71. package/dist/src/providers/groq/types.d.ts +44 -0
  72. package/dist/src/providers/groq/types.js +3 -0
  73. package/dist/src/providers/groq/util.d.ts +15 -0
  74. package/dist/src/providers/groq/util.js +28 -0
  75. package/dist/src/providers/mcp/client.d.ts +8 -0
  76. package/dist/src/providers/mcp/client.js +60 -10
  77. package/dist/src/providers/mcp/types.d.ts +21 -0
  78. package/dist/src/providers/openai/chatkit-pool.d.ts +114 -0
  79. package/dist/src/providers/openai/chatkit-pool.js +548 -0
  80. package/dist/src/providers/openai/chatkit-types.d.ts +73 -0
  81. package/dist/src/providers/openai/chatkit-types.js +3 -0
  82. package/dist/src/providers/openai/chatkit.d.ts +76 -0
  83. package/dist/src/providers/openai/chatkit.js +879 -0
  84. package/dist/src/providers/openai/codex-sdk.d.ts +109 -0
  85. package/dist/src/providers/openai/codex-sdk.js +346 -0
  86. package/dist/src/providers/openai/defaults.d.ts +2 -0
  87. package/dist/src/providers/openai/defaults.js +10 -4
  88. package/dist/src/providers/registry.js +48 -9
  89. package/dist/src/providers/responses/types.d.ts +1 -1
  90. package/dist/src/providers/sagemaker.d.ts +2 -2
  91. package/dist/src/providers/webSearchUtils.d.ts +17 -0
  92. package/dist/src/providers/webSearchUtils.js +169 -0
  93. package/dist/src/providers/xai/chat.d.ts +61 -0
  94. package/dist/src/providers/xai/chat.js +68 -3
  95. package/dist/src/providers/xai/responses.d.ts +189 -0
  96. package/dist/src/providers/xai/responses.js +268 -0
  97. package/dist/src/redteam/constants/plugins.d.ts +1 -1
  98. package/dist/src/redteam/constants/plugins.js +1 -1
  99. package/dist/src/redteam/constants/strategies.d.ts +1 -1
  100. package/dist/src/redteam/constants/strategies.js +1 -0
  101. package/dist/src/redteam/plugins/vlguard.d.ts +53 -4
  102. package/dist/src/redteam/plugins/vlguard.js +362 -46
  103. package/dist/src/redteam/providers/constants.d.ts +2 -2
  104. package/dist/src/redteam/providers/constants.js +2 -2
  105. package/dist/src/redteam/providers/crescendo/index.d.ts +1 -1
  106. package/dist/src/redteam/providers/crescendo/index.js +5 -3
  107. package/dist/src/redteam/providers/hydra/index.js +1 -1
  108. package/dist/src/server/routes/modelAudit.js +4 -4
  109. package/dist/src/share.js +4 -2
  110. package/dist/src/telemetry.js +44 -8
  111. package/dist/src/types/env.d.ts +3 -0
  112. package/dist/src/types/env.js +1 -0
  113. package/dist/src/types/index.d.ts +896 -615
  114. package/dist/src/types/index.js +1 -0
  115. package/dist/src/types/providers.d.ts +1 -0
  116. package/dist/src/types/tracing.d.ts +3 -0
  117. package/dist/src/util/database.d.ts +6 -4
  118. package/dist/src/util/file.js +6 -4
  119. package/dist/src/util/modelAuditCliParser.d.ts +4 -4
  120. package/dist/src/util/xlsx.js +52 -26
  121. package/dist/src/validators/providers.d.ts +142 -122
  122. package/dist/src/validators/providers.js +4 -6
  123. package/dist/src/validators/redteam.d.ts +36 -28
  124. package/dist/src/validators/redteam.js +9 -3
  125. package/dist/tsconfig.tsbuildinfo +1 -1
  126. package/package.json +28 -26
  127. package/dist/drizzle/CLAUDE.md +0 -65
  128. package/dist/src/app/assets/index-DifT6VGT.js +0 -51
  129. package/dist/src/app/assets/sync-Oo-W_Rbj.js +0 -1
  130. package/dist/src/app/assets/vendor-mui-x-C2xF-yiO.js +0 -45
  131. package/dist/src/providers/groq.js +0 -48
@@ -1,19 +1,26 @@
1
- import { OpenAiChatCompletionProvider } from './openai/chat';
2
- import type { ProviderOptions } from '../types/providers';
3
- import type { OpenAiCompletionOptions } from './openai/types';
4
- type GroqCompletionOptions = OpenAiCompletionOptions & {
5
- systemPrompt?: string;
6
- parallel_tool_calls?: boolean | null;
7
- reasoning_format?: string | null;
8
- };
9
- type GroqProviderOptions = ProviderOptions & {
10
- config?: GroqCompletionOptions;
11
- };
1
+ import type { CallApiContextParams, CallApiOptionsParams } from '../../types/index';
2
+ import { OpenAiChatCompletionProvider } from '../openai/chat';
3
+ import type { GroqProviderOptions } from './types';
4
+ /**
5
+ * Groq Chat Completions API Provider
6
+ *
7
+ * Extends OpenAI Chat Completions provider with Groq-specific configuration.
8
+ * Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
9
+ *
10
+ * Usage:
11
+ * groq:llama-3.3-70b-versatile
12
+ * groq:openai/gpt-oss-120b
13
+ * groq:qwen/qwen3-32b
14
+ */
12
15
  export declare class GroqProvider extends OpenAiChatCompletionProvider {
13
16
  protected get apiKey(): string | undefined;
14
17
  protected isReasoningModel(): boolean;
15
18
  protected supportsTemperature(): boolean;
16
19
  constructor(modelName: string, providerOptions: GroqProviderOptions);
20
+ getOpenAiBody(prompt: string, context?: CallApiContextParams, callApiOptions?: CallApiOptionsParams): Promise<{
21
+ body: any;
22
+ config: any;
23
+ }>;
17
24
  id(): string;
18
25
  toString(): string;
19
26
  toJSON(): {
@@ -38,11 +45,11 @@ export declare class GroqProvider extends OpenAiChatCompletionProvider {
38
45
  frequency_penalty?: number;
39
46
  presence_penalty?: number;
40
47
  best_of?: number;
41
- functions?: import("./openai/util").OpenAiFunction[];
48
+ functions?: import("../openai/util").OpenAiFunction[];
42
49
  function_call?: "none" | "auto" | {
43
50
  name: string;
44
51
  };
45
- tools?: (import("./openai/util").OpenAiTool | import("./openai/types").OpenAiMCPTool | import("./openai/types").OpenAiWebSearchTool | import("./openai/types").OpenAiCodeInterpreterTool)[];
52
+ tools?: (import("../openai/util").OpenAiTool | import("../openai/types").OpenAiMCPTool | import("../openai/types").OpenAiWebSearchTool | import("../openai/types").OpenAiCodeInterpreterTool)[];
46
53
  tool_choice?: "none" | "auto" | "required" | {
47
54
  type: "function";
48
55
  function?: {
@@ -69,8 +76,8 @@ export declare class GroqProvider extends OpenAiChatCompletionProvider {
69
76
  stop?: string[];
70
77
  seed?: number;
71
78
  passthrough?: object;
72
- reasoning_effort?: import("./openai/types").ReasoningEffort;
73
- reasoning?: import("./openai/types").Reasoning | import("./openai/types").GPT5Reasoning;
79
+ reasoning_effort?: import("../openai/types").ReasoningEffort;
80
+ reasoning?: import("../openai/types").Reasoning | import("../openai/types").GPT5Reasoning;
74
81
  service_tier?: ("auto" | "default" | "premium") | null;
75
82
  modalities?: string[];
76
83
  audio?: {
@@ -91,11 +98,10 @@ export declare class GroqProvider extends OpenAiChatCompletionProvider {
91
98
  user?: string;
92
99
  background?: boolean;
93
100
  webhook_url?: string;
94
- functionToolCallbacks?: Record<import("openai").default.FunctionDefinition["name"], import("./openai/types").AssistantFunctionCallback | string>;
95
- mcp?: import("./mcp/types").MCPConfig;
96
- verbosity?: import("./openai/types").GPT5Verbosity;
101
+ functionToolCallbacks?: Record<import("openai").default.FunctionDefinition["name"], import("../openai/types").AssistantFunctionCallback | string>;
102
+ mcp?: import("../mcp/types").MCPConfig;
103
+ verbosity?: import("../openai/types").GPT5Verbosity;
97
104
  };
98
105
  };
99
106
  }
100
- export {};
101
- //# sourceMappingURL=groq.d.ts.map
107
+ //# sourceMappingURL=chat.d.ts.map
@@ -0,0 +1,79 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.GroqProvider = void 0;
4
+ const chat_1 = require("../openai/chat");
5
+ const util_1 = require("./util");
6
+ const GROQ_API_BASE_URL = 'https://api.groq.com/openai/v1';
7
+ /**
8
+ * Groq Chat Completions API Provider
9
+ *
10
+ * Extends OpenAI Chat Completions provider with Groq-specific configuration.
11
+ * Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
12
+ *
13
+ * Usage:
14
+ * groq:llama-3.3-70b-versatile
15
+ * groq:openai/gpt-oss-120b
16
+ * groq:qwen/qwen3-32b
17
+ */
18
+ class GroqProvider extends chat_1.OpenAiChatCompletionProvider {
19
+ get apiKey() {
20
+ return this.config?.apiKey;
21
+ }
22
+ isReasoningModel() {
23
+ return (0, util_1.isGroqReasoningModel)(this.modelName) || super.isReasoningModel();
24
+ }
25
+ supportsTemperature() {
26
+ // Groq's reasoning models support temperature, unlike OpenAI's o1 models
27
+ if ((0, util_1.groqSupportsTemperature)(this.modelName)) {
28
+ return true;
29
+ }
30
+ return super.supportsTemperature();
31
+ }
32
+ constructor(modelName, providerOptions) {
33
+ super(modelName, {
34
+ ...providerOptions,
35
+ config: {
36
+ ...providerOptions.config,
37
+ apiKeyEnvar: 'GROQ_API_KEY',
38
+ apiBaseUrl: GROQ_API_BASE_URL,
39
+ },
40
+ });
41
+ }
42
+ async getOpenAiBody(prompt, context, callApiOptions) {
43
+ const { body, config } = await super.getOpenAiBody(prompt, context, callApiOptions);
44
+ const groqConfig = this.config;
45
+ // Add Groq-specific reasoning parameters
46
+ if (groqConfig.reasoning_format !== undefined) {
47
+ body.reasoning_format = groqConfig.reasoning_format;
48
+ }
49
+ if (groqConfig.include_reasoning !== undefined) {
50
+ body.include_reasoning = groqConfig.include_reasoning;
51
+ }
52
+ // Add Compound model parameters
53
+ if (groqConfig.compound_custom) {
54
+ body.compound_custom = groqConfig.compound_custom;
55
+ }
56
+ if (groqConfig.search_settings) {
57
+ body.search_settings = groqConfig.search_settings;
58
+ }
59
+ return { body, config };
60
+ }
61
+ id() {
62
+ return `groq:${this.modelName}`;
63
+ }
64
+ toString() {
65
+ return `[Groq Provider ${this.modelName}]`;
66
+ }
67
+ toJSON() {
68
+ return {
69
+ provider: 'groq',
70
+ model: this.modelName,
71
+ config: {
72
+ ...this.config,
73
+ ...(this.apiKey && { apiKey: undefined }),
74
+ },
75
+ };
76
+ }
77
+ }
78
+ exports.GroqProvider = GroqProvider;
79
+ //# sourceMappingURL=chat.js.map
@@ -0,0 +1,5 @@
1
+ export { GroqProvider } from './chat';
2
+ export { GroqResponsesProvider } from './responses';
3
+ export * from './types';
4
+ export * from './util';
5
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,24 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.GroqResponsesProvider = exports.GroqProvider = void 0;
18
+ var chat_1 = require("./chat");
19
+ Object.defineProperty(exports, "GroqProvider", { enumerable: true, get: function () { return chat_1.GroqProvider; } });
20
+ var responses_1 = require("./responses");
21
+ Object.defineProperty(exports, "GroqResponsesProvider", { enumerable: true, get: function () { return responses_1.GroqResponsesProvider; } });
22
+ __exportStar(require("./types"), exports);
23
+ __exportStar(require("./util"), exports);
24
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,106 @@
1
+ import { OpenAiResponsesProvider } from '../openai/responses';
2
+ import type { GroqResponsesProviderOptions } from './types';
3
+ /**
4
+ * Groq Responses API Provider
5
+ *
6
+ * Extends OpenAI Responses API provider with Groq-specific configuration.
7
+ * Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
8
+ *
9
+ * Note: Unlike the Chat Completions API, the Responses API does NOT support
10
+ * `reasoning_format` or `include_reasoning` parameters. Reasoning is controlled
11
+ * via the `reasoning.effort` parameter inherited from OpenAiCompletionOptions.
12
+ *
13
+ * Usage:
14
+ * groq:responses:llama-3.3-70b-versatile
15
+ * groq:responses:openai/gpt-oss-120b
16
+ * groq:responses:qwen/qwen3-32b
17
+ */
18
+ export declare class GroqResponsesProvider extends OpenAiResponsesProvider {
19
+ protected get apiKey(): string | undefined;
20
+ protected isReasoningModel(): boolean;
21
+ protected supportsTemperature(): boolean;
22
+ constructor(modelName: string, providerOptions: GroqResponsesProviderOptions);
23
+ id(): string;
24
+ toString(): string;
25
+ toJSON(): {
26
+ provider: string;
27
+ model: string;
28
+ config: {
29
+ apiKey?: string;
30
+ apiKeyEnvar?: string;
31
+ apiKeyRequired?: boolean;
32
+ apiHost?: string;
33
+ apiBaseUrl?: string;
34
+ organization?: string;
35
+ cost?: number;
36
+ headers?: {
37
+ [key: string]: string;
38
+ };
39
+ maxRetries?: number;
40
+ temperature?: number;
41
+ max_completion_tokens?: number;
42
+ max_tokens?: number;
43
+ top_p?: number;
44
+ frequency_penalty?: number;
45
+ presence_penalty?: number;
46
+ best_of?: number;
47
+ functions?: import("../openai/util").OpenAiFunction[];
48
+ function_call?: "none" | "auto" | {
49
+ name: string;
50
+ };
51
+ tools?: (import("../openai/util").OpenAiTool | import("../openai/types").OpenAiMCPTool | import("../openai/types").OpenAiWebSearchTool | import("../openai/types").OpenAiCodeInterpreterTool)[];
52
+ tool_choice?: "none" | "auto" | "required" | {
53
+ type: "function";
54
+ function?: {
55
+ name: string;
56
+ };
57
+ };
58
+ tool_resources?: Record<string, any>;
59
+ showThinking?: boolean;
60
+ response_format?: {
61
+ type: "json_object";
62
+ } | {
63
+ type: "json_schema";
64
+ json_schema: {
65
+ name: string;
66
+ strict: boolean;
67
+ schema: {
68
+ type: "object";
69
+ properties: Record<string, any>;
70
+ required?: string[];
71
+ additionalProperties: false;
72
+ };
73
+ };
74
+ };
75
+ stop?: string[];
76
+ seed?: number;
77
+ passthrough?: object;
78
+ reasoning_effort?: import("../openai/types").ReasoningEffort;
79
+ reasoning?: import("../openai/types").Reasoning | import("../openai/types").GPT5Reasoning;
80
+ service_tier?: ("auto" | "default" | "premium") | null;
81
+ modalities?: string[];
82
+ audio?: {
83
+ bitrate?: string;
84
+ format?: string | "wav" | "mp3" | "flac" | "opus" | "pcm16" | "aac";
85
+ speed?: number;
86
+ voice?: string;
87
+ };
88
+ instructions?: string;
89
+ max_output_tokens?: number;
90
+ max_tool_calls?: number;
91
+ metadata?: Record<string, string>;
92
+ parallel_tool_calls?: boolean;
93
+ previous_response_id?: string;
94
+ store?: boolean;
95
+ stream?: boolean;
96
+ truncation?: "auto" | "disabled";
97
+ user?: string;
98
+ background?: boolean;
99
+ webhook_url?: string;
100
+ functionToolCallbacks?: Record<import("openai").default.FunctionDefinition["name"], import("../openai/types").AssistantFunctionCallback | string>;
101
+ mcp?: import("../mcp/types").MCPConfig;
102
+ verbosity?: import("../openai/types").GPT5Verbosity;
103
+ };
104
+ };
105
+ }
106
+ //# sourceMappingURL=responses.d.ts.map
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.GroqResponsesProvider = void 0;
4
+ const responses_1 = require("../openai/responses");
5
+ const util_1 = require("./util");
6
+ const GROQ_API_BASE_URL = 'https://api.groq.com/openai/v1';
7
+ /**
8
+ * Groq Responses API Provider
9
+ *
10
+ * Extends OpenAI Responses API provider with Groq-specific configuration.
11
+ * Supports reasoning models (DeepSeek R1, GPT-OSS, Qwen) with temperature control.
12
+ *
13
+ * Note: Unlike the Chat Completions API, the Responses API does NOT support
14
+ * `reasoning_format` or `include_reasoning` parameters. Reasoning is controlled
15
+ * via the `reasoning.effort` parameter inherited from OpenAiCompletionOptions.
16
+ *
17
+ * Usage:
18
+ * groq:responses:llama-3.3-70b-versatile
19
+ * groq:responses:openai/gpt-oss-120b
20
+ * groq:responses:qwen/qwen3-32b
21
+ */
22
+ class GroqResponsesProvider extends responses_1.OpenAiResponsesProvider {
23
+ get apiKey() {
24
+ return this.config?.apiKey;
25
+ }
26
+ isReasoningModel() {
27
+ return (0, util_1.isGroqReasoningModel)(this.modelName) || super.isReasoningModel();
28
+ }
29
+ supportsTemperature() {
30
+ // Groq's reasoning models support temperature, unlike OpenAI's o1 models
31
+ if ((0, util_1.groqSupportsTemperature)(this.modelName)) {
32
+ return true;
33
+ }
34
+ return super.supportsTemperature();
35
+ }
36
+ constructor(modelName, providerOptions) {
37
+ super(modelName, {
38
+ ...providerOptions,
39
+ config: {
40
+ ...providerOptions.config,
41
+ apiKeyEnvar: 'GROQ_API_KEY',
42
+ apiBaseUrl: GROQ_API_BASE_URL,
43
+ },
44
+ });
45
+ }
46
+ id() {
47
+ return `groq:responses:${this.modelName}`;
48
+ }
49
+ toString() {
50
+ return `[Groq Responses Provider ${this.modelName}]`;
51
+ }
52
+ toJSON() {
53
+ return {
54
+ provider: 'groq:responses',
55
+ model: this.modelName,
56
+ config: {
57
+ ...this.config,
58
+ ...(this.apiKey && { apiKey: undefined }),
59
+ },
60
+ };
61
+ }
62
+ }
63
+ exports.GroqResponsesProvider = GroqResponsesProvider;
64
+ //# sourceMappingURL=responses.js.map
@@ -0,0 +1,44 @@
1
+ import type { ProviderOptions } from '../../types/providers';
2
+ import type { OpenAiCompletionOptions } from '../openai/types';
3
+ /**
4
+ * Groq-specific completion options for Chat Completions API.
5
+ */
6
+ export type GroqCompletionOptions = OpenAiCompletionOptions & {
7
+ systemPrompt?: string;
8
+ parallel_tool_calls?: boolean | null;
9
+ /** Controls how reasoning is presented. Options: 'parsed', 'raw', 'hidden'. */
10
+ reasoning_format?: 'parsed' | 'raw' | 'hidden' | null;
11
+ /** For GPT-OSS models, set to false to hide reasoning output. */
12
+ include_reasoning?: boolean;
13
+ /** Compound model tool configuration. */
14
+ compound_custom?: {
15
+ tools?: {
16
+ enabled_tools?: string[];
17
+ wolfram_settings?: {
18
+ authorization?: string;
19
+ };
20
+ };
21
+ };
22
+ /** Web search customization settings. */
23
+ search_settings?: {
24
+ exclude_domains?: string[];
25
+ include_domains?: string[];
26
+ country?: string;
27
+ };
28
+ };
29
+ /**
30
+ * Groq Responses API options.
31
+ * Note: Unlike Chat Completions API, does NOT support reasoning_format or include_reasoning.
32
+ * Reasoning is controlled via reasoning.effort inherited from OpenAiCompletionOptions.
33
+ */
34
+ export type GroqResponsesOptions = OpenAiCompletionOptions & {
35
+ systemPrompt?: string;
36
+ parallel_tool_calls?: boolean | null;
37
+ };
38
+ export type GroqProviderOptions = ProviderOptions & {
39
+ config?: GroqCompletionOptions;
40
+ };
41
+ export type GroqResponsesProviderOptions = ProviderOptions & {
42
+ config?: GroqResponsesOptions;
43
+ };
44
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Groq-specific utility functions shared between Chat and Responses providers.
3
+ */
4
+ /**
5
+ * Check if a model name corresponds to a Groq reasoning model.
6
+ * Groq's reasoning models include DeepSeek R1, GPT-OSS, and Qwen variants.
7
+ */
8
+ export declare function isGroqReasoningModel(modelName: string): boolean;
9
+ /**
10
+ * Check if a Groq model supports temperature configuration.
11
+ * Unlike some reasoning models (e.g., o1), Groq's reasoning models
12
+ * (DeepSeek R1, GPT-OSS, Qwen) support temperature settings.
13
+ */
14
+ export declare function groqSupportsTemperature(modelName: string): boolean;
15
+ //# sourceMappingURL=util.d.ts.map
@@ -0,0 +1,28 @@
1
+ "use strict";
2
+ /**
3
+ * Groq-specific utility functions shared between Chat and Responses providers.
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.isGroqReasoningModel = isGroqReasoningModel;
7
+ exports.groqSupportsTemperature = groqSupportsTemperature;
8
+ /**
9
+ * Groq reasoning models that support extended thinking capabilities.
10
+ * These models include DeepSeek R1 variants, OpenAI GPT-OSS, and Qwen models.
11
+ */
12
+ const GROQ_REASONING_MODEL_PATTERNS = ['deepseek-r1', 'gpt-oss', 'qwen'];
13
+ /**
14
+ * Check if a model name corresponds to a Groq reasoning model.
15
+ * Groq's reasoning models include DeepSeek R1, GPT-OSS, and Qwen variants.
16
+ */
17
+ function isGroqReasoningModel(modelName) {
18
+ return GROQ_REASONING_MODEL_PATTERNS.some((pattern) => modelName.includes(pattern));
19
+ }
20
+ /**
21
+ * Check if a Groq model supports temperature configuration.
22
+ * Unlike some reasoning models (e.g., o1), Groq's reasoning models
23
+ * (DeepSeek R1, GPT-OSS, Qwen) support temperature settings.
24
+ */
25
+ function groqSupportsTemperature(modelName) {
26
+ return isGroqReasoningModel(modelName);
27
+ }
28
+ //# sourceMappingURL=util.js.map
@@ -6,6 +6,14 @@ export declare class MCPClient {
6
6
  private transports;
7
7
  get hasInitialized(): boolean;
8
8
  get connectedServers(): string[];
9
+ /**
10
+ * Check if debug mode is enabled (config takes priority over env var)
11
+ */
12
+ private get isDebugEnabled();
13
+ /**
14
+ * Check if verbose mode is enabled (config takes priority over env var)
15
+ */
16
+ private get isVerboseEnabled();
9
17
  constructor(config: MCPConfig);
10
18
  initialize(): Promise<void>;
11
19
  private connectToServer;
@@ -38,8 +38,31 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
38
38
  Object.defineProperty(exports, "__esModule", { value: true });
39
39
  exports.MCPClient = void 0;
40
40
  const index_js_1 = require("@modelcontextprotocol/sdk/client/index.js");
41
+ const envars_1 = require("../../envars");
41
42
  const logger_1 = __importDefault(require("../../logger"));
42
43
  const util_1 = require("./util");
44
+ /**
45
+ * Get the effective request options for MCP requests.
46
+ * Priority: config values > MCP_REQUEST_TIMEOUT_MS env var > undefined (SDK default of 60s)
47
+ */
48
+ function getEffectiveRequestOptions(config) {
49
+ const timeout = config.timeout ?? (0, envars_1.getEnvInt)('MCP_REQUEST_TIMEOUT_MS');
50
+ // If no timeout options are set, return undefined to use SDK defaults
51
+ if (!timeout && !config.resetTimeoutOnProgress && !config.maxTotalTimeout) {
52
+ return undefined;
53
+ }
54
+ const options = {};
55
+ if (timeout) {
56
+ options.timeout = timeout;
57
+ }
58
+ if (config.resetTimeoutOnProgress) {
59
+ options.resetTimeoutOnProgress = config.resetTimeoutOnProgress;
60
+ }
61
+ if (config.maxTotalTimeout) {
62
+ options.maxTotalTimeout = config.maxTotalTimeout;
63
+ }
64
+ return options;
65
+ }
43
66
  class MCPClient {
44
67
  get hasInitialized() {
45
68
  return this.clients.size > 0;
@@ -47,6 +70,18 @@ class MCPClient {
47
70
  get connectedServers() {
48
71
  return Array.from(this.clients.keys());
49
72
  }
73
+ /**
74
+ * Check if debug mode is enabled (config takes priority over env var)
75
+ */
76
+ get isDebugEnabled() {
77
+ return this.config.debug ?? (0, envars_1.getEnvBool)('MCP_DEBUG') ?? false;
78
+ }
79
+ /**
80
+ * Check if verbose mode is enabled (config takes priority over env var)
81
+ */
82
+ get isVerboseEnabled() {
83
+ return this.config.verbose ?? (0, envars_1.getEnvBool)('MCP_VERBOSE') ?? false;
84
+ }
50
85
  constructor(config) {
51
86
  this.clients = new Map();
52
87
  this.tools = new Map();
@@ -69,6 +104,7 @@ class MCPClient {
69
104
  const client = new index_js_1.Client({ name: 'promptfoo-MCP', version: '1.0.0' });
70
105
  let transport;
71
106
  try {
107
+ const requestOptions = getEffectiveRequestOptions(this.config);
72
108
  if (server.command && server.args) {
73
109
  const { StdioClientTransport } = await Promise.resolve().then(() => __importStar(require('@modelcontextprotocol/sdk/client/stdio.js')));
74
110
  // NPM package or other command execution
@@ -77,7 +113,7 @@ class MCPClient {
77
113
  args: server.args,
78
114
  env: process.env,
79
115
  });
80
- await client.connect(transport);
116
+ await client.connect(transport, requestOptions);
81
117
  }
82
118
  else if (server.path) {
83
119
  // Local server file
@@ -97,7 +133,7 @@ class MCPClient {
97
133
  args: [server.path],
98
134
  env: process.env,
99
135
  });
100
- await client.connect(transport);
136
+ await client.connect(transport, requestOptions);
101
137
  }
102
138
  else if (server.url) {
103
139
  // Get auth headers and combine with custom headers
@@ -111,22 +147,34 @@ class MCPClient {
111
147
  try {
112
148
  const { StreamableHTTPClientTransport } = await Promise.resolve().then(() => __importStar(require('@modelcontextprotocol/sdk/client/streamableHttp.js')));
113
149
  transport = new StreamableHTTPClientTransport(new URL(server.url), options);
114
- await client.connect(transport);
150
+ await client.connect(transport, requestOptions);
115
151
  logger_1.default.debug('Connected using Streamable HTTP transport');
116
152
  }
117
153
  catch (error) {
118
154
  logger_1.default.debug(`Failed to connect to MCP server with Streamable HTTP transport ${serverKey}: ${error}`);
119
155
  const { SSEClientTransport } = await Promise.resolve().then(() => __importStar(require('@modelcontextprotocol/sdk/client/sse.js')));
120
156
  transport = new SSEClientTransport(new URL(server.url), options);
121
- await client.connect(transport);
157
+ await client.connect(transport, requestOptions);
122
158
  logger_1.default.debug('Connected using SSE transport');
123
159
  }
124
160
  }
125
161
  else {
126
162
  throw new Error('Either command+args or path or url must be specified for MCP server');
127
163
  }
164
+ // Ping server to verify connection if configured
165
+ if (this.config.pingOnConnect) {
166
+ try {
167
+ await client.ping(requestOptions);
168
+ logger_1.default.debug(`MCP server ${serverKey} ping successful`);
169
+ }
170
+ catch (pingError) {
171
+ const pingErrorMessage = pingError instanceof Error ? pingError.message : String(pingError);
172
+ throw new Error(`MCP server ${serverKey} ping failed: ${pingErrorMessage}`);
173
+ }
174
+ }
128
175
  // List available tools
129
- const toolsResult = await client.listTools();
176
+ const toolsResult = await client.listTools(undefined, // no pagination params
177
+ requestOptions);
130
178
  const serverTools = toolsResult?.tools?.map((tool) => ({
131
179
  name: tool.name,
132
180
  description: tool.description || '',
@@ -143,13 +191,13 @@ class MCPClient {
143
191
  this.transports.set(serverKey, transport);
144
192
  this.clients.set(serverKey, client);
145
193
  this.tools.set(serverKey, filteredTools);
146
- if (this.config.verbose) {
194
+ if (this.isVerboseEnabled) {
147
195
  console.log(`Connected to MCP server ${serverKey} with tools:`, filteredTools.map((tool) => tool.name));
148
196
  }
149
197
  }
150
198
  catch (error) {
151
199
  const errorMessage = error instanceof Error ? error.message : String(error);
152
- if (this.config.debug) {
200
+ if (this.isDebugEnabled) {
153
201
  logger_1.default.error(`Failed to connect to MCP server ${serverKey}: ${errorMessage}`);
154
202
  }
155
203
  throw new Error(`Failed to connect to MCP server ${serverKey}: ${errorMessage}`);
@@ -159,12 +207,14 @@ class MCPClient {
159
207
  return Array.from(this.tools.values()).flat();
160
208
  }
161
209
  async callTool(name, args) {
210
+ const requestOptions = getEffectiveRequestOptions(this.config);
162
211
  // Find which server has this tool
163
212
  for (const [serverKey, client] of this.clients.entries()) {
164
213
  const serverTools = this.tools.get(serverKey) || [];
165
214
  if (serverTools.some((tool) => tool.name === name)) {
166
215
  try {
167
- const result = await client.callTool({ name, arguments: args });
216
+ const result = await client.callTool({ name, arguments: args }, undefined, // use default result schema
217
+ requestOptions);
168
218
  // Handle different content types appropriately
169
219
  let content = '';
170
220
  if (result?.content) {
@@ -191,7 +241,7 @@ class MCPClient {
191
241
  }
192
242
  catch (error) {
193
243
  const errorMessage = error instanceof Error ? error.message : String(error);
194
- if (this.config.debug) {
244
+ if (this.isDebugEnabled) {
195
245
  logger_1.default.error(`Error calling tool ${name}: ${errorMessage}`);
196
246
  }
197
247
  return {
@@ -213,7 +263,7 @@ class MCPClient {
213
263
  await client.close();
214
264
  }
215
265
  catch (error) {
216
- if (this.config.debug) {
266
+ if (this.isDebugEnabled) {
217
267
  logger_1.default.error(`Error during cleanup: ${error instanceof Error ? error.message : String(error)}`);
218
268
  }
219
269
  }
@@ -19,7 +19,28 @@ export interface MCPConfig {
19
19
  enabled: boolean;
20
20
  server?: MCPServerConfig;
21
21
  servers?: MCPServerConfig[];
22
+ /**
23
+ * Request timeout in milliseconds for MCP operations.
24
+ * Defaults to 60000 (60 seconds) if not specified.
25
+ */
22
26
  timeout?: number;
27
+ /**
28
+ * If true, receiving progress notifications will reset the request timeout.
29
+ * Useful for long-running operations that send periodic progress updates.
30
+ * Default: false
31
+ */
32
+ resetTimeoutOnProgress?: boolean;
33
+ /**
34
+ * Maximum total time in milliseconds to wait for a response, regardless of progress.
35
+ * If exceeded, the request will timeout even if progress notifications are received.
36
+ * If not specified, there is no maximum total timeout.
37
+ */
38
+ maxTotalTimeout?: number;
39
+ /**
40
+ * If true, ping the MCP server after connecting to verify it's responsive.
41
+ * Default: false
42
+ */
43
+ pingOnConnect?: boolean;
23
44
  tools?: string[];
24
45
  exclude_tools?: string[];
25
46
  debug?: boolean;