promptfoo 0.113.0 → 0.113.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/dist/package.json +28 -30
  2. package/dist/src/app/assets/{index-BBF3o-At.js → index-DDgDcSbp.js} +176 -176
  3. package/dist/src/app/assets/{index.es-CIVyj4Ly.js → index.es-CTuWilQA.js} +1 -1
  4. package/dist/src/app/assets/{source-map-support-DmE-TbSd.js → source-map-support-CMD9DZ3t.js} +1 -1
  5. package/dist/src/app/assets/{sync-DKyeSqmb.js → sync-DXeGLZqh.js} +1 -1
  6. package/dist/src/app/index.html +1 -1
  7. package/dist/src/commands/auth.d.ts.map +1 -1
  8. package/dist/src/commands/auth.js +2 -0
  9. package/dist/src/commands/auth.js.map +1 -1
  10. package/dist/src/commands/cache.d.ts.map +1 -1
  11. package/dist/src/commands/cache.js +1 -0
  12. package/dist/src/commands/cache.js.map +1 -1
  13. package/dist/src/commands/config.d.ts.map +1 -1
  14. package/dist/src/commands/config.js +3 -0
  15. package/dist/src/commands/config.js.map +1 -1
  16. package/dist/src/commands/delete.d.ts.map +1 -1
  17. package/dist/src/commands/delete.js +1 -0
  18. package/dist/src/commands/delete.js.map +1 -1
  19. package/dist/src/commands/eval.d.ts.map +1 -1
  20. package/dist/src/commands/eval.js +2 -0
  21. package/dist/src/commands/eval.js.map +1 -1
  22. package/dist/src/commands/export.d.ts.map +1 -1
  23. package/dist/src/commands/export.js +1 -0
  24. package/dist/src/commands/export.js.map +1 -1
  25. package/dist/src/commands/generate/dataset.d.ts.map +1 -1
  26. package/dist/src/commands/generate/dataset.js +2 -0
  27. package/dist/src/commands/generate/dataset.js.map +1 -1
  28. package/dist/src/commands/import.d.ts.map +1 -1
  29. package/dist/src/commands/import.js +1 -0
  30. package/dist/src/commands/import.js.map +1 -1
  31. package/dist/src/commands/init.d.ts.map +1 -1
  32. package/dist/src/commands/init.js +1 -0
  33. package/dist/src/commands/init.js.map +1 -1
  34. package/dist/src/commands/list.d.ts.map +1 -1
  35. package/dist/src/commands/list.js +3 -0
  36. package/dist/src/commands/list.js.map +1 -1
  37. package/dist/src/commands/share.d.ts.map +1 -1
  38. package/dist/src/commands/share.js +3 -2
  39. package/dist/src/commands/share.js.map +1 -1
  40. package/dist/src/commands/show.d.ts.map +1 -1
  41. package/dist/src/commands/show.js +4 -0
  42. package/dist/src/commands/show.js.map +1 -1
  43. package/dist/src/commands/view.d.ts.map +1 -1
  44. package/dist/src/commands/view.js +1 -0
  45. package/dist/src/commands/view.js.map +1 -1
  46. package/dist/src/configTypes.d.ts +0 -1
  47. package/dist/src/configTypes.d.ts.map +1 -1
  48. package/dist/src/evaluator.js +1 -1
  49. package/dist/src/evaluator.js.map +1 -1
  50. package/dist/src/evaluatorHelpers.js +3 -3
  51. package/dist/src/evaluatorHelpers.js.map +1 -1
  52. package/dist/src/globalConfig/accounts.d.ts.map +1 -1
  53. package/dist/src/globalConfig/accounts.js +0 -1
  54. package/dist/src/globalConfig/accounts.js.map +1 -1
  55. package/dist/src/globalConfig/globalConfig.d.ts +1 -1
  56. package/dist/src/globalConfig/globalConfig.d.ts.map +1 -1
  57. package/dist/src/globalConfig/globalConfig.js +9 -17
  58. package/dist/src/globalConfig/globalConfig.js.map +1 -1
  59. package/dist/src/index.d.ts.map +1 -1
  60. package/dist/src/index.js +2 -0
  61. package/dist/src/index.js.map +1 -1
  62. package/dist/src/main.js +1 -1
  63. package/dist/src/main.js.map +1 -1
  64. package/dist/src/onboarding.js +1 -1
  65. package/dist/src/onboarding.js.map +1 -1
  66. package/dist/src/providers/bam.d.ts +22 -16
  67. package/dist/src/providers/bam.d.ts.map +1 -1
  68. package/dist/src/providers/bedrock.js +1 -1
  69. package/dist/src/providers/bedrock.js.map +1 -1
  70. package/dist/src/providers/bedrockKnowledgeBase.js +1 -1
  71. package/dist/src/providers/bedrockKnowledgeBase.js.map +1 -1
  72. package/dist/src/providers/http.d.ts.map +1 -1
  73. package/dist/src/providers/http.js +17 -5
  74. package/dist/src/providers/http.js.map +1 -1
  75. package/dist/src/providers/sagemaker.js +1 -1
  76. package/dist/src/providers/sagemaker.js.map +1 -1
  77. package/dist/src/redteam/commands/discover.d.ts +117 -79
  78. package/dist/src/redteam/commands/discover.d.ts.map +1 -1
  79. package/dist/src/redteam/commands/discover.js +139 -179
  80. package/dist/src/redteam/commands/discover.js.map +1 -1
  81. package/dist/src/redteam/commands/generate.d.ts.map +1 -1
  82. package/dist/src/redteam/commands/generate.js +10 -11
  83. package/dist/src/redteam/commands/generate.js.map +1 -1
  84. package/dist/src/redteam/commands/init.js +1 -1
  85. package/dist/src/redteam/commands/init.js.map +1 -1
  86. package/dist/src/redteam/commands/poison.d.ts.map +1 -1
  87. package/dist/src/redteam/commands/poison.js +1 -0
  88. package/dist/src/redteam/commands/poison.js.map +1 -1
  89. package/dist/src/redteam/commands/report.d.ts.map +1 -1
  90. package/dist/src/redteam/commands/report.js +1 -0
  91. package/dist/src/redteam/commands/report.js.map +1 -1
  92. package/dist/src/redteam/commands/run.d.ts.map +1 -1
  93. package/dist/src/redteam/commands/run.js +1 -0
  94. package/dist/src/redteam/commands/run.js.map +1 -1
  95. package/dist/src/redteam/commands/setup.d.ts.map +1 -1
  96. package/dist/src/redteam/commands/setup.js +1 -0
  97. package/dist/src/redteam/commands/setup.js.map +1 -1
  98. package/dist/src/redteam/constants.d.ts +1 -1
  99. package/dist/src/redteam/constants.d.ts.map +1 -1
  100. package/dist/src/redteam/constants.js +1 -1
  101. package/dist/src/redteam/constants.js.map +1 -1
  102. package/dist/src/redteam/index.d.ts.map +1 -1
  103. package/dist/src/redteam/index.js +11 -9
  104. package/dist/src/redteam/index.js.map +1 -1
  105. package/dist/src/redteam/plugins/index.js +1 -1
  106. package/dist/src/redteam/plugins/index.js.map +1 -1
  107. package/dist/src/redteam/plugins/intent.d.ts +1 -1
  108. package/dist/src/redteam/plugins/intent.d.ts.map +1 -1
  109. package/dist/src/redteam/plugins/intent.js +44 -22
  110. package/dist/src/redteam/plugins/intent.js.map +1 -1
  111. package/dist/src/redteam/providers/crescendo/index.d.ts.map +1 -1
  112. package/dist/src/redteam/providers/crescendo/index.js +7 -2
  113. package/dist/src/redteam/providers/crescendo/index.js.map +1 -1
  114. package/dist/src/redteam/providers/goat.js +1 -1
  115. package/dist/src/redteam/providers/goat.js.map +1 -1
  116. package/dist/src/redteam/providers/iterative.d.ts.map +1 -1
  117. package/dist/src/redteam/providers/iterative.js +1 -1
  118. package/dist/src/redteam/providers/iterative.js.map +1 -1
  119. package/dist/src/redteam/providers/iterativeTree.js +1 -1
  120. package/dist/src/redteam/providers/iterativeTree.js.map +1 -1
  121. package/dist/src/server/routes/user.js +1 -1
  122. package/dist/src/server/routes/user.js.map +1 -1
  123. package/dist/src/server/server.js +1 -1
  124. package/dist/src/server/server.js.map +1 -1
  125. package/dist/src/telemetry.d.ts +12 -6
  126. package/dist/src/telemetry.d.ts.map +1 -1
  127. package/dist/src/telemetry.js +75 -78
  128. package/dist/src/telemetry.js.map +1 -1
  129. package/dist/src/util/config/load.js +4 -4
  130. package/dist/src/util/config/load.js.map +1 -1
  131. package/dist/src/util/testCaseReader.js +9 -9
  132. package/dist/src/util/testCaseReader.js.map +1 -1
  133. package/dist/tsconfig.tsbuildinfo +1 -1
  134. package/package.json +28 -30
@@ -1,97 +1,135 @@
1
1
  import { type Command } from 'commander';
2
2
  import { z } from 'zod';
3
- import type { ApiProvider } from '../../types';
4
- export declare const ArgsSchema: z.ZodEffects<z.ZodEffects<z.ZodEffects<z.ZodEffects<z.ZodEffects<z.ZodObject<{
5
- config: z.ZodOptional<z.ZodString>;
6
- output: z.ZodOptional<z.ZodString>;
7
- target: z.ZodOptional<z.ZodString>;
8
- preview: z.ZodBoolean;
9
- turns: z.ZodOptional<z.ZodNumber>;
10
- overwrite: z.ZodBoolean;
3
+ import type { ApiProvider, Prompt, UnifiedConfig } from '../../types';
4
+ export declare const TargetPurposeDiscoveryStateSchema: z.ZodObject<{
5
+ currentQuestionIndex: z.ZodNumber;
6
+ answers: z.ZodArray<z.ZodString, "many">;
11
7
  }, "strip", z.ZodTypeAny, {
12
- preview: boolean;
13
- overwrite: boolean;
14
- config?: string | undefined;
15
- output?: string | undefined;
16
- target?: string | undefined;
17
- turns?: number | undefined;
8
+ currentQuestionIndex: number;
9
+ answers: string[];
18
10
  }, {
19
- preview: boolean;
20
- overwrite: boolean;
21
- config?: string | undefined;
22
- output?: string | undefined;
23
- target?: string | undefined;
24
- turns?: number | undefined;
25
- }>, {
26
- preview: boolean;
27
- overwrite: boolean;
28
- config?: string | undefined;
29
- output?: string | undefined;
30
- target?: string | undefined;
31
- turns?: number | undefined;
11
+ currentQuestionIndex: number;
12
+ answers: string[];
13
+ }>;
14
+ export declare const TargetPurposeDiscoveryRequestSchema: z.ZodObject<{
15
+ state: z.ZodObject<{
16
+ currentQuestionIndex: z.ZodNumber;
17
+ answers: z.ZodArray<z.ZodString, "many">;
18
+ }, "strip", z.ZodTypeAny, {
19
+ currentQuestionIndex: number;
20
+ answers: string[];
21
+ }, {
22
+ currentQuestionIndex: number;
23
+ answers: string[];
24
+ }>;
25
+ task: z.ZodLiteral<"target-purpose-discovery">;
26
+ version: z.ZodString;
27
+ email: z.ZodNullable<z.ZodOptional<z.ZodString>>;
28
+ }, "strip", z.ZodTypeAny, {
29
+ version: string;
30
+ state: {
31
+ currentQuestionIndex: number;
32
+ answers: string[];
33
+ };
34
+ task: "target-purpose-discovery";
35
+ email?: string | null | undefined;
32
36
  }, {
33
- preview: boolean;
34
- overwrite: boolean;
35
- config?: string | undefined;
36
- output?: string | undefined;
37
- target?: string | undefined;
38
- turns?: number | undefined;
39
- }>, {
40
- preview: boolean;
41
- overwrite: boolean;
42
- config?: string | undefined;
43
- output?: string | undefined;
44
- target?: string | undefined;
45
- turns?: number | undefined;
37
+ version: string;
38
+ state: {
39
+ currentQuestionIndex: number;
40
+ answers: string[];
41
+ };
42
+ task: "target-purpose-discovery";
43
+ email?: string | null | undefined;
44
+ }>;
45
+ export declare const DiscoveredPurposeSchema: z.ZodObject<{
46
+ purpose: z.ZodNullable<z.ZodString>;
47
+ limitations: z.ZodNullable<z.ZodString>;
48
+ user: z.ZodNullable<z.ZodString>;
49
+ tools: z.ZodNullable<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodAny>, "many">>;
50
+ }, "strip", z.ZodTypeAny, {
51
+ purpose: string | null;
52
+ user: string | null;
53
+ tools: Record<string, any>[] | null;
54
+ limitations: string | null;
46
55
  }, {
47
- preview: boolean;
48
- overwrite: boolean;
49
- config?: string | undefined;
50
- output?: string | undefined;
51
- target?: string | undefined;
52
- turns?: number | undefined;
53
- }>, {
54
- preview: boolean;
55
- overwrite: boolean;
56
- config?: string | undefined;
57
- output?: string | undefined;
58
- target?: string | undefined;
59
- turns?: number | undefined;
56
+ purpose: string | null;
57
+ user: string | null;
58
+ tools: Record<string, any>[] | null;
59
+ limitations: string | null;
60
+ }>;
61
+ export declare const TargetPurposeDiscoveryResponseSchema: z.ZodObject<{
62
+ done: z.ZodBoolean;
63
+ question: z.ZodOptional<z.ZodString>;
64
+ purpose: z.ZodOptional<z.ZodObject<{
65
+ purpose: z.ZodNullable<z.ZodString>;
66
+ limitations: z.ZodNullable<z.ZodString>;
67
+ user: z.ZodNullable<z.ZodString>;
68
+ tools: z.ZodNullable<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodAny>, "many">>;
69
+ }, "strip", z.ZodTypeAny, {
70
+ purpose: string | null;
71
+ user: string | null;
72
+ tools: Record<string, any>[] | null;
73
+ limitations: string | null;
74
+ }, {
75
+ purpose: string | null;
76
+ user: string | null;
77
+ tools: Record<string, any>[] | null;
78
+ limitations: string | null;
79
+ }>>;
80
+ state: z.ZodObject<{
81
+ currentQuestionIndex: z.ZodNumber;
82
+ answers: z.ZodArray<z.ZodString, "many">;
83
+ }, "strip", z.ZodTypeAny, {
84
+ currentQuestionIndex: number;
85
+ answers: string[];
86
+ }, {
87
+ currentQuestionIndex: number;
88
+ answers: string[];
89
+ }>;
90
+ }, "strip", z.ZodTypeAny, {
91
+ done: boolean;
92
+ state: {
93
+ currentQuestionIndex: number;
94
+ answers: string[];
95
+ };
96
+ purpose?: {
97
+ purpose: string | null;
98
+ user: string | null;
99
+ tools: Record<string, any>[] | null;
100
+ limitations: string | null;
101
+ } | undefined;
102
+ question?: string | undefined;
60
103
  }, {
61
- preview: boolean;
62
- overwrite: boolean;
63
- config?: string | undefined;
64
- output?: string | undefined;
65
- target?: string | undefined;
66
- turns?: number | undefined;
67
- }>, {
68
- preview: boolean;
69
- overwrite: boolean;
104
+ done: boolean;
105
+ state: {
106
+ currentQuestionIndex: number;
107
+ answers: string[];
108
+ };
109
+ purpose?: {
110
+ purpose: string | null;
111
+ user: string | null;
112
+ tools: Record<string, any>[] | null;
113
+ limitations: string | null;
114
+ } | undefined;
115
+ question?: string | undefined;
116
+ }>;
117
+ export type DiscoveredPurpose = z.infer<typeof DiscoveredPurposeSchema>;
118
+ export declare const ArgsSchema: z.ZodEffects<z.ZodObject<{
119
+ config: z.ZodOptional<z.ZodString>;
120
+ target: z.ZodOptional<z.ZodString>;
121
+ }, "strip", z.ZodTypeAny, {
70
122
  config?: string | undefined;
71
- output?: string | undefined;
72
123
  target?: string | undefined;
73
- turns?: number | undefined;
74
124
  }, {
75
- preview: boolean;
76
- overwrite: boolean;
77
125
  config?: string | undefined;
78
- output?: string | undefined;
79
126
  target?: string | undefined;
80
- turns?: number | undefined;
81
127
  }>, {
82
- preview: boolean;
83
- overwrite: boolean;
84
128
  config?: string | undefined;
85
- output?: string | undefined;
86
129
  target?: string | undefined;
87
- turns?: number | undefined;
88
130
  }, {
89
- preview: boolean;
90
- overwrite: boolean;
91
131
  config?: string | undefined;
92
- output?: string | undefined;
93
132
  target?: string | undefined;
94
- turns?: number | undefined;
95
133
  }>;
96
134
  export declare const DEFAULT_TURN_COUNT = 5;
97
135
  /**
@@ -102,10 +140,10 @@ export declare const DEFAULT_TURN_COUNT = 5;
102
140
  * @param maxTurns - The maximum number of turns to run the discovery process.
103
141
  * @returns The purpose of the target.
104
142
  */
105
- export declare function doTargetPurposeDiscovery(target: ApiProvider, maxTurns?: number): Promise<string>;
106
- export declare function mergePurposes(humanDefinedPurpose: string, discoveredPurpose: string): string;
143
+ export declare function doTargetPurposeDiscovery(target: ApiProvider, prompt?: Prompt): Promise<DiscoveredPurpose | undefined>;
144
+ export declare function mergePurposes(humanDefinedPurpose: string | undefined, discoveredPurpose: Record<string, any> | undefined): string;
107
145
  /**
108
146
  * Registers the `discover` command with the CLI.
109
147
  */
110
- export declare function discoverCommand(program: Command): void;
148
+ export declare function discoverCommand(program: Command, defaultConfig: Partial<UnifiedConfig>, defaultConfigPath: string | undefined): void;
111
149
  //# sourceMappingURL=discover.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../../../src/redteam/commands/discover.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,OAAO,EAAU,MAAM,WAAW,CAAC;AAKjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,KAAK,EAAE,WAAW,EAAiB,MAAM,aAAa,CAAC;AAS9D,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiCnB,CAAC;AAOL,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,WAAW,EACnB,QAAQ,GAAE,MAA2B,GACpC,OAAO,CAAC,MAAM,CAAC,CAkFjB;AAkCD,wBAAgB,aAAa,CAAC,mBAAmB,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,UAEnF;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,QAwL/C"}
1
+ {"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../../../src/redteam/commands/discover.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAOtE,eAAO,MAAM,iCAAiC;;;;;;;;;EAG5C,CAAC;AAEH,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAK9C,CAAC;AACH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;EAKlC,CAAC;AAEH,eAAO,MAAM,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAK/C,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;EASnB,CAAC;AAOL,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,WAAW,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CA2GxC;AAED,wBAAgB,aAAa,CAC3B,mBAAmB,EAAE,MAAM,GAAG,SAAS,EACvC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,UAqBnD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EACrC,iBAAiB,EAAE,MAAM,GAAG,SAAS,QAqItC"}