promptfoo 0.17.4 → 0.17.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/package.json +3 -2
- package/dist/src/assertions.d.ts.map +1 -1
- package/dist/src/assertions.js +14 -2
- package/dist/src/assertions.js.map +1 -1
- package/dist/src/cache.d.ts +3 -0
- package/dist/src/cache.d.ts.map +1 -1
- package/dist/src/cache.js +6 -1
- package/dist/src/cache.js.map +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +21 -17
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/main.js +66 -36
- package/dist/src/main.js.map +1 -1
- package/dist/src/providers/anthropic.js.map +1 -1
- package/dist/src/providers/azureopenai.d.ts +34 -0
- package/dist/src/providers/azureopenai.d.ts.map +1 -0
- package/dist/src/providers/azureopenai.js +222 -0
- package/dist/src/providers/azureopenai.js.map +1 -0
- package/dist/src/providers/localai.d.ts.map +1 -1
- package/dist/src/providers/localai.js +2 -1
- package/dist/src/providers/localai.js.map +1 -1
- package/dist/src/providers/openai.d.ts.map +1 -1
- package/dist/src/providers/openai.js +1 -13
- package/dist/src/providers/openai.js.map +1 -1
- package/dist/src/providers/replicate.d.ts +11 -0
- package/dist/src/providers/replicate.d.ts.map +1 -0
- package/dist/src/providers/replicate.js +78 -0
- package/dist/src/providers/replicate.js.map +1 -0
- package/dist/src/providers/shared.d.ts +5 -0
- package/dist/src/providers/shared.d.ts.map +1 -1
- package/dist/src/providers/shared.js +33 -1
- package/dist/src/providers/shared.js.map +1 -1
- package/dist/src/providers.d.ts +2 -0
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +24 -0
- package/dist/src/providers.js.map +1 -1
- package/dist/src/types.d.ts +4 -1
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/updates.d.ts.map +1 -1
- package/dist/src/updates.js +3 -0
- package/dist/src/updates.js.map +1 -1
- package/dist/src/web/client/assets/{index-58a0e3e3.js → index-13198388.js} +23 -23
- package/dist/src/web/client/assets/index-f9b230d1.css +1 -0
- package/dist/src/web/client/index.html +2 -2
- package/package.json +3 -2
- package/src/assertions.ts +18 -2
- package/src/cache.ts +5 -1
- package/src/evaluator.ts +23 -17
- package/src/main.ts +87 -38
- package/src/providers/anthropic.ts +1 -1
- package/src/providers/azureopenai.ts +264 -0
- package/src/providers/localai.ts +3 -2
- package/src/providers/openai.ts +3 -18
- package/src/providers/replicate.ts +86 -0
- package/src/providers/shared.ts +29 -0
- package/src/providers.ts +27 -0
- package/src/types.ts +6 -0
- package/src/updates.ts +4 -0
- package/src/web/client/src/App.tsx +6 -0
- package/src/web/client/src/EvalOutputPromptDialog.tsx +6 -2
- package/src/web/client/src/ResultsTable.tsx +5 -0
- package/src/web/client/src/ResultsView.tsx +14 -11
- package/src/web/client/src/index.css +1 -12
- package/src/web/client/src/types.ts +1 -1
- package/dist/src/web/client/assets/index-b82d0138.css +0 -1
package/dist/src/types.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ export interface CommandLineOptions {
|
|
|
3
3
|
providers: string[];
|
|
4
4
|
output: string;
|
|
5
5
|
maxConcurrency: string;
|
|
6
|
+
repeat: string;
|
|
6
7
|
vars?: string;
|
|
7
8
|
tests?: string;
|
|
8
9
|
config?: string;
|
|
@@ -59,6 +60,7 @@ export interface EvaluateOptions {
|
|
|
59
60
|
maxConcurrency?: number;
|
|
60
61
|
showProgressBar?: boolean;
|
|
61
62
|
generateSuggestions?: boolean;
|
|
63
|
+
repeat?: number;
|
|
62
64
|
}
|
|
63
65
|
export interface Prompt {
|
|
64
66
|
raw: string;
|
|
@@ -105,7 +107,7 @@ export interface GradingResult {
|
|
|
105
107
|
reason: string;
|
|
106
108
|
tokensUsed?: TokenUsage;
|
|
107
109
|
}
|
|
108
|
-
type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'similar' | 'llm-rubric' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l';
|
|
110
|
+
type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'starts-with' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'similar' | 'llm-rubric' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l';
|
|
109
111
|
type NotPrefixed<T extends string> = `not-${T}`;
|
|
110
112
|
export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
|
|
111
113
|
export interface Assertion {
|
|
@@ -140,6 +142,7 @@ export interface TestSuiteConfig {
|
|
|
140
142
|
tests: string | TestCase[];
|
|
141
143
|
defaultTest?: Omit<TestCase, 'description'>;
|
|
142
144
|
outputPath?: string;
|
|
145
|
+
sharing?: boolean;
|
|
143
146
|
}
|
|
144
147
|
export type UnifiedConfig = TestSuiteConfig & {
|
|
145
148
|
evaluateOptions: EvaluateOptions;
|
package/dist/src/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IAGf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,CAAC,EAAE,GAAG,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,CAAC;AAEd,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG1B,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB;AAGD,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;IAGlD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,aAAa,CAAC;CACxC;AAGD,MAAM,WAAW,cAAe,SAAQ,QAAQ;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAGlB,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;CACjC;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;AAG/E,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,iBAAiB,EAAE,CAAC;IAG3D,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,QAAQ,EAAE,CAAC;IAG3B,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"updates.d.ts","sourceRoot":"","sources":["../../src/updates.ts"],"names":[],"mappings":"AASA,wBAAsB,gBAAgB,CAAC,WAAW,EAAE,MAAM,gBAOzD;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"updates.d.ts","sourceRoot":"","sources":["../../src/updates.ts"],"names":[],"mappings":"AASA,wBAAsB,gBAAgB,CAAC,WAAW,EAAE,MAAM,gBAOzD;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC,CA2BxD"}
|
package/dist/src/updates.js
CHANGED
|
@@ -20,6 +20,9 @@ async function getLatestVersion(packageName) {
|
|
|
20
20
|
}
|
|
21
21
|
exports.getLatestVersion = getLatestVersion;
|
|
22
22
|
async function checkForUpdates() {
|
|
23
|
+
if (process.env.PROMPTFOO_DISABLE_UPDATE) {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
23
26
|
let latestVersion;
|
|
24
27
|
try {
|
|
25
28
|
latestVersion = await getLatestVersion('promptfoo');
|
package/dist/src/updates.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"updates.js","sourceRoot":"","sources":["../../src/updates.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAC1B,6DAA2C;AAE3C,sDAA8B;AAC9B,iCAA0C;AAC1C,mEAA0C;AAE1C,MAAM,OAAO,GAAG,sBAAW,CAAC,OAAO,CAAC;AAE7B,KAAK,UAAU,gBAAgB,CAAC,WAAmB;IACxD,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAgB,EAAC,8BAA8B,WAAW,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/F,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;QAChB,MAAM,IAAI,KAAK,CAAC,2CAA2C,WAAW,EAAE,CAAC,CAAC;KAC3E;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;AAClC,CAAC;AAPD,4CAOC;AAEM,KAAK,UAAU,eAAe;IACnC,IAAI,aAAqB,CAAC;IAC1B,IAAI;QACF,aAAa,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;KACrD;IAAC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;IACD,IAAI,IAAA,YAAQ,EAAC,aAAa,EAAE,OAAO,CAAC,EAAE;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACvD,gBAAM,CAAC,IAAI,CACT,KAAK,MAAM;EACf,eAAK,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,eAAK,CAAC,MAAM,CAC7D,OAAO,CACR,+CAA+C,eAAK,CAAC,KAAK,CAAC,aAAa,CAAC;;aAEnE,eAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,OAAO,eAAK,CAAC,KAAK,CAC1D,iCAAiC,CAClC;EACL,MAAM,IAAI,CACP,CAAC;QACF,OAAO,IAAI,CAAC;KACb;IACD,OAAO,KAAK,CAAC;AACf,CAAC;
|
|
1
|
+
{"version":3,"file":"updates.js","sourceRoot":"","sources":["../../src/updates.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAC1B,6DAA2C;AAE3C,sDAA8B;AAC9B,iCAA0C;AAC1C,mEAA0C;AAE1C,MAAM,OAAO,GAAG,sBAAW,CAAC,OAAO,CAAC;AAE7B,KAAK,UAAU,gBAAgB,CAAC,WAAmB;IACxD,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAgB,EAAC,8BAA8B,WAAW,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/F,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;QAChB,MAAM,IAAI,KAAK,CAAC,2CAA2C,WAAW,EAAE,CAAC,CAAC;KAC3E;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;AAClC,CAAC;AAPD,4CAOC;AAEM,KAAK,UAAU,eAAe;IACnC,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE;QACxC,OAAO,KAAK,CAAC;KACd;IAED,IAAI,aAAqB,CAAC;IAC1B,IAAI;QACF,aAAa,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;KACrD;IAAC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;IACD,IAAI,IAAA,YAAQ,EAAC,aAAa,EAAE,OAAO,CAAC,EAAE;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACvD,gBAAM,CAAC,IAAI,CACT,KAAK,MAAM;EACf,eAAK,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,eAAK,CAAC,MAAM,CAC7D,OAAO,CACR,+CAA+C,eAAK,CAAC,KAAK,CAAC,aAAa,CAAC;;aAEnE,eAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,OAAO,eAAK,CAAC,KAAK,CAC1D,iCAAiC,CAClC;EACL,MAAM,IAAI,CACP,CAAC;QACF,OAAO,IAAI,CAAC;KACb;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AA3BD,0CA2BC"}
|