promptfoo 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/README.md +23 -4
  2. package/dist/evaluator.d.ts.map +1 -1
  3. package/dist/evaluator.js +1 -0
  4. package/dist/evaluator.js.map +1 -1
  5. package/dist/main.js +139 -72
  6. package/dist/main.js.map +1 -1
  7. package/dist/providers.d.ts +1 -1
  8. package/dist/providers.d.ts.map +1 -1
  9. package/dist/providers.js +5 -4
  10. package/dist/providers.js.map +1 -1
  11. package/dist/tableOutput.html +2 -2
  12. package/dist/types.d.ts +2 -2
  13. package/dist/types.d.ts.map +1 -1
  14. package/package.json +1 -1
  15. package/src/evaluator.ts +1 -0
  16. package/src/main.ts +183 -87
  17. package/src/providers.ts +5 -12
  18. package/src/types.ts +2 -2
package/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # promptfoo
2
2
 
3
+ [![npm](https://img.shields.io/npm/v/promptfoo)](https://npmjs.com/package/promptfoo)
4
+ ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/typpo/promptfoo/main.yml)
5
+
3
6
  `promptfoo` is a library and command-line tool that helps you evaluate LLM prompt & model quality with a systematic approach to comparing model outputs.
4
7
 
5
8
  With promptfoo, you can:
@@ -12,15 +15,29 @@ With promptfoo, you can:
12
15
 
13
16
  ## Usage (command line)
14
17
 
15
- To evaluate prompts using `promptfoo`, use the following command:
18
+ To get started, run the following command:
19
+
20
+ ```
21
+ npx promptfoo init
22
+ ```
23
+
24
+ This will create some templates in your current directory: `prompts.txt`, `vars.csv`, and `promptfooconfig.js`.
25
+
26
+ After editing the prompts and variables to your liking, run the eval command to kick off an evaluation:
27
+
28
+ ```
29
+ npx promptfoo eval
30
+ ```
31
+
32
+ If you're looking to customize your usage, you have the full set of parameters at your disposal:
16
33
 
17
34
  ```bash
18
- npx promptfoo eval -p <prompt_paths...> -o <output_path> -r <provider> [-v <vars_path>] [-j <max_concurrency] [-c <config_path>]
35
+ npx promptfoo eval -p <prompt_paths...> -o <output_path> -r <providers> [-v <vars_path>] [-j <max_concurrency] [-c <config_path>]
19
36
  ```
20
37
 
21
38
  - `<prompt_paths...>`: Paths to prompt file(s)
22
39
  - `<output_path>`: Path to output CSV, JSON, YAML, or HTML file. Defaults to terminal output
23
- - `<provider>`: One or more of: `openai:<model_name>`, or filesystem path to custom API caller module
40
+ - `<providers>`: One or more of: `openai:<model_name>`, or filesystem path to custom API caller module
24
41
  - `<vars_path>` (optional): Path to CSV, JSON, or YAML file with prompt variables
25
42
  - `<max_concurrency>` (optional): Number of simultaneous API requests. Defaults to 3
26
43
  - `<config_path>` (optional): Path to configuration file
@@ -306,6 +323,8 @@ Below is an example of a custom API provider that returns a predefined output an
306
323
 
307
324
  ```javascript
308
325
  // customApiProvider.js
326
+ import fetch from 'node-fetch';
327
+
309
328
  class CustomApiProvider {
310
329
  id() {
311
330
  return 'my-custom-api';
@@ -328,7 +347,7 @@ class CustomApiProvider {
328
347
  }
329
348
  }
330
349
 
331
- module.exports.default = CustomApiProvider;
350
+ export default CustomApiProvider;
332
351
  ```
333
352
 
334
353
  To use the custom API provider with `promptfoo`, pass the path to the module as the `provider` option in the CLI invocation:
package/dist/evaluator.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAuC,MAAM,YAAY,CAAC;AAiEnG,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAoIjF"}
1
+ {"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAuC,MAAM,YAAY,CAAC;AAiEnG,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAqIjF"}
package/dist/evaluator.js CHANGED
@@ -147,6 +147,7 @@ export async function evaluate(options) {
147
147
  if (progressbar) {
148
148
  progressbar.stop();
149
149
  }
150
+ // TODO(ian): Display errors in table UI.
150
151
  if (isTest) {
151
152
  table.push(...combinedOutputs.map((output, index) => [
152
153
  results[index].success ? 'PASS' : `FAIL: ${results[index].error}`,
package/dist/evaluator.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAahC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,SAAS,kBAAkB,CAAC,QAAgB,EAAE,MAAc;IAC1D,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,YAAY,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,UAAU,QAAQ,EAAE,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;KAC7B;SAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;SAAM;QACL,OAAO,QAAQ,KAAK,MAAM,CAAC;KAC5B;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,EACrB,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;IACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;IAClB,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE3D,4DAA4D;IAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAElF,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE;YACN,GAAG,EAAE,cAAc;YACnB,OAAO,EAAE,aAAa;SACvB;QACD,IAAI;KACL,CAAC;IAEF,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9F,MAAM,GAAG,GAAmB;YAC1B,GAAG,KAAK;YACR,QAAQ;YACR,OAAO;SACR,CAAC;QACF,IAAI,CAAC,OAAO,EAAE;YACZ,GAAG,CAAC,KAAK,GAAG,YAAY,IAAI,CAAC,UAAU,UAAU,QAAQ,CAAC,MAAM,GAAG,CAAC;SACrE;QACD,OAAO,GAAG,CAAC;KACZ;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,GAAG,KAAK;YACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;YAClB,OAAO,EAAE,KAAK;SACf,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB;IACrD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;QAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,aAAa;gBAClB,OAAO,EACL,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa;aACvF,CAAC,CAAC;SACJ;KACF;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC,UAAU,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAClC,MAAM,KAAK,GAAe;QACxB,MAAM;YACJ,CAAC,CAAC;gBACE,QAAQ;gBACR,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;aAClF,CAAC,IAAI,EAAE;YACV,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;KACtF,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,CAAC;SACd;KACF,CAAC;IAEF,IAAI,WAAkC,CAAC;IACvC,IAAI,OAAO,CAAC,eAAe,EAAE;QAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;YACE,MAAM,EACJ,4FAA4F;SAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;YACjC,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,EAAE;YACV,IAAI,EAAE,EAAE;SACT,CAAC,CAAC;KACJ;IAED,MAAM,cAAc,GAAqB,EAAE,CAAC;IAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,cAAc,CAAC,IAAI,CAAC;oBAClB,QAAQ;oBACR,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;iBAChD,CAAC,CAAC;aACJ;SACF;KACF;IAED,MAAM,eAAe,GAAe,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,KAAK,CAAC,QAAQ,EAAE,CAAC;SAClB;aAAM;YACL,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,KAAK,CAAC,SAAS,EAAE,CAAC;aACnB;iBAAM;gBACL,KAAK,CAAC,QAAQ,EAAE,CAAC;aAClB;YACD,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;SAC1E;QAED,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,SAAS,CAAC;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;qBAC5B,IAAI,CAAC,GAAG,CAAC;qBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aAChB,CAAC,CAAC;SACJ;QAED,wBAAwB;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;SAClD;QACD,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/D,eAAe,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC,CACF,CAAC;IAEF,IAAI,WAAW,EAAE;QACf,WAAW,CAAC,IAAI,EAAE,CAAC;KACpB;IAED,IAAI,MAAM,EAAE;QACV,KAAK,CAAC,IAAI,CACR,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE;YACjE,GAAG,MAAM;YACT,GAAG,MAAM,CAAC,MAAM,CAAC,0BAA0B,CAAC,KAAK,CAAC,CAAC;SACpD,CAAC,CACH,CAAC;KACH;SAAM;QACL,KAAK,CAAC,IAAI,CACR,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;KACH;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC"}
1
+ {"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAahC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,SAAS,kBAAkB,CAAC,QAAgB,EAAE,MAAc;IAC1D,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAChC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,YAAY,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,UAAU,QAAQ,EAAE,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;KAC7B;SAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;SAAM;QACL,OAAO,QAAQ,KAAK,MAAM,CAAC;KAC5B;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,EACrB,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;IACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;IAClB,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE3D,4DAA4D;IAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAElF,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE;YACN,GAAG,EAAE,cAAc;YACnB,OAAO,EAAE,aAAa;SACvB;QACD,IAAI;KACL,CAAC;IAEF,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9F,MAAM,GAAG,GAAmB;YAC1B,GAAG,KAAK;YACR,QAAQ;YACR,OAAO;SACR,CAAC;QACF,IAAI,CAAC,OAAO,EAAE;YACZ,GAAG,CAAC,KAAK,GAAG,YAAY,IAAI,CAAC,UAAU,UAAU,QAAQ,CAAC,MAAM,GAAG,CAAC;SACrE;QACD,OAAO,GAAG,CAAC;KACZ;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,GAAG,KAAK;YACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;YAClB,OAAO,EAAE,KAAK;SACf,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB;IACrD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;QAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,aAAa;gBAClB,OAAO,EACL,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa;aACvF,CAAC,CAAC;SACJ;KACF;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC,UAAU,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAClC,MAAM,KAAK,GAAe;QACxB,MAAM;YACJ,CAAC,CAAC;gBACE,QAAQ;gBACR,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;aAClF,CAAC,IAAI,EAAE;YACV,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;KACtF,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,CAAC;SACd;KACF,CAAC;IAEF,IAAI,WAAkC,CAAC;IACvC,IAAI,OAAO,CAAC,eAAe,EAAE;QAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;YACE,MAAM,EACJ,4FAA4F;SAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;YACjC,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,EAAE;YACV,IAAI,EAAE,EAAE;SACT,CAAC,CAAC;KACJ;IAED,MAAM,cAAc,GAAqB,EAAE,CAAC;IAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,cAAc,CAAC,IAAI,CAAC;oBAClB,QAAQ;oBACR,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;iBAChD,CAAC,CAAC;aACJ;SACF;KACF;IAED,MAAM,eAAe,GAAe,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;QACxD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,GAAG,CAAC,KAAK,EAAE;YACb,KAAK,CAAC,QAAQ,EAAE,CAAC;SAClB;aAAM;YACL,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,KAAK,CAAC,SAAS,EAAE,CAAC;aACnB;iBAAM;gBACL,KAAK,CAAC,QAAQ,EAAE,CAAC;aAClB;YACD,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;SAC1E;QAED,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,SAAS,CAAC;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;qBAC5B,IAAI,CAAC,GAAG,CAAC;qBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aAChB,CAAC,CAAC;SACJ;QAED,wBAAwB;QACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;SAClD;QACD,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/D,eAAe,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC,CACF,CAAC;IAEF,IAAI,WAAW,EAAE;QACf,WAAW,CAAC,IAAI,EAAE,CAAC;KACpB;IAED,yCAAyC;IACzC,IAAI,MAAM,EAAE;QACV,KAAK,CAAC,IAAI,CACR,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE;YACjE,GAAG,MAAM;YACT,GAAG,MAAM,CAAC,MAAM,CAAC,0BAA0B,CAAC,KAAK,CAAC,CAAC;SACpD,CAAC,CACH,CAAC;KACH;SAAM;QACL,KAAK,CAAC,IAAI,CACR,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;KACH;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC"}
package/dist/main.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
- import { readFileSync } from 'fs';
3
- import { parse } from 'path';
2
+ import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
3
+ import { parse, join as pathJoin } from 'path';
4
4
  import Table from 'cli-table3';
5
5
  import chalk from 'chalk';
6
6
  import { Command } from 'commander';
@@ -8,83 +8,150 @@ import logger, { setLogLevel } from './logger.js';
8
8
  import { loadApiProvider } from './providers.js';
9
9
  import { evaluate } from './evaluator.js';
10
10
  import { readPrompts, readVars, writeOutput } from './util.js';
11
- const program = new Command();
12
- program
13
- .command('eval')
14
- .description('Evaluate prompts')
15
- .requiredOption('-p, --prompt <paths...>', 'Paths to prompt files (.txt)')
16
- .requiredOption('-r, --provider <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module')
17
- .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)')
18
- .option('-v, --vars <path>', 'Path to file with prompt variables (csv, json, yaml)')
19
- .option('-c, --config <path>', 'Path to configuration file')
20
- .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls')
21
- .option('--verbose', 'Show debug logs')
22
- .action(async (cmdObj) => {
23
- if (cmdObj.verbose) {
24
- setLogLevel('debug');
11
+ function createDummyFiles(directory) {
12
+ if (directory) {
13
+ // Make the directory if it doesn't exist
14
+ if (!existsSync(directory)) {
15
+ mkdirSync(directory);
16
+ }
25
17
  }
26
- const configPath = cmdObj.config;
27
- let config = {};
28
- if (configPath) {
29
- const ext = parse(configPath).ext;
30
- switch (ext) {
31
- case '.json':
32
- const content = readFileSync(configPath, 'utf-8');
33
- config = JSON.parse(content);
34
- break;
35
- case '.js':
36
- config = require(configPath);
37
- break;
38
- default:
39
- throw new Error(`Unsupported configuration file format: ${ext}`);
18
+ const dummyPrompts = `Your first prompt goes here
19
+ ---
20
+ Next prompt goes here. You can substitute variables like this: {{var1}} {{var2}} {{var3}}
21
+ ---
22
+ This is the next prompt.
23
+
24
+ These prompts are nunjucks templates, so you can use logic like this:
25
+ {% if var1 %}
26
+ {{ var1 }}
27
+ {% endif %}`;
28
+ const dummyVars = 'var1,var2,var3\nvalue1,value2,value3\nanother value1,another value2,another value3';
29
+ const dummyConfig = `export default {
30
+ prompts: ['prompts.txt'],
31
+ providers: ['openai:gpt-3.5-turbo'],
32
+ vars: 'vars.csv',
33
+ maxConcurrency: 3,
34
+ };`;
35
+ const readme = `To get started, set your OPENAI_API_KEY environment variable. Then run:
36
+ \`\`\`
37
+ promptfoo eval
38
+ \`\`\`
39
+
40
+ You'll probably want to change a few of the prompts in prompts.txt and the variables in vars.csv before letting it rip.
41
+ `;
42
+ if (directory) {
43
+ if (!existsSync(directory)) {
44
+ logger.info(`Creating directory ${directory} ...`);
45
+ mkdirSync(directory);
40
46
  }
41
47
  }
42
- let vars = [];
43
- if (cmdObj.vars) {
44
- vars = readVars(cmdObj.vars);
48
+ else {
49
+ directory = '.';
45
50
  }
46
- const providers = cmdObj.provider.map((p) => loadApiProvider(p));
47
- const options = {
48
- prompts: readPrompts(cmdObj.prompt),
49
- vars,
50
- providers,
51
- showProgressBar: true,
52
- maxConcurrency: cmdObj.maxConcurrency && cmdObj.maxConcurrency > 0 ? cmdObj.maxConcurrency : undefined,
53
- ...config,
54
- };
55
- const summary = await evaluate(options);
56
- if (cmdObj.output) {
57
- logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
58
- writeOutput(cmdObj.output, summary);
51
+ writeFileSync(pathJoin(process.cwd(), directory, 'prompts.txt'), dummyPrompts);
52
+ writeFileSync(pathJoin(process.cwd(), directory, 'vars.csv'), dummyVars);
53
+ writeFileSync(pathJoin(process.cwd(), directory, 'promptfooconfig.js'), dummyConfig);
54
+ writeFileSync(pathJoin(process.cwd(), directory, 'README.md'), readme);
55
+ if (directory) {
56
+ logger.info(`Wrote prompts.txt, vars.csv, and promptfooconfig.js to ${directory}`);
57
+ logger.info(`\`cd ${directory}\` and open README.md to get started!`);
59
58
  }
60
59
  else {
61
- // Output table by default
62
- const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
63
- const head = summary.table[0];
64
- const table = new Table({
65
- head,
66
- colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
67
- wordWrap: true,
68
- wrapOnWordBoundary: true,
69
- style: {
70
- head: ['blue', 'bold'],
71
- },
72
- });
73
- // Skip first row (header) and add the rest. Color the first column green if it's a success, red if it's a failure.
74
- for (const row of summary.table.slice(1)) {
75
- const color = row[0] === 'PASS' ? 'green' : row[0].startsWith('FAIL') ? 'red' : undefined;
76
- table.push(row.map((col, i) => (i === 0 && color ? chalk[color](col) : col)));
60
+ logger.info('Wrote prompts.txt, vars.csv, and promptfooconfig.js');
61
+ }
62
+ }
63
+ async function main() {
64
+ let defaultConfig = {};
65
+ if (existsSync('promptfooconfig.js')) {
66
+ // @ts-ignore
67
+ defaultConfig = (await import(pathJoin(process.cwd(), './promptfooconfig.js'))).default;
68
+ logger.info('Loaded default config from promptfooconfig.js');
69
+ }
70
+ const program = new Command();
71
+ program
72
+ .command('init [directory]')
73
+ .description('Initialize project with dummy files')
74
+ .action((directory) => {
75
+ createDummyFiles(directory);
76
+ });
77
+ program
78
+ .command('eval')
79
+ .description('Evaluate prompts')
80
+ .requiredOption('-p, --prompts <paths...>', 'Paths to prompt files (.txt)', defaultConfig.prompts)
81
+ .requiredOption('-r, --providers <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module', defaultConfig.providers)
82
+ .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)', defaultConfig.output)
83
+ .option('-v, --vars <path>', 'Path to file with prompt variables (csv, json, yaml)', defaultConfig.vars)
84
+ .option('-c, --config <path>', 'Path to configuration file. Automatically loads promptfooconfig.js', defaultConfig.config)
85
+ .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', String(defaultConfig.maxConcurrency))
86
+ .option('--verbose', 'Show debug logs', defaultConfig.verbose)
87
+ .action(async (cmdObj) => {
88
+ if (cmdObj.verbose) {
89
+ setLogLevel('debug');
90
+ }
91
+ const configPath = cmdObj.config;
92
+ let config = {};
93
+ if (configPath) {
94
+ const ext = parse(configPath).ext;
95
+ switch (ext) {
96
+ case '.json':
97
+ const content = readFileSync(configPath, 'utf-8');
98
+ config = JSON.parse(content);
99
+ break;
100
+ case '.js':
101
+ config = require(configPath);
102
+ break;
103
+ default:
104
+ throw new Error(`Unsupported configuration file format: ${ext}`);
105
+ }
106
+ }
107
+ let vars = [];
108
+ if (cmdObj.vars) {
109
+ vars = readVars(cmdObj.vars);
110
+ }
111
+ const providers = await Promise.all(cmdObj.providers.map(async (p) => await loadApiProvider(p)));
112
+ const options = {
113
+ prompts: readPrompts(cmdObj.prompts),
114
+ vars,
115
+ providers,
116
+ showProgressBar: true,
117
+ maxConcurrency: cmdObj.maxConcurrency && cmdObj.maxConcurrency > 0 ? cmdObj.maxConcurrency : undefined,
118
+ ...config,
119
+ };
120
+ const summary = await evaluate(options);
121
+ if (cmdObj.output) {
122
+ logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
123
+ writeOutput(cmdObj.output, summary);
124
+ }
125
+ else {
126
+ // Output table by default
127
+ const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
128
+ const head = summary.table[0];
129
+ const table = new Table({
130
+ head,
131
+ colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
132
+ wordWrap: true,
133
+ wrapOnWordBoundary: true,
134
+ style: {
135
+ head: ['blue', 'bold'],
136
+ },
137
+ });
138
+ // Skip first row (header) and add the rest. Color the first column green if it's a success, red if it's a failure.
139
+ for (const row of summary.table.slice(1)) {
140
+ const color = row[0] === 'PASS' ? 'green' : row[0].startsWith('FAIL') ? 'red' : undefined;
141
+ table.push(row.map((col, i) => (i === 0 && color ? chalk[color](col) : col)));
142
+ }
143
+ logger.info('\n' + table.toString());
77
144
  }
78
- logger.info('\n' + table.toString());
145
+ logger.info('Evaluation complete');
146
+ logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
147
+ logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
148
+ logger.info(`Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`);
149
+ logger.info('Done.');
150
+ });
151
+ program.parse(process.argv);
152
+ if (!process.argv.slice(2).length) {
153
+ program.outputHelp();
79
154
  }
80
- logger.info('Evaluation complete');
81
- logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
82
- logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
83
- logger.info(`Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`);
84
- logger.info('Done.');
85
- });
86
- program.parse(process.argv);
87
- if (!process.argv.slice(2).length) {
88
- program.outputHelp();
89
155
  }
156
+ main();
90
157
  //# sourceMappingURL=main.js.map
package/dist/main.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE7B,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI/D,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,kBAAkB,CAAC;KAC/B,cAAc,CAAC,yBAAyB,EAAE,8BAA8B,CAAC;KACzE,cAAc,CACb,kCAAkC,EAClC,kGAAkG,CACnG;KACA,MAAM,CAAC,qBAAqB,EAAE,6CAA6C,CAAC;KAC5E,MAAM,CAAC,mBAAmB,EAAE,sDAAsD,CAAC;KACnF,MAAM,CAAC,qBAAqB,EAAE,4BAA4B,CAAC;KAC3D,MAAM,CAAC,gCAAgC,EAAE,wCAAwC,CAAC;KAClF,MAAM,CAAC,WAAW,EAAE,iBAAiB,CAAC;KACtC,MAAM,CAAC,KAAK,EAAE,MAAoC,EAAE,EAAE;IACrD,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,WAAW,CAAC,OAAO,CAAC,CAAC;KACtB;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,UAAU,EAAE;QACd,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;QAClC,QAAQ,GAAG,EAAE;YACX,KAAK,OAAO;gBACV,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAClD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM;YACR,KAAK,KAAK;gBACR,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7B,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;SACpE;KACF;IAED,IAAI,IAAI,GAAiB,EAAE,CAAC;IAC5B,IAAI,MAAM,CAAC,IAAI,EAAE;QACf,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAoB;QAC/B,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;QACnC,IAAI;QACJ,SAAS;QACT,eAAe,EAAE,IAAI;QACrB,cAAc,EACZ,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;QACxF,GAAG,MAAM;KACV,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,MAAM,CAAC,MAAM,EAAE;QACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACrC;SAAM;QACL,0BAA0B;QAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACtB,IAAI;YACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YACtE,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,IAAI;YACxB,KAAK,EAAE;gBACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;aACvB;SACF,CAAC,CAAC;QACH,mHAAmH;QACnH,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACxC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YAC1F,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC/E;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;KACtC;IACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,WAAW,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,eAAe,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,CACnJ,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;IACjC,OAAO,CAAC,UAAU,EAAE,CAAC;CACtB"}
1
+ {"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,KAAK,EAAE,IAAI,IAAI,QAAQ,EAAE,MAAM,MAAM,CAAC;AAE/C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,MAAM,EAAE,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI/D,SAAS,gBAAgB,CAAC,SAAwB;IAChD,IAAI,SAAS,EAAE;QACb,yCAAyC;QACzC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC1B,SAAS,CAAC,SAAS,CAAC,CAAC;SACtB;KACF;IACD,MAAM,YAAY,GAAG;;;;;;;;;YASX,CAAC;IACX,MAAM,SAAS,GACb,oFAAoF,CAAC;IACvF,MAAM,WAAW,GAAG;;;;;GAKnB,CAAC;IACF,MAAM,MAAM,GAAG;;;;;;CAMhB,CAAC;IAEA,IAAI,SAAS,EAAE;QACb,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC1B,MAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,MAAM,CAAC,CAAC;YACnD,SAAS,CAAC,SAAS,CAAC,CAAC;SACtB;KACF;SAAM;QACL,SAAS,GAAG,GAAG,CAAC;KACjB;IAED,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,YAAY,CAAC,CAAC;IAC/E,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC;IACzE,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,oBAAoB,CAAC,EAAE,WAAW,CAAC,CAAC;IACrF,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;IAEvE,IAAI,SAAS,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,0DAA0D,SAAS,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,IAAI,CAAC,QAAQ,SAAS,uCAAuC,CAAC,CAAC;KACvE;SAAM;QACL,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;KACpE;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,aAAa,GAAgC,EAAE,CAAC;IACpD,IAAI,UAAU,CAAC,oBAAoB,CAAC,EAAE;QACpC,aAAa;QACb,aAAa,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;KAC9D;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,qCAAqC,CAAC;SAClD,MAAM,CAAC,CAAC,SAAwB,EAAE,EAAE;QACnC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEL,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,kBAAkB,CAAC;SAC/B,cAAc,CACb,0BAA0B,EAC1B,8BAA8B,EAC9B,aAAa,CAAC,OAAO,CACtB;SACA,cAAc,CACb,mCAAmC,EACnC,kGAAkG,EAClG,aAAa,CAAC,SAAS,CACxB;SACA,MAAM,CACL,qBAAqB,EACrB,6CAA6C,EAC7C,aAAa,CAAC,MAAM,CACrB;SACA,MAAM,CACL,mBAAmB,EACnB,sDAAsD,EACtD,aAAa,CAAC,IAAI,CACnB;SACA,MAAM,CACL,qBAAqB,EACrB,oEAAoE,EACpE,aAAa,CAAC,MAAM,CACrB;SACA,MAAM,CACL,gCAAgC,EAChC,wCAAwC,EACxC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CACrC;SACA,MAAM,CAAC,WAAW,EAAE,iBAAiB,EAAE,aAAa,CAAC,OAAO,CAAC;SAC7D,MAAM,CAAC,KAAK,EAAE,MAAoC,EAAE,EAAE;QACrD,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,WAAW,CAAC,OAAO,CAAC,CAAC;SACtB;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,UAAU,EAAE;YACd,MAAM,GAAG,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;YAClC,QAAQ,GAAG,EAAE;gBACX,KAAK,OAAO;oBACV,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;oBAClD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,MAAM;gBACR,KAAK,KAAK;oBACR,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;oBAC7B,MAAM;gBACR;oBACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;aACpE;SACF;QAED,IAAI,IAAI,GAAiB,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,IAAI,EAAE;YACf,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SAC9B;QAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,eAAe,CAAC,CAAC,CAAC,CAAC,CAC5D,CAAC;QACF,MAAM,OAAO,GAAoB;YAC/B,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;YACpC,IAAI;YACJ,SAAS;YACT,eAAe,EAAE,IAAI;YACrB,cAAc,EACZ,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YACxF,GAAG,MAAM;SACV,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QAExC,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChE,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC;aAAM;YACL,0BAA0B;YAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;gBACtB,IAAI;gBACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtE,QAAQ,EAAE,IAAI;gBACd,kBAAkB,EAAE,IAAI;gBACxB,KAAK,EAAE;oBACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;iBACvB;aACF,CAAC,CAAC;YACH,mHAAmH;YACnH,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;gBACxC,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC1F,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aAC/E;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;SACtC;QACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,WAAW,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,eAAe,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,EAAE,CACnJ,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEL,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;QACjC,OAAO,CAAC,UAAU,EAAE,CAAC;KACtB;AACH,CAAC;AAED,IAAI,EAAE,CAAC"}
package/dist/providers.d.ts CHANGED
@@ -17,5 +17,5 @@ export declare class OpenAiChatCompletionProvider extends OpenAiGenericProvider
17
17
  constructor(modelName: string, apiKey?: string);
18
18
  callApi(prompt: string): Promise<ProviderResponse>;
19
19
  }
20
- export declare function loadApiProvider(providerPath: string): ApiProvider;
20
+ export declare function loadApiProvider(providerPath: string): Promise<ApiProvider>;
21
21
  //# sourceMappingURL=providers.d.ts.map
package/dist/providers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG3D,qBAAa,qBAAsB,YAAW,WAAW;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;gBAEH,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAY9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAWxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA4BzD;AAED,qBAAa,4BAA6B,SAAQ,qBAAqB;IACrE,MAAM,CAAC,kBAAkB,WAOvB;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAWxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoCzD;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,CAyBjE"}
1
+ {"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG3D,qBAAa,qBAAsB,YAAW,WAAW;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;gBAEH,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAY9C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAKZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAGzD;AAED,qBAAa,wBAAyB,SAAQ,qBAAqB;IACjE,MAAM,CAAC,wBAAwB,WAM7B;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA4BzD;AAED,qBAAa,4BAA6B,SAAQ,qBAAqB;IACrE,MAAM,CAAC,kBAAkB,WAOvB;gBAEU,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;IAOxC,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoCzD;AAED,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAyBhF"}
package/dist/providers.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import fetch from 'node-fetch';
2
+ import path from 'node:path';
2
3
  import logger from './logger.js';
3
4
  export class OpenAiGenericProvider {
4
5
  constructor(modelName, apiKey) {
@@ -23,7 +24,7 @@ export class OpenAiGenericProvider {
23
24
  class OpenAiCompletionProvider extends OpenAiGenericProvider {
24
25
  constructor(modelName, apiKey) {
25
26
  if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
26
- throw new Error(`Unknown OpenAI completion model name: ${modelName}. Use one of the following: ${OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.join(', ')}`);
27
+ logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
27
28
  }
28
29
  super(modelName, apiKey);
29
30
  }
@@ -66,7 +67,7 @@ export { OpenAiCompletionProvider };
66
67
  class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
67
68
  constructor(modelName, apiKey) {
68
69
  if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
69
- throw new Error(`Unknown OpenAI completion model name: ${modelName}. Use one of the following: ${OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.join(', ')}`);
70
+ logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
70
71
  }
71
72
  super(modelName, apiKey);
72
73
  }
@@ -116,7 +117,7 @@ OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS = [
116
117
  'gpt-3.5-turbo-0301',
117
118
  ];
118
119
  export { OpenAiChatCompletionProvider };
119
- export function loadApiProvider(providerPath) {
120
+ export async function loadApiProvider(providerPath) {
120
121
  if (providerPath?.startsWith('openai:')) {
121
122
  // Load OpenAI module
122
123
  const options = providerPath.split(':');
@@ -139,7 +140,7 @@ export function loadApiProvider(providerPath) {
139
140
  }
140
141
  }
141
142
  // Load custom module
142
- const CustomApiProvider = require(providerPath).default;
143
+ const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
143
144
  return new CustomApiProvider();
144
145
  }
145
146
  //# sourceMappingURL=providers.js.map
package/dist/providers.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAG/B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,OAAO,qBAAqB;IAIhC,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjD,IAAI,CAAC,GAAG,EAAE;YACR,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,+BAA+B,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACrI,IAAI,CACL,EAAE,CACJ,CAAC;SACH;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC;SACjD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5B,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;gBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;aACzC;SACF,CAAC;IACJ,CAAC;;AA9CM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;SAPS,wBAAwB;AAkDrC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,MAAM,IAAI,KAAK,CACb,yCAAyC,SAAS,+BAA+B,wBAAwB,CAAC,wBAAwB,CAAC,IAAI,CACrI,IAAI,CACL,EAAE,CACJ,CAAC;SACH;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,CAAC,EAAE;YACV,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC;SACrD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YACzE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;YACvC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;gBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;aACzC;SACF,CAAC;IACJ,CAAC;;AAvDM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;SARS,4BAA4B;AA2DzC,MAAM,UAAU,eAAe,CAAC,YAAoB;IAClD,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QACvC,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,4BAA4B,CAAC,SAAS,IAAI,eAAe,CAAC,CAAC;SACvE;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,wBAAwB,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;SACtE;aAAM,IAAI,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,4BAA4B,CAAC,SAAS,CAAC,CAAC;SACpD;aAAM,IAAI,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,wBAAwB,CAAC,SAAS,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;IACxD,OAAO,IAAI,iBAAiB,EAAE,CAAC;AACjC,CAAC"}
1
+ {"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,OAAO,qBAAqB;IAIhC,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACjD,IAAI,CAAC,GAAG,EAAE;YACR,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;SACH;QACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IACpB,CAAC;IAED,EAAE;QACA,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,QAAQ;QACN,OAAO,oBAAoB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC/C,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AAED,MAAa,wBAAyB,SAAQ,qBAAqB;IASjE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1E,MAAM,CAAC,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;SACpE;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,MAAM;YACN,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC;SACjD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5B,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;gBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;aACzC;SACF,CAAC;IACJ,CAAC;;AA1CM,iDAAwB,GAAG;IAChC,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;IAClB,cAAc;CACf,CAAC;SAPS,wBAAwB;AA8CrC,MAAa,4BAA6B,SAAQ,qBAAqB;IAUrE,YAAY,SAAiB,EAAE,MAAe;QAC5C,IAAI,CAAC,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACxE,MAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;SAC9D;QACD,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,IAAI,QAA6C,CAAC;QAClD,IAAI;YACF,qEAAqE;YACrE,yCAAyC;YACzC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SAC/B;QAAC,OAAO,CAAC,EAAE;YACV,QAAQ,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;SAChD;QACD,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAI,CAAC,SAAS;YACrB,QAAQ,EAAE,QAAQ;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC;SACrD,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YACzE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;YACvC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;gBAC9B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;aACzC;SACF,CAAC;IACJ,CAAC;;AAnDM,+CAAkB,GAAG;IAC1B,OAAO;IACP,YAAY;IACZ,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,oBAAoB;CACrB,CAAC;SARS,4BAA4B;AAuDzC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,YAAoB;IACxD,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QACvC,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,4BAA4B,CAAC,SAAS,IAAI,eAAe,CAAC,CAAC;SACvE;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,wBAAwB,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;SACtE;aAAM,IAAI,4BAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,4BAA4B,CAAC,SAAS,CAAC,CAAC;SACpD;aAAM,IAAI,wBAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,wBAAwB,CAAC,SAAS,CAAC,CAAC;SAChD;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,EAAE,CAAC;AACjC,CAAC"}
package/dist/tableOutput.html CHANGED
@@ -22,13 +22,13 @@
22
22
  padding: 5px;
23
23
  }
24
24
  /* If data-content is exactly "PASS", set font color to green */
25
- tr > td[data-content="PASS"]:first-child {
25
+ tr > td[data-content='PASS']:first-child {
26
26
  color: green;
27
27
  font-weight: bold;
28
28
  }
29
29
 
30
30
  /* If data-content starts with "FAIL", set font color to red */
31
- tr > td[data-content^="FAIL"]:first-child {
31
+ tr > td[data-content^='FAIL']:first-child {
32
32
  color: red;
33
33
  font-weight: bold;
34
34
  }
package/dist/types.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  export interface CommandLineOptions {
2
- prompt: string[];
3
- provider: string[];
2
+ prompts: string[];
3
+ providers: string[];
4
4
  output?: string;
5
5
  vars?: string;
6
6
  config?: string;
package/dist/types.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,UAAU,UAAU;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,KAAK,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;CACH"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,UAAU,UAAU;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IAEpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,KAAK,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;CACH"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "promptfoo",
3
3
  "author": "Ian Webster",
4
- "version": "0.1.0",
4
+ "version": "0.2.0",
5
5
  "license": "MIT",
6
6
  "type": "module",
7
7
  "main": "dist/index.js",
package/src/evaluator.ts CHANGED
@@ -185,6 +185,7 @@ export async function evaluate(options: EvaluateOptions): Promise<EvaluateSummar
185
185
  progressbar.stop();
186
186
  }
187
187
 
188
+ // TODO(ian): Display errors in table UI.
188
189
  if (isTest) {
189
190
  table.push(
190
191
  ...combinedOutputs.map((output, index) => [
package/src/main.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
- import { readFileSync } from 'fs';
3
- import { parse } from 'path';
2
+ import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
3
+ import { parse, join as pathJoin } from 'path';
4
4
 
5
5
  import Table from 'cli-table3';
6
6
  import chalk from 'chalk';
@@ -13,96 +13,192 @@ import { readPrompts, readVars, writeOutput } from './util.js';
13
13
 
14
14
  import type { CommandLineOptions, EvaluateOptions, VarMapping } from './types.js';
15
15
 
16
- const program = new Command();
17
-
18
- program
19
- .command('eval')
20
- .description('Evaluate prompts')
21
- .requiredOption('-p, --prompt <paths...>', 'Paths to prompt files (.txt)')
22
- .requiredOption(
23
- '-r, --provider <name or path...>',
24
- 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module',
25
- )
26
- .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)')
27
- .option('-v, --vars <path>', 'Path to file with prompt variables (csv, json, yaml)')
28
- .option('-c, --config <path>', 'Path to configuration file')
29
- .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls')
30
- .option('--verbose', 'Show debug logs')
31
- .action(async (cmdObj: CommandLineOptions & Command) => {
32
- if (cmdObj.verbose) {
33
- setLogLevel('debug');
16
+ function createDummyFiles(directory: string | null) {
17
+ if (directory) {
18
+ // Make the directory if it doesn't exist
19
+ if (!existsSync(directory)) {
20
+ mkdirSync(directory);
34
21
  }
22
+ }
23
+ const dummyPrompts = `Your first prompt goes here
24
+ ---
25
+ Next prompt goes here. You can substitute variables like this: {{var1}} {{var2}} {{var3}}
26
+ ---
27
+ This is the next prompt.
35
28
 
36
- const configPath = cmdObj.config;
37
- let config = {};
38
- if (configPath) {
39
- const ext = parse(configPath).ext;
40
- switch (ext) {
41
- case '.json':
42
- const content = readFileSync(configPath, 'utf-8');
43
- config = JSON.parse(content);
44
- break;
45
- case '.js':
46
- config = require(configPath);
47
- break;
48
- default:
49
- throw new Error(`Unsupported configuration file format: ${ext}`);
50
- }
51
- }
29
+ These prompts are nunjucks templates, so you can use logic like this:
30
+ {% if var1 %}
31
+ {{ var1 }}
32
+ {% endif %}`;
33
+ const dummyVars =
34
+ 'var1,var2,var3\nvalue1,value2,value3\nanother value1,another value2,another value3';
35
+ const dummyConfig = `export default {
36
+ prompts: ['prompts.txt'],
37
+ providers: ['openai:gpt-3.5-turbo'],
38
+ vars: 'vars.csv',
39
+ maxConcurrency: 3,
40
+ };`;
41
+ const readme = `To get started, set your OPENAI_API_KEY environment variable. Then run:
42
+ \`\`\`
43
+ promptfoo eval
44
+ \`\`\`
45
+
46
+ You'll probably want to change a few of the prompts in prompts.txt and the variables in vars.csv before letting it rip.
47
+ `;
52
48
 
53
- let vars: VarMapping[] = [];
54
- if (cmdObj.vars) {
55
- vars = readVars(cmdObj.vars);
49
+ if (directory) {
50
+ if (!existsSync(directory)) {
51
+ logger.info(`Creating directory ${directory} ...`);
52
+ mkdirSync(directory);
56
53
  }
54
+ } else {
55
+ directory = '.';
56
+ }
57
+
58
+ writeFileSync(pathJoin(process.cwd(), directory, 'prompts.txt'), dummyPrompts);
59
+ writeFileSync(pathJoin(process.cwd(), directory, 'vars.csv'), dummyVars);
60
+ writeFileSync(pathJoin(process.cwd(), directory, 'promptfooconfig.js'), dummyConfig);
61
+ writeFileSync(pathJoin(process.cwd(), directory, 'README.md'), readme);
62
+
63
+ if (directory) {
64
+ logger.info(`Wrote prompts.txt, vars.csv, and promptfooconfig.js to ${directory}`);
65
+ logger.info(`\`cd ${directory}\` and open README.md to get started!`);
66
+ } else {
67
+ logger.info('Wrote prompts.txt, vars.csv, and promptfooconfig.js');
68
+ }
69
+ }
57
70
 
58
- const providers = cmdObj.provider.map((p) => loadApiProvider(p));
59
- const options: EvaluateOptions = {
60
- prompts: readPrompts(cmdObj.prompt),
61
- vars,
62
- providers,
63
- showProgressBar: true,
64
- maxConcurrency:
65
- cmdObj.maxConcurrency && cmdObj.maxConcurrency > 0 ? cmdObj.maxConcurrency : undefined,
66
- ...config,
67
- };
68
-
69
- const summary = await evaluate(options);
70
-
71
- if (cmdObj.output) {
72
- logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
73
- writeOutput(cmdObj.output, summary);
74
- } else {
75
- // Output table by default
76
- const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
77
- const head = summary.table[0];
78
- const table = new Table({
79
- head,
80
- colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
81
- wordWrap: true,
82
- wrapOnWordBoundary: true,
83
- style: {
84
- head: ['blue', 'bold'],
85
- },
86
- });
87
- // Skip first row (header) and add the rest. Color the first column green if it's a success, red if it's a failure.
88
- for (const row of summary.table.slice(1)) {
89
- const color = row[0] === 'PASS' ? 'green' : row[0].startsWith('FAIL') ? 'red' : undefined;
90
- table.push(row.map((col, i) => (i === 0 && color ? chalk[color](col) : col)));
71
+ async function main() {
72
+ let defaultConfig: Partial<CommandLineOptions> = {};
73
+ if (existsSync('promptfooconfig.js')) {
74
+ // @ts-ignore
75
+ defaultConfig = (await import(pathJoin(process.cwd(), './promptfooconfig.js'))).default;
76
+ logger.info('Loaded default config from promptfooconfig.js');
77
+ }
78
+
79
+ const program = new Command();
80
+
81
+ program
82
+ .command('init [directory]')
83
+ .description('Initialize project with dummy files')
84
+ .action((directory: string | null) => {
85
+ createDummyFiles(directory);
86
+ });
87
+
88
+ program
89
+ .command('eval')
90
+ .description('Evaluate prompts')
91
+ .requiredOption(
92
+ '-p, --prompts <paths...>',
93
+ 'Paths to prompt files (.txt)',
94
+ defaultConfig.prompts,
95
+ )
96
+ .requiredOption(
97
+ '-r, --providers <name or path...>',
98
+ 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module',
99
+ defaultConfig.providers,
100
+ )
101
+ .option(
102
+ '-o, --output <path>',
103
+ 'Path to output file (csv, json, yaml, html)',
104
+ defaultConfig.output,
105
+ )
106
+ .option(
107
+ '-v, --vars <path>',
108
+ 'Path to file with prompt variables (csv, json, yaml)',
109
+ defaultConfig.vars,
110
+ )
111
+ .option(
112
+ '-c, --config <path>',
113
+ 'Path to configuration file. Automatically loads promptfooconfig.js',
114
+ defaultConfig.config,
115
+ )
116
+ .option(
117
+ '-j, --max-concurrency <number>',
118
+ 'Maximum number of concurrent API calls',
119
+ String(defaultConfig.maxConcurrency),
120
+ )
121
+ .option('--verbose', 'Show debug logs', defaultConfig.verbose)
122
+ .action(async (cmdObj: CommandLineOptions & Command) => {
123
+ if (cmdObj.verbose) {
124
+ setLogLevel('debug');
91
125
  }
92
126
 
93
- logger.info('\n' + table.toString());
94
- }
95
- logger.info('Evaluation complete');
96
- logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
97
- logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
98
- logger.info(
99
- `Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`,
100
- );
101
- logger.info('Done.');
102
- });
103
-
104
- program.parse(process.argv);
105
-
106
- if (!process.argv.slice(2).length) {
107
- program.outputHelp();
127
+ const configPath = cmdObj.config;
128
+ let config = {};
129
+ if (configPath) {
130
+ const ext = parse(configPath).ext;
131
+ switch (ext) {
132
+ case '.json':
133
+ const content = readFileSync(configPath, 'utf-8');
134
+ config = JSON.parse(content);
135
+ break;
136
+ case '.js':
137
+ config = require(configPath);
138
+ break;
139
+ default:
140
+ throw new Error(`Unsupported configuration file format: ${ext}`);
141
+ }
142
+ }
143
+
144
+ let vars: VarMapping[] = [];
145
+ if (cmdObj.vars) {
146
+ vars = readVars(cmdObj.vars);
147
+ }
148
+
149
+ const providers = await Promise.all(
150
+ cmdObj.providers.map(async (p) => await loadApiProvider(p)),
151
+ );
152
+ const options: EvaluateOptions = {
153
+ prompts: readPrompts(cmdObj.prompts),
154
+ vars,
155
+ providers,
156
+ showProgressBar: true,
157
+ maxConcurrency:
158
+ cmdObj.maxConcurrency && cmdObj.maxConcurrency > 0 ? cmdObj.maxConcurrency : undefined,
159
+ ...config,
160
+ };
161
+
162
+ const summary = await evaluate(options);
163
+
164
+ if (cmdObj.output) {
165
+ logger.info(chalk.yellow(`Writing output to ${cmdObj.output}`));
166
+ writeOutput(cmdObj.output, summary);
167
+ } else {
168
+ // Output table by default
169
+ const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
170
+ const head = summary.table[0];
171
+ const table = new Table({
172
+ head,
173
+ colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
174
+ wordWrap: true,
175
+ wrapOnWordBoundary: true,
176
+ style: {
177
+ head: ['blue', 'bold'],
178
+ },
179
+ });
180
+ // Skip first row (header) and add the rest. Color the first column green if it's a success, red if it's a failure.
181
+ for (const row of summary.table.slice(1)) {
182
+ const color = row[0] === 'PASS' ? 'green' : row[0].startsWith('FAIL') ? 'red' : undefined;
183
+ table.push(row.map((col, i) => (i === 0 && color ? chalk[color](col) : col)));
184
+ }
185
+
186
+ logger.info('\n' + table.toString());
187
+ }
188
+ logger.info('Evaluation complete');
189
+ logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
190
+ logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
191
+ logger.info(
192
+ `Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`,
193
+ );
194
+ logger.info('Done.');
195
+ });
196
+
197
+ program.parse(process.argv);
198
+
199
+ if (!process.argv.slice(2).length) {
200
+ program.outputHelp();
201
+ }
108
202
  }
203
+
204
+ main();
package/src/providers.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import fetch from 'node-fetch';
2
+ import path from 'node:path';
2
3
 
3
4
  import { ApiProvider, ProviderResponse } from './types.js';
4
5
  import logger from './logger.js';
@@ -44,11 +45,7 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
44
45
 
45
46
  constructor(modelName: string, apiKey?: string) {
46
47
  if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
47
- throw new Error(
48
- `Unknown OpenAI completion model name: ${modelName}. Use one of the following: ${OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.join(
49
- ', ',
50
- )}`,
51
- );
48
+ logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
52
49
  }
53
50
  super(modelName, apiKey);
54
51
  }
@@ -95,11 +92,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
95
92
 
96
93
  constructor(modelName: string, apiKey?: string) {
97
94
  if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
98
- throw new Error(
99
- `Unknown OpenAI completion model name: ${modelName}. Use one of the following: ${OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.join(
100
- ', ',
101
- )}`,
102
- );
95
+ logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
103
96
  }
104
97
  super(modelName, apiKey);
105
98
  }
@@ -142,7 +135,7 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
142
135
  }
143
136
  }
144
137
 
145
- export function loadApiProvider(providerPath: string): ApiProvider {
138
+ export async function loadApiProvider(providerPath: string): Promise<ApiProvider> {
146
139
  if (providerPath?.startsWith('openai:')) {
147
140
  // Load OpenAI module
148
141
  const options = providerPath.split(':');
@@ -165,6 +158,6 @@ export function loadApiProvider(providerPath: string): ApiProvider {
165
158
  }
166
159
 
167
160
  // Load custom module
168
- const CustomApiProvider = require(providerPath).default;
161
+ const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
169
162
  return new CustomApiProvider();
170
163
  }
package/src/types.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  export interface CommandLineOptions {
2
- prompt: string[];
3
- provider: string[];
2
+ prompts: string[];
3
+ providers: string[];
4
4
  output?: string;
5
5
  vars?: string;
6
6
  config?: string;