vskill 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/find.test.js +48 -0
- package/dist/commands/find.test.js.map +1 -1
- package/dist/eval/__tests__/benchmark.test.js +29 -0
- package/dist/eval/__tests__/benchmark.test.js.map +1 -1
- package/dist/eval/__tests__/comparator.test.js +143 -0
- package/dist/eval/__tests__/comparator.test.js.map +1 -1
- package/dist/eval/__tests__/judge.test.js +75 -1
- package/dist/eval/__tests__/judge.test.js.map +1 -1
- package/dist/eval/__tests__/mcp-detector.test.js +55 -0
- package/dist/eval/__tests__/mcp-detector.test.js.map +1 -1
- package/dist/eval/__tests__/prompt-builder.test.js +18 -0
- package/dist/eval/__tests__/prompt-builder.test.js.map +1 -1
- package/dist/eval/action-items.d.ts +21 -0
- package/dist/eval/action-items.js +97 -0
- package/dist/eval/action-items.js.map +1 -0
- package/dist/eval/benchmark-history.d.ts +4 -4
- package/dist/eval/benchmark-history.js.map +1 -1
- package/dist/eval/benchmark.d.ts +18 -1
- package/dist/eval/benchmark.js.map +1 -1
- package/dist/eval/comparator.d.ts +5 -3
- package/dist/eval/comparator.js +16 -6
- package/dist/eval/comparator.js.map +1 -1
- package/dist/eval/judge.d.ts +3 -1
- package/dist/eval/judge.js +18 -2
- package/dist/eval/judge.js.map +1 -1
- package/dist/eval/llm.js +5 -7
- package/dist/eval/llm.js.map +1 -1
- package/dist/eval/mcp-detector.js +30 -0
- package/dist/eval/mcp-detector.js.map +1 -1
- package/dist/eval/prompt-builder.js +22 -1
- package/dist/eval/prompt-builder.js.map +1 -1
- package/dist/eval/skill-scanner.d.ts +6 -0
- package/dist/eval/skill-scanner.js +55 -9
- package/dist/eval/skill-scanner.js.map +1 -1
- package/dist/eval-server/__tests__/sse-helpers.test.d.ts +1 -0
- package/dist/eval-server/__tests__/sse-helpers.test.js +128 -0
- package/dist/eval-server/__tests__/sse-helpers.test.js.map +1 -0
- package/dist/eval-server/api-routes.js +42 -8
- package/dist/eval-server/api-routes.js.map +1 -1
- package/dist/eval-server/skill-create-routes.js +15 -0
- package/dist/eval-server/skill-create-routes.js.map +1 -1
- package/dist/eval-server/sse-helpers.d.ts +10 -0
- package/dist/eval-server/sse-helpers.js +42 -0
- package/dist/eval-server/sse-helpers.js.map +1 -1
- package/dist/eval-ui/assets/index-DP0_fnIN.js +75 -0
- package/dist/eval-ui/index.html +1 -1
- package/dist/utils/__tests__/resolve-binary.integration.test.d.ts +1 -0
- package/dist/utils/__tests__/resolve-binary.integration.test.js +138 -0
- package/dist/utils/__tests__/resolve-binary.integration.test.js.map +1 -0
- package/dist/utils/resolve-binary.js +27 -8
- package/dist/utils/resolve-binary.js.map +1 -1
- package/package.json +1 -1
- package/dist/eval-ui/assets/index-BDZxYaAi.js +0 -73
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-builder.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/prompt-builder.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,MAAM,YAAY,GAAG,6CAA6C,CAAC;IAEnE,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,oCAAoC;AACpC,8EAA8E;AAE9E,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,qBAAqB,CAAC,mCAAmC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,2DAA2D,CAAC,CAAC;QACtF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,UAAU,GAAG,uEAAuE,CAAC;QAC3F,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,WAAW,GAAG,+CAA+C,CAAC;QACpE,MAAM,MAAM,GAAG,qBAAqB,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,UAAU,GAAG,6FAA6F,CAAC;QACjH,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,MAAM,KAAK,GAAG,0CAA0C,CAAC;QACzD,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,0CAA0C,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,CAAC,yBAAyB,EAAE,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;uBAoBO,CAAC;QAEpB,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,GAAG,GAAG,6CAA6C,CAAC;QAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,GAAG,GAAG,gCAAgC,CAAC;QAC7C,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,wCAAwC,CAAC;QACrD,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB;IAC1E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"prompt-builder.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/prompt-builder.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,MAAM,YAAY,GAAG,6CAA6C,CAAC;IAEnE,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,UAAU,GAAG,4DAA4D,CAAC;QAChF,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,UAAU,GAAG,8CAA8C,CAAC;QAClE,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,UAAU,GAAG,+DAA+D,CAAC;QACnF,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,oCAAoC;AACpC,8EAA8E;AAE9E,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,qBAAqB,CAAC,mCAAmC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,2DAA2D,CAAC,CAAC;QACtF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,UAAU,GAAG,uEAAuE,CAAC;QAC3F,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,WAAW,GAAG,+CAA+C,CAAC;QACpE,MAAM,MAAM,GAAG,qBAAqB,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,UAAU,GAAG,6FAA6F,CAAC;QACjH,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,MAAM,KAAK,GAAG,0CAA0C,CAAC;QACzD,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,0CAA0C,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,CAAC,yBAAyB,EAAE,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;uBAoBO,CAAC;QAEpB,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,GAAG,GAAG,6CAA6C,CAAC;QAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,GAAG,GAAG,gCAAgC,CAAC;QAC7C,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,wCAAwC,CAAC;QACrD,MAAM,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB;IAC1E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { LlmClient } from "./llm.js";
|
|
2
|
+
import type { EvalVerdict } from "./verdict.js";
|
|
3
|
+
import type { ActionItems, BenchmarkAssertionResult } from "./benchmark.js";
|
|
4
|
+
interface ComparisonStats {
|
|
5
|
+
passRate: number;
|
|
6
|
+
skillRubricAvg: number;
|
|
7
|
+
baselineRubricAvg: number;
|
|
8
|
+
delta: number;
|
|
9
|
+
}
|
|
10
|
+
interface CaseResult {
|
|
11
|
+
eval_id: number;
|
|
12
|
+
eval_name: string;
|
|
13
|
+
winner: "skill" | "baseline" | "tie";
|
|
14
|
+
skillContentScore: number;
|
|
15
|
+
skillStructureScore: number;
|
|
16
|
+
baselineContentScore: number;
|
|
17
|
+
baselineStructureScore: number;
|
|
18
|
+
assertionResults: BenchmarkAssertionResult[];
|
|
19
|
+
}
|
|
20
|
+
export declare function generateActionItems(client: LlmClient, verdict: EvalVerdict, stats: ComparisonStats, cases: CaseResult[], skillContent: string): Promise<ActionItems>;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// action-items.ts -- generate actionable recommendations from A/B comparison
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
const SYSTEM_PROMPT = `You are an expert skill evaluator analyzing A/B comparison results.
|
|
5
|
+
|
|
6
|
+
A skill file (SKILL.md) guides an AI assistant's behavior. An A/B comparison runs the same prompts with and without the skill, then blind-scores both outputs on content (1-5) and structure (1-5).
|
|
7
|
+
|
|
8
|
+
Analyze the results and produce concrete, actionable recommendations.
|
|
9
|
+
|
|
10
|
+
Respond with ONLY valid JSON (no code fences, no preamble):
|
|
11
|
+
{
|
|
12
|
+
"recommendation": "keep" | "improve" | "rewrite" | "remove",
|
|
13
|
+
"summary": "<1-2 sentences: what happened and what to do>",
|
|
14
|
+
"weaknesses": ["<specific weakness 1>", ...],
|
|
15
|
+
"strengths": ["<specific strength 1>", ...],
|
|
16
|
+
"suggestedFocus": "<the single most impactful change to make>"
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
Recommendation criteria:
|
|
20
|
+
- "keep": Skill clearly beats baseline — high pass rate (>=80%), consistent wins, delta > +1
|
|
21
|
+
- "improve": Skill shows promise but has fixable weaknesses — moderate pass rate, some wins
|
|
22
|
+
- "rewrite": Skill barely helps or is inconsistent — low pass rate, mixed wins/losses
|
|
23
|
+
- "remove": Skill actively degrades output — baseline consistently wins, negative delta
|
|
24
|
+
|
|
25
|
+
Be specific in weaknesses and strengths — reference actual eval cases and scores, not generic advice.
|
|
26
|
+
Keep suggestedFocus to one concrete, actionable sentence.`;
|
|
27
|
+
function buildUserPrompt(verdict, stats, cases, skillContent) {
|
|
28
|
+
const caseBreakdown = cases.map((c) => {
|
|
29
|
+
const failed = c.assertionResults.filter((a) => !a.pass);
|
|
30
|
+
const failedSection = failed.length > 0
|
|
31
|
+
? failed.map((a) => ` - FAIL: ${a.text} — ${a.reasoning}`).join("\n")
|
|
32
|
+
: " (all passed)";
|
|
33
|
+
return `### Case: "${c.eval_name}" (eval #${c.eval_id})
|
|
34
|
+
- Winner: ${c.winner}
|
|
35
|
+
- Skill scores: content=${c.skillContentScore}/5, structure=${c.skillStructureScore}/5
|
|
36
|
+
- Baseline scores: content=${c.baselineContentScore}/5, structure=${c.baselineStructureScore}/5
|
|
37
|
+
- Assertions: ${c.assertionResults.filter((a) => a.pass).length}/${c.assertionResults.length} passed
|
|
38
|
+
${failedSection}`;
|
|
39
|
+
}).join("\n\n");
|
|
40
|
+
// Truncate skill content to avoid token bloat (keep first 2000 chars)
|
|
41
|
+
const truncatedSkill = skillContent.length > 2000
|
|
42
|
+
? skillContent.slice(0, 2000) + "\n\n[... truncated ...]"
|
|
43
|
+
: skillContent;
|
|
44
|
+
return `## Verdict: ${verdict}
|
|
45
|
+
|
|
46
|
+
## Comparison Statistics
|
|
47
|
+
- Assertion pass rate: ${Math.round(stats.passRate * 100)}%
|
|
48
|
+
- Skill rubric average: ${stats.skillRubricAvg.toFixed(1)}/5
|
|
49
|
+
- Baseline rubric average: ${stats.baselineRubricAvg.toFixed(1)}/5
|
|
50
|
+
- Delta: ${stats.delta > 0 ? "+" : ""}${stats.delta.toFixed(1)}
|
|
51
|
+
|
|
52
|
+
## Per-Case Breakdown
|
|
53
|
+
${caseBreakdown}
|
|
54
|
+
|
|
55
|
+
## SKILL.md Content
|
|
56
|
+
${truncatedSkill}
|
|
57
|
+
|
|
58
|
+
Analyze these results and provide your recommendation.`;
|
|
59
|
+
}
|
|
60
|
+
function parseActionItems(raw) {
|
|
61
|
+
let json;
|
|
62
|
+
try {
|
|
63
|
+
const fenceMatch = raw.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
64
|
+
const jsonStr = fenceMatch ? fenceMatch[1] : raw;
|
|
65
|
+
json = JSON.parse(jsonStr.trim());
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return {
|
|
69
|
+
recommendation: "improve",
|
|
70
|
+
summary: "Could not parse action items from LLM response.",
|
|
71
|
+
weaknesses: [],
|
|
72
|
+
strengths: [],
|
|
73
|
+
suggestedFocus: "Re-run the comparison for actionable recommendations.",
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
const validRecs = new Set(["keep", "improve", "rewrite", "remove"]);
|
|
77
|
+
const rec = typeof json.recommendation === "string" && validRecs.has(json.recommendation)
|
|
78
|
+
? json.recommendation
|
|
79
|
+
: "improve";
|
|
80
|
+
return {
|
|
81
|
+
recommendation: rec,
|
|
82
|
+
summary: typeof json.summary === "string" ? json.summary : "",
|
|
83
|
+
weaknesses: Array.isArray(json.weaknesses)
|
|
84
|
+
? json.weaknesses.filter((w) => typeof w === "string")
|
|
85
|
+
: [],
|
|
86
|
+
strengths: Array.isArray(json.strengths)
|
|
87
|
+
? json.strengths.filter((s) => typeof s === "string")
|
|
88
|
+
: [],
|
|
89
|
+
suggestedFocus: typeof json.suggestedFocus === "string" ? json.suggestedFocus : "",
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
export async function generateActionItems(client, verdict, stats, cases, skillContent) {
|
|
93
|
+
const userPrompt = buildUserPrompt(verdict, stats, cases, skillContent);
|
|
94
|
+
const { text } = await client.generate(SYSTEM_PROMPT, userPrompt);
|
|
95
|
+
return parseActionItems(text);
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=action-items.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action-items.js","sourceRoot":"","sources":["../../src/eval/action-items.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6EAA6E;AAC7E,8EAA8E;AAwB9E,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;0DAsBoC,CAAC;AAE3D,SAAS,eAAe,CACtB,OAAoB,EACpB,KAAsB,EACtB,KAAmB,EACnB,YAAoB;IAEpB,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;YACrC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACtE,CAAC,CAAC,gBAAgB,CAAC;QACrB,OAAO,cAAc,CAAC,CAAC,SAAS,YAAY,CAAC,CAAC,OAAO;YAC7C,CAAC,CAAC,MAAM;0BACM,CAAC,CAAC,iBAAiB,iBAAiB,CAAC,CAAC,mBAAmB;6BACtD,CAAC,CAAC,oBAAoB,iBAAiB,CAAC,CAAC,sBAAsB;gBAC5E,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,gBAAgB,CAAC,MAAM;EAC1F,aAAa,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,sEAAsE;IACtE,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,GAAG,IAAI;QAC/C,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,yBAAyB;QACzD,CAAC,CAAC,YAAY,CAAC;IAEjB,OAAO,eAAe,OAAO;;;yBAGN,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC;0BAC/B,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;6BAC5B,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;WACpD,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;;;EAG5D,aAAa;;;EAGb,cAAc;;uDAEuC,CAAC;AACxD,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,IAAI,IAA6B,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACjD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,cAAc,EAAE,SAAS;YACzB,OAAO,EAAE,iDAAiD;YAC1D,UAAU,EAAE,EAAE;YACd,SAAS,EAAE,EAAE;YACb,cAAc,EAAE,uDAAuD;SACxE,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;IACpE,MAAM,GAAG,GAAG,OAAO,IAAI,CAAC,cAAc,KAAK,QAAQ,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;QACvF,CAAC,CAAC,IAAI,CAAC,cAA+C;QACtD,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAC7D,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;YACxC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;YACnE,CAAC,CAAC,EAAE;QACN,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;YAClE,CAAC,CAAC,EAAE;QACN,cAAc,EAAE,OAAO,IAAI,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE;KACnF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,MAAiB,EACjB,OAAoB,EACpB,KAAsB,EACtB,KAAmB,EACnB,YAAoB;IAEpB,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IACxE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;IAClE,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC"}
|
|
@@ -5,7 +5,7 @@ export interface HistorySummary {
|
|
|
5
5
|
model: string;
|
|
6
6
|
skillName: string;
|
|
7
7
|
passRate: number;
|
|
8
|
-
type: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct";
|
|
8
|
+
type: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct" | "ai-generate" | "eval-generate";
|
|
9
9
|
caseCount: number;
|
|
10
10
|
totalDurationMs: number;
|
|
11
11
|
totalTokens: number | null;
|
|
@@ -14,14 +14,14 @@ export interface HistorySummary {
|
|
|
14
14
|
}
|
|
15
15
|
export interface HistoryFilter {
|
|
16
16
|
model?: string;
|
|
17
|
-
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct";
|
|
17
|
+
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct" | "ai-generate" | "eval-generate";
|
|
18
18
|
from?: string;
|
|
19
19
|
to?: string;
|
|
20
20
|
}
|
|
21
21
|
export interface CaseHistoryEntry {
|
|
22
22
|
timestamp: string;
|
|
23
23
|
model: string;
|
|
24
|
-
type: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct";
|
|
24
|
+
type: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct" | "ai-generate" | "eval-generate";
|
|
25
25
|
provider?: string;
|
|
26
26
|
pass_rate: number;
|
|
27
27
|
durationMs?: number;
|
|
@@ -39,7 +39,7 @@ export interface RegressionEntry {
|
|
|
39
39
|
change: "regression" | "improvement";
|
|
40
40
|
}
|
|
41
41
|
export declare function writeHistoryEntry(skillDir: string, result: BenchmarkResult & {
|
|
42
|
-
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct";
|
|
42
|
+
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct" | "ai-generate" | "eval-generate";
|
|
43
43
|
}): Promise<string>;
|
|
44
44
|
export declare function deleteHistoryEntry(skillDir: string, timestamp: string): Promise<boolean>;
|
|
45
45
|
export declare function listHistory(skillDir: string, filter?: HistoryFilter): Promise<HistorySummary[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark-history.js","sourceRoot":"","sources":["../../src/eval/benchmark-history.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA6ChD,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,0CAA0C;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3C,yDAAyD;IACzD,OAAO,EAAE,CAAC,OAAO,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAuH;IAEvH,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,uDAAuD;IACvD,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEvC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAsB;IAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAE1E,2EAA2E;IAC3E,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,QAAQ,IAAI,EAAE,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC5C,IAAI,MAAM,IAAI,EAAE,GAAG,MAAM;gBAAE,OAAO,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,MAAM,SAAS,GAAI,IAAI,CAAC,IAA+B,IAAI,WAAW,CAAC;YAEvE,gCAAgC;YAChC,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAC3D,IAAI,MAAM,EAAE,IAAI,IAAI,SAAS,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAExD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAC3D,CAAC,CACF,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS;gBAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrD,CAAC,CAAC,IAAI,CAAC;YAET,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,QAAQ,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBAC5B,eAAe;gBACf,WAAW;gBACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAc,EACd,MAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAE3D,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAG,IAAI,CAAC,IAAiC,IAAI,WAAW;gBAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,UAAU,EAAE,YAAY,CAAC,UAAU;gBACnC,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,UAAU,EAAE,YAAY,CAAC,UAAU;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAyBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElE,qDAAqD;IACrD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyG,CAAC;IACtI,qBAAqB;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4E,CAAC;IACrG,eAAe;IACf,MAAM,WAAW,GAA+B,EAAE,CAAC;IAEnD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YACxE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;YAC3C,IAAI,SAAS,KAAK,WAAW;gBAAE,SAAS,CAAC,sCAAsC;YAE/E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvG,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEhF,QAAQ;YACR,WAAW,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;YAC/F,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC;YACnC,QAAQ,CAAC,eAAe,IAAI,eAAe,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAEnC,sBAAsB;YACtB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;oBAChI,IAAI,CAAC,KAAK,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI;wBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC1B,mBAAmB;oBACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;SACrD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,SAAS,EAAE,CAAC,CAAC,KAAK;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,yBAAyB;IAErE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;SAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpB,KAAK;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtD,aAAa,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC3D,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO;QACL,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,cAAc;QACd,UAAU;QACV,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAwB,EACxB,QAAyB;IAEzB,MAAM,WAAW,GAAsB,EAAE,CAAC;IAE1C,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS,CAAC,sBAAsB;YAExD,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,IAAI;oBACpB,aAAa,EAAE,KAAK;oBACpB,MAAM,EAAE,YAAY;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,KAAK;oBACrB,aAAa,EAAE,IAAI;oBACnB,MAAM,EAAE,aAAa;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|
|
1
|
+
{"version":3,"file":"benchmark-history.js","sourceRoot":"","sources":["../../src/eval/benchmark-history.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA6ChD,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,0CAA0C;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3C,yDAAyD;IACzD,OAAO,EAAE,CAAC,OAAO,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAyJ;IAEzJ,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,uDAAuD;IACvD,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEvC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAsB;IAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAE1E,2EAA2E;IAC3E,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,QAAQ,IAAI,EAAE,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC5C,IAAI,MAAM,IAAI,EAAE,GAAG,MAAM;gBAAE,OAAO,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,MAAM,SAAS,GAAI,IAAI,CAAC,IAA+B,IAAI,WAAW,CAAC;YAEvE,gCAAgC;YAChC,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAC3D,IAAI,MAAM,EAAE,IAAI,IAAI,SAAS,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAExD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAC3D,CAAC,CACF,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS;gBAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrD,CAAC,CAAC,IAAI,CAAC;YAET,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,QAAQ,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBAC5B,eAAe;gBACf,WAAW;gBACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAc,EACd,MAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAE3D,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAG,IAAI,CAAC,IAAiC,IAAI,WAAW;gBAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,UAAU,EAAE,YAAY,CAAC,UAAU;gBACnC,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,UAAU,EAAE,YAAY,CAAC,UAAU;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAyBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElE,qDAAqD;IACrD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyG,CAAC;IACtI,qBAAqB;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4E,CAAC;IACrG,eAAe;IACf,MAAM,WAAW,GAA+B,EAAE,CAAC;IAEnD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YACxE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;YAC3C,IAAI,SAAS,KAAK,WAAW;gBAAE,SAAS,CAAC,sCAAsC;YAE/E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvG,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEhF,QAAQ;YACR,WAAW,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;YAC/F,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC;YACnC,QAAQ,CAAC,eAAe,IAAI,eAAe,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAEnC,sBAAsB;YACtB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;oBAChI,IAAI,CAAC,KAAK,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI;wBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC1B,mBAAmB;oBACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;SACrD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,SAAS,EAAE,CAAC,CAAC,KAAK;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,yBAAyB;IAErE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;SAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpB,KAAK;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtD,aAAa,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC3D,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO;QACL,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,cAAc;QACd,UAAU;QACV,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAwB,EACxB,QAAyB;IAEzB,MAAM,WAAW,GAAsB,EAAE,CAAC;IAE1C,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS,CAAC,sBAAsB;YAExD,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,IAAI;oBACpB,aAAa,EAAE,KAAK;oBACpB,MAAM,EAAE,YAAY;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,KAAK;oBACrB,aAAa,EAAE,IAAI;oBACnB,MAAM,EAAE,aAAa;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|
package/dist/eval/benchmark.d.ts
CHANGED
|
@@ -33,13 +33,21 @@ export interface BenchmarkCase {
|
|
|
33
33
|
assertions: BenchmarkAssertionResult[];
|
|
34
34
|
comparisonDetail?: ComparisonCaseDetail;
|
|
35
35
|
}
|
|
36
|
+
export type ActionRecommendation = "keep" | "improve" | "rewrite" | "remove";
|
|
37
|
+
export interface ActionItems {
|
|
38
|
+
recommendation: ActionRecommendation;
|
|
39
|
+
summary: string;
|
|
40
|
+
weaknesses: string[];
|
|
41
|
+
strengths: string[];
|
|
42
|
+
suggestedFocus: string;
|
|
43
|
+
}
|
|
36
44
|
export interface BenchmarkResult {
|
|
37
45
|
timestamp: string;
|
|
38
46
|
model: string;
|
|
39
47
|
skill_name: string;
|
|
40
48
|
cases: BenchmarkCase[];
|
|
41
49
|
overall_pass_rate?: number;
|
|
42
|
-
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct";
|
|
50
|
+
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct" | "ai-generate" | "eval-generate";
|
|
43
51
|
provider?: string;
|
|
44
52
|
totalDurationMs?: number;
|
|
45
53
|
totalInputTokens?: number | null;
|
|
@@ -58,6 +66,15 @@ export interface BenchmarkResult {
|
|
|
58
66
|
improved: string;
|
|
59
67
|
reasoning: string;
|
|
60
68
|
};
|
|
69
|
+
generate?: {
|
|
70
|
+
prompt: string;
|
|
71
|
+
result: string;
|
|
72
|
+
};
|
|
73
|
+
actionItems?: ActionItems;
|
|
74
|
+
mcpSimulation?: {
|
|
75
|
+
active: boolean;
|
|
76
|
+
servers: string[];
|
|
77
|
+
};
|
|
61
78
|
}
|
|
62
79
|
export declare function writeBenchmark(skillDir: string, result: BenchmarkResult): Promise<void>;
|
|
63
80
|
export declare function readBenchmark(skillDir: string): Promise<BenchmarkResult | null>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAsFjC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAuB;IAEvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAClD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC3D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { LlmClient } from "./llm.js";
|
|
2
|
+
import { type McpDependency } from "./mcp-detector.js";
|
|
3
|
+
export type ComparisonProgressCallback = (phase: string, message: string) => void;
|
|
2
4
|
export interface ComparisonOutput {
|
|
3
5
|
skillOutput: string;
|
|
4
6
|
skillDurationMs: number;
|
|
@@ -28,6 +30,6 @@ export interface ComparisonResult {
|
|
|
28
30
|
baselineStructureScore: number;
|
|
29
31
|
winner: "skill" | "baseline" | "tie";
|
|
30
32
|
}
|
|
31
|
-
export declare function generateComparisonOutputs(prompt: string, skillContent: string, client: LlmClient): Promise<ComparisonOutput>;
|
|
32
|
-
export declare function scoreComparison(outputA: string, outputB: string, prompt: string, client: LlmClient): Promise<ComparisonScore>;
|
|
33
|
-
export declare function runComparison(prompt: string, skillContent: string, client: LlmClient): Promise<ComparisonResult>;
|
|
33
|
+
export declare function generateComparisonOutputs(prompt: string, skillContent: string, client: LlmClient, onProgress?: ComparisonProgressCallback): Promise<ComparisonOutput>;
|
|
34
|
+
export declare function scoreComparison(outputA: string, outputB: string, prompt: string, client: LlmClient, mcpDeps?: McpDependency[]): Promise<ComparisonScore>;
|
|
35
|
+
export declare function runComparison(prompt: string, skillContent: string, client: LlmClient, onProgress?: ComparisonProgressCallback): Promise<ComparisonResult>;
|
package/dist/eval/comparator.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// comparator.ts -- blind A/B comparison (with/without skill)
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
import { buildEvalSystemPrompt, buildBaselineSystemPrompt } from "./prompt-builder.js";
|
|
5
|
+
import { detectMcpDependencies } from "./mcp-detector.js";
|
|
5
6
|
const COMPARATOR_SYSTEM_PROMPT = `You are a blind quality evaluator. You will receive two AI assistant responses labeled "Response A" and "Response B". You do NOT know which response used a skill and which did not.
|
|
6
7
|
|
|
7
8
|
Evaluate each response on two dimensions:
|
|
@@ -19,11 +20,13 @@ Respond with ONLY valid JSON:
|
|
|
19
20
|
"winner": "first" | "second" | "tie",
|
|
20
21
|
"reasoning": "<brief explanation>"
|
|
21
22
|
}`;
|
|
22
|
-
export async function generateComparisonOutputs(prompt, skillContent, client) {
|
|
23
|
+
export async function generateComparisonOutputs(prompt, skillContent, client, onProgress) {
|
|
23
24
|
const skillSystemPrompt = buildEvalSystemPrompt(skillContent);
|
|
24
25
|
const baselineSystemPrompt = buildBaselineSystemPrompt();
|
|
25
26
|
// Run sequentially (claude-cli can only handle one at a time)
|
|
27
|
+
onProgress?.("generating_skill", "Generating skill output...");
|
|
26
28
|
const skillResult = await client.generate(skillSystemPrompt, prompt);
|
|
29
|
+
onProgress?.("generating_baseline", "Generating baseline output...");
|
|
27
30
|
const baselineResult = await client.generate(baselineSystemPrompt, prompt);
|
|
28
31
|
const totalTokens = (r) => r.inputTokens != null && r.outputTokens != null ? r.inputTokens + r.outputTokens : null;
|
|
29
32
|
return {
|
|
@@ -35,7 +38,12 @@ export async function generateComparisonOutputs(prompt, skillContent, client) {
|
|
|
35
38
|
baselineTokens: totalTokens(baselineResult),
|
|
36
39
|
};
|
|
37
40
|
}
|
|
38
|
-
export async function scoreComparison(outputA, outputB, prompt, client) {
|
|
41
|
+
export async function scoreComparison(outputA, outputB, prompt, client, mcpDeps) {
|
|
42
|
+
let systemPrompt = COMPARATOR_SYSTEM_PROMPT;
|
|
43
|
+
if (mcpDeps && mcpDeps.length > 0) {
|
|
44
|
+
const serverList = mcpDeps.map((d) => d.server).join(", ");
|
|
45
|
+
systemPrompt += `\n\nNOTE: One or both responses may contain SIMULATED MCP tool interactions (tools: ${serverList}).\nSimulated tool calls are valid demonstrations. Evaluate the quality and realism of the\nsimulation (correct tool names, realistic parameters, plausible responses) rather than\npenalizing simulated output vs. real output.`;
|
|
46
|
+
}
|
|
39
47
|
const userPrompt = `## Original Prompt
|
|
40
48
|
${prompt}
|
|
41
49
|
|
|
@@ -46,7 +54,7 @@ ${outputA}
|
|
|
46
54
|
${outputB}
|
|
47
55
|
|
|
48
56
|
Evaluate both responses.`;
|
|
49
|
-
const { text: response } = await client.generate(
|
|
57
|
+
const { text: response } = await client.generate(systemPrompt, userPrompt);
|
|
50
58
|
// Parse JSON from response (may be in code fence or plain)
|
|
51
59
|
let json;
|
|
52
60
|
try {
|
|
@@ -67,13 +75,15 @@ Evaluate both responses.`;
|
|
|
67
75
|
: "tie",
|
|
68
76
|
};
|
|
69
77
|
}
|
|
70
|
-
export async function runComparison(prompt, skillContent, client) {
|
|
71
|
-
const outputs = await generateComparisonOutputs(prompt, skillContent, client);
|
|
78
|
+
export async function runComparison(prompt, skillContent, client, onProgress) {
|
|
79
|
+
const outputs = await generateComparisonOutputs(prompt, skillContent, client, onProgress);
|
|
72
80
|
// Randomize order for blind comparison
|
|
73
81
|
const skillIsA = Math.random() < 0.5;
|
|
74
82
|
const outputA = skillIsA ? outputs.skillOutput : outputs.baselineOutput;
|
|
75
83
|
const outputB = skillIsA ? outputs.baselineOutput : outputs.skillOutput;
|
|
76
|
-
const
|
|
84
|
+
const mcpDeps = detectMcpDependencies(skillContent);
|
|
85
|
+
onProgress?.("scoring", "Scoring comparison...");
|
|
86
|
+
const scores = await scoreComparison(outputA, outputB, prompt, client, mcpDeps.length > 0 ? mcpDeps : undefined);
|
|
77
87
|
// Map scores back to skill/baseline
|
|
78
88
|
const skillContentScore = skillIsA ? scores.contentScoreA : scores.contentScoreB;
|
|
79
89
|
const skillStructureScore = skillIsA ? scores.structureScoreA : scores.structureScoreB;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../src/eval/comparator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../src/eval/comparator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAE,qBAAqB,EAAsB,MAAM,mBAAmB,CAAC;AAoC9E,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;EAgB/B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,YAAoB,EACpB,MAAiB,EACjB,UAAuC;IAEvC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAC9D,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAC;IAEzD,8DAA8D;IAC9D,UAAU,EAAE,CAAC,kBAAkB,EAAE,4BAA4B,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IACrE,UAAU,EAAE,CAAC,qBAAqB,EAAE,+BAA+B,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IAE3E,MAAM,WAAW,GAAG,CAAC,CAAqB,EAAE,EAAE,CAC5C,CAAC,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1F,OAAO;QACL,WAAW,EAAE,WAAW,CAAC,IAAI;QAC7B,eAAe,EAAE,WAAW,CAAC,UAAU;QACvC,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC;QACrC,cAAc,EAAE,cAAc,CAAC,IAAI;QACnC,kBAAkB,EAAE,cAAc,CAAC,UAAU;QAC7C,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAAe,EACf,MAAc,EACd,MAAiB,EACjB,OAAyB;IAEzB,IAAI,YAAY,GAAG,wBAAwB,CAAC;IAE5C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,YAAY,IAAI,uFAAuF,UAAU,kOAAkO,CAAC;IACtV,CAAC;IAED,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,OAAO;;;EAGP,OAAO;;yBAEgB,CAAC;IAExB,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAE3E,2DAA2D;IAC3D,IAAI,IAA6B,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,OAAO;QACL,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC;QAC/C,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACnD,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC;QAC/C,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACnD,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK;YAClF,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,KAAK;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,YAAoB,EACpB,MAAiB,EACjB,UAAuC;IAEvC,MAAM,OAAO,GAAG,MAAM,yBAAyB,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAE1F,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IACxE,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;IAExE,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,UAAU,EAAE,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEjH,oCAAoC;IACpC,MAAM,iBAAiB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IACjF,MAAM,mBAAmB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;IACvF,MAAM,oBAAoB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IACpF,MAAM,sBAAsB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;IAE1F,kBAAkB;IAClB,IAAI,MAAoC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,GAAG,KAAK,CAAC;IACjB,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACrC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3C,CAAC;IAED,OAAO;QACL,MAAM;QACN,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,iBAAiB;QACjB,mBAAmB;QACnB,oBAAoB;QACpB,sBAAsB;QACtB,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,GAAY;IAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC"}
|
package/dist/eval/judge.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { Assertion } from "./schema.js";
|
|
2
2
|
import type { LlmClient } from "./llm.js";
|
|
3
|
+
import type { McpDependency } from "./mcp-detector.js";
|
|
3
4
|
export interface AssertionResult {
|
|
4
5
|
id: string;
|
|
5
6
|
text: string;
|
|
6
7
|
pass: boolean;
|
|
7
8
|
reasoning: string;
|
|
8
9
|
}
|
|
9
|
-
export declare function
|
|
10
|
+
export declare function buildJudgeSystemPrompt(mcpDeps?: McpDependency[]): string;
|
|
11
|
+
export declare function judgeAssertion(output: string, assertion: Assertion, client: LlmClient, mcpDeps?: McpDependency[]): Promise<AssertionResult>;
|
package/dist/eval/judge.js
CHANGED
|
@@ -2,7 +2,23 @@
|
|
|
2
2
|
// Assertion-level LLM judge
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
const JUDGE_SYSTEM = `You are a binary assertion evaluator. Given an LLM output and an assertion, determine if the output satisfies the assertion. Respond with ONLY a JSON object: { "pass": boolean, "reasoning": "brief explanation" }`;
|
|
5
|
-
export
|
|
5
|
+
export function buildJudgeSystemPrompt(mcpDeps) {
|
|
6
|
+
if (!mcpDeps || mcpDeps.length === 0) {
|
|
7
|
+
return JUDGE_SYSTEM;
|
|
8
|
+
}
|
|
9
|
+
const serverList = mcpDeps.map((d) => d.server).join(", ");
|
|
10
|
+
return `You are a binary assertion evaluator. Given an LLM output and an assertion, determine if the output satisfies the assertion.
|
|
11
|
+
|
|
12
|
+
IMPORTANT: This output was generated in MCP SIMULATION MODE. The following tools were simulated (not connected to real services): ${serverList}.
|
|
13
|
+
When evaluating assertions:
|
|
14
|
+
- A simulated tool call is valid if it names the correct tool and provides realistic parameters
|
|
15
|
+
- A simulated response is valid if it contains plausible data (realistic IDs, timestamps, content)
|
|
16
|
+
- Do NOT fail assertions just because data is simulated rather than real
|
|
17
|
+
|
|
18
|
+
Respond with ONLY a JSON object: { "pass": boolean, "reasoning": "brief explanation" }`;
|
|
19
|
+
}
|
|
20
|
+
export async function judgeAssertion(output, assertion, client, mcpDeps) {
|
|
21
|
+
const systemPrompt = buildJudgeSystemPrompt(mcpDeps);
|
|
6
22
|
const userPrompt = `## LLM Output
|
|
7
23
|
${output}
|
|
8
24
|
|
|
@@ -10,7 +26,7 @@ ${output}
|
|
|
10
26
|
${assertion.text}
|
|
11
27
|
|
|
12
28
|
Does the LLM output satisfy this assertion? Respond with JSON only: { "pass": boolean, "reasoning": "..." }`;
|
|
13
|
-
const { text: raw } = await client.generate(
|
|
29
|
+
const { text: raw } = await client.generate(systemPrompt, userPrompt);
|
|
14
30
|
const parsed = parseJudgeResponse(raw);
|
|
15
31
|
return {
|
|
16
32
|
id: assertion.id,
|
package/dist/eval/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAa9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3D,OAAO;;oIAE2H,UAAU;;;;;;uFAMvD,CAAC;AACxF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB,EACjB,OAAyB;IAEzB,MAAM,YAAY,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAErD,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAEtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAEvC,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,4CAA4C;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,sEAAsE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/dist/eval/llm.js
CHANGED
|
@@ -123,13 +123,15 @@ function createAnthropicClient(modelOverride) {
|
|
|
123
123
|
function createCliClient(config) {
|
|
124
124
|
// Resolve binary path once at client creation time (cached internally)
|
|
125
125
|
const resolvedBinary = resolveCliBinary(config.binary);
|
|
126
|
+
// Pre-compute enhanced PATH once — avoids repeated execSync per generate() call
|
|
127
|
+
const cachedEnhancedPath = enhancedPath();
|
|
126
128
|
return {
|
|
127
129
|
model: config.displayModel,
|
|
128
130
|
async generate(systemPrompt, userPrompt) {
|
|
129
131
|
const combinedPrompt = `${systemPrompt}\n\n${userPrompt}`;
|
|
130
132
|
const start = Date.now();
|
|
131
133
|
const text = await new Promise((resolve, reject) => {
|
|
132
|
-
// Build env: strip prefix vars if needed, always
|
|
134
|
+
// Build env: strip prefix vars if needed, always use enhanced PATH
|
|
133
135
|
let env;
|
|
134
136
|
if (config.stripEnvPrefix) {
|
|
135
137
|
env = {};
|
|
@@ -139,14 +141,10 @@ function createCliClient(config) {
|
|
|
139
141
|
env[k] = v;
|
|
140
142
|
}
|
|
141
143
|
}
|
|
142
|
-
// Enhance PATH in the child process env so any sub-processes also
|
|
143
|
-
// have access to common binary directories (macOS, Linux, Windows)
|
|
144
|
-
if (env) {
|
|
145
|
-
env.PATH = enhancedPath(env.PATH);
|
|
146
|
-
}
|
|
147
144
|
else {
|
|
148
|
-
env = { ...process.env
|
|
145
|
+
env = { ...process.env };
|
|
149
146
|
}
|
|
147
|
+
env.PATH = cachedEnhancedPath;
|
|
150
148
|
// On Windows, .cmd/.bat files require shell:true to execute via spawn
|
|
151
149
|
const needsShell = process.platform === "win32" && /\.(cmd|bat)$/i.test(resolvedBinary);
|
|
152
150
|
const proc = spawn(resolvedBinary, config.args, {
|
package/dist/eval/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAgB5E,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,2EAA2E,CACtH,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAiBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAgB5E,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,2EAA2E,CACtH,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAiBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,gFAAgF;IAChF,MAAM,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE1C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,mEAAmE;gBACnE,IAAI,GAA2B,CAAC;gBAChC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,GAAG,GAAG,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;oBACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAA6B,EAAE,CAAC;gBACrD,CAAC;gBACD,GAAG,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAE9B,sEAAsE;gBACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAExF,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE;oBAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG;oBACH,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,wBAAwB,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/E,CAAC,EAAE,SAAS,CAAC,CAAC;gBAEd,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;oBACxC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CACd,GAAG,MAAM,CAAC,IAAI,yBAAyB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QACzF,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACzE,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,UAAU,KAAK,EAAE;QAC/B,cAAc,EAAE,QAAQ;QACxB,WAAW,EACT,qJAAqJ;KACxJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS,CAAC;IAC1E,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;QAChC,YAAY,EAAE,SAAS,KAAK,EAAE;QAC9B,WAAW,EACT,4IAA4I;KAC/I,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,kFAAkF;AAClF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;IACjF,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,kJAAkJ;KACrJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;aACtC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -26,6 +26,36 @@ const MCP_REGISTRY = [
|
|
|
26
26
|
url: "https://mcp.google.com/mcp",
|
|
27
27
|
transport: "http",
|
|
28
28
|
},
|
|
29
|
+
{
|
|
30
|
+
server: "Notion",
|
|
31
|
+
prefixes: ["notion_"],
|
|
32
|
+
url: "https://mcp.notion.com/mcp",
|
|
33
|
+
transport: "http",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
server: "Jira",
|
|
37
|
+
prefixes: ["jira_"],
|
|
38
|
+
url: "https://mcp.atlassian.com/jira/mcp",
|
|
39
|
+
transport: "http",
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
server: "Confluence",
|
|
43
|
+
prefixes: ["confluence_"],
|
|
44
|
+
url: "https://mcp.atlassian.com/confluence/mcp",
|
|
45
|
+
transport: "http",
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
server: "Figma",
|
|
49
|
+
prefixes: ["figma_"],
|
|
50
|
+
url: "https://mcp.figma.com/mcp",
|
|
51
|
+
transport: "http",
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
server: "Sentry",
|
|
55
|
+
prefixes: ["sentry_"],
|
|
56
|
+
url: "https://mcp.sentry.dev/mcp",
|
|
57
|
+
transport: "http",
|
|
58
|
+
},
|
|
29
59
|
];
|
|
30
60
|
function buildConfigSnippet(serverKey, url) {
|
|
31
61
|
return JSON.stringify({ mcpServers: { [serverKey.toLowerCase()]: { type: "http", url } } }, null, 2);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-detector.js","sourceRoot":"","sources":["../../src/eval/mcp-detector.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oFAAoF;AACpF,8EAA8E;AAsB9E,MAAM,YAAY,GAAuB;IACvC;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;QACvE,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;CACF,CAAC;AAEF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,GAAW;IACxD,OAAO,IAAI,CAAC,SAAS,CACnB,EAAE,UAAU,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,EACpE,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAEtB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,gCAAgC,CAAC,CAAC,CAAC;IACjF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,CAAC,CAAC;aAChB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,uDAAuD;IACvD,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC,CAAC;IAC9E,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC,CAAC,CAAC;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;IAErC,iCAAiC;IACjC,KAAK,MAAM,IAAI,IAAI,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IACxD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC;YACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC3C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACxC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CACzD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEnC,IAAI,CAAC,IAAI,CAAC;YACR,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,aAAa,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,sCAAsC;IACtC,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAExD,sEAAsE;IACtE,MAAM,QAAQ,GAAG;QACf,sCAAsC;QACtC,iCAAiC;QACjC,0CAA0C;QAC1C,8CAA8C;KAC/C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
1
|
+
{"version":3,"file":"mcp-detector.js","sourceRoot":"","sources":["../../src/eval/mcp-detector.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oFAAoF;AACpF,8EAA8E;AAsB9E,MAAM,YAAY,GAAuB;IACvC;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;QACvE,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,aAAa,CAAC;QACzB,GAAG,EAAE,0CAA0C;QAC/C,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;CACF,CAAC;AAEF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,GAAW;IACxD,OAAO,IAAI,CAAC,SAAS,CACnB,EAAE,UAAU,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,EACpE,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAEtB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,gCAAgC,CAAC,CAAC,CAAC;IACjF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,CAAC,CAAC;aAChB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,uDAAuD;IACvD,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC,CAAC;IAC9E,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC,CAAC,CAAC;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;IAErC,iCAAiC;IACjC,KAAK,MAAM,IAAI,IAAI,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IACxD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC;YACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC3C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACxC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CACzD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEnC,IAAI,CAAC,IAAI,CAAC;YACR,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,aAAa,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,sCAAsC;IACtC,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAExD,sEAAsE;IACtE,MAAM,QAAQ,GAAG;QACf,sCAAsC;QACtC,iCAAiC;QACjC,0CAA0C;QAC1C,8CAA8C;KAC/C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|