vskill 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eval/__tests__/benchmark.test.js +29 -0
- package/dist/eval/__tests__/benchmark.test.js.map +1 -1
- package/dist/eval/__tests__/comparator.test.js +80 -0
- package/dist/eval/__tests__/comparator.test.js.map +1 -1
- package/dist/eval/__tests__/judge.test.js +75 -1
- package/dist/eval/__tests__/judge.test.js.map +1 -1
- package/dist/eval/__tests__/mcp-detector.test.js +55 -0
- package/dist/eval/__tests__/mcp-detector.test.js.map +1 -1
- package/dist/eval/__tests__/prompt-builder.test.js +18 -0
- package/dist/eval/__tests__/prompt-builder.test.js.map +1 -1
- package/dist/eval/action-items.d.ts +21 -0
- package/dist/eval/action-items.js +97 -0
- package/dist/eval/action-items.js.map +1 -0
- package/dist/eval/benchmark-history.d.ts +4 -4
- package/dist/eval/benchmark-history.js.map +1 -1
- package/dist/eval/benchmark.d.ts +18 -1
- package/dist/eval/benchmark.js.map +1 -1
- package/dist/eval/comparator.d.ts +2 -1
- package/dist/eval/comparator.js +10 -3
- package/dist/eval/comparator.js.map +1 -1
- package/dist/eval/judge.d.ts +3 -1
- package/dist/eval/judge.js +18 -2
- package/dist/eval/judge.js.map +1 -1
- package/dist/eval/llm.js +5 -7
- package/dist/eval/llm.js.map +1 -1
- package/dist/eval/mcp-detector.js +30 -0
- package/dist/eval/mcp-detector.js.map +1 -1
- package/dist/eval/prompt-builder.js +22 -1
- package/dist/eval/prompt-builder.js.map +1 -1
- package/dist/eval/skill-scanner.d.ts +6 -0
- package/dist/eval/skill-scanner.js +55 -9
- package/dist/eval/skill-scanner.js.map +1 -1
- package/dist/eval-server/api-routes.js +37 -1
- package/dist/eval-server/api-routes.js.map +1 -1
- package/dist/eval-server/skill-create-routes.js +17 -2
- package/dist/eval-server/skill-create-routes.js.map +1 -1
- package/dist/eval-ui/assets/index-BHqTxODT.js +75 -0
- package/dist/eval-ui/index.html +1 -1
- package/dist/utils/__tests__/resolve-binary.integration.test.d.ts +1 -0
- package/dist/utils/__tests__/resolve-binary.integration.test.js +138 -0
- package/dist/utils/__tests__/resolve-binary.integration.test.js.map +1 -0
- package/dist/utils/resolve-binary.js +27 -8
- package/dist/utils/resolve-binary.js.map +1 -1
- package/package.json +1 -1
- package/dist/eval-ui/assets/index-CU39Pi0h.js +0 -75
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark-history.js","sourceRoot":"","sources":["../../src/eval/benchmark-history.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA6ChD,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,0CAA0C;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3C,yDAAyD;IACzD,OAAO,EAAE,CAAC,OAAO,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAuH;IAEvH,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,uDAAuD;IACvD,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEvC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAsB;IAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAE1E,2EAA2E;IAC3E,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,QAAQ,IAAI,EAAE,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC5C,IAAI,MAAM,IAAI,EAAE,GAAG,MAAM;gBAAE,OAAO,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,MAAM,SAAS,GAAI,IAAI,CAAC,IAA+B,IAAI,WAAW,CAAC;YAEvE,gCAAgC;YAChC,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAC3D,IAAI,MAAM,EAAE,IAAI,IAAI,SAAS,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAExD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAC3D,CAAC,CACF,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS;gBAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrD,CAAC,CAAC,IAAI,CAAC;YAET,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,QAAQ,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBAC5B,eAAe;gBACf,WAAW;gBACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAc,EACd,MAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAE3D,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAG,IAAI,CAAC,IAAiC,IAAI,WAAW;gBAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,UAAU,EAAE,YAAY,CAAC,UAAU;gBACnC,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,UAAU,EAAE,YAAY,CAAC,UAAU;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAyBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElE,qDAAqD;IACrD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyG,CAAC;IACtI,qBAAqB;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4E,CAAC;IACrG,eAAe;IACf,MAAM,WAAW,GAA+B,EAAE,CAAC;IAEnD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YACxE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;YAC3C,IAAI,SAAS,KAAK,WAAW;gBAAE,SAAS,CAAC,sCAAsC;YAE/E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvG,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEhF,QAAQ;YACR,WAAW,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;YAC/F,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC;YACnC,QAAQ,CAAC,eAAe,IAAI,eAAe,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAEnC,sBAAsB;YACtB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;oBAChI,IAAI,CAAC,KAAK,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI;wBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC1B,mBAAmB;oBACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;SACrD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,SAAS,EAAE,CAAC,CAAC,KAAK;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,yBAAyB;IAErE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;SAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpB,KAAK;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtD,aAAa,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC3D,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO;QACL,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,cAAc;QACd,UAAU;QACV,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAwB,EACxB,QAAyB;IAEzB,MAAM,WAAW,GAAsB,EAAE,CAAC;IAE1C,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS,CAAC,sBAAsB;YAExD,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,IAAI;oBACpB,aAAa,EAAE,KAAK;oBACpB,MAAM,EAAE,YAAY;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,KAAK;oBACrB,aAAa,EAAE,IAAI;oBACnB,MAAM,EAAE,aAAa;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|
|
1
|
+
{"version":3,"file":"benchmark-history.js","sourceRoot":"","sources":["../../src/eval/benchmark-history.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA6ChD,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,0CAA0C;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC3C,yDAAyD;IACzD,OAAO,EAAE,CAAC,OAAO,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAyJ;IAEzJ,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAE5C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE3D,uDAAuD;IACvD,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEvC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,MAAsB;IAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAE1E,2EAA2E;IAC3E,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,EAAE,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtE,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,QAAQ,IAAI,EAAE,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAC;YAC5C,IAAI,MAAM,IAAI,EAAE,GAAG,MAAM;gBAAE,OAAO,KAAK,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,MAAM,SAAS,GAAI,IAAI,CAAC,IAA+B,IAAI,WAAW,CAAC;YAEvE,gCAAgC;YAChC,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAC3D,IAAI,MAAM,EAAE,IAAI,IAAI,SAAS,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAExD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACpF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAC3D,CAAC,CACF,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;YAC3D,MAAM,WAAW,GAAG,SAAS;gBAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrD,CAAC,CAAC,IAAI,CAAC;YAET,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,QAAQ,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBAC5B,eAAe;gBACf,WAAW;gBACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC;IAC1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAc,EACd,MAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7E,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YAExE,IAAI,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,SAAS;YAE3D,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,OAAO,CAAC,IAAI,CAAC;gBACX,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAG,IAAI,CAAC,IAAiC,IAAI,WAAW;gBAC5D,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,UAAU,EAAE,YAAY,CAAC,UAAU;gBACnC,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,UAAU,EAAE,YAAY,CAAC,UAAU;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAyBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAElE,qDAAqD;IACrD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyG,CAAC;IACtI,qBAAqB;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4E,CAAC;IACrG,eAAe;IACf,MAAM,WAAW,GAA+B,EAAE,CAAC;IAEnD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAwC,CAAC;YACxE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC;YAC3C,IAAI,SAAS,KAAK,WAAW;gBAAE,SAAS,CAAC,sCAAsC;YAE/E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAChF,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvG,MAAM,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEhF,QAAQ;YACR,WAAW,CAAC,IAAI,CAAC;gBACf,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC;gBACtC,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;YAC/F,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC;YACnC,QAAQ,CAAC,eAAe,IAAI,eAAe,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAEnC,sBAAsB;YACtB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC;oBAChI,IAAI,CAAC,KAAK,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI;wBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC1B,mBAAmB;oBACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;SACrD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9C,SAAS,EAAE,CAAC,CAAC,KAAK;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,yBAAyB;IAErE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;SAC9C,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpB,KAAK;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,WAAW,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtD,aAAa,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC3D,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO;QACL,SAAS,EAAE,WAAW,CAAC,MAAM;QAC7B,cAAc;QACd,UAAU;QACV,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAwB,EACxB,QAAyB;IAEzB,MAAM,WAAW,GAAsB,EAAE,CAAC;IAE1C,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS,CAAC,sBAAsB;YAExD,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,IAAI;oBACpB,aAAa,EAAE,KAAK;oBACpB,MAAM,EAAE,YAAY;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC;oBACf,WAAW,EAAE,CAAC,CAAC,EAAE;oBACjB,MAAM,EAAE,CAAC,CAAC,OAAO;oBACjB,QAAQ,EAAE,CAAC,CAAC,SAAS;oBACrB,cAAc,EAAE,KAAK;oBACrB,aAAa,EAAE,IAAI;oBACnB,MAAM,EAAE,aAAa;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|
package/dist/eval/benchmark.d.ts
CHANGED
|
@@ -33,13 +33,21 @@ export interface BenchmarkCase {
|
|
|
33
33
|
assertions: BenchmarkAssertionResult[];
|
|
34
34
|
comparisonDetail?: ComparisonCaseDetail;
|
|
35
35
|
}
|
|
36
|
+
export type ActionRecommendation = "keep" | "improve" | "rewrite" | "remove";
|
|
37
|
+
export interface ActionItems {
|
|
38
|
+
recommendation: ActionRecommendation;
|
|
39
|
+
summary: string;
|
|
40
|
+
weaknesses: string[];
|
|
41
|
+
strengths: string[];
|
|
42
|
+
suggestedFocus: string;
|
|
43
|
+
}
|
|
36
44
|
export interface BenchmarkResult {
|
|
37
45
|
timestamp: string;
|
|
38
46
|
model: string;
|
|
39
47
|
skill_name: string;
|
|
40
48
|
cases: BenchmarkCase[];
|
|
41
49
|
overall_pass_rate?: number;
|
|
42
|
-
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct";
|
|
50
|
+
type?: "benchmark" | "comparison" | "baseline" | "model-compare" | "improve" | "instruct" | "ai-generate" | "eval-generate";
|
|
43
51
|
provider?: string;
|
|
44
52
|
totalDurationMs?: number;
|
|
45
53
|
totalInputTokens?: number | null;
|
|
@@ -58,6 +66,15 @@ export interface BenchmarkResult {
|
|
|
58
66
|
improved: string;
|
|
59
67
|
reasoning: string;
|
|
60
68
|
};
|
|
69
|
+
generate?: {
|
|
70
|
+
prompt: string;
|
|
71
|
+
result: string;
|
|
72
|
+
};
|
|
73
|
+
actionItems?: ActionItems;
|
|
74
|
+
mcpSimulation?: {
|
|
75
|
+
active: boolean;
|
|
76
|
+
servers: string[];
|
|
77
|
+
};
|
|
61
78
|
}
|
|
62
79
|
export declare function writeBenchmark(skillDir: string, result: BenchmarkResult): Promise<void>;
|
|
63
80
|
export declare function readBenchmark(skillDir: string): Promise<BenchmarkResult | null>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/eval/benchmark.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAsFjC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAuB;IAEvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAClD,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC3D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { LlmClient } from "./llm.js";
|
|
2
|
+
import { type McpDependency } from "./mcp-detector.js";
|
|
2
3
|
export type ComparisonProgressCallback = (phase: string, message: string) => void;
|
|
3
4
|
export interface ComparisonOutput {
|
|
4
5
|
skillOutput: string;
|
|
@@ -30,5 +31,5 @@ export interface ComparisonResult {
|
|
|
30
31
|
winner: "skill" | "baseline" | "tie";
|
|
31
32
|
}
|
|
32
33
|
export declare function generateComparisonOutputs(prompt: string, skillContent: string, client: LlmClient, onProgress?: ComparisonProgressCallback): Promise<ComparisonOutput>;
|
|
33
|
-
export declare function scoreComparison(outputA: string, outputB: string, prompt: string, client: LlmClient): Promise<ComparisonScore>;
|
|
34
|
+
export declare function scoreComparison(outputA: string, outputB: string, prompt: string, client: LlmClient, mcpDeps?: McpDependency[]): Promise<ComparisonScore>;
|
|
34
35
|
export declare function runComparison(prompt: string, skillContent: string, client: LlmClient, onProgress?: ComparisonProgressCallback): Promise<ComparisonResult>;
|
package/dist/eval/comparator.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// comparator.ts -- blind A/B comparison (with/without skill)
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
import { buildEvalSystemPrompt, buildBaselineSystemPrompt } from "./prompt-builder.js";
|
|
5
|
+
import { detectMcpDependencies } from "./mcp-detector.js";
|
|
5
6
|
const COMPARATOR_SYSTEM_PROMPT = `You are a blind quality evaluator. You will receive two AI assistant responses labeled "Response A" and "Response B". You do NOT know which response used a skill and which did not.
|
|
6
7
|
|
|
7
8
|
Evaluate each response on two dimensions:
|
|
@@ -37,7 +38,12 @@ export async function generateComparisonOutputs(prompt, skillContent, client, on
|
|
|
37
38
|
baselineTokens: totalTokens(baselineResult),
|
|
38
39
|
};
|
|
39
40
|
}
|
|
40
|
-
export async function scoreComparison(outputA, outputB, prompt, client) {
|
|
41
|
+
export async function scoreComparison(outputA, outputB, prompt, client, mcpDeps) {
|
|
42
|
+
let systemPrompt = COMPARATOR_SYSTEM_PROMPT;
|
|
43
|
+
if (mcpDeps && mcpDeps.length > 0) {
|
|
44
|
+
const serverList = mcpDeps.map((d) => d.server).join(", ");
|
|
45
|
+
systemPrompt += `\n\nNOTE: One or both responses may contain SIMULATED MCP tool interactions (tools: ${serverList}).\nSimulated tool calls are valid demonstrations. Evaluate the quality and realism of the\nsimulation (correct tool names, realistic parameters, plausible responses) rather than\npenalizing simulated output vs. real output.`;
|
|
46
|
+
}
|
|
41
47
|
const userPrompt = `## Original Prompt
|
|
42
48
|
${prompt}
|
|
43
49
|
|
|
@@ -48,7 +54,7 @@ ${outputA}
|
|
|
48
54
|
${outputB}
|
|
49
55
|
|
|
50
56
|
Evaluate both responses.`;
|
|
51
|
-
const { text: response } = await client.generate(
|
|
57
|
+
const { text: response } = await client.generate(systemPrompt, userPrompt);
|
|
52
58
|
// Parse JSON from response (may be in code fence or plain)
|
|
53
59
|
let json;
|
|
54
60
|
try {
|
|
@@ -75,8 +81,9 @@ export async function runComparison(prompt, skillContent, client, onProgress) {
|
|
|
75
81
|
const skillIsA = Math.random() < 0.5;
|
|
76
82
|
const outputA = skillIsA ? outputs.skillOutput : outputs.baselineOutput;
|
|
77
83
|
const outputB = skillIsA ? outputs.baselineOutput : outputs.skillOutput;
|
|
84
|
+
const mcpDeps = detectMcpDependencies(skillContent);
|
|
78
85
|
onProgress?.("scoring", "Scoring comparison...");
|
|
79
|
-
const scores = await scoreComparison(outputA, outputB, prompt, client);
|
|
86
|
+
const scores = await scoreComparison(outputA, outputB, prompt, client, mcpDeps.length > 0 ? mcpDeps : undefined);
|
|
80
87
|
// Map scores back to skill/baseline
|
|
81
88
|
const skillContentScore = skillIsA ? scores.contentScoreA : scores.contentScoreB;
|
|
82
89
|
const skillStructureScore = skillIsA ? scores.structureScoreA : scores.structureScoreB;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../src/eval/comparator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../src/eval/comparator.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAE,qBAAqB,EAAsB,MAAM,mBAAmB,CAAC;AAoC9E,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;EAgB/B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,YAAoB,EACpB,MAAiB,EACjB,UAAuC;IAEvC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAC9D,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAC;IAEzD,8DAA8D;IAC9D,UAAU,EAAE,CAAC,kBAAkB,EAAE,4BAA4B,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IACrE,UAAU,EAAE,CAAC,qBAAqB,EAAE,+BAA+B,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IAE3E,MAAM,WAAW,GAAG,CAAC,CAAqB,EAAE,EAAE,CAC5C,CAAC,CAAC,WAAW,IAAI,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1F,OAAO;QACL,WAAW,EAAE,WAAW,CAAC,IAAI;QAC7B,eAAe,EAAE,WAAW,CAAC,UAAU;QACvC,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC;QACrC,cAAc,EAAE,cAAc,CAAC,IAAI;QACnC,kBAAkB,EAAE,cAAc,CAAC,UAAU;QAC7C,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAAe,EACf,MAAc,EACd,MAAiB,EACjB,OAAyB;IAEzB,IAAI,YAAY,GAAG,wBAAwB,CAAC;IAE5C,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,YAAY,IAAI,uFAAuF,UAAU,kOAAkO,CAAC;IACtV,CAAC;IAED,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,OAAO;;;EAGP,OAAO;;yBAEgB,CAAC;IAExB,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAE3E,2DAA2D;IAC3D,IAAI,IAA6B,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,OAAO;QACL,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC;QAC/C,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACnD,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC;QAC/C,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACnD,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK;YAClF,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,KAAK;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,YAAoB,EACpB,MAAiB,EACjB,UAAuC;IAEvC,MAAM,OAAO,GAAG,MAAM,yBAAyB,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAE1F,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IACxE,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;IAExE,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,UAAU,EAAE,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEjH,oCAAoC;IACpC,MAAM,iBAAiB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IACjF,MAAM,mBAAmB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;IACvF,MAAM,oBAAoB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IACpF,MAAM,sBAAsB,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC;IAE1F,kBAAkB;IAClB,IAAI,MAAoC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,GAAG,KAAK,CAAC;IACjB,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACrC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3C,CAAC;IAED,OAAO;QACL,MAAM;QACN,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,iBAAiB;QACjB,mBAAmB;QACnB,oBAAoB;QACpB,sBAAsB;QACtB,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,GAAY;IAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACvB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC"}
|
package/dist/eval/judge.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { Assertion } from "./schema.js";
|
|
2
2
|
import type { LlmClient } from "./llm.js";
|
|
3
|
+
import type { McpDependency } from "./mcp-detector.js";
|
|
3
4
|
export interface AssertionResult {
|
|
4
5
|
id: string;
|
|
5
6
|
text: string;
|
|
6
7
|
pass: boolean;
|
|
7
8
|
reasoning: string;
|
|
8
9
|
}
|
|
9
|
-
export declare function
|
|
10
|
+
export declare function buildJudgeSystemPrompt(mcpDeps?: McpDependency[]): string;
|
|
11
|
+
export declare function judgeAssertion(output: string, assertion: Assertion, client: LlmClient, mcpDeps?: McpDependency[]): Promise<AssertionResult>;
|
package/dist/eval/judge.js
CHANGED
|
@@ -2,7 +2,23 @@
|
|
|
2
2
|
// Assertion-level LLM judge
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
const JUDGE_SYSTEM = `You are a binary assertion evaluator. Given an LLM output and an assertion, determine if the output satisfies the assertion. Respond with ONLY a JSON object: { "pass": boolean, "reasoning": "brief explanation" }`;
|
|
5
|
-
export
|
|
5
|
+
export function buildJudgeSystemPrompt(mcpDeps) {
|
|
6
|
+
if (!mcpDeps || mcpDeps.length === 0) {
|
|
7
|
+
return JUDGE_SYSTEM;
|
|
8
|
+
}
|
|
9
|
+
const serverList = mcpDeps.map((d) => d.server).join(", ");
|
|
10
|
+
return `You are a binary assertion evaluator. Given an LLM output and an assertion, determine if the output satisfies the assertion.
|
|
11
|
+
|
|
12
|
+
IMPORTANT: This output was generated in MCP SIMULATION MODE. The following tools were simulated (not connected to real services): ${serverList}.
|
|
13
|
+
When evaluating assertions:
|
|
14
|
+
- A simulated tool call is valid if it names the correct tool and provides realistic parameters
|
|
15
|
+
- A simulated response is valid if it contains plausible data (realistic IDs, timestamps, content)
|
|
16
|
+
- Do NOT fail assertions just because data is simulated rather than real
|
|
17
|
+
|
|
18
|
+
Respond with ONLY a JSON object: { "pass": boolean, "reasoning": "brief explanation" }`;
|
|
19
|
+
}
|
|
20
|
+
export async function judgeAssertion(output, assertion, client, mcpDeps) {
|
|
21
|
+
const systemPrompt = buildJudgeSystemPrompt(mcpDeps);
|
|
6
22
|
const userPrompt = `## LLM Output
|
|
7
23
|
${output}
|
|
8
24
|
|
|
@@ -10,7 +26,7 @@ ${output}
|
|
|
10
26
|
${assertion.text}
|
|
11
27
|
|
|
12
28
|
Does the LLM output satisfy this assertion? Respond with JSON only: { "pass": boolean, "reasoning": "..." }`;
|
|
13
|
-
const { text: raw } = await client.generate(
|
|
29
|
+
const { text: raw } = await client.generate(systemPrompt, userPrompt);
|
|
14
30
|
const parsed = parseJudgeResponse(raw);
|
|
15
31
|
return {
|
|
16
32
|
id: assertion.id,
|
package/dist/eval/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAa9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3D,OAAO;;oIAE2H,UAAU;;;;;;uFAMvD,CAAC;AACxF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB,EACjB,OAAyB;IAEzB,MAAM,YAAY,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAErD,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAEtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAEvC,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,4CAA4C;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,sEAAsE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/dist/eval/llm.js
CHANGED
|
@@ -123,13 +123,15 @@ function createAnthropicClient(modelOverride) {
|
|
|
123
123
|
function createCliClient(config) {
|
|
124
124
|
// Resolve binary path once at client creation time (cached internally)
|
|
125
125
|
const resolvedBinary = resolveCliBinary(config.binary);
|
|
126
|
+
// Pre-compute enhanced PATH once — avoids repeated execSync per generate() call
|
|
127
|
+
const cachedEnhancedPath = enhancedPath();
|
|
126
128
|
return {
|
|
127
129
|
model: config.displayModel,
|
|
128
130
|
async generate(systemPrompt, userPrompt) {
|
|
129
131
|
const combinedPrompt = `${systemPrompt}\n\n${userPrompt}`;
|
|
130
132
|
const start = Date.now();
|
|
131
133
|
const text = await new Promise((resolve, reject) => {
|
|
132
|
-
// Build env: strip prefix vars if needed, always
|
|
134
|
+
// Build env: strip prefix vars if needed, always use enhanced PATH
|
|
133
135
|
let env;
|
|
134
136
|
if (config.stripEnvPrefix) {
|
|
135
137
|
env = {};
|
|
@@ -139,14 +141,10 @@ function createCliClient(config) {
|
|
|
139
141
|
env[k] = v;
|
|
140
142
|
}
|
|
141
143
|
}
|
|
142
|
-
// Enhance PATH in the child process env so any sub-processes also
|
|
143
|
-
// have access to common binary directories (macOS, Linux, Windows)
|
|
144
|
-
if (env) {
|
|
145
|
-
env.PATH = enhancedPath(env.PATH);
|
|
146
|
-
}
|
|
147
144
|
else {
|
|
148
|
-
env = { ...process.env
|
|
145
|
+
env = { ...process.env };
|
|
149
146
|
}
|
|
147
|
+
env.PATH = cachedEnhancedPath;
|
|
150
148
|
// On Windows, .cmd/.bat files require shell:true to execute via spawn
|
|
151
149
|
const needsShell = process.platform === "win32" && /\.(cmd|bat)$/i.test(resolvedBinary);
|
|
152
150
|
const proc = spawn(resolvedBinary, config.args, {
|
package/dist/eval/llm.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAgB5E,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,2EAA2E,CACtH,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAiBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAgB5E,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,2EAA2E,CACtH,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAiBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,gFAAgF;IAChF,MAAM,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE1C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,mEAAmE;gBACnE,IAAI,GAA2B,CAAC;gBAChC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,GAAG,GAAG,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;oBACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAA6B,EAAE,CAAC;gBACrD,CAAC;gBACD,GAAG,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAE9B,sEAAsE;gBACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAExF,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE;oBAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG;oBACH,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,wBAAwB,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/E,CAAC,EAAE,SAAS,CAAC,CAAC;gBAEd,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;oBACxC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CACd,GAAG,MAAM,CAAC,IAAI,yBAAyB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QACzF,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACzE,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,UAAU,KAAK,EAAE;QAC/B,cAAc,EAAE,QAAQ;QACxB,WAAW,EACT,qJAAqJ;KACxJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS,CAAC;IAC1E,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;QAChC,YAAY,EAAE,SAAS,KAAK,EAAE;QAC9B,WAAW,EACT,4IAA4I;KAC/I,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,kFAAkF;AAClF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;IACjF,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,kJAAkJ;KACrJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;aACtC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -26,6 +26,36 @@ const MCP_REGISTRY = [
|
|
|
26
26
|
url: "https://mcp.google.com/mcp",
|
|
27
27
|
transport: "http",
|
|
28
28
|
},
|
|
29
|
+
{
|
|
30
|
+
server: "Notion",
|
|
31
|
+
prefixes: ["notion_"],
|
|
32
|
+
url: "https://mcp.notion.com/mcp",
|
|
33
|
+
transport: "http",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
server: "Jira",
|
|
37
|
+
prefixes: ["jira_"],
|
|
38
|
+
url: "https://mcp.atlassian.com/jira/mcp",
|
|
39
|
+
transport: "http",
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
server: "Confluence",
|
|
43
|
+
prefixes: ["confluence_"],
|
|
44
|
+
url: "https://mcp.atlassian.com/confluence/mcp",
|
|
45
|
+
transport: "http",
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
server: "Figma",
|
|
49
|
+
prefixes: ["figma_"],
|
|
50
|
+
url: "https://mcp.figma.com/mcp",
|
|
51
|
+
transport: "http",
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
server: "Sentry",
|
|
55
|
+
prefixes: ["sentry_"],
|
|
56
|
+
url: "https://mcp.sentry.dev/mcp",
|
|
57
|
+
transport: "http",
|
|
58
|
+
},
|
|
29
59
|
];
|
|
30
60
|
function buildConfigSnippet(serverKey, url) {
|
|
31
61
|
return JSON.stringify({ mcpServers: { [serverKey.toLowerCase()]: { type: "http", url } } }, null, 2);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-detector.js","sourceRoot":"","sources":["../../src/eval/mcp-detector.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oFAAoF;AACpF,8EAA8E;AAsB9E,MAAM,YAAY,GAAuB;IACvC;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;QACvE,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;CACF,CAAC;AAEF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,GAAW;IACxD,OAAO,IAAI,CAAC,SAAS,CACnB,EAAE,UAAU,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,EACpE,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAEtB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,gCAAgC,CAAC,CAAC,CAAC;IACjF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,CAAC,CAAC;aAChB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,uDAAuD;IACvD,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC,CAAC;IAC9E,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC,CAAC,CAAC;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;IAErC,iCAAiC;IACjC,KAAK,MAAM,IAAI,IAAI,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IACxD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC;YACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC3C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACxC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CACzD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEnC,IAAI,CAAC,IAAI,CAAC;YACR,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,aAAa,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,sCAAsC;IACtC,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAExD,sEAAsE;IACtE,MAAM,QAAQ,GAAG;QACf,sCAAsC;QACtC,iCAAiC;QACjC,0CAA0C;QAC1C,8CAA8C;KAC/C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
1
|
+
{"version":3,"file":"mcp-detector.js","sourceRoot":"","sources":["../../src/eval/mcp-detector.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oFAAoF;AACpF,8EAA8E;AAsB9E,MAAM,YAAY,GAAuB;IACvC;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,kBAAkB;QAC1B,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;QACvE,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,oCAAoC;QACzC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,aAAa,CAAC;QACzB,GAAG,EAAE,0CAA0C;QAC/C,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,GAAG,EAAE,2BAA2B;QAChC,SAAS,EAAE,MAAM;KAClB;IACD;QACE,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,GAAG,EAAE,4BAA4B;QACjC,SAAS,EAAE,MAAM;KAClB;CACF,CAAC;AAEF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,GAAW;IACxD,OAAO,IAAI,CAAC,SAAS,CACnB,EAAE,UAAU,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,EACpE,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe,EAAE,KAAa;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAEtB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,gCAAgC,CAAC,CAAC,CAAC;IACjF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,CAAC,CAAC;aAChB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5C,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,uDAAuD;IACvD,MAAM,WAAW,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC,CAAC;IAC9E,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC,CAAC,CAAC;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,MAAM,KAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;IAErC,iCAAiC;IACjC,KAAK,MAAM,IAAI,IAAI,qBAAqB,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAED,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IACxD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC;YACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC3C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACnD,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACxC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CACzD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEnC,IAAI,CAAC,IAAI,CAAC;YACR,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,aAAa,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAe;IACrD,MAAM,IAAI,GAAsB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,sCAAsC;IACtC,KAAK,MAAM,GAAG,IAAI,qBAAqB,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QACjE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAExD,sEAAsE;IACtE,MAAM,QAAQ,GAAG;QACf,sCAAsC;QACtC,iCAAiC;QACjC,0CAA0C;QAC1C,8CAA8C;KAC/C,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -100,6 +100,27 @@ export function buildBaselineSystemPrompt() {
|
|
|
100
100
|
// Eval generation prompt (for creating evals.json from SKILL.md)
|
|
101
101
|
// ---------------------------------------------------------------------------
|
|
102
102
|
export function buildEvalInitPrompt(skillContent) {
|
|
103
|
+
const mcpDeps = detectMcpDependencies(skillContent);
|
|
104
|
+
let mcpSection = "";
|
|
105
|
+
if (mcpDeps.length > 0) {
|
|
106
|
+
const toolList = mcpDeps
|
|
107
|
+
.map((d) => `- **${d.server}**: ${d.matchedTools.join(", ")}`)
|
|
108
|
+
.join("\n");
|
|
109
|
+
mcpSection = `
|
|
110
|
+
|
|
111
|
+
## MCP Simulation Context
|
|
112
|
+
|
|
113
|
+
This skill uses MCP tools that will be SIMULATED during evaluation (not connected to real services).
|
|
114
|
+
Tools detected:
|
|
115
|
+
${toolList}
|
|
116
|
+
|
|
117
|
+
When generating assertions:
|
|
118
|
+
- Assert that the output demonstrates the correct tool call workflow (tool name, parameters)
|
|
119
|
+
- Assert that simulated responses are realistic and complete
|
|
120
|
+
- Do NOT assert on real API artifacts (actual URLs, real IDs, live data)
|
|
121
|
+
- Use assertions like "demonstrates calling slack_send_message with channel and message params"
|
|
122
|
+
`;
|
|
123
|
+
}
|
|
103
124
|
return `You are an expert eval generator for AI skills. Your task is to create an evals.json file for the skill described below.
|
|
104
125
|
|
|
105
126
|
## Skill Content (SKILL.md)
|
|
@@ -111,7 +132,7 @@ ${SCHEMA_REFERENCE}
|
|
|
111
132
|
${EXAMPLE_EVALS}
|
|
112
133
|
|
|
113
134
|
${BEST_PRACTICES}
|
|
114
|
-
|
|
135
|
+
${mcpSection}
|
|
115
136
|
## Instructions
|
|
116
137
|
|
|
117
138
|
Generate a complete evals.json for this skill. Output ONLY the JSON inside a \`\`\`json code fence. Generate 2-3 eval cases with realistic, specific prompts and objectively verifiable assertions. Each case must have at least 2 assertions.`;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-builder.js","sourceRoot":"","sources":["../../src/eval/prompt-builder.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BxB,CAAC;AAEF,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBrB,CAAC;AAEF,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,cAAc,GAAG;;;;;;;;;;;;CAYtB,CAAC;AAEF,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,YAAoB;IACxD,IAAI,CAAC,YAAY;QAAE,OAAO,iCAAiC,CAAC;IAE5D,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,iEAAiE,YAAY,EAAE,CAAC;IACzF,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO;SACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;SAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,iEAAiE,YAAY,sPAAsP,QAAQ,qhBAAqhB,CAAC;AAC12B,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,OAAO;;;;EAIP,YAAY;;EAEZ,gBAAgB;;EAEhB,aAAa;;EAEb,cAAc
|
|
1
|
+
{"version":3,"file":"prompt-builder.js","sourceRoot":"","sources":["../../src/eval/prompt-builder.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BxB,CAAC;AAEF,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBrB,CAAC;AAEF,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,cAAc,GAAG;;;;;;;;;;;;CAYtB,CAAC;AAEF,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,YAAoB;IACxD,IAAI,CAAC,YAAY;QAAE,OAAO,iCAAiC,CAAC;IAE5D,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,iEAAiE,YAAY,EAAE,CAAC;IACzF,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO;SACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;SAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,iEAAiE,YAAY,sPAAsP,QAAQ,qhBAAqhB,CAAC;AAC12B,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,UAAU,GAAG;;;;;;EAMf,QAAQ;;;;;;;CAOT,CAAC;IACA,CAAC;IAED,OAAO;;;;EAIP,YAAY;;EAEZ,gBAAgB;;EAEhB,aAAa;;EAEb,cAAc;EACd,UAAU;;;+OAGmO,CAAC;AAChP,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,+CAA+C;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,+BAAgC,GAAa,CAAC,OAAO,EAAE,CACxD,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE5B,YAAY;IACZ,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;aAC1B,CAAC,CAAC;SACJ,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACzD,IAAI,CAAC,CAAC,CAAC,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC"}
|
|
@@ -4,5 +4,11 @@ export interface SkillInfo {
|
|
|
4
4
|
dir: string;
|
|
5
5
|
hasEvals: boolean;
|
|
6
6
|
hasBenchmark: boolean;
|
|
7
|
+
origin: "source" | "installed";
|
|
7
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Classify a skill's origin based on its filesystem path relative to root.
|
|
11
|
+
* Skills inside agent config directories or plugin caches are "installed".
|
|
12
|
+
*/
|
|
13
|
+
export declare function classifyOrigin(skillDir: string, root: string): "source" | "installed";
|
|
8
14
|
export declare function scanSkills(root: string): Promise<SkillInfo[]>;
|
|
@@ -9,7 +9,52 @@
|
|
|
9
9
|
// 5. Flat: {root}/{skill}/SKILL.md (skills as direct children of root)
|
|
10
10
|
// ---------------------------------------------------------------------------
|
|
11
11
|
import { readdirSync, existsSync } from "node:fs";
|
|
12
|
-
import { join, basename, dirname } from "node:path";
|
|
12
|
+
import { join, basename, dirname, relative, sep } from "node:path";
|
|
13
|
+
import { AGENTS_REGISTRY } from "../agents/agents-registry.js";
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// Origin classification — detect if a skill lives inside an agent config
|
|
16
|
+
// directory (installed/consumed) vs. a user's own project (source/editable).
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
/** Extra known config dirs not covered by agents-registry localSkillsDir. */
|
|
19
|
+
const EXTRA_CONFIG_DIRS = [
|
|
20
|
+
".specweave", ".vscode", ".idea", ".zed", ".devcontainer",
|
|
21
|
+
".github", ".agents", ".agent",
|
|
22
|
+
];
|
|
23
|
+
/** Lazily built set of all known agent config directory prefixes. */
|
|
24
|
+
let _installedPrefixes = null;
|
|
25
|
+
function getInstalledPrefixes() {
|
|
26
|
+
if (_installedPrefixes)
|
|
27
|
+
return _installedPrefixes;
|
|
28
|
+
const prefixes = new Set();
|
|
29
|
+
for (const agent of AGENTS_REGISTRY) {
|
|
30
|
+
// Extract first path segment: ".claude/skills" → ".claude"
|
|
31
|
+
const first = agent.localSkillsDir.split("/")[0];
|
|
32
|
+
if (first)
|
|
33
|
+
prefixes.add(first);
|
|
34
|
+
}
|
|
35
|
+
for (const dir of EXTRA_CONFIG_DIRS) {
|
|
36
|
+
prefixes.add(dir);
|
|
37
|
+
}
|
|
38
|
+
_installedPrefixes = prefixes;
|
|
39
|
+
return prefixes;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Classify a skill's origin based on its filesystem path relative to root.
|
|
43
|
+
* Skills inside agent config directories or plugin caches are "installed".
|
|
44
|
+
*/
|
|
45
|
+
export function classifyOrigin(skillDir, root) {
|
|
46
|
+
const rel = relative(root, skillDir).split(sep).join("/");
|
|
47
|
+
// Check if the relative path starts with a known agent config prefix
|
|
48
|
+
const firstSegment = rel.split("/")[0];
|
|
49
|
+
if (firstSegment && getInstalledPrefixes().has(firstSegment)) {
|
|
50
|
+
return "installed";
|
|
51
|
+
}
|
|
52
|
+
// Check for plugin cache directories (e.g. plugins/cache/...)
|
|
53
|
+
if (rel.includes("plugins/cache/")) {
|
|
54
|
+
return "installed";
|
|
55
|
+
}
|
|
56
|
+
return "source";
|
|
57
|
+
}
|
|
13
58
|
export async function scanSkills(root) {
|
|
14
59
|
const skills = [];
|
|
15
60
|
if (!existsSync(root))
|
|
@@ -38,29 +83,30 @@ export async function scanSkills(root) {
|
|
|
38
83
|
dir: root,
|
|
39
84
|
hasEvals,
|
|
40
85
|
hasBenchmark,
|
|
86
|
+
origin: classifyOrigin(root, root),
|
|
41
87
|
});
|
|
42
88
|
return skills;
|
|
43
89
|
}
|
|
44
90
|
// Layout 3: root-level skills/ directory → {root}/skills/{skill}/SKILL.md
|
|
45
|
-
scanSkillsDir(basename(root) || "default", join(root, "skills"), skills);
|
|
91
|
+
scanSkillsDir(basename(root) || "default", join(root, "skills"), skills, root);
|
|
46
92
|
// Layout 1: direct plugin subdirs → {root}/{plugin}/skills/{skill}/SKILL.md
|
|
47
93
|
// Exclude "skills" (Layout 3) and "plugins" (Layout 2) from being treated as plugins
|
|
48
|
-
scanPluginDirs(root, skills, ["skills", "plugins"]);
|
|
94
|
+
scanPluginDirs(root, skills, ["skills", "plugins"], root);
|
|
49
95
|
// Layout 2: nested plugins/ dir → {root}/plugins/{plugin}/skills/{skill}/SKILL.md
|
|
50
96
|
// Only exclude "plugins" to prevent recursion; "skills" is a valid plugin name here
|
|
51
97
|
const pluginsDir = join(root, "plugins");
|
|
52
98
|
if (existsSync(pluginsDir)) {
|
|
53
|
-
scanPluginDirs(pluginsDir, skills, ["plugins"]);
|
|
99
|
+
scanPluginDirs(pluginsDir, skills, ["plugins"], root);
|
|
54
100
|
}
|
|
55
101
|
// Layout 5: flat → {root}/{skill}/SKILL.md (skills as direct children of root)
|
|
56
102
|
// Only pick up directories not already matched by layouts 1-3.
|
|
57
103
|
if (skills.length === 0) {
|
|
58
104
|
const defaultPlugin = basename(root) || "default";
|
|
59
|
-
scanSkillsDir(defaultPlugin, root, skills);
|
|
105
|
+
scanSkillsDir(defaultPlugin, root, skills, root);
|
|
60
106
|
}
|
|
61
107
|
return skills;
|
|
62
108
|
}
|
|
63
|
-
function scanPluginDirs(dir, skills, exclude) {
|
|
109
|
+
function scanPluginDirs(dir, skills, exclude, root) {
|
|
64
110
|
let entries;
|
|
65
111
|
try {
|
|
66
112
|
entries = readdirSync(dir, { withFileTypes: true })
|
|
@@ -71,10 +117,10 @@ function scanPluginDirs(dir, skills, exclude) {
|
|
|
71
117
|
return;
|
|
72
118
|
}
|
|
73
119
|
for (const plugin of entries) {
|
|
74
|
-
scanSkillsDir(plugin, join(dir, plugin, "skills"), skills);
|
|
120
|
+
scanSkillsDir(plugin, join(dir, plugin, "skills"), skills, root);
|
|
75
121
|
}
|
|
76
122
|
}
|
|
77
|
-
function scanSkillsDir(plugin, skillsDir, skills) {
|
|
123
|
+
function scanSkillsDir(plugin, skillsDir, skills, root) {
|
|
78
124
|
if (!existsSync(skillsDir))
|
|
79
125
|
return;
|
|
80
126
|
let skillDirs;
|
|
@@ -93,7 +139,7 @@ function scanSkillsDir(plugin, skillsDir, skills) {
|
|
|
93
139
|
continue;
|
|
94
140
|
const hasEvals = existsSync(join(skillDir, "evals", "evals.json"));
|
|
95
141
|
const hasBenchmark = existsSync(join(skillDir, "evals", "benchmark.json"));
|
|
96
|
-
skills.push({ plugin, skill, dir: skillDir, hasEvals, hasBenchmark });
|
|
142
|
+
skills.push({ plugin, skill, dir: skillDir, hasEvals, hasBenchmark, origin: classifyOrigin(skillDir, root) });
|
|
97
143
|
}
|
|
98
144
|
}
|
|
99
145
|
//# sourceMappingURL=skill-scanner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-scanner.js","sourceRoot":"","sources":["../../src/eval/skill-scanner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,qDAAqD;AACrD,yDAAyD;AACzD,iEAAiE;AACjE,0EAA0E;AAC1E,uEAAuE;AACvE,8EAA8E;AAC9E,8EAA8E;AAE9E,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"skill-scanner.js","sourceRoot":"","sources":["../../src/eval/skill-scanner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,qDAAqD;AACrD,yDAAyD;AACzD,iEAAiE;AACjE,0EAA0E;AAC1E,uEAAuE;AACvE,8EAA8E;AAC9E,8EAA8E;AAE9E,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAW/D,8EAA8E;AAC9E,yEAAyE;AACzE,6EAA6E;AAC7E,8EAA8E;AAE9E,6EAA6E;AAC7E,MAAM,iBAAiB,GAAG;IACxB,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,eAAe;IACzD,SAAS,EAAE,SAAS,EAAE,QAAQ;CAC/B,CAAC;AAEF,qEAAqE;AACrE,IAAI,kBAAkB,GAAuB,IAAI,CAAC;AAElD,SAAS,oBAAoB;IAC3B,IAAI,kBAAkB;QAAE,OAAO,kBAAkB,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;QACpC,2DAA2D;QAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACpC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,kBAAkB,GAAG,QAAQ,CAAC;IAC9B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,IAAY;IAC3D,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1D,qEAAqE;IACrE,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,IAAI,YAAY,IAAI,oBAAoB,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7D,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,8DAA8D;IAC9D,IAAI,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAErC,iEAAiE;IACjE,sDAAsD;IACtD,2EAA2E;IAC3E,4EAA4E;IAC5E,wDAAwD;IACxD,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAgB;QACxD,IAAI,UAAkB,CAAC;QACvB,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,wFAAwF;YACxF,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,MAAM,IAAI,SAAS,CAAC;QACnC,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CAAC;YACV,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,SAAS;YAChB,GAAG,EAAE,IAAI;YACT,QAAQ;YACR,YAAY;YACZ,MAAM,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC;SACnC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,0EAA0E;IAC1E,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IAE/E,4EAA4E;IAC5E,qFAAqF;IACrF,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;IAE1D,kFAAkF;IAClF,oFAAoF;IACpF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACzC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,+EAA+E;IAC/E,+DAA+D;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC;QAClD,aAAa,CAAC,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,MAAmB,EAAE,OAAiB,EAAE,IAAY;IACvF,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,MAAc,EACd,SAAiB,EACjB,MAAmB,EACnB,IAAY;IAEZ,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO;IAEnC,IAAI,SAAmB,CAAC;IACxB,IAAI,CAAC;QACH,SAAS,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aACxD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE3C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAS;QAEnC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACnE,MAAM,YAAY,GAAG,UAAU,CAC7B,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAC1C,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAChH,CAAC;AACH,CAAC"}
|