promptfoo 0.120.27 → 0.121.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/{ListApp-8WOe2nT6.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-B0pgC1oV.js +206 -0
- package/dist/src/{accounts-DVINui-2.js → accounts-Bm2D8Db9.js} +39 -34
- package/dist/src/{accounts-CPDRAMND.js → accounts-CiBLOnA7.js} +38 -33
- package/dist/src/{accounts-Fl2J3_Fu.cjs → accounts-gtkH-5KX.cjs} +77 -78
- package/dist/src/{agentic-utils-D922n6mm.js → agentic-utils-DS1g3GLF.js} +9 -10
- package/dist/src/{agents-BcsN_BgB.js → agents-9qiOy0ho.js} +16 -12
- package/dist/src/{agents-BXLmVsxR.js → agents-CBr9A01V.js} +37 -37
- package/dist/src/{agents-pMfppv9Z.js → agents-CmvBq8LV.js} +16 -18
- package/dist/src/{agents-hqgSV-3o.js → agents-D__IdAlg.js} +39 -40
- package/dist/src/{agents-BO2n8Z0d.cjs → agents-DbRtpYxR.cjs} +37 -40
- package/dist/src/{agents-BdUTAwi-.js → agents-DgF2zDag.js} +37 -42
- package/dist/src/{agents-DgJf2-ez.cjs → agents-Di9DKPzn.cjs} +16 -17
- package/dist/src/{agents-DNvSH78i.js → agents-cLXA8a_8.js} +17 -19
- package/dist/src/{aimlapi-DtgPI0nE.js → aimlapi-B4rcnZgv.js} +15 -17
- package/dist/src/{aimlapi-BE_Tg9Fl.cjs → aimlapi-BvlNH0gr.cjs} +15 -16
- package/dist/src/{aimlapi-DOib86oE.js → aimlapi-CnkC2HqE.js} +16 -18
- package/dist/src/{aimlapi-DTPACCB1.js → aimlapi-DHJU_kcV.js} +15 -4
- package/dist/src/app/assets/index-4LKxG2CG.js +439 -0
- package/dist/src/app/assets/{index-NCn4eVBv.css → index-C3zcsZFQ.css} +1 -1
- package/dist/src/app/assets/vendor-charts-BnDWwBlI.js +36 -0
- package/dist/src/app/index.html +3 -3
- package/dist/src/app/tsconfig.app.tsbuildinfo +1 -1
- package/dist/src/{audio-BnRUGAm_.js → audio-Bkv46et0.js} +6 -5
- package/dist/src/{audio-Cwo68yZS.cjs → audio-CGMyULza.cjs} +6 -7
- package/dist/src/{audio-MSRki4JU.js → audio-ClI_AFre.js} +6 -8
- package/dist/src/{audio-BRYU0BFo.js → audio-Dz3z7s3J.js} +7 -9
- package/dist/src/{base-pGVmXNl4.cjs → base-CGrhspbK.cjs} +36 -38
- package/dist/src/{base-h961VXYk.js → base-CpjcHe4e.js} +11 -13
- package/dist/src/base-DLKtKMFh.js +193 -0
- package/dist/src/{base-XB2tDJrB.js → base-Dy1V8--Z.js} +11 -13
- package/dist/src/blobs-BDbfYdrJ.js +236 -0
- package/dist/src/{blobs-CR5C4Ihh.js → blobs-CBO20krR.js} +9 -12
- package/dist/src/{blobs-BM_e6hCa.js → blobs-CMHN0Qcz.js} +9 -12
- package/dist/src/{blobs-B-KQAFhX.cjs → blobs-D23XLin-.cjs} +34 -37
- package/dist/src/{cache-jsiwsAJv.js → cache-BVeDlD87.js} +132 -117
- package/dist/src/{cache-CIpsoBZR.js → cache-C4Nxf52C.js} +132 -118
- package/dist/src/cache-CeUpFm3M.cjs +5 -0
- package/dist/src/{cache-BTVYfbka.cjs → cache-Dh5WtQps.cjs} +182 -168
- package/dist/src/cache-i1P6crbO.js +756 -0
- package/dist/src/cache-n-RCJ-hL.js +6 -0
- package/dist/src/{chat-BcPjZXIp.js → chat-BiKyneZl.js} +45 -46
- package/dist/src/{chat-D31K7C4u.cjs → chat-C1Qst7jL.cjs} +20 -21
- package/dist/src/{chat-B84t99NW.js → chat-C2jrdPMx.js} +20 -9
- package/dist/src/{chat-BE44YOc6.cjs → chat-CgF-J-Jj.cjs} +65 -66
- package/dist/src/{chat-DwWifjxi.js → chat-CzkrVDfz.js} +20 -22
- package/dist/src/chat-DJIw17u0.js +766 -0
- package/dist/src/{chat-CcUCysjU.js → chat-DqxYYtWA.js} +45 -46
- package/dist/src/{chat-DZM2GUHO.js → chat-qmatte1u.js} +21 -23
- package/dist/src/{chatkit-D67HS_0b.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DAB_qfzI.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-Biqb_wsD.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-PGG4ZYIn.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-SVM6AdBu.js → claude-agent-sdk-Apiy0iaz.js} +31 -31
- package/dist/src/{claude-agent-sdk-C-IOTPfo.js → claude-agent-sdk-D2bJee9S.js} +31 -29
- package/dist/src/{claude-agent-sdk-C9SiaQub.cjs → claude-agent-sdk-D9Z5Pr9X.cjs} +31 -28
- package/dist/src/{claude-agent-sdk-CiluSyW1.js → claude-agent-sdk-DfCoW0E6.js} +33 -20
- package/dist/src/cloud-BBh91EUK.js +4 -0
- package/dist/src/{cloud-CZ-q9Ier.js → cloud-C0dlstV_.js} +7 -9
- package/dist/src/{cloudflare-ai-BahKHyhh.js → cloudflare-ai-8TDxHR0x.js} +16 -18
- package/dist/src/{cloudflare-ai-v_qZD6_q.js → cloudflare-ai-BxAGvfju.js} +17 -19
- package/dist/src/{cloudflare-ai-Dfahv5SY.cjs → cloudflare-ai-CknbZ5LJ.cjs} +16 -17
- package/dist/src/{cloudflare-ai-Dxyt50Nl.js → cloudflare-ai-g7PB6VHR.js} +16 -4
- package/dist/src/{cloudflare-gateway-Bi_FpOFy.js → cloudflare-gateway-B9HWA5wf.js} +23 -23
- package/dist/src/{cloudflare-gateway-BPWoZIzJ.cjs → cloudflare-gateway-BSnDmHYo.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-C0guUNwk.js → cloudflare-gateway-CKDb4dJ8.js} +26 -14
- package/dist/src/{cloudflare-gateway-btS7h1OZ.js → cloudflare-gateway-CP9QEWYS.js} +21 -25
- package/dist/src/{codex-sdk-DSxAnbfT.js → codex-sdk-C6UMlxwV.js} +28 -29
- package/dist/src/{codex-sdk-IYVi9fuM.js → codex-sdk-DUwKWezN.js} +28 -27
- package/dist/src/{codex-sdk-DulY0ZRq.js → codex-sdk-GGAw0qbD.js} +28 -29
- package/dist/src/{codex-sdk-DFKMtAyf.cjs → codex-sdk-fAO0c3yA.cjs} +28 -29
- package/dist/src/{cometapi-DzrR3SR_.js → cometapi-BL9yvj_f.js} +16 -4
- package/dist/src/{cometapi-DIO64tf4.cjs → cometapi-C4xSqeID.cjs} +21 -22
- package/dist/src/{cometapi-C9EEpJzT.js → cometapi-CUQq3H_a.js} +21 -24
- package/dist/src/{cometapi-DkNBMk0G.js → cometapi-DFNiKmSz.js} +17 -19
- package/dist/src/{completion-CG29bfKX.js → completion-5MzrpJxT.js} +11 -13
- package/dist/src/{completion-CCRT4kX1.cjs → completion-CM6oK8PS.cjs} +21 -23
- package/dist/src/{completion-Bgf1VJoq.js → completion-DZ083F31.js} +11 -13
- package/dist/src/completion-qRoZAYRB.js +120 -0
- package/dist/src/{createHash-Dw_iLu31.js → createHash-CTQmL3G2.js} +2 -3
- package/dist/src/{createHash-CYQy4YeL.cjs → createHash-CfZSc0b4.cjs} +13 -14
- package/dist/src/{createHash-CJcfskIZ.js → createHash-Da8fMwqB.js} +2 -3
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-D-ayp2FW.js → docker-Bb5dcxr8.js} +18 -20
- package/dist/src/{docker-B81N0t4e.js → docker-BvfL2BrW.js} +19 -21
- package/dist/src/{docker-DNcLR4Ig.cjs → docker-DcF2pRrj.cjs} +18 -19
- package/dist/src/{docker-egERKxCF.js → docker-ExVyLp0S.js} +18 -7
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-B9dPm_BF.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-D2pZ87fL.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-Ct-Joyue.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-B3r2CVXr.js +15 -0
- package/dist/src/{eval-C-Nr6wX_.js → eval-Dg2nG4v2.js} +47 -54
- package/dist/src/evalResult-5xwYnECe.js +12 -0
- package/dist/src/evalResult-71lY93Kj.cjs +10 -0
- package/dist/src/{evalResult-DXMWJ3sx.js → evalResult-BBRNtX4I.js} +10 -11
- package/dist/src/{evalResult-4BzI2tmj.js → evalResult-BDMqrapS.js} +16 -12
- package/dist/src/evalResult-Dx5P5cIv.js +10 -0
- package/dist/src/{evalResult-CX8wQecI.cjs → evalResult-fuaI8HkH.cjs} +20 -21
- package/dist/src/{evaluator-8aGyV12L.js → evaluator-BhoWwp5b.js} +211 -235
- package/dist/src/evaluator-Jx6bRZV6.js +36 -0
- package/dist/src/{extractor-V5x_m1i0.js → extractor-C0EVHewb.js} +22 -24
- package/dist/src/extractor-D25qpmGX.js +374 -0
- package/dist/src/{extractor-CD5yKL-G.js → extractor-DReVID0K.js} +22 -24
- package/dist/src/{extractor-C031XmTA.cjs → extractor-pYLLi3wS.cjs} +37 -39
- package/dist/src/{fetch-BmbD-v1L.cjs → fetch-BPkYtG8K.cjs} +244 -277
- package/dist/src/fetch-BxNb_Lp3.js +5 -0
- package/dist/src/{fetch-D3OHf-lV.js → fetch-Cwxnd8zz.js} +36 -44
- package/dist/src/{fetch-CXZI9RRr.js → fetch-Dxpd4_sr.js} +23 -35
- package/dist/src/fetch-HaqdX7U1.js +780 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-CdevI3An.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-Dres3qrN.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/{graders--1y2u9HO.js → graders-BTeBGqjJ.js} +349 -397
- package/dist/src/graders-B_pgMLS2.js +34 -0
- package/dist/src/{graders-DTeBrzWp.js → graders-Bj_Odv7c.js} +349 -397
- package/dist/src/graders-DErokPDO.cjs +32 -0
- package/dist/src/graders-DP7KFFo-.js +13466 -0
- package/dist/src/graders-DR_uNe54.js +32 -0
- package/dist/src/{graders-DohM2dir.cjs → graders-DU49_J8Y.cjs} +684 -732
- package/dist/src/graders-w3176Wz-.js +32 -0
- package/dist/src/{image-B0U4Hqll.js → image-B02ogr_b.js} +7 -9
- package/dist/src/{image-DmE-niFE.js → image-B0h9VEMc.js} +6 -5
- package/dist/src/{image-CuKHuccK.cjs → image-BLmROtN3.cjs} +29 -30
- package/dist/src/{image-DNEIf_aI.js → image-Bb4vWQLM.js} +6 -8
- package/dist/src/{image-DpKl2F15.cjs → image-C1madmKh.cjs} +6 -7
- package/dist/src/{image-C3wHC9_h.js → image-CHfWvljl.js} +9 -10
- package/dist/src/{image-O1u4bCFg.js → image-DS-o-0ph.js} +9 -10
- package/dist/src/image-Dpxa1Jt6.js +257 -0
- package/dist/src/index.cjs +615 -695
- package/dist/src/index.d.cts +271 -7
- package/dist/src/index.d.ts +271 -3
- package/dist/src/index.js +580 -664
- package/dist/src/{interactiveCheck-Bxj1Swex.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-CEzQobWX.js → knowledgeBase-B3OoKIej.js} +14 -9
- package/dist/src/{knowledgeBase-Be_zyW4L.js → knowledgeBase-CYTLHOt1.js} +16 -16
- package/dist/src/{knowledgeBase-BZ41IFwq.js → knowledgeBase-D33Ty2l6.js} +14 -18
- package/dist/src/{knowledgeBase-D-5BMXlr.cjs → knowledgeBase-DOO_BM9b.cjs} +14 -15
- package/dist/src/{litellm-DnbRJ2if.js → litellm-AaeZcZQF.js} +18 -19
- package/dist/src/{litellm-hUSNM_M2.cjs → litellm-I_hbp_dc.cjs} +17 -17
- package/dist/src/{litellm-CRDqPhNI.js → litellm-NbjknEh6.js} +17 -18
- package/dist/src/{litellm-9vR8zpfU.js → litellm-TrljxD9G.js} +17 -5
- package/dist/src/{logger-CG1uZPbQ.js → logger-CT3IKMKA.js} +10 -29
- package/dist/src/{logger-B7sBeGa0.cjs → logger-Cp1GPUjj.cjs} +152 -180
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-LSBxlt7a.js → logger-KkObSCzq.js} +13 -31
- package/dist/src/{luma-ray-4blv9iZ2.js → luma-ray-BS2_tY8L.js} +22 -21
- package/dist/src/{luma-ray-drvgdpP9.js → luma-ray-DDsjcgZZ.js} +20 -13
- package/dist/src/{luma-ray-Hm3d6VJE.cjs → luma-ray-Due0n7di.cjs} +20 -21
- package/dist/src/{luma-ray-B2__8lYH.js → luma-ray-f6I2fft-.js} +20 -23
- package/dist/src/main.js +1170 -1321
- package/dist/src/{messages-Uee41Mj5.js → messages-BS17jdMx.js} +22 -24
- package/dist/src/{messages-XhiwCbi4.cjs → messages-Bs1kC7P4.cjs} +32 -34
- package/dist/src/{messages-CGPPidQr.js → messages-D0lx5qK7.js} +22 -24
- package/dist/src/messages-ZJk778GH.js +240 -0
- package/dist/src/{meteor-BYykdXrV.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-CsopaHrH.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-e-E-2vVl.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-C8lGP6P4.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/{modelslab-yKz-ZNB4.js → modelslab-Bmni6skY.js} +17 -10
- package/dist/src/{modelslab-E9gO-bYd.js → modelslab-Bx9IrZfS.js} +18 -20
- package/dist/src/{modelslab-lUVW0cmB.cjs → modelslab-CoUX6Jc_.cjs} +17 -18
- package/dist/src/{modelslab-ClBkr8_9.js → modelslab-DRb74SP4.js} +17 -19
- package/dist/src/{nova-reel-Dk8jNpId.js → nova-reel-BfPq-0Yk.js} +20 -13
- package/dist/src/{nova-reel-D8CuO6QH.cjs → nova-reel-C_QM18Xn.cjs} +20 -21
- package/dist/src/{nova-reel-u2eF2Cxm.js → nova-reel-D_W1tjMH.js} +22 -21
- package/dist/src/{nova-reel-P9bwvtYX.js → nova-reel-bgjxilYW.js} +20 -23
- package/dist/src/{nova-sonic-CK2rAiKi.js → nova-sonic-CFb5GYhg.js} +30 -26
- package/dist/src/{nova-sonic-BaqWlkds.js → nova-sonic-DIGQNR07.js} +30 -31
- package/dist/src/{nova-sonic-yZapPLv7.js → nova-sonic-De1HW5fD.js} +31 -32
- package/dist/src/{nova-sonic-Ds1C-dpm.cjs → nova-sonic-zfcljeRp.cjs} +30 -31
- package/dist/src/{openai-DUFopMrH.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-PblZ3jUE.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CcN1B8Sb.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-B6qqDr_u.cjs → openclaw-CSugPYAr.cjs} +188 -130
- package/dist/src/{openclaw-A-3_loM7.js → openclaw-DiSz3I5L.js} +180 -109
- package/dist/src/{openclaw-a3lylB-V.js → openclaw-DuvJKEW5.js} +178 -124
- package/dist/src/{openclaw-COn6QzDi.js → openclaw-tiVYRtr-.js} +178 -122
- package/dist/src/opencode-sdk-0j6rTWNb.js +562 -0
- package/dist/src/opencode-sdk-B3CWY9h_.js +560 -0
- package/dist/src/opencode-sdk-BL764Jdi.cjs +564 -0
- package/dist/src/opencode-sdk-C2y6UkP2.js +560 -0
- package/dist/src/{otlpReceiver-oyf5wLGC.js → otlpReceiver-C99PPb48.js} +53 -51
- package/dist/src/{otlpReceiver-lXsYVbpj.cjs → otlpReceiver-CGq6LspY.cjs} +53 -55
- package/dist/src/{otlpReceiver-94URx7UW.js → otlpReceiver-CdNBdbsk.js} +53 -55
- package/dist/src/{otlpReceiver-BmmTiMjA.js → otlpReceiver-D89fR-rC.js} +53 -55
- package/dist/src/{providerRegistry-Cq_JK_CJ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-DSSHjMKf.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-CvHEVJad.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B7V0njNs.js +32 -0
- package/dist/src/providers-BEwbhv0X.js +30 -0
- package/dist/src/{providers-Iil64vk9.js → providers-BlqUifFg.js} +1543 -1676
- package/dist/src/providers-CH3C7zf7.js +30 -0
- package/dist/src/{providers-DHbjzW2e.cjs → providers-CgKOSgTR.cjs} +1896 -2029
- package/dist/src/providers-D8lF1sqW.js +33246 -0
- package/dist/src/{providers-BnFpbY_s.js → providers-Dk_6ocUX.js} +1536 -1669
- package/dist/src/providers-zyB6k_38.cjs +31 -0
- package/dist/src/{pythonUtils-CcT5LH1M.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DBbuI3QJ.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-hZ8LeQLv.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/{quiverai-BuI0tE39.js → quiverai-BbOUOn2L.js} +8 -7
- package/dist/src/{quiverai-DCGSZt4U.js → quiverai-CIaELU_m.js} +8 -10
- package/dist/src/{quiverai-DiMVJQDz.cjs → quiverai-PdShCPox.cjs} +8 -9
- package/dist/src/{quiverai-fQNkExW4.js → quiverai-uH-dcTIr.js} +9 -11
- package/dist/src/{render-Dj1smHEb.js → render-Drod8m7K.js} +4 -5
- package/dist/src/responses-CB2jwoAr.js +660 -0
- package/dist/src/{responses-ghR3IOfy.cjs → responses-D8SBTL64.cjs} +39 -42
- package/dist/src/{responses-DOAFFENS.js → responses-DIR9Ud3j.js} +24 -27
- package/dist/src/{responses-CxzoQoBe.js → responses-WNGNYe3K.js} +24 -27
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-CwbGmgYN.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-DudlFZed.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-C8MhKGHb.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-gmskuyre.js → sagemaker-CNBxx5CJ.js} +75 -70
- package/dist/src/{sagemaker-CcxhlOAR.js → sagemaker-CemTFp2h.js} +75 -79
- package/dist/src/{sagemaker-77zbJ2Q2.cjs → sagemaker-Cl28mZU2.cjs} +75 -76
- package/dist/src/{sagemaker-DuM71dVU.js → sagemaker-YSyBXQQh.js} +77 -77
- package/dist/src/{scanner-DJYiSXQj.js → scanner-BsBlNXNn.js} +100 -121
- package/dist/src/server/index.js +5520 -67427
- package/dist/src/{server-B5v33lvE.cjs → server-C_7Ax-hA.cjs} +57 -67
- package/dist/src/{server-BJ4m4f1D.js → server-CqzrVGpF.js} +26 -29
- package/dist/src/server-CuxBbeSY.js +229 -0
- package/dist/src/server-DA4Cyrrq.js +7 -0
- package/dist/src/server-Dulb-4-K.cjs +5 -0
- package/dist/src/{server-RV_i_YX5.js → server-VWgWb00X.js} +19 -24
- package/dist/src/{signal-BW33JuId.js → signal-4U3mfRvL.js} +9 -11
- package/dist/src/{slack-DEURelTy.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-BQYeW9L3.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-BB6yuZzp.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-2pRrhhgJ.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-CXGFv4aR.js +228 -0
- package/dist/src/store-CXS-Q_91.js +6 -0
- package/dist/src/{store-D7CgQzAR.cjs → store-DLlFCC4h.cjs} +44 -45
- package/dist/src/{store-DJNsD1iC.js → store-DXilxTl-.js} +40 -36
- package/dist/src/{store-s3SftUwF.js → store-Dim__MDd.js} +34 -35
- package/dist/src/store-eYkaKMwq.cjs +5 -0
- package/dist/src/{tables-DfTsNN7X.js → tables-6YKwjN9-.js} +19 -21
- package/dist/src/tables-DLJPUdUE.js +288 -0
- package/dist/src/{tables-BKTmd6u7.cjs → tables-DPi7wKeM.cjs} +89 -91
- package/dist/src/{tables-DMegD0Xf.js → tables-gftXzE9I.js} +21 -23
- package/dist/src/telemetry-BpMfhthR.cjs +5 -0
- package/dist/src/{telemetry--WAdAfVi.js → telemetry-CMrFgtPB.js} +11 -13
- package/dist/src/telemetry-Cps3mIU-.js +171 -0
- package/dist/src/{telemetry-DQgVBCAb.cjs → telemetry-DaX14Chu.cjs} +21 -24
- package/dist/src/{telemetry-BedSm-bZ.js → telemetry-Dthj_BbD.js} +17 -14
- package/dist/src/telemetry-Dw38hanS.js +7 -0
- package/dist/src/{text-oiSbwSOI.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-oKzCBnK6.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-B_IrO4GZ.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-mYS9vd5v.js → transcription-BNYURcXg.js} +14 -7
- package/dist/src/{transcription-X2-B4vkX.js → transcription-B_OdaHp7.js} +14 -16
- package/dist/src/{transcription-BO1AHegO.cjs → transcription-NLVG9MT1.cjs} +14 -15
- package/dist/src/{transcription-lzBLiTFJ.js → transcription-s6A-bNrZ.js} +15 -17
- package/dist/src/{transform-B1Hi5lWS.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-DeGlxb0D.js → transform-D5HsjduX.js} +39 -47
- package/dist/src/{transform-CYDILYDe.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-BEgStbHK.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/{transform-D5PjiWiZ.cjs → transform-DuHvhZpj.cjs} +179 -187
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/transform-uAytVuyX.js +1506 -0
- package/dist/src/{transform-Dfl89yi4.js → transform-vNucnNr0.js} +39 -47
- package/dist/src/{transformersAvailability-SZnTS3pJ.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-D-glmEy7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-CjeFXhuJ.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-DWNf48sT.cjs → types-C_7nyzr1.cjs} +538 -574
- package/dist/src/{types-CXQduE9o.js → types-Cbd8uOMq.js} +68 -100
- package/dist/src/types-CzW2QFyi.js +3288 -0
- package/dist/src/{types-C5hEkb-x.js → types-DmyIJ-sR.js} +63 -99
- package/dist/src/{util-CoQjmE3u.js → util-B3xGByQh.js} +4 -5
- package/dist/src/{util-aLhtl3fe.cjs → util-B9vlHIIh.cjs} +208 -223
- package/dist/src/{util-Du96oyYS.js → util-BHGHw5G1.js} +4 -5
- package/dist/src/{util-DQ984syk.js → util-BRYkYPTd.js} +36 -51
- package/dist/src/{util-D9eLdGfa.js → util-BV4XUC0n.js} +5 -6
- package/dist/src/util-Bv6uGDfH.js +293 -0
- package/dist/src/{util-1wWM599Z.cjs → util-BzMcevZc.cjs} +50 -51
- package/dist/src/{util-_h4pVqrz.js → util-C1CeHl-P.js} +36 -51
- package/dist/src/{util-Bm_-UMD_.js → util-CMy69ZgQ.js} +5 -6
- package/dist/src/{util-CyUdMzV0.cjs → util-DGNOS1db.cjs} +34 -35
- package/dist/src/util-Dnmk2mBQ.js +599 -0
- package/dist/src/util-ZzmqNPlg.js +1426 -0
- package/dist/src/{utils-BjLy-Q72.cjs → utils-Cz9qXqII.cjs} +29 -32
- package/dist/src/{utils-CFMn2yHW.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-DvWMzuMx.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +38 -38
- package/dist/src/app/assets/index-B2D0bCSI.js +0 -439
- package/dist/src/app/assets/vendor-charts-CCl15Imd.js +0 -36
- package/dist/src/cache-ChPcurj7.js +0 -6
- package/dist/src/cache-VVu_W-yg.js +0 -8
- package/dist/src/cache-YLNCFEM2.cjs +0 -6
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-BndfXy4H.js +0 -5
- package/dist/src/eval-BhHvMY82.js +0 -17
- package/dist/src/evalResult-Dq2gFNQY.js +0 -12
- package/dist/src/evalResult-nmcP5VKH.cjs +0 -12
- package/dist/src/evalResult-trqZjVYh.js +0 -14
- package/dist/src/evaluator-CnfPstzT.js +0 -39
- package/dist/src/fetch-IDPDue6F.cjs +0 -4
- package/dist/src/fetch-hKJ-It8q.js +0 -6
- package/dist/src/fetch-ouKnrWK-.js +0 -4
- package/dist/src/graders-CQn7WUsd.cjs +0 -34
- package/dist/src/graders-DC6QAbpW.js +0 -35
- package/dist/src/graders-DUWz3Y7j.js +0 -37
- package/dist/src/opencode-sdk-4bL9n-Gk.js +0 -382
- package/dist/src/opencode-sdk-BfC2zWcR.js +0 -376
- package/dist/src/opencode-sdk-DMJyuwMg.js +0 -380
- package/dist/src/opencode-sdk-Da-9adza.cjs +0 -383
- package/dist/src/providers-CsXB2Ix-.js +0 -35
- package/dist/src/providers-DO8ltjLC.js +0 -33
- package/dist/src/providers-Dtq-xnXd.cjs +0 -33
- package/dist/src/rubyUtils-BUbcND2f.js +0 -6
- package/dist/src/rubyUtils-Cr55X_KE.js +0 -5
- package/dist/src/rubyUtils-DlIiqoYo.cjs +0 -5
- package/dist/src/server-C2eQH4Gu.js +0 -6
- package/dist/src/server-CXWycu7H.cjs +0 -6
- package/dist/src/server-Q6OGlxxT.js +0 -8
- package/dist/src/store-B3EDO9Q3.js +0 -7
- package/dist/src/store-Dl9F8aw5.js +0 -6
- package/dist/src/store-SnrGrlt9.cjs +0 -6
- package/dist/src/telemetry-BGhiPZtl.js +0 -8
- package/dist/src/telemetry-CFfiYan6.cjs +0 -6
- package/dist/src/telemetry-DHzEduxX.js +0 -6
- package/dist/src/transform-C1x1ZlMQ.cjs +0 -6
- package/dist/src/transform-DYHjFmQu.js +0 -8
- package/dist/src/transform-rmwJT5JQ.js +0 -7
- package/dist/src/transformersAvailability-eJooj0gX.js +0 -35
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const require_logger = require(
|
|
2
|
-
const require_invariant = require(
|
|
3
|
-
const require_esm = require(
|
|
4
|
-
const require_pythonUtils = require(
|
|
5
|
-
const require_fileExtensions = require(
|
|
6
|
-
const require_types = require(
|
|
7
|
-
const require_fetch = require(
|
|
1
|
+
const require_logger = require("./logger-Cp1GPUjj.cjs");
|
|
2
|
+
const require_invariant = require("./invariant-kfQ8Bu82.cjs");
|
|
3
|
+
const require_esm = require("./esm-CnNt7sI4.cjs");
|
|
4
|
+
const require_pythonUtils = require("./pythonUtils-CTU3Y3lw.cjs");
|
|
5
|
+
const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
|
|
6
|
+
const require_types = require("./types-C_7nyzr1.cjs");
|
|
7
|
+
const require_fetch = require("./fetch-BPkYtG8K.cjs");
|
|
8
8
|
let fs = require("fs");
|
|
9
9
|
fs = require_logger.__toESM(fs);
|
|
10
10
|
let path = require("path");
|
|
@@ -27,7 +27,6 @@ let nunjucks = require("nunjucks");
|
|
|
27
27
|
nunjucks = require_logger.__toESM(nunjucks);
|
|
28
28
|
let fast_xml_parser = require("fast-xml-parser");
|
|
29
29
|
let csv_stringify_sync = require("csv-stringify/sync");
|
|
30
|
-
|
|
31
30
|
//#region src/util/provider.ts
|
|
32
31
|
function canonicalizeProviderId(id) {
|
|
33
32
|
if (id.startsWith("file://")) {
|
|
@@ -198,7 +197,6 @@ function isGoogleProvider(providerId) {
|
|
|
198
197
|
}
|
|
199
198
|
return false;
|
|
200
199
|
}
|
|
201
|
-
|
|
202
200
|
//#endregion
|
|
203
201
|
//#region src/util/comparison.ts
|
|
204
202
|
/**
|
|
@@ -297,7 +295,7 @@ function resultIsForTestCase(result, testCase) {
|
|
|
297
295
|
const isMatch = doVarsMatch && providersMatch;
|
|
298
296
|
if (!isMatch) {
|
|
299
297
|
const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
|
|
300
|
-
require_logger.
|
|
298
|
+
require_logger.logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
|
|
301
299
|
testProvider: testProviderId || "none",
|
|
302
300
|
resultProvider: resultProviderId || "none",
|
|
303
301
|
testVarKeys: varKeys
|
|
@@ -305,7 +303,6 @@ function resultIsForTestCase(result, testCase) {
|
|
|
305
303
|
}
|
|
306
304
|
return isMatch;
|
|
307
305
|
}
|
|
308
|
-
|
|
309
306
|
//#endregion
|
|
310
307
|
//#region src/util/env.ts
|
|
311
308
|
/**
|
|
@@ -322,8 +319,8 @@ function setupEnv(envPath) {
|
|
|
322
319
|
return;
|
|
323
320
|
}
|
|
324
321
|
for (const p of paths) if (!fs.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
|
|
325
|
-
if (paths.length === 1) require_logger.
|
|
326
|
-
else require_logger.
|
|
322
|
+
if (paths.length === 1) require_logger.logger.info(`Loading environment variables from ${paths[0]}`);
|
|
323
|
+
else require_logger.logger.info(`Loading environment variables from: ${paths.join(", ")}`);
|
|
327
324
|
const pathArg = paths.length === 1 ? paths[0] : paths;
|
|
328
325
|
dotenv.default.config({
|
|
329
326
|
path: pathArg,
|
|
@@ -332,7 +329,6 @@ function setupEnv(envPath) {
|
|
|
332
329
|
});
|
|
333
330
|
} else dotenv.default.config({ quiet: true });
|
|
334
331
|
}
|
|
335
|
-
|
|
336
332
|
//#endregion
|
|
337
333
|
//#region src/util/functions/loadFunction.ts
|
|
338
334
|
const functionCache = {};
|
|
@@ -341,7 +337,7 @@ const functionCache = {};
|
|
|
341
337
|
* @param options Options for loading the function
|
|
342
338
|
* @returns The loaded function
|
|
343
339
|
*/
|
|
344
|
-
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.
|
|
340
|
+
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.state.basePath, useCache = true }) {
|
|
345
341
|
const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
|
|
346
342
|
if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
|
|
347
343
|
const resolvedPath = basePath ? path.default.resolve(basePath, filePath) : filePath;
|
|
@@ -362,7 +358,7 @@ async function loadFunction({ filePath, functionName, defaultFunctionName = "fun
|
|
|
362
358
|
if (useCache) functionCache[cacheKey] = func;
|
|
363
359
|
return func;
|
|
364
360
|
} catch (err) {
|
|
365
|
-
require_logger.
|
|
361
|
+
require_logger.logger.error(`Failed to load function: ${err.message}`);
|
|
366
362
|
throw err;
|
|
367
363
|
}
|
|
368
364
|
}
|
|
@@ -381,7 +377,6 @@ function parseFileUrl(fileUrl) {
|
|
|
381
377
|
};
|
|
382
378
|
return { filePath: urlWithoutProtocol };
|
|
383
379
|
}
|
|
384
|
-
|
|
385
380
|
//#endregion
|
|
386
381
|
//#region src/util/templates.ts
|
|
387
382
|
/**
|
|
@@ -400,7 +395,7 @@ function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false)
|
|
|
400
395
|
});
|
|
401
396
|
const envGlobals = {
|
|
402
397
|
...require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", require_logger.getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
|
|
403
|
-
...require_logger.
|
|
398
|
+
...require_logger.state.config?.env
|
|
404
399
|
};
|
|
405
400
|
env.addGlobal("env", envGlobals);
|
|
406
401
|
env.addFilter("load", function(str) {
|
|
@@ -440,7 +435,6 @@ function extractVariablesFromTemplates(templates) {
|
|
|
440
435
|
for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
|
|
441
436
|
return Array.from(variableSet);
|
|
442
437
|
}
|
|
443
|
-
|
|
444
438
|
//#endregion
|
|
445
439
|
//#region src/util/render.ts
|
|
446
440
|
/**
|
|
@@ -469,10 +463,6 @@ function extractVariablesFromTemplates(templates) {
|
|
|
469
463
|
function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
470
464
|
if (require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
|
|
471
465
|
if (typeof obj === "string") {
|
|
472
|
-
if (obj.length > 5e4) {
|
|
473
|
-
require_logger.logger_default.warn(`String too long (${obj.length} chars) for template matching. Skipping env var rendering.`);
|
|
474
|
-
return obj;
|
|
475
|
-
}
|
|
476
466
|
const nunjucks = getNunjucksEngine();
|
|
477
467
|
const baseEnvGlobals = nunjucks.getGlobal("env");
|
|
478
468
|
const envGlobals = replaceBase ? envOverrides ?? {} : envOverrides ? {
|
|
@@ -486,7 +476,7 @@ function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
|
486
476
|
if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
|
|
487
477
|
return nunjucks.renderString(match, { env: envGlobals });
|
|
488
478
|
} catch (error) {
|
|
489
|
-
require_logger.
|
|
479
|
+
require_logger.logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
|
|
490
480
|
return match;
|
|
491
481
|
}
|
|
492
482
|
return match;
|
|
@@ -511,7 +501,6 @@ function renderVarsInObject(obj, vars) {
|
|
|
511
501
|
} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
|
|
512
502
|
return obj;
|
|
513
503
|
}
|
|
514
|
-
|
|
515
504
|
//#endregion
|
|
516
505
|
//#region src/util/file.ts
|
|
517
506
|
/**
|
|
@@ -522,7 +511,7 @@ function getNunjucksEngineForFilePath() {
|
|
|
522
511
|
const env = nunjucks.default.configure({ autoescape: false });
|
|
523
512
|
env.addGlobal("env", {
|
|
524
513
|
...process.env,
|
|
525
|
-
...require_logger.
|
|
514
|
+
...require_logger.state.config?.env
|
|
526
515
|
});
|
|
527
516
|
return env;
|
|
528
517
|
}
|
|
@@ -541,24 +530,24 @@ function getNunjucksEngineForFilePath() {
|
|
|
541
530
|
* @throws {Error} If the specified file does not exist.
|
|
542
531
|
*/
|
|
543
532
|
function maybeLoadFromExternalFile(filePath, context) {
|
|
544
|
-
if (Array.isArray(filePath)) return filePath.map((path) => {
|
|
545
|
-
return maybeLoadFromExternalFile(path, context);
|
|
533
|
+
if (Array.isArray(filePath)) return filePath.map((path$2) => {
|
|
534
|
+
return maybeLoadFromExternalFile(path$2, context);
|
|
546
535
|
});
|
|
547
536
|
if (typeof filePath !== "string") return filePath;
|
|
548
537
|
if (!filePath.startsWith("file://")) return filePath;
|
|
549
538
|
const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
|
|
550
539
|
const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
|
|
551
540
|
if (context === "assertion" && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) {
|
|
552
|
-
require_logger.
|
|
541
|
+
require_logger.logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
|
|
553
542
|
return renderedFilePath;
|
|
554
543
|
}
|
|
555
544
|
if (context === "vars") {
|
|
556
|
-
require_logger.
|
|
545
|
+
require_logger.logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
|
|
557
546
|
return renderedFilePath;
|
|
558
547
|
}
|
|
559
548
|
if (functionName && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) return renderedFilePath;
|
|
560
549
|
const pathToUse = functionName && !(cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
|
|
561
|
-
const resolvedPath = path.resolve(require_logger.
|
|
550
|
+
const resolvedPath = path.resolve(require_logger.state.basePath || "", pathToUse);
|
|
562
551
|
if ((0, glob.hasMagic)(pathToUse)) {
|
|
563
552
|
const matchedFiles = (0, glob.globSync)(resolvedPath, { windowsPathsNoEscape: true });
|
|
564
553
|
if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
|
|
@@ -569,7 +558,7 @@ function maybeLoadFromExternalFile(filePath, context) {
|
|
|
569
558
|
contents = fs.readFileSync(matchedFile, "utf8");
|
|
570
559
|
} catch (error) {
|
|
571
560
|
if (error.code === "ENOENT") {
|
|
572
|
-
require_logger.
|
|
561
|
+
require_logger.logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
|
|
573
562
|
continue;
|
|
574
563
|
}
|
|
575
564
|
throw error;
|
|
@@ -779,37 +768,37 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
779
768
|
const { filePath, functionName } = parseFileUrl(rendered);
|
|
780
769
|
if (functionName && (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath))) {
|
|
781
770
|
const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
782
|
-
require_logger.
|
|
771
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
|
|
783
772
|
try {
|
|
784
773
|
let toolDefinitions;
|
|
785
774
|
if (filePath.endsWith(".py")) {
|
|
786
|
-
const absPath = require_esm.safeResolve(require_logger.
|
|
787
|
-
require_logger.
|
|
775
|
+
const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
|
|
776
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
|
|
788
777
|
toolDefinitions = await require_pythonUtils.runPython(absPath, functionName, []);
|
|
789
778
|
} else {
|
|
790
|
-
const absPath = require_esm.safeResolve(require_logger.
|
|
791
|
-
require_logger.
|
|
779
|
+
const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
|
|
780
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
|
|
792
781
|
const module = await require_esm.importModule(absPath);
|
|
793
782
|
const fn = module[functionName] || module.default?.[functionName];
|
|
794
783
|
if (typeof fn !== "function") {
|
|
795
784
|
const availableExports = Object.keys(module).filter((k) => k !== "default");
|
|
796
|
-
const basePath = require_logger.
|
|
785
|
+
const basePath = require_logger.state.basePath || process.cwd();
|
|
797
786
|
throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
|
|
798
787
|
}
|
|
799
788
|
toolDefinitions = await Promise.resolve(fn());
|
|
800
789
|
}
|
|
801
790
|
if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
|
|
802
|
-
require_logger.
|
|
791
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
|
|
803
792
|
return toolDefinitions;
|
|
804
793
|
} catch (err) {
|
|
805
794
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
806
|
-
const basePath = require_logger.
|
|
795
|
+
const basePath = require_logger.state.basePath || process.cwd();
|
|
807
796
|
throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
|
|
808
797
|
}
|
|
809
798
|
}
|
|
810
799
|
if (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath)) {
|
|
811
800
|
const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
812
|
-
const basePath = require_logger.
|
|
801
|
+
const basePath = require_logger.state.basePath || process.cwd();
|
|
813
802
|
throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? ` def get_tools():\n return [{"type": "function", "function": {...}}]` : ` module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
|
|
814
803
|
}
|
|
815
804
|
}
|
|
@@ -827,7 +816,6 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
827
816
|
}
|
|
828
817
|
return loaded;
|
|
829
818
|
}
|
|
830
|
-
|
|
831
819
|
//#endregion
|
|
832
820
|
//#region src/googleSheets.ts
|
|
833
821
|
async function checkGoogleSheetAccess(url) {
|
|
@@ -842,7 +830,7 @@ async function checkGoogleSheetAccess(url) {
|
|
|
842
830
|
status: response.status
|
|
843
831
|
};
|
|
844
832
|
} catch (error) {
|
|
845
|
-
require_logger.
|
|
833
|
+
require_logger.logger.error(`Error checking sheet access: ${error}`);
|
|
846
834
|
return { public: false };
|
|
847
835
|
}
|
|
848
836
|
}
|
|
@@ -894,7 +882,7 @@ async function fetchCsvFromGoogleSheetAuthenticated(url) {
|
|
|
894
882
|
}
|
|
895
883
|
async function fetchCsvFromGoogleSheet(url) {
|
|
896
884
|
const { public: isPublic } = await checkGoogleSheetAccess(url);
|
|
897
|
-
require_logger.
|
|
885
|
+
require_logger.logger.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
|
|
898
886
|
if (isPublic) return fetchCsvFromGoogleSheetUnauthenticated(url);
|
|
899
887
|
return fetchCsvFromGoogleSheetAuthenticated(url);
|
|
900
888
|
}
|
|
@@ -937,7 +925,7 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
937
925
|
});
|
|
938
926
|
range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
|
|
939
927
|
}
|
|
940
|
-
require_logger.
|
|
928
|
+
require_logger.logger.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
|
|
941
929
|
await sheets.spreadsheets.values.update({
|
|
942
930
|
spreadsheetId,
|
|
943
931
|
range,
|
|
@@ -946,7 +934,6 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
946
934
|
requestBody: { values }
|
|
947
935
|
});
|
|
948
936
|
}
|
|
949
|
-
|
|
950
937
|
//#endregion
|
|
951
938
|
//#region src/server/utils/evalTableUtils.ts
|
|
952
939
|
/**
|
|
@@ -1145,7 +1132,6 @@ async function streamEvalCsv(eval_, options) {
|
|
|
1145
1132
|
isRedteam
|
|
1146
1133
|
})]));
|
|
1147
1134
|
}
|
|
1148
|
-
|
|
1149
1135
|
//#endregion
|
|
1150
1136
|
//#region src/util/output.ts
|
|
1151
1137
|
const outputToSimpleString = (output) => {
|
|
@@ -1210,7 +1196,7 @@ async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
|
|
|
1210
1196
|
const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
|
|
1211
1197
|
if (isStringLen || isHeapOOM) {
|
|
1212
1198
|
const resultCount = await evalRecord.getResultsCount();
|
|
1213
|
-
require_logger.
|
|
1199
|
+
require_logger.logger.error(`Dataset too large for JSON export (${resultCount} results).`);
|
|
1214
1200
|
throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
|
|
1215
1201
|
} else throw error;
|
|
1216
1202
|
}
|
|
@@ -1229,7 +1215,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1229
1215
|
});
|
|
1230
1216
|
return csvRow;
|
|
1231
1217
|
});
|
|
1232
|
-
require_logger.
|
|
1218
|
+
require_logger.logger.info(`Writing ${rows.length} rows to Google Sheets...`);
|
|
1233
1219
|
await writeCsvToGoogleSheet(rows, outputPath);
|
|
1234
1220
|
return;
|
|
1235
1221
|
}
|
|
@@ -1311,217 +1297,216 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1311
1297
|
async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
|
|
1312
1298
|
await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
|
|
1313
1299
|
}
|
|
1314
|
-
|
|
1315
1300
|
//#endregion
|
|
1316
1301
|
//#region src/util/runtime.ts
|
|
1317
1302
|
function printBorder() {
|
|
1318
1303
|
const border = "=".repeat(require_fetch.TERMINAL_MAX_WIDTH);
|
|
1319
|
-
require_logger.
|
|
1304
|
+
require_logger.logger.info(border);
|
|
1320
1305
|
}
|
|
1321
|
-
|
|
1322
1306
|
//#endregion
|
|
1323
|
-
Object.defineProperty(exports,
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1307
|
+
Object.defineProperty(exports, "checkProviderApiKeys", {
|
|
1308
|
+
enumerable: true,
|
|
1309
|
+
get: function() {
|
|
1310
|
+
return checkProviderApiKeys;
|
|
1311
|
+
}
|
|
1328
1312
|
});
|
|
1329
|
-
Object.defineProperty(exports,
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1313
|
+
Object.defineProperty(exports, "deduplicateTestCases", {
|
|
1314
|
+
enumerable: true,
|
|
1315
|
+
get: function() {
|
|
1316
|
+
return deduplicateTestCases;
|
|
1317
|
+
}
|
|
1334
1318
|
});
|
|
1335
|
-
Object.defineProperty(exports,
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1319
|
+
Object.defineProperty(exports, "doesProviderRefMatch", {
|
|
1320
|
+
enumerable: true,
|
|
1321
|
+
get: function() {
|
|
1322
|
+
return doesProviderRefMatch;
|
|
1323
|
+
}
|
|
1340
1324
|
});
|
|
1341
|
-
Object.defineProperty(exports,
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1325
|
+
Object.defineProperty(exports, "extractRuntimeVars", {
|
|
1326
|
+
enumerable: true,
|
|
1327
|
+
get: function() {
|
|
1328
|
+
return extractRuntimeVars;
|
|
1329
|
+
}
|
|
1346
1330
|
});
|
|
1347
|
-
Object.defineProperty(exports,
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1331
|
+
Object.defineProperty(exports, "extractVariablesFromTemplate", {
|
|
1332
|
+
enumerable: true,
|
|
1333
|
+
get: function() {
|
|
1334
|
+
return extractVariablesFromTemplate;
|
|
1335
|
+
}
|
|
1352
1336
|
});
|
|
1353
|
-
Object.defineProperty(exports,
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1337
|
+
Object.defineProperty(exports, "extractVariablesFromTemplates", {
|
|
1338
|
+
enumerable: true,
|
|
1339
|
+
get: function() {
|
|
1340
|
+
return extractVariablesFromTemplates;
|
|
1341
|
+
}
|
|
1358
1342
|
});
|
|
1359
|
-
Object.defineProperty(exports,
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1343
|
+
Object.defineProperty(exports, "fetchCsvFromGoogleSheet", {
|
|
1344
|
+
enumerable: true,
|
|
1345
|
+
get: function() {
|
|
1346
|
+
return fetchCsvFromGoogleSheet;
|
|
1347
|
+
}
|
|
1364
1348
|
});
|
|
1365
|
-
Object.defineProperty(exports,
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1349
|
+
Object.defineProperty(exports, "filterRuntimeVars", {
|
|
1350
|
+
enumerable: true,
|
|
1351
|
+
get: function() {
|
|
1352
|
+
return filterRuntimeVars;
|
|
1353
|
+
}
|
|
1370
1354
|
});
|
|
1371
|
-
Object.defineProperty(exports,
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1355
|
+
Object.defineProperty(exports, "getNunjucksEngine", {
|
|
1356
|
+
enumerable: true,
|
|
1357
|
+
get: function() {
|
|
1358
|
+
return getNunjucksEngine;
|
|
1359
|
+
}
|
|
1376
1360
|
});
|
|
1377
|
-
Object.defineProperty(exports,
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1361
|
+
Object.defineProperty(exports, "getNunjucksEngineForFilePath", {
|
|
1362
|
+
enumerable: true,
|
|
1363
|
+
get: function() {
|
|
1364
|
+
return getNunjucksEngineForFilePath;
|
|
1365
|
+
}
|
|
1382
1366
|
});
|
|
1383
|
-
Object.defineProperty(exports,
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1367
|
+
Object.defineProperty(exports, "getProviderDescription", {
|
|
1368
|
+
enumerable: true,
|
|
1369
|
+
get: function() {
|
|
1370
|
+
return getProviderDescription;
|
|
1371
|
+
}
|
|
1388
1372
|
});
|
|
1389
|
-
Object.defineProperty(exports,
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1373
|
+
Object.defineProperty(exports, "getResolvedRelativePath", {
|
|
1374
|
+
enumerable: true,
|
|
1375
|
+
get: function() {
|
|
1376
|
+
return getResolvedRelativePath;
|
|
1377
|
+
}
|
|
1394
1378
|
});
|
|
1395
|
-
Object.defineProperty(exports,
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1379
|
+
Object.defineProperty(exports, "getTestCaseDeduplicationKey", {
|
|
1380
|
+
enumerable: true,
|
|
1381
|
+
get: function() {
|
|
1382
|
+
return getTestCaseDeduplicationKey;
|
|
1383
|
+
}
|
|
1400
1384
|
});
|
|
1401
|
-
Object.defineProperty(exports,
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1385
|
+
Object.defineProperty(exports, "isAnthropicProvider", {
|
|
1386
|
+
enumerable: true,
|
|
1387
|
+
get: function() {
|
|
1388
|
+
return isAnthropicProvider;
|
|
1389
|
+
}
|
|
1406
1390
|
});
|
|
1407
|
-
Object.defineProperty(exports,
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1391
|
+
Object.defineProperty(exports, "isGoogleProvider", {
|
|
1392
|
+
enumerable: true,
|
|
1393
|
+
get: function() {
|
|
1394
|
+
return isGoogleProvider;
|
|
1395
|
+
}
|
|
1412
1396
|
});
|
|
1413
|
-
Object.defineProperty(exports,
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1397
|
+
Object.defineProperty(exports, "isOpenAiProvider", {
|
|
1398
|
+
enumerable: true,
|
|
1399
|
+
get: function() {
|
|
1400
|
+
return isOpenAiProvider;
|
|
1401
|
+
}
|
|
1418
1402
|
});
|
|
1419
|
-
Object.defineProperty(exports,
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1403
|
+
Object.defineProperty(exports, "isProviderAllowed", {
|
|
1404
|
+
enumerable: true,
|
|
1405
|
+
get: function() {
|
|
1406
|
+
return isProviderAllowed;
|
|
1407
|
+
}
|
|
1424
1408
|
});
|
|
1425
|
-
Object.defineProperty(exports,
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1409
|
+
Object.defineProperty(exports, "loadFunction", {
|
|
1410
|
+
enumerable: true,
|
|
1411
|
+
get: function() {
|
|
1412
|
+
return loadFunction;
|
|
1413
|
+
}
|
|
1430
1414
|
});
|
|
1431
|
-
Object.defineProperty(exports,
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1415
|
+
Object.defineProperty(exports, "maybeLoadConfigFromExternalFile", {
|
|
1416
|
+
enumerable: true,
|
|
1417
|
+
get: function() {
|
|
1418
|
+
return maybeLoadConfigFromExternalFile;
|
|
1419
|
+
}
|
|
1436
1420
|
});
|
|
1437
|
-
Object.defineProperty(exports,
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1421
|
+
Object.defineProperty(exports, "maybeLoadFromExternalFile", {
|
|
1422
|
+
enumerable: true,
|
|
1423
|
+
get: function() {
|
|
1424
|
+
return maybeLoadFromExternalFile;
|
|
1425
|
+
}
|
|
1442
1426
|
});
|
|
1443
|
-
Object.defineProperty(exports,
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1427
|
+
Object.defineProperty(exports, "maybeLoadFromExternalFileWithVars", {
|
|
1428
|
+
enumerable: true,
|
|
1429
|
+
get: function() {
|
|
1430
|
+
return maybeLoadFromExternalFileWithVars;
|
|
1431
|
+
}
|
|
1448
1432
|
});
|
|
1449
|
-
Object.defineProperty(exports,
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1433
|
+
Object.defineProperty(exports, "maybeLoadResponseFormatFromExternalFile", {
|
|
1434
|
+
enumerable: true,
|
|
1435
|
+
get: function() {
|
|
1436
|
+
return maybeLoadResponseFormatFromExternalFile;
|
|
1437
|
+
}
|
|
1454
1438
|
});
|
|
1455
|
-
Object.defineProperty(exports,
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1439
|
+
Object.defineProperty(exports, "maybeLoadToolsFromExternalFile", {
|
|
1440
|
+
enumerable: true,
|
|
1441
|
+
get: function() {
|
|
1442
|
+
return maybeLoadToolsFromExternalFile;
|
|
1443
|
+
}
|
|
1460
1444
|
});
|
|
1461
|
-
Object.defineProperty(exports,
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1445
|
+
Object.defineProperty(exports, "parseFileUrl", {
|
|
1446
|
+
enumerable: true,
|
|
1447
|
+
get: function() {
|
|
1448
|
+
return parseFileUrl;
|
|
1449
|
+
}
|
|
1466
1450
|
});
|
|
1467
|
-
Object.defineProperty(exports,
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1451
|
+
Object.defineProperty(exports, "parsePathOrGlob", {
|
|
1452
|
+
enumerable: true,
|
|
1453
|
+
get: function() {
|
|
1454
|
+
return parsePathOrGlob;
|
|
1455
|
+
}
|
|
1472
1456
|
});
|
|
1473
|
-
Object.defineProperty(exports,
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1457
|
+
Object.defineProperty(exports, "printBorder", {
|
|
1458
|
+
enumerable: true,
|
|
1459
|
+
get: function() {
|
|
1460
|
+
return printBorder;
|
|
1461
|
+
}
|
|
1478
1462
|
});
|
|
1479
|
-
Object.defineProperty(exports,
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1463
|
+
Object.defineProperty(exports, "readFilters", {
|
|
1464
|
+
enumerable: true,
|
|
1465
|
+
get: function() {
|
|
1466
|
+
return readFilters;
|
|
1467
|
+
}
|
|
1484
1468
|
});
|
|
1485
|
-
Object.defineProperty(exports,
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1469
|
+
Object.defineProperty(exports, "readOutput", {
|
|
1470
|
+
enumerable: true,
|
|
1471
|
+
get: function() {
|
|
1472
|
+
return readOutput;
|
|
1473
|
+
}
|
|
1490
1474
|
});
|
|
1491
|
-
Object.defineProperty(exports,
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1475
|
+
Object.defineProperty(exports, "renderEnvOnlyInObject", {
|
|
1476
|
+
enumerable: true,
|
|
1477
|
+
get: function() {
|
|
1478
|
+
return renderEnvOnlyInObject;
|
|
1479
|
+
}
|
|
1496
1480
|
});
|
|
1497
|
-
Object.defineProperty(exports,
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1481
|
+
Object.defineProperty(exports, "renderVarsInObject", {
|
|
1482
|
+
enumerable: true,
|
|
1483
|
+
get: function() {
|
|
1484
|
+
return renderVarsInObject;
|
|
1485
|
+
}
|
|
1502
1486
|
});
|
|
1503
|
-
Object.defineProperty(exports,
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1487
|
+
Object.defineProperty(exports, "resultIsForTestCase", {
|
|
1488
|
+
enumerable: true,
|
|
1489
|
+
get: function() {
|
|
1490
|
+
return resultIsForTestCase;
|
|
1491
|
+
}
|
|
1508
1492
|
});
|
|
1509
|
-
Object.defineProperty(exports,
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1493
|
+
Object.defineProperty(exports, "setupEnv", {
|
|
1494
|
+
enumerable: true,
|
|
1495
|
+
get: function() {
|
|
1496
|
+
return setupEnv;
|
|
1497
|
+
}
|
|
1514
1498
|
});
|
|
1515
|
-
Object.defineProperty(exports,
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1499
|
+
Object.defineProperty(exports, "writeMultipleOutputs", {
|
|
1500
|
+
enumerable: true,
|
|
1501
|
+
get: function() {
|
|
1502
|
+
return writeMultipleOutputs;
|
|
1503
|
+
}
|
|
1520
1504
|
});
|
|
1521
|
-
Object.defineProperty(exports,
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1505
|
+
Object.defineProperty(exports, "writeOutput", {
|
|
1506
|
+
enumerable: true,
|
|
1507
|
+
get: function() {
|
|
1508
|
+
return writeOutput;
|
|
1509
|
+
}
|
|
1526
1510
|
});
|
|
1527
|
-
|
|
1511
|
+
|
|
1512
|
+
//# sourceMappingURL=util-B9vlHIIh.cjs.map
|