promptfoo 0.120.27 → 0.121.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/{ListApp-8WOe2nT6.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-B0pgC1oV.js +206 -0
- package/dist/src/{accounts-DVINui-2.js → accounts-Bm2D8Db9.js} +39 -34
- package/dist/src/{accounts-CPDRAMND.js → accounts-CiBLOnA7.js} +38 -33
- package/dist/src/{accounts-Fl2J3_Fu.cjs → accounts-gtkH-5KX.cjs} +77 -78
- package/dist/src/{agentic-utils-D922n6mm.js → agentic-utils-DS1g3GLF.js} +9 -10
- package/dist/src/{agents-BcsN_BgB.js → agents-9qiOy0ho.js} +16 -12
- package/dist/src/{agents-BXLmVsxR.js → agents-CBr9A01V.js} +37 -37
- package/dist/src/{agents-pMfppv9Z.js → agents-CmvBq8LV.js} +16 -18
- package/dist/src/{agents-hqgSV-3o.js → agents-D__IdAlg.js} +39 -40
- package/dist/src/{agents-BO2n8Z0d.cjs → agents-DbRtpYxR.cjs} +37 -40
- package/dist/src/{agents-BdUTAwi-.js → agents-DgF2zDag.js} +37 -42
- package/dist/src/{agents-DgJf2-ez.cjs → agents-Di9DKPzn.cjs} +16 -17
- package/dist/src/{agents-DNvSH78i.js → agents-cLXA8a_8.js} +17 -19
- package/dist/src/{aimlapi-DtgPI0nE.js → aimlapi-B4rcnZgv.js} +15 -17
- package/dist/src/{aimlapi-BE_Tg9Fl.cjs → aimlapi-BvlNH0gr.cjs} +15 -16
- package/dist/src/{aimlapi-DOib86oE.js → aimlapi-CnkC2HqE.js} +16 -18
- package/dist/src/{aimlapi-DTPACCB1.js → aimlapi-DHJU_kcV.js} +15 -4
- package/dist/src/app/assets/index-4LKxG2CG.js +439 -0
- package/dist/src/app/assets/{index-NCn4eVBv.css → index-C3zcsZFQ.css} +1 -1
- package/dist/src/app/assets/vendor-charts-BnDWwBlI.js +36 -0
- package/dist/src/app/index.html +3 -3
- package/dist/src/app/tsconfig.app.tsbuildinfo +1 -1
- package/dist/src/{audio-BnRUGAm_.js → audio-Bkv46et0.js} +6 -5
- package/dist/src/{audio-Cwo68yZS.cjs → audio-CGMyULza.cjs} +6 -7
- package/dist/src/{audio-MSRki4JU.js → audio-ClI_AFre.js} +6 -8
- package/dist/src/{audio-BRYU0BFo.js → audio-Dz3z7s3J.js} +7 -9
- package/dist/src/{base-pGVmXNl4.cjs → base-CGrhspbK.cjs} +36 -38
- package/dist/src/{base-h961VXYk.js → base-CpjcHe4e.js} +11 -13
- package/dist/src/base-DLKtKMFh.js +193 -0
- package/dist/src/{base-XB2tDJrB.js → base-Dy1V8--Z.js} +11 -13
- package/dist/src/blobs-BDbfYdrJ.js +236 -0
- package/dist/src/{blobs-CR5C4Ihh.js → blobs-CBO20krR.js} +9 -12
- package/dist/src/{blobs-BM_e6hCa.js → blobs-CMHN0Qcz.js} +9 -12
- package/dist/src/{blobs-B-KQAFhX.cjs → blobs-D23XLin-.cjs} +34 -37
- package/dist/src/{cache-jsiwsAJv.js → cache-BVeDlD87.js} +132 -117
- package/dist/src/{cache-CIpsoBZR.js → cache-C4Nxf52C.js} +132 -118
- package/dist/src/cache-CeUpFm3M.cjs +5 -0
- package/dist/src/{cache-BTVYfbka.cjs → cache-Dh5WtQps.cjs} +182 -168
- package/dist/src/cache-i1P6crbO.js +756 -0
- package/dist/src/cache-n-RCJ-hL.js +6 -0
- package/dist/src/{chat-BcPjZXIp.js → chat-BiKyneZl.js} +45 -46
- package/dist/src/{chat-D31K7C4u.cjs → chat-C1Qst7jL.cjs} +20 -21
- package/dist/src/{chat-B84t99NW.js → chat-C2jrdPMx.js} +20 -9
- package/dist/src/{chat-BE44YOc6.cjs → chat-CgF-J-Jj.cjs} +65 -66
- package/dist/src/{chat-DwWifjxi.js → chat-CzkrVDfz.js} +20 -22
- package/dist/src/chat-DJIw17u0.js +766 -0
- package/dist/src/{chat-CcUCysjU.js → chat-DqxYYtWA.js} +45 -46
- package/dist/src/{chat-DZM2GUHO.js → chat-qmatte1u.js} +21 -23
- package/dist/src/{chatkit-D67HS_0b.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DAB_qfzI.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-Biqb_wsD.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-PGG4ZYIn.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-SVM6AdBu.js → claude-agent-sdk-Apiy0iaz.js} +31 -31
- package/dist/src/{claude-agent-sdk-C-IOTPfo.js → claude-agent-sdk-D2bJee9S.js} +31 -29
- package/dist/src/{claude-agent-sdk-C9SiaQub.cjs → claude-agent-sdk-D9Z5Pr9X.cjs} +31 -28
- package/dist/src/{claude-agent-sdk-CiluSyW1.js → claude-agent-sdk-DfCoW0E6.js} +33 -20
- package/dist/src/cloud-BBh91EUK.js +4 -0
- package/dist/src/{cloud-CZ-q9Ier.js → cloud-C0dlstV_.js} +7 -9
- package/dist/src/{cloudflare-ai-BahKHyhh.js → cloudflare-ai-8TDxHR0x.js} +16 -18
- package/dist/src/{cloudflare-ai-v_qZD6_q.js → cloudflare-ai-BxAGvfju.js} +17 -19
- package/dist/src/{cloudflare-ai-Dfahv5SY.cjs → cloudflare-ai-CknbZ5LJ.cjs} +16 -17
- package/dist/src/{cloudflare-ai-Dxyt50Nl.js → cloudflare-ai-g7PB6VHR.js} +16 -4
- package/dist/src/{cloudflare-gateway-Bi_FpOFy.js → cloudflare-gateway-B9HWA5wf.js} +23 -23
- package/dist/src/{cloudflare-gateway-BPWoZIzJ.cjs → cloudflare-gateway-BSnDmHYo.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-C0guUNwk.js → cloudflare-gateway-CKDb4dJ8.js} +26 -14
- package/dist/src/{cloudflare-gateway-btS7h1OZ.js → cloudflare-gateway-CP9QEWYS.js} +21 -25
- package/dist/src/{codex-sdk-DSxAnbfT.js → codex-sdk-C6UMlxwV.js} +28 -29
- package/dist/src/{codex-sdk-IYVi9fuM.js → codex-sdk-DUwKWezN.js} +28 -27
- package/dist/src/{codex-sdk-DulY0ZRq.js → codex-sdk-GGAw0qbD.js} +28 -29
- package/dist/src/{codex-sdk-DFKMtAyf.cjs → codex-sdk-fAO0c3yA.cjs} +28 -29
- package/dist/src/{cometapi-DzrR3SR_.js → cometapi-BL9yvj_f.js} +16 -4
- package/dist/src/{cometapi-DIO64tf4.cjs → cometapi-C4xSqeID.cjs} +21 -22
- package/dist/src/{cometapi-C9EEpJzT.js → cometapi-CUQq3H_a.js} +21 -24
- package/dist/src/{cometapi-DkNBMk0G.js → cometapi-DFNiKmSz.js} +17 -19
- package/dist/src/{completion-CG29bfKX.js → completion-5MzrpJxT.js} +11 -13
- package/dist/src/{completion-CCRT4kX1.cjs → completion-CM6oK8PS.cjs} +21 -23
- package/dist/src/{completion-Bgf1VJoq.js → completion-DZ083F31.js} +11 -13
- package/dist/src/completion-qRoZAYRB.js +120 -0
- package/dist/src/{createHash-Dw_iLu31.js → createHash-CTQmL3G2.js} +2 -3
- package/dist/src/{createHash-CYQy4YeL.cjs → createHash-CfZSc0b4.cjs} +13 -14
- package/dist/src/{createHash-CJcfskIZ.js → createHash-Da8fMwqB.js} +2 -3
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-D-ayp2FW.js → docker-Bb5dcxr8.js} +18 -20
- package/dist/src/{docker-B81N0t4e.js → docker-BvfL2BrW.js} +19 -21
- package/dist/src/{docker-DNcLR4Ig.cjs → docker-DcF2pRrj.cjs} +18 -19
- package/dist/src/{docker-egERKxCF.js → docker-ExVyLp0S.js} +18 -7
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-B9dPm_BF.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-D2pZ87fL.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-Ct-Joyue.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-B3r2CVXr.js +15 -0
- package/dist/src/{eval-C-Nr6wX_.js → eval-Dg2nG4v2.js} +47 -54
- package/dist/src/evalResult-5xwYnECe.js +12 -0
- package/dist/src/evalResult-71lY93Kj.cjs +10 -0
- package/dist/src/{evalResult-DXMWJ3sx.js → evalResult-BBRNtX4I.js} +10 -11
- package/dist/src/{evalResult-4BzI2tmj.js → evalResult-BDMqrapS.js} +16 -12
- package/dist/src/evalResult-Dx5P5cIv.js +10 -0
- package/dist/src/{evalResult-CX8wQecI.cjs → evalResult-fuaI8HkH.cjs} +20 -21
- package/dist/src/{evaluator-8aGyV12L.js → evaluator-BhoWwp5b.js} +211 -235
- package/dist/src/evaluator-Jx6bRZV6.js +36 -0
- package/dist/src/{extractor-V5x_m1i0.js → extractor-C0EVHewb.js} +22 -24
- package/dist/src/extractor-D25qpmGX.js +374 -0
- package/dist/src/{extractor-CD5yKL-G.js → extractor-DReVID0K.js} +22 -24
- package/dist/src/{extractor-C031XmTA.cjs → extractor-pYLLi3wS.cjs} +37 -39
- package/dist/src/{fetch-BmbD-v1L.cjs → fetch-BPkYtG8K.cjs} +244 -277
- package/dist/src/fetch-BxNb_Lp3.js +5 -0
- package/dist/src/{fetch-D3OHf-lV.js → fetch-Cwxnd8zz.js} +36 -44
- package/dist/src/{fetch-CXZI9RRr.js → fetch-Dxpd4_sr.js} +23 -35
- package/dist/src/fetch-HaqdX7U1.js +780 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-CdevI3An.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-Dres3qrN.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/{graders--1y2u9HO.js → graders-BTeBGqjJ.js} +349 -397
- package/dist/src/graders-B_pgMLS2.js +34 -0
- package/dist/src/{graders-DTeBrzWp.js → graders-Bj_Odv7c.js} +349 -397
- package/dist/src/graders-DErokPDO.cjs +32 -0
- package/dist/src/graders-DP7KFFo-.js +13466 -0
- package/dist/src/graders-DR_uNe54.js +32 -0
- package/dist/src/{graders-DohM2dir.cjs → graders-DU49_J8Y.cjs} +684 -732
- package/dist/src/graders-w3176Wz-.js +32 -0
- package/dist/src/{image-B0U4Hqll.js → image-B02ogr_b.js} +7 -9
- package/dist/src/{image-DmE-niFE.js → image-B0h9VEMc.js} +6 -5
- package/dist/src/{image-CuKHuccK.cjs → image-BLmROtN3.cjs} +29 -30
- package/dist/src/{image-DNEIf_aI.js → image-Bb4vWQLM.js} +6 -8
- package/dist/src/{image-DpKl2F15.cjs → image-C1madmKh.cjs} +6 -7
- package/dist/src/{image-C3wHC9_h.js → image-CHfWvljl.js} +9 -10
- package/dist/src/{image-O1u4bCFg.js → image-DS-o-0ph.js} +9 -10
- package/dist/src/image-Dpxa1Jt6.js +257 -0
- package/dist/src/index.cjs +615 -695
- package/dist/src/index.d.cts +271 -7
- package/dist/src/index.d.ts +271 -3
- package/dist/src/index.js +580 -664
- package/dist/src/{interactiveCheck-Bxj1Swex.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-CEzQobWX.js → knowledgeBase-B3OoKIej.js} +14 -9
- package/dist/src/{knowledgeBase-Be_zyW4L.js → knowledgeBase-CYTLHOt1.js} +16 -16
- package/dist/src/{knowledgeBase-BZ41IFwq.js → knowledgeBase-D33Ty2l6.js} +14 -18
- package/dist/src/{knowledgeBase-D-5BMXlr.cjs → knowledgeBase-DOO_BM9b.cjs} +14 -15
- package/dist/src/{litellm-DnbRJ2if.js → litellm-AaeZcZQF.js} +18 -19
- package/dist/src/{litellm-hUSNM_M2.cjs → litellm-I_hbp_dc.cjs} +17 -17
- package/dist/src/{litellm-CRDqPhNI.js → litellm-NbjknEh6.js} +17 -18
- package/dist/src/{litellm-9vR8zpfU.js → litellm-TrljxD9G.js} +17 -5
- package/dist/src/{logger-CG1uZPbQ.js → logger-CT3IKMKA.js} +10 -29
- package/dist/src/{logger-B7sBeGa0.cjs → logger-Cp1GPUjj.cjs} +152 -180
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-LSBxlt7a.js → logger-KkObSCzq.js} +13 -31
- package/dist/src/{luma-ray-4blv9iZ2.js → luma-ray-BS2_tY8L.js} +22 -21
- package/dist/src/{luma-ray-drvgdpP9.js → luma-ray-DDsjcgZZ.js} +20 -13
- package/dist/src/{luma-ray-Hm3d6VJE.cjs → luma-ray-Due0n7di.cjs} +20 -21
- package/dist/src/{luma-ray-B2__8lYH.js → luma-ray-f6I2fft-.js} +20 -23
- package/dist/src/main.js +1170 -1321
- package/dist/src/{messages-Uee41Mj5.js → messages-BS17jdMx.js} +22 -24
- package/dist/src/{messages-XhiwCbi4.cjs → messages-Bs1kC7P4.cjs} +32 -34
- package/dist/src/{messages-CGPPidQr.js → messages-D0lx5qK7.js} +22 -24
- package/dist/src/messages-ZJk778GH.js +240 -0
- package/dist/src/{meteor-BYykdXrV.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-CsopaHrH.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-e-E-2vVl.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-C8lGP6P4.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/{modelslab-yKz-ZNB4.js → modelslab-Bmni6skY.js} +17 -10
- package/dist/src/{modelslab-E9gO-bYd.js → modelslab-Bx9IrZfS.js} +18 -20
- package/dist/src/{modelslab-lUVW0cmB.cjs → modelslab-CoUX6Jc_.cjs} +17 -18
- package/dist/src/{modelslab-ClBkr8_9.js → modelslab-DRb74SP4.js} +17 -19
- package/dist/src/{nova-reel-Dk8jNpId.js → nova-reel-BfPq-0Yk.js} +20 -13
- package/dist/src/{nova-reel-D8CuO6QH.cjs → nova-reel-C_QM18Xn.cjs} +20 -21
- package/dist/src/{nova-reel-u2eF2Cxm.js → nova-reel-D_W1tjMH.js} +22 -21
- package/dist/src/{nova-reel-P9bwvtYX.js → nova-reel-bgjxilYW.js} +20 -23
- package/dist/src/{nova-sonic-CK2rAiKi.js → nova-sonic-CFb5GYhg.js} +30 -26
- package/dist/src/{nova-sonic-BaqWlkds.js → nova-sonic-DIGQNR07.js} +30 -31
- package/dist/src/{nova-sonic-yZapPLv7.js → nova-sonic-De1HW5fD.js} +31 -32
- package/dist/src/{nova-sonic-Ds1C-dpm.cjs → nova-sonic-zfcljeRp.cjs} +30 -31
- package/dist/src/{openai-DUFopMrH.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-PblZ3jUE.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CcN1B8Sb.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-B6qqDr_u.cjs → openclaw-CSugPYAr.cjs} +188 -130
- package/dist/src/{openclaw-A-3_loM7.js → openclaw-DiSz3I5L.js} +180 -109
- package/dist/src/{openclaw-a3lylB-V.js → openclaw-DuvJKEW5.js} +178 -124
- package/dist/src/{openclaw-COn6QzDi.js → openclaw-tiVYRtr-.js} +178 -122
- package/dist/src/opencode-sdk-0j6rTWNb.js +562 -0
- package/dist/src/opencode-sdk-B3CWY9h_.js +560 -0
- package/dist/src/opencode-sdk-BL764Jdi.cjs +564 -0
- package/dist/src/opencode-sdk-C2y6UkP2.js +560 -0
- package/dist/src/{otlpReceiver-oyf5wLGC.js → otlpReceiver-C99PPb48.js} +53 -51
- package/dist/src/{otlpReceiver-lXsYVbpj.cjs → otlpReceiver-CGq6LspY.cjs} +53 -55
- package/dist/src/{otlpReceiver-94URx7UW.js → otlpReceiver-CdNBdbsk.js} +53 -55
- package/dist/src/{otlpReceiver-BmmTiMjA.js → otlpReceiver-D89fR-rC.js} +53 -55
- package/dist/src/{providerRegistry-Cq_JK_CJ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-DSSHjMKf.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-CvHEVJad.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B7V0njNs.js +32 -0
- package/dist/src/providers-BEwbhv0X.js +30 -0
- package/dist/src/{providers-Iil64vk9.js → providers-BlqUifFg.js} +1543 -1676
- package/dist/src/providers-CH3C7zf7.js +30 -0
- package/dist/src/{providers-DHbjzW2e.cjs → providers-CgKOSgTR.cjs} +1896 -2029
- package/dist/src/providers-D8lF1sqW.js +33246 -0
- package/dist/src/{providers-BnFpbY_s.js → providers-Dk_6ocUX.js} +1536 -1669
- package/dist/src/providers-zyB6k_38.cjs +31 -0
- package/dist/src/{pythonUtils-CcT5LH1M.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DBbuI3QJ.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-hZ8LeQLv.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/{quiverai-BuI0tE39.js → quiverai-BbOUOn2L.js} +8 -7
- package/dist/src/{quiverai-DCGSZt4U.js → quiverai-CIaELU_m.js} +8 -10
- package/dist/src/{quiverai-DiMVJQDz.cjs → quiverai-PdShCPox.cjs} +8 -9
- package/dist/src/{quiverai-fQNkExW4.js → quiverai-uH-dcTIr.js} +9 -11
- package/dist/src/{render-Dj1smHEb.js → render-Drod8m7K.js} +4 -5
- package/dist/src/responses-CB2jwoAr.js +660 -0
- package/dist/src/{responses-ghR3IOfy.cjs → responses-D8SBTL64.cjs} +39 -42
- package/dist/src/{responses-DOAFFENS.js → responses-DIR9Ud3j.js} +24 -27
- package/dist/src/{responses-CxzoQoBe.js → responses-WNGNYe3K.js} +24 -27
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-CwbGmgYN.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-DudlFZed.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-C8MhKGHb.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-gmskuyre.js → sagemaker-CNBxx5CJ.js} +75 -70
- package/dist/src/{sagemaker-CcxhlOAR.js → sagemaker-CemTFp2h.js} +75 -79
- package/dist/src/{sagemaker-77zbJ2Q2.cjs → sagemaker-Cl28mZU2.cjs} +75 -76
- package/dist/src/{sagemaker-DuM71dVU.js → sagemaker-YSyBXQQh.js} +77 -77
- package/dist/src/{scanner-DJYiSXQj.js → scanner-BsBlNXNn.js} +100 -121
- package/dist/src/server/index.js +5520 -67427
- package/dist/src/{server-B5v33lvE.cjs → server-C_7Ax-hA.cjs} +57 -67
- package/dist/src/{server-BJ4m4f1D.js → server-CqzrVGpF.js} +26 -29
- package/dist/src/server-CuxBbeSY.js +229 -0
- package/dist/src/server-DA4Cyrrq.js +7 -0
- package/dist/src/server-Dulb-4-K.cjs +5 -0
- package/dist/src/{server-RV_i_YX5.js → server-VWgWb00X.js} +19 -24
- package/dist/src/{signal-BW33JuId.js → signal-4U3mfRvL.js} +9 -11
- package/dist/src/{slack-DEURelTy.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-BQYeW9L3.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-BB6yuZzp.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-2pRrhhgJ.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-CXGFv4aR.js +228 -0
- package/dist/src/store-CXS-Q_91.js +6 -0
- package/dist/src/{store-D7CgQzAR.cjs → store-DLlFCC4h.cjs} +44 -45
- package/dist/src/{store-DJNsD1iC.js → store-DXilxTl-.js} +40 -36
- package/dist/src/{store-s3SftUwF.js → store-Dim__MDd.js} +34 -35
- package/dist/src/store-eYkaKMwq.cjs +5 -0
- package/dist/src/{tables-DfTsNN7X.js → tables-6YKwjN9-.js} +19 -21
- package/dist/src/tables-DLJPUdUE.js +288 -0
- package/dist/src/{tables-BKTmd6u7.cjs → tables-DPi7wKeM.cjs} +89 -91
- package/dist/src/{tables-DMegD0Xf.js → tables-gftXzE9I.js} +21 -23
- package/dist/src/telemetry-BpMfhthR.cjs +5 -0
- package/dist/src/{telemetry--WAdAfVi.js → telemetry-CMrFgtPB.js} +11 -13
- package/dist/src/telemetry-Cps3mIU-.js +171 -0
- package/dist/src/{telemetry-DQgVBCAb.cjs → telemetry-DaX14Chu.cjs} +21 -24
- package/dist/src/{telemetry-BedSm-bZ.js → telemetry-Dthj_BbD.js} +17 -14
- package/dist/src/telemetry-Dw38hanS.js +7 -0
- package/dist/src/{text-oiSbwSOI.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-oKzCBnK6.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-B_IrO4GZ.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-mYS9vd5v.js → transcription-BNYURcXg.js} +14 -7
- package/dist/src/{transcription-X2-B4vkX.js → transcription-B_OdaHp7.js} +14 -16
- package/dist/src/{transcription-BO1AHegO.cjs → transcription-NLVG9MT1.cjs} +14 -15
- package/dist/src/{transcription-lzBLiTFJ.js → transcription-s6A-bNrZ.js} +15 -17
- package/dist/src/{transform-B1Hi5lWS.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-DeGlxb0D.js → transform-D5HsjduX.js} +39 -47
- package/dist/src/{transform-CYDILYDe.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-BEgStbHK.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/{transform-D5PjiWiZ.cjs → transform-DuHvhZpj.cjs} +179 -187
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/transform-uAytVuyX.js +1506 -0
- package/dist/src/{transform-Dfl89yi4.js → transform-vNucnNr0.js} +39 -47
- package/dist/src/{transformersAvailability-SZnTS3pJ.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-D-glmEy7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-CjeFXhuJ.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-DWNf48sT.cjs → types-C_7nyzr1.cjs} +538 -574
- package/dist/src/{types-CXQduE9o.js → types-Cbd8uOMq.js} +68 -100
- package/dist/src/types-CzW2QFyi.js +3288 -0
- package/dist/src/{types-C5hEkb-x.js → types-DmyIJ-sR.js} +63 -99
- package/dist/src/{util-CoQjmE3u.js → util-B3xGByQh.js} +4 -5
- package/dist/src/{util-aLhtl3fe.cjs → util-B9vlHIIh.cjs} +208 -223
- package/dist/src/{util-Du96oyYS.js → util-BHGHw5G1.js} +4 -5
- package/dist/src/{util-DQ984syk.js → util-BRYkYPTd.js} +36 -51
- package/dist/src/{util-D9eLdGfa.js → util-BV4XUC0n.js} +5 -6
- package/dist/src/util-Bv6uGDfH.js +293 -0
- package/dist/src/{util-1wWM599Z.cjs → util-BzMcevZc.cjs} +50 -51
- package/dist/src/{util-_h4pVqrz.js → util-C1CeHl-P.js} +36 -51
- package/dist/src/{util-Bm_-UMD_.js → util-CMy69ZgQ.js} +5 -6
- package/dist/src/{util-CyUdMzV0.cjs → util-DGNOS1db.cjs} +34 -35
- package/dist/src/util-Dnmk2mBQ.js +599 -0
- package/dist/src/util-ZzmqNPlg.js +1426 -0
- package/dist/src/{utils-BjLy-Q72.cjs → utils-Cz9qXqII.cjs} +29 -32
- package/dist/src/{utils-CFMn2yHW.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-DvWMzuMx.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +38 -38
- package/dist/src/app/assets/index-B2D0bCSI.js +0 -439
- package/dist/src/app/assets/vendor-charts-CCl15Imd.js +0 -36
- package/dist/src/cache-ChPcurj7.js +0 -6
- package/dist/src/cache-VVu_W-yg.js +0 -8
- package/dist/src/cache-YLNCFEM2.cjs +0 -6
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-BndfXy4H.js +0 -5
- package/dist/src/eval-BhHvMY82.js +0 -17
- package/dist/src/evalResult-Dq2gFNQY.js +0 -12
- package/dist/src/evalResult-nmcP5VKH.cjs +0 -12
- package/dist/src/evalResult-trqZjVYh.js +0 -14
- package/dist/src/evaluator-CnfPstzT.js +0 -39
- package/dist/src/fetch-IDPDue6F.cjs +0 -4
- package/dist/src/fetch-hKJ-It8q.js +0 -6
- package/dist/src/fetch-ouKnrWK-.js +0 -4
- package/dist/src/graders-CQn7WUsd.cjs +0 -34
- package/dist/src/graders-DC6QAbpW.js +0 -35
- package/dist/src/graders-DUWz3Y7j.js +0 -37
- package/dist/src/opencode-sdk-4bL9n-Gk.js +0 -382
- package/dist/src/opencode-sdk-BfC2zWcR.js +0 -376
- package/dist/src/opencode-sdk-DMJyuwMg.js +0 -380
- package/dist/src/opencode-sdk-Da-9adza.cjs +0 -383
- package/dist/src/providers-CsXB2Ix-.js +0 -35
- package/dist/src/providers-DO8ltjLC.js +0 -33
- package/dist/src/providers-Dtq-xnXd.cjs +0 -33
- package/dist/src/rubyUtils-BUbcND2f.js +0 -6
- package/dist/src/rubyUtils-Cr55X_KE.js +0 -5
- package/dist/src/rubyUtils-DlIiqoYo.cjs +0 -5
- package/dist/src/server-C2eQH4Gu.js +0 -6
- package/dist/src/server-CXWycu7H.cjs +0 -6
- package/dist/src/server-Q6OGlxxT.js +0 -8
- package/dist/src/store-B3EDO9Q3.js +0 -7
- package/dist/src/store-Dl9F8aw5.js +0 -6
- package/dist/src/store-SnrGrlt9.cjs +0 -6
- package/dist/src/telemetry-BGhiPZtl.js +0 -8
- package/dist/src/telemetry-CFfiYan6.cjs +0 -6
- package/dist/src/telemetry-DHzEduxX.js +0 -6
- package/dist/src/transform-C1x1ZlMQ.cjs +0 -6
- package/dist/src/transform-DYHjFmQu.js +0 -8
- package/dist/src/transform-rmwJT5JQ.js +0 -7
- package/dist/src/transformersAvailability-eJooj0gX.js +0 -35
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
const require_logger = require(
|
|
2
|
-
const require_util = require(
|
|
3
|
-
const require_fetch = require(
|
|
1
|
+
const require_logger = require("./logger-Cp1GPUjj.cjs");
|
|
2
|
+
const require_util = require("./util-B9vlHIIh.cjs");
|
|
3
|
+
const require_fetch = require("./fetch-BPkYtG8K.cjs");
|
|
4
4
|
let openai = require("openai");
|
|
5
5
|
openai = require_logger.__toESM(openai);
|
|
6
|
-
|
|
7
6
|
//#region src/providers/openai/util.ts
|
|
8
7
|
const ajv = require_logger.getAjv();
|
|
9
8
|
const OPENAI_CHAT_MODELS = [
|
|
@@ -595,60 +594,60 @@ function formatOpenAiError(data) {
|
|
|
595
594
|
errorMessage += "\n\n" + require_logger.safeJsonStringify(data, true);
|
|
596
595
|
return errorMessage;
|
|
597
596
|
}
|
|
598
|
-
|
|
599
597
|
//#endregion
|
|
600
|
-
Object.defineProperty(exports,
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
598
|
+
Object.defineProperty(exports, "OPENAI_CHAT_MODELS", {
|
|
599
|
+
enumerable: true,
|
|
600
|
+
get: function() {
|
|
601
|
+
return OPENAI_CHAT_MODELS;
|
|
602
|
+
}
|
|
605
603
|
});
|
|
606
|
-
Object.defineProperty(exports,
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
604
|
+
Object.defineProperty(exports, "OPENAI_COMPLETION_MODELS", {
|
|
605
|
+
enumerable: true,
|
|
606
|
+
get: function() {
|
|
607
|
+
return OPENAI_COMPLETION_MODELS;
|
|
608
|
+
}
|
|
611
609
|
});
|
|
612
|
-
Object.defineProperty(exports,
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
610
|
+
Object.defineProperty(exports, "OPENAI_REALTIME_MODELS", {
|
|
611
|
+
enumerable: true,
|
|
612
|
+
get: function() {
|
|
613
|
+
return OPENAI_REALTIME_MODELS;
|
|
614
|
+
}
|
|
617
615
|
});
|
|
618
|
-
Object.defineProperty(exports,
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
616
|
+
Object.defineProperty(exports, "OPENAI_TRANSCRIPTION_MODELS", {
|
|
617
|
+
enumerable: true,
|
|
618
|
+
get: function() {
|
|
619
|
+
return OPENAI_TRANSCRIPTION_MODELS;
|
|
620
|
+
}
|
|
623
621
|
});
|
|
624
|
-
Object.defineProperty(exports,
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
622
|
+
Object.defineProperty(exports, "calculateOpenAICost", {
|
|
623
|
+
enumerable: true,
|
|
624
|
+
get: function() {
|
|
625
|
+
return calculateOpenAICost;
|
|
626
|
+
}
|
|
629
627
|
});
|
|
630
|
-
Object.defineProperty(exports,
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
628
|
+
Object.defineProperty(exports, "failApiCall", {
|
|
629
|
+
enumerable: true,
|
|
630
|
+
get: function() {
|
|
631
|
+
return failApiCall;
|
|
632
|
+
}
|
|
635
633
|
});
|
|
636
|
-
Object.defineProperty(exports,
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
634
|
+
Object.defineProperty(exports, "formatOpenAiError", {
|
|
635
|
+
enumerable: true,
|
|
636
|
+
get: function() {
|
|
637
|
+
return formatOpenAiError;
|
|
638
|
+
}
|
|
641
639
|
});
|
|
642
|
-
Object.defineProperty(exports,
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
640
|
+
Object.defineProperty(exports, "getTokenUsage", {
|
|
641
|
+
enumerable: true,
|
|
642
|
+
get: function() {
|
|
643
|
+
return getTokenUsage;
|
|
644
|
+
}
|
|
647
645
|
});
|
|
648
|
-
Object.defineProperty(exports,
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
646
|
+
Object.defineProperty(exports, "validateFunctionCall", {
|
|
647
|
+
enumerable: true,
|
|
648
|
+
get: function() {
|
|
649
|
+
return validateFunctionCall;
|
|
650
|
+
}
|
|
653
651
|
});
|
|
654
|
-
|
|
652
|
+
|
|
653
|
+
//# sourceMappingURL=util-BzMcevZc.cjs.map
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { O as
|
|
3
|
-
import {
|
|
4
|
-
import { t as invariant } from "./invariant-
|
|
5
|
-
import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-
|
|
6
|
-
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-
|
|
7
|
-
import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-
|
|
8
|
-
import { r as runPython } from "./pythonUtils-
|
|
2
|
+
import { O as state, b as getEnvBool, l as sanitizeObject, o as logger } from "./logger-KkObSCzq.js";
|
|
3
|
+
import { D as TERMINAL_MAX_WIDTH, N as VERSION, n as fetchWithProxy } from "./fetch-Dxpd4_sr.js";
|
|
4
|
+
import { t as invariant } from "./invariant-BtWWVVhl.js";
|
|
5
|
+
import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-Cbd8uOMq.js";
|
|
6
|
+
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-Ds-foDzt.js";
|
|
7
|
+
import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-C03C-mv3.js";
|
|
8
|
+
import { r as runPython } from "./pythonUtils-C3py6GC1.js";
|
|
9
9
|
import dotenv from "dotenv";
|
|
10
10
|
import * as fs$2 from "fs";
|
|
11
11
|
import * as path$1 from "path";
|
|
@@ -20,7 +20,6 @@ import { globSync, hasMagic } from "glob";
|
|
|
20
20
|
import nunjucks from "nunjucks";
|
|
21
21
|
import { XMLBuilder } from "fast-xml-parser";
|
|
22
22
|
import { stringify } from "csv-stringify/sync";
|
|
23
|
-
|
|
24
23
|
//#region src/util/provider.ts
|
|
25
24
|
function canonicalizeProviderId(id) {
|
|
26
25
|
if (id.startsWith("file://")) {
|
|
@@ -191,7 +190,6 @@ function isGoogleProvider(providerId) {
|
|
|
191
190
|
}
|
|
192
191
|
return false;
|
|
193
192
|
}
|
|
194
|
-
|
|
195
193
|
//#endregion
|
|
196
194
|
//#region src/util/comparison.ts
|
|
197
195
|
/**
|
|
@@ -290,7 +288,7 @@ function resultIsForTestCase(result, testCase) {
|
|
|
290
288
|
const isMatch = doVarsMatch && providersMatch;
|
|
291
289
|
if (!isMatch) {
|
|
292
290
|
const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
|
|
293
|
-
|
|
291
|
+
logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
|
|
294
292
|
testProvider: testProviderId || "none",
|
|
295
293
|
resultProvider: resultProviderId || "none",
|
|
296
294
|
testVarKeys: varKeys
|
|
@@ -298,7 +296,6 @@ function resultIsForTestCase(result, testCase) {
|
|
|
298
296
|
}
|
|
299
297
|
return isMatch;
|
|
300
298
|
}
|
|
301
|
-
|
|
302
299
|
//#endregion
|
|
303
300
|
//#region src/util/env.ts
|
|
304
301
|
/**
|
|
@@ -315,8 +312,8 @@ function setupEnv(envPath) {
|
|
|
315
312
|
return;
|
|
316
313
|
}
|
|
317
314
|
for (const p of paths) if (!fs$2.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
|
|
318
|
-
if (paths.length === 1)
|
|
319
|
-
else
|
|
315
|
+
if (paths.length === 1) logger.info(`Loading environment variables from ${paths[0]}`);
|
|
316
|
+
else logger.info(`Loading environment variables from: ${paths.join(", ")}`);
|
|
320
317
|
const pathArg = paths.length === 1 ? paths[0] : paths;
|
|
321
318
|
dotenv.config({
|
|
322
319
|
path: pathArg,
|
|
@@ -325,7 +322,6 @@ function setupEnv(envPath) {
|
|
|
325
322
|
});
|
|
326
323
|
} else dotenv.config({ quiet: true });
|
|
327
324
|
}
|
|
328
|
-
|
|
329
325
|
//#endregion
|
|
330
326
|
//#region src/util/functions/loadFunction.ts
|
|
331
327
|
const functionCache = {};
|
|
@@ -334,7 +330,7 @@ const functionCache = {};
|
|
|
334
330
|
* @param options Options for loading the function
|
|
335
331
|
* @returns The loaded function
|
|
336
332
|
*/
|
|
337
|
-
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath =
|
|
333
|
+
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = state.basePath, useCache = true }) {
|
|
338
334
|
const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
|
|
339
335
|
if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
|
|
340
336
|
const resolvedPath = basePath ? path.resolve(basePath, filePath) : filePath;
|
|
@@ -355,7 +351,7 @@ async function loadFunction({ filePath, functionName, defaultFunctionName = "fun
|
|
|
355
351
|
if (useCache) functionCache[cacheKey] = func;
|
|
356
352
|
return func;
|
|
357
353
|
} catch (err) {
|
|
358
|
-
|
|
354
|
+
logger.error(`Failed to load function: ${err.message}`);
|
|
359
355
|
throw err;
|
|
360
356
|
}
|
|
361
357
|
}
|
|
@@ -374,7 +370,6 @@ function parseFileUrl(fileUrl) {
|
|
|
374
370
|
};
|
|
375
371
|
return { filePath: urlWithoutProtocol };
|
|
376
372
|
}
|
|
377
|
-
|
|
378
373
|
//#endregion
|
|
379
374
|
//#region src/util/templates.ts
|
|
380
375
|
/**
|
|
@@ -393,7 +388,7 @@ function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false)
|
|
|
393
388
|
});
|
|
394
389
|
const envGlobals = {
|
|
395
390
|
...getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
|
|
396
|
-
...
|
|
391
|
+
...state.config?.env
|
|
397
392
|
};
|
|
398
393
|
env.addGlobal("env", envGlobals);
|
|
399
394
|
env.addFilter("load", function(str) {
|
|
@@ -433,7 +428,6 @@ function extractVariablesFromTemplates(templates) {
|
|
|
433
428
|
for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
|
|
434
429
|
return Array.from(variableSet);
|
|
435
430
|
}
|
|
436
|
-
|
|
437
431
|
//#endregion
|
|
438
432
|
//#region src/util/render.ts
|
|
439
433
|
/**
|
|
@@ -462,10 +456,6 @@ function extractVariablesFromTemplates(templates) {
|
|
|
462
456
|
function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
463
457
|
if (getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
|
|
464
458
|
if (typeof obj === "string") {
|
|
465
|
-
if (obj.length > 5e4) {
|
|
466
|
-
logger_default.warn(`String too long (${obj.length} chars) for template matching. Skipping env var rendering.`);
|
|
467
|
-
return obj;
|
|
468
|
-
}
|
|
469
459
|
const nunjucks = getNunjucksEngine();
|
|
470
460
|
const baseEnvGlobals = nunjucks.getGlobal("env");
|
|
471
461
|
const envGlobals = replaceBase ? envOverrides ?? {} : envOverrides ? {
|
|
@@ -479,7 +469,7 @@ function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
|
479
469
|
if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
|
|
480
470
|
return nunjucks.renderString(match, { env: envGlobals });
|
|
481
471
|
} catch (error) {
|
|
482
|
-
|
|
472
|
+
logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
|
|
483
473
|
return match;
|
|
484
474
|
}
|
|
485
475
|
return match;
|
|
@@ -504,7 +494,6 @@ function renderVarsInObject(obj, vars) {
|
|
|
504
494
|
} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
|
|
505
495
|
return obj;
|
|
506
496
|
}
|
|
507
|
-
|
|
508
497
|
//#endregion
|
|
509
498
|
//#region src/util/file.ts
|
|
510
499
|
/**
|
|
@@ -515,7 +504,7 @@ function getNunjucksEngineForFilePath() {
|
|
|
515
504
|
const env = nunjucks.configure({ autoescape: false });
|
|
516
505
|
env.addGlobal("env", {
|
|
517
506
|
...process.env,
|
|
518
|
-
...
|
|
507
|
+
...state.config?.env
|
|
519
508
|
});
|
|
520
509
|
return env;
|
|
521
510
|
}
|
|
@@ -542,16 +531,16 @@ function maybeLoadFromExternalFile(filePath, context) {
|
|
|
542
531
|
const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
|
|
543
532
|
const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
|
|
544
533
|
if (context === "assertion" && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) {
|
|
545
|
-
|
|
534
|
+
logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
|
|
546
535
|
return renderedFilePath;
|
|
547
536
|
}
|
|
548
537
|
if (context === "vars") {
|
|
549
|
-
|
|
538
|
+
logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
|
|
550
539
|
return renderedFilePath;
|
|
551
540
|
}
|
|
552
541
|
if (functionName && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) return renderedFilePath;
|
|
553
542
|
const pathToUse = functionName && !(cleanPath.endsWith(".py") || isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
|
|
554
|
-
const resolvedPath = path$1.resolve(
|
|
543
|
+
const resolvedPath = path$1.resolve(state.basePath || "", pathToUse);
|
|
555
544
|
if (hasMagic(pathToUse)) {
|
|
556
545
|
const matchedFiles = globSync(resolvedPath, { windowsPathsNoEscape: true });
|
|
557
546
|
if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
|
|
@@ -562,7 +551,7 @@ function maybeLoadFromExternalFile(filePath, context) {
|
|
|
562
551
|
contents = fs$2.readFileSync(matchedFile, "utf8");
|
|
563
552
|
} catch (error) {
|
|
564
553
|
if (error.code === "ENOENT") {
|
|
565
|
-
|
|
554
|
+
logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
|
|
566
555
|
continue;
|
|
567
556
|
}
|
|
568
557
|
throw error;
|
|
@@ -772,37 +761,37 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
772
761
|
const { filePath, functionName } = parseFileUrl(rendered);
|
|
773
762
|
if (functionName && (filePath.endsWith(".py") || isJavascriptFile(filePath))) {
|
|
774
763
|
const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
775
|
-
|
|
764
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
|
|
776
765
|
try {
|
|
777
766
|
let toolDefinitions;
|
|
778
767
|
if (filePath.endsWith(".py")) {
|
|
779
|
-
const absPath = safeResolve(
|
|
780
|
-
|
|
768
|
+
const absPath = safeResolve(state.basePath || process.cwd(), filePath);
|
|
769
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
|
|
781
770
|
toolDefinitions = await runPython(absPath, functionName, []);
|
|
782
771
|
} else {
|
|
783
|
-
const absPath = safeResolve(
|
|
784
|
-
|
|
772
|
+
const absPath = safeResolve(state.basePath || process.cwd(), filePath);
|
|
773
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
|
|
785
774
|
const module = await importModule(absPath);
|
|
786
775
|
const fn = module[functionName] || module.default?.[functionName];
|
|
787
776
|
if (typeof fn !== "function") {
|
|
788
777
|
const availableExports = Object.keys(module).filter((k) => k !== "default");
|
|
789
|
-
const basePath =
|
|
778
|
+
const basePath = state.basePath || process.cwd();
|
|
790
779
|
throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
|
|
791
780
|
}
|
|
792
781
|
toolDefinitions = await Promise.resolve(fn());
|
|
793
782
|
}
|
|
794
783
|
if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
|
|
795
|
-
|
|
784
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
|
|
796
785
|
return toolDefinitions;
|
|
797
786
|
} catch (err) {
|
|
798
787
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
799
|
-
const basePath =
|
|
788
|
+
const basePath = state.basePath || process.cwd();
|
|
800
789
|
throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
|
|
801
790
|
}
|
|
802
791
|
}
|
|
803
792
|
if (filePath.endsWith(".py") || isJavascriptFile(filePath)) {
|
|
804
793
|
const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
805
|
-
const basePath =
|
|
794
|
+
const basePath = state.basePath || process.cwd();
|
|
806
795
|
throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? ` def get_tools():\n return [{"type": "function", "function": {...}}]` : ` module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
|
|
807
796
|
}
|
|
808
797
|
}
|
|
@@ -820,7 +809,6 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
820
809
|
}
|
|
821
810
|
return loaded;
|
|
822
811
|
}
|
|
823
|
-
|
|
824
812
|
//#endregion
|
|
825
813
|
//#region src/googleSheets.ts
|
|
826
814
|
async function checkGoogleSheetAccess(url) {
|
|
@@ -835,7 +823,7 @@ async function checkGoogleSheetAccess(url) {
|
|
|
835
823
|
status: response.status
|
|
836
824
|
};
|
|
837
825
|
} catch (error) {
|
|
838
|
-
|
|
826
|
+
logger.error(`Error checking sheet access: ${error}`);
|
|
839
827
|
return { public: false };
|
|
840
828
|
}
|
|
841
829
|
}
|
|
@@ -887,7 +875,7 @@ async function fetchCsvFromGoogleSheetAuthenticated(url) {
|
|
|
887
875
|
}
|
|
888
876
|
async function fetchCsvFromGoogleSheet(url) {
|
|
889
877
|
const { public: isPublic } = await checkGoogleSheetAccess(url);
|
|
890
|
-
|
|
878
|
+
logger.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
|
|
891
879
|
if (isPublic) return fetchCsvFromGoogleSheetUnauthenticated(url);
|
|
892
880
|
return fetchCsvFromGoogleSheetAuthenticated(url);
|
|
893
881
|
}
|
|
@@ -930,7 +918,7 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
930
918
|
});
|
|
931
919
|
range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
|
|
932
920
|
}
|
|
933
|
-
|
|
921
|
+
logger.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
|
|
934
922
|
await sheets.spreadsheets.values.update({
|
|
935
923
|
spreadsheetId,
|
|
936
924
|
range,
|
|
@@ -939,7 +927,6 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
939
927
|
requestBody: { values }
|
|
940
928
|
});
|
|
941
929
|
}
|
|
942
|
-
|
|
943
930
|
//#endregion
|
|
944
931
|
//#region src/server/utils/evalTableUtils.ts
|
|
945
932
|
/**
|
|
@@ -1263,7 +1250,6 @@ async function streamEvalCsv(eval_, options) {
|
|
|
1263
1250
|
isRedteam
|
|
1264
1251
|
})]));
|
|
1265
1252
|
}
|
|
1266
|
-
|
|
1267
1253
|
//#endregion
|
|
1268
1254
|
//#region src/util/output.ts
|
|
1269
1255
|
const outputToSimpleString = (output) => {
|
|
@@ -1328,7 +1314,7 @@ async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
|
|
|
1328
1314
|
const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
|
|
1329
1315
|
if (isStringLen || isHeapOOM) {
|
|
1330
1316
|
const resultCount = await evalRecord.getResultsCount();
|
|
1331
|
-
|
|
1317
|
+
logger.error(`Dataset too large for JSON export (${resultCount} results).`);
|
|
1332
1318
|
throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
|
|
1333
1319
|
} else throw error;
|
|
1334
1320
|
}
|
|
@@ -1347,7 +1333,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1347
1333
|
});
|
|
1348
1334
|
return csvRow;
|
|
1349
1335
|
});
|
|
1350
|
-
|
|
1336
|
+
logger.info(`Writing ${rows.length} rows to Google Sheets...`);
|
|
1351
1337
|
await writeCsvToGoogleSheet(rows, outputPath);
|
|
1352
1338
|
return;
|
|
1353
1339
|
}
|
|
@@ -1429,14 +1415,13 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1429
1415
|
async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
|
|
1430
1416
|
await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
|
|
1431
1417
|
}
|
|
1432
|
-
|
|
1433
1418
|
//#endregion
|
|
1434
1419
|
//#region src/util/runtime.ts
|
|
1435
1420
|
function printBorder() {
|
|
1436
1421
|
const border = "=".repeat(TERMINAL_MAX_WIDTH);
|
|
1437
|
-
|
|
1422
|
+
logger.info(border);
|
|
1438
1423
|
}
|
|
1439
|
-
|
|
1440
1424
|
//#endregion
|
|
1441
1425
|
export { filterRuntimeVars as A, extractVariablesFromTemplates as C, setupEnv as D, parseFileUrl as E, getProviderDescription as F, isAnthropicProvider as I, isGoogleProvider as L, resultIsForTestCase as M, checkProviderApiKeys as N, deduplicateTestCases as O, doesProviderRefMatch as P, isOpenAiProvider as R, extractVariablesFromTemplate as S, loadFunction as T, parsePathOrGlob as _, ComparisonEvalNotFoundError as a, renderEnvOnlyInObject as b, mergeComparisonTables as c, getResolvedRelativePath as d, maybeLoadConfigFromExternalFile as f, maybeLoadToolsFromExternalFile as g, maybeLoadResponseFormatFromExternalFile as h, writeOutput as i, getTestCaseDeduplicationKey as j, extractRuntimeVars as k, fetchCsvFromGoogleSheet as l, maybeLoadFromExternalFileWithVars as m, createOutputMetadata as n, evalTableToJson as o, maybeLoadFromExternalFile as p, writeMultipleOutputs as r, generateEvalCsv as s, printBorder as t, getNunjucksEngineForFilePath as u, readFilters as v, getNunjucksEngine as w, renderVarsInObject as x, readOutput as y, isProviderAllowed as z };
|
|
1442
|
-
|
|
1426
|
+
|
|
1427
|
+
//# sourceMappingURL=util-C1CeHl-P.js.map
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { g as safeJsonStringify, p as getAjv } from "./logger-
|
|
3
|
-
import { p as calculateCost } from "./fetch-
|
|
4
|
-
import { m as maybeLoadFromExternalFileWithVars } from "./util-
|
|
2
|
+
import { g as safeJsonStringify, p as getAjv } from "./logger-KkObSCzq.js";
|
|
3
|
+
import { p as calculateCost } from "./fetch-Dxpd4_sr.js";
|
|
4
|
+
import { m as maybeLoadFromExternalFileWithVars } from "./util-C1CeHl-P.js";
|
|
5
5
|
import OpenAI from "openai";
|
|
6
|
-
|
|
7
6
|
//#region src/providers/openai/util.ts
|
|
8
7
|
const ajv = getAjv();
|
|
9
8
|
const OPENAI_CHAT_MODELS = [
|
|
@@ -595,7 +594,7 @@ function formatOpenAiError(data) {
|
|
|
595
594
|
errorMessage += "\n\n" + safeJsonStringify(data, true);
|
|
596
595
|
return errorMessage;
|
|
597
596
|
}
|
|
598
|
-
|
|
599
597
|
//#endregion
|
|
600
598
|
export { calculateOpenAICost as a, getTokenUsage as c, OPENAI_TRANSCRIPTION_MODELS as i, validateFunctionCall as l, OPENAI_COMPLETION_MODELS as n, failApiCall as o, OPENAI_REALTIME_MODELS as r, formatOpenAiError as s, OPENAI_CHAT_MODELS as t };
|
|
601
|
-
|
|
599
|
+
|
|
600
|
+
//# sourceMappingURL=util-CMy69ZgQ.js.map
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
const require_fetch = require(
|
|
2
|
-
const require_transform = require(
|
|
3
|
-
|
|
1
|
+
const require_fetch = require("./fetch-BPkYtG8K.cjs");
|
|
2
|
+
const require_transform = require("./transform-DuHvhZpj.cjs");
|
|
4
3
|
//#region src/providers/anthropic/util.ts
|
|
5
4
|
const ANTHROPIC_MODELS = [
|
|
6
5
|
...["claude-sonnet-4-6", "claude-sonnet-4-6-latest"].map((model) => ({
|
|
@@ -288,42 +287,42 @@ function transformWebSearchTool(config) {
|
|
|
288
287
|
if (config.cache_control) tool.cache_control = config.cache_control;
|
|
289
288
|
return tool;
|
|
290
289
|
}
|
|
291
|
-
|
|
292
290
|
//#endregion
|
|
293
|
-
Object.defineProperty(exports,
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
291
|
+
Object.defineProperty(exports, "ANTHROPIC_MODELS", {
|
|
292
|
+
enumerable: true,
|
|
293
|
+
get: function() {
|
|
294
|
+
return ANTHROPIC_MODELS;
|
|
295
|
+
}
|
|
298
296
|
});
|
|
299
|
-
Object.defineProperty(exports,
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
297
|
+
Object.defineProperty(exports, "calculateAnthropicCost", {
|
|
298
|
+
enumerable: true,
|
|
299
|
+
get: function() {
|
|
300
|
+
return calculateAnthropicCost;
|
|
301
|
+
}
|
|
304
302
|
});
|
|
305
|
-
Object.defineProperty(exports,
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
303
|
+
Object.defineProperty(exports, "getTokenUsage", {
|
|
304
|
+
enumerable: true,
|
|
305
|
+
get: function() {
|
|
306
|
+
return getTokenUsage;
|
|
307
|
+
}
|
|
310
308
|
});
|
|
311
|
-
Object.defineProperty(exports,
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
309
|
+
Object.defineProperty(exports, "outputFromMessage", {
|
|
310
|
+
enumerable: true,
|
|
311
|
+
get: function() {
|
|
312
|
+
return outputFromMessage;
|
|
313
|
+
}
|
|
316
314
|
});
|
|
317
|
-
Object.defineProperty(exports,
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
315
|
+
Object.defineProperty(exports, "parseMessages", {
|
|
316
|
+
enumerable: true,
|
|
317
|
+
get: function() {
|
|
318
|
+
return parseMessages;
|
|
319
|
+
}
|
|
322
320
|
});
|
|
323
|
-
Object.defineProperty(exports,
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
321
|
+
Object.defineProperty(exports, "processAnthropicTools", {
|
|
322
|
+
enumerable: true,
|
|
323
|
+
get: function() {
|
|
324
|
+
return processAnthropicTools;
|
|
325
|
+
}
|
|
328
326
|
});
|
|
329
|
-
|
|
327
|
+
|
|
328
|
+
//# sourceMappingURL=util-DGNOS1db.cjs.map
|