promptfoo 0.120.27 → 0.121.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/{ListApp-8WOe2nT6.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-BgNJDBE6.js +206 -0
- package/dist/src/{accounts-Fl2J3_Fu.cjs → accounts-Bx-x3bmW.cjs} +77 -78
- package/dist/src/{accounts-DVINui-2.js → accounts-CMqkzrVf.js} +39 -34
- package/dist/src/{accounts-CPDRAMND.js → accounts-xrUGFA6n.js} +38 -33
- package/dist/src/{agentic-utils-D922n6mm.js → agentic-utils-BKIN5PKu.js} +9 -10
- package/dist/src/{agents-BO2n8Z0d.cjs → agents-B0f4HICh.cjs} +37 -40
- package/dist/src/{agents-BXLmVsxR.js → agents-C-dDThPK.js} +37 -37
- package/dist/src/{agents-DgJf2-ez.cjs → agents-CErsqg5U.cjs} +16 -17
- package/dist/src/{agents-BcsN_BgB.js → agents-CVIn-Utx.js} +16 -12
- package/dist/src/{agents-hqgSV-3o.js → agents-CXknwsFX.js} +37 -40
- package/dist/src/{agents-pMfppv9Z.js → agents-DeH4Gu94.js} +18 -18
- package/dist/src/{agents-BdUTAwi-.js → agents-Dy2YpZpa.js} +38 -41
- package/dist/src/{agents-DNvSH78i.js → agents-aF4-T121.js} +16 -20
- package/dist/src/{aimlapi-DOib86oE.js → aimlapi-BAGZDo5G.js} +16 -18
- package/dist/src/{aimlapi-DtgPI0nE.js → aimlapi-BNfTBexL.js} +15 -17
- package/dist/src/{aimlapi-DTPACCB1.js → aimlapi-DHRKlBEA.js} +15 -4
- package/dist/src/{aimlapi-BE_Tg9Fl.cjs → aimlapi-tg0Gkcvr.cjs} +15 -16
- package/dist/src/app/assets/index-BFCZg7hQ.js +439 -0
- package/dist/src/app/index.html +1 -1
- package/dist/src/{audio-BRYU0BFo.js → audio-BRODU0UK.js} +7 -9
- package/dist/src/{audio-Cwo68yZS.cjs → audio-BWeaWovU.cjs} +6 -7
- package/dist/src/{audio-BnRUGAm_.js → audio-CHQ4r-RV.js} +6 -5
- package/dist/src/{audio-MSRki4JU.js → audio-tf_NBjlC.js} +6 -8
- package/dist/src/{base-h961VXYk.js → base-B0tcrnq_.js} +11 -13
- package/dist/src/{base-XB2tDJrB.js → base-B4QJRyFS.js} +11 -13
- package/dist/src/{base-pGVmXNl4.cjs → base-DBtwl2FR.cjs} +36 -38
- package/dist/src/base-fEDN28WM.js +193 -0
- package/dist/src/{blobs-BM_e6hCa.js → blobs-BAU-dXan.js} +9 -12
- package/dist/src/{blobs-CR5C4Ihh.js → blobs-Bpg5rH6i.js} +9 -12
- package/dist/src/{blobs-B-KQAFhX.cjs → blobs-DvS-O6be.cjs} +34 -37
- package/dist/src/blobs-qTYm-1PY.js +236 -0
- package/dist/src/{cache-CIpsoBZR.js → cache-8XhNqPKW.js} +64 -67
- package/dist/src/cache-Bbn1Nyrd.cjs +5 -0
- package/dist/src/cache-BwsMSda7.js +6 -0
- package/dist/src/{cache-jsiwsAJv.js → cache-CG0SlR1d.js} +64 -66
- package/dist/src/{cache-BTVYfbka.cjs → cache-COish3-W.cjs} +114 -117
- package/dist/src/cache-D3eqDYGU.js +739 -0
- package/dist/src/{chat-D31K7C4u.cjs → chat-2K608PeQ.cjs} +20 -21
- package/dist/src/chat-BKm79wib.js +764 -0
- package/dist/src/{chat-B84t99NW.js → chat-CM_kyI8B.js} +20 -9
- package/dist/src/{chat-BcPjZXIp.js → chat-CRWNNq73.js} +41 -44
- package/dist/src/{chat-CcUCysjU.js → chat-CznLWr_D.js} +41 -44
- package/dist/src/{chat-DwWifjxi.js → chat-DHMH-N64.js} +20 -22
- package/dist/src/{chat-BE44YOc6.cjs → chat-DaqekjFr.cjs} +61 -64
- package/dist/src/{chat-DZM2GUHO.js → chat-DxysjBvt.js} +21 -23
- package/dist/src/{chatkit-D67HS_0b.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DAB_qfzI.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-Biqb_wsD.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-PGG4ZYIn.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-SVM6AdBu.js → claude-agent-sdk-BLTu0WBO.js} +31 -31
- package/dist/src/{claude-agent-sdk-C9SiaQub.cjs → claude-agent-sdk-CJH22shf.cjs} +31 -28
- package/dist/src/{claude-agent-sdk-C-IOTPfo.js → claude-agent-sdk-D6_k9FKA.js} +31 -29
- package/dist/src/{claude-agent-sdk-CiluSyW1.js → claude-agent-sdk-Dy5lT-Tx.js} +33 -20
- package/dist/src/{cloud-CZ-q9Ier.js → cloud-Bc9526yV.js} +7 -9
- package/dist/src/cloud-DmE0EwsY.js +4 -0
- package/dist/src/{cloudflare-ai-BahKHyhh.js → cloudflare-ai-C9r2sRhw.js} +16 -18
- package/dist/src/{cloudflare-ai-Dxyt50Nl.js → cloudflare-ai-CWWJCRim.js} +16 -4
- package/dist/src/{cloudflare-ai-Dfahv5SY.cjs → cloudflare-ai-ClWSdor4.cjs} +16 -17
- package/dist/src/{cloudflare-ai-v_qZD6_q.js → cloudflare-ai-ICsOuD-z.js} +17 -19
- package/dist/src/{cloudflare-gateway-BPWoZIzJ.cjs → cloudflare-gateway-C2_-KG5o.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-Bi_FpOFy.js → cloudflare-gateway-D6O7AlYb.js} +23 -23
- package/dist/src/{cloudflare-gateway-btS7h1OZ.js → cloudflare-gateway-D6xFc5pa.js} +21 -25
- package/dist/src/{cloudflare-gateway-C0guUNwk.js → cloudflare-gateway-pXGHxJ47.js} +26 -14
- package/dist/src/{codex-sdk-DSxAnbfT.js → codex-sdk-C6UMlxwV.js} +28 -29
- package/dist/src/{codex-sdk-IYVi9fuM.js → codex-sdk-DUwKWezN.js} +28 -27
- package/dist/src/{codex-sdk-DulY0ZRq.js → codex-sdk-GGAw0qbD.js} +28 -29
- package/dist/src/{codex-sdk-DFKMtAyf.cjs → codex-sdk-fAO0c3yA.cjs} +28 -29
- package/dist/src/{cometapi-DkNBMk0G.js → cometapi-BasUi7-_.js} +17 -19
- package/dist/src/{cometapi-DzrR3SR_.js → cometapi-Bbjp5V4x.js} +16 -4
- package/dist/src/{cometapi-C9EEpJzT.js → cometapi-DkXrKi5z.js} +21 -24
- package/dist/src/{cometapi-DIO64tf4.cjs → cometapi-vY6aDZgo.cjs} +21 -22
- package/dist/src/{completion-CG29bfKX.js → completion-6Mx_iXxK.js} +11 -13
- package/dist/src/{completion-Bgf1VJoq.js → completion-C5rtR_9P.js} +11 -13
- package/dist/src/{completion-CCRT4kX1.cjs → completion-CDOouNzq.cjs} +21 -23
- package/dist/src/completion-C_P3ypkJ.js +120 -0
- package/dist/src/{createHash-Dw_iLu31.js → createHash-CTQmL3G2.js} +2 -3
- package/dist/src/{createHash-CYQy4YeL.cjs → createHash-CfZSc0b4.cjs} +13 -14
- package/dist/src/{createHash-CJcfskIZ.js → createHash-Da8fMwqB.js} +2 -3
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-D-ayp2FW.js → docker-5KcG-_86.js} +18 -20
- package/dist/src/{docker-DNcLR4Ig.cjs → docker-BwsKwxFs.cjs} +18 -19
- package/dist/src/{docker-egERKxCF.js → docker-CZnqU1XV.js} +18 -7
- package/dist/src/{docker-B81N0t4e.js → docker-DzxyDPIj.js} +19 -21
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-B9dPm_BF.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-D2pZ87fL.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-Ct-Joyue.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-17JizQIv.js +15 -0
- package/dist/src/{eval-C-Nr6wX_.js → eval-DmFyWU7i.js} +47 -54
- package/dist/src/{evalResult-4BzI2tmj.js → evalResult-CDQiuUuf.js} +16 -12
- package/dist/src/{evalResult-DXMWJ3sx.js → evalResult-CTG2AHOS.js} +10 -11
- package/dist/src/evalResult-Cqj8pldJ.js +12 -0
- package/dist/src/{evalResult-CX8wQecI.cjs → evalResult-Dap2CekP.cjs} +20 -21
- package/dist/src/evalResult-DvcJAWJU.cjs +10 -0
- package/dist/src/evalResult-Hftn-S_i.js +10 -0
- package/dist/src/evaluator-B2CFNt-P.js +36 -0
- package/dist/src/{evaluator-8aGyV12L.js → evaluator-DPFRbFIL.js} +201 -229
- package/dist/src/{extractor-CD5yKL-G.js → extractor-CFG6bcWJ.js} +22 -24
- package/dist/src/{extractor-C031XmTA.cjs → extractor-DX36oYEv.cjs} +37 -39
- package/dist/src/{extractor-V5x_m1i0.js → extractor-M67RUtg6.js} +22 -24
- package/dist/src/extractor-YMU_Gvt8.js +374 -0
- package/dist/src/{fetch-D3OHf-lV.js → fetch-4M3YRaqL.js} +40 -45
- package/dist/src/fetch-60Gzydls.js +777 -0
- package/dist/src/{fetch-CXZI9RRr.js → fetch-BMv0O527.js} +23 -35
- package/dist/src/{fetch-BmbD-v1L.cjs → fetch-BxUk8odA.cjs} +244 -277
- package/dist/src/fetch-KV5kNASw.js +5 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-CdevI3An.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-Dres3qrN.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/graders-Bu0H9nXi.js +32 -0
- package/dist/src/{graders-DTeBrzWp.js → graders-CHO8EPM4.js} +349 -397
- package/dist/src/graders-Cfhkvx-e.js +34 -0
- package/dist/src/{graders--1y2u9HO.js → graders-CpdqD9PI.js} +349 -397
- package/dist/src/graders-DClJVpGP.cjs +32 -0
- package/dist/src/{graders-DohM2dir.cjs → graders-DOXycdlG.cjs} +684 -732
- package/dist/src/graders-DcnJsrMO.js +32 -0
- package/dist/src/graders-R9rYUM0d.js +13466 -0
- package/dist/src/{image-C3wHC9_h.js → image-BmEZqVmk.js} +9 -10
- package/dist/src/{image-O1u4bCFg.js → image-CBBVXWuT.js} +9 -10
- package/dist/src/{image-DpKl2F15.cjs → image-CDLQOcqT.cjs} +6 -7
- package/dist/src/{image-DmE-niFE.js → image-DJEvKveK.js} +6 -5
- package/dist/src/{image-CuKHuccK.cjs → image-DTedmQPg.cjs} +29 -30
- package/dist/src/{image-B0U4Hqll.js → image-gvmivTEe.js} +7 -9
- package/dist/src/image-pAX56tPG.js +257 -0
- package/dist/src/{image-DNEIf_aI.js → image-tL5hIOFh.js} +6 -8
- package/dist/src/index.cjs +605 -689
- package/dist/src/index.d.cts +11 -7
- package/dist/src/index.d.ts +11 -3
- package/dist/src/index.js +570 -658
- package/dist/src/{interactiveCheck-Bxj1Swex.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-Be_zyW4L.js → knowledgeBase-CLJybhnF.js} +16 -16
- package/dist/src/{knowledgeBase-CEzQobWX.js → knowledgeBase-CoU-UQBg.js} +14 -9
- package/dist/src/{knowledgeBase-BZ41IFwq.js → knowledgeBase-DjWPVqSb.js} +14 -18
- package/dist/src/{knowledgeBase-D-5BMXlr.cjs → knowledgeBase-wkxuRFhA.cjs} +14 -15
- package/dist/src/{litellm-DnbRJ2if.js → litellm-B9Hysuri.js} +16 -18
- package/dist/src/{litellm-CRDqPhNI.js → litellm-CTfa0hqi.js} +15 -17
- package/dist/src/{litellm-hUSNM_M2.cjs → litellm-NYpQ8RQu.cjs} +15 -16
- package/dist/src/{litellm-9vR8zpfU.js → litellm-ePxtr9F1.js} +15 -4
- package/dist/src/{logger-CG1uZPbQ.js → logger-CT3IKMKA.js} +10 -29
- package/dist/src/{logger-B7sBeGa0.cjs → logger-Cp1GPUjj.cjs} +152 -180
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-LSBxlt7a.js → logger-KkObSCzq.js} +13 -31
- package/dist/src/{luma-ray-Hm3d6VJE.cjs → luma-ray-B0GGNRc1.cjs} +20 -21
- package/dist/src/{luma-ray-drvgdpP9.js → luma-ray-BE2mOt6N.js} +20 -13
- package/dist/src/{luma-ray-4blv9iZ2.js → luma-ray-BW9IRGIc.js} +22 -21
- package/dist/src/{luma-ray-B2__8lYH.js → luma-ray-Cm1KZBhs.js} +20 -23
- package/dist/src/main.js +1170 -1321
- package/dist/src/{messages-XhiwCbi4.cjs → messages-1JrJs91T.cjs} +32 -34
- package/dist/src/{messages-CGPPidQr.js → messages-1x9atZmP.js} +22 -24
- package/dist/src/{messages-Uee41Mj5.js → messages-BLbWdsyt.js} +22 -24
- package/dist/src/messages-D8EA0oDc.js +240 -0
- package/dist/src/{meteor-BYykdXrV.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-CsopaHrH.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-e-E-2vVl.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-C8lGP6P4.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/{modelslab-yKz-ZNB4.js → modelslab-C1OLRmVX.js} +17 -10
- package/dist/src/{modelslab-E9gO-bYd.js → modelslab-CqXBy3U8.js} +18 -20
- package/dist/src/{modelslab-lUVW0cmB.cjs → modelslab-DcOSFwKh.cjs} +17 -18
- package/dist/src/{modelslab-ClBkr8_9.js → modelslab-X5-4LroM.js} +17 -19
- package/dist/src/{nova-reel-Dk8jNpId.js → nova-reel-BgS1ZWuK.js} +20 -13
- package/dist/src/{nova-reel-u2eF2Cxm.js → nova-reel-D2ZkOSyr.js} +22 -21
- package/dist/src/{nova-reel-D8CuO6QH.cjs → nova-reel-D9xfaMBs.cjs} +20 -21
- package/dist/src/{nova-reel-P9bwvtYX.js → nova-reel-DihqLeol.js} +20 -23
- package/dist/src/{nova-sonic-Ds1C-dpm.cjs → nova-sonic-DVu3mMIy.cjs} +30 -31
- package/dist/src/{nova-sonic-CK2rAiKi.js → nova-sonic-DezhVUYT.js} +30 -26
- package/dist/src/{nova-sonic-BaqWlkds.js → nova-sonic-P-CdUMlV.js} +30 -31
- package/dist/src/{nova-sonic-yZapPLv7.js → nova-sonic-Q3BOJeig.js} +31 -32
- package/dist/src/{openai-DUFopMrH.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-PblZ3jUE.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CcN1B8Sb.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-A-3_loM7.js → openclaw-BiSZPL7J.js} +20 -14
- package/dist/src/{openclaw-COn6QzDi.js → openclaw-Bv1DINsX.js} +20 -27
- package/dist/src/{openclaw-a3lylB-V.js → openclaw-D1D_ej1z.js} +21 -28
- package/dist/src/{openclaw-B6qqDr_u.cjs → openclaw-DAfWQn-o.cjs} +33 -39
- package/dist/src/opencode-sdk-C7m-wRfI.js +560 -0
- package/dist/src/opencode-sdk-CfaLN8PY.cjs +564 -0
- package/dist/src/opencode-sdk-D95s6SnR.js +562 -0
- package/dist/src/opencode-sdk-DxUPkLT7.js +560 -0
- package/dist/src/{otlpReceiver-oyf5wLGC.js → otlpReceiver--AIRW_S4.js} +53 -51
- package/dist/src/{otlpReceiver-BmmTiMjA.js → otlpReceiver-Bn5wGB1v.js} +53 -55
- package/dist/src/{otlpReceiver-lXsYVbpj.cjs → otlpReceiver-Diec4cln.cjs} +53 -55
- package/dist/src/{otlpReceiver-94URx7UW.js → otlpReceiver-g3ByGaXs.js} +53 -55
- package/dist/src/{providerRegistry-Cq_JK_CJ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-DSSHjMKf.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-CvHEVJad.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B3HvufyI.js +33246 -0
- package/dist/src/{providers-BnFpbY_s.js → providers-BKRJTjBz.js} +1536 -1669
- package/dist/src/providers-C1rOSHiR.js +32 -0
- package/dist/src/{providers-Iil64vk9.js → providers-CFLy1_ji.js} +1543 -1676
- package/dist/src/{providers-DHbjzW2e.cjs → providers-CFu-TZl-.cjs} +1896 -2029
- package/dist/src/providers-CxmDwEFf.cjs +31 -0
- package/dist/src/providers-Dodakqr0.js +30 -0
- package/dist/src/providers-GIQ2TcsA.js +30 -0
- package/dist/src/{pythonUtils-CcT5LH1M.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DBbuI3QJ.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-hZ8LeQLv.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/{quiverai-BuI0tE39.js → quiverai-C2jVwbH1.js} +8 -7
- package/dist/src/{quiverai-DCGSZt4U.js → quiverai-CI6gYJVI.js} +8 -10
- package/dist/src/{quiverai-DiMVJQDz.cjs → quiverai-CLkWkyZc.cjs} +8 -9
- package/dist/src/{quiverai-fQNkExW4.js → quiverai-MHSxbmmZ.js} +9 -11
- package/dist/src/{render-Dj1smHEb.js → render-Drod8m7K.js} +4 -5
- package/dist/src/{responses-DOAFFENS.js → responses-BKqJmhhc.js} +22 -25
- package/dist/src/{responses-CxzoQoBe.js → responses-CGw0DCzh.js} +22 -25
- package/dist/src/responses-jxdehPkC.js +660 -0
- package/dist/src/{responses-ghR3IOfy.cjs → responses-tD4Bd4dc.cjs} +37 -40
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-CwbGmgYN.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-DudlFZed.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-C8MhKGHb.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-gmskuyre.js → sagemaker-BK4Zb993.js} +75 -70
- package/dist/src/{sagemaker-DuM71dVU.js → sagemaker-BfiWTmvn.js} +77 -77
- package/dist/src/{sagemaker-77zbJ2Q2.cjs → sagemaker-CcQHM1jV.cjs} +75 -76
- package/dist/src/{sagemaker-CcxhlOAR.js → sagemaker-D2Q1c-sD.js} +75 -79
- package/dist/src/{scanner-DJYiSXQj.js → scanner-J8CA3LsV.js} +100 -121
- package/dist/src/server/index.js +5505 -67416
- package/dist/src/{server-B5v33lvE.cjs → server-B0PPuDw-.cjs} +57 -67
- package/dist/src/server-B1vi21hA.js +7 -0
- package/dist/src/{server-RV_i_YX5.js → server-BC7XJFgr.js} +19 -24
- package/dist/src/server-Cm9Kai_h.cjs +5 -0
- package/dist/src/{server-BJ4m4f1D.js → server-DbFphssR.js} +26 -29
- package/dist/src/server-OAs3nBRT.js +229 -0
- package/dist/src/{signal-BW33JuId.js → signal-BOTbd53Z.js} +9 -11
- package/dist/src/{slack-DEURelTy.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-BQYeW9L3.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-BB6yuZzp.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-2pRrhhgJ.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-BNmZ1KAz.cjs +5 -0
- package/dist/src/{store-D7CgQzAR.cjs → store-BSc-TF2w.cjs} +44 -45
- package/dist/src/store-BltJg2cd.js +6 -0
- package/dist/src/{store-s3SftUwF.js → store-D1tv90v3.js} +34 -35
- package/dist/src/{store-DJNsD1iC.js → store-DQLEjuEO.js} +40 -36
- package/dist/src/store-Ub2vaGJ1.js +228 -0
- package/dist/src/{tables-DfTsNN7X.js → tables-5EvT_Bwn.js} +19 -21
- package/dist/src/{tables-BKTmd6u7.cjs → tables-C7K-XKWp.cjs} +89 -91
- package/dist/src/{tables-DMegD0Xf.js → tables-D36WTqKX.js} +21 -23
- package/dist/src/tables-xKANLRBD.js +288 -0
- package/dist/src/telemetry-5BCRNBbe.cjs +5 -0
- package/dist/src/{telemetry-BedSm-bZ.js → telemetry-C15ziL8u.js} +17 -14
- package/dist/src/{telemetry--WAdAfVi.js → telemetry-C2YDkUQH.js} +11 -13
- package/dist/src/{telemetry-DQgVBCAb.cjs → telemetry-CbrnxHp_.cjs} +21 -24
- package/dist/src/telemetry-D4W5hboe.js +7 -0
- package/dist/src/telemetry-DMb2Mpfm.js +171 -0
- package/dist/src/{text-oiSbwSOI.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-oKzCBnK6.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-B_IrO4GZ.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-BO1AHegO.cjs → transcription-CL78qbOU.cjs} +14 -15
- package/dist/src/{transcription-mYS9vd5v.js → transcription-DAtxHhAM.js} +14 -7
- package/dist/src/{transcription-X2-B4vkX.js → transcription-LNZTNUUL.js} +14 -16
- package/dist/src/{transcription-lzBLiTFJ.js → transcription-QHh3AH6Z.js} +15 -17
- package/dist/src/{transform-DeGlxb0D.js → transform-Cgi24fJ7.js} +39 -47
- package/dist/src/{transform-B1Hi5lWS.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-CYDILYDe.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/{transform-Dfl89yi4.js → transform-DGLazrMm.js} +39 -47
- package/dist/src/transform-DGxXocjk.js +1506 -0
- package/dist/src/{transform-D5PjiWiZ.cjs → transform-DOcQeLld.cjs} +179 -187
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-BEgStbHK.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/{transformersAvailability-SZnTS3pJ.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-D-glmEy7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-CjeFXhuJ.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-CXQduE9o.js → types-CH3Ge2sE.js} +30 -90
- package/dist/src/{types-C5hEkb-x.js → types-CLKiCBW3.js} +25 -89
- package/dist/src/types-CN_TZ2GJ.js +3260 -0
- package/dist/src/{types-DWNf48sT.cjs → types-LJ0r3wbR.cjs} +500 -564
- package/dist/src/util-5cB-L7U3.js +1430 -0
- package/dist/src/util-6-GqIvzS.js +599 -0
- package/dist/src/{util-CoQjmE3u.js → util-B7T3SiBS.js} +4 -5
- package/dist/src/{util-D9eLdGfa.js → util-Betm42rL.js} +5 -6
- package/dist/src/{util-Bm_-UMD_.js → util-C-PPYSMq.js} +5 -6
- package/dist/src/{util-CyUdMzV0.cjs → util-CchiqXh_.cjs} +34 -35
- package/dist/src/{util-Du96oyYS.js → util-DaWTWKBK.js} +4 -5
- package/dist/src/{util-1wWM599Z.cjs → util-Db0a0AFH.cjs} +50 -51
- package/dist/src/{util-DQ984syk.js → util-Dlz_Wvgm.js} +37 -48
- package/dist/src/{util-_h4pVqrz.js → util-YT5HPZaS.js} +37 -48
- package/dist/src/{util-aLhtl3fe.cjs → util-Yz-1aEhW.cjs} +209 -220
- package/dist/src/util-ZZH-3QZz.js +293 -0
- package/dist/src/{utils-BjLy-Q72.cjs → utils-Cz9qXqII.cjs} +29 -32
- package/dist/src/{utils-CFMn2yHW.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-DvWMzuMx.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +30 -30
- package/dist/src/app/assets/index-B2D0bCSI.js +0 -439
- package/dist/src/app/tsconfig.app.tsbuildinfo +0 -1
- package/dist/src/cache-ChPcurj7.js +0 -6
- package/dist/src/cache-VVu_W-yg.js +0 -8
- package/dist/src/cache-YLNCFEM2.cjs +0 -6
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-BndfXy4H.js +0 -5
- package/dist/src/eval-BhHvMY82.js +0 -17
- package/dist/src/evalResult-Dq2gFNQY.js +0 -12
- package/dist/src/evalResult-nmcP5VKH.cjs +0 -12
- package/dist/src/evalResult-trqZjVYh.js +0 -14
- package/dist/src/evaluator-CnfPstzT.js +0 -39
- package/dist/src/fetch-IDPDue6F.cjs +0 -4
- package/dist/src/fetch-hKJ-It8q.js +0 -6
- package/dist/src/fetch-ouKnrWK-.js +0 -4
- package/dist/src/graders-CQn7WUsd.cjs +0 -34
- package/dist/src/graders-DC6QAbpW.js +0 -35
- package/dist/src/graders-DUWz3Y7j.js +0 -37
- package/dist/src/opencode-sdk-4bL9n-Gk.js +0 -382
- package/dist/src/opencode-sdk-BfC2zWcR.js +0 -376
- package/dist/src/opencode-sdk-DMJyuwMg.js +0 -380
- package/dist/src/opencode-sdk-Da-9adza.cjs +0 -383
- package/dist/src/providers-CsXB2Ix-.js +0 -35
- package/dist/src/providers-DO8ltjLC.js +0 -33
- package/dist/src/providers-Dtq-xnXd.cjs +0 -33
- package/dist/src/rubyUtils-BUbcND2f.js +0 -6
- package/dist/src/rubyUtils-Cr55X_KE.js +0 -5
- package/dist/src/rubyUtils-DlIiqoYo.cjs +0 -5
- package/dist/src/server-C2eQH4Gu.js +0 -6
- package/dist/src/server-CXWycu7H.cjs +0 -6
- package/dist/src/server-Q6OGlxxT.js +0 -8
- package/dist/src/store-B3EDO9Q3.js +0 -7
- package/dist/src/store-Dl9F8aw5.js +0 -6
- package/dist/src/store-SnrGrlt9.cjs +0 -6
- package/dist/src/telemetry-BGhiPZtl.js +0 -8
- package/dist/src/telemetry-CFfiYan6.cjs +0 -6
- package/dist/src/telemetry-DHzEduxX.js +0 -6
- package/dist/src/transform-C1x1ZlMQ.cjs +0 -6
- package/dist/src/transform-DYHjFmQu.js +0 -8
- package/dist/src/transform-rmwJT5JQ.js +0 -7
- package/dist/src/transformersAvailability-eJooj0gX.js +0 -35
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const require_logger = require(
|
|
2
|
-
const require_invariant = require(
|
|
3
|
-
const require_esm = require(
|
|
4
|
-
const require_pythonUtils = require(
|
|
5
|
-
const require_fileExtensions = require(
|
|
6
|
-
const require_types = require(
|
|
7
|
-
const require_fetch = require(
|
|
1
|
+
const require_logger = require("./logger-Cp1GPUjj.cjs");
|
|
2
|
+
const require_invariant = require("./invariant-kfQ8Bu82.cjs");
|
|
3
|
+
const require_esm = require("./esm-CnNt7sI4.cjs");
|
|
4
|
+
const require_pythonUtils = require("./pythonUtils-CTU3Y3lw.cjs");
|
|
5
|
+
const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
|
|
6
|
+
const require_types = require("./types-LJ0r3wbR.cjs");
|
|
7
|
+
const require_fetch = require("./fetch-BxUk8odA.cjs");
|
|
8
8
|
let fs = require("fs");
|
|
9
9
|
fs = require_logger.__toESM(fs);
|
|
10
10
|
let path = require("path");
|
|
@@ -27,7 +27,6 @@ let nunjucks = require("nunjucks");
|
|
|
27
27
|
nunjucks = require_logger.__toESM(nunjucks);
|
|
28
28
|
let fast_xml_parser = require("fast-xml-parser");
|
|
29
29
|
let csv_stringify_sync = require("csv-stringify/sync");
|
|
30
|
-
|
|
31
30
|
//#region src/util/provider.ts
|
|
32
31
|
function canonicalizeProviderId(id) {
|
|
33
32
|
if (id.startsWith("file://")) {
|
|
@@ -198,7 +197,6 @@ function isGoogleProvider(providerId) {
|
|
|
198
197
|
}
|
|
199
198
|
return false;
|
|
200
199
|
}
|
|
201
|
-
|
|
202
200
|
//#endregion
|
|
203
201
|
//#region src/util/comparison.ts
|
|
204
202
|
/**
|
|
@@ -297,7 +295,7 @@ function resultIsForTestCase(result, testCase) {
|
|
|
297
295
|
const isMatch = doVarsMatch && providersMatch;
|
|
298
296
|
if (!isMatch) {
|
|
299
297
|
const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
|
|
300
|
-
require_logger.
|
|
298
|
+
require_logger.logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
|
|
301
299
|
testProvider: testProviderId || "none",
|
|
302
300
|
resultProvider: resultProviderId || "none",
|
|
303
301
|
testVarKeys: varKeys
|
|
@@ -305,7 +303,6 @@ function resultIsForTestCase(result, testCase) {
|
|
|
305
303
|
}
|
|
306
304
|
return isMatch;
|
|
307
305
|
}
|
|
308
|
-
|
|
309
306
|
//#endregion
|
|
310
307
|
//#region src/util/env.ts
|
|
311
308
|
/**
|
|
@@ -322,8 +319,8 @@ function setupEnv(envPath) {
|
|
|
322
319
|
return;
|
|
323
320
|
}
|
|
324
321
|
for (const p of paths) if (!fs.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
|
|
325
|
-
if (paths.length === 1) require_logger.
|
|
326
|
-
else require_logger.
|
|
322
|
+
if (paths.length === 1) require_logger.logger.info(`Loading environment variables from ${paths[0]}`);
|
|
323
|
+
else require_logger.logger.info(`Loading environment variables from: ${paths.join(", ")}`);
|
|
327
324
|
const pathArg = paths.length === 1 ? paths[0] : paths;
|
|
328
325
|
dotenv.default.config({
|
|
329
326
|
path: pathArg,
|
|
@@ -332,7 +329,6 @@ function setupEnv(envPath) {
|
|
|
332
329
|
});
|
|
333
330
|
} else dotenv.default.config({ quiet: true });
|
|
334
331
|
}
|
|
335
|
-
|
|
336
332
|
//#endregion
|
|
337
333
|
//#region src/util/functions/loadFunction.ts
|
|
338
334
|
const functionCache = {};
|
|
@@ -341,7 +337,7 @@ const functionCache = {};
|
|
|
341
337
|
* @param options Options for loading the function
|
|
342
338
|
* @returns The loaded function
|
|
343
339
|
*/
|
|
344
|
-
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.
|
|
340
|
+
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.state.basePath, useCache = true }) {
|
|
345
341
|
const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
|
|
346
342
|
if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
|
|
347
343
|
const resolvedPath = basePath ? path.default.resolve(basePath, filePath) : filePath;
|
|
@@ -362,7 +358,7 @@ async function loadFunction({ filePath, functionName, defaultFunctionName = "fun
|
|
|
362
358
|
if (useCache) functionCache[cacheKey] = func;
|
|
363
359
|
return func;
|
|
364
360
|
} catch (err) {
|
|
365
|
-
require_logger.
|
|
361
|
+
require_logger.logger.error(`Failed to load function: ${err.message}`);
|
|
366
362
|
throw err;
|
|
367
363
|
}
|
|
368
364
|
}
|
|
@@ -381,7 +377,6 @@ function parseFileUrl(fileUrl) {
|
|
|
381
377
|
};
|
|
382
378
|
return { filePath: urlWithoutProtocol };
|
|
383
379
|
}
|
|
384
|
-
|
|
385
380
|
//#endregion
|
|
386
381
|
//#region src/util/templates.ts
|
|
387
382
|
/**
|
|
@@ -400,7 +395,7 @@ function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false)
|
|
|
400
395
|
});
|
|
401
396
|
const envGlobals = {
|
|
402
397
|
...require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", require_logger.getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
|
|
403
|
-
...require_logger.
|
|
398
|
+
...require_logger.state.config?.env
|
|
404
399
|
};
|
|
405
400
|
env.addGlobal("env", envGlobals);
|
|
406
401
|
env.addFilter("load", function(str) {
|
|
@@ -440,7 +435,6 @@ function extractVariablesFromTemplates(templates) {
|
|
|
440
435
|
for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
|
|
441
436
|
return Array.from(variableSet);
|
|
442
437
|
}
|
|
443
|
-
|
|
444
438
|
//#endregion
|
|
445
439
|
//#region src/util/render.ts
|
|
446
440
|
/**
|
|
@@ -470,7 +464,7 @@ function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
|
470
464
|
if (require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
|
|
471
465
|
if (typeof obj === "string") {
|
|
472
466
|
if (obj.length > 5e4) {
|
|
473
|
-
require_logger.
|
|
467
|
+
require_logger.logger.warn(`String too long (${obj.length} chars) for template matching. Skipping env var rendering.`);
|
|
474
468
|
return obj;
|
|
475
469
|
}
|
|
476
470
|
const nunjucks = getNunjucksEngine();
|
|
@@ -486,7 +480,7 @@ function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
|
486
480
|
if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
|
|
487
481
|
return nunjucks.renderString(match, { env: envGlobals });
|
|
488
482
|
} catch (error) {
|
|
489
|
-
require_logger.
|
|
483
|
+
require_logger.logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
|
|
490
484
|
return match;
|
|
491
485
|
}
|
|
492
486
|
return match;
|
|
@@ -511,7 +505,6 @@ function renderVarsInObject(obj, vars) {
|
|
|
511
505
|
} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
|
|
512
506
|
return obj;
|
|
513
507
|
}
|
|
514
|
-
|
|
515
508
|
//#endregion
|
|
516
509
|
//#region src/util/file.ts
|
|
517
510
|
/**
|
|
@@ -522,7 +515,7 @@ function getNunjucksEngineForFilePath() {
|
|
|
522
515
|
const env = nunjucks.default.configure({ autoescape: false });
|
|
523
516
|
env.addGlobal("env", {
|
|
524
517
|
...process.env,
|
|
525
|
-
...require_logger.
|
|
518
|
+
...require_logger.state.config?.env
|
|
526
519
|
});
|
|
527
520
|
return env;
|
|
528
521
|
}
|
|
@@ -541,24 +534,24 @@ function getNunjucksEngineForFilePath() {
|
|
|
541
534
|
* @throws {Error} If the specified file does not exist.
|
|
542
535
|
*/
|
|
543
536
|
function maybeLoadFromExternalFile(filePath, context) {
|
|
544
|
-
if (Array.isArray(filePath)) return filePath.map((path) => {
|
|
545
|
-
return maybeLoadFromExternalFile(path, context);
|
|
537
|
+
if (Array.isArray(filePath)) return filePath.map((path$2) => {
|
|
538
|
+
return maybeLoadFromExternalFile(path$2, context);
|
|
546
539
|
});
|
|
547
540
|
if (typeof filePath !== "string") return filePath;
|
|
548
541
|
if (!filePath.startsWith("file://")) return filePath;
|
|
549
542
|
const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
|
|
550
543
|
const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
|
|
551
544
|
if (context === "assertion" && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) {
|
|
552
|
-
require_logger.
|
|
545
|
+
require_logger.logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
|
|
553
546
|
return renderedFilePath;
|
|
554
547
|
}
|
|
555
548
|
if (context === "vars") {
|
|
556
|
-
require_logger.
|
|
549
|
+
require_logger.logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
|
|
557
550
|
return renderedFilePath;
|
|
558
551
|
}
|
|
559
552
|
if (functionName && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) return renderedFilePath;
|
|
560
553
|
const pathToUse = functionName && !(cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
|
|
561
|
-
const resolvedPath = path.resolve(require_logger.
|
|
554
|
+
const resolvedPath = path.resolve(require_logger.state.basePath || "", pathToUse);
|
|
562
555
|
if ((0, glob.hasMagic)(pathToUse)) {
|
|
563
556
|
const matchedFiles = (0, glob.globSync)(resolvedPath, { windowsPathsNoEscape: true });
|
|
564
557
|
if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
|
|
@@ -569,7 +562,7 @@ function maybeLoadFromExternalFile(filePath, context) {
|
|
|
569
562
|
contents = fs.readFileSync(matchedFile, "utf8");
|
|
570
563
|
} catch (error) {
|
|
571
564
|
if (error.code === "ENOENT") {
|
|
572
|
-
require_logger.
|
|
565
|
+
require_logger.logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
|
|
573
566
|
continue;
|
|
574
567
|
}
|
|
575
568
|
throw error;
|
|
@@ -779,37 +772,37 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
779
772
|
const { filePath, functionName } = parseFileUrl(rendered);
|
|
780
773
|
if (functionName && (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath))) {
|
|
781
774
|
const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
782
|
-
require_logger.
|
|
775
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
|
|
783
776
|
try {
|
|
784
777
|
let toolDefinitions;
|
|
785
778
|
if (filePath.endsWith(".py")) {
|
|
786
|
-
const absPath = require_esm.safeResolve(require_logger.
|
|
787
|
-
require_logger.
|
|
779
|
+
const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
|
|
780
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
|
|
788
781
|
toolDefinitions = await require_pythonUtils.runPython(absPath, functionName, []);
|
|
789
782
|
} else {
|
|
790
|
-
const absPath = require_esm.safeResolve(require_logger.
|
|
791
|
-
require_logger.
|
|
783
|
+
const absPath = require_esm.safeResolve(require_logger.state.basePath || process.cwd(), filePath);
|
|
784
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
|
|
792
785
|
const module = await require_esm.importModule(absPath);
|
|
793
786
|
const fn = module[functionName] || module.default?.[functionName];
|
|
794
787
|
if (typeof fn !== "function") {
|
|
795
788
|
const availableExports = Object.keys(module).filter((k) => k !== "default");
|
|
796
|
-
const basePath = require_logger.
|
|
789
|
+
const basePath = require_logger.state.basePath || process.cwd();
|
|
797
790
|
throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
|
|
798
791
|
}
|
|
799
792
|
toolDefinitions = await Promise.resolve(fn());
|
|
800
793
|
}
|
|
801
794
|
if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
|
|
802
|
-
require_logger.
|
|
795
|
+
require_logger.logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
|
|
803
796
|
return toolDefinitions;
|
|
804
797
|
} catch (err) {
|
|
805
798
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
806
|
-
const basePath = require_logger.
|
|
799
|
+
const basePath = require_logger.state.basePath || process.cwd();
|
|
807
800
|
throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
|
|
808
801
|
}
|
|
809
802
|
}
|
|
810
803
|
if (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath)) {
|
|
811
804
|
const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
812
|
-
const basePath = require_logger.
|
|
805
|
+
const basePath = require_logger.state.basePath || process.cwd();
|
|
813
806
|
throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? ` def get_tools():\n return [{"type": "function", "function": {...}}]` : ` module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
|
|
814
807
|
}
|
|
815
808
|
}
|
|
@@ -827,7 +820,6 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
827
820
|
}
|
|
828
821
|
return loaded;
|
|
829
822
|
}
|
|
830
|
-
|
|
831
823
|
//#endregion
|
|
832
824
|
//#region src/googleSheets.ts
|
|
833
825
|
async function checkGoogleSheetAccess(url) {
|
|
@@ -842,7 +834,7 @@ async function checkGoogleSheetAccess(url) {
|
|
|
842
834
|
status: response.status
|
|
843
835
|
};
|
|
844
836
|
} catch (error) {
|
|
845
|
-
require_logger.
|
|
837
|
+
require_logger.logger.error(`Error checking sheet access: ${error}`);
|
|
846
838
|
return { public: false };
|
|
847
839
|
}
|
|
848
840
|
}
|
|
@@ -894,7 +886,7 @@ async function fetchCsvFromGoogleSheetAuthenticated(url) {
|
|
|
894
886
|
}
|
|
895
887
|
async function fetchCsvFromGoogleSheet(url) {
|
|
896
888
|
const { public: isPublic } = await checkGoogleSheetAccess(url);
|
|
897
|
-
require_logger.
|
|
889
|
+
require_logger.logger.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
|
|
898
890
|
if (isPublic) return fetchCsvFromGoogleSheetUnauthenticated(url);
|
|
899
891
|
return fetchCsvFromGoogleSheetAuthenticated(url);
|
|
900
892
|
}
|
|
@@ -937,7 +929,7 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
937
929
|
});
|
|
938
930
|
range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
|
|
939
931
|
}
|
|
940
|
-
require_logger.
|
|
932
|
+
require_logger.logger.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
|
|
941
933
|
await sheets.spreadsheets.values.update({
|
|
942
934
|
spreadsheetId,
|
|
943
935
|
range,
|
|
@@ -946,7 +938,6 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
946
938
|
requestBody: { values }
|
|
947
939
|
});
|
|
948
940
|
}
|
|
949
|
-
|
|
950
941
|
//#endregion
|
|
951
942
|
//#region src/server/utils/evalTableUtils.ts
|
|
952
943
|
/**
|
|
@@ -1145,7 +1136,6 @@ async function streamEvalCsv(eval_, options) {
|
|
|
1145
1136
|
isRedteam
|
|
1146
1137
|
})]));
|
|
1147
1138
|
}
|
|
1148
|
-
|
|
1149
1139
|
//#endregion
|
|
1150
1140
|
//#region src/util/output.ts
|
|
1151
1141
|
const outputToSimpleString = (output) => {
|
|
@@ -1210,7 +1200,7 @@ async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
|
|
|
1210
1200
|
const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
|
|
1211
1201
|
if (isStringLen || isHeapOOM) {
|
|
1212
1202
|
const resultCount = await evalRecord.getResultsCount();
|
|
1213
|
-
require_logger.
|
|
1203
|
+
require_logger.logger.error(`Dataset too large for JSON export (${resultCount} results).`);
|
|
1214
1204
|
throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
|
|
1215
1205
|
} else throw error;
|
|
1216
1206
|
}
|
|
@@ -1229,7 +1219,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1229
1219
|
});
|
|
1230
1220
|
return csvRow;
|
|
1231
1221
|
});
|
|
1232
|
-
require_logger.
|
|
1222
|
+
require_logger.logger.info(`Writing ${rows.length} rows to Google Sheets...`);
|
|
1233
1223
|
await writeCsvToGoogleSheet(rows, outputPath);
|
|
1234
1224
|
return;
|
|
1235
1225
|
}
|
|
@@ -1311,217 +1301,216 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1311
1301
|
async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
|
|
1312
1302
|
await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
|
|
1313
1303
|
}
|
|
1314
|
-
|
|
1315
1304
|
//#endregion
|
|
1316
1305
|
//#region src/util/runtime.ts
|
|
1317
1306
|
function printBorder() {
|
|
1318
1307
|
const border = "=".repeat(require_fetch.TERMINAL_MAX_WIDTH);
|
|
1319
|
-
require_logger.
|
|
1308
|
+
require_logger.logger.info(border);
|
|
1320
1309
|
}
|
|
1321
|
-
|
|
1322
1310
|
//#endregion
|
|
1323
|
-
Object.defineProperty(exports,
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1311
|
+
Object.defineProperty(exports, "checkProviderApiKeys", {
|
|
1312
|
+
enumerable: true,
|
|
1313
|
+
get: function() {
|
|
1314
|
+
return checkProviderApiKeys;
|
|
1315
|
+
}
|
|
1328
1316
|
});
|
|
1329
|
-
Object.defineProperty(exports,
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1317
|
+
Object.defineProperty(exports, "deduplicateTestCases", {
|
|
1318
|
+
enumerable: true,
|
|
1319
|
+
get: function() {
|
|
1320
|
+
return deduplicateTestCases;
|
|
1321
|
+
}
|
|
1334
1322
|
});
|
|
1335
|
-
Object.defineProperty(exports,
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1323
|
+
Object.defineProperty(exports, "doesProviderRefMatch", {
|
|
1324
|
+
enumerable: true,
|
|
1325
|
+
get: function() {
|
|
1326
|
+
return doesProviderRefMatch;
|
|
1327
|
+
}
|
|
1340
1328
|
});
|
|
1341
|
-
Object.defineProperty(exports,
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1329
|
+
Object.defineProperty(exports, "extractRuntimeVars", {
|
|
1330
|
+
enumerable: true,
|
|
1331
|
+
get: function() {
|
|
1332
|
+
return extractRuntimeVars;
|
|
1333
|
+
}
|
|
1346
1334
|
});
|
|
1347
|
-
Object.defineProperty(exports,
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1335
|
+
Object.defineProperty(exports, "extractVariablesFromTemplate", {
|
|
1336
|
+
enumerable: true,
|
|
1337
|
+
get: function() {
|
|
1338
|
+
return extractVariablesFromTemplate;
|
|
1339
|
+
}
|
|
1352
1340
|
});
|
|
1353
|
-
Object.defineProperty(exports,
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1341
|
+
Object.defineProperty(exports, "extractVariablesFromTemplates", {
|
|
1342
|
+
enumerable: true,
|
|
1343
|
+
get: function() {
|
|
1344
|
+
return extractVariablesFromTemplates;
|
|
1345
|
+
}
|
|
1358
1346
|
});
|
|
1359
|
-
Object.defineProperty(exports,
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1347
|
+
Object.defineProperty(exports, "fetchCsvFromGoogleSheet", {
|
|
1348
|
+
enumerable: true,
|
|
1349
|
+
get: function() {
|
|
1350
|
+
return fetchCsvFromGoogleSheet;
|
|
1351
|
+
}
|
|
1364
1352
|
});
|
|
1365
|
-
Object.defineProperty(exports,
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1353
|
+
Object.defineProperty(exports, "filterRuntimeVars", {
|
|
1354
|
+
enumerable: true,
|
|
1355
|
+
get: function() {
|
|
1356
|
+
return filterRuntimeVars;
|
|
1357
|
+
}
|
|
1370
1358
|
});
|
|
1371
|
-
Object.defineProperty(exports,
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1359
|
+
Object.defineProperty(exports, "getNunjucksEngine", {
|
|
1360
|
+
enumerable: true,
|
|
1361
|
+
get: function() {
|
|
1362
|
+
return getNunjucksEngine;
|
|
1363
|
+
}
|
|
1376
1364
|
});
|
|
1377
|
-
Object.defineProperty(exports,
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1365
|
+
Object.defineProperty(exports, "getNunjucksEngineForFilePath", {
|
|
1366
|
+
enumerable: true,
|
|
1367
|
+
get: function() {
|
|
1368
|
+
return getNunjucksEngineForFilePath;
|
|
1369
|
+
}
|
|
1382
1370
|
});
|
|
1383
|
-
Object.defineProperty(exports,
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1371
|
+
Object.defineProperty(exports, "getProviderDescription", {
|
|
1372
|
+
enumerable: true,
|
|
1373
|
+
get: function() {
|
|
1374
|
+
return getProviderDescription;
|
|
1375
|
+
}
|
|
1388
1376
|
});
|
|
1389
|
-
Object.defineProperty(exports,
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1377
|
+
Object.defineProperty(exports, "getResolvedRelativePath", {
|
|
1378
|
+
enumerable: true,
|
|
1379
|
+
get: function() {
|
|
1380
|
+
return getResolvedRelativePath;
|
|
1381
|
+
}
|
|
1394
1382
|
});
|
|
1395
|
-
Object.defineProperty(exports,
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1383
|
+
Object.defineProperty(exports, "getTestCaseDeduplicationKey", {
|
|
1384
|
+
enumerable: true,
|
|
1385
|
+
get: function() {
|
|
1386
|
+
return getTestCaseDeduplicationKey;
|
|
1387
|
+
}
|
|
1400
1388
|
});
|
|
1401
|
-
Object.defineProperty(exports,
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1389
|
+
Object.defineProperty(exports, "isAnthropicProvider", {
|
|
1390
|
+
enumerable: true,
|
|
1391
|
+
get: function() {
|
|
1392
|
+
return isAnthropicProvider;
|
|
1393
|
+
}
|
|
1406
1394
|
});
|
|
1407
|
-
Object.defineProperty(exports,
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1395
|
+
Object.defineProperty(exports, "isGoogleProvider", {
|
|
1396
|
+
enumerable: true,
|
|
1397
|
+
get: function() {
|
|
1398
|
+
return isGoogleProvider;
|
|
1399
|
+
}
|
|
1412
1400
|
});
|
|
1413
|
-
Object.defineProperty(exports,
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1401
|
+
Object.defineProperty(exports, "isOpenAiProvider", {
|
|
1402
|
+
enumerable: true,
|
|
1403
|
+
get: function() {
|
|
1404
|
+
return isOpenAiProvider;
|
|
1405
|
+
}
|
|
1418
1406
|
});
|
|
1419
|
-
Object.defineProperty(exports,
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1407
|
+
Object.defineProperty(exports, "isProviderAllowed", {
|
|
1408
|
+
enumerable: true,
|
|
1409
|
+
get: function() {
|
|
1410
|
+
return isProviderAllowed;
|
|
1411
|
+
}
|
|
1424
1412
|
});
|
|
1425
|
-
Object.defineProperty(exports,
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1413
|
+
Object.defineProperty(exports, "loadFunction", {
|
|
1414
|
+
enumerable: true,
|
|
1415
|
+
get: function() {
|
|
1416
|
+
return loadFunction;
|
|
1417
|
+
}
|
|
1430
1418
|
});
|
|
1431
|
-
Object.defineProperty(exports,
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1419
|
+
Object.defineProperty(exports, "maybeLoadConfigFromExternalFile", {
|
|
1420
|
+
enumerable: true,
|
|
1421
|
+
get: function() {
|
|
1422
|
+
return maybeLoadConfigFromExternalFile;
|
|
1423
|
+
}
|
|
1436
1424
|
});
|
|
1437
|
-
Object.defineProperty(exports,
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1425
|
+
Object.defineProperty(exports, "maybeLoadFromExternalFile", {
|
|
1426
|
+
enumerable: true,
|
|
1427
|
+
get: function() {
|
|
1428
|
+
return maybeLoadFromExternalFile;
|
|
1429
|
+
}
|
|
1442
1430
|
});
|
|
1443
|
-
Object.defineProperty(exports,
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1431
|
+
Object.defineProperty(exports, "maybeLoadFromExternalFileWithVars", {
|
|
1432
|
+
enumerable: true,
|
|
1433
|
+
get: function() {
|
|
1434
|
+
return maybeLoadFromExternalFileWithVars;
|
|
1435
|
+
}
|
|
1448
1436
|
});
|
|
1449
|
-
Object.defineProperty(exports,
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1437
|
+
Object.defineProperty(exports, "maybeLoadResponseFormatFromExternalFile", {
|
|
1438
|
+
enumerable: true,
|
|
1439
|
+
get: function() {
|
|
1440
|
+
return maybeLoadResponseFormatFromExternalFile;
|
|
1441
|
+
}
|
|
1454
1442
|
});
|
|
1455
|
-
Object.defineProperty(exports,
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1443
|
+
Object.defineProperty(exports, "maybeLoadToolsFromExternalFile", {
|
|
1444
|
+
enumerable: true,
|
|
1445
|
+
get: function() {
|
|
1446
|
+
return maybeLoadToolsFromExternalFile;
|
|
1447
|
+
}
|
|
1460
1448
|
});
|
|
1461
|
-
Object.defineProperty(exports,
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1449
|
+
Object.defineProperty(exports, "parseFileUrl", {
|
|
1450
|
+
enumerable: true,
|
|
1451
|
+
get: function() {
|
|
1452
|
+
return parseFileUrl;
|
|
1453
|
+
}
|
|
1466
1454
|
});
|
|
1467
|
-
Object.defineProperty(exports,
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1455
|
+
Object.defineProperty(exports, "parsePathOrGlob", {
|
|
1456
|
+
enumerable: true,
|
|
1457
|
+
get: function() {
|
|
1458
|
+
return parsePathOrGlob;
|
|
1459
|
+
}
|
|
1472
1460
|
});
|
|
1473
|
-
Object.defineProperty(exports,
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1461
|
+
Object.defineProperty(exports, "printBorder", {
|
|
1462
|
+
enumerable: true,
|
|
1463
|
+
get: function() {
|
|
1464
|
+
return printBorder;
|
|
1465
|
+
}
|
|
1478
1466
|
});
|
|
1479
|
-
Object.defineProperty(exports,
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1467
|
+
Object.defineProperty(exports, "readFilters", {
|
|
1468
|
+
enumerable: true,
|
|
1469
|
+
get: function() {
|
|
1470
|
+
return readFilters;
|
|
1471
|
+
}
|
|
1484
1472
|
});
|
|
1485
|
-
Object.defineProperty(exports,
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1473
|
+
Object.defineProperty(exports, "readOutput", {
|
|
1474
|
+
enumerable: true,
|
|
1475
|
+
get: function() {
|
|
1476
|
+
return readOutput;
|
|
1477
|
+
}
|
|
1490
1478
|
});
|
|
1491
|
-
Object.defineProperty(exports,
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1479
|
+
Object.defineProperty(exports, "renderEnvOnlyInObject", {
|
|
1480
|
+
enumerable: true,
|
|
1481
|
+
get: function() {
|
|
1482
|
+
return renderEnvOnlyInObject;
|
|
1483
|
+
}
|
|
1496
1484
|
});
|
|
1497
|
-
Object.defineProperty(exports,
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1485
|
+
Object.defineProperty(exports, "renderVarsInObject", {
|
|
1486
|
+
enumerable: true,
|
|
1487
|
+
get: function() {
|
|
1488
|
+
return renderVarsInObject;
|
|
1489
|
+
}
|
|
1502
1490
|
});
|
|
1503
|
-
Object.defineProperty(exports,
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1491
|
+
Object.defineProperty(exports, "resultIsForTestCase", {
|
|
1492
|
+
enumerable: true,
|
|
1493
|
+
get: function() {
|
|
1494
|
+
return resultIsForTestCase;
|
|
1495
|
+
}
|
|
1508
1496
|
});
|
|
1509
|
-
Object.defineProperty(exports,
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1497
|
+
Object.defineProperty(exports, "setupEnv", {
|
|
1498
|
+
enumerable: true,
|
|
1499
|
+
get: function() {
|
|
1500
|
+
return setupEnv;
|
|
1501
|
+
}
|
|
1514
1502
|
});
|
|
1515
|
-
Object.defineProperty(exports,
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1503
|
+
Object.defineProperty(exports, "writeMultipleOutputs", {
|
|
1504
|
+
enumerable: true,
|
|
1505
|
+
get: function() {
|
|
1506
|
+
return writeMultipleOutputs;
|
|
1507
|
+
}
|
|
1520
1508
|
});
|
|
1521
|
-
Object.defineProperty(exports,
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1509
|
+
Object.defineProperty(exports, "writeOutput", {
|
|
1510
|
+
enumerable: true,
|
|
1511
|
+
get: function() {
|
|
1512
|
+
return writeOutput;
|
|
1513
|
+
}
|
|
1526
1514
|
});
|
|
1527
|
-
|
|
1515
|
+
|
|
1516
|
+
//# sourceMappingURL=util-Yz-1aEhW.cjs.map
|