promptfoo 0.120.26 → 0.121.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/drizzle/0023_wooden_mandrill.sql +2 -0
- package/dist/drizzle/meta/0023_snapshot.json +1496 -0
- package/dist/drizzle/meta/_journal.json +7 -0
- package/dist/src/{ListApp-D3DG0F8h.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-BgNJDBE6.js +206 -0
- package/dist/src/{accounts-BzEY8H3v.cjs → accounts-Bx-x3bmW.cjs} +99 -80
- package/dist/src/{accounts-DHHiXsy6.js → accounts-CMqkzrVf.js} +61 -36
- package/dist/src/{accounts-R3gfCR_g.js → accounts-xrUGFA6n.js} +60 -35
- package/dist/src/{agentic-utils-D6_gzOUF.js → agentic-utils-BKIN5PKu.js} +9 -10
- package/dist/src/{agents-CwM7re15.cjs → agents-B0f4HICh.cjs} +37 -40
- package/dist/src/{agents-Cnph5GLD.js → agents-C-dDThPK.js} +37 -37
- package/dist/src/{agents-C7BiinFI.cjs → agents-CErsqg5U.cjs} +19 -27
- package/dist/src/{agents-v4cW_ZgC.js → agents-CVIn-Utx.js} +19 -22
- package/dist/src/{agents-GiUianme.js → agents-CXknwsFX.js} +37 -40
- package/dist/src/{agents-DETIQHqF.js → agents-DeH4Gu94.js} +21 -28
- package/dist/src/{agents-DYIT-hQy.js → agents-Dy2YpZpa.js} +38 -41
- package/dist/src/{agents-Cao4i7AX.js → agents-aF4-T121.js} +19 -30
- package/dist/src/{aimlapi-DMF6v_vb.js → aimlapi-BAGZDo5G.js} +16 -18
- package/dist/src/{aimlapi-CMJpKK-B.js → aimlapi-BNfTBexL.js} +15 -17
- package/dist/src/{aimlapi-DtSf1ykJ.js → aimlapi-DHRKlBEA.js} +15 -4
- package/dist/src/{aimlapi-DoGLcQW_.cjs → aimlapi-tg0Gkcvr.cjs} +15 -16
- package/dist/src/app/assets/index-BFCZg7hQ.js +439 -0
- package/dist/src/app/assets/index-NCn4eVBv.css +1 -0
- package/dist/src/app/assets/{vendor-charts-CYyo8R8v.js → vendor-charts-CCl15Imd.js} +1 -1
- package/dist/src/app/assets/{vendor-markdown-DSmzq4Jh.js → vendor-markdown-0tekx3KX.js} +1 -1
- package/dist/src/app/index.html +4 -34
- package/dist/src/{audio-DUH4q0Xq.js → audio-BRODU0UK.js} +7 -9
- package/dist/src/{audio-BWjyvHn9.cjs → audio-BWeaWovU.cjs} +6 -7
- package/dist/src/{audio-U580w8jM.js → audio-CHQ4r-RV.js} +6 -5
- package/dist/src/{audio-BrJBFN2b.js → audio-tf_NBjlC.js} +6 -8
- package/dist/src/base-B0tcrnq_.js +193 -0
- package/dist/src/base-B4QJRyFS.js +194 -0
- package/dist/src/base-DBtwl2FR.cjs +222 -0
- package/dist/src/base-fEDN28WM.js +193 -0
- package/dist/src/{blobs-kt8v3UyH.js → blobs-BAU-dXan.js} +9 -12
- package/dist/src/{blobs-C9J2mVgo.js → blobs-Bpg5rH6i.js} +9 -12
- package/dist/src/{blobs-673H0jCl.cjs → blobs-DvS-O6be.cjs} +34 -37
- package/dist/src/blobs-qTYm-1PY.js +236 -0
- package/dist/src/{cache-BLLayYEN.js → cache-8XhNqPKW.js} +64 -67
- package/dist/src/cache-Bbn1Nyrd.cjs +5 -0
- package/dist/src/cache-BwsMSda7.js +6 -0
- package/dist/src/{cache-mIszOnuz.js → cache-CG0SlR1d.js} +64 -66
- package/dist/src/{cache-7xULbvt3.cjs → cache-COish3-W.cjs} +114 -117
- package/dist/src/cache-D3eqDYGU.js +739 -0
- package/dist/src/{chat-Fl6TZJRS.cjs → chat-2K608PeQ.cjs} +20 -21
- package/dist/src/chat-BKm79wib.js +764 -0
- package/dist/src/{chat-XPN9YHhr.js → chat-CM_kyI8B.js} +20 -9
- package/dist/src/{chat-DIywASPG.js → chat-CRWNNq73.js} +49 -49
- package/dist/src/{chat-C8Ei4f87.js → chat-CznLWr_D.js} +49 -49
- package/dist/src/{chat-CgyGj2hC.js → chat-DHMH-N64.js} +20 -22
- package/dist/src/{chat-C4zqjObh.cjs → chat-DaqekjFr.cjs} +69 -69
- package/dist/src/{chat-Cpz3O-Xl.js → chat-DxysjBvt.js} +21 -23
- package/dist/src/{chatkit-Dpxrq4eD.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DIrJX8xk.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-DEls11hE.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-e25Ziu17.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-6-xTaLwM.js → claude-agent-sdk-BLTu0WBO.js} +45 -31
- package/dist/src/{claude-agent-sdk-BzxF6NIJ.cjs → claude-agent-sdk-CJH22shf.cjs} +44 -29
- package/dist/src/{claude-agent-sdk-CmkTnKGH.js → claude-agent-sdk-D6_k9FKA.js} +45 -33
- package/dist/src/{claude-agent-sdk-rXCBLK_o.js → claude-agent-sdk-Dy5lT-Tx.js} +46 -21
- package/dist/src/{cloud-BMbRVJFw.js → cloud-Bc9526yV.js} +32 -12
- package/dist/src/cloud-DmE0EwsY.js +4 -0
- package/dist/src/{cloudflare-ai-CUg4BTcj.js → cloudflare-ai-C9r2sRhw.js} +16 -18
- package/dist/src/{cloudflare-ai-Z9X219gp.js → cloudflare-ai-CWWJCRim.js} +16 -4
- package/dist/src/{cloudflare-ai-BAQ0u_dg.cjs → cloudflare-ai-ClWSdor4.cjs} +16 -17
- package/dist/src/{cloudflare-ai-CobxMTR4.js → cloudflare-ai-ICsOuD-z.js} +17 -19
- package/dist/src/{cloudflare-gateway-C0sgfr_z.cjs → cloudflare-gateway-C2_-KG5o.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-_itGuXry.js → cloudflare-gateway-D6O7AlYb.js} +23 -23
- package/dist/src/{cloudflare-gateway-D2_yi-Fh.js → cloudflare-gateway-D6xFc5pa.js} +21 -25
- package/dist/src/{cloudflare-gateway-Djf3F3_H.js → cloudflare-gateway-pXGHxJ47.js} +26 -14
- package/dist/src/{codex-sdk-ibXwdglL.js → codex-sdk-C6UMlxwV.js} +49 -32
- package/dist/src/{codex-sdk-BASDNkIl.js → codex-sdk-DUwKWezN.js} +49 -30
- package/dist/src/{codex-sdk-dSnGdgIp.js → codex-sdk-GGAw0qbD.js} +49 -32
- package/dist/src/{codex-sdk-wTEpMM_X.cjs → codex-sdk-fAO0c3yA.cjs} +49 -32
- package/dist/src/{cometapi-B01btbfb.js → cometapi-BasUi7-_.js} +17 -19
- package/dist/src/{cometapi-DHUAH6nK.js → cometapi-Bbjp5V4x.js} +16 -4
- package/dist/src/{cometapi-ChAaRjg5.js → cometapi-DkXrKi5z.js} +21 -24
- package/dist/src/{cometapi-JbvOJSCO.cjs → cometapi-vY6aDZgo.cjs} +21 -22
- package/dist/src/{completion-D9_MDlnd.js → completion-6Mx_iXxK.js} +11 -13
- package/dist/src/{completion-BBJ6zmG3.js → completion-C5rtR_9P.js} +11 -13
- package/dist/src/{completion-DDyL3Cb2.cjs → completion-CDOouNzq.cjs} +21 -23
- package/dist/src/completion-C_P3ypkJ.js +120 -0
- package/dist/src/createHash-CTQmL3G2.js +15 -0
- package/dist/src/createHash-CfZSc0b4.cjs +27 -0
- package/dist/src/createHash-Da8fMwqB.js +16 -0
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-JAAubMw3.js → docker-5KcG-_86.js} +18 -20
- package/dist/src/{docker-Ckw-j7Rr.cjs → docker-BwsKwxFs.cjs} +18 -19
- package/dist/src/{docker-vnOg96gi.js → docker-CZnqU1XV.js} +18 -7
- package/dist/src/{docker-BuButc4D.js → docker-DzxyDPIj.js} +19 -21
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-CYhseqj4.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-rDtG_2rg.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-BQkx5roy.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-17JizQIv.js +15 -0
- package/dist/src/{eval-CYrbG57o.js → eval-DmFyWU7i.js} +49 -55
- package/dist/src/{evalResult-COsVttMA.js → evalResult-CDQiuUuf.js} +16 -12
- package/dist/src/{evalResult-6JaUIStC.js → evalResult-CTG2AHOS.js} +10 -11
- package/dist/src/evalResult-Cqj8pldJ.js +12 -0
- package/dist/src/{evalResult-DlRfu_Rq.cjs → evalResult-Dap2CekP.cjs} +20 -21
- package/dist/src/evalResult-DvcJAWJU.cjs +10 -0
- package/dist/src/evalResult-Hftn-S_i.js +10 -0
- package/dist/src/evaluator-B2CFNt-P.js +36 -0
- package/dist/src/{evaluator-3EJCMTs0.js → evaluator-DPFRbFIL.js} +210 -232
- package/dist/src/{extractor-LSYjrhK0.js → extractor-CFG6bcWJ.js} +23 -38
- package/dist/src/{extractor-DbhlYEeo.cjs → extractor-DX36oYEv.cjs} +37 -64
- package/dist/src/{extractor-Hs7la_19.js → extractor-M67RUtg6.js} +23 -38
- package/dist/src/extractor-YMU_Gvt8.js +374 -0
- package/dist/src/fetch-4M3YRaqL.js +780 -0
- package/dist/src/{fetch-18MuNu9i.js → fetch-60Gzydls.js} +60 -46
- package/dist/src/{fetch-SRsE6Ctl.js → fetch-BMv0O527.js} +41 -35
- package/dist/src/{fetch-ZMn_oemb.cjs → fetch-BxUk8odA.cjs} +268 -279
- package/dist/src/fetch-KV5kNASw.js +5 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-Doo0xq-z.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-CQlpZkrp.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/graders-Bu0H9nXi.js +32 -0
- package/dist/src/{graders-BaMCwIKp.js → graders-CHO8EPM4.js} +385 -417
- package/dist/src/graders-Cfhkvx-e.js +34 -0
- package/dist/src/{graders-QsALpIdy.js → graders-CpdqD9PI.js} +385 -417
- package/dist/src/graders-DClJVpGP.cjs +32 -0
- package/dist/src/{graders-DzUUnUjC.cjs → graders-DOXycdlG.cjs} +721 -753
- package/dist/src/graders-DcnJsrMO.js +32 -0
- package/dist/src/graders-R9rYUM0d.js +13466 -0
- package/dist/src/{image-BiEVdpdP.js → image-BmEZqVmk.js} +57 -18
- package/dist/src/{image-mhAGP07h.js → image-CBBVXWuT.js} +57 -18
- package/dist/src/{image-D10zEe1f.cjs → image-CDLQOcqT.cjs} +6 -7
- package/dist/src/{image-COCWy5dX.js → image-DJEvKveK.js} +6 -5
- package/dist/src/{image-C3BjJUAU.cjs → image-DTedmQPg.cjs} +77 -32
- package/dist/src/{image-DB4sHxdJ.js → image-gvmivTEe.js} +7 -9
- package/dist/src/image-pAX56tPG.js +257 -0
- package/dist/src/{image-BXt_7u0v.js → image-tL5hIOFh.js} +6 -8
- package/dist/src/index.cjs +696 -693
- package/dist/src/index.d.cts +113 -10
- package/dist/src/index.d.ts +113 -6
- package/dist/src/index.js +657 -658
- package/dist/src/{interactiveCheck-DU-MAhp5.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-DotRBzUE.js → knowledgeBase-CLJybhnF.js} +19 -34
- package/dist/src/{knowledgeBase-XJQ0Qyez.js → knowledgeBase-CoU-UQBg.js} +17 -41
- package/dist/src/{knowledgeBase-CMvMlLZR.js → knowledgeBase-DjWPVqSb.js} +17 -43
- package/dist/src/{knowledgeBase-Bnb00xKs.cjs → knowledgeBase-wkxuRFhA.cjs} +17 -40
- package/dist/src/{litellm-CHrRmPAe.js → litellm-B9Hysuri.js} +16 -18
- package/dist/src/{litellm-CrLJrPIm.js → litellm-CTfa0hqi.js} +15 -17
- package/dist/src/{litellm-BrnZhMcL.cjs → litellm-NYpQ8RQu.cjs} +15 -16
- package/dist/src/{litellm-BECdjOTx.js → litellm-ePxtr9F1.js} +15 -4
- package/dist/src/{logger-w8Ozp0Td.js → logger-CT3IKMKA.js} +24 -41
- package/dist/src/{logger-BdZ-IqBc.cjs → logger-Cp1GPUjj.cjs} +166 -192
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-BotXmWKW.js → logger-KkObSCzq.js} +27 -43
- package/dist/src/{luma-ray-C0RkI3lt.cjs → luma-ray-B0GGNRc1.cjs} +20 -21
- package/dist/src/{luma-ray-C-w6EsJm.js → luma-ray-BE2mOt6N.js} +20 -13
- package/dist/src/{luma-ray-BOeX-h0M.js → luma-ray-BW9IRGIc.js} +22 -21
- package/dist/src/{luma-ray-DgKLS0BF.js → luma-ray-Cm1KZBhs.js} +20 -23
- package/dist/src/main.js +1985 -2055
- package/dist/src/{messages-DXV3Qh8_.cjs → messages-1JrJs91T.cjs} +35 -34
- package/dist/src/{messages-D61tPFQo.js → messages-1x9atZmP.js} +25 -24
- package/dist/src/{messages-CDZYGNlS.js → messages-BLbWdsyt.js} +25 -24
- package/dist/src/messages-D8EA0oDc.js +240 -0
- package/dist/src/{meteor-P2rUE-Uz.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-SLNTgmXm.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-odmwVbyG.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-Dj8cTkU_.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/modelslab-C1OLRmVX.js +166 -0
- package/dist/src/modelslab-CqXBy3U8.js +168 -0
- package/dist/src/modelslab-DcOSFwKh.cjs +166 -0
- package/dist/src/modelslab-X5-4LroM.js +166 -0
- package/dist/src/{nova-reel-C2LFfVTf.js → nova-reel-BgS1ZWuK.js} +20 -13
- package/dist/src/{nova-reel-DtCjbD5O.js → nova-reel-D2ZkOSyr.js} +22 -21
- package/dist/src/{nova-reel-D9FXq3Mt.cjs → nova-reel-D9xfaMBs.cjs} +20 -21
- package/dist/src/{nova-reel-Bk5npr2q.js → nova-reel-DihqLeol.js} +20 -23
- package/dist/src/{nova-sonic-BoRSY_U6.cjs → nova-sonic-DVu3mMIy.cjs} +30 -31
- package/dist/src/{nova-sonic-D_qERM-K.js → nova-sonic-DezhVUYT.js} +30 -26
- package/dist/src/{nova-sonic-CgaWLDM1.js → nova-sonic-P-CdUMlV.js} +30 -31
- package/dist/src/{nova-sonic-BXRfQyF-.js → nova-sonic-Q3BOJeig.js} +31 -32
- package/dist/src/{openai-Bigwjgo1.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-Dz3surb_.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CT5fwbve.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-dHLcXUWZ.js → openclaw-BiSZPL7J.js} +20 -14
- package/dist/src/{openclaw-CpPrXwf6.js → openclaw-Bv1DINsX.js} +20 -27
- package/dist/src/{openclaw-B6XY2kUf.js → openclaw-D1D_ej1z.js} +21 -28
- package/dist/src/{openclaw-DDSfq5fp.cjs → openclaw-DAfWQn-o.cjs} +33 -39
- package/dist/src/opencode-sdk-C7m-wRfI.js +560 -0
- package/dist/src/opencode-sdk-CfaLN8PY.cjs +564 -0
- package/dist/src/opencode-sdk-D95s6SnR.js +562 -0
- package/dist/src/opencode-sdk-DxUPkLT7.js +560 -0
- package/dist/src/{otlpReceiver-DmRb0NBj.js → otlpReceiver--AIRW_S4.js} +53 -51
- package/dist/src/{otlpReceiver-Dg817agV.js → otlpReceiver-Bn5wGB1v.js} +53 -55
- package/dist/src/{otlpReceiver-B6Xo4KZM.cjs → otlpReceiver-Diec4cln.cjs} +53 -55
- package/dist/src/{otlpReceiver-BO0rbDzh.js → otlpReceiver-g3ByGaXs.js} +53 -55
- package/dist/src/{providerRegistry-Xf0qdqGQ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-wCWd7sKQ.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-lc7a7utN.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B3HvufyI.js +33246 -0
- package/dist/src/{providers-BiNq_Iyc.js → providers-BKRJTjBz.js} +1743 -1795
- package/dist/src/providers-C1rOSHiR.js +32 -0
- package/dist/src/{providers-BlEhY5mi.js → providers-CFLy1_ji.js} +1750 -1802
- package/dist/src/{providers-BNKVY53V.cjs → providers-CFu-TZl-.cjs} +2111 -2163
- package/dist/src/providers-CxmDwEFf.cjs +31 -0
- package/dist/src/providers-Dodakqr0.js +30 -0
- package/dist/src/providers-GIQ2TcsA.js +30 -0
- package/dist/src/{pythonUtils-r1uBuA0n.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DZ6EbdY4.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-vMlk9Qp5.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/quiverai-C2jVwbH1.js +213 -0
- package/dist/src/quiverai-CI6gYJVI.js +213 -0
- package/dist/src/quiverai-CLkWkyZc.cjs +213 -0
- package/dist/src/quiverai-MHSxbmmZ.js +215 -0
- package/dist/src/{render-CAZvKKkB.js → render-Drod8m7K.js} +4 -5
- package/dist/src/{responses-DLLjADw5.js → responses-BKqJmhhc.js} +34 -27
- package/dist/src/{responses-TsdODUpm.js → responses-CGw0DCzh.js} +34 -27
- package/dist/src/responses-jxdehPkC.js +660 -0
- package/dist/src/{responses-zOtKtnY_.cjs → responses-tD4Bd4dc.cjs} +49 -42
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-Cs35SDYa.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-BtjlqyXt.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-DCVaJ3mc.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-Du4LIR97.js → sagemaker-BK4Zb993.js} +75 -70
- package/dist/src/{sagemaker-CLdUAv5z.js → sagemaker-BfiWTmvn.js} +77 -77
- package/dist/src/{sagemaker-DwNnEVYt.cjs → sagemaker-CcQHM1jV.cjs} +75 -76
- package/dist/src/{sagemaker-BcgLu0U4.js → sagemaker-D2Q1c-sD.js} +75 -79
- package/dist/src/{scanner-Dyw21Wg_.js → scanner-J8CA3LsV.js} +149 -122
- package/dist/src/server/index.js +5620 -67302
- package/dist/src/{server-CgUQ25qW.cjs → server-B0PPuDw-.cjs} +57 -67
- package/dist/src/server-B1vi21hA.js +7 -0
- package/dist/src/{server-CbMTRQkg.js → server-BC7XJFgr.js} +19 -24
- package/dist/src/server-Cm9Kai_h.cjs +5 -0
- package/dist/src/{server-DWmZLfCy.js → server-DbFphssR.js} +26 -29
- package/dist/src/server-OAs3nBRT.js +229 -0
- package/dist/src/{signal-Bl32q42d.js → signal-BOTbd53Z.js} +9 -11
- package/dist/src/{slack-BtMkB6xP.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-OZYxoVON.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-DPqj42Ts.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-BfdBx2tO.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-BNmZ1KAz.cjs +5 -0
- package/dist/src/{store-BqwfFEyF.cjs → store-BSc-TF2w.cjs} +44 -45
- package/dist/src/store-BltJg2cd.js +6 -0
- package/dist/src/{store-D4gdn9ih.js → store-D1tv90v3.js} +34 -35
- package/dist/src/{store-2ocbYY9D.js → store-DQLEjuEO.js} +40 -36
- package/dist/src/store-Ub2vaGJ1.js +228 -0
- package/dist/src/{tables-D-NSwNIb.js → tables-5EvT_Bwn.js} +23 -23
- package/dist/src/{tables-B9E1kRp-.cjs → tables-C7K-XKWp.cjs} +93 -93
- package/dist/src/{tables-C7TT2XVn.js → tables-D36WTqKX.js} +25 -25
- package/dist/src/tables-xKANLRBD.js +288 -0
- package/dist/src/telemetry-5BCRNBbe.cjs +5 -0
- package/dist/src/{telemetry-DZ_7PaVq.js → telemetry-C15ziL8u.js} +17 -14
- package/dist/src/{telemetry-BXyVqyAg.js → telemetry-C2YDkUQH.js} +11 -13
- package/dist/src/{telemetry-D0_yFdtU.cjs → telemetry-CbrnxHp_.cjs} +21 -24
- package/dist/src/telemetry-D4W5hboe.js +7 -0
- package/dist/src/telemetry-DMb2Mpfm.js +171 -0
- package/dist/src/{text-Dm78AVGG.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-DF2hMKdg.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-DgMr_tiM.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-FNIz3YOe.cjs → transcription-CL78qbOU.cjs} +14 -15
- package/dist/src/{transcription-C-M81iDA.js → transcription-DAtxHhAM.js} +14 -7
- package/dist/src/{transcription-CYuY5sFO.js → transcription-LNZTNUUL.js} +14 -16
- package/dist/src/{transcription-Ch7S-LWw.js → transcription-QHh3AH6Z.js} +15 -17
- package/dist/src/{transform-CoP2bJ7P.js → transform-Cgi24fJ7.js} +94 -66
- package/dist/src/{transform-Kd6u-oNm.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-D8dILpfZ.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/{transform-DMaxQwDx.js → transform-DGLazrMm.js} +94 -66
- package/dist/src/transform-DGxXocjk.js +1506 -0
- package/dist/src/{transform-ivxEY4f7.cjs → transform-DOcQeLld.cjs} +234 -206
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-CqTFr7KR.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/{transformersAvailability-DEU2naS1.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-Bkep3ka7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-DwmezkVe.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-t52w-XsS.js → types-CH3Ge2sE.js} +103 -92
- package/dist/src/{types-DMVjYLpx.js → types-CLKiCBW3.js} +98 -91
- package/dist/src/types-CN_TZ2GJ.js +3260 -0
- package/dist/src/{types-BIfttHrT.cjs → types-LJ0r3wbR.cjs} +573 -566
- package/dist/src/util-5cB-L7U3.js +1430 -0
- package/dist/src/util-6-GqIvzS.js +599 -0
- package/dist/src/{util-vjscpUzy.js → util-B7T3SiBS.js} +5 -6
- package/dist/src/{util-Cl0zfT3V.js → util-Betm42rL.js} +44 -17
- package/dist/src/{util-CUEt0Vum.js → util-C-PPYSMq.js} +44 -17
- package/dist/src/{util-DkFTvieG.cjs → util-CchiqXh_.cjs} +35 -36
- package/dist/src/{util-mJ58qbbw.js → util-DaWTWKBK.js} +5 -6
- package/dist/src/{util-C08Kns6-.cjs → util-Db0a0AFH.cjs} +89 -62
- package/dist/src/{util-DiCePfDu.js → util-Dlz_Wvgm.js} +102 -53
- package/dist/src/{util-BSh4a_Q8.js → util-YT5HPZaS.js} +102 -53
- package/dist/src/{util-DUYOvxAy.cjs → util-Yz-1aEhW.cjs} +274 -219
- package/dist/src/util-ZZH-3QZz.js +293 -0
- package/dist/src/{utils-DFaZa6Rf.cjs → utils-Cz9qXqII.cjs} +32 -35
- package/dist/src/{utils-CVzb4YiI.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-JaY9veb5.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +59 -53
- package/dist/src/app/assets/index-BOgkICuY.css +0 -1
- package/dist/src/app/assets/index-CSgqn_Vd.js +0 -428
- package/dist/src/app/tsconfig.app.tsbuildinfo +0 -1
- package/dist/src/base-BaXmtXYp.js +0 -107
- package/dist/src/base-Dtp8b4_N.js +0 -106
- package/dist/src/base-f71xxWai.cjs +0 -111
- package/dist/src/cache-BUPcq0Ad.js +0 -6
- package/dist/src/cache-CVfRb-HD.cjs +0 -6
- package/dist/src/cache-O4EuX2JV.js +0 -8
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-CZ4hytdm.js +0 -5
- package/dist/src/eval-CKHWqG9f.js +0 -16
- package/dist/src/evalResult-CxTP-LMm.cjs +0 -11
- package/dist/src/evalResult-CzLURDcP.js +0 -13
- package/dist/src/evalResult-DyttNQ_G.js +0 -11
- package/dist/src/evaluator-0PvfeBYh.js +0 -38
- package/dist/src/fetch-Bi0o-fdp.js +0 -4
- package/dist/src/fetch-CMptBDVg.cjs +0 -4
- package/dist/src/fetch-DAZkv3gV.js +0 -6
- package/dist/src/graders-BCytzXrb.js +0 -34
- package/dist/src/graders-CGZQShfJ.cjs +0 -33
- package/dist/src/graders-spkuVC-E.js +0 -36
- package/dist/src/opencode-sdk-CImWVqy9.js +0 -382
- package/dist/src/opencode-sdk-CuCztr4P.js +0 -380
- package/dist/src/opencode-sdk-DhcfRbBH.js +0 -376
- package/dist/src/opencode-sdk-mqF-Oj3f.cjs +0 -383
- package/dist/src/providers-BMZZmPBJ.cjs +0 -32
- package/dist/src/providers-CQQrNaJk.js +0 -32
- package/dist/src/providers-Ck8HyrC-.js +0 -34
- package/dist/src/quiverai-BNfIwKCO.cjs +0 -54
- package/dist/src/quiverai-BQigKdIH.js +0 -57
- package/dist/src/quiverai-Bfy2WnE2.js +0 -55
- package/dist/src/quiverai-CedIP0PJ.js +0 -43
- package/dist/src/rubyUtils-D7--T12C.js +0 -6
- package/dist/src/rubyUtils-DRRiMFV2.js +0 -5
- package/dist/src/rubyUtils-vb8OYFC-.cjs +0 -5
- package/dist/src/server-BUbS0Qfh.js +0 -6
- package/dist/src/server-XpGXFHkS.cjs +0 -6
- package/dist/src/server-gfOx5Zrk.js +0 -8
- package/dist/src/store-5u2yriTV.js +0 -7
- package/dist/src/store-D_lq_8oQ.js +0 -6
- package/dist/src/store-m5KT6Ly7.cjs +0 -6
- package/dist/src/telemetry-5RHFoCJh.js +0 -6
- package/dist/src/telemetry-Do8wMnA-.js +0 -8
- package/dist/src/telemetry-LojxPoFq.cjs +0 -6
- package/dist/src/transform-8eGmaH-7.js +0 -7
- package/dist/src/transform-BRVvWaG4.cjs +0 -6
- package/dist/src/transform-GybT0X0u.js +0 -8
- package/dist/src/transformersAvailability-DkAWaK5B.js +0 -35
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { _ as getEnvBool, i as
|
|
2
|
-
import { t as invariant } from "./invariant-
|
|
3
|
-
import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-
|
|
4
|
-
import { r as runPython } from "./pythonUtils-
|
|
5
|
-
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-
|
|
6
|
-
import { A as isApiProvider, g as ResultFailureReason, h as OutputFileExtension, j as isProviderOptions } from "./types-
|
|
7
|
-
import {
|
|
1
|
+
import { _ as getEnvBool, i as logger, s as sanitizeObject, w as state } from "./logger-CT3IKMKA.js";
|
|
2
|
+
import { t as invariant } from "./invariant-Ddh24eXh.js";
|
|
3
|
+
import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-Cd1AjG1D.js";
|
|
4
|
+
import { r as runPython } from "./pythonUtils-D5nxkQ0P.js";
|
|
5
|
+
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DnqA1y9x.js";
|
|
6
|
+
import { A as isApiProvider, g as ResultFailureReason, h as OutputFileExtension, j as isProviderOptions } from "./types-CLKiCBW3.js";
|
|
7
|
+
import { N as VERSION, O as TERMINAL_MAX_WIDTH, t as fetchWithProxy } from "./fetch-60Gzydls.js";
|
|
8
8
|
import * as fs$1 from "fs";
|
|
9
9
|
import * as path$1 from "path";
|
|
10
10
|
import path from "path";
|
|
@@ -19,7 +19,6 @@ import { parse as parse$1 } from "csv-parse/sync";
|
|
|
19
19
|
import nunjucks from "nunjucks";
|
|
20
20
|
import { XMLBuilder } from "fast-xml-parser";
|
|
21
21
|
import { stringify } from "csv-stringify/sync";
|
|
22
|
-
|
|
23
22
|
//#region src/util/provider.ts
|
|
24
23
|
function canonicalizeProviderId(id) {
|
|
25
24
|
if (id.startsWith("file://")) {
|
|
@@ -128,6 +127,55 @@ function isAnthropicProvider(providerId) {
|
|
|
128
127
|
}
|
|
129
128
|
return false;
|
|
130
129
|
}
|
|
130
|
+
const KNOWN_ENV_VARS = {
|
|
131
|
+
openai: "OPENAI_API_KEY",
|
|
132
|
+
anthropic: "ANTHROPIC_API_KEY",
|
|
133
|
+
google: "GOOGLE_API_KEY",
|
|
134
|
+
mistral: "MISTRAL_API_KEY",
|
|
135
|
+
cohere: "COHERE_API_KEY",
|
|
136
|
+
replicate: "REPLICATE_API_TOKEN",
|
|
137
|
+
voyage: "VOYAGE_API_KEY",
|
|
138
|
+
ai21: "AI21_API_KEY",
|
|
139
|
+
xai: "XAI_API_KEY",
|
|
140
|
+
groq: "GROQ_API_KEY",
|
|
141
|
+
deepseek: "DEEPSEEK_API_KEY",
|
|
142
|
+
perplexity: "PERPLEXITY_API_KEY",
|
|
143
|
+
hyperbolic: "HYPERBOLIC_API_KEY",
|
|
144
|
+
cerebras: "CEREBRAS_API_KEY",
|
|
145
|
+
togetherai: "TOGETHER_API_KEY",
|
|
146
|
+
fal: "FAL_KEY",
|
|
147
|
+
huggingface: "HF_TOKEN",
|
|
148
|
+
"cloudflare-ai": "CLOUDFLARE_API_KEY"
|
|
149
|
+
};
|
|
150
|
+
function getDefaultEnvVar(providerId) {
|
|
151
|
+
const prefix = providerId.split(":")[0];
|
|
152
|
+
return KNOWN_ENV_VARS[prefix] || `${prefix.toUpperCase()}_API_KEY`;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Pre-checks providers for missing API keys before evaluation starts.
|
|
156
|
+
* Assumes getApiKey() is side-effect free (no network calls or token refresh).
|
|
157
|
+
*/
|
|
158
|
+
function checkProviderApiKeys(providers) {
|
|
159
|
+
const missingApiKeys = /* @__PURE__ */ new Map();
|
|
160
|
+
for (const provider of providers) {
|
|
161
|
+
const p = provider;
|
|
162
|
+
if (typeof p.getApiKey !== "function") continue;
|
|
163
|
+
if (provider.id().startsWith("azure:")) continue;
|
|
164
|
+
const requiresKey = typeof p.requiresApiKey === "function" ? p.requiresApiKey() : p.config?.apiKeyRequired !== false;
|
|
165
|
+
let apiKey;
|
|
166
|
+
try {
|
|
167
|
+
apiKey = p.getApiKey();
|
|
168
|
+
} catch {
|
|
169
|
+
apiKey = void 0;
|
|
170
|
+
}
|
|
171
|
+
if (requiresKey && !apiKey) {
|
|
172
|
+
const envVar = p.config?.apiKeyEnvar || getDefaultEnvVar(provider.id());
|
|
173
|
+
if (!missingApiKeys.has(envVar)) missingApiKeys.set(envVar, []);
|
|
174
|
+
missingApiKeys.get(envVar).push(provider.id());
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return missingApiKeys;
|
|
178
|
+
}
|
|
131
179
|
/**
|
|
132
180
|
* Detects if a provider uses Google models.
|
|
133
181
|
* This includes direct Google/Vertex providers with Gemini and other Google models.
|
|
@@ -141,7 +189,6 @@ function isGoogleProvider(providerId) {
|
|
|
141
189
|
}
|
|
142
190
|
return false;
|
|
143
191
|
}
|
|
144
|
-
|
|
145
192
|
//#endregion
|
|
146
193
|
//#region src/util/comparison.ts
|
|
147
194
|
/**
|
|
@@ -240,7 +287,7 @@ function resultIsForTestCase(result, testCase) {
|
|
|
240
287
|
const isMatch = doVarsMatch && providersMatch;
|
|
241
288
|
if (!isMatch) {
|
|
242
289
|
const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
|
|
243
|
-
|
|
290
|
+
logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
|
|
244
291
|
testProvider: testProviderId || "none",
|
|
245
292
|
resultProvider: resultProviderId || "none",
|
|
246
293
|
testVarKeys: varKeys
|
|
@@ -248,7 +295,6 @@ function resultIsForTestCase(result, testCase) {
|
|
|
248
295
|
}
|
|
249
296
|
return isMatch;
|
|
250
297
|
}
|
|
251
|
-
|
|
252
298
|
//#endregion
|
|
253
299
|
//#region src/util/env.ts
|
|
254
300
|
/**
|
|
@@ -265,8 +311,8 @@ function setupEnv(envPath) {
|
|
|
265
311
|
return;
|
|
266
312
|
}
|
|
267
313
|
for (const p of paths) if (!fs$1.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
|
|
268
|
-
if (paths.length === 1)
|
|
269
|
-
else
|
|
314
|
+
if (paths.length === 1) logger.info(`Loading environment variables from ${paths[0]}`);
|
|
315
|
+
else logger.info(`Loading environment variables from: ${paths.join(", ")}`);
|
|
270
316
|
const pathArg = paths.length === 1 ? paths[0] : paths;
|
|
271
317
|
dotenv.config({
|
|
272
318
|
path: pathArg,
|
|
@@ -275,7 +321,6 @@ function setupEnv(envPath) {
|
|
|
275
321
|
});
|
|
276
322
|
} else dotenv.config({ quiet: true });
|
|
277
323
|
}
|
|
278
|
-
|
|
279
324
|
//#endregion
|
|
280
325
|
//#region src/util/functions/loadFunction.ts
|
|
281
326
|
const functionCache = {};
|
|
@@ -284,7 +329,7 @@ const functionCache = {};
|
|
|
284
329
|
* @param options Options for loading the function
|
|
285
330
|
* @returns The loaded function
|
|
286
331
|
*/
|
|
287
|
-
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath =
|
|
332
|
+
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = state.basePath, useCache = true }) {
|
|
288
333
|
const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
|
|
289
334
|
if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
|
|
290
335
|
const resolvedPath = basePath ? path.resolve(basePath, filePath) : filePath;
|
|
@@ -305,7 +350,7 @@ async function loadFunction({ filePath, functionName, defaultFunctionName = "fun
|
|
|
305
350
|
if (useCache) functionCache[cacheKey] = func;
|
|
306
351
|
return func;
|
|
307
352
|
} catch (err) {
|
|
308
|
-
|
|
353
|
+
logger.error(`Failed to load function: ${err.message}`);
|
|
309
354
|
throw err;
|
|
310
355
|
}
|
|
311
356
|
}
|
|
@@ -324,7 +369,6 @@ function parseFileUrl(fileUrl) {
|
|
|
324
369
|
};
|
|
325
370
|
return { filePath: urlWithoutProtocol };
|
|
326
371
|
}
|
|
327
|
-
|
|
328
372
|
//#endregion
|
|
329
373
|
//#region src/util/templates.ts
|
|
330
374
|
/**
|
|
@@ -343,7 +387,7 @@ function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false)
|
|
|
343
387
|
});
|
|
344
388
|
const envGlobals = {
|
|
345
389
|
...getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
|
|
346
|
-
...
|
|
390
|
+
...state.config?.env
|
|
347
391
|
};
|
|
348
392
|
env.addGlobal("env", envGlobals);
|
|
349
393
|
env.addFilter("load", function(str) {
|
|
@@ -383,7 +427,6 @@ function extractVariablesFromTemplates(templates) {
|
|
|
383
427
|
for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
|
|
384
428
|
return Array.from(variableSet);
|
|
385
429
|
}
|
|
386
|
-
|
|
387
430
|
//#endregion
|
|
388
431
|
//#region src/util/render.ts
|
|
389
432
|
/**
|
|
@@ -413,7 +456,7 @@ function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
|
413
456
|
if (getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
|
|
414
457
|
if (typeof obj === "string") {
|
|
415
458
|
if (obj.length > 5e4) {
|
|
416
|
-
|
|
459
|
+
logger.warn(`String too long (${obj.length} chars) for template matching. Skipping env var rendering.`);
|
|
417
460
|
return obj;
|
|
418
461
|
}
|
|
419
462
|
const nunjucks = getNunjucksEngine();
|
|
@@ -429,7 +472,7 @@ function renderEnvOnlyInObject(obj, envOverrides, replaceBase) {
|
|
|
429
472
|
if (match.includes("|") || varName && varName in envGlobals && envGlobals[varName] !== void 0) try {
|
|
430
473
|
return nunjucks.renderString(match, { env: envGlobals });
|
|
431
474
|
} catch (error) {
|
|
432
|
-
|
|
475
|
+
logger.debug(`Failed to render env template "${match}": ${error instanceof Error ? error.message : String(error)}`);
|
|
433
476
|
return match;
|
|
434
477
|
}
|
|
435
478
|
return match;
|
|
@@ -454,7 +497,6 @@ function renderVarsInObject(obj, vars) {
|
|
|
454
497
|
} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
|
|
455
498
|
return obj;
|
|
456
499
|
}
|
|
457
|
-
|
|
458
500
|
//#endregion
|
|
459
501
|
//#region src/util/file.ts
|
|
460
502
|
/**
|
|
@@ -465,7 +507,7 @@ function getNunjucksEngineForFilePath() {
|
|
|
465
507
|
const env = nunjucks.configure({ autoescape: false });
|
|
466
508
|
env.addGlobal("env", {
|
|
467
509
|
...process.env,
|
|
468
|
-
...
|
|
510
|
+
...state.config?.env
|
|
469
511
|
});
|
|
470
512
|
return env;
|
|
471
513
|
}
|
|
@@ -492,16 +534,16 @@ function maybeLoadFromExternalFile(filePath, context) {
|
|
|
492
534
|
const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
|
|
493
535
|
const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
|
|
494
536
|
if (context === "assertion" && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) {
|
|
495
|
-
|
|
537
|
+
logger.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
|
|
496
538
|
return renderedFilePath;
|
|
497
539
|
}
|
|
498
540
|
if (context === "vars") {
|
|
499
|
-
|
|
541
|
+
logger.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
|
|
500
542
|
return renderedFilePath;
|
|
501
543
|
}
|
|
502
544
|
if (functionName && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) return renderedFilePath;
|
|
503
545
|
const pathToUse = functionName && !(cleanPath.endsWith(".py") || isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
|
|
504
|
-
const resolvedPath = path$1.resolve(
|
|
546
|
+
const resolvedPath = path$1.resolve(state.basePath || "", pathToUse);
|
|
505
547
|
if (hasMagic(pathToUse)) {
|
|
506
548
|
const matchedFiles = globSync(resolvedPath, { windowsPathsNoEscape: true });
|
|
507
549
|
if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
|
|
@@ -512,7 +554,7 @@ function maybeLoadFromExternalFile(filePath, context) {
|
|
|
512
554
|
contents = fs$1.readFileSync(matchedFile, "utf8");
|
|
513
555
|
} catch (error) {
|
|
514
556
|
if (error.code === "ENOENT") {
|
|
515
|
-
|
|
557
|
+
logger.debug(`File disappeared during glob expansion: ${matchedFile}`);
|
|
516
558
|
continue;
|
|
517
559
|
}
|
|
518
560
|
throw error;
|
|
@@ -722,37 +764,37 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
722
764
|
const { filePath, functionName } = parseFileUrl(rendered);
|
|
723
765
|
if (functionName && (filePath.endsWith(".py") || isJavascriptFile(filePath))) {
|
|
724
766
|
const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
725
|
-
|
|
767
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
|
|
726
768
|
try {
|
|
727
769
|
let toolDefinitions;
|
|
728
770
|
if (filePath.endsWith(".py")) {
|
|
729
|
-
const absPath = safeResolve(
|
|
730
|
-
|
|
771
|
+
const absPath = safeResolve(state.basePath || process.cwd(), filePath);
|
|
772
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
|
|
731
773
|
toolDefinitions = await runPython(absPath, functionName, []);
|
|
732
774
|
} else {
|
|
733
|
-
const absPath = safeResolve(
|
|
734
|
-
|
|
775
|
+
const absPath = safeResolve(state.basePath || process.cwd(), filePath);
|
|
776
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
|
|
735
777
|
const module = await importModule(absPath);
|
|
736
778
|
const fn = module[functionName] || module.default?.[functionName];
|
|
737
779
|
if (typeof fn !== "function") {
|
|
738
780
|
const availableExports = Object.keys(module).filter((k) => k !== "default");
|
|
739
|
-
const basePath =
|
|
781
|
+
const basePath = state.basePath || process.cwd();
|
|
740
782
|
throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
|
|
741
783
|
}
|
|
742
784
|
toolDefinitions = await Promise.resolve(fn());
|
|
743
785
|
}
|
|
744
786
|
if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
|
|
745
|
-
|
|
787
|
+
logger.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
|
|
746
788
|
return toolDefinitions;
|
|
747
789
|
} catch (err) {
|
|
748
790
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
749
|
-
const basePath =
|
|
791
|
+
const basePath = state.basePath || process.cwd();
|
|
750
792
|
throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
|
|
751
793
|
}
|
|
752
794
|
}
|
|
753
795
|
if (filePath.endsWith(".py") || isJavascriptFile(filePath)) {
|
|
754
796
|
const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
|
|
755
|
-
const basePath =
|
|
797
|
+
const basePath = state.basePath || process.cwd();
|
|
756
798
|
throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? ` def get_tools():\n return [{"type": "function", "function": {...}}]` : ` module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
|
|
757
799
|
}
|
|
758
800
|
}
|
|
@@ -770,7 +812,6 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
|
|
|
770
812
|
}
|
|
771
813
|
return loaded;
|
|
772
814
|
}
|
|
773
|
-
|
|
774
815
|
//#endregion
|
|
775
816
|
//#region src/googleSheets.ts
|
|
776
817
|
async function checkGoogleSheetAccess(url) {
|
|
@@ -785,7 +826,7 @@ async function checkGoogleSheetAccess(url) {
|
|
|
785
826
|
status: response.status
|
|
786
827
|
};
|
|
787
828
|
} catch (error) {
|
|
788
|
-
|
|
829
|
+
logger.error(`Error checking sheet access: ${error}`);
|
|
789
830
|
return { public: false };
|
|
790
831
|
}
|
|
791
832
|
}
|
|
@@ -837,7 +878,7 @@ async function fetchCsvFromGoogleSheetAuthenticated(url) {
|
|
|
837
878
|
}
|
|
838
879
|
async function fetchCsvFromGoogleSheet(url) {
|
|
839
880
|
const { public: isPublic } = await checkGoogleSheetAccess(url);
|
|
840
|
-
|
|
881
|
+
logger.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
|
|
841
882
|
if (isPublic) return fetchCsvFromGoogleSheetUnauthenticated(url);
|
|
842
883
|
return fetchCsvFromGoogleSheetAuthenticated(url);
|
|
843
884
|
}
|
|
@@ -880,7 +921,7 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
880
921
|
});
|
|
881
922
|
range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
|
|
882
923
|
}
|
|
883
|
-
|
|
924
|
+
logger.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
|
|
884
925
|
await sheets.spreadsheets.values.update({
|
|
885
926
|
spreadsheetId,
|
|
886
927
|
range,
|
|
@@ -889,7 +930,6 @@ async function writeCsvToGoogleSheet(rows, url) {
|
|
|
889
930
|
requestBody: { values }
|
|
890
931
|
});
|
|
891
932
|
}
|
|
892
|
-
|
|
893
933
|
//#endregion
|
|
894
934
|
//#region src/server/utils/evalTableUtils.ts
|
|
895
935
|
/**
|
|
@@ -1088,7 +1128,6 @@ async function streamEvalCsv(eval_, options) {
|
|
|
1088
1128
|
isRedteam
|
|
1089
1129
|
})]));
|
|
1090
1130
|
}
|
|
1091
|
-
|
|
1092
1131
|
//#endregion
|
|
1093
1132
|
//#region src/util/output.ts
|
|
1094
1133
|
const outputToSimpleString = (output) => {
|
|
@@ -1104,6 +1143,13 @@ const outputToSimpleString = (output) => {
|
|
|
1104
1143
|
${gradingResultText}
|
|
1105
1144
|
`.trim();
|
|
1106
1145
|
};
|
|
1146
|
+
function sanitizeConfigForOutput(config) {
|
|
1147
|
+
return sanitizeObject(config, {
|
|
1148
|
+
context: "output config",
|
|
1149
|
+
throwOnError: true,
|
|
1150
|
+
maxDepth: Number.POSITIVE_INFINITY
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1107
1153
|
function createOutputMetadata(evalRecord) {
|
|
1108
1154
|
let evaluationCreatedAt;
|
|
1109
1155
|
if (evalRecord.createdAt) try {
|
|
@@ -1130,10 +1176,11 @@ async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
|
|
|
1130
1176
|
const metadata = createOutputMetadata(evalRecord);
|
|
1131
1177
|
try {
|
|
1132
1178
|
const summary = await evalRecord.toEvaluateSummary();
|
|
1179
|
+
const redactedConfig = sanitizeConfigForOutput(evalRecord.config);
|
|
1133
1180
|
const outputData = {
|
|
1134
1181
|
evalId: evalRecord.id,
|
|
1135
1182
|
results: summary,
|
|
1136
|
-
config:
|
|
1183
|
+
config: redactedConfig,
|
|
1137
1184
|
shareableUrl,
|
|
1138
1185
|
metadata
|
|
1139
1186
|
};
|
|
@@ -1145,7 +1192,7 @@ async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
|
|
|
1145
1192
|
const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
|
|
1146
1193
|
if (isStringLen || isHeapOOM) {
|
|
1147
1194
|
const resultCount = await evalRecord.getResultsCount();
|
|
1148
|
-
|
|
1195
|
+
logger.error(`Dataset too large for JSON export (${resultCount} results).`);
|
|
1149
1196
|
throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
|
|
1150
1197
|
} else throw error;
|
|
1151
1198
|
}
|
|
@@ -1164,7 +1211,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1164
1211
|
});
|
|
1165
1212
|
return csvRow;
|
|
1166
1213
|
});
|
|
1167
|
-
|
|
1214
|
+
logger.info(`Writing ${rows.length} rows to Google Sheets...`);
|
|
1168
1215
|
await writeCsvToGoogleSheet(rows, outputPath);
|
|
1169
1216
|
return;
|
|
1170
1217
|
}
|
|
@@ -1188,10 +1235,11 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1188
1235
|
} else if (outputExtension === "json") await writeJsonOutputSafely(outputPath, evalRecord, shareableUrl);
|
|
1189
1236
|
else if (outputExtension === "yaml" || outputExtension === "yml" || outputExtension === "txt") {
|
|
1190
1237
|
const summary = await evalRecord.toEvaluateSummary();
|
|
1238
|
+
const redactedConfig = sanitizeConfigForOutput(evalRecord.config);
|
|
1191
1239
|
await fsPromises.writeFile(outputPath, yaml.dump({
|
|
1192
1240
|
evalId: evalRecord.id,
|
|
1193
1241
|
results: summary,
|
|
1194
|
-
config:
|
|
1242
|
+
config: redactedConfig,
|
|
1195
1243
|
shareableUrl,
|
|
1196
1244
|
metadata
|
|
1197
1245
|
}));
|
|
@@ -1199,10 +1247,11 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1199
1247
|
const table = await evalRecord.getTable();
|
|
1200
1248
|
invariant(table, "Table is required");
|
|
1201
1249
|
const summary = await evalRecord.toEvaluateSummary();
|
|
1250
|
+
const redactedConfig = sanitizeConfigForOutput(evalRecord.config);
|
|
1202
1251
|
const template = await fsPromises.readFile(path$1.join(getDirectory(), "tableOutput.html"), "utf-8");
|
|
1203
1252
|
const htmlTable = [[...table.head.vars, ...table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`)], ...table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)])];
|
|
1204
1253
|
const htmlOutput = getNunjucksEngine().renderString(template, {
|
|
1205
|
-
config:
|
|
1254
|
+
config: redactedConfig,
|
|
1206
1255
|
table: htmlTable,
|
|
1207
1256
|
results: summary
|
|
1208
1257
|
});
|
|
@@ -1215,6 +1264,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1215
1264
|
}
|
|
1216
1265
|
} else if (outputExtension === "xml") {
|
|
1217
1266
|
const summary = await evalRecord.toEvaluateSummary();
|
|
1267
|
+
const redactedConfig = sanitizeConfigForOutput(evalRecord.config);
|
|
1218
1268
|
const sanitizeForXml = (obj) => {
|
|
1219
1269
|
if (obj === null || obj === void 0) return "";
|
|
1220
1270
|
if (typeof obj === "boolean" || typeof obj === "number") return String(obj);
|
|
@@ -1234,7 +1284,7 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1234
1284
|
}).build({ promptfoo: {
|
|
1235
1285
|
evalId: evalRecord.id,
|
|
1236
1286
|
results: sanitizeForXml(summary),
|
|
1237
|
-
config: sanitizeForXml(
|
|
1287
|
+
config: sanitizeForXml(redactedConfig),
|
|
1238
1288
|
shareableUrl: shareableUrl || ""
|
|
1239
1289
|
} });
|
|
1240
1290
|
await fsPromises.writeFile(outputPath, xmlData);
|
|
@@ -1243,14 +1293,13 @@ async function writeOutput(outputPath, evalRecord, shareableUrl) {
|
|
|
1243
1293
|
async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
|
|
1244
1294
|
await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
|
|
1245
1295
|
}
|
|
1246
|
-
|
|
1247
1296
|
//#endregion
|
|
1248
1297
|
//#region src/util/runtime.ts
|
|
1249
1298
|
function printBorder() {
|
|
1250
1299
|
const border = "=".repeat(TERMINAL_MAX_WIDTH);
|
|
1251
|
-
|
|
1300
|
+
logger.info(border);
|
|
1252
1301
|
}
|
|
1253
|
-
|
|
1254
1302
|
//#endregion
|
|
1255
|
-
export {
|
|
1256
|
-
|
|
1303
|
+
export { getProviderDescription as A, deduplicateTestCases as C, resultIsForTestCase as D, getTestCaseDeduplicationKey as E, isGoogleProvider as M, isOpenAiProvider as N, checkProviderApiKeys as O, isProviderAllowed as P, setupEnv as S, filterRuntimeVars as T, extractVariablesFromTemplate as _, getNunjucksEngineForFilePath as a, loadFunction as b, maybeLoadFromExternalFile as c, maybeLoadToolsFromExternalFile as d, parsePathOrGlob as f, renderVarsInObject as g, renderEnvOnlyInObject as h, fetchCsvFromGoogleSheet as i, isAnthropicProvider as j, doesProviderRefMatch as k, maybeLoadFromExternalFileWithVars as l, readOutput as m, writeMultipleOutputs as n, getResolvedRelativePath as o, readFilters as p, writeOutput as r, maybeLoadConfigFromExternalFile as s, printBorder as t, maybeLoadResponseFormatFromExternalFile as u, extractVariablesFromTemplates as v, extractRuntimeVars as w, parseFileUrl as x, getNunjucksEngine as y };
|
|
1304
|
+
|
|
1305
|
+
//# sourceMappingURL=util-Dlz_Wvgm.js.map
|