promptfoo 0.121.1 → 0.121.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/src/{accounts-xrUGFA6n.js → accounts-B2XmGjty.js} +5 -5
- package/dist/src/{accounts-Bx-x3bmW.cjs → accounts-BPyfpSeU.cjs} +5 -5
- package/dist/src/{accounts-CMqkzrVf.js → accounts-CFLK3mnD.js} +6 -6
- package/dist/src/{accounts-BgNJDBE6.js → accounts-Xatc0RYb.js} +5 -5
- package/dist/src/{agentic-utils-BKIN5PKu.js → agentic-utils-36epdqwB.js} +3 -3
- package/dist/src/{cometapi-DkXrKi5z.js → agentic-utils-D8yXo5Lm.js} +4 -61
- package/dist/src/{cometapi-vY6aDZgo.cjs → agentic-utils-DAVsChuB.cjs} +24 -62
- package/dist/src/agentic-utils-DIYAAYE7.js +153 -0
- package/dist/src/{agents-C-dDThPK.js → agents-BBVJCIYr.js} +226 -13
- package/dist/src/{agents-CErsqg5U.cjs → agents-BBWxKSM0.cjs} +7 -7
- package/dist/src/{agents-Dy2YpZpa.js → agents-Bqgfdokm.js} +227 -14
- package/dist/src/{agents-B0f4HICh.cjs → agents-CAYbM7qD.cjs} +226 -13
- package/dist/src/{agents-CVIn-Utx.js → agents-CLQ-P15P.js} +7 -7
- package/dist/src/{agents-DeH4Gu94.js → agents-CgBniSlI.js} +8 -8
- package/dist/src/{agents-CXknwsFX.js → agents-DSSTV4bv.js} +226 -13
- package/dist/src/{agents-aF4-T121.js → agents-wg3ohknq.js} +7 -7
- package/dist/src/{aimlapi-tg0Gkcvr.cjs → aimlapi-Bv8Fmc-b.cjs} +14 -14
- package/dist/src/{aimlapi-BNfTBexL.js → aimlapi-BwGC1TtS.js} +13 -13
- package/dist/src/{aimlapi-BAGZDo5G.js → aimlapi-DaC3qZ-o.js} +14 -14
- package/dist/src/{aimlapi-DHRKlBEA.js → aimlapi-MgSLdvy7.js} +13 -13
- package/dist/src/app/assets/index-B6l9CVVb.js +439 -0
- package/dist/src/app/assets/index-DyZ0Ep37.css +1 -0
- package/dist/src/app/assets/sync-CStkzc6u.js +4 -0
- package/dist/src/app/assets/vendor-charts-BnDWwBlI.js +36 -0
- package/dist/src/app/assets/vendor-markdown-Bz7N-ca6.js +29 -0
- package/dist/src/app/index.html +4 -4
- package/dist/src/{audio-tf_NBjlC.js → audio-Bn44pQxv.js} +4 -4
- package/dist/src/{audio-CHQ4r-RV.js → audio-DDA5WHdx.js} +4 -4
- package/dist/src/{audio-BWeaWovU.cjs → audio-DVFjQ67_.cjs} +4 -4
- package/dist/src/{audio-BRODU0UK.js → audio-DjU9GswO.js} +5 -5
- package/dist/src/{base-DBtwl2FR.cjs → base-BboXIF_0.cjs} +3 -3
- package/dist/src/{base-B4QJRyFS.js → base-CKjwebIH.js} +3 -3
- package/dist/src/{base-B0tcrnq_.js → base-CqzQ4K8j.js} +3 -3
- package/dist/src/{base-fEDN28WM.js → base-Cz2ZC_iA.js} +3 -3
- package/dist/src/{blobs-BAU-dXan.js → blobs-B1JriOyi.js} +3 -3
- package/dist/src/{blobs-qTYm-1PY.js → blobs-BUWmKWzo.js} +3 -3
- package/dist/src/{blobs-DvS-O6be.cjs → blobs-C6j0bvFz.cjs} +3 -3
- package/dist/src/{blobs-Bpg5rH6i.js → blobs-DXTl6J3H.js} +3 -3
- package/dist/src/{cache-COish3-W.cjs → cache-C5yFZ4gC.cjs} +75 -58
- package/dist/src/{cache-8XhNqPKW.js → cache-CaT5tPgo.js} +75 -58
- package/dist/src/cache-CyCanoMu.js +6 -0
- package/dist/src/{cache-CG0SlR1d.js → cache-DSqR6ezl.js} +75 -58
- package/dist/src/cache-Df_QFDNu.cjs +5 -0
- package/dist/src/{cache-D3eqDYGU.js → cache-HP0NP4k3.js} +75 -58
- package/dist/src/{chat-DHMH-N64.js → chat-B-52XYI1.js} +12 -12
- package/dist/src/{chat-BKm79wib.js → chat-B0iaWhoh.js} +16 -14
- package/dist/src/{chat-DxysjBvt.js → chat-BE0qTA8e.js} +13 -13
- package/dist/src/{chat-CRWNNq73.js → chat-BEwdgGEg.js} +16 -14
- package/dist/src/{chat-2K608PeQ.cjs → chat-BtIKkLKx.cjs} +13 -13
- package/dist/src/{chat-DaqekjFr.cjs → chat-CM8qWR3_.cjs} +17 -15
- package/dist/src/{chat-CM_kyI8B.js → chat-DK1U-eZ-.js} +12 -12
- package/dist/src/{chat-CznLWr_D.js → chat-pxmiVpWe.js} +16 -14
- package/dist/src/{chatkit-65VXf5SR.js → chatkit-BYGQlHlV.js} +4 -4
- package/dist/src/{chatkit-DKyPi1Gs.cjs → chatkit-Cx174XI3.cjs} +4 -4
- package/dist/src/{chatkit-BxFvW8KY.js → chatkit-_8eJqKcD.js} +4 -4
- package/dist/src/{chatkit-Be-Q-a9F.js → chatkit-a2D6mY6s.js} +4 -4
- package/dist/src/{claude-agent-sdk-CJH22shf.cjs → claude-agent-sdk-8ddRp1L2.cjs} +35 -17
- package/dist/src/{claude-agent-sdk-Dy5lT-Tx.js → claude-agent-sdk-Bq5EArsX.js} +33 -15
- package/dist/src/{claude-agent-sdk-BLTu0WBO.js → claude-agent-sdk-CMjh4LFH.js} +33 -15
- package/dist/src/{claude-agent-sdk-D6_k9FKA.js → claude-agent-sdk-HgbFioFw.js} +33 -15
- package/dist/src/cloud-DE3t1-ZI.js +4 -0
- package/dist/src/{cloud-Bc9526yV.js → cloud-z8KZpUoa.js} +3 -3
- package/dist/src/{cloudflare-ai-CWWJCRim.js → cloudflare-ai-BGyXlpXJ.js} +13 -13
- package/dist/src/{cloudflare-ai-C9r2sRhw.js → cloudflare-ai-Bbp26N0L.js} +13 -13
- package/dist/src/{cloudflare-ai-ClWSdor4.cjs → cloudflare-ai-C62x6MQG.cjs} +14 -14
- package/dist/src/{cloudflare-ai-ICsOuD-z.js → cloudflare-ai-DdKP9TKT.js} +14 -14
- package/dist/src/{cloudflare-gateway-D6xFc5pa.js → cloudflare-gateway-BwAaUgeW.js} +14 -14
- package/dist/src/{cloudflare-gateway-D6O7AlYb.js → cloudflare-gateway-D-e9i1Sn.js} +15 -15
- package/dist/src/{cloudflare-gateway-pXGHxJ47.js → cloudflare-gateway-DXhtXDRb.js} +15 -163
- package/dist/src/{cloudflare-gateway-C2_-KG5o.cjs → cloudflare-gateway-Dx36ftqF.cjs} +15 -15
- package/dist/src/{codex-sdk-DUwKWezN.js → codex-sdk-BQEw16R_.js} +180 -11
- package/dist/src/{codex-sdk-C6UMlxwV.js → codex-sdk-C_07GuVS.js} +180 -11
- package/dist/src/{codex-sdk-GGAw0qbD.js → codex-sdk-DE5G18dx.js} +180 -11
- package/dist/src/{codex-sdk-fAO0c3yA.cjs → codex-sdk-ZLKfDjqP.cjs} +181 -12
- package/dist/src/cometapi-BDyV-NNm.js +62 -0
- package/dist/src/cometapi-C3hOlM7-.cjs +62 -0
- package/dist/src/{cometapi-Bbjp5V4x.js → cometapi-hhL4TAh3.js} +14 -14
- package/dist/src/{cometapi-BasUi7-_.js → cometapi-sp7sJpBD.js} +15 -15
- package/dist/src/{completion-C_P3ypkJ.js → completion-BCimtq-h.js} +6 -6
- package/dist/src/{completion-6Mx_iXxK.js → completion-DCjv7RZ3.js} +6 -6
- package/dist/src/{completion-CDOouNzq.cjs → completion-DlXUhj5c.cjs} +6 -6
- package/dist/src/{completion-C5rtR_9P.js → completion-DoYy49ti.js} +6 -6
- package/dist/src/{createHash-CfZSc0b4.cjs → createHash-BYwImsYv.cjs} +2 -2
- package/dist/src/{docker-BwsKwxFs.cjs → docker-Cqj2-QVi.cjs} +14 -14
- package/dist/src/{docker-CZnqU1XV.js → docker-CxCkwMzc.js} +13 -13
- package/dist/src/{docker-DzxyDPIj.js → docker-DpguQj-w.js} +14 -14
- package/dist/src/{docker-5KcG-_86.js → docker-FeBni2dw.js} +13 -13
- package/dist/src/{esm-C03C-mv3.js → esm-7UIl0pPM.js} +2 -2
- package/dist/src/{esm-Cd1AjG1D.js → esm-CKWP3u_P.js} +3 -3
- package/dist/src/{esm-CnNt7sI4.cjs → esm-CipptfDu.cjs} +2 -2
- package/dist/src/{esm-CaIwzWR5.js → esm-SUNIX1x3.js} +3 -3
- package/dist/src/eval-7aEqoMs3.js +15 -0
- package/dist/src/{eval-DmFyWU7i.js → eval-BTqTn7lb.js} +10 -10
- package/dist/src/{evalResult-CDQiuUuf.js → evalResult-BkIhRdTe.js} +7 -7
- package/dist/src/evalResult-CYNHkk5A.js +12 -0
- package/dist/src/evalResult-CuvJeNiM.js +10 -0
- package/dist/src/{evalResult-CTG2AHOS.js → evalResult-DUDShQrm.js} +7 -7
- package/dist/src/{evalResult-Dap2CekP.cjs → evalResult-DpARzUCb.cjs} +7 -7
- package/dist/src/evalResult-tGdilrWt.cjs +10 -0
- package/dist/src/evaluator-BBUqRhz1.js +36 -0
- package/dist/src/{evaluator-DPFRbFIL.js → evaluator-BcvOGaam.js} +833 -79
- package/dist/src/{extractor-YMU_Gvt8.js → extractor-C8XwivI9.js} +6 -6
- package/dist/src/{extractor-CFG6bcWJ.js → extractor-CAZ2G3Kh.js} +6 -6
- package/dist/src/{extractor-DX36oYEv.cjs → extractor-DG3sSfXE.cjs} +6 -6
- package/dist/src/{extractor-M67RUtg6.js → extractor-D_wd8jxt.js} +6 -6
- package/dist/src/{fetch-4M3YRaqL.js → fetch-BiYv2BZc.js} +3 -3
- package/dist/src/{fetch-BxUk8odA.cjs → fetch-BnR9wSnm.cjs} +3 -3
- package/dist/src/{fetch-60Gzydls.js → fetch-CVAtKnI3.js} +3 -3
- package/dist/src/{fetch-BMv0O527.js → fetch-DoVRJZhJ.js} +4 -4
- package/dist/src/fetch-UWU706qb.js +5 -0
- package/dist/src/{genaiTracer-DN4dQywX.cjs → genaiTracer-BfxrvSUb.cjs} +2 -2
- package/dist/src/{graders-DOXycdlG.cjs → graders-BElhu9ZY.cjs} +126 -55
- package/dist/src/{graders-R9rYUM0d.js → graders-BXAJ0sbS.js} +120 -55
- package/dist/src/graders-BxfEguVY.js +32 -0
- package/dist/src/graders-CzVMbEnv.js +34 -0
- package/dist/src/{graders-CpdqD9PI.js → graders-DG7mhg-b.js} +120 -55
- package/dist/src/graders-DjCXfj0l.cjs +32 -0
- package/dist/src/{graders-CHO8EPM4.js → graders-RjHF8VfG.js} +120 -55
- package/dist/src/graders-kHzIWOKu.js +32 -0
- package/dist/src/{image-DTedmQPg.cjs → image--F58eEIn.cjs} +6 -6
- package/dist/src/{image-DJEvKveK.js → image-6WQXK8m8.js} +4 -4
- package/dist/src/{image-pAX56tPG.js → image-B8b6f36E.js} +6 -6
- package/dist/src/{image-BmEZqVmk.js → image-CoxZp9PZ.js} +6 -6
- package/dist/src/{image-gvmivTEe.js → image-DO0RYnjH.js} +5 -5
- package/dist/src/{image-CBBVXWuT.js → image-PoF6DN3x.js} +6 -6
- package/dist/src/{image-CDLQOcqT.cjs → image-fza3zuKs.cjs} +4 -4
- package/dist/src/{image-tL5hIOFh.js → image-xNbw5ph2.js} +4 -4
- package/dist/src/index.cjs +863 -110
- package/dist/src/index.d.cts +833 -60
- package/dist/src/index.d.ts +833 -60
- package/dist/src/index.js +860 -108
- package/dist/src/{interactiveCheck-BgLZUIt3.js → interactiveCheck-BnMYOjMu.js} +2 -2
- package/dist/src/{knowledgeBase-CoU-UQBg.js → knowledgeBase-Bi7CmDbx.js} +7 -7
- package/dist/src/{knowledgeBase-CLJybhnF.js → knowledgeBase-Ce3ofVan.js} +8 -8
- package/dist/src/{knowledgeBase-DjWPVqSb.js → knowledgeBase-DFRXPZl_.js} +7 -7
- package/dist/src/{knowledgeBase-wkxuRFhA.cjs → knowledgeBase-DqrLX8fy.cjs} +7 -7
- package/dist/src/{litellm-B9Hysuri.js → litellm-Bo2gQXpo.js} +16 -15
- package/dist/src/{litellm-ePxtr9F1.js → litellm-CKiAxnoM.js} +15 -14
- package/dist/src/{litellm-NYpQ8RQu.cjs → litellm-CnHI69aj.cjs} +16 -15
- package/dist/src/{litellm-CTfa0hqi.js → litellm-Tc294Jhj.js} +15 -14
- package/dist/src/{logger-KkObSCzq.js → logger-BcJBzSSA.js} +10 -14
- package/dist/src/{logger-DLcq4dWf.js → logger-BnkjG2jt.js} +10 -14
- package/dist/src/{logger-Cp1GPUjj.cjs → logger-D5iKBpu_.cjs} +27 -13
- package/dist/src/{logger-CT3IKMKA.js → logger-DO8_zM18.js} +10 -14
- package/dist/src/{luma-ray-BW9IRGIc.js → luma-ray-0ehMPt5N.js} +10 -10
- package/dist/src/{luma-ray-BE2mOt6N.js → luma-ray-C9q8rdQe.js} +9 -9
- package/dist/src/{luma-ray-Cm1KZBhs.js → luma-ray-DP0QA9qn.js} +9 -9
- package/dist/src/{luma-ray-B0GGNRc1.cjs → luma-ray-m9Ku2meV.cjs} +9 -9
- package/dist/src/main.js +69 -71
- package/dist/src/{messages-1x9atZmP.js → messages-DJNo37Ko.js} +14 -9
- package/dist/src/{messages-BLbWdsyt.js → messages-Dy9QecMs.js} +14 -9
- package/dist/src/{messages-1JrJs91T.cjs → messages-HJsyEh4o.cjs} +15 -10
- package/dist/src/{messages-D8EA0oDc.js → messages-biC_ex-p.js} +14 -9
- package/dist/src/{modelslab-C1OLRmVX.js → modelslab-B5J-ZM5c.js} +9 -9
- package/dist/src/{modelslab-CqXBy3U8.js → modelslab-BI458moT.js} +10 -10
- package/dist/src/{modelslab-X5-4LroM.js → modelslab-BTOT8FUO.js} +9 -9
- package/dist/src/{modelslab-DcOSFwKh.cjs → modelslab-IQbNg-r7.cjs} +9 -9
- package/dist/src/{nova-reel-DihqLeol.js → nova-reel-BZ9y-Y5s.js} +9 -9
- package/dist/src/{nova-reel-D9xfaMBs.cjs → nova-reel-CE5etkv9.cjs} +9 -9
- package/dist/src/{nova-reel-D2ZkOSyr.js → nova-reel-DEeQlnOJ.js} +10 -10
- package/dist/src/{nova-reel-BgS1ZWuK.js → nova-reel-Xw1SXLpg.js} +9 -9
- package/dist/src/{nova-sonic-Q3BOJeig.js → nova-sonic-DWswpN1E.js} +7 -7
- package/dist/src/{nova-sonic-DezhVUYT.js → nova-sonic-DXTLpi-r.js} +6 -6
- package/dist/src/{nova-sonic-DVu3mMIy.cjs → nova-sonic-N0yCm0vb.cjs} +6 -6
- package/dist/src/{nova-sonic-P-CdUMlV.js → nova-sonic-Ogqf-csn.js} +6 -6
- package/dist/src/{openai-DhbB7eWK.js → openai-BMcwgD5C.js} +2 -2
- package/dist/src/{openai-j-sE2O7r.js → openai-BcB5KlTk.js} +2 -2
- package/dist/src/{openai-Cuif0GEt.cjs → openai-CoxGAQwn.cjs} +2 -2
- package/dist/src/{openai-DElQ-fPX.js → openai-D6wITiVn.js} +2 -2
- package/dist/src/{openclaw-Bv1DINsX.js → openclaw-0Sv7AK3O.js} +172 -109
- package/dist/src/{openclaw-DAfWQn-o.cjs → openclaw-CXxbKgDH.cjs} +174 -110
- package/dist/src/{openclaw-BiSZPL7J.js → openclaw-D1FSCps-.js} +172 -109
- package/dist/src/{openclaw-D1D_ej1z.js → openclaw-D2ENvu7a.js} +173 -110
- package/dist/src/{opencode-sdk-D95s6SnR.js → opencode-sdk-C71Z0ehR.js} +13 -13
- package/dist/src/{opencode-sdk-DxUPkLT7.js → opencode-sdk-CHCs7dEb.js} +12 -12
- package/dist/src/{opencode-sdk-C7m-wRfI.js → opencode-sdk-DDxj4QqH.js} +12 -12
- package/dist/src/{opencode-sdk-CfaLN8PY.cjs → opencode-sdk-WWJhnbKr.cjs} +16 -16
- package/dist/src/{otlpReceiver-g3ByGaXs.js → otlpReceiver-C9KlUtxh.js} +6 -6
- package/dist/src/{otlpReceiver--AIRW_S4.js → otlpReceiver-CZL48YfC.js} +6 -6
- package/dist/src/{otlpReceiver-Bn5wGB1v.js → otlpReceiver-CavGAA6k.js} +6 -6
- package/dist/src/{otlpReceiver-Diec4cln.cjs → otlpReceiver-DHKqJlsz.cjs} +6 -6
- package/dist/src/{providerRegistry-B0RUOLI_.js → providerRegistry-B9lh-_tx.js} +2 -2
- package/dist/src/{providerRegistry-Civky8Ar.cjs → providerRegistry-BTDgfV5h.cjs} +2 -2
- package/dist/src/{providerRegistry-CD8MEar9.js → providerRegistry-BkzVH5Ba.js} +2 -2
- package/dist/src/{providerRegistry-DM8rZYol.js → providerRegistry-CUWki5mQ.js} +2 -2
- package/dist/src/providers-BSLEaIQG.js +32 -0
- package/dist/src/{providers-CFu-TZl-.cjs → providers-CScd1wN6.cjs} +733 -464
- package/dist/src/{providers-CFLy1_ji.js → providers-Ch6Mr0gn.js} +795 -526
- package/dist/src/{providers-BKRJTjBz.js → providers-Cn73d5sr.js} +795 -526
- package/dist/src/providers-D-FnDg8k.cjs +31 -0
- package/dist/src/providers-DEYiFVAo.js +30 -0
- package/dist/src/{providers-B3HvufyI.js → providers-DvddrgxL.js} +795 -526
- package/dist/src/providers-sS2WI8YD.js +30 -0
- package/dist/src/{pythonUtils-D6fwaDSg.js → pythonUtils-Bzwbgpbg.js} +3 -3
- package/dist/src/{pythonUtils-D5nxkQ0P.js → pythonUtils-Cpo0Ez1p.js} +3 -3
- package/dist/src/{pythonUtils-CTU3Y3lw.cjs → pythonUtils-dAVigVK-.cjs} +3 -3
- package/dist/src/{pythonUtils-C3py6GC1.js → pythonUtils-wIqk7zAf.js} +3 -3
- package/dist/src/{quiverai-CI6gYJVI.js → quiverai-BeofbLVc.js} +4 -4
- package/dist/src/{quiverai-MHSxbmmZ.js → quiverai-CCQn73lq.js} +5 -5
- package/dist/src/{quiverai-CLkWkyZc.cjs → quiverai-CcUhPIBg.cjs} +4 -4
- package/dist/src/{quiverai-C2jVwbH1.js → quiverai-DVSEqJiq.js} +4 -4
- package/dist/src/{render-Drod8m7K.js → render-BHl6QVq9.js} +3 -3
- package/dist/src/{responses-CGw0DCzh.js → responses-BKP_WYis.js} +16 -12
- package/dist/src/{responses-BKqJmhhc.js → responses-CQb1Tj69.js} +16 -12
- package/dist/src/{responses-jxdehPkC.js → responses-CgNyTPsY.js} +16 -12
- package/dist/src/{responses-tD4Bd4dc.cjs → responses-mo0KQDbu.cjs} +16 -12
- package/dist/src/rubyUtils-B1HXG4ej.cjs +4 -0
- package/dist/src/{rubyUtils-DhCAlxZr.cjs → rubyUtils-CGeUtCfW.cjs} +3 -3
- package/dist/src/{rubyUtils-Boc4HZzX.js → rubyUtils-CiVfln3g.js} +3 -3
- package/dist/src/{rubyUtils-BcuGX77l.js → rubyUtils-DECSbsfY.js} +3 -3
- package/dist/src/{rubyUtils-BUVePouc.js → rubyUtils-PgU-gHmx.js} +3 -3
- package/dist/src/rubyUtils-Rt6pKA96.js +5 -0
- package/dist/src/{sagemaker-BK4Zb993.js → sagemaker-CVv8W7so.js} +17 -17
- package/dist/src/{sagemaker-D2Q1c-sD.js → sagemaker-CqeASYE5.js} +17 -17
- package/dist/src/{sagemaker-BfiWTmvn.js → sagemaker-MUbD5V3v.js} +18 -18
- package/dist/src/{sagemaker-CcQHM1jV.cjs → sagemaker-jiw1wQa-.cjs} +17 -17
- package/dist/src/{scanner-J8CA3LsV.js → scanner-DVDeUz1r.js} +10 -10
- package/dist/src/server/index.js +864 -112
- package/dist/src/server-B0Xh1Gx-.js +7 -0
- package/dist/src/{server-B0PPuDw-.cjs → server-BtoCXeXI.cjs} +4 -4
- package/dist/src/{server-BC7XJFgr.js → server-CP9qKM40.js} +4 -4
- package/dist/src/{server-OAs3nBRT.js → server-Cns05F1j.js} +5 -5
- package/dist/src/server-DJTKu9IR.cjs +5 -0
- package/dist/src/{server-DbFphssR.js → server-DZ9MtCn0.js} +6 -6
- package/dist/src/{signal-BOTbd53Z.js → signal-C3ZTsUgi.js} +3 -3
- package/dist/src/{slack-DXMKtA-f.js → slack-2sdpGzbt.js} +2 -2
- package/dist/src/{slack-BmVAVGaK.cjs → slack-94iG3T0s.cjs} +2 -2
- package/dist/src/{slack-DCUPTzS2.js → slack-BR0HtO3K.js} +2 -2
- package/dist/src/{slack-DOdy_kyv.js → slack-DCEV-vWP.js} +2 -2
- package/dist/src/store-C5u6MgC8.js +6 -0
- package/dist/src/{store-BSc-TF2w.cjs → store-CLyU7AtI.cjs} +17 -5
- package/dist/src/store-CNHk-De4.cjs +5 -0
- package/dist/src/{store-DQLEjuEO.js → store-Cj258DgL.js} +17 -5
- package/dist/src/{store-D1tv90v3.js → store-P8OKm19S.js} +17 -5
- package/dist/src/{store-Ub2vaGJ1.js → store-VB0GP46K.js} +17 -5
- package/dist/src/{tables-xKANLRBD.js → tables-BEIFz2tM.js} +3 -3
- package/dist/src/{tables-C7K-XKWp.cjs → tables-BdZQEpRz.cjs} +3 -3
- package/dist/src/{tables-D36WTqKX.js → tables-DmzvLbeZ.js} +3 -3
- package/dist/src/{tables-5EvT_Bwn.js → tables-kC7R5kiK.js} +3 -3
- package/dist/src/{telemetry-C2YDkUQH.js → telemetry-BnH5VJAU.js} +4 -4
- package/dist/src/{telemetry-C15ziL8u.js → telemetry-BugWqKiu.js} +4 -4
- package/dist/src/{telemetry-DMb2Mpfm.js → telemetry-DPXLd7UE.js} +4 -4
- package/dist/src/telemetry-Yig0Tino.js +7 -0
- package/dist/src/telemetry-p8Pwqm1i.cjs +5 -0
- package/dist/src/{telemetry-CbrnxHp_.cjs → telemetry-re627Lre.cjs} +4 -4
- package/dist/src/{transcription-CL78qbOU.cjs → transcription-BvtsrzRG.cjs} +13 -13
- package/dist/src/{transcription-DAtxHhAM.js → transcription-CaMivnjG.js} +13 -13
- package/dist/src/{transcription-QHh3AH6Z.js → transcription-DOMMTu01.js} +14 -14
- package/dist/src/{transcription-LNZTNUUL.js → transcription-Hb3VnC4M.js} +13 -13
- package/dist/src/{transform-DOcQeLld.cjs → transform-0BwoBsvO.cjs} +19 -5
- package/dist/src/{transform-DGxXocjk.js → transform-B2-jIv68.js} +8 -6
- package/dist/src/{transform-DECvGmzp.js → transform-BqPkNPYm.js} +4 -4
- package/dist/src/{transform-aa6tmVpZ.js → transform-BzK09Q_9.js} +4 -4
- package/dist/src/transform-ChNIpHz7.js +6 -0
- package/dist/src/{transform-Cgi24fJ7.js → transform-DrleutM3.js} +8 -6
- package/dist/src/{transform-DGLazrMm.js → transform-DyDAwEpE.js} +8 -6
- package/dist/src/transform-PtQ6rAE3.cjs +5 -0
- package/dist/src/{transform-CzK1Q0zl.cjs → transform-ZrG2dvlo.cjs} +4 -4
- package/dist/src/{transform-DilY9wbS.js → transform-ljLYHEPh.js} +4 -4
- package/dist/src/{transformersAvailability-CEVM2GNQ.js → transformersAvailability-BGkzavwb.js} +1 -1
- package/dist/src/{transformersAvailability-CwayUSlh.cjs → transformersAvailability-DKoRtQLy.cjs} +1 -1
- package/dist/src/{types-CH3Ge2sE.js → types-CIhFeUC4.js} +45 -11
- package/dist/src/{types-CN_TZ2GJ.js → types-Cd3ygw8W.js} +45 -11
- package/dist/src/{types-LJ0r3wbR.cjs → types-D8cGDZbL.cjs} +46 -12
- package/dist/src/{types-CLKiCBW3.js → types-q8GXGF65.js} +45 -11
- package/dist/src/{util-CchiqXh_.cjs → util--9u9UVCt.cjs} +3 -3
- package/dist/src/{util-5cB-L7U3.js → util-BLvy9qfE.js} +7 -11
- package/dist/src/{util-YT5HPZaS.js → util-Bm3E9jpK.js} +7 -11
- package/dist/src/{util-6-GqIvzS.js → util-BtoGs5Cb.js} +18 -4
- package/dist/src/{util-Db0a0AFH.cjs → util-CFj4YKIn.cjs} +18 -4
- package/dist/src/{util-Dlz_Wvgm.js → util-CMMkIxfU.js} +7 -11
- package/dist/src/{util-Betm42rL.js → util-CgDCK4KI.js} +18 -4
- package/dist/src/{util-Yz-1aEhW.cjs → util-CuLo2pMR.cjs} +7 -11
- package/dist/src/{util-C-PPYSMq.js → util-DM2rTn_6.js} +18 -4
- package/dist/src/{util-B7T3SiBS.js → util-DMFeUvLz.js} +3 -3
- package/dist/src/{util-ZZH-3QZz.js → util-DbVG-yZU.js} +3 -3
- package/dist/src/{util-DaWTWKBK.js → util-vNmDL5DT.js} +3 -3
- package/dist/src/{utils-XiOAgly5.js → utils-CFxO9KGo.js} +2 -2
- package/dist/src/{utils-f2-Moju7.js → utils-DEuL4VNB.js} +2 -2
- package/dist/src/{utils-Cz9qXqII.cjs → utils-DKw8mrgr.cjs} +3 -3
- package/dist/src/{utils-dLokC-eR.js → utils-DOjD4dTC.js} +2 -2
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +38 -38
- package/dist/src/app/assets/index-BFCZg7hQ.js +0 -439
- package/dist/src/app/assets/index-NCn4eVBv.css +0 -1
- package/dist/src/app/assets/sync-9qqYcY-B.js +0 -4
- package/dist/src/app/assets/vendor-charts-CCl15Imd.js +0 -36
- package/dist/src/app/assets/vendor-markdown-0tekx3KX.js +0 -29
- package/dist/src/cache-Bbn1Nyrd.cjs +0 -5
- package/dist/src/cache-BwsMSda7.js +0 -6
- package/dist/src/cloud-DmE0EwsY.js +0 -4
- package/dist/src/eval-17JizQIv.js +0 -15
- package/dist/src/evalResult-Cqj8pldJ.js +0 -12
- package/dist/src/evalResult-DvcJAWJU.cjs +0 -10
- package/dist/src/evalResult-Hftn-S_i.js +0 -10
- package/dist/src/evaluator-B2CFNt-P.js +0 -36
- package/dist/src/fetch-KV5kNASw.js +0 -5
- package/dist/src/graders-Bu0H9nXi.js +0 -32
- package/dist/src/graders-Cfhkvx-e.js +0 -34
- package/dist/src/graders-DClJVpGP.cjs +0 -32
- package/dist/src/graders-DcnJsrMO.js +0 -32
- package/dist/src/providers-C1rOSHiR.js +0 -32
- package/dist/src/providers-CxmDwEFf.cjs +0 -31
- package/dist/src/providers-Dodakqr0.js +0 -30
- package/dist/src/providers-GIQ2TcsA.js +0 -30
- package/dist/src/rubyUtils-BUHu6PhO.js +0 -5
- package/dist/src/rubyUtils-CP42kMvq.cjs +0 -4
- package/dist/src/server-B1vi21hA.js +0 -7
- package/dist/src/server-Cm9Kai_h.cjs +0 -5
- package/dist/src/store-BNmZ1KAz.cjs +0 -5
- package/dist/src/store-BltJg2cd.js +0 -6
- package/dist/src/telemetry-5BCRNBbe.cjs +0 -5
- package/dist/src/telemetry-D4W5hboe.js +0 -7
- package/dist/src/transform-DTGDnAzW.js +0 -6
- package/dist/src/transform-m3qNw4KP.cjs +0 -5
package/dist/src/index.d.cts
CHANGED
|
@@ -88,7 +88,12 @@ declare const ProviderEnvOverridesSchema: z.ZodObject<{
|
|
|
88
88
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
89
89
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
90
90
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
91
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
92
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
93
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
91
94
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
95
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
96
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
92
97
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
93
98
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
94
99
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -237,6 +242,13 @@ interface ChatMessage {
|
|
|
237
242
|
role: 'system' | 'user' | 'assistant' | 'tool' | 'function';
|
|
238
243
|
content: string;
|
|
239
244
|
}
|
|
245
|
+
interface SkillCallEntry {
|
|
246
|
+
name: string;
|
|
247
|
+
input?: unknown;
|
|
248
|
+
path?: string;
|
|
249
|
+
source?: 'heuristic' | 'tool';
|
|
250
|
+
is_error?: boolean;
|
|
251
|
+
}
|
|
240
252
|
type ProviderTypeMap = Partial<Record<ProviderType, string | ProviderOptions | ApiProvider>>;
|
|
241
253
|
interface AtomicTestCase$1 {
|
|
242
254
|
description?: string;
|
|
@@ -574,7 +586,12 @@ declare const ProvidersSchema: z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Cal
|
|
|
574
586
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
575
587
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
576
588
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
589
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
590
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
591
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
577
592
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
593
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
594
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
578
595
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
579
596
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
580
597
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -701,7 +718,12 @@ declare const ProvidersSchema: z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Cal
|
|
|
701
718
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
702
719
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
703
720
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
721
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
722
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
723
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
704
724
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
725
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
726
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
705
727
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
706
728
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
707
729
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -1646,6 +1668,12 @@ declare const BaseAssertionTypesSchema: z.ZodEnum<{
|
|
|
1646
1668
|
"similar:euclidean": "similar:euclidean";
|
|
1647
1669
|
"starts-with": "starts-with";
|
|
1648
1670
|
"tool-call-f1": "tool-call-f1";
|
|
1671
|
+
"skill-used": "skill-used";
|
|
1672
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
1673
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
1674
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
1675
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
1676
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
1649
1677
|
"trace-error-spans": "trace-error-spans";
|
|
1650
1678
|
"trace-span-count": "trace-span-count";
|
|
1651
1679
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -1714,13 +1742,19 @@ declare const NotPrefixedAssertionTypesSchema: z.ZodPipe<z.ZodEnum<{
|
|
|
1714
1742
|
"similar:euclidean": "similar:euclidean";
|
|
1715
1743
|
"starts-with": "starts-with";
|
|
1716
1744
|
"tool-call-f1": "tool-call-f1";
|
|
1745
|
+
"skill-used": "skill-used";
|
|
1746
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
1747
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
1748
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
1749
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
1750
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
1717
1751
|
"trace-error-spans": "trace-error-spans";
|
|
1718
1752
|
"trace-span-count": "trace-span-count";
|
|
1719
1753
|
"trace-span-duration": "trace-span-duration";
|
|
1720
1754
|
"search-rubric": "search-rubric";
|
|
1721
1755
|
webhook: "webhook";
|
|
1722
1756
|
"word-count": "word-count";
|
|
1723
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>;
|
|
1757
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>;
|
|
1724
1758
|
declare const AssertionTypeSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
1725
1759
|
"answer-relevance": "answer-relevance";
|
|
1726
1760
|
bleu: "bleu";
|
|
@@ -1775,6 +1809,12 @@ declare const AssertionTypeSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
|
1775
1809
|
"similar:euclidean": "similar:euclidean";
|
|
1776
1810
|
"starts-with": "starts-with";
|
|
1777
1811
|
"tool-call-f1": "tool-call-f1";
|
|
1812
|
+
"skill-used": "skill-used";
|
|
1813
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
1814
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
1815
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
1816
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
1817
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
1778
1818
|
"trace-error-spans": "trace-error-spans";
|
|
1779
1819
|
"trace-span-count": "trace-span-count";
|
|
1780
1820
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -1835,13 +1875,19 @@ declare const AssertionTypeSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
|
1835
1875
|
"similar:euclidean": "similar:euclidean";
|
|
1836
1876
|
"starts-with": "starts-with";
|
|
1837
1877
|
"tool-call-f1": "tool-call-f1";
|
|
1878
|
+
"skill-used": "skill-used";
|
|
1879
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
1880
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
1881
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
1882
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
1883
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
1838
1884
|
"trace-error-spans": "trace-error-spans";
|
|
1839
1885
|
"trace-span-count": "trace-span-count";
|
|
1840
1886
|
"trace-span-duration": "trace-span-duration";
|
|
1841
1887
|
"search-rubric": "search-rubric";
|
|
1842
1888
|
webhook: "webhook";
|
|
1843
1889
|
"word-count": "word-count";
|
|
1844
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
1890
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
1845
1891
|
"select-best": "select-best";
|
|
1846
1892
|
human: "human";
|
|
1847
1893
|
"max-score": "max-score";
|
|
@@ -1904,6 +1950,12 @@ declare const AssertionSetSchema: z.ZodObject<{
|
|
|
1904
1950
|
"similar:euclidean": "similar:euclidean";
|
|
1905
1951
|
"starts-with": "starts-with";
|
|
1906
1952
|
"tool-call-f1": "tool-call-f1";
|
|
1953
|
+
"skill-used": "skill-used";
|
|
1954
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
1955
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
1956
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
1957
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
1958
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
1907
1959
|
"trace-error-spans": "trace-error-spans";
|
|
1908
1960
|
"trace-span-count": "trace-span-count";
|
|
1909
1961
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -1964,13 +2016,19 @@ declare const AssertionSetSchema: z.ZodObject<{
|
|
|
1964
2016
|
"similar:euclidean": "similar:euclidean";
|
|
1965
2017
|
"starts-with": "starts-with";
|
|
1966
2018
|
"tool-call-f1": "tool-call-f1";
|
|
2019
|
+
"skill-used": "skill-used";
|
|
2020
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2021
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2022
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2023
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2024
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
1967
2025
|
"trace-error-spans": "trace-error-spans";
|
|
1968
2026
|
"trace-span-count": "trace-span-count";
|
|
1969
2027
|
"trace-span-duration": "trace-span-duration";
|
|
1970
2028
|
"search-rubric": "search-rubric";
|
|
1971
2029
|
webhook: "webhook";
|
|
1972
2030
|
"word-count": "word-count";
|
|
1973
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2031
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
1974
2032
|
"select-best": "select-best";
|
|
1975
2033
|
human: "human";
|
|
1976
2034
|
"max-score": "max-score";
|
|
@@ -2052,6 +2110,12 @@ declare const AssertionSchema: z.ZodObject<{
|
|
|
2052
2110
|
"similar:euclidean": "similar:euclidean";
|
|
2053
2111
|
"starts-with": "starts-with";
|
|
2054
2112
|
"tool-call-f1": "tool-call-f1";
|
|
2113
|
+
"skill-used": "skill-used";
|
|
2114
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2115
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2116
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2117
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2118
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2055
2119
|
"trace-error-spans": "trace-error-spans";
|
|
2056
2120
|
"trace-span-count": "trace-span-count";
|
|
2057
2121
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -2112,13 +2176,19 @@ declare const AssertionSchema: z.ZodObject<{
|
|
|
2112
2176
|
"similar:euclidean": "similar:euclidean";
|
|
2113
2177
|
"starts-with": "starts-with";
|
|
2114
2178
|
"tool-call-f1": "tool-call-f1";
|
|
2179
|
+
"skill-used": "skill-used";
|
|
2180
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2181
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2182
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2183
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2184
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2115
2185
|
"trace-error-spans": "trace-error-spans";
|
|
2116
2186
|
"trace-span-count": "trace-span-count";
|
|
2117
2187
|
"trace-span-duration": "trace-span-duration";
|
|
2118
2188
|
"search-rubric": "search-rubric";
|
|
2119
2189
|
webhook: "webhook";
|
|
2120
2190
|
"word-count": "word-count";
|
|
2121
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2191
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2122
2192
|
"select-best": "select-best";
|
|
2123
2193
|
human: "human";
|
|
2124
2194
|
"max-score": "max-score";
|
|
@@ -2201,6 +2271,12 @@ declare const AssertionOrSetSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
2201
2271
|
"similar:euclidean": "similar:euclidean";
|
|
2202
2272
|
"starts-with": "starts-with";
|
|
2203
2273
|
"tool-call-f1": "tool-call-f1";
|
|
2274
|
+
"skill-used": "skill-used";
|
|
2275
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2276
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2277
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2278
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2279
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2204
2280
|
"trace-error-spans": "trace-error-spans";
|
|
2205
2281
|
"trace-span-count": "trace-span-count";
|
|
2206
2282
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -2261,13 +2337,19 @@ declare const AssertionOrSetSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
2261
2337
|
"similar:euclidean": "similar:euclidean";
|
|
2262
2338
|
"starts-with": "starts-with";
|
|
2263
2339
|
"tool-call-f1": "tool-call-f1";
|
|
2340
|
+
"skill-used": "skill-used";
|
|
2341
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2342
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2343
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2344
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2345
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2264
2346
|
"trace-error-spans": "trace-error-spans";
|
|
2265
2347
|
"trace-span-count": "trace-span-count";
|
|
2266
2348
|
"trace-span-duration": "trace-span-duration";
|
|
2267
2349
|
"search-rubric": "search-rubric";
|
|
2268
2350
|
webhook: "webhook";
|
|
2269
2351
|
"word-count": "word-count";
|
|
2270
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2352
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2271
2353
|
"select-best": "select-best";
|
|
2272
2354
|
human: "human";
|
|
2273
2355
|
"max-score": "max-score";
|
|
@@ -2347,6 +2429,12 @@ declare const AssertionOrSetSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
2347
2429
|
"similar:euclidean": "similar:euclidean";
|
|
2348
2430
|
"starts-with": "starts-with";
|
|
2349
2431
|
"tool-call-f1": "tool-call-f1";
|
|
2432
|
+
"skill-used": "skill-used";
|
|
2433
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2434
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2435
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2436
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2437
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2350
2438
|
"trace-error-spans": "trace-error-spans";
|
|
2351
2439
|
"trace-span-count": "trace-span-count";
|
|
2352
2440
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -2407,13 +2495,19 @@ declare const AssertionOrSetSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
2407
2495
|
"similar:euclidean": "similar:euclidean";
|
|
2408
2496
|
"starts-with": "starts-with";
|
|
2409
2497
|
"tool-call-f1": "tool-call-f1";
|
|
2498
|
+
"skill-used": "skill-used";
|
|
2499
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2500
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2501
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2502
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2503
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2410
2504
|
"trace-error-spans": "trace-error-spans";
|
|
2411
2505
|
"trace-span-count": "trace-span-count";
|
|
2412
2506
|
"trace-span-duration": "trace-span-duration";
|
|
2413
2507
|
"search-rubric": "search-rubric";
|
|
2414
2508
|
webhook: "webhook";
|
|
2415
2509
|
"word-count": "word-count";
|
|
2416
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2510
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2417
2511
|
"select-best": "select-best";
|
|
2418
2512
|
human: "human";
|
|
2419
2513
|
"max-score": "max-score";
|
|
@@ -2621,7 +2715,12 @@ declare const TestCaseSchema: z.ZodObject<{
|
|
|
2621
2715
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
2622
2716
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
2623
2717
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
2718
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
2719
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
2720
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
2624
2721
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
2722
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
2723
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
2625
2724
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
2626
2725
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
2627
2726
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -2740,6 +2839,12 @@ declare const TestCaseSchema: z.ZodObject<{
|
|
|
2740
2839
|
"similar:euclidean": "similar:euclidean";
|
|
2741
2840
|
"starts-with": "starts-with";
|
|
2742
2841
|
"tool-call-f1": "tool-call-f1";
|
|
2842
|
+
"skill-used": "skill-used";
|
|
2843
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2844
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2845
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2846
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2847
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2743
2848
|
"trace-error-spans": "trace-error-spans";
|
|
2744
2849
|
"trace-span-count": "trace-span-count";
|
|
2745
2850
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -2800,13 +2905,19 @@ declare const TestCaseSchema: z.ZodObject<{
|
|
|
2800
2905
|
"similar:euclidean": "similar:euclidean";
|
|
2801
2906
|
"starts-with": "starts-with";
|
|
2802
2907
|
"tool-call-f1": "tool-call-f1";
|
|
2908
|
+
"skill-used": "skill-used";
|
|
2909
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
2910
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
2911
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
2912
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
2913
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2803
2914
|
"trace-error-spans": "trace-error-spans";
|
|
2804
2915
|
"trace-span-count": "trace-span-count";
|
|
2805
2916
|
"trace-span-duration": "trace-span-duration";
|
|
2806
2917
|
"search-rubric": "search-rubric";
|
|
2807
2918
|
webhook: "webhook";
|
|
2808
2919
|
"word-count": "word-count";
|
|
2809
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2920
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2810
2921
|
"select-best": "select-best";
|
|
2811
2922
|
human: "human";
|
|
2812
2923
|
"max-score": "max-score";
|
|
@@ -2886,6 +2997,12 @@ declare const TestCaseSchema: z.ZodObject<{
|
|
|
2886
2997
|
"similar:euclidean": "similar:euclidean";
|
|
2887
2998
|
"starts-with": "starts-with";
|
|
2888
2999
|
"tool-call-f1": "tool-call-f1";
|
|
3000
|
+
"skill-used": "skill-used";
|
|
3001
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3002
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3003
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3004
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3005
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2889
3006
|
"trace-error-spans": "trace-error-spans";
|
|
2890
3007
|
"trace-span-count": "trace-span-count";
|
|
2891
3008
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -2946,13 +3063,19 @@ declare const TestCaseSchema: z.ZodObject<{
|
|
|
2946
3063
|
"similar:euclidean": "similar:euclidean";
|
|
2947
3064
|
"starts-with": "starts-with";
|
|
2948
3065
|
"tool-call-f1": "tool-call-f1";
|
|
3066
|
+
"skill-used": "skill-used";
|
|
3067
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3068
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3069
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3070
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3071
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
2949
3072
|
"trace-error-spans": "trace-error-spans";
|
|
2950
3073
|
"trace-span-count": "trace-span-count";
|
|
2951
3074
|
"trace-span-duration": "trace-span-duration";
|
|
2952
3075
|
"search-rubric": "search-rubric";
|
|
2953
3076
|
webhook: "webhook";
|
|
2954
3077
|
"word-count": "word-count";
|
|
2955
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
3078
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
2956
3079
|
"select-best": "select-best";
|
|
2957
3080
|
human: "human";
|
|
2958
3081
|
"max-score": "max-score";
|
|
@@ -3166,7 +3289,12 @@ declare const TestCaseWithVarsFileSchema: z.ZodObject<{
|
|
|
3166
3289
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
3167
3290
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
3168
3291
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
3292
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
3293
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
3294
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
3169
3295
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
3296
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
3297
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
3170
3298
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
3171
3299
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
3172
3300
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -3285,6 +3413,12 @@ declare const TestCaseWithVarsFileSchema: z.ZodObject<{
|
|
|
3285
3413
|
"similar:euclidean": "similar:euclidean";
|
|
3286
3414
|
"starts-with": "starts-with";
|
|
3287
3415
|
"tool-call-f1": "tool-call-f1";
|
|
3416
|
+
"skill-used": "skill-used";
|
|
3417
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3418
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3419
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3420
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3421
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3288
3422
|
"trace-error-spans": "trace-error-spans";
|
|
3289
3423
|
"trace-span-count": "trace-span-count";
|
|
3290
3424
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -3345,13 +3479,19 @@ declare const TestCaseWithVarsFileSchema: z.ZodObject<{
|
|
|
3345
3479
|
"similar:euclidean": "similar:euclidean";
|
|
3346
3480
|
"starts-with": "starts-with";
|
|
3347
3481
|
"tool-call-f1": "tool-call-f1";
|
|
3482
|
+
"skill-used": "skill-used";
|
|
3483
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3484
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3485
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3486
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3487
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3348
3488
|
"trace-error-spans": "trace-error-spans";
|
|
3349
3489
|
"trace-span-count": "trace-span-count";
|
|
3350
3490
|
"trace-span-duration": "trace-span-duration";
|
|
3351
3491
|
"search-rubric": "search-rubric";
|
|
3352
3492
|
webhook: "webhook";
|
|
3353
3493
|
"word-count": "word-count";
|
|
3354
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
3494
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
3355
3495
|
"select-best": "select-best";
|
|
3356
3496
|
human: "human";
|
|
3357
3497
|
"max-score": "max-score";
|
|
@@ -3431,6 +3571,12 @@ declare const TestCaseWithVarsFileSchema: z.ZodObject<{
|
|
|
3431
3571
|
"similar:euclidean": "similar:euclidean";
|
|
3432
3572
|
"starts-with": "starts-with";
|
|
3433
3573
|
"tool-call-f1": "tool-call-f1";
|
|
3574
|
+
"skill-used": "skill-used";
|
|
3575
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3576
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3577
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3578
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3579
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3434
3580
|
"trace-error-spans": "trace-error-spans";
|
|
3435
3581
|
"trace-span-count": "trace-span-count";
|
|
3436
3582
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -3491,13 +3637,19 @@ declare const TestCaseWithVarsFileSchema: z.ZodObject<{
|
|
|
3491
3637
|
"similar:euclidean": "similar:euclidean";
|
|
3492
3638
|
"starts-with": "starts-with";
|
|
3493
3639
|
"tool-call-f1": "tool-call-f1";
|
|
3640
|
+
"skill-used": "skill-used";
|
|
3641
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3642
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3643
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3644
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3645
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3494
3646
|
"trace-error-spans": "trace-error-spans";
|
|
3495
3647
|
"trace-span-count": "trace-span-count";
|
|
3496
3648
|
"trace-span-duration": "trace-span-duration";
|
|
3497
3649
|
"search-rubric": "search-rubric";
|
|
3498
3650
|
webhook: "webhook";
|
|
3499
3651
|
"word-count": "word-count";
|
|
3500
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
3652
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
3501
3653
|
"select-best": "select-best";
|
|
3502
3654
|
human: "human";
|
|
3503
3655
|
"max-score": "max-score";
|
|
@@ -3710,7 +3862,12 @@ declare const TestCasesWithMetadataSchema: z.ZodObject<{
|
|
|
3710
3862
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
3711
3863
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
3712
3864
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
3865
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
3866
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
3867
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
3713
3868
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
3869
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
3870
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
3714
3871
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
3715
3872
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
3716
3873
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -3829,6 +3986,12 @@ declare const TestCasesWithMetadataSchema: z.ZodObject<{
|
|
|
3829
3986
|
"similar:euclidean": "similar:euclidean";
|
|
3830
3987
|
"starts-with": "starts-with";
|
|
3831
3988
|
"tool-call-f1": "tool-call-f1";
|
|
3989
|
+
"skill-used": "skill-used";
|
|
3990
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
3991
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
3992
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
3993
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
3994
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3832
3995
|
"trace-error-spans": "trace-error-spans";
|
|
3833
3996
|
"trace-span-count": "trace-span-count";
|
|
3834
3997
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -3889,13 +4052,19 @@ declare const TestCasesWithMetadataSchema: z.ZodObject<{
|
|
|
3889
4052
|
"similar:euclidean": "similar:euclidean";
|
|
3890
4053
|
"starts-with": "starts-with";
|
|
3891
4054
|
"tool-call-f1": "tool-call-f1";
|
|
4055
|
+
"skill-used": "skill-used";
|
|
4056
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4057
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4058
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4059
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4060
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3892
4061
|
"trace-error-spans": "trace-error-spans";
|
|
3893
4062
|
"trace-span-count": "trace-span-count";
|
|
3894
4063
|
"trace-span-duration": "trace-span-duration";
|
|
3895
4064
|
"search-rubric": "search-rubric";
|
|
3896
4065
|
webhook: "webhook";
|
|
3897
4066
|
"word-count": "word-count";
|
|
3898
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4067
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
3899
4068
|
"select-best": "select-best";
|
|
3900
4069
|
human: "human";
|
|
3901
4070
|
"max-score": "max-score";
|
|
@@ -3975,6 +4144,12 @@ declare const TestCasesWithMetadataSchema: z.ZodObject<{
|
|
|
3975
4144
|
"similar:euclidean": "similar:euclidean";
|
|
3976
4145
|
"starts-with": "starts-with";
|
|
3977
4146
|
"tool-call-f1": "tool-call-f1";
|
|
4147
|
+
"skill-used": "skill-used";
|
|
4148
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4149
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4150
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4151
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4152
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
3978
4153
|
"trace-error-spans": "trace-error-spans";
|
|
3979
4154
|
"trace-span-count": "trace-span-count";
|
|
3980
4155
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -4035,13 +4210,19 @@ declare const TestCasesWithMetadataSchema: z.ZodObject<{
|
|
|
4035
4210
|
"similar:euclidean": "similar:euclidean";
|
|
4036
4211
|
"starts-with": "starts-with";
|
|
4037
4212
|
"tool-call-f1": "tool-call-f1";
|
|
4213
|
+
"skill-used": "skill-used";
|
|
4214
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4215
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4216
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4217
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4218
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
4038
4219
|
"trace-error-spans": "trace-error-spans";
|
|
4039
4220
|
"trace-span-count": "trace-span-count";
|
|
4040
4221
|
"trace-span-duration": "trace-span-duration";
|
|
4041
4222
|
"search-rubric": "search-rubric";
|
|
4042
4223
|
webhook: "webhook";
|
|
4043
4224
|
"word-count": "word-count";
|
|
4044
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4225
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4045
4226
|
"select-best": "select-best";
|
|
4046
4227
|
human: "human";
|
|
4047
4228
|
"max-score": "max-score";
|
|
@@ -4312,7 +4493,12 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4312
4493
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
4313
4494
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
4314
4495
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
4496
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
4497
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
4498
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
4315
4499
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
4500
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
4501
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
4316
4502
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
4317
4503
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
4318
4504
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -4431,6 +4617,12 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4431
4617
|
"similar:euclidean": "similar:euclidean";
|
|
4432
4618
|
"starts-with": "starts-with";
|
|
4433
4619
|
"tool-call-f1": "tool-call-f1";
|
|
4620
|
+
"skill-used": "skill-used";
|
|
4621
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4622
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4623
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4624
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4625
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
4434
4626
|
"trace-error-spans": "trace-error-spans";
|
|
4435
4627
|
"trace-span-count": "trace-span-count";
|
|
4436
4628
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -4491,13 +4683,19 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4491
4683
|
"similar:euclidean": "similar:euclidean";
|
|
4492
4684
|
"starts-with": "starts-with";
|
|
4493
4685
|
"tool-call-f1": "tool-call-f1";
|
|
4686
|
+
"skill-used": "skill-used";
|
|
4687
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4688
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4689
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4690
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4691
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
4494
4692
|
"trace-error-spans": "trace-error-spans";
|
|
4495
4693
|
"trace-span-count": "trace-span-count";
|
|
4496
4694
|
"trace-span-duration": "trace-span-duration";
|
|
4497
4695
|
"search-rubric": "search-rubric";
|
|
4498
4696
|
webhook: "webhook";
|
|
4499
4697
|
"word-count": "word-count";
|
|
4500
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4698
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4501
4699
|
"select-best": "select-best";
|
|
4502
4700
|
human: "human";
|
|
4503
4701
|
"max-score": "max-score";
|
|
@@ -4577,6 +4775,12 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4577
4775
|
"similar:euclidean": "similar:euclidean";
|
|
4578
4776
|
"starts-with": "starts-with";
|
|
4579
4777
|
"tool-call-f1": "tool-call-f1";
|
|
4778
|
+
"skill-used": "skill-used";
|
|
4779
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4780
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4781
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4782
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4783
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
4580
4784
|
"trace-error-spans": "trace-error-spans";
|
|
4581
4785
|
"trace-span-count": "trace-span-count";
|
|
4582
4786
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -4637,13 +4841,19 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4637
4841
|
"similar:euclidean": "similar:euclidean";
|
|
4638
4842
|
"starts-with": "starts-with";
|
|
4639
4843
|
"tool-call-f1": "tool-call-f1";
|
|
4844
|
+
"skill-used": "skill-used";
|
|
4845
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
4846
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
4847
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
4848
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
4849
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
4640
4850
|
"trace-error-spans": "trace-error-spans";
|
|
4641
4851
|
"trace-span-count": "trace-span-count";
|
|
4642
4852
|
"trace-span-duration": "trace-span-duration";
|
|
4643
4853
|
"search-rubric": "search-rubric";
|
|
4644
4854
|
webhook: "webhook";
|
|
4645
4855
|
"word-count": "word-count";
|
|
4646
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4856
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
4647
4857
|
"select-best": "select-best";
|
|
4648
4858
|
human: "human";
|
|
4649
4859
|
"max-score": "max-score";
|
|
@@ -4852,7 +5062,12 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4852
5062
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
4853
5063
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
4854
5064
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
5065
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
5066
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
5067
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
4855
5068
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
5069
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
5070
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
4856
5071
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
4857
5072
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
4858
5073
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -4971,6 +5186,12 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
4971
5186
|
"similar:euclidean": "similar:euclidean";
|
|
4972
5187
|
"starts-with": "starts-with";
|
|
4973
5188
|
"tool-call-f1": "tool-call-f1";
|
|
5189
|
+
"skill-used": "skill-used";
|
|
5190
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5191
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5192
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5193
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5194
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
4974
5195
|
"trace-error-spans": "trace-error-spans";
|
|
4975
5196
|
"trace-span-count": "trace-span-count";
|
|
4976
5197
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -5031,13 +5252,19 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
5031
5252
|
"similar:euclidean": "similar:euclidean";
|
|
5032
5253
|
"starts-with": "starts-with";
|
|
5033
5254
|
"tool-call-f1": "tool-call-f1";
|
|
5255
|
+
"skill-used": "skill-used";
|
|
5256
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5257
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5258
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5259
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5260
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5034
5261
|
"trace-error-spans": "trace-error-spans";
|
|
5035
5262
|
"trace-span-count": "trace-span-count";
|
|
5036
5263
|
"trace-span-duration": "trace-span-duration";
|
|
5037
5264
|
"search-rubric": "search-rubric";
|
|
5038
5265
|
webhook: "webhook";
|
|
5039
5266
|
"word-count": "word-count";
|
|
5040
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5267
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5041
5268
|
"select-best": "select-best";
|
|
5042
5269
|
human: "human";
|
|
5043
5270
|
"max-score": "max-score";
|
|
@@ -5117,6 +5344,12 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
5117
5344
|
"similar:euclidean": "similar:euclidean";
|
|
5118
5345
|
"starts-with": "starts-with";
|
|
5119
5346
|
"tool-call-f1": "tool-call-f1";
|
|
5347
|
+
"skill-used": "skill-used";
|
|
5348
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5349
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5350
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5351
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5352
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5120
5353
|
"trace-error-spans": "trace-error-spans";
|
|
5121
5354
|
"trace-span-count": "trace-span-count";
|
|
5122
5355
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -5177,13 +5410,19 @@ declare const ScenarioSchema: z.ZodObject<{
|
|
|
5177
5410
|
"similar:euclidean": "similar:euclidean";
|
|
5178
5411
|
"starts-with": "starts-with";
|
|
5179
5412
|
"tool-call-f1": "tool-call-f1";
|
|
5413
|
+
"skill-used": "skill-used";
|
|
5414
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5415
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5416
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5417
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5418
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5180
5419
|
"trace-error-spans": "trace-error-spans";
|
|
5181
5420
|
"trace-span-count": "trace-span-count";
|
|
5182
5421
|
"trace-span-duration": "trace-span-duration";
|
|
5183
5422
|
"search-rubric": "search-rubric";
|
|
5184
5423
|
webhook: "webhook";
|
|
5185
5424
|
"word-count": "word-count";
|
|
5186
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5425
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5187
5426
|
"select-best": "select-best";
|
|
5188
5427
|
human: "human";
|
|
5189
5428
|
"max-score": "max-score";
|
|
@@ -5393,7 +5632,12 @@ declare const AtomicTestCaseSchema: z.ZodObject<{
|
|
|
5393
5632
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
5394
5633
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
5395
5634
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
5635
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
5636
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
5637
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
5396
5638
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
5639
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
5640
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
5397
5641
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
5398
5642
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
5399
5643
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -5512,6 +5756,12 @@ declare const AtomicTestCaseSchema: z.ZodObject<{
|
|
|
5512
5756
|
"similar:euclidean": "similar:euclidean";
|
|
5513
5757
|
"starts-with": "starts-with";
|
|
5514
5758
|
"tool-call-f1": "tool-call-f1";
|
|
5759
|
+
"skill-used": "skill-used";
|
|
5760
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5761
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5762
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5763
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5764
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5515
5765
|
"trace-error-spans": "trace-error-spans";
|
|
5516
5766
|
"trace-span-count": "trace-span-count";
|
|
5517
5767
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -5572,13 +5822,19 @@ declare const AtomicTestCaseSchema: z.ZodObject<{
|
|
|
5572
5822
|
"similar:euclidean": "similar:euclidean";
|
|
5573
5823
|
"starts-with": "starts-with";
|
|
5574
5824
|
"tool-call-f1": "tool-call-f1";
|
|
5825
|
+
"skill-used": "skill-used";
|
|
5826
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5827
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5828
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5829
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5830
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5575
5831
|
"trace-error-spans": "trace-error-spans";
|
|
5576
5832
|
"trace-span-count": "trace-span-count";
|
|
5577
5833
|
"trace-span-duration": "trace-span-duration";
|
|
5578
5834
|
"search-rubric": "search-rubric";
|
|
5579
5835
|
webhook: "webhook";
|
|
5580
5836
|
"word-count": "word-count";
|
|
5581
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5837
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5582
5838
|
"select-best": "select-best";
|
|
5583
5839
|
human: "human";
|
|
5584
5840
|
"max-score": "max-score";
|
|
@@ -5658,6 +5914,12 @@ declare const AtomicTestCaseSchema: z.ZodObject<{
|
|
|
5658
5914
|
"similar:euclidean": "similar:euclidean";
|
|
5659
5915
|
"starts-with": "starts-with";
|
|
5660
5916
|
"tool-call-f1": "tool-call-f1";
|
|
5917
|
+
"skill-used": "skill-used";
|
|
5918
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5919
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5920
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5921
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5922
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5661
5923
|
"trace-error-spans": "trace-error-spans";
|
|
5662
5924
|
"trace-span-count": "trace-span-count";
|
|
5663
5925
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -5718,13 +5980,19 @@ declare const AtomicTestCaseSchema: z.ZodObject<{
|
|
|
5718
5980
|
"similar:euclidean": "similar:euclidean";
|
|
5719
5981
|
"starts-with": "starts-with";
|
|
5720
5982
|
"tool-call-f1": "tool-call-f1";
|
|
5983
|
+
"skill-used": "skill-used";
|
|
5984
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
5985
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
5986
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
5987
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
5988
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
5721
5989
|
"trace-error-spans": "trace-error-spans";
|
|
5722
5990
|
"trace-span-count": "trace-span-count";
|
|
5723
5991
|
"trace-span-duration": "trace-span-duration";
|
|
5724
5992
|
"search-rubric": "search-rubric";
|
|
5725
5993
|
webhook: "webhook";
|
|
5726
5994
|
"word-count": "word-count";
|
|
5727
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5995
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
5728
5996
|
"select-best": "select-best";
|
|
5729
5997
|
human: "human";
|
|
5730
5998
|
"max-score": "max-score";
|
|
@@ -5981,7 +6249,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
5981
6249
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
5982
6250
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
5983
6251
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
6252
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
6253
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
6254
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
5984
6255
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
6256
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
6257
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
5985
6258
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
5986
6259
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
5987
6260
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -6100,6 +6373,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6100
6373
|
"similar:euclidean": "similar:euclidean";
|
|
6101
6374
|
"starts-with": "starts-with";
|
|
6102
6375
|
"tool-call-f1": "tool-call-f1";
|
|
6376
|
+
"skill-used": "skill-used";
|
|
6377
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
6378
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
6379
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
6380
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
6381
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6103
6382
|
"trace-error-spans": "trace-error-spans";
|
|
6104
6383
|
"trace-span-count": "trace-span-count";
|
|
6105
6384
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -6160,13 +6439,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6160
6439
|
"similar:euclidean": "similar:euclidean";
|
|
6161
6440
|
"starts-with": "starts-with";
|
|
6162
6441
|
"tool-call-f1": "tool-call-f1";
|
|
6442
|
+
"skill-used": "skill-used";
|
|
6443
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
6444
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
6445
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
6446
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
6447
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6163
6448
|
"trace-error-spans": "trace-error-spans";
|
|
6164
6449
|
"trace-span-count": "trace-span-count";
|
|
6165
6450
|
"trace-span-duration": "trace-span-duration";
|
|
6166
6451
|
"search-rubric": "search-rubric";
|
|
6167
6452
|
webhook: "webhook";
|
|
6168
6453
|
"word-count": "word-count";
|
|
6169
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
6454
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
6170
6455
|
"select-best": "select-best";
|
|
6171
6456
|
human: "human";
|
|
6172
6457
|
"max-score": "max-score";
|
|
@@ -6246,6 +6531,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6246
6531
|
"similar:euclidean": "similar:euclidean";
|
|
6247
6532
|
"starts-with": "starts-with";
|
|
6248
6533
|
"tool-call-f1": "tool-call-f1";
|
|
6534
|
+
"skill-used": "skill-used";
|
|
6535
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
6536
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
6537
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
6538
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
6539
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6249
6540
|
"trace-error-spans": "trace-error-spans";
|
|
6250
6541
|
"trace-span-count": "trace-span-count";
|
|
6251
6542
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -6306,13 +6597,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6306
6597
|
"similar:euclidean": "similar:euclidean";
|
|
6307
6598
|
"starts-with": "starts-with";
|
|
6308
6599
|
"tool-call-f1": "tool-call-f1";
|
|
6600
|
+
"skill-used": "skill-used";
|
|
6601
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
6602
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
6603
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
6604
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
6605
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6309
6606
|
"trace-error-spans": "trace-error-spans";
|
|
6310
6607
|
"trace-span-count": "trace-span-count";
|
|
6311
6608
|
"trace-span-duration": "trace-span-duration";
|
|
6312
6609
|
"search-rubric": "search-rubric";
|
|
6313
6610
|
webhook: "webhook";
|
|
6314
6611
|
"word-count": "word-count";
|
|
6315
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
6612
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
6316
6613
|
"select-best": "select-best";
|
|
6317
6614
|
human: "human";
|
|
6318
6615
|
"max-score": "max-score";
|
|
@@ -6523,7 +6820,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6523
6820
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
6524
6821
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
6525
6822
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
6823
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
6824
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
6825
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
6526
6826
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
6827
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
6828
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
6527
6829
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
6528
6830
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
6529
6831
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -6642,6 +6944,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6642
6944
|
"similar:euclidean": "similar:euclidean";
|
|
6643
6945
|
"starts-with": "starts-with";
|
|
6644
6946
|
"tool-call-f1": "tool-call-f1";
|
|
6947
|
+
"skill-used": "skill-used";
|
|
6948
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
6949
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
6950
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
6951
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
6952
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6645
6953
|
"trace-error-spans": "trace-error-spans";
|
|
6646
6954
|
"trace-span-count": "trace-span-count";
|
|
6647
6955
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -6702,13 +7010,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6702
7010
|
"similar:euclidean": "similar:euclidean";
|
|
6703
7011
|
"starts-with": "starts-with";
|
|
6704
7012
|
"tool-call-f1": "tool-call-f1";
|
|
7013
|
+
"skill-used": "skill-used";
|
|
7014
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7015
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7016
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7017
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7018
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6705
7019
|
"trace-error-spans": "trace-error-spans";
|
|
6706
7020
|
"trace-span-count": "trace-span-count";
|
|
6707
7021
|
"trace-span-duration": "trace-span-duration";
|
|
6708
7022
|
"search-rubric": "search-rubric";
|
|
6709
7023
|
webhook: "webhook";
|
|
6710
7024
|
"word-count": "word-count";
|
|
6711
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7025
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
6712
7026
|
"select-best": "select-best";
|
|
6713
7027
|
human: "human";
|
|
6714
7028
|
"max-score": "max-score";
|
|
@@ -6788,6 +7102,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6788
7102
|
"similar:euclidean": "similar:euclidean";
|
|
6789
7103
|
"starts-with": "starts-with";
|
|
6790
7104
|
"tool-call-f1": "tool-call-f1";
|
|
7105
|
+
"skill-used": "skill-used";
|
|
7106
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7107
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7108
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7109
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7110
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6791
7111
|
"trace-error-spans": "trace-error-spans";
|
|
6792
7112
|
"trace-span-count": "trace-span-count";
|
|
6793
7113
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -6848,13 +7168,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
6848
7168
|
"similar:euclidean": "similar:euclidean";
|
|
6849
7169
|
"starts-with": "starts-with";
|
|
6850
7170
|
"tool-call-f1": "tool-call-f1";
|
|
7171
|
+
"skill-used": "skill-used";
|
|
7172
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7173
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7174
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7175
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7176
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
6851
7177
|
"trace-error-spans": "trace-error-spans";
|
|
6852
7178
|
"trace-span-count": "trace-span-count";
|
|
6853
7179
|
"trace-span-duration": "trace-span-duration";
|
|
6854
7180
|
"search-rubric": "search-rubric";
|
|
6855
7181
|
webhook: "webhook";
|
|
6856
7182
|
"word-count": "word-count";
|
|
6857
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7183
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
6858
7184
|
"select-best": "select-best";
|
|
6859
7185
|
human: "human";
|
|
6860
7186
|
"max-score": "max-score";
|
|
@@ -7063,7 +7389,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7063
7389
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
7064
7390
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
7065
7391
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
7392
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
7393
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
7394
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
7066
7395
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
7396
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
7397
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
7067
7398
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
7068
7399
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
7069
7400
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -7182,6 +7513,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7182
7513
|
"similar:euclidean": "similar:euclidean";
|
|
7183
7514
|
"starts-with": "starts-with";
|
|
7184
7515
|
"tool-call-f1": "tool-call-f1";
|
|
7516
|
+
"skill-used": "skill-used";
|
|
7517
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7518
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7519
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7520
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7521
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7185
7522
|
"trace-error-spans": "trace-error-spans";
|
|
7186
7523
|
"trace-span-count": "trace-span-count";
|
|
7187
7524
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -7242,13 +7579,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7242
7579
|
"similar:euclidean": "similar:euclidean";
|
|
7243
7580
|
"starts-with": "starts-with";
|
|
7244
7581
|
"tool-call-f1": "tool-call-f1";
|
|
7582
|
+
"skill-used": "skill-used";
|
|
7583
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7584
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7585
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7586
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7587
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7245
7588
|
"trace-error-spans": "trace-error-spans";
|
|
7246
7589
|
"trace-span-count": "trace-span-count";
|
|
7247
7590
|
"trace-span-duration": "trace-span-duration";
|
|
7248
7591
|
"search-rubric": "search-rubric";
|
|
7249
7592
|
webhook: "webhook";
|
|
7250
7593
|
"word-count": "word-count";
|
|
7251
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7594
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7252
7595
|
"select-best": "select-best";
|
|
7253
7596
|
human: "human";
|
|
7254
7597
|
"max-score": "max-score";
|
|
@@ -7328,6 +7671,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7328
7671
|
"similar:euclidean": "similar:euclidean";
|
|
7329
7672
|
"starts-with": "starts-with";
|
|
7330
7673
|
"tool-call-f1": "tool-call-f1";
|
|
7674
|
+
"skill-used": "skill-used";
|
|
7675
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7676
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7677
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7678
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7679
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7331
7680
|
"trace-error-spans": "trace-error-spans";
|
|
7332
7681
|
"trace-span-count": "trace-span-count";
|
|
7333
7682
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -7388,13 +7737,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7388
7737
|
"similar:euclidean": "similar:euclidean";
|
|
7389
7738
|
"starts-with": "starts-with";
|
|
7390
7739
|
"tool-call-f1": "tool-call-f1";
|
|
7740
|
+
"skill-used": "skill-used";
|
|
7741
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
7742
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
7743
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
7744
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
7745
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7391
7746
|
"trace-error-spans": "trace-error-spans";
|
|
7392
7747
|
"trace-span-count": "trace-span-count";
|
|
7393
7748
|
"trace-span-duration": "trace-span-duration";
|
|
7394
7749
|
"search-rubric": "search-rubric";
|
|
7395
7750
|
webhook: "webhook";
|
|
7396
7751
|
"word-count": "word-count";
|
|
7397
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7752
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7398
7753
|
"select-best": "select-best";
|
|
7399
7754
|
human: "human";
|
|
7400
7755
|
"max-score": "max-score";
|
|
@@ -7605,7 +7960,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7605
7960
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
7606
7961
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
7607
7962
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
7963
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
7964
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
7965
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
7608
7966
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
7967
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
7968
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
7609
7969
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
7610
7970
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
7611
7971
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -7722,6 +8082,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7722
8082
|
"similar:euclidean": "similar:euclidean";
|
|
7723
8083
|
"starts-with": "starts-with";
|
|
7724
8084
|
"tool-call-f1": "tool-call-f1";
|
|
8085
|
+
"skill-used": "skill-used";
|
|
8086
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
8087
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
8088
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
8089
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
8090
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7725
8091
|
"trace-error-spans": "trace-error-spans";
|
|
7726
8092
|
"trace-span-count": "trace-span-count";
|
|
7727
8093
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -7782,13 +8148,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7782
8148
|
"similar:euclidean": "similar:euclidean";
|
|
7783
8149
|
"starts-with": "starts-with";
|
|
7784
8150
|
"tool-call-f1": "tool-call-f1";
|
|
8151
|
+
"skill-used": "skill-used";
|
|
8152
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
8153
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
8154
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
8155
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
8156
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7785
8157
|
"trace-error-spans": "trace-error-spans";
|
|
7786
8158
|
"trace-span-count": "trace-span-count";
|
|
7787
8159
|
"trace-span-duration": "trace-span-duration";
|
|
7788
8160
|
"search-rubric": "search-rubric";
|
|
7789
8161
|
webhook: "webhook";
|
|
7790
8162
|
"word-count": "word-count";
|
|
7791
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
8163
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7792
8164
|
"select-best": "select-best";
|
|
7793
8165
|
human: "human";
|
|
7794
8166
|
"max-score": "max-score";
|
|
@@ -7868,6 +8240,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7868
8240
|
"similar:euclidean": "similar:euclidean";
|
|
7869
8241
|
"starts-with": "starts-with";
|
|
7870
8242
|
"tool-call-f1": "tool-call-f1";
|
|
8243
|
+
"skill-used": "skill-used";
|
|
8244
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
8245
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
8246
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
8247
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
8248
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7871
8249
|
"trace-error-spans": "trace-error-spans";
|
|
7872
8250
|
"trace-span-count": "trace-span-count";
|
|
7873
8251
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -7928,13 +8306,19 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
7928
8306
|
"similar:euclidean": "similar:euclidean";
|
|
7929
8307
|
"starts-with": "starts-with";
|
|
7930
8308
|
"tool-call-f1": "tool-call-f1";
|
|
8309
|
+
"skill-used": "skill-used";
|
|
8310
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
8311
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
8312
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
8313
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
8314
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
7931
8315
|
"trace-error-spans": "trace-error-spans";
|
|
7932
8316
|
"trace-span-count": "trace-span-count";
|
|
7933
8317
|
"trace-span-duration": "trace-span-duration";
|
|
7934
8318
|
"search-rubric": "search-rubric";
|
|
7935
8319
|
webhook: "webhook";
|
|
7936
8320
|
"word-count": "word-count";
|
|
7937
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
8321
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
7938
8322
|
"select-best": "select-best";
|
|
7939
8323
|
human: "human";
|
|
7940
8324
|
"max-score": "max-score";
|
|
@@ -8134,7 +8518,12 @@ declare const TestSuiteSchema: z.ZodObject<{
|
|
|
8134
8518
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8135
8519
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
8136
8520
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
8521
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8522
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8523
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8137
8524
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8525
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
8526
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8138
8527
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8139
8528
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8140
8529
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -8303,7 +8692,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8303
8692
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8304
8693
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
8305
8694
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
8695
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8696
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8697
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8306
8698
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8699
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
8700
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8307
8701
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8308
8702
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8309
8703
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -8430,7 +8824,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8430
8824
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8431
8825
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
8432
8826
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
8827
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8828
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8829
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8433
8830
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8831
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
8832
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8434
8833
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8435
8834
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8436
8835
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -8573,7 +8972,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8573
8972
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8574
8973
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
8575
8974
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
8975
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8976
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8977
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8576
8978
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8979
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
8980
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
8577
8981
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
8578
8982
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
8579
8983
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -8692,6 +9096,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8692
9096
|
"similar:euclidean": "similar:euclidean";
|
|
8693
9097
|
"starts-with": "starts-with";
|
|
8694
9098
|
"tool-call-f1": "tool-call-f1";
|
|
9099
|
+
"skill-used": "skill-used";
|
|
9100
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9101
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9102
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9103
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9104
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
8695
9105
|
"trace-error-spans": "trace-error-spans";
|
|
8696
9106
|
"trace-span-count": "trace-span-count";
|
|
8697
9107
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -8752,13 +9162,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8752
9162
|
"similar:euclidean": "similar:euclidean";
|
|
8753
9163
|
"starts-with": "starts-with";
|
|
8754
9164
|
"tool-call-f1": "tool-call-f1";
|
|
9165
|
+
"skill-used": "skill-used";
|
|
9166
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9167
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9168
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9169
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9170
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
8755
9171
|
"trace-error-spans": "trace-error-spans";
|
|
8756
9172
|
"trace-span-count": "trace-span-count";
|
|
8757
9173
|
"trace-span-duration": "trace-span-duration";
|
|
8758
9174
|
"search-rubric": "search-rubric";
|
|
8759
9175
|
webhook: "webhook";
|
|
8760
9176
|
"word-count": "word-count";
|
|
8761
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9177
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
8762
9178
|
"select-best": "select-best";
|
|
8763
9179
|
human: "human";
|
|
8764
9180
|
"max-score": "max-score";
|
|
@@ -8838,6 +9254,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8838
9254
|
"similar:euclidean": "similar:euclidean";
|
|
8839
9255
|
"starts-with": "starts-with";
|
|
8840
9256
|
"tool-call-f1": "tool-call-f1";
|
|
9257
|
+
"skill-used": "skill-used";
|
|
9258
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9259
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9260
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9261
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9262
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
8841
9263
|
"trace-error-spans": "trace-error-spans";
|
|
8842
9264
|
"trace-span-count": "trace-span-count";
|
|
8843
9265
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -8898,13 +9320,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
8898
9320
|
"similar:euclidean": "similar:euclidean";
|
|
8899
9321
|
"starts-with": "starts-with";
|
|
8900
9322
|
"tool-call-f1": "tool-call-f1";
|
|
9323
|
+
"skill-used": "skill-used";
|
|
9324
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9325
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9326
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9327
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9328
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
8901
9329
|
"trace-error-spans": "trace-error-spans";
|
|
8902
9330
|
"trace-span-count": "trace-span-count";
|
|
8903
9331
|
"trace-span-duration": "trace-span-duration";
|
|
8904
9332
|
"search-rubric": "search-rubric";
|
|
8905
9333
|
webhook: "webhook";
|
|
8906
9334
|
"word-count": "word-count";
|
|
8907
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9335
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
8908
9336
|
"select-best": "select-best";
|
|
8909
9337
|
human: "human";
|
|
8910
9338
|
"max-score": "max-score";
|
|
@@ -9121,7 +9549,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9121
9549
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
9122
9550
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
9123
9551
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
9552
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
9553
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
9554
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
9124
9555
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
9556
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
9557
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
9125
9558
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
9126
9559
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
9127
9560
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -9240,6 +9673,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9240
9673
|
"similar:euclidean": "similar:euclidean";
|
|
9241
9674
|
"starts-with": "starts-with";
|
|
9242
9675
|
"tool-call-f1": "tool-call-f1";
|
|
9676
|
+
"skill-used": "skill-used";
|
|
9677
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9678
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9679
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9680
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9681
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9243
9682
|
"trace-error-spans": "trace-error-spans";
|
|
9244
9683
|
"trace-span-count": "trace-span-count";
|
|
9245
9684
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -9300,13 +9739,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9300
9739
|
"similar:euclidean": "similar:euclidean";
|
|
9301
9740
|
"starts-with": "starts-with";
|
|
9302
9741
|
"tool-call-f1": "tool-call-f1";
|
|
9742
|
+
"skill-used": "skill-used";
|
|
9743
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9744
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9745
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9746
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9747
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9303
9748
|
"trace-error-spans": "trace-error-spans";
|
|
9304
9749
|
"trace-span-count": "trace-span-count";
|
|
9305
9750
|
"trace-span-duration": "trace-span-duration";
|
|
9306
9751
|
"search-rubric": "search-rubric";
|
|
9307
9752
|
webhook: "webhook";
|
|
9308
9753
|
"word-count": "word-count";
|
|
9309
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9754
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9310
9755
|
"select-best": "select-best";
|
|
9311
9756
|
human: "human";
|
|
9312
9757
|
"max-score": "max-score";
|
|
@@ -9386,6 +9831,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9386
9831
|
"similar:euclidean": "similar:euclidean";
|
|
9387
9832
|
"starts-with": "starts-with";
|
|
9388
9833
|
"tool-call-f1": "tool-call-f1";
|
|
9834
|
+
"skill-used": "skill-used";
|
|
9835
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9836
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9837
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9838
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9839
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9389
9840
|
"trace-error-spans": "trace-error-spans";
|
|
9390
9841
|
"trace-span-count": "trace-span-count";
|
|
9391
9842
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -9446,13 +9897,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9446
9897
|
"similar:euclidean": "similar:euclidean";
|
|
9447
9898
|
"starts-with": "starts-with";
|
|
9448
9899
|
"tool-call-f1": "tool-call-f1";
|
|
9900
|
+
"skill-used": "skill-used";
|
|
9901
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
9902
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
9903
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
9904
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
9905
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9449
9906
|
"trace-error-spans": "trace-error-spans";
|
|
9450
9907
|
"trace-span-count": "trace-span-count";
|
|
9451
9908
|
"trace-span-duration": "trace-span-duration";
|
|
9452
9909
|
"search-rubric": "search-rubric";
|
|
9453
9910
|
webhook: "webhook";
|
|
9454
9911
|
"word-count": "word-count";
|
|
9455
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9912
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9456
9913
|
"select-best": "select-best";
|
|
9457
9914
|
human: "human";
|
|
9458
9915
|
"max-score": "max-score";
|
|
@@ -9661,7 +10118,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9661
10118
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
9662
10119
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
9663
10120
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
10121
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
10122
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
10123
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
9664
10124
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
10125
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
10126
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
9665
10127
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
9666
10128
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
9667
10129
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -9780,6 +10242,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9780
10242
|
"similar:euclidean": "similar:euclidean";
|
|
9781
10243
|
"starts-with": "starts-with";
|
|
9782
10244
|
"tool-call-f1": "tool-call-f1";
|
|
10245
|
+
"skill-used": "skill-used";
|
|
10246
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10247
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10248
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10249
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10250
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9783
10251
|
"trace-error-spans": "trace-error-spans";
|
|
9784
10252
|
"trace-span-count": "trace-span-count";
|
|
9785
10253
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -9840,13 +10308,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9840
10308
|
"similar:euclidean": "similar:euclidean";
|
|
9841
10309
|
"starts-with": "starts-with";
|
|
9842
10310
|
"tool-call-f1": "tool-call-f1";
|
|
10311
|
+
"skill-used": "skill-used";
|
|
10312
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10313
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10314
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10315
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10316
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9843
10317
|
"trace-error-spans": "trace-error-spans";
|
|
9844
10318
|
"trace-span-count": "trace-span-count";
|
|
9845
10319
|
"trace-span-duration": "trace-span-duration";
|
|
9846
10320
|
"search-rubric": "search-rubric";
|
|
9847
10321
|
webhook: "webhook";
|
|
9848
10322
|
"word-count": "word-count";
|
|
9849
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
10323
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9850
10324
|
"select-best": "select-best";
|
|
9851
10325
|
human: "human";
|
|
9852
10326
|
"max-score": "max-score";
|
|
@@ -9926,6 +10400,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9926
10400
|
"similar:euclidean": "similar:euclidean";
|
|
9927
10401
|
"starts-with": "starts-with";
|
|
9928
10402
|
"tool-call-f1": "tool-call-f1";
|
|
10403
|
+
"skill-used": "skill-used";
|
|
10404
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10405
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10406
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10407
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10408
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9929
10409
|
"trace-error-spans": "trace-error-spans";
|
|
9930
10410
|
"trace-span-count": "trace-span-count";
|
|
9931
10411
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -9986,13 +10466,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
9986
10466
|
"similar:euclidean": "similar:euclidean";
|
|
9987
10467
|
"starts-with": "starts-with";
|
|
9988
10468
|
"tool-call-f1": "tool-call-f1";
|
|
10469
|
+
"skill-used": "skill-used";
|
|
10470
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10471
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10472
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10473
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10474
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
9989
10475
|
"trace-error-spans": "trace-error-spans";
|
|
9990
10476
|
"trace-span-count": "trace-span-count";
|
|
9991
10477
|
"trace-span-duration": "trace-span-duration";
|
|
9992
10478
|
"search-rubric": "search-rubric";
|
|
9993
10479
|
webhook: "webhook";
|
|
9994
10480
|
"word-count": "word-count";
|
|
9995
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
10481
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
9996
10482
|
"select-best": "select-best";
|
|
9997
10483
|
human: "human";
|
|
9998
10484
|
"max-score": "max-score";
|
|
@@ -10203,7 +10689,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10203
10689
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
10204
10690
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
10205
10691
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
10692
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
10693
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
10694
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
10206
10695
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
10696
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
10697
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
10207
10698
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
10208
10699
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
10209
10700
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -10320,6 +10811,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10320
10811
|
"similar:euclidean": "similar:euclidean";
|
|
10321
10812
|
"starts-with": "starts-with";
|
|
10322
10813
|
"tool-call-f1": "tool-call-f1";
|
|
10814
|
+
"skill-used": "skill-used";
|
|
10815
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10816
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10817
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10818
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10819
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
10323
10820
|
"trace-error-spans": "trace-error-spans";
|
|
10324
10821
|
"trace-span-count": "trace-span-count";
|
|
10325
10822
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -10380,13 +10877,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10380
10877
|
"similar:euclidean": "similar:euclidean";
|
|
10381
10878
|
"starts-with": "starts-with";
|
|
10382
10879
|
"tool-call-f1": "tool-call-f1";
|
|
10880
|
+
"skill-used": "skill-used";
|
|
10881
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10882
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10883
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10884
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10885
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
10383
10886
|
"trace-error-spans": "trace-error-spans";
|
|
10384
10887
|
"trace-span-count": "trace-span-count";
|
|
10385
10888
|
"trace-span-duration": "trace-span-duration";
|
|
10386
10889
|
"search-rubric": "search-rubric";
|
|
10387
10890
|
webhook: "webhook";
|
|
10388
10891
|
"word-count": "word-count";
|
|
10389
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
10892
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
10390
10893
|
"select-best": "select-best";
|
|
10391
10894
|
human: "human";
|
|
10392
10895
|
"max-score": "max-score";
|
|
@@ -10466,6 +10969,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10466
10969
|
"similar:euclidean": "similar:euclidean";
|
|
10467
10970
|
"starts-with": "starts-with";
|
|
10468
10971
|
"tool-call-f1": "tool-call-f1";
|
|
10972
|
+
"skill-used": "skill-used";
|
|
10973
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
10974
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
10975
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
10976
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
10977
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
10469
10978
|
"trace-error-spans": "trace-error-spans";
|
|
10470
10979
|
"trace-span-count": "trace-span-count";
|
|
10471
10980
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -10526,13 +11035,19 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10526
11035
|
"similar:euclidean": "similar:euclidean";
|
|
10527
11036
|
"starts-with": "starts-with";
|
|
10528
11037
|
"tool-call-f1": "tool-call-f1";
|
|
11038
|
+
"skill-used": "skill-used";
|
|
11039
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
11040
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
11041
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
11042
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
11043
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
10529
11044
|
"trace-error-spans": "trace-error-spans";
|
|
10530
11045
|
"trace-span-count": "trace-span-count";
|
|
10531
11046
|
"trace-span-duration": "trace-span-duration";
|
|
10532
11047
|
"search-rubric": "search-rubric";
|
|
10533
11048
|
webhook: "webhook";
|
|
10534
11049
|
"word-count": "word-count";
|
|
10535
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
11050
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
10536
11051
|
"select-best": "select-best";
|
|
10537
11052
|
human: "human";
|
|
10538
11053
|
"max-score": "max-score";
|
|
@@ -10737,7 +11252,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10737
11252
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
10738
11253
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
10739
11254
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
11255
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
11256
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
11257
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
10740
11258
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
11259
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
11260
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
10741
11261
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
10742
11262
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
10743
11263
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -10883,7 +11403,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
10883
11403
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
10884
11404
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
10885
11405
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
11406
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
11407
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
11408
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
10886
11409
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
11410
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
11411
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
10887
11412
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
10888
11413
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
10889
11414
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -11078,7 +11603,12 @@ declare const TestSuiteConfigSchema: z.ZodObject<{
|
|
|
11078
11603
|
OPENAI_API_KEY?: string | undefined;
|
|
11079
11604
|
OPENAI_BASE_URL?: string | undefined;
|
|
11080
11605
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
11606
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
11607
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
11608
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
11081
11609
|
CODEX_API_KEY?: string | undefined;
|
|
11610
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
11611
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
11082
11612
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
11083
11613
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
11084
11614
|
PALM_API_HOST?: string | undefined;
|
|
@@ -11286,7 +11816,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11286
11816
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
11287
11817
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
11288
11818
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
11819
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
11820
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
11821
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
11289
11822
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
11823
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
11824
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
11290
11825
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
11291
11826
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
11292
11827
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -11405,6 +11940,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11405
11940
|
"similar:euclidean": "similar:euclidean";
|
|
11406
11941
|
"starts-with": "starts-with";
|
|
11407
11942
|
"tool-call-f1": "tool-call-f1";
|
|
11943
|
+
"skill-used": "skill-used";
|
|
11944
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
11945
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
11946
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
11947
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
11948
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
11408
11949
|
"trace-error-spans": "trace-error-spans";
|
|
11409
11950
|
"trace-span-count": "trace-span-count";
|
|
11410
11951
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -11465,13 +12006,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11465
12006
|
"similar:euclidean": "similar:euclidean";
|
|
11466
12007
|
"starts-with": "starts-with";
|
|
11467
12008
|
"tool-call-f1": "tool-call-f1";
|
|
12009
|
+
"skill-used": "skill-used";
|
|
12010
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12011
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12012
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12013
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12014
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
11468
12015
|
"trace-error-spans": "trace-error-spans";
|
|
11469
12016
|
"trace-span-count": "trace-span-count";
|
|
11470
12017
|
"trace-span-duration": "trace-span-duration";
|
|
11471
12018
|
"search-rubric": "search-rubric";
|
|
11472
12019
|
webhook: "webhook";
|
|
11473
12020
|
"word-count": "word-count";
|
|
11474
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12021
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
11475
12022
|
"select-best": "select-best";
|
|
11476
12023
|
human: "human";
|
|
11477
12024
|
"max-score": "max-score";
|
|
@@ -11551,6 +12098,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11551
12098
|
"similar:euclidean": "similar:euclidean";
|
|
11552
12099
|
"starts-with": "starts-with";
|
|
11553
12100
|
"tool-call-f1": "tool-call-f1";
|
|
12101
|
+
"skill-used": "skill-used";
|
|
12102
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12103
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12104
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12105
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12106
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
11554
12107
|
"trace-error-spans": "trace-error-spans";
|
|
11555
12108
|
"trace-span-count": "trace-span-count";
|
|
11556
12109
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -11611,13 +12164,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11611
12164
|
"similar:euclidean": "similar:euclidean";
|
|
11612
12165
|
"starts-with": "starts-with";
|
|
11613
12166
|
"tool-call-f1": "tool-call-f1";
|
|
12167
|
+
"skill-used": "skill-used";
|
|
12168
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12169
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12170
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12171
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12172
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
11614
12173
|
"trace-error-spans": "trace-error-spans";
|
|
11615
12174
|
"trace-span-count": "trace-span-count";
|
|
11616
12175
|
"trace-span-duration": "trace-span-duration";
|
|
11617
12176
|
"search-rubric": "search-rubric";
|
|
11618
12177
|
webhook: "webhook";
|
|
11619
12178
|
"word-count": "word-count";
|
|
11620
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12179
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
11621
12180
|
"select-best": "select-best";
|
|
11622
12181
|
human: "human";
|
|
11623
12182
|
"max-score": "max-score";
|
|
@@ -11834,7 +12393,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11834
12393
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
11835
12394
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
11836
12395
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
12396
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
12397
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
12398
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
11837
12399
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
12400
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
12401
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
11838
12402
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
11839
12403
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
11840
12404
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -11953,6 +12517,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
11953
12517
|
"similar:euclidean": "similar:euclidean";
|
|
11954
12518
|
"starts-with": "starts-with";
|
|
11955
12519
|
"tool-call-f1": "tool-call-f1";
|
|
12520
|
+
"skill-used": "skill-used";
|
|
12521
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12522
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12523
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12524
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12525
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
11956
12526
|
"trace-error-spans": "trace-error-spans";
|
|
11957
12527
|
"trace-span-count": "trace-span-count";
|
|
11958
12528
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -12013,13 +12583,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12013
12583
|
"similar:euclidean": "similar:euclidean";
|
|
12014
12584
|
"starts-with": "starts-with";
|
|
12015
12585
|
"tool-call-f1": "tool-call-f1";
|
|
12586
|
+
"skill-used": "skill-used";
|
|
12587
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12588
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12589
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12590
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12591
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12016
12592
|
"trace-error-spans": "trace-error-spans";
|
|
12017
12593
|
"trace-span-count": "trace-span-count";
|
|
12018
12594
|
"trace-span-duration": "trace-span-duration";
|
|
12019
12595
|
"search-rubric": "search-rubric";
|
|
12020
12596
|
webhook: "webhook";
|
|
12021
12597
|
"word-count": "word-count";
|
|
12022
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12598
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12023
12599
|
"select-best": "select-best";
|
|
12024
12600
|
human: "human";
|
|
12025
12601
|
"max-score": "max-score";
|
|
@@ -12099,6 +12675,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12099
12675
|
"similar:euclidean": "similar:euclidean";
|
|
12100
12676
|
"starts-with": "starts-with";
|
|
12101
12677
|
"tool-call-f1": "tool-call-f1";
|
|
12678
|
+
"skill-used": "skill-used";
|
|
12679
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12680
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12681
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12682
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12683
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12102
12684
|
"trace-error-spans": "trace-error-spans";
|
|
12103
12685
|
"trace-span-count": "trace-span-count";
|
|
12104
12686
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -12159,13 +12741,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12159
12741
|
"similar:euclidean": "similar:euclidean";
|
|
12160
12742
|
"starts-with": "starts-with";
|
|
12161
12743
|
"tool-call-f1": "tool-call-f1";
|
|
12744
|
+
"skill-used": "skill-used";
|
|
12745
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
12746
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
12747
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
12748
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
12749
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12162
12750
|
"trace-error-spans": "trace-error-spans";
|
|
12163
12751
|
"trace-span-count": "trace-span-count";
|
|
12164
12752
|
"trace-span-duration": "trace-span-duration";
|
|
12165
12753
|
"search-rubric": "search-rubric";
|
|
12166
12754
|
webhook: "webhook";
|
|
12167
12755
|
"word-count": "word-count";
|
|
12168
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12756
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12169
12757
|
"select-best": "select-best";
|
|
12170
12758
|
human: "human";
|
|
12171
12759
|
"max-score": "max-score";
|
|
@@ -12374,7 +12962,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12374
12962
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
12375
12963
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
12376
12964
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
12965
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
12966
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
12967
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
12377
12968
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
12969
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
12970
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
12378
12971
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
12379
12972
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
12380
12973
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -12493,6 +13086,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12493
13086
|
"similar:euclidean": "similar:euclidean";
|
|
12494
13087
|
"starts-with": "starts-with";
|
|
12495
13088
|
"tool-call-f1": "tool-call-f1";
|
|
13089
|
+
"skill-used": "skill-used";
|
|
13090
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13091
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13092
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13093
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13094
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12496
13095
|
"trace-error-spans": "trace-error-spans";
|
|
12497
13096
|
"trace-span-count": "trace-span-count";
|
|
12498
13097
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -12553,13 +13152,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12553
13152
|
"similar:euclidean": "similar:euclidean";
|
|
12554
13153
|
"starts-with": "starts-with";
|
|
12555
13154
|
"tool-call-f1": "tool-call-f1";
|
|
13155
|
+
"skill-used": "skill-used";
|
|
13156
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13157
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13158
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13159
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13160
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12556
13161
|
"trace-error-spans": "trace-error-spans";
|
|
12557
13162
|
"trace-span-count": "trace-span-count";
|
|
12558
13163
|
"trace-span-duration": "trace-span-duration";
|
|
12559
13164
|
"search-rubric": "search-rubric";
|
|
12560
13165
|
webhook: "webhook";
|
|
12561
13166
|
"word-count": "word-count";
|
|
12562
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
13167
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12563
13168
|
"select-best": "select-best";
|
|
12564
13169
|
human: "human";
|
|
12565
13170
|
"max-score": "max-score";
|
|
@@ -12639,6 +13244,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12639
13244
|
"similar:euclidean": "similar:euclidean";
|
|
12640
13245
|
"starts-with": "starts-with";
|
|
12641
13246
|
"tool-call-f1": "tool-call-f1";
|
|
13247
|
+
"skill-used": "skill-used";
|
|
13248
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13249
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13250
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13251
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13252
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12642
13253
|
"trace-error-spans": "trace-error-spans";
|
|
12643
13254
|
"trace-span-count": "trace-span-count";
|
|
12644
13255
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -12699,13 +13310,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12699
13310
|
"similar:euclidean": "similar:euclidean";
|
|
12700
13311
|
"starts-with": "starts-with";
|
|
12701
13312
|
"tool-call-f1": "tool-call-f1";
|
|
13313
|
+
"skill-used": "skill-used";
|
|
13314
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13315
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13316
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13317
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13318
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
12702
13319
|
"trace-error-spans": "trace-error-spans";
|
|
12703
13320
|
"trace-span-count": "trace-span-count";
|
|
12704
13321
|
"trace-span-duration": "trace-span-duration";
|
|
12705
13322
|
"search-rubric": "search-rubric";
|
|
12706
13323
|
webhook: "webhook";
|
|
12707
13324
|
"word-count": "word-count";
|
|
12708
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
13325
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
12709
13326
|
"select-best": "select-best";
|
|
12710
13327
|
human: "human";
|
|
12711
13328
|
"max-score": "max-score";
|
|
@@ -12916,7 +13533,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
12916
13533
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
12917
13534
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
12918
13535
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
13536
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
13537
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
13538
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
12919
13539
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
13540
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
13541
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
12920
13542
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
12921
13543
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
12922
13544
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -13033,6 +13655,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13033
13655
|
"similar:euclidean": "similar:euclidean";
|
|
13034
13656
|
"starts-with": "starts-with";
|
|
13035
13657
|
"tool-call-f1": "tool-call-f1";
|
|
13658
|
+
"skill-used": "skill-used";
|
|
13659
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13660
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13661
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13662
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13663
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
13036
13664
|
"trace-error-spans": "trace-error-spans";
|
|
13037
13665
|
"trace-span-count": "trace-span-count";
|
|
13038
13666
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -13093,13 +13721,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13093
13721
|
"similar:euclidean": "similar:euclidean";
|
|
13094
13722
|
"starts-with": "starts-with";
|
|
13095
13723
|
"tool-call-f1": "tool-call-f1";
|
|
13724
|
+
"skill-used": "skill-used";
|
|
13725
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13726
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13727
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13728
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13729
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
13096
13730
|
"trace-error-spans": "trace-error-spans";
|
|
13097
13731
|
"trace-span-count": "trace-span-count";
|
|
13098
13732
|
"trace-span-duration": "trace-span-duration";
|
|
13099
13733
|
"search-rubric": "search-rubric";
|
|
13100
13734
|
webhook: "webhook";
|
|
13101
13735
|
"word-count": "word-count";
|
|
13102
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
13736
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
13103
13737
|
"select-best": "select-best";
|
|
13104
13738
|
human: "human";
|
|
13105
13739
|
"max-score": "max-score";
|
|
@@ -13179,6 +13813,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13179
13813
|
"similar:euclidean": "similar:euclidean";
|
|
13180
13814
|
"starts-with": "starts-with";
|
|
13181
13815
|
"tool-call-f1": "tool-call-f1";
|
|
13816
|
+
"skill-used": "skill-used";
|
|
13817
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13818
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13819
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13820
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13821
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
13182
13822
|
"trace-error-spans": "trace-error-spans";
|
|
13183
13823
|
"trace-span-count": "trace-span-count";
|
|
13184
13824
|
"trace-span-duration": "trace-span-duration";
|
|
@@ -13239,13 +13879,19 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13239
13879
|
"similar:euclidean": "similar:euclidean";
|
|
13240
13880
|
"starts-with": "starts-with";
|
|
13241
13881
|
"tool-call-f1": "tool-call-f1";
|
|
13882
|
+
"skill-used": "skill-used";
|
|
13883
|
+
"trajectory:goal-success": "trajectory:goal-success";
|
|
13884
|
+
"trajectory:tool-args-match": "trajectory:tool-args-match";
|
|
13885
|
+
"trajectory:step-count": "trajectory:step-count";
|
|
13886
|
+
"trajectory:tool-sequence": "trajectory:tool-sequence";
|
|
13887
|
+
"trajectory:tool-used": "trajectory:tool-used";
|
|
13242
13888
|
"trace-error-spans": "trace-error-spans";
|
|
13243
13889
|
"trace-span-count": "trace-span-count";
|
|
13244
13890
|
"trace-span-duration": "trace-span-duration";
|
|
13245
13891
|
"search-rubric": "search-rubric";
|
|
13246
13892
|
webhook: "webhook";
|
|
13247
13893
|
"word-count": "word-count";
|
|
13248
|
-
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
13894
|
+
}>, z.ZodTransform<"not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count", "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count">>, z.ZodEnum<{
|
|
13249
13895
|
"select-best": "select-best";
|
|
13250
13896
|
human: "human";
|
|
13251
13897
|
"max-score": "max-score";
|
|
@@ -13450,7 +14096,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13450
14096
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
13451
14097
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
13452
14098
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
14099
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14100
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14101
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
13453
14102
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14103
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
14104
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
13454
14105
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
13455
14106
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
13456
14107
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -13596,7 +14247,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13596
14247
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
13597
14248
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
13598
14249
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
14250
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14251
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14252
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
13599
14253
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14254
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
14255
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
13600
14256
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
13601
14257
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
13602
14258
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -13791,7 +14447,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
13791
14447
|
OPENAI_API_KEY?: string | undefined;
|
|
13792
14448
|
OPENAI_BASE_URL?: string | undefined;
|
|
13793
14449
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
14450
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
14451
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
14452
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
13794
14453
|
CODEX_API_KEY?: string | undefined;
|
|
14454
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
14455
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
13795
14456
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
13796
14457
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
13797
14458
|
PALM_API_HOST?: string | undefined;
|
|
@@ -14043,7 +14704,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14043
14704
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14044
14705
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
14045
14706
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
14707
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14708
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14709
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14046
14710
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14711
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
14712
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14047
14713
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14048
14714
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14049
14715
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -14170,7 +14836,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14170
14836
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14171
14837
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
14172
14838
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
14839
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14840
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14841
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14173
14842
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14843
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
14844
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14174
14845
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14175
14846
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14176
14847
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -14306,7 +14977,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14306
14977
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14307
14978
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
14308
14979
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
14980
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14981
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14982
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14309
14983
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14984
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
14985
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14310
14986
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14311
14987
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14312
14988
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -14433,7 +15109,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14433
15109
|
OPENAI_API_KEY: z.ZodOptional<z.ZodString>;
|
|
14434
15110
|
OPENAI_BASE_URL: z.ZodOptional<z.ZodString>;
|
|
14435
15111
|
OPENAI_ORGANIZATION: z.ZodOptional<z.ZodString>;
|
|
15112
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
15113
|
+
CLAWDBOT_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
15114
|
+
CLAWDBOT_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14436
15115
|
CODEX_API_KEY: z.ZodOptional<z.ZodString>;
|
|
15116
|
+
OPENCLAW_CONFIG_PATH: z.ZodOptional<z.ZodString>;
|
|
15117
|
+
OPENCLAW_GATEWAY_PASSWORD: z.ZodOptional<z.ZodString>;
|
|
14437
15118
|
OPENCLAW_GATEWAY_TOKEN: z.ZodOptional<z.ZodString>;
|
|
14438
15119
|
OPENCLAW_GATEWAY_URL: z.ZodOptional<z.ZodString>;
|
|
14439
15120
|
PALM_API_HOST: z.ZodOptional<z.ZodString>;
|
|
@@ -14585,7 +15266,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14585
15266
|
OPENAI_API_KEY?: string | undefined;
|
|
14586
15267
|
OPENAI_BASE_URL?: string | undefined;
|
|
14587
15268
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
15269
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
15270
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
15271
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
14588
15272
|
CODEX_API_KEY?: string | undefined;
|
|
15273
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
15274
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
14589
15275
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
14590
15276
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
14591
15277
|
PALM_API_HOST?: string | undefined;
|
|
@@ -14648,7 +15334,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14648
15334
|
prompts?: string[] | undefined;
|
|
14649
15335
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
14650
15336
|
assert?: ({
|
|
14651
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15337
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
14652
15338
|
value?: AssertionValue | undefined;
|
|
14653
15339
|
config?: Record<string, any> | undefined;
|
|
14654
15340
|
threshold?: number | undefined;
|
|
@@ -14664,7 +15350,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14664
15350
|
} | {
|
|
14665
15351
|
type: "assert-set";
|
|
14666
15352
|
assert: {
|
|
14667
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15353
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
14668
15354
|
value?: AssertionValue | undefined;
|
|
14669
15355
|
config?: Record<string, any> | undefined;
|
|
14670
15356
|
threshold?: number | undefined;
|
|
@@ -14836,7 +15522,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14836
15522
|
OPENAI_API_KEY?: string | undefined;
|
|
14837
15523
|
OPENAI_BASE_URL?: string | undefined;
|
|
14838
15524
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
15525
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
15526
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
15527
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
14839
15528
|
CODEX_API_KEY?: string | undefined;
|
|
15529
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
15530
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
14840
15531
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
14841
15532
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
14842
15533
|
PALM_API_HOST?: string | undefined;
|
|
@@ -14899,7 +15590,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14899
15590
|
prompts?: string[] | undefined;
|
|
14900
15591
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
14901
15592
|
assert?: ({
|
|
14902
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15593
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
14903
15594
|
value?: AssertionValue | undefined;
|
|
14904
15595
|
config?: Record<string, any> | undefined;
|
|
14905
15596
|
threshold?: number | undefined;
|
|
@@ -14915,7 +15606,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
14915
15606
|
} | {
|
|
14916
15607
|
type: "assert-set";
|
|
14917
15608
|
assert: {
|
|
14918
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15609
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
14919
15610
|
value?: AssertionValue | undefined;
|
|
14920
15611
|
config?: Record<string, any> | undefined;
|
|
14921
15612
|
threshold?: number | undefined;
|
|
@@ -15086,7 +15777,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15086
15777
|
OPENAI_API_KEY?: string | undefined;
|
|
15087
15778
|
OPENAI_BASE_URL?: string | undefined;
|
|
15088
15779
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
15780
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
15781
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
15782
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
15089
15783
|
CODEX_API_KEY?: string | undefined;
|
|
15784
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
15785
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
15090
15786
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
15091
15787
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
15092
15788
|
PALM_API_HOST?: string | undefined;
|
|
@@ -15149,7 +15845,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15149
15845
|
prompts?: string[] | undefined;
|
|
15150
15846
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
15151
15847
|
assert?: ({
|
|
15152
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15848
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15153
15849
|
value?: AssertionValue | undefined;
|
|
15154
15850
|
config?: Record<string, any> | undefined;
|
|
15155
15851
|
threshold?: number | undefined;
|
|
@@ -15165,7 +15861,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15165
15861
|
} | {
|
|
15166
15862
|
type: "assert-set";
|
|
15167
15863
|
assert: {
|
|
15168
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15864
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15169
15865
|
value?: AssertionValue | undefined;
|
|
15170
15866
|
config?: Record<string, any> | undefined;
|
|
15171
15867
|
threshold?: number | undefined;
|
|
@@ -15339,7 +16035,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15339
16035
|
OPENAI_API_KEY?: string | undefined;
|
|
15340
16036
|
OPENAI_BASE_URL?: string | undefined;
|
|
15341
16037
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
16038
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
16039
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
16040
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
15342
16041
|
CODEX_API_KEY?: string | undefined;
|
|
16042
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
16043
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
15343
16044
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
15344
16045
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
15345
16046
|
PALM_API_HOST?: string | undefined;
|
|
@@ -15400,7 +16101,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15400
16101
|
} | undefined;
|
|
15401
16102
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
15402
16103
|
assert?: ({
|
|
15403
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16104
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15404
16105
|
value?: AssertionValue | undefined;
|
|
15405
16106
|
config?: Record<string, any> | undefined;
|
|
15406
16107
|
threshold?: number | undefined;
|
|
@@ -15416,7 +16117,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15416
16117
|
} | {
|
|
15417
16118
|
type: "assert-set";
|
|
15418
16119
|
assert: {
|
|
15419
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16120
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
15420
16121
|
value?: AssertionValue | undefined;
|
|
15421
16122
|
config?: Record<string, any> | undefined;
|
|
15422
16123
|
threshold?: number | undefined;
|
|
@@ -15583,7 +16284,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15583
16284
|
OPENAI_API_KEY?: string | undefined;
|
|
15584
16285
|
OPENAI_BASE_URL?: string | undefined;
|
|
15585
16286
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
16287
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
16288
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
16289
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
15586
16290
|
CODEX_API_KEY?: string | undefined;
|
|
16291
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
16292
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
15587
16293
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
15588
16294
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
15589
16295
|
PALM_API_HOST?: string | undefined;
|
|
@@ -15801,7 +16507,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15801
16507
|
OPENAI_API_KEY?: string | undefined;
|
|
15802
16508
|
OPENAI_BASE_URL?: string | undefined;
|
|
15803
16509
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
16510
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
16511
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
16512
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
15804
16513
|
CODEX_API_KEY?: string | undefined;
|
|
16514
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
16515
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
15805
16516
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
15806
16517
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
15807
16518
|
PALM_API_HOST?: string | undefined;
|
|
@@ -15928,7 +16639,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
15928
16639
|
OPENAI_API_KEY?: string | undefined;
|
|
15929
16640
|
OPENAI_BASE_URL?: string | undefined;
|
|
15930
16641
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
16642
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
16643
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
16644
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
15931
16645
|
CODEX_API_KEY?: string | undefined;
|
|
16646
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
16647
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
15932
16648
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
15933
16649
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
15934
16650
|
PALM_API_HOST?: string | undefined;
|
|
@@ -16060,7 +16776,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16060
16776
|
OPENAI_API_KEY?: string | undefined;
|
|
16061
16777
|
OPENAI_BASE_URL?: string | undefined;
|
|
16062
16778
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
16779
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
16780
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
16781
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
16063
16782
|
CODEX_API_KEY?: string | undefined;
|
|
16783
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
16784
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
16064
16785
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
16065
16786
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
16066
16787
|
PALM_API_HOST?: string | undefined;
|
|
@@ -16187,7 +16908,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16187
16908
|
OPENAI_API_KEY?: string | undefined;
|
|
16188
16909
|
OPENAI_BASE_URL?: string | undefined;
|
|
16189
16910
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
16911
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
16912
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
16913
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
16190
16914
|
CODEX_API_KEY?: string | undefined;
|
|
16915
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
16916
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
16191
16917
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
16192
16918
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
16193
16919
|
PALM_API_HOST?: string | undefined;
|
|
@@ -16339,7 +17065,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16339
17065
|
OPENAI_API_KEY?: string | undefined;
|
|
16340
17066
|
OPENAI_BASE_URL?: string | undefined;
|
|
16341
17067
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
17068
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
17069
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
17070
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
16342
17071
|
CODEX_API_KEY?: string | undefined;
|
|
17072
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
17073
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
16343
17074
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
16344
17075
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
16345
17076
|
PALM_API_HOST?: string | undefined;
|
|
@@ -16402,7 +17133,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16402
17133
|
prompts?: string[] | undefined;
|
|
16403
17134
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
16404
17135
|
assert?: ({
|
|
16405
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17136
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16406
17137
|
value?: AssertionValue | undefined;
|
|
16407
17138
|
config?: Record<string, any> | undefined;
|
|
16408
17139
|
threshold?: number | undefined;
|
|
@@ -16418,7 +17149,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16418
17149
|
} | {
|
|
16419
17150
|
type: "assert-set";
|
|
16420
17151
|
assert: {
|
|
16421
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17152
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16422
17153
|
value?: AssertionValue | undefined;
|
|
16423
17154
|
config?: Record<string, any> | undefined;
|
|
16424
17155
|
threshold?: number | undefined;
|
|
@@ -16590,7 +17321,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16590
17321
|
OPENAI_API_KEY?: string | undefined;
|
|
16591
17322
|
OPENAI_BASE_URL?: string | undefined;
|
|
16592
17323
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
17324
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
17325
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
17326
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
16593
17327
|
CODEX_API_KEY?: string | undefined;
|
|
17328
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
17329
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
16594
17330
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
16595
17331
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
16596
17332
|
PALM_API_HOST?: string | undefined;
|
|
@@ -16653,7 +17389,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16653
17389
|
prompts?: string[] | undefined;
|
|
16654
17390
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
16655
17391
|
assert?: ({
|
|
16656
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17392
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16657
17393
|
value?: AssertionValue | undefined;
|
|
16658
17394
|
config?: Record<string, any> | undefined;
|
|
16659
17395
|
threshold?: number | undefined;
|
|
@@ -16669,7 +17405,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16669
17405
|
} | {
|
|
16670
17406
|
type: "assert-set";
|
|
16671
17407
|
assert: {
|
|
16672
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17408
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16673
17409
|
value?: AssertionValue | undefined;
|
|
16674
17410
|
config?: Record<string, any> | undefined;
|
|
16675
17411
|
threshold?: number | undefined;
|
|
@@ -16840,7 +17576,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16840
17576
|
OPENAI_API_KEY?: string | undefined;
|
|
16841
17577
|
OPENAI_BASE_URL?: string | undefined;
|
|
16842
17578
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
17579
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
17580
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
17581
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
16843
17582
|
CODEX_API_KEY?: string | undefined;
|
|
17583
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
17584
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
16844
17585
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
16845
17586
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
16846
17587
|
PALM_API_HOST?: string | undefined;
|
|
@@ -16903,7 +17644,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16903
17644
|
prompts?: string[] | undefined;
|
|
16904
17645
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
16905
17646
|
assert?: ({
|
|
16906
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17647
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16907
17648
|
value?: AssertionValue | undefined;
|
|
16908
17649
|
config?: Record<string, any> | undefined;
|
|
16909
17650
|
threshold?: number | undefined;
|
|
@@ -16919,7 +17660,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
16919
17660
|
} | {
|
|
16920
17661
|
type: "assert-set";
|
|
16921
17662
|
assert: {
|
|
16922
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17663
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
16923
17664
|
value?: AssertionValue | undefined;
|
|
16924
17665
|
config?: Record<string, any> | undefined;
|
|
16925
17666
|
threshold?: number | undefined;
|
|
@@ -17093,7 +17834,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17093
17834
|
OPENAI_API_KEY?: string | undefined;
|
|
17094
17835
|
OPENAI_BASE_URL?: string | undefined;
|
|
17095
17836
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
17837
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
17838
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
17839
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
17096
17840
|
CODEX_API_KEY?: string | undefined;
|
|
17841
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
17842
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
17097
17843
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
17098
17844
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
17099
17845
|
PALM_API_HOST?: string | undefined;
|
|
@@ -17154,7 +17900,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17154
17900
|
} | undefined;
|
|
17155
17901
|
providerOutput?: string | Record<string, unknown> | undefined;
|
|
17156
17902
|
assert?: ({
|
|
17157
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17903
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17158
17904
|
value?: AssertionValue | undefined;
|
|
17159
17905
|
config?: Record<string, any> | undefined;
|
|
17160
17906
|
threshold?: number | undefined;
|
|
@@ -17170,7 +17916,7 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17170
17916
|
} | {
|
|
17171
17917
|
type: "assert-set";
|
|
17172
17918
|
assert: {
|
|
17173
|
-
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17919
|
+
type: `promptfoo:redteam:${string}` | "answer-relevance" | "bleu" | "classifier" | "contains" | "contains-all" | "contains-any" | "contains-html" | "contains-json" | "contains-sql" | "contains-xml" | "context-faithfulness" | "context-recall" | "context-relevance" | "conversation-relevance" | "cost" | "equals" | "factuality" | "finish-reason" | "g-eval" | "gleu" | "guardrails" | "icontains" | "icontains-all" | "icontains-any" | "is-html" | "is-json" | "is-refusal" | "is-sql" | "is-valid-function-call" | "is-valid-openai-function-call" | "is-valid-openai-tools-call" | "is-xml" | "javascript" | "latency" | "levenshtein" | "llm-rubric" | "pi" | "meteor" | "model-graded-closedqa" | "model-graded-factuality" | "moderation" | "perplexity" | "perplexity-score" | "python" | "regex" | "rouge-n" | "ruby" | "similar" | "similar:cosine" | "similar:dot" | "similar:euclidean" | "starts-with" | "tool-call-f1" | "skill-used" | "trajectory:goal-success" | "trajectory:tool-args-match" | "trajectory:step-count" | "trajectory:tool-sequence" | "trajectory:tool-used" | "trace-error-spans" | "trace-span-count" | "trace-span-duration" | "search-rubric" | "webhook" | "word-count" | "not-answer-relevance" | "not-bleu" | "not-classifier" | "not-contains" | "not-contains-all" | "not-contains-any" | "not-contains-html" | "not-contains-json" | "not-contains-sql" | "not-contains-xml" | "not-context-faithfulness" | "not-context-recall" | "not-context-relevance" | "not-conversation-relevance" | "not-cost" | "not-equals" | "not-factuality" | "not-finish-reason" | "not-g-eval" | "not-gleu" | "not-guardrails" | "not-icontains" | "not-icontains-all" | "not-icontains-any" | "not-is-html" | "not-is-json" | "not-is-refusal" | "not-is-sql" | "not-is-valid-function-call" | "not-is-valid-openai-function-call" | "not-is-valid-openai-tools-call" | "not-is-xml" | "not-javascript" | "not-latency" | "not-levenshtein" | "not-llm-rubric" | "not-pi" | "not-meteor" | "not-model-graded-closedqa" | "not-model-graded-factuality" | "not-moderation" | "not-perplexity" | "not-perplexity-score" | "not-python" | "not-regex" | "not-rouge-n" | "not-ruby" | "not-similar" | "not-similar:cosine" | "not-similar:dot" | "not-similar:euclidean" | "not-starts-with" | "not-tool-call-f1" | "not-skill-used" | "not-trajectory:goal-success" | "not-trajectory:tool-args-match" | "not-trajectory:step-count" | "not-trajectory:tool-sequence" | "not-trajectory:tool-used" | "not-trace-error-spans" | "not-trace-span-count" | "not-trace-span-duration" | "not-search-rubric" | "not-webhook" | "not-word-count" | "select-best" | "human" | "max-score";
|
|
17174
17920
|
value?: AssertionValue | undefined;
|
|
17175
17921
|
config?: Record<string, any> | undefined;
|
|
17176
17922
|
threshold?: number | undefined;
|
|
@@ -17337,7 +18083,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17337
18083
|
OPENAI_API_KEY?: string | undefined;
|
|
17338
18084
|
OPENAI_BASE_URL?: string | undefined;
|
|
17339
18085
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
18086
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
18087
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
18088
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
17340
18089
|
CODEX_API_KEY?: string | undefined;
|
|
18090
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
18091
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
17341
18092
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
17342
18093
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
17343
18094
|
PALM_API_HOST?: string | undefined;
|
|
@@ -17555,7 +18306,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17555
18306
|
OPENAI_API_KEY?: string | undefined;
|
|
17556
18307
|
OPENAI_BASE_URL?: string | undefined;
|
|
17557
18308
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
18309
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
18310
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
18311
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
17558
18312
|
CODEX_API_KEY?: string | undefined;
|
|
18313
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
18314
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
17559
18315
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
17560
18316
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
17561
18317
|
PALM_API_HOST?: string | undefined;
|
|
@@ -17682,7 +18438,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17682
18438
|
OPENAI_API_KEY?: string | undefined;
|
|
17683
18439
|
OPENAI_BASE_URL?: string | undefined;
|
|
17684
18440
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
18441
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
18442
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
18443
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
17685
18444
|
CODEX_API_KEY?: string | undefined;
|
|
18445
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
18446
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
17686
18447
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
17687
18448
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
17688
18449
|
PALM_API_HOST?: string | undefined;
|
|
@@ -17814,7 +18575,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17814
18575
|
OPENAI_API_KEY?: string | undefined;
|
|
17815
18576
|
OPENAI_BASE_URL?: string | undefined;
|
|
17816
18577
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
18578
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
18579
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
18580
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
17817
18581
|
CODEX_API_KEY?: string | undefined;
|
|
18582
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
18583
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
17818
18584
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
17819
18585
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
17820
18586
|
PALM_API_HOST?: string | undefined;
|
|
@@ -17941,7 +18707,12 @@ declare const UnifiedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
17941
18707
|
OPENAI_API_KEY?: string | undefined;
|
|
17942
18708
|
OPENAI_BASE_URL?: string | undefined;
|
|
17943
18709
|
OPENAI_ORGANIZATION?: string | undefined;
|
|
18710
|
+
CLAWDBOT_GATEWAY_PASSWORD?: string | undefined;
|
|
18711
|
+
CLAWDBOT_GATEWAY_TOKEN?: string | undefined;
|
|
18712
|
+
CLAWDBOT_GATEWAY_URL?: string | undefined;
|
|
17944
18713
|
CODEX_API_KEY?: string | undefined;
|
|
18714
|
+
OPENCLAW_CONFIG_PATH?: string | undefined;
|
|
18715
|
+
OPENCLAW_GATEWAY_PASSWORD?: string | undefined;
|
|
17945
18716
|
OPENCLAW_GATEWAY_TOKEN?: string | undefined;
|
|
17946
18717
|
OPENCLAW_GATEWAY_URL?: string | undefined;
|
|
17947
18718
|
PALM_API_HOST?: string | undefined;
|
|
@@ -18157,7 +18928,8 @@ declare function runAssertion({
|
|
|
18157
18928
|
vars,
|
|
18158
18929
|
latencyMs,
|
|
18159
18930
|
providerResponse,
|
|
18160
|
-
traceId
|
|
18931
|
+
traceId,
|
|
18932
|
+
traceData
|
|
18161
18933
|
}: {
|
|
18162
18934
|
prompt?: string;
|
|
18163
18935
|
provider?: ApiProvider;
|
|
@@ -18168,6 +18940,7 @@ declare function runAssertion({
|
|
|
18168
18940
|
latencyMs?: number;
|
|
18169
18941
|
assertIndex?: number;
|
|
18170
18942
|
traceId?: string;
|
|
18943
|
+
traceData?: TraceData | null;
|
|
18171
18944
|
}): Promise<GradingResult>;
|
|
18172
18945
|
declare function runAssertions({
|
|
18173
18946
|
assertScoringFunction,
|
|
@@ -18833,5 +19606,5 @@ declare const _default$1: {
|
|
|
18833
19606
|
};
|
|
18834
19607
|
};
|
|
18835
19608
|
//#endregion
|
|
18836
|
-
export { type AfterAllExtensionHookContext, type AfterEachExtensionHookContext, AgentCancelledPayload, AgentErrorPayload, ApiClassificationProvider, ApiEmbeddingProvider, ApiModerationProvider, ApiProvider, ApiSimilarityProvider, Assertion, AssertionOrSet, AssertionOrSetSchema, AssertionParams, AssertionSchema, AssertionSet, AssertionSetSchema, AssertionType, AssertionTypeSchema, AssertionValue, AssertionValueFunction, AssertionValueFunctionContext, AssertionValueFunctionResult, AtomicTestCase, AtomicTestCaseSchema, AudioGradingConfig, BaseAssertionTypes, BaseAssertionTypesSchema, BaseRedteamMetadata, BaseTokenUsage, BaseTokenUsageSchema, type BeforeAllExtensionHookContext, type BeforeEachExtensionHookContext, CallApiContextParams, CallApiFunction, CallApiOptionsParams, ChatMessage, CommandLineOptions, CommandLineOptionsSchema, CompletedPrompt, CompletedPromptSchema, CompletionTokenDetails, CompletionTokenDetailsSchema, ConversationMessage, ConversationMessageSchema, CsvRow, DefaultProviders, DerivedMetric, DerivedMetricSchema, type EnvOverrides, EvalConversations, EvalRegisters, EvalResultsFilterMode, EvalSummary, EvalTableDTO, EvalWithMetadata, EvaluateOptions, EvaluateOptionsSchema, EvaluateResult, EvaluateStats, EvaluateSummaryV2, EvaluateSummaryV3, EvaluateTable, EvaluateTableOutput, EvaluateTableRow, EvaluateTestSuite, EvaluateTestSuiteWithEvaluateOptions, type ExtensionHookContextMap, FailedPluginInfo, FilePath, GradingConfig, GradingConfigSchema, GradingResult, GuardrailResponse, ImageOutput, InferSchema, type Inputs, InputsSchema, Intent, Job, LoadApiProviderContext, ModerationFlag, Modifier, NotPrefixedAssertionTypesSchema, NunjucksFilterMap, OutputConfig, OutputConfigSchema, OutputFile, OutputFileExtension, OutputMetadata, PartialGenerationError, PluginActionParams, PluginConfig, PluginConfigSchema, PoliciesById, Policy, PolicyObject, PolicyObjectSchema, Prompt, PromptConfig, PromptContent, PromptFunction, PromptFunctionContext, PromptFunctionResult, PromptMetrics, PromptWithMetadata, ProviderClassificationResponse, ProviderEmbeddingResponse, ProviderFunction, ProviderId, ProviderLabel, ProviderModerationResponse, ProviderOptions, ProviderOptionsMap, ProviderResponse, ProviderSimilarityResponse, ProviderTestResponse, ProviderType, ProviderTypeMap, ProvidersSchema, RateLimitRegistryRef, RedteamAssertionTypes, RedteamCliGenerateOptions, RedteamContext, RedteamFileConfig, RedteamGenerateOptions, RedteamHistoryEntry, RedteamMediaData, RedteamObjectConfig, RedteamPlugin, RedteamPluginObject, RedteamRunOptions, RedteamStrategy, RedteamStrategyObject, ResultFailureReason, ResultLightweight, ResultLightweightWithLabel, ResultSuggestion, ResultsFile, RunEvalOptions, SavedRedteamConfig, Scenario, ScenarioSchema, ScoringFunction, ServerPromptWithMetadata, SharedResults, SpecialAssertionTypes, SpecialAssertionTypesSchema, StrategyConfig, StrategyConfigSchema, SynthesizeOptions, TestCase, TestCaseSchema, TestCaseWithPlugin, TestCaseWithVarsFile, TestCaseWithVarsFileSchema, TestCasesWithMetadata, TestCasesWithMetadataPrompt, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfig, TestGeneratorConfigSchema, TestSuite, TestSuiteConfig, TestSuiteConfigSchema, TestSuiteSchema, type TokenUsage, TraceData, TraceSpan, TracingConfig, UnifiedConfig, UnifiedConfigSchema, VarMapping, type VarValue, Vars, VarsSchema, ZodLikeSchema, _default as assertions, cache_d_exports as cache, _default$1 as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
|
|
19609
|
+
export { type AfterAllExtensionHookContext, type AfterEachExtensionHookContext, AgentCancelledPayload, AgentErrorPayload, ApiClassificationProvider, ApiEmbeddingProvider, ApiModerationProvider, ApiProvider, ApiSimilarityProvider, Assertion, AssertionOrSet, AssertionOrSetSchema, AssertionParams, AssertionSchema, AssertionSet, AssertionSetSchema, AssertionType, AssertionTypeSchema, AssertionValue, AssertionValueFunction, AssertionValueFunctionContext, AssertionValueFunctionResult, AtomicTestCase, AtomicTestCaseSchema, AudioGradingConfig, BaseAssertionTypes, BaseAssertionTypesSchema, BaseRedteamMetadata, BaseTokenUsage, BaseTokenUsageSchema, type BeforeAllExtensionHookContext, type BeforeEachExtensionHookContext, CallApiContextParams, CallApiFunction, CallApiOptionsParams, ChatMessage, CommandLineOptions, CommandLineOptionsSchema, CompletedPrompt, CompletedPromptSchema, CompletionTokenDetails, CompletionTokenDetailsSchema, ConversationMessage, ConversationMessageSchema, CsvRow, DefaultProviders, DerivedMetric, DerivedMetricSchema, type EnvOverrides, EvalConversations, EvalRegisters, EvalResultsFilterMode, EvalSummary, EvalTableDTO, EvalWithMetadata, EvaluateOptions, EvaluateOptionsSchema, EvaluateResult, EvaluateStats, EvaluateSummaryV2, EvaluateSummaryV3, EvaluateTable, EvaluateTableOutput, EvaluateTableRow, EvaluateTestSuite, EvaluateTestSuiteWithEvaluateOptions, type ExtensionHookContextMap, FailedPluginInfo, FilePath, GradingConfig, GradingConfigSchema, GradingResult, GuardrailResponse, ImageOutput, InferSchema, type Inputs, InputsSchema, Intent, Job, LoadApiProviderContext, ModerationFlag, Modifier, NotPrefixedAssertionTypesSchema, NunjucksFilterMap, OutputConfig, OutputConfigSchema, OutputFile, OutputFileExtension, OutputMetadata, PartialGenerationError, PluginActionParams, PluginConfig, PluginConfigSchema, PoliciesById, Policy, PolicyObject, PolicyObjectSchema, Prompt, PromptConfig, PromptContent, PromptFunction, PromptFunctionContext, PromptFunctionResult, PromptMetrics, PromptWithMetadata, ProviderClassificationResponse, ProviderEmbeddingResponse, ProviderFunction, ProviderId, ProviderLabel, ProviderModerationResponse, ProviderOptions, ProviderOptionsMap, ProviderResponse, ProviderSimilarityResponse, ProviderTestResponse, ProviderType, ProviderTypeMap, ProvidersSchema, RateLimitRegistryRef, RedteamAssertionTypes, RedteamCliGenerateOptions, RedteamContext, RedteamFileConfig, RedteamGenerateOptions, RedteamHistoryEntry, RedteamMediaData, RedteamObjectConfig, RedteamPlugin, RedteamPluginObject, RedteamRunOptions, RedteamStrategy, RedteamStrategyObject, ResultFailureReason, ResultLightweight, ResultLightweightWithLabel, ResultSuggestion, ResultsFile, RunEvalOptions, SavedRedteamConfig, Scenario, ScenarioSchema, ScoringFunction, ServerPromptWithMetadata, SharedResults, SkillCallEntry, SpecialAssertionTypes, SpecialAssertionTypesSchema, StrategyConfig, StrategyConfigSchema, SynthesizeOptions, TestCase, TestCaseSchema, TestCaseWithPlugin, TestCaseWithVarsFile, TestCaseWithVarsFileSchema, TestCasesWithMetadata, TestCasesWithMetadataPrompt, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfig, TestGeneratorConfigSchema, TestSuite, TestSuiteConfig, TestSuiteConfigSchema, TestSuiteSchema, type TokenUsage, TraceData, TraceSpan, TracingConfig, UnifiedConfig, UnifiedConfigSchema, VarMapping, type VarValue, Vars, VarsSchema, ZodLikeSchema, _default as assertions, cache_d_exports as cache, _default$1 as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
|
|
18837
19610
|
//# sourceMappingURL=index.d.cts.map
|