promptfoo 0.121.2 → 0.121.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/src/{ListApp-Du7YVwj5.js → ListApp-DQkFNqE9.js} +1 -1
- package/dist/src/{accounts-B0pgC1oV.js → accounts-DdJ2pHMI.js} +5 -5
- package/dist/src/{accounts-CiBLOnA7.js → accounts-DhMYUUbu.js} +5 -5
- package/dist/src/{accounts-gtkH-5KX.cjs → accounts-Dy17bs4D.cjs} +5 -5
- package/dist/src/{accounts-Bm2D8Db9.js → accounts-F9d_5sMC.js} +6 -6
- package/dist/src/{cometapi-C4xSqeID.cjs → agentic-utils-BpX5b23w.cjs} +24 -62
- package/dist/src/{cometapi-CUQq3H_a.js → agentic-utils-P172hM8B.js} +4 -61
- package/dist/src/agentic-utils-qFlm6zes.js +153 -0
- package/dist/src/{agentic-utils-DS1g3GLF.js → agentic-utils-w68v6_Dz.js} +3 -3
- package/dist/src/{agents-CmvBq8LV.js → agents-8FDnTriG.js} +6 -7
- package/dist/src/{agents-DbRtpYxR.cjs → agents-BahDpe5G.cjs} +255 -20
- package/dist/src/{agents-DgF2zDag.js → agents-C-R_jfzI.js} +255 -20
- package/dist/src/{agents-9qiOy0ho.js → agents-CgaMXvLM.js} +5 -7
- package/dist/src/{agents-Di9DKPzn.cjs → agents-D7-HGxUj.cjs} +5 -7
- package/dist/src/{agents-CBr9A01V.js → agents-DJ35I3Nt.js} +255 -20
- package/dist/src/{agents-cLXA8a_8.js → agents-aYPQLf8W.js} +5 -9
- package/dist/src/{agents-D__IdAlg.js → agents-pQeBEXMm.js} +255 -21
- package/dist/src/{aimlapi-B4rcnZgv.js → aimlapi-BCq3MHeL.js} +8 -14
- package/dist/src/{aimlapi-DHJU_kcV.js → aimlapi-BD6J9oKt.js} +7 -14
- package/dist/src/{aimlapi-BvlNH0gr.cjs → aimlapi-qcK4OT55.cjs} +8 -15
- package/dist/src/{aimlapi-CnkC2HqE.js → aimlapi-sgYnkE54.js} +7 -16
- package/dist/src/app/app/tsconfig.app.tsbuildinfo +1 -0
- package/dist/src/app/assets/Report-CQYFezYu.js +1 -0
- package/dist/src/app/assets/index-BXGkeMwh.css +1 -0
- package/dist/src/app/assets/index-BzJt18Jz.js +385 -0
- package/dist/src/app/assets/rolldown-runtime-COnpUsM8.js +1 -0
- package/dist/src/app/assets/scroll-timeline-D9IT_e8Z.js +1 -0
- package/dist/src/app/assets/sync-IjzpWrOE.js +4 -0
- package/dist/src/app/assets/vendor-charts-BNdH8TCw.js +36 -0
- package/dist/src/app/assets/vendor-markdown-Ch00wnNI.js +29 -0
- package/dist/src/app/assets/vendor-react-CVvmk1UB.js +9 -0
- package/dist/src/app/assets/vendor-utils-BnEYbx2Q.js +37 -0
- package/dist/src/app/index.html +7 -7
- package/dist/src/{audio-Dz3z7s3J.js → audio-B7izf48x.js} +4 -5
- package/dist/src/{audio-CGMyULza.cjs → audio-BQtNuYBj.cjs} +4 -4
- package/dist/src/{audio-Bkv46et0.js → audio-COrn8rM6.js} +4 -4
- package/dist/src/{audio-ClI_AFre.js → audio-DcVKoInv.js} +4 -4
- package/dist/src/{base-CGrhspbK.cjs → base-D-670DX8.cjs} +3 -3
- package/dist/src/{base-CpjcHe4e.js → base-PYJvBE1i.js} +3 -3
- package/dist/src/{base-Dy1V8--Z.js → base-fZ9wgg50.js} +3 -3
- package/dist/src/{base-DLKtKMFh.js → base-yrI1Yal4.js} +3 -3
- package/dist/src/{blobs-BDbfYdrJ.js → blobs-BCZavS8s.js} +4 -4
- package/dist/src/{blobs-CMHN0Qcz.js → blobs-BQWqnnvL.js} +4 -4
- package/dist/src/{blobs-CBO20krR.js → blobs-C-F78Kfn.js} +3 -3
- package/dist/src/{blobs-D23XLin-.cjs → blobs-D2FAd1Q5.cjs} +3 -3
- package/dist/src/cache-BIyPcp5v.cjs +376 -0
- package/dist/src/cache-C4Xb-hNb.js +310 -0
- package/dist/src/cache-Cr9oLMUa.js +3 -0
- package/dist/src/cache-D5NZmMiT.js +310 -0
- package/dist/src/cache-DbLsVWB2.cjs +3 -0
- package/dist/src/cache-mb7c8hbp.js +280 -0
- package/dist/src/{chat-C2jrdPMx.js → chat-0bwXjVP0.js} +3 -13
- package/dist/src/{chat-C1Qst7jL.cjs → chat-BPXSW8Bv.cjs} +3 -13
- package/dist/src/{chat-DJIw17u0.js → chat-BfPaS15_.js} +68 -42
- package/dist/src/{chat-CgF-J-Jj.cjs → chat-CclRbxGf.cjs} +68 -42
- package/dist/src/{chat-BiKyneZl.js → chat-Dr3DUQ0D.js} +68 -42
- package/dist/src/{chat-DqxYYtWA.js → chat-I9izLm49.js} +67 -41
- package/dist/src/{chat-CzkrVDfz.js → chat-MKxMnZJZ.js} +3 -13
- package/dist/src/{chat-qmatte1u.js → chat-mW0ORo8G.js} +3 -14
- package/dist/src/{chatkit-DKyPi1Gs.cjs → chatkit-BoWoSgXl.cjs} +4 -4
- package/dist/src/{chatkit-65VXf5SR.js → chatkit-CJnHRRMM.js} +4 -4
- package/dist/src/{chatkit-Be-Q-a9F.js → chatkit-Cv6AhukM.js} +4 -4
- package/dist/src/{chatkit-BxFvW8KY.js → chatkit-zUIVoDos.js} +4 -4
- package/dist/src/{claude-agent-sdk-Apiy0iaz.js → claude-agent-sdk-BQNuLaAK.js} +41 -18
- package/dist/src/{claude-agent-sdk-D9Z5Pr9X.cjs → claude-agent-sdk-CPJo3dBQ.cjs} +45 -22
- package/dist/src/{claude-agent-sdk-D2bJee9S.js → claude-agent-sdk-Dtq_L-Sc.js} +40 -17
- package/dist/src/{claude-agent-sdk-DfCoW0E6.js → claude-agent-sdk-nfAIcxNf.js} +42 -20
- package/dist/src/{cloud-C0dlstV_.js → cloud-DQZ5sVjW.js} +25 -13
- package/dist/src/cloud-Hphvo8kr.js +3 -0
- package/dist/src/{cloudflare-ai-8TDxHR0x.js → cloudflare-ai-BIB567w6.js} +5 -14
- package/dist/src/{cloudflare-ai-g7PB6VHR.js → cloudflare-ai-DGLte7Py.js} +5 -14
- package/dist/src/{cloudflare-ai-CknbZ5LJ.cjs → cloudflare-ai-Dl3N9OVD.cjs} +6 -15
- package/dist/src/{cloudflare-ai-BxAGvfju.js → cloudflare-ai-DlKr0rY7.js} +5 -15
- package/dist/src/{cloudflare-gateway-B9HWA5wf.js → cloudflare-gateway-BDZrYydE.js} +4 -16
- package/dist/src/{cloudflare-gateway-BSnDmHYo.cjs → cloudflare-gateway-BYDp495F.cjs} +4 -15
- package/dist/src/{cloudflare-gateway-CKDb4dJ8.js → cloudflare-gateway-CiIZHU0Q.js} +5 -164
- package/dist/src/{cloudflare-gateway-CP9QEWYS.js → cloudflare-gateway-DI1HNP5F.js} +4 -15
- package/dist/src/codex-sdk-BAmYE7qy.js +3 -0
- package/dist/src/codex-sdk-C2_M2pl_.cjs +1172 -0
- package/dist/src/codex-sdk-CErXn7qh.js +1165 -0
- package/dist/src/codex-sdk-CWEnH70W.cjs +2 -0
- package/dist/src/codex-sdk-CpqiOqDO.js +1164 -0
- package/dist/src/codex-sdk-Rtky3M4I.js +1165 -0
- package/dist/src/{cometapi-BL9yvj_f.js → cometapi-BUlt_ELa.js} +8 -15
- package/dist/src/{cometapi-DFNiKmSz.js → cometapi-CtJ-mS8R.js} +8 -16
- package/dist/src/cometapi-DT-jlVCB.js +55 -0
- package/dist/src/cometapi-UVOryo4W.cjs +55 -0
- package/dist/src/{completion-CM6oK8PS.cjs → completion-BozdoXba.cjs} +7 -62
- package/dist/src/{completion-5MzrpJxT.js → completion-Dnxn7E-j.js} +8 -57
- package/dist/src/{completion-qRoZAYRB.js → completion-HUe8wDhZ.js} +8 -57
- package/dist/src/{completion-DZ083F31.js → completion-x0a_c2y1.js} +8 -57
- package/dist/src/{createHash-CTQmL3G2.js → createHash-4gFQpDDv.js} +3 -3
- package/dist/src/{createHash-CfZSc0b4.cjs → createHash-B7KvgoOD.cjs} +4 -4
- package/dist/src/{createHash-Da8fMwqB.js → createHash-ChI45QR1.js} +3 -3
- package/dist/src/{createHash-DmPQkvBh.js → createHash-CwDVU5xr.js} +3 -3
- package/dist/src/{docker-ExVyLp0S.js → docker-CQmlA2NU.js} +7 -14
- package/dist/src/{docker-Bb5dcxr8.js → docker-ClnmCf1Z.js} +6 -14
- package/dist/src/{docker-BvfL2BrW.js → docker-DCgsveLD.js} +6 -16
- package/dist/src/{docker-DcF2pRrj.cjs → docker-DS4_Osau.cjs} +7 -15
- package/dist/src/embedding-D3xTseo7.js +59 -0
- package/dist/src/embedding-DD9wa3ae.js +58 -0
- package/dist/src/embedding-I45KG3o7.cjs +63 -0
- package/dist/src/embedding-nFbumxcv.js +58 -0
- package/dist/src/entrypoint.js +69 -6
- package/dist/src/{errors-P6ll7XSJ.js → errors-Cw810C93.js} +1 -1
- package/dist/src/{esm-CaIwzWR5.js → esm-C7PnfdF8.js} +16 -7
- package/dist/src/{esm-CnNt7sI4.cjs → esm-CtEPLdAj.cjs} +15 -6
- package/dist/src/{esm-Cd1AjG1D.js → esm-Dh4dOLlt.js} +15 -6
- package/dist/src/{esm-C03C-mv3.js → esm-tVgYPY-f.js} +27 -18
- package/dist/src/eval-CzJFfFO9.js +3 -0
- package/dist/src/{eval-Dg2nG4v2.js → eval-u4UVafl6.js} +49 -20
- package/dist/src/{evalResult-BDMqrapS.js → evalResult-Bgm9ZH31.js} +7 -7
- package/dist/src/{evalResult-BBRNtX4I.js → evalResult-D3hVYFis.js} +7 -7
- package/dist/src/evalResult-D8MT9p0s.js +3 -0
- package/dist/src/evalResult-DElBuddX.js +2 -0
- package/dist/src/evalResult-Dvc-iucu.cjs +2 -0
- package/dist/src/{evalResult-fuaI8HkH.cjs → evalResult-KZqXl4XP.cjs} +7 -7
- package/dist/src/evaluator-CVessDWe.js +3 -0
- package/dist/src/{evaluator-BhoWwp5b.js → evaluator-IvuDYSvQ.js} +3080 -1254
- package/dist/src/{extractor-D25qpmGX.js → extractor-CAfTSraf.js} +6 -6
- package/dist/src/{extractor-DReVID0K.js → extractor-DNSeBVOJ.js} +6 -6
- package/dist/src/{extractor-C0EVHewb.js → extractor-Dk6bRWkv.js} +6 -6
- package/dist/src/{extractor-pYLLi3wS.cjs → extractor-WVPOrH43.cjs} +6 -6
- package/dist/src/{fetch-HaqdX7U1.js → fetch-B0Z3Oe4k.js} +218 -55
- package/dist/src/{fetch-Dxpd4_sr.js → fetch-BEWnXrrG.js} +195 -45
- package/dist/src/fetch-C7bGKDlQ.js +3 -0
- package/dist/src/{fetch-BPkYtG8K.cjs → fetch-CJU5ELPa.cjs} +223 -48
- package/dist/src/{fetch-Cwxnd8zz.js → fetch-Di00EQrc.js} +218 -55
- package/dist/src/{fileExtensions-Ds-foDzt.js → fileExtensions-AWa2ZML4.js} +1 -1
- package/dist/src/{fileExtensions-LcDYkU4v.js → fileExtensions-BArZuxsI.js} +1 -1
- package/dist/src/{formatDuration-DgBVMN65.js → formatDuration-DZzPsexs.js} +1 -1
- package/dist/src/{genaiTracer-D3fD9dNV.js → genaiTracer-COYDi-tC.js} +6 -2
- package/dist/src/{genaiTracer-C1rxGO8Q.js → genaiTracer-DWdZ28hY.js} +6 -2
- package/dist/src/{genaiTracer-70Z8BIuV.js → genaiTracer-XnrcgDCe.js} +6 -2
- package/dist/src/{genaiTracer-DN4dQywX.cjs → genaiTracer-yRuxj9-L.cjs} +7 -3
- package/dist/src/golang/wrapper.go +1 -1
- package/dist/src/{graders-DU49_J8Y.cjs → graders--zknU_uk.cjs} +5747 -3206
- package/dist/src/graders-BOAzQEUe.cjs +2 -0
- package/dist/src/graders-D4BTsZdG2.js +3 -0
- package/dist/src/graders-DOJK1XpV.js +2 -0
- package/dist/src/graders-NAv9LcBn.js +2 -0
- package/dist/src/{graders-BTeBGqjJ.js → graders-Zy3x0zqX.js} +5727 -3218
- package/dist/src/{graders-Bj_Odv7c.js → graders-eIHhRqoC.js} +5719 -3210
- package/dist/src/{graders-DP7KFFo-.js → graders-pvbReLLn.js} +5728 -3219
- package/dist/src/{image-B0h9VEMc.js → image-9302QVqR.js} +4 -4
- package/dist/src/{image-CHfWvljl.js → image-B5Mv-Z3h.js} +8 -8
- package/dist/src/{image-DS-o-0ph.js → image-DVz2RiMF.js} +8 -8
- package/dist/src/{image-C1madmKh.cjs → image-De2FBmYV.cjs} +4 -4
- package/dist/src/{image-B02ogr_b.js → image-dnoUgPrC.js} +4 -5
- package/dist/src/{image-Dpxa1Jt6.js → image-qUpPvmNZ.js} +8 -8
- package/dist/src/{image-Bb4vWQLM.js → image-u7-rKnYU.js} +4 -4
- package/dist/src/{image-BLmROtN3.cjs → image-x6KqLQl4.cjs} +8 -8
- package/dist/src/index.cjs +3728 -1482
- package/dist/src/index.d.cts +3232 -79
- package/dist/src/index.d.ts +3232 -79
- package/dist/src/index.js +3735 -1490
- package/dist/src/{interactiveCheck-BgLZUIt3.js → interactiveCheck-CLERUB0c.js} +2 -2
- package/dist/src/{knowledgeBase-DOO_BM9b.cjs → knowledgeBase-Bpoe_nLu.cjs} +6 -8
- package/dist/src/{knowledgeBase-D33Ty2l6.js → knowledgeBase-Dgc7CBWF.js} +6 -8
- package/dist/src/{knowledgeBase-B3OoKIej.js → knowledgeBase-RhFPGWDc.js} +6 -8
- package/dist/src/{knowledgeBase-CYTLHOt1.js → knowledgeBase-lm9RXSAm.js} +6 -9
- package/dist/src/{litellm-NbjknEh6.js → litellm-C2kqjxqp.js} +6 -14
- package/dist/src/{litellm-I_hbp_dc.cjs → litellm-CoyI4IAl.cjs} +6 -15
- package/dist/src/{litellm-TrljxD9G.js → litellm-DRjpcSa7.js} +5 -14
- package/dist/src/{litellm-AaeZcZQF.js → litellm-p37R1dzQ.js} +5 -16
- package/dist/src/{logger-DLcq4dWf.js → logger-B88EkIn6.js} +48 -23
- package/dist/src/{logger-Cp1GPUjj.cjs → logger-COuQb2xB.cjs} +77 -22
- package/dist/src/{logger-CT3IKMKA.js → logger-Ct2S6Yx-.js} +48 -23
- package/dist/src/{logger-KkObSCzq.js → logger-DksKw1Qc.js} +48 -23
- package/dist/src/{luma-ray-f6I2fft-.js → luma-ray-B863CmuZ.js} +6 -10
- package/dist/src/{luma-ray-DDsjcgZZ.js → luma-ray-BTTLtqQ8.js} +7 -10
- package/dist/src/{luma-ray-Due0n7di.cjs → luma-ray-BxVKaW2a.cjs} +6 -10
- package/dist/src/{luma-ray-BS2_tY8L.js → luma-ray-KgTCXrZC.js} +6 -12
- package/dist/src/main.d.ts +1 -26
- package/dist/src/main.js +1011 -548
- package/dist/src/{messages-Bs1kC7P4.cjs → messages-811uVVW5.cjs} +74 -19
- package/dist/src/{messages-BS17jdMx.js → messages-BTQz42fn.js} +74 -19
- package/dist/src/{messages-ZJk778GH.js → messages-MYTQ2TWp.js} +74 -19
- package/dist/src/{messages-D0lx5qK7.js → messages-zWbkLLHz.js} +74 -19
- package/dist/src/{meteor-D-SotUw9.js → meteor-CU5UAE-H.js} +1 -1
- package/dist/src/{meteor-DLZZ3osF.cjs → meteor-Co1VQ1u5.cjs} +1 -1
- package/dist/src/{meteor-44VjEACX.js → meteor-DHdzY1Ss.js} +1 -1
- package/dist/src/{meteor-DUiCJRC-.js → meteor-DuAFv6gF.js} +1 -1
- package/dist/src/{modelslab-Bmni6skY.js → modelslab-D0erNWKe.js} +7 -10
- package/dist/src/{modelslab-DRb74SP4.js → modelslab-DIq-6y7x.js} +7 -10
- package/dist/src/{modelslab-CoUX6Jc_.cjs → modelslab-Dk1JAtVo.cjs} +7 -10
- package/dist/src/{modelslab-Bx9IrZfS.js → modelslab-wu9yi5GE.js} +7 -11
- package/dist/src/{nova-reel-BfPq-0Yk.js → nova-reel-CCFRfeRb.js} +7 -10
- package/dist/src/{nova-reel-C_QM18Xn.cjs → nova-reel-CrLXVKQf.cjs} +6 -10
- package/dist/src/{nova-reel-bgjxilYW.js → nova-reel-DQrm74ng.js} +6 -10
- package/dist/src/{nova-reel-D_W1tjMH.js → nova-reel-gr11WG7f.js} +6 -12
- package/dist/src/{nova-sonic-DIGQNR07.js → nova-sonic-BYdp-QLs.js} +5 -7
- package/dist/src/{nova-sonic-CFb5GYhg.js → nova-sonic-B_ZXcUJB.js} +4 -7
- package/dist/src/{nova-sonic-De1HW5fD.js → nova-sonic-TDgrlTk7.js} +4 -9
- package/dist/src/{nova-sonic-zfcljeRp.cjs → nova-sonic-i5tUvXKn.cjs} +4 -7
- package/dist/src/{openai-DElQ-fPX.js → openai-DhVEmgeZ.js} +6 -3
- package/dist/src/{openai-DhbB7eWK.js → openai-Qsvz25mV.js} +6 -3
- package/dist/src/{openai-Cuif0GEt.cjs → openai-URNyItar.cjs} +6 -3
- package/dist/src/{openai-j-sE2O7r.js → openai-iYtrXzOX.js} +6 -3
- package/dist/src/openclaw-CLWrW03k.js +1200 -0
- package/dist/src/openclaw-CnQ363Wi.js +1199 -0
- package/dist/src/openclaw-CwzlQSQX.js +1199 -0
- package/dist/src/openclaw-wX9rtfke.cjs +1205 -0
- package/dist/src/{opencode-sdk-B3CWY9h_.js → opencode-sdk-BUu5Nevv.js} +12 -14
- package/dist/src/{opencode-sdk-BL764Jdi.cjs → opencode-sdk-BZ2idgYA.cjs} +16 -18
- package/dist/src/{opencode-sdk-0j6rTWNb.js → opencode-sdk-BxD8vXp_.js} +12 -15
- package/dist/src/{opencode-sdk-C2y6UkP2.js → opencode-sdk-GI2KaAXq.js} +12 -14
- package/dist/src/{otlpReceiver-C99PPb48.js → otlpReceiver-B2z58l4e.js} +154 -98
- package/dist/src/{otlpReceiver-CGq6LspY.cjs → otlpReceiver-BfcVq2Nq.cjs} +154 -98
- package/dist/src/{otlpReceiver-D89fR-rC.js → otlpReceiver-BntK801g.js} +154 -98
- package/dist/src/{otlpReceiver-CdNBdbsk.js → otlpReceiver-DmVulbhC.js} +154 -98
- package/dist/src/{providerRegistry-CD8MEar9.js → providerRegistry-Bvh8mv85.js} +2 -2
- package/dist/src/{providerRegistry-DM8rZYol.js → providerRegistry-CPQ_CmVO.js} +2 -2
- package/dist/src/{providerRegistry-Civky8Ar.cjs → providerRegistry-CQMdTmHP.cjs} +2 -2
- package/dist/src/{providerRegistry-B0RUOLI_.js → providerRegistry-CWoPjKFZ.js} +2 -2
- package/dist/src/{providers-CgKOSgTR.cjs → providers-1eKkXBKp.cjs} +1435 -930
- package/dist/src/{providers-BlqUifFg.js → providers-BV_KMZje.js} +1419 -944
- package/dist/src/providers-Bp4S-FvO.js +2 -0
- package/dist/src/providers-DV3ax9e_.cjs +3 -0
- package/dist/src/{providers-D8lF1sqW.js → providers-Domz_llv.js} +1427 -952
- package/dist/src/{providers-Dk_6ocUX.js → providers-DruaQfwu.js} +1424 -949
- package/dist/src/providers-iUt5fbAN.js +3 -0
- package/dist/src/providers-u9Enmfok.js +2 -0
- package/dist/src/python/persistent_wrapper.py +0 -5
- package/dist/src/{pythonUtils-D6fwaDSg.js → pythonUtils-C2UQ30Rz.js} +4 -4
- package/dist/src/{pythonUtils-D5nxkQ0P.js → pythonUtils-Cldx7huE.js} +4 -4
- package/dist/src/{pythonUtils-C3py6GC1.js → pythonUtils-CnndUbW-.js} +3 -3
- package/dist/src/{pythonUtils-CTU3Y3lw.cjs → pythonUtils-tAJvvpS-.cjs} +3 -3
- package/dist/src/{quiverai-CIaELU_m.js → quiverai-CtWi6x_g.js} +4 -4
- package/dist/src/{quiverai-PdShCPox.cjs → quiverai-DFotyafY.cjs} +4 -4
- package/dist/src/{quiverai-BbOUOn2L.js → quiverai-DR0SnIQV.js} +4 -4
- package/dist/src/{quiverai-uH-dcTIr.js → quiverai-aPPvXOgn.js} +4 -5
- package/dist/src/render-CH-62LbA.js +135 -0
- package/dist/src/render-CMEpfLaO.js +136 -0
- package/dist/src/{render-Drod8m7K.js → render-CgVDrJmM.js} +2 -3
- package/dist/src/render-DHIZ6_k8.js +135 -0
- package/dist/src/render-DfQSFxGE.cjs +165 -0
- package/dist/src/{responses-DIR9Ud3j.js → responses--OsX2aYW.js} +23 -14
- package/dist/src/{responses-D8SBTL64.cjs → responses-Bi9vBuW_.cjs} +24 -15
- package/dist/src/{responses-CB2jwoAr.js → responses-C-flexAY.js} +24 -15
- package/dist/src/{responses-WNGNYe3K.js → responses-DL9m8CyY.js} +24 -15
- package/dist/src/{rubyUtils-DhCAlxZr.cjs → rubyUtils-B6eljPuh.cjs} +3 -3
- package/dist/src/{rubyUtils-BcuGX77l.js → rubyUtils-CYSQEG4a.js} +3 -3
- package/dist/src/rubyUtils-D1L2d3jb.js +3 -0
- package/dist/src/rubyUtils-DUbq4tff.cjs +2 -0
- package/dist/src/{rubyUtils-BUVePouc.js → rubyUtils-DVLeA2jg.js} +3 -3
- package/dist/src/{rubyUtils-Boc4HZzX.js → rubyUtils-DsGrTx8R.js} +3 -3
- package/dist/src/{sagemaker-CNBxx5CJ.js → sagemaker-BVkaG2-l.js} +14 -18
- package/dist/src/{sagemaker-CemTFp2h.js → sagemaker-BveBvuxm.js} +14 -18
- package/dist/src/{sagemaker-YSyBXQQh.js → sagemaker-D67yzMzs.js} +14 -19
- package/dist/src/{sagemaker-Cl28mZU2.cjs → sagemaker-XnfhheQv.cjs} +14 -18
- package/dist/src/{scanner-BsBlNXNn.js → scanner-1DqWi1Ej.js} +130 -35
- package/dist/src/server/golang/wrapper.go +1 -1
- package/dist/src/server/index.js +3757 -1511
- package/dist/src/server/python/persistent_wrapper.py +0 -5
- package/dist/src/{server-CqzrVGpF.js → server-BNYztJkh.js} +128 -9
- package/dist/src/{server-CuxBbeSY.js → server-BSB45Nt9.js} +127 -8
- package/dist/src/{server-VWgWb00X.js → server-D6Il2Sob.js} +126 -7
- package/dist/src/server-DCtHUqlp.js +3 -0
- package/dist/src/server-DaA2eR26.cjs +2 -0
- package/dist/src/{server-C_7Ax-hA.cjs → server-Dx2TyCH2.cjs} +140 -6
- package/dist/src/{signal-4U3mfRvL.js → signal-CE5G3a7x.js} +3 -3
- package/dist/src/{slack-BmVAVGaK.cjs → slack-1Rhq0EoV.cjs} +2 -2
- package/dist/src/{slack-DCUPTzS2.js → slack-D5Wpy8LM.js} +2 -2
- package/dist/src/{slack-DXMKtA-f.js → slack-DDUe-5MC.js} +2 -2
- package/dist/src/{slack-DOdy_kyv.js → slack-acRb0IqQ.js} +2 -2
- package/dist/src/store-CWOSz6D_.cjs +2 -0
- package/dist/src/{store-Dim__MDd.js → store-CYEy5J2D.js} +17 -5
- package/dist/src/{store-DLlFCC4h.cjs → store-DAAyxcy6.cjs} +17 -5
- package/dist/src/store-DCDBhv7B.js +3 -0
- package/dist/src/{store-CXGFv4aR.js → store-Dn9HUkdW.js} +17 -5
- package/dist/src/{store-DXilxTl-.js → store-M0b1WfYb.js} +17 -5
- package/dist/src/{tables-gftXzE9I.js → tables-C4CH3zRr.js} +3 -3
- package/dist/src/{tables-DLJPUdUE.js → tables-CsWou1Bx.js} +3 -3
- package/dist/src/{tables-6YKwjN9-.js → tables-DQ4WU5tX.js} +3 -3
- package/dist/src/{tables-DPi7wKeM.cjs → tables-DUfh1F7Z.cjs} +3 -3
- package/dist/src/telemetry-C1IqxcdW.js +3 -0
- package/dist/src/telemetry-C4ZEa_es.cjs +2 -0
- package/dist/src/{telemetry-CMrFgtPB.js → telemetry-CQPez_Jp.js} +4 -4
- package/dist/src/{telemetry-DaX14Chu.cjs → telemetry-Dsw_faFj.cjs} +4 -4
- package/dist/src/{telemetry-Dthj_BbD.js → telemetry-Dvqxv3YC.js} +4 -4
- package/dist/src/{telemetry-Cps3mIU-.js → telemetry-dbaJ0E98.js} +4 -4
- package/dist/src/{text-CW1cyrwj.cjs → text-BVi-cLPJ.cjs} +1 -1
- package/dist/src/{text-B_UCRPp2.js → text-CZr46tp_.js} +1 -1
- package/dist/src/{text-TIv0QYnd.js → text-DHxdyQqT.js} +1 -1
- package/dist/src/{text-Db-Wt2u2.js → text-KvuD2Iko.js} +1 -1
- package/dist/src/{tokenUsageUtils-bVa1ga6f.cjs → tokenUsageUtils-Bb7DkZPz.cjs} +7 -3
- package/dist/src/{tokenUsageUtils-NYT-WKS6.js → tokenUsageUtils-C-bmyHoE.js} +7 -3
- package/dist/src/{tokenUsageUtils-DflFMjS0.js → tokenUsageUtils-CXrvO-wA.js} +7 -3
- package/dist/src/{transcription-NLVG9MT1.cjs → transcription-BvjmiYB1.cjs} +12 -16
- package/dist/src/{transcription-BNYURcXg.js → transcription-CJspiD2c.js} +11 -14
- package/dist/src/{transcription-B_OdaHp7.js → transcription-DuWDupG7.js} +10 -14
- package/dist/src/{transcription-s6A-bNrZ.js → transcription-V2HaAmy2.js} +10 -16
- package/dist/src/{transform-DECvGmzp.js → transform-Bbg6A8Jk.js} +4 -4
- package/dist/src/{transform-vNucnNr0.js → transform-CG0ehZNG.js} +11 -7
- package/dist/src/{transform-DuHvhZpj.cjs → transform-CTeuTR3S.cjs} +31 -9
- package/dist/src/{transform-CzK1Q0zl.cjs → transform-CUnzlsbn.cjs} +4 -4
- package/dist/src/{transform-aa6tmVpZ.js → transform-DYX1_Xnh.js} +5 -5
- package/dist/src/transform-DgKlRr73.cjs +2 -0
- package/dist/src/transform-M6ITAESf.js +3 -0
- package/dist/src/{transform-DilY9wbS.js → transform-UN5UGu8U.js} +5 -5
- package/dist/src/{transform-uAytVuyX.js → transform-lQrDE1BQ.js} +11 -7
- package/dist/src/{transform-D5HsjduX.js → transform-zDhMmzwX.js} +11 -7
- package/dist/src/{transformersAvailability-CEVM2GNQ.js → transformersAvailability-CcHusyhw.js} +1 -1
- package/dist/src/{transformersAvailability-CwayUSlh.cjs → transformersAvailability-Cju9mHgR.cjs} +1 -1
- package/dist/src/{transformersAvailability-D6c6ROpT.js → transformersAvailability-DLlROWhg.js} +1 -1
- package/dist/src/{types-DmyIJ-sR.js → types-BGQDAP8i.js} +357 -22
- package/dist/src/{types-CzW2QFyi.js → types-Bgh5SOn6.js} +358 -24
- package/dist/src/{types-C_7nyzr1.cjs → types-CeaeaZdP.cjs} +393 -22
- package/dist/src/{types-Cbd8uOMq.js → types-Dm9JM6Vb.js} +368 -23
- package/dist/src/{util-BHGHw5G1.js → util-BYvQUPp7.js} +138 -36
- package/dist/src/{util-B9vlHIIh.cjs → util-Bxn8emtE.cjs} +15 -168
- package/dist/src/{util-ZzmqNPlg.js → util-C8e5uydV.js} +19 -142
- package/dist/src/{util-CMy69ZgQ.js → util-C9J8ahRn.js} +18 -4
- package/dist/src/{util-BzMcevZc.cjs → util-CN3SrLT4.cjs} +18 -4
- package/dist/src/{util-BV4XUC0n.js → util-D3q0WQ-0.js} +18 -4
- package/dist/src/{util-Dnmk2mBQ.js → util-D9TisOyk.js} +18 -4
- package/dist/src/{util-B3xGByQh.js → util-DDs-7g6-.js} +138 -36
- package/dist/src/{util-Bv6uGDfH.js → util-DvU2Pw8c.js} +138 -36
- package/dist/src/{util-C1CeHl-P.js → util-DxWpWjhc.js} +13 -136
- package/dist/src/{util-BRYkYPTd.js → util-oGMLA7vc.js} +17 -140
- package/dist/src/{util-DGNOS1db.cjs → util-olYL5C6N.cjs} +143 -35
- package/dist/src/{utils-Cz9qXqII.cjs → utils-B05gLxER.cjs} +6 -4
- package/dist/src/{utils-f2-Moju7.js → utils-BLJKfv0y.js} +6 -4
- package/dist/src/{utils-dLokC-eR.js → utils-DJfvjyMj.js} +6 -4
- package/dist/src/{utils-XiOAgly5.js → utils-hXtCYanr.js} +6 -4
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +66 -57
- package/dist/src/app/assets/index-4LKxG2CG.js +0 -439
- package/dist/src/app/assets/index-C3zcsZFQ.css +0 -1
- package/dist/src/app/assets/scroll-timeline-BdJZVXlz.js +0 -1
- package/dist/src/app/assets/sync-9qqYcY-B.js +0 -4
- package/dist/src/app/assets/vendor-charts-BnDWwBlI.js +0 -36
- package/dist/src/app/assets/vendor-markdown-0tekx3KX.js +0 -29
- package/dist/src/app/assets/vendor-react-AtKqiNEf.js +0 -4
- package/dist/src/app/assets/vendor-syntax-D06x6TQF.js +0 -2
- package/dist/src/app/assets/vendor-utils-BvMHZmO7.js +0 -37
- package/dist/src/app/tsconfig.app.tsbuildinfo +0 -1
- package/dist/src/cache-BVeDlD87.js +0 -726
- package/dist/src/cache-C4Nxf52C.js +0 -756
- package/dist/src/cache-CeUpFm3M.cjs +0 -5
- package/dist/src/cache-Dh5WtQps.cjs +0 -816
- package/dist/src/cache-i1P6crbO.js +0 -756
- package/dist/src/cache-n-RCJ-hL.js +0 -6
- package/dist/src/cloud-BBh91EUK.js +0 -4
- package/dist/src/codex-sdk-C6UMlxwV.js +0 -665
- package/dist/src/codex-sdk-DUwKWezN.js +0 -665
- package/dist/src/codex-sdk-GGAw0qbD.js +0 -666
- package/dist/src/codex-sdk-fAO0c3yA.cjs +0 -669
- package/dist/src/eval-B3r2CVXr.js +0 -15
- package/dist/src/evalResult-5xwYnECe.js +0 -12
- package/dist/src/evalResult-71lY93Kj.cjs +0 -10
- package/dist/src/evalResult-Dx5P5cIv.js +0 -10
- package/dist/src/evaluator-Jx6bRZV6.js +0 -36
- package/dist/src/fetch-BxNb_Lp3.js +0 -5
- package/dist/src/graders-B_pgMLS2.js +0 -34
- package/dist/src/graders-DErokPDO.cjs +0 -32
- package/dist/src/graders-DR_uNe54.js +0 -32
- package/dist/src/graders-w3176Wz-.js +0 -32
- package/dist/src/openclaw-CSugPYAr.cjs +0 -586
- package/dist/src/openclaw-DiSz3I5L.js +0 -582
- package/dist/src/openclaw-DuvJKEW5.js +0 -580
- package/dist/src/openclaw-tiVYRtr-.js +0 -580
- package/dist/src/providers-B7V0njNs.js +0 -32
- package/dist/src/providers-BEwbhv0X.js +0 -30
- package/dist/src/providers-CH3C7zf7.js +0 -30
- package/dist/src/providers-zyB6k_38.cjs +0 -31
- package/dist/src/rubyUtils-BUHu6PhO.js +0 -5
- package/dist/src/rubyUtils-CP42kMvq.cjs +0 -4
- package/dist/src/server-DA4Cyrrq.js +0 -7
- package/dist/src/server-Dulb-4-K.cjs +0 -5
- package/dist/src/store-CXS-Q_91.js +0 -6
- package/dist/src/store-eYkaKMwq.cjs +0 -5
- package/dist/src/telemetry-BpMfhthR.cjs +0 -5
- package/dist/src/telemetry-Dw38hanS.js +0 -7
- package/dist/src/tokenUsageUtils-BDGe-iyI.js +0 -138
- package/dist/src/transform-DTGDnAzW.js +0 -6
- package/dist/src/transform-m3qNw4KP.cjs +0 -5
|
@@ -0,0 +1,1164 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { N as state, O as getEnvString, m as sanitizeObject, p as normalizeFieldName, s as logger, u as REDACTED } from "./logger-DksKw1Qc.js";
|
|
3
|
+
import { i as resolvePackageEntryPoint, r as importModule, t as getDirectory } from "./esm-tVgYPY-f.js";
|
|
4
|
+
import { n as withGenAISpan, t as getTraceparent } from "./genaiTracer-DWdZ28hY.js";
|
|
5
|
+
import { n as renderVarsInObject } from "./render-CMEpfLaO.js";
|
|
6
|
+
import { t as providerRegistry } from "./providerRegistry-CWoPjKFZ.js";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import fs from "fs";
|
|
9
|
+
import path from "path";
|
|
10
|
+
import dedent from "dedent";
|
|
11
|
+
import crypto from "crypto";
|
|
12
|
+
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
13
|
+
//#region src/providers/openai/codex-sdk.ts
|
|
14
|
+
const MINIMAL_CLI_ENV_KEYS = [
|
|
15
|
+
"PATH",
|
|
16
|
+
"Path",
|
|
17
|
+
"HOME",
|
|
18
|
+
"USER",
|
|
19
|
+
"USERNAME",
|
|
20
|
+
"USERPROFILE",
|
|
21
|
+
"TMPDIR",
|
|
22
|
+
"TMP",
|
|
23
|
+
"TEMP",
|
|
24
|
+
"SHELL",
|
|
25
|
+
"COMSPEC",
|
|
26
|
+
"SystemRoot",
|
|
27
|
+
"PATHEXT",
|
|
28
|
+
"LANG",
|
|
29
|
+
"LC_ALL",
|
|
30
|
+
"TERM"
|
|
31
|
+
];
|
|
32
|
+
const COMMON_OPTIONAL_PROCESS_ENV_KEYS = [
|
|
33
|
+
"CODEX_HOME",
|
|
34
|
+
"HTTP_PROXY",
|
|
35
|
+
"HTTPS_PROXY",
|
|
36
|
+
"ALL_PROXY",
|
|
37
|
+
"NO_PROXY",
|
|
38
|
+
"SSL_CERT_FILE",
|
|
39
|
+
"SSL_CERT_DIR",
|
|
40
|
+
"REQUESTS_CA_BUNDLE",
|
|
41
|
+
"NODE_EXTRA_CA_CERTS",
|
|
42
|
+
"SSH_AUTH_SOCK",
|
|
43
|
+
"GIT_SSH_COMMAND"
|
|
44
|
+
];
|
|
45
|
+
const CodexCliEnvValueSchema = z.union([
|
|
46
|
+
z.string(),
|
|
47
|
+
z.number(),
|
|
48
|
+
z.boolean()
|
|
49
|
+
]).transform(String);
|
|
50
|
+
const OpenAICodexSDKConfigShape = {
|
|
51
|
+
basePath: z.string().optional(),
|
|
52
|
+
prefix: z.string().optional(),
|
|
53
|
+
suffix: z.string().optional(),
|
|
54
|
+
provider: z.unknown().optional(),
|
|
55
|
+
linkedTargetId: z.string().optional(),
|
|
56
|
+
apiKey: z.string().min(1).optional(),
|
|
57
|
+
base_url: z.string().min(1).optional(),
|
|
58
|
+
working_dir: z.string().min(1).optional(),
|
|
59
|
+
additional_directories: z.array(z.string().min(1)).optional(),
|
|
60
|
+
skip_git_repo_check: z.boolean().optional(),
|
|
61
|
+
codex_path_override: z.string().min(1).optional(),
|
|
62
|
+
model: z.string().min(1).optional(),
|
|
63
|
+
sandbox_mode: z.enum([
|
|
64
|
+
"read-only",
|
|
65
|
+
"workspace-write",
|
|
66
|
+
"danger-full-access"
|
|
67
|
+
]).optional(),
|
|
68
|
+
model_reasoning_effort: z.enum([
|
|
69
|
+
"minimal",
|
|
70
|
+
"low",
|
|
71
|
+
"medium",
|
|
72
|
+
"high",
|
|
73
|
+
"xhigh"
|
|
74
|
+
]).optional(),
|
|
75
|
+
network_access_enabled: z.boolean().optional(),
|
|
76
|
+
web_search_enabled: z.boolean().optional(),
|
|
77
|
+
web_search_mode: z.enum([
|
|
78
|
+
"disabled",
|
|
79
|
+
"cached",
|
|
80
|
+
"live"
|
|
81
|
+
]).optional(),
|
|
82
|
+
collaboration_mode: z.enum(["coding", "plan"]).optional(),
|
|
83
|
+
approval_policy: z.enum([
|
|
84
|
+
"never",
|
|
85
|
+
"on-request",
|
|
86
|
+
"on-failure",
|
|
87
|
+
"untrusted"
|
|
88
|
+
]).optional(),
|
|
89
|
+
thread_id: z.string().min(1).optional(),
|
|
90
|
+
persist_threads: z.boolean().optional(),
|
|
91
|
+
thread_pool_size: z.number().int().positive().optional(),
|
|
92
|
+
output_schema: z.record(z.string(), z.unknown()).optional(),
|
|
93
|
+
cli_env: z.record(z.string(), CodexCliEnvValueSchema).optional(),
|
|
94
|
+
inherit_process_env: z.boolean().optional(),
|
|
95
|
+
enable_streaming: z.boolean().optional(),
|
|
96
|
+
deep_tracing: z.boolean().optional(),
|
|
97
|
+
cli_config: z.record(z.string(), z.unknown()).optional()
|
|
98
|
+
};
|
|
99
|
+
const OpenAICodexSDKConfigSchema = z.object(OpenAICodexSDKConfigShape).strict();
|
|
100
|
+
const OpenAICodexSDKMergedPromptConfigSchema = z.object(OpenAICodexSDKConfigShape).strip();
|
|
101
|
+
function parseCodexConfig(config, options = {}) {
|
|
102
|
+
const schema = options.stripUnknownKeys ? OpenAICodexSDKMergedPromptConfigSchema : OpenAICodexSDKConfigSchema;
|
|
103
|
+
try {
|
|
104
|
+
return schema.parse(config ?? {});
|
|
105
|
+
} catch (error) {
|
|
106
|
+
if (error instanceof z.ZodError) {
|
|
107
|
+
const issues = error.issues.map((issue) => {
|
|
108
|
+
return `${issue.path.length > 0 ? issue.path.join(".") : "(root)"}: ${issue.message}`;
|
|
109
|
+
}).join("; ");
|
|
110
|
+
throw new Error(`Invalid OpenAI Codex SDK config: ${issues}`);
|
|
111
|
+
}
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function getMinimalProcessEnv() {
|
|
116
|
+
const env = {};
|
|
117
|
+
for (const key of MINIMAL_CLI_ENV_KEYS) {
|
|
118
|
+
const value = process.env[key];
|
|
119
|
+
if (typeof value === "string" && value.length > 0) env[key] = value;
|
|
120
|
+
}
|
|
121
|
+
return env;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Helper to load the OpenAI Codex SDK ESM module
|
|
125
|
+
* Uses resolvePackageEntryPoint to handle ESM-only packages with restrictive exports
|
|
126
|
+
*/
|
|
127
|
+
async function loadCodexSDK() {
|
|
128
|
+
const basePaths = [
|
|
129
|
+
state.basePath ? path.resolve(state.basePath) : void 0,
|
|
130
|
+
process.cwd(),
|
|
131
|
+
path.resolve(getDirectory(), ".."),
|
|
132
|
+
path.resolve(getDirectory(), "../..")
|
|
133
|
+
].filter((candidate) => Boolean(candidate));
|
|
134
|
+
let codexPath = null;
|
|
135
|
+
for (const basePath of new Set(basePaths)) {
|
|
136
|
+
codexPath = resolvePackageEntryPoint("@openai/codex-sdk", basePath);
|
|
137
|
+
if (codexPath) break;
|
|
138
|
+
}
|
|
139
|
+
if (!codexPath) throw new Error(dedent`The @openai/codex-sdk package is required but not installed.
|
|
140
|
+
|
|
141
|
+
To use the OpenAI Codex SDK provider, install it with:
|
|
142
|
+
npm install @openai/codex-sdk
|
|
143
|
+
|
|
144
|
+
Requires Node.js 20.20+ or 22.22+.
|
|
145
|
+
|
|
146
|
+
For more information, see: https://www.promptfoo.dev/docs/providers/openai-codex-sdk/`);
|
|
147
|
+
try {
|
|
148
|
+
return await importModule(codexPath);
|
|
149
|
+
} catch (err) {
|
|
150
|
+
logger.error(`Failed to load OpenAI Codex SDK: ${err}`);
|
|
151
|
+
if (err.stack) logger.error(err.stack);
|
|
152
|
+
throw new Error(dedent`Failed to load @openai/codex-sdk.
|
|
153
|
+
|
|
154
|
+
The package was found but could not be loaded. This may be due to:
|
|
155
|
+
- Incompatible Node.js version (requires Node.js 20.20+ or 22.22+)
|
|
156
|
+
- Corrupted installation
|
|
157
|
+
|
|
158
|
+
Try reinstalling:
|
|
159
|
+
npm install @openai/codex-sdk
|
|
160
|
+
|
|
161
|
+
For more information, see: https://www.promptfoo.dev/docs/providers/openai-codex-sdk/`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const CODEX_MODEL_PRICING = {
|
|
165
|
+
"gpt-5.4": {
|
|
166
|
+
input: 2.5,
|
|
167
|
+
output: 15,
|
|
168
|
+
cache_read: .25
|
|
169
|
+
},
|
|
170
|
+
"gpt-5.4-pro": {
|
|
171
|
+
input: 30,
|
|
172
|
+
output: 180,
|
|
173
|
+
cache_read: 30
|
|
174
|
+
},
|
|
175
|
+
"gpt-5.3-codex": {
|
|
176
|
+
input: 1.75,
|
|
177
|
+
output: 14,
|
|
178
|
+
cache_read: .175
|
|
179
|
+
},
|
|
180
|
+
"gpt-5.3-codex-spark": {
|
|
181
|
+
input: .5,
|
|
182
|
+
output: 4,
|
|
183
|
+
cache_read: .05
|
|
184
|
+
},
|
|
185
|
+
"gpt-5.2": {
|
|
186
|
+
input: 1.75,
|
|
187
|
+
output: 14,
|
|
188
|
+
cache_read: .175
|
|
189
|
+
},
|
|
190
|
+
"gpt-5.2-codex": {
|
|
191
|
+
input: 1.75,
|
|
192
|
+
output: 14,
|
|
193
|
+
cache_read: .175
|
|
194
|
+
},
|
|
195
|
+
"gpt-5.1-codex": {
|
|
196
|
+
input: 2,
|
|
197
|
+
output: 8,
|
|
198
|
+
cache_read: .2
|
|
199
|
+
},
|
|
200
|
+
"gpt-5.1-codex-max": {
|
|
201
|
+
input: 3,
|
|
202
|
+
output: 12,
|
|
203
|
+
cache_read: .3
|
|
204
|
+
},
|
|
205
|
+
"gpt-5.1-codex-mini": {
|
|
206
|
+
input: .5,
|
|
207
|
+
output: 2,
|
|
208
|
+
cache_read: .05
|
|
209
|
+
},
|
|
210
|
+
"gpt-5-codex": {
|
|
211
|
+
input: 2,
|
|
212
|
+
output: 8,
|
|
213
|
+
cache_read: .2
|
|
214
|
+
},
|
|
215
|
+
"gpt-5-codex-mini": {
|
|
216
|
+
input: .5,
|
|
217
|
+
output: 2,
|
|
218
|
+
cache_read: .05
|
|
219
|
+
},
|
|
220
|
+
"gpt-5": {
|
|
221
|
+
input: 2,
|
|
222
|
+
output: 8,
|
|
223
|
+
cache_read: .2
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
var OpenAICodexSDKProvider = class OpenAICodexSDKProvider {
|
|
227
|
+
static OPENAI_MODELS = [
|
|
228
|
+
"gpt-5.4",
|
|
229
|
+
"gpt-5.4-pro",
|
|
230
|
+
"gpt-5.3-codex",
|
|
231
|
+
"gpt-5.3-codex-spark",
|
|
232
|
+
"gpt-5.2",
|
|
233
|
+
"gpt-5.2-codex",
|
|
234
|
+
"gpt-5.1-codex",
|
|
235
|
+
"gpt-5.1-codex-max",
|
|
236
|
+
"gpt-5.1-codex-mini",
|
|
237
|
+
"gpt-5-codex",
|
|
238
|
+
"gpt-5-codex-mini",
|
|
239
|
+
"gpt-5"
|
|
240
|
+
];
|
|
241
|
+
config;
|
|
242
|
+
env;
|
|
243
|
+
apiKey;
|
|
244
|
+
providerId = "openai:codex-sdk";
|
|
245
|
+
codexModule;
|
|
246
|
+
codexInstances = /* @__PURE__ */ new Map();
|
|
247
|
+
threads = /* @__PURE__ */ new Map();
|
|
248
|
+
threadRunQueues = /* @__PURE__ */ new Map();
|
|
249
|
+
deepTracingWarningShown = false;
|
|
250
|
+
ignoredProviderEnvWarningShown = false;
|
|
251
|
+
omittedProcessEnvWarningShown = false;
|
|
252
|
+
constructor(options = {}) {
|
|
253
|
+
const { config, env, id } = options;
|
|
254
|
+
this.config = parseCodexConfig(config);
|
|
255
|
+
this.env = env;
|
|
256
|
+
this.apiKey = this.getApiKey();
|
|
257
|
+
this.providerId = id ?? this.providerId;
|
|
258
|
+
providerRegistry.register(this);
|
|
259
|
+
if (this.config.model && !OpenAICodexSDKProvider.OPENAI_MODELS.includes(this.config.model)) logger.warn(`Using unknown model for OpenAI Codex SDK: ${this.config.model}`);
|
|
260
|
+
}
|
|
261
|
+
id() {
|
|
262
|
+
return this.providerId;
|
|
263
|
+
}
|
|
264
|
+
getApiKey(config = this.config) {
|
|
265
|
+
return config?.apiKey || this.env?.OPENAI_API_KEY || this.env?.CODEX_API_KEY || getEnvString("OPENAI_API_KEY") || getEnvString("CODEX_API_KEY");
|
|
266
|
+
}
|
|
267
|
+
requiresApiKey() {
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
toString() {
|
|
271
|
+
return "[OpenAI Codex SDK Provider]";
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Safely tear down a Codex instance by calling its cleanup method
|
|
275
|
+
* (destroy, cleanup, or close -- whichever is available).
|
|
276
|
+
*/
|
|
277
|
+
async destroyInstance(instance) {
|
|
278
|
+
if (typeof instance.destroy === "function") await instance.destroy();
|
|
279
|
+
else if (typeof instance.cleanup === "function") await instance.cleanup();
|
|
280
|
+
else if (typeof instance.close === "function") await instance.close();
|
|
281
|
+
}
|
|
282
|
+
async cleanup() {
|
|
283
|
+
this.threads.clear();
|
|
284
|
+
this.threadRunQueues.clear();
|
|
285
|
+
for (const instance of this.codexInstances.values()) try {
|
|
286
|
+
await this.destroyInstance(instance);
|
|
287
|
+
} catch (error) {
|
|
288
|
+
logger.warn("[CodexSDK] Error during cleanup", { error });
|
|
289
|
+
}
|
|
290
|
+
this.codexInstances.clear();
|
|
291
|
+
}
|
|
292
|
+
async shutdown() {
|
|
293
|
+
try {
|
|
294
|
+
await this.cleanup();
|
|
295
|
+
} finally {
|
|
296
|
+
providerRegistry.unregister(this);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
prepareEnvironment(config, traceparent, apiKey = this.getApiKey(config)) {
|
|
300
|
+
const inheritProcessEnv = config.inherit_process_env === true;
|
|
301
|
+
const cliEnv = Object.fromEntries(Object.entries(config.cli_env ?? {}).map(([key, value]) => [key, String(value)]));
|
|
302
|
+
const env = {
|
|
303
|
+
...inheritProcessEnv ? process.env : getMinimalProcessEnv(),
|
|
304
|
+
...cliEnv
|
|
305
|
+
};
|
|
306
|
+
const ignoredProviderEnvKeys = Object.keys(this.env ?? {}).filter((key) => key !== "OPENAI_API_KEY" && key !== "CODEX_API_KEY" && !(key in (config.cli_env ?? {}))).sort();
|
|
307
|
+
if (ignoredProviderEnvKeys.length > 0 && !this.ignoredProviderEnvWarningShown) {
|
|
308
|
+
logger.warn("[CodexSDK] Ignoring promptfoo-level env overrides for the Codex CLI process. Move these keys into config.cli_env if Codex shell commands need them.", { envKeys: ignoredProviderEnvKeys });
|
|
309
|
+
this.ignoredProviderEnvWarningShown = true;
|
|
310
|
+
}
|
|
311
|
+
if (!inheritProcessEnv && !this.omittedProcessEnvWarningShown) {
|
|
312
|
+
const omittedProcessEnvKeys = this.getOmittedOptionalProcessEnvKeys(config, env);
|
|
313
|
+
if (omittedProcessEnvKeys.length > 0) {
|
|
314
|
+
logger.warn("[CodexSDK] Optional Codex CLI process env vars are not inherited by default. Move these keys into config.cli_env or set inherit_process_env: true if Codex CLI commands need them.", { envKeys: omittedProcessEnvKeys });
|
|
315
|
+
this.omittedProcessEnvWarningShown = true;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
const sortedEnv = {};
|
|
319
|
+
for (const key of Object.keys(env).sort()) if (env[key] !== void 0) sortedEnv[key] = env[key];
|
|
320
|
+
if (apiKey) {
|
|
321
|
+
sortedEnv.OPENAI_API_KEY = apiKey;
|
|
322
|
+
sortedEnv.CODEX_API_KEY = apiKey;
|
|
323
|
+
}
|
|
324
|
+
if (config.deep_tracing) {
|
|
325
|
+
if (!sortedEnv.OTEL_EXPORTER_OTLP_ENDPOINT) sortedEnv.OTEL_EXPORTER_OTLP_ENDPOINT = "http://127.0.0.1:4318";
|
|
326
|
+
if (!sortedEnv.OTEL_EXPORTER_OTLP_PROTOCOL) sortedEnv.OTEL_EXPORTER_OTLP_PROTOCOL = "http/json";
|
|
327
|
+
if (!sortedEnv.OTEL_SERVICE_NAME) sortedEnv.OTEL_SERVICE_NAME = "codex-cli";
|
|
328
|
+
if (!sortedEnv.OTEL_TRACES_EXPORTER) sortedEnv.OTEL_TRACES_EXPORTER = "otlp";
|
|
329
|
+
if (traceparent) sortedEnv.TRACEPARENT = traceparent;
|
|
330
|
+
logger.debug("[CodexSDK] Injecting OTEL config for deep tracing", {
|
|
331
|
+
traceparent: traceparent || "(none - CLI will start own trace)",
|
|
332
|
+
endpoint: sortedEnv.OTEL_EXPORTER_OTLP_ENDPOINT,
|
|
333
|
+
userConfigured: {
|
|
334
|
+
endpoint: !!env.OTEL_EXPORTER_OTLP_ENDPOINT,
|
|
335
|
+
protocol: !!env.OTEL_EXPORTER_OTLP_PROTOCOL,
|
|
336
|
+
serviceName: !!env.OTEL_SERVICE_NAME
|
|
337
|
+
}
|
|
338
|
+
});
|
|
339
|
+
} else delete sortedEnv.TRACEPARENT;
|
|
340
|
+
return sortedEnv;
|
|
341
|
+
}
|
|
342
|
+
getOmittedOptionalProcessEnvKeys(config, env) {
|
|
343
|
+
const shouldWarnForSshEnv = config.network_access_enabled === true || config.web_search_enabled === true || config.web_search_mode === "live";
|
|
344
|
+
return COMMON_OPTIONAL_PROCESS_ENV_KEYS.filter((key) => typeof process.env[key] === "string" && !(key in env) && (shouldWarnForSshEnv || key !== "SSH_AUTH_SOCK" && key !== "GIT_SSH_COMMAND"));
|
|
345
|
+
}
|
|
346
|
+
getResolvedCliConfig(config) {
|
|
347
|
+
if (!config.cli_config && !config.collaboration_mode) return;
|
|
348
|
+
return {
|
|
349
|
+
...config.cli_config ?? {},
|
|
350
|
+
...config.collaboration_mode ? { collaboration_mode: config.collaboration_mode } : {}
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
getSkillRootPrefixes(env, workingDir) {
|
|
354
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
355
|
+
const addPrefix = (candidate) => {
|
|
356
|
+
if (!candidate) return;
|
|
357
|
+
const normalized = candidate.replace(/\\/g, "/").replace(/\/+$/g, "");
|
|
358
|
+
if (normalized) prefixes.add(normalized);
|
|
359
|
+
};
|
|
360
|
+
addPrefix(env.CODEX_HOME);
|
|
361
|
+
addPrefix("/etc/codex");
|
|
362
|
+
if (workingDir) {
|
|
363
|
+
const resolvedWorkingDir = path.resolve(workingDir).replace(/\\/g, "/");
|
|
364
|
+
addPrefix(path.posix.join(resolvedWorkingDir, ".agents"));
|
|
365
|
+
const gitRoot = this.findGitRepositoryRoot(resolvedWorkingDir);
|
|
366
|
+
if (gitRoot) addPrefix(path.posix.join(gitRoot.replace(/\\/g, "/"), ".agents"));
|
|
367
|
+
}
|
|
368
|
+
const homeDir = env.HOME || env.USERPROFILE || process.env.HOME || process.env.USERPROFILE;
|
|
369
|
+
if (homeDir) addPrefix(path.posix.join(homeDir.replace(/\\/g, "/"), ".codex"));
|
|
370
|
+
return Array.from(prefixes);
|
|
371
|
+
}
|
|
372
|
+
isValidCodexSkillName(name) {
|
|
373
|
+
return /^[A-Za-z0-9._:-]+$/.test(name);
|
|
374
|
+
}
|
|
375
|
+
extractSkillPathCandidates(text, skillRootPrefixes = []) {
|
|
376
|
+
const matches = /* @__PURE__ */ new Map();
|
|
377
|
+
for (const rawToken of text.split(/\s+/)) {
|
|
378
|
+
const token = rawToken.replace(/^[`"'([{<]+|[`"',;:)\]}>]+$/g, "").trim();
|
|
379
|
+
if (!token) continue;
|
|
380
|
+
const normalizedPath = token.replace(/\\/g, "/");
|
|
381
|
+
const repoMatch = normalizedPath.match(/^\.agents\/skills\/([^/\s]+)\/SKILL\.md$/);
|
|
382
|
+
if (repoMatch) {
|
|
383
|
+
if (this.isValidCodexSkillName(repoMatch[1])) matches.set(normalizedPath, {
|
|
384
|
+
name: repoMatch[1],
|
|
385
|
+
path: normalizedPath
|
|
386
|
+
});
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
const matchingRoot = skillRootPrefixes.find((prefix) => normalizedPath.startsWith(`${prefix}/skills/`));
|
|
390
|
+
if (!matchingRoot) continue;
|
|
391
|
+
const customRootMatch = normalizedPath.slice(matchingRoot.length + 1).match(/^skills\/([^/\s]+)\/SKILL\.md$/);
|
|
392
|
+
if (customRootMatch && this.isValidCodexSkillName(customRootMatch[1])) matches.set(normalizedPath, {
|
|
393
|
+
name: customRootMatch[1],
|
|
394
|
+
path: normalizedPath
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
return Array.from(matches.values());
|
|
398
|
+
}
|
|
399
|
+
extractSkillCallsFromItems(items, skillRootPrefixes = [], options = {}) {
|
|
400
|
+
const skillCalls = /* @__PURE__ */ new Map();
|
|
401
|
+
for (const item of items) {
|
|
402
|
+
if (item?.type !== "command_execution") continue;
|
|
403
|
+
if (options.requireSuccessfulCommand && !this.isSuccessfulCommandExecution(item)) continue;
|
|
404
|
+
const command = typeof item.command === "string" && item.command.trim() ? item.command : void 0;
|
|
405
|
+
if (!command) continue;
|
|
406
|
+
for (const skillPath of this.extractSkillPathCandidates(command, skillRootPrefixes)) {
|
|
407
|
+
const existing = skillCalls.get(skillPath.path) ?? {
|
|
408
|
+
name: skillPath.name,
|
|
409
|
+
path: skillPath.path
|
|
410
|
+
};
|
|
411
|
+
skillCalls.set(skillPath.path, existing);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return Array.from(skillCalls.values()).map((skillCall) => ({
|
|
415
|
+
name: skillCall.name,
|
|
416
|
+
path: skillCall.path,
|
|
417
|
+
source: "heuristic"
|
|
418
|
+
}));
|
|
419
|
+
}
|
|
420
|
+
buildSkillMetadata(items, skillRootPrefixes = []) {
|
|
421
|
+
if (!Array.isArray(items) || items.length === 0) return;
|
|
422
|
+
const attemptedSkillCalls = this.extractSkillCallsFromItems(items, skillRootPrefixes);
|
|
423
|
+
const skillCalls = this.extractSkillCallsFromItems(items, skillRootPrefixes, { requireSuccessfulCommand: true });
|
|
424
|
+
if (skillCalls.length === 0 && attemptedSkillCalls.length <= skillCalls.length) return;
|
|
425
|
+
return {
|
|
426
|
+
attemptedSkillCalls,
|
|
427
|
+
skillCalls
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
isSuccessfulCommandExecution(item) {
|
|
431
|
+
if (item?.type !== "command_execution") return false;
|
|
432
|
+
if (typeof item.status === "string" && item.status !== "completed") return false;
|
|
433
|
+
if (typeof item.exit_code === "number" && item.exit_code !== 0) return false;
|
|
434
|
+
return true;
|
|
435
|
+
}
|
|
436
|
+
validateWorkingDirectory(workingDir, skipGitCheck = false) {
|
|
437
|
+
let stats;
|
|
438
|
+
try {
|
|
439
|
+
stats = fs.statSync(workingDir);
|
|
440
|
+
} catch (err) {
|
|
441
|
+
throw new Error(`Working directory ${workingDir} does not exist or isn't accessible: ${err.message}`);
|
|
442
|
+
}
|
|
443
|
+
if (!stats.isDirectory()) throw new Error(`Working directory ${workingDir} is not a directory`);
|
|
444
|
+
if (!skipGitCheck && !this.isInsideGitRepository(workingDir)) throw new Error(dedent`Working directory ${workingDir} is not inside a Git repository.
|
|
445
|
+
|
|
446
|
+
Codex requires a Git repository by default to prevent unrecoverable errors.
|
|
447
|
+
|
|
448
|
+
To bypass this check, set skip_git_repo_check: true in your provider config.`);
|
|
449
|
+
}
|
|
450
|
+
isInsideGitRepository(workingDir) {
|
|
451
|
+
return this.findGitRepositoryRoot(workingDir) !== void 0;
|
|
452
|
+
}
|
|
453
|
+
findGitRepositoryRoot(workingDir) {
|
|
454
|
+
let currentDir = path.resolve(workingDir);
|
|
455
|
+
while (true) {
|
|
456
|
+
if (fs.existsSync(path.join(currentDir, ".git"))) return currentDir;
|
|
457
|
+
const parentDir = path.dirname(currentDir);
|
|
458
|
+
if (parentDir === currentDir) return;
|
|
459
|
+
currentDir = parentDir;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Build Codex constructor options from provider config.
|
|
464
|
+
* Used when creating both local (deep-tracing) and cached instances.
|
|
465
|
+
*/
|
|
466
|
+
buildCodexOptions(env, config, apiKey = this.getApiKey(config)) {
|
|
467
|
+
const cliConfig = this.getResolvedCliConfig(config);
|
|
468
|
+
return {
|
|
469
|
+
env,
|
|
470
|
+
...apiKey ? { apiKey } : {},
|
|
471
|
+
...config.codex_path_override ? { codexPathOverride: config.codex_path_override } : {},
|
|
472
|
+
...config.base_url ? { baseUrl: config.base_url } : {},
|
|
473
|
+
...cliConfig ? { config: cliConfig } : {}
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
buildThreadOptions(config) {
|
|
477
|
+
return {
|
|
478
|
+
workingDirectory: config.working_dir,
|
|
479
|
+
skipGitRepoCheck: config.skip_git_repo_check ?? false,
|
|
480
|
+
...config.model ? { model: config.model } : {},
|
|
481
|
+
...config.additional_directories?.length ? { additionalDirectories: config.additional_directories } : {},
|
|
482
|
+
...config.sandbox_mode ? { sandboxMode: config.sandbox_mode } : {},
|
|
483
|
+
...config.model_reasoning_effort ? { modelReasoningEffort: config.model_reasoning_effort } : {},
|
|
484
|
+
...config.network_access_enabled === void 0 ? {} : { networkAccessEnabled: config.network_access_enabled },
|
|
485
|
+
...config.web_search_mode ? { webSearchMode: config.web_search_mode } : {},
|
|
486
|
+
...config.web_search_enabled !== void 0 && !config.web_search_mode ? { webSearchEnabled: config.web_search_enabled } : {},
|
|
487
|
+
...config.approval_policy ? { approvalPolicy: config.approval_policy } : {}
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
async getOrCreateThread(config, cacheKey, instanceKey, instance) {
|
|
491
|
+
const threadOptions = this.buildThreadOptions(config);
|
|
492
|
+
if (config.deep_tracing) return instance.startThread(threadOptions);
|
|
493
|
+
if (config.thread_id) {
|
|
494
|
+
const threadIdCacheKey = `${instanceKey}:${config.thread_id}`;
|
|
495
|
+
const cached = this.threads.get(threadIdCacheKey);
|
|
496
|
+
if (cached) return cached;
|
|
497
|
+
const thread = instance.resumeThread(config.thread_id, threadOptions);
|
|
498
|
+
if (config.persist_threads) this.threads.set(threadIdCacheKey, thread);
|
|
499
|
+
return thread;
|
|
500
|
+
}
|
|
501
|
+
if (config.persist_threads && cacheKey) {
|
|
502
|
+
const cached = this.threads.get(cacheKey);
|
|
503
|
+
if (cached) return cached;
|
|
504
|
+
const poolSize = config.thread_pool_size ?? 1;
|
|
505
|
+
if (this.threads.size >= poolSize) {
|
|
506
|
+
const oldestKey = this.threads.keys().next().value;
|
|
507
|
+
if (oldestKey) this.threads.delete(oldestKey);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
const thread = instance.startThread(threadOptions);
|
|
511
|
+
if (config.persist_threads && cacheKey) this.threads.set(cacheKey, thread);
|
|
512
|
+
return thread;
|
|
513
|
+
}
|
|
514
|
+
async runStreaming(thread, prompt, runOptions, callOptions, skillRootPrefixes = []) {
|
|
515
|
+
const { events } = await thread.runStreamed(prompt, runOptions);
|
|
516
|
+
const tracer = trace.getTracer("promptfoo.codex-sdk");
|
|
517
|
+
const state = this.createCodexStreamingState(prompt);
|
|
518
|
+
try {
|
|
519
|
+
for await (const event of events) {
|
|
520
|
+
const eventTime = Date.now();
|
|
521
|
+
if (callOptions?.abortSignal?.aborted) throw this.createAbortError("OpenAI Codex SDK call aborted");
|
|
522
|
+
this.handleStreamingEvent(event, state, tracer, eventTime, skillRootPrefixes);
|
|
523
|
+
state.lastEventTime = eventTime;
|
|
524
|
+
}
|
|
525
|
+
} finally {
|
|
526
|
+
this.endUnclosedStreamingSpans(state);
|
|
527
|
+
}
|
|
528
|
+
return this.buildStreamingTurnResult(state);
|
|
529
|
+
}
|
|
530
|
+
createCodexStreamingState(prompt) {
|
|
531
|
+
return {
|
|
532
|
+
items: [],
|
|
533
|
+
usage: void 0,
|
|
534
|
+
activeSpans: /* @__PURE__ */ new Map(),
|
|
535
|
+
itemStartTimes: /* @__PURE__ */ new Map(),
|
|
536
|
+
lastEventTime: Date.now(),
|
|
537
|
+
reasoningTexts: [],
|
|
538
|
+
conversationMessages: [{
|
|
539
|
+
role: "user",
|
|
540
|
+
content: this.formatPromptInputForTrace(prompt)
|
|
541
|
+
}]
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
handleStreamingEvent(event, state, tracer, eventTime, skillRootPrefixes) {
|
|
545
|
+
switch (event.type) {
|
|
546
|
+
case "item.started":
|
|
547
|
+
this.handleStreamingItemStarted(event, state, tracer, eventTime);
|
|
548
|
+
return;
|
|
549
|
+
case "item.completed":
|
|
550
|
+
this.handleStreamingItemCompleted(event, state, tracer, eventTime, skillRootPrefixes);
|
|
551
|
+
return;
|
|
552
|
+
case "item.updated":
|
|
553
|
+
this.handleStreamingItemUpdated(event, state, skillRootPrefixes);
|
|
554
|
+
return;
|
|
555
|
+
case "turn.completed":
|
|
556
|
+
state.usage = event.usage;
|
|
557
|
+
logger.debug("Codex turn completed", { usage: state.usage });
|
|
558
|
+
return;
|
|
559
|
+
case "turn.failed": {
|
|
560
|
+
const errorMsg = event.error?.message || "Turn failed";
|
|
561
|
+
logger.error("Codex turn failed", { error: errorMsg });
|
|
562
|
+
throw new Error(`Codex turn failed: ${errorMsg}`);
|
|
563
|
+
}
|
|
564
|
+
case "error": {
|
|
565
|
+
const errorMsg = typeof event.message === "string" && event.message ? event.message : "Stream failed";
|
|
566
|
+
logger.error("Codex stream error", { error: errorMsg });
|
|
567
|
+
throw new Error(`Codex stream error: ${errorMsg}`);
|
|
568
|
+
}
|
|
569
|
+
case "thread.started":
|
|
570
|
+
case "turn.started": return;
|
|
571
|
+
default: logger.debug("Codex unknown event type", { type: event.type });
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
handleStreamingItemStarted(event, state, tracer, eventTime) {
|
|
575
|
+
const item = event.item;
|
|
576
|
+
if (!item) {
|
|
577
|
+
logger.warn("Codex item.started event missing item", { event });
|
|
578
|
+
return;
|
|
579
|
+
}
|
|
580
|
+
if (!item.id) {
|
|
581
|
+
logger.debug("Codex item.started without id, will create span at completion", { type: item.type });
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
const itemId = String(item.id);
|
|
585
|
+
const span = this.startStreamingItemSpan(tracer, item, itemId);
|
|
586
|
+
state.activeSpans.set(itemId, span);
|
|
587
|
+
state.itemStartTimes.set(itemId, eventTime);
|
|
588
|
+
logger.debug("Codex item started", {
|
|
589
|
+
itemId,
|
|
590
|
+
type: item.type
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
handleStreamingItemCompleted(event, state, tracer, eventTime, skillRootPrefixes) {
|
|
594
|
+
const item = event.item;
|
|
595
|
+
if (!item) {
|
|
596
|
+
logger.warn("Codex item.completed event missing item", { event });
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const itemId = item.id ? String(item.id) : crypto.randomUUID();
|
|
600
|
+
state.items.push(item);
|
|
601
|
+
this.collectStreamingItemText(item, state);
|
|
602
|
+
const span = state.activeSpans.get(itemId) ?? this.startStreamingItemSpan(tracer, item, itemId, state.lastEventTime);
|
|
603
|
+
const hadStartEvent = state.activeSpans.has(itemId);
|
|
604
|
+
const startTime = state.itemStartTimes.get(itemId) ?? state.lastEventTime;
|
|
605
|
+
const durationMs = eventTime - startTime;
|
|
606
|
+
this.applyStreamingCompletionAttributes(span, item, skillRootPrefixes, eventTime, startTime, hadStartEvent);
|
|
607
|
+
span.end();
|
|
608
|
+
state.activeSpans.delete(itemId);
|
|
609
|
+
state.itemStartTimes.delete(itemId);
|
|
610
|
+
logger.debug("Codex item completed", {
|
|
611
|
+
itemId,
|
|
612
|
+
type: item.type,
|
|
613
|
+
durationMs
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
handleStreamingItemUpdated(event, state, skillRootPrefixes) {
|
|
617
|
+
const item = event.item;
|
|
618
|
+
if (item?.id) {
|
|
619
|
+
const itemId = String(item.id);
|
|
620
|
+
const span = state.activeSpans.get(itemId);
|
|
621
|
+
if (span) this.setStreamingCompletionAttributes(span, item, skillRootPrefixes);
|
|
622
|
+
}
|
|
623
|
+
logger.debug("Codex item updated", {
|
|
624
|
+
itemId: item?.id,
|
|
625
|
+
type: item?.type
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
startStreamingItemSpan(tracer, item, itemId, startTime) {
|
|
629
|
+
return tracer.startSpan(this.getSpanNameForItem(item), {
|
|
630
|
+
kind: SpanKind.INTERNAL,
|
|
631
|
+
...startTime === void 0 ? {} : { startTime },
|
|
632
|
+
attributes: {
|
|
633
|
+
"codex.item.id": itemId,
|
|
634
|
+
"codex.item.type": item.type,
|
|
635
|
+
...startTime === void 0 ? {} : { "codex.timing.estimated": true },
|
|
636
|
+
...this.getAttributesForItem(item)
|
|
637
|
+
}
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
collectStreamingItemText(item, state) {
|
|
641
|
+
if (item.type === "reasoning" && typeof item.text === "string") {
|
|
642
|
+
const sanitizedReasoning = this.sanitizeTraceText(item.text, "Codex reasoning trace event");
|
|
643
|
+
if (sanitizedReasoning) state.reasoningTexts.push(sanitizedReasoning);
|
|
644
|
+
}
|
|
645
|
+
if (item.type === "agent_message" && typeof item.text === "string") state.conversationMessages.push({
|
|
646
|
+
role: "assistant",
|
|
647
|
+
content: this.sanitizeTraceText(item.text, "Codex agent message trace event") ?? ""
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
applyStreamingCompletionAttributes(span, item, skillRootPrefixes, eventTime, startTime, hadStartEvent) {
|
|
651
|
+
this.setStreamingCompletionAttributes(span, item, skillRootPrefixes);
|
|
652
|
+
span.setAttribute("codex.duration_ms", eventTime - startTime);
|
|
653
|
+
span.setAttribute("codex.had_start_event", hadStartEvent);
|
|
654
|
+
this.addStreamingSpanEvents(span, item);
|
|
655
|
+
this.setStreamingSpanStatus(span, item);
|
|
656
|
+
}
|
|
657
|
+
setStreamingCompletionAttributes(span, item, skillRootPrefixes) {
|
|
658
|
+
const completionAttrs = this.getCompletionAttributesForItem(item, skillRootPrefixes);
|
|
659
|
+
for (const [key, value] of Object.entries(completionAttrs)) span.setAttribute(key, value);
|
|
660
|
+
}
|
|
661
|
+
addStreamingSpanEvents(span, item) {
|
|
662
|
+
if (item.type === "reasoning" && typeof item.text === "string") span.addEvent("reasoning", { "codex.reasoning.text": this.sanitizeTraceText(item.text, "Codex reasoning span event") ?? "" });
|
|
663
|
+
if (item.type === "agent_message" && typeof item.text === "string") span.addEvent("message", { "codex.message.text": this.sanitizeTraceText(item.text, "Codex agent message span event") ?? "" });
|
|
664
|
+
if (item.type === "command_execution" && typeof item.aggregated_output === "string") span.addEvent("output", { "codex.command.output": this.sanitizeTraceText(item.aggregated_output, "Codex command output span event") ?? "" });
|
|
665
|
+
}
|
|
666
|
+
setStreamingSpanStatus(span, item) {
|
|
667
|
+
if (!this.isStreamingItemError(item)) {
|
|
668
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
669
|
+
return;
|
|
670
|
+
}
|
|
671
|
+
span.setStatus({
|
|
672
|
+
code: SpanStatusCode.ERROR,
|
|
673
|
+
message: this.getStreamingItemErrorMessage(item)
|
|
674
|
+
});
|
|
675
|
+
}
|
|
676
|
+
isStreamingItemError(item) {
|
|
677
|
+
return item.status === "failed" || item.type === "error" || item.error !== void 0 || item.type === "command_execution" && typeof item.exit_code === "number" && item.exit_code !== 0;
|
|
678
|
+
}
|
|
679
|
+
getStreamingItemErrorMessage(item) {
|
|
680
|
+
return (typeof item.message === "string" ? item.message : null) || (typeof item.error?.message === "string" ? item.error.message : null) || (item.type === "command_execution" && item.exit_code !== 0 ? `Command exited with code ${item.exit_code}` : null) || "Item failed";
|
|
681
|
+
}
|
|
682
|
+
endUnclosedStreamingSpans(state) {
|
|
683
|
+
for (const [itemId, span] of state.activeSpans) {
|
|
684
|
+
logger.warn("Codex item span not properly closed", { itemId });
|
|
685
|
+
span.setStatus({
|
|
686
|
+
code: SpanStatusCode.ERROR,
|
|
687
|
+
message: "Span not properly closed"
|
|
688
|
+
});
|
|
689
|
+
span.end();
|
|
690
|
+
}
|
|
691
|
+
state.activeSpans.clear();
|
|
692
|
+
state.itemStartTimes.clear();
|
|
693
|
+
}
|
|
694
|
+
buildStreamingTurnResult(state) {
|
|
695
|
+
const agentMessages = state.items.filter((item) => item.type === "agent_message");
|
|
696
|
+
return {
|
|
697
|
+
finalResponse: agentMessages.length > 0 ? agentMessages.map((item) => item.text).join("\n") : "",
|
|
698
|
+
items: state.items,
|
|
699
|
+
usage: state.usage,
|
|
700
|
+
reasoningTexts: state.reasoningTexts,
|
|
701
|
+
conversationMessages: state.conversationMessages
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
parsePromptInput(prompt) {
|
|
705
|
+
let parsedPrompt;
|
|
706
|
+
try {
|
|
707
|
+
parsedPrompt = JSON.parse(prompt);
|
|
708
|
+
} catch {
|
|
709
|
+
return prompt;
|
|
710
|
+
}
|
|
711
|
+
if (!Array.isArray(parsedPrompt) || parsedPrompt.length === 0 || !parsedPrompt.every((item) => this.isCodexPromptInputItem(item))) return prompt;
|
|
712
|
+
return parsedPrompt;
|
|
713
|
+
}
|
|
714
|
+
isCodexPromptInputItem(item) {
|
|
715
|
+
if (!item || typeof item !== "object" || Array.isArray(item)) return false;
|
|
716
|
+
const keys = Object.keys(item);
|
|
717
|
+
if ("type" in item && item.type === "text") return keys.length === 2 && keys.includes("type") && "text" in item && typeof item.text === "string";
|
|
718
|
+
if ("type" in item && item.type === "local_image") return keys.length === 2 && keys.includes("type") && "path" in item && typeof item.path === "string";
|
|
719
|
+
return false;
|
|
720
|
+
}
|
|
721
|
+
formatPromptInputForTrace(prompt) {
|
|
722
|
+
if (typeof prompt === "string") return prompt;
|
|
723
|
+
return prompt.map((item) => item.type === "text" ? item.text : `[local_image: ${item.path}]`).join("\n");
|
|
724
|
+
}
|
|
725
|
+
/**
|
|
726
|
+
* Get a descriptive span name for a Codex item
|
|
727
|
+
*/
|
|
728
|
+
getSpanNameForItem(item) {
|
|
729
|
+
switch (item.type) {
|
|
730
|
+
case "command_execution": return `exec ${typeof item.command === "string" ? item.command.split(" ")[0] || "command" : "command"}`;
|
|
731
|
+
case "file_change": return `file ${item.changes?.[0]?.kind || "change"}`;
|
|
732
|
+
case "mcp_tool_call": return `mcp ${typeof item.server === "string" ? item.server : "unknown"}/${typeof item.tool === "string" ? item.tool : "unknown"}`;
|
|
733
|
+
case "agent_message": return "agent response";
|
|
734
|
+
case "reasoning": return "reasoning";
|
|
735
|
+
case "web_search": return `search "${typeof item.query === "string" ? (this.sanitizeTraceText(item.query, "Codex web search span name") ?? "").slice(0, 30) : ""}"`;
|
|
736
|
+
case "todo_list": return "todo update";
|
|
737
|
+
case "error": return "error";
|
|
738
|
+
case "collaboration_tool_call": return `collab ${typeof item.tool === "string" ? item.tool : "unknown"}`;
|
|
739
|
+
case "spawn_agent": return `spawn ${typeof item.role === "string" ? item.role : "agent"}`;
|
|
740
|
+
case "send_input": return "send input";
|
|
741
|
+
case "agent_wait": return "wait";
|
|
742
|
+
default: return `codex.${item.type || "unknown"}`;
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* Get attributes for a Codex item at start
|
|
747
|
+
*/
|
|
748
|
+
getSkillTraceAttributes(item, skillRootPrefixes = [], options = {}) {
|
|
749
|
+
if (item?.type !== "command_execution") return {};
|
|
750
|
+
if (options.requireSuccessfulCommand && !this.isSuccessfulCommandExecution(item)) return {};
|
|
751
|
+
const command = typeof item.command === "string" && item.command.trim() ? item.command : void 0;
|
|
752
|
+
const skillCandidates = /* @__PURE__ */ new Map();
|
|
753
|
+
if (command) for (const skill of this.extractSkillPathCandidates(command, skillRootPrefixes)) skillCandidates.set(skill.path, skill);
|
|
754
|
+
if (skillCandidates.size === 0) return {};
|
|
755
|
+
const skills = Array.from(skillCandidates.values());
|
|
756
|
+
const attrs = {
|
|
757
|
+
"promptfoo.skill.count": skills.length,
|
|
758
|
+
"promptfoo.skill.names": skills.map((skill) => skill.name).join(","),
|
|
759
|
+
"promptfoo.skill.paths": skills.map((skill) => skill.path).join(",")
|
|
760
|
+
};
|
|
761
|
+
if (skills.length === 1) {
|
|
762
|
+
attrs["promptfoo.skill.name"] = skills[0].name;
|
|
763
|
+
attrs["promptfoo.skill.path"] = skills[0].path;
|
|
764
|
+
}
|
|
765
|
+
return attrs;
|
|
766
|
+
}
|
|
767
|
+
getAttributesForItem(item) {
|
|
768
|
+
const attrs = {};
|
|
769
|
+
switch (item.type) {
|
|
770
|
+
case "command_execution":
|
|
771
|
+
if (typeof item.command === "string") attrs["codex.command"] = this.sanitizeTraceText(item.command, "Codex command trace attribute") ?? "";
|
|
772
|
+
break;
|
|
773
|
+
case "mcp_tool_call":
|
|
774
|
+
if (typeof item.server === "string") attrs["codex.mcp.server"] = item.server;
|
|
775
|
+
if (typeof item.tool === "string") attrs["codex.mcp.tool"] = item.tool;
|
|
776
|
+
{
|
|
777
|
+
const serializedArgs = this.serializeItemValue(item.arguments ?? item.args ?? item.input);
|
|
778
|
+
if (serializedArgs) attrs["codex.mcp.input"] = serializedArgs;
|
|
779
|
+
}
|
|
780
|
+
break;
|
|
781
|
+
case "web_search":
|
|
782
|
+
if (typeof item.query === "string") attrs["codex.search.query"] = this.sanitizeTraceText(item.query, "Codex web search query trace attribute") ?? "";
|
|
783
|
+
break;
|
|
784
|
+
case "collaboration_tool_call":
|
|
785
|
+
if (typeof item.tool === "string") attrs["codex.collab.tool"] = item.tool;
|
|
786
|
+
if (typeof item.target_thread_id === "string") attrs["codex.collab.target_thread"] = item.target_thread_id;
|
|
787
|
+
break;
|
|
788
|
+
case "spawn_agent":
|
|
789
|
+
if (typeof item.role === "string") attrs["codex.collab.role"] = item.role;
|
|
790
|
+
if (typeof item.thread_id === "string") attrs["codex.collab.spawned_thread"] = item.thread_id;
|
|
791
|
+
break;
|
|
792
|
+
case "send_input":
|
|
793
|
+
if (typeof item.target_thread_id === "string") attrs["codex.collab.target_thread"] = item.target_thread_id;
|
|
794
|
+
break;
|
|
795
|
+
}
|
|
796
|
+
return attrs;
|
|
797
|
+
}
|
|
798
|
+
serializeItemValue(value) {
|
|
799
|
+
if (typeof value === "string") {
|
|
800
|
+
const trimmed = value.trim();
|
|
801
|
+
if (!trimmed) return;
|
|
802
|
+
try {
|
|
803
|
+
return JSON.stringify(this.redactTracePii(sanitizeObject(JSON.parse(trimmed))));
|
|
804
|
+
} catch {
|
|
805
|
+
return this.redactTracePii(sanitizeObject(trimmed, { context: "Codex MCP trace input" }));
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
if (value === void 0 || value === null) return;
|
|
809
|
+
try {
|
|
810
|
+
return JSON.stringify(this.redactTracePii(sanitizeObject(value, { context: "Codex MCP trace input" })));
|
|
811
|
+
} catch {
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
sanitizeTraceText(value, context) {
|
|
816
|
+
const sanitized = this.redactTracePii(sanitizeObject(value, { context }));
|
|
817
|
+
if (typeof sanitized === "string") return sanitized;
|
|
818
|
+
if (sanitized === void 0 || sanitized === null) return;
|
|
819
|
+
try {
|
|
820
|
+
return JSON.stringify(sanitized);
|
|
821
|
+
} catch {
|
|
822
|
+
return;
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
redactTracePii(value) {
|
|
826
|
+
if (typeof value === "string") return value.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, REDACTED).replace(/\b(?:sk-(?:proj-)?[A-Za-z0-9_-]{20,}|sk-ant-[A-Za-z0-9_-]{20,}|AKIA[A-Z0-9]{16}|AIza[A-Za-z0-9_-]{35}|Bearer\s+[A-Za-z0-9._~+/-]{20,}|Basic\s+[A-Za-z0-9+/=]{20,})\b/g, REDACTED).replace(/\b(api[_-]?key|token|password|secret|authorization|auth)\s*([=:])(\s*)(["']?)[^\s"'`]+(\4)/gi, (_match, key, separator, spacing, quote) => `${key}${separator}${spacing}${quote}${REDACTED}${quote}`);
|
|
827
|
+
if (Array.isArray(value)) return value.map((item) => this.redactTracePii(item));
|
|
828
|
+
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, entryValue]) => [key, normalizeFieldName(key).includes("email") ? REDACTED : this.redactTracePii(entryValue)]));
|
|
829
|
+
return value;
|
|
830
|
+
}
|
|
831
|
+
/**
|
|
832
|
+
* Get attributes for a Codex item at completion
|
|
833
|
+
*/
|
|
834
|
+
getCompletionAttributesForItem(item, skillRootPrefixes = []) {
|
|
835
|
+
const attrs = {};
|
|
836
|
+
switch (item.type) {
|
|
837
|
+
case "command_execution":
|
|
838
|
+
if (typeof item.exit_code === "number") attrs["codex.exit_code"] = item.exit_code;
|
|
839
|
+
if (typeof item.status === "string") attrs["codex.status"] = item.status;
|
|
840
|
+
if (typeof item.aggregated_output === "string") attrs["codex.output"] = this.sanitizeTraceText(item.aggregated_output, "Codex command output trace attribute") ?? "";
|
|
841
|
+
Object.assign(attrs, this.getSkillTraceAttributes(item, skillRootPrefixes, { requireSuccessfulCommand: true }));
|
|
842
|
+
break;
|
|
843
|
+
case "file_change":
|
|
844
|
+
if (typeof item.status === "string") attrs["codex.status"] = item.status;
|
|
845
|
+
if (Array.isArray(item.changes) && item.changes.length) {
|
|
846
|
+
attrs["codex.files_changed"] = item.changes.length;
|
|
847
|
+
attrs["codex.files"] = item.changes.map((c) => typeof c?.path === "string" ? c.path : "").filter(Boolean).join(", ");
|
|
848
|
+
}
|
|
849
|
+
break;
|
|
850
|
+
case "mcp_tool_call":
|
|
851
|
+
if (typeof item.status === "string") attrs["codex.status"] = item.status;
|
|
852
|
+
if (typeof item.error?.message === "string") attrs["codex.error"] = this.sanitizeTraceText(item.error.message, "Codex MCP error trace attribute") ?? "";
|
|
853
|
+
{
|
|
854
|
+
const serializedArgs = this.serializeItemValue(item.arguments ?? item.args ?? item.input);
|
|
855
|
+
if (serializedArgs) attrs["codex.mcp.input"] = serializedArgs;
|
|
856
|
+
}
|
|
857
|
+
break;
|
|
858
|
+
case "agent_message":
|
|
859
|
+
if (typeof item.text === "string") attrs["codex.message"] = this.sanitizeTraceText(item.text, "Codex agent message trace attribute") ?? "";
|
|
860
|
+
break;
|
|
861
|
+
case "reasoning":
|
|
862
|
+
if (typeof item.text === "string") attrs["codex.reasoning"] = this.sanitizeTraceText(item.text, "Codex reasoning trace attribute") ?? "";
|
|
863
|
+
break;
|
|
864
|
+
case "error":
|
|
865
|
+
if (typeof item.message === "string") attrs["codex.error"] = this.sanitizeTraceText(item.message, "Codex error trace attribute") ?? "";
|
|
866
|
+
break;
|
|
867
|
+
}
|
|
868
|
+
return attrs;
|
|
869
|
+
}
|
|
870
|
+
generateInstanceKey(env, config) {
|
|
871
|
+
const keyData = {
|
|
872
|
+
env,
|
|
873
|
+
base_url: config.base_url,
|
|
874
|
+
cli_config: this.getResolvedCliConfig(config),
|
|
875
|
+
codex_path_override: config.codex_path_override
|
|
876
|
+
};
|
|
877
|
+
return `openai:codex-sdk:instance:${crypto.createHash("sha256").update(JSON.stringify(keyData)).digest("hex")}`;
|
|
878
|
+
}
|
|
879
|
+
generateCacheKey(config, prompt, instanceKey) {
|
|
880
|
+
const keyData = {
|
|
881
|
+
instanceKey,
|
|
882
|
+
working_dir: config.working_dir,
|
|
883
|
+
additional_directories: config.additional_directories,
|
|
884
|
+
model: config.model,
|
|
885
|
+
output_schema: config.output_schema,
|
|
886
|
+
sandbox_mode: config.sandbox_mode,
|
|
887
|
+
model_reasoning_effort: config.model_reasoning_effort,
|
|
888
|
+
network_access_enabled: config.network_access_enabled,
|
|
889
|
+
web_search_enabled: config.web_search_enabled,
|
|
890
|
+
web_search_mode: config.web_search_mode,
|
|
891
|
+
approval_policy: config.approval_policy,
|
|
892
|
+
prompt
|
|
893
|
+
};
|
|
894
|
+
return `openai:codex-sdk:${crypto.createHash("sha256").update(JSON.stringify(keyData)).digest("hex")}`;
|
|
895
|
+
}
|
|
896
|
+
getThreadRunQueueKey(config, cacheKey, instanceKey) {
|
|
897
|
+
if (config.deep_tracing) return;
|
|
898
|
+
if (config.thread_id) return `${instanceKey}:${config.thread_id}`;
|
|
899
|
+
if (config.persist_threads && cacheKey) return cacheKey;
|
|
900
|
+
}
|
|
901
|
+
async runSerializedThreadTurn(queueKey, abortSignal, executeTurn) {
|
|
902
|
+
if (!queueKey) return executeTurn();
|
|
903
|
+
const previousRun = this.threadRunQueues.get(queueKey) ?? Promise.resolve();
|
|
904
|
+
let releaseCurrentRun = () => {};
|
|
905
|
+
const currentRun = new Promise((resolve) => {
|
|
906
|
+
releaseCurrentRun = resolve;
|
|
907
|
+
});
|
|
908
|
+
const queuedRun = previousRun.catch(() => void 0).then(() => currentRun);
|
|
909
|
+
this.threadRunQueues.set(queueKey, queuedRun);
|
|
910
|
+
queuedRun.finally(() => {
|
|
911
|
+
if (this.threadRunQueues.get(queueKey) === queuedRun) this.threadRunQueues.delete(queueKey);
|
|
912
|
+
});
|
|
913
|
+
try {
|
|
914
|
+
await this.waitForPreviousThreadRun(previousRun, abortSignal);
|
|
915
|
+
return await executeTurn();
|
|
916
|
+
} finally {
|
|
917
|
+
releaseCurrentRun();
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
async waitForPreviousThreadRun(previousRun, abortSignal) {
|
|
921
|
+
const previousRunDone = previousRun.catch(() => void 0);
|
|
922
|
+
if (!abortSignal) {
|
|
923
|
+
await previousRunDone;
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
if (abortSignal.aborted) throw this.createAbortError("Codex thread turn wait aborted");
|
|
927
|
+
let onAbort;
|
|
928
|
+
const abortPromise = new Promise((_, reject) => {
|
|
929
|
+
onAbort = () => reject(this.createAbortError("Codex thread turn wait aborted"));
|
|
930
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
931
|
+
});
|
|
932
|
+
try {
|
|
933
|
+
await Promise.race([previousRunDone, abortPromise]);
|
|
934
|
+
} finally {
|
|
935
|
+
if (onAbort) abortSignal.removeEventListener("abort", onAbort);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
createAbortError(message) {
|
|
939
|
+
const error = new Error(message);
|
|
940
|
+
error.name = "AbortError";
|
|
941
|
+
return error;
|
|
942
|
+
}
|
|
943
|
+
async callApi(prompt, context, callOptions) {
|
|
944
|
+
const config = renderVarsInObject({
|
|
945
|
+
...this.config,
|
|
946
|
+
...context?.prompt?.config
|
|
947
|
+
}, context?.vars);
|
|
948
|
+
const requestedModel = typeof config.model === "string" && config.model ? config.model : void 0;
|
|
949
|
+
return withGenAISpan(this.buildCodexSpanContext(prompt, context, requestedModel), () => this.callApiInternal(prompt, context, callOptions, config), (response) => this.extractCodexSpanResult(response, requestedModel));
|
|
950
|
+
}
|
|
951
|
+
buildCodexSpanContext(prompt, context, requestedModel) {
|
|
952
|
+
return {
|
|
953
|
+
system: "openai",
|
|
954
|
+
operationName: "chat",
|
|
955
|
+
model: requestedModel ?? "codex",
|
|
956
|
+
providerId: this.id(),
|
|
957
|
+
evalId: context?.evaluationId || context?.test?.metadata?.evaluationId,
|
|
958
|
+
testIndex: typeof context?.test?.vars?.__testIdx === "number" ? context.test.vars.__testIdx : void 0,
|
|
959
|
+
promptLabel: context?.prompt?.label,
|
|
960
|
+
traceparent: context?.traceparent,
|
|
961
|
+
requestBody: prompt
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
extractCodexSpanResult(response, requestedModel) {
|
|
965
|
+
const result = {};
|
|
966
|
+
if (response.tokenUsage) result.tokenUsage = response.tokenUsage;
|
|
967
|
+
if (response.sessionId) result.responseId = response.sessionId;
|
|
968
|
+
if (response.cached !== void 0) result.cacheHit = response.cached;
|
|
969
|
+
if (requestedModel) result.responseModel = requestedModel;
|
|
970
|
+
this.setCodexSpanResponseBody(result, response.output);
|
|
971
|
+
this.setCodexSpanRawAttributes(result, response.raw);
|
|
972
|
+
return result;
|
|
973
|
+
}
|
|
974
|
+
setCodexSpanResponseBody(result, output) {
|
|
975
|
+
if (output === void 0) return;
|
|
976
|
+
try {
|
|
977
|
+
result.responseBody = typeof output === "string" ? output : JSON.stringify(output);
|
|
978
|
+
} catch {
|
|
979
|
+
result.responseBody = "[unable to serialize output]";
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
setCodexSpanRawAttributes(result, raw) {
|
|
983
|
+
if (!raw) return;
|
|
984
|
+
try {
|
|
985
|
+
const rawData = typeof raw === "string" ? JSON.parse(raw) : raw;
|
|
986
|
+
if (rawData.reasoningTexts?.length > 0) result.additionalAttributes = {
|
|
987
|
+
...result.additionalAttributes,
|
|
988
|
+
"codex.reasoning.count": rawData.reasoningTexts.length,
|
|
989
|
+
"codex.reasoning.summary": rawData.reasoningTexts.join("\n---\n").slice(0, 2e3)
|
|
990
|
+
};
|
|
991
|
+
if (rawData.conversationMessages?.length > 0) result.additionalAttributes = {
|
|
992
|
+
...result.additionalAttributes,
|
|
993
|
+
"codex.conversation.message_count": rawData.conversationMessages.length
|
|
994
|
+
};
|
|
995
|
+
if (rawData.items?.length > 0) result.additionalAttributes = {
|
|
996
|
+
...result.additionalAttributes,
|
|
997
|
+
"codex.items.total": rawData.items.length,
|
|
998
|
+
"codex.items.breakdown": JSON.stringify(this.getCodexItemCounts(rawData.items))
|
|
999
|
+
};
|
|
1000
|
+
} catch {}
|
|
1001
|
+
}
|
|
1002
|
+
getCodexItemCounts(items) {
|
|
1003
|
+
const itemCounts = {};
|
|
1004
|
+
for (const item of items) itemCounts[item.type] = (itemCounts[item.type] || 0) + 1;
|
|
1005
|
+
return itemCounts;
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Internal implementation of callApi without tracing wrapper.
|
|
1009
|
+
* Context is available for future use (e.g., _context?.vars for template rendering,
|
|
1010
|
+
* _context?.bustCache for cache control, _context?.debug for debug mode).
|
|
1011
|
+
*/
|
|
1012
|
+
async callApiInternal(prompt, context, callOptions, rawConfig) {
|
|
1013
|
+
let config;
|
|
1014
|
+
try {
|
|
1015
|
+
config = parseCodexConfig(rawConfig, { stripUnknownKeys: true });
|
|
1016
|
+
} catch (error) {
|
|
1017
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1018
|
+
logger.error("Error calling OpenAI Codex SDK", { error: errorMessage });
|
|
1019
|
+
return { error: `Error calling OpenAI Codex SDK: ${errorMessage}` };
|
|
1020
|
+
}
|
|
1021
|
+
const currentTraceparent = getTraceparent();
|
|
1022
|
+
const apiKey = this.getApiKey(config);
|
|
1023
|
+
const workingDirectory = config.working_dir ?? process.cwd();
|
|
1024
|
+
const resolvedConfig = {
|
|
1025
|
+
...config,
|
|
1026
|
+
working_dir: workingDirectory
|
|
1027
|
+
};
|
|
1028
|
+
const env = this.prepareEnvironment(resolvedConfig, currentTraceparent, apiKey);
|
|
1029
|
+
const skillRootPrefixes = this.getSkillRootPrefixes(env, resolvedConfig.working_dir);
|
|
1030
|
+
const promptInput = this.parsePromptInput(prompt);
|
|
1031
|
+
if (apiKey) logger.debug("[CodexSDK] Using explicit API credentials from promptfoo config/environment");
|
|
1032
|
+
else logger.debug("[CodexSDK] No explicit API credentials configured; deferring auth resolution to Codex SDK login state");
|
|
1033
|
+
if (callOptions?.abortSignal?.aborted) return { error: "OpenAI Codex SDK call aborted before it started" };
|
|
1034
|
+
let localInstance = void 0;
|
|
1035
|
+
let useLocalInstance = false;
|
|
1036
|
+
let cacheKey = void 0;
|
|
1037
|
+
try {
|
|
1038
|
+
this.validateWorkingDirectory(resolvedConfig.working_dir, resolvedConfig.skip_git_repo_check);
|
|
1039
|
+
const codexInstance = await this.getCodexInstanceForTurn(env, resolvedConfig, apiKey);
|
|
1040
|
+
const activeInstance = codexInstance.activeInstance;
|
|
1041
|
+
localInstance = codexInstance.localInstance;
|
|
1042
|
+
useLocalInstance = codexInstance.useLocalInstance;
|
|
1043
|
+
const promptCacheBasis = context?.prompt?.raw ?? prompt;
|
|
1044
|
+
cacheKey = this.generateCacheKey(resolvedConfig, promptCacheBasis, codexInstance.instanceKey);
|
|
1045
|
+
const { turn, sessionId } = await this.executeCodexTurn(activeInstance, codexInstance.instanceKey, cacheKey, promptInput, resolvedConfig, callOptions, skillRootPrefixes);
|
|
1046
|
+
return this.buildCodexProviderResponse(turn, sessionId, skillRootPrefixes, resolvedConfig);
|
|
1047
|
+
} catch (error) {
|
|
1048
|
+
if (error instanceof Error && error.name === "AbortError" || callOptions?.abortSignal?.aborted) {
|
|
1049
|
+
logger.warn("OpenAI Codex SDK call aborted");
|
|
1050
|
+
return { error: "OpenAI Codex SDK call aborted" };
|
|
1051
|
+
}
|
|
1052
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
1053
|
+
logger.error("Error calling OpenAI Codex SDK", { error: errorMessage });
|
|
1054
|
+
return { error: `Error calling OpenAI Codex SDK: ${errorMessage}` };
|
|
1055
|
+
} finally {
|
|
1056
|
+
await this.cleanupCodexTurn(resolvedConfig, cacheKey, useLocalInstance, localInstance);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
buildCodexRunOptions(config, callOptions) {
|
|
1060
|
+
return {
|
|
1061
|
+
...config.output_schema ? { outputSchema: config.output_schema } : {},
|
|
1062
|
+
...callOptions?.abortSignal ? { signal: callOptions.abortSignal } : {}
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
async getCodexInstanceForTurn(env, resolvedConfig, apiKey) {
|
|
1066
|
+
if (!this.codexModule) this.codexModule = await loadCodexSDK();
|
|
1067
|
+
const stableEnv = { ...env };
|
|
1068
|
+
delete stableEnv.TRACEPARENT;
|
|
1069
|
+
const instanceKey = this.generateInstanceKey(stableEnv, resolvedConfig);
|
|
1070
|
+
if (resolvedConfig.deep_tracing) {
|
|
1071
|
+
this.warnOnceForDeepTracingThreadOptions(resolvedConfig);
|
|
1072
|
+
const localInstance = new this.codexModule.Codex(this.buildCodexOptions(env, resolvedConfig, apiKey));
|
|
1073
|
+
return {
|
|
1074
|
+
activeInstance: localInstance,
|
|
1075
|
+
instanceKey,
|
|
1076
|
+
localInstance,
|
|
1077
|
+
useLocalInstance: true
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
let activeInstance = this.codexInstances.get(instanceKey);
|
|
1081
|
+
if (!activeInstance) {
|
|
1082
|
+
activeInstance = new this.codexModule.Codex(this.buildCodexOptions(env, resolvedConfig, apiKey));
|
|
1083
|
+
this.codexInstances.set(instanceKey, activeInstance);
|
|
1084
|
+
}
|
|
1085
|
+
return {
|
|
1086
|
+
activeInstance,
|
|
1087
|
+
instanceKey,
|
|
1088
|
+
localInstance: void 0,
|
|
1089
|
+
useLocalInstance: false
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
warnOnceForDeepTracingThreadOptions(resolvedConfig) {
|
|
1093
|
+
if (this.deepTracingWarningShown || !resolvedConfig.persist_threads && !resolvedConfig.thread_id && (resolvedConfig.thread_pool_size ?? 0) <= 1) return;
|
|
1094
|
+
logger.warn("[CodexSDK] deep_tracing is incompatible with thread persistence. Thread options (persist_threads, thread_id, thread_pool_size) are ignored when deep_tracing is enabled.");
|
|
1095
|
+
this.deepTracingWarningShown = true;
|
|
1096
|
+
}
|
|
1097
|
+
async executeCodexTurn(activeInstance, instanceKey, cacheKey, promptInput, resolvedConfig, callOptions, skillRootPrefixes) {
|
|
1098
|
+
const queueKey = this.getThreadRunQueueKey(resolvedConfig, cacheKey, instanceKey);
|
|
1099
|
+
const runOptions = this.buildCodexRunOptions(resolvedConfig, callOptions);
|
|
1100
|
+
return this.runSerializedThreadTurn(queueKey, callOptions?.abortSignal, async () => {
|
|
1101
|
+
const thread = await this.getOrCreateThread(resolvedConfig, cacheKey, instanceKey, activeInstance);
|
|
1102
|
+
return {
|
|
1103
|
+
turn: resolvedConfig.enable_streaming ? await this.runStreaming(thread, promptInput, runOptions, callOptions, skillRootPrefixes) : await thread.run(promptInput, runOptions),
|
|
1104
|
+
sessionId: thread.id || "unknown"
|
|
1105
|
+
};
|
|
1106
|
+
});
|
|
1107
|
+
}
|
|
1108
|
+
buildCodexProviderResponse(turn, sessionId, skillRootPrefixes, resolvedConfig) {
|
|
1109
|
+
const output = turn.finalResponse || "";
|
|
1110
|
+
const tokenUsage = this.buildCodexTokenUsage(turn.usage);
|
|
1111
|
+
logger.debug("OpenAI Codex SDK response", {
|
|
1112
|
+
output,
|
|
1113
|
+
usage: turn.usage
|
|
1114
|
+
});
|
|
1115
|
+
return {
|
|
1116
|
+
output,
|
|
1117
|
+
tokenUsage,
|
|
1118
|
+
cost: this.calculateCodexResponseCost(tokenUsage, resolvedConfig.model),
|
|
1119
|
+
metadata: this.buildCodexResponseMetadata(turn.items, skillRootPrefixes),
|
|
1120
|
+
raw: JSON.stringify(turn),
|
|
1121
|
+
sessionId
|
|
1122
|
+
};
|
|
1123
|
+
}
|
|
1124
|
+
buildCodexResponseMetadata(items, skillRootPrefixes) {
|
|
1125
|
+
const skillMetadata = this.buildSkillMetadata(items, skillRootPrefixes);
|
|
1126
|
+
if (!skillMetadata) return;
|
|
1127
|
+
return {
|
|
1128
|
+
...skillMetadata.skillCalls.length > 0 ? { skillCalls: skillMetadata.skillCalls } : {},
|
|
1129
|
+
...skillMetadata.attemptedSkillCalls.length > skillMetadata.skillCalls.length ? { attemptedSkillCalls: skillMetadata.attemptedSkillCalls } : {}
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
buildCodexTokenUsage(turnUsage) {
|
|
1133
|
+
if (!turnUsage) return;
|
|
1134
|
+
return {
|
|
1135
|
+
prompt: turnUsage.input_tokens,
|
|
1136
|
+
completion: turnUsage.output_tokens,
|
|
1137
|
+
total: turnUsage.input_tokens + turnUsage.output_tokens,
|
|
1138
|
+
cached: turnUsage.cached_input_tokens || 0
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
calculateCodexResponseCost(tokenUsage, model) {
|
|
1142
|
+
if (!tokenUsage || !model) return 0;
|
|
1143
|
+
const pricing = CODEX_MODEL_PRICING[model];
|
|
1144
|
+
if (!pricing) return 0;
|
|
1145
|
+
const cachedTokens = tokenUsage.cached || 0;
|
|
1146
|
+
const inputCost = ((tokenUsage.prompt || 0) - cachedTokens) * (pricing.input / 1e6);
|
|
1147
|
+
const cacheReadCost = cachedTokens * (pricing.cache_read / 1e6);
|
|
1148
|
+
const outputCost = (tokenUsage.completion || 0) * (pricing.output / 1e6);
|
|
1149
|
+
return inputCost + cacheReadCost + outputCost;
|
|
1150
|
+
}
|
|
1151
|
+
async cleanupCodexTurn(resolvedConfig, cacheKey, useLocalInstance, localInstance) {
|
|
1152
|
+
if (!resolvedConfig.deep_tracing && !resolvedConfig.persist_threads && !resolvedConfig.thread_id && cacheKey) this.threads.delete(cacheKey);
|
|
1153
|
+
if (!useLocalInstance || !localInstance) return;
|
|
1154
|
+
try {
|
|
1155
|
+
await this.destroyInstance(localInstance);
|
|
1156
|
+
} catch (cleanupError) {
|
|
1157
|
+
logger.debug("[CodexSDK] Error cleaning up local instance", { error: cleanupError });
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
1161
|
+
//#endregion
|
|
1162
|
+
export { OpenAICodexSDKProvider as t };
|
|
1163
|
+
|
|
1164
|
+
//# sourceMappingURL=codex-sdk-CpqiOqDO.js.map
|