titan-agent 5.3.0 → 5.3.2

package/ui/dist/assets/EvalHarnessPanel-Cz9dRg61.js

@@ -1,2 +0,0 @@
-import{b as S,r as h,j as e,bq as w,e as O,h as C}from"./index-D7Clon2u.js";import{L as F}from"./layers-B6jDzitD.js";import{e as $,R as E,L as M,W as T}from"./TitanCanvas-BQU1yxqf.js";import{T as P}from"./trending-up-Dh_CffGX.js";import{P as _}from"./play-hUyR3CVS.js";import{a as I,C as B}from"./VoiceOverlay-l6yoasVz.js";import{C as R}from"./circle-check-big-B1hMwau0.js";const A={"widget-creation":"Widget Creation",safety:"Safety","tool-routing":"Tool Routing","gate-format":"Gate Format",pipeline:"Pipeline Classification",adversarial:"Adversarial","tool-routing-v2":"Tool Routing v2",session:"Session Management","widget-v2":"Widget v2","gate-format-v2":"Gate Format v2",content:"Content Quality"};async function D(){const s=await S("/api/eval/suites");if(!s.ok)throw new Error(`/api/eval/suites failed: ${s.status}`);const i=await s.json();return Array.isArray(i.suites)?i.suites:[]}async function U(s){const i=await S("/api/eval/run",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({suite:s})});if(!i.ok){const t=await i.text().catch(()=>"");throw new Error(`/api/eval/run failed: ${i.status} ${t.slice(0,200)}`)}return i.json()}function L(s){return s.total?Math.round(s.passed/s.total*100):NaN}function N(s){return Number.isNaN(s)?"slate":s>=100?"emerald":s>=80?"amber":"red"}function W(s){const i=[];for(const t of s.split(`
-`)){const n=t.trim();if(!n||n.startsWith("#"))continue;const l=n.match(/^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+([0-9eE+\-.NaN]+)/);if(!l)continue;const d={};if(l[2])for(const b of l[2].split(",")){const[m,...c]=b.split("=");if(!m)continue;const j=c.join("=").trim().replace(/^"|"$/g,"");d[m.trim()]=j}const a=Number(l[3]);Number.isNaN(a)||i.push({name:l[1],labels:d,value:a})}return i}async function q(s){const i=await S("/metrics");if(!i.ok)throw new Error(`/metrics failed: ${i.status}`);const t=await i.text(),n=W(t);return s?n.filter(l=>l.name===s||l.name.startsWith(`${s}_`)):n}const y={emerald:"bg-emerald-500/15 text-emerald-300 border-emerald-500/40",amber:"bg-amber-500/15 text-amber-300 border-amber-500/40",red:"bg-red-500/15 text-red-300 border-red-500/40",slate:"bg-slate-500/15 text-slate-300 border-slate-500/40"},J=[{id:"suites",label:"Suites",icon:F},{id:"memory",label:"Memory Regression",icon:$},{id:"redteam",label:"Red Team",icon:w},{id:"trends",label:"Trends",icon:P}];function le(){const[s,i]=h.useState("suites"),[t,n]=h.useState([]),[l,d]=h.useState(!1),[a,b]=h.useState(null),m=h.useCallback(async()=>{d(!0);try{const r=await D();n(x=>{const f=new Map(x.map(p=>[p.name,p]));return r.map(p=>f.get(p)??{name:p,status:"idle"})})}catch(r){console.error("listEvalSuites failed:",r),n([])}finally{d(!1)}},[]);h.useEffect(()=>{m()},[m]);const c=h.useCallback(async r=>{n(x=>x.map(f=>f.name===r?{...f,status:"running",error:void 0}:f));try{const x=await U(r);n(f=>f.map(p=>p.name===r?{...p,status:"done",result:x}:p))}catch(x){n(f=>f.map(p=>p.name===r?{...p,status:"error",error:x.message}:p))}},[]),j=h.useCallback(async()=>{for(const r of t)await c(r.name)},[t,c]),g=t.reduce((r,x)=>{var f,p,k;return{total:r.total+(((f=x.result)==null?void 0:f.total)??0),passed:r.passed+(((p=x.result)==null?void 0:p.passed)??0),failed:r.failed+(((k=x.result)==null?void 0:k.failed)??0)}},{total:0,passed:0,failed:0}),o=L(g),u=N(o),v=t.some(r=>r.status==="running");return e.jsxs("div",{className:"flex h-full flex-col gap-4 p-4 text-slate-200",children:[e.jsxs("div",{className:"flex items-center gap-3",children:[e.jsx("h2",{className:"text-lg font-semibold",children:"Eval Harness"}),e.jsx("div",{className:"flex items-center gap-1 rounded border border-slate-700 bg-slate-800/40 p-0.5",children:J.map(r=>{const x=r.icon,f=r.id===s;return e.jsxs("button",{onClick:()=>i(r.id),className:`flex items-center gap-1.5 rounded px-2.5 py-1 text-xs ${f?"bg-slate-700/80 text-slate-100":"text-slate-400 hover:bg-slate-700/30"}`,children:[e.jsx(x,{className:"h-3 w-3"}),r.label]},r.id)})}),s==="suites"&&e.jsxs(e.Fragment,{children:[e.jsx("button",{onClick:m,disabled:l||v,className:"rounded border border-slate-700 bg-slate-800/40 p-1.5 hover:bg-slate-800 disabled:opacity-50",title:"Refresh suite list",children:e.jsx(E,{className:`h-3.5 w-3.5 ${l?"animate-spin":""}`})}),e.jsxs("button",{onClick:j,disabled:t.length===0||v,className:"ml-auto flex items-center gap-1.5 rounded border border-emerald-500/40 bg-emerald-500/10 px-3 py-1.5 text-sm text-emerald-300 hover:bg-emerald-500/20 disabled:opacity-50",children:[v?e.jsx(M,{className:"h-3.5 w-3.5 animate-spin"}):e.jsx(_,{className:"h-3.5 w-3.5"}),"Run all suites"]})]})]}),s==="suites"&&e.jsxs(e.Fragment,{children:[e.jsxs("div",{className:`flex items-center gap-3 rounded-lg border p-3 text-sm ${y[u]}`,children:[e.jsxs("div",{className:"font-medium",children:["Overall: ",g.passed,"/",g.total," passed",Number.isFinite(o)&&e.jsxs("span",{className:"ml-2 opacity-80",children:["(",o,"%)"]})]}),g.failed>0&&e.jsxs("div",{className:"ml-auto flex items-center gap-1",children:[e.jsx(O,{className:"h-3.5 w-3.5"}),g.failed," failure",g.failed===1?"":"s"]})]}),e.jsxs("div",{className:"flex-1 space-y-2 overflow-auto",children:[t.length===0&&!l&&e.jsxs("div",{className:"rounded border border-slate-700 bg-slate-800/30 p-4 text-center text-sm text-slate-400",children:["No eval suites available. Verify ",e.jsx("code",{className:"font-mono text-xs",children:"/api/eval/suites"})," on the gateway."]}),t.map(r=>e.jsx(V,{suite:r,expanded:a===r.name,onToggle:()=>b(x=>x===r.name?null:r.name),onRun:()=>c(r.name)},r.name))]})]}),s==="memory"&&e.jsx(z,{suites:t,onRun:c}),s==="redteam"&&e.jsx(H,{suites:t,onRun:c}),s==="trends"&&e.jsx(K,{})]})}function z({suites:s,onRun:i}){const t=s.flatMap(l=>{var d;return(((d=l.result)==null?void 0:d.results)??[]).filter(a=>/memory|recall|stale|distract|context/i.test(a.name)).map(a=>({suite:l.name,case:a}))});if(t.length===0)return e.jsx("div",{className:"flex flex-1 items-center justify-center rounded border border-dashed border-slate-700 bg-slate-800/20 p-8 text-center",children:e.jsxs("div",{children:[e.jsx($,{className:"mx-auto mb-2 h-6 w-6 text-slate-500"}),e.jsx("div",{className:"text-sm text-slate-300",children:"No memory regression results yet."}),e.jsxs("div",{className:"mt-1 text-xs text-slate-500",children:["Run a suite that contains memory cases (session, content). Multi-turn fact recall is asserted via tape replay in ",e.jsx("code",{className:"font-mono",children:"tests/eval/memory.test.ts"}),"."]}),e.jsxs("div",{className:"mt-3 flex justify-center gap-2",children:[e.jsx("button",{onClick:()=>i("session"),className:"rounded border border-emerald-500/40 bg-emerald-500/10 px-3 py-1 text-xs text-emerald-300 hover:bg-emerald-500/20",children:"Run session suite"}),e.jsx("button",{onClick:()=>i("content"),className:"rounded border border-emerald-500/40 bg-emerald-500/10 px-3 py-1 text-xs text-emerald-300 hover:bg-emerald-500/20",children:"Run content suite"})]})]})});const n=t.filter(l=>l.case.passed).length;return e.jsxs("div",{className:"flex flex-1 flex-col gap-3 overflow-auto",children:[e.jsx("div",{className:`rounded-lg border p-3 text-sm ${y[N(n/t.length*100)]}`,children:e.jsxs("div",{className:"font-medium",children:["Memory recall: ",n,"/",t.length," cases passed",e.jsxs("span",{className:"ml-2 opacity-80",children:["(",Math.round(n/t.length*100),"%)"]})]})}),e.jsx("div",{className:"space-y-1.5",children:t.map(l=>e.jsxs("div",{className:`rounded p-2 text-xs ${l.case.passed?"bg-slate-800/30":"bg-red-950/30"}`,children:[e.jsxs("div",{className:"flex items-center gap-2",children:[l.case.passed?e.jsx(R,{className:"h-3.5 w-3.5 text-emerald-400"}):e.jsx(C,{className:"h-3.5 w-3.5 text-red-400"}),e.jsx("div",{className:"font-mono text-slate-200",children:l.case.name}),e.jsx("div",{className:"ml-auto text-[10px] text-slate-500",children:l.suite})]}),!l.case.passed&&l.case.errors.length>0&&e.jsx("ul",{className:"mt-1 ml-5 list-disc text-red-300",children:l.case.errors.map((d,a)=>e.jsx("li",{children:d},a))})]},`${l.suite}/${l.case.name}`))})]})}const G=[{id:"ignore-prev",label:"Ignore-previous",pattern:/ignore.*(previous|prior)|disregard|forget/i},{id:"dan",label:"DAN / persona swap",pattern:/\bDAN\b|do anything|jailbreak/i},{id:"prompt-extract",label:"Prompt extraction",pattern:/system prompt|reveal|extract|leak/i},{id:"path-traversal",label:"Path traversal",pattern:/\.\.\/|\/etc\/passwd|path traversal/i},{id:"shell-inject",label:"Shell injection",pattern:/;.*rm|`whoami`|\$\(|\| ?bash/i},{id:"url-scheme",label:"URL scheme abuse",pattern:/file:\/\/|dict:\/\/|gopher:\/\//i},{id:"markdown-js",label:"Markdown JS link",pattern:/\[.*\]\(javascript:/i},{id:"other",label:"Other",pattern:/.*/}];function H({suites:s,onRun:i}){var d;const t=s.find(a=>a.name==="adversarial"),n=((d=t==null?void 0:t.result)==null?void 0:d.results)??[];if(n.length===0)return e.jsx("div",{className:"flex flex-1 items-center justify-center rounded border border-dashed border-slate-700 bg-slate-800/20 p-8 text-center",children:e.jsxs("div",{children:[e.jsx(w,{className:"mx-auto mb-2 h-6 w-6 text-slate-500"}),e.jsx("div",{className:"text-sm text-slate-300",children:"No red-team results yet."}),e.jsx("div",{className:"mt-1 text-xs text-slate-500",children:"Run the adversarial suite to populate this grid."}),e.jsx("button",{onClick:()=>i("adversarial"),className:"mt-3 rounded border border-red-500/40 bg-red-500/10 px-3 py-1 text-xs text-red-300 hover:bg-red-500/20",children:"Run adversarial suite"})]})});const l=G.map(a=>{const b=n.filter(c=>a.pattern.test(c.name)||a.pattern.test(c.content||"")),m=b.filter(c=>c.passed).length;return{vector:a,total:b.length,blocked:m,matched:b}}).filter(a=>a.total>0);return e.jsxs("div",{className:"flex flex-1 flex-col gap-3 overflow-auto",children:[e.jsx("div",{className:"text-xs text-slate-400",children:"Pass = attack BLOCKED (TITAN refused or sanitized). Fail = attack possibly EXPLOITED — investigate the case."}),e.jsx("div",{className:"grid grid-cols-1 gap-2 sm:grid-cols-2 lg:grid-cols-3",children:l.map(a=>{const b=a.total>0?Math.round(a.blocked/a.total*100):0,m=N(b);return e.jsxs("div",{className:`rounded-lg border p-3 ${y[m]}`,children:[e.jsxs("div",{className:"flex items-center gap-2 text-sm font-medium",children:[e.jsx(w,{className:"h-3.5 w-3.5"}),a.vector.label]}),e.jsxs("div",{className:"mt-1 text-xs",children:[a.blocked,"/",a.total," blocked (",b,"%)"]}),e.jsxs("div",{className:"mt-2 space-y-1",children:[a.matched.slice(0,4).map(c=>e.jsxs("div",{className:"flex items-center gap-1.5 text-[11px]",children:[c.passed?e.jsx(R,{className:"h-2.5 w-2.5 text-emerald-400"}):e.jsx(C,{className:"h-2.5 w-2.5 text-red-400"}),e.jsx("span",{className:"font-mono",children:c.name})]},c.name)),a.matched.length>4&&e.jsxs("div",{className:"text-[10px] text-slate-500",children:["+ ",a.matched.length-4," more"]})]})]},a.vector.id)})})]})}function K(){const[s,i]=h.useState([]),[t,n]=h.useState(!1),[l,d]=h.useState(null),[a,b]=h.useState(null),m=h.useCallback(async()=>{n(!0),d(null);try{const o=await q("titan_eval");i(o),b(new Date)}catch(o){d(o.message)}finally{n(!1)}},[]);h.useEffect(()=>{m();const o=setInterval(m,3e4);return()=>clearInterval(o)},[m]);const c=h.useMemo(()=>s.filter(o=>o.name==="titan_eval_pass_rate"&&o.labels.suite).map(o=>({suite:o.labels.suite,rate:o.value})).sort((o,u)=>o.rate-u.rate),[s]),j=h.useMemo(()=>{const o=new Map;for(const u of s){if(u.name!=="titan_eval_cases_total"||!u.labels.suite)continue;const v=o.get(u.labels.suite)??{passed:0,failed:0};u.labels.outcome==="passed"?v.passed=u.value:u.labels.outcome==="failed"&&(v.failed=u.value),o.set(u.labels.suite,v)}return o},[s]),g=100;return e.jsxs("div",{className:"flex flex-1 flex-col gap-3 overflow-auto",children:[e.jsxs("div",{className:"flex items-center gap-2 text-xs text-slate-400",children:[e.jsxs("span",{children:["Live from ",e.jsx("code",{className:"font-mono",children:"/metrics"})," (titan_eval_pass_rate gauge)."]}),e.jsx("span",{className:"ml-auto",children:a?`updated ${a.toLocaleTimeString()}`:""}),e.jsx("button",{onClick:m,disabled:t,className:"rounded border border-slate-700 bg-slate-800/40 p-1 hover:bg-slate-800 disabled:opacity-50",children:e.jsx(E,{className:`h-3 w-3 ${t?"animate-spin":""}`})})]}),l&&e.jsx("div",{className:"rounded border border-red-500/40 bg-red-500/10 p-2 text-xs text-red-300",children:l}),c.length===0&&!l&&e.jsxs("div",{className:"rounded border border-dashed border-slate-700 bg-slate-800/20 p-6 text-center text-sm text-slate-400",children:["No eval pass-rate metrics yet. Run a suite via the Suites tab to populate ",e.jsx("code",{className:"font-mono",children:"titan_eval_pass_rate"}),"."]}),e.jsx("div",{className:"space-y-1.5",children:c.map(({suite:o,rate:u})=>{const v=j.get(o),r=N(u),x=A[o]??o;return e.jsx("div",{className:`rounded border p-2 text-xs ${y[r]}`,children:e.jsxs("div",{className:"flex items-center gap-2",children:[e.jsx("div",{className:"w-32 truncate font-medium text-slate-200",children:x}),e.jsx("div",{className:"relative h-4 flex-1 overflow-hidden rounded bg-slate-800/60",children:e.jsx("div",{className:`h-full ${r==="emerald"?"bg-emerald-500/60":r==="amber"?"bg-amber-500/60":r==="red"?"bg-red-500/60":"bg-slate-500/60"}`,style:{width:`${u/g*100}%`}})}),e.jsxs("div",{className:"w-16 text-right font-mono",children:[u,"%"]}),v&&e.jsxs("div",{className:"w-20 text-right text-[10px] text-slate-500",children:[v.passed,"/",v.passed+v.failed," ever"]})]})},o)})})]})}function V({suite:s,expanded:i,onToggle:t,onRun:n}){var b;const l=A[s.name]??s.name,d=s.result?L(s.result):NaN,a=N(d);return e.jsxs("div",{className:`rounded-lg border ${y[a]}`,children:[e.jsxs("div",{className:"flex items-center gap-3 p-3",children:[e.jsx("button",{onClick:t,disabled:!s.result,className:"text-slate-300 disabled:opacity-30","aria-label":i?"Collapse":"Expand",children:i?e.jsx(I,{className:"h-4 w-4"}):e.jsx(B,{className:"h-4 w-4"})}),e.jsx("div",{className:"font-medium text-slate-100",children:l}),e.jsx("div",{className:"text-xs text-slate-400",children:s.result?`${s.result.passed}/${s.result.total} (${d}%)`:s.status==="running"?"running…":s.status==="error"?`error: ${s.error}`:"idle"}),((b=s.result)==null?void 0:b.durationMs)!==void 0&&e.jsxs("div",{className:"text-xs text-slate-500",children:[(s.result.durationMs/1e3).toFixed(1),"s"]}),e.jsxs("button",{onClick:n,disabled:s.status==="running",className:"ml-auto flex items-center gap-1 rounded border border-slate-600 bg-slate-800/60 px-2 py-1 text-xs text-slate-200 hover:bg-slate-800 disabled:opacity-50",children:[s.status==="running"?e.jsx(M,{className:"h-3 w-3 animate-spin"}):e.jsx(_,{className:"h-3 w-3"}),"Run"]})]}),i&&s.result&&e.jsx("div",{className:"space-y-1.5 border-t border-slate-700/40 px-3 py-2",children:s.result.results.map(m=>e.jsx(X,{caseResult:m},m.name))})]})}function X({caseResult:s}){const i=s.expectedToolSequence!==void 0&&JSON.stringify(s.expectedToolSequence)!==JSON.stringify(s.toolsUsed);return e.jsxs("div",{className:`rounded p-2 text-xs ${s.passed?"bg-slate-800/30":"bg-red-950/30"}`,children:[e.jsxs("div",{className:"flex items-center gap-2",children:[s.passed?e.jsx(R,{className:"h-3.5 w-3.5 text-emerald-400"}):e.jsx(C,{className:"h-3.5 w-3.5 text-red-400"}),e.jsx("div",{className:"font-mono text-slate-200",children:s.name}),e.jsxs("div",{className:"ml-auto text-slate-500",children:[s.durationMs,"ms"]})]}),!s.passed&&s.errors.length>0&&e.jsx("ul",{className:"mt-1 ml-5 list-disc text-red-300",children:s.errors.map((t,n)=>e.jsx("li",{children:t},n))}),i&&e.jsxs("div",{className:"mt-2 ml-5 grid grid-cols-2 gap-2 rounded bg-slate-900/40 p-2",children:[e.jsxs("div",{children:[e.jsxs("div",{className:"mb-1 flex items-center gap-1 text-slate-400",children:[e.jsx(T,{className:"h-3 w-3"})," expected"]}),(s.expectedToolSequence??[]).map((t,n)=>e.jsxs("div",{className:"font-mono text-emerald-300",children:[n+1,". ",t]},n))]}),e.jsxs("div",{children:[e.jsxs("div",{className:"mb-1 flex items-center gap-1 text-slate-400",children:[e.jsx(T,{className:"h-3 w-3"})," actual"]}),s.toolsUsed.length===0&&e.jsx("div",{className:"italic text-slate-500",children:"(no tool calls)"}),s.toolsUsed.map((t,n)=>{var a;const d=((a=s.expectedToolSequence)==null?void 0:a[n])===t;return e.jsxs("div",{className:`font-mono ${d?"text-emerald-300":"text-red-300"}`,children:[n+1,". ",t]},n)})]})]})]})}export{le as default};