stable-harness 0.0.21 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/guides/quality-gates.md +13 -0
- package/package.json +1 -1
- package/packages/core/dist/quality/event-evidence.d.ts +6 -0
- package/packages/core/dist/quality/event-evidence.js +1 -1
- package/packages/core/dist/quality/index.d.ts +1 -0
- package/packages/core/dist/quality/index.js +1 -1
- package/packages/core/dist/quality/profile.js +1 -1
- package/packages/core/dist/quality/runtime.js +1 -1
- package/packages/core/dist/quality/synthesis.d.ts +2 -0
- package/packages/core/dist/quality/synthesis.js +1 -0
- package/packages/core/dist/quality/types.d.ts +5 -0
- package/packages/core/dist/runtime/events.d.ts +6 -0
|
@@ -39,6 +39,11 @@ runtime:
|
|
|
39
39
|
verdict.
|
|
40
40
|
executionReview:
|
|
41
41
|
requireToolEvidence: true
|
|
42
|
+
rejectUngroundedNumbers: true
|
|
43
|
+
synthesis:
|
|
44
|
+
enabled: true
|
|
45
|
+
mode: evidence_only
|
|
46
|
+
maxEvidenceItems: 5
|
|
42
47
|
recovery:
|
|
43
48
|
maxLoops: 2
|
|
44
49
|
```
|
|
@@ -75,6 +80,14 @@ declared inventory. If execution review fails, the runtime asks the agent to
|
|
|
75
80
|
continue from the current state with ReAct: inspect the evidence gap, choose the
|
|
76
81
|
next declared tool or subagent action, execute it, and synthesize from evidence.
|
|
77
82
|
|
|
83
|
+
When `synthesis.enabled` is true, the runtime may recover from a rejected final
|
|
84
|
+
answer by building an `evidence_only` report from successful observed tool or
|
|
85
|
+
delegated-task outputs. This is a control-plane synthesis path, not a domain
|
|
86
|
+
writer: it does not call more tools, does not add pretrained facts, and runs the
|
|
87
|
+
same execution review again before delivery. Control outputs such as invalid
|
|
88
|
+
input, approval blocks, or repeated-call limits are preserved as blockers or
|
|
89
|
+
evidence gaps rather than treated as factual evidence.
|
|
90
|
+
|
|
78
91
|
BetterCall remains focused on individual tool-call reliability. Quality gates
|
|
79
92
|
consume tool and repair diagnostics as evidence, but they live in Stable Harness
|
|
80
93
|
because they govern the run lifecycle rather than a single tool call.
|
package/package.json
CHANGED
|
@@ -2,4 +2,10 @@ import type { RuntimeEvent } from "../types.js";
|
|
|
2
2
|
export declare function hasPlanningEvidence(events: RuntimeEvent[]): boolean;
|
|
3
3
|
export declare function successfulEvidenceToolIds(events: RuntimeEvent[]): string[];
|
|
4
4
|
export declare function successfulEvidenceOutputs(events: RuntimeEvent[]): string[];
|
|
5
|
+
export type SuccessfulEvidenceItem = {
|
|
6
|
+
source: string;
|
|
7
|
+
output: string;
|
|
8
|
+
};
|
|
9
|
+
export declare function successfulEvidenceItems(events: RuntimeEvent[]): SuccessfulEvidenceItem[];
|
|
5
10
|
export declare function controlBlockers(events: RuntimeEvent[]): string[];
|
|
11
|
+
export declare function controlGaps(events: RuntimeEvent[]): string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export function hasPlanningEvidence(t){return t.some(t=>{const e=readAdapterEvent(t);return!!e&&("plan"===e.traceType||String(e.traceLabel??"").startsWith("plan.")||"write_todos"===e.toolId&&String(e.phase??"").startsWith("agent.tool."))})}export function successfulEvidenceToolIds(t){const e=t.flatMap(t=>{if("runtime.tool.direct.completed"===t.type)return[t.toolId];const e=readAdapterEvent(t);return e&&"agent.tool.result"===e.phase&&"string"==typeof e.toolId&&isSuccessfulEvidenceEvent(e)?[e.toolId]:[]});return[...new Set(e)]}export function successfulEvidenceOutputs(t){return t.flatMap(t=>{if("runtime.tool.direct.completed"===t.type)return stringifyEvidence(t.output);const e=readAdapterEvent(t);return e&&"agent.tool.result"===e.phase&&"string"==typeof e.toolId?!isSuccessfulEvidenceEvent(e)||function isPlanningTool(t){return/^(?:write_todos|read_todos)$/u.test(t)}(e.toolId)?[]:stringifyEvidence(e.output):[]})}export function controlBlockers(t){return t.flatMap(t=>{const e=readAdapterEvent(t),r=readString(e?.controlStatus)??readOutputStatus(e?.output);return r&&function isBlockerStatus(t){return/^(?:blocked|approval_required|schema_repair_failed|tool_argument_error|invalid_input)$/iu.test(t)}(r)?[`${readString(e?.toolId)??"tool"}:${r}`]:[]})}function stringifyEvidence(t){return"string"==typeof t?t.trim()?[t]:[]:null==t?[]:[JSON.stringify(t)]}function readAdapterEvent(t){return"runtime.adapter.event"===t.type&&isRecord(t.event)?t.event:void 0}function isSuccessfulEvidenceEvent(t){const e=readString(t.controlStatus)??readOutputStatus(t.output);return!e||/^(?:completed|success|ok|recorded)$/iu.test(e)}function readOutputStatus(t){if("string"!=typeof t)return;const e=function parseJsonRecord(t){try{const e=JSON.parse(t);return isRecord(e)?e:void 0}catch{return}}(t);return"string"==typeof e?.status?e.status:t.match(/^Status:\s*([A-Za-z0-9_-]+)/imu)?.[1]}function isRecord(t){return"object"==typeof t&&null!==t&&!Array.isArray(t)}function readString(t){return"string"==typeof t&&t.trim()?t:void 0}
|
|
1
|
+
export function hasPlanningEvidence(t){return t.some(t=>{const e=readAdapterEvent(t);return!!e&&("plan"===e.traceType||String(e.traceLabel??"").startsWith("plan.")||"write_todos"===e.toolId&&String(e.phase??"").startsWith("agent.tool."))})}export function successfulEvidenceToolIds(t){const e=t.flatMap(t=>{if("runtime.tool.direct.completed"===t.type)return[t.toolId];const e=readAdapterEvent(t);return e&&"agent.tool.result"===e.phase&&"string"==typeof e.toolId&&isSuccessfulEvidenceEvent(e)?[e.toolId]:[]});return[...new Set(e)]}export function successfulEvidenceOutputs(t){return successfulEvidenceItems(t).map(t=>t.output)}export function successfulEvidenceItems(t){return t.flatMap(t=>{if("runtime.tool.direct.completed"===t.type)return stringifyEvidence(t.output).map(e=>({source:t.toolId,output:e}));const e=readAdapterEvent(t);return e&&"agent.tool.result"===e.phase&&"string"==typeof e.toolId?!isSuccessfulEvidenceEvent(e)||function isPlanningTool(t){return/^(?:write_todos|read_todos)$/u.test(t)}(e.toolId)?[]:stringifyEvidence(e.output).map(t=>({source:e.toolId,output:t})):[]})}export function controlBlockers(t){return t.flatMap(t=>{const e=readAdapterEvent(t),r=readString(e?.controlStatus)??readOutputStatus(e?.output);return r&&function isBlockerStatus(t){return/^(?:blocked|approval_required|schema_repair_failed|tool_argument_error|invalid_input)$/iu.test(t)}(r)?[`${readString(e?.toolId)??"tool"}:${r}`]:[]})}export function controlGaps(t){return t.flatMap(t=>{const e=readAdapterEvent(t),r=readString(e?.controlStatus)??readOutputStatus(e?.output);return r&&function isGapStatus(t){return/^(?:dependency_required|plan_required|repeated_tool_call_limit|duplicate_tool_call)$/iu.test(t)}(r)?[`${readString(e?.toolId)??"tool"}:${r}`]:[]})}function stringifyEvidence(t){return"string"==typeof t?t.trim()?[t]:[]:null==t?[]:[JSON.stringify(t)]}function readAdapterEvent(t){return"runtime.adapter.event"===t.type&&isRecord(t.event)?t.event:void 0}function isSuccessfulEvidenceEvent(t){const e=readString(t.controlStatus)??readOutputStatus(t.output);return!e||/^(?:completed|success|ok|recorded)$/iu.test(e)}function readOutputStatus(t){if("string"!=typeof t)return;const e=function parseJsonRecord(t){try{const e=JSON.parse(t);return isRecord(e)?e:void 0}catch{return}}(t);return"string"==typeof e?.status?e.status:t.match(/^Status:\s*([A-Za-z0-9_-]+)/imu)?.[1]}function isRecord(t){return"object"==typeof t&&null!==t&&!Array.isArray(t)}function readString(t){return"string"==typeof t&&t.trim()?t:void 0}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export*from"./types.js";export*from"./profile.js";export*from"./planning-review.js";export*from"./execution-review.js";export*from"./llm-review.js";export*from"./recovery-policy.js";export*from"./runtime.js";
|
|
1
|
+
export*from"./types.js";export*from"./profile.js";export*from"./planning-review.js";export*from"./execution-review.js";export*from"./llm-review.js";export*from"./recovery-policy.js";export*from"./synthesis.js";export*from"./runtime.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export function resolveQualityPolicy(e,n){const r=n.config.quality??e.quality,
|
|
1
|
+
export function resolveQualityPolicy(e,n){const r=n.config.quality??e.quality,i=function readProfile(e){const n="string"==typeof e?e:isRecord(e)?e.profile:void 0;return"fast"===n||"balanced"===n||"strict"===n?n:"off"}(r),o=function profileDefaults(e,n){const r=n.tools.length>0||n.subagents.length>0||(n.skills?.length??0)>0;return"off"===e?function disabledPolicy(e){return{enabled:!1,profile:e,reviewer:{mode:"deterministic"},planningReview:{enabled:!1,requirePlan:!1},executionReview:{enabled:!1,requireToolEvidence:!1,rejectEmptyFinal:!1,stopOnBlocker:!1,rejectUngroundedNumbers:!1},synthesis:{enabled:!1,mode:"evidence_only",maxEvidenceItems:5},recovery:{enabled:!1,maxLoops:0}}}(e):{enabled:!0,profile:e,planningReview:{enabled:"fast"!==e,requirePlan:"fast"!==e&&r},executionReview:{enabled:!0,requireToolEvidence:"strict"===e&&r,rejectEmptyFinal:!0,stopOnBlocker:!0,rejectUngroundedNumbers:"strict"===e},synthesis:{enabled:!1,mode:"evidence_only",maxEvidenceItems:5},recovery:{enabled:"fast"!==e,maxLoops:"strict"===e?3:2}}}(i,n),t=isRecord(r)?r:{},a=function readReviewer(e){const n=isRecord(e.reviewer)?e.reviewer:e,r="llm"===n.mode||"string"==typeof n.modelRef?"llm":"deterministic",i=isRecord(n.prompts)?n.prompts:{},o=readString(n.modelRef);return{mode:r,...o?{modelRef:o}:{},...readString(n.planningPrompt)??readString(i.planning)?{planningPrompt:readString(n.planningPrompt)??readString(i.planning)}:{},...readString(n.executionPrompt)??readString(i.execution)?{executionPrompt:readString(n.executionPrompt)??readString(i.execution)}:{}}}(t);return{enabled:o.enabled,profile:i,...a?{reviewer:a}:{},planningReview:{enabled:readBoolean(t.planningReview,"enabled")??o.planningReview.enabled,requirePlan:readBoolean(t.planningReview,"requirePlan")??o.planningReview.requirePlan},executionReview:{enabled:readBoolean(t.executionReview,"enabled")??o.executionReview.enabled,requireToolEvidence:readBoolean(t.executionReview,"requireToolEvidence")??o.executionReview.requireToolEvidence,rejectEmptyFinal:readBoolean(t.executionReview,"rejectEmptyFinal")??o.executionReview.rejectEmptyFinal,stopOnBlocker:readBoolean(t.executionReview,"stopOnBlocker")??o.executionReview.stopOnBlocker,rejectUngroundedNumbers:readBoolean(t.executionReview,"rejectUngroundedNumbers")??o.executionReview.rejectUngroundedNumbers},synthesis:{enabled:readBoolean(t.synthesis,"enabled")??o.synthesis.enabled,mode:(d=t.synthesis,("evidence_only"===(isRecord(d)?d:{}).mode?"evidence_only":void 0)??o.synthesis.mode),maxEvidenceItems:readPositiveInteger(t.synthesis,"maxEvidenceItems")??o.synthesis.maxEvidenceItems},recovery:{enabled:readBoolean(t.recovery,"enabled")??o.recovery.enabled,maxLoops:readPositiveInteger(t.recovery,"maxLoops")??o.recovery.maxLoops}};var d}function readString(e){return"string"==typeof e&&e.trim()?e.trim():void 0}function readBoolean(e,n){const r=isRecord(e)?e:{};return"boolean"==typeof r[n]?r[n]:void 0}function readPositiveInteger(e,n){const r=(isRecord(e)?e:{})[n];return"number"==typeof r&&Number.isInteger(r)&&r>0?r:void 0}function isRecord(e){return"object"==typeof e&&null!==e&&!Array.isArray(e)}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{successfulEvidenceOutputs as e}from"./event-evidence.js";import{buildQualityRecoveryRequest as
|
|
1
|
+
import{successfulEvidenceOutputs as e}from"./event-evidence.js";import{buildQualityRecoveryRequest as t}from"./recovery-policy.js";import{reviewExecutionEvidence as i}from"./execution-review.js";import{reviewWithLlm as n}from"./llm-review.js";import{reviewPlanningEvidence as s}from"./planning-review.js";import{synthesizeEvidenceOnlyReport as r}from"./synthesis.js";export async function recoverQualityReview(e,t,i,n){if(!n.enabled)return i;let s=t,r=i;for(let t=0;t<n.recovery.maxLoops+1;t+=1){const i=await emitPlanningReview(e,s,r,n);if("blocked"===i.verdict)return qualityFailureOutput("planning",i);const u=buildQualityRecovery(e,s,i,"planning",n,t);if(u){s=u,r=await e.runAdapter(s);continue}const a=await emitExecutionReview(e,s,r,n),o=buildQualityRecovery(e,s,a,"execution",n,t);if(!o)return"pass"===a.verdict?r:await trySynthesizeExecution(e,s,a,n)??qualityFailureOutput("execution",a);s=o,r=await e.runAdapter(s)}return qualityFailureOutput("execution",{verdict:"blocked",issues:[{code:"quality_recovery_exhausted",message:`Quality recovery exceeded maxLoops=${n.recovery.maxLoops}.`,recoverable:!1}]})}async function trySynthesizeExecution(e,t,i,n){const s=r({...reviewInputFor(e,t),output:void 0},i,n);if(!s)return;e.emit({type:"runtime.quality.synthesis.created",requestId:e.requestId,sessionId:e.sessionId,agentId:e.agent.id,mode:n.synthesis.mode});const u={text:s};return"pass"===(await emitExecutionReview(e,t,u,n)).verdict?u:void 0}function emitPlanningReview(e,t,i,n){return emitReview(e,"planning",s,t,i,n)}function emitExecutionReview(e,t,n,s){return emitReview(e,"execution",i,t,n,s)}async function emitReview(e,t,i,s,r,u){const a={...reviewInputFor(e,s),output:r},o="planning"===t?u.planningReview.enabled:u.executionReview.enabled;if(!o)return i(a,u);const c=i(a,u),d=await n({phase:t,review:a,policy:u,model:e.reviewModel}),v="pass"===c.verdict?d??c:c;return o&&function emitReviewEvent(e,t,i){"planning"!==t?e.emit({type:"runtime.quality.execution.reviewed",requestId:e.requestId,sessionId:e.sessionId,agentId:e.agent.id,verdict:i.verdict,issues:i.issues}):e.emit({type:"runtime.quality.planning.reviewed",requestId:e.requestId,sessionId:e.sessionId,agentId:e.agent.id,verdict:i.verdict,issues:i.issues})}(e,t,v),v}function buildQualityRecovery(i,n,s,r,u,a){if(a>=u.recovery.maxLoops)return;const o=t({request:n,result:s,phase:r,policy:u,availableToolIds:i.agent.tools,availableSubagentIds:i.agent.subagents,observedEvidence:"execution"===r?e(i.getEvents()):[]});return o&&i.emit({type:"runtime.quality.recovery.started",requestId:i.requestId,sessionId:i.sessionId,agentId:i.agent.id,phase:r,attempt:a+1,verdict:s.verdict}),o}function reviewInputFor(e,t){return{workspace:e.workspace,agent:e.agent,request:t,events:e.getEvents()}}function qualityFailureOutput(e,t){return{text:[`Stable runtime quality review blocked final delivery during ${e}.`,"",...t.issues.length>0?t.issues.map(e=>`- ${e.code}: ${e.message}`):["- quality_review_failed: Quality review did not pass."]].join("\n")}}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{controlBlockers as e,controlGaps as n,successfulEvidenceItems as s}from"./event-evidence.js";export function synthesizeEvidenceOnlyReport(t,o,i){if(!i.enabled||!i.synthesis.enabled||"evidence_only"!==i.synthesis.mode)return;if("pass"===o.verdict||!function hasRecoverableSynthesisIssue(e){return e.issues.some(e=>"control_blocker"!==e.code)}(o))return;const r=s(t.events).slice(-i.synthesis.maxEvidenceItems),c=e(t.events),a=n(t.events);return 0!==r.length||0!==c.length||0!==a.length?["# Grounded report","","Stable Harness rejected the previous final answer because it was not fully supported by observed runtime evidence. The report below is synthesized only from completed tool or delegated-task evidence.","","## Evidence",...evidenceLines(r),"","## Evidence gaps and blockers",...gapLines(c,a)].join("\n"):void 0}function evidenceLines(e){return 0===e.length?["- Evidence gap: no successful tool or delegated-task evidence was available."]:e.map(e=>`- ${e.source}: ${function formatEvidence(e){const n=e.replace(/\s+/gu," ").trim();return n.length>1200?`${n.slice(0,1197)}...`:n}(e.output)}`)}function gapLines(e,n){return 0===e.length&&0===n.length?["- Evidence gap: any requested category not present in the evidence above remains unsupported."]:[...e.map(e=>`- Blocked: ${e}`),...n.map(e=>`- Evidence gap: ${e}`)]}
|
|
@@ -30,6 +30,11 @@ export type QualityPolicy = {
|
|
|
30
30
|
stopOnBlocker: boolean;
|
|
31
31
|
rejectUngroundedNumbers: boolean;
|
|
32
32
|
};
|
|
33
|
+
synthesis: {
|
|
34
|
+
enabled: boolean;
|
|
35
|
+
mode: "evidence_only";
|
|
36
|
+
maxEvidenceItems: number;
|
|
37
|
+
};
|
|
33
38
|
recovery: {
|
|
34
39
|
enabled: boolean;
|
|
35
40
|
maxLoops: number;
|
|
@@ -222,6 +222,12 @@ export type RuntimeEvent = {
|
|
|
222
222
|
phase: "planning" | "execution";
|
|
223
223
|
attempt: number;
|
|
224
224
|
verdict: string;
|
|
225
|
+
} | {
|
|
226
|
+
type: "runtime.quality.synthesis.created";
|
|
227
|
+
requestId: string;
|
|
228
|
+
sessionId: string;
|
|
229
|
+
agentId: string;
|
|
230
|
+
mode: "evidence_only";
|
|
225
231
|
} | {
|
|
226
232
|
type: "runtime.adapter.event";
|
|
227
233
|
requestId: string;
|