synergyspec-selfevolving 1.4.0 → 2.1.0

package/dist/core/self-evolution/reward-agent.d.ts

@@ -0,0 +1,379 @@
+/**
+ * 奖励智能体 REWARD AGENT — loop v2 (self-evolution as in-context RL).
+ *
+ * LLM as judge. Reads ONE episode's two arms from the {@link import('./episode-store.js')}
+ * episode store and CALCULATES 算分 reward(主臂) and reward(基线臂), each in
+ * [0,1], anchored on the objective evidence on disk (tests · health · 轨迹度量).
+ * advantage ＝ reward(主臂) − reward(基线臂). It finds errors / names gaps with a
+ * suggested direction ＝ 文本梯度 textual gradient; it NEVER edits any file; when
+ * there is no nameable gap it 弃权 abstains.
+ *
+ * It is a SIBLING of the 演进智能体 EVOLVING AGENT (optimizer.step; ONE bounded
+ * edit ≤L; never scores) — never parent-child. The reward agent runs FIRST (it
+ * scores), the 演进智能体 runs AFTER (it edits); each is code-spawned in a fresh
+ * context via {@link runHeadlessAgent} from `./host-harness.js` — NOT a skill,
+ * the same headless-spawn seam the canonical proposer uses.
+ *
+ * Write boundary: this module's ONLY write path is the episode dir — it calls
+ * {@link writeDiagnosis} and {@link advanceEpisodeStage} (to `scored`). It never
+ * touches a canonical target file, a 策略 POLICY snapshot, or the change dir
+ * (which it reads, never copies). The loss/health numbers from
+ * `src/core/fitness/loss.ts` stay on disk as ANCHORS — the reward score itself
+ * is the JUDGE'S OWN, anchored on but not equal to the loss.
+ *
+ * Uses the same fenced-block agent idiom as the EVOLVING AGENT: one fenced-block
+ * output discipline, a bounded repair loop on parse/validation failure,
+ * fail-closed plain (Zod-free) validation, and atomic episode-store writes.
+ */
+import { spawn as nodeSpawn } from 'node:child_process';
+import { type EpisodeRecord } from './episode-store.js';
+export declare class RewardAgentOutputInvalid extends Error {
+    constructor(message: string);
+}
+export declare class RewardAgentInvocationError extends Error {
+    constructor(stderr: string);
+}
+/**
+ * The objective record each arm persists as `objective.json`, as the 奖励智能体
+ * REWARD AGENT READS it. A null-safe SUPERSET of the canonical on-disk
+ * {@link import('./critic-agent.js').ArmObjective} (which is the flat shape both
+ * arms WRITE): the MAIN ARM may instead carry the nested-learn fallback fields
+ * (`testMetrics`, `healthSignal`, a nested {@link import('../fitness/loss.js').PerChangeLoss}
+ * `loss`) when an older capture path wrote a raw {@link import('../fitness/sample.js').FitnessSample}.
+ * This module reads ONLY the fields it maps to anchors and is null-safe to
+ * either shape, so a divergence in the exact layout cannot crash the judge.
+ *
+ * Named distinctly from the canonical `ArmObjective` (the barrel re-exports the
+ * critic-agent declaration) so there is no ambiguous duplicate export.
+ */
+export interface RewardArmObjectiveInput {
+    /** Flat pass rate in [0,1] (CRITIC AGENT shape). */
+    passRate?: number | null;
+    testsTotal?: number | null;
+    testsFailed?: number | null;
+    /** Flat normalized health penalty in [0,1] (CRITIC AGENT shape). */
+    healthPenalty?: number | null;
+    /** Flat blended loss in [0,1] (CRITIC AGENT shape) OR nested PerChangeLoss (MAIN ARM shape). */
+    loss?: number | null | {
+        functionalLoss?: number | null;
+        healthPenalty?: number | null;
+        loss?: number | null;
+    };
+    verified?: boolean;
+    observedStatus?: string | null;
+    measuredAt?: string | null;
+    /** Whether a real test-runner invocation was OBSERVED (P2 confidence calibration). */
+    testRunObserved?: boolean;
+    /** Failing test ids + assertion lines parsed from the observed runner output (P1 contrast). */
+    observedFailures?: {
+        testId: string;
+        file?: string;
+        assertion?: string;
+    }[];
+    /** MAIN ARM shape (FitnessSample): functional metrics live under testMetrics. */
+    testMetrics?: {
+        passRate?: number | null;
+    } | null;
+    /** MAIN ARM shape (FitnessSample): the raw health signal, distinct from loss.healthPenalty. */
+    healthSignal?: number | null;
+    /** Additive/forward-compatible fields pass through unread. */
+    [key: string]: unknown;
+}
+/** The anchors block mapped from both arms' objectives (nulls where skipped). */
+export interface DiagnosisAnchors {
+    mainLoss: number | null;
+    baselineLoss: number | null;
+    mainPassRate: number | null;
+    baselinePassRate: number | null;
+    mainHealthPenalty: number | null;
+    baselineHealthPenalty: number | null;
+    /**
+     * The VERBOSITY component of the code-health signal, surfaced separately so
+     * the judge can apply a compression term (① composite reward). `null` when no
+     * verbosity sub-signal was captured — the rubric then judges verbosity from
+     * the artifacts alone. Additive/optional: absent on schemaVersion-1 captures.
+     */
+    mainVerbosity?: number | null;
+    baselineVerbosity?: number | null;
+}
+/** One named error the judge found, addressed to a quoted span in a real file. */
+export interface DiagnosisError {
+    arm: 'main' | 'baseline';
+    description: string;
+    evidence: {
+        file: string;
+        quote: string;
+    };
+}
+/**
+ * The failure mode of a gap (⑥ weakness-class), enabling the 演进智能体 EVOLVING
+ * AGENT to aim its bounded edit: `forgetting` (a capability the baseline had and
+ * the main arm lost), `boundary` (an edge/limit case), `rare` (a low-frequency
+ * scenario), `logic` (an outright wrong behavior), `verbosity` (bloat/redundancy
+ * to prune), or `other`.
+ */
+export type WeaknessClass = 'forgetting' | 'boundary' | 'rare' | 'logic' | 'verbosity' | 'other';
+export type GapSeverity = 'high' | 'medium' | 'low';
+/** One nameable gap, addressed to a heading (`section`) in a target file. */
+export interface DiagnosisGap {
+    file: string;
+    /** A heading in the target file; the `'*'` wildcard is allowed. */
+    section: string;
+    description: string;
+    /** ⑥ Optional weakness-class — the failure mode this gap represents. */
+    weaknessClass?: WeaknessClass;
+    /** ⑥ Optional severity — high-severity gaps are addressed first. */
+    severity?: GapSeverity;
+}
+/**
+ * The judge's overall verdict (⑤ ternary). `no-gap` is the existing abstain
+ * (nothing to improve); `insufficient-signal` is the NEW honest abstain (the
+ * judge cannot tell — set by the statistical layer when the advantage is within
+ * the A/A noise floor, or by the tamper check in block mode). `main-better` /
+ * `baseline-better` / `tie` are scored outcomes.
+ */
+export type RewardVerdict = 'main-better' | 'baseline-better' | 'tie' | 'insufficient-signal' | 'no-gap';
+/** ② Statistical summary, populated by the reward aggregator (absent for single-sample). */
+export interface DiagnosisStats {
+    samples: number;
+    advantageMean: number | null;
+    advantageStdev: number | null;
+    /** Judge jitter measured by an A/A pair; `|advantageMean| < noiseFloor` ⇒ insufficient-signal. */
+    noiseFloor: number | null;
+    sequentialDecision: 'accept' | 'reject' | 'continue' | 'single';
+}
+/** ④ Integrity / anti-hacking signals. */
+export interface DiagnosisIntegrity {
+    /** A test-tamper signal was detected on the main arm (Batch 4 wires the detector). */
+    testTamperSuspected: boolean;
+    /**
+     * Signed disagreement between the judge's advantage and the loss-implied
+     * advantage (baselineLoss − mainLoss): the judge loved an arm the verifier
+     * dislikes. `null` when either loss anchor is missing.
+     */
+    judgeVerifierDivergence: number | null;
+    /** Human-readable integrity flags (empty when clean). */
+    flags: string[];
+}
+/**
+ * The `diagnosis.json` the 奖励智能体 REWARD AGENT writes. schemaVersion 2 adds
+ * the OPTIONAL `verdict` / `confidence` / `stats` / `integrity` fields (a
+ * schemaVersion-1 reader ignores them; a schemaVersion-2 reader tolerates their
+ * absence). advantage ＝ reward(主臂) − reward(基线臂); `null` when the baseline
+ * arm was skipped (no comparison possible) OR when the judge 弃权 abstained.
+ */
+export interface RewardDiagnosis {
+    schemaVersion: 1 | 2;
+    episodeId: string;
+    changeName: string;
+    targetId: string;
+    policyVersions: {
+        main: number | null;
+        baseline: number | null;
+    };
+    rewardMain: number;
+    /** null when the baseline arm was skipped. */
+    rewardBaseline: number | null;
+    /** reward(主臂) − reward(基线臂); null when baseline skipped or abstained. */
+    advantage: number | null;
+    anchors: DiagnosisAnchors;
+    errors: DiagnosisError[];
+    gaps: DiagnosisGap[];
+    /** 文本梯度 textual gradient; null only when abstained. */
+    textualGradient: string | null;
+    abstained: boolean;
+    abstainReason?: string;
+    /** ⑤ Overall verdict (optional; derived single-sample, set by the aggregator otherwise). */
+    verdict?: RewardVerdict;
+    /** ⑤/② Confidence in [0,1] (optional; set by the statistical layer). */
+    confidence?: number | null;
+    /** ② Statistical summary (optional; absent for single-sample). */
+    stats?: DiagnosisStats;
+    /** ④ Integrity / anti-hacking signals (optional). */
+    integrity?: DiagnosisIntegrity;
+}
+/** Assembled input for {@link assembleRewardAgentPrompt}. */
+export interface RewardAgentPromptInput {
+    changeName: string;
+    targetId: string;
+    policyVersions: {
+        main: number | null;
+        baseline: number | null;
+    };
+    /** 主智能体 MAIN AGENT (policy vN+1) capture. */
+    mainArm: {
+        skeleton: object | null;
+        /** Raw transcript text (jsonl) — bounded by {@link assembleRewardAgentPrompt}. */
+        transcript: string | null;
+        objective: RewardArmObjectiveInput;
+    };
+    /**
+     * CRITIC AGENT（基线智能体 baseline agent, policy vN）capture. `null` when the
+     * baseline arm was SKIPPED — the BASELINE ARM block is OMITTED entirely and
+     * the prompt states no comparison is possible (null rewardBaseline/advantage).
+     */
+    baselineArm: {
+        skeleton: object | null;
+        transcript: string | null;
+        objective: RewardArmObjectiveInput;
+    } | null;
+    /** Bounded excerpts of the 5 artifacts + test-report.md read from the change dir. */
+    artifacts: {
+        file: string;
+        content: string;
+    }[];
+    /** Pre-mapped anchors (loss/health/passRate) from both arms. */
+    anchors: DiagnosisAnchors;
+    /**
+     * ③ Order in which the two arms are presented to the judge. Swapping the order
+     * across samples cancels the LLM's position bias. Defaults to `main-first`
+     * (the historical order; single-sample callers keep byte-identical prompts).
+     */
+    armOrder?: 'main-first' | 'baseline-first';
+    /**
+     * ④ An optional tamper signal computed BEFORE scoring (Batch 4). When present
+     * and `suspected`, the judge is told not to reward passing tests that were
+     * weakened. Omitted ⇒ the prompt is unchanged.
+     */
+    integrityHint?: {
+        suspected: boolean;
+        flags: string[];
+    } | null;
+}
+/**
+ * Assemble the 奖励智能体 REWARD AGENT prompt. Pure (no I/O); exported for golden
+ * tests. Ordered blocks:
+ *   1. PRELUDE — the judge contract.
+ *   2. MAIN ARM (主智能体, policy vN+1) — skeleton + bounded transcript excerpt.
+ *   3. BASELINE ARM (CRITIC AGENT（基线智能体）, policy vN) — same; OMITTED
+ *      ENTIRELY when the baseline arm was skipped (a one-line note replaces it,
+ *      demanding null rewardBaseline/advantage).
+ *   4. ARTIFACTS — the 5 artifacts + test-report.md, bounded excerpts.
+ *   5. OBJECTIVE EVIDENCE — both arms' anchors verbatim (the on-disk loss/health
+ *      numbers that anchor the score).
+ */
+export declare function assembleRewardAgentPrompt(input: RewardAgentPromptInput): string;
+interface ParsedDiagnosis {
+    rewardMain: number;
+    rewardBaseline: number | null;
+    advantage: number | null;
+    errors: DiagnosisError[];
+    gaps: DiagnosisGap[];
+    textualGradient: string | null;
+    abstained: boolean;
+    abstainReason?: string;
+    /** ⑤ Optional judge-emitted verdict (validated against the enum when present). */
+    verdict?: RewardVerdict;
+    /** ⑤ Optional judge-emitted confidence in [0,1]. */
+    confidence?: number | null;
+}
+/**
+ * Parse the judge's `json:diagnosis` block with a strict one-block discipline:
+ * exactly one fenced block, well-formed JSON, then fail-closed shape + range
+ * validation.
+ *
+ * Throws {@link RewardAgentOutputInvalid} on any violation (the repair loop
+ * re-prompts with the concrete message appended).
+ */
+export declare function parseRewardAgentResponse(text: string): ParsedDiagnosis;
+/** Map an arm's objective to its (loss, passRate, healthPenalty, verbosity) anchors. */
+export declare function mapArmAnchors(objective: RewardArmObjectiveInput | null | undefined): {
+    loss: number | null;
+    passRate: number | null;
+    healthPenalty: number | null;
+    verbosity: number | null;
+};
+/** Build the {@link DiagnosisAnchors} block from both arms' objectives. */
+export declare function buildAnchors(mainObjective: RewardArmObjectiveInput, baselineObjective: RewardArmObjectiveInput | null): DiagnosisAnchors;
+export interface RunRewardAgentOptions {
+    repoRoot: string;
+    episodeId: string;
+    /** Injected for tests; defaults to node's spawn. */
+    spawn?: typeof nodeSpawn;
+    /** Override the agent binary; defaults to the host harness's default. */
+    binary?: string;
+    /** Bounded re-prompts on parse/validation failure (default 2 ⇒ at most 3 spawns). */
+    maxRepairAttempts?: number;
+}
+export interface RunRewardAgentResult {
+    diagnosis: RewardDiagnosis;
+    /** Absolute path of the written `diagnosis.json`. */
+    diagnosisPath: string;
+    /** The episode record after advancing to `scored`. */
+    episode: EpisodeRecord;
+}
+export interface ScoreOnceOptions {
+    /** The fully-assembled prompt input (carries arms, anchors, armOrder, integrityHint). */
+    promptInput: RewardAgentPromptInput;
+    /** Whether the baseline arm was skipped (drives the null-baseline contract). */
+    baselineSkipped: boolean;
+    /** cwd for the headless spawn (the repo/change root). */
+    repoRoot: string;
+    spawn?: typeof nodeSpawn;
+    binary?: string;
+    /** Bounded re-prompts on parse/validation failure (default 2 ⇒ at most 3 spawns). */
+    maxRepairAttempts?: number;
+}
+export interface ScoreOnceResult {
+    parsed: ParsedDiagnosis;
+    /** ④ Integrity signals computed from this duel's parse + anchors + tamper hint. */
+    integrity: DiagnosisIntegrity;
+}
+/**
+ * Score ONE judged duel: spawn the judge (fresh context), parse with the bounded
+ * repair loop (re-prompting with the concrete error appended), recompute and
+ * validate the advantage (incl. ① gate-not-blend), and compute ④ integrity
+ * signals. Does NOT write `diagnosis.json` — the caller (single-sample
+ * {@link runRewardAgent}, or the statistical aggregator) owns the write. This is
+ * the unit the aggregator calls k times for the A/A noise floor + SPRT.
+ */
+export declare function scoreOnce(opts: ScoreOnceOptions): Promise<ScoreOnceResult>;
+/** The read-side scoring context for one episode: arms, anchors, assembled prompt input. */
+export interface RewardScoringContext {
+    episode: EpisodeRecord;
+    baselineSkipped: boolean;
+    /** Prompt input with `armOrder` defaulting to `main-first` and no integrity hint set. */
+    promptInput: RewardAgentPromptInput;
+}
+/**
+ * Load everything the judge needs to score one episode WITHOUT spawning: read
+ * the episode + both arms (baseline omitted when skipped), map anchors, and read
+ * the change artifacts. Shared by {@link runRewardAgent} (single sample) and the
+ * statistical aggregator (which calls {@link scoreOnce} k times over the same
+ * context with swapped `armOrder`).
+ */
+export declare function loadRewardScoringContext(repoRoot: string, episodeId: string): Promise<RewardScoringContext>;
+/**
+ * Run the 奖励智能体 REWARD AGENT end-to-end for one episode (single sample):
+ *   1. read the episode + both arms (baseline omitted when skipped),
+ *   2. map anchors and assemble the prompt,
+ *   3. {@link scoreOnce} — spawn the judge, parse with the bounded repair loop,
+ *      recompute/validate the advantage (incl. ① gate-not-blend), compute ④
+ *      integrity,
+ *   4. derive the ⑤ single-sample verdict and write `diagnosis.json` (schema 2)
+ *      via {@link writeDiagnosis}, then advance the episode stage to `scored`.
+ *
+ * Behaviour is byte-compatible with the historical single-call path: no extra
+ * spawns, `armOrder` defaults to `main-first`, and the new schema-2 fields are
+ * OPTIONAL (a reader that ignores them sees the same diagnosis). The statistical
+ * layer (Batch 3) wraps {@link scoreOnce} instead of calling this directly.
+ *
+ * The ONLY write path is the episode dir. Invocation errors (agent crash) are
+ * NOT repaired — they propagate as {@link RewardAgentInvocationError}.
+ */
+export declare function runRewardAgent(opts: RunRewardAgentOptions): Promise<RunRewardAgentResult>;
+/**
+ * Derive the ⑤ single-sample verdict. A judge-emitted `verdict` wins; otherwise
+ * it is read off the advantage sign (no-gap when abstained; undefined when the
+ * baseline was skipped and only the main arm was scored). The statistical layer
+ * overrides this with `insufficient-signal` when the advantage is within the
+ * A/A noise floor.
+ */
+export declare function deriveSingleSampleVerdict(parsed: ParsedDiagnosis): RewardVerdict | undefined;
+/** ④ Compute integrity signals from a parsed duel, its anchors, and the tamper hint. */
+export declare function computeIntegrity(parsed: ParsedDiagnosis, anchors: DiagnosisAnchors, integrityHint: {
+    suspected: boolean;
+    flags: string[];
+} | null): DiagnosisIntegrity;
+export {};
+//# sourceMappingURL=reward-agent.d.ts.map