pi-continual-harness 0.9.0 → 0.11.0

package/README.md

@@ -53,8 +53,9 @@ Or drop `src/index.ts` into `~/.pi/agent/extensions/`.
 ```
 /refine              # review last 25 turns, propose deltas
 /refine 50           # review last 50 turns
-/refine 25 --commit  # also export durable state to ~/.pi/agent/harness-state.md
+/refine 25 --commit  # also export durable state (global + project layers)
 /refine --proposer dedupe  # run the rule-based dedupe proposer instead of steering
+/refine --proposer dedupe --threshold 0.75  # dedupe with a one-shot threshold
 ```
 
 Durable I/O (round-trip with pi-reflect):
@@ -62,6 +63,7 @@ Durable I/O (round-trip with pi-reflect):
 ```
 /harness status                  # counts + durable layer presence/mtime
 /harness export [path]           # layered export (or full snapshot to a path)
+/harness export-corpus [path]    # calibration corpus JSONL for pi-reflex (local-only; dedupe pairs + lifecycle)
 /harness import [--prune] [path] # layered import (or single file from a path)
 /harness prune [--decay <days>]  # drop items below the importance floor
 /harness keep <id>               # nudge importance up (+0.1)
@@ -106,6 +108,9 @@ The model-facing tools:
 - `harness_mutate { deltas: [...] }` — apply a batch of `create` / `update` /
   `delete` deltas. Every `create` requires `evidence`. New items are stamped
   automatically with the active model (see [Model binding](#model-binding-per-model-isolation)).
+  `create`/`update` deltas also accept `scope: "global" | "project"` (the
+  project slug is stamped server-side from the session cwd — used by
+  `/harness split`; normally managed via `/harness move`).
 
 Active items are injected into the system prompt each turn as a structured
 block, appended to (never replacing) the base prompt.
@@ -137,6 +142,7 @@ Optional config at `~/.pi/agent/harness.json` (missing or malformed → defaults
 {
   "autoImport": false,
   "proposer": "steering",
+  "dedupe": { "threshold": 0.6, "merge": true },
   "injection": { "enabled": true, "maxTokens": 1500, "maxPerKind": 10, "charsPerToken": 4 },
   "remindRefine": { "enabled": false, "everyTurns": 50 },
   "autoRefine": { "enabled": false, "everyTurns": 100, "commit": false },
@@ -160,7 +166,19 @@ Optional config at `~/.pi/agent/harness.json` (missing or malformed → defaults
   restore the legacy "inject all items, in store order" behaviour.
 - **`proposer`** — which delta proposer `/refine` and auto-refine use. Defaults
   to `steering` (the agent reasons via a steering message). `dedupe` applies a
-  rule-based dedupe directly. See [Proposers](#proposers).
+  rule-based merge directly. See [Proposers](#proposers).
+- **`dedupe`** — the rule-based dedupe proposer's policy. `threshold` (default
+  `0.6`, valid `(0,1]`) is the token-overlap level at which two items sharing
+  the key fields (kind, owner model, durable layer) count as duplicates;
+  `merge` (default `true`) **merges** each duplicate into its keeper — the
+  keeper keeps its content verbatim, its `evidence` becomes the line-wise union
+  of both, applied as one audited `update` + `delete` pair — while `false`
+  restores the pre-0.10 delete-only behavior. `/refine --proposer dedupe
+  --threshold 0.75` overrides the threshold for one run. A fourth key,
+  `similarity`, is API-only (functions cannot come from `harness.json`):
+  companion packages inject a richer comparator whose return may **abstain**
+  per pair (`{ score, abstain }` — see [`SimilarityResult`](#proposers)); an
+  abstaining pair is always kept, never merged or deleted.
 - **`remindRefine`** — opt-in `turn_end` nudge. `{ "enabled": true,
   "everyTurns": 50 }` notifies you to run `/refine` on a cadence. It is
   informational only — it never mutates state.
@@ -235,9 +253,9 @@ How the binding is set and respected:
   `model: provider/id`; `/harness import` restores it. An item whose tag
   pi-reflect stripped becomes an orphan and is adopted by the active model.
 
-Manual commands (`export`, `import`, `keep`, `drop`, `prune`, `push-mem`,
-`status`) operate on the **whole store** by design — they are explicit human
-actions with full control. Isolation is enforced only where pollution would
+Manual commands (`export`, `import`, `move`, `split`, `keep`, `drop`, `prune`,
+`push-mem`, `status`) operate on the **whole store** by design — they are
+explicit human actions with full control. Isolation is enforced only where pollution would
 leak automatically: injection, listing, create-stamping, and the outcome loop.
 In particular, `/harness push-mem` pushes *every* model's active items into
 pi-mem by default (which can yield near-duplicate memories across models); pass
@@ -294,7 +312,16 @@ deltas directly). The propose stage is pluggable via a registry
 | Name | What it does |
 |---|---|
 | `steering` (default) | Delegates reasoning to the agent via a steering message — reuses the agent loop, model-agnostic, fully visible. |
-| `dedupe` | Rule-based: drops near-duplicate active items (token-overlap ≥ 0.6), keeping the higher-importance one. No model call. |
+| `dedupe` | Rule-based: **merges** near-duplicate active items (token-overlap ≥ the configured threshold, same kind / owner model / durable layer) into the higher-importance keeper — one evidence-union update per keeper, then deletes the duplicates. `"dedupe": { "merge": false }` restores delete-only. No model call. |
+
+The dedupe planner's `similarity` seam (`DedupeOptions.similarity`, re-exported
+from the package entry) is how a companion package upgrades the comparison —
+e.g. embedding cosine similarity, or a local decision engine such as pi-reflex
+(a local-only project for now). A comparator may return
+`{ score, abstain }` instead of a plain number: **an abstaining pair is
+treated as not duplicates** — both items are kept — so an engine with
+conformal uncertainty guarantees can safely decline to merge. Plain numeric
+returns keep working (the default remains token Jaccard).
 
 Select a proposer per run with `/refine --proposer <name>`, or set the default
 for auto-refine via `proposer` in the [config](#configuration). Both paths are

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-continual-harness",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "Online self-improvement layer for pi. A unified, in-trajectory harness-state store (prompt notes, memory, skill descriptions, sub-agent specs) with a manual /refine that proposes evidence-backed structured CRUD deltas. Composes with pi-reflect (offline) and pi-mem (storage); owns only the online optimizer layer. Inspired by Continual Harness (arXiv 2605.09998) and ACE (arXiv 2510.04618).",
   "keywords": [
     "pi-package",

package/src/config.ts

@@ -20,12 +20,22 @@ import { homedir } from "node:os";
 import { basename, dirname, join } from "node:path";
 import { DEFAULT_DURABLE_PATH, PROJECT_DURABLE_DIR, type LayerFile, type ScopeInfo } from "./store.js";
 import { DEFAULT_INJECTION, normalizeInjection, type NormalizedInjection } from "./select.js";
+import { DEDUPE_THRESHOLD } from "./proposer.js";
 
 export const DEFAULT_EVERY_TURNS = 50;
 export const DEFAULT_AUTO_EVERY_TURNS = 100;
 /** Default per-reference importance bump for the opt-in outcome loop. */
 export const DEFAULT_REF_BUMP = 0.03;
 
+/** Resolved dedupe policy (merge-capable, 0.10.0). Populated by loadConfig. */
+export interface NormalizedDedupe {
+  /** Token-overlap threshold in (0,1]; pairs at/above it merge. Default 0.6. */
+  threshold: number;
+  /** Merge duplicates into the keeper (evidence union) instead of delete-only.
+   *  Default true; `false` restores the pre-0.10 delete-only behavior. */
+  merge: boolean;
+}
+
 export interface HarnessConfig {
   /** DEPRECATED (0.9.0): items carry their own scope now (see /harness move);
    *  durable I/O is always layered. Parsed for compatibility, ignored. */
@@ -53,6 +63,9 @@ export interface HarnessConfig {
   /** Injection selection policy (on by default). Resolved by loadConfig, so the
    *  value here is always fully-populated. See src/select.ts. */
   injection?: NormalizedInjection;
+  /** Dedupe proposer policy (merge-capable). Resolved by loadConfig, so the
+   *  value here is always fully-populated. See src/proposer.ts. */
+  dedupe?: NormalizedDedupe;
 }
 
 export const DEFAULT_CONFIG: HarnessConfig = {
@@ -62,6 +75,10 @@ export const DEFAULT_CONFIG: HarnessConfig = {
   autoRefine: { enabled: false, everyTurns: DEFAULT_AUTO_EVERY_TURNS, commit: false },
   proposer: "steering",
   outcomeImportance: { enabled: false, bump: DEFAULT_REF_BUMP },
+  // Dedupe (merge-capable): merge ON at the historical 0.6 threshold — a
+  // near-duplicate merges into its keeper (evidence unioned) instead of being
+  // deleted cold. `merge: false` restores delete-only.
+  dedupe: { threshold: DEDUPE_THRESHOLD, merge: true },
   // ON by default: importance-ordered, maxPerKind 10, maxTokens 1500. A no-op
   // for small stores; protective as the harness accumulates. Opt out with
   // `injection.enabled: false`. See src/select.ts.
@@ -95,6 +112,10 @@ function mergeConfig(over: Partial<HarnessConfig>): HarnessConfig {
     // malformed `injection` object degrades to the shipped defaults rather than
     // corrupting the block sizing arithmetic.
     injection: normalizeInjection(over.injection as Partial<{ enabled: boolean; maxTokens: number; maxPerKind: number; charsPerToken: number }> | undefined),
+    dedupe: {
+      threshold: coerceThreshold(over.dedupe?.threshold),
+      merge: over.dedupe?.merge !== false,
+    },
   };
 }
 
@@ -106,6 +127,14 @@ function coerceBump(raw: unknown): number {
   return typeof raw === "number" && Number.isFinite(raw) ? raw : DEFAULT_REF_BUMP;
 }
 
+/** Coerce a user-provided dedupe threshold to a finite number in (0,1], else
+ *  the default. The threshold is a COMPARISON operand (similarity >= threshold);
+ *  out-of-range or non-numeric values must degrade to the default rather than
+ *  silently matching everything (0/-1) or nothing (>1). */
+function coerceThreshold(raw: unknown): number {
+  return typeof raw === "number" && Number.isFinite(raw) && raw > 0 && raw <= 1 ? raw : DEDUPE_THRESHOLD;
+}
+
 /** Load the config, merged over defaults. Tolerant: missing/malformed → defaults.
  *  Cached for the process lifetime (config is read once per session). */
 export async function loadConfig(path: string = CONFIG_PATH): Promise<HarnessConfig> {

package/src/corpus.ts

@@ -0,0 +1,310 @@
+// Calibration corpus exporter — the S1 half of the pi-reflex consumer contract
+// (formerly pi-jev; CONTRACT-harness.md §4 in that project; issue #12 here).
+//
+// Turns the session's OWN audit trail — `harness-state` snapshots and
+// `harness-refinement` entries — into two JSONL corpora for calibrating and
+// evaluating a semantic decision engine (pi-reflex) against real harness
+// decisions:
+//
+//   dedupe-pairs.jsonl  one record per compared item pair
+//     - label "dup"     parsed from dedupe delete reasons (merged / near-
+//                       duplicate); the run's ground truth
+//     - label "not_dup" same-key-field pairs with token overlap in
+//                       [NOT_DUP_FLOOR, threshold) that the planner did NOT
+//                       merge; heuristic negatives, needs_review until a human
+//                       confirms (the planner's threshold, not a label, decided)
+//   lifecycle.jsonl     one record per item lifecycle event, classified from
+//                       consecutive snapshot diffs:
+//     created (id appears) · kept (+0.1) · dropped (−0.1) · cited (small
+//     positive bump, the outcome loop's default +0.03) · pruned (removed with
+//     last importance < IMPORTANCE_FLOOR) · deleted (removed otherwise)
+//
+// Pure: takes an entry iterable (e.g. ctx.sessionManager.getBranch()), returns
+// records; no I/O here. The `/harness export-corpus` subcommand is thin glue.
+//
+// CONTRACT INVARIANT (local-only): records describe item CONTENTS. Export
+// writes local files and nothing else — no network, no telemetry. The corpus
+// leaves the machine only if the user pushes it.
+//
+// Determinism: same entries in, same records out (stable order, hashes of
+// content only). Overlaps are rounded to 4 decimals for stable JSON.
+
+import { createHash } from "node:crypto";
+import { IMPORTANCE_FLOOR, REFINE_ENTRY, STATE_ENTRY } from "./store.js";
+import { DEDUPE_THRESHOLD, tokenOverlap } from "./proposer.js";
+import { DEFAULT_REF_BUMP } from "./config.js";
+import type { ComponentKind, HarnessItem, HarnessState } from "./types.js";
+
+// ---- record types (contract §4; `v` follows the contract's major.minor) -----
+
+export type LifecycleEvent = "created" | "cited" | "kept" | "dropped" | "pruned" | "deleted";
+
+export interface LifecycleRecord {
+  v: 1;
+  kind: "lifecycle";
+  item_kind: ComponentKind;
+  /** sha256 of the item's content, prefixed — contents are not carried here. */
+  content_hash: string;
+  event: LifecycleEvent;
+  /** Post-event importance. */
+  importance: number;
+}
+
+export interface DedupePairRecord {
+  v: 1;
+  kind: "dedupe_pair";
+  /** Keeper (dup label) / first-by-store-order (not_dup) content. */
+  a: string;
+  /** Duplicate (dup label) / second-by-store-order (not_dup) content. */
+  b: string;
+  label: "dup" | "not_dup";
+  similarity: number;
+  source: "tokenOverlap";
+  /** Present on "not_dup" records: the label is inferred from non-merge, not
+   *  human-confirmed. */
+  needs_review?: true;
+  meta: { kinds: ComponentKind[]; owners: string[] };
+}
+
+export interface CorpusResult {
+  pairs: DedupePairRecord[];
+  lifecycle: LifecycleRecord[];
+}
+
+/** Options for buildCorpus. */
+export interface CorpusOptions {
+  /** The CONFIGURED outcome-loop citation bump (harness.json
+   *  `outcomeImportance.bump`; default 0.03 = DEFAULT_REF_BUMP). Classification
+   *  is config-aware: keep/drop (±0.1) are checked FIRST, so a configured
+   *  bump of exactly 0.1 collides and keep/drop win — documented behavior. */
+  citeBump?: number | undefined;
+}
+
+// ---- tuning knobs -----------------------------------------------------------
+
+/** Overlap floor for "not_dup" candidates: below it a pair is uninformative. */
+export const NOT_DUP_FLOOR = 0.3;
+/** keep/drop bump magnitude (harness.ts handleBump). */
+const KEEP_DROP_BUMP = 0.1;
+/** Classification tolerance around each bump (float safety, clamped bumps). */
+const BUMP_TOL = 0.02;
+/** Smallest importance delta worth classifying at all. */
+const EPSILON = 0.005;
+
+// ---- helpers ----------------------------------------------------------------
+
+/** Loose entry shape, kept decoupled from pi's internal session types.
+ *  `applied` is a legacy COUNT; the full delta list lives in `appliedDeltas`
+ *  (0.11.0+ audit entries — see refine.ts). */
+type AnyEntry = {
+  type?: string;
+  customType?: string;
+  data?: {
+    state?: HarnessState;
+    proposer?: string;
+    applied?: unknown;
+    appliedDeltas?: unknown[];
+  };
+};
+
+/** Parse a dedupe delete reason ("merged into h_x (overlap 0.71)" or the
+ *  delete-only "near-duplicate of h_x (overlap 0.71)"). */
+export function parseDedupeDelete(
+  reason: string,
+): { keeperId: string; overlap: number } | undefined {
+  const m = /^(?:merged into|near-duplicate of) (h_[A-Za-z0-9_]+) \(overlap (0?\.\d+|1(?:\.0+)?)\)$/.exec(
+    reason,
+  );
+  if (!m) return undefined;
+  return { keeperId: m[1]!, overlap: Number(m[2]) };
+}
+
+function round4(x: number): number {
+  return Math.round(x * 1e4) / 1e4;
+}
+
+function hash(content: string): string {
+  return `sha256:${createHash("sha256").update(content).digest("hex")}`;
+}
+
+/** Durable placement, mirroring planDedupe's private key: only items in the
+ *  same layer were ever compared by the planner. */
+function durableLayer(i: HarnessItem): string {
+  return i.scope === "project" ? `project:${i.project ?? ""}` : "global";
+}
+
+// ---- lifecycle ---------------------------------------------------------------
+
+/** Classify one snapshot → next-snapshot transition into lifecycle records.
+ *  `citeBump` is the CONFIGURED outcome-loop bump (see CorpusOptions). */
+function diffSnapshots(
+  prev: HarnessState,
+  next: HarnessState,
+  citeBump: number,
+  out: LifecycleRecord[],
+): void {
+  const prevById = new Map(prev.items.map((i) => [i.id, i]));
+  const nextById = new Map(next.items.map((i) => [i.id, i]));
+
+  for (const item of next.items) {
+    const before = prevById.get(item.id);
+    if (!before) {
+      out.push(lifecycle(item, "created", item.importance));
+      continue;
+    }
+    const d = item.importance - before.importance;
+    if (Math.abs(d) < EPSILON) continue;
+    // keep/drop FIRST: a configured citation bump of exactly ±0.1 collides and
+    // keep/drop win (documented in CorpusOptions).
+    if (Math.abs(d - KEEP_DROP_BUMP) <= BUMP_TOL) out.push(lifecycle(item, "kept", item.importance));
+    else if (Math.abs(d + KEEP_DROP_BUMP) <= BUMP_TOL)
+      out.push(lifecycle(item, "dropped", item.importance));
+    else if (citeBump > 0 && Math.abs(d - citeBump) <= BUMP_TOL)
+      out.push(lifecycle(item, "cited", item.importance));
+    // else: explicit harness_mutate importance set — not a lifecycle signal.
+  }
+  for (const item of prev.items) {
+    if (nextById.has(item.id)) continue;
+    const event = item.importance < IMPORTANCE_FLOOR ? "pruned" : "deleted";
+    out.push(lifecycle(item, event, item.importance));
+  }
+}
+
+function lifecycle(item: HarnessItem, event: LifecycleEvent, importance: number): LifecycleRecord {
+  return { v: 1, kind: "lifecycle", item_kind: item.kind, content_hash: hash(item.content), event, importance };
+}
+
+// ---- dedupe pairs --------------------------------------------------------------
+
+/** Emit pair records for one dedupe refinement run.
+ *
+ *  `current` is the store state AT AUDIT TIME — a run that applied nothing
+ *  writes NO harness-state snapshot, so "last snapshot" is correct whether or
+ *  not this run wrote one (no-merge runs: current IS the pre-run state).
+ *  `pre` (previous snapshot, only meaningful when the run applied deletes)
+ *  supplies DELETED items' contents; keepers are read from `current`.
+ *
+ *  `deltas` comes from the audit entry's `appliedDeltas` (0.11.0+). Legacy
+ *  entries carry only a numeric count — pass [] there: dup reconstruction is
+ *  impossible without delete reasons, but not_dup candidates still emit. */
+function collectDedupePairs(
+  current: HarnessState,
+  pre: HarnessState,
+  deltas: unknown[],
+  seen: Set<string>,
+  out: DedupePairRecord[],
+): void {
+  const currentById = new Map(current.items.map((i) => [i.id, i]));
+  const preById = new Map(pre.items.map((i) => [i.id, i]));
+
+  const deletes = (deltas as Array<{ op?: string; id?: string; reason?: string }>).filter(
+    (d) => d && d.op === "delete" && d.id && typeof d.reason === "string",
+  );
+  const deletedIds = new Set(deletes.map((d) => d.id!));
+
+  for (const d of deletes) {
+    const parsed = parseDedupeDelete(d.reason!);
+    if (!parsed) continue;
+    const keeper = currentById.get(parsed.keeperId) ?? preById.get(parsed.keeperId);
+    const dup = preById.get(d.id!);
+    if (!keeper || !dup) continue;
+    pushPair(
+      out,
+      seen,
+      keeper.content,
+      dup.content,
+      "dup",
+      round4(parsed.overlap),
+      { kinds: [keeper.kind], owners: [keeper.ownerModel, dup.ownerModel] },
+    );
+  }
+
+  // not_dup candidates: everything the planner COULD have compared (same key
+  // fields, active, not deleted this run) but did not merge, with an
+  // informative overlap. Always relative to the CURRENT state.
+  const candidates = current.items.filter((i) => i.active && !deletedIds.has(i.id));
+  for (let i = 0; i < candidates.length; i++) {
+    for (let j = i + 1; j < candidates.length; j++) {
+      const a = candidates[i]!;
+      const b = candidates[j]!;
+      if (a.kind !== b.kind) continue;
+      if (a.ownerModel !== b.ownerModel) continue;
+      if (durableLayer(a) !== durableLayer(b)) continue;
+      const sim = tokenOverlap(a.content, b.content);
+      if (sim < NOT_DUP_FLOOR || sim >= DEDUPE_THRESHOLD) continue;
+      pushPair(
+        out,
+        seen,
+        a.content,
+        b.content,
+        "not_dup",
+        round4(sim),
+        { kinds: [a.kind], owners: [a.ownerModel] },
+        true,
+      );
+    }
+  }
+}
+
+function pushPair(
+  out: DedupePairRecord[],
+  seen: Set<string>,
+  a: string,
+  b: string,
+  label: DedupePairRecord["label"],
+  similarity: number,
+  meta: DedupePairRecord["meta"],
+  needsReview = false,
+): void {
+  const key = `${label}\x00${a}\x00${b}`;
+  if (seen.has(key)) return; // one record per unique pair across runs
+  seen.add(key);
+  const rec: DedupePairRecord = { v: 1, kind: "dedupe_pair", a, b, label, similarity, source: "tokenOverlap", meta };
+  if (needsReview) rec.needs_review = true;
+  out.push(rec);
+}
+
+// ---- entry-point ---------------------------------------------------------------
+
+/**
+ * Build the calibration corpora from an ordered session-branch entry iterable
+ * (e.g. `ctx.sessionManager.getBranch()`). Pure and deterministic.
+ *
+ * Walk notes:
+ *  - A dedupe run that APPLIED deltas appends its post-run `harness-state`
+ *    snapshot immediately before its `harness-refinement` audit entry, so at
+ *    an audit entry the last snapshot is the CURRENT state either way.
+ *  - A no-merge run writes NO snapshot: the last snapshot is an older state
+ *    that is still the current store (candidates emit from it).
+ *  - Legacy audit entries (pre-0.11.0) carry `applied` as a COUNT with no
+ *    delta details: dup reconstruction is skipped, candidates still emit.
+ */
+export function buildCorpus(entries: Iterable<unknown>, opts: CorpusOptions = {}): CorpusResult {
+  const citeBump = opts.citeBump ?? DEFAULT_REF_BUMP;
+  const pairs: DedupePairRecord[] = [];
+  const lifecycle: LifecycleRecord[] = [];
+  const seen = new Set<string>();
+  const snapshots: HarnessState[] = [];
+  for (const raw of entries) {
+    const entry = raw as AnyEntry;
+    if (entry.type === "custom" && entry.customType === STATE_ENTRY && entry.data?.state) {
+      const next = entry.data.state;
+      const prev = snapshots.at(-1);
+      if (prev) diffSnapshots(prev, next, citeBump, lifecycle);
+      snapshots.push(next);
+      continue;
+    }
+    if (
+      entry.type === "custom" &&
+      entry.customType === REFINE_ENTRY &&
+      entry.data?.proposer === "dedupe"
+    ) {
+      const current = snapshots.at(-1);
+      if (!current) continue; // no state seen yet — nothing to compare
+      const deltas = Array.isArray(entry.data.appliedDeltas) ? entry.data.appliedDeltas : [];
+      const pre = snapshots.at(-2) ?? { items: [] };
+      collectDedupePairs(current, pre, deltas, seen, pairs);
+    }
+  }
+  return { pairs, lifecycle };
+}

package/src/harness.ts

@@ -25,6 +25,8 @@
 
 import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
 import { stat } from "node:fs/promises";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
 import {
   applyDeltas,
   bumpImportance,
@@ -41,11 +43,13 @@ import {
 import {
   defaultScopeForPath,
   layerFilesFor,
+  loadConfig,
   projectDurablePath,
   projectSlug,
 } from "./config.js";
 import { KIND_LABEL } from "./types.js";
 import type { ComponentKind, Delta, HarnessItem } from "./types.js";
+import { buildCorpus } from "./corpus.js";
 
 /** One row of the /harness subcommand menu (label = name, value = name). */
 interface CompletionEntry {
@@ -56,6 +60,7 @@ interface CompletionEntry {
 const SUBCOMMANDS: CompletionEntry[] = [
   { name: "import", description: "Import durable state, layered (--prune to prune stale items)" },
   { name: "export", description: "Export active items to durable layers (or one file with a path)" },
+  { name: "export-corpus", description: "Export calibration corpus JSONL for pi-reflex (dedupe pairs + lifecycle)" },
   { name: "status", description: "Show harness status (active/total, per-kind counts, durable layers)" },
   { name: "prune", description: "Decay & prune inactive items (--decay <days>)" },
   { name: "keep", description: "Bump item importance (+0.1)" },
@@ -218,6 +223,9 @@ export function registerHarness(pi: ExtensionAPI): void {
         case "export":
           await handleExport(ctx, rest);
           return;
+        case "export-corpus":
+          await handleExportCorpus(ctx, rest);
+          return;
         case "prune":
           await handlePrune(pi, ctx, rest);
           return;
@@ -320,6 +328,38 @@ async function handleExport(ctx: ExtensionCommandContext, rest: string[]): Promi
   ctx.ui.setStatus("harness", undefined);
 }
 
+function toJsonl(rows: unknown[]): string {
+  return rows.length > 0 ? `${rows.map((r) => JSON.stringify(r)).join("\n")}\n` : "";
+}
+
+/** `/harness export-corpus [path]` — write the calibration corpora (contract
+ *  §4) from THIS session branch's audit trail. Default dir:
+ *  ./harness-corpus/<yyyy-mm-dd>/. Local file output only — contract invariant:
+ *  nothing leaves the machine. */
+async function handleExportCorpus(ctx: ExtensionCommandContext, rest: string[]): Promise<void> {
+  const explicit = explicitPath(rest);
+  const dir = explicit ?? join("harness-corpus", new Date().toISOString().slice(0, 10));
+  ctx.ui.setStatus("harness", "Exporting calibration corpus…");
+  try {
+    // Config-aware citation classification: the outcome loop's bump is
+    // configurable (default 0.03), so the classifier must not hardcode it.
+    const { outcomeImportance } = await loadConfig();
+    const corpus = buildCorpus(ctx.sessionManager.getBranch() as Iterable<unknown>, {
+      citeBump: outcomeImportance?.bump,
+    });
+    await mkdir(dir, { recursive: true });
+    await writeFile(join(dir, "dedupe-pairs.jsonl"), toJsonl(corpus.pairs));
+    await writeFile(join(dir, "lifecycle.jsonl"), toJsonl(corpus.lifecycle));
+    ctx.ui.notify(
+      `Calibration corpus → ${dir}: ${corpus.pairs.length} dedupe pair(s), ${corpus.lifecycle.length} lifecycle event(s). Local files only — nothing leaves the machine.`,
+      "info",
+    );
+  } catch (err) {
+    ctx.ui.notify(`/harness export-corpus failed: ${(err as Error).message}`, "error");
+  }
+  ctx.ui.setStatus("harness", undefined);
+}
+
 async function handlePrune(
   pi: ExtensionAPI,
   ctx: ExtensionCommandContext,

package/src/index.ts

@@ -74,17 +74,37 @@ export {
 // (see src/proposer.ts) and it becomes selectable via /refine --proposer <name>
 // or the `proposer` config key.
 export {
+  DEFAULT_DEDUPE,
+  DEDUPE_THRESHOLD,
+  EVIDENCE_MERGE_CAP,
   listProposers,
+  planDedupe,
   registerProposer,
+  unionEvidence,
   type CompleteOptions,
   type CompleteResult,
+  type DedupeOptions,
   type DeltaProposer,
   type ModelCallTelemetry,
   type ProposeInput,
   type ProposedDelta,
   type ProposeResult,
+  type SimilarityResult,
 } from "./proposer.js";
 
+// Calibration corpus exporter (pi-reflex contract §4, formerly pi-jev). Re-exported so companion
+// packages can build corpora from exported session branches without reaching
+// into internal paths.
+export {
+  buildCorpus,
+  NOT_DUP_FLOOR,
+  parseDedupeDelete,
+  type CorpusResult,
+  type DedupePairRecord,
+  type LifecycleEvent,
+  type LifecycleRecord,
+} from "./corpus.js";
+
 // Injection selection policy (on by default). Re-exported so companion packages
 // and tests can reuse the pure selection/normalization without reaching into
 // internal paths.

package/src/proposer.ts

@@ -10,9 +10,10 @@
 // original behavior exactly (delegates to the agent via a steering message,
 // reusing the agent loop — model-agnostic, fully visible). Alternate proposers
 // produce deltas directly:
-//   - `dedupeProposer` (rule-based): drops near-duplicate active items by token
-//     overlap, keeping the higher-importance one. Pure function of state —
-//     deterministic, no model call, fully testable.
+//   - `dedupeProposer` (rule-based): merges near-duplicate active items by
+//     token overlap (threshold configurable), unioning their evidence into the
+//     higher-importance keeper; `merge: false` restores delete-only. Pure
+//     function of state — deterministic, no model call, fully testable.
 //
 // The interface supports a dedicated-model proposer — one that makes its own
 // hidden LLM call to produce deltas directly, instead of delegating to the
@@ -27,6 +28,7 @@
 // named proposer, then select it via /refine --proposer <name> or config.
 
 import type { Delta, HarnessItem, HarnessState } from "./types.js";
+import type { HarnessConfig } from "./config.js";
 
 /** Options for the one-shot model completion injected into ProposeInput. */
 export interface CompleteOptions {
@@ -69,6 +71,11 @@ export interface ProposeInput {
   state: HarnessState;
   /** Lookback window in turns. */
   lookback: number;
+  /** Loaded harness config (harness.json), passed by runRefine so proposers
+   *  can read tuned knobs (e.g. the dedupe threshold / merge policy) without
+   *  file I/O. Optional — direct/legacy callers omit it and proposers fall
+   *  back to their built-in defaults. */
+  config?: HarnessConfig;
   /** One-shot model completion, injected by runRefine when a model is available.
    *  Dedicated-model proposers call this to make a hidden completion; rule-based
    *  and steering proposers ignore it. Undefined when no model is resolvable, so
@@ -131,10 +138,44 @@ export const steeringProposer: DeltaProposer = {
   },
 };
 
-// ---- rule-based alternate: dedupe -----------------------------------------
+// ---- rule-based alternate: dedupe (merge-capable) -------------------------
 
 export const DEDUPE_THRESHOLD = 0.6;
 
+/** Enriched similarity return: a score plus an explicit per-pair abstain.
+ *  The shape pinned by the pi-jev consumer contract (a conformal decision
+ *  engine abstains when its prediction set is not a singleton). */
+export interface SimilarityResult {
+  /** Similarity in [0,1], compared against DedupeOptions.threshold. */
+  score: number;
+  /** True = the engine abstains on this pair (uncertain). An abstaining
+   *  pair is conservatively treated as NOT duplicates: both items are kept. */
+  abstain?: boolean;
+}
+
+/** Options for the dedupe planner. */
+export interface DedupeOptions {
+  /** Token-overlap threshold in (0,1]: pairs at/above it are duplicates.
+   *  1 means "identical token sets only". */
+  threshold: number;
+  /** Merge duplicates into their keeper (one evidence-union update + deletes)
+   *  instead of delete-only. */
+  merge: boolean;
+  /** Similarity in [0,1]; defaults to tokenOverlap. The seam for the semantic
+   *  upgrade path (see docs/PLAN-dedupe-merge.md, Research grounding): a
+   *  companion package injects cosine similarity over embeddings without
+   *  forking planDedupe. May return a SimilarityResult to ABSTAIN per pair
+   *  (e.g. a conformal engine below confidence): an abstaining pair is never
+   *  treated as duplicates, whatever the score. */
+  similarity?: (a: string, b: string) => number | SimilarityResult;
+}
+
+/** Shipped defaults: merge ON at the historical 0.6 threshold. */
+export const DEFAULT_DEDUPE: DedupeOptions = { threshold: DEDUPE_THRESHOLD, merge: true };
+
+/** Cap (chars) on a merged item's evidence so durable layers stay lean. */
+export const EVIDENCE_MERGE_CAP = 2000;
+
 /** Tokenize for overlap comparison: lowercase alnum tokens. */
 export function tokenize(text: string): string[] {
   return text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
@@ -154,50 +195,123 @@ function truncate(s: string, n = 48): string {
   return s.length > n ? `${s.slice(0, n - 1)}…` : s;
 }
 
+/** Durable placement of an item: "global" or "project:<slug>". Only items in
+ *  the SAME layer merge — a merge must never silently move an item between
+ *  durable files or projects. */
+function durableLayer(i: HarnessItem): string {
+  return i.scope === "project" ? `project:${i.project ?? ""}` : "global";
+}
+
+/** Normalize a similarity return: plain number or { score, abstain }. */
+function asSimilarity(r: number | SimilarityResult): SimilarityResult {
+  return typeof r === "number" ? { score: r } : r;
+}
+
+/** Union evidence strings line-wise: trim, drop empty and exact-duplicate
+ *  lines (first occurrence wins, keeper's lines first), cap the total. */
+export function unionEvidence(sources: string[]): string {
+  const seen = new Set<string>();
+  const lines: string[] = [];
+  for (const src of sources) {
+    for (const raw of src.split("\n")) {
+      const line = raw.trim();
+      if (!line || seen.has(line)) continue;
+      seen.add(line);
+      lines.push(line);
+    }
+  }
+  const joined = lines.join("\n");
+  return joined.length > EVIDENCE_MERGE_CAP
+    ? `${joined.slice(0, EVIDENCE_MERGE_CAP)}\n[…merged evidence truncated…]`
+    : joined;
+}
+
 /**
- * Rule-based proposer: drops near-duplicate ACTIVE items (same kind, Jaccard
- * token overlap >= DEDUPE_THRESHOLD), keeping the higher-importance one.
+ * Pure dedupe planner: merges (or, with merge:false, drops) near-duplicate
+ * ACTIVE items. Two items are duplicates iff they share the key fields —
+ * kind, ownerModel, durable layer (scope+project) — and their content
+ * similarity is >= opts.threshold. A similarity engine MAY abstain per pair
+ * (see SimilarityResult): an abstaining pair is treated as NOT duplicates.
+ *
+ * Merge semantics (ACE-style deterministic merge, never a prose rewrite):
+ * the higher-importance item is the KEEPER and keeps its content verbatim;
+ * each keeper that absorbed duplicates gets ONE update whose evidence is the
+ * line-wise union of its own and every absorbed duplicate's evidence (one
+ * final union — applyOne replaces evidence wholesale, so per-duplicate
+ * updates would clobber each other); each absorbed duplicate is deleted with
+ * a reason naming its keeper. Updates are emitted before deletes.
  *
  * Greedy and contradiction-free: items are considered in importance-descending
- * order; the first item is always a keeper, and each later item is compared
- * only against keepers, so a keeper is never subsequently dropped.
+ * order (ties keep store order); the first item is always a keeper, and each
+ * later item is compared only against keepers, so a keeper is never
+ * subsequently dropped. A duplicate matching several keepers joins the
+ * best-overlap one.
  */
-export const dedupeProposer: DeltaProposer = {
-  name: "dedupe",
-  async propose({ state }): Promise<ProposeResult> {
-    const ordered = state.items
-      .filter((i) => i.active)
-      .sort((a, b) => b.importance - a.importance);
-    const keepers: HarnessItem[] = [];
-    const proposals: ProposedDelta[] = [];
-    for (const cand of ordered) {
-      let dup: HarnessItem | undefined;
-      let best = 0;
-      for (const k of keepers) {
-        if (k.kind !== cand.kind) continue;
-        // Per-model isolation: each model keeps its own copy, so two near-
-        // identical items bound to different models are NOT duplicates.
-        if (k.ownerModel !== cand.ownerModel) continue;
-        const sim = tokenOverlap(k.content, cand.content);
-        if (sim >= DEDUPE_THRESHOLD && sim > best) {
-          best = sim;
-          dup = k;
-        }
+export function planDedupe(state: HarnessState, opts: DedupeOptions = DEFAULT_DEDUPE): ProposedDelta[] {
+  const similarity = opts.similarity ?? tokenOverlap;
+  const ordered = state.items
+    .filter((i) => i.active)
+    .sort((a, b) => b.importance - a.importance);
+  const keepers: Array<{ item: HarnessItem; absorbed: Array<{ dup: HarnessItem; overlap: number }> }> = [];
+  for (const cand of ordered) {
+    let target: (typeof keepers)[number] | undefined;
+    let best = 0;
+    for (const k of keepers) {
+      if (k.item.kind !== cand.kind) continue;
+      // Per-model isolation: each model keeps its own copy, so two near-
+      // identical items bound to different models are NOT duplicates.
+      if (k.item.ownerModel !== cand.ownerModel) continue;
+      // Same durable layer only (see durableLayer).
+      if (durableLayer(k.item) !== durableLayer(cand)) continue;
+      const { score: sim, abstain } = asSimilarity(similarity(k.item.content, cand.content));
+      // Abstain = keep both: an uncertain engine must never cause a merge.
+      if (!abstain && sim >= opts.threshold && sim > best) {
+        best = sim;
+        target = k;
       }
-      if (dup) {
-        proposals.push({
-          delta: {
-            op: "delete",
-            id: cand.id,
-            reason: `near-duplicate of ${dup.id} (overlap ${best.toFixed(2)})`,
-          },
-          rationale: `dedupe: "${truncate(cand.content)}" ≈ keeper "${truncate(dup.content)}" (Jaccard ${best.toFixed(2)}); kept higher-importance ${dup.id}.`,
+    }
+    if (target) target.absorbed.push({ dup: cand, overlap: best });
+    else keepers.push({ item: cand, absorbed: [] });
+  }
+
+  const updates: ProposedDelta[] = [];
+  const deletes: ProposedDelta[] = [];
+  for (const k of keepers) {
+    if (k.absorbed.length === 0) continue;
+    if (opts.merge) {
+      const merged = unionEvidence([k.item.evidence, ...k.absorbed.map((a) => a.dup.evidence)]);
+      // Skip the no-op: identical evidence → the merge degenerates to a plain
+      // delete for this pair (no update delta).
+      if (merged !== k.item.evidence) {
+        updates.push({
+          delta: { op: "update", id: k.item.id, evidence: merged },
+          rationale: `dedupe: merged ${k.absorbed.length} duplicate(s) (${k.absorbed.map((a) => a.dup.id).join(", ")}) into ${k.item.id}; evidence unioned (${k.absorbed.length + 1} sources).`,
         });
-      } else {
-        keepers.push(cand);
       }
     }
-    return proposals.length ? { deltas: proposals } : {};
+    for (const { dup, overlap } of k.absorbed) {
+      deletes.push({
+        delta: {
+          op: "delete",
+          id: dup.id,
+          reason: opts.merge
+            ? `merged into ${k.item.id} (overlap ${overlap.toFixed(2)})`
+            : `near-duplicate of ${k.item.id} (overlap ${overlap.toFixed(2)})`,
+        },
+        rationale: `dedupe: "${truncate(dup.content)}" ≈ keeper "${truncate(k.item.content)}" (Jaccard ${overlap.toFixed(2)}); ${opts.merge ? `merged into ${k.item.id}.` : `kept higher-importance ${k.item.id}.`}`,
+      });
+    }
+  }
+  return [...updates, ...deletes];
+}
+
+/** Rule-based proposer wrapping planDedupe with the configured (or default)
+ *  options — merges near-duplicates; `merge: false` restores delete-only. */
+export const dedupeProposer: DeltaProposer = {
+  name: "dedupe",
+  async propose({ state, config }): Promise<ProposeResult> {
+    const deltas = planDedupe(state, config?.dedupe ?? DEFAULT_DEDUPE);
+    return deltas.length ? { deltas } : {};
   },
 };
 

package/src/refine.ts

@@ -20,9 +20,10 @@
 import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import type { Context, Model, TextContent } from "@earendil-works/pi-ai";
 import { applyDeltas, DEFAULT_DURABLE_PATH, exportDurableLayers, modelKey, PROJECT_DURABLE_DIR, REFINE_ENTRY, snapshotState } from "./store.js";
-import { getProposer } from "./proposer.js";
+import { getProposer, DEFAULT_DEDUPE } from "./proposer.js";
 import type { CompleteOptions, CompleteResult } from "./proposer.js";
-import { projectSlug } from "./config.js";
+import type { AppliedDelta } from "./types.js";
+import { loadConfig, projectSlug } from "./config.js";
 
 const DEFAULT_EVIDENCE_BYTES = 16000;
 export const DEFAULT_LOOKBACK_TURNS = 25;
@@ -41,10 +42,18 @@ type AnyEntry = {
 export function registerRefine(pi: ExtensionAPI): void {
   pi.registerCommand("refine", {
     description:
-      "Online self-improvement: review recent trajectory and propose evidence-backed CRUD deltas to the harness state. Usage: /refine [lookback-turns] [--commit] [--proposer steering|dedupe]",
+      "Online self-improvement: review recent trajectory and propose evidence-backed CRUD deltas to the harness state. Usage: /refine [lookback-turns] [--commit] [--proposer steering|dedupe] [--threshold 0.75]",
     handler: async (args, ctx) => {
-      const { lookback, commit, proposer } = parseArgs(args);
-      await runRefine(pi, ctx, { lookback, commit, ...(proposer ? { proposer } : {}) });
+      const { lookback, commit, proposer, threshold, thresholdInvalid } = parseArgs(args);
+      if (thresholdInvalid) {
+        ctx.ui.notify("Invalid --threshold ignored (expected 0 < t ≤ 1).", "warning");
+      }
+      await runRefine(pi, ctx, {
+        lookback,
+        commit,
+        ...(proposer ? { proposer } : {}),
+        ...(threshold !== undefined ? { threshold } : {}),
+      });
     },
   });
 }
@@ -54,6 +63,10 @@ export interface RefineOptions {
   commit?: boolean;
   /** Proposer name (see proposer.ts registry). Defaults to "steering". */
   proposer?: string;
+  /** Dedupe similarity threshold override in (0,1], applied for this run only
+   *  (overrides `dedupe.threshold` in harness.json). Affects the rule-based
+   *  `dedupe` proposer; ignored by others. */
+  threshold?: number;
 }
 
 export interface RefineResult {
@@ -85,6 +98,14 @@ export async function runRefine(
   ctx.ui.setStatus("harness", `Refining (last ${lookback} turns)… [${proposer.name}]`);
 
   const evidence = gatherEvidence(ctx, lookback);
+  // Load the (cached) config and thread it into ProposeInput so proposers read
+  // tuned knobs (e.g. the dedupe threshold/merge policy) without file I/O. An
+  // explicit --threshold overrides the configured dedupe threshold for this run.
+  const config = await loadConfig();
+  const proposerConfig =
+    options.threshold !== undefined
+      ? { ...config, dedupe: { ...(config.dedupe ?? DEFAULT_DEDUPE), threshold: options.threshold } }
+      : config;
   // Inject a one-shot model completion (built from ctx) so dedicated-model
   // proposers can make a hidden LLM call. Undefined when no model is resolvable.
   const complete = buildComplete(ctx);
@@ -94,6 +115,7 @@ export async function runRefine(
     evidence,
     state: snapshotState(),
     lookback,
+    config: proposerConfig,
     ...(complete ? { complete } : {}),
   });
   const proposedDeltas = result.deltas ?? [];
@@ -104,6 +126,7 @@ export async function runRefine(
   const ownerKey = modelKey(ctx.model);
 
   let applied = 0;
+  let appliedDeltas: AppliedDelta[] = [];
   let applyError: string | undefined;
   if (proposedDeltas.length > 0) {
     // Direct-apply path (rule-based / model proposers): persist each batch the
@@ -113,7 +136,7 @@ export async function runRefine(
     // race during the await above. It rolls back in-memory state before throwing,
     // so we surface the failure as an audited no-op rather than crashing /refine.
     try {
-      const appliedDeltas = applyDeltas(
+      appliedDeltas = applyDeltas(
         proposedDeltas.map((d) => {
           const delta = d.delta;
           return ownerKey !== undefined && delta.op === "create"
@@ -143,6 +166,10 @@ export async function runRefine(
     source,
     proposer: proposer.name,
     applied,
+    // Full applied-delta list (incl. delete reasons) — richer than the count:
+    // the calibration-corpus exporter (corpus.ts) reconstructs dedupe pairs
+    // from it. Legacy readers only consuming `applied` are unaffected.
+    appliedDeltas,
     rationales: proposedDeltas.map((d) => d.rationale),
     // Hidden model spend made visible: dedicated-model proposers report what the
     // call cost (model, tokens, latency, ok/error). Absent for rule-based/steering.
@@ -184,19 +211,30 @@ function parseArgs(args: string): {
   lookback: number;
   commit: boolean;
   proposer: string | undefined;
+  threshold: number | undefined;
+  thresholdInvalid: boolean;
 } {
   const parts = args.trim().split(/\s+/).filter(Boolean);
   let lookback = DEFAULT_LOOKBACK_TURNS;
   let commit = false;
   let proposer: string | undefined;
+  let threshold: number | undefined;
+  let thresholdInvalid = false;
+  const readThreshold = (raw: string): void => {
+    const v = Number(raw);
+    if (Number.isFinite(v) && v > 0 && v <= 1) threshold = v;
+    else thresholdInvalid = true;
+  };
   for (let i = 0; i < parts.length; i++) {
     const p = parts[i]!;
     if (p === "--commit") commit = true;
     else if (p === "--proposer" && i + 1 < parts.length) proposer = parts[++i];
     else if (p.startsWith("--proposer=")) proposer = p.slice("--proposer=".length);
+    else if (p === "--threshold" && i + 1 < parts.length) readThreshold(parts[++i]!);
+    else if (p.startsWith("--threshold=")) readThreshold(p.slice("--threshold=".length));
     else if (/^\d+$/.test(p)) lookback = Math.max(1, Math.min(200, Number(p)));
   }
-  return { lookback, commit, proposer };
+  return { lookback, commit, proposer, threshold, thresholdInvalid };
 }
 
 function gatherEvidence(ctx: ExtensionContext, lookback: number): string {