shieldcortex 4.28.1 → 4.30.0

package/dist/database/migrations.js

@@ -528,4 +528,130 @@ export function runMigrations(database) {
     catch (err) {
         logIfUnexpectedDdlError(err, 'session_events.sensitivity_level column + index');
     }
+    // ---------------------------------------------------------------------------
+    // Migration: v4.29.0 — one-time salience-wall backfill (clamp-only).
+    //
+    // Historically ~80% of stored memories landed at salience=1.0 (the "wall"),
+    // collapsing salience as a ranking signal. That stale 1.0 data is all
+    // machine-generated (auto / legacy-migrate / plugin captures). Forward-only
+    // fixes never cleaned it, so this self-healing auto-migration clamps those
+    // rows down to a flat 0.6 on the next process startup after updating.
+    //
+    // SCOPE IS CLAMP-ONLY. We deliberately do NOT re-derive category or
+    // memory_purpose — live data proved the title-prefix heuristic is not
+    // authoritative and re-deriving corrupts already-correct labels. The ONLY
+    // mutation is lowering salience.
+    //
+    // Untouched: manual/user rows (explicit user input), pinned rows, and
+    // canonical rows. updated_at is intentionally NOT re-stamped — these rows
+    // were not user-edited, and re-stamping would trigger a fleet-wide cloud
+    // re-sync burst.
+    //
+    // Run-once guard = existence of `memories_backfill_backup` (NOT user_version,
+    // which `.dump`-based disaster recovery does not preserve; the table does).
+    // The backup table also powers the revert path (Task 5).
+    //
+    // This block owns a LOUD catch (NOT logIfUnexpectedDdlError) — a data
+    // backfill failure must be visible and retried, not swallowed as idempotent.
+    try {
+        // Clamp predicate — flat 0.6 for machine rows; manual/user/pinned/canonical
+        // untouched. COALESCE guards old fleet DBs with NULL pinned/status.
+        const CLAMP_WHERE = `
+        salience > 0.6
+        AND COALESCE(pinned, 0) = 0
+        AND COALESCE(status, 'active') != 'canonical'
+        AND capture_method IN ('auto', 'legacy-migrate', 'plugin')`;
+        // Run-once guard (outside the txn): if the box already self-healed, bail
+        // before doing any FTS or write work.
+        const alreadyDone = database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memories_backfill_backup'").get();
+        if (!alreadyDone) {
+            // FTS rebuild FIRST, in its own try/catch (B3). On a drifted-FTS DB the
+            // per-row memories_au trigger throws "database disk image is malformed"
+            // mid-UPDATE and aborts everything. Repairing the index up front means
+            // the trigger firings during the clamp won't throw. A rebuild failure
+            // here must NOT abort the backfill, so it is isolated.
+            try {
+                database.exec("INSERT INTO memories_fts(memories_fts) VALUES('rebuild')");
+            }
+            catch (ftsErr) {
+                const msg = ftsErr instanceof Error ? ftsErr.message : String(ftsErr);
+                console.error(`[backfill v4.29.0] FTS rebuild skipped (continuing): ${msg}`);
+            }
+            // Backup-table creation + the clamp UPDATE in a SINGLE IMMEDIATE
+            // transaction (B4). IMMEDIATE grabs the write lock at BEGIN, closing the
+            // read-gate -> write race so five concurrent runMigrations processes
+            // serialise cleanly. The guard is RE-READ inside the txn: a concurrent
+            // winner may have created the table between our outer check and the lock.
+            let clamped = 0;
+            const run = database.transaction(() => {
+                const winnerCreatedTable = database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memories_backfill_backup'").get();
+                if (winnerCreatedTable) {
+                    return; // a concurrent winner already created the table; no-op
+                }
+                // Back up just the mutated column (clamp-only). Doubles as the run-once
+                // guard: even a zero-row match creates the (empty) table so the box
+                // never re-scans on subsequent startups.
+                database.exec(`
+          CREATE TABLE IF NOT EXISTS memories_backfill_backup AS
+          SELECT id, salience, datetime('now') AS backed_up_at
+          FROM memories
+          WHERE ${CLAMP_WHERE}
+        `);
+                const res = database.prepare(`UPDATE memories SET salience = 0.6 WHERE ${CLAMP_WHERE}`).run();
+                clamped = res.changes;
+            });
+            run.immediate();
+            // SUCCESS marker (after commit) so the fleet rollout is observable via the
+            // existing cloud audit ingest. firewall_result is constrained to
+            // ALLOW/BLOCK/QUARANTINE — this is an informational ALLOW marker.
+            // ORDERING DEPENDENCY: both markers INSERT into `defence_audit`, which is
+            // created earlier in this same function — keep the table migration ahead
+            // of this backfill block or the markers will silently no-op (caught below).
+            //
+            // GATED on clamped > 0: a run that healed nothing (fresh DB, or a box with
+            // no stale machine rows) still creates the run-once guard table, but must
+            // NOT emit a "healed" telemetry row. An unconditional marker was genuine
+            // noise AND tripped the defence-pipeline-bypass invariant (every
+            // saveAutoExtractedMemory must produce exactly one defence_audit row —
+            // a spurious migration marker made it two).
+            if (clamped > 0) {
+                try {
+                    database.prepare(`
+            INSERT INTO defence_audit (
+              memory_id, project, timestamp,
+              source_type, source_identifier,
+              trust_score, sensitivity_level, firewall_result,
+              anomaly_score, threat_indicators, blocked_patterns,
+              reason, fragmentation_score, pipeline_duration_ms
+            ) VALUES (NULL, NULL, ?, 'migration', 'backfill-v4.29.0', 1.0, 'INTERNAL', 'ALLOW', 0, '[]', '[]', ?, NULL, 0)
+          `).run(new Date().toISOString(), JSON.stringify({ status: 'success', clamped, version: 'v4.29.0' }));
+                }
+                catch {
+                    // Older schemas may predate the audit columns. The success path itself
+                    // already committed; the marker is best-effort observability only.
+                }
+            }
+        }
+    }
+    catch (err) {
+        // LOUD catch — do NOT route through logIfUnexpectedDdlError. Because the
+        // guard table is created INSIDE the rolled-back transaction, a failure here
+        // leaves NO guard, so the box cleanly retries on next startup.
+        const msg = err instanceof Error ? err.message : String(err);
+        console.error(`[backfill v4.29.0] FAILED, will retry next startup: ${msg}`);
+        try {
+            database.prepare(`
+        INSERT INTO defence_audit (
+          memory_id, project, timestamp,
+          source_type, source_identifier,
+          trust_score, sensitivity_level, firewall_result,
+          anomaly_score, threat_indicators, blocked_patterns,
+          reason, fragmentation_score, pipeline_duration_ms
+        ) VALUES (NULL, NULL, ?, 'migration', 'backfill-v4.29.0', 0, 'INTERNAL', 'BLOCK', 0, '[]', '[]', ?, NULL, 0)
+      `).run(new Date().toISOString(), JSON.stringify({ status: 'failed', error: msg, version: 'v4.29.0' }));
+        }
+        catch {
+            // Best-effort marker; the stderr line above carries the signal.
+        }
+    }
 }

package/dist/memory/consolidate.d.ts

@@ -27,8 +27,42 @@ export declare function findDuplicateMemoryPairs(options?: {
     limit?: number;
 }): DuplicateMemoryPair[];
 /**
- * Find memories with very similar titles/content and merge them.
- * Keeps the newer one, appends unique content from the older one.
+ * De-duplicate long-term memories found as duplicate PAIRS by
+ * findDuplicateMemoryPairs (same project|category, similar title + >0.5 content
+ * overlap). Runs every ~4h on the auto path: server.ts setInterval → fullCleanup
+ * → consolidate → deduplicateMemories.
+ *
+ * B11 (memory-quality fix): the previous implementation built
+ * `kept.content + "\n\nMerged from duplicate:\n" + uniqueSentences` and wrote it
+ * back with `UPDATE memories SET content = ?`. That was the SECOND lossy-append
+ * site (mergeSimilarMemories was the first) and kept generating ever-growing
+ * "frankenmemories" on this path even after the first site was fixed. The kept
+ * memory's `content` is now NEVER modified.
+ *
+ * New policy (identical to mergeSimilarMemories — one source of truth):
+ *   - KEEP the higher-EFFECTIVE-salience member of the pair (computeEffectiveSalience
+ *     via the shared salience.mjs — recency/access/pin/downvote, same ranking as
+ *     recall and the downvote CLI). NOTE: this can differ from the pair's raw
+ *     `recommendedKeepId`, which only considered raw salience + recency.
+ *   - Dispose the loser via the shared disposeLoser: combined similarity
+ *     (re-measured vs the kept row) ≥ 0.85 → DELETE; below → DOWNVOTE (reversible).
+ *   - Tags are unioned onto the kept row (metadata only, non-lossy). The kept
+ *     content and salience are untouched.
+ *
+ * Defence re-scan removed (B11): the old re-scan existed ONLY to validate the
+ * SYNTHESIZED concatenated merged body before persisting it (two individually-
+ * clean rows could straddle a credential pattern across the join). Now that no
+ * concatenation happens here, there is no synthesized content to scan — the kept
+ * row's bytes are written verbatim and unchanged, exactly as they already exist
+ * in `memories`. So this pass introduces no new bytes to validate. (This is a
+ * narrow claim about THIS path: it does not assume every row was scanned at write
+ * time — e.g. importMemories inserts rows with no scan — only that the dedup no
+ * longer fabricates a new body that would need scanning.) Dropping the re-scan
+ * also avoids re-auditing unchanged content on every 4h pass.
+ *
+ * Only rows clustered in THIS pass are acted on; historical merged rows are not
+ * rewritten (effective salience demotes them over time). STM→LTM promotion is
+ * handled separately by processDecay/promoteMemory in consolidate().
  */
 export declare function deduplicateMemories(options?: {
     dryRun?: boolean;
@@ -55,10 +89,32 @@ export declare function clusterAndSummarise(options?: {
  */
 export declare function enforceMemoryLimits(config?: MemoryConfig): number;
 /**
- * Find and merge similar short-term memories into coherent long-term entries.
- * Groups memories by project|category, then clusters by Jaccard similarity
- * on content (0.6 weight) + title (0.4 weight).
- * Returns count of deleted (merged) memories.
+ * Find clusters of similar short-term memories within each project|category,
+ * KEEP the single highest-effective-salience member, and demote the rest
+ * WITHOUT destroying or concatenating their content.
+ *
+ * B11 (memory-quality fix): the previous implementation appended every loser's
+ * body into the kept memory ("Consolidated context:" bullets) and deleted the
+ * losers. Over ~417 live rows this produced 188 ever-growing "frankenmemories"
+ * — exactly the garbled content users complained about. The kept memory's
+ * `content` is now NEVER modified.
+ *
+ * Disposition per loser (see NEAR_IDENTICAL_DELETE_THRESHOLD):
+ *   - combined ≥ 0.85  → DELETE (near-identical; content already in the kept row)
+ *   - combined < 0.85  → DOWNVOTE (reversible; sinks via effective salience)
+ *
+ * Why the split exists at all: NO reaping path (decay.ts shouldDelete,
+ * prune.ts, expiry.ts, enforceMemoryLimits) reads `downvote_count` or effective
+ * salience — they all order by raw `salience`. So downvote-ONLY would leave the
+ * store unbounded (the dups sink in recall but never get pruned). Deleting the
+ * near-identical losers keeps the store bounded with zero information loss; the
+ * moderate near-dups are preserved and merely demoted.
+ *
+ * Tags are unioned onto the kept row and access counts summed (metadata only,
+ * non-lossy). STM→LTM promotion is handled separately by processDecay/
+ * promoteMemory in consolidate(); this function only de-duplicates.
+ *
+ * Returns the count of deleted (near-identical) losers.
  */
 export declare function mergeSimilarMemories(project?: string, similarityThreshold?: number): number;
 /**
@@ -144,5 +200,25 @@ export interface DreamModeResult {
  * 1. Find near-duplicates by embedding similarity >0.9, merge them
  * 2. Flag memories >30 days old with no access as archival candidates
  * 3. Detect and flag contradictions
+ *
+ * DISPOSAL POLICY — deliberately DIVERGES from the auto path:
+ * This is the EXPLICIT, user-invoked `shieldcortex consolidate` command (and the
+ * public `consolidateMemories` export). Because the user has actively asked for a
+ * consolidation, aggressive cleanup is the intended behaviour: the loser of every
+ * duplicate pair is HARD-DELETED. This is on purpose and must NOT be replaced with
+ * the gentle `disposeLoser` policy.
+ *
+ * Contrast with the 4-hourly AUTO path (deduplicateMemories / mergeSimilarMemories),
+ * which runs UNATTENDED and therefore uses the conservative `disposeLoser`: it only
+ * deletes near-identical losers (combined ≥ 0.85, content already contained in the
+ * kept row) and merely DOWNVOTES (reversible) the moderate near-dups. That caution
+ * is right for an unsupervised background job, but wrong for a deliberate manual
+ * cleanup the user explicitly triggered.
+ *
+ * The two paths DO share keep-selection: both pick the kept row by EFFECTIVE
+ * salience (effectiveSalienceOfRow — recency/access/pin/downvote), so the row that
+ * survives is consistent across all three dedup sites. Effective salience also
+ * discriminates where the pair's raw `recommendedKeepId` (raw salience + recency)
+ * ties at 1.0.
  */
 export declare function consolidateMemories(): DreamModeResult;