pi-mega-compact 0.20.5 → 0.20.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/dist/config/vector-cortex.js +12 -0
  2. package/dist/config.js +1 -1
  3. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +3 -31
  4. package/dist/extensions/dashboard-server/routes-rag-settings-types.js +11 -0
  5. package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +43 -0
  6. package/dist/extensions/dashboard-server/routes-vector-cortex-rollout.js +43 -0
  7. package/dist/extensions/dashboard-server/routes-vector-cortex.js +1 -0
  8. package/dist/extensions/dashboard-server/routes.js +1 -1
  9. package/dist/extensions/dashboard-server/server.js +3 -1
  10. package/dist/extensions/mega-runtime/vector-cortex-live.js +105 -0
  11. package/dist/src/config/dedup.js +1 -0
  12. package/dist/src/config/vector-cortex.js +12 -0
  13. package/dist/src/config.js +1 -1
  14. package/dist/src/dedup-audit.test-helpers.js +43 -0
  15. package/dist/src/monitoring.js +9 -0
  16. package/dist/src/vector-cortex/rollout/_acceptance-fixture.js +55 -0
  17. package/dist/src/vector-cortex/rollout/_acceptance-helpers.js +8 -0
  18. package/dist/src/vector-cortex/rollout/_acceptance-scenario.js +70 -0
  19. package/dist/src/vector-cortex/rollout/assign.js +56 -0
  20. package/dist/src/vector-cortex/rollout/emit.js +50 -0
  21. package/dist/src/vector-cortex/rollout/gate.js +75 -0
  22. package/dist/src/vector-cortex/rollout/types.js +39 -0
  23. package/dist/src/vectorStore/add-l0.js +108 -0
  24. package/dist/src/vectorStore/add-l1.js +35 -0
  25. package/dist/src/vectorStore/add.js +229 -0
  26. package/dist/src/vectorStore/class.js +15 -291
  27. package/dist/src/vectorStore/dedup-audit.js +104 -0
  28. package/dist/src/vectorStore.js +1 -0
  29. package/dist/vector-cortex/rollout/_acceptance-fixture.js +55 -0
  30. package/dist/vector-cortex/rollout/_acceptance-helpers.js +8 -0
  31. package/dist/vector-cortex/rollout/_acceptance-scenario.js +70 -0
  32. package/dist/vector-cortex/rollout/assign.js +56 -0
  33. package/dist/vector-cortex/rollout/emit.js +50 -0
  34. package/dist/vector-cortex/rollout/gate.js +75 -0
  35. package/dist/vector-cortex/rollout/types.js +39 -0
  36. package/extensions/dashboard-client/dist/assets/{AreaChart-CGL29Tcl.js → AreaChart-DqTbaSRA.js} +2 -2
  37. package/extensions/dashboard-client/dist/assets/{AreaChart-CGL29Tcl.js.map → AreaChart-DqTbaSRA.js.map} +1 -1
  38. package/extensions/dashboard-client/dist/assets/{BarChart-OxhalIZa.js → BarChart-BfNBeOvF.js} +2 -2
  39. package/extensions/dashboard-client/dist/assets/{BarChart-OxhalIZa.js.map → BarChart-BfNBeOvF.js.map} +1 -1
  40. package/extensions/dashboard-client/dist/assets/{CacheTab-VcMFwL-6.js → CacheTab-C9SOMYB5.js} +2 -2
  41. package/extensions/dashboard-client/dist/assets/{CacheTab-VcMFwL-6.js.map → CacheTab-C9SOMYB5.js.map} +1 -1
  42. package/extensions/dashboard-client/dist/assets/{EventsTab-DePofXDM.js → EventsTab-GsMKP10_.js} +2 -2
  43. package/extensions/dashboard-client/dist/assets/{EventsTab-DePofXDM.js.map → EventsTab-GsMKP10_.js.map} +1 -1
  44. package/extensions/dashboard-client/dist/assets/{HealthTab-nQd1tjSF.js → HealthTab-DAOCXwRP.js} +2 -2
  45. package/extensions/dashboard-client/dist/assets/{HealthTab-nQd1tjSF.js.map → HealthTab-DAOCXwRP.js.map} +1 -1
  46. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CzvA8-F6.js → MaintenanceTab-BtBeMkB5.js} +2 -2
  47. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CzvA8-F6.js.map → MaintenanceTab-BtBeMkB5.js.map} +1 -1
  48. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-3lxgWBvU.js → MemoryMapTab-DzndtgkZ.js} +2 -2
  49. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-3lxgWBvU.js.map → MemoryMapTab-DzndtgkZ.js.map} +1 -1
  50. package/extensions/dashboard-client/dist/assets/{MetricsTab-D_FcA6kN.js → MetricsTab-DyxJ4eb8.js} +2 -2
  51. package/extensions/dashboard-client/dist/assets/{MetricsTab-D_FcA6kN.js.map → MetricsTab-DyxJ4eb8.js.map} +1 -1
  52. package/extensions/dashboard-client/dist/assets/{OverviewTab-BU6R4aEs.js → OverviewTab-DCiEdVWR.js} +2 -2
  53. package/extensions/dashboard-client/dist/assets/{OverviewTab-BU6R4aEs.js.map → OverviewTab-DCiEdVWR.js.map} +1 -1
  54. package/extensions/dashboard-client/dist/assets/{ReposTab-DFjy1isz.js → ReposTab-B5SIqVHA.js} +2 -2
  55. package/extensions/dashboard-client/dist/assets/{ReposTab-DFjy1isz.js.map → ReposTab-B5SIqVHA.js.map} +1 -1
  56. package/extensions/dashboard-client/dist/assets/{SessionsTab-UGIzIH25.js → SessionsTab-DLWFJ9xI.js} +2 -2
  57. package/extensions/dashboard-client/dist/assets/{SessionsTab-UGIzIH25.js.map → SessionsTab-DLWFJ9xI.js.map} +1 -1
  58. package/extensions/dashboard-client/dist/assets/{SetupTab-CECrgMKI.js → SetupTab-CvGXZyej.js} +2 -2
  59. package/extensions/dashboard-client/dist/assets/{SetupTab-CECrgMKI.js.map → SetupTab-CvGXZyej.js.map} +1 -1
  60. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dyy4SCDk.js → TimeSavedCard-wZuQGxLS.js} +2 -2
  61. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dyy4SCDk.js.map → TimeSavedCard-wZuQGxLS.js.map} +1 -1
  62. package/extensions/dashboard-client/dist/assets/{TurnsTab-B0LPcVs1.js → TurnsTab-B7aOejch.js} +2 -2
  63. package/extensions/dashboard-client/dist/assets/{TurnsTab-B0LPcVs1.js.map → TurnsTab-B7aOejch.js.map} +1 -1
  64. package/extensions/dashboard-client/dist/assets/VectorCortexTab-eHVt9yDx.js +2 -0
  65. package/extensions/dashboard-client/dist/assets/VectorCortexTab-eHVt9yDx.js.map +1 -0
  66. package/extensions/dashboard-client/dist/assets/{WikiTab-DWk9Qvut.js → WikiTab-DhWHjAtt.js} +2 -2
  67. package/extensions/dashboard-client/dist/assets/{WikiTab-DWk9Qvut.js.map → WikiTab-DhWHjAtt.js.map} +1 -1
  68. package/extensions/dashboard-client/dist/assets/{button-O59wPr5H.js → button-BrC-W87h.js} +2 -2
  69. package/extensions/dashboard-client/dist/assets/{button-O59wPr5H.js.map → button-BrC-W87h.js.map} +1 -1
  70. package/extensions/dashboard-client/dist/assets/{card-C-DxXv1-.js → card-0rgfRgr3.js} +2 -2
  71. package/extensions/dashboard-client/dist/assets/{card-C-DxXv1-.js.map → card-0rgfRgr3.js.map} +1 -1
  72. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-Dw5YnE5s.js → generateCategoricalChart-DRresOQF.js} +2 -2
  73. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-Dw5YnE5s.js.map → generateCategoricalChart-DRresOQF.js.map} +1 -1
  74. package/extensions/dashboard-client/dist/assets/{index-i9oaPE0a.js → index-DohmtZWL.js} +3 -3
  75. package/extensions/dashboard-client/dist/assets/{index-i9oaPE0a.js.map → index-DohmtZWL.js.map} +1 -1
  76. package/extensions/dashboard-client/dist/assets/{switch-B69QY3IM.js → switch-Bs8g_zB5.js} +2 -2
  77. package/extensions/dashboard-client/dist/assets/{switch-B69QY3IM.js.map → switch-Bs8g_zB5.js.map} +1 -1
  78. package/extensions/dashboard-client/dist/assets/{toggle-BJ4Kwggu.js → toggle-DVVauIo8.js} +2 -2
  79. package/extensions/dashboard-client/dist/assets/{toggle-BJ4Kwggu.js.map → toggle-DVVauIo8.js.map} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{useSSE-BpuA8sPd.js → useSSE-DxH1PTAR.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{useSSE-BpuA8sPd.js.map → useSSE-DxH1PTAR.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/index.html +1 -1
  83. package/extensions/dashboard-client/src/api/vector-cortex.ts +9 -0
  84. package/extensions/dashboard-client/src/tabs/VectorCortexRolloutCard.tsx +44 -0
  85. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +8 -0
  86. package/extensions/dashboard-client/src/types/vector-cortex.ts +17 -0
  87. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +32 -1
  88. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +11 -129
  89. package/extensions/dashboard-server/routes-rag-settings-types.ts +34 -0
  90. package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +142 -0
  91. package/extensions/dashboard-server/routes-vector-cortex-rollout.ts +52 -0
  92. package/extensions/dashboard-server/routes-vector-cortex.ts +1 -0
  93. package/extensions/dashboard-server/routes.ts +1 -0
  94. package/extensions/dashboard-server/server.ts +2 -0
  95. package/extensions/mega-runtime/vector-cortex-live.ts +165 -0
  96. package/package.json +1 -1
  97. package/src/config/dedup.ts +9 -0
  98. package/src/config/vector-cortex.ts +13 -0
  99. package/src/config.ts +1 -0
  100. package/src/dedup-audit.test-helpers.ts +49 -0
  101. package/src/monitoring.ts +10 -0
  102. package/src/vector-cortex/rollout/_acceptance-fixture.ts +94 -0
  103. package/src/vector-cortex/rollout/_acceptance-helpers.ts +24 -0
  104. package/src/vector-cortex/rollout/_acceptance-scenario.ts +96 -0
  105. package/src/vector-cortex/rollout/assign.ts +67 -0
  106. package/src/vector-cortex/rollout/emit.ts +81 -0
  107. package/src/vector-cortex/rollout/gate.ts +119 -0
  108. package/src/vector-cortex/rollout/types.ts +111 -0
  109. package/src/vectorStore/add-l0.ts +152 -0
  110. package/src/vectorStore/add-l1.ts +57 -0
  111. package/src/vectorStore/add.ts +269 -0
  112. package/src/vectorStore/class.ts +17 -327
  113. package/src/vectorStore/dedup-audit.ts +199 -0
  114. package/src/vectorStore.ts +1 -0
  115. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BWi3u_gp.js +0 -2
  116. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BWi3u_gp.js.map +0 -1
@@ -0,0 +1,50 @@
1
+ /**
2
+ * vector-cortex/rollout/emit.ts — VC5C rollout emit seam.
3
+ *
4
+ * The structured-event surface for the live graduated rollout: assignment and
5
+ * promotion-blocked observability. Mirrors the VC0C resilience emit seam and
6
+ * the VC0B replay emit seam: the same injected `(event, fields)` callback, the
7
+ * same non-fatal `safe()` best-effort wrapper, and the same per-call
8
+ * `VC5C_ENABLED()` gate. Absent emitter (or flag OFF) degrades to a no-op with
9
+ * byte-identical predecessor behavior (PREVENT-PI-004, structured `ts`+`event`).
10
+ *
11
+ * Carries exactly the two events the sprint names:
12
+ * - vector_cortex_rollout_assigned (a session was assigned a bucket)
13
+ * - vector_cortex_rollout_promotion_blocked (a hard fault froze promotion)
14
+ */
15
+ import { VC5C_ENABLED } from "../../config/vector-cortex.js";
16
+ function safe(fn) {
17
+ return (event, fields) => {
18
+ try {
19
+ fn(event, fields);
20
+ }
21
+ catch {
22
+ /* non-fatal observability — never break the agent loop */
23
+ }
24
+ };
25
+ }
26
+ /**
27
+ * Build a typed reporter over an injected emit callback. Emission is gated on
28
+ * `VC5C_ENABLED()` — flag OFF (or absent emitter) means zero rollout
29
+ * observability writes (pre-VC predecessor parity). The flag is read per-call so
30
+ * a live toggle takes effect immediately.
31
+ */
32
+ export function createRolloutReporter(emit) {
33
+ const fire = safe((event, fields) => {
34
+ if (VC5C_ENABLED())
35
+ emit?.(event, fields);
36
+ });
37
+ return {
38
+ assigned(fields) {
39
+ fire("vector_cortex_rollout_assigned", fields);
40
+ },
41
+ promotionBlocked(fields) {
42
+ fire("vector_cortex_rollout_promotion_blocked", fields);
43
+ },
44
+ };
45
+ }
46
+ /** Export the event names for consumers that match on them. */
47
+ export const ROLLOUT_EVENT_NAMES = [
48
+ "vector_cortex_rollout_assigned",
49
+ "vector_cortex_rollout_promotion_blocked",
50
+ ];
@@ -0,0 +1,75 @@
1
+ /**
2
+ * vector-cortex/rollout/gate.ts — monotonic graduated-gate advance decision.
3
+ *
4
+ * A gate advances exactly ONE step only when ALL conjuncts hold:
5
+ * (elapsedWallClock >= 72h) — measured on the MONOTONIC clock, not wall time
6
+ * AND (powered sample available)
7
+ * AND (events >= 10000)
8
+ * AND (sessions >= 200)
9
+ *
10
+ * Promotion is strictly monotonic by one gate step. The unique failure
11
+ * injection (TRIAD_RESILIENCE.md): a wall-clock jump while the monotonic clock
12
+ * is unchanged does NOT advance — because the 72h eligibility is the MONOTONIC
13
+ * delta, not wall time. We therefore take an injected clock seam with SEPARATE
14
+ * monotonic and wall readings and never call Date.now() directly for the safety
15
+ * decision (PREVENT decorum: the host injects a testable fake clock).
16
+ *
17
+ * Pure arithmetic over injected evidence + clock; no I/O, no network.
18
+ * PREVENT-011: no `any`.
19
+ */
20
+ import { ROLLOUT_GATES, } from "./types.js";
21
+ /** The 72h monotonic residency requirement (milliseconds). */
22
+ export const GATE_MIN_ELAPSED_MS = 72 * 60 * 60 * 1000;
23
+ /** Powered-sample availability gate (minimum events for a one-sided CI read). */
24
+ export const GATE_MIN_EVENTS = 10_000;
25
+ /** Minimum distinct sessions before a gate may advance. */
26
+ export const GATE_MIN_SESSIONS = 200;
27
+ /** Default clock: real monotonic (performance.now) + wall (ISO). */
28
+ export function defaultClock() {
29
+ return {
30
+ now: () => performance.now(),
31
+ wallNow: () => new Date().toISOString(),
32
+ };
33
+ }
34
+ /**
35
+ * Decide the next live outcome for a decision epoch. Returns the (possibly
36
+ * advanced) gate index + the full evidence snapshot. Promotion monotonic by one
37
+ * step; a hard fault freezes promotion (promotionBlocked=true) and is reported
38
+ * for C-selection by the live seam.
39
+ *
40
+ * @param currentGate the gate index BEFORE this decision (0..4).
41
+ * @param evidence the observed window evidence.
42
+ * @param clock the injected monotonic/wall clock seam.
43
+ */
44
+ export function decideGate(currentGate, evidence, clock) {
45
+ const elapsedMs = Math.max(0, clock.now() - evidence.windowStartMs);
46
+ const hardFault = evidence.hardFaults.length > 0;
47
+ const eligible = !hardFault &&
48
+ currentGate < (ROLLOUT_GATES.length - 1) &&
49
+ elapsedMs >= GATE_MIN_ELAPSED_MS &&
50
+ evidence.powered &&
51
+ evidence.events >= GATE_MIN_EVENTS &&
52
+ evidence.sessions >= GATE_MIN_SESSIONS;
53
+ // Monotonic by ONE step only.
54
+ const nextGate = eligible
55
+ ? (currentGate + 1)
56
+ : currentGate;
57
+ return {
58
+ schema: "live-outcome-v1",
59
+ gateIndex: nextGate,
60
+ gatePct: ROLLOUT_GATES[nextGate],
61
+ elapsedMs,
62
+ powered: evidence.powered,
63
+ events: evidence.events,
64
+ sessions: evidence.sessions,
65
+ promotionBlocked: hardFault,
66
+ decidedAt: clock.wallNow(),
67
+ };
68
+ }
69
+ /**
70
+ * Whether a hard fault should SELECT triad C (pre-VC path). Any hard causal/
71
+ * tool/anchor/exact failure does. Exposed so the live seam can branch cleanly.
72
+ */
73
+ export function selectsPreVcPath(fault) {
74
+ return fault !== undefined;
75
+ }
@@ -0,0 +1,39 @@
1
+ /**
2
+ * vector-cortex/rollout/types.ts — VC5C live graduated-rollout contracts.
3
+ *
4
+ * A session is deterministically hashed into one of 10,000 stable buckets
5
+ * (0..9999). The current rollout gate is one of 1% / 5% / 25% / 50% / 100%.
6
+ * A bucket is "in" the current gate iff `bucket < gatePct * 100`. Assignment is
7
+ * a PURE function of the session id: it NEVER changes across process restart
8
+ * (no Date.now / Math.random — see `assign.ts`).
9
+ *
10
+ * Pure types + registered conformance IDs: no storage, no console, no network
11
+ * (PREVENT-PI-004 / PREVENT-011).
12
+ */
13
+ /** Total stable buckets. Reassignment on restart is impossible by construction. */
14
+ export const ROLLOUT_BUCKETS = 10_000;
15
+ /**
16
+ * The five graduated gates, in promotion order. Index = step. Promotion is
17
+ * strictly monotonic by ONE step (never skipping a gate).
18
+ */
19
+ export const ROLLOUT_GATES = [1, 5, 25, 50, 100];
20
+ /**
21
+ * The two conformance events VC5C emits to the monitoring/events path. Mirrors
22
+ * how prior sprints emit (resilience/emit.ts, render/emit.ts).
23
+ */
24
+ export const ROLLOUT_EVENTS = [
25
+ "vector_cortex_rollout_assigned",
26
+ "vector_cortex_rollout_promotion_blocked",
27
+ ];
28
+ /**
29
+ * Registered rollout conformance ID range (ROL-001..020). The acceptance
30
+ * aggregator reads these rows from the v2 manifest and asserts each returns its
31
+ * manifest `ok`/`code`.
32
+ */
33
+ export const ROL_IDS = Array.from({ length: 20 }, (_v, i) => `ROL-${String(i + 1).padStart(3, "0")}`);
34
+ /** Named VC5C rollout conformance assertions (the sprint's headline rows). */
35
+ export const ROL_NAMED_IDS = [
36
+ "ROL-BUCKET-001",
37
+ "ROL-POWER-002",
38
+ "ROL-SAFETY-003",
39
+ ];
@@ -0,0 +1,108 @@
1
+ /**
2
+ * add-l0.ts — the L0 exact-match tier of the dedup cascade.
3
+ *
4
+ * L0 is three exact-hash probes sharing one enable flag and one MARK_ONLY flag:
5
+ * 1. contentHash — normalized dual-hash, bloom-accelerated (Sprint 9/10)
6
+ * 2. regionHash — the legacy pre-normalization hash (backward-compat)
7
+ * 3. summaryHash — same-topic incremental compactions
8
+ *
9
+ * Split out of add.ts so both files stay under the 300-line src/ soft limit.
10
+ * The tier is a pure decision: it either produces a collapse (returning the
11
+ * matched result) or reports the fall-through, and it never writes a new
12
+ * checkpoint — that stays in add.ts.
13
+ *
14
+ * All three probes are hash comparisons, so none of them computes a similarity
15
+ * score and none is invented for the audit trail.
16
+ */
17
+ import { createHash } from "node:crypto";
18
+ import { upsertCheckpoint, addTokensSaved, bumpDedupStats } from "../store/sqlite.js";
19
+ /** Run the L0 exact-match tier. */
20
+ export function runL0Tier(ctx) {
21
+ const { store, input, all, digest, regionHash, origTokens, bloom, t0, audit } = ctx;
22
+ const cfg = store.cfg;
23
+ const onTier = input.onTier;
24
+ let markOnly = null;
25
+ // 0. Content-hash dedup (Sprint 9) — catches identical content arriving under
26
+ // different regionText. Normalization handles case/whitespace/ANSI so
27
+ // variants collapse to one row. Dual-hash guards a single-hash collision.
28
+ // Sprint 10: bloom is the accelerator — a miss means "definitely new" and
29
+ // skips the scan; a hit is only a candidate, confirmed against `all` here.
30
+ // MARK_ONLY_L0 records the decision but does not collapse.
31
+ onTier?.({ tier: "L0", status: "scanning" });
32
+ if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
33
+ const contentMatch = all.find((cp) => cp.contentHash === digest.contentHash &&
34
+ cp.contentHash2 === digest.contentHash2);
35
+ if (contentMatch) {
36
+ if (cfg.MARK_ONLY_L0) {
37
+ markOnly = "L0"; // Record-but-don't-collapse: fall through.
38
+ }
39
+ else {
40
+ contentMatch.timestamp = input.timestamp;
41
+ upsertCheckpoint(contentMatch, store.stateDir);
42
+ bumpDedupStats(true, store.stateDir);
43
+ // Deduped: whole original region discarded, nothing new stored.
44
+ addTokensSaved(origTokens, store.stateDir);
45
+ store.record("L0", "deduped", "contentHash", Date.now() - t0, 1, contentMatch.checkpointId);
46
+ audit.deduped("L0", contentMatch.checkpointId, "contentHash");
47
+ onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
48
+ return {
49
+ result: { checkpoint: contentMatch, deduped: true, reason: "contentHash" },
50
+ markOnly,
51
+ };
52
+ }
53
+ }
54
+ }
55
+ // 1. Legacy regionHash dedup (backward-compat) — part of L0 tier gating.
56
+ if (cfg.L0_ENABLED) {
57
+ const regionMatch = all.find((cp) => cp.regionHash === regionHash);
58
+ if (regionMatch) {
59
+ if (cfg.MARK_ONLY_L0) {
60
+ markOnly = "L0"; // fall through
61
+ }
62
+ else {
63
+ bumpDedupStats(true, store.stateDir);
64
+ // Deduped: whole original region discarded, nothing new stored.
65
+ addTokensSaved(origTokens, store.stateDir);
66
+ store.record("L0", "deduped", "regionHash", Date.now() - t0, 1, regionMatch.checkpointId);
67
+ audit.deduped("L0", regionMatch.checkpointId, "regionHash");
68
+ onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
69
+ return {
70
+ result: { checkpoint: regionMatch, deduped: true, reason: "regionHash" },
71
+ markOnly,
72
+ };
73
+ }
74
+ }
75
+ }
76
+ // 2. SummaryHash dedup — catches same-topic incremental compactions.
77
+ if (ctx.summaryHash && cfg.L0_ENABLED) {
78
+ const summaryMatch = all.find((cp) => cp.summaryHash === ctx.summaryHash);
79
+ if (summaryMatch) {
80
+ if (cfg.MARK_ONLY_L0) {
81
+ markOnly = "L0"; // fall through
82
+ }
83
+ else {
84
+ summaryMatch.timestamp = input.timestamp;
85
+ upsertCheckpoint(summaryMatch, store.stateDir);
86
+ bumpDedupStats(true, store.stateDir);
87
+ // Deduped: whole original region discarded, nothing new stored.
88
+ addTokensSaved(origTokens, store.stateDir);
89
+ store.record("L0", "deduped", "summaryHash", Date.now() - t0, 1, summaryMatch.checkpointId);
90
+ audit.deduped("L0", summaryMatch.checkpointId, "summaryHash");
91
+ onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
92
+ return {
93
+ result: { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" },
94
+ markOnly,
95
+ };
96
+ }
97
+ }
98
+ }
99
+ // L0 did not collapse this region.
100
+ onTier?.({ tier: "L0", status: "passed" });
101
+ return { markOnly };
102
+ }
103
+ /** SHA-256 of the topic summary (full 64-hex; 16-hex was collision-prone). */
104
+ export function computeSummaryHash(topicSummary) {
105
+ return topicSummary
106
+ ? createHash("sha256").update(topicSummary).digest("hex")
107
+ : undefined;
108
+ }
@@ -0,0 +1,35 @@
1
+ import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES, } from "../dedup/l1-minhash.js";
2
+ import { lshBands } from "../dedup/l1-lsh.js";
3
+ import { isNearDuplicate } from "../dedup/l1-verify.js";
4
+ import { lshCandidateChunks } from "../store/sqlite.js";
5
+ export /**
6
+ * L1 near-duplicate lookup: MinHash → LSH candidate retrieval → trigram verify.
7
+ * Returns the matching checkpoint or undefined. Bounded by a 100-candidate cap
8
+ * and a 20ms verify budget (QA #7/#15) so it never hangs a large session.
9
+ */ function findL1Duplicate(store, sessionId, regionText, all) {
10
+ if (all.length === 0)
11
+ return undefined;
12
+ const sig = minhashSignature(regionText);
13
+ if (sig.length !== NUM_HASHES)
14
+ return undefined;
15
+ const bands = lshBands(sig, sessionId, SIGNATURE_VERSION);
16
+ // Cheap candidate retrieval (single query, capped). Exclude nothing yet —
17
+ // the new checkpoint has no id, so pass a sentinel that never matches.
18
+ const candidateIds = lshCandidateChunks(bands, sessionId, "__new__", store.stateDir, 100);
19
+ if (candidateIds.length === 0)
20
+ return undefined;
21
+ const byId = new Map(all.map((cp) => [cp.checkpointId, cp]));
22
+ const VERIFY_BUDGET_MS = 20;
23
+ const start = Date.now();
24
+ for (const id of candidateIds) {
25
+ if (Date.now() - start > VERIFY_BUDGET_MS)
26
+ break; // QA #15: abort → "not dup"
27
+ const cand = byId.get(id);
28
+ if (!cand)
29
+ continue;
30
+ const candText = cand.normalizedText ?? cand.summary ?? "";
31
+ if (isNearDuplicate(regionText, candText))
32
+ return cand;
33
+ }
34
+ return undefined;
35
+ }
@@ -0,0 +1,229 @@
1
+ /**
2
+ * add.ts — the dedup cascade behind VectorStore.add() (extracted from class.ts).
3
+ *
4
+ * Free function taking the store as its first argument, matching the sibling
5
+ * pattern used by vector-read.ts / vector-search.ts / vector-dedup.ts: the
6
+ * store's fields are package-public by design (see the comment in class.ts) so
7
+ * the helpers can read them without casts, and class.ts stays a thin shell.
8
+ *
9
+ * Cascade order (unchanged by the extraction):
10
+ * 1. L0 contentHash exact match (bloom-accelerated)
11
+ * 2. L0 legacy regionHash exact match (backward-compat)
12
+ * 3. L0 summaryHash exact match (same-topic incremental compactions)
13
+ * 4. L1 MinHash/LSH near-duplicate
14
+ * 5. L2 cosine similarity >= threshold
15
+ * 6. None matched → create a new checkpoint
16
+ *
17
+ * Sprint-14 monitoring (`store.record`) and the live `onTier` UI hook are
18
+ * untouched. The dedup audit trail (external-audit item #2) is layered on top as
19
+ * PURE instrumentation: every emit is best-effort and flag-gated, so a flag-off
20
+ * or a failing write leaves the dedup outcome byte-identical.
21
+ */
22
+ import { cosineSimilarity } from "../embedder.js";
23
+ import { normalizeSessionId, compressSmart } from "../store.js";
24
+ import { computeContentDigest } from "../dedup/digest.js";
25
+ import { minhashSignature, SIGNATURE_VERSION } from "../dedup/l1-minhash.js";
26
+ import { lshBands } from "../dedup/l1-lsh.js";
27
+ import { openBloom, saveBloom } from "../store/bloom.js";
28
+ import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, addTokensSaved, bumpDedupStats, } from "../store/sqlite.js";
29
+ import { computeRegionHash } from "./hash.js";
30
+ import { runL0Tier, computeSummaryHash } from "./add-l0.js";
31
+ import { findL1Duplicate } from "./add-l1.js";
32
+ import { dedupAuditRecorder } from "./dedup-audit.js";
33
+ /**
34
+ * Add a checkpoint, running the full dedup cascade.
35
+ *
36
+ * Returns the matched checkpoint with `deduped: true` when a tier collapsed the
37
+ * region, otherwise the newly created checkpoint with `deduped: false`.
38
+ */
39
+ export function addCheckpoint(store, input) {
40
+ const t0 = Date.now();
41
+ const sessionId = normalizeSessionId(input.sessionId);
42
+ const regionHash = computeRegionHash(input.regionText);
43
+ const all = listCheckpoints(sessionId, store.stateDir);
44
+ // Honest "tokens saved" base for this region. For a deduped add the whole
45
+ // original region is discarded (nothing new stored); for a new checkpoint
46
+ // we persist (orig − stored). Falls back to stored when orig is unknown.
47
+ const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
48
+ const cfg = store.cfg;
49
+ // Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
50
+ // so the UI can paint "L0 ✓ → L1 ✓ → L2 0.91 → stored" during a compaction.
51
+ const onTier = input.onTier;
52
+ // Durable decision trail (external-audit item #2). Bound to this one cascade
53
+ // so each call site is a single line; a no-op when cfg.DEDUP_AUDIT is off.
54
+ const audit = dedupAuditRecorder({
55
+ stateDir: store.stateDir,
56
+ eventsPath: store.eventsPath,
57
+ auditEnabled: cfg.DEDUP_AUDIT,
58
+ }, {
59
+ sessionId,
60
+ originalTokenEstimate: input.originalTokenEstimate,
61
+ tokenEstimate: input.tokenEstimate,
62
+ });
63
+ // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
64
+ // and which tier.
65
+ let markOnly = null;
66
+ // L0 exact-match tier (contentHash / regionHash / summaryHash) — see add-l0.ts.
67
+ const digest = computeContentDigest(input.regionText);
68
+ const bloom = openBloom(store.stateDir);
69
+ const summaryHash = computeSummaryHash(input.topicSummary);
70
+ const l0 = runL0Tier({
71
+ store,
72
+ input,
73
+ all,
74
+ digest,
75
+ regionHash,
76
+ summaryHash,
77
+ origTokens,
78
+ bloom,
79
+ t0,
80
+ audit,
81
+ });
82
+ if (l0.result)
83
+ return l0.result;
84
+ if (l0.markOnly)
85
+ markOnly = l0.markOnly;
86
+ // 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) — catches one-word
87
+ // edits / rewordings that L0's exact hash misses. Cheap LSH bucket
88
+ // retrieval → trigram verification (pg_trgm-equivalent) as the final gate.
89
+ // Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
90
+ onTier?.({ tier: "L1", status: "scanning" });
91
+ if (cfg.L1_ENABLED) {
92
+ const l1 = findL1Duplicate(store, sessionId, input.regionText, all);
93
+ if (l1 && !cfg.MARK_ONLY_L1) {
94
+ l1.timestamp = input.timestamp;
95
+ upsertCheckpoint(l1, store.stateDir);
96
+ bumpDedupStats(true, store.stateDir);
97
+ const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
98
+ store.record("L1", "deduped", "l1MinHash", Date.now() - t0, 1, l1.checkpointId);
99
+ // Verify tier: the trigram gate is a boolean, so no score is invented.
100
+ audit.deduped("L1", l1.checkpointId, "l1MinHash");
101
+ onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
102
+ return r;
103
+ }
104
+ if (l1 && cfg.MARK_ONLY_L1)
105
+ markOnly = "L1";
106
+ }
107
+ onTier?.({ tier: "L1", status: "passed" });
108
+ // 3. L2 semantic dedup — catches near-identical / semantically-similar regions
109
+ // via cosine over the embedding. topicSummary is used for summaryHash dedup
110
+ // (tier 2); the vector index is keyed on the original region for backward-
111
+ // compat search semantics. Threshold from cfg (L2_COSINE trigram honest
112
+ // firing point). QA #13 timeout guard: if the O(n) scan exceeds the budget,
113
+ // degrade to "store without dedup this pass" so we never lose a checkpoint.
114
+ // Gated by L2_ENABLED (Sprint 14); MARK_ONLY_L2 records but doesn't collapse.
115
+ const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
116
+ const simThreshold = store.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
117
+ const embedding = store.embedder.embed(input.regionText);
118
+ onTier?.({ tier: "L2", status: "scanning" });
119
+ if (cfg.L2_ENABLED && all.length > 0) {
120
+ const start = Date.now();
121
+ let timedOut = false;
122
+ const nearest = all.reduce((best, cp) => {
123
+ if (!timedOut && Date.now() - start > SIMILARITY_BUDGET_MS)
124
+ timedOut = true;
125
+ if (timedOut)
126
+ return best;
127
+ const sim = cosineSimilarity(embedding, cp.embedding);
128
+ return sim > best.sim ? { checkpoint: cp, sim } : best;
129
+ }, { checkpoint: all[0], sim: -1 });
130
+ if (!timedOut && nearest.sim >= simThreshold) {
131
+ if (!cfg.MARK_ONLY_L2) {
132
+ // Near-identical — update timestamp on existing checkpoint
133
+ nearest.checkpoint.timestamp = input.timestamp;
134
+ upsertCheckpoint(nearest.checkpoint, store.stateDir);
135
+ bumpDedupStats(true, store.stateDir);
136
+ // Deduped: whole original region discarded, nothing new stored.
137
+ addTokensSaved(origTokens, store.stateDir);
138
+ const r = {
139
+ checkpoint: nearest.checkpoint,
140
+ deduped: true,
141
+ reason: "contentSimilarity",
142
+ };
143
+ store.record("L2", "deduped", "contentSimilarity", Date.now() - t0, nearest.sim, nearest.checkpoint.checkpointId);
144
+ // The real cosine that cleared the threshold — the tuning datum.
145
+ audit.deduped("L2", nearest.checkpoint.checkpointId, "contentSimilarity", nearest.sim);
146
+ onTier?.({
147
+ tier: "L2",
148
+ status: "deduped",
149
+ detail: nearest.sim.toFixed(2),
150
+ });
151
+ return r;
152
+ }
153
+ markOnly = "L2";
154
+ }
155
+ onTier?.({
156
+ tier: "L2",
157
+ status: "passed",
158
+ detail: `best ${nearest.sim.toFixed(2)}`,
159
+ });
160
+ // Near-miss: how close did we come to collapsing? Only emitted when the
161
+ // scan actually completed and scored a candidate — a timed-out scan has no
162
+ // honest best to report.
163
+ if (!timedOut && nearest.sim >= 0) {
164
+ audit.passed("L2", nearest.checkpoint.checkpointId, nearest.sim);
165
+ }
166
+ }
167
+ // 4. Genuinely new — create checkpoint
168
+ const checkpointId = nextCheckpointId(sessionId, store.stateDir);
169
+ const checkpoint = {
170
+ checkpointId,
171
+ sessionId,
172
+ repoId: store.repoId,
173
+ summary: input.summary,
174
+ topicSummary: input.topicSummary,
175
+ summaryHash,
176
+ keyDecisions: input.keyDecisions ?? [],
177
+ nextSteps: input.nextSteps ?? [],
178
+ filesModified: input.filesModified ?? [],
179
+ tokenEstimate: input.tokenEstimate ?? 0,
180
+ originalTokenEstimate: input.originalTokenEstimate,
181
+ regionHash,
182
+ contentHash: digest.contentHash,
183
+ contentHash2: digest.contentHash2,
184
+ contentHashVersion: digest.contentHashVersion,
185
+ normalizedText: digest.normalizedText,
186
+ compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8"), input.compressionPressure),
187
+ embedding,
188
+ timestamp: input.timestamp,
189
+ };
190
+ // Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
191
+ // idempotent-by-id semantics the old JSON append implied.
192
+ upsertCheckpoint(checkpoint, store.stateDir);
193
+ // Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
194
+ // checkpoint the saved amount is (original − stored); for a deduped add the
195
+ // whole original region is discarded (handled in the deduped return paths
196
+ // above). Survives sessions and travels with the repo.
197
+ const stored = input.tokenEstimate ?? 0;
198
+ addTokensSaved(Math.max(0, origTokens - stored), store.stateDir);
199
+ // L1: persist this checkpoint's MinHash signature + LSH buckets so future
200
+ // near-duplicate inserts can find it. Deterministic given the seed.
201
+ const sig = minhashSignature(input.regionText);
202
+ upsertMinhashSignature(checkpointId, sessionId, SIGNATURE_VERSION, sig, store.stateDir);
203
+ insertLshBuckets(checkpointId, sessionId, SIGNATURE_VERSION, lshBands(sig, sessionId, SIGNATURE_VERSION), store.stateDir);
204
+ // Bloom accelerator: record the new content_hash so a future add() can short-
205
+ // circuit the scan on a hit (still confirmed by the SELECT-based `all` above).
206
+ bloom.add(digest.contentHash);
207
+ saveBloom(store.stateDir);
208
+ // Track the region hash in session state for fast sentinel checks.
209
+ const state = loadSessionState(sessionId, store.stateDir);
210
+ if (!state.storedRegionHashes.includes(regionHash)) {
211
+ state.storedRegionHashes.push(regionHash);
212
+ saveSessionState(sessionId, state, store.stateDir);
213
+ }
214
+ // A new checkpoint. If a tier matched while MARK_ONLY, record that (the
215
+ // decision fired but we intentionally did not collapse).
216
+ if (markOnly) {
217
+ store.record(markOnly, "mark_only", "mark_only", Date.now() - t0);
218
+ }
219
+ else {
220
+ store.record("L0", "new", undefined, Date.now() - t0);
221
+ }
222
+ // Final outcome. `mark_only` distinguishes "a tier matched but policy said
223
+ // store anyway" from a genuinely novel region.
224
+ audit.stored(checkpointId, markOnly ? "mark_only" : "new", stored);
225
+ // Cumulative store-wide dedup accounting (attempt, not collapsed).
226
+ bumpDedupStats(false, store.stateDir);
227
+ onTier?.({ tier: "new", status: "stored" });
228
+ return { checkpoint, deduped: false };
229
+ }