pi-mega-compact 0.13.6 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -6
- package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js +6 -0
- package/dist/extensions/dashboard-server/index-reader.js +50 -25
- package/dist/extensions/dashboard-server/resolve-snapshot.js +81 -0
- package/dist/extensions/dashboard-server/routes-core.js +8 -3
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +7 -0
- package/dist/extensions/dashboard-server/routes-raptor.js +60 -0
- package/dist/extensions/dashboard-server/routes-repo.js +5 -1
- package/dist/extensions/dashboard-server/routes-sessions.js +6 -3
- package/dist/extensions/dashboard-server/routes.js +1 -1
- package/dist/extensions/dashboard-server/server.js +3 -1
- package/dist/extensions/mega-pipeline/recall.js +24 -16
- package/dist/extensions/mega-runtime/capture-model.js +5 -2
- package/dist/src/config/dedup.js +2 -1
- package/dist/src/costApi.js +113 -0
- package/dist/src/dedup/raptor/incremental.js +31 -3
- package/dist/src/dedup/raptor/index.js +10 -6
- package/dist/src/pricing.js +8 -0
- package/dist/src/vector-search.js +3 -3
- package/extensions/dashboard-client/dist/assets/{AreaChart-KAmp_6bL.js → AreaChart-C68tGr2J.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AreaChart-KAmp_6bL.js.map → AreaChart-C68tGr2J.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{CacheTab-Bk-gm-R0.js → CacheTab-CDyZqMO-.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{CacheTab-Bk-gm-R0.js.map → CacheTab-CDyZqMO-.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/EventsTab-ByPYMJVG.js +2 -0
- package/extensions/dashboard-client/dist/assets/EventsTab-ByPYMJVG.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{HealthTab-D-IzmV29.js → HealthTab-CeHQR0py.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{HealthTab-D-IzmV29.js.map → HealthTab-CeHQR0py.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CrlkCFVj.js → MaintenanceTab-ceTD_c1A.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CrlkCFVj.js.map → MaintenanceTab-ceTD_c1A.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-CRUdVR7u.js +2 -0
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-DHUdZ7RR.js.map → MemoryMapTab-CRUdVR7u.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MetricsTab-B7I4HWVg.js → MetricsTab-DP7Kmdu9.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-B7I4HWVg.js.map → MetricsTab-DP7Kmdu9.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-K6j4ZI7u.js → OverviewTab-4CGP0djv.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-K6j4ZI7u.js.map → OverviewTab-4CGP0djv.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-BSRWhn56.js → ReposTab-DUrP7Efa.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-BSRWhn56.js.map → ReposTab-DUrP7Efa.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BXdizHN1.js → SessionsTab-Cg-IsijD.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BXdizHN1.js.map → SessionsTab-Cg-IsijD.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-CAgyfqY3.js → SetupTab-ZE1daXjN.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-CAgyfqY3.js.map → SetupTab-ZE1daXjN.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-DwgCzPD9.js → TimeSavedCard-BkTgi3Af.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-DwgCzPD9.js.map → TimeSavedCard-BkTgi3Af.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CVKcStBp.js → TurnsTab-CZvudHQF.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-CVKcStBp.js.map → TurnsTab-CZvudHQF.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{WikiTab-BNohxTbs.js → WikiTab-BVAnGp8_.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{WikiTab-BNohxTbs.js.map → WikiTab-BVAnGp8_.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{button-B4A-_dCA.js → button-CUrzhi2A.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{button-B4A-_dCA.js.map → button-CUrzhi2A.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{card-Cjn7Ifoe.js → card-BsfojHfp.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{card-Cjn7Ifoe.js.map → card-BsfojHfp.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-CQjCLvv7.js → index-OM3w-l2T.js} +11 -11
- package/extensions/dashboard-client/dist/assets/index-OM3w-l2T.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/pricing-DkYp8-vi.js +2 -0
- package/extensions/dashboard-client/dist/assets/pricing-DkYp8-vi.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{switch-CcdXX0Zm.js → switch-qSz2Un3t.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{switch-CcdXX0Zm.js.map → switch-qSz2Un3t.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{toggle-KdBnFLu3.js → toggle-DyBKcLe-.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{toggle-KdBnFLu3.js.map → toggle-DyBKcLe-.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-B6t44vzc.js → useSSE-DqVouqww.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-B6t44vzc.js.map → useSSE-DqVouqww.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-client/src/api/client.ts +10 -0
- package/extensions/dashboard-client/src/components/EventStream.tsx +1 -1
- package/extensions/dashboard-client/src/tabs/MemoryMapTab/RaptorTreeView.tsx +58 -4
- package/extensions/dashboard-server/api-contracts/endpoints/registry.ts +9 -1
- package/extensions/dashboard-server/api-contracts/index.ts +2 -2
- package/extensions/dashboard-server/api-contracts/raptor.ts +19 -0
- package/extensions/dashboard-server/index-reader.ts +29 -4
- package/extensions/dashboard-server/resolve-snapshot.ts +98 -0
- package/extensions/dashboard-server/routes-core.ts +8 -3
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +17 -0
- package/extensions/dashboard-server/routes-raptor.ts +88 -1
- package/extensions/dashboard-server/routes-repo.ts +5 -1
- package/extensions/dashboard-server/routes-sessions.ts +6 -3
- package/extensions/dashboard-server/routes.ts +1 -1
- package/extensions/dashboard-server/server.ts +2 -0
- package/extensions/mega-pipeline/recall.ts +30 -21
- package/extensions/mega-runtime/capture-model.ts +5 -2
- package/package.json +1 -1
- package/src/config/dedup.ts +2 -1
- package/src/costApi.ts +131 -0
- package/src/dedup/raptor/incremental.ts +48 -3
- package/src/dedup/raptor/index.ts +10 -6
- package/src/pricing.ts +8 -0
- package/src/vector-search.ts +3 -3
- package/extensions/dashboard-client/dist/assets/EventsTab-Cyylo0of.js +0 -2
- package/extensions/dashboard-client/dist/assets/EventsTab-Cyylo0of.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-DHUdZ7RR.js +0 -2
- package/extensions/dashboard-client/dist/assets/index-CQjCLvv7.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/pricing-CvE4NllQ.js +0 -2
- package/extensions/dashboard-client/dist/assets/pricing-CvE4NllQ.js.map +0 -1
|
@@ -241,8 +241,8 @@ export type {
|
|
|
241
241
|
GraphValidationReport,
|
|
242
242
|
} from "./memory-map.js";
|
|
243
243
|
|
|
244
|
-
// Part B — RAPTOR tree
|
|
245
|
-
export type { RaptorNodeDTO, RaptorTreeResponse } from "./raptor.js";
|
|
244
|
+
// Part B — RAPTOR tree + build history
|
|
245
|
+
export type { RaptorNodeDTO, RaptorTreeResponse, RaptorBuildHistoryDTO, RaptorBuildHistoryResponse } from "./raptor.js";
|
|
246
246
|
|
|
247
247
|
// A3 — cache stripe distribution + health
|
|
248
248
|
export type {
|
|
@@ -47,3 +47,22 @@ export interface RaptorTreeResponse {
|
|
|
47
47
|
/** True when there are no RAPTOR nodes for the resolved session. */
|
|
48
48
|
readonly empty: boolean;
|
|
49
49
|
}
|
|
50
|
+
|
|
51
|
+
/** A single build-history row (client-safe). */
|
|
52
|
+
export interface RaptorBuildHistoryDTO {
|
|
53
|
+
readonly buildId: string;
|
|
54
|
+
readonly sessionId: string;
|
|
55
|
+
readonly startedAt: number;
|
|
56
|
+
readonly completedAt: number;
|
|
57
|
+
readonly nodeCount: number;
|
|
58
|
+
readonly leafCount: number;
|
|
59
|
+
readonly depth: number;
|
|
60
|
+
readonly coherenceScore: number | null;
|
|
61
|
+
readonly timedOut: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Response body for GET /api/raptor-build-history. */
|
|
65
|
+
export interface RaptorBuildHistoryResponse {
|
|
66
|
+
readonly builds: RaptorBuildHistoryDTO[];
|
|
67
|
+
readonly empty: boolean;
|
|
68
|
+
}
|
|
@@ -17,6 +17,8 @@ import type { IndexRepo, IndexIndex } from "./types.js";
|
|
|
17
17
|
import {
|
|
18
18
|
readProviderCacheForRepo,
|
|
19
19
|
} from "../../src/store/sqlite/perf-samples.js";
|
|
20
|
+
import { lookupModelInputRate } from "../../src/pricing.js";
|
|
21
|
+
import { costApiConfig, lookupCostApiPricing } from "../../src/costApi.js";
|
|
20
22
|
|
|
21
23
|
export function getIndexDir(): string {
|
|
22
24
|
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
@@ -40,7 +42,29 @@ export function readIndex(): IndexIndex | null {
|
|
|
40
42
|
const rows = db
|
|
41
43
|
.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC")
|
|
42
44
|
.all() as Record<string, unknown>[];
|
|
43
|
-
|
|
45
|
+
// Lazily fetch cost-API pricing once per readIndex call (cached for 1 hour
|
|
46
|
+
// inside costApi.ts). Avoids per-row network calls.
|
|
47
|
+
const costApi = costApiConfig();
|
|
48
|
+
const mapped: IndexRepo[] = rows.map((r) => {
|
|
49
|
+
const modelName = String(r.model_name ?? "");
|
|
50
|
+
const localRate = (r.input_rate as number | null) ||
|
|
51
|
+
lookupModelInputRate(modelName) ||
|
|
52
|
+
null;
|
|
53
|
+
let inputRate = localRate;
|
|
54
|
+
let outputRate = (r.output_rate as number | null) ?? null;
|
|
55
|
+
// Fall back to the cost API when the local table has no data.
|
|
56
|
+
if (costApi.enabled && (inputRate == null || outputRate == null) && modelName) {
|
|
57
|
+
const apiPricing = lookupCostApiPricing(modelName);
|
|
58
|
+
if (apiPricing) {
|
|
59
|
+
if (inputRate == null && apiPricing.inputRate != null) {
|
|
60
|
+
inputRate = apiPricing.inputRate;
|
|
61
|
+
}
|
|
62
|
+
if (outputRate == null && apiPricing.outputRate != null) {
|
|
63
|
+
outputRate = apiPricing.outputRate;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
44
68
|
repoRoot: String(r.repo_root ?? ""),
|
|
45
69
|
displayName: String(r.display_name ?? ""),
|
|
46
70
|
stateDir: String(r.state_dir ?? ""),
|
|
@@ -51,8 +75,8 @@ export function readIndex(): IndexIndex | null {
|
|
|
51
75
|
provider: (r.provider as string | null) ?? null,
|
|
52
76
|
providerName: (r.provider_name as string | null) ?? null,
|
|
53
77
|
modelName: (r.model_name as string | null) ?? null,
|
|
54
|
-
inputRate
|
|
55
|
-
outputRate
|
|
78
|
+
inputRate,
|
|
79
|
+
outputRate,
|
|
56
80
|
lastSeen: Number(r.last_seen ?? 0),
|
|
57
81
|
// Defaults — enriched below from each repo's own store.
|
|
58
82
|
tokensKept: 0,
|
|
@@ -64,7 +88,8 @@ export function readIndex(): IndexIndex | null {
|
|
|
64
88
|
providerCachePct: null,
|
|
65
89
|
providerCacheRead: null,
|
|
66
90
|
providerCacheWrite: null,
|
|
67
|
-
}
|
|
91
|
+
};
|
|
92
|
+
});
|
|
68
93
|
// Enrich each repo with per-store token + model detail read directly via
|
|
69
94
|
// node:sqlite (same zero-dependency invariant as readIndex; no store graph
|
|
70
95
|
// import). Best-effort: a missing/corrupt store degrades to the defaults
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dashboard-server/resolve-snapshot.ts — dynamic per-repo snapshot resolver.
|
|
3
|
+
*
|
|
4
|
+
* The dashboard server is launched with a single static stateDir, but the
|
|
5
|
+
* active pi session may be running in a different repo. This module queries
|
|
6
|
+
* the repo registry to find the most recently active repo's dashboard.json,
|
|
7
|
+
* so the Overview always shows live data from the repo pi is actually running
|
|
8
|
+
* in — not the stale launcher dir.
|
|
9
|
+
*
|
|
10
|
+
* Resolution order:
|
|
11
|
+
* 1. Most recently seen repo (by last_seen in repo_registry) whose
|
|
12
|
+
* dashboard.json exists and has a non-null updatedAt
|
|
13
|
+
* 2. The static launch stateDir's dashboard.json (original behavior)
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { existsSync } from "node:fs";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { createRequire } from "node:module";
|
|
19
|
+
import { readSnapshot } from "./snapshot.js";
|
|
20
|
+
import type { Snapshot } from "./types.js";
|
|
21
|
+
import { getIndexDir } from "./index-reader.js";
|
|
22
|
+
|
|
23
|
+
export interface ResolvedSnapshot {
|
|
24
|
+
snapshot: Snapshot;
|
|
25
|
+
/** The stateDir that produced this snapshot. */
|
|
26
|
+
stateDir: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const require = createRequire(import.meta.url);
|
|
30
|
+
|
|
31
|
+
interface RepoRegistryRow {
|
|
32
|
+
state_dir: string;
|
|
33
|
+
last_seen: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Find the most recently active repo's stateDir by scanning the repo registry
|
|
38
|
+
* for repos whose dashboard.json exists and was written recently. Returns null
|
|
39
|
+
* if no suitable repo is found.
|
|
40
|
+
*/
|
|
41
|
+
function mostRecentRepoStateDir(staticStateDir: string): string | null {
|
|
42
|
+
const indexPath = join(getIndexDir(), "index.sqlite");
|
|
43
|
+
if (!existsSync(indexPath)) return null;
|
|
44
|
+
let db;
|
|
45
|
+
try {
|
|
46
|
+
const { DatabaseSync } = require("node:sqlite") as typeof import("node:sqlite");
|
|
47
|
+
db = new DatabaseSync(indexPath, { readOnly: true });
|
|
48
|
+
const rows = db
|
|
49
|
+
.prepare(
|
|
50
|
+
`SELECT state_dir, last_seen FROM repo_registry
|
|
51
|
+
WHERE state_dir IS NOT NULL AND state_dir != ?
|
|
52
|
+
ORDER BY last_seen DESC`,
|
|
53
|
+
)
|
|
54
|
+
.all(staticStateDir) as unknown as RepoRegistryRow[];
|
|
55
|
+
for (const r of rows) {
|
|
56
|
+
const dashPath = join(r.state_dir, "dashboard.json");
|
|
57
|
+
if (!existsSync(dashPath)) continue;
|
|
58
|
+
const snap = readSnapshot(dashPath);
|
|
59
|
+
if (snap.updatedAt) {
|
|
60
|
+
return r.state_dir;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
} catch {
|
|
65
|
+
return null;
|
|
66
|
+
} finally {
|
|
67
|
+
try {
|
|
68
|
+
db?.close();
|
|
69
|
+
} catch {
|
|
70
|
+
/* ignore */
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Resolve the best snapshot to serve. Tries the most recently active repo's
|
|
77
|
+
* dashboard.json first, then falls back to the static launch path.
|
|
78
|
+
*/
|
|
79
|
+
export function resolveSnapshot(
|
|
80
|
+
staticSnapshotPath: string,
|
|
81
|
+
staticStateDir: string,
|
|
82
|
+
): ResolvedSnapshot {
|
|
83
|
+
// Try the most recently active repo first.
|
|
84
|
+
const activeDir = mostRecentRepoStateDir(staticStateDir);
|
|
85
|
+
if (activeDir) {
|
|
86
|
+
const activePath = join(activeDir, "dashboard.json");
|
|
87
|
+
const snap = readSnapshot(activePath);
|
|
88
|
+
if (snap.updatedAt) {
|
|
89
|
+
return { snapshot: snap, stateDir: activeDir };
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// Fall back to the static launch path.
|
|
93
|
+
return {
|
|
94
|
+
snapshot: readSnapshot(staticSnapshotPath),
|
|
95
|
+
stateDir: staticStateDir,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import type { ServerResponse } from "node:http";
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { resolveSnapshot } from "./resolve-snapshot.js";
|
|
12
12
|
import type { IndexIndex, Snapshot } from "./types.js";
|
|
13
13
|
|
|
14
14
|
// ---------------------------------------------------------------------------
|
|
@@ -53,14 +53,19 @@ function makeOverlayCurrentRepo(
|
|
|
53
53
|
): (idx: IndexIndex | null) => void {
|
|
54
54
|
return (idx: IndexIndex | null): void => {
|
|
55
55
|
if (!idx || !idx.repos.length) return;
|
|
56
|
+
// Dynamically resolve the most recently active repo's snapshot so the
|
|
57
|
+
// overlay matches what /api/snapshot serves (not the stale launch dir).
|
|
56
58
|
let snap: Snapshot | null = null;
|
|
59
|
+
let activeStateDir = stateDir;
|
|
57
60
|
try {
|
|
58
|
-
|
|
61
|
+
const resolved = resolveSnapshot(snapshotPath, stateDir);
|
|
62
|
+
snap = resolved.snapshot;
|
|
63
|
+
activeStateDir = resolved.stateDir;
|
|
59
64
|
} catch {
|
|
60
65
|
return;
|
|
61
66
|
}
|
|
62
67
|
if (!snap || !snap.repo) return;
|
|
63
|
-
const cur = idx.repos.find((r) => r.stateDir ===
|
|
68
|
+
const cur = idx.repos.find((r) => r.stateDir === activeStateDir);
|
|
64
69
|
if (!cur) return;
|
|
65
70
|
const prevSaved = cur.tokensSaved;
|
|
66
71
|
const prevCp = cur.checkpointCount;
|
|
@@ -272,6 +272,23 @@ export const SETTINGS: ReadonlyArray<{
|
|
|
272
272
|
num("MEGACOMPACT_EMBED_CACHE", "Embed Cache Size", "Embedding cache entries (0 = disabled)", 256, 0, 10000),
|
|
273
273
|
],
|
|
274
274
|
},
|
|
275
|
+
{
|
|
276
|
+
name: "Cost API",
|
|
277
|
+
settings: [
|
|
278
|
+
boolDirect(
|
|
279
|
+
"MEGACOMPACT_COST_API_ENABLED",
|
|
280
|
+
"Cost API Lookup",
|
|
281
|
+
"Fetch model pricing from an external API (PREVENT-PI-004 network exception — opt-in). Enriches the dashboard Cache/Repos tabs with real $/token rates for models not in the local pricing table.",
|
|
282
|
+
false,
|
|
283
|
+
),
|
|
284
|
+
str(
|
|
285
|
+
"MEGACOMPACT_COST_API_URL",
|
|
286
|
+
"Cost API URL",
|
|
287
|
+
"OpenRouter-compatible model pricing endpoint (e.g. https://openrouter.ai/api/v1/models). Only contacted when Cost API Lookup is ON.",
|
|
288
|
+
"",
|
|
289
|
+
),
|
|
290
|
+
],
|
|
291
|
+
},
|
|
275
292
|
];
|
|
276
293
|
|
|
277
294
|
/** Flat index of every setting by key for O(1) lookup + validation. */
|
|
@@ -10,7 +10,7 @@ import { createRequire } from "node:module";
|
|
|
10
10
|
import type { ServerResponse, IncomingMessage } from "node:http";
|
|
11
11
|
import { parse as parseUrl } from "node:url";
|
|
12
12
|
import type { RouteContext } from "./routes-core.js";
|
|
13
|
-
import type { RaptorTreeResponse, RaptorNodeDTO } from "./api-contracts/raptor.js";
|
|
13
|
+
import type { RaptorTreeResponse, RaptorNodeDTO, RaptorBuildHistoryResponse, RaptorBuildHistoryDTO } from "./api-contracts/raptor.js";
|
|
14
14
|
import { openStore } from "../../src/store/sqlite.js";
|
|
15
15
|
|
|
16
16
|
const _req = createRequire(import.meta.url);
|
|
@@ -126,3 +126,90 @@ export function handleRaptorTree(
|
|
|
126
126
|
return true;
|
|
127
127
|
}
|
|
128
128
|
}
|
|
129
|
+
|
|
130
|
+
/** Find the most recent session that has build history rows (else null). */
|
|
131
|
+
function latestBuildSession(stateDir: string): string | null {
|
|
132
|
+
try {
|
|
133
|
+
const db = openStore(stateDir);
|
|
134
|
+
const row = db
|
|
135
|
+
.prepare(
|
|
136
|
+
"SELECT DISTINCT session_id FROM raptor_build_history ORDER BY completed_at DESC LIMIT 1",
|
|
137
|
+
)
|
|
138
|
+
.get() as { session_id: string } | undefined;
|
|
139
|
+
return row?.session_id ?? null;
|
|
140
|
+
} catch {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function handleRaptorBuildHistory(
|
|
146
|
+
req: IncomingMessage,
|
|
147
|
+
res: ServerResponse,
|
|
148
|
+
ctx: RouteContext,
|
|
149
|
+
): boolean {
|
|
150
|
+
if (req.method !== "GET") return false;
|
|
151
|
+
const parsed = parseUrl(req.url ?? "", true);
|
|
152
|
+
if (parsed.pathname !== "/api/raptor-build-history") return false;
|
|
153
|
+
|
|
154
|
+
try {
|
|
155
|
+
const { listBuildHistory } = _req(
|
|
156
|
+
"../../src/dedup/raptor/buildHistory.js",
|
|
157
|
+
) as {
|
|
158
|
+
listBuildHistory: (
|
|
159
|
+
sessionId: string,
|
|
160
|
+
stateDir: string,
|
|
161
|
+
) => Array<{
|
|
162
|
+
buildId: string;
|
|
163
|
+
sessionId: string;
|
|
164
|
+
startedAt: number;
|
|
165
|
+
completedAt: number;
|
|
166
|
+
nodeCount: number;
|
|
167
|
+
leafCount: number;
|
|
168
|
+
depth: number;
|
|
169
|
+
coherenceScore: number | null;
|
|
170
|
+
timedOut: boolean;
|
|
171
|
+
}>;
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
let sessionId =
|
|
175
|
+
typeof parsed.query.sessionId === "string" ? parsed.query.sessionId : "";
|
|
176
|
+
|
|
177
|
+
if (!sessionId) {
|
|
178
|
+
sessionId = latestBuildSession(ctx.stateDir) ?? "";
|
|
179
|
+
if (!sessionId) {
|
|
180
|
+
const body: RaptorBuildHistoryResponse = { builds: [], empty: true };
|
|
181
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
182
|
+
res.end(JSON.stringify(body));
|
|
183
|
+
return true;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const rows = listBuildHistory(sessionId, ctx.stateDir);
|
|
188
|
+
if (rows.length === 0) {
|
|
189
|
+
const body: RaptorBuildHistoryResponse = { builds: [], empty: true };
|
|
190
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
191
|
+
res.end(JSON.stringify(body));
|
|
192
|
+
return true;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const dtos: RaptorBuildHistoryDTO[] = rows.map((r) => ({
|
|
196
|
+
buildId: r.buildId,
|
|
197
|
+
sessionId: r.sessionId,
|
|
198
|
+
startedAt: r.startedAt,
|
|
199
|
+
completedAt: r.completedAt,
|
|
200
|
+
nodeCount: r.nodeCount,
|
|
201
|
+
leafCount: r.leafCount,
|
|
202
|
+
depth: r.depth,
|
|
203
|
+
coherenceScore: r.coherenceScore,
|
|
204
|
+
timedOut: r.timedOut,
|
|
205
|
+
}));
|
|
206
|
+
const body: RaptorBuildHistoryResponse = { builds: dtos, empty: false };
|
|
207
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
208
|
+
res.end(JSON.stringify(body));
|
|
209
|
+
return true;
|
|
210
|
+
} catch (e) {
|
|
211
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
212
|
+
res.end(JSON.stringify({ error: String(e) }));
|
|
213
|
+
return true;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
@@ -7,6 +7,7 @@ import { join } from "node:path";
|
|
|
7
7
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
8
8
|
|
|
9
9
|
import { readSnapshot } from "./snapshot.js";
|
|
10
|
+
import { resolveSnapshot } from "./resolve-snapshot.js";
|
|
10
11
|
import { dashboardHtml } from "./html.js";
|
|
11
12
|
import { readIndex, getIndexDir } from "./index-reader.js";
|
|
12
13
|
import { ACTIVE_WINDOW_SEC } from "./types.js";
|
|
@@ -41,7 +42,10 @@ export function handleIndex(
|
|
|
41
42
|
}
|
|
42
43
|
|
|
43
44
|
if (req.url === "/api/snapshot") {
|
|
44
|
-
|
|
45
|
+
// Dynamically resolve the most recently active session's dashboard.json
|
|
46
|
+
// so the Overview shows live data from the repo pi is running in — not
|
|
47
|
+
// the stale launcher dir the server was started with.
|
|
48
|
+
const { snapshot: snap } = resolveSnapshot(snapshotPath, ctx.stateDir);
|
|
45
49
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
46
50
|
res.end(JSON.stringify(snap));
|
|
47
51
|
return true;
|
|
@@ -28,10 +28,13 @@ export function handleEvents(
|
|
|
28
28
|
Connection: "keep-alive",
|
|
29
29
|
});
|
|
30
30
|
|
|
31
|
-
// Drain existing events so the client starts with history
|
|
31
|
+
// Drain existing events so the client starts with history.
|
|
32
|
+
// Only stream events with a "type" field — events.log also contains
|
|
33
|
+
// internal monitoring rows (e.g. captureModel:recorded) that use "event"
|
|
34
|
+
// instead of "type" and aren't part of the SseEvent contract.
|
|
32
35
|
const { data: existing, offset: initialOffset } = readFrom(eventsPath, 0);
|
|
33
36
|
eventOffsetRef.value = initialOffset;
|
|
34
|
-
const lines = existing.split("\n").filter((l: string) => l.trim());
|
|
37
|
+
const lines = existing.split("\n").filter((l: string) => l.trim() && l.includes('"type"'));
|
|
35
38
|
for (const line of lines) {
|
|
36
39
|
res.write(`data: ${line}\n\n`);
|
|
37
40
|
}
|
|
@@ -44,7 +47,7 @@ export function handleEvents(
|
|
|
44
47
|
watchTimer = null;
|
|
45
48
|
const { data, offset } = readFrom(eventsPath, eventOffsetRef.value);
|
|
46
49
|
eventOffsetRef.value = offset;
|
|
47
|
-
const newLines = data.split("\n").filter((l: string) => l.trim());
|
|
50
|
+
const newLines = data.split("\n").filter((l: string) => l.trim() && l.includes('"type"'));
|
|
48
51
|
for (const line of newLines) {
|
|
49
52
|
res.write(`data: ${line}\n\n`);
|
|
50
53
|
}
|
|
@@ -23,7 +23,7 @@ export { handleProviderCache } from "./routes-cache.js";
|
|
|
23
23
|
export { handleMemoryStatus } from "./routes-memory.js";
|
|
24
24
|
export { handleSetupStatus, handleSetupDetect, handleSetupConfigure } from "./routes-setup.js";
|
|
25
25
|
export { handleMemoryMap } from "./routes-memory-map.js";
|
|
26
|
-
export { handleRaptorTree } from "./routes-raptor.js";
|
|
26
|
+
export { handleRaptorTree, handleRaptorBuildHistory } from "./routes-raptor.js";
|
|
27
27
|
export { handleCacheStripes } from "./routes-cache.js";
|
|
28
28
|
export { handleContextHealth, handleCachePoison, handleHealthSettings } from "./routes-health.js";
|
|
29
29
|
export { handleEmbedderHealth } from "./routes-embedder-health.js";
|
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
handleSetupConfigure,
|
|
47
47
|
handleMemoryMap,
|
|
48
48
|
handleRaptorTree,
|
|
49
|
+
handleRaptorBuildHistory,
|
|
49
50
|
handleContextHealth,
|
|
50
51
|
handleCachePoison,
|
|
51
52
|
handleHealthSettings,
|
|
@@ -260,6 +261,7 @@ export async function launchDashboardServer(
|
|
|
260
261
|
if (handleSetupConfigure(req, res, ctx)) return;
|
|
261
262
|
if (handleMemoryMap(req, res, ctx)) return;
|
|
262
263
|
if (handleRaptorTree(req, res, ctx)) return;
|
|
264
|
+
if (handleRaptorBuildHistory(req, res, ctx)) return;
|
|
263
265
|
if (handleContextHealth(req, res, ctx)) return;
|
|
264
266
|
if (handleCachePoison(req, res, ctx)) return;
|
|
265
267
|
if (handleHealthSettings(req, res, ctx)) return;
|
|
@@ -107,16 +107,23 @@ export function doRecall(
|
|
|
107
107
|
);
|
|
108
108
|
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
109
109
|
}
|
|
110
|
+
let sumTokens = 0;
|
|
111
|
+
for (const h of result.toInject) sumTokens += h.checkpoint.tokenEstimate;
|
|
110
112
|
if (result.toInject.length > 0) {
|
|
111
|
-
let sumTokens = 0;
|
|
112
|
-
for (const h of result.toInject) sumTokens += h.checkpoint.tokenEstimate;
|
|
113
113
|
runtime.rt.recallInjections += result.toInject.length;
|
|
114
114
|
runtime.rt.cacheHitTokens += sumTokens;
|
|
115
115
|
incRecallInjected(result.toInject.length, runtime.currentStateDir);
|
|
116
116
|
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
}
|
|
118
|
+
// S43: record recall provenance — which checkpoints/summaries served this
|
|
119
|
+
// turn, their score + source path. Linked to the turn row written at
|
|
120
|
+
// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
|
|
121
|
+
// Persists telemetry (HyDE + recall metrics) even when recall returned
|
|
122
|
+
// no hits, so empty-recall HyDE invocations are still visible in the
|
|
123
|
+
// dashboard Turns/Metrics tabs.
|
|
124
|
+
const hasTelemetry =
|
|
125
|
+
result.hydeInfo != null || result.recallMetrics != null;
|
|
126
|
+
if (result.toInject.length > 0 || hasTelemetry) {
|
|
120
127
|
try {
|
|
121
128
|
const convId = ensureConversationIdFor(
|
|
122
129
|
config,
|
|
@@ -136,22 +143,24 @@ export function doRecall(
|
|
|
136
143
|
},
|
|
137
144
|
runtime.currentStateDir,
|
|
138
145
|
);
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
146
|
+
if (result.toInject.length > 0) {
|
|
147
|
+
recordRecallWrite(
|
|
148
|
+
config,
|
|
149
|
+
turnId,
|
|
150
|
+
result.toInject.map((h) => ({
|
|
151
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
152
|
+
score: h.score,
|
|
153
|
+
source:
|
|
154
|
+
h.raptorLevel !== undefined
|
|
155
|
+
? "raptor"
|
|
156
|
+
: h.repoId
|
|
157
|
+
? "cross-repo"
|
|
158
|
+
: "flat",
|
|
159
|
+
raptorLevel: h.raptorLevel,
|
|
160
|
+
})),
|
|
161
|
+
runtime.currentStateDir,
|
|
162
|
+
);
|
|
163
|
+
}
|
|
155
164
|
} catch {
|
|
156
165
|
/* non-fatal: recall provenance never breaks the recall path */
|
|
157
166
|
}
|
|
@@ -10,6 +10,7 @@ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
|
10
10
|
import { resolveRepoRoot } from "../mega-config.js";
|
|
11
11
|
import { recordModelSnapshot, recordRepoModel } from "../../src/store/sqlite.js";
|
|
12
12
|
import type { ModelSnapshot } from "../../src/store/sqlite.js";
|
|
13
|
+
import { lookupModelInputRate } from "../../src/pricing.js";
|
|
13
14
|
|
|
14
15
|
// ---------------------------------------------------------------------- types
|
|
15
16
|
|
|
@@ -42,12 +43,14 @@ export function captureModelImpl(ctx: CaptureModelContext, ectx: ExtensionContex
|
|
|
42
43
|
} catch {
|
|
43
44
|
/* optional */
|
|
44
45
|
}
|
|
46
|
+
const modelId = m.id;
|
|
47
|
+
const fallbackInput = lookupModelInputRate(modelId);
|
|
45
48
|
const snap: Omit<ModelSnapshot, "capturedAt"> = {
|
|
46
49
|
provider: m.provider,
|
|
47
50
|
providerName,
|
|
48
|
-
modelId
|
|
51
|
+
modelId,
|
|
49
52
|
modelName: m.name ?? null,
|
|
50
|
-
inputRate: m.cost?.input
|
|
53
|
+
inputRate: m.cost?.input || fallbackInput || 0,
|
|
51
54
|
outputRate: m.cost?.output ?? 0,
|
|
52
55
|
contextWindow: m.contextWindow ?? 0,
|
|
53
56
|
maxTokens: m.maxTokens ?? 0,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
package/src/config/dedup.ts
CHANGED
|
@@ -89,7 +89,8 @@ export function loadDedupConfig(): DedupConfigShape {
|
|
|
89
89
|
L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
|
|
90
90
|
// Fix D: RAPTOR promoted to live recall. Default ON; canary.ts sequences it
|
|
91
91
|
// last (L0→L1→L2→RAPTOR) and auto-disables on p95 breach, so promotion is
|
|
92
|
-
// safe.
|
|
92
|
+
// safe. Shadow mode (RAPTOR_SHADOW_MODE=true) opts out of serving while
|
|
93
|
+
// still building + persisting the tree (transition/eval use).
|
|
93
94
|
RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
|
|
94
95
|
MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
|
|
95
96
|
MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
|
package/src/costApi.ts
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* costApi.ts — optional model pricing lookup from an external API.
|
|
3
|
+
*
|
|
4
|
+
* PREVENT-PI-004 EXEMPTION: This module makes network calls to fetch model
|
|
5
|
+
* pricing data. It is disabled by default and must be explicitly opted in via
|
|
6
|
+
* MEGACOMPACT_COST_API_ENABLED=true + MEGACOMPACT_COST_API_URL=<url>.
|
|
7
|
+
*
|
|
8
|
+
* The dashboard server uses this to enrich the inputRate/outputRate fields
|
|
9
|
+
* when the local pricing table (pricing.ts) has no data for a model.
|
|
10
|
+
*
|
|
11
|
+
* Results are cached for 1 hour to minimize network calls.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: user-opted-in cost API — fetches model pricing from a user-configured endpoint, never called unless MEGACOMPACT_COST_API_ENABLED=true
|
|
15
|
+
|
|
16
|
+
export interface ModelPricing {
|
|
17
|
+
inputRate: number | null;
|
|
18
|
+
outputRate: number | null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface CostApiConfig {
|
|
22
|
+
enabled: boolean;
|
|
23
|
+
url: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let _cache: { timestamp: number; data: Map<string, ModelPricing> } | null = null;
|
|
27
|
+
const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
28
|
+
|
|
29
|
+
export function costApiConfig(): CostApiConfig {
|
|
30
|
+
const enabled =
|
|
31
|
+
process.env.MEGACOMPACT_COST_API_ENABLED === "true" ||
|
|
32
|
+
process.env.MEGACOMPACT_COST_API_ENABLED === "1";
|
|
33
|
+
const url = process.env.MEGACOMPACT_COST_API_URL ?? "";
|
|
34
|
+
return { enabled, url };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Fetch model pricing from the configured API endpoint.
|
|
39
|
+
*
|
|
40
|
+
* Expects a JSON response of shape:
|
|
41
|
+
* { "data": [{ "id": "model-name", "pricing": { "prompt": "0.000003", "completion": "0.000015" } }] }
|
|
42
|
+
* (OpenRouter-compatible format). Values are USD per token as strings.
|
|
43
|
+
*
|
|
44
|
+
* Returns a map keyed by lowercase model id → { inputRate, outputRate }.
|
|
45
|
+
* Returns an empty map on any error. Never throws.
|
|
46
|
+
*/
|
|
47
|
+
export function fetchModelPricing(): Map<string, ModelPricing> {
|
|
48
|
+
const cfg = costApiConfig();
|
|
49
|
+
if (!cfg.enabled || !cfg.url) return new Map();
|
|
50
|
+
|
|
51
|
+
// Return cached data if fresh.
|
|
52
|
+
if (_cache && Date.now() - _cache.timestamp < CACHE_TTL_MS) {
|
|
53
|
+
return _cache.data;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const result = _doFetch(cfg.url);
|
|
57
|
+
_cache = { timestamp: Date.now(), data: result };
|
|
58
|
+
return result;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function _doFetch(url: string): Map<string, ModelPricing> {
|
|
62
|
+
// Synchronous bridge: spawnSync an inline worker so the call blocks without
|
|
63
|
+
// deadlocking fetch (mirrors HttpEmbedder / RAPTOR summarizer pattern).
|
|
64
|
+
const WORKER = String.raw`
|
|
65
|
+
try {
|
|
66
|
+
const r = await fetch(process.env.COST_API_URL, { // guardrails-allow PREVENT-PI-004: user-opted-in cost API — fetches model pricing from a user-configured endpoint
|
|
67
|
+
method: "GET",
|
|
68
|
+
headers: { "accept": "application/json" },
|
|
69
|
+
});
|
|
70
|
+
const j = await r.json();
|
|
71
|
+
process.stdout.write(JSON.stringify({ ok: r.ok, data: j }));
|
|
72
|
+
} catch (e) {
|
|
73
|
+
process.stdout.write(JSON.stringify({ ok: false, error: String(e && e.message ? e.message : e) }));
|
|
74
|
+
}
|
|
75
|
+
`;
|
|
76
|
+
const res = spawnSync(process.execPath, ["-e", WORKER], {
|
|
77
|
+
encoding: "utf8",
|
|
78
|
+
timeout: 15_000,
|
|
79
|
+
env: { ...process.env, COST_API_URL: url },
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
const out = new Map<string, ModelPricing>();
|
|
83
|
+
|
|
84
|
+
let parsed: { ok: boolean; data?: unknown; error?: string };
|
|
85
|
+
try {
|
|
86
|
+
parsed = JSON.parse(res.stdout);
|
|
87
|
+
} catch {
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (!parsed.ok || !parsed.data) return out;
|
|
92
|
+
|
|
93
|
+
// OpenRouter-compatible: { data: [{ id, pricing: { prompt, completion } }] }
|
|
94
|
+
const models = (parsed.data as { data?: Array<{ id?: string; pricing?: { prompt?: string; completion?: string } }> })?.data;
|
|
95
|
+
if (!Array.isArray(models)) return out;
|
|
96
|
+
|
|
97
|
+
for (const m of models) {
|
|
98
|
+
if (!m.id) continue;
|
|
99
|
+
const prompt = m.pricing?.prompt;
|
|
100
|
+
const completion = m.pricing?.completion;
|
|
101
|
+
const inputRate = prompt != null ? parseFloat(prompt) : null;
|
|
102
|
+
const outputRate = completion != null ? parseFloat(completion) : null;
|
|
103
|
+
if (inputRate != null || outputRate != null) {
|
|
104
|
+
out.set(m.id.toLowerCase(), { inputRate, outputRate });
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Look up pricing for a model name from the cost API cache.
|
|
113
|
+
* Tries exact match, then prefix match (same pattern as lookupModelInputRate).
|
|
114
|
+
*/
|
|
115
|
+
export function lookupCostApiPricing(model: string): ModelPricing | null {
|
|
116
|
+
const data = fetchModelPricing();
|
|
117
|
+
if (data.size === 0) return null;
|
|
118
|
+
const lower = model.toLowerCase();
|
|
119
|
+
// Exact match.
|
|
120
|
+
if (data.has(lower)) return data.get(lower) ?? null;
|
|
121
|
+
// Prefix match.
|
|
122
|
+
for (const [key, pricing] of data) {
|
|
123
|
+
if (lower.startsWith(key)) return pricing;
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Clear the pricing cache (useful for testing). */
|
|
129
|
+
export function clearCostApiCache(): void {
|
|
130
|
+
_cache = null;
|
|
131
|
+
}
|