tokentrace 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +45 -14
- package/TOKENTRACE_AGENT.md +1 -1
- package/app/api/evidence-pack/route.ts +64 -0
- package/app/api/import-profile-preview/route.ts +26 -0
- package/app/api/operating-metadata/route.ts +13 -0
- package/app/api/reports/route.ts +72 -0
- package/app/api/scan/route.ts +11 -0
- package/app/api/settings/route.ts +3 -1
- package/app/diagnostics/page.tsx +60 -2
- package/app/discovery/page.tsx +1 -1
- package/app/evidence/page.tsx +186 -15
- package/app/guide/page.tsx +2 -2
- package/app/layout.tsx +4 -0
- package/app/loading.tsx +20 -0
- package/app/models/page.tsx +1 -1
- package/app/page.tsx +98 -7
- package/app/repair/page.tsx +23 -7
- package/app/settings/page.tsx +19 -3
- package/bin/tokentrace.js +1 -1
- package/components/charts/rank-bar-chart.tsx +19 -1
- package/components/charts/trend-chart.tsx +18 -1
- package/components/navigation-progress.tsx +74 -0
- package/components/period-filter.tsx +49 -11
- package/components/scan-health-summary.tsx +1 -1
- package/components/scan-now-button.tsx +23 -7
- package/components/session-explorer.tsx +56 -3
- package/components/settings-panel.tsx +440 -49
- package/dist/runtime/agent.mjs +3 -3
- package/dist/runtime/db-migrate.mjs +63 -2
- package/dist/runtime/db-seed.mjs +63 -2
- package/dist/runtime/digest.mjs +298 -64
- package/dist/runtime/doctor.mjs +301 -67
- package/dist/runtime/evidence.mjs +101 -28
- package/dist/runtime/insights.mjs +297 -63
- package/dist/runtime/pricing-refresh.mjs +63 -2
- package/dist/runtime/repair.mjs +102 -23
- package/dist/runtime/report.mjs +591 -69
- package/dist/runtime/reset.mjs +63 -2
- package/dist/runtime/review.mjs +297 -63
- package/dist/runtime/roadmap.mjs +205 -91
- package/dist/runtime/scan.mjs +418 -11
- package/dist/runtime/status.mjs +65 -4
- package/docs/assets/evidence-0.12.0.png +0 -0
- package/docs/assets/overview-0.12.0.png +0 -0
- package/docs/assets/repair-0.12.0.png +0 -0
- package/docs/assets/scan-health-0.12.0.png +0 -0
- package/llms.txt +1 -1
- package/package.json +1 -1
- package/scripts/report.ts +37 -3
- package/scripts/roadmap.ts +1 -1
- package/scripts/smoke-cli.mjs +2 -2
- package/scripts/smoke-packed-install.mjs +2 -2
- package/src/db/migrate-core.ts +12 -0
- package/src/db/schema.ts +51 -2
- package/src/db/settings.ts +56 -4
- package/src/ingestion/adapters/cursor-chat.ts +99 -0
- package/src/ingestion/adapters/helpers.ts +30 -1
- package/src/ingestion/adapters/index.ts +4 -0
- package/src/ingestion/adapters/structured-usage-log.ts +95 -0
- package/src/ingestion/scan.ts +2 -0
- package/src/lib/accounting-invariants.ts +1 -1
- package/src/lib/agent-discovery.ts +3 -3
- package/src/lib/analytics.ts +106 -45
- package/src/lib/daily-digest.ts +1 -1
- package/src/lib/doctor.ts +3 -3
- package/src/lib/evidence-pack.ts +119 -0
- package/src/lib/evidence-trail.ts +39 -26
- package/src/lib/first-run-status.ts +4 -4
- package/src/lib/import-profile-preview.ts +125 -0
- package/src/lib/import-profiles.ts +22 -2
- package/src/lib/live-status.ts +1 -1
- package/src/lib/operating-metadata.ts +24 -0
- package/src/lib/recommendations.ts +1 -1
- package/src/lib/report-cli.ts +18 -1
- package/src/lib/review-queue.ts +1 -1
- package/src/lib/roadmap-status.ts +234 -92
- package/src/lib/saved-reports.ts +102 -0
- package/src/lib/scan-health.ts +5 -5
- package/src/lib/scan-schedule.ts +89 -0
- package/src/lib/scheduled-scan.ts +58 -0
- package/src/lib/source-catalog.ts +148 -0
- package/src/lib/support-matrix.ts +1 -1
- package/src/lib/unknown-cost-repair.ts +56 -18
- package/src/lib/usage-guardrails.ts +106 -5
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,48 @@ All notable changes to TokenTrace are documented here.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
## [0.12.0] - 2026-05-19
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- 0.12.0 Local Sources & Trust roadmap, rolling the 0.13.0 Evidence Portability, 0.14.0 Local Operations, 0.15.0 Governance & Guardrails, 0.16.0 Parser Studio, 0.17.0 Reports, and 0.18.0 Agent Handoff themes into one larger release.
|
|
12
|
+
- Native structured usage log ingestion for local wrappers and team JSONL/NDJSON logs with session, model, token, and source-cost fields.
|
|
13
|
+
- Native Cursor-style chat/composer export ingestion with local source evidence and no raw prompt storage by default.
|
|
14
|
+
- Source Catalog and Source Coverage in Scan Health so users can distinguish native, profile-assisted, fallback, and unsupported local files.
|
|
15
|
+
- Privacy-safe Evidence Packs exported as JSON or Markdown with totals, confidence drivers, source files, parser notes, model-rate state, repair links, and an explicit redaction manifest.
|
|
16
|
+
- Local scan scheduling settings for manual-only, on-open, hourly, and daily scans, plus scan-history retention and last scheduled scan status.
|
|
17
|
+
- Scoped project/model/tool guardrails with per-guardrail warning thresholds and anomaly notes.
|
|
18
|
+
- Import Profile preview for sampling a local file, checking parser fit, reviewing detected fields, and applying recommended matchers without exposing raw content.
|
|
19
|
+
- Saved report definitions and export endpoints for weekly usage, high-cost sessions, unknown-cost repair, confidence trends, guardrail status, and source coverage in Markdown, JSON, and CSV.
|
|
20
|
+
- Operating metadata export for settings, source catalog, schedules, guardrails, report definitions, and roadmap status without raw usage records.
|
|
21
|
+
- Agent-readable Roadmap V2 with current release, next planned release, rolled-up release themes, action recipes, evidence paths, verification gates, and release status.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- Settings now includes Scan Scheduling, Scoped Guardrails, Import Profile preview, and Local Exports.
|
|
26
|
+
- Evidence pages now offer one-click JSON and Markdown evidence-pack exports.
|
|
27
|
+
- Evidence remains a contextual drill-down from Overview, Sessions, Repair, and exports, with direct-entry guidance for users who open `/evidence` manually.
|
|
28
|
+
- Scan, setup, guardrail, package-trust, folder, import-profile, and export CTAs now deep-link to the exact Settings section instead of dropping users at the top of Settings.
|
|
29
|
+
- Settings scan feedback now reports files checked, records imported, warnings, errors, recalculated costs, unknown cost, stale support imports, model aliases, and next-step links to Scan Health, Repair, Discovery, and Model Rates.
|
|
30
|
+
- Browser-triggered Scan now responses now return compact warning/error previews with full counts, keeping large duplicate-file scans from shipping megabytes of warning text back to the UI.
|
|
31
|
+
- Settings now has sticky section navigation, and Scan Controls shows a persisted Last scan result from local scan history after reload.
|
|
32
|
+
- Overview now includes a compact Last verified trust strip near the accounting totals for latest scan, package IOC, model-rate coverage, and evidence-pack availability.
|
|
33
|
+
- Evidence pages now include opened-from breadcrumbs, safe return behavior, and explicit Export JSON pack / Export Markdown pack actions.
|
|
34
|
+
- Route transitions now show a subtle top loading bar, and route loading states explain what is happening plus the next useful action.
|
|
35
|
+
- Scan and evidence follow-up actions now consistently use Scan now, Open Scan Health, Open repair, Set model rate, View evidence, and Export pack language.
|
|
36
|
+
- Period filters stay attached to the current page, preserve Evidence context, and keep desktop controls in one compact row while retaining horizontal scrolling for narrow widths.
|
|
37
|
+
- Chart cards now show lightweight loading placeholders during client hydration instead of appearing blank while Recharts initializes.
|
|
38
|
+
- Session Explorer now paginates large local result sets to keep dense session tables responsive.
|
|
39
|
+
- Repair now opens as a capped workbench of the top visible unknown-cost groups, while keeping full summary counts and focused repair links for deep review.
|
|
40
|
+
- Roadmap CLI/API now report the 0.12.0 Local Sources & Trust release contract and the next planned 0.19.0 direction.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
|
|
44
|
+
- Overview, Evidence, and Settings now avoid full raw-row scans and oversized scan-health payloads on large local databases by using covering indexes, scoped scan-file reads, and a summary session path on Overview.
|
|
45
|
+
- Repair no longer serializes the entire unknown-cost queue on first load and uses a dedicated cost-repair index for large local databases.
|
|
46
|
+
- Scan Health no longer loads all scan-file evidence and full session details just to render summary panels.
|
|
47
|
+
- Overview no longer blocks initial render on due scheduled scans, so opening localhost does not wait for scan work before showing the dashboard.
|
|
48
|
+
|
|
7
49
|
## [0.11.0] - 2026-05-18
|
|
8
50
|
|
|
9
51
|
### Added
|
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ TokenTrace is designed for local development machines first, with macOS-oriented
|
|
|
10
10
|
|
|
11
11
|
[Website](https://www.abhiyoheswaran.com/apps/tokentrace) · [Source](https://github.com/abhiyoheswaran1/tokentrace)
|
|
12
12
|
|
|
13
|
-

|
|
14
14
|
|
|
15
15
|
## Start In Seconds
|
|
16
16
|
|
|
@@ -40,7 +40,7 @@ tokentrace agent --json # Print machine-readable agent discovery manifest
|
|
|
40
40
|
tokentrace capabilities --json
|
|
41
41
|
# Alias for agent discovery manifest
|
|
42
42
|
tokentrace roadmap --json
|
|
43
|
-
# Print
|
|
43
|
+
# Print Local Sources & Trust release handoff
|
|
44
44
|
tokentrace scan # Scan local AI CLI usage logs
|
|
45
45
|
tokentrace doctor --json
|
|
46
46
|
# Inspect scan health and repair recommendations
|
|
@@ -114,7 +114,7 @@ curl http://127.0.0.1:3030/api/agent
|
|
|
114
114
|
curl http://127.0.0.1:3030/api/capabilities
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
-
The
|
|
117
|
+
The Local Sources & Trust release handoff is also machine-readable:
|
|
118
118
|
|
|
119
119
|
```bash
|
|
120
120
|
tokentrace roadmap --json
|
|
@@ -154,9 +154,34 @@ npm run security:ioc
|
|
|
154
154
|
npm run smoke:packed
|
|
155
155
|
# Inspect packed tarball and smoke test packed CLI
|
|
156
156
|
tokentrace roadmap --json
|
|
157
|
-
# Inspect
|
|
157
|
+
# Inspect roadmap handoff, action recipes, and release status
|
|
158
158
|
```
|
|
159
159
|
|
|
160
|
+
## Local Sources And Trust
|
|
161
|
+
|
|
162
|
+
TokenTrace 0.12.0 rolls the next several roadmap themes into one larger
|
|
163
|
+
release: local source expansion, evidence portability, local operations,
|
|
164
|
+
scoped guardrails, parser profile preview, saved reports, and agent-readable
|
|
165
|
+
roadmap handoff.
|
|
166
|
+
|
|
167
|
+
New trust surfaces include:
|
|
168
|
+
|
|
169
|
+
- native structured usage log and Cursor-style chat export ingestion
|
|
170
|
+
- Source Coverage in Scan Health for native, profile-assisted, fallback, and
|
|
171
|
+
unsupported files
|
|
172
|
+
- privacy-safe Evidence Packs as JSON or Markdown
|
|
173
|
+
- local scan scheduling: manual, on-open, hourly, or daily
|
|
174
|
+
- project/model/tool scoped guardrails with warning thresholds
|
|
175
|
+
- Import Profile preview before saving matchers
|
|
176
|
+
- saved report exports for weekly usage, source coverage, guardrails, unknown
|
|
177
|
+
cost, high-cost sessions, and confidence trends
|
|
178
|
+
- operating metadata export without raw usage records
|
|
179
|
+
|
|
180
|
+
The 0.12.0 dashboard also tightens the daily operator path: setup buttons now
|
|
181
|
+
open the exact Settings section, scan results show what changed and where to go
|
|
182
|
+
next, and Evidence explains when it is being opened as a contextual drill-down
|
|
183
|
+
rather than a sidebar destination.
|
|
184
|
+
|
|
160
185
|
## Accuracy And Evidence
|
|
161
186
|
|
|
162
187
|
TokenTrace labels the trust level behind imported numbers:
|
|
@@ -302,13 +327,13 @@ Codex CLI status-line integration is intentionally deferred until its status-lin
|
|
|
302
327
|
|
|
303
328
|
Dashboard views:
|
|
304
329
|
|
|
305
|
-

|
|
306
331
|
|
|
307
|
-

|
|
308
333
|
|
|
309
|
-

|
|
310
335
|
|
|
311
|
-

|
|
312
337
|
|
|
313
338
|
CLI startup and help:
|
|
314
339
|
|
|
@@ -388,6 +413,9 @@ Adapters live under `src/ingestion/adapters/`:
|
|
|
388
413
|
|
|
389
414
|
- `claude-code.ts`
|
|
390
415
|
- `codex-cli.ts`
|
|
416
|
+
- `structured-usage-log.ts`
|
|
417
|
+
- `cursor-chat.ts`
|
|
418
|
+
- `sqlite-history.ts`
|
|
391
419
|
- `generic-jsonl.ts`
|
|
392
420
|
- `generic-json.ts`
|
|
393
421
|
- `generic-log.ts`
|
|
@@ -402,6 +430,9 @@ TokenTrace keeps a visible support contract so daily scans are easier to trust:
|
|
|
402
430
|
| --- | --- | --- |
|
|
403
431
|
| Claude Code project transcripts | Stable | Primary local CLI ingestion source. |
|
|
404
432
|
| Codex CLI session artifacts | Best-effort | Parsed defensively while CLI formats evolve. |
|
|
433
|
+
| Structured usage JSONL/NDJSON | Stable | Local wrapper and team logs with session, model, token, and source-cost fields. |
|
|
434
|
+
| Cursor-style chat exports | Best-effort | Imports local editor chat/composer exports without storing raw prompt text by default. |
|
|
435
|
+
| Usage-shaped SQLite histories | Best-effort | Reads local databases that expose session, model, token, or cost-like columns. |
|
|
405
436
|
| Generic JSONL, JSON, and text logs | Best-effort | Conservative usage-shaped records only. |
|
|
406
437
|
| Claude/Codex cache, plugin, todo, config, and support files | Ignored | Tracked as non-usage files, not parser failures. |
|
|
407
438
|
| Editable model pricing | Stable | Local pricing rows drive costs and unknown-cost repair queues. |
|
|
@@ -426,8 +457,8 @@ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for local setu
|
|
|
426
457
|
## Known Limitations
|
|
427
458
|
|
|
428
459
|
- Claude Code and Codex CLI log formats are inferred defensively and may need refinement with real sample logs.
|
|
429
|
-
-
|
|
430
|
-
- SQLite
|
|
460
|
+
- Tokenizer-backed estimates are available for recognized OpenAI/Codex and Claude-family model names. Unrecognized text-only records still fall back to a conservative simple estimate.
|
|
461
|
+
- SQLite-history ingestion expects usage-shaped local tables and skips arbitrary databases that do not expose session, model, token, or cost-like fields.
|
|
431
462
|
- Seed prices are editable and should be verified manually for your account, region, and provider plan.
|
|
432
463
|
|
|
433
464
|
## License
|
|
@@ -436,7 +467,7 @@ Open source by [Abhi Yoheswaran](https://www.abhiyoheswaran.com). Released under
|
|
|
436
467
|
|
|
437
468
|
## Next Improvements
|
|
438
469
|
|
|
439
|
-
-
|
|
440
|
-
- Add
|
|
441
|
-
-
|
|
442
|
-
-
|
|
470
|
+
- Expand first-class native adapters for more local AI tools and editor histories.
|
|
471
|
+
- Add provider-specific tokenizer refinements where public tokenizer behavior is stable enough to label clearly.
|
|
472
|
+
- Make Import Profile preview more interactive for teams with custom wrapper logs.
|
|
473
|
+
- Stream scan progress into the UI for very large local folders.
|
package/TOKENTRACE_AGENT.md
CHANGED
|
@@ -28,7 +28,7 @@ curl http://127.0.0.1:3030/api/capabilities
|
|
|
28
28
|
|
|
29
29
|
## Roadmap Status
|
|
30
30
|
|
|
31
|
-
Inspect the current 0.
|
|
31
|
+
Inspect the current 0.12.0 Local Sources & Trust handoff, action recipes, and release blockers:
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
tokentrace roadmap --json
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAnalyticsData } from "@/src/lib/analytics";
|
|
3
|
+
import { buildEvidencePack, renderEvidencePackMarkdown } from "@/src/lib/evidence-pack";
|
|
4
|
+
import { buildEvidenceTrail, parseEvidenceMetric } from "@/src/lib/evidence-trail";
|
|
5
|
+
|
|
6
|
+
export const dynamic = "force-dynamic";
|
|
7
|
+
|
|
8
|
+
export async function GET(request: Request) {
|
|
9
|
+
const url = new URL(request.url);
|
|
10
|
+
const format = url.searchParams.get("format") === "markdown" ? "markdown" : "json";
|
|
11
|
+
const metric = parseEvidenceMetric(url.searchParams.get("metric"));
|
|
12
|
+
const trail = buildEvidenceTrail({ metric });
|
|
13
|
+
const analytics = getAnalyticsData();
|
|
14
|
+
const pack = buildEvidencePack({
|
|
15
|
+
scope: {
|
|
16
|
+
type: "metric",
|
|
17
|
+
id: metric,
|
|
18
|
+
label: trail.title
|
|
19
|
+
},
|
|
20
|
+
totals: trail.totals,
|
|
21
|
+
confidenceDrivers: [
|
|
22
|
+
`${trail.confidence.exact.toLocaleString()} exact interactions`,
|
|
23
|
+
`${trail.confidence.estimated.toLocaleString()} estimated interactions`,
|
|
24
|
+
`${trail.confidence.unknown.toLocaleString()} unknown interactions`,
|
|
25
|
+
`Data confidence ${analytics.dataConfidence.score}/100`
|
|
26
|
+
],
|
|
27
|
+
sourceFiles: trail.sourceFiles.map((source) => source.sourceFile),
|
|
28
|
+
parserNotes: trail.sessions
|
|
29
|
+
.map((session) => `${session.parser ?? "unknown parser"}: ${session.parserStatus ?? "unknown status"}`)
|
|
30
|
+
.slice(0, 20),
|
|
31
|
+
modelRateState: trail.sessions
|
|
32
|
+
.map((session) =>
|
|
33
|
+
session.pricingHref ? `${session.model}: model-rate link available` : `${session.model}: no model-rate link`
|
|
34
|
+
)
|
|
35
|
+
.slice(0, 20),
|
|
36
|
+
repairLinks: trail.sessions
|
|
37
|
+
.filter((session) => session.unknownCostInteractions > 0)
|
|
38
|
+
.map((session) => `/repair?source=${encodeURIComponent(session.sourceFile)}`),
|
|
39
|
+
records: trail.sessions.map((session) => ({
|
|
40
|
+
id: session.id,
|
|
41
|
+
role: "session",
|
|
42
|
+
model: session.model,
|
|
43
|
+
sourceFile: session.sourceFile,
|
|
44
|
+
totalTokens: session.totalTokens,
|
|
45
|
+
cost: session.cost,
|
|
46
|
+
interactions: session.interactions
|
|
47
|
+
}))
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
if (format === "markdown") {
|
|
51
|
+
return new NextResponse(renderEvidencePackMarkdown(pack), {
|
|
52
|
+
headers: {
|
|
53
|
+
"content-type": "text/markdown; charset=utf-8",
|
|
54
|
+
"content-disposition": `attachment; filename="tokentrace-${metric}-evidence.md"`
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return NextResponse.json(pack, {
|
|
60
|
+
headers: {
|
|
61
|
+
"content-disposition": `attachment; filename="tokentrace-${metric}-evidence.json"`
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { readJsonObject } from "@/src/lib/api-json";
|
|
3
|
+
import { buildImportProfilePreview } from "@/src/lib/import-profile-preview";
|
|
4
|
+
|
|
5
|
+
export const dynamic = "force-dynamic";
|
|
6
|
+
|
|
7
|
+
export async function POST(request: Request) {
|
|
8
|
+
const parsed = await readJsonObject(request);
|
|
9
|
+
if (!parsed.ok) return NextResponse.json({ error: parsed.error }, { status: 400 });
|
|
10
|
+
const filePath = parsed.body.filePath;
|
|
11
|
+
if (typeof filePath !== "string" || !filePath.trim()) {
|
|
12
|
+
return NextResponse.json({ error: "filePath is required." }, { status: 400 });
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
const preview = await buildImportProfilePreview({
|
|
16
|
+
filePath: filePath.trim(),
|
|
17
|
+
storeRawMessageContent: parsed.body.storeRawMessageContent === true
|
|
18
|
+
});
|
|
19
|
+
return NextResponse.json(preview);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
return NextResponse.json(
|
|
22
|
+
{ error: error instanceof Error ? error.message : "Preview failed." },
|
|
23
|
+
{ status: 400 }
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAppVersion } from "@/src/lib/app-version";
|
|
3
|
+
import { buildOperatingMetadata } from "@/src/lib/operating-metadata";
|
|
4
|
+
|
|
5
|
+
export const dynamic = "force-dynamic";
|
|
6
|
+
|
|
7
|
+
export async function GET() {
|
|
8
|
+
return NextResponse.json(buildOperatingMetadata(getAppVersion()), {
|
|
9
|
+
headers: {
|
|
10
|
+
"content-disposition": "attachment; filename=\"tokentrace-operating-metadata.json\""
|
|
11
|
+
}
|
|
12
|
+
});
|
|
13
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAnalyticsData, getScanTrustData } from "@/src/lib/analytics";
|
|
3
|
+
import { buildSourceCatalog, summarizeSourceCoverage } from "@/src/lib/source-catalog";
|
|
4
|
+
import { buildSavedReportDefinitions, renderSavedReport, type SavedReportFormat } from "@/src/lib/saved-reports";
|
|
5
|
+
|
|
6
|
+
export const dynamic = "force-dynamic";
|
|
7
|
+
|
|
8
|
+
function reportRows(definitionId: string) {
|
|
9
|
+
const analytics = getAnalyticsData();
|
|
10
|
+
const trust = getScanTrustData();
|
|
11
|
+
const sourceCoverage = summarizeSourceCoverage(trust.scanFiles);
|
|
12
|
+
if (definitionId === "source-coverage") {
|
|
13
|
+
return [
|
|
14
|
+
{ label: "Native files", value: sourceCoverage.nativeFiles.toLocaleString(), detail: "First-class adapters" },
|
|
15
|
+
{ label: "Profile-assisted files", value: sourceCoverage.profileAssistedFiles.toLocaleString(), detail: "Import profile or generic parser" },
|
|
16
|
+
{ label: "Fallback files", value: sourceCoverage.fallbackFiles.toLocaleString(), detail: "Low-confidence text or generic fallback" },
|
|
17
|
+
{ label: "Imported records", value: sourceCoverage.importedRecords.toLocaleString(), detail: "Records imported from scan files" }
|
|
18
|
+
];
|
|
19
|
+
}
|
|
20
|
+
if (definitionId === "guardrail-status") {
|
|
21
|
+
return [
|
|
22
|
+
{ label: "Cost guardrail", value: analytics.usageGuardrails.cost.status, detail: `${analytics.usageGuardrails.cost.used.toFixed(2)} used` },
|
|
23
|
+
{ label: "Token guardrail", value: analytics.usageGuardrails.tokens.status, detail: `${analytics.usageGuardrails.tokens.used.toLocaleString()} tokens used` },
|
|
24
|
+
{ label: "Scoped guardrails", value: analytics.usageGuardrails.scoped.length.toLocaleString(), detail: "Project/model/tool limits" },
|
|
25
|
+
{ label: "Anomalies", value: analytics.usageGuardrails.anomalies.length.toLocaleString(), detail: "Warning or exceeded scoped guardrails" }
|
|
26
|
+
];
|
|
27
|
+
}
|
|
28
|
+
return [
|
|
29
|
+
{ label: "Tokens", value: analytics.summary.totalTokens.toLocaleString(), detail: "Selected local data" },
|
|
30
|
+
{ label: "Cost", value: `$${analytics.summary.totalCost.toFixed(2)}`, detail: "Provider estimate or source cost" },
|
|
31
|
+
{ label: "Sessions", value: analytics.summary.sessions.toLocaleString(), detail: "Imported sessions" },
|
|
32
|
+
{ label: "Unknown cost", value: analytics.summary.unknownCostInteractions.toLocaleString(), detail: "Repair queue candidates" },
|
|
33
|
+
{ label: "Source catalog", value: buildSourceCatalog().entries.length.toLocaleString(), detail: "Known import paths" }
|
|
34
|
+
];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function GET(request: Request) {
|
|
38
|
+
const url = new URL(request.url);
|
|
39
|
+
const definitions = buildSavedReportDefinitions();
|
|
40
|
+
const definitionId = url.searchParams.get("type") ?? "weekly-usage";
|
|
41
|
+
const definition = definitions.find((item) => item.id === definitionId);
|
|
42
|
+
if (!definition) return NextResponse.json({ error: "Unknown report type." }, { status: 400 });
|
|
43
|
+
const format = (url.searchParams.get("format") ?? "json") as SavedReportFormat;
|
|
44
|
+
if (!definition.formats.includes(format)) {
|
|
45
|
+
return NextResponse.json({ error: "Unsupported report format." }, { status: 400 });
|
|
46
|
+
}
|
|
47
|
+
const rendered = renderSavedReport({
|
|
48
|
+
definitionId,
|
|
49
|
+
format,
|
|
50
|
+
generatedAt: new Date().toISOString(),
|
|
51
|
+
rows: reportRows(definitionId)
|
|
52
|
+
});
|
|
53
|
+
if (format === "json") {
|
|
54
|
+
return new NextResponse(rendered, {
|
|
55
|
+
headers: { "content-type": "application/json; charset=utf-8" }
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
if (format === "csv") {
|
|
59
|
+
return new NextResponse(rendered, {
|
|
60
|
+
headers: {
|
|
61
|
+
"content-type": "text/csv; charset=utf-8",
|
|
62
|
+
"content-disposition": `attachment; filename="tokentrace-${definitionId}.csv"`
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return new NextResponse(rendered, {
|
|
67
|
+
headers: {
|
|
68
|
+
"content-type": "text/markdown; charset=utf-8",
|
|
69
|
+
"content-disposition": `attachment; filename="tokentrace-${definitionId}.md"`
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
package/app/api/scan/route.ts
CHANGED
|
@@ -23,5 +23,16 @@ export async function POST(request: Request) {
|
|
|
23
23
|
folders: stringList(body.folders),
|
|
24
24
|
force: jsonBooleanFlag(body.force)
|
|
25
25
|
});
|
|
26
|
+
if (jsonBooleanFlag(body.compact)) {
|
|
27
|
+
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
28
|
+
const errors = Array.isArray(result.errors) ? result.errors : [];
|
|
29
|
+
return NextResponse.json({
|
|
30
|
+
...result,
|
|
31
|
+
warningCount: warnings.length,
|
|
32
|
+
errorCount: errors.length,
|
|
33
|
+
warnings: warnings.slice(0, 25),
|
|
34
|
+
errors: errors.slice(0, 25)
|
|
35
|
+
});
|
|
36
|
+
}
|
|
26
37
|
return NextResponse.json(result);
|
|
27
38
|
}
|
|
@@ -3,6 +3,7 @@ import { getDatabasePath } from "@/src/db/client";
|
|
|
3
3
|
import { getAppSettings, normalizeUsageGuardrails, saveAppSettings } from "@/src/db/settings";
|
|
4
4
|
import { jsonBooleanFlag, readJsonObject } from "@/src/lib/api-json";
|
|
5
5
|
import { normalizeImportProfiles } from "@/src/lib/import-profiles";
|
|
6
|
+
import { normalizeScanSchedule } from "@/src/lib/scan-schedule";
|
|
6
7
|
|
|
7
8
|
export const dynamic = "force-dynamic";
|
|
8
9
|
|
|
@@ -29,7 +30,8 @@ export async function PUT(request: Request) {
|
|
|
29
30
|
customFolders,
|
|
30
31
|
storeRawMessageContent: jsonBooleanFlag(body.storeRawMessageContent),
|
|
31
32
|
usageGuardrails: normalizeUsageGuardrails(body.usageGuardrails),
|
|
32
|
-
importProfiles: normalizeImportProfiles(body.importProfiles)
|
|
33
|
+
importProfiles: normalizeImportProfiles(body.importProfiles),
|
|
34
|
+
scanSchedule: normalizeScanSchedule(body.scanSchedule)
|
|
33
35
|
});
|
|
34
36
|
|
|
35
37
|
return NextResponse.json(saved);
|
package/app/diagnostics/page.tsx
CHANGED
|
@@ -11,6 +11,7 @@ import { buildScanHealth } from "@/src/lib/scan-health";
|
|
|
11
11
|
import { getDefaultSearchRoots } from "@/src/ingestion/discovery";
|
|
12
12
|
import { formatDate } from "@/src/lib/format";
|
|
13
13
|
import { getSupplyChainHealth } from "@/src/lib/supply-chain-health";
|
|
14
|
+
import { buildSourceCatalog, summarizeSourceCoverage } from "@/src/lib/source-catalog";
|
|
14
15
|
|
|
15
16
|
export const dynamic = "force-dynamic";
|
|
16
17
|
|
|
@@ -462,8 +463,63 @@ function ScanHistoryPanel({ scanRuns }: { scanRuns: DebugScanRun[] }) {
|
|
|
462
463
|
);
|
|
463
464
|
}
|
|
464
465
|
|
|
466
|
+
function SourceCoveragePanel({ scanFiles }: { scanFiles: ReturnType<typeof getScanTrustData>["scanFiles"] }) {
|
|
467
|
+
const catalog = buildSourceCatalog();
|
|
468
|
+
const coverage = summarizeSourceCoverage(scanFiles);
|
|
469
|
+
const summary = [
|
|
470
|
+
["Native", coverage.nativeFiles],
|
|
471
|
+
["Profile-assisted", coverage.profileAssistedFiles],
|
|
472
|
+
["Fallback", coverage.fallbackFiles],
|
|
473
|
+
["Unsupported", coverage.unsupportedFiles]
|
|
474
|
+
];
|
|
475
|
+
|
|
476
|
+
return (
|
|
477
|
+
<Card>
|
|
478
|
+
<CardHeader className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
|
479
|
+
<div>
|
|
480
|
+
<CardTitle>Source Coverage</CardTitle>
|
|
481
|
+
<CardDescription>
|
|
482
|
+
Import support is grouped by native adapters, profile-assisted parsers, fallback parsers, and unsupported files.
|
|
483
|
+
</CardDescription>
|
|
484
|
+
</div>
|
|
485
|
+
<Link href="/api/reports?type=source-coverage&format=markdown" className="text-sm font-medium text-primary underline-offset-4 hover:underline">
|
|
486
|
+
Export source report
|
|
487
|
+
</Link>
|
|
488
|
+
</CardHeader>
|
|
489
|
+
<CardContent className="space-y-4">
|
|
490
|
+
<div className="grid border-y sm:grid-cols-2 lg:grid-cols-4">
|
|
491
|
+
{summary.map(([label, value]) => (
|
|
492
|
+
<div key={label} className="p-3">
|
|
493
|
+
<FieldLabel>{label}</FieldLabel>
|
|
494
|
+
<DataValue className="mt-1" size="md">{Number(value).toLocaleString()}</DataValue>
|
|
495
|
+
</div>
|
|
496
|
+
))}
|
|
497
|
+
</div>
|
|
498
|
+
<div className="grid gap-2 lg:grid-cols-2">
|
|
499
|
+
{catalog.entries.map((entry) => (
|
|
500
|
+
<div key={entry.id} className="rounded-md border p-3">
|
|
501
|
+
<div className="flex flex-wrap items-center gap-2">
|
|
502
|
+
<div className="text-sm font-semibold">{entry.label}</div>
|
|
503
|
+
<Badge variant={entry.tier === "native" ? "success" : entry.tier === "profile-assisted" ? "warning" : "secondary"}>
|
|
504
|
+
{entry.tier}
|
|
505
|
+
</Badge>
|
|
506
|
+
</div>
|
|
507
|
+
<p className="mt-1 text-xs leading-relaxed text-muted-foreground">{entry.description}</p>
|
|
508
|
+
<div className="mt-2 flex flex-wrap gap-1">
|
|
509
|
+
{entry.matchers.map((matcher) => (
|
|
510
|
+
<code key={matcher} className="rounded bg-muted px-1.5 py-0.5 text-xs">{matcher}</code>
|
|
511
|
+
))}
|
|
512
|
+
</div>
|
|
513
|
+
</div>
|
|
514
|
+
))}
|
|
515
|
+
</div>
|
|
516
|
+
</CardContent>
|
|
517
|
+
</Card>
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
|
|
465
521
|
export default async function DiagnosticsPage() {
|
|
466
|
-
const baseData = getScanTrustData();
|
|
522
|
+
const baseData = getScanTrustData({}, { scanFileScope: "recent" });
|
|
467
523
|
const supplyChain = getSupplyChainHealth();
|
|
468
524
|
const data = {
|
|
469
525
|
...baseData,
|
|
@@ -474,7 +530,7 @@ export default async function DiagnosticsPage() {
|
|
|
474
530
|
supplyChain
|
|
475
531
|
})
|
|
476
532
|
};
|
|
477
|
-
const analytics = getAnalyticsData();
|
|
533
|
+
const analytics = getAnalyticsData({}, { scanFileScope: "none", sessionDetail: "summary" });
|
|
478
534
|
const roots = await getDefaultSearchRoots();
|
|
479
535
|
const doctorReport = buildDoctorReport({
|
|
480
536
|
...data,
|
|
@@ -519,6 +575,8 @@ export default async function DiagnosticsPage() {
|
|
|
519
575
|
|
|
520
576
|
<ScanHistoryPanel scanRuns={data.scanRuns} />
|
|
521
577
|
|
|
578
|
+
<SourceCoveragePanel scanFiles={data.scanFiles} />
|
|
579
|
+
|
|
522
580
|
<ScanHealthSummary health={data.health} />
|
|
523
581
|
|
|
524
582
|
<div className="grid gap-4 md:grid-cols-3">
|
package/app/discovery/page.tsx
CHANGED
|
@@ -58,7 +58,7 @@ export default function DiscoveryPage() {
|
|
|
58
58
|
title="No files discovered yet"
|
|
59
59
|
description="Run a scan from Settings to populate Discovery. If expected folders are missing, add them in Settings."
|
|
60
60
|
actions={[
|
|
61
|
-
{ label: "Add folder", href: "/settings", variant: "outline" }
|
|
61
|
+
{ label: "Add folder", href: "/settings#custom-folders", variant: "outline" }
|
|
62
62
|
]}
|
|
63
63
|
>
|
|
64
64
|
<ScanNowButton size="sm" />
|