tokentrace 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -1
- package/README.md +47 -17
- package/TOKENTRACE_AGENT.md +2 -2
- package/app/api/evidence-pack/route.ts +64 -0
- package/app/api/import-profile-preview/route.ts +26 -0
- package/app/api/operating-metadata/route.ts +13 -0
- package/app/api/reports/route.ts +72 -0
- package/app/api/scan/route.ts +11 -0
- package/app/api/settings/route.ts +3 -1
- package/app/diagnostics/page.tsx +60 -2
- package/app/discovery/page.tsx +1 -1
- package/app/evidence/page.tsx +186 -15
- package/app/guide/page.tsx +2 -2
- package/app/layout.tsx +4 -0
- package/app/loading.tsx +20 -0
- package/app/models/page.tsx +1 -1
- package/app/page.tsx +98 -7
- package/app/repair/page.tsx +23 -7
- package/app/settings/page.tsx +19 -3
- package/bin/tokentrace.js +1 -1
- package/components/charts/rank-bar-chart.tsx +19 -1
- package/components/charts/trend-chart.tsx +18 -1
- package/components/navigation-progress.tsx +74 -0
- package/components/period-filter.tsx +49 -11
- package/components/scan-health-summary.tsx +1 -1
- package/components/scan-now-button.tsx +23 -7
- package/components/session-explorer.tsx +56 -3
- package/components/settings-panel.tsx +440 -49
- package/dist/runtime/agent.mjs +3 -3
- package/dist/runtime/db-migrate.mjs +63 -2
- package/dist/runtime/db-seed.mjs +63 -2
- package/dist/runtime/digest.mjs +298 -64
- package/dist/runtime/doctor.mjs +301 -67
- package/dist/runtime/evidence.mjs +101 -28
- package/dist/runtime/insights.mjs +297 -63
- package/dist/runtime/pricing-refresh.mjs +63 -2
- package/dist/runtime/repair.mjs +102 -23
- package/dist/runtime/report.mjs +591 -69
- package/dist/runtime/reset.mjs +63 -2
- package/dist/runtime/review.mjs +297 -63
- package/dist/runtime/roadmap.mjs +155 -91
- package/dist/runtime/scan.mjs +418 -11
- package/dist/runtime/status.mjs +65 -4
- package/docs/assets/evidence-0.12.0.png +0 -0
- package/docs/assets/overview-0.12.0.png +0 -0
- package/docs/assets/repair-0.12.0.png +0 -0
- package/docs/assets/scan-health-0.12.0.png +0 -0
- package/llms.txt +1 -1
- package/package.json +1 -1
- package/scripts/report.ts +37 -3
- package/scripts/roadmap.ts +1 -1
- package/scripts/smoke-cli.mjs +2 -2
- package/scripts/smoke-packed-install.mjs +2 -2
- package/src/db/migrate-core.ts +12 -0
- package/src/db/schema.ts +51 -2
- package/src/db/settings.ts +56 -4
- package/src/ingestion/adapters/cursor-chat.ts +99 -0
- package/src/ingestion/adapters/helpers.ts +30 -1
- package/src/ingestion/adapters/index.ts +4 -0
- package/src/ingestion/adapters/structured-usage-log.ts +95 -0
- package/src/ingestion/scan.ts +2 -0
- package/src/lib/accounting-invariants.ts +1 -1
- package/src/lib/agent-discovery.ts +3 -3
- package/src/lib/analytics.ts +106 -45
- package/src/lib/daily-digest.ts +1 -1
- package/src/lib/doctor.ts +3 -3
- package/src/lib/evidence-pack.ts +119 -0
- package/src/lib/evidence-trail.ts +39 -26
- package/src/lib/first-run-status.ts +4 -4
- package/src/lib/import-profile-preview.ts +125 -0
- package/src/lib/import-profiles.ts +22 -2
- package/src/lib/live-status.ts +1 -1
- package/src/lib/operating-metadata.ts +24 -0
- package/src/lib/recommendations.ts +1 -1
- package/src/lib/report-cli.ts +18 -1
- package/src/lib/review-queue.ts +1 -1
- package/src/lib/roadmap-status.ts +170 -92
- package/src/lib/saved-reports.ts +102 -0
- package/src/lib/scan-health.ts +5 -5
- package/src/lib/scan-schedule.ts +89 -0
- package/src/lib/scheduled-scan.ts +58 -0
- package/src/lib/source-catalog.ts +148 -0
- package/src/lib/support-matrix.ts +1 -1
- package/src/lib/unknown-cost-repair.ts +56 -18
- package/src/lib/usage-guardrails.ts +106 -5
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,55 @@ All notable changes to TokenTrace are documented here.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
## [0.12.1] - 2026-05-19
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
|
|
11
|
+
- Public docs, CLI help, package runtime output, and agent-readable release status now describe shipped TokenTrace behavior only.
|
|
12
|
+
- Future-version roadmap labels no longer appear in public package surfaces.
|
|
13
|
+
|
|
14
|
+
## [0.12.0] - 2026-05-19
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- Local Sources & Trust release bundle covering native local ingestion, evidence packs, scan scheduling, scoped guardrails, import-profile preview, saved reports, operating metadata export, and agent-readable release status.
|
|
19
|
+
- Native structured usage log ingestion for local wrappers and team JSONL/NDJSON logs with session, model, token, and source-cost fields.
|
|
20
|
+
- Native Cursor-style chat/composer export ingestion with local source evidence and no raw prompt storage by default.
|
|
21
|
+
- Source Catalog and Source Coverage in Scan Health so users can distinguish native, profile-assisted, fallback, and unsupported local files.
|
|
22
|
+
- Privacy-safe Evidence Packs exported as JSON or Markdown with totals, confidence drivers, source files, parser notes, model-rate state, repair links, and an explicit redaction manifest.
|
|
23
|
+
- Local scan scheduling settings for manual-only, on-open, hourly, and daily scans, plus scan-history retention and last scheduled scan status.
|
|
24
|
+
- Scoped project/model/tool guardrails with per-guardrail warning thresholds and anomaly notes.
|
|
25
|
+
- Import Profile preview for sampling a local file, checking parser fit, reviewing detected fields, and applying recommended matchers without exposing raw content.
|
|
26
|
+
- Saved report definitions and export endpoints for weekly usage, high-cost sessions, unknown-cost repair, confidence trends, guardrail status, and source coverage in Markdown, JSON, and CSV.
|
|
27
|
+
- Operating metadata export for settings, source catalog, schedules, guardrails, report definitions, and roadmap status without raw usage records.
|
|
28
|
+
- Agent-readable release status with implemented cards, action recipes, evidence paths, verification gates, and release status.
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- Settings now includes Scan Scheduling, Scoped Guardrails, Import Profile preview, and Local Exports.
|
|
33
|
+
- Evidence pages now offer one-click JSON and Markdown evidence-pack exports.
|
|
34
|
+
- Evidence remains a contextual drill-down from Overview, Sessions, Repair, and exports, with direct-entry guidance for users who open `/evidence` manually.
|
|
35
|
+
- Scan, setup, guardrail, package-trust, folder, import-profile, and export CTAs now deep-link to the exact Settings section instead of dropping users at the top of Settings.
|
|
36
|
+
- Settings scan feedback now reports files checked, records imported, warnings, errors, recalculated costs, unknown cost, stale support imports, model aliases, and next-step links to Scan Health, Repair, Discovery, and Model Rates.
|
|
37
|
+
- Browser-triggered Scan now responses now return compact warning/error previews with full counts, keeping large duplicate-file scans from shipping megabytes of warning text back to the UI.
|
|
38
|
+
- Settings now has sticky section navigation, and Scan Controls shows a persisted Last scan result from local scan history after reload.
|
|
39
|
+
- Overview now includes a compact Last verified trust strip near the accounting totals for latest scan, package IOC, model-rate coverage, and evidence-pack availability.
|
|
40
|
+
- Evidence pages now include opened-from breadcrumbs, safe return behavior, and explicit Export JSON pack / Export Markdown pack actions.
|
|
41
|
+
- Route transitions now show a subtle top loading bar, and route loading states explain what is happening plus the next useful action.
|
|
42
|
+
- Scan and evidence follow-up actions now consistently use Scan now, Open Scan Health, Open repair, Set model rate, View evidence, and Export pack language.
|
|
43
|
+
- Period filters stay attached to the current page, preserve Evidence context, and keep desktop controls in one compact row while retaining horizontal scrolling for narrow widths.
|
|
44
|
+
- Chart cards now show lightweight loading placeholders during client hydration instead of appearing blank while Recharts initializes.
|
|
45
|
+
- Session Explorer now paginates large local result sets to keep dense session tables responsive.
|
|
46
|
+
- Repair now opens as a capped workbench of the top visible unknown-cost groups, while keeping full summary counts and focused repair links for deep review.
|
|
47
|
+
- Roadmap CLI/API now report the 0.12.0 release contract, implemented cards, action recipes, evidence paths, verification gates, and release status.
|
|
48
|
+
|
|
49
|
+
### Fixed
|
|
50
|
+
|
|
51
|
+
- Overview, Evidence, and Settings now avoid full raw-row scans and oversized scan-health payloads on large local databases by using covering indexes, scoped scan-file reads, and a summary session path on Overview.
|
|
52
|
+
- Repair no longer serializes the entire unknown-cost queue on first load and uses a dedicated cost-repair index for large local databases.
|
|
53
|
+
- Scan Health no longer loads all scan-file evidence and full session details just to render summary panels.
|
|
54
|
+
- Overview no longer blocks initial render on due scheduled scans, so opening localhost does not wait for scan work before showing the dashboard.
|
|
55
|
+
|
|
7
56
|
## [0.11.0] - 2026-05-18
|
|
8
57
|
|
|
9
58
|
### Added
|
|
@@ -253,7 +302,7 @@ All notable changes to TokenTrace are documented here.
|
|
|
253
302
|
### Added
|
|
254
303
|
|
|
255
304
|
- Visible running TokenTrace version in the desktop sidebar, mobile header, and Settings.
|
|
256
|
-
- 0.4.0 roadmap and release checklist
|
|
305
|
+
- 0.4.0 roadmap and release checklist for maintainer-approved public releases.
|
|
257
306
|
- `tokentrace doctor --json` plus a shared Doctor report model for scan health and repair recommendations.
|
|
258
307
|
- Bundled parser provenance metadata on imported scan files, including parser id, display name, source, and version.
|
|
259
308
|
- Unknown-cost repair queue on Overview, grouped by cause, model, tool, source file, and repair path.
|
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ TokenTrace is designed for local development machines first, with macOS-oriented
|
|
|
10
10
|
|
|
11
11
|
[Website](https://www.abhiyoheswaran.com/apps/tokentrace) · [Source](https://github.com/abhiyoheswaran1/tokentrace)
|
|
12
12
|
|
|
13
|
-

|
|
14
14
|
|
|
15
15
|
## Start In Seconds
|
|
16
16
|
|
|
@@ -40,7 +40,7 @@ tokentrace agent --json # Print machine-readable agent discovery manifest
|
|
|
40
40
|
tokentrace capabilities --json
|
|
41
41
|
# Alias for agent discovery manifest
|
|
42
42
|
tokentrace roadmap --json
|
|
43
|
-
# Print
|
|
43
|
+
# Print release status handoff
|
|
44
44
|
tokentrace scan # Scan local AI CLI usage logs
|
|
45
45
|
tokentrace doctor --json
|
|
46
46
|
# Inspect scan health and repair recommendations
|
|
@@ -114,7 +114,7 @@ curl http://127.0.0.1:3030/api/agent
|
|
|
114
114
|
curl http://127.0.0.1:3030/api/capabilities
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
-
The
|
|
117
|
+
The Local Sources & Trust release handoff is also machine-readable:
|
|
118
118
|
|
|
119
119
|
```bash
|
|
120
120
|
tokentrace roadmap --json
|
|
@@ -154,9 +154,33 @@ npm run security:ioc
|
|
|
154
154
|
npm run smoke:packed
|
|
155
155
|
# Inspect packed tarball and smoke test packed CLI
|
|
156
156
|
tokentrace roadmap --json
|
|
157
|
-
# Inspect
|
|
157
|
+
# Inspect roadmap handoff, action recipes, and release status
|
|
158
158
|
```
|
|
159
159
|
|
|
160
|
+
## Local Sources And Trust
|
|
161
|
+
|
|
162
|
+
TokenTrace 0.12.0 bundles local source expansion, evidence exports, scan
|
|
163
|
+
scheduling, scoped guardrails, parser profile preview, saved reports, and
|
|
164
|
+
agent-readable release status.
|
|
165
|
+
|
|
166
|
+
New trust surfaces include:
|
|
167
|
+
|
|
168
|
+
- native structured usage log and Cursor-style chat export ingestion
|
|
169
|
+
- Source Coverage in Scan Health for native, profile-assisted, fallback, and
|
|
170
|
+
unsupported files
|
|
171
|
+
- privacy-safe Evidence Packs as JSON or Markdown
|
|
172
|
+
- local scan scheduling: manual, on-open, hourly, or daily
|
|
173
|
+
- project/model/tool scoped guardrails with warning thresholds
|
|
174
|
+
- Import Profile preview before saving matchers
|
|
175
|
+
- saved report exports for weekly usage, source coverage, guardrails, unknown
|
|
176
|
+
cost, high-cost sessions, and confidence trends
|
|
177
|
+
- operating metadata export without raw usage records
|
|
178
|
+
|
|
179
|
+
The 0.12.0 dashboard also tightens the daily operator path: setup buttons now
|
|
180
|
+
open the exact Settings section, scan results show what changed and where to go
|
|
181
|
+
next, and Evidence explains when it is being opened as a contextual drill-down
|
|
182
|
+
rather than a sidebar destination.
|
|
183
|
+
|
|
160
184
|
## Accuracy And Evidence
|
|
161
185
|
|
|
162
186
|
TokenTrace labels the trust level behind imported numbers:
|
|
@@ -171,9 +195,9 @@ The dashboard surfaces a Data Confidence score on Overview, Projects, Sessions,
|
|
|
171
195
|
and Session Timeline pages. Scan Health also includes a supply-chain IOC check
|
|
172
196
|
so package trust is visible in the product, not only in release scripts.
|
|
173
197
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
198
|
+
Public releases require maintainer approval. See
|
|
199
|
+
[docs/RELEASE_CHECKLIST.md](docs/RELEASE_CHECKLIST.md) before bumping versions,
|
|
200
|
+
tagging, creating GitHub releases, or publishing npm.
|
|
177
201
|
|
|
178
202
|
In local development, the SQLite database defaults to `.tokentrace/tokentrace.db`. Override it with:
|
|
179
203
|
|
|
@@ -302,13 +326,13 @@ Codex CLI status-line integration is intentionally deferred until its status-lin
|
|
|
302
326
|
|
|
303
327
|
Dashboard views:
|
|
304
328
|
|
|
305
|
-

|
|
306
330
|
|
|
307
|
-

|
|
308
332
|
|
|
309
|
-

|
|
310
334
|
|
|
311
|
-

|
|
312
336
|
|
|
313
337
|
CLI startup and help:
|
|
314
338
|
|
|
@@ -388,6 +412,9 @@ Adapters live under `src/ingestion/adapters/`:
|
|
|
388
412
|
|
|
389
413
|
- `claude-code.ts`
|
|
390
414
|
- `codex-cli.ts`
|
|
415
|
+
- `structured-usage-log.ts`
|
|
416
|
+
- `cursor-chat.ts`
|
|
417
|
+
- `sqlite-history.ts`
|
|
391
418
|
- `generic-jsonl.ts`
|
|
392
419
|
- `generic-json.ts`
|
|
393
420
|
- `generic-log.ts`
|
|
@@ -402,6 +429,9 @@ TokenTrace keeps a visible support contract so daily scans are easier to trust:
|
|
|
402
429
|
| --- | --- | --- |
|
|
403
430
|
| Claude Code project transcripts | Stable | Primary local CLI ingestion source. |
|
|
404
431
|
| Codex CLI session artifacts | Best-effort | Parsed defensively while CLI formats evolve. |
|
|
432
|
+
| Structured usage JSONL/NDJSON | Stable | Local wrapper and team logs with session, model, token, and source-cost fields. |
|
|
433
|
+
| Cursor-style chat exports | Best-effort | Imports local editor chat/composer exports without storing raw prompt text by default. |
|
|
434
|
+
| Usage-shaped SQLite histories | Best-effort | Reads local databases that expose session, model, token, or cost-like columns. |
|
|
405
435
|
| Generic JSONL, JSON, and text logs | Best-effort | Conservative usage-shaped records only. |
|
|
406
436
|
| Claude/Codex cache, plugin, todo, config, and support files | Ignored | Tracked as non-usage files, not parser failures. |
|
|
407
437
|
| Editable model pricing | Stable | Local pricing rows drive costs and unknown-cost repair queues. |
|
|
@@ -426,8 +456,8 @@ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for local setu
|
|
|
426
456
|
## Known Limitations
|
|
427
457
|
|
|
428
458
|
- Claude Code and Codex CLI log formats are inferred defensively and may need refinement with real sample logs.
|
|
429
|
-
-
|
|
430
|
-
- SQLite
|
|
459
|
+
- Tokenizer-backed estimates are available for recognized OpenAI/Codex and Claude-family model names. Unrecognized text-only records still fall back to a conservative simple estimate.
|
|
460
|
+
- SQLite-history ingestion expects usage-shaped local tables and skips arbitrary databases that do not expose session, model, token, or cost-like fields.
|
|
431
461
|
- Seed prices are editable and should be verified manually for your account, region, and provider plan.
|
|
432
462
|
|
|
433
463
|
## License
|
|
@@ -436,7 +466,7 @@ Open source by [Abhi Yoheswaran](https://www.abhiyoheswaran.com). Released under
|
|
|
436
466
|
|
|
437
467
|
## Next Improvements
|
|
438
468
|
|
|
439
|
-
-
|
|
440
|
-
- Add
|
|
441
|
-
-
|
|
442
|
-
-
|
|
469
|
+
- Expand first-class native adapters for more local AI tools and editor histories.
|
|
470
|
+
- Add provider-specific tokenizer refinements where public tokenizer behavior is stable enough to label clearly.
|
|
471
|
+
- Make Import Profile preview more interactive for teams with custom wrapper logs.
|
|
472
|
+
- Stream scan progress into the UI for very large local folders.
|
package/TOKENTRACE_AGENT.md
CHANGED
|
@@ -26,9 +26,9 @@ curl http://127.0.0.1:3030/api/agent
|
|
|
26
26
|
curl http://127.0.0.1:3030/api/capabilities
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
##
|
|
29
|
+
## Release Status
|
|
30
30
|
|
|
31
|
-
Inspect the current
|
|
31
|
+
Inspect the current release handoff, action recipes, and release status:
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
tokentrace roadmap --json
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAnalyticsData } from "@/src/lib/analytics";
|
|
3
|
+
import { buildEvidencePack, renderEvidencePackMarkdown } from "@/src/lib/evidence-pack";
|
|
4
|
+
import { buildEvidenceTrail, parseEvidenceMetric } from "@/src/lib/evidence-trail";
|
|
5
|
+
|
|
6
|
+
export const dynamic = "force-dynamic";
|
|
7
|
+
|
|
8
|
+
export async function GET(request: Request) {
|
|
9
|
+
const url = new URL(request.url);
|
|
10
|
+
const format = url.searchParams.get("format") === "markdown" ? "markdown" : "json";
|
|
11
|
+
const metric = parseEvidenceMetric(url.searchParams.get("metric"));
|
|
12
|
+
const trail = buildEvidenceTrail({ metric });
|
|
13
|
+
const analytics = getAnalyticsData();
|
|
14
|
+
const pack = buildEvidencePack({
|
|
15
|
+
scope: {
|
|
16
|
+
type: "metric",
|
|
17
|
+
id: metric,
|
|
18
|
+
label: trail.title
|
|
19
|
+
},
|
|
20
|
+
totals: trail.totals,
|
|
21
|
+
confidenceDrivers: [
|
|
22
|
+
`${trail.confidence.exact.toLocaleString()} exact interactions`,
|
|
23
|
+
`${trail.confidence.estimated.toLocaleString()} estimated interactions`,
|
|
24
|
+
`${trail.confidence.unknown.toLocaleString()} unknown interactions`,
|
|
25
|
+
`Data confidence ${analytics.dataConfidence.score}/100`
|
|
26
|
+
],
|
|
27
|
+
sourceFiles: trail.sourceFiles.map((source) => source.sourceFile),
|
|
28
|
+
parserNotes: trail.sessions
|
|
29
|
+
.map((session) => `${session.parser ?? "unknown parser"}: ${session.parserStatus ?? "unknown status"}`)
|
|
30
|
+
.slice(0, 20),
|
|
31
|
+
modelRateState: trail.sessions
|
|
32
|
+
.map((session) =>
|
|
33
|
+
session.pricingHref ? `${session.model}: model-rate link available` : `${session.model}: no model-rate link`
|
|
34
|
+
)
|
|
35
|
+
.slice(0, 20),
|
|
36
|
+
repairLinks: trail.sessions
|
|
37
|
+
.filter((session) => session.unknownCostInteractions > 0)
|
|
38
|
+
.map((session) => `/repair?source=${encodeURIComponent(session.sourceFile)}`),
|
|
39
|
+
records: trail.sessions.map((session) => ({
|
|
40
|
+
id: session.id,
|
|
41
|
+
role: "session",
|
|
42
|
+
model: session.model,
|
|
43
|
+
sourceFile: session.sourceFile,
|
|
44
|
+
totalTokens: session.totalTokens,
|
|
45
|
+
cost: session.cost,
|
|
46
|
+
interactions: session.interactions
|
|
47
|
+
}))
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
if (format === "markdown") {
|
|
51
|
+
return new NextResponse(renderEvidencePackMarkdown(pack), {
|
|
52
|
+
headers: {
|
|
53
|
+
"content-type": "text/markdown; charset=utf-8",
|
|
54
|
+
"content-disposition": `attachment; filename="tokentrace-${metric}-evidence.md"`
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return NextResponse.json(pack, {
|
|
60
|
+
headers: {
|
|
61
|
+
"content-disposition": `attachment; filename="tokentrace-${metric}-evidence.json"`
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { readJsonObject } from "@/src/lib/api-json";
|
|
3
|
+
import { buildImportProfilePreview } from "@/src/lib/import-profile-preview";
|
|
4
|
+
|
|
5
|
+
export const dynamic = "force-dynamic";
|
|
6
|
+
|
|
7
|
+
export async function POST(request: Request) {
|
|
8
|
+
const parsed = await readJsonObject(request);
|
|
9
|
+
if (!parsed.ok) return NextResponse.json({ error: parsed.error }, { status: 400 });
|
|
10
|
+
const filePath = parsed.body.filePath;
|
|
11
|
+
if (typeof filePath !== "string" || !filePath.trim()) {
|
|
12
|
+
return NextResponse.json({ error: "filePath is required." }, { status: 400 });
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
const preview = await buildImportProfilePreview({
|
|
16
|
+
filePath: filePath.trim(),
|
|
17
|
+
storeRawMessageContent: parsed.body.storeRawMessageContent === true
|
|
18
|
+
});
|
|
19
|
+
return NextResponse.json(preview);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
return NextResponse.json(
|
|
22
|
+
{ error: error instanceof Error ? error.message : "Preview failed." },
|
|
23
|
+
{ status: 400 }
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAppVersion } from "@/src/lib/app-version";
|
|
3
|
+
import { buildOperatingMetadata } from "@/src/lib/operating-metadata";
|
|
4
|
+
|
|
5
|
+
export const dynamic = "force-dynamic";
|
|
6
|
+
|
|
7
|
+
export async function GET() {
|
|
8
|
+
return NextResponse.json(buildOperatingMetadata(getAppVersion()), {
|
|
9
|
+
headers: {
|
|
10
|
+
"content-disposition": "attachment; filename=\"tokentrace-operating-metadata.json\""
|
|
11
|
+
}
|
|
12
|
+
});
|
|
13
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAnalyticsData, getScanTrustData } from "@/src/lib/analytics";
|
|
3
|
+
import { buildSourceCatalog, summarizeSourceCoverage } from "@/src/lib/source-catalog";
|
|
4
|
+
import { buildSavedReportDefinitions, renderSavedReport, type SavedReportFormat } from "@/src/lib/saved-reports";
|
|
5
|
+
|
|
6
|
+
export const dynamic = "force-dynamic";
|
|
7
|
+
|
|
8
|
+
function reportRows(definitionId: string) {
|
|
9
|
+
const analytics = getAnalyticsData();
|
|
10
|
+
const trust = getScanTrustData();
|
|
11
|
+
const sourceCoverage = summarizeSourceCoverage(trust.scanFiles);
|
|
12
|
+
if (definitionId === "source-coverage") {
|
|
13
|
+
return [
|
|
14
|
+
{ label: "Native files", value: sourceCoverage.nativeFiles.toLocaleString(), detail: "First-class adapters" },
|
|
15
|
+
{ label: "Profile-assisted files", value: sourceCoverage.profileAssistedFiles.toLocaleString(), detail: "Import profile or generic parser" },
|
|
16
|
+
{ label: "Fallback files", value: sourceCoverage.fallbackFiles.toLocaleString(), detail: "Low-confidence text or generic fallback" },
|
|
17
|
+
{ label: "Imported records", value: sourceCoverage.importedRecords.toLocaleString(), detail: "Records imported from scan files" }
|
|
18
|
+
];
|
|
19
|
+
}
|
|
20
|
+
if (definitionId === "guardrail-status") {
|
|
21
|
+
return [
|
|
22
|
+
{ label: "Cost guardrail", value: analytics.usageGuardrails.cost.status, detail: `${analytics.usageGuardrails.cost.used.toFixed(2)} used` },
|
|
23
|
+
{ label: "Token guardrail", value: analytics.usageGuardrails.tokens.status, detail: `${analytics.usageGuardrails.tokens.used.toLocaleString()} tokens used` },
|
|
24
|
+
{ label: "Scoped guardrails", value: analytics.usageGuardrails.scoped.length.toLocaleString(), detail: "Project/model/tool limits" },
|
|
25
|
+
{ label: "Anomalies", value: analytics.usageGuardrails.anomalies.length.toLocaleString(), detail: "Warning or exceeded scoped guardrails" }
|
|
26
|
+
];
|
|
27
|
+
}
|
|
28
|
+
return [
|
|
29
|
+
{ label: "Tokens", value: analytics.summary.totalTokens.toLocaleString(), detail: "Selected local data" },
|
|
30
|
+
{ label: "Cost", value: `$${analytics.summary.totalCost.toFixed(2)}`, detail: "Provider estimate or source cost" },
|
|
31
|
+
{ label: "Sessions", value: analytics.summary.sessions.toLocaleString(), detail: "Imported sessions" },
|
|
32
|
+
{ label: "Unknown cost", value: analytics.summary.unknownCostInteractions.toLocaleString(), detail: "Repair queue candidates" },
|
|
33
|
+
{ label: "Source catalog", value: buildSourceCatalog().entries.length.toLocaleString(), detail: "Known import paths" }
|
|
34
|
+
];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function GET(request: Request) {
|
|
38
|
+
const url = new URL(request.url);
|
|
39
|
+
const definitions = buildSavedReportDefinitions();
|
|
40
|
+
const definitionId = url.searchParams.get("type") ?? "weekly-usage";
|
|
41
|
+
const definition = definitions.find((item) => item.id === definitionId);
|
|
42
|
+
if (!definition) return NextResponse.json({ error: "Unknown report type." }, { status: 400 });
|
|
43
|
+
const format = (url.searchParams.get("format") ?? "json") as SavedReportFormat;
|
|
44
|
+
if (!definition.formats.includes(format)) {
|
|
45
|
+
return NextResponse.json({ error: "Unsupported report format." }, { status: 400 });
|
|
46
|
+
}
|
|
47
|
+
const rendered = renderSavedReport({
|
|
48
|
+
definitionId,
|
|
49
|
+
format,
|
|
50
|
+
generatedAt: new Date().toISOString(),
|
|
51
|
+
rows: reportRows(definitionId)
|
|
52
|
+
});
|
|
53
|
+
if (format === "json") {
|
|
54
|
+
return new NextResponse(rendered, {
|
|
55
|
+
headers: { "content-type": "application/json; charset=utf-8" }
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
if (format === "csv") {
|
|
59
|
+
return new NextResponse(rendered, {
|
|
60
|
+
headers: {
|
|
61
|
+
"content-type": "text/csv; charset=utf-8",
|
|
62
|
+
"content-disposition": `attachment; filename="tokentrace-${definitionId}.csv"`
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return new NextResponse(rendered, {
|
|
67
|
+
headers: {
|
|
68
|
+
"content-type": "text/markdown; charset=utf-8",
|
|
69
|
+
"content-disposition": `attachment; filename="tokentrace-${definitionId}.md"`
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
package/app/api/scan/route.ts
CHANGED
|
@@ -23,5 +23,16 @@ export async function POST(request: Request) {
|
|
|
23
23
|
folders: stringList(body.folders),
|
|
24
24
|
force: jsonBooleanFlag(body.force)
|
|
25
25
|
});
|
|
26
|
+
if (jsonBooleanFlag(body.compact)) {
|
|
27
|
+
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
28
|
+
const errors = Array.isArray(result.errors) ? result.errors : [];
|
|
29
|
+
return NextResponse.json({
|
|
30
|
+
...result,
|
|
31
|
+
warningCount: warnings.length,
|
|
32
|
+
errorCount: errors.length,
|
|
33
|
+
warnings: warnings.slice(0, 25),
|
|
34
|
+
errors: errors.slice(0, 25)
|
|
35
|
+
});
|
|
36
|
+
}
|
|
26
37
|
return NextResponse.json(result);
|
|
27
38
|
}
|
|
@@ -3,6 +3,7 @@ import { getDatabasePath } from "@/src/db/client";
|
|
|
3
3
|
import { getAppSettings, normalizeUsageGuardrails, saveAppSettings } from "@/src/db/settings";
|
|
4
4
|
import { jsonBooleanFlag, readJsonObject } from "@/src/lib/api-json";
|
|
5
5
|
import { normalizeImportProfiles } from "@/src/lib/import-profiles";
|
|
6
|
+
import { normalizeScanSchedule } from "@/src/lib/scan-schedule";
|
|
6
7
|
|
|
7
8
|
export const dynamic = "force-dynamic";
|
|
8
9
|
|
|
@@ -29,7 +30,8 @@ export async function PUT(request: Request) {
|
|
|
29
30
|
customFolders,
|
|
30
31
|
storeRawMessageContent: jsonBooleanFlag(body.storeRawMessageContent),
|
|
31
32
|
usageGuardrails: normalizeUsageGuardrails(body.usageGuardrails),
|
|
32
|
-
importProfiles: normalizeImportProfiles(body.importProfiles)
|
|
33
|
+
importProfiles: normalizeImportProfiles(body.importProfiles),
|
|
34
|
+
scanSchedule: normalizeScanSchedule(body.scanSchedule)
|
|
33
35
|
});
|
|
34
36
|
|
|
35
37
|
return NextResponse.json(saved);
|
package/app/diagnostics/page.tsx
CHANGED
|
@@ -11,6 +11,7 @@ import { buildScanHealth } from "@/src/lib/scan-health";
|
|
|
11
11
|
import { getDefaultSearchRoots } from "@/src/ingestion/discovery";
|
|
12
12
|
import { formatDate } from "@/src/lib/format";
|
|
13
13
|
import { getSupplyChainHealth } from "@/src/lib/supply-chain-health";
|
|
14
|
+
import { buildSourceCatalog, summarizeSourceCoverage } from "@/src/lib/source-catalog";
|
|
14
15
|
|
|
15
16
|
export const dynamic = "force-dynamic";
|
|
16
17
|
|
|
@@ -462,8 +463,63 @@ function ScanHistoryPanel({ scanRuns }: { scanRuns: DebugScanRun[] }) {
|
|
|
462
463
|
);
|
|
463
464
|
}
|
|
464
465
|
|
|
466
|
+
function SourceCoveragePanel({ scanFiles }: { scanFiles: ReturnType<typeof getScanTrustData>["scanFiles"] }) {
|
|
467
|
+
const catalog = buildSourceCatalog();
|
|
468
|
+
const coverage = summarizeSourceCoverage(scanFiles);
|
|
469
|
+
const summary = [
|
|
470
|
+
["Native", coverage.nativeFiles],
|
|
471
|
+
["Profile-assisted", coverage.profileAssistedFiles],
|
|
472
|
+
["Fallback", coverage.fallbackFiles],
|
|
473
|
+
["Unsupported", coverage.unsupportedFiles]
|
|
474
|
+
];
|
|
475
|
+
|
|
476
|
+
return (
|
|
477
|
+
<Card>
|
|
478
|
+
<CardHeader className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
|
479
|
+
<div>
|
|
480
|
+
<CardTitle>Source Coverage</CardTitle>
|
|
481
|
+
<CardDescription>
|
|
482
|
+
Import support is grouped by native adapters, profile-assisted parsers, fallback parsers, and unsupported files.
|
|
483
|
+
</CardDescription>
|
|
484
|
+
</div>
|
|
485
|
+
<Link href="/api/reports?type=source-coverage&format=markdown" className="text-sm font-medium text-primary underline-offset-4 hover:underline">
|
|
486
|
+
Export source report
|
|
487
|
+
</Link>
|
|
488
|
+
</CardHeader>
|
|
489
|
+
<CardContent className="space-y-4">
|
|
490
|
+
<div className="grid border-y sm:grid-cols-2 lg:grid-cols-4">
|
|
491
|
+
{summary.map(([label, value]) => (
|
|
492
|
+
<div key={label} className="p-3">
|
|
493
|
+
<FieldLabel>{label}</FieldLabel>
|
|
494
|
+
<DataValue className="mt-1" size="md">{Number(value).toLocaleString()}</DataValue>
|
|
495
|
+
</div>
|
|
496
|
+
))}
|
|
497
|
+
</div>
|
|
498
|
+
<div className="grid gap-2 lg:grid-cols-2">
|
|
499
|
+
{catalog.entries.map((entry) => (
|
|
500
|
+
<div key={entry.id} className="rounded-md border p-3">
|
|
501
|
+
<div className="flex flex-wrap items-center gap-2">
|
|
502
|
+
<div className="text-sm font-semibold">{entry.label}</div>
|
|
503
|
+
<Badge variant={entry.tier === "native" ? "success" : entry.tier === "profile-assisted" ? "warning" : "secondary"}>
|
|
504
|
+
{entry.tier}
|
|
505
|
+
</Badge>
|
|
506
|
+
</div>
|
|
507
|
+
<p className="mt-1 text-xs leading-relaxed text-muted-foreground">{entry.description}</p>
|
|
508
|
+
<div className="mt-2 flex flex-wrap gap-1">
|
|
509
|
+
{entry.matchers.map((matcher) => (
|
|
510
|
+
<code key={matcher} className="rounded bg-muted px-1.5 py-0.5 text-xs">{matcher}</code>
|
|
511
|
+
))}
|
|
512
|
+
</div>
|
|
513
|
+
</div>
|
|
514
|
+
))}
|
|
515
|
+
</div>
|
|
516
|
+
</CardContent>
|
|
517
|
+
</Card>
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
|
|
465
521
|
export default async function DiagnosticsPage() {
|
|
466
|
-
const baseData = getScanTrustData();
|
|
522
|
+
const baseData = getScanTrustData({}, { scanFileScope: "recent" });
|
|
467
523
|
const supplyChain = getSupplyChainHealth();
|
|
468
524
|
const data = {
|
|
469
525
|
...baseData,
|
|
@@ -474,7 +530,7 @@ export default async function DiagnosticsPage() {
|
|
|
474
530
|
supplyChain
|
|
475
531
|
})
|
|
476
532
|
};
|
|
477
|
-
const analytics = getAnalyticsData();
|
|
533
|
+
const analytics = getAnalyticsData({}, { scanFileScope: "none", sessionDetail: "summary" });
|
|
478
534
|
const roots = await getDefaultSearchRoots();
|
|
479
535
|
const doctorReport = buildDoctorReport({
|
|
480
536
|
...data,
|
|
@@ -519,6 +575,8 @@ export default async function DiagnosticsPage() {
|
|
|
519
575
|
|
|
520
576
|
<ScanHistoryPanel scanRuns={data.scanRuns} />
|
|
521
577
|
|
|
578
|
+
<SourceCoveragePanel scanFiles={data.scanFiles} />
|
|
579
|
+
|
|
522
580
|
<ScanHealthSummary health={data.health} />
|
|
523
581
|
|
|
524
582
|
<div className="grid gap-4 md:grid-cols-3">
|
package/app/discovery/page.tsx
CHANGED
|
@@ -58,7 +58,7 @@ export default function DiscoveryPage() {
|
|
|
58
58
|
title="No files discovered yet"
|
|
59
59
|
description="Run a scan from Settings to populate Discovery. If expected folders are missing, add them in Settings."
|
|
60
60
|
actions={[
|
|
61
|
-
{ label: "Add folder", href: "/settings", variant: "outline" }
|
|
61
|
+
{ label: "Add folder", href: "/settings#custom-folders", variant: "outline" }
|
|
62
62
|
]}
|
|
63
63
|
>
|
|
64
64
|
<ScanNowButton size="sm" />
|