tokentrace 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -1
- package/README.md +63 -15
- package/SECURITY.md +6 -2
- package/TOKENTRACE_AGENT.md +1 -1
- package/app/api/evidence-pack/route.ts +64 -0
- package/app/api/import-profile-preview/route.ts +26 -0
- package/app/api/operating-metadata/route.ts +13 -0
- package/app/api/repair-items/route.ts +22 -0
- package/app/api/reports/route.ts +72 -0
- package/app/api/scan/route.ts +11 -0
- package/app/api/settings/route.ts +5 -1
- package/app/diagnostics/page.tsx +72 -2
- package/app/discovery/page.tsx +1 -1
- package/app/evidence/page.tsx +186 -15
- package/app/guide/page.tsx +10 -5
- package/app/layout.tsx +4 -0
- package/app/loading.tsx +20 -0
- package/app/models/page.tsx +1 -1
- package/app/page.tsx +136 -7
- package/app/projects/page.tsx +13 -0
- package/app/repair/page.tsx +113 -97
- package/app/sessions/[id]/page.tsx +59 -0
- package/app/settings/page.tsx +19 -3
- package/bin/tokentrace.js +1 -1
- package/components/charts/rank-bar-chart.tsx +19 -1
- package/components/charts/trend-chart.tsx +18 -1
- package/components/navigation-progress.tsx +74 -0
- package/components/period-filter.tsx +49 -11
- package/components/repair-bulk-actions.tsx +84 -0
- package/components/scan-health-summary.tsx +23 -2
- package/components/scan-now-button.tsx +23 -7
- package/components/session-explorer.tsx +70 -4
- package/components/settings-panel.tsx +540 -47
- package/dist/runtime/agent.mjs +3 -3
- package/dist/runtime/db-migrate.mjs +63 -2
- package/dist/runtime/db-seed.mjs +89 -3
- package/dist/runtime/digest.mjs +535 -89
- package/dist/runtime/doctor.mjs +555 -110
- package/dist/runtime/evidence.mjs +101 -28
- package/dist/runtime/insights.mjs +534 -88
- package/dist/runtime/pricing-refresh.mjs +89 -3
- package/dist/runtime/repair.mjs +102 -23
- package/dist/runtime/report.mjs +828 -94
- package/dist/runtime/reset.mjs +89 -3
- package/dist/runtime/review.mjs +534 -88
- package/dist/runtime/roadmap.mjs +211 -85
- package/dist/runtime/scan.mjs +840 -82
- package/dist/runtime/status.mjs +65 -4
- package/docs/assets/evidence-0.12.0.png +0 -0
- package/docs/assets/overview-0.12.0.png +0 -0
- package/docs/assets/repair-0.12.0.png +0 -0
- package/docs/assets/scan-health-0.12.0.png +0 -0
- package/llms.txt +1 -1
- package/package.json +3 -2
- package/scripts/report.ts +37 -3
- package/scripts/roadmap.ts +1 -1
- package/scripts/security-ioc.mjs +384 -0
- package/scripts/smoke-cli.mjs +2 -2
- package/scripts/smoke-packed-install.mjs +2 -2
- package/src/db/migrate-core.ts +12 -0
- package/src/db/schema.ts +51 -2
- package/src/db/settings.ts +60 -4
- package/src/ingestion/adapters/cursor-chat.ts +99 -0
- package/src/ingestion/adapters/helpers.ts +30 -1
- package/src/ingestion/adapters/index.ts +6 -0
- package/src/ingestion/adapters/sqlite-history.ts +201 -0
- package/src/ingestion/adapters/structured-usage-log.ts +95 -0
- package/src/ingestion/discovery.ts +15 -18
- package/src/ingestion/persist.ts +32 -9
- package/src/ingestion/scan.ts +5 -1
- package/src/ingestion/types.ts +9 -1
- package/src/lib/accounting-invariants.ts +1 -1
- package/src/lib/agent-discovery.ts +3 -3
- package/src/lib/analytics.ts +207 -59
- package/src/lib/cost-recalculation.ts +28 -1
- package/src/lib/daily-digest.ts +1 -1
- package/src/lib/data-confidence.ts +83 -0
- package/src/lib/doctor.ts +3 -3
- package/src/lib/evidence-pack.ts +119 -0
- package/src/lib/evidence-trail.ts +39 -26
- package/src/lib/first-run-status.ts +4 -4
- package/src/lib/import-profile-preview.ts +125 -0
- package/src/lib/import-profiles.ts +144 -0
- package/src/lib/live-status.ts +1 -1
- package/src/lib/operating-metadata.ts +24 -0
- package/src/lib/recommendations.ts +1 -1
- package/src/lib/report-cli.ts +18 -1
- package/src/lib/review-queue.ts +1 -1
- package/src/lib/roadmap-status.ts +240 -86
- package/src/lib/saved-reports.ts +102 -0
- package/src/lib/scan-health.ts +38 -11
- package/src/lib/scan-schedule.ts +89 -0
- package/src/lib/scheduled-scan.ts +58 -0
- package/src/lib/session-timeline.ts +64 -0
- package/src/lib/source-catalog.ts +148 -0
- package/src/lib/supply-chain-health.ts +31 -0
- package/src/lib/support-matrix.ts +1 -1
- package/src/lib/token-estimator.ts +72 -5
- package/src/lib/unknown-cost-repair.ts +75 -18
- package/src/lib/usage-guardrails.ts +106 -5
package/CHANGELOG.md
CHANGED
|
@@ -4,7 +4,69 @@ All notable changes to TokenTrace are documented here.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
## [0.12.0] - 2026-05-19
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- 0.12.0 Local Sources & Trust roadmap, rolling the 0.13.0 Evidence Portability, 0.14.0 Local Operations, 0.15.0 Governance & Guardrails, 0.16.0 Parser Studio, 0.17.0 Reports, and 0.18.0 Agent Handoff themes into one larger release.
|
|
12
|
+
- Native structured usage log ingestion for local wrappers and team JSONL/NDJSON logs with session, model, token, and source-cost fields.
|
|
13
|
+
- Native Cursor-style chat/composer export ingestion with local source evidence and no raw prompt storage by default.
|
|
14
|
+
- Source Catalog and Source Coverage in Scan Health so users can distinguish native, profile-assisted, fallback, and unsupported local files.
|
|
15
|
+
- Privacy-safe Evidence Packs exported as JSON or Markdown with totals, confidence drivers, source files, parser notes, model-rate state, repair links, and an explicit redaction manifest.
|
|
16
|
+
- Local scan scheduling settings for manual-only, on-open, hourly, and daily scans, plus scan-history retention and last scheduled scan status.
|
|
17
|
+
- Scoped project/model/tool guardrails with per-guardrail warning thresholds and anomaly notes.
|
|
18
|
+
- Import Profile preview for sampling a local file, checking parser fit, reviewing detected fields, and applying recommended matchers without exposing raw content.
|
|
19
|
+
- Saved report definitions and export endpoints for weekly usage, high-cost sessions, unknown-cost repair, confidence trends, guardrail status, and source coverage in Markdown, JSON, and CSV.
|
|
20
|
+
- Operating metadata export for settings, source catalog, schedules, guardrails, report definitions, and roadmap status without raw usage records.
|
|
21
|
+
- Agent-readable Roadmap V2 with current release, next planned release, rolled-up release themes, action recipes, evidence paths, verification gates, and release status.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
|
|
25
|
+
- Settings now includes Scan Scheduling, Scoped Guardrails, Import Profile preview, and Local Exports.
|
|
26
|
+
- Evidence pages now offer one-click JSON and Markdown evidence-pack exports.
|
|
27
|
+
- Evidence remains a contextual drill-down from Overview, Sessions, Repair, and exports, with direct-entry guidance for users who open `/evidence` manually.
|
|
28
|
+
- Scan, setup, guardrail, package-trust, folder, import-profile, and export CTAs now deep-link to the exact Settings section instead of dropping users at the top of Settings.
|
|
29
|
+
- Settings scan feedback now reports files checked, records imported, warnings, errors, recalculated costs, unknown cost, stale support imports, model aliases, and next-step links to Scan Health, Repair, Discovery, and Model Rates.
|
|
30
|
+
- Browser-triggered Scan now responses now return compact warning/error previews with full counts, keeping large duplicate-file scans from shipping megabytes of warning text back to the UI.
|
|
31
|
+
- Settings now has sticky section navigation, and Scan Controls shows a persisted Last scan result from local scan history after reload.
|
|
32
|
+
- Overview now includes a compact Last verified trust strip near the accounting totals for latest scan, package IOC, model-rate coverage, and evidence-pack availability.
|
|
33
|
+
- Evidence pages now include opened-from breadcrumbs, safe return behavior, and explicit Export JSON pack / Export Markdown pack actions.
|
|
34
|
+
- Route transitions now show a subtle top loading bar, and route loading states explain what is happening plus the next useful action.
|
|
35
|
+
- Scan and evidence follow-up actions now consistently use Scan now, Open Scan Health, Open repair, Set model rate, View evidence, and Export pack language.
|
|
36
|
+
- Period filters stay attached to the current page, preserve Evidence context, and keep desktop controls in one compact row while retaining horizontal scrolling for narrow widths.
|
|
37
|
+
- Chart cards now show lightweight loading placeholders during client hydration instead of appearing blank while Recharts initializes.
|
|
38
|
+
- Session Explorer now paginates large local result sets to keep dense session tables responsive.
|
|
39
|
+
- Repair now opens as a capped workbench of the top visible unknown-cost groups, while keeping full summary counts and focused repair links for deep review.
|
|
40
|
+
- Roadmap CLI/API now report the 0.12.0 Local Sources & Trust release contract and the next planned 0.19.0 direction.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
|
|
44
|
+
- Overview, Evidence, and Settings now avoid full raw-row scans and oversized scan-health payloads on large local databases by using covering indexes, scoped scan-file reads, and a summary session path on Overview.
|
|
45
|
+
- Repair no longer serializes the entire unknown-cost queue on first load and uses a dedicated cost-repair index for large local databases.
|
|
46
|
+
- Scan Health no longer loads all scan-file evidence and full session details just to render summary panels.
|
|
47
|
+
- Overview no longer blocks initial render on due scheduled scans, so opening localhost does not wait for scan work before showing the dashboard.
|
|
48
|
+
|
|
49
|
+
## [0.11.0] - 2026-05-18
|
|
50
|
+
|
|
51
|
+
### Added
|
|
52
|
+
|
|
53
|
+
- `npm run security:ioc` scans project files and local Claude/VS Code hook files for high-signal Mini Shai-Hulud/TanStack supply-chain indicators before release.
|
|
54
|
+
- Tokenizer-backed estimates for recognized OpenAI/Codex and Claude-family model names, with explicit `tokenizer estimate` and `simple estimate` confidence labels.
|
|
55
|
+
- Native SQLite history ingestion for local usage databases with usage-shaped tables, source-file evidence, import-profile metadata, and source-provided cost preservation.
|
|
56
|
+
- Data Confidence scoring on Overview, Projects, Sessions, and Session Timeline views, combining token source, cost coverage, parser confidence, and scan freshness.
|
|
57
|
+
- Import Profiles in Settings so users can define safe local wrapper-log matchers without writing parser code.
|
|
58
|
+
- Repair Workbench bulk actions for marking visible unknown-cost groups verified, parser-review, ignored, or reopened.
|
|
59
|
+
- Supply-chain IOC status in Scan Health, backed by the same local `security:ioc` scanner used by release checks.
|
|
60
|
+
|
|
61
|
+
### Changed
|
|
62
|
+
|
|
63
|
+
- Session Timeline pages now explain token spikes, model changes, cache activity, cost coverage, confidence drivers, and direct unknown-cost repair links.
|
|
64
|
+
- Scan Health now distinguishes exact, tokenizer-estimated, simple-estimated, high-confidence, low-confidence, and unknown token rows.
|
|
65
|
+
- The roadmap API and CLI now report the 0.11.0 Accuracy & Evidence release contract.
|
|
66
|
+
|
|
67
|
+
### Fixed
|
|
68
|
+
|
|
69
|
+
- Source-provided interaction costs are no longer overwritten as unknown during post-scan cost recalculation when model rates are missing.
|
|
8
70
|
|
|
9
71
|
## [0.10.1] - 2026-05-18
|
|
10
72
|
|
package/README.md
CHANGED
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
|
|
5
5
|
# TokenTrace CLI
|
|
6
6
|
|
|
7
|
-
Local-first AI CLI usage analytics. TokenTrace scans local CLI logs, normalizes token usage, estimates missing counts, and shows cost, model, project, and
|
|
7
|
+
Local-first AI CLI usage analytics. TokenTrace scans local CLI logs and local usage databases, normalizes token usage, estimates missing counts with confidence labels, and shows cost, model, project, session, repair, and evidence analytics in a browser dashboard.
|
|
8
8
|
|
|
9
9
|
TokenTrace is designed for local development machines first, with macOS-oriented defaults. It does not require a cloud account and does not send telemetry or logs anywhere.
|
|
10
10
|
|
|
11
11
|
[Website](https://www.abhiyoheswaran.com/apps/tokentrace) · [Source](https://github.com/abhiyoheswaran1/tokentrace)
|
|
12
12
|
|
|
13
|
-

|
|
14
14
|
|
|
15
15
|
## Start In Seconds
|
|
16
16
|
|
|
@@ -40,7 +40,7 @@ tokentrace agent --json # Print machine-readable agent discovery manifest
|
|
|
40
40
|
tokentrace capabilities --json
|
|
41
41
|
# Alias for agent discovery manifest
|
|
42
42
|
tokentrace roadmap --json
|
|
43
|
-
# Print
|
|
43
|
+
# Print Local Sources & Trust release handoff
|
|
44
44
|
tokentrace scan # Scan local AI CLI usage logs
|
|
45
45
|
tokentrace doctor --json
|
|
46
46
|
# Inspect scan health and repair recommendations
|
|
@@ -114,7 +114,7 @@ curl http://127.0.0.1:3030/api/agent
|
|
|
114
114
|
curl http://127.0.0.1:3030/api/capabilities
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
-
The
|
|
117
|
+
The Local Sources & Trust release handoff is also machine-readable:
|
|
118
118
|
|
|
119
119
|
```bash
|
|
120
120
|
tokentrace roadmap --json
|
|
@@ -149,12 +149,53 @@ npm run verify # Run Vitest, TypeScript, and ESLint checks
|
|
|
149
149
|
npm run package:test # Verify, build, and dry-run the npm package
|
|
150
150
|
npm run package:inspect
|
|
151
151
|
# Check package transparency guardrails
|
|
152
|
+
npm run security:ioc
|
|
153
|
+
# Scan lockfiles, workflows, and local AI-tool hooks for supply-chain IOCs
|
|
152
154
|
npm run smoke:packed
|
|
153
155
|
# Inspect packed tarball and smoke test packed CLI
|
|
154
156
|
tokentrace roadmap --json
|
|
155
|
-
# Inspect
|
|
157
|
+
# Inspect roadmap handoff, action recipes, and release status
|
|
156
158
|
```
|
|
157
159
|
|
|
160
|
+
## Local Sources And Trust
|
|
161
|
+
|
|
162
|
+
TokenTrace 0.12.0 rolls the next several roadmap themes into one larger
|
|
163
|
+
release: local source expansion, evidence portability, local operations,
|
|
164
|
+
scoped guardrails, parser profile preview, saved reports, and agent-readable
|
|
165
|
+
roadmap handoff.
|
|
166
|
+
|
|
167
|
+
New trust surfaces include:
|
|
168
|
+
|
|
169
|
+
- native structured usage log and Cursor-style chat export ingestion
|
|
170
|
+
- Source Coverage in Scan Health for native, profile-assisted, fallback, and
|
|
171
|
+
unsupported files
|
|
172
|
+
- privacy-safe Evidence Packs as JSON or Markdown
|
|
173
|
+
- local scan scheduling: manual, on-open, hourly, or daily
|
|
174
|
+
- project/model/tool scoped guardrails with warning thresholds
|
|
175
|
+
- Import Profile preview before saving matchers
|
|
176
|
+
- saved report exports for weekly usage, source coverage, guardrails, unknown
|
|
177
|
+
cost, high-cost sessions, and confidence trends
|
|
178
|
+
- operating metadata export without raw usage records
|
|
179
|
+
|
|
180
|
+
The 0.12.0 dashboard also tightens the daily operator path: setup buttons now
|
|
181
|
+
open the exact Settings section, scan results show what changed and where to go
|
|
182
|
+
next, and Evidence explains when it is being opened as a contextual drill-down
|
|
183
|
+
rather than a sidebar destination.
|
|
184
|
+
|
|
185
|
+
## Accuracy And Evidence
|
|
186
|
+
|
|
187
|
+
TokenTrace labels the trust level behind imported numbers:
|
|
188
|
+
|
|
189
|
+
- exact provider token counts
|
|
190
|
+
- tokenizer estimates for recognized OpenAI/Codex and Claude-family model names
|
|
191
|
+
- simple estimates when only text-like content is available
|
|
192
|
+
- source-provided costs from local SQLite histories
|
|
193
|
+
- unknown cost repair groups when model, token, or rate evidence is missing
|
|
194
|
+
|
|
195
|
+
The dashboard surfaces a Data Confidence score on Overview, Projects, Sessions,
|
|
196
|
+
and Session Timeline pages. Scan Health also includes a supply-chain IOC check
|
|
197
|
+
so package trust is visible in the product, not only in release scripts.
|
|
198
|
+
|
|
158
199
|
Release work uses internal milestone commits until the next public minor
|
|
159
200
|
release. See [docs/RELEASE_CHECKLIST.md](docs/RELEASE_CHECKLIST.md) before
|
|
160
201
|
bumping versions, tagging, creating GitHub releases, or publishing npm.
|
|
@@ -286,13 +327,13 @@ Codex CLI status-line integration is intentionally deferred until its status-lin
|
|
|
286
327
|
|
|
287
328
|
Dashboard views:
|
|
288
329
|
|
|
289
|
-

|
|
290
331
|
|
|
291
|
-

|
|
292
333
|
|
|
293
|
-

|
|
294
335
|
|
|
295
|
-

|
|
296
337
|
|
|
297
338
|
CLI startup and help:
|
|
298
339
|
|
|
@@ -324,6 +365,7 @@ Stop the server with `Ctrl+C` in the terminal where `tokentrace` is running.
|
|
|
324
365
|
- The published package ships readable application source and the compiled CLI runtime, not generated `.next/server` route bundles.
|
|
325
366
|
- `tokentrace serve` prepares the local dashboard build in the user's TokenTrace app-data directory the first time it is needed.
|
|
326
367
|
- `npm run package:inspect` fails if generated Next.js build output appears in the published tarball.
|
|
368
|
+
- `npm run security:ioc` scans lockfiles, workflows, and local Claude/VS Code hook files for high-signal supply-chain compromise indicators.
|
|
327
369
|
- Public npm publishing is configured through GitHub Actions Trusted Publishing and provenance from version tags.
|
|
328
370
|
- Socket GitHub checks and ProjScan are used as release guardrails, alongside `npm audit --audit-level=moderate`.
|
|
329
371
|
- Release notes are published directly in GitHub Releases from the relevant changelog section, not as a link-only summary.
|
|
@@ -371,6 +413,9 @@ Adapters live under `src/ingestion/adapters/`:
|
|
|
371
413
|
|
|
372
414
|
- `claude-code.ts`
|
|
373
415
|
- `codex-cli.ts`
|
|
416
|
+
- `structured-usage-log.ts`
|
|
417
|
+
- `cursor-chat.ts`
|
|
418
|
+
- `sqlite-history.ts`
|
|
374
419
|
- `generic-jsonl.ts`
|
|
375
420
|
- `generic-json.ts`
|
|
376
421
|
- `generic-log.ts`
|
|
@@ -385,6 +430,9 @@ TokenTrace keeps a visible support contract so daily scans are easier to trust:
|
|
|
385
430
|
| --- | --- | --- |
|
|
386
431
|
| Claude Code project transcripts | Stable | Primary local CLI ingestion source. |
|
|
387
432
|
| Codex CLI session artifacts | Best-effort | Parsed defensively while CLI formats evolve. |
|
|
433
|
+
| Structured usage JSONL/NDJSON | Stable | Local wrapper and team logs with session, model, token, and source-cost fields. |
|
|
434
|
+
| Cursor-style chat exports | Best-effort | Imports local editor chat/composer exports without storing raw prompt text by default. |
|
|
435
|
+
| Usage-shaped SQLite histories | Best-effort | Reads local databases that expose session, model, token, or cost-like columns. |
|
|
388
436
|
| Generic JSONL, JSON, and text logs | Best-effort | Conservative usage-shaped records only. |
|
|
389
437
|
| Claude/Codex cache, plugin, todo, config, and support files | Ignored | Tracked as non-usage files, not parser failures. |
|
|
390
438
|
| Editable model pricing | Stable | Local pricing rows drive costs and unknown-cost repair queues. |
|
|
@@ -409,8 +457,8 @@ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for local setu
|
|
|
409
457
|
## Known Limitations
|
|
410
458
|
|
|
411
459
|
- Claude Code and Codex CLI log formats are inferred defensively and may need refinement with real sample logs.
|
|
412
|
-
-
|
|
413
|
-
- SQLite
|
|
460
|
+
- Tokenizer-backed estimates are available for recognized OpenAI/Codex and Claude-family model names. Unrecognized text-only records still fall back to a conservative simple estimate.
|
|
461
|
+
- SQLite-history ingestion expects usage-shaped local tables and skips arbitrary databases that do not expose session, model, token, or cost-like fields.
|
|
414
462
|
- Seed prices are editable and should be verified manually for your account, region, and provider plan.
|
|
415
463
|
|
|
416
464
|
## License
|
|
@@ -419,7 +467,7 @@ Open source by [Abhi Yoheswaran](https://www.abhiyoheswaran.com). Released under
|
|
|
419
467
|
|
|
420
468
|
## Next Improvements
|
|
421
469
|
|
|
422
|
-
-
|
|
423
|
-
- Add
|
|
424
|
-
-
|
|
425
|
-
-
|
|
470
|
+
- Expand first-class native adapters for more local AI tools and editor histories.
|
|
471
|
+
- Add provider-specific tokenizer refinements where public tokenizer behavior is stable enough to label clearly.
|
|
472
|
+
- Make Import Profile preview more interactive for teams with custom wrapper logs.
|
|
473
|
+
- Stream scan progress into the UI for very large local folders.
|
package/SECURITY.md
CHANGED
|
@@ -40,8 +40,12 @@ When running from source, the default development database is
|
|
|
40
40
|
## Release Trust
|
|
41
41
|
|
|
42
42
|
Public package releases are built from Git tags. The release flow includes
|
|
43
|
-
package inspection, production dependency audit,
|
|
44
|
-
Trusted Publishing through GitHub Actions.
|
|
43
|
+
package inspection, supply-chain IOC scanning, production dependency audit,
|
|
44
|
+
ProjScan checks, and npm Trusted Publishing through GitHub Actions.
|
|
45
|
+
|
|
46
|
+
Run `npm run security:ioc` to check lockfiles, GitHub Actions workflows, and
|
|
47
|
+
local Claude/VS Code hook files for high-signal Mini Shai-Hulud/TanStack-style
|
|
48
|
+
indicators before release or after a dependency scare.
|
|
45
49
|
|
|
46
50
|
## Reporting
|
|
47
51
|
|
package/TOKENTRACE_AGENT.md
CHANGED
|
@@ -28,7 +28,7 @@ curl http://127.0.0.1:3030/api/capabilities
|
|
|
28
28
|
|
|
29
29
|
## Roadmap Status
|
|
30
30
|
|
|
31
|
-
Inspect the current 0.
|
|
31
|
+
Inspect the current 0.12.0 Local Sources & Trust handoff, action recipes, and release blockers:
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
tokentrace roadmap --json
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAnalyticsData } from "@/src/lib/analytics";
|
|
3
|
+
import { buildEvidencePack, renderEvidencePackMarkdown } from "@/src/lib/evidence-pack";
|
|
4
|
+
import { buildEvidenceTrail, parseEvidenceMetric } from "@/src/lib/evidence-trail";
|
|
5
|
+
|
|
6
|
+
export const dynamic = "force-dynamic";
|
|
7
|
+
|
|
8
|
+
export async function GET(request: Request) {
|
|
9
|
+
const url = new URL(request.url);
|
|
10
|
+
const format = url.searchParams.get("format") === "markdown" ? "markdown" : "json";
|
|
11
|
+
const metric = parseEvidenceMetric(url.searchParams.get("metric"));
|
|
12
|
+
const trail = buildEvidenceTrail({ metric });
|
|
13
|
+
const analytics = getAnalyticsData();
|
|
14
|
+
const pack = buildEvidencePack({
|
|
15
|
+
scope: {
|
|
16
|
+
type: "metric",
|
|
17
|
+
id: metric,
|
|
18
|
+
label: trail.title
|
|
19
|
+
},
|
|
20
|
+
totals: trail.totals,
|
|
21
|
+
confidenceDrivers: [
|
|
22
|
+
`${trail.confidence.exact.toLocaleString()} exact interactions`,
|
|
23
|
+
`${trail.confidence.estimated.toLocaleString()} estimated interactions`,
|
|
24
|
+
`${trail.confidence.unknown.toLocaleString()} unknown interactions`,
|
|
25
|
+
`Data confidence ${analytics.dataConfidence.score}/100`
|
|
26
|
+
],
|
|
27
|
+
sourceFiles: trail.sourceFiles.map((source) => source.sourceFile),
|
|
28
|
+
parserNotes: trail.sessions
|
|
29
|
+
.map((session) => `${session.parser ?? "unknown parser"}: ${session.parserStatus ?? "unknown status"}`)
|
|
30
|
+
.slice(0, 20),
|
|
31
|
+
modelRateState: trail.sessions
|
|
32
|
+
.map((session) =>
|
|
33
|
+
session.pricingHref ? `${session.model}: model-rate link available` : `${session.model}: no model-rate link`
|
|
34
|
+
)
|
|
35
|
+
.slice(0, 20),
|
|
36
|
+
repairLinks: trail.sessions
|
|
37
|
+
.filter((session) => session.unknownCostInteractions > 0)
|
|
38
|
+
.map((session) => `/repair?source=${encodeURIComponent(session.sourceFile)}`),
|
|
39
|
+
records: trail.sessions.map((session) => ({
|
|
40
|
+
id: session.id,
|
|
41
|
+
role: "session",
|
|
42
|
+
model: session.model,
|
|
43
|
+
sourceFile: session.sourceFile,
|
|
44
|
+
totalTokens: session.totalTokens,
|
|
45
|
+
cost: session.cost,
|
|
46
|
+
interactions: session.interactions
|
|
47
|
+
}))
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
if (format === "markdown") {
|
|
51
|
+
return new NextResponse(renderEvidencePackMarkdown(pack), {
|
|
52
|
+
headers: {
|
|
53
|
+
"content-type": "text/markdown; charset=utf-8",
|
|
54
|
+
"content-disposition": `attachment; filename="tokentrace-${metric}-evidence.md"`
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return NextResponse.json(pack, {
|
|
60
|
+
headers: {
|
|
61
|
+
"content-disposition": `attachment; filename="tokentrace-${metric}-evidence.json"`
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { readJsonObject } from "@/src/lib/api-json";
|
|
3
|
+
import { buildImportProfilePreview } from "@/src/lib/import-profile-preview";
|
|
4
|
+
|
|
5
|
+
export const dynamic = "force-dynamic";
|
|
6
|
+
|
|
7
|
+
export async function POST(request: Request) {
|
|
8
|
+
const parsed = await readJsonObject(request);
|
|
9
|
+
if (!parsed.ok) return NextResponse.json({ error: parsed.error }, { status: 400 });
|
|
10
|
+
const filePath = parsed.body.filePath;
|
|
11
|
+
if (typeof filePath !== "string" || !filePath.trim()) {
|
|
12
|
+
return NextResponse.json({ error: "filePath is required." }, { status: 400 });
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
const preview = await buildImportProfilePreview({
|
|
16
|
+
filePath: filePath.trim(),
|
|
17
|
+
storeRawMessageContent: parsed.body.storeRawMessageContent === true
|
|
18
|
+
});
|
|
19
|
+
return NextResponse.json(preview);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
return NextResponse.json(
|
|
22
|
+
{ error: error instanceof Error ? error.message : "Preview failed." },
|
|
23
|
+
{ status: 400 }
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAppVersion } from "@/src/lib/app-version";
|
|
3
|
+
import { buildOperatingMetadata } from "@/src/lib/operating-metadata";
|
|
4
|
+
|
|
5
|
+
export const dynamic = "force-dynamic";
|
|
6
|
+
|
|
7
|
+
export async function GET() {
|
|
8
|
+
return NextResponse.json(buildOperatingMetadata(getAppVersion()), {
|
|
9
|
+
headers: {
|
|
10
|
+
"content-disposition": "attachment; filename=\"tokentrace-operating-metadata.json\""
|
|
11
|
+
}
|
|
12
|
+
});
|
|
13
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { NextResponse } from "next/server";
|
|
2
2
|
import {
|
|
3
|
+
bulkUpdateUnknownCostRepairs,
|
|
3
4
|
buildUnknownCostRepairWorkbench,
|
|
4
5
|
getUnknownCostReview,
|
|
5
6
|
saveUnknownCostReview,
|
|
@@ -40,9 +41,30 @@ export async function PUT(request: Request) {
|
|
|
40
41
|
return NextResponse.json({ error: parsed.error }, { status: 400 });
|
|
41
42
|
}
|
|
42
43
|
const body = parsed.body;
|
|
44
|
+
const keys = Array.isArray(body.keys)
|
|
45
|
+
? body.keys.map((value) => text(value, 1000)).filter(Boolean)
|
|
46
|
+
: [];
|
|
43
47
|
const key = text(body.key, 1000);
|
|
44
48
|
const status = reviewState(body.status ?? body.state);
|
|
45
49
|
|
|
50
|
+
if (keys.length) {
|
|
51
|
+
if (!status) {
|
|
52
|
+
return NextResponse.json({ error: "status must be unresolved, ignored, resolved, or needs-parser-review" }, { status: 400 });
|
|
53
|
+
}
|
|
54
|
+
const currentKeys = new Set(buildUnknownCostRepairWorkbench().groups.map((group) => group.key));
|
|
55
|
+
const missingKey = keys.find((item) => !currentKeys.has(item) && !getUnknownCostReview(item)?.updatedAt);
|
|
56
|
+
if (missingKey) {
|
|
57
|
+
return NextResponse.json({ error: "one or more repair keys were not found in current workbench evidence" }, { status: 404 });
|
|
58
|
+
}
|
|
59
|
+
return NextResponse.json(
|
|
60
|
+
bulkUpdateUnknownCostRepairs({
|
|
61
|
+
keys,
|
|
62
|
+
status,
|
|
63
|
+
notes: text(body.notes ?? body.note, 500)
|
|
64
|
+
})
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
46
68
|
if (!key) {
|
|
47
69
|
return NextResponse.json({ error: "key is required" }, { status: 400 });
|
|
48
70
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { NextResponse } from "next/server";
|
|
2
|
+
import { getAnalyticsData, getScanTrustData } from "@/src/lib/analytics";
|
|
3
|
+
import { buildSourceCatalog, summarizeSourceCoverage } from "@/src/lib/source-catalog";
|
|
4
|
+
import { buildSavedReportDefinitions, renderSavedReport, type SavedReportFormat } from "@/src/lib/saved-reports";
|
|
5
|
+
|
|
6
|
+
export const dynamic = "force-dynamic";
|
|
7
|
+
|
|
8
|
+
function reportRows(definitionId: string) {
|
|
9
|
+
const analytics = getAnalyticsData();
|
|
10
|
+
const trust = getScanTrustData();
|
|
11
|
+
const sourceCoverage = summarizeSourceCoverage(trust.scanFiles);
|
|
12
|
+
if (definitionId === "source-coverage") {
|
|
13
|
+
return [
|
|
14
|
+
{ label: "Native files", value: sourceCoverage.nativeFiles.toLocaleString(), detail: "First-class adapters" },
|
|
15
|
+
{ label: "Profile-assisted files", value: sourceCoverage.profileAssistedFiles.toLocaleString(), detail: "Import profile or generic parser" },
|
|
16
|
+
{ label: "Fallback files", value: sourceCoverage.fallbackFiles.toLocaleString(), detail: "Low-confidence text or generic fallback" },
|
|
17
|
+
{ label: "Imported records", value: sourceCoverage.importedRecords.toLocaleString(), detail: "Records imported from scan files" }
|
|
18
|
+
];
|
|
19
|
+
}
|
|
20
|
+
if (definitionId === "guardrail-status") {
|
|
21
|
+
return [
|
|
22
|
+
{ label: "Cost guardrail", value: analytics.usageGuardrails.cost.status, detail: `${analytics.usageGuardrails.cost.used.toFixed(2)} used` },
|
|
23
|
+
{ label: "Token guardrail", value: analytics.usageGuardrails.tokens.status, detail: `${analytics.usageGuardrails.tokens.used.toLocaleString()} tokens used` },
|
|
24
|
+
{ label: "Scoped guardrails", value: analytics.usageGuardrails.scoped.length.toLocaleString(), detail: "Project/model/tool limits" },
|
|
25
|
+
{ label: "Anomalies", value: analytics.usageGuardrails.anomalies.length.toLocaleString(), detail: "Warning or exceeded scoped guardrails" }
|
|
26
|
+
];
|
|
27
|
+
}
|
|
28
|
+
return [
|
|
29
|
+
{ label: "Tokens", value: analytics.summary.totalTokens.toLocaleString(), detail: "Selected local data" },
|
|
30
|
+
{ label: "Cost", value: `$${analytics.summary.totalCost.toFixed(2)}`, detail: "Provider estimate or source cost" },
|
|
31
|
+
{ label: "Sessions", value: analytics.summary.sessions.toLocaleString(), detail: "Imported sessions" },
|
|
32
|
+
{ label: "Unknown cost", value: analytics.summary.unknownCostInteractions.toLocaleString(), detail: "Repair queue candidates" },
|
|
33
|
+
{ label: "Source catalog", value: buildSourceCatalog().entries.length.toLocaleString(), detail: "Known import paths" }
|
|
34
|
+
];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function GET(request: Request) {
|
|
38
|
+
const url = new URL(request.url);
|
|
39
|
+
const definitions = buildSavedReportDefinitions();
|
|
40
|
+
const definitionId = url.searchParams.get("type") ?? "weekly-usage";
|
|
41
|
+
const definition = definitions.find((item) => item.id === definitionId);
|
|
42
|
+
if (!definition) return NextResponse.json({ error: "Unknown report type." }, { status: 400 });
|
|
43
|
+
const format = (url.searchParams.get("format") ?? "json") as SavedReportFormat;
|
|
44
|
+
if (!definition.formats.includes(format)) {
|
|
45
|
+
return NextResponse.json({ error: "Unsupported report format." }, { status: 400 });
|
|
46
|
+
}
|
|
47
|
+
const rendered = renderSavedReport({
|
|
48
|
+
definitionId,
|
|
49
|
+
format,
|
|
50
|
+
generatedAt: new Date().toISOString(),
|
|
51
|
+
rows: reportRows(definitionId)
|
|
52
|
+
});
|
|
53
|
+
if (format === "json") {
|
|
54
|
+
return new NextResponse(rendered, {
|
|
55
|
+
headers: { "content-type": "application/json; charset=utf-8" }
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
if (format === "csv") {
|
|
59
|
+
return new NextResponse(rendered, {
|
|
60
|
+
headers: {
|
|
61
|
+
"content-type": "text/csv; charset=utf-8",
|
|
62
|
+
"content-disposition": `attachment; filename="tokentrace-${definitionId}.csv"`
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return new NextResponse(rendered, {
|
|
67
|
+
headers: {
|
|
68
|
+
"content-type": "text/markdown; charset=utf-8",
|
|
69
|
+
"content-disposition": `attachment; filename="tokentrace-${definitionId}.md"`
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
package/app/api/scan/route.ts
CHANGED
|
@@ -23,5 +23,16 @@ export async function POST(request: Request) {
|
|
|
23
23
|
folders: stringList(body.folders),
|
|
24
24
|
force: jsonBooleanFlag(body.force)
|
|
25
25
|
});
|
|
26
|
+
if (jsonBooleanFlag(body.compact)) {
|
|
27
|
+
const warnings = Array.isArray(result.warnings) ? result.warnings : [];
|
|
28
|
+
const errors = Array.isArray(result.errors) ? result.errors : [];
|
|
29
|
+
return NextResponse.json({
|
|
30
|
+
...result,
|
|
31
|
+
warningCount: warnings.length,
|
|
32
|
+
errorCount: errors.length,
|
|
33
|
+
warnings: warnings.slice(0, 25),
|
|
34
|
+
errors: errors.slice(0, 25)
|
|
35
|
+
});
|
|
36
|
+
}
|
|
26
37
|
return NextResponse.json(result);
|
|
27
38
|
}
|
|
@@ -2,6 +2,8 @@ import { NextResponse } from "next/server";
|
|
|
2
2
|
import { getDatabasePath } from "@/src/db/client";
|
|
3
3
|
import { getAppSettings, normalizeUsageGuardrails, saveAppSettings } from "@/src/db/settings";
|
|
4
4
|
import { jsonBooleanFlag, readJsonObject } from "@/src/lib/api-json";
|
|
5
|
+
import { normalizeImportProfiles } from "@/src/lib/import-profiles";
|
|
6
|
+
import { normalizeScanSchedule } from "@/src/lib/scan-schedule";
|
|
5
7
|
|
|
6
8
|
export const dynamic = "force-dynamic";
|
|
7
9
|
|
|
@@ -27,7 +29,9 @@ export async function PUT(request: Request) {
|
|
|
27
29
|
const saved = saveAppSettings({
|
|
28
30
|
customFolders,
|
|
29
31
|
storeRawMessageContent: jsonBooleanFlag(body.storeRawMessageContent),
|
|
30
|
-
usageGuardrails: normalizeUsageGuardrails(body.usageGuardrails)
|
|
32
|
+
usageGuardrails: normalizeUsageGuardrails(body.usageGuardrails),
|
|
33
|
+
importProfiles: normalizeImportProfiles(body.importProfiles),
|
|
34
|
+
scanSchedule: normalizeScanSchedule(body.scanSchedule)
|
|
31
35
|
});
|
|
32
36
|
|
|
33
37
|
return NextResponse.json(saved);
|
package/app/diagnostics/page.tsx
CHANGED
|
@@ -7,8 +7,11 @@ import { DataValue, FieldLabel, MonoText, PageHeader } from "@/components/ui/typ
|
|
|
7
7
|
import { ScanHealthSummary } from "@/components/scan-health-summary";
|
|
8
8
|
import { getAnalyticsData, getScanTrustData, type DebugScanRun } from "@/src/lib/analytics";
|
|
9
9
|
import { buildDoctorReport, type DoctorReport } from "@/src/lib/doctor";
|
|
10
|
+
import { buildScanHealth } from "@/src/lib/scan-health";
|
|
10
11
|
import { getDefaultSearchRoots } from "@/src/ingestion/discovery";
|
|
11
12
|
import { formatDate } from "@/src/lib/format";
|
|
13
|
+
import { getSupplyChainHealth } from "@/src/lib/supply-chain-health";
|
|
14
|
+
import { buildSourceCatalog, summarizeSourceCoverage } from "@/src/lib/source-catalog";
|
|
12
15
|
|
|
13
16
|
export const dynamic = "force-dynamic";
|
|
14
17
|
|
|
@@ -460,9 +463,74 @@ function ScanHistoryPanel({ scanRuns }: { scanRuns: DebugScanRun[] }) {
|
|
|
460
463
|
);
|
|
461
464
|
}
|
|
462
465
|
|
|
466
|
+
function SourceCoveragePanel({ scanFiles }: { scanFiles: ReturnType<typeof getScanTrustData>["scanFiles"] }) {
|
|
467
|
+
const catalog = buildSourceCatalog();
|
|
468
|
+
const coverage = summarizeSourceCoverage(scanFiles);
|
|
469
|
+
const summary = [
|
|
470
|
+
["Native", coverage.nativeFiles],
|
|
471
|
+
["Profile-assisted", coverage.profileAssistedFiles],
|
|
472
|
+
["Fallback", coverage.fallbackFiles],
|
|
473
|
+
["Unsupported", coverage.unsupportedFiles]
|
|
474
|
+
];
|
|
475
|
+
|
|
476
|
+
return (
|
|
477
|
+
<Card>
|
|
478
|
+
<CardHeader className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
|
479
|
+
<div>
|
|
480
|
+
<CardTitle>Source Coverage</CardTitle>
|
|
481
|
+
<CardDescription>
|
|
482
|
+
Import support is grouped by native adapters, profile-assisted parsers, fallback parsers, and unsupported files.
|
|
483
|
+
</CardDescription>
|
|
484
|
+
</div>
|
|
485
|
+
<Link href="/api/reports?type=source-coverage&format=markdown" className="text-sm font-medium text-primary underline-offset-4 hover:underline">
|
|
486
|
+
Export source report
|
|
487
|
+
</Link>
|
|
488
|
+
</CardHeader>
|
|
489
|
+
<CardContent className="space-y-4">
|
|
490
|
+
<div className="grid border-y sm:grid-cols-2 lg:grid-cols-4">
|
|
491
|
+
{summary.map(([label, value]) => (
|
|
492
|
+
<div key={label} className="p-3">
|
|
493
|
+
<FieldLabel>{label}</FieldLabel>
|
|
494
|
+
<DataValue className="mt-1" size="md">{Number(value).toLocaleString()}</DataValue>
|
|
495
|
+
</div>
|
|
496
|
+
))}
|
|
497
|
+
</div>
|
|
498
|
+
<div className="grid gap-2 lg:grid-cols-2">
|
|
499
|
+
{catalog.entries.map((entry) => (
|
|
500
|
+
<div key={entry.id} className="rounded-md border p-3">
|
|
501
|
+
<div className="flex flex-wrap items-center gap-2">
|
|
502
|
+
<div className="text-sm font-semibold">{entry.label}</div>
|
|
503
|
+
<Badge variant={entry.tier === "native" ? "success" : entry.tier === "profile-assisted" ? "warning" : "secondary"}>
|
|
504
|
+
{entry.tier}
|
|
505
|
+
</Badge>
|
|
506
|
+
</div>
|
|
507
|
+
<p className="mt-1 text-xs leading-relaxed text-muted-foreground">{entry.description}</p>
|
|
508
|
+
<div className="mt-2 flex flex-wrap gap-1">
|
|
509
|
+
{entry.matchers.map((matcher) => (
|
|
510
|
+
<code key={matcher} className="rounded bg-muted px-1.5 py-0.5 text-xs">{matcher}</code>
|
|
511
|
+
))}
|
|
512
|
+
</div>
|
|
513
|
+
</div>
|
|
514
|
+
))}
|
|
515
|
+
</div>
|
|
516
|
+
</CardContent>
|
|
517
|
+
</Card>
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
|
|
463
521
|
export default async function DiagnosticsPage() {
|
|
464
|
-
const
|
|
465
|
-
const
|
|
522
|
+
const baseData = getScanTrustData({}, { scanFileScope: "recent" });
|
|
523
|
+
const supplyChain = getSupplyChainHealth();
|
|
524
|
+
const data = {
|
|
525
|
+
...baseData,
|
|
526
|
+
health: buildScanHealth({
|
|
527
|
+
scanRuns: baseData.scanRuns,
|
|
528
|
+
scanFiles: baseData.scanFiles,
|
|
529
|
+
confidence: baseData.confidence,
|
|
530
|
+
supplyChain
|
|
531
|
+
})
|
|
532
|
+
};
|
|
533
|
+
const analytics = getAnalyticsData({}, { scanFileScope: "none", sessionDetail: "summary" });
|
|
466
534
|
const roots = await getDefaultSearchRoots();
|
|
467
535
|
const doctorReport = buildDoctorReport({
|
|
468
536
|
...data,
|
|
@@ -507,6 +575,8 @@ export default async function DiagnosticsPage() {
|
|
|
507
575
|
|
|
508
576
|
<ScanHistoryPanel scanRuns={data.scanRuns} />
|
|
509
577
|
|
|
578
|
+
<SourceCoveragePanel scanFiles={data.scanFiles} />
|
|
579
|
+
|
|
510
580
|
<ScanHealthSummary health={data.health} />
|
|
511
581
|
|
|
512
582
|
<div className="grid gap-4 md:grid-cols-3">
|
package/app/discovery/page.tsx
CHANGED
|
@@ -58,7 +58,7 @@ export default function DiscoveryPage() {
|
|
|
58
58
|
title="No files discovered yet"
|
|
59
59
|
description="Run a scan from Settings to populate Discovery. If expected folders are missing, add them in Settings."
|
|
60
60
|
actions={[
|
|
61
|
-
{ label: "Add folder", href: "/settings", variant: "outline" }
|
|
61
|
+
{ label: "Add folder", href: "/settings#custom-folders", variant: "outline" }
|
|
62
62
|
]}
|
|
63
63
|
>
|
|
64
64
|
<ScanNowButton size="sm" />
|