token-studio 4.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/.nvmrc +1 -0
  2. package/CHANGELOG.md +89 -0
  3. package/Dockerfile +17 -0
  4. package/LICENSE +22 -0
  5. package/NOTICE.md +21 -0
  6. package/PRIVACY.md +68 -0
  7. package/README.en.md +220 -0
  8. package/README.md +220 -0
  9. package/config/collectors.json +54 -0
  10. package/data/.gitkeep +1 -0
  11. package/docker-compose.yml +17 -0
  12. package/docs/assets/.gitkeep +1 -0
  13. package/docs/assets/token-studio-v44-dashboard.png +0 -0
  14. package/docs/assets/token-studio-v44-live.png +0 -0
  15. package/docs/assets/token-studio-v44-review-mobile.png +0 -0
  16. package/docs/assets/token-studio-v44-review.png +0 -0
  17. package/docs/assets/token-studio-v45-dashboard.png +0 -0
  18. package/docs/assets/token-studio-v45-live.png +0 -0
  19. package/docs/assets/token-studio-v45-review-mobile.png +0 -0
  20. package/docs/assets/token-studio-v45-review.png +0 -0
  21. package/docs/blog-case-study.md +34 -0
  22. package/docs/collector-support-matrix.md +65 -0
  23. package/docs/competitive-notes.md +87 -0
  24. package/docs/demo-data/README.md +12 -0
  25. package/docs/demo-data/token-studio-v2-demo.json +146 -0
  26. package/docs/demo-flow.md +39 -0
  27. package/docs/first-run.md +95 -0
  28. package/docs/local-collectors.md +49 -0
  29. package/docs/public-launch-checklist.md +45 -0
  30. package/docs/resume-bullets.md +7 -0
  31. package/docs/statusline.md +52 -0
  32. package/index.html +16 -0
  33. package/package.json +36 -0
  34. package/render.yaml +17 -0
  35. package/src/auto-attribution.mjs +396 -0
  36. package/src/ccusage-bridge.mjs +74 -0
  37. package/src/ccusage-import.mjs +415 -0
  38. package/src/cli.mjs +643 -0
  39. package/src/client/dashboard/App.jsx +1734 -0
  40. package/src/client/dashboard/annotation-presets.js +138 -0
  41. package/src/client/dashboard/attribution.js +328 -0
  42. package/src/client/dashboard/components-charts.jsx +622 -0
  43. package/src/client/dashboard/components-tables.jsx +1531 -0
  44. package/src/client/dashboard/components-top.jsx +307 -0
  45. package/src/client/dashboard/import-budget.js +41 -0
  46. package/src/client/dashboard/model-usage.js +108 -0
  47. package/src/client/dashboard/onboarding.js +80 -0
  48. package/src/client/dashboard/styles.css +2606 -0
  49. package/src/client/live/LiveApp.jsx +226 -0
  50. package/src/client/live/styles.css +446 -0
  51. package/src/client/main.jsx +20 -0
  52. package/src/client/review/ReviewApp.jsx +507 -0
  53. package/src/client/review/closure-progress.js +165 -0
  54. package/src/client/review/markdown-report.js +401 -0
  55. package/src/client/review/model-strategy.js +273 -0
  56. package/src/client/review/roi-advisor.js +255 -0
  57. package/src/client/review/roi-evidence.js +78 -0
  58. package/src/client/review/savings-simulator.js +252 -0
  59. package/src/client/review/sections-1.jsx +277 -0
  60. package/src/client/review/sections-2.jsx +927 -0
  61. package/src/client/review/styles.css +2321 -0
  62. package/src/client/review/utils.js +345 -0
  63. package/src/client/shared/utils.js +236 -0
  64. package/src/closure-check.mjs +537 -0
  65. package/src/closure-import.mjs +646 -0
  66. package/src/collect.mjs +247 -0
  67. package/src/collector-config.mjs +82 -0
  68. package/src/collector-registry.mjs +333 -0
  69. package/src/collectors/claude-code.mjs +355 -0
  70. package/src/collectors/codex.mjs +418 -0
  71. package/src/collectors/copilot.mjs +19 -0
  72. package/src/collectors/cursor.mjs +23 -0
  73. package/src/collectors/gemini.mjs +530 -0
  74. package/src/collectors/goose.mjs +15 -0
  75. package/src/collectors/hermes.mjs +206 -0
  76. package/src/collectors/kimi.mjs +15 -0
  77. package/src/collectors/openclaw.mjs +400 -0
  78. package/src/collectors/opencode.mjs +349 -0
  79. package/src/collectors/qwen.mjs +15 -0
  80. package/src/collectors/structured-usage.mjs +437 -0
  81. package/src/collectors/utils.mjs +93 -0
  82. package/src/db.mjs +1397 -0
  83. package/src/demo-seed.mjs +39 -0
  84. package/src/dev.mjs +43 -0
  85. package/src/live.mjs +428 -0
  86. package/src/model-policy.mjs +147 -0
  87. package/src/pricing.mjs +434 -0
  88. package/src/privacy-check.mjs +126 -0
  89. package/src/server.mjs +1240 -0
  90. package/src/source-health.mjs +195 -0
  91. package/src/statusline.mjs +156 -0
  92. package/src/terminal-report.mjs +245 -0
  93. package/src/update-pricing.mjs +8 -0
  94. package/test/annotation-presets.test.mjs +137 -0
  95. package/test/api-annotations.test.mjs +202 -0
  96. package/test/api-auto-attribution.test.mjs +169 -0
  97. package/test/api-source-health.test.mjs +109 -0
  98. package/test/api-v2.test.mjs +278 -0
  99. package/test/api-v43.test.mjs +151 -0
  100. package/test/api-v44.test.mjs +128 -0
  101. package/test/attribution-summary.test.mjs +164 -0
  102. package/test/auto-attribution.test.mjs +116 -0
  103. package/test/ccusage-bridge.test.mjs +36 -0
  104. package/test/ccusage-import.test.mjs +93 -0
  105. package/test/cli-v43.test.mjs +64 -0
  106. package/test/cli-v45.test.mjs +34 -0
  107. package/test/cli-v46.test.mjs +129 -0
  108. package/test/cli-v47.test.mjs +98 -0
  109. package/test/closure-check.test.mjs +202 -0
  110. package/test/closure-import.test.mjs +263 -0
  111. package/test/collector-config.test.mjs +25 -0
  112. package/test/collector-registry.test.mjs +56 -0
  113. package/test/csv.test.mjs +19 -0
  114. package/test/db-annotations.test.mjs +186 -0
  115. package/test/db-v2.test.mjs +200 -0
  116. package/test/db-v4.test.mjs +178 -0
  117. package/test/experimental-collectors.test.mjs +103 -0
  118. package/test/fixtures/collectors/copilot/usage.jsonl +2 -0
  119. package/test/fixtures/collectors/cursor/usage.jsonl +2 -0
  120. package/test/fixtures/collectors/goose/usage.jsonl +2 -0
  121. package/test/fixtures/collectors/kimi/usage.jsonl +2 -0
  122. package/test/fixtures/collectors/qwen/usage.jsonl +2 -0
  123. package/test/import-budget.test.mjs +40 -0
  124. package/test/live.test.mjs +256 -0
  125. package/test/markdown-report.test.mjs +193 -0
  126. package/test/model-policy.test.mjs +34 -0
  127. package/test/model-strategy.test.mjs +116 -0
  128. package/test/model-usage.test.mjs +99 -0
  129. package/test/official-pricing.test.mjs +70 -0
  130. package/test/onboarding.test.mjs +55 -0
  131. package/test/privacy-check.test.mjs +33 -0
  132. package/test/review-closure-progress.test.mjs +99 -0
  133. package/test/roi-advisor.test.mjs +188 -0
  134. package/test/roi-evidence.test.mjs +48 -0
  135. package/test/roi-summary.test.mjs +101 -0
  136. package/test/savings-simulator.test.mjs +141 -0
  137. package/test/source-health.test.mjs +62 -0
  138. package/test/statusline.test.mjs +148 -0
  139. package/vite.config.js +23 -0
@@ -0,0 +1,98 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { spawn } from 'node:child_process';
4
+ import { mkdtempSync, rmSync } from 'node:fs';
5
+ import { tmpdir } from 'node:os';
6
+ import { join } from 'node:path';
7
+ import { openDb, upsertSession, upsertSessionAnnotation } from '../src/db.mjs';
8
+
9
+ test('v4.7 budget CLI supports fixed windows and warning thresholds', async () => {
10
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-cli-v47-budget-'));
11
+ const dbPath = join(dir, 'usage.sqlite');
12
+ try {
13
+ const saved = await runCli([
14
+ 'budget', 'set',
15
+ '--db', dbPath,
16
+ '--source', 'Codex CLI',
17
+ '--label', 'Codex 5h',
18
+ '--window-type', 'fixed',
19
+ '--window-minutes', '300',
20
+ '--reset-anchor', '2026-06-17T00:00:00Z',
21
+ '--warning-threshold', '0.7',
22
+ '--token-budget', '500000',
23
+ '--json'
24
+ ]);
25
+ assert.equal(saved.code, 0, saved.stderr);
26
+ const body = JSON.parse(saved.stdout);
27
+ assert.equal(body.profile.windowType, 'fixed');
28
+ assert.equal(body.profile.warningThreshold, 0.7);
29
+
30
+ const listed = await runCli(['budget', 'list', '--db', dbPath]);
31
+ assert.equal(listed.code, 0, listed.stderr);
32
+ assert.match(listed.stdout, /fixed:300m/);
33
+ assert.match(listed.stdout, /warn=70%/);
34
+ } finally {
35
+ rmSync(dir, { recursive: true, force: true });
36
+ }
37
+ });
38
+
39
+ test('v4.7 policy CLI exports markdown snippets without writing files', async () => {
40
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-cli-v47-policy-'));
41
+ const dbPath = join(dir, 'usage.sqlite');
42
+ const db = openDb(dbPath);
43
+ try {
44
+ upsertSession(db, {
45
+ device: 'devbox',
46
+ source: 'Codex CLI',
47
+ sessionId: 's1',
48
+ lastActivity: '2026-06-17T02:00:00Z',
49
+ totalTokens: 1200
50
+ });
51
+ upsertSessionAnnotation(db, {
52
+ device: 'devbox',
53
+ source: 'Codex CLI',
54
+ sessionId: 's1',
55
+ taskType: '功能开发',
56
+ outputStatus: '已完成',
57
+ workPurpose: '测试验证',
58
+ workStage: '验证',
59
+ valueLevel: '中'
60
+ });
61
+ } finally {
62
+ db.close();
63
+ }
64
+
65
+ try {
66
+ const agents = await runCli(['policy', '--db', dbPath, '--format=agents-md']);
67
+ assert.equal(agents.code, 0, agents.stderr);
68
+ assert.match(agents.stdout, /Token Studio ROI Agent Policy/);
69
+ assert.match(agents.stdout, /测试验证/);
70
+ assert.match(agents.stdout, /Do not automatically edit/);
71
+ assert.doesNotMatch(agents.stdout, /secret prompt|private response|actual transcript/i);
72
+
73
+ const claude = await runCli(['policy', '--db', join(dir, 'missing.sqlite'), '--format=claude-md']);
74
+ assert.equal(claude.code, 0, claude.stderr);
75
+ assert.match(claude.stdout, /Reviewed sessions|No matching annotated sessions|Claude Code/);
76
+ } finally {
77
+ rmSync(dir, { recursive: true, force: true });
78
+ }
79
+ });
80
+
81
+ function runCli(argv) {
82
+ return new Promise(resolve => {
83
+ const child = spawn(process.execPath, ['src/cli.mjs', ...argv], {
84
+ cwd: process.cwd(),
85
+ env: process.env,
86
+ stdio: ['ignore', 'pipe', 'pipe'],
87
+ windowsHide: true
88
+ });
89
+ let stdout = '';
90
+ let stderr = '';
91
+ child.stdout.setEncoding('utf8');
92
+ child.stderr.setEncoding('utf8');
93
+ child.stdout.on('data', chunk => { stdout += chunk; });
94
+ child.stderr.on('data', chunk => { stderr += chunk; });
95
+ child.on('close', code => resolve({ code, stdout, stderr }));
96
+ child.on('error', error => resolve({ code: 1, stdout, stderr: `${stderr}${error.message}` }));
97
+ });
98
+ }
@@ -0,0 +1,202 @@
1
+ import assert from 'node:assert/strict';
2
+ import { mkdtempSync, rmSync } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import test from 'node:test';
6
+ import {
7
+ buildClosureAuditFromDb,
8
+ buildClosureImportTemplate,
9
+ formatClosureAudit,
10
+ formatClosureImportTemplate,
11
+ formatClosureWorklist,
12
+ openReadOnlyDb,
13
+ parseArgs
14
+ } from '../src/closure-check.mjs';
15
+ import {
16
+ openDb,
17
+ upsertDaily,
18
+ upsertSession,
19
+ upsertSessionAnnotation,
20
+ upsertSessionOutput
21
+ } from '../src/db.mjs';
22
+
23
+ test('buildClosureAuditFromDb reports complete P0 gate from structured data', () => withDb((db) => {
24
+ seedCompleteClosureData(db);
25
+
26
+ const audit = buildClosureAuditFromDb(db, { dbPath: 'temp.sqlite' });
27
+ assert.equal(audit.status, 'complete');
28
+ assert.equal(audit.counts.sessions, 10);
29
+ assert.equal(audit.counts.annotations, 10);
30
+ assert.equal(audit.counts.outputs, 3);
31
+ assert.equal(audit.progress.checks.find(check => check.id === 'real-attribution').complete, true);
32
+ assert.equal(audit.progress.checks.find(check => check.id === 'output-links').complete, true);
33
+ assert.equal(audit.progress.checks.find(check => check.id === 'non-label-advice').complete, true);
34
+ assert.equal(audit.roiAdvice.some(item => item.category !== '补标注'), true);
35
+ assert.equal(audit.progress.annotatedSessions[0].project, 'Token Studio');
36
+ assert.equal('note' in audit.progress.annotatedSessions[0], false);
37
+ }));
38
+
39
+ test('formatClosureAudit includes gaps and privacy boundary without conversation content', () => withDb((db) => {
40
+ upsertSession(db, {
41
+ device: 'local',
42
+ source: 'Codex CLI',
43
+ sessionId: 'gap-1',
44
+ projectPath: 'D:/Projects/token-studio-roi',
45
+ inputTokens: 2000,
46
+ outputTokens: 100,
47
+ totalTokens: 2100,
48
+ costUSD: 2
49
+ });
50
+
51
+ const audit = buildClosureAuditFromDb(db, { dbPath: 'temp.sqlite' });
52
+ const text = formatClosureAudit(audit);
53
+ assert.equal('session' in audit.progress.topGaps[0], false);
54
+ assert.match(text, /Status: needs-work/);
55
+ assert.match(text, /Top gaps:/);
56
+ assert.match(text, /missing 项目别名, 任务类型, 产出状态, 工作目的, 工作阶段, 产出价值/);
57
+ assert.match(text, /Privacy: read-only SQLite audit/);
58
+ assert.equal(text.includes('对话正文'), false);
59
+ }));
60
+
61
+ test('formatClosureWorklist prints fillable rows and escapes markdown formula cells', () => withDb((db) => {
62
+ upsertSession(db, {
63
+ device: 'local',
64
+ source: 'Codex CLI',
65
+ sessionId: '=session',
66
+ projectPath: '+project|name',
67
+ inputTokens: 2000,
68
+ outputTokens: 100,
69
+ totalTokens: 2100,
70
+ costUSD: 2
71
+ });
72
+
73
+ const audit = buildClosureAuditFromDb(db, { dbPath: 'temp.sqlite', topGapLimit: 10 });
74
+ const text = formatClosureWorklist(audit, { limit: 10 });
75
+ assert.match(text, /^# Token Studio P0 Attribution Worklist/);
76
+ assert.match(text, /Project alias \| Task type \| Output status/);
77
+ assert.match(text, /'\+project\\\|name/);
78
+ assert.match(text, /'\=session/);
79
+ assert.match(text, /Privacy: no conversation content/);
80
+ assert.equal(text.includes('conversation body'), false);
81
+ }));
82
+
83
+ test('formatClosureImportTemplate prints import-ready blank JSON for real gaps', () => withDb((db) => {
84
+ upsertSession(db, {
85
+ device: 'local',
86
+ source: 'Codex CLI',
87
+ sessionId: '=session',
88
+ projectPath: '+project|name',
89
+ inputTokens: 2000,
90
+ outputTokens: 100,
91
+ totalTokens: 2100,
92
+ costUSD: 2
93
+ });
94
+
95
+ const audit = buildClosureAuditFromDb(db, { dbPath: 'temp.sqlite', topGapLimit: 10 });
96
+ const template = buildClosureImportTemplate(audit, { limit: 10 });
97
+ const text = formatClosureImportTemplate(audit, { limit: 10 });
98
+ const parsed = JSON.parse(text);
99
+
100
+ assert.equal(template.sessions.length, 1);
101
+ assert.equal(template.sessions[0].device, 'local');
102
+ assert.equal(template.sessions[0].source, 'Codex CLI');
103
+ assert.equal(template.sessions[0].sessionId, '=session');
104
+ assert.equal(template.sessions[0].projectAlias, '');
105
+ assert.equal(template.sessions[0].taskType, '');
106
+ assert.equal(template.sessions[0].outputUrl, '');
107
+ assert.deepEqual(template.allowedValues.taskType.includes('功能开发'), true);
108
+ assert.equal(parsed.sessions[0].sessionId, '=session');
109
+ assert.match(text, /No conversation content was read/);
110
+ assert.equal(text.includes('conversation body'), false);
111
+ }));
112
+
113
+ test('parseArgs supports json and closure gate options', () => {
114
+ const options = parseArgs([
115
+ '--db', 'custom.sqlite',
116
+ '--json',
117
+ '--worklist',
118
+ '--template-json',
119
+ '--out=labels.json',
120
+ '--limit=8',
121
+ '--fail-on-incomplete',
122
+ '--target-sessions=12',
123
+ '--target-outputs=4'
124
+ ]);
125
+
126
+ assert.equal(options.dbPath, 'custom.sqlite');
127
+ assert.equal(options.json, true);
128
+ assert.equal(options.worklist, true);
129
+ assert.equal(options.templateJson, true);
130
+ assert.equal(options.outPath, 'labels.json');
131
+ assert.equal(options.worklistLimit, 8);
132
+ assert.equal(options.topGapLimit, 8);
133
+ assert.equal(options.failOnIncomplete, true);
134
+ assert.equal(options.targetAttributedSessions, 12);
135
+ assert.equal(options.targetOutputLinks, 4);
136
+ });
137
+
138
+ test('openReadOnlyDb rejects missing database instead of creating one', () => {
139
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-missing-db-'));
140
+ const dbPath = join(dir, 'missing.sqlite');
141
+ try {
142
+ assert.throws(() => openReadOnlyDb(dbPath), /not found/);
143
+ } finally {
144
+ rmSync(dir, { recursive: true, force: true });
145
+ }
146
+ });
147
+
148
+ function withDb(fn) {
149
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-closure-check-'));
150
+ const dbPath = join(dir, 'usage.sqlite');
151
+ const db = openDb(dbPath);
152
+ try {
153
+ return fn(db, dbPath);
154
+ } finally {
155
+ db.close();
156
+ rmSync(dir, { recursive: true, force: true });
157
+ }
158
+ }
159
+
160
+ function seedCompleteClosureData(db) {
161
+ upsertDaily(db, {
162
+ device: 'local',
163
+ source: 'Codex CLI',
164
+ usageDate: '2026-06-12',
165
+ model: 'gpt-5.3-codex',
166
+ inputTokens: 130000,
167
+ outputTokens: 10000,
168
+ totalTokens: 140000,
169
+ costUSD: 1
170
+ });
171
+
172
+ for (let index = 0; index < 10; index += 1) {
173
+ const session = {
174
+ device: 'local',
175
+ source: 'Codex CLI',
176
+ sessionId: `complete-${index + 1}`,
177
+ projectPath: 'D:/Projects/token-studio-roi',
178
+ inputTokens: 13000,
179
+ outputTokens: 800,
180
+ totalTokens: 13800,
181
+ costUSD: 1 + index / 10
182
+ };
183
+ upsertSession(db, session);
184
+ upsertSessionAnnotation(db, {
185
+ ...session,
186
+ projectAlias: 'Token Studio',
187
+ taskType: '功能开发',
188
+ outputStatus: index < 3 ? '已发布' : '已完成',
189
+ workPurpose: '功能开发',
190
+ workStage: '实现',
191
+ valueLevel: '高'
192
+ });
193
+ if (index < 3) {
194
+ upsertSessionOutput(db, {
195
+ ...session,
196
+ outputUrl: `https://example.com/output/${index + 1}`,
197
+ outputLabel: `Output ${index + 1}`,
198
+ outputType: '文档'
199
+ });
200
+ }
201
+ }
202
+ }
@@ -0,0 +1,263 @@
1
+ import assert from 'node:assert/strict';
2
+ import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import test from 'node:test';
6
+ import {
7
+ applyClosureImport,
8
+ buildClosureFillGuide,
9
+ buildClosureImportReport,
10
+ formatApplyValidResult,
11
+ formatClosureFillGuide,
12
+ formatImportReport,
13
+ formatImportPlan,
14
+ loadClosureImportFile,
15
+ parseImportArgs,
16
+ planClosureImport,
17
+ planValidClosureImport
18
+ } from '../src/closure-import.mjs';
19
+ import { openDb, upsertSession } from '../src/db.mjs';
20
+
21
+ const baseSession = {
22
+ device: 'local',
23
+ source: 'Codex CLI',
24
+ sessionId: 'session-1',
25
+ projectPath: 'D:/Projects/token-studio-roi',
26
+ inputTokens: 1000,
27
+ outputTokens: 100,
28
+ totalTokens: 1100,
29
+ costUSD: 1
30
+ };
31
+
32
+ const filledRow = {
33
+ sessionId: 'session-1',
34
+ projectAlias: 'Token Studio',
35
+ taskType: '功能开发',
36
+ outputStatus: '已发布',
37
+ workPurpose: '功能开发',
38
+ workStage: '发布',
39
+ valueLevel: '高',
40
+ note: '真实标注',
41
+ outputUrl: 'https://example.com/pr/1',
42
+ outputLabel: 'PR #1',
43
+ outputType: 'PR'
44
+ };
45
+
46
+ test('planClosureImport validates filled rows without writing SQLite', () => withDb(({ db }) => {
47
+ upsertSession(db, baseSession);
48
+ const plan = planClosureImport(db, [filledRow]);
49
+
50
+ assert.equal(plan.mode, 'dry-run');
51
+ assert.equal(plan.annotationCount, 1);
52
+ assert.equal(plan.outputCount, 1);
53
+ assert.equal(plan.sessions[0].device, 'local');
54
+ assert.equal(plan.sessions[0].hasOutput, true);
55
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_annotations').get().total, 0);
56
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_outputs').get().total, 0);
57
+ }));
58
+
59
+ test('applyClosureImport writes labels only after explicit apply and creates backup', () => withDb(({ db, dbPath, dir }) => {
60
+ upsertSession(db, baseSession);
61
+ const plan = planClosureImport(db, [filledRow]);
62
+ const result = applyClosureImport(db, plan, { dbPath, backupDir: join(dir, 'backups') });
63
+
64
+ assert.equal(result.applied, true);
65
+ assert.equal(result.annotationCount, 1);
66
+ assert.equal(result.outputCount, 1);
67
+ assert.equal(existsSync(result.backup.path), true);
68
+
69
+ const annotation = db.prepare('SELECT project_alias AS projectAlias, value_level AS valueLevel FROM session_annotations').get();
70
+ const output = db.prepare('SELECT output_url AS outputUrl, output_type AS outputType FROM session_outputs').get();
71
+ assert.equal(annotation.projectAlias, 'Token Studio');
72
+ assert.equal(annotation.valueLevel, '高');
73
+ assert.equal(output.outputUrl, 'https://example.com/pr/1');
74
+ assert.equal(output.outputType, 'PR');
75
+ }));
76
+
77
+ test('planClosureImport rejects partial labels and invalid output status for links', () => withDb(({ db }) => {
78
+ upsertSession(db, baseSession);
79
+ assert.throws(() => planClosureImport(db, [{ ...filledRow, valueLevel: '未评估' }]), /valueLevel/);
80
+ assert.throws(() => planClosureImport(db, [{ ...filledRow, outputStatus: '进行中' }]), /outputUrl requires/);
81
+ assert.throws(() => planClosureImport(db, [{ ...filledRow, taskType: '不存在' }]), /taskType must be one of/);
82
+ }));
83
+
84
+ test('buildClosureImportReport returns every invalid row without writing SQLite', () => withDb(({ db }) => {
85
+ upsertSession(db, baseSession);
86
+ upsertSession(db, { ...baseSession, sessionId: 'session-2' });
87
+
88
+ const report = buildClosureImportReport(db, [
89
+ { ...filledRow, valueLevel: '未评估' },
90
+ { ...filledRow, sessionId: 'session-2', outputStatus: '进行中' },
91
+ { ...filledRow, sessionId: 'missing-session' }
92
+ ]);
93
+ const text = formatImportReport(report);
94
+
95
+ assert.equal(report.mode, 'report');
96
+ assert.equal(report.valid, false);
97
+ assert.equal(report.rowCount, 3);
98
+ assert.equal(report.validCount, 0);
99
+ assert.equal(report.errorCount, 3);
100
+ assert.match(report.errors[0].error, /valueLevel/);
101
+ assert.match(report.errors[1].error, /outputUrl requires/);
102
+ assert.match(report.errors[2].error, /does not exist/);
103
+ assert.match(text, /Invalid: 3/);
104
+ assert.match(text, /does not run collect, write SQLite, or read conversation content/);
105
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_annotations').get().total, 0);
106
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_outputs').get().total, 0);
107
+ }));
108
+
109
+ test('buildClosureImportReport reports a valid filled file', () => withDb(({ db }) => {
110
+ upsertSession(db, baseSession);
111
+ const report = buildClosureImportReport(db, [filledRow]);
112
+ const text = formatImportReport(report);
113
+
114
+ assert.equal(report.valid, true);
115
+ assert.equal(report.validCount, 1);
116
+ assert.equal(report.errorCount, 0);
117
+ assert.equal(report.outputCount, 1);
118
+ assert.match(text, /All rows are valid/);
119
+ }));
120
+
121
+ test('buildClosureFillGuide prints missing fields and enum choices without writing', () => withDb(({ db }) => {
122
+ upsertSession(db, baseSession);
123
+ const guide = buildClosureFillGuide(db, [{
124
+ ...baseSession,
125
+ projectHint: 'Token Studio',
126
+ officialCostUSD: 1,
127
+ projectAlias: '',
128
+ taskType: '',
129
+ outputStatus: '',
130
+ workPurpose: '',
131
+ workStage: '',
132
+ valueLevel: ''
133
+ }]);
134
+ const text = formatClosureFillGuide(guide);
135
+
136
+ assert.equal(guide.mode, 'fill-guide');
137
+ assert.equal(guide.readyCount, 0);
138
+ assert.equal(guide.needsInputCount, 1);
139
+ assert.equal(guide.rows[0].missingFields.includes('projectAlias'), true);
140
+ assert.equal(guide.allowedValues.taskType.includes('功能开发'), true);
141
+ assert.match(text, /Token Studio Closure Fill Guide/);
142
+ assert.match(text, /taskType: 未分类 \/ 功能开发/);
143
+ assert.match(text, /fill required fields/);
144
+ assert.match(text, /no collect, no SQLite writes, no conversation content/);
145
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_annotations').get().total, 0);
146
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_outputs').get().total, 0);
147
+ }));
148
+
149
+ test('buildClosureFillGuide reports no rows when labels are complete', () => withDb(({ db }) => {
150
+ upsertSession(db, baseSession);
151
+ const guide = buildClosureFillGuide(db, [filledRow]);
152
+ const text = formatClosureFillGuide(guide);
153
+
154
+ assert.equal(guide.readyCount, 1);
155
+ assert.equal(guide.needsInputCount, 0);
156
+ assert.equal(guide.rows.length, 0);
157
+ assert.match(text, /No rows need input/);
158
+ }));
159
+
160
+ test('planValidClosureImport writes valid rows and skips invalid rows', () => withDb(({ db, dbPath, dir }) => {
161
+ upsertSession(db, baseSession);
162
+ upsertSession(db, { ...baseSession, sessionId: 'session-2' });
163
+
164
+ const plan = planValidClosureImport(db, [
165
+ filledRow,
166
+ { ...filledRow, sessionId: 'session-2', valueLevel: '未评估' }
167
+ ]);
168
+ const result = applyClosureImport(db, plan, {
169
+ dbPath,
170
+ backupDir: join(dir, 'backups'),
171
+ mode: 'apply-valid',
172
+ reason: 'closure-import-valid'
173
+ });
174
+ const text = formatApplyValidResult(plan, result);
175
+
176
+ assert.equal(plan.annotationCount, 1);
177
+ assert.equal(plan.outputCount, 1);
178
+ assert.equal(plan.skippedCount, 1);
179
+ assert.equal(result.mode, 'apply-valid');
180
+ assert.equal(result.applied, true);
181
+ assert.equal(existsSync(result.backup.path), true);
182
+ assert.match(text, /Applied annotations: 1/);
183
+ assert.match(text, /Skipped invalid rows: 1/);
184
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_annotations').get().total, 1);
185
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_outputs').get().total, 1);
186
+ }));
187
+
188
+ test('planValidClosureImport does not write or back up when no rows are valid', () => withDb(({ db, dbPath, dir }) => {
189
+ upsertSession(db, baseSession);
190
+
191
+ const plan = planValidClosureImport(db, [{ ...filledRow, valueLevel: '未评估' }]);
192
+ const result = applyClosureImport(db, plan, {
193
+ dbPath,
194
+ backupDir: join(dir, 'backups'),
195
+ mode: 'apply-valid',
196
+ reason: 'closure-import-valid'
197
+ });
198
+ const text = formatApplyValidResult(plan, result);
199
+
200
+ assert.equal(plan.annotationCount, 0);
201
+ assert.equal(plan.skippedCount, 1);
202
+ assert.equal(result.applied, false);
203
+ assert.equal(result.backup, null);
204
+ assert.match(text, /No valid rows were found/);
205
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_annotations').get().total, 0);
206
+ assert.equal(db.prepare('SELECT COUNT(*) AS total FROM session_outputs').get().total, 0);
207
+ }));
208
+
209
+ test('planClosureImport requires device and source for ambiguous session ids', () => withDb(({ db }) => {
210
+ upsertSession(db, baseSession);
211
+ upsertSession(db, { ...baseSession, device: 'other', source: 'Claude Code' });
212
+
213
+ assert.throws(() => planClosureImport(db, [filledRow]), /ambiguous/);
214
+ const plan = planClosureImport(db, [{ ...filledRow, device: 'local', source: 'Codex CLI' }]);
215
+ assert.equal(plan.sessions[0].source, 'Codex CLI');
216
+ }));
217
+
218
+ test('loadClosureImportFile and parseImportArgs support dry-run workflow', () => {
219
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-closure-import-file-'));
220
+ try {
221
+ const filePath = join(dir, 'labels.json');
222
+ writeFileSync(filePath, JSON.stringify({ sessions: [filledRow] }), 'utf8');
223
+ const loaded = loadClosureImportFile(filePath);
224
+ const args = parseImportArgs(['--file', filePath, '--db=usage.sqlite', '--json', '--report', '--limit=3']);
225
+
226
+ assert.equal(loaded.rows.length, 1);
227
+ assert.equal(args.file, filePath);
228
+ assert.equal(args.dbPath, 'usage.sqlite');
229
+ assert.equal(args.json, true);
230
+ assert.equal(args.report, true);
231
+ assert.equal(args.limit, 3);
232
+ assert.equal(parseImportArgs(['--file', filePath, '--fill-guide']).fillGuide, true);
233
+ assert.equal(parseImportArgs(['--file', filePath, '--apply-valid']).applyValid, true);
234
+ assert.equal(args.apply, false);
235
+ assert.throws(() => parseImportArgs(['--file', filePath, '--report', '--apply']), /cannot be combined/);
236
+ assert.throws(() => parseImportArgs(['--file', filePath, '--report', '--apply-valid']), /cannot be combined/);
237
+ assert.throws(() => parseImportArgs(['--file', filePath, '--fill-guide', '--apply-valid']), /cannot be combined/);
238
+ assert.throws(() => parseImportArgs(['--file', filePath, '--limit=0']), /positive integer/);
239
+ } finally {
240
+ rmSync(dir, { recursive: true, force: true });
241
+ }
242
+ });
243
+
244
+ test('formatImportPlan makes dry-run status explicit', () => withDb(({ db }) => {
245
+ upsertSession(db, baseSession);
246
+ const plan = planClosureImport(db, [filledRow]);
247
+ const text = formatImportPlan(plan);
248
+ assert.match(text, /Mode: dry-run/);
249
+ assert.match(text, /Dry run only/);
250
+ assert.match(text, /does not run collect or read conversation content/);
251
+ }));
252
+
253
+ function withDb(fn) {
254
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-closure-import-'));
255
+ const dbPath = join(dir, 'usage.sqlite');
256
+ const db = openDb(dbPath);
257
+ try {
258
+ return fn({ db, dbPath, dir });
259
+ } finally {
260
+ db.close();
261
+ rmSync(dir, { recursive: true, force: true });
262
+ }
263
+ }
@@ -0,0 +1,25 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { mkdtempSync, writeFileSync } from 'node:fs';
4
+ import { tmpdir } from 'node:os';
5
+ import { join } from 'node:path';
6
+
7
+ test('collector config accepts UTF-8 BOM files instead of falling back to defaults', async () => {
8
+ const dir = mkdtempSync(join(tmpdir(), 'token-studio-config-'));
9
+ const configPath = join(dir, 'collectors.json');
10
+ writeFileSync(configPath, `\uFEFF${JSON.stringify({
11
+ collectors: { cursor: { roots: ['D:/fixture/cursor'] } },
12
+ enabledCollectors: ['cursor']
13
+ })}`, 'utf8');
14
+
15
+ const old = process.env.TOKEN_STUDIO_CONFIG;
16
+ process.env.TOKEN_STUDIO_CONFIG = configPath;
17
+ try {
18
+ const moduleUrl = `../src/collector-config.mjs?case=${Date.now()}`;
19
+ const { configuredPaths } = await import(moduleUrl);
20
+ assert.deepEqual(configuredPaths('cursor', 'roots'), ['D:/fixture/cursor']);
21
+ } finally {
22
+ if (old == null) delete process.env.TOKEN_STUDIO_CONFIG;
23
+ else process.env.TOKEN_STUDIO_CONFIG = old;
24
+ }
25
+ });
@@ -0,0 +1,56 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import {
4
+ collectableCollectors,
5
+ collectorLabel,
6
+ detectCollectors,
7
+ enabledCollectorIds,
8
+ stableCollectors
9
+ } from '../src/collector-registry.mjs';
10
+
11
+ test('collector registry exposes six stable v4 sources', () => {
12
+ const stable = stableCollectors().map(item => item.id).sort();
13
+ assert.deepEqual(stable, ['claude', 'codex', 'gemini', 'hermes', 'openclaw', 'opencode']);
14
+ assert.equal(collectorLabel('codex'), 'Codex CLI');
15
+ });
16
+
17
+ test('collector detection includes v4.1 experimental source metadata', () => {
18
+ const rows = detectCollectors();
19
+ const cursor = rows.find(item => item.id === 'cursor');
20
+ const copilot = rows.find(item => item.id === 'copilot');
21
+ assert.equal(cursor.supportStatus, 'experimental');
22
+ assert.equal(copilot.defaultEnabled, false);
23
+ assert.equal(cursor.readsConversationContent, false);
24
+ assert.equal(cursor.tokenReliability, 'explicit-token-fields-only');
25
+ assert.ok(cursor.dataFields.includes('input_tokens'));
26
+ });
27
+
28
+ test('enabled collectors ignore experimental ids by default', () => {
29
+ const old = process.env.TOKEN_STUDIO_COLLECTORS;
30
+ process.env.TOKEN_STUDIO_COLLECTORS = 'claude,cursor,codex';
31
+ try {
32
+ assert.deepEqual(Array.from(enabledCollectorIds()).sort(), ['claude', 'codex']);
33
+ assert.deepEqual(Array.from(enabledCollectorIds({ includeExperimental: true })).sort(), ['claude', 'codex', 'cursor']);
34
+ } finally {
35
+ if (old == null) delete process.env.TOKEN_STUDIO_COLLECTORS;
36
+ else process.env.TOKEN_STUDIO_COLLECTORS = old;
37
+ }
38
+ });
39
+
40
+ test('collectable collectors include experimental modules only when requested', () => {
41
+ assert.equal(collectableCollectors().some(item => item.id === 'cursor'), false);
42
+ assert.equal(collectableCollectors({ includeExperimental: true }).some(item => item.id === 'cursor'), true);
43
+ });
44
+
45
+ test('collector matrix includes import-only and detected-only entries without collect modules', () => {
46
+ const rows = detectCollectors();
47
+ const ccusage = rows.find(item => item.id === 'ccusage');
48
+ const amp = rows.find(item => item.id === 'amp');
49
+ const roo = rows.find(item => item.id === 'roo-code');
50
+ assert.equal(ccusage.supportStatus, 'import-only');
51
+ assert.equal(ccusage.module, null);
52
+ assert.equal(ccusage.readsConversationContent, false);
53
+ assert.equal(amp.supportStatus, 'detected-only');
54
+ assert.equal(roo.tokenReliability, 'unknown-no-usage-import');
55
+ assert.equal(collectableCollectors({ includeExperimental: true }).some(item => item.id === 'ccusage'), false);
56
+ });
@@ -0,0 +1,19 @@
1
+ import assert from 'node:assert/strict';
2
+ import test from 'node:test';
3
+ import { U } from '../src/client/shared/utils.js';
4
+
5
+ test('csvCell escapes spreadsheet formula prefixes', () => {
6
+ assert.equal(U.csvCell('=IMPORTXML("https://example.com")'), '"\'=IMPORTXML(""https://example.com"")"');
7
+ assert.equal(U.csvCell('+cmd|calc'), "'+cmd|calc");
8
+ assert.equal(U.csvCell('-1'), "'-1");
9
+ assert.equal(U.csvCell('@SUM(A1:A2)'), "'@SUM(A1:A2)");
10
+ assert.equal(U.csvCell('\t=SUM(A1:A2)'), "'\t=SUM(A1:A2)");
11
+ assert.equal(U.csvCell('\r=SUM(A1:A2)'), '"\'\r=SUM(A1:A2)"');
12
+ });
13
+
14
+ test('csvCell still applies RFC-style quoting for commas and quotes', () => {
15
+ assert.equal(U.csvCell('safe text'), 'safe text');
16
+ assert.equal(U.csvCell('hello,world'), '"hello,world"');
17
+ assert.equal(U.csvCell('say "hi"'), '"say ""hi"""');
18
+ assert.equal(U.csvCell(null), '');
19
+ });