tokenusage-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +22 -0
- package/README.md +94 -0
- package/THIRD_PARTY_NOTICES.md +140 -0
- package/bin/tokenusage.js +69 -0
- package/package.json +59 -0
- package/src/cli.js +134 -0
- package/src/commands/device-login.js +205 -0
- package/src/commands/diagnostics.js +38 -0
- package/src/commands/doctor.js +110 -0
- package/src/commands/init.js +2436 -0
- package/src/commands/sessions.js +29 -0
- package/src/commands/status.js +1747 -0
- package/src/commands/sync.js +7934 -0
- package/src/commands/uninstall.js +312 -0
- package/src/lib/account-usage-dedup.js +267 -0
- package/src/lib/antigravity-paths.js +48 -0
- package/src/lib/ark-agent-plan-limits.js +438 -0
- package/src/lib/ark-coding-plan-limits.js +537 -0
- package/src/lib/async-queue-reader.js +138 -0
- package/src/lib/atomic-json.js +27 -0
- package/src/lib/bearer-token.js +11 -0
- package/src/lib/categorizer-utils.js +232 -0
- package/src/lib/claude-categorizer.js +1343 -0
- package/src/lib/claude-config.js +279 -0
- package/src/lib/claude-user-identity.js +56 -0
- package/src/lib/cli-ui.js +179 -0
- package/src/lib/cloud-account.js +541 -0
- package/src/lib/cloud-device-token.js +141 -0
- package/src/lib/cloud-sync-prefs.js +14 -0
- package/src/lib/cloud-usage-reset.js +115 -0
- package/src/lib/codex-config.js +595 -0
- package/src/lib/codex-context-breakdown.js +806 -0
- package/src/lib/codex-model-attribution.js +128 -0
- package/src/lib/codex-rollout-parser.js +1471 -0
- package/src/lib/codex-service-tier.js +49 -0
- package/src/lib/codex-token-refresh.js +140 -0
- package/src/lib/codex-token-usage.js +195 -0
- package/src/lib/codex-usage-record.js +195 -0
- package/src/lib/command-runner.js +313 -0
- package/src/lib/commandcode-limits.js +317 -0
- package/src/lib/context-health.js +124 -0
- package/src/lib/cursor-config.js +605 -0
- package/src/lib/cursor-store.js +1918 -0
- package/src/lib/debug-flags.js +7 -0
- package/src/lib/devin-limits.js +249 -0
- package/src/lib/devin-usage.js +212 -0
- package/src/lib/diagnostics.js +383 -0
- package/src/lib/doctor.js +385 -0
- package/src/lib/dumate.js +95 -0
- package/src/lib/fs.js +455 -0
- package/src/lib/function-url.js +14 -0
- package/src/lib/gemini-config.js +291 -0
- package/src/lib/git-outcomes.js +243 -0
- package/src/lib/grok-context-breakdown.js +890 -0
- package/src/lib/grok-hook.js +360 -0
- package/src/lib/grok-limits.js +865 -0
- package/src/lib/grok-usage.js +98 -0
- package/src/lib/heatmap-compact.js +114 -0
- package/src/lib/ingest-sanitize.js +41 -0
- package/src/lib/init-flow.js +42 -0
- package/src/lib/install-resolver.js +170 -0
- package/src/lib/jsonl-lines.js +134 -0
- package/src/lib/kimi-model-repair.js +173 -0
- package/src/lib/kimi-profile.js +61 -0
- package/src/lib/kimi-session.js +216 -0
- package/src/lib/local-api.js +3588 -0
- package/src/lib/machine-id.js +238 -0
- package/src/lib/multi-install-parser.js +113 -0
- package/src/lib/notify-command.js +38 -0
- package/src/lib/omp-hook.js +589 -0
- package/src/lib/openclaw-cli.js +104 -0
- package/src/lib/openclaw-hook.js +375 -0
- package/src/lib/openclaw-session-plugin.js +560 -0
- package/src/lib/opencode-config.js +121 -0
- package/src/lib/opencode-go-limits.js +807 -0
- package/src/lib/outcomes-engine.js +253 -0
- package/src/lib/passive-mode.js +219 -0
- package/src/lib/pet-packages.js +722 -0
- package/src/lib/pricing/curated-overrides.json +822 -0
- package/src/lib/pricing/index.js +379 -0
- package/src/lib/pricing/litellm-fetcher.js +172 -0
- package/src/lib/pricing/matcher.js +456 -0
- package/src/lib/pricing/seed-snapshot.json +1 -0
- package/src/lib/progress.js +76 -0
- package/src/lib/project-usage-purge.js +117 -0
- package/src/lib/prompt.js +24 -0
- package/src/lib/provider-status.js +104 -0
- package/src/lib/proxy-env.js +489 -0
- package/src/lib/proxy-settings.js +163 -0
- package/src/lib/qoder-cn-cli-limits.js +119 -0
- package/src/lib/qoder-limits.js +880 -0
- package/src/lib/quiet-warnings.js +26 -0
- package/src/lib/rollout.js +25966 -0
- package/src/lib/runtime-config.js +259 -0
- package/src/lib/scan-roots.js +277 -0
- package/src/lib/sentry.js +231 -0
- package/src/lib/session-analytics.js +2972 -0
- package/src/lib/session-performance.js +237 -0
- package/src/lib/session-upload.js +379 -0
- package/src/lib/skill-usage.js +267 -0
- package/src/lib/skills-manager.js +1591 -0
- package/src/lib/source-metadata.js +48 -0
- package/src/lib/sqlite-reader.js +215 -0
- package/src/lib/star-cta.js +73 -0
- package/src/lib/subscription-manager.js +293 -0
- package/src/lib/subscriptions.js +508 -0
- package/src/lib/telemetry.js +193 -0
- package/src/lib/tracker-paths.js +16 -0
- package/src/lib/trae-cn-config.js +548 -0
- package/src/lib/trae-db.js +127 -0
- package/src/lib/trae-snapshot.js +230 -0
- package/src/lib/trae-sqlite.js +143 -0
- package/src/lib/trae-usage.js +121 -0
- package/src/lib/upload-queue.js +134 -0
- package/src/lib/upload-throttle.js +171 -0
- package/src/lib/usage-limits.js +4552 -0
- package/src/lib/wrapped-aggregator.js +236 -0
- package/src/lib/wsl-probe.js +281 -0
- package/src/lib/zcode-limits.js +1172 -0
- package/src/lib/zcode-reset-credits.js +66 -0
- package/test/cli-smoke.test.js +53 -0
- package/test/ingest-sanitize.test.js +162 -0
- package/test/quiet-warnings.test.js +53 -0
- package/test/session-upload.test.js +427 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Estimates fixed context overhead without returning file contents, prompts,
|
|
4
|
+
// tool schemas, secrets, or command arguments.
|
|
5
|
+
|
|
6
|
+
const fs = require("node:fs");
|
|
7
|
+
const os = require("node:os");
|
|
8
|
+
const path = require("node:path");
|
|
9
|
+
const contextCache = new Map();
|
|
10
|
+
|
|
11
|
+
function estimateTokens(text) {
|
|
12
|
+
const value = String(text || "");
|
|
13
|
+
let cjk = 0;
|
|
14
|
+
for (const char of value) if (/\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}/u.test(char)) cjk += 1;
|
|
15
|
+
return Math.ceil(cjk + (value.length - cjk) / 4);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function readMetadata(filePath, kind, root) {
|
|
19
|
+
try {
|
|
20
|
+
const text = fs.readFileSync(filePath, "utf8");
|
|
21
|
+
return { kind, name: path.relative(root, filePath) || path.basename(filePath), bytes: Buffer.byteLength(text), estimated_tokens: estimateTokens(text) };
|
|
22
|
+
} catch { return null; }
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function walkNamedFiles(root, targetName, limit = 500) {
|
|
26
|
+
const out = [];
|
|
27
|
+
const stack = [root];
|
|
28
|
+
while (stack.length && out.length < limit) {
|
|
29
|
+
const dir = stack.pop();
|
|
30
|
+
let entries;
|
|
31
|
+
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { continue; }
|
|
32
|
+
for (const entry of entries) {
|
|
33
|
+
if ([".git", "node_modules", "dist", "build"].includes(entry.name)) continue;
|
|
34
|
+
const filePath = path.join(dir, entry.name);
|
|
35
|
+
if (entry.isDirectory()) stack.push(filePath);
|
|
36
|
+
else if (entry.isFile() && entry.name === targetName) out.push(filePath);
|
|
37
|
+
if (out.length >= limit) break;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function countMcpServers(home, cwd) {
|
|
44
|
+
const files = [
|
|
45
|
+
path.join(home, ".claude.json"),
|
|
46
|
+
path.join(home, ".claude", "settings.json"),
|
|
47
|
+
path.join(cwd, ".mcp.json"),
|
|
48
|
+
];
|
|
49
|
+
const names = new Set();
|
|
50
|
+
for (const filePath of files) {
|
|
51
|
+
try {
|
|
52
|
+
const data = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
53
|
+
const servers = data.mcpServers || data.mcp_servers || {};
|
|
54
|
+
for (const name of Object.keys(servers)) names.add(name);
|
|
55
|
+
} catch { /* absent or not JSON */ }
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const toml = fs.readFileSync(path.join(home, ".codex", "config.toml"), "utf8");
|
|
59
|
+
for (const match of toml.matchAll(/^\s*\[mcp_servers\.([^\]]+)\]/gm)) names.add(match[1]);
|
|
60
|
+
} catch { /* absent */ }
|
|
61
|
+
return names.size;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function computeContextHealth({ home = os.homedir(), cwd = process.cwd(), env = process.env, cacheTtlMs = 5 * 60_000 } = {}) {
|
|
65
|
+
const cacheKey = `${home}\0${cwd}\0${env.TOKENUSAGE_MCP_TOOL_SCHEMA_TOKENS || ""}\0${env.TOKENUSAGE_MCP_TOOLS_PER_SERVER || ""}`;
|
|
66
|
+
const cached = contextCache.get(cacheKey);
|
|
67
|
+
if (cached && Date.now() - cached.at < cacheTtlMs) return cached.value;
|
|
68
|
+
const files = [];
|
|
69
|
+
const known = [
|
|
70
|
+
[path.join(home, ".claude", "CLAUDE.md"), "instruction", home],
|
|
71
|
+
[path.join(home, ".codex", "AGENTS.md"), "instruction", home],
|
|
72
|
+
[path.join(cwd, "CLAUDE.md"), "instruction", cwd],
|
|
73
|
+
[path.join(cwd, "AGENTS.md"), "instruction", cwd],
|
|
74
|
+
];
|
|
75
|
+
for (const [filePath, kind, root] of known) {
|
|
76
|
+
const item = readMetadata(filePath, kind, root);
|
|
77
|
+
if (item) files.push(item);
|
|
78
|
+
}
|
|
79
|
+
const skillRoots = [
|
|
80
|
+
path.join(home, ".claude", "skills"),
|
|
81
|
+
path.join(home, ".codex", "skills"),
|
|
82
|
+
path.join(cwd, ".claude", "skills"),
|
|
83
|
+
path.join(cwd, ".agents", "skills"),
|
|
84
|
+
];
|
|
85
|
+
for (const root of skillRoots) {
|
|
86
|
+
for (const filePath of walkNamedFiles(root, "SKILL.md")) {
|
|
87
|
+
const item = readMetadata(filePath, "skill", root);
|
|
88
|
+
if (item) files.push(item);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const instructionTokens = files.filter((row) => row.kind === "instruction").reduce((sum, row) => sum + row.estimated_tokens, 0);
|
|
92
|
+
const skillTokens = files.filter((row) => row.kind === "skill").reduce((sum, row) => sum + row.estimated_tokens, 0);
|
|
93
|
+
const mcpServers = countMcpServers(home, cwd);
|
|
94
|
+
const perToolSchemaTokens = Math.max(0, Number(env.TOKENUSAGE_MCP_TOOL_SCHEMA_TOKENS || 400));
|
|
95
|
+
const assumedToolsPerServer = Math.max(1, Number(env.TOKENUSAGE_MCP_TOOLS_PER_SERVER || 5));
|
|
96
|
+
const mcpSchemaTokens = mcpServers * perToolSchemaTokens * assumedToolsPerServer;
|
|
97
|
+
const total = instructionTokens + skillTokens + mcpSchemaTokens;
|
|
98
|
+
const severity = total >= 50_000 ? "high" : total >= 20_000 ? "medium" : "low";
|
|
99
|
+
const result = {
|
|
100
|
+
generated_at: new Date().toISOString(),
|
|
101
|
+
estimated_fixed_tokens: total,
|
|
102
|
+
severity,
|
|
103
|
+
breakdown: {
|
|
104
|
+
instruction_files: files.filter((row) => row.kind === "instruction").length,
|
|
105
|
+
instruction_tokens: instructionTokens,
|
|
106
|
+
skills: files.filter((row) => row.kind === "skill").length,
|
|
107
|
+
skill_tokens: skillTokens,
|
|
108
|
+
mcp_servers: mcpServers,
|
|
109
|
+
mcp_schema_tokens: mcpSchemaTokens,
|
|
110
|
+
},
|
|
111
|
+
largest_items: files.sort((a, b) => b.estimated_tokens - a.estimated_tokens).slice(0, 10),
|
|
112
|
+
provenance: {
|
|
113
|
+
confidence: "inferred",
|
|
114
|
+
source: "local-config-metadata",
|
|
115
|
+
mcp_assumption: { tools_per_server: assumedToolsPerServer, tokens_per_tool_schema: perToolSchemaTokens },
|
|
116
|
+
content_retained: false,
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
contextCache.set(cacheKey, { at: Date.now(), value: result });
|
|
120
|
+
if (contextCache.size > 8) contextCache.delete(contextCache.keys().next().value);
|
|
121
|
+
return result;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
module.exports = { estimateTokens, computeContextHealth, countMcpServers };
|