tokenusage-cli 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -19
- package/LICENSE +22 -22
- package/README.md +123 -94
- package/THIRD_PARTY_NOTICES.md +140 -140
- package/bin/tokenusage.js +69 -69
- package/package.json +59 -59
- package/src/cli.js +146 -134
- package/src/commands/device-login.js +205 -205
- package/src/commands/diagnostics.js +38 -38
- package/src/commands/doctor.js +110 -110
- package/src/commands/init.js +2484 -2436
- package/src/commands/pause.js +77 -0
- package/src/commands/sessions.js +29 -29
- package/src/commands/status.js +1758 -1747
- package/src/commands/sync.js +8048 -7934
- package/src/commands/uninstall.js +322 -312
- package/src/lib/account-usage-dedup.js +267 -267
- package/src/lib/antigravity-paths.js +48 -48
- package/src/lib/ark-agent-plan-limits.js +438 -438
- package/src/lib/ark-coding-plan-limits.js +537 -537
- package/src/lib/async-queue-reader.js +138 -138
- package/src/lib/atomic-json.js +27 -27
- package/src/lib/bearer-token.js +11 -11
- package/src/lib/categorizer-utils.js +232 -232
- package/src/lib/claude-categorizer.js +1343 -1343
- package/src/lib/claude-config.js +279 -279
- package/src/lib/claude-user-identity.js +56 -56
- package/src/lib/cli-ui.js +179 -179
- package/src/lib/cloud-account.js +541 -541
- package/src/lib/cloud-device-token.js +141 -141
- package/src/lib/cloud-sync-prefs.js +14 -14
- package/src/lib/cloud-usage-reset.js +115 -115
- package/src/lib/codex-config.js +595 -595
- package/src/lib/codex-context-breakdown.js +806 -806
- package/src/lib/codex-model-attribution.js +128 -128
- package/src/lib/codex-rollout-parser.js +1471 -1471
- package/src/lib/codex-service-tier.js +49 -49
- package/src/lib/codex-token-refresh.js +140 -140
- package/src/lib/codex-token-usage.js +195 -195
- package/src/lib/codex-usage-record.js +195 -195
- package/src/lib/collection-pause.js +77 -0
- package/src/lib/command-runner.js +313 -313
- package/src/lib/commandcode-limits.js +317 -317
- package/src/lib/context-health.js +124 -124
- package/src/lib/cursor-config.js +605 -605
- package/src/lib/cursor-store.js +1918 -1918
- package/src/lib/debug-flags.js +7 -7
- package/src/lib/devin-limits.js +249 -249
- package/src/lib/devin-usage.js +212 -212
- package/src/lib/diagnostics.js +383 -383
- package/src/lib/doctor.js +385 -385
- package/src/lib/dumate.js +95 -95
- package/src/lib/fs.js +455 -455
- package/src/lib/function-url.js +14 -14
- package/src/lib/gemini-config.js +291 -291
- package/src/lib/git-outcomes.js +243 -243
- package/src/lib/grok-context-breakdown.js +890 -890
- package/src/lib/grok-hook.js +360 -360
- package/src/lib/grok-limits.js +865 -865
- package/src/lib/grok-usage.js +98 -98
- package/src/lib/heatmap-compact.js +114 -114
- package/src/lib/ingest-sanitize.js +41 -41
- package/src/lib/init-flow.js +42 -42
- package/src/lib/install-resolver.js +170 -170
- package/src/lib/jsonl-lines.js +134 -134
- package/src/lib/kimi-model-repair.js +173 -173
- package/src/lib/kimi-profile.js +61 -61
- package/src/lib/kimi-session.js +216 -216
- package/src/lib/local-api.js +3588 -3588
- package/src/lib/machine-id.js +238 -238
- package/src/lib/multi-install-parser.js +113 -113
- package/src/lib/notify-command.js +38 -38
- package/src/lib/omp-hook.js +589 -589
- package/src/lib/openclaw-cli.js +104 -104
- package/src/lib/openclaw-hook.js +375 -375
- package/src/lib/openclaw-session-plugin.js +560 -560
- package/src/lib/opencode-config.js +121 -121
- package/src/lib/opencode-go-limits.js +807 -807
- package/src/lib/outcomes-engine.js +253 -253
- package/src/lib/passive-mode.js +219 -219
- package/src/lib/pet-packages.js +722 -722
- package/src/lib/pricing/curated-overrides.json +822 -822
- package/src/lib/pricing/index.js +379 -379
- package/src/lib/pricing/litellm-fetcher.js +172 -172
- package/src/lib/pricing/matcher.js +456 -456
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/progress.js +76 -76
- package/src/lib/project-usage-purge.js +117 -117
- package/src/lib/prompt.js +24 -24
- package/src/lib/provider-status.js +104 -104
- package/src/lib/proxy-env.js +489 -489
- package/src/lib/proxy-settings.js +163 -163
- package/src/lib/qoder-cn-cli-limits.js +119 -119
- package/src/lib/qoder-limits.js +880 -880
- package/src/lib/quiet-warnings.js +26 -26
- package/src/lib/rollout.js +26005 -25966
- package/src/lib/runtime-config.js +9 -4
- package/src/lib/scan-roots.js +277 -277
- package/src/lib/scheduled-task.js +145 -0
- package/src/lib/sentry.js +231 -231
- package/src/lib/session-analytics.js +2972 -2972
- package/src/lib/session-performance.js +237 -237
- package/src/lib/session-upload.js +379 -379
- package/src/lib/skill-usage.js +267 -267
- package/src/lib/skills-manager.js +1591 -1591
- package/src/lib/source-metadata.js +48 -48
- package/src/lib/sqlite-reader.js +215 -215
- package/src/lib/star-cta.js +73 -73
- package/src/lib/subscription-manager.js +293 -293
- package/src/lib/subscriptions.js +508 -508
- package/src/lib/sync-report.js +138 -0
- package/src/lib/telemetry.js +193 -193
- package/src/lib/tracker-paths.js +16 -16
- package/src/lib/trae-cn-config.js +548 -548
- package/src/lib/trae-db.js +127 -127
- package/src/lib/trae-snapshot.js +230 -230
- package/src/lib/trae-sqlite.js +143 -143
- package/src/lib/trae-usage.js +121 -121
- package/src/lib/upload-queue.js +134 -134
- package/src/lib/upload-throttle.js +171 -171
- package/src/lib/usage-limits.js +4552 -4552
- package/src/lib/wrapped-aggregator.js +236 -236
- package/src/lib/wsl-probe.js +281 -281
- package/src/lib/zcode-limits.js +1172 -1172
- package/src/lib/zcode-reset-credits.js +66 -66
- package/test/audit-fixes.test.js +200 -0
- package/test/cli-smoke.test.js +53 -53
- package/test/collection-controls.test.js +210 -0
- package/test/ingest-sanitize.test.js +162 -162
- package/test/quiet-warnings.test.js +53 -53
- package/test/session-upload.test.js +427 -427
|
@@ -31,22 +31,27 @@ function resolveRuntimeConfig({ cli = {}, config = {}, env = process.env, defaul
|
|
|
31
31
|
// isolation bug is fixed, but existing installs must recover instead of
|
|
32
32
|
// backing off cloud uploads forever against the reserved placeholder host.
|
|
33
33
|
const persistedBaseUrl = normalizePersistedBaseUrl(config.baseUrl);
|
|
34
|
+
// envBaseNewName wins over the retired upstream-era INSFORGE_* names (audit
|
|
35
|
+
// fix: the backend is token-usage's own Supabase deployment, not InsForge);
|
|
36
|
+
// the old names stay as a legacy fallback so existing scripts keep working.
|
|
37
|
+
const envBaseUrl = env?.TOKENUSAGE_BASE_URL ?? env?.TOKENUSAGE_INSFORGE_BASE_URL;
|
|
38
|
+
const envAnonKey = env?.TOKENUSAGE_ANON_KEY ?? env?.TOKENUSAGE_INSFORGE_ANON_KEY;
|
|
34
39
|
const baseUrl = pickString(
|
|
35
40
|
cli.baseUrl,
|
|
36
41
|
persistedBaseUrl,
|
|
37
|
-
|
|
42
|
+
envBaseUrl,
|
|
38
43
|
defaults.baseUrl,
|
|
39
44
|
DEFAULT_BASE_URL,
|
|
40
45
|
);
|
|
41
46
|
const selectedBase = normalizeInstanceBaseUrl(baseUrl.value);
|
|
42
47
|
const configBase = normalizeInstanceBaseUrl(persistedBaseUrl || DEFAULT_BASE_URL);
|
|
43
|
-
const envBase = normalizeInstanceBaseUrl(
|
|
48
|
+
const envBase = normalizeInstanceBaseUrl(envBaseUrl || baseUrl.value);
|
|
44
49
|
const defaultBase = normalizeInstanceBaseUrl(defaults.baseUrl || DEFAULT_BASE_URL);
|
|
45
50
|
const custom = selectedBase !== DEFAULT_BASE_URL;
|
|
46
51
|
const anonKey = pickString(
|
|
47
52
|
cli.anonKey,
|
|
48
53
|
configBase === selectedBase ? config.anonKey : undefined,
|
|
49
|
-
envBase === selectedBase ?
|
|
54
|
+
envBase === selectedBase ? envAnonKey : undefined,
|
|
50
55
|
defaultBase === selectedBase ? defaults.anonKey : undefined,
|
|
51
56
|
custom ? null : DEFAULT_ANON_KEY,
|
|
52
57
|
);
|
|
@@ -61,7 +66,7 @@ function resolveRuntimeConfig({ cli = {}, config = {}, env = process.env, defaul
|
|
|
61
66
|
cli.deviceToken,
|
|
62
67
|
configBase === selectedBase && (!custom || normalizeInstanceBaseUrl(config.deviceTokenBaseUrl) === selectedBase)
|
|
63
68
|
? config.deviceToken : undefined,
|
|
64
|
-
envBase === selectedBase && (!custom || normalizeInstanceBaseUrl(
|
|
69
|
+
envBase === selectedBase && (!custom || normalizeInstanceBaseUrl(envBaseUrl) === selectedBase)
|
|
65
70
|
? env?.TOKENUSAGE_DEVICE_TOKEN : undefined,
|
|
66
71
|
defaultBase === selectedBase ? defaults.deviceToken : undefined,
|
|
67
72
|
null,
|
package/src/lib/scan-roots.js
CHANGED
|
@@ -1,277 +1,277 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
// Persistent extra scan roots for Codex and Claude Code (#657).
|
|
4
|
-
//
|
|
5
|
-
// Agent harnesses run Codex / Claude Code with a private home
|
|
6
|
-
// (CODEX_HOME=..., CLAUDE_CONFIG_DIR=...). Those sessions were invisible to
|
|
7
|
-
// Token Usage unless the sync happened to be spawned by a process that
|
|
8
|
-
// inherited the same environment: a hook-fired sync saw them, the native app's
|
|
9
|
-
// background refresh did not. Worse, the two producers then disagreed about
|
|
10
|
-
// where a rollout's cursor lives (the cursor store classifies a rollout as a
|
|
11
|
-
// Codex path — per-day shard — only when it sits under one of ITS codexRoots,
|
|
12
|
-
// everything else goes to core), never saw each other's cursor, and re-parsed
|
|
13
|
-
// the same file from byte 0 on every alternation (#639: one 30-minute bucket
|
|
14
|
-
// re-emitted 138 times, 59B tokens for a 1.2B day).
|
|
15
|
-
//
|
|
16
|
-
// So roots come from ONE persistent list that every producer reads, and the
|
|
17
|
-
// cursor store's codexRoots are derived from that same list.
|
|
18
|
-
//
|
|
19
|
-
// Per provider, in order (first occurrence wins; later duplicates by realpath
|
|
20
|
-
// are dropped):
|
|
21
|
-
// 1. the roots the caller already scans (native home, WSL install)
|
|
22
|
-
// 2. the spawning process's implicit root — CODEX_HOME / CLAUDE_CONFIG_DIR
|
|
23
|
-
// 3. config.scanRoots.<provider>[] from ~/.tokenusage/tracker/config.json
|
|
24
|
-
//
|
|
25
|
-
// Config shape:
|
|
26
|
-
// { "scanRoots": { "codex": ["~/.local/share/agent/codex"],
|
|
27
|
-
// "claude": ["~/.local/share/agent/claude", "~/.claude-native"] } }
|
|
28
|
-
|
|
29
|
-
const fs = require("node:fs");
|
|
30
|
-
const os = require("node:os");
|
|
31
|
-
const path = require("node:path");
|
|
32
|
-
|
|
33
|
-
const { readJson } = require("./fs");
|
|
34
|
-
const { resolveTrackerPaths } = require("./tracker-paths");
|
|
35
|
-
|
|
36
|
-
const PROVIDERS = Object.freeze({
|
|
37
|
-
codex: Object.freeze({ envVar: "CODEX_HOME" }),
|
|
38
|
-
claude: Object.freeze({ envVar: "CLAUDE_CONFIG_DIR" }),
|
|
39
|
-
});
|
|
40
|
-
const PROVIDER_NAMES = Object.freeze(Object.keys(PROVIDERS));
|
|
41
|
-
|
|
42
|
-
// Turn a configured root into a stable absolute path. `~` expands against
|
|
43
|
-
// `home`, and a RELATIVE entry is resolved against `home` too — never against
|
|
44
|
-
// process.cwd(), which differs between a hook-fired sync, the CLI and the
|
|
45
|
-
// desktop app and would make producers scan different directories.
|
|
46
|
-
function expandHome(value, home) {
|
|
47
|
-
if (typeof value !== "string") return null;
|
|
48
|
-
const trimmed = value.trim();
|
|
49
|
-
if (!trimmed) return null;
|
|
50
|
-
if (trimmed === "~") return path.resolve(home);
|
|
51
|
-
if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) {
|
|
52
|
-
return path.resolve(home, trimmed.slice(2));
|
|
53
|
-
}
|
|
54
|
-
return path.isAbsolute(trimmed) ? path.resolve(trimmed) : path.resolve(home, trimmed);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// The spawning process's implicit root — CODEX_HOME / CLAUDE_CONFIG_DIR —
|
|
58
|
-
// normalized ONCE, the same way configured entries are: `~` and relative
|
|
59
|
-
// values are anchored to `home`, never to process.cwd(). Every producer
|
|
60
|
-
// (sync discovery, the cursor store's codexRoots, status, diagnostics, the
|
|
61
|
-
// session browser) must use this value, or a relative CODEX_HOME such as
|
|
62
|
-
// `profiles/codex` is scanned under one directory and cursor-classified
|
|
63
|
-
// under another, and the same rollout replays. Returns null when unset.
|
|
64
|
-
function resolveEnvRoot(provider, { env = process.env, home = os.homedir() } = {}) {
|
|
65
|
-
const spec = PROVIDERS[provider];
|
|
66
|
-
if (!spec) return null;
|
|
67
|
-
const raw = env && typeof env[spec.envVar] === "string" ? env[spec.envVar] : "";
|
|
68
|
-
return expandHome(raw, home);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// Accepts { codex: string | string[], claude: string | string[] }. Anything
|
|
72
|
-
// else — wrong type, empty strings, unknown providers — is ignored rather than
|
|
73
|
-
// failing the sync: a malformed config must not stop collection.
|
|
74
|
-
function normalizeScanRootsConfig(raw) {
|
|
75
|
-
const out = {};
|
|
76
|
-
for (const provider of PROVIDER_NAMES) out[provider] = [];
|
|
77
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return out;
|
|
78
|
-
for (const provider of PROVIDER_NAMES) {
|
|
79
|
-
const value = raw[provider];
|
|
80
|
-
const list = Array.isArray(value) ? value : (typeof value === "string" ? [value] : []);
|
|
81
|
-
for (const entry of list) {
|
|
82
|
-
if (typeof entry !== "string" || !entry.trim()) continue;
|
|
83
|
-
out[provider].push(entry.trim());
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
return out;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// { exists, error }: `exists` is true only for a directory that can be
|
|
90
|
-
// LISTED, not merely stat'ed — a stat-able directory whose listing fails
|
|
91
|
-
// (mode 000, ACL) would otherwise be admitted to the scan, yield no files,
|
|
92
|
-
// read as "present" in status/doctor, and slip past the deferred-repair
|
|
93
|
-
// guard in sync. ENOENT and ENOTDIR (or a regular file in a directory's
|
|
94
|
-
// place) mean the root is absent (error null); any other failure (EACCES,
|
|
95
|
-
// ELOOP, EIO, ...) is reported as `error` so callers can say "unreadable"
|
|
96
|
-
// instead of "missing".
|
|
97
|
-
function scanRootDirState(target, deps = {}) {
|
|
98
|
-
const statSync = deps.statSync || fs.statSync;
|
|
99
|
-
const opendirSync = deps.opendirSync || fs.opendirSync;
|
|
100
|
-
try {
|
|
101
|
-
if (!statSync(target).isDirectory()) return { exists: false, error: null };
|
|
102
|
-
// Probe listability without reading entries; large roots stay cheap.
|
|
103
|
-
opendirSync(target).closeSync();
|
|
104
|
-
return { exists: true, error: null };
|
|
105
|
-
} catch (e) {
|
|
106
|
-
const code = e && typeof e.code === "string" ? e.code : "EUNKNOWN";
|
|
107
|
-
if (code === "ENOENT" || code === "ENOTDIR") return { exists: false, error: null };
|
|
108
|
-
return { exists: false, error: code };
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
function isDirectorySync(target, deps) {
|
|
113
|
-
return scanRootDirState(target, deps).exists;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
function realpathOrSelf(target, deps) {
|
|
117
|
-
const realpathSync = deps.realpathSync || fs.realpathSync;
|
|
118
|
-
try {
|
|
119
|
-
return realpathSync(target);
|
|
120
|
-
} catch {
|
|
121
|
-
return target;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function identityKey(target, deps) {
|
|
126
|
-
const resolved = path.resolve(target);
|
|
127
|
-
const real = realpathOrSelf(resolved, deps);
|
|
128
|
-
return real.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
// Drop later entries that resolve (realpath) to a directory already listed.
|
|
132
|
-
// Used at the projects/ level too: two Claude profiles may symlink one
|
|
133
|
-
// projects/ dir, and reading it under both spellings would double-parse every
|
|
134
|
-
// file, leaving correctness to the bounded claudeHashes layer.
|
|
135
|
-
function dedupeDirsByRealpath(dirs, deps = {}) {
|
|
136
|
-
const seen = new Set();
|
|
137
|
-
const out = [];
|
|
138
|
-
for (const dir of Array.isArray(dirs) ? dirs : []) {
|
|
139
|
-
if (typeof dir !== "string" || !dir) continue;
|
|
140
|
-
const key = identityKey(dir, deps);
|
|
141
|
-
if (seen.has(key)) continue;
|
|
142
|
-
seen.add(key);
|
|
143
|
-
out.push(dir);
|
|
144
|
-
}
|
|
145
|
-
return out;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
// Resolve the full, ordered, deduplicated root list per provider.
|
|
149
|
-
//
|
|
150
|
-
// base: { codex: string[], claude: string[] } — roots the caller scans
|
|
151
|
-
// anyway (native home, WSL install); they keep their position and
|
|
152
|
-
// are labelled origin "native".
|
|
153
|
-
// env: CODEX_HOME / CLAUDE_CONFIG_DIR become origin "env" unless already
|
|
154
|
-
// covered by base.
|
|
155
|
-
// config.scanRoots.<provider>[] become origin "config".
|
|
156
|
-
//
|
|
157
|
-
// Every entry: { path, realPath, exists, error, origin }. `exists` is a
|
|
158
|
-
// readable-directory check and `error` carries a stat error code other than
|
|
159
|
-
// absence (see scanRootDirState); non-existent or unreadable configured roots
|
|
160
|
-
// are kept in the list (so cursor-path classification stays stable while a
|
|
161
|
-
// volume is unmounted) and callers that walk directories skip them via
|
|
162
|
-
// `exists`.
|
|
163
|
-
function resolveScanRoots({
|
|
164
|
-
home = os.homedir(),
|
|
165
|
-
env = process.env,
|
|
166
|
-
config = null,
|
|
167
|
-
base = {},
|
|
168
|
-
deps = {},
|
|
169
|
-
} = {}) {
|
|
170
|
-
const configured = normalizeScanRootsConfig(config && typeof config === "object" ? config.scanRoots : null);
|
|
171
|
-
const out = {};
|
|
172
|
-
for (const provider of PROVIDER_NAMES) {
|
|
173
|
-
const candidates = [];
|
|
174
|
-
for (const root of Array.isArray(base[provider]) ? base[provider] : []) {
|
|
175
|
-
if (typeof root === "string" && root.trim()) candidates.push({ raw: root, origin: "native" });
|
|
176
|
-
}
|
|
177
|
-
const envRoot = resolveEnvRoot(provider, { env, home });
|
|
178
|
-
if (envRoot) candidates.push({ raw: envRoot, origin: "env" });
|
|
179
|
-
for (const root of configured[provider]) candidates.push({ raw: root, origin: "config" });
|
|
180
|
-
|
|
181
|
-
const seen = new Set();
|
|
182
|
-
const entries = [];
|
|
183
|
-
for (const candidate of candidates) {
|
|
184
|
-
const expanded = expandHome(candidate.raw, home);
|
|
185
|
-
if (!expanded) continue;
|
|
186
|
-
const resolved = expanded;
|
|
187
|
-
const key = identityKey(resolved, deps);
|
|
188
|
-
if (seen.has(key)) continue;
|
|
189
|
-
seen.add(key);
|
|
190
|
-
const state = scanRootDirState(resolved, deps);
|
|
191
|
-
entries.push({
|
|
192
|
-
path: resolved,
|
|
193
|
-
realPath: realpathOrSelf(resolved, deps),
|
|
194
|
-
exists: state.exists,
|
|
195
|
-
error: state.error,
|
|
196
|
-
origin: candidate.origin,
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
out[provider] = entries;
|
|
200
|
-
}
|
|
201
|
-
return out;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
// Append `extraDirs` to `baseDirs`, dropping any extra that resolves (realpath)
|
|
205
|
-
// to a directory already present. Base entries are never collapsed against
|
|
206
|
-
// each other: the native + WSL pair is listed on purpose (see sync.js) and
|
|
207
|
-
// their dedup is handled downstream by file-identity hashes.
|
|
208
|
-
function appendUniqueDirs(baseDirs, extraDirs, deps = {}) {
|
|
209
|
-
const out = [];
|
|
210
|
-
const seen = new Set();
|
|
211
|
-
for (const dir of Array.isArray(baseDirs) ? baseDirs : []) {
|
|
212
|
-
if (typeof dir !== "string" || !dir) continue;
|
|
213
|
-
seen.add(identityKey(dir, deps));
|
|
214
|
-
out.push(dir);
|
|
215
|
-
}
|
|
216
|
-
for (const dir of Array.isArray(extraDirs) ? extraDirs : []) {
|
|
217
|
-
if (typeof dir !== "string" || !dir) continue;
|
|
218
|
-
const key = identityKey(dir, deps);
|
|
219
|
-
if (seen.has(key)) continue;
|
|
220
|
-
seen.add(key);
|
|
221
|
-
out.push(dir);
|
|
222
|
-
}
|
|
223
|
-
return out;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
// Roots beyond what the caller already scans: the "env" and "config" entries
|
|
227
|
-
// that exist on disk. Paths only, in list order.
|
|
228
|
-
function extraScanRootPaths(entries) {
|
|
229
|
-
return (Array.isArray(entries) ? entries : [])
|
|
230
|
-
.filter((entry) => entry && entry.origin !== "native" && entry.exists)
|
|
231
|
-
.map((entry) => entry.path);
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
function hasAnyScanChild(root, children, deps = {}) {
|
|
235
|
-
for (const child of Array.isArray(children) ? children : [children]) {
|
|
236
|
-
if (isDirectorySync(path.join(root, child), deps)) return true;
|
|
237
|
-
}
|
|
238
|
-
return false;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
// Human label for status / doctor output.
|
|
242
|
-
function describeScanRootOrigin(entry, provider) {
|
|
243
|
-
if (!entry || entry.origin === "native") return "native";
|
|
244
|
-
if (entry.origin === "env") return PROVIDERS[provider]?.envVar || "env";
|
|
245
|
-
return "scanRoots";
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
// "" for a readable root, " (missing)" for an absent one, " (unreadable: CODE)"
|
|
249
|
-
// for a stat failure other than absence.
|
|
250
|
-
function describeScanRootState(entry) {
|
|
251
|
-
if (!entry || entry.exists) return "";
|
|
252
|
-
return entry.error ? ` (unreadable: ${entry.error})` : " (missing)";
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
// config.scanRoots as persisted in ~/.tokenusage/tracker/config.json for
|
|
256
|
-
// callers that do not already hold the config (the session browser).
|
|
257
|
-
async function loadScanRootsConfig({ home = os.homedir() } = {}) {
|
|
258
|
-
const { trackerDir } = await resolveTrackerPaths({ home });
|
|
259
|
-
const config = await readJson(path.join(trackerDir, "config.json")).catch(() => null);
|
|
260
|
-
return normalizeScanRootsConfig(config && typeof config === "object" ? config.scanRoots : null);
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
module.exports = {
|
|
264
|
-
PROVIDER_NAMES,
|
|
265
|
-
appendUniqueDirs,
|
|
266
|
-
dedupeDirsByRealpath,
|
|
267
|
-
describeScanRootOrigin,
|
|
268
|
-
describeScanRootState,
|
|
269
|
-
expandHome,
|
|
270
|
-
extraScanRootPaths,
|
|
271
|
-
hasAnyScanChild,
|
|
272
|
-
loadScanRootsConfig,
|
|
273
|
-
normalizeScanRootsConfig,
|
|
274
|
-
resolveEnvRoot,
|
|
275
|
-
resolveScanRoots,
|
|
276
|
-
scanRootDirState,
|
|
277
|
-
};
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Persistent extra scan roots for Codex and Claude Code (#657).
|
|
4
|
+
//
|
|
5
|
+
// Agent harnesses run Codex / Claude Code with a private home
|
|
6
|
+
// (CODEX_HOME=..., CLAUDE_CONFIG_DIR=...). Those sessions were invisible to
|
|
7
|
+
// Token Usage unless the sync happened to be spawned by a process that
|
|
8
|
+
// inherited the same environment: a hook-fired sync saw them, the native app's
|
|
9
|
+
// background refresh did not. Worse, the two producers then disagreed about
|
|
10
|
+
// where a rollout's cursor lives (the cursor store classifies a rollout as a
|
|
11
|
+
// Codex path — per-day shard — only when it sits under one of ITS codexRoots,
|
|
12
|
+
// everything else goes to core), never saw each other's cursor, and re-parsed
|
|
13
|
+
// the same file from byte 0 on every alternation (#639: one 30-minute bucket
|
|
14
|
+
// re-emitted 138 times, 59B tokens for a 1.2B day).
|
|
15
|
+
//
|
|
16
|
+
// So roots come from ONE persistent list that every producer reads, and the
|
|
17
|
+
// cursor store's codexRoots are derived from that same list.
|
|
18
|
+
//
|
|
19
|
+
// Per provider, in order (first occurrence wins; later duplicates by realpath
|
|
20
|
+
// are dropped):
|
|
21
|
+
// 1. the roots the caller already scans (native home, WSL install)
|
|
22
|
+
// 2. the spawning process's implicit root — CODEX_HOME / CLAUDE_CONFIG_DIR
|
|
23
|
+
// 3. config.scanRoots.<provider>[] from ~/.tokenusage/tracker/config.json
|
|
24
|
+
//
|
|
25
|
+
// Config shape:
|
|
26
|
+
// { "scanRoots": { "codex": ["~/.local/share/agent/codex"],
|
|
27
|
+
// "claude": ["~/.local/share/agent/claude", "~/.claude-native"] } }
|
|
28
|
+
|
|
29
|
+
const fs = require("node:fs");
|
|
30
|
+
const os = require("node:os");
|
|
31
|
+
const path = require("node:path");
|
|
32
|
+
|
|
33
|
+
const { readJson } = require("./fs");
|
|
34
|
+
const { resolveTrackerPaths } = require("./tracker-paths");
|
|
35
|
+
|
|
36
|
+
const PROVIDERS = Object.freeze({
|
|
37
|
+
codex: Object.freeze({ envVar: "CODEX_HOME" }),
|
|
38
|
+
claude: Object.freeze({ envVar: "CLAUDE_CONFIG_DIR" }),
|
|
39
|
+
});
|
|
40
|
+
const PROVIDER_NAMES = Object.freeze(Object.keys(PROVIDERS));
|
|
41
|
+
|
|
42
|
+
// Turn a configured root into a stable absolute path. `~` expands against
|
|
43
|
+
// `home`, and a RELATIVE entry is resolved against `home` too — never against
|
|
44
|
+
// process.cwd(), which differs between a hook-fired sync, the CLI and the
|
|
45
|
+
// desktop app and would make producers scan different directories.
|
|
46
|
+
function expandHome(value, home) {
|
|
47
|
+
if (typeof value !== "string") return null;
|
|
48
|
+
const trimmed = value.trim();
|
|
49
|
+
if (!trimmed) return null;
|
|
50
|
+
if (trimmed === "~") return path.resolve(home);
|
|
51
|
+
if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) {
|
|
52
|
+
return path.resolve(home, trimmed.slice(2));
|
|
53
|
+
}
|
|
54
|
+
return path.isAbsolute(trimmed) ? path.resolve(trimmed) : path.resolve(home, trimmed);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// The spawning process's implicit root — CODEX_HOME / CLAUDE_CONFIG_DIR —
|
|
58
|
+
// normalized ONCE, the same way configured entries are: `~` and relative
|
|
59
|
+
// values are anchored to `home`, never to process.cwd(). Every producer
|
|
60
|
+
// (sync discovery, the cursor store's codexRoots, status, diagnostics, the
|
|
61
|
+
// session browser) must use this value, or a relative CODEX_HOME such as
|
|
62
|
+
// `profiles/codex` is scanned under one directory and cursor-classified
|
|
63
|
+
// under another, and the same rollout replays. Returns null when unset.
|
|
64
|
+
function resolveEnvRoot(provider, { env = process.env, home = os.homedir() } = {}) {
|
|
65
|
+
const spec = PROVIDERS[provider];
|
|
66
|
+
if (!spec) return null;
|
|
67
|
+
const raw = env && typeof env[spec.envVar] === "string" ? env[spec.envVar] : "";
|
|
68
|
+
return expandHome(raw, home);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Accepts { codex: string | string[], claude: string | string[] }. Anything
|
|
72
|
+
// else — wrong type, empty strings, unknown providers — is ignored rather than
|
|
73
|
+
// failing the sync: a malformed config must not stop collection.
|
|
74
|
+
function normalizeScanRootsConfig(raw) {
|
|
75
|
+
const out = {};
|
|
76
|
+
for (const provider of PROVIDER_NAMES) out[provider] = [];
|
|
77
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return out;
|
|
78
|
+
for (const provider of PROVIDER_NAMES) {
|
|
79
|
+
const value = raw[provider];
|
|
80
|
+
const list = Array.isArray(value) ? value : (typeof value === "string" ? [value] : []);
|
|
81
|
+
for (const entry of list) {
|
|
82
|
+
if (typeof entry !== "string" || !entry.trim()) continue;
|
|
83
|
+
out[provider].push(entry.trim());
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// { exists, error }: `exists` is true only for a directory that can be
|
|
90
|
+
// LISTED, not merely stat'ed — a stat-able directory whose listing fails
|
|
91
|
+
// (mode 000, ACL) would otherwise be admitted to the scan, yield no files,
|
|
92
|
+
// read as "present" in status/doctor, and slip past the deferred-repair
|
|
93
|
+
// guard in sync. ENOENT and ENOTDIR (or a regular file in a directory's
|
|
94
|
+
// place) mean the root is absent (error null); any other failure (EACCES,
|
|
95
|
+
// ELOOP, EIO, ...) is reported as `error` so callers can say "unreadable"
|
|
96
|
+
// instead of "missing".
|
|
97
|
+
function scanRootDirState(target, deps = {}) {
|
|
98
|
+
const statSync = deps.statSync || fs.statSync;
|
|
99
|
+
const opendirSync = deps.opendirSync || fs.opendirSync;
|
|
100
|
+
try {
|
|
101
|
+
if (!statSync(target).isDirectory()) return { exists: false, error: null };
|
|
102
|
+
// Probe listability without reading entries; large roots stay cheap.
|
|
103
|
+
opendirSync(target).closeSync();
|
|
104
|
+
return { exists: true, error: null };
|
|
105
|
+
} catch (e) {
|
|
106
|
+
const code = e && typeof e.code === "string" ? e.code : "EUNKNOWN";
|
|
107
|
+
if (code === "ENOENT" || code === "ENOTDIR") return { exists: false, error: null };
|
|
108
|
+
return { exists: false, error: code };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function isDirectorySync(target, deps) {
|
|
113
|
+
return scanRootDirState(target, deps).exists;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function realpathOrSelf(target, deps) {
|
|
117
|
+
const realpathSync = deps.realpathSync || fs.realpathSync;
|
|
118
|
+
try {
|
|
119
|
+
return realpathSync(target);
|
|
120
|
+
} catch {
|
|
121
|
+
return target;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function identityKey(target, deps) {
|
|
126
|
+
const resolved = path.resolve(target);
|
|
127
|
+
const real = realpathOrSelf(resolved, deps);
|
|
128
|
+
return real.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Drop later entries that resolve (realpath) to a directory already listed.
|
|
132
|
+
// Used at the projects/ level too: two Claude profiles may symlink one
|
|
133
|
+
// projects/ dir, and reading it under both spellings would double-parse every
|
|
134
|
+
// file, leaving correctness to the bounded claudeHashes layer.
|
|
135
|
+
function dedupeDirsByRealpath(dirs, deps = {}) {
|
|
136
|
+
const seen = new Set();
|
|
137
|
+
const out = [];
|
|
138
|
+
for (const dir of Array.isArray(dirs) ? dirs : []) {
|
|
139
|
+
if (typeof dir !== "string" || !dir) continue;
|
|
140
|
+
const key = identityKey(dir, deps);
|
|
141
|
+
if (seen.has(key)) continue;
|
|
142
|
+
seen.add(key);
|
|
143
|
+
out.push(dir);
|
|
144
|
+
}
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Resolve the full, ordered, deduplicated root list per provider.
|
|
149
|
+
//
|
|
150
|
+
// base: { codex: string[], claude: string[] } — roots the caller scans
|
|
151
|
+
// anyway (native home, WSL install); they keep their position and
|
|
152
|
+
// are labelled origin "native".
|
|
153
|
+
// env: CODEX_HOME / CLAUDE_CONFIG_DIR become origin "env" unless already
|
|
154
|
+
// covered by base.
|
|
155
|
+
// config.scanRoots.<provider>[] become origin "config".
|
|
156
|
+
//
|
|
157
|
+
// Every entry: { path, realPath, exists, error, origin }. `exists` is a
|
|
158
|
+
// readable-directory check and `error` carries a stat error code other than
|
|
159
|
+
// absence (see scanRootDirState); non-existent or unreadable configured roots
|
|
160
|
+
// are kept in the list (so cursor-path classification stays stable while a
|
|
161
|
+
// volume is unmounted) and callers that walk directories skip them via
|
|
162
|
+
// `exists`.
|
|
163
|
+
function resolveScanRoots({
|
|
164
|
+
home = os.homedir(),
|
|
165
|
+
env = process.env,
|
|
166
|
+
config = null,
|
|
167
|
+
base = {},
|
|
168
|
+
deps = {},
|
|
169
|
+
} = {}) {
|
|
170
|
+
const configured = normalizeScanRootsConfig(config && typeof config === "object" ? config.scanRoots : null);
|
|
171
|
+
const out = {};
|
|
172
|
+
for (const provider of PROVIDER_NAMES) {
|
|
173
|
+
const candidates = [];
|
|
174
|
+
for (const root of Array.isArray(base[provider]) ? base[provider] : []) {
|
|
175
|
+
if (typeof root === "string" && root.trim()) candidates.push({ raw: root, origin: "native" });
|
|
176
|
+
}
|
|
177
|
+
const envRoot = resolveEnvRoot(provider, { env, home });
|
|
178
|
+
if (envRoot) candidates.push({ raw: envRoot, origin: "env" });
|
|
179
|
+
for (const root of configured[provider]) candidates.push({ raw: root, origin: "config" });
|
|
180
|
+
|
|
181
|
+
const seen = new Set();
|
|
182
|
+
const entries = [];
|
|
183
|
+
for (const candidate of candidates) {
|
|
184
|
+
const expanded = expandHome(candidate.raw, home);
|
|
185
|
+
if (!expanded) continue;
|
|
186
|
+
const resolved = expanded;
|
|
187
|
+
const key = identityKey(resolved, deps);
|
|
188
|
+
if (seen.has(key)) continue;
|
|
189
|
+
seen.add(key);
|
|
190
|
+
const state = scanRootDirState(resolved, deps);
|
|
191
|
+
entries.push({
|
|
192
|
+
path: resolved,
|
|
193
|
+
realPath: realpathOrSelf(resolved, deps),
|
|
194
|
+
exists: state.exists,
|
|
195
|
+
error: state.error,
|
|
196
|
+
origin: candidate.origin,
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
out[provider] = entries;
|
|
200
|
+
}
|
|
201
|
+
return out;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Append `extraDirs` to `baseDirs`, dropping any extra that resolves (realpath)
|
|
205
|
+
// to a directory already present. Base entries are never collapsed against
|
|
206
|
+
// each other: the native + WSL pair is listed on purpose (see sync.js) and
|
|
207
|
+
// their dedup is handled downstream by file-identity hashes.
|
|
208
|
+
function appendUniqueDirs(baseDirs, extraDirs, deps = {}) {
|
|
209
|
+
const out = [];
|
|
210
|
+
const seen = new Set();
|
|
211
|
+
for (const dir of Array.isArray(baseDirs) ? baseDirs : []) {
|
|
212
|
+
if (typeof dir !== "string" || !dir) continue;
|
|
213
|
+
seen.add(identityKey(dir, deps));
|
|
214
|
+
out.push(dir);
|
|
215
|
+
}
|
|
216
|
+
for (const dir of Array.isArray(extraDirs) ? extraDirs : []) {
|
|
217
|
+
if (typeof dir !== "string" || !dir) continue;
|
|
218
|
+
const key = identityKey(dir, deps);
|
|
219
|
+
if (seen.has(key)) continue;
|
|
220
|
+
seen.add(key);
|
|
221
|
+
out.push(dir);
|
|
222
|
+
}
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Roots beyond what the caller already scans: the "env" and "config" entries
|
|
227
|
+
// that exist on disk. Paths only, in list order.
|
|
228
|
+
function extraScanRootPaths(entries) {
|
|
229
|
+
return (Array.isArray(entries) ? entries : [])
|
|
230
|
+
.filter((entry) => entry && entry.origin !== "native" && entry.exists)
|
|
231
|
+
.map((entry) => entry.path);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function hasAnyScanChild(root, children, deps = {}) {
|
|
235
|
+
for (const child of Array.isArray(children) ? children : [children]) {
|
|
236
|
+
if (isDirectorySync(path.join(root, child), deps)) return true;
|
|
237
|
+
}
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Human label for status / doctor output.
|
|
242
|
+
function describeScanRootOrigin(entry, provider) {
|
|
243
|
+
if (!entry || entry.origin === "native") return "native";
|
|
244
|
+
if (entry.origin === "env") return PROVIDERS[provider]?.envVar || "env";
|
|
245
|
+
return "scanRoots";
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// "" for a readable root, " (missing)" for an absent one, " (unreadable: CODE)"
|
|
249
|
+
// for a stat failure other than absence.
|
|
250
|
+
function describeScanRootState(entry) {
|
|
251
|
+
if (!entry || entry.exists) return "";
|
|
252
|
+
return entry.error ? ` (unreadable: ${entry.error})` : " (missing)";
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// config.scanRoots as persisted in ~/.tokenusage/tracker/config.json for
|
|
256
|
+
// callers that do not already hold the config (the session browser).
|
|
257
|
+
async function loadScanRootsConfig({ home = os.homedir() } = {}) {
|
|
258
|
+
const { trackerDir } = await resolveTrackerPaths({ home });
|
|
259
|
+
const config = await readJson(path.join(trackerDir, "config.json")).catch(() => null);
|
|
260
|
+
return normalizeScanRootsConfig(config && typeof config === "object" ? config.scanRoots : null);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
module.exports = {
|
|
264
|
+
PROVIDER_NAMES,
|
|
265
|
+
appendUniqueDirs,
|
|
266
|
+
dedupeDirsByRealpath,
|
|
267
|
+
describeScanRootOrigin,
|
|
268
|
+
describeScanRootState,
|
|
269
|
+
expandHome,
|
|
270
|
+
extraScanRootPaths,
|
|
271
|
+
hasAnyScanChild,
|
|
272
|
+
loadScanRootsConfig,
|
|
273
|
+
normalizeScanRootsConfig,
|
|
274
|
+
resolveEnvRoot,
|
|
275
|
+
resolveScanRoots,
|
|
276
|
+
scanRootDirState,
|
|
277
|
+
};
|