tokenusage-cli 0.2.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +70 -0
  2. package/README.md +52 -11
  3. package/bin/tokenusage.js +14 -2
  4. package/package.json +1 -2
  5. package/src/cli.js +325 -78
  6. package/src/commands/device-login.js +3 -3
  7. package/src/commands/init.js +343 -153
  8. package/src/commands/pause.js +11 -23
  9. package/src/commands/reconcile.js +485 -0
  10. package/src/commands/sessions.js +57 -14
  11. package/src/commands/status.js +5 -2
  12. package/src/commands/sync.js +77 -18
  13. package/src/commands/uninstall.js +6 -9
  14. package/src/commands/upload.js +147 -0
  15. package/src/lib/cli-errors.js +33 -0
  16. package/src/lib/cli-ui.js +93 -0
  17. package/src/lib/ingest-sanitize.js +9 -0
  18. package/src/lib/queue-row-normalize.js +55 -0
  19. package/src/lib/rollout.js +39 -2
  20. package/src/lib/scheduled-task.js +597 -75
  21. package/src/lib/session-analytics.js +171 -3
  22. package/src/lib/session-upload.js +6 -0
  23. package/src/lib/sync-report.js +8 -2
  24. package/src/lib/trae-db.js +6 -1
  25. package/src/lib/antigravity-paths.js +0 -48
  26. package/src/lib/async-queue-reader.js +0 -138
  27. package/src/lib/atomic-json.js +0 -27
  28. package/src/lib/bearer-token.js +0 -11
  29. package/src/lib/cloud-account.js +0 -541
  30. package/src/lib/codex-context-breakdown.js +0 -806
  31. package/src/lib/context-health.js +0 -124
  32. package/src/lib/grok-context-breakdown.js +0 -890
  33. package/src/lib/heatmap-compact.js +0 -114
  34. package/src/lib/local-api.js +0 -3574
  35. package/src/lib/outcomes-engine.js +0 -253
  36. package/src/lib/pet-packages.js +0 -722
  37. package/src/lib/skill-usage.js +0 -267
  38. package/src/lib/skills-manager.js +0 -1591
  39. package/src/lib/source-metadata.js +0 -48
  40. package/src/lib/subscription-manager.js +0 -293
  41. package/src/lib/wrapped-aggregator.js +0 -236
  42. package/test/audit-fixes.test.js +0 -249
  43. package/test/branding.test.js +0 -24
  44. package/test/cli-options.test.js +0 -43
  45. package/test/cli-smoke.test.js +0 -53
  46. package/test/cli-ui.test.js +0 -47
  47. package/test/collection-controls.test.js +0 -255
  48. package/test/ingest-sanitize.test.js +0 -162
  49. package/test/packaging.test.js +0 -107
  50. package/test/quiet-warnings.test.js +0 -53
  51. package/test/runtime-copy.test.js +0 -111
  52. package/test/session-upload.test.js +0 -427
  53. package/test/zstd-native.test.js +0 -36
@@ -87,7 +87,11 @@ const {
87
87
  // v23 excludes database updates and user records from request durations.
88
88
  // v24 keeps OpenCode usage dates at event time and fork history on its ancestor.
89
89
  // v25 adds Kimi wire sessions, including official/legacy migration dedup.
90
- const SIDECAR_VERSION = 25;
90
+ // 26: qoder session rows now apply the same billable gate as the token-side
91
+ // bucket parser — billable:false rows carry un-billed "would-be" credits
92
+ // (observed: 3718 of 4591 credits on this machine) and must not inflate the
93
+ // session view. Bumped so existing sidecars rebuild once.
94
+ const SIDECAR_VERSION = 26;
91
95
  const EDIT_TOOLS = new Set([
92
96
  "apply_patch",
93
97
  "edit",
@@ -1935,6 +1939,122 @@ async function scanKimiSessions(filePaths) {
1935
1939
  }));
1936
1940
  }
1937
1941
 
1942
+ // Qoder's flat per-session JSONL: one file per session id, assistant messages
1943
+ // carry Anthropic-style usage — credits-only on current builds (2026-10-31).
1944
+ // Sessions sum both dimensions; the credits figure rides the row as
1945
+ // display-only telemetry and never feeds the token identity.
1946
+ function clampQoderCounter(value) {
1947
+ const parsed = Number(value);
1948
+ return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : 0;
1949
+ }
1950
+
1951
+ function qoderTimestampMs(record) {
1952
+ const raw = record?.timestamp ?? record?.message?.timestamp;
1953
+ if (typeof raw === "number" && Number.isFinite(raw)) {
1954
+ return raw < 1e12 ? Math.trunc(raw * 1000) : Math.trunc(raw);
1955
+ }
1956
+ if (typeof raw === "string") {
1957
+ const parsed = Date.parse(raw);
1958
+ if (Number.isFinite(parsed)) return parsed;
1959
+ const numeric = Number(raw);
1960
+ if (Number.isFinite(numeric) && raw.trim() !== "") {
1961
+ return numeric < 1e12 ? Math.trunc(numeric * 1000) : Math.trunc(numeric);
1962
+ }
1963
+ }
1964
+ return null;
1965
+ }
1966
+
1967
+ function scanQoderSessionFile(filePath, source) {
1968
+ let lines;
1969
+ try {
1970
+ lines = fs.readFileSync(filePath, "utf8").split("\n");
1971
+ } catch (error) {
1972
+ if (isRecoverableSessionReadError(error)) throw error;
1973
+ throw error;
1974
+ }
1975
+ const sessionId = path.basename(filePath).replace(/\.jsonl$/i, "");
1976
+ if (!sessionId) throw new Error("qoder session file without an id");
1977
+
1978
+ let projectRef = null;
1979
+ let startedMs = null;
1980
+ let endedMs = null;
1981
+ let credits = 0;
1982
+ const tokens = {
1983
+ input_tokens: 0,
1984
+ cached_input_tokens: 0,
1985
+ cache_creation_input_tokens: 0,
1986
+ output_tokens: 0,
1987
+ reasoning_output_tokens: 0,
1988
+ };
1989
+
1990
+ for (const line of lines) {
1991
+ if (!line.trim()) continue;
1992
+ let record;
1993
+ try { record = JSON.parse(line); } catch { continue; }
1994
+ if (record?.type === "workspace-directories") {
1995
+ const cwd = Array.isArray(record.directories) ? record.directories[0] : null;
1996
+ if (typeof cwd === "string" && cwd.trim()) projectRef = cwd.trim();
1997
+ continue;
1998
+ }
1999
+ if (record?.type !== "assistant" || record?.message?.role !== "assistant") continue;
2000
+ const usage = record.message.usage;
2001
+ if (!usage || typeof usage !== "object") continue;
2002
+ const timestamp = qoderTimestampMs(record);
2003
+ if (timestamp == null) continue;
2004
+
2005
+ // Same gate as the bucket parser (normalizeQoderNewTokens + its caller):
2006
+ // a billable:false row with no tokens is un-billed internal activity —
2007
+ // neither conversation nor credits — and must not reach either view.
2008
+ const rowTokens = {
2009
+ input_tokens: clampQoderCounter(usage.input_tokens),
2010
+ cached_input_tokens: clampQoderCounter(usage.cache_read_input_tokens ?? usage.cached_tokens),
2011
+ cache_creation_input_tokens: clampQoderCounter(usage.cache_creation_input_tokens),
2012
+ output_tokens: clampQoderCounter(usage.output_tokens),
2013
+ reasoning_output_tokens: 0,
2014
+ };
2015
+ const hasTokens =
2016
+ rowTokens.input_tokens > 0 || rowTokens.cached_input_tokens > 0 ||
2017
+ rowTokens.cache_creation_input_tokens > 0 || rowTokens.output_tokens > 0;
2018
+ const isBillable = usage.billable !== false;
2019
+ if (!hasTokens && !isBillable) continue;
2020
+
2021
+ tokens.input_tokens += rowTokens.input_tokens;
2022
+ tokens.cached_input_tokens += rowTokens.cached_input_tokens;
2023
+ tokens.cache_creation_input_tokens += rowTokens.cache_creation_input_tokens;
2024
+ tokens.output_tokens += rowTokens.output_tokens;
2025
+ tokens.reasoning_output_tokens += rowTokens.reasoning_output_tokens;
2026
+ const creditValue = Number(usage.credits ?? usage.original_credits ?? 0);
2027
+ credits = Math.round((credits + (Number.isFinite(creditValue) && creditValue > 0 ? creditValue : 0)) * 1e6) / 1e6;
2028
+
2029
+ if (startedMs == null || timestamp < startedMs) startedMs = timestamp;
2030
+ if (endedMs == null || timestamp > endedMs) endedMs = timestamp;
2031
+ }
2032
+
2033
+ if (endedMs == null) return null; // no timestamped usage in this file
2034
+
2035
+ const totalTokens =
2036
+ tokens.input_tokens + tokens.cached_input_tokens +
2037
+ tokens.cache_creation_input_tokens + tokens.output_tokens +
2038
+ tokens.reasoning_output_tokens;
2039
+
2040
+ return finalizeRecord({
2041
+ version: SIDECAR_VERSION,
2042
+ session_hash: sessionHash(source, sessionId),
2043
+ session_id: sessionId,
2044
+ source,
2045
+ ...sessionStorageUsage([filePath]),
2046
+ project_key: projectKey(projectRef, filePath),
2047
+ project_ref: projectRef || null,
2048
+ tokens,
2049
+ total_tokens: totalTokens,
2050
+ credits,
2051
+ started_at: new Date(startedMs).toISOString(),
2052
+ ended_at: new Date(endedMs).toISOString(),
2053
+ active_ms: Math.max(0, endedMs - startedMs),
2054
+ provenance: { source: "local-session-log", confidence: "observed", retry_confidence: "inferred", content_retained: false },
2055
+ });
2056
+ }
2057
+
1938
2058
  async function discoverSessionFiles(home, env = process.env, deps = {}) {
1939
2059
  const grokHome = resolveGrokHome(home);
1940
2060
  // config.scanRoots (#657): read from the tracker config unless the caller
@@ -2027,7 +2147,42 @@ async function discoverSessionFiles(home, env = process.env, deps = {}) {
2027
2147
  return roots;
2028
2148
  });
2029
2149
  const kimi = discoverKimiWireFiles(dedupeDirsByRealpath(kimiRoots), incomplete);
2030
- return { claude, codex: groupCodexFiles([...codex, ...archived]), grok, zcode, opencode, kimi, incomplete };
2150
+ // Qoder's flat per-session JSONL (one file = one session id) carries the
2151
+ // same usage shape as the token side — including credits-only rows, so the
2152
+ // session rows sum credits alongside tokens (2026-10-31). listRolloutFilesDeep
2153
+ // is codex-specific (rollout-* names), so qoder walks its own tree.
2154
+ const qoder = (await Promise.all(
2155
+ [
2156
+ ...providerRoots(home, ".qoder", env, deps).map((root) => ({ root, source: "qoder" })),
2157
+ ...providerRoots(home, ".qoder-cn", env, deps).map((root) => ({ root, source: "qoder-cn" })),
2158
+ ].map(async ({ root, source }) =>
2159
+ (await listQoderSessionFiles(path.join(root, "projects")))
2160
+ .map((filePath) => ({ filePath, source })),
2161
+ ),
2162
+ )).flat();
2163
+ return { claude, codex: groupCodexFiles([...codex, ...archived]), grok, zcode, opencode, kimi, qoder, incomplete };
2164
+ }
2165
+
2166
+ // Qoder session files: `<projects>/<slug>/<sessionId>.jsonl` — any .jsonl is
2167
+ // a session transcript, so the walk is generic (unlike codex's rollout-* rule).
2168
+ async function listQoderSessionFiles(projectsDir) {
2169
+ const out = [];
2170
+ async function walk(dir) {
2171
+ let entries;
2172
+ try {
2173
+ entries = await fsp.readdir(dir, { withFileTypes: true });
2174
+ } catch {
2175
+ return; // absent or unreadable: an empty result, reported as such upstream
2176
+ }
2177
+ for (const entry of entries) {
2178
+ const p = path.join(dir, entry.name);
2179
+ if (entry.isDirectory()) await walk(p);
2180
+ else if (entry.isFile() && entry.name.endsWith(".jsonl")) out.push(p);
2181
+ }
2182
+ }
2183
+ await walk(projectsDir);
2184
+ out.sort((a, b) => a.localeCompare(b));
2185
+ return out;
2031
2186
  }
2032
2187
 
2033
2188
  // Warn once per unreadable directory per process: the dashboard polls this
@@ -2215,6 +2370,7 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
2215
2370
  ...discovered.grok.map(grokSignalsPathFor),
2216
2371
  // db + its WAL sidecars: the trio is one logical file for change detection.
2217
2372
  ...[...(discovered.zcode || []), ...(discovered.opencode || [])].flatMap((dbPath) => [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]),
2373
+ ...discovered.qoder.map(({ filePath }) => filePath),
2218
2374
  ]);
2219
2375
  if (!force && !retryZcodeScan && !retryOpencodeScan && !retryKimiScan && previousMeta?.version === SIDECAR_VERSION && previousMeta.signature === signature) {
2220
2376
  await writeAtomic(metaPath, `${JSON.stringify({ ...previousMeta, checked_at: new Date().toISOString() })}\n`);
@@ -2239,6 +2395,11 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
2239
2395
  ...(discovered.zcode.length ? [{ source: "zcode", filePath: discovered.zcode, scan: scanZcodeSessions }] : []),
2240
2396
  ...(discovered.opencode.length ? [{ source: "opencode", filePath: discovered.opencode, scan: scanOpencodeSessions }] : []),
2241
2397
  ...(discovered.kimi.length ? [{ source: "kimi", filePath: discovered.kimi, scan: scanKimiSessions }] : []),
2398
+ ...discovered.qoder.map(({ filePath, source }) => ({
2399
+ source,
2400
+ filePath,
2401
+ scan: (file) => scanQoderSessionFile(file, source),
2402
+ })),
2242
2403
  ];
2243
2404
  // Files we could not turn into a row (permission denied, half-written line,
2244
2405
  // vanished mid-scan). Swallowing these silently made sessions disappear with
@@ -2247,6 +2408,7 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
2247
2408
  let incompleteZcodeScan = false;
2248
2409
  let incompleteOpencodeScan = false;
2249
2410
  let incompleteKimiScan = false;
2411
+ let incompleteQoderScan = false;
2250
2412
  for (const entry of entries) {
2251
2413
  const cacheKey = sessionFileCacheKey(entry.source, entry.filePath);
2252
2414
  const statKey = analyticsEntryStatKey(entry.source, entry.filePath);
@@ -2255,6 +2417,7 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
2255
2417
  if (entry.source === "zcode") incompleteZcodeScan = true;
2256
2418
  if (entry.source === "opencode") incompleteOpencodeScan = true;
2257
2419
  if (entry.source === "kimi") incompleteKimiScan = true;
2420
+ if (entry.source === "qoder" || entry.source === "qoder-cn") incompleteQoderScan = true;
2258
2421
  continue;
2259
2422
  }
2260
2423
  let rows = null;
@@ -2280,6 +2443,7 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
2280
2443
  if (entry.source === "zcode") incompleteZcodeScan = true;
2281
2444
  if (entry.source === "opencode") incompleteOpencodeScan = true;
2282
2445
  if (entry.source === "kimi") incompleteKimiScan = true;
2446
+ if (entry.source === "qoder" || entry.source === "qoder-cn") incompleteQoderScan = true;
2283
2447
  if (!process.env.NODE_TEST_CONTEXT) {
2284
2448
  console.warn(`[session-analytics] skipped ${entry.source} session file: ${error?.message || error}`);
2285
2449
  }
@@ -2317,10 +2481,14 @@ async function buildSessionAnalyticsInternal({ home = os.homedir(), force = fals
2317
2481
  await writeAtomic(incompleteKimiPath, "1\n");
2318
2482
  sessions.push(...readSidecar(sidecarPath).filter((row) => row.source === "kimi" && row.version === SIDECAR_VERSION));
2319
2483
  }
2484
+ if (incompleteQoderScan) {
2485
+ sessions.push(...readSidecar(sidecarPath).filter((row) =>
2486
+ (row.source === "qoder" || row.source === "qoder-cn") && row.version === SIDECAR_VERSION));
2487
+ }
2320
2488
  sessions.sort((a, b) => String(b.ended_at || "").localeCompare(String(a.ended_at || "")));
2321
2489
  // A failed database scan is an incomplete inventory. Keep the last complete
2322
2490
  // snapshot and signature so an unchanged database is retried next build.
2323
- if (incompleteDirs.length === 0 && !incompleteZcodeScan && !incompleteOpencodeScan && !incompleteKimiScan) {
2491
+ if (incompleteDirs.length === 0 && !incompleteZcodeScan && !incompleteOpencodeScan && !incompleteKimiScan && !incompleteQoderScan) {
2324
2492
  const content = sessions.map(serializeSessionRecord).join("\n") + (sessions.length ? "\n" : "");
2325
2493
  await writeAtomic(sidecarPath, content);
2326
2494
  const generatedAt = new Date().toISOString();
@@ -186,6 +186,12 @@ function mapSessionRow(row) {
186
186
  project_key: basenameKey(row.project_key ?? row.project_ref),
187
187
  ...tokens,
188
188
  billable_total_tokens: null,
189
+ // Session-level credits (docs/cli-contract.md §2.4, 2026-10-31): optional
190
+ // display-only telemetry; old backends ignore it, so it rides only when
191
+ // the sidecar carried a positive figure.
192
+ ...(Number(row.credits) > 0
193
+ ? { credits: Math.round(Number(row.credits) * 1e6) / 1e6 }
194
+ : {}),
189
195
  ...(models.length > 0 ? { models } : {}),
190
196
  };
191
197
  }
@@ -49,6 +49,7 @@ function createSyncRunTelemetry({
49
49
  let parsedFiles = 0;
50
50
  let bucketsUploaded = 0;
51
51
  let tokensIngested = 0;
52
+ let creditsIngested = 0;
52
53
  let sessionsUploaded = 0;
53
54
  let sessionsSkipped = 0;
54
55
  let queueBytes = null;
@@ -64,10 +65,14 @@ function createSyncRunTelemetry({
64
65
  },
65
66
  // Server-acked drain result: buckets placed into successful ingest
66
67
  // requests plus their total_tokens sum (client-side telemetry semantics
67
- // per §2.11 — a larger replay may override these buckets later).
68
- noteUpload({ inserted = 0, tokens = 0 } = {}) {
68
+ // per §2.11 — a larger replay may override these buckets later). Credits
69
+ // (docs/sources-capture-audit.md §2) sum identically as display-only
70
+ // telemetry.
71
+ noteUpload({ inserted = 0, tokens = 0, credits = 0 } = {}) {
69
72
  bucketsUploaded += Math.max(0, Number(inserted) || 0);
70
73
  tokensIngested += Math.max(0, Number(tokens) || 0);
74
+ creditsIngested =
75
+ Math.round((creditsIngested + Math.max(0, Number(credits) || 0)) * 1e6) / 1e6;
71
76
  },
72
77
  noteSessions({ upserted = 0, skipped = 0 } = {}) {
73
78
  sessionsUploaded += Math.max(0, Number(upserted) || 0);
@@ -91,6 +96,7 @@ function createSyncRunTelemetry({
91
96
  parsed_files: parsedFiles,
92
97
  buckets_uploaded: bucketsUploaded,
93
98
  tokens_ingested: tokensIngested,
99
+ ...(creditsIngested > 0 ? { credits_ingested: creditsIngested } : {}),
94
100
  sessions_uploaded: sessionsUploaded,
95
101
  sessions_skipped: sessionsSkipped,
96
102
  queue_bytes: queueBytes,
@@ -21,7 +21,12 @@ function resolveTraeDbPaths(env = process.env, { platform = process.platform, ho
21
21
  : platform === "darwin"
22
22
  ? path.join(home, "Library", "Application Support")
23
23
  : env.XDG_CONFIG_HOME || path.join(home, ".config");
24
- const roots = customHome ? [customHome] : [path.join(parent, "Trae"), path.join(parent, "TRAE SOLO")];
24
+ // The CN IDE build installs as "Trae CN" with the same ModularData schema;
25
+ // its database joins the same source ledger as a separate store (per-store
26
+ // turn identity in parseTraeIncremental), so no separate source id is needed.
27
+ const roots = customHome
28
+ ? [customHome]
29
+ : [path.join(parent, "Trae"), path.join(parent, "TRAE SOLO"), path.join(parent, "Trae CN")];
25
30
  const result = [];
26
31
  for (const root of roots) {
27
32
  // A decrypted sibling may be an old export. Only an explicit DB override
@@ -1,48 +0,0 @@
1
- const fs = require("node:fs");
2
- const os = require("node:os");
3
- const path = require("node:path");
4
-
5
- const MAIN_APP_SUBDIR = "antigravity";
6
- const IDE_SUBDIR = "antigravity-ide";
7
-
8
- function geminiHome(env) {
9
- return path.join(env.HOME || env.USERPROFILE || os.homedir(), ".gemini");
10
- }
11
-
12
- function trimmed(value) {
13
- return typeof value === "string" ? value.trim() : "";
14
- }
15
-
16
- function existsDir(dirPath) {
17
- try {
18
- return fs.statSync(dirPath).isDirectory();
19
- } catch (_e) {
20
- return false;
21
- }
22
- }
23
-
24
- function resolveAntigravitySkillDirs(env = process.env) {
25
- const override = trimmed(env.TOKENUSAGE_ANTIGRAVITY_HOME);
26
- if (override) return [path.join(override, "skills")];
27
-
28
- const home = geminiHome(env);
29
- const mainSkills = path.join(home, MAIN_APP_SUBDIR, "skills");
30
- const ideSkills = path.join(home, IDE_SUBDIR, "skills");
31
- const mainParent = path.join(home, MAIN_APP_SUBDIR);
32
- const ideParent = path.join(home, IDE_SUBDIR);
33
-
34
- const dirs = [];
35
- if (existsDir(mainParent)) dirs.push(mainSkills);
36
- if (existsDir(ideParent)) dirs.push(ideSkills);
37
-
38
- // Neither Antigravity install present: fall back to the main-app path so
39
- // targetList still surfaces a stable path string and the user can install
40
- // skills now and run Antigravity later. The empty parent will be created on
41
- // demand by ensureDir; cleanup happens naturally via uninstallSkill.
42
- if (dirs.length > 0) return dirs;
43
- return [mainSkills];
44
- }
45
-
46
- module.exports = {
47
- resolveAntigravitySkillDirs,
48
- };
@@ -1,138 +0,0 @@
1
- const fs = require("node:fs/promises");
2
- const { createHash } = require("node:crypto");
3
- const { StringDecoder } = require("node:string_decoder");
4
- const { setImmediate: yieldToRequests } = require("node:timers/promises");
5
-
6
- const CHUNK_BYTES = 64 * 1024;
7
-
8
- function signature(stat) {
9
- return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
10
- }
11
-
12
- // One in-flight read per path shares the dashboard's request fan-out. Parsing
13
- // yields between bounded chunks so a cold/replaced queue cannot monopolize the
14
- // HTTP event loop. Keep committed newline-terminated rows separate from a tail
15
- // that a concurrent writer may still be completing.
16
- function createAsyncQueueReader({ parseRow, label, maxEntries = 4 }) {
17
- const cache = new Map();
18
- const pending = new Map();
19
-
20
- async function hashPrefix(file, length) {
21
- const hash = createHash("sha256");
22
- const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
23
- let position = 0;
24
- while (position < length) {
25
- const { bytesRead } = await file.read(buffer, 0, Math.min(CHUNK_BYTES, length - position), position);
26
- if (!bytesRead) throw new Error("Queue truncated during validation");
27
- hash.update(buffer.subarray(0, bytesRead));
28
- position += bytesRead;
29
- await yieldToRequests();
30
- }
31
- return hash;
32
- }
33
-
34
- async function load(queuePath) {
35
- const cached = cache.get(queuePath);
36
- let file;
37
- try {
38
- file = await fs.open(queuePath, "r");
39
- // Stat the opened file, rather than a path which can be atomically replaced
40
- // between stat and open. Its signature always describes the bytes we read.
41
- const stat = await file.stat({ bigint: true });
42
- const nextSignature = signature(stat);
43
- const identity = `${stat.dev}:${stat.ino}`;
44
- const size = Number(stat.size);
45
- if (cached?.signature === nextSignature) {
46
- // Windows can reuse mtime/ctime for same-size writes anywhere in the
47
- // file. Validate every byte before reusing the parsed snapshot.
48
- const hash = await hashPrefix(file, size);
49
- if (hash.digest("hex") === cached.digest) return cached.rows;
50
- }
51
- let offset = 0;
52
- let seen = new Map();
53
- let hash = createHash("sha256");
54
- if (cached && cached.identity === identity && size > cached.size && cached.consumedBytes > 0) {
55
- // Growth can include an interior rewrite, so verify the entire old
56
- // committed prefix before parsing only the new tail.
57
- const prefixHash = await hashPrefix(file, cached.consumedBytes);
58
- if (prefixHash.copy().digest("hex") === cached.committedDigest) {
59
- offset = cached.consumedBytes;
60
- seen = cached.seen;
61
- hash = prefixHash;
62
- }
63
- }
64
- let committedHash = hash.copy();
65
-
66
- const changes = new Map();
67
- let malformed = 0;
68
- function parse(line, target) {
69
- if (!line.trim()) return;
70
- try {
71
- const entry = parseRow(JSON.parse(line));
72
- if (entry) target.set(entry.key, entry.row);
73
- } catch { malformed++; }
74
- }
75
-
76
- const decoder = new StringDecoder("utf8");
77
- const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
78
- let remainder = "";
79
- let position = offset;
80
- let consumedBytes = offset;
81
- while (position < size) {
82
- const length = Math.min(CHUNK_BYTES, size - position);
83
- const { bytesRead } = await file.read(buffer, 0, length, position);
84
- if (!bytesRead) throw new Error("Queue truncated during read");
85
- const newline = buffer.lastIndexOf(0x0a, bytesRead - 1);
86
- const chunk = buffer.subarray(0, bytesRead);
87
- if (newline >= 0) {
88
- consumedBytes = position + newline + 1;
89
- committedHash = hash.copy().update(chunk.subarray(0, newline + 1));
90
- }
91
- hash.update(chunk);
92
- position += bytesRead;
93
- remainder += decoder.write(chunk);
94
- let start = 0;
95
- let end;
96
- while ((end = remainder.indexOf("\n", start)) !== -1) {
97
- parse(remainder.slice(start, end), changes);
98
- start = end + 1;
99
- }
100
- remainder = remainder.slice(start);
101
- await yieldToRequests();
102
- }
103
- remainder += decoder.end();
104
- for (const [key, row] of changes) seen.set(key, row);
105
- const tail = new Map();
106
- parse(remainder, tail);
107
- // Legacy files without a final newline are readable, but their provisional
108
- // key never enters the committed map (completion may change that key).
109
- const rows = tail.size ? Array.from(new Map([...seen, ...tail]).values()) : Array.from(seen.values());
110
- cache.delete(queuePath);
111
- cache.set(queuePath, {
112
- signature: nextSignature, identity, size, consumedBytes,
113
- digest: hash.digest("hex"), committedDigest: committedHash.digest("hex"), seen, rows,
114
- });
115
- while (cache.size > maxEntries) cache.delete(cache.keys().next().value);
116
- if (malformed) console.error(`[LocalAPI] ${label}: skipped ${malformed} malformed line(s)`);
117
- return rows;
118
- } catch (error) {
119
- if (error.code === "ENOENT") {
120
- cache.delete(queuePath);
121
- return [];
122
- }
123
- console.error(`[LocalAPI] ${label}: failed to read queue:`, error.message);
124
- return cached?.rows || [];
125
- } finally {
126
- if (file) await file.close();
127
- }
128
- }
129
-
130
- return function read(queuePath) {
131
- if (pending.has(queuePath)) return pending.get(queuePath);
132
- const task = load(queuePath).finally(() => pending.delete(queuePath));
133
- pending.set(queuePath, task);
134
- return task;
135
- };
136
- }
137
-
138
- module.exports = { createAsyncQueueReader };
@@ -1,27 +0,0 @@
1
- const fs = require("node:fs");
2
- const path = require("node:path");
3
- const crypto = require("node:crypto");
4
-
5
- function writeJsonAtomicSync(filePath, value) {
6
- const contents = JSON.stringify(value, null, 2) + "\n";
7
- fs.mkdirSync(path.dirname(filePath), { recursive: true });
8
- const tmp = `${filePath}.tmp.${crypto.randomUUID()}`;
9
- let fd;
10
- let created = false;
11
- try {
12
- fd = fs.openSync(tmp, "wx", 0o600);
13
- created = true;
14
- fs.writeFileSync(fd, contents, "utf8");
15
- fs.closeSync(fd);
16
- fd = undefined;
17
- // Replace the final directory entry rather than following a link that
18
- // appeared after the caller read or checked the previous configuration.
19
- fs.renameSync(tmp, filePath);
20
- created = false;
21
- } finally {
22
- if (fd !== undefined) fs.closeSync(fd);
23
- if (created) fs.unlinkSync(tmp);
24
- }
25
- }
26
-
27
- module.exports = { writeJsonAtomicSync };
@@ -1,11 +0,0 @@
1
- function parseBearerToken(authorization) {
2
- // Match only the prefix so an all-whitespace header cannot make two
3
- // overlapping quantifiers repeatedly backtrack over the same characters.
4
- const value = String(authorization || "");
5
- const prefix = /^Bearer[ \t]+/i.exec(value);
6
- if (!prefix) return null;
7
- const token = value.slice(prefix[0].length);
8
- return token && !/[\r\n]/.test(token) ? token : null;
9
- }
10
-
11
- module.exports = { parseBearerToken };