tokenusage-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -19
- package/LICENSE +22 -22
- package/README.md +94 -94
- package/THIRD_PARTY_NOTICES.md +140 -140
- package/bin/tokenusage.js +69 -69
- package/package.json +59 -59
- package/src/cli.js +146 -134
- package/src/commands/device-login.js +205 -205
- package/src/commands/diagnostics.js +38 -38
- package/src/commands/doctor.js +110 -110
- package/src/commands/init.js +2484 -2436
- package/src/commands/pause.js +77 -0
- package/src/commands/sessions.js +29 -29
- package/src/commands/status.js +1758 -1747
- package/src/commands/sync.js +8048 -7934
- package/src/commands/uninstall.js +322 -312
- package/src/lib/account-usage-dedup.js +267 -267
- package/src/lib/antigravity-paths.js +48 -48
- package/src/lib/ark-agent-plan-limits.js +438 -438
- package/src/lib/ark-coding-plan-limits.js +537 -537
- package/src/lib/async-queue-reader.js +138 -138
- package/src/lib/atomic-json.js +27 -27
- package/src/lib/bearer-token.js +11 -11
- package/src/lib/categorizer-utils.js +232 -232
- package/src/lib/claude-categorizer.js +1343 -1343
- package/src/lib/claude-config.js +279 -279
- package/src/lib/claude-user-identity.js +56 -56
- package/src/lib/cli-ui.js +179 -179
- package/src/lib/cloud-account.js +541 -541
- package/src/lib/cloud-device-token.js +141 -141
- package/src/lib/cloud-sync-prefs.js +14 -14
- package/src/lib/cloud-usage-reset.js +115 -115
- package/src/lib/codex-config.js +595 -595
- package/src/lib/codex-context-breakdown.js +806 -806
- package/src/lib/codex-model-attribution.js +128 -128
- package/src/lib/codex-rollout-parser.js +1471 -1471
- package/src/lib/codex-service-tier.js +49 -49
- package/src/lib/codex-token-refresh.js +140 -140
- package/src/lib/codex-token-usage.js +195 -195
- package/src/lib/codex-usage-record.js +195 -195
- package/src/lib/collection-pause.js +77 -0
- package/src/lib/command-runner.js +313 -313
- package/src/lib/commandcode-limits.js +317 -317
- package/src/lib/context-health.js +124 -124
- package/src/lib/cursor-config.js +605 -605
- package/src/lib/cursor-store.js +1918 -1918
- package/src/lib/debug-flags.js +7 -7
- package/src/lib/devin-limits.js +249 -249
- package/src/lib/devin-usage.js +212 -212
- package/src/lib/diagnostics.js +383 -383
- package/src/lib/doctor.js +385 -385
- package/src/lib/dumate.js +95 -95
- package/src/lib/fs.js +455 -455
- package/src/lib/function-url.js +14 -14
- package/src/lib/gemini-config.js +291 -291
- package/src/lib/git-outcomes.js +243 -243
- package/src/lib/grok-context-breakdown.js +890 -890
- package/src/lib/grok-hook.js +360 -360
- package/src/lib/grok-limits.js +865 -865
- package/src/lib/grok-usage.js +98 -98
- package/src/lib/heatmap-compact.js +114 -114
- package/src/lib/ingest-sanitize.js +41 -41
- package/src/lib/init-flow.js +42 -42
- package/src/lib/install-resolver.js +170 -170
- package/src/lib/jsonl-lines.js +134 -134
- package/src/lib/kimi-model-repair.js +173 -173
- package/src/lib/kimi-profile.js +61 -61
- package/src/lib/kimi-session.js +216 -216
- package/src/lib/local-api.js +3588 -3588
- package/src/lib/machine-id.js +238 -238
- package/src/lib/multi-install-parser.js +113 -113
- package/src/lib/notify-command.js +38 -38
- package/src/lib/omp-hook.js +589 -589
- package/src/lib/openclaw-cli.js +104 -104
- package/src/lib/openclaw-hook.js +375 -375
- package/src/lib/openclaw-session-plugin.js +560 -560
- package/src/lib/opencode-config.js +121 -121
- package/src/lib/opencode-go-limits.js +807 -807
- package/src/lib/outcomes-engine.js +253 -253
- package/src/lib/passive-mode.js +219 -219
- package/src/lib/pet-packages.js +722 -722
- package/src/lib/pricing/curated-overrides.json +822 -822
- package/src/lib/pricing/index.js +379 -379
- package/src/lib/pricing/litellm-fetcher.js +172 -172
- package/src/lib/pricing/matcher.js +456 -456
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/progress.js +76 -76
- package/src/lib/project-usage-purge.js +117 -117
- package/src/lib/prompt.js +24 -24
- package/src/lib/provider-status.js +104 -104
- package/src/lib/proxy-env.js +489 -489
- package/src/lib/proxy-settings.js +163 -163
- package/src/lib/qoder-cn-cli-limits.js +119 -119
- package/src/lib/qoder-limits.js +880 -880
- package/src/lib/quiet-warnings.js +26 -26
- package/src/lib/rollout.js +26005 -25966
- package/src/lib/runtime-config.js +9 -4
- package/src/lib/scan-roots.js +277 -277
- package/src/lib/scheduled-task.js +145 -0
- package/src/lib/sentry.js +231 -231
- package/src/lib/session-analytics.js +2972 -2972
- package/src/lib/session-performance.js +237 -237
- package/src/lib/session-upload.js +379 -379
- package/src/lib/skill-usage.js +267 -267
- package/src/lib/skills-manager.js +1591 -1591
- package/src/lib/source-metadata.js +48 -48
- package/src/lib/sqlite-reader.js +215 -215
- package/src/lib/star-cta.js +73 -73
- package/src/lib/subscription-manager.js +293 -293
- package/src/lib/subscriptions.js +508 -508
- package/src/lib/sync-report.js +138 -0
- package/src/lib/telemetry.js +193 -193
- package/src/lib/tracker-paths.js +16 -16
- package/src/lib/trae-cn-config.js +548 -548
- package/src/lib/trae-db.js +127 -127
- package/src/lib/trae-snapshot.js +230 -230
- package/src/lib/trae-sqlite.js +143 -143
- package/src/lib/trae-usage.js +121 -121
- package/src/lib/upload-queue.js +134 -134
- package/src/lib/upload-throttle.js +171 -171
- package/src/lib/usage-limits.js +4552 -4552
- package/src/lib/wrapped-aggregator.js +236 -236
- package/src/lib/wsl-probe.js +281 -281
- package/src/lib/zcode-limits.js +1172 -1172
- package/src/lib/zcode-reset-credits.js +66 -66
- package/test/audit-fixes.test.js +200 -0
- package/test/cli-smoke.test.js +53 -53
- package/test/collection-controls.test.js +181 -0
- package/test/ingest-sanitize.test.js +162 -162
- package/test/quiet-warnings.test.js +53 -53
- package/test/session-upload.test.js +427 -427
package/src/lib/upload-queue.js
CHANGED
|
@@ -1,134 +1,134 @@
|
|
|
1
|
-
const fs = require("node:fs/promises");
|
|
2
|
-
const crypto = require("node:crypto");
|
|
3
|
-
const { isUtf8 } = require("node:buffer");
|
|
4
|
-
|
|
5
|
-
const BACKLOG_BYTES = 1_000_000;
|
|
6
|
-
const CHUNK_BYTES = 64 * 1024;
|
|
7
|
-
const { TOKEN_COLUMNS } = require("./account-usage-dedup");
|
|
8
|
-
|
|
9
|
-
function sessionSnapshotMs(row) {
|
|
10
|
-
const verifiedMs = Date.parse(row?.snapshot_verified_at);
|
|
11
|
-
if (typeof row?.source !== "string" || row.source.trim().toLowerCase() !== "trae-cn" ||
|
|
12
|
-
typeof row.session_id !== "string" || !row.session_id.trim() || row.session_id.trim().length > 256 ||
|
|
13
|
-
typeof row.model !== "string" || !row.model.trim() || row.model.trim().length > 200 ||
|
|
14
|
-
!Number.isFinite(Date.parse(row.bucket_start)) || !Number.isFinite(verifiedMs) ||
|
|
15
|
-
TOKEN_COLUMNS.some((column) => !Number.isSafeInteger(Number(row[column])) || Number(row[column]) < 0) ||
|
|
16
|
-
Number(row.total_tokens) !== Number(row.input_tokens) + Number(row.output_tokens) + Number(row.cached_input_tokens) + Number(row.cache_creation_input_tokens)) return null;
|
|
17
|
-
return verifiedMs;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
// Keep raw bytes, including CRLF and incomplete tails. Upload offsets are
|
|
21
|
-
// byte positions, not JavaScript string lengths.
|
|
22
|
-
async function* iterateQueueLines(handle, start, end) {
|
|
23
|
-
let position = start;
|
|
24
|
-
let lineStart = start;
|
|
25
|
-
let carry = Buffer.alloc(0);
|
|
26
|
-
while (position < end) {
|
|
27
|
-
const chunk = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, end - position));
|
|
28
|
-
const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
|
|
29
|
-
if (!bytesRead) throw new Error("Queue changed while reading");
|
|
30
|
-
position += bytesRead;
|
|
31
|
-
const data = carry.length ? Buffer.concat([carry, chunk.subarray(0, bytesRead)]) : chunk.subarray(0, bytesRead);
|
|
32
|
-
let consumed = 0;
|
|
33
|
-
let newline;
|
|
34
|
-
while ((newline = data.indexOf(10, consumed)) !== -1) {
|
|
35
|
-
const bytes = data.subarray(consumed, newline + 1);
|
|
36
|
-
yield { bytes, start: lineStart, complete: true };
|
|
37
|
-
lineStart += bytes.length;
|
|
38
|
-
consumed = newline + 1;
|
|
39
|
-
}
|
|
40
|
-
carry = Buffer.from(data.subarray(consumed));
|
|
41
|
-
}
|
|
42
|
-
if (carry.length) yield { bytes: carry, start: lineStart, complete: false };
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function snapshotKey(bytes) {
|
|
46
|
-
if (!isUtf8(bytes)) return null;
|
|
47
|
-
let row;
|
|
48
|
-
try { row = JSON.parse(bytes.toString("utf8")); } catch { return null; }
|
|
49
|
-
if (!row || typeof row !== "object" || Array.isArray(row)) return null;
|
|
50
|
-
if (row.kind === "account_session_state") {
|
|
51
|
-
const verifiedMs = sessionSnapshotMs(row);
|
|
52
|
-
if (verifiedMs === null) return null;
|
|
53
|
-
return { key: JSON.stringify(["session", row.source.trim().toLowerCase(), row.session_id.trim()]), verifiedMs };
|
|
54
|
-
}
|
|
55
|
-
// Preserve unknown control records even when they happen to carry a bucket.
|
|
56
|
-
if (Object.hasOwn(row, "kind") || typeof row.hour_start !== "string" || !Number.isFinite(Date.parse(row.hour_start))) return null;
|
|
57
|
-
const source = (typeof row.source === "string" ? row.source.trim().toLowerCase() : "") || "codex";
|
|
58
|
-
const model = (typeof row.model === "string" ? row.model.trim() : "") || "unknown";
|
|
59
|
-
return { key: JSON.stringify(["hourly", source, model, row.hour_start]), verifiedMs: null };
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// Caller holds sync.lock. Only rewrite pending snapshots; uploaded history and
|
|
63
|
-
// queue.state.json stay byte-for-byte unchanged across an atomic replacement.
|
|
64
|
-
async function compactPendingQueue({ queuePath, offset = 0, minPendingBytes = BACKLOG_BYTES }) {
|
|
65
|
-
let input;
|
|
66
|
-
let output;
|
|
67
|
-
let temporary;
|
|
68
|
-
try {
|
|
69
|
-
input = await fs.open(queuePath, "r");
|
|
70
|
-
const before = await input.stat();
|
|
71
|
-
const pendingBytesBefore = Math.max(0, before.size - offset);
|
|
72
|
-
const unchanged = { compacted: false, removedRecords: 0, pendingBytesBefore, pendingBytesAfter: pendingBytesBefore };
|
|
73
|
-
if (!before.isFile() || !Number.isSafeInteger(offset) || offset < 0 || offset >= before.size || pendingBytesBefore < minPendingBytes) return unchanged;
|
|
74
|
-
if (offset > 0) {
|
|
75
|
-
const boundary = Buffer.alloc(1);
|
|
76
|
-
await input.read(boundary, 0, 1, offset - 1);
|
|
77
|
-
if (boundary[0] !== 10) return unchanged;
|
|
78
|
-
}
|
|
79
|
-
const latest = new Map();
|
|
80
|
-
let removedRecords = 0;
|
|
81
|
-
for await (const line of iterateQueueLines(input, offset, before.size)) {
|
|
82
|
-
const snapshot = line.complete ? snapshotKey(line.bytes) : null;
|
|
83
|
-
if (snapshot == null) continue;
|
|
84
|
-
const previous = latest.get(snapshot.key);
|
|
85
|
-
if (previous) removedRecords += 1;
|
|
86
|
-
// Account truth uses a strictly newer fetch stamp; equal stamps retain
|
|
87
|
-
// the first observation, even after a replay or local clock rollback.
|
|
88
|
-
if (!previous || snapshot.verifiedMs === null || snapshot.verifiedMs > previous.verifiedMs) {
|
|
89
|
-
latest.set(snapshot.key, { start: line.start, verifiedMs: snapshot.verifiedMs });
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
if (!removedRecords) return unchanged;
|
|
93
|
-
|
|
94
|
-
temporary = `${queuePath}.compact.${crypto.randomUUID()}`;
|
|
95
|
-
output = await fs.open(temporary, "wx", before.mode & 0o777);
|
|
96
|
-
await output.chmod(before.mode & 0o777);
|
|
97
|
-
for (let position = 0; position < offset;) {
|
|
98
|
-
const chunk = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, offset - position));
|
|
99
|
-
const { bytesRead } = await input.read(chunk, 0, chunk.length, position);
|
|
100
|
-
if (!bytesRead) throw new Error("Queue changed while copying uploaded history");
|
|
101
|
-
await output.writeFile(chunk.subarray(0, bytesRead));
|
|
102
|
-
position += bytesRead;
|
|
103
|
-
}
|
|
104
|
-
let pendingBytesAfter = 0;
|
|
105
|
-
for await (const line of iterateQueueLines(input, offset, before.size)) {
|
|
106
|
-
const snapshot = line.complete ? snapshotKey(line.bytes) : null;
|
|
107
|
-
if (snapshot != null && latest.get(snapshot.key).start !== line.start) continue;
|
|
108
|
-
await output.writeFile(line.bytes);
|
|
109
|
-
pendingBytesAfter += line.bytes.length;
|
|
110
|
-
}
|
|
111
|
-
await output.sync();
|
|
112
|
-
await output.close();
|
|
113
|
-
output = null;
|
|
114
|
-
// Windows cannot replace the destination while our read handle is open.
|
|
115
|
-
await input.close();
|
|
116
|
-
input = null;
|
|
117
|
-
const current = await fs.stat(queuePath);
|
|
118
|
-
if (current.ino !== before.ino || current.size !== before.size || current.mtimeMs !== before.mtimeMs) {
|
|
119
|
-
throw new Error("Queue changed during pending compaction");
|
|
120
|
-
}
|
|
121
|
-
await fs.rename(temporary, queuePath);
|
|
122
|
-
temporary = null;
|
|
123
|
-
return { compacted: true, removedRecords, pendingBytesBefore, pendingBytesAfter };
|
|
124
|
-
} catch (error) {
|
|
125
|
-
if (error.code === "ENOENT" && !input && !temporary) return { compacted: false, removedRecords: 0, pendingBytesBefore: 0, pendingBytesAfter: 0 };
|
|
126
|
-
throw error;
|
|
127
|
-
} finally {
|
|
128
|
-
await input?.close();
|
|
129
|
-
await output?.close();
|
|
130
|
-
if (temporary) await fs.unlink(temporary).catch(() => {});
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
module.exports = { BACKLOG_BYTES, compactPendingQueue, iterateQueueLines, sessionSnapshotMs };
|
|
1
|
+
const fs = require("node:fs/promises");
|
|
2
|
+
const crypto = require("node:crypto");
|
|
3
|
+
const { isUtf8 } = require("node:buffer");
|
|
4
|
+
|
|
5
|
+
const BACKLOG_BYTES = 1_000_000;
|
|
6
|
+
const CHUNK_BYTES = 64 * 1024;
|
|
7
|
+
const { TOKEN_COLUMNS } = require("./account-usage-dedup");
|
|
8
|
+
|
|
9
|
+
function sessionSnapshotMs(row) {
|
|
10
|
+
const verifiedMs = Date.parse(row?.snapshot_verified_at);
|
|
11
|
+
if (typeof row?.source !== "string" || row.source.trim().toLowerCase() !== "trae-cn" ||
|
|
12
|
+
typeof row.session_id !== "string" || !row.session_id.trim() || row.session_id.trim().length > 256 ||
|
|
13
|
+
typeof row.model !== "string" || !row.model.trim() || row.model.trim().length > 200 ||
|
|
14
|
+
!Number.isFinite(Date.parse(row.bucket_start)) || !Number.isFinite(verifiedMs) ||
|
|
15
|
+
TOKEN_COLUMNS.some((column) => !Number.isSafeInteger(Number(row[column])) || Number(row[column]) < 0) ||
|
|
16
|
+
Number(row.total_tokens) !== Number(row.input_tokens) + Number(row.output_tokens) + Number(row.cached_input_tokens) + Number(row.cache_creation_input_tokens)) return null;
|
|
17
|
+
return verifiedMs;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Keep raw bytes, including CRLF and incomplete tails. Upload offsets are
|
|
21
|
+
// byte positions, not JavaScript string lengths.
|
|
22
|
+
async function* iterateQueueLines(handle, start, end) {
|
|
23
|
+
let position = start;
|
|
24
|
+
let lineStart = start;
|
|
25
|
+
let carry = Buffer.alloc(0);
|
|
26
|
+
while (position < end) {
|
|
27
|
+
const chunk = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, end - position));
|
|
28
|
+
const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
|
|
29
|
+
if (!bytesRead) throw new Error("Queue changed while reading");
|
|
30
|
+
position += bytesRead;
|
|
31
|
+
const data = carry.length ? Buffer.concat([carry, chunk.subarray(0, bytesRead)]) : chunk.subarray(0, bytesRead);
|
|
32
|
+
let consumed = 0;
|
|
33
|
+
let newline;
|
|
34
|
+
while ((newline = data.indexOf(10, consumed)) !== -1) {
|
|
35
|
+
const bytes = data.subarray(consumed, newline + 1);
|
|
36
|
+
yield { bytes, start: lineStart, complete: true };
|
|
37
|
+
lineStart += bytes.length;
|
|
38
|
+
consumed = newline + 1;
|
|
39
|
+
}
|
|
40
|
+
carry = Buffer.from(data.subarray(consumed));
|
|
41
|
+
}
|
|
42
|
+
if (carry.length) yield { bytes: carry, start: lineStart, complete: false };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function snapshotKey(bytes) {
|
|
46
|
+
if (!isUtf8(bytes)) return null;
|
|
47
|
+
let row;
|
|
48
|
+
try { row = JSON.parse(bytes.toString("utf8")); } catch { return null; }
|
|
49
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) return null;
|
|
50
|
+
if (row.kind === "account_session_state") {
|
|
51
|
+
const verifiedMs = sessionSnapshotMs(row);
|
|
52
|
+
if (verifiedMs === null) return null;
|
|
53
|
+
return { key: JSON.stringify(["session", row.source.trim().toLowerCase(), row.session_id.trim()]), verifiedMs };
|
|
54
|
+
}
|
|
55
|
+
// Preserve unknown control records even when they happen to carry a bucket.
|
|
56
|
+
if (Object.hasOwn(row, "kind") || typeof row.hour_start !== "string" || !Number.isFinite(Date.parse(row.hour_start))) return null;
|
|
57
|
+
const source = (typeof row.source === "string" ? row.source.trim().toLowerCase() : "") || "codex";
|
|
58
|
+
const model = (typeof row.model === "string" ? row.model.trim() : "") || "unknown";
|
|
59
|
+
return { key: JSON.stringify(["hourly", source, model, row.hour_start]), verifiedMs: null };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Caller holds sync.lock. Only rewrite pending snapshots; uploaded history and
|
|
63
|
+
// queue.state.json stay byte-for-byte unchanged across an atomic replacement.
|
|
64
|
+
async function compactPendingQueue({ queuePath, offset = 0, minPendingBytes = BACKLOG_BYTES }) {
|
|
65
|
+
let input;
|
|
66
|
+
let output;
|
|
67
|
+
let temporary;
|
|
68
|
+
try {
|
|
69
|
+
input = await fs.open(queuePath, "r");
|
|
70
|
+
const before = await input.stat();
|
|
71
|
+
const pendingBytesBefore = Math.max(0, before.size - offset);
|
|
72
|
+
const unchanged = { compacted: false, removedRecords: 0, pendingBytesBefore, pendingBytesAfter: pendingBytesBefore };
|
|
73
|
+
if (!before.isFile() || !Number.isSafeInteger(offset) || offset < 0 || offset >= before.size || pendingBytesBefore < minPendingBytes) return unchanged;
|
|
74
|
+
if (offset > 0) {
|
|
75
|
+
const boundary = Buffer.alloc(1);
|
|
76
|
+
await input.read(boundary, 0, 1, offset - 1);
|
|
77
|
+
if (boundary[0] !== 10) return unchanged;
|
|
78
|
+
}
|
|
79
|
+
const latest = new Map();
|
|
80
|
+
let removedRecords = 0;
|
|
81
|
+
for await (const line of iterateQueueLines(input, offset, before.size)) {
|
|
82
|
+
const snapshot = line.complete ? snapshotKey(line.bytes) : null;
|
|
83
|
+
if (snapshot == null) continue;
|
|
84
|
+
const previous = latest.get(snapshot.key);
|
|
85
|
+
if (previous) removedRecords += 1;
|
|
86
|
+
// Account truth uses a strictly newer fetch stamp; equal stamps retain
|
|
87
|
+
// the first observation, even after a replay or local clock rollback.
|
|
88
|
+
if (!previous || snapshot.verifiedMs === null || snapshot.verifiedMs > previous.verifiedMs) {
|
|
89
|
+
latest.set(snapshot.key, { start: line.start, verifiedMs: snapshot.verifiedMs });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (!removedRecords) return unchanged;
|
|
93
|
+
|
|
94
|
+
temporary = `${queuePath}.compact.${crypto.randomUUID()}`;
|
|
95
|
+
output = await fs.open(temporary, "wx", before.mode & 0o777);
|
|
96
|
+
await output.chmod(before.mode & 0o777);
|
|
97
|
+
for (let position = 0; position < offset;) {
|
|
98
|
+
const chunk = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, offset - position));
|
|
99
|
+
const { bytesRead } = await input.read(chunk, 0, chunk.length, position);
|
|
100
|
+
if (!bytesRead) throw new Error("Queue changed while copying uploaded history");
|
|
101
|
+
await output.writeFile(chunk.subarray(0, bytesRead));
|
|
102
|
+
position += bytesRead;
|
|
103
|
+
}
|
|
104
|
+
let pendingBytesAfter = 0;
|
|
105
|
+
for await (const line of iterateQueueLines(input, offset, before.size)) {
|
|
106
|
+
const snapshot = line.complete ? snapshotKey(line.bytes) : null;
|
|
107
|
+
if (snapshot != null && latest.get(snapshot.key).start !== line.start) continue;
|
|
108
|
+
await output.writeFile(line.bytes);
|
|
109
|
+
pendingBytesAfter += line.bytes.length;
|
|
110
|
+
}
|
|
111
|
+
await output.sync();
|
|
112
|
+
await output.close();
|
|
113
|
+
output = null;
|
|
114
|
+
// Windows cannot replace the destination while our read handle is open.
|
|
115
|
+
await input.close();
|
|
116
|
+
input = null;
|
|
117
|
+
const current = await fs.stat(queuePath);
|
|
118
|
+
if (current.ino !== before.ino || current.size !== before.size || current.mtimeMs !== before.mtimeMs) {
|
|
119
|
+
throw new Error("Queue changed during pending compaction");
|
|
120
|
+
}
|
|
121
|
+
await fs.rename(temporary, queuePath);
|
|
122
|
+
temporary = null;
|
|
123
|
+
return { compacted: true, removedRecords, pendingBytesBefore, pendingBytesAfter };
|
|
124
|
+
} catch (error) {
|
|
125
|
+
if (error.code === "ENOENT" && !input && !temporary) return { compacted: false, removedRecords: 0, pendingBytesBefore: 0, pendingBytesAfter: 0 };
|
|
126
|
+
throw error;
|
|
127
|
+
} finally {
|
|
128
|
+
await input?.close();
|
|
129
|
+
await output?.close();
|
|
130
|
+
if (temporary) await fs.unlink(temporary).catch(() => {});
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
module.exports = { BACKLOG_BYTES, compactPendingQueue, iterateQueueLines, sessionSnapshotMs };
|
|
@@ -1,171 +1,171 @@
|
|
|
1
|
-
const DEFAULTS = {
|
|
2
|
-
intervalMs: 30 * 60_000,
|
|
3
|
-
jitterMsMax: 60_000,
|
|
4
|
-
backlogBytes: 1_000_000,
|
|
5
|
-
batchSize: 300,
|
|
6
|
-
maxBatchesSmall: 2,
|
|
7
|
-
maxBatchesLarge: 4,
|
|
8
|
-
backoffInitialMs: 60_000,
|
|
9
|
-
backoffMaxMs: 30 * 60_000,
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
function normalizeState(raw) {
|
|
13
|
-
const s = raw && typeof raw === "object" ? raw : {};
|
|
14
|
-
return {
|
|
15
|
-
version: 1,
|
|
16
|
-
lastSuccessMs: toSafeInt(s.lastSuccessMs),
|
|
17
|
-
lastSuccessAttemptId: typeof s.lastSuccessAttemptId === "string" ? s.lastSuccessAttemptId : null,
|
|
18
|
-
nextAllowedAtMs: toSafeInt(s.nextAllowedAtMs),
|
|
19
|
-
serverNextAllowedAtMs: toSafeInt(s.serverNextAllowedAtMs),
|
|
20
|
-
backoffUntilMs: toSafeInt(s.backoffUntilMs),
|
|
21
|
-
backoffStep: toSafeInt(s.backoffStep),
|
|
22
|
-
lastErrorAt: typeof s.lastErrorAt === "string" ? s.lastErrorAt : null,
|
|
23
|
-
lastError: typeof s.lastError === "string" ? s.lastError : null,
|
|
24
|
-
lastErrorStatus: toSafeInt(s.lastErrorStatus),
|
|
25
|
-
lastErrorCode: typeof s.lastErrorCode === "string" ? s.lastErrorCode : null,
|
|
26
|
-
lastErrorAttemptId: typeof s.lastErrorAttemptId === "string" ? s.lastErrorAttemptId : null,
|
|
27
|
-
updatedAt: typeof s.updatedAt === "string" ? s.updatedAt : null,
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
function decideAutoUpload({ nowMs, pendingBytes, state, config }) {
|
|
32
|
-
const cfg = { ...DEFAULTS, ...(config || {}) };
|
|
33
|
-
const s = normalizeState(state);
|
|
34
|
-
const pending = Number(pendingBytes || 0);
|
|
35
|
-
|
|
36
|
-
if (pending <= 0) {
|
|
37
|
-
return {
|
|
38
|
-
allowed: false,
|
|
39
|
-
reason: "no-pending",
|
|
40
|
-
maxBatches: 0,
|
|
41
|
-
batchSize: cfg.batchSize,
|
|
42
|
-
blockedUntilMs: 0,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
const blockedUntilMs = Math.max(s.nextAllowedAtMs || 0, s.backoffUntilMs || 0);
|
|
47
|
-
if (blockedUntilMs > 0 && nowMs < blockedUntilMs) {
|
|
48
|
-
return {
|
|
49
|
-
allowed: false,
|
|
50
|
-
reason: "throttled",
|
|
51
|
-
maxBatches: 0,
|
|
52
|
-
batchSize: cfg.batchSize,
|
|
53
|
-
blockedUntilMs,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
const maxBatches = pending >= cfg.backlogBytes ? cfg.maxBatchesLarge : cfg.maxBatchesSmall;
|
|
58
|
-
return {
|
|
59
|
-
allowed: true,
|
|
60
|
-
reason: "allowed",
|
|
61
|
-
maxBatches,
|
|
62
|
-
batchSize: cfg.batchSize,
|
|
63
|
-
blockedUntilMs: 0,
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
function recordUploadSuccess({ nowMs, state, config, randInt, cloudAccess, attemptId }) {
|
|
68
|
-
const cfg = { ...DEFAULTS, ...(config || {}) };
|
|
69
|
-
const s = normalizeState(state);
|
|
70
|
-
const jitter =
|
|
71
|
-
typeof randInt === "function" ? randInt(0, cfg.jitterMsMax) : randomInt(0, cfg.jitterMsMax);
|
|
72
|
-
const serverNextAllowedAtMs = toSafeInt(Date.parse(cloudAccess?.next_allowed_at));
|
|
73
|
-
const rawInterval = cloudAccess?.sync_interval_seconds;
|
|
74
|
-
const seconds = typeof rawInterval === "number" ? rawInterval
|
|
75
|
-
: typeof rawInterval === "string" && /^\d+$/.test(rawInterval.trim()) ? Number(rawInterval) : NaN;
|
|
76
|
-
const intervalMs = Number.isFinite(seconds) && seconds >= 0 ? Math.floor(seconds) * 1000 : cfg.intervalMs;
|
|
77
|
-
const nextAllowedAtMs = Math.max(serverNextAllowedAtMs, nowMs + intervalMs + jitter);
|
|
78
|
-
|
|
79
|
-
return {
|
|
80
|
-
...s,
|
|
81
|
-
lastSuccessMs: nowMs,
|
|
82
|
-
lastSuccessAttemptId: typeof attemptId === "string" && attemptId ? attemptId : null,
|
|
83
|
-
nextAllowedAtMs,
|
|
84
|
-
serverNextAllowedAtMs,
|
|
85
|
-
backoffUntilMs: 0,
|
|
86
|
-
backoffStep: 0,
|
|
87
|
-
lastErrorAt: null,
|
|
88
|
-
lastError: null,
|
|
89
|
-
lastErrorStatus: 0,
|
|
90
|
-
lastErrorCode: null,
|
|
91
|
-
lastErrorAttemptId: null,
|
|
92
|
-
updatedAt: new Date(nowMs).toISOString(),
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function recordUploadFailure({ nowMs, state, error, config, attemptId }) {
|
|
97
|
-
const cfg = { ...DEFAULTS, ...(config || {}) };
|
|
98
|
-
const s = normalizeState(state);
|
|
99
|
-
|
|
100
|
-
const retryAfterMs = toSafeInt(error?.retryAfterMs);
|
|
101
|
-
const status = toSafeInt(error?.status);
|
|
102
|
-
|
|
103
|
-
let backoffMs = 0;
|
|
104
|
-
if (retryAfterMs > 0) {
|
|
105
|
-
// Retry-After is the server's deadline. The configured cap applies only
|
|
106
|
-
// to exponential retries without an explicit server deadline.
|
|
107
|
-
backoffMs = Math.max(cfg.backoffInitialMs, retryAfterMs);
|
|
108
|
-
} else {
|
|
109
|
-
const step = Math.min(10, Math.max(0, s.backoffStep || 0));
|
|
110
|
-
backoffMs = Math.min(cfg.backoffMaxMs, cfg.backoffInitialMs * Math.pow(2, step));
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
const backoffUntilMs = nowMs + backoffMs;
|
|
114
|
-
const hasServerDeadline = status === 429 && error?.code === "cloud_sync_throttled" && retryAfterMs > 0;
|
|
115
|
-
const nextAllowedAtMs = hasServerDeadline
|
|
116
|
-
? backoffUntilMs : Math.max(s.nextAllowedAtMs || 0, backoffUntilMs);
|
|
117
|
-
|
|
118
|
-
return {
|
|
119
|
-
...s,
|
|
120
|
-
nextAllowedAtMs,
|
|
121
|
-
serverNextAllowedAtMs: hasServerDeadline ? backoffUntilMs : s.serverNextAllowedAtMs,
|
|
122
|
-
backoffUntilMs,
|
|
123
|
-
backoffStep: Math.min(20, (s.backoffStep || 0) + 1),
|
|
124
|
-
lastErrorAt: new Date(nowMs).toISOString(),
|
|
125
|
-
lastError: truncate(String(error?.message || "upload failed"), 200),
|
|
126
|
-
lastErrorStatus: status,
|
|
127
|
-
lastErrorCode: typeof error?.code === "string" ? error.code : null,
|
|
128
|
-
lastErrorAttemptId: typeof attemptId === "string" && attemptId ? attemptId : null,
|
|
129
|
-
updatedAt: new Date(nowMs).toISOString(),
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function parseRetryAfterMs(headerValue, nowMs = Date.now()) {
|
|
134
|
-
if (typeof headerValue !== "string" || headerValue.trim().length === 0) return null;
|
|
135
|
-
const v = headerValue.trim();
|
|
136
|
-
const seconds = Number(v);
|
|
137
|
-
if (Number.isFinite(seconds) && seconds >= 0) return Math.floor(seconds * 1000);
|
|
138
|
-
const d = new Date(v);
|
|
139
|
-
if (Number.isNaN(d.getTime())) return null;
|
|
140
|
-
const delta = d.getTime() - nowMs;
|
|
141
|
-
return delta > 0 ? delta : 0;
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function randomInt(min, maxInclusive) {
|
|
145
|
-
const lo = Math.floor(min);
|
|
146
|
-
const hi = Math.floor(maxInclusive);
|
|
147
|
-
if (hi <= lo) return lo;
|
|
148
|
-
return lo + Math.floor(Math.random() * (hi - lo + 1));
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function toSafeInt(v) {
|
|
152
|
-
const n = Number(v);
|
|
153
|
-
if (!Number.isFinite(n)) return 0;
|
|
154
|
-
if (n <= 0) return 0;
|
|
155
|
-
return Math.floor(n);
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
function truncate(s, maxLen) {
|
|
159
|
-
if (typeof s !== "string") return "";
|
|
160
|
-
if (s.length <= maxLen) return s;
|
|
161
|
-
return s.slice(0, maxLen - 1) + "…";
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
module.exports = {
|
|
165
|
-
DEFAULTS,
|
|
166
|
-
normalizeState,
|
|
167
|
-
decideAutoUpload,
|
|
168
|
-
recordUploadSuccess,
|
|
169
|
-
recordUploadFailure,
|
|
170
|
-
parseRetryAfterMs,
|
|
171
|
-
};
|
|
1
|
+
const DEFAULTS = {
|
|
2
|
+
intervalMs: 30 * 60_000,
|
|
3
|
+
jitterMsMax: 60_000,
|
|
4
|
+
backlogBytes: 1_000_000,
|
|
5
|
+
batchSize: 300,
|
|
6
|
+
maxBatchesSmall: 2,
|
|
7
|
+
maxBatchesLarge: 4,
|
|
8
|
+
backoffInitialMs: 60_000,
|
|
9
|
+
backoffMaxMs: 30 * 60_000,
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
function normalizeState(raw) {
|
|
13
|
+
const s = raw && typeof raw === "object" ? raw : {};
|
|
14
|
+
return {
|
|
15
|
+
version: 1,
|
|
16
|
+
lastSuccessMs: toSafeInt(s.lastSuccessMs),
|
|
17
|
+
lastSuccessAttemptId: typeof s.lastSuccessAttemptId === "string" ? s.lastSuccessAttemptId : null,
|
|
18
|
+
nextAllowedAtMs: toSafeInt(s.nextAllowedAtMs),
|
|
19
|
+
serverNextAllowedAtMs: toSafeInt(s.serverNextAllowedAtMs),
|
|
20
|
+
backoffUntilMs: toSafeInt(s.backoffUntilMs),
|
|
21
|
+
backoffStep: toSafeInt(s.backoffStep),
|
|
22
|
+
lastErrorAt: typeof s.lastErrorAt === "string" ? s.lastErrorAt : null,
|
|
23
|
+
lastError: typeof s.lastError === "string" ? s.lastError : null,
|
|
24
|
+
lastErrorStatus: toSafeInt(s.lastErrorStatus),
|
|
25
|
+
lastErrorCode: typeof s.lastErrorCode === "string" ? s.lastErrorCode : null,
|
|
26
|
+
lastErrorAttemptId: typeof s.lastErrorAttemptId === "string" ? s.lastErrorAttemptId : null,
|
|
27
|
+
updatedAt: typeof s.updatedAt === "string" ? s.updatedAt : null,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function decideAutoUpload({ nowMs, pendingBytes, state, config }) {
|
|
32
|
+
const cfg = { ...DEFAULTS, ...(config || {}) };
|
|
33
|
+
const s = normalizeState(state);
|
|
34
|
+
const pending = Number(pendingBytes || 0);
|
|
35
|
+
|
|
36
|
+
if (pending <= 0) {
|
|
37
|
+
return {
|
|
38
|
+
allowed: false,
|
|
39
|
+
reason: "no-pending",
|
|
40
|
+
maxBatches: 0,
|
|
41
|
+
batchSize: cfg.batchSize,
|
|
42
|
+
blockedUntilMs: 0,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const blockedUntilMs = Math.max(s.nextAllowedAtMs || 0, s.backoffUntilMs || 0);
|
|
47
|
+
if (blockedUntilMs > 0 && nowMs < blockedUntilMs) {
|
|
48
|
+
return {
|
|
49
|
+
allowed: false,
|
|
50
|
+
reason: "throttled",
|
|
51
|
+
maxBatches: 0,
|
|
52
|
+
batchSize: cfg.batchSize,
|
|
53
|
+
blockedUntilMs,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const maxBatches = pending >= cfg.backlogBytes ? cfg.maxBatchesLarge : cfg.maxBatchesSmall;
|
|
58
|
+
return {
|
|
59
|
+
allowed: true,
|
|
60
|
+
reason: "allowed",
|
|
61
|
+
maxBatches,
|
|
62
|
+
batchSize: cfg.batchSize,
|
|
63
|
+
blockedUntilMs: 0,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function recordUploadSuccess({ nowMs, state, config, randInt, cloudAccess, attemptId }) {
|
|
68
|
+
const cfg = { ...DEFAULTS, ...(config || {}) };
|
|
69
|
+
const s = normalizeState(state);
|
|
70
|
+
const jitter =
|
|
71
|
+
typeof randInt === "function" ? randInt(0, cfg.jitterMsMax) : randomInt(0, cfg.jitterMsMax);
|
|
72
|
+
const serverNextAllowedAtMs = toSafeInt(Date.parse(cloudAccess?.next_allowed_at));
|
|
73
|
+
const rawInterval = cloudAccess?.sync_interval_seconds;
|
|
74
|
+
const seconds = typeof rawInterval === "number" ? rawInterval
|
|
75
|
+
: typeof rawInterval === "string" && /^\d+$/.test(rawInterval.trim()) ? Number(rawInterval) : NaN;
|
|
76
|
+
const intervalMs = Number.isFinite(seconds) && seconds >= 0 ? Math.floor(seconds) * 1000 : cfg.intervalMs;
|
|
77
|
+
const nextAllowedAtMs = Math.max(serverNextAllowedAtMs, nowMs + intervalMs + jitter);
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
...s,
|
|
81
|
+
lastSuccessMs: nowMs,
|
|
82
|
+
lastSuccessAttemptId: typeof attemptId === "string" && attemptId ? attemptId : null,
|
|
83
|
+
nextAllowedAtMs,
|
|
84
|
+
serverNextAllowedAtMs,
|
|
85
|
+
backoffUntilMs: 0,
|
|
86
|
+
backoffStep: 0,
|
|
87
|
+
lastErrorAt: null,
|
|
88
|
+
lastError: null,
|
|
89
|
+
lastErrorStatus: 0,
|
|
90
|
+
lastErrorCode: null,
|
|
91
|
+
lastErrorAttemptId: null,
|
|
92
|
+
updatedAt: new Date(nowMs).toISOString(),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function recordUploadFailure({ nowMs, state, error, config, attemptId }) {
|
|
97
|
+
const cfg = { ...DEFAULTS, ...(config || {}) };
|
|
98
|
+
const s = normalizeState(state);
|
|
99
|
+
|
|
100
|
+
const retryAfterMs = toSafeInt(error?.retryAfterMs);
|
|
101
|
+
const status = toSafeInt(error?.status);
|
|
102
|
+
|
|
103
|
+
let backoffMs = 0;
|
|
104
|
+
if (retryAfterMs > 0) {
|
|
105
|
+
// Retry-After is the server's deadline. The configured cap applies only
|
|
106
|
+
// to exponential retries without an explicit server deadline.
|
|
107
|
+
backoffMs = Math.max(cfg.backoffInitialMs, retryAfterMs);
|
|
108
|
+
} else {
|
|
109
|
+
const step = Math.min(10, Math.max(0, s.backoffStep || 0));
|
|
110
|
+
backoffMs = Math.min(cfg.backoffMaxMs, cfg.backoffInitialMs * Math.pow(2, step));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const backoffUntilMs = nowMs + backoffMs;
|
|
114
|
+
const hasServerDeadline = status === 429 && error?.code === "cloud_sync_throttled" && retryAfterMs > 0;
|
|
115
|
+
const nextAllowedAtMs = hasServerDeadline
|
|
116
|
+
? backoffUntilMs : Math.max(s.nextAllowedAtMs || 0, backoffUntilMs);
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
...s,
|
|
120
|
+
nextAllowedAtMs,
|
|
121
|
+
serverNextAllowedAtMs: hasServerDeadline ? backoffUntilMs : s.serverNextAllowedAtMs,
|
|
122
|
+
backoffUntilMs,
|
|
123
|
+
backoffStep: Math.min(20, (s.backoffStep || 0) + 1),
|
|
124
|
+
lastErrorAt: new Date(nowMs).toISOString(),
|
|
125
|
+
lastError: truncate(String(error?.message || "upload failed"), 200),
|
|
126
|
+
lastErrorStatus: status,
|
|
127
|
+
lastErrorCode: typeof error?.code === "string" ? error.code : null,
|
|
128
|
+
lastErrorAttemptId: typeof attemptId === "string" && attemptId ? attemptId : null,
|
|
129
|
+
updatedAt: new Date(nowMs).toISOString(),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function parseRetryAfterMs(headerValue, nowMs = Date.now()) {
|
|
134
|
+
if (typeof headerValue !== "string" || headerValue.trim().length === 0) return null;
|
|
135
|
+
const v = headerValue.trim();
|
|
136
|
+
const seconds = Number(v);
|
|
137
|
+
if (Number.isFinite(seconds) && seconds >= 0) return Math.floor(seconds * 1000);
|
|
138
|
+
const d = new Date(v);
|
|
139
|
+
if (Number.isNaN(d.getTime())) return null;
|
|
140
|
+
const delta = d.getTime() - nowMs;
|
|
141
|
+
return delta > 0 ? delta : 0;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function randomInt(min, maxInclusive) {
|
|
145
|
+
const lo = Math.floor(min);
|
|
146
|
+
const hi = Math.floor(maxInclusive);
|
|
147
|
+
if (hi <= lo) return lo;
|
|
148
|
+
return lo + Math.floor(Math.random() * (hi - lo + 1));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function toSafeInt(v) {
|
|
152
|
+
const n = Number(v);
|
|
153
|
+
if (!Number.isFinite(n)) return 0;
|
|
154
|
+
if (n <= 0) return 0;
|
|
155
|
+
return Math.floor(n);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function truncate(s, maxLen) {
|
|
159
|
+
if (typeof s !== "string") return "";
|
|
160
|
+
if (s.length <= maxLen) return s;
|
|
161
|
+
return s.slice(0, maxLen - 1) + "…";
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
module.exports = {
|
|
165
|
+
DEFAULTS,
|
|
166
|
+
normalizeState,
|
|
167
|
+
decideAutoUpload,
|
|
168
|
+
recordUploadSuccess,
|
|
169
|
+
recordUploadFailure,
|
|
170
|
+
parseRetryAfterMs,
|
|
171
|
+
};
|