tokenusage-cli 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/package.json +1 -1
- package/src/commands/sync.js +30 -0
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/project-repos.js +259 -0
- package/src/lib/rollout.js +63 -4
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Detected repository URLs for projects (docs/cli-contract.md §2.4 `projects`,
|
|
4
|
+
// contract v3, 2026-11-06). The CLI only ever reports public GitHub / Gitee
|
|
5
|
+
// remotes it already canonicalised, and only the entries that differ from what
|
|
6
|
+
// this device last had confirmed — the server stores one row per
|
|
7
|
+
// (user, device, project_key), so sending the full map on every sync would
|
|
8
|
+
// rewrite rows for unchanged projects.
|
|
9
|
+
//
|
|
10
|
+
// Wire rules mirrored from the contract (the server validates and drops
|
|
11
|
+
// non-conforming entries instead of failing the batch):
|
|
12
|
+
// • https://github.com/<owner>/<repo> or https://gitee.com/<owner>/<repo>
|
|
13
|
+
// • exactly two path segments of [A-Za-z0-9._-], no credentials/port/query
|
|
14
|
+
// • `null` clears this device's previously reported value for the project
|
|
15
|
+
|
|
16
|
+
const fs = require("node:fs");
|
|
17
|
+
const fsp = require("node:fs/promises");
|
|
18
|
+
const path = require("node:path");
|
|
19
|
+
const crypto = require("node:crypto");
|
|
20
|
+
|
|
21
|
+
const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
|
|
22
|
+
|
|
23
|
+
const INGEST_SLUG = "token-usage-ingest";
|
|
24
|
+
const STATE_VERSION = 1;
|
|
25
|
+
const MAX_PROJECTS_PER_BATCH = 200;
|
|
26
|
+
const REPO_HOSTS = new Set(["github.com", "gitee.com"]);
|
|
27
|
+
const PATH_SEGMENT = /^[A-Za-z0-9._-]{1,100}$/;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Canonicalise a git remote into the contract's only accepted shape.
|
|
31
|
+
* Returns null for anything that is not a public GitHub / Gitee repository
|
|
32
|
+
* (self-hosted GitLab, enterprise hosts, local paths, credentials, ports).
|
|
33
|
+
*/
|
|
34
|
+
function normalizeRepoUrl(value) {
|
|
35
|
+
if (typeof value !== "string") return null;
|
|
36
|
+
let raw = value.trim();
|
|
37
|
+
if (!raw) return null;
|
|
38
|
+
|
|
39
|
+
const scp = raw.match(/^[A-Za-z0-9._-]+@([^:/\\]+):(.+)$/);
|
|
40
|
+
if (scp) {
|
|
41
|
+
raw = `https://${scp[1]}/${scp[2]}`;
|
|
42
|
+
} else if (/^ssh:\/\//i.test(raw)) {
|
|
43
|
+
try {
|
|
44
|
+
const parsed = new URL(raw);
|
|
45
|
+
raw = `https://${parsed.hostname}${parsed.pathname}`;
|
|
46
|
+
} catch {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
} else if (/^git:\/\//i.test(raw)) {
|
|
50
|
+
raw = `https://${raw.slice("git://".length)}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
let parsed;
|
|
54
|
+
try {
|
|
55
|
+
parsed = new URL(raw);
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return null;
|
|
60
|
+
if (parsed.username || parsed.password || parsed.port) return null;
|
|
61
|
+
if (parsed.search || parsed.hash) return null;
|
|
62
|
+
|
|
63
|
+
const host = parsed.hostname.toLowerCase();
|
|
64
|
+
if (!REPO_HOSTS.has(host)) return null;
|
|
65
|
+
|
|
66
|
+
const segments = parsed.pathname.split("/").filter((segment) => segment.length > 0);
|
|
67
|
+
if (segments.length !== 2) return null;
|
|
68
|
+
|
|
69
|
+
const owner = segments[0];
|
|
70
|
+
let repo = segments[1];
|
|
71
|
+
if (/\.git$/i.test(repo)) repo = repo.slice(0, -4);
|
|
72
|
+
if (!PATH_SEGMENT.test(owner) || !PATH_SEGMENT.test(repo)) return null;
|
|
73
|
+
if (owner === "." || owner === ".." || repo === "." || repo === "..") return null;
|
|
74
|
+
|
|
75
|
+
const canonical = `https://${host}/${owner}/${repo}`;
|
|
76
|
+
return canonical.length <= 200 ? canonical : null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function emptyState() {
|
|
80
|
+
return { version: STATE_VERSION, confirmed: {} };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function loadProjectRepoState(filePath) {
|
|
84
|
+
try {
|
|
85
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
86
|
+
if (!parsed || typeof parsed !== "object") return emptyState();
|
|
87
|
+
const confirmed = {};
|
|
88
|
+
for (const [key, value] of Object.entries(
|
|
89
|
+
parsed.confirmed && typeof parsed.confirmed === "object" ? parsed.confirmed : {},
|
|
90
|
+
)) {
|
|
91
|
+
if (typeof key !== "string" || !key) continue;
|
|
92
|
+
if (value === null || typeof value === "string") confirmed[key] = value;
|
|
93
|
+
}
|
|
94
|
+
return { version: STATE_VERSION, confirmed };
|
|
95
|
+
} catch {
|
|
96
|
+
return emptyState();
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function saveProjectRepoState(filePath, state) {
|
|
101
|
+
const payload = {
|
|
102
|
+
version: STATE_VERSION,
|
|
103
|
+
confirmed: Object.fromEntries(
|
|
104
|
+
Object.entries(state.confirmed).sort(([a], [b]) => a.localeCompare(b)),
|
|
105
|
+
),
|
|
106
|
+
};
|
|
107
|
+
await fsp.mkdir(path.dirname(filePath), { recursive: true });
|
|
108
|
+
const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
|
|
109
|
+
await fsp.writeFile(tmp, JSON.stringify(payload, null, 2) + "\n", { mode: 0o600 });
|
|
110
|
+
await fsp.rename(tmp, filePath);
|
|
111
|
+
await fsp.chmod(filePath, 0o600).catch(() => {});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Entries this device should (re)report: new detections, moved repositories,
|
|
116
|
+
* and clears (a project whose remote disappeared or moved off GitHub/Gitee).
|
|
117
|
+
* Projects that were never confirmed and still have no repository are skipped —
|
|
118
|
+
* sending `null` for every local-only project would only burn requests.
|
|
119
|
+
*/
|
|
120
|
+
function collectProjectRepoChanges({ projects, confirmed = {} }) {
|
|
121
|
+
const entries = [];
|
|
122
|
+
for (const [key, meta] of Object.entries(
|
|
123
|
+
projects && typeof projects === "object" ? projects : {},
|
|
124
|
+
)) {
|
|
125
|
+
if (typeof key !== "string" || !key || !meta || typeof meta !== "object") continue;
|
|
126
|
+
const hasConfirmed = Object.prototype.hasOwnProperty.call(confirmed, key);
|
|
127
|
+
const previous = hasConfirmed ? confirmed[key] : undefined;
|
|
128
|
+
const status = typeof meta.status === "string" ? meta.status : null;
|
|
129
|
+
const detected = status === "public_verified" || status == null
|
|
130
|
+
? normalizeRepoUrl(meta.project_ref)
|
|
131
|
+
: null;
|
|
132
|
+
|
|
133
|
+
if (detected) {
|
|
134
|
+
if (detected !== previous) entries.push({ project_key: key, repo_url: detected });
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (hasConfirmed && previous !== null) {
|
|
138
|
+
entries.push({ project_key: key, repo_url: null });
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return entries;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** New confirmed map after the server acknowledged these entries. */
|
|
145
|
+
function applyProjectRepoConfirmation(confirmed, sent) {
|
|
146
|
+
const next = { ...confirmed };
|
|
147
|
+
for (const entry of sent) next[entry.project_key] = entry.repo_url;
|
|
148
|
+
return next;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function postProjects({
|
|
152
|
+
baseUrl,
|
|
153
|
+
anonKey,
|
|
154
|
+
deviceToken,
|
|
155
|
+
entries,
|
|
156
|
+
fetchImpl,
|
|
157
|
+
signal,
|
|
158
|
+
}) {
|
|
159
|
+
const headers = {
|
|
160
|
+
"Content-Type": "application/json",
|
|
161
|
+
Accept: "application/json",
|
|
162
|
+
Authorization: `Bearer ${deviceToken}`,
|
|
163
|
+
};
|
|
164
|
+
if (anonKey) headers.apikey = anonKey;
|
|
165
|
+
|
|
166
|
+
return fetchFunctionResponse(
|
|
167
|
+
functionUrlFor(baseUrl, INGEST_SLUG),
|
|
168
|
+
{
|
|
169
|
+
method: "POST",
|
|
170
|
+
headers,
|
|
171
|
+
...(signal ? { signal } : {}),
|
|
172
|
+
body: JSON.stringify({ projects: entries }),
|
|
173
|
+
},
|
|
174
|
+
fetchImpl,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Upload pending repository changes in batches of <=200. Returns
|
|
180
|
+
* `unsupported: true` (and leaves the state untouched) when the backend
|
|
181
|
+
* predates contract v3 — an old server answers a projects-only request with
|
|
182
|
+
* `400 No usage buckets provided`, and a retired route with 404/405. Both are
|
|
183
|
+
* "retry later after upgrading the backend", never a sync failure.
|
|
184
|
+
*/
|
|
185
|
+
async function uploadProjectRepos({
|
|
186
|
+
baseUrl,
|
|
187
|
+
anonKey,
|
|
188
|
+
deviceToken,
|
|
189
|
+
projects,
|
|
190
|
+
statePath,
|
|
191
|
+
canUpload = () => true,
|
|
192
|
+
batchSize = MAX_PROJECTS_PER_BATCH,
|
|
193
|
+
fetchImpl,
|
|
194
|
+
signal,
|
|
195
|
+
}) {
|
|
196
|
+
const state = loadProjectRepoState(statePath);
|
|
197
|
+
const pending = collectProjectRepoChanges({
|
|
198
|
+
projects,
|
|
199
|
+
confirmed: state.confirmed,
|
|
200
|
+
});
|
|
201
|
+
if (pending.length === 0) return { pending: 0, uploaded: 0, skipped: 0, batches: 0 };
|
|
202
|
+
if (!canUpload()) return { pending: pending.length, uploaded: 0, skipped: 0, batches: 0, paused: true };
|
|
203
|
+
|
|
204
|
+
const limit = Math.min(Math.max(1, Math.trunc(Number(batchSize) || 1)), MAX_PROJECTS_PER_BATCH);
|
|
205
|
+
let uploaded = 0;
|
|
206
|
+
let skipped = 0;
|
|
207
|
+
let batches = 0;
|
|
208
|
+
|
|
209
|
+
for (let index = 0; index < pending.length; index += limit) {
|
|
210
|
+
const batch = pending.slice(index, index + limit);
|
|
211
|
+
if (!canUpload()) break;
|
|
212
|
+
const res = await postProjects({ baseUrl, anonKey, deviceToken, entries: batch, fetchImpl, signal });
|
|
213
|
+
|
|
214
|
+
if (!res.ok) {
|
|
215
|
+
const text = await res.text().catch(() => "");
|
|
216
|
+
if (
|
|
217
|
+
res.status === 404 ||
|
|
218
|
+
res.status === 405 ||
|
|
219
|
+
(res.status === 400 && text.includes("No usage buckets provided"))
|
|
220
|
+
) {
|
|
221
|
+
return { pending: pending.length, uploaded, skipped, batches, unsupported: true };
|
|
222
|
+
}
|
|
223
|
+
const error = new Error(`repo upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`);
|
|
224
|
+
error.status = res.status;
|
|
225
|
+
throw error;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const data = await res.json().catch(() => null);
|
|
229
|
+
const isCount = (value) =>
|
|
230
|
+
(typeof value === "number" || typeof value === "string") &&
|
|
231
|
+
value !== "" &&
|
|
232
|
+
Number.isFinite(Number(value));
|
|
233
|
+
if (!data || typeof data !== "object" || !isCount(data.projects_upserted)) {
|
|
234
|
+
const error = new Error(
|
|
235
|
+
`repo upload was not confirmed (HTTP ${res.status}): unexpected response body`,
|
|
236
|
+
);
|
|
237
|
+
error.status = res.status;
|
|
238
|
+
throw error;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
state.confirmed = applyProjectRepoConfirmation(state.confirmed, batch);
|
|
242
|
+
await saveProjectRepoState(statePath, state);
|
|
243
|
+
uploaded += batch.length;
|
|
244
|
+
skipped += Number(data.projects_skipped || 0);
|
|
245
|
+
batches += 1;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return { pending: pending.length, uploaded, skipped, batches };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
module.exports = {
|
|
252
|
+
MAX_PROJECTS_PER_BATCH,
|
|
253
|
+
applyProjectRepoConfirmation,
|
|
254
|
+
collectProjectRepoChanges,
|
|
255
|
+
loadProjectRepoState,
|
|
256
|
+
normalizeRepoUrl,
|
|
257
|
+
saveProjectRepoState,
|
|
258
|
+
uploadProjectRepos,
|
|
259
|
+
};
|
package/src/lib/rollout.js
CHANGED
|
@@ -11314,6 +11314,34 @@ const WORKBUDDY_VARIANTS = {
|
|
|
11314
11314
|
"workbuddy-ai": { dir: ".workbuddy-ai", envVar: "WORKBUDDY_AI_HOME" },
|
|
11315
11315
|
};
|
|
11316
11316
|
|
|
11317
|
+
// Bump when the project pass changes how it derives a bucket's model. Version 2
|
|
11318
|
+
// pairs project usage with the hourly model, which is what enqueueTouchedBuckets
|
|
11319
|
+
// needs to emit project + residual rows; version-1 buckets have no model and can
|
|
11320
|
+
// never be split. An install below the current version resets only WorkBuddy's
|
|
11321
|
+
// project cursors and drops those model-less buckets — the totals cursors stay
|
|
11322
|
+
// untouched, so history is re-attributed without re-counting a single token.
|
|
11323
|
+
const WORKBUDDY_PROJECT_ATTRIBUTION_VERSION = 2;
|
|
11324
|
+
|
|
11325
|
+
// Remove project buckets written before the model dimension existed. Only the
|
|
11326
|
+
// buckets owned by the given WorkBuddy variant are touched; other sources and
|
|
11327
|
+
// every model-bearing bucket are left exactly as they are.
|
|
11328
|
+
function dropLegacyWorkbuddyProjectBuckets(projectState, variant) {
|
|
11329
|
+
if (!projectState || !projectState.buckets || typeof projectState.buckets !== "object") {
|
|
11330
|
+
return 0;
|
|
11331
|
+
}
|
|
11332
|
+
const source = normalizeSourceInput(variant) || DEFAULT_SOURCE;
|
|
11333
|
+
let removed = 0;
|
|
11334
|
+
for (const [key, bucket] of Object.entries(projectState.buckets)) {
|
|
11335
|
+
if (!bucket || typeof bucket !== "object") continue;
|
|
11336
|
+
// A legacy bucket is exactly the one getProjectBucket wrote with model == null.
|
|
11337
|
+
if (bucket.model != null) continue;
|
|
11338
|
+
if ((normalizeSourceInput(bucket.source) || DEFAULT_SOURCE) !== source) continue;
|
|
11339
|
+
delete projectState.buckets[key];
|
|
11340
|
+
removed += 1;
|
|
11341
|
+
}
|
|
11342
|
+
return removed;
|
|
11343
|
+
}
|
|
11344
|
+
|
|
11317
11345
|
function resolveWorkbuddyVariant(variant) {
|
|
11318
11346
|
return Object.prototype.hasOwnProperty.call(WORKBUDDY_VARIANTS, variant)
|
|
11319
11347
|
? variant
|
|
@@ -11617,13 +11645,27 @@ async function parseWorkbuddyIncremental({
|
|
|
11617
11645
|
const projectTouchedBuckets = projectEnabled ? new Set() : null;
|
|
11618
11646
|
const projectMetaCache = projectEnabled ? new Map() : null;
|
|
11619
11647
|
const publicRepoCache = projectEnabled ? new Map() : null;
|
|
11648
|
+
// Version 2 pairs every project bucket with the hourly model so the split in
|
|
11649
|
+
// enqueueTouchedBuckets can emit project rows (a model-less bucket is never
|
|
11650
|
+
// splittable). An install still carrying version-1 state rescan its WorkBuddy
|
|
11651
|
+
// history in the project pass alone — the totals cursors are untouched, so no
|
|
11652
|
+
// token is counted twice.
|
|
11653
|
+
const projectAttributionNeedsRescan =
|
|
11654
|
+
Number(workbuddyState.projectAttributionVersion || 0) < WORKBUDDY_PROJECT_ATTRIBUTION_VERSION;
|
|
11620
11655
|
const projectSeenIds = new Set(
|
|
11621
|
-
Array.isArray(workbuddyState.projectSeenIds)
|
|
11656
|
+
!projectAttributionNeedsRescan && Array.isArray(workbuddyState.projectSeenIds)
|
|
11657
|
+
? workbuddyState.projectSeenIds
|
|
11658
|
+
: [],
|
|
11622
11659
|
);
|
|
11623
11660
|
const projectFileOffsets =
|
|
11624
|
-
|
|
11661
|
+
!projectAttributionNeedsRescan &&
|
|
11662
|
+
workbuddyState.projectFileOffsets &&
|
|
11663
|
+
typeof workbuddyState.projectFileOffsets === "object"
|
|
11625
11664
|
? { ...workbuddyState.projectFileOffsets }
|
|
11626
11665
|
: {};
|
|
11666
|
+
if (projectState && projectAttributionNeedsRescan) {
|
|
11667
|
+
dropLegacyWorkbuddyProjectBuckets(projectState, variant);
|
|
11668
|
+
}
|
|
11627
11669
|
let projectBucketsQueued = 0;
|
|
11628
11670
|
let projectCursorUpdated = false;
|
|
11629
11671
|
|
|
@@ -11644,6 +11686,7 @@ async function parseWorkbuddyIncremental({
|
|
|
11644
11686
|
if (files.length === 0 && !dbExists) {
|
|
11645
11687
|
cursors[variant] = {
|
|
11646
11688
|
...workbuddyState,
|
|
11689
|
+
projectAttributionVersion: WORKBUDDY_PROJECT_ATTRIBUTION_VERSION,
|
|
11647
11690
|
seenIds: Array.from(seenIds),
|
|
11648
11691
|
seenTraceIds: Array.from(seenTraceIds),
|
|
11649
11692
|
tracedSessionIds: Array.from(tracedSessionIds),
|
|
@@ -11919,16 +11962,27 @@ async function parseWorkbuddyIncremental({
|
|
|
11919
11962
|
const bucketStart = toUtcHalfHourStart(new Date(tsMs).toISOString());
|
|
11920
11963
|
if (!bucketStart) return;
|
|
11921
11964
|
|
|
11965
|
+
// Same model expression as the totals pass, so the project bucket and
|
|
11966
|
+
// the hourly bucket it belongs to always share one model key. Without
|
|
11967
|
+
// this the split can never match the hour and WorkBuddy usage stays
|
|
11968
|
+
// unattributed forever.
|
|
11969
|
+
const projectModel =
|
|
11970
|
+
normalizeModelInput(provider.model) ||
|
|
11971
|
+
normalizeModelInput(provider.requestModelId) ||
|
|
11972
|
+
normalizeModelInput(entry.model) ||
|
|
11973
|
+
fallbackModel;
|
|
11922
11974
|
const projectBucket = getProjectBucket(
|
|
11923
11975
|
projectState,
|
|
11924
11976
|
projectKey,
|
|
11925
11977
|
source,
|
|
11926
|
-
|
|
11978
|
+
projectModel,
|
|
11927
11979
|
bucketStart,
|
|
11928
11980
|
projectRef,
|
|
11929
11981
|
);
|
|
11930
11982
|
addTotals(projectBucket.totals, delta);
|
|
11931
|
-
projectTouchedBuckets.add(
|
|
11983
|
+
projectTouchedBuckets.add(
|
|
11984
|
+
projectBucketKey(projectKey, source, projectModel, bucketStart),
|
|
11985
|
+
);
|
|
11932
11986
|
},
|
|
11933
11987
|
);
|
|
11934
11988
|
|
|
@@ -12236,6 +12290,7 @@ async function parseWorkbuddyIncremental({
|
|
|
12236
12290
|
}
|
|
12237
12291
|
cursors[variant] = {
|
|
12238
12292
|
...workbuddyState,
|
|
12293
|
+
projectAttributionVersion: WORKBUDDY_PROJECT_ATTRIBUTION_VERSION,
|
|
12239
12294
|
seenIds: cappedSeen,
|
|
12240
12295
|
seenTraceIds: cappedSeenTraceIds,
|
|
12241
12296
|
tracedSessionIds: cappedTracedSessionIds,
|
|
@@ -26309,4 +26364,8 @@ module.exports = {
|
|
|
26309
26364
|
commandCodeUsageToTotals,
|
|
26310
26365
|
extractCommandCodeSessionUsage,
|
|
26311
26366
|
parseCommandCodeIncremental,
|
|
26367
|
+
|
|
26368
|
+
// WorkBuddy project attribution (v2: project buckets carry the hourly model)
|
|
26369
|
+
WORKBUDDY_PROJECT_ATTRIBUTION_VERSION,
|
|
26370
|
+
dropLegacyWorkbuddyProjectBuckets,
|
|
26312
26371
|
};
|