tokenusage-cli 0.4.7 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,259 @@
1
+ "use strict";
2
+
3
+ // Detected repository URLs for projects (docs/cli-contract.md §2.4 `projects`,
4
+ // contract v3, 2026-11-06). The CLI only ever reports public GitHub / Gitee
5
+ // remotes it already canonicalised, and only the entries that differ from what
6
+ // this device last had confirmed — the server stores one row per
7
+ // (user, device, project_key), so sending the full map on every sync would
8
+ // rewrite rows for unchanged projects.
9
+ //
10
+ // Wire rules mirrored from the contract (the server validates and drops
11
+ // non-conforming entries instead of failing the batch):
12
+ // • https://github.com/<owner>/<repo> or https://gitee.com/<owner>/<repo>
13
+ // • exactly two path segments of [A-Za-z0-9._-], no credentials/port/query
14
+ // • `null` clears this device's previously reported value for the project
15
+
16
+ const fs = require("node:fs");
17
+ const fsp = require("node:fs/promises");
18
+ const path = require("node:path");
19
+ const crypto = require("node:crypto");
20
+
21
+ const { functionUrlFor, fetchFunctionResponse } = require("./function-url");
22
+
23
+ const INGEST_SLUG = "token-usage-ingest";
24
+ const STATE_VERSION = 1;
25
+ const MAX_PROJECTS_PER_BATCH = 200;
26
+ const REPO_HOSTS = new Set(["github.com", "gitee.com"]);
27
+ const PATH_SEGMENT = /^[A-Za-z0-9._-]{1,100}$/;
28
+
29
+ /**
30
+ * Canonicalise a git remote into the contract's only accepted shape.
31
+ * Returns null for anything that is not a public GitHub / Gitee repository
32
+ * (self-hosted GitLab, enterprise hosts, local paths, credentials, ports).
33
+ */
34
+ function normalizeRepoUrl(value) {
35
+ if (typeof value !== "string") return null;
36
+ let raw = value.trim();
37
+ if (!raw) return null;
38
+
39
+ const scp = raw.match(/^[A-Za-z0-9._-]+@([^:/\\]+):(.+)$/);
40
+ if (scp) {
41
+ raw = `https://${scp[1]}/${scp[2]}`;
42
+ } else if (/^ssh:\/\//i.test(raw)) {
43
+ try {
44
+ const parsed = new URL(raw);
45
+ raw = `https://${parsed.hostname}${parsed.pathname}`;
46
+ } catch {
47
+ return null;
48
+ }
49
+ } else if (/^git:\/\//i.test(raw)) {
50
+ raw = `https://${raw.slice("git://".length)}`;
51
+ }
52
+
53
+ let parsed;
54
+ try {
55
+ parsed = new URL(raw);
56
+ } catch {
57
+ return null;
58
+ }
59
+ if (parsed.protocol !== "https:" && parsed.protocol !== "http:") return null;
60
+ if (parsed.username || parsed.password || parsed.port) return null;
61
+ if (parsed.search || parsed.hash) return null;
62
+
63
+ const host = parsed.hostname.toLowerCase();
64
+ if (!REPO_HOSTS.has(host)) return null;
65
+
66
+ const segments = parsed.pathname.split("/").filter((segment) => segment.length > 0);
67
+ if (segments.length !== 2) return null;
68
+
69
+ const owner = segments[0];
70
+ let repo = segments[1];
71
+ if (/\.git$/i.test(repo)) repo = repo.slice(0, -4);
72
+ if (!PATH_SEGMENT.test(owner) || !PATH_SEGMENT.test(repo)) return null;
73
+ if (owner === "." || owner === ".." || repo === "." || repo === "..") return null;
74
+
75
+ const canonical = `https://${host}/${owner}/${repo}`;
76
+ return canonical.length <= 200 ? canonical : null;
77
+ }
78
+
79
+ function emptyState() {
80
+ return { version: STATE_VERSION, confirmed: {} };
81
+ }
82
+
83
+ function loadProjectRepoState(filePath) {
84
+ try {
85
+ const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
86
+ if (!parsed || typeof parsed !== "object") return emptyState();
87
+ const confirmed = {};
88
+ for (const [key, value] of Object.entries(
89
+ parsed.confirmed && typeof parsed.confirmed === "object" ? parsed.confirmed : {},
90
+ )) {
91
+ if (typeof key !== "string" || !key) continue;
92
+ if (value === null || typeof value === "string") confirmed[key] = value;
93
+ }
94
+ return { version: STATE_VERSION, confirmed };
95
+ } catch {
96
+ return emptyState();
97
+ }
98
+ }
99
+
100
+ async function saveProjectRepoState(filePath, state) {
101
+ const payload = {
102
+ version: STATE_VERSION,
103
+ confirmed: Object.fromEntries(
104
+ Object.entries(state.confirmed).sort(([a], [b]) => a.localeCompare(b)),
105
+ ),
106
+ };
107
+ await fsp.mkdir(path.dirname(filePath), { recursive: true });
108
+ const tmp = `${filePath}.${crypto.randomBytes(4).toString("hex")}.tmp`;
109
+ await fsp.writeFile(tmp, JSON.stringify(payload, null, 2) + "\n", { mode: 0o600 });
110
+ await fsp.rename(tmp, filePath);
111
+ await fsp.chmod(filePath, 0o600).catch(() => {});
112
+ }
113
+
114
+ /**
115
+ * Entries this device should (re)report: new detections, moved repositories,
116
+ * and clears (a project whose remote disappeared or moved off GitHub/Gitee).
117
+ * Projects that were never confirmed and still have no repository are skipped —
118
+ * sending `null` for every local-only project would only burn requests.
119
+ */
120
+ function collectProjectRepoChanges({ projects, confirmed = {} }) {
121
+ const entries = [];
122
+ for (const [key, meta] of Object.entries(
123
+ projects && typeof projects === "object" ? projects : {},
124
+ )) {
125
+ if (typeof key !== "string" || !key || !meta || typeof meta !== "object") continue;
126
+ const hasConfirmed = Object.prototype.hasOwnProperty.call(confirmed, key);
127
+ const previous = hasConfirmed ? confirmed[key] : undefined;
128
+ const status = typeof meta.status === "string" ? meta.status : null;
129
+ const detected = status === "public_verified" || status == null
130
+ ? normalizeRepoUrl(meta.project_ref)
131
+ : null;
132
+
133
+ if (detected) {
134
+ if (detected !== previous) entries.push({ project_key: key, repo_url: detected });
135
+ continue;
136
+ }
137
+ if (hasConfirmed && previous !== null) {
138
+ entries.push({ project_key: key, repo_url: null });
139
+ }
140
+ }
141
+ return entries;
142
+ }
143
+
144
+ /** New confirmed map after the server acknowledged these entries. */
145
+ function applyProjectRepoConfirmation(confirmed, sent) {
146
+ const next = { ...confirmed };
147
+ for (const entry of sent) next[entry.project_key] = entry.repo_url;
148
+ return next;
149
+ }
150
+
151
+ async function postProjects({
152
+ baseUrl,
153
+ anonKey,
154
+ deviceToken,
155
+ entries,
156
+ fetchImpl,
157
+ signal,
158
+ }) {
159
+ const headers = {
160
+ "Content-Type": "application/json",
161
+ Accept: "application/json",
162
+ Authorization: `Bearer ${deviceToken}`,
163
+ };
164
+ if (anonKey) headers.apikey = anonKey;
165
+
166
+ return fetchFunctionResponse(
167
+ functionUrlFor(baseUrl, INGEST_SLUG),
168
+ {
169
+ method: "POST",
170
+ headers,
171
+ ...(signal ? { signal } : {}),
172
+ body: JSON.stringify({ projects: entries }),
173
+ },
174
+ fetchImpl,
175
+ );
176
+ }
177
+
178
+ /**
179
+ * Upload pending repository changes in batches of <=200. Returns
180
+ * `unsupported: true` (and leaves the state untouched) when the backend
181
+ * predates contract v3 — an old server answers a projects-only request with
182
+ * `400 No usage buckets provided`, and a retired route with 404/405. Both are
183
+ * "retry later after upgrading the backend", never a sync failure.
184
+ */
185
+ async function uploadProjectRepos({
186
+ baseUrl,
187
+ anonKey,
188
+ deviceToken,
189
+ projects,
190
+ statePath,
191
+ canUpload = () => true,
192
+ batchSize = MAX_PROJECTS_PER_BATCH,
193
+ fetchImpl,
194
+ signal,
195
+ }) {
196
+ const state = loadProjectRepoState(statePath);
197
+ const pending = collectProjectRepoChanges({
198
+ projects,
199
+ confirmed: state.confirmed,
200
+ });
201
+ if (pending.length === 0) return { pending: 0, uploaded: 0, skipped: 0, batches: 0 };
202
+ if (!canUpload()) return { pending: pending.length, uploaded: 0, skipped: 0, batches: 0, paused: true };
203
+
204
+ const limit = Math.min(Math.max(1, Math.trunc(Number(batchSize) || 1)), MAX_PROJECTS_PER_BATCH);
205
+ let uploaded = 0;
206
+ let skipped = 0;
207
+ let batches = 0;
208
+
209
+ for (let index = 0; index < pending.length; index += limit) {
210
+ const batch = pending.slice(index, index + limit);
211
+ if (!canUpload()) break;
212
+ const res = await postProjects({ baseUrl, anonKey, deviceToken, entries: batch, fetchImpl, signal });
213
+
214
+ if (!res.ok) {
215
+ const text = await res.text().catch(() => "");
216
+ if (
217
+ res.status === 404 ||
218
+ res.status === 405 ||
219
+ (res.status === 400 && text.includes("No usage buckets provided"))
220
+ ) {
221
+ return { pending: pending.length, uploaded, skipped, batches, unsupported: true };
222
+ }
223
+ const error = new Error(`repo upload failed (HTTP ${res.status}): ${text.slice(0, 300)}`);
224
+ error.status = res.status;
225
+ throw error;
226
+ }
227
+
228
+ const data = await res.json().catch(() => null);
229
+ const isCount = (value) =>
230
+ (typeof value === "number" || typeof value === "string") &&
231
+ value !== "" &&
232
+ Number.isFinite(Number(value));
233
+ if (!data || typeof data !== "object" || !isCount(data.projects_upserted)) {
234
+ const error = new Error(
235
+ `repo upload was not confirmed (HTTP ${res.status}): unexpected response body`,
236
+ );
237
+ error.status = res.status;
238
+ throw error;
239
+ }
240
+
241
+ state.confirmed = applyProjectRepoConfirmation(state.confirmed, batch);
242
+ await saveProjectRepoState(statePath, state);
243
+ uploaded += batch.length;
244
+ skipped += Number(data.projects_skipped || 0);
245
+ batches += 1;
246
+ }
247
+
248
+ return { pending: pending.length, uploaded, skipped, batches };
249
+ }
250
+
251
+ module.exports = {
252
+ MAX_PROJECTS_PER_BATCH,
253
+ applyProjectRepoConfirmation,
254
+ collectProjectRepoChanges,
255
+ loadProjectRepoState,
256
+ normalizeRepoUrl,
257
+ saveProjectRepoState,
258
+ uploadProjectRepos,
259
+ };
@@ -11314,6 +11314,34 @@ const WORKBUDDY_VARIANTS = {
11314
11314
  "workbuddy-ai": { dir: ".workbuddy-ai", envVar: "WORKBUDDY_AI_HOME" },
11315
11315
  };
11316
11316
 
11317
+ // Bump when the project pass changes how it derives a bucket's model. Version 2
11318
+ // pairs project usage with the hourly model, which is what enqueueTouchedBuckets
11319
+ // needs to emit project + residual rows; version-1 buckets have no model and can
11320
+ // never be split. An install below the current version resets only WorkBuddy's
11321
+ // project cursors and drops those model-less buckets — the totals cursors stay
11322
+ // untouched, so history is re-attributed without re-counting a single token.
11323
+ const WORKBUDDY_PROJECT_ATTRIBUTION_VERSION = 2;
11324
+
11325
+ // Remove project buckets written before the model dimension existed. Only the
11326
+ // buckets owned by the given WorkBuddy variant are touched; other sources and
11327
+ // every model-bearing bucket are left exactly as they are.
11328
+ function dropLegacyWorkbuddyProjectBuckets(projectState, variant) {
11329
+ if (!projectState || !projectState.buckets || typeof projectState.buckets !== "object") {
11330
+ return 0;
11331
+ }
11332
+ const source = normalizeSourceInput(variant) || DEFAULT_SOURCE;
11333
+ let removed = 0;
11334
+ for (const [key, bucket] of Object.entries(projectState.buckets)) {
11335
+ if (!bucket || typeof bucket !== "object") continue;
11336
+ // A legacy bucket is exactly the one getProjectBucket wrote with model == null.
11337
+ if (bucket.model != null) continue;
11338
+ if ((normalizeSourceInput(bucket.source) || DEFAULT_SOURCE) !== source) continue;
11339
+ delete projectState.buckets[key];
11340
+ removed += 1;
11341
+ }
11342
+ return removed;
11343
+ }
11344
+
11317
11345
  function resolveWorkbuddyVariant(variant) {
11318
11346
  return Object.prototype.hasOwnProperty.call(WORKBUDDY_VARIANTS, variant)
11319
11347
  ? variant
@@ -11617,13 +11645,27 @@ async function parseWorkbuddyIncremental({
11617
11645
  const projectTouchedBuckets = projectEnabled ? new Set() : null;
11618
11646
  const projectMetaCache = projectEnabled ? new Map() : null;
11619
11647
  const publicRepoCache = projectEnabled ? new Map() : null;
11648
+ // Version 2 pairs every project bucket with the hourly model so the split in
11649
+ // enqueueTouchedBuckets can emit project rows (a model-less bucket is never
11650
+ // splittable). An install still carrying version-1 state rescan its WorkBuddy
11651
+ // history in the project pass alone — the totals cursors are untouched, so no
11652
+ // token is counted twice.
11653
+ const projectAttributionNeedsRescan =
11654
+ Number(workbuddyState.projectAttributionVersion || 0) < WORKBUDDY_PROJECT_ATTRIBUTION_VERSION;
11620
11655
  const projectSeenIds = new Set(
11621
- Array.isArray(workbuddyState.projectSeenIds) ? workbuddyState.projectSeenIds : [],
11656
+ !projectAttributionNeedsRescan && Array.isArray(workbuddyState.projectSeenIds)
11657
+ ? workbuddyState.projectSeenIds
11658
+ : [],
11622
11659
  );
11623
11660
  const projectFileOffsets =
11624
- workbuddyState.projectFileOffsets && typeof workbuddyState.projectFileOffsets === "object"
11661
+ !projectAttributionNeedsRescan &&
11662
+ workbuddyState.projectFileOffsets &&
11663
+ typeof workbuddyState.projectFileOffsets === "object"
11625
11664
  ? { ...workbuddyState.projectFileOffsets }
11626
11665
  : {};
11666
+ if (projectState && projectAttributionNeedsRescan) {
11667
+ dropLegacyWorkbuddyProjectBuckets(projectState, variant);
11668
+ }
11627
11669
  let projectBucketsQueued = 0;
11628
11670
  let projectCursorUpdated = false;
11629
11671
 
@@ -11644,6 +11686,7 @@ async function parseWorkbuddyIncremental({
11644
11686
  if (files.length === 0 && !dbExists) {
11645
11687
  cursors[variant] = {
11646
11688
  ...workbuddyState,
11689
+ projectAttributionVersion: WORKBUDDY_PROJECT_ATTRIBUTION_VERSION,
11647
11690
  seenIds: Array.from(seenIds),
11648
11691
  seenTraceIds: Array.from(seenTraceIds),
11649
11692
  tracedSessionIds: Array.from(tracedSessionIds),
@@ -11919,16 +11962,27 @@ async function parseWorkbuddyIncremental({
11919
11962
  const bucketStart = toUtcHalfHourStart(new Date(tsMs).toISOString());
11920
11963
  if (!bucketStart) return;
11921
11964
 
11965
+ // Same model expression as the totals pass, so the project bucket and
11966
+ // the hourly bucket it belongs to always share one model key. Without
11967
+ // this the split can never match the hour and WorkBuddy usage stays
11968
+ // unattributed forever.
11969
+ const projectModel =
11970
+ normalizeModelInput(provider.model) ||
11971
+ normalizeModelInput(provider.requestModelId) ||
11972
+ normalizeModelInput(entry.model) ||
11973
+ fallbackModel;
11922
11974
  const projectBucket = getProjectBucket(
11923
11975
  projectState,
11924
11976
  projectKey,
11925
11977
  source,
11926
- null,
11978
+ projectModel,
11927
11979
  bucketStart,
11928
11980
  projectRef,
11929
11981
  );
11930
11982
  addTotals(projectBucket.totals, delta);
11931
- projectTouchedBuckets.add(projectBucketKey(projectKey, source, null, bucketStart));
11983
+ projectTouchedBuckets.add(
11984
+ projectBucketKey(projectKey, source, projectModel, bucketStart),
11985
+ );
11932
11986
  },
11933
11987
  );
11934
11988
 
@@ -12236,6 +12290,7 @@ async function parseWorkbuddyIncremental({
12236
12290
  }
12237
12291
  cursors[variant] = {
12238
12292
  ...workbuddyState,
12293
+ projectAttributionVersion: WORKBUDDY_PROJECT_ATTRIBUTION_VERSION,
12239
12294
  seenIds: cappedSeen,
12240
12295
  seenTraceIds: cappedSeenTraceIds,
12241
12296
  tracedSessionIds: cappedTracedSessionIds,
@@ -26309,4 +26364,8 @@ module.exports = {
26309
26364
  commandCodeUsageToTotals,
26310
26365
  extractCommandCodeSessionUsage,
26311
26366
  parseCommandCodeIncremental,
26367
+
26368
+ // WorkBuddy project attribution (v2: project buckets carry the hourly model)
26369
+ WORKBUDDY_PROJECT_ATTRIBUTION_VERSION,
26370
+ dropLegacyWorkbuddyProjectBuckets,
26312
26371
  };