tokentracker-cli 0.2.23

package/src/lib/uploader.js

@@ -0,0 +1,224 @@
+const fs = require("node:fs/promises");
+const fssync = require("node:fs");
+const readline = require("node:readline");
+
+const { ensureDir, readJson, writeJson } = require("./fs");
+const { ingestHourly } = require("./vibeusage-api");
+
+const DEFAULT_SOURCE = "codex";
+const DEFAULT_MODEL = "unknown";
+const BUCKET_SEPARATOR = "|";
+const MAX_INGEST_BUCKETS = 500;
+
+async function drainQueueToCloud({
+  baseUrl,
+  deviceToken,
+  deviceSubscriptions,
+  queuePath,
+  queueStatePath,
+  projectQueuePath,
+  projectQueueStatePath,
+  maxBatches,
+  batchSize,
+  onProgress,
+}) {
+  await ensureDir(require("node:path").dirname(queueStatePath));
+  const projectQueueEnabled = typeof projectQueuePath === "string" && projectQueuePath.length > 0;
+  if (projectQueueEnabled && (!projectQueueStatePath || projectQueueStatePath.length === 0)) {
+    throw new Error("projectQueueStatePath is required when projectQueuePath is set");
+  }
+  if (projectQueueEnabled) {
+    await ensureDir(require("node:path").dirname(projectQueueStatePath));
+  }
+
+  const state = (await readJson(queueStatePath)) || { offset: 0 };
+  let offset = Number(state.offset || 0);
+  const projectStatePath = projectQueueEnabled ? projectQueueStatePath : null;
+  const projectState = projectQueueEnabled
+    ? (await readJson(projectStatePath)) || { offset: 0 }
+    : { offset: 0 };
+  let projectOffset = Number(projectState.offset || 0);
+  let inserted = 0;
+  let skipped = 0;
+  let attempted = 0;
+
+  const cb = typeof onProgress === "function" ? onProgress : null;
+  const queueSize = await safeFileSize(queuePath);
+  const projectQueueSize = projectQueueEnabled ? await safeFileSize(projectQueuePath) : 0;
+  const maxBuckets = Math.max(1, Math.floor(Number(batchSize || 200)));
+  const totalLimit = Math.min(maxBuckets, MAX_INGEST_BUCKETS);
+
+  const normalizedSubscriptions = Array.isArray(deviceSubscriptions)
+    ? deviceSubscriptions.filter((entry) => entry && typeof entry === "object")
+    : [];
+
+  for (let batch = 0; batch < maxBatches; batch++) {
+    const hasHourly = queueSize > offset;
+    const hasProject = projectQueueEnabled && projectQueueSize > projectOffset;
+    let hourlyLimit = 0;
+    let projectLimit = 0;
+
+    if (hasHourly && hasProject) {
+      hourlyLimit = Math.ceil(totalLimit / 2);
+      projectLimit = totalLimit - hourlyLimit;
+    } else if (hasHourly) {
+      hourlyLimit = totalLimit;
+    } else if (hasProject) {
+      projectLimit = totalLimit;
+    }
+
+    const res =
+      hourlyLimit > 0
+        ? await readBatch(queuePath, offset, hourlyLimit)
+        : { buckets: [], nextOffset: offset };
+    const projectRes =
+      projectLimit > 0
+        ? await readProjectBatch(projectQueuePath, projectOffset, projectLimit)
+        : { buckets: [], nextOffset: projectOffset };
+    if (res.buckets.length === 0 && projectRes.buckets.length === 0) break;
+
+    attempted += res.buckets.length + projectRes.buckets.length;
+    const ingest = await ingestHourly({
+      baseUrl,
+      deviceToken,
+      hourly: res.buckets,
+      project_hourly: projectRes.buckets,
+      device_subscriptions: batch === 0 ? normalizedSubscriptions : undefined,
+    });
+    inserted += ingest.inserted || 0;
+    skipped += ingest.skipped || 0;
+
+    offset = res.nextOffset;
+    state.offset = offset;
+    state.updatedAt = new Date().toISOString();
+    await writeJson(queueStatePath, state);
+    if (projectQueuePath) {
+      projectOffset = projectRes.nextOffset;
+      projectState.offset = projectOffset;
+      projectState.updatedAt = new Date().toISOString();
+      await writeJson(projectStatePath, projectState);
+    }
+
+    if (cb) {
+      const combinedOffset = offset + projectOffset;
+      cb({
+        batch: batch + 1,
+        maxBatches,
+        offset: combinedOffset,
+        queueSize: queueSize + projectQueueSize,
+        inserted,
+        skipped,
+      });
+    }
+  }
+
+  return { inserted, skipped, attempted };
+}
+
+async function readBatch(queuePath, startOffset, maxBuckets) {
+  const st = await fs.stat(queuePath).catch(() => null);
+  if (!st || !st.isFile()) return { buckets: [], nextOffset: startOffset };
+  if (startOffset >= st.size) return { buckets: [], nextOffset: startOffset };
+
+  const stream = fssync.createReadStream(queuePath, { encoding: "utf8", start: startOffset });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  const bucketMap = new Map();
+  let offset = startOffset;
+  let linesRead = 0;
+  for await (const line of rl) {
+    const bytes = Buffer.byteLength(line, "utf8") + 1;
+    offset += bytes;
+    if (!line.trim()) continue;
+    let bucket;
+    try {
+      bucket = JSON.parse(line);
+    } catch (_e) {
+      continue;
+    }
+    const hourStart = typeof bucket?.hour_start === "string" ? bucket.hour_start : null;
+    if (!hourStart) continue;
+    const source = normalizeSource(bucket?.source) || DEFAULT_SOURCE;
+    const model = normalizeModel(bucket?.model) || DEFAULT_MODEL;
+    bucket.source = source;
+    bucket.model = model;
+    bucketMap.set(bucketKey(source, model, hourStart), bucket);
+    linesRead += 1;
+    if (linesRead >= maxBuckets) break;
+  }
+
+  rl.close();
+  stream.close?.();
+  return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
+}
+
+async function readProjectBatch(queuePath, startOffset, maxBuckets) {
+  if (!queuePath) return { buckets: [], nextOffset: startOffset };
+  const st = await fs.stat(queuePath).catch(() => null);
+  if (!st || !st.isFile()) return { buckets: [], nextOffset: startOffset };
+  if (startOffset >= st.size) return { buckets: [], nextOffset: startOffset };
+
+  const stream = fssync.createReadStream(queuePath, { encoding: "utf8", start: startOffset });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  const bucketMap = new Map();
+  let offset = startOffset;
+  let linesRead = 0;
+  for await (const line of rl) {
+    const bytes = Buffer.byteLength(line, "utf8") + 1;
+    offset += bytes;
+    if (!line.trim()) continue;
+    let bucket;
+    try {
+      bucket = JSON.parse(line);
+    } catch (_e) {
+      continue;
+    }
+    const hourStart = typeof bucket?.hour_start === "string" ? bucket.hour_start : null;
+    const projectKey = typeof bucket?.project_key === "string" ? bucket.project_key : null;
+    const projectRef = typeof bucket?.project_ref === "string" ? bucket.project_ref : null;
+    if (!hourStart || !projectKey || !projectRef) continue;
+    const source = normalizeSource(bucket?.source) || DEFAULT_SOURCE;
+    bucket.source = source;
+    bucket.project_key = projectKey;
+    bucket.project_ref = projectRef;
+    bucketMap.set(projectBucketKey(projectKey, source, hourStart), bucket);
+    linesRead += 1;
+    if (linesRead >= maxBuckets) break;
+  }
+
+  rl.close();
+  stream.close?.();
+  return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
+}
+
+async function safeFileSize(p) {
+  try {
+    const st = await fs.stat(p);
+    return st && st.isFile() ? st.size : 0;
+  } catch (_e) {
+    return 0;
+  }
+}
+
+function bucketKey(source, model, hourStart) {
+  return `${source}${BUCKET_SEPARATOR}${model}${BUCKET_SEPARATOR}${hourStart}`;
+}
+
+function projectBucketKey(projectKey, source, hourStart) {
+  return `${projectKey}${BUCKET_SEPARATOR}${source}${BUCKET_SEPARATOR}${hourStart}`;
+}
+
+function normalizeSource(value) {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim().toLowerCase();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function normalizeModel(value) {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+module.exports = { drainQueueToCloud };

package/src/lib/vibeusage-api.js

@@ -0,0 +1,243 @@
+"use strict";
+
+const { createInsforgeClient } = require("./insforge-client");
+
+async function signInWithPassword({ baseUrl, email, password }) {
+  const client = createInsforgeClient({ baseUrl });
+  const { data, error } = await client.auth.signInWithPassword({ email, password });
+  if (error) throw normalizeSdkError(error, "Sign-in failed");
+
+  const accessToken = data?.accessToken;
+  if (typeof accessToken !== "string" || accessToken.length === 0) {
+    throw new Error("Sign-in failed: missing accessToken");
+  }
+
+  return accessToken;
+}
+
+async function issueDeviceToken({ baseUrl, accessToken, deviceName, platform = "macos" }) {
+  const data = await invokeFunction({
+    baseUrl,
+    accessToken,
+    slug: "tokentracker-device-token-issue",
+    method: "POST",
+    body: { device_name: deviceName, platform },
+    errorPrefix: "Device token issue failed",
+  });
+
+  const token = data?.token;
+  const deviceId = data?.device_id;
+  if (typeof token !== "string" || token.length === 0)
+    throw new Error("Device token issue failed: missing token");
+  if (typeof deviceId !== "string" || deviceId.length === 0) {
+    throw new Error("Device token issue failed: missing device_id");
+  }
+
+  return { token, deviceId };
+}
+
+async function exchangeLinkCode({ baseUrl, linkCode, requestId, deviceName, platform = "macos" }) {
+  const data = await invokeFunction({
+    baseUrl,
+    accessToken: null,
+    slug: "vibeusage-link-code-exchange",
+    method: "POST",
+    body: {
+      link_code: linkCode,
+      request_id: requestId,
+      device_name: deviceName,
+      platform,
+    },
+    errorPrefix: "Link code exchange failed",
+  });
+
+  const token = data?.token;
+  const deviceId = data?.device_id;
+  if (typeof token !== "string" || token.length === 0) {
+    throw new Error("Link code exchange failed: missing token");
+  }
+  if (typeof deviceId !== "string" || deviceId.length === 0) {
+    throw new Error("Link code exchange failed: missing device_id");
+  }
+
+  return { token, deviceId, userId: data?.user_id || null };
+}
+
+async function ingestHourly({
+  baseUrl,
+  deviceToken,
+  hourly,
+  project_hourly,
+  device_subscriptions,
+}) {
+  const body = { hourly, project_hourly };
+  if (Array.isArray(device_subscriptions) && device_subscriptions.length > 0) {
+    body.device_subscriptions = device_subscriptions;
+  }
+
+  const data = await invokeFunctionWithRetry({
+    baseUrl,
+    accessToken: deviceToken,
+    slug: "vibeusage-ingest",
+    method: "POST",
+    body,
+    errorPrefix: "Ingest failed",
+    retry: { maxRetries: 3, baseDelayMs: 500, maxDelayMs: 5000 },
+  });
+
+  return {
+    inserted: Number(data?.inserted || 0),
+    skipped: Number(data?.skipped || 0),
+  };
+}
+
+async function syncHeartbeat({ baseUrl, deviceToken }) {
+  const data = await invokeFunction({
+    baseUrl,
+    accessToken: deviceToken,
+    slug: "vibeusage-sync-ping",
+    method: "POST",
+    body: {},
+    errorPrefix: "Sync heartbeat failed",
+  });
+
+  return {
+    updated: Boolean(data?.updated),
+    last_sync_at: typeof data?.last_sync_at === "string" ? data.last_sync_at : null,
+    min_interval_minutes: Number(data?.min_interval_minutes || 0),
+  };
+}
+
+module.exports = {
+  signInWithPassword,
+  issueDeviceToken,
+  exchangeLinkCode,
+  ingestHourly,
+  syncHeartbeat,
+};
+
+async function invokeFunction({ baseUrl, accessToken, slug, method, body, errorPrefix }) {
+  const client = createInsforgeClient({ baseUrl, accessToken });
+  const { data, error } = await client.functions.invoke(slug, { method, body });
+  if (error) throw normalizeSdkError(error, errorPrefix);
+  return data;
+}
+
+async function invokeFunctionWithRetry({
+  baseUrl,
+  accessToken,
+  slug,
+  method,
+  body,
+  errorPrefix,
+  retry,
+}) {
+  const retryOptions = normalizeRetryOptions(retry);
+  let attempt = 0;
+
+  while (true) {
+    try {
+      return await invokeFunction({ baseUrl, accessToken, slug, method, body, errorPrefix });
+    } catch (e) {
+      if (!shouldRetry({ err: e, attempt, retryOptions })) throw e;
+      const delayMs = computeRetryDelayMs({ retryOptions, attempt, err: e });
+      await sleep(delayMs);
+      attempt += 1;
+    }
+  }
+}
+
+function normalizeSdkError(error, errorPrefix) {
+  const raw = extractSdkErrorMessage(error);
+  const msg = normalizeBackendErrorMessage(raw);
+  const err = new Error(errorPrefix ? `${errorPrefix}: ${msg}` : msg);
+  const status = error?.statusCode ?? error?.status;
+  const code = typeof error?.error === "string" ? error.error.trim() : "";
+  if (typeof status === "number") err.status = status;
+  if (code) err.code = code;
+  err.retryable = isRetryableStatus(status) || isRetryableMessage(raw);
+  if (msg !== raw) err.originalMessage = raw;
+  if (error?.nextActions) err.nextActions = error.nextActions;
+  return err;
+}
+
+function extractSdkErrorMessage(error) {
+  if (!error) return "Unknown error";
+  const message = typeof error.message === "string" ? error.message.trim() : "";
+  const code = typeof error.error === "string" ? error.error.trim() : "";
+  if (message && message !== "InsForgeError") return message;
+  if (code && code !== "REQUEST_FAILED") return code;
+  if (message) return message;
+  if (code) return code;
+  return String(error);
+}
+
+function normalizeBackendErrorMessage(message) {
+  if (!isBackendRuntimeDownMessage(message)) return String(message || "Unknown error");
+  return "Backend runtime unavailable (InsForge). Please retry later.";
+}
+
+function isBackendRuntimeDownMessage(message) {
+  const s = String(message || "").toLowerCase();
+  if (!s) return false;
+  if (s.includes("deno:") || s.includes("deno")) return true;
+  if (s.includes("econnreset") || s.includes("econnrefused")) return true;
+  if (s.includes("etimedout")) return true;
+  if (s.includes("timeout") && s.includes("request")) return true;
+  if (s.includes("upstream") && (s.includes("deno") || s.includes("connect"))) return true;
+  return false;
+}
+
+function isRetryableStatus(status) {
+  return status === 429 || status === 502 || status === 503 || status === 504;
+}
+
+function isRetryableMessage(message) {
+  const s = String(message || "").toLowerCase();
+  if (!s) return false;
+  if (isBackendRuntimeDownMessage(s)) return true;
+  if (s.includes("econnreset") || s.includes("econnrefused")) return true;
+  if (s.includes("etimedout") || s.includes("timeout")) return true;
+  if (s.includes("networkerror") || s.includes("failed to fetch")) return true;
+  if (s.includes("socket hang up") || s.includes("connection reset")) return true;
+  return false;
+}
+
+function normalizeRetryOptions(retry) {
+  if (!retry) {
+    return { maxRetries: 0, baseDelayMs: 0, maxDelayMs: 0, jitterRatio: 0.0 };
+  }
+  const maxRetries = clampInt(retry.maxRetries, 0, 10);
+  const baseDelayMs = clampInt(retry.baseDelayMs ?? 300, 50, 60_000);
+  const maxDelayMs = clampInt(retry.maxDelayMs ?? baseDelayMs * 4, baseDelayMs, 120_000);
+  const jitterRatio =
+    typeof retry.jitterRatio === "number" ? Math.max(0, Math.min(0.5, retry.jitterRatio)) : 0.2;
+  return { maxRetries, baseDelayMs, maxDelayMs, jitterRatio };
+}
+
+function shouldRetry({ err, attempt, retryOptions }) {
+  if (!retryOptions || retryOptions.maxRetries <= 0) return false;
+  if (attempt >= retryOptions.maxRetries) return false;
+  return Boolean(err && err.retryable);
+}
+
+function computeRetryDelayMs({ retryOptions, attempt, err }) {
+  if (!retryOptions || retryOptions.maxRetries <= 0) return 0;
+  const exp = retryOptions.baseDelayMs * Math.pow(2, attempt);
+  const capped = Math.min(retryOptions.maxDelayMs, exp);
+  const jitter = capped * retryOptions.jitterRatio * Math.random();
+  const backoff = Math.round(capped + jitter);
+  const retryAfter = typeof err?.retryAfterMs === "number" ? err.retryAfterMs : 0;
+  return Math.max(backoff, retryAfter || 0);
+}
+
+function clampInt(value, min, max) {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return min;
+  return Math.min(max, Math.max(min, Math.floor(n)));
+}
+
+function sleep(ms) {
+  if (!ms || ms <= 0) return Promise.resolve();
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/lib/vibeusage-public-repo.js

@@ -0,0 +1,105 @@
+const crypto = require("node:crypto");
+
+const GITHUB_HOST = "github.com";
+const GITHUB_API_BASE = "https://api.github.com";
+
+function parseGitHubRepoId(projectRef) {
+  if (typeof projectRef !== "string") return null;
+  let parsed;
+  try {
+    parsed = new URL(projectRef);
+  } catch (_err) {
+    return null;
+  }
+  if (!parsed.hostname || parsed.hostname.toLowerCase() !== GITHUB_HOST) return null;
+  const segments = parsed.pathname.split("/").filter(Boolean);
+  if (segments.length < 2) return null;
+  const owner = segments[0].trim().toLowerCase();
+  const repo = segments[1]
+    .trim()
+    .replace(/\.git$/i, "")
+    .toLowerCase();
+  if (!owner || !repo) return null;
+  return { owner, repo, repoId: `${owner}/${repo}` };
+}
+
+function hashRepoRoot(repoRoot) {
+  return crypto.createHash("sha256").update(String(repoRoot)).digest("hex");
+}
+
+function isRateLimited(res, body) {
+  if (!res) return false;
+  const remaining = res.headers?.get?.("x-ratelimit-remaining");
+  if (remaining === "0") return true;
+  if (res.status === 429) return true;
+  if (
+    body &&
+    typeof body.message === "string" &&
+    body.message.toLowerCase().includes("rate limit")
+  ) {
+    return true;
+  }
+  return false;
+}
+
+async function resolveGitHubPublicStatus(projectRef, fetchImpl) {
+  const parsed = parseGitHubRepoId(projectRef);
+  if (!parsed) {
+    return { status: "blocked", projectKey: null, projectRef, reason: "non_github" };
+  }
+  const fetchFn = fetchImpl || fetch;
+  const apiUrl = `${GITHUB_API_BASE}/repos/${parsed.repoId}`;
+
+  let res;
+  let body = null;
+  try {
+    res = await fetchFn(apiUrl, {
+      headers: {
+        Accept: "application/vnd.github+json",
+        "User-Agent": "vibeusage-cli",
+      },
+    });
+  } catch (_err) {
+    return {
+      status: "pending_public",
+      projectKey: parsed.repoId,
+      projectRef,
+      reason: "fetch_failed",
+    };
+  }
+
+  if (res.status === 200) {
+    body = await res.json().catch(() => ({}));
+    const isPrivate = body && typeof body.private === "boolean" ? body.private : null;
+    const visibility = typeof body?.visibility === "string" ? body.visibility : null;
+    const isPublic = isPrivate === false || visibility === "public";
+    return {
+      status: isPublic ? "public_verified" : "blocked",
+      projectKey: parsed.repoId,
+      projectRef,
+      reason: isPublic ? "public" : "private",
+    };
+  }
+
+  if (res.status === 404) {
+    return { status: "blocked", projectKey: parsed.repoId, projectRef, reason: "not_found" };
+  }
+
+  body = await res.json().catch(() => ({}));
+  if (isRateLimited(res, body) || res.status === 401 || res.status === 403 || res.status >= 500) {
+    return {
+      status: "pending_public",
+      projectKey: parsed.repoId,
+      projectRef,
+      reason: "rate_limited",
+    };
+  }
+
+  return { status: "pending_public", projectKey: parsed.repoId, projectRef, reason: "unknown" };
+}
+
+module.exports = {
+  parseGitHubRepoId,
+  hashRepoRoot,
+  resolveGitHubPublicStatus,
+};