vibeusage 0.2.15 → 0.2.16

package/src/lib/uploader.js

@@ -8,26 +8,73 @@ const { ingestHourly } = require('./vibeusage-api');
 const DEFAULT_SOURCE = 'codex';
 const DEFAULT_MODEL = 'unknown';
 const BUCKET_SEPARATOR = '|';
-
-async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePath, maxBatches, batchSize, onProgress }) {
+const MAX_INGEST_BUCKETS = 500;
+
+async function drainQueueToCloud({
+  baseUrl,
+  deviceToken,
+  queuePath,
+  queueStatePath,
+  projectQueuePath,
+  projectQueueStatePath,
+  maxBatches,
+  batchSize,
+  onProgress
+}) {
   await ensureDir(require('node:path').dirname(queueStatePath));
+  const projectQueueEnabled = typeof projectQueuePath === 'string' && projectQueuePath.length > 0;
+  if (projectQueueEnabled && (!projectQueueStatePath || projectQueueStatePath.length === 0)) {
+    throw new Error('projectQueueStatePath is required when projectQueuePath is set');
+  }
+  if (projectQueueEnabled) {
+    await ensureDir(require('node:path').dirname(projectQueueStatePath));
+  }
 
   const state = (await readJson(queueStatePath)) || { offset: 0 };
   let offset = Number(state.offset || 0);
+  const projectStatePath = projectQueueEnabled ? projectQueueStatePath : null;
+  const projectState = projectQueueEnabled ? (await readJson(projectStatePath)) || { offset: 0 } : { offset: 0 };
+  let projectOffset = Number(projectState.offset || 0);
   let inserted = 0;
   let skipped = 0;
   let attempted = 0;
 
   const cb = typeof onProgress === 'function' ? onProgress : null;
   const queueSize = await safeFileSize(queuePath);
+  const projectQueueSize = projectQueueEnabled ? await safeFileSize(projectQueuePath) : 0;
   const maxBuckets = Math.max(1, Math.floor(Number(batchSize || 200)));
+  const totalLimit = Math.min(maxBuckets, MAX_INGEST_BUCKETS);
 
   for (let batch = 0; batch < maxBatches; batch++) {
-    const res = await readBatch(queuePath, offset, maxBuckets);
-    if (res.buckets.length === 0) break;
+    const hasHourly = queueSize > offset;
+    const hasProject = projectQueueEnabled && projectQueueSize > projectOffset;
+    let hourlyLimit = 0;
+    let projectLimit = 0;
+
+    if (hasHourly && hasProject) {
+      hourlyLimit = Math.ceil(totalLimit / 2);
+      projectLimit = totalLimit - hourlyLimit;
+    } else if (hasHourly) {
+      hourlyLimit = totalLimit;
+    } else if (hasProject) {
+      projectLimit = totalLimit;
+    }
 
-    attempted += res.buckets.length;
-    const ingest = await ingestHourly({ baseUrl, deviceToken, hourly: res.buckets });
+    const res =
+      hourlyLimit > 0 ? await readBatch(queuePath, offset, hourlyLimit) : { buckets: [], nextOffset: offset };
+    const projectRes =
+      projectLimit > 0
+        ? await readProjectBatch(projectQueuePath, projectOffset, projectLimit)
+        : { buckets: [], nextOffset: projectOffset };
+    if (res.buckets.length === 0 && projectRes.buckets.length === 0) break;
+
+    attempted += res.buckets.length + projectRes.buckets.length;
+    const ingest = await ingestHourly({
+      baseUrl,
+      deviceToken,
+      hourly: res.buckets,
+      project_hourly: projectRes.buckets
+    });
     inserted += ingest.inserted || 0;
     skipped += ingest.skipped || 0;
 
@@ -35,13 +82,20 @@ async function drainQueueToCloud({ baseUrl, deviceToken, queuePath, queueStatePa
     state.offset = offset;
     state.updatedAt = new Date().toISOString();
     await writeJson(queueStatePath, state);
+    if (projectQueuePath) {
+      projectOffset = projectRes.nextOffset;
+      projectState.offset = projectOffset;
+      projectState.updatedAt = new Date().toISOString();
+      await writeJson(projectStatePath, projectState);
+    }
 
     if (cb) {
+      const combinedOffset = offset + projectOffset;
       cb({
         batch: batch + 1,
         maxBatches,
-        offset,
-        queueSize,
+        offset: combinedOffset,
+        queueSize: queueSize + projectQueueSize,
         inserted,
         skipped
       });
@@ -88,6 +142,46 @@ async function readBatch(queuePath, startOffset, maxBuckets) {
   return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
 }
 
+async function readProjectBatch(queuePath, startOffset, maxBuckets) {
+  if (!queuePath) return { buckets: [], nextOffset: startOffset };
+  const st = await fs.stat(queuePath).catch(() => null);
+  if (!st || !st.isFile()) return { buckets: [], nextOffset: startOffset };
+  if (startOffset >= st.size) return { buckets: [], nextOffset: startOffset };
+
+  const stream = fssync.createReadStream(queuePath, { encoding: 'utf8', start: startOffset });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  const bucketMap = new Map();
+  let offset = startOffset;
+  let linesRead = 0;
+  for await (const line of rl) {
+    const bytes = Buffer.byteLength(line, 'utf8') + 1;
+    offset += bytes;
+    if (!line.trim()) continue;
+    let bucket;
+    try {
+      bucket = JSON.parse(line);
+    } catch (_e) {
+      continue;
+    }
+    const hourStart = typeof bucket?.hour_start === 'string' ? bucket.hour_start : null;
+    const projectKey = typeof bucket?.project_key === 'string' ? bucket.project_key : null;
+    const projectRef = typeof bucket?.project_ref === 'string' ? bucket.project_ref : null;
+    if (!hourStart || !projectKey || !projectRef) continue;
+    const source = normalizeSource(bucket?.source) || DEFAULT_SOURCE;
+    bucket.source = source;
+    bucket.project_key = projectKey;
+    bucket.project_ref = projectRef;
+    bucketMap.set(projectBucketKey(projectKey, source, hourStart), bucket);
+    linesRead += 1;
+    if (linesRead >= maxBuckets) break;
+  }
+
+  rl.close();
+  stream.close?.();
+  return { buckets: Array.from(bucketMap.values()), nextOffset: offset };
+}
+
 async function safeFileSize(p) {
   try {
     const st = await fs.stat(p);
@@ -101,6 +195,10 @@ function bucketKey(source, model, hourStart) {
   return `${source}${BUCKET_SEPARATOR}${model}${BUCKET_SEPARATOR}${hourStart}`;
 }
 
+function projectBucketKey(projectKey, source, hourStart) {
+  return `${projectKey}${BUCKET_SEPARATOR}${source}${BUCKET_SEPARATOR}${hourStart}`;
+}
+
 function normalizeSource(value) {
   if (typeof value !== 'string') return null;
   const trimmed = value.trim().toLowerCase();

package/src/lib/vibeusage-api.js

@@ -62,13 +62,13 @@ async function exchangeLinkCode({ baseUrl, linkCode, requestId, deviceName, plat
   return { token, deviceId, userId: data?.user_id || null };
 }
 
-async function ingestHourly({ baseUrl, deviceToken, hourly }) {
+async function ingestHourly({ baseUrl, deviceToken, hourly, project_hourly }) {
   const data = await invokeFunctionWithRetry({
     baseUrl,
     accessToken: deviceToken,
     slug: 'vibeusage-ingest',
     method: 'POST',
-    body: { hourly },
+    body: { hourly, project_hourly },
     errorPrefix: 'Ingest failed',
     retry: { maxRetries: 3, baseDelayMs: 500, maxDelayMs: 5000 }
   });

package/src/lib/vibeusage-public-repo.js

@@ -0,0 +1,88 @@
+const crypto = require('node:crypto');
+
+const GITHUB_HOST = 'github.com';
+const GITHUB_API_BASE = 'https://api.github.com';
+
+function parseGitHubRepoId(projectRef) {
+  if (typeof projectRef !== 'string') return null;
+  let parsed;
+  try {
+    parsed = new URL(projectRef);
+  } catch (_err) {
+    return null;
+  }
+  if (!parsed.hostname || parsed.hostname.toLowerCase() !== GITHUB_HOST) return null;
+  const segments = parsed.pathname.split('/').filter(Boolean);
+  if (segments.length < 2) return null;
+  const owner = segments[0].trim().toLowerCase();
+  const repo = segments[1].trim().replace(/\.git$/i, '').toLowerCase();
+  if (!owner || !repo) return null;
+  return { owner, repo, repoId: `${owner}/${repo}` };
+}
+
+function hashRepoRoot(repoRoot) {
+  return crypto.createHash('sha256').update(String(repoRoot)).digest('hex');
+}
+
+function isRateLimited(res, body) {
+  if (!res) return false;
+  const remaining = res.headers?.get?.('x-ratelimit-remaining');
+  if (remaining === '0') return true;
+  if (res.status === 429) return true;
+  if (body && typeof body.message === 'string' && body.message.toLowerCase().includes('rate limit')) {
+    return true;
+  }
+  return false;
+}
+
+async function resolveGitHubPublicStatus(projectRef, fetchImpl) {
+  const parsed = parseGitHubRepoId(projectRef);
+  if (!parsed) {
+    return { status: 'blocked', projectKey: null, projectRef, reason: 'non_github' };
+  }
+  const fetchFn = fetchImpl || fetch;
+  const apiUrl = `${GITHUB_API_BASE}/repos/${parsed.repoId}`;
+
+  let res;
+  let body = null;
+  try {
+    res = await fetchFn(apiUrl, {
+      headers: {
+        Accept: 'application/vnd.github+json',
+        'User-Agent': 'vibeusage-cli'
+      }
+    });
+  } catch (_err) {
+    return { status: 'pending_public', projectKey: parsed.repoId, projectRef, reason: 'fetch_failed' };
+  }
+
+  if (res.status === 200) {
+    body = await res.json().catch(() => ({}));
+    const isPrivate = body && typeof body.private === 'boolean' ? body.private : null;
+    const visibility = typeof body?.visibility === 'string' ? body.visibility : null;
+    const isPublic = isPrivate === false || visibility === 'public';
+    return {
+      status: isPublic ? 'public_verified' : 'blocked',
+      projectKey: parsed.repoId,
+      projectRef,
+      reason: isPublic ? 'public' : 'private'
+    };
+  }
+
+  if (res.status === 404) {
+    return { status: 'blocked', projectKey: parsed.repoId, projectRef, reason: 'not_found' };
+  }
+
+  body = await res.json().catch(() => ({}));
+  if (isRateLimited(res, body) || res.status === 401 || res.status === 403 || res.status >= 500) {
+    return { status: 'pending_public', projectKey: parsed.repoId, projectRef, reason: 'rate_limited' };
+  }
+
+  return { status: 'pending_public', projectKey: parsed.repoId, projectRef, reason: 'unknown' };
+}
+
+module.exports = {
+  parseGitHubRepoId,
+  hashRepoRoot,
+  resolveGitHubPublicStatus
+};