vibeusage 0.2.16 → 0.2.17

package/src/commands/sync.js

@@ -12,9 +12,11 @@ const {
   parseRolloutIncremental,
   parseClaudeIncremental,
   parseGeminiIncremental,
-  parseOpencodeIncremental
+  parseOpencodeIncremental,
+  parseOpenclawIncremental
 } = require('../lib/rollout');
 const { drainQueueToCloud } = require('../lib/uploader');
+const { collectLocalSubscriptions } = require('../lib/subscriptions');
 const { createProgress, renderBar, formatNumber, formatBytes } = require('../lib/progress');
 const { syncHeartbeat } = require('../lib/vibeusage-api');
 const {
@@ -34,6 +36,9 @@ async function cmdSync(argv) {
   const { trackerDir } = await resolveTrackerPaths({ home });
 
   await ensureDir(trackerDir);
+  if (opts.fromOpenclaw) {
+    await writeOpenclawSignal(trackerDir);
+  }
 
   const lockPath = path.join(trackerDir, 'sync.lock');
   const lock = await openLock(lockPath, { quietIfLocked: opts.auto });
@@ -64,10 +69,16 @@ async function cmdSync(argv) {
     const opencodeHome = process.env.OPENCODE_HOME || path.join(xdgDataHome, 'opencode');
     const opencodeStorageDir = path.join(opencodeHome, 'storage');
 
-    const sources = [
-      { source: 'codex', sessionsDir: path.join(codexHome, 'sessions') },
-      { source: 'every-code', sessionsDir: path.join(codeHome, 'sessions') }
-    ];
+    // OpenClaw hook integration: allow a hook to request incremental parsing for a single session jsonl.
+    // When present, we skip all other sources to keep hook-triggered sync fast and deterministic.
+    const openclawSignal = opts.fromOpenclaw ? resolveOpenclawSignal({ home, env: process.env }) : null;
+
+    const sources = openclawSignal
+      ? []
+      : [
+          { source: 'codex', sessionsDir: path.join(codexHome, 'sessions') },
+          { source: 'every-code', sessionsDir: path.join(codeHome, 'sessions') }
+        ];
 
     const rolloutFiles = [];
     const seenSessions = new Set();
@@ -80,6 +91,8 @@ async function cmdSync(argv) {
       }
     }
 
+    const openclawFiles = openclawSignal?.sessionFile ? [{ path: openclawSignal.sessionFile, source: 'openclaw' }] : [];
+
     if (progress?.enabled) {
       progress.start(`Parsing ${renderBar(0)} 0/${formatNumber(rolloutFiles.length)} files | buckets 0`);
     }
@@ -100,9 +113,21 @@ async function cmdSync(argv) {
       }
     });
 
-    const claudeFiles = await listClaudeProjectFiles(claudeProjectsDir);
+    let openclawResult = { filesProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+    if (openclawFiles.length > 0) {
+      // Only runs when explicitly triggered by OpenClaw hooks.
+      openclawResult = await parseOpenclawIncremental({
+        sessionFiles: openclawFiles,
+        cursors,
+        queuePath,
+        projectQueuePath,
+        source: 'openclaw'
+      });
+    }
+
+    const claudeFiles = openclawSignal ? [] : await listClaudeProjectFiles(claudeProjectsDir);
     let claudeResult = { filesProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
-    if (claudeFiles.length > 0) {
+    if (!openclawSignal && claudeFiles.length > 0) {
       if (progress?.enabled) {
         progress.start(`Parsing Claude ${renderBar(0)} 0/${formatNumber(claudeFiles.length)} files | buckets 0`);
       }
@@ -124,9 +149,9 @@ async function cmdSync(argv) {
       });
     }
 
-    const geminiFiles = await listGeminiSessionFiles(geminiTmpDir);
+    const geminiFiles = openclawSignal ? [] : await listGeminiSessionFiles(geminiTmpDir);
     let geminiResult = { filesProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
-    if (geminiFiles.length > 0) {
+    if (!openclawSignal && geminiFiles.length > 0) {
       if (progress?.enabled) {
         progress.start(`Parsing Gemini ${renderBar(0)} 0/${formatNumber(geminiFiles.length)} files | buckets 0`);
       }
@@ -148,9 +173,9 @@ async function cmdSync(argv) {
       });
     }
 
-    const opencodeFiles = await listOpencodeMessageFiles(opencodeStorageDir);
+    const opencodeFiles = openclawSignal ? [] : await listOpencodeMessageFiles(opencodeStorageDir);
     let opencodeResult = { filesProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
-    if (opencodeFiles.length > 0) {
+    if (!openclawSignal && opencodeFiles.length > 0) {
       if (progress?.enabled) {
         progress.start(`Parsing Opencode ${renderBar(0)} 0/${formatNumber(opencodeFiles.length)} files | buckets 0`);
       }
@@ -244,10 +269,17 @@ async function cmdSync(argv) {
 
       if (allowUpload && maxBatches > 0) {
         uploadAttempted = true;
+        const deviceSubscriptions = await collectLocalSubscriptions({
+          home,
+          env: process.env,
+          probeKeychain: true,
+          probeKeychainDetails: true
+        });
         try {
           uploadResult = await drainQueueToCloud({
             baseUrl,
             deviceToken,
+            deviceSubscriptions,
             queuePath,
             queueStatePath,
             projectQueuePath,
@@ -330,11 +362,13 @@ async function cmdSync(argv) {
     if (!opts.auto) {
       const totalParsed =
         parseResult.filesProcessed +
+        openclawResult.filesProcessed +
         claudeResult.filesProcessed +
         geminiResult.filesProcessed +
         opencodeResult.filesProcessed;
       const totalBuckets =
         parseResult.bucketsQueued +
+        openclawResult.bucketsQueued +
         claudeResult.bucketsQueued +
         geminiResult.bucketsQueued +
         opencodeResult.bucketsQueued;
@@ -367,6 +401,7 @@ function parseArgs(argv) {
     auto: false,
     fromNotify: false,
     fromRetry: false,
+    fromOpenclaw: false,
     drain: false
   };
   for (let i = 0; i < argv.length; i++) {
@@ -374,6 +409,7 @@ function parseArgs(argv) {
     if (a === '--auto') out.auto = true;
     else if (a === '--from-notify') out.fromNotify = true;
     else if (a === '--from-retry') out.fromRetry = true;
+    else if (a === '--from-openclaw') out.fromOpenclaw = true;
     else if (a === '--drain') out.drain = true;
     else throw new Error(`Unknown option: ${a}`);
   }
@@ -382,6 +418,26 @@ function parseArgs(argv) {
 
 module.exports = { cmdSync };
 
+function normalizeString(value) {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function resolveOpenclawSignal({ home, env } = {}) {
+  if (!env) return null;
+
+  const agentId = normalizeString(env.VIBEUSAGE_OPENCLAW_AGENT_ID);
+  const sessionId = normalizeString(env.VIBEUSAGE_OPENCLAW_PREV_SESSION_ID);
+  if (!agentId || !sessionId) return null;
+
+  const openclawHome = normalizeString(env.VIBEUSAGE_OPENCLAW_HOME) || path.join(home || os.homedir(), '.openclaw');
+  const sessionFile = path.join(openclawHome, 'agents', agentId, 'sessions', `${sessionId}.jsonl`);
+
+  return { agentId, sessionId, openclawHome, sessionFile };
+}
+
+
 async function safeStatSize(p) {
   try {
     const st = await fs.stat(p);
@@ -509,6 +565,15 @@ function coerceRetryMs(v) {
   return Math.floor(n);
 }
 
+async function writeOpenclawSignal(trackerDir) {
+  const openclawSignalPath = path.join(trackerDir, 'openclaw.signal');
+  try {
+    await fs.writeFile(openclawSignalPath, new Date().toISOString(), 'utf8');
+  } catch (_e) {
+    // best-effort marker
+  }
+}
+
 const HEARTBEAT_MIN_INTERVAL_MINUTES = 30;
 const HEARTBEAT_MIN_INTERVAL_MS = HEARTBEAT_MIN_INTERVAL_MINUTES * 60 * 1000;
 const AUTO_RETRY_FILENAME = 'auto.retry.json';

package/src/lib/codex-config.js

@@ -117,14 +117,25 @@ async function readEveryCodeNotify(codeConfigPath) {
 
 function extractNotify(text) {
   // Heuristic parse: find a line that starts with "notify =".
+  // Supports single-line arrays:
+  // - notify = ["a", "b"]
+  // And multi-line arrays:
+  // - notify = [
+  //     "a",
+  //     "b"
+  //   ]
   const lines = text.split(/\r?\n/);
-  for (const line of lines) {
-    const m = line.match(/^\s*notify\s*=\s*(.+)\s*$/);
-    if (m) {
-      const rhs = m[1].trim();
-      const parsed = parseTomlStringArray(rhs);
-      if (parsed) return parsed;
-    }
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const m = line.match(/^\s*notify\s*=\s*(.*)\s*$/);
+    if (!m) continue;
+
+    const rhs = (m[1] || '').trim();
+    const literal = readTomlArrayLiteral(lines, i, rhs);
+    if (!literal) continue;
+
+    const parsed = parseTomlStringArray(literal);
+    if (parsed) return parsed;
   }
   return null;
 }
@@ -137,12 +148,15 @@ function setNotify(text, notifyCmd) {
   let replaced = false;
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
-    const isNotify = /^\s*notify\s*=/.test(line);
-    if (isNotify) {
+    const m = line.match(/^\s*notify\s*=\s*(.*)\s*$/);
+    if (m) {
       if (!replaced) {
         out.push(notifyLine);
         replaced = true;
       }
+
+      const rhs = (m[1] || '').trim();
+      i = findTomlArrayBlockEnd(lines, i, rhs);
       continue;
     }
     out.push(line);
@@ -160,7 +174,17 @@ function setNotify(text, notifyCmd) {
 
 function removeNotify(text) {
   const lines = text.split(/\r?\n/);
-  const out = lines.filter((l) => !/^\s*notify\s*=/.test(l));
+  const out = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const m = line.match(/^\s*notify\s*=\s*(.*)\s*$/);
+    if (m) {
+      const rhs = (m[1] || '').trim();
+      i = findTomlArrayBlockEnd(lines, i, rhs);
+      continue;
+    }
+    out.push(line);
+  }
   return out.join('\n').replace(/\n+$/, '\n');
 }
 
@@ -201,6 +225,104 @@ function formatTomlStringArray(arr) {
   return `[${arr.map((s) => JSON.stringify(String(s))).join(', ')}]`;
 }
 
+function readTomlArrayLiteral(lines, startIndex, rhs) {
+  const first = rhs.trim();
+  if (!first.startsWith('[')) return null;
+
+  let inString = false;
+  let quote = null;
+  let depth = 0;
+  let sawOpen = false;
+
+  function scanChunk(chunk) {
+    for (let i = 0; i < chunk.length; i++) {
+      const ch = chunk[i];
+      if (!inString) {
+        if (ch === '"' || ch === "'") {
+          inString = true;
+          quote = ch;
+          continue;
+        }
+        if (ch === '[') {
+          depth += 1;
+          sawOpen = true;
+          continue;
+        }
+        if (ch === ']') {
+          depth -= 1;
+          if (sawOpen && depth === 0) return i;
+        }
+        continue;
+      }
+      if (ch === quote) {
+        inString = false;
+        quote = null;
+      }
+    }
+    return -1;
+  }
+
+  const parts = [first];
+  let endPos = scanChunk(first);
+  if (endPos !== -1) return first.slice(0, endPos + 1).trim();
+
+  for (let j = startIndex + 1; j < lines.length; j++) {
+    const line = lines[j];
+    endPos = scanChunk(line);
+    if (endPos !== -1) {
+      parts.push(line.slice(0, endPos + 1));
+      return parts.join('\n').trim();
+    }
+    parts.push(line);
+  }
+
+  return null;
+}
+
+function findTomlArrayBlockEnd(lines, startIndex, rhs) {
+  const first = rhs.trim();
+  if (!first.startsWith('[')) return startIndex;
+
+  let inString = false;
+  let quote = null;
+  let depth = 0;
+  let sawOpen = false;
+
+  function scanChunk(chunk) {
+    for (let i = 0; i < chunk.length; i++) {
+      const ch = chunk[i];
+      if (!inString) {
+        if (ch === '"' || ch === "'") {
+          inString = true;
+          quote = ch;
+          continue;
+        }
+        if (ch === '[') {
+          depth += 1;
+          sawOpen = true;
+          continue;
+        }
+        if (ch === ']') {
+          depth -= 1;
+          if (sawOpen && depth === 0) return true;
+        }
+        continue;
+      }
+      if (ch === quote) {
+        inString = false;
+        quote = null;
+      }
+    }
+    return false;
+  }
+
+  if (scanChunk(first)) return startIndex;
+  for (let j = startIndex + 1; j < lines.length; j++) {
+    if (scanChunk(lines[j])) return j;
+  }
+  return startIndex;
+}
+
 function arraysEqual(a, b) {
   if (!Array.isArray(a) || !Array.isArray(b)) return false;
   if (a.length !== b.length) return false;

package/src/lib/diagnostics.js

@@ -26,6 +26,7 @@ async function collectTrackerDiagnostics({
   const queueStatePath = path.join(trackerDir, 'queue.state.json');
   const cursorsPath = path.join(trackerDir, 'cursors.json');
   const notifySignalPath = path.join(trackerDir, 'notify.signal');
+  const openclawSignalPath = path.join(trackerDir, 'openclaw.signal');
   const throttlePath = path.join(trackerDir, 'sync.throttle');
   const uploadThrottlePath = path.join(trackerDir, 'upload.throttle.json');
   const autoRetryPath = path.join(trackerDir, 'auto.retry.json');
@@ -47,6 +48,7 @@ async function collectTrackerDiagnostics({
   const pendingBytes = Math.max(0, queueSize - offsetBytes);
 
   const lastNotify = (await safeReadText(notifySignalPath))?.trim() || null;
+  const lastOpenclawSync = (await safeReadText(openclawSignalPath))?.trim() || null;
   const lastNotifySpawn = parseEpochMsToIso((await safeReadText(throttlePath))?.trim() || null);
 
   const codexNotifyRaw = await readCodexNotify(codexConfigPath);
@@ -107,6 +109,7 @@ async function collectTrackerDiagnostics({
     },
     notify: {
       last_notify: lastNotify,
+      last_openclaw_triggered_sync: lastOpenclawSync,
       last_notify_triggered_sync: lastNotifySpawn,
       codex_notify_configured: notifyConfigured,
       codex_notify: codexNotify,

package/src/lib/rollout.js

@@ -527,6 +527,159 @@ async function parseOpencodeIncremental({
   return { filesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
 }
 
+async function parseOpenclawIncremental({
+  sessionFiles,
+  cursors,
+  queuePath,
+  projectQueuePath,
+  onProgress,
+  source
+}) {
+  await ensureDir(path.dirname(queuePath));
+  let filesProcessed = 0;
+  let eventsAggregated = 0;
+
+  const cb = typeof onProgress === 'function' ? onProgress : null;
+  const files = Array.isArray(sessionFiles) ? sessionFiles : [];
+  const totalFiles = files.length;
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const projectEnabled = typeof projectQueuePath === 'string' && projectQueuePath.length > 0;
+  const projectState = projectEnabled ? normalizeProjectState(cursors?.projectHourly) : null;
+  const projectTouchedBuckets = projectEnabled ? new Set() : null;
+  const touchedBuckets = new Set();
+  const defaultSource = normalizeSourceInput(source) || 'openclaw';
+
+  if (!cursors.files || typeof cursors.files !== 'object') {
+    cursors.files = {};
+  }
+
+  for (let idx = 0; idx < files.length; idx++) {
+    const entry = files[idx];
+    const filePath = typeof entry === 'string' ? entry : entry?.path;
+    if (!filePath) continue;
+    const fileSource =
+      typeof entry === 'string' ? defaultSource : normalizeSourceInput(entry?.source) || defaultSource;
+    const st = await fs.stat(filePath).catch(() => null);
+    if (!st || !st.isFile()) continue;
+
+    const key = filePath;
+    const prev = cursors.files[key] || null;
+    const inode = st.ino || 0;
+    const startOffset = prev && prev.inode === inode ? prev.offset || 0 : 0;
+
+    const result = await parseOpenclawSessionFile({
+      filePath,
+      startOffset,
+      hourlyState,
+      touchedBuckets,
+      source: fileSource,
+      projectState,
+      projectTouchedBuckets
+    });
+
+    cursors.files[key] = {
+      inode,
+      offset: result.endOffset,
+      updatedAt: new Date().toISOString()
+    };
+
+    filesProcessed += 1;
+    eventsAggregated += result.eventsAggregated;
+
+    if (cb) {
+      cb({
+        index: idx + 1,
+        total: totalFiles,
+        filePath,
+        filesProcessed,
+        eventsAggregated,
+        bucketsQueued: touchedBuckets.size
+      });
+    }
+  }
+
+  const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
+  const projectBucketsQueued = projectEnabled
+    ? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
+    : 0;
+  hourlyState.updatedAt = new Date().toISOString();
+  cursors.hourly = hourlyState;
+  if (projectState) {
+    projectState.updatedAt = new Date().toISOString();
+    cursors.projectHourly = projectState;
+  }
+
+  return { filesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
+}
+
+async function parseOpenclawSessionFile({
+  filePath,
+  startOffset,
+  hourlyState,
+  touchedBuckets,
+  source,
+  projectState,
+  projectTouchedBuckets
+}) {
+  const st = await fs.stat(filePath);
+  const endOffset = st.size;
+  if (startOffset >= endOffset) return { endOffset, eventsAggregated: 0 };
+
+  const stream = fssync.createReadStream(filePath, { encoding: 'utf8', start: startOffset });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  let eventsAggregated = 0;
+  for await (const line of rl) {
+    if (!line) continue;
+    // Fast-path filter: OpenClaw assistant messages include message.usage.totalTokens.
+    if (!line.includes('"usage"') || !line.includes('totalTokens')) continue;
+
+    let obj;
+    try {
+      obj = JSON.parse(line);
+    } catch (_e) {
+      continue;
+    }
+
+    if (obj?.type !== 'message') continue;
+    const msg = obj?.message;
+    if (!msg || typeof msg !== 'object') continue;
+
+    const usage = msg.usage;
+    if (!usage || typeof usage !== 'object') continue;
+
+    const tokenTimestamp = typeof obj?.timestamp === 'string' ? obj.timestamp : null;
+    if (!tokenTimestamp) continue;
+
+    const model = normalizeModelInput(msg.model) || DEFAULT_MODEL;
+
+    const delta = {
+      input_tokens: Number(usage.input || 0),
+      cached_input_tokens: Number((usage.cacheRead || 0) + (usage.cacheWrite || 0)),
+      output_tokens: Number(usage.output || 0),
+      reasoning_output_tokens: 0,
+      total_tokens: Number(usage.totalTokens || 0)
+    };
+
+    if (isAllZeroUsage(delta)) continue;
+
+    const bucketStart = toUtcHalfHourStart(tokenTimestamp);
+    if (!bucketStart) continue;
+
+    const bucket = getHourlyBucket(hourlyState, source, model, bucketStart);
+    addTotals(bucket.totals, delta);
+    touchedBuckets.add(bucketKey(source, model, bucketStart));
+
+    // Project-level OpenClaw attribution is not supported yet (no stable cwd info).
+    // If OpenClaw later records cwd per event, we can mirror rollout's project logic.
+    eventsAggregated += 1;
+  }
+
+  rl.close();
+  stream.close?.();
+  return { endOffset, eventsAggregated };
+}
+
 async function parseRolloutFile({
   filePath,
   startOffset,
@@ -1972,5 +2125,6 @@ module.exports = {
   parseRolloutIncremental,
   parseClaudeIncremental,
   parseGeminiIncremental,
-  parseOpencodeIncremental
+  parseOpencodeIncremental,
+  parseOpenclawIncremental
 };