tokentracker-cli 1.3.8 → 1.3.10

package/src/commands/sync.js

@@ -8,6 +8,7 @@ const { functionUrlFor, fetchFunctionResponse } = require("../lib/function-url")
 
 const { resolveInstallPaths, resolveZcodeNativeDbPath, resolveMimoNativeDbPath, ensureFlatCursor } = require("../lib/install-resolver");
 const { multiInstallParse, mergeBothFileSources } = require("../lib/multi-install-parser");
+const { resolveDumateDbPaths, parseDumateIncremental } = require("../lib/dumate");
 const wsl = require("../lib/wsl-probe");
 const {
   ensureDir,
@@ -155,12 +156,14 @@ const {
   recordUploadSuccess,
   parseRetryAfterMs,
 } = require("../lib/upload-throttle");
+const { compactPendingQueue, iterateQueueLines, sessionSnapshotMs } = require("../lib/upload-queue");
 const AUTO_UPLOAD_CONFIG = {
   intervalMs: 15 * 60_000,
   batchSize: 200,
   maxBatchesSmall: 5,
-  maxBatchesLarge: 5,
+  maxBatchesLarge: 100,
 };
+const UPLOAD_BUDGET_MS = 90_000;
 const { maybeSendHeartbeat } = require("../lib/telemetry");
 const {
   isCursorInstalled,
@@ -327,6 +330,7 @@ const AUTO_SYNC_SOURCES = new Set([
   "cursor",
   "devin",
   "droid",
+  "dumate",
   "dsh",
   "every-code",
   "gemini",
@@ -1686,6 +1690,20 @@ async function cmdSync(argv, context = {}) {
       }
     }
 
+    let dumateResult = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+    if (sourceAllowed("dumate")) {
+      const dumateDbPaths = resolveDumateDbPaths({ home });
+      if (dumateDbPaths.length > 0) {
+        if (progress?.enabled) progress.start(`Parsing DuMate ${renderBar(0)} | buckets 0`);
+        try {
+          dumateResult = await parseDumateIncremental({
+            dbPaths: dumateDbPaths, cursors, queuePath, projectQueuePath,
+            onProgress: makeProviderProgress("DuMate"),
+          });
+        } catch (err) { warnProviderParseFailure("DuMate", err, opts); }
+      }
+    }
+
     // ── Mimo (mimocode — OpenCode-fork SQLite) ──
     // readMimoDbMessages filters out mirrored Claude Code rows to avoid
     // double-counting usage already counted as source=claude.
@@ -3319,6 +3337,7 @@ async function cmdSync(argv, context = {}) {
       anythingllmResult.recordsProcessed +
       devinResult.recordsProcessed +
       kiloResult.recordsProcessed +
+      dumateResult.recordsProcessed +
       mimoResult.recordsProcessed +
       zcodeResult.recordsProcessed +
       kilocodeResult.recordsProcessed +
@@ -3364,6 +3383,7 @@ async function cmdSync(argv, context = {}) {
       anythingllmResult.bucketsQueued +
       devinResult.bucketsQueued +
       kiloResult.bucketsQueued +
+      dumateResult.bucketsQueued +
       mimoResult.bucketsQueued +
       zcodeResult.bucketsQueued +
       kilocodeResult.bucketsQueued +
@@ -3490,6 +3510,10 @@ async function cmdSync(argv, context = {}) {
     if (canUpload() && runtime.deviceToken && runtime.baseUrl &&
         (!isBackgroundLightweightSync || opts.publishAccount) &&
         (!autoUploadDecision || autoUploadDecision.allowed)) {
+      // Decide the window budget from the original backlog. Compact only once
+      // upload is allowed so a cooldown run cannot shrink away that budget.
+      const state = (await readJson(queueStatePath)) || { offset: 0 };
+      await compactPendingQueue({ queuePath, offset: Number(state.offset || 0) });
       uploadAttempted = true;
       // Mirror the machine identity into the purge-surviving seed file so a
       // future `uninstall --purge` + reinstall recovers the same cloud device
@@ -3511,6 +3535,7 @@ async function cmdSync(argv, context = {}) {
             queueStatePath,
             maxBatches: opts.drain ? 100 : (autoUploadDecision?.maxBatches || 5),
             batchSize: autoUploadDecision?.batchSize || 200,
+            maxDurationMs: UPLOAD_BUDGET_MS,
             canUpload,
           });
         try {
@@ -3570,6 +3595,14 @@ async function cmdSync(argv, context = {}) {
           await writeJson(uploadThrottlePath, uploadThrottleState);
         }
       } catch (e) {
+        if (e.uploadResult?.batches > 0) {
+          uploadResult = e.uploadResult;
+          uploadThrottleState = recordUploadSuccess({
+            nowMs: Date.now(), state: uploadThrottleState, config: AUTO_UPLOAD_CONFIG,
+            cloudAccess: uploadResult.cloudAccess,
+            attemptId: process.env.TOKENTRACKER_LOCAL_SYNC_ATTEMPT_ID,
+          });
+        }
         // Persist a backoff on 429 / 5xx so the next auto-sync waits instead
         // of retrying immediately and making the rate-limit worse. The
         // throttle module already parses Retry-After when we surface it on
@@ -4252,7 +4285,7 @@ const AUTO_RETRY_MAX_DELAY_MS = 2 * 60 * 60 * 1000;
 const INGEST_SLUG = "tokentracker-ingest";
 const MAX_INGEST_BUCKETS = 500;
 
-async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200, canUpload = () => true }) {
+async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, queueStatePath, maxBatches = 5, batchSize = 200, maxDurationMs = UPLOAD_BUDGET_MS, canUpload = () => true }) {
   const state = (await readJson(queueStatePath)) || { offset: 0 };
   let offset = Number(state.offset || 0);
   let inserted = 0;
@@ -4260,84 +4293,113 @@ async function drainQueueToCloud({ baseUrl, anonKey, deviceToken, queuePath, que
   let batches = 0;
   let cloudAccess = null;
   const uploadId = crypto.randomUUID();
+  const deadline = Date.now() + maxDurationMs;
+  let batchBudget = Math.min(maxBatches, 100);
+  let requestSignal = null;
+  const resultSoFar = () => ({ inserted, skipped, batches, ...(cloudAccess?.next_allowed_at ? { cloudAccess } : {}) });
 
   const queueSize = await safeStatSize(queuePath);
   const limit = Math.min(Math.max(1, Math.floor(Number(batchSize || 200))), MAX_INGEST_BUCKETS);
 
-  for (let batch = 0; batch < maxBatches; batch++) {
-    if (offset >= queueSize) break;
-    const result = await readQueueBatch(queuePath, offset, limit);
-    // A states-only tail (e.g. a retry batch whose bucket rows already
-    // uploaded) is still a meaningful upload: it advances the canonical
-    // session corrections cloud-side.
-    if (result.buckets.length === 0 && result.sessionStates.length === 0) break;
-
-    const root = baseUrl.replace(/\/$/, "");
-    const headers = {
-      "Content-Type": "application/json",
-      Accept: "application/json",
-      Authorization: `Bearer ${deviceToken}`,
-    };
-    if (anonKey) headers.apikey = anonKey;
-    // Re-read after parsing/each batch so switching off stops an active drain.
-    if (!canUpload()) break;
-    const res = await fetchFunctionResponse(functionUrlFor(root, INGEST_SLUG), {
-      method: "POST",
-      headers,
-      body: JSON.stringify({
-        upload_id: uploadId,
-        hourly: result.buckets,
-        ...(result.sessionStates.length > 0
-          ? { account_session_states: result.sessionStates }
-          : {}),
-      }),
-    });
+  try {
+    for (let batch = 0; batch < batchBudget; batch++) {
+      if (Date.now() >= deadline) break;
+      if (offset >= queueSize) break;
+      const result = await readQueueBatch(queuePath, offset, limit);
+      // A states-only tail (e.g. a retry batch whose bucket rows already
+      // uploaded) is still a meaningful upload: it advances the canonical
+      // session corrections cloud-side.
+      if (result.buckets.length === 0 && result.sessionStates.length === 0) {
+        // Complete ignored control/malformed rows need no cloud request, but
+        // must not leave a permanent pending tail. Incomplete bytes stay put.
+        if (result.nextOffset > offset) {
+          offset = result.nextOffset;
+          state.offset = offset;
+          state.updatedAt = new Date().toISOString();
+          await writeJson(queueStatePath, state);
+        }
+        break;
+      }
 
-    const rawText = await res.text().catch(() => "");
-    let data = {};
-    try { data = JSON.parse(rawText); } catch { data = {}; }
-    if (!res.ok) {
-      const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
-      err.status = res.status;
-      err.code = res.status === 401 ? "CLOUD_DEVICE_TOKEN_REJECTED"
-        : typeof data?.code === "string" ? data.code
-          : res.status === 403 ? "CLOUD_UPLOAD_FORBIDDEN" : "CLOUD_UPLOAD_FAILED";
-      const retryAfter = res.headers?.get?.("Retry-After") ?? null;
-      const retryAfterMs = parseRetryAfterMs(retryAfter);
-      if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
-      throw err;
-    }
-
-    cloudAccess = data;
-    inserted += Number(data?.inserted || 0);
-    skipped += Number(data?.skipped || 0);
-    batches += 1;
-
-    offset = result.nextOffset;
-    state.offset = offset;
-    state.updatedAt = new Date().toISOString();
-    await writeJson(queueStatePath, state);
-  }
+      const root = baseUrl.replace(/\/$/, "");
+      const headers = {
+        "Content-Type": "application/json",
+        Accept: "application/json",
+        Authorization: `Bearer ${deviceToken}`,
+      };
+      if (anonKey) headers.apikey = anonKey;
+      // Re-read after parsing/each batch so switching off stops an active drain.
+      if (!canUpload()) break;
+      const remainingMs = deadline - Date.now();
+      if (remainingMs <= 0) break;
+      requestSignal = AbortSignal.timeout(Math.max(1, Math.ceil(remainingMs)));
+      const res = await fetchFunctionResponse(functionUrlFor(root, INGEST_SLUG), {
+        method: "POST",
+        headers,
+        signal: requestSignal,
+        body: JSON.stringify({
+          upload_id: uploadId,
+          hourly: result.buckets,
+          ...(result.sessionStates.length > 0
+            ? { account_session_states: result.sessionStates }
+            : {}),
+        }),
+      });
 
-  return { inserted, skipped, batches, ...(cloudAccess?.next_allowed_at ? { cloudAccess } : {}) };
+      const rawText = await res.text();
+      let data = {};
+      try { data = JSON.parse(rawText); } catch { data = {}; }
+      if (!res.ok) {
+        const err = new Error(`HTTP ${res.status}: ${rawText.substring(0, 500)}`);
+        err.status = res.status;
+        err.code = res.status === 401 ? "CLOUD_DEVICE_TOKEN_REJECTED"
+          : typeof data?.code === "string" ? data.code
+            : res.status === 403 ? "CLOUD_UPLOAD_FORBIDDEN" : "CLOUD_UPLOAD_FAILED";
+        const retryAfter = res.headers?.get?.("Retry-After") ?? null;
+        const retryAfterMs = parseRetryAfterMs(retryAfter);
+        if (retryAfterMs !== null) err.retryAfterMs = retryAfterMs;
+        throw err;
+      }
+
+      cloudAccess = data;
+      if (data.upload_scope === "community") batchBudget = Math.min(batchBudget, 10);
+      inserted += Number(data?.inserted || 0);
+      skipped += Number(data?.skipped || 0);
+      batches += 1;
+
+      offset = result.nextOffset;
+      state.offset = offset;
+      state.updatedAt = new Date().toISOString();
+      await writeJson(queueStatePath, state);
+    }
+  } catch (error) {
+    // A bounded drain can finish partially without discarding successful
+    // offsets or the server cooldown. Other failures retain their retry path.
+    const requestTimedOut = requestSignal?.aborted && requestSignal.reason?.name === "TimeoutError" &&
+      (error === requestSignal.reason || error.name === "TimeoutError" || error.name === "AbortError");
+    if (!requestTimedOut || batches === 0) {
+      error.uploadResult = resultSoFar();
+      throw error;
+    }
+  }
+  return resultSoFar();
 }
 
 async function readQueueBatch(queuePath, startOffset, maxBuckets) {
   const st = await fs.stat(queuePath).catch(() => null);
-  if (!st || !st.isFile()) return { buckets: [], nextOffset: startOffset };
-  if (startOffset >= st.size) return { buckets: [], nextOffset: startOffset };
+  if (!st || !st.isFile()) return { buckets: [], sessionStates: [], nextOffset: startOffset };
+  if (startOffset >= st.size) return { buckets: [], sessionStates: [], nextOffset: startOffset };
 
-  const stream = fssync.createReadStream(queuePath, { encoding: "utf8", start: startOffset });
-  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+  const handle = await fs.open(queuePath, "r");
 
   const bucketMap = new Map();
   // Account session states (kind: "account_session_state") are queue
   // control records, not usage rows: collect them separately so they ride
   // the same upload (after the bucket rows in file order) without entering
-  // the bucket stream. Keyed by (source, session_id) LAST-WINS: the
-  // append-only queue writes a session's newer observation after its older
-  // one, and only the latest per session may reach the ingest batch (a
-  // duplicate would make the cloud ON CONFLICT affect one row twice).
+  // the bucket stream. Keyed by (source, session_id), use the strictly newer
+  // snapshot_verified_at and retain the first row on ties, matching account
+  // truth even after a clock rollback. Only one observation per session may
+  // reach the ingest batch (duplicates would affect one ON CONFLICT row twice).
   // snapshot_verified_at rides along verbatim from the ORIGINAL fetch, so a
   // transport retry never fakes logical freshness.
   //
@@ -4351,68 +4413,81 @@ async function readQueueBatch(queuePath, startOffset, maxBuckets) {
   // (and the MAX_INGEST_BUCKETS=500 clamp above), every request stays under
   // the edge cap while batching follows the existing design.
   const sessionStateMap = new Map();
+  const sessionStateStamps = new Map();
   let offset = startOffset;
   let linesRead = 0;
-  for await (const line of rl) {
-    const bytes = Buffer.byteLength(line, "utf8") + 1;
-    offset += bytes;
-    if (!line.trim()) continue;
-    let bucket;
-    try {
-      bucket = JSON.parse(line);
-    } catch (_e) {
-      continue;
-    }
-    if (bucket?.kind === "account_session_state") {
-      if (typeof bucket.source === "string" && typeof bucket.session_id === "string" && bucket.session_id.trim()) {
-        const ssSource = bucket.source.trim().toLowerCase();
-        sessionStateMap.set(ssSource + "|" + bucket.session_id.trim(), {
-          source: ssSource,
-          session_id: bucket.session_id.trim(),
-          model: typeof bucket.model === "string" ? bucket.model : "",
-          bucket_start: typeof bucket.bucket_start === "string" ? bucket.bucket_start : "",
-          input_tokens: Number(bucket.input_tokens) || 0,
-          output_tokens: Number(bucket.output_tokens) || 0,
-          cached_input_tokens: Number(bucket.cached_input_tokens) || 0,
-          cache_creation_input_tokens: Number(bucket.cache_creation_input_tokens) || 0,
-          reasoning_output_tokens: Number(bucket.reasoning_output_tokens) || 0,
-          total_tokens: Number(bucket.total_tokens) || 0,
-          ...(typeof bucket.snapshot_verified_at === "string" && bucket.snapshot_verified_at
-            ? { snapshot_verified_at: bucket.snapshot_verified_at }
-            : {}),
-        });
-        // Counted like a bucket row: the record rides THIS batch and its
-        // bytes are covered by nextOffset, so the next batch resumes right
-        // after it.
-        linesRead += 1;
-        if (linesRead >= maxBuckets) break;
+  try {
+    for await (const record of iterateQueueLines(handle, startOffset, st.size)) {
+      // A producer may still be writing the last row; retry it once complete.
+      if (!record.complete) break;
+      const line = record.bytes.toString("utf8");
+      offset += record.bytes.length;
+      if (!line.trim()) continue;
+      let bucket;
+      try {
+        bucket = JSON.parse(line);
+      } catch (_e) {
+        continue;
       }
-      continue;
+      if (bucket?.kind === "account_session_state") {
+        if (typeof bucket.source === "string" && typeof bucket.session_id === "string" && bucket.session_id.trim()) {
+          const ssSource = bucket.source.trim().toLowerCase();
+          const stateKey = ssSource + "|" + bucket.session_id.trim();
+          const sessionState = {
+            source: ssSource,
+            session_id: bucket.session_id.trim(),
+            model: typeof bucket.model === "string" ? bucket.model : "",
+            bucket_start: typeof bucket.bucket_start === "string" ? bucket.bucket_start : "",
+            input_tokens: Number(bucket.input_tokens) || 0,
+            output_tokens: Number(bucket.output_tokens) || 0,
+            cached_input_tokens: Number(bucket.cached_input_tokens) || 0,
+            cache_creation_input_tokens: Number(bucket.cache_creation_input_tokens) || 0,
+            reasoning_output_tokens: Number(bucket.reasoning_output_tokens) || 0,
+            total_tokens: Number(bucket.total_tokens) || 0,
+            ...(typeof bucket.snapshot_verified_at === "string" && bucket.snapshot_verified_at
+              ? { snapshot_verified_at: bucket.snapshot_verified_at }
+              : {}),
+          };
+          const priorState = sessionStateMap.get(stateKey);
+          const verifiedMs = sessionSnapshotMs(bucket);
+          const priorVerifiedMs = sessionStateStamps.get(stateKey) ?? null;
+          if (!priorState || (verifiedMs !== null && (priorVerifiedMs === null || verifiedMs > priorVerifiedMs))) {
+            sessionStateMap.set(stateKey, sessionState);
+            sessionStateStamps.set(stateKey, verifiedMs);
+          }
+          // Counted like a bucket row: the record rides THIS batch and its
+          // bytes are covered by nextOffset, so the next batch resumes right
+          // after it.
+          linesRead += 1;
+          if (linesRead >= maxBuckets) break;
+        }
+        continue;
+      }
+      // Pre-release watermark records from this feature branch's dev queues
+      // are obsolete control records: drop them instead of letting them enter
+      // the bucket stream (they carry no hour_start and would be skipped
+      // downstream anyway, but be explicit).
+      if (bucket?.kind === "account_sync_watermark") continue;
+      const hourStart = typeof bucket?.hour_start === "string" ? bucket.hour_start : null;
+      if (!hourStart) continue;
+      const source = (typeof bucket?.source === "string" ? bucket.source.trim().toLowerCase() : "") || "codex";
+      const model = (typeof bucket?.model === "string" ? bucket.model.trim() : "") || "unknown";
+      bucket.source = source;
+      bucket.model = model;
+      // Apply the same legacy-row corrections every local reader applies
+      // (local-api readQueueData / project queue / wrapped aggregator). Without
+      // this the cloud permanently kept the RAW legacy values — e.g. old Codex
+      // rows whose input_tokens still include cached tokens (6-7x inflated) —
+      // while the local dashboard showed corrected numbers.
+      bucket = require("../lib/local-api").normalizeQueueRow(bucket);
+      bucketMap.set(`${source}|${model}|${hourStart}`, bucket);
+      linesRead += 1;
+      if (linesRead >= maxBuckets) break;
     }
-    // Pre-release watermark records from this feature branch's dev queues
-    // are obsolete control records: drop them instead of letting them enter
-    // the bucket stream (they carry no hour_start and would be skipped
-    // downstream anyway, but be explicit).
-    if (bucket?.kind === "account_sync_watermark") continue;
-    const hourStart = typeof bucket?.hour_start === "string" ? bucket.hour_start : null;
-    if (!hourStart) continue;
-    const source = (typeof bucket?.source === "string" ? bucket.source.trim().toLowerCase() : "") || "codex";
-    const model = (typeof bucket?.model === "string" ? bucket.model.trim() : "") || "unknown";
-    bucket.source = source;
-    bucket.model = model;
-    // Apply the same legacy-row corrections every local reader applies
-    // (local-api readQueueData / project queue / wrapped aggregator). Without
-    // this the cloud permanently kept the RAW legacy values — e.g. old Codex
-    // rows whose input_tokens still include cached tokens (6-7x inflated) —
-    // while the local dashboard showed corrected numbers.
-    bucket = require("../lib/local-api").normalizeQueueRow(bucket);
-    bucketMap.set(`${source}|${model}|${hourStart}`, bucket);
-    linesRead += 1;
-    if (linesRead >= maxBuckets) break;
-  }
-
-  rl.close();
-  stream.close?.();
+
+  } finally {
+    await handle.close();
+  }
   return {
     buckets: Array.from(bucketMap.values()),
     sessionStates: Array.from(sessionStateMap.values()),

package/src/lib/dumate.js

@@ -0,0 +1,95 @@
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { expandHome } = require("./scan-roots");
+const { readOpencodeDbMessages, parseOpencodeDbIncremental, normalizeOpencodeTokens } = require("./rollout");
+
+function resolveDumateDbPaths({ home = os.homedir(), env = process.env, platform = process.platform } = {}) {
+  const override = expandHome(env.TOKENTRACKER_DUMATE_HOME, home);
+  const appdata = typeof env.APPDATA === "string" && env.APPDATA.trim()
+    ? env.APPDATA.trim() : path.join(home, "AppData", "Roaming");
+  if (!override && platform !== "win32") return [];
+  const root = override || path.join(appdata, "qianfan-desktop-app", "qianfan_desk_xdg");
+  // The override accepts the data/opencode directory or an install's XDG root.
+  const candidates = [path.join(root, "opencode.db"), path.join(root, "data", "opencode", "opencode.db")];
+  try {
+    for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+      if (entry.isDirectory()) candidates.push(path.join(root, entry.name, "data", "opencode", "opencode.db"));
+    }
+  } catch (_e) { }
+  const seen = new Set();
+  return candidates.sort().filter((candidate) => {
+    try {
+      if (!fs.statSync(candidate).isFile()) return false;
+      const realPath = fs.realpathSync(candidate);
+      if (seen.has(realPath)) return false;
+      seen.add(realPath);
+      return true;
+    } catch (_e) { return false; }
+  });
+}
+
+async function parseDumateIncremental({ dbPaths, ...options }) {
+  // Reinstalls can copy the same sessions into a new install-id directory.
+  // Share a message cursor across installations and choose the newest copy
+  // before parsing, so stale copies cannot roll back a completed turn.
+  const messages = new Map();
+  const includeZeroMessage = (message) => {
+    const provider = message.data?.providerID || message.data?.model?.providerID;
+    return String(provider || "").toLowerCase() === "qianfanpersonalquota";
+  };
+  for (const dbPath of dbPaths) {
+    for (const message of readOpencodeDbMessages(dbPath, {}, { propagateReadFailure: true, includeZeroMessage })) {
+      // DuMate 1.0.83 bundles OpenCode's import command without an import
+      // marker. Its own runtime uses QianfanPersonalQuota; other providers
+      // can be imported history already counted under Claude or OpenCode.
+      const provider = message.data?.providerID || message.data?.model?.providerID;
+      if (String(provider || "").toLowerCase() !== "qianfanpersonalquota") continue;
+      const id = message.data?.id || message.id;
+      const session = message.data?.sessionID || message.sessionID || "";
+      if (!id) continue;
+      const key = `${session}|${id}`;
+      if (Number(message.timeUpdated || 0) < Number(options.cursors?.dumate?.messages?.[key]?.sourceUpdatedAt || 0)) continue;
+      const previous = messages.get(key);
+      if (!previous || Number(message.timeUpdated || 0) > Number(previous.message.timeUpdated || 0)) {
+        messages.set(key, { message, dbPath });
+      }
+    }
+  }
+  const byDatabase = new Map();
+  for (const [key, { message, dbPath }] of messages) {
+    // DuMate's OpenAI-compatible adapter retains reasoning inside output.
+    // The shared parser expects separate columns, so subtract that subset once.
+    const usage = normalizeOpencodeTokens(message.data?.tokens);
+    // Zero copies participate in latest-copy selection so a stale install
+    // cannot resurrect usage. Only a previously counted message needs a
+    // queued retraction; a new empty message contributes nothing.
+    const previous = options.cursors?.dumate?.messages?.[key];
+    if (usage?.total_tokens === 0 && (!previous?.lastTotals || previous.dedupedForkCopy === true)) continue;
+    if (!byDatabase.has(dbPath)) byDatabase.set(dbPath, []);
+    const normalized = usage ? {
+      ...message,
+      data: { ...message.data, tokens: { ...message.data.tokens,
+        output: Math.max(0, usage.output_tokens - usage.reasoning_output_tokens),
+      } },
+    } : message;
+    byDatabase.get(dbPath).push(normalized);
+  }
+  const result = { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  for (const [dbPath, dbMessages] of byDatabase) {
+    const parsed = await parseOpencodeDbIncremental({
+      ...options, dbPath, dbMessages, source: "dumate", cursorKey: "dumate",
+    });
+    for (const message of dbMessages) {
+      const key = `${message.data?.sessionID || message.sessionID || ""}|${message.data?.id || message.id}`;
+      const cursor = options.cursors?.dumate?.messages?.[key];
+      if (cursor) cursor.sourceUpdatedAt = Number(message.timeUpdated || 0);
+    }
+    result.recordsProcessed += parsed.messagesProcessed || 0;
+    result.eventsAggregated += parsed.eventsAggregated || 0;
+    result.bucketsQueued += parsed.bucketsQueued || 0;
+  }
+  return result;
+}
+
+module.exports = { resolveDumateDbPaths, parseDumateIncremental };