tokentracker-cli 0.94.1 → 0.95.0

package/src/lib/rollout.js

@@ -4900,6 +4900,42 @@ function isZcodeNativeMessage(data) {
   );
 }
 
+// ZCode persists inclusive parent counters in both its legacy OpenCode tables
+// and the newer model_usage table: cache read/write are already included in
+// input, and reasoning is already included in output. The shared OpenCode
+// parser expects disjoint columns, so split the subsets before it computes
+// queue totals and cost (issue #554).
+function normalizeZcodeInclusiveTokens(tokens) {
+  if (!tokens || typeof tokens !== "object") return tokens;
+  const rawInput = toNonNegativeInt(tokens.input);
+  const rawOutput = toNonNegativeInt(tokens.output);
+  const cacheRead = toNonNegativeInt(tokens.cache?.read);
+  const cacheWrite = toNonNegativeInt(tokens.cache?.write);
+  const reasoning = toNonNegativeInt(tokens.reasoning);
+  return {
+    ...tokens,
+    input: Math.max(0, rawInput - cacheRead - cacheWrite),
+    output: Math.max(0, rawOutput - reasoning),
+    reasoning,
+    cache: {
+      ...(tokens.cache && typeof tokens.cache === "object" ? tokens.cache : {}),
+      read: cacheRead,
+      write: cacheWrite,
+    },
+  };
+}
+
+function normalizeZcodeLegacyMessage(message) {
+  if (!message?.data?.tokens) return message;
+  return {
+    ...message,
+    data: {
+      ...message.data,
+      tokens: normalizeZcodeInclusiveTokens(message.data.tokens),
+    },
+  };
+}
+
 const ZCODE_NATIVE_USAGE_COLUMNS = new Set([
   "id",
   "logical_request_id",
@@ -5045,18 +5081,6 @@ function readZcodeNativeUsageMessages(dbPath, sqliteOptions = {}) {
     const startedAt = coerceEpochMs(row?.started_at);
     if (!providerID || !modelID || !sessionID || !id || !startedAt) continue;
 
-    // ZCode's persisted input/output columns are inclusive parents:
-    // cache read/write are subsets of input, and reasoning is a subset of
-    // output. TokenTracker stores those five columns disjointly, so split the
-    // subsets here before entering the shared OpenCode parser. This preserves
-    // sum(input + cache read + cache write + output + reasoning) without
-    // billing either subset twice.
-    const rawInput = toNonNegativeInt(row?.input_tokens);
-    const rawOutput = toNonNegativeInt(row?.output_tokens);
-    const cacheRead = toNonNegativeInt(row?.cache_read_input_tokens);
-    const cacheWrite = toNonNegativeInt(row?.cache_creation_input_tokens);
-    const reasoning = toNonNegativeInt(row?.reasoning_tokens);
-
     const data = {
       id,
       sessionID,
@@ -5064,15 +5088,15 @@ function readZcodeNativeUsageMessages(dbPath, sqliteOptions = {}) {
       providerID,
       modelID,
       time: { created: startedAt, completed: startedAt },
-      tokens: {
-        input: Math.max(0, rawInput - cacheRead - cacheWrite),
-        output: Math.max(0, rawOutput - reasoning),
-        reasoning,
+      tokens: normalizeZcodeInclusiveTokens({
+        input: row?.input_tokens,
+        output: row?.output_tokens,
+        reasoning: row?.reasoning_tokens,
         cache: {
-          read: cacheRead,
-          write: cacheWrite,
+          read: row?.cache_read_input_tokens,
+          write: row?.cache_creation_input_tokens,
         },
-      },
+      }),
     };
     if (typeof row?.directory === "string" && row.directory.trim()) {
       data.path = { cwd: row.directory.trim() };
@@ -5085,12 +5109,31 @@ function readZcodeNativeUsageMessages(dbPath, sqliteOptions = {}) {
 
 // Read only genuine ZCode assistant messages (its own GLM models via Z.ai /
 // BigModel), dropping any bundled sub-agent turns. See isZcodeNativeMessage.
+//
+// ZCode started writing model_usage after many installations had already
+// accumulated months of history in the OpenCode message tables. The native
+// table is authoritative from its first completed row onward, but it is not a
+// historical backfill. Keep legacy rows before that boundary so merely adding
+// the new table cannot make older usage disappear from TokenTracker.
 function readZcodeDbMessages(dbPath, sqliteOptions = {}) {
   if (!dbPath || !fssync.existsSync(dbPath)) return [];
   const nativeMessages = readZcodeNativeUsageMessages(dbPath, sqliteOptions);
-  if (nativeMessages !== null) return nativeMessages;
-  const all = readOpencodeDbMessages(dbPath, sqliteOptions);
-  return all.filter((m) => isZcodeNativeMessage(m.data));
+  const legacyMessages = readOpencodeDbMessages(dbPath, sqliteOptions)
+    .filter((message) => isZcodeNativeMessage(message.data))
+    .map(normalizeZcodeLegacyMessage);
+  if (nativeMessages === null) return legacyMessages;
+
+  const nativeStartMs = nativeMessages.reduce((earliest, message) => {
+    const timestampMs = coerceEpochMs(message?.timeUpdated);
+    return timestampMs > 0 ? Math.min(earliest, timestampMs) : earliest;
+  }, Number.POSITIVE_INFINITY);
+  if (!Number.isFinite(nativeStartMs)) return legacyMessages;
+
+  const historicalMessages = legacyMessages.filter((message) => {
+    const timestampMs = coerceEpochMs(message?.timeUpdated);
+    return timestampMs > 0 && timestampMs < nativeStartMs;
+  });
+  return [...historicalMessages, ...nativeMessages];
 }
 
 async function parseOpencodeDbIncremental({
@@ -11160,6 +11203,955 @@ async function parseZedIncremental({
   return { recordsProcessed, eventsAggregated, bucketsQueued };
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// LM Studio
+//
+// Data: pretty-printed OpenAI-compatible final responses beneath
+// `~/.lmstudio/server-logs/`. The reader recognizes both Chat Completions and
+// Responses API usage shapes. It scans only response identity, model, timestamp,
+// and the balanced `usage` object; prompt and response bodies are never parsed
+// or persisted.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const LMSTUDIO_SOURCE = "lmstudio";
+const LMSTUDIO_WINDOW_BYTES = 8 * 1024 * 1024;
+const LMSTUDIO_CHUNK_BYTES = 64 * 1024;
+const LMSTUDIO_IDENTITY_OVERLAP_BYTES = 512;
+const LMSTUDIO_MESSAGE_LIMIT = 10_000;
+const LMSTUDIO_USAGE_MARKER = Buffer.from('"usage"');
+
+function resolveLmstudioHome(env = process.env) {
+  const override = typeof env.TOKENTRACKER_LMSTUDIO_HOME === "string"
+    ? env.TOKENTRACKER_LMSTUDIO_HOME.trim()
+    : "";
+  if (override) return path.resolve(override);
+  const nativeHome = typeof env.LM_STUDIO_HOME === "string"
+    ? env.LM_STUDIO_HOME.trim()
+    : "";
+  if (nativeHome) return path.resolve(nativeHome);
+  return path.join(env.HOME || env.USERPROFILE || os.homedir(), ".lmstudio");
+}
+
+async function resolveLmstudioLogFiles(env = process.env) {
+  const root = path.join(resolveLmstudioHome(env), "server-logs");
+  const files = [];
+  const pending = [root];
+  while (pending.length > 0) {
+    const dir = pending.pop();
+    let entries;
+    try {
+      entries = await fs.readdir(dir, { withFileTypes: true });
+    } catch (_e) {
+      continue;
+    }
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) pending.push(fullPath);
+      else if (entry.isFile() && entry.name.toLowerCase().endsWith(".log")) files.push(fullPath);
+    }
+  }
+  return files.sort((a, b) => a.localeCompare(b));
+}
+
+function skipLmstudioWhitespace(buffer, index) {
+  while (index < buffer.length && /\s/.test(String.fromCharCode(buffer[index]))) index += 1;
+  return index;
+}
+
+function scanLmstudioUsageObjectStart(buffer, from = 0) {
+  let cursor = Math.max(0, from);
+  while (cursor < buffer.length) {
+    const marker = buffer.indexOf(LMSTUDIO_USAGE_MARKER, cursor);
+    if (marker < 0) break;
+    cursor = marker + LMSTUDIO_USAGE_MARKER.length;
+    if (marker > 0 && buffer[marker - 1] === 0x5c) continue;
+    const colon = skipLmstudioWhitespace(buffer, cursor);
+    if (colon >= buffer.length) return { found: null, certainTo: marker };
+    if (buffer[colon] !== 0x3a) continue;
+    const brace = skipLmstudioWhitespace(buffer, colon + 1);
+    if (brace >= buffer.length) return { found: null, certainTo: marker };
+    if (buffer[brace] === 0x7b) return { found: { marker, objectStart: brace }, certainTo: marker };
+  }
+  return {
+    found: null,
+    certainTo: Math.max(0, buffer.length - Math.max(0, LMSTUDIO_USAGE_MARKER.length - 1)),
+  };
+}
+
+function lmstudioBalancedObjectEnd(buffer, start) {
+  if (buffer[start] !== 0x7b) return -1;
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let index = start; index < buffer.length; index++) {
+    const byte = buffer[index];
+    if (inString) {
+      if (escaped) escaped = false;
+      else if (byte === 0x5c) escaped = true;
+      else if (byte === 0x22) inString = false;
+      continue;
+    }
+    if (byte === 0x22) inString = true;
+    else if (byte === 0x7b) depth += 1;
+    else if (byte === 0x7d) {
+      depth -= 1;
+      if (depth === 0) return index + 1;
+    }
+  }
+  return -1;
+}
+
+function lmstudioJsonStringEnd(buffer, start) {
+  if (buffer[start] !== 0x22) return -1;
+  let escaped = false;
+  for (let index = start + 1; index < buffer.length; index++) {
+    const byte = buffer[index];
+    if (escaped) escaped = false;
+    else if (byte === 0x5c) escaped = true;
+    else if (byte === 0x22) return index + 1;
+  }
+  return -1;
+}
+
+function lastLmstudioJsonStringField(buffer, field) {
+  const markerBuffer = Buffer.from(`"${field}"`);
+  let cursor = 0;
+  let found = null;
+  while (cursor < buffer.length) {
+    const index = buffer.indexOf(markerBuffer, cursor);
+    if (index < 0) break;
+    cursor = index + markerBuffer.length;
+    if (index > 0 && buffer[index - 1] === 0x5c) continue;
+    let valueStart = skipLmstudioWhitespace(buffer, cursor);
+    if (buffer[valueStart] !== 0x3a) continue;
+    valueStart = skipLmstudioWhitespace(buffer, valueStart + 1);
+    const valueEnd = lmstudioJsonStringEnd(buffer, valueStart);
+    if (valueEnd < 0) continue;
+    try {
+      const value = JSON.parse(buffer.subarray(valueStart, valueEnd).toString("utf8"));
+      if (typeof value === "string") found = value;
+    } catch (_e) { }
+  }
+  return found;
+}
+
+function lastLmstudioTimestamp(buffer) {
+  const text = buffer.toString("utf8");
+  let timestamp = null;
+  for (const rawLine of text.split("\n")) {
+    const line = rawLine.startsWith("\r") ? rawLine.slice(1) : rawLine;
+    const match = /^\[(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\]\[/.exec(line);
+    if (!match) continue;
+    const date = new Date(
+      Number(match[1]),
+      Number(match[2]) - 1,
+      Number(match[3]),
+      Number(match[4]),
+      Number(match[5]),
+      Number(match[6]),
+    );
+    if (!Number.isNaN(date.getTime())) timestamp = date.getTime();
+  }
+  return timestamp;
+}
+
+function normalizeLocalStudioTokens(usage) {
+  if (!usage || typeof usage !== "object") return null;
+  const prompt = toNonNegativeInt(
+    usage.prompt_tokens ?? usage.promptTokens ?? usage.input_tokens ?? usage.inputTokens,
+  );
+  const completion = toNonNegativeInt(
+    usage.completion_tokens ?? usage.completionTokens ?? usage.output_tokens ?? usage.outputTokens,
+  );
+  const total = Math.max(
+    toNonNegativeInt(usage.total_tokens ?? usage.totalTokens),
+    prompt + completion,
+  );
+  if (total <= 0) return null;
+  const promptDetails = usage.prompt_tokens_details
+    || usage.input_tokens_details
+    || usage.inputTokensDetails
+    || {};
+  const outputDetails = usage.completion_tokens_details
+    || usage.output_tokens_details
+    || usage.completionTokensDetails
+    || usage.outputTokensDetails
+    || {};
+  const cacheRead = Math.min(
+    prompt,
+    Math.max(
+      toNonNegativeInt(promptDetails.cached_tokens ?? promptDetails.cache_read_tokens),
+      toNonNegativeInt(usage.cached_tokens),
+    ),
+  );
+  const cacheWrite = Math.min(
+    Math.max(0, prompt - cacheRead),
+    Math.max(
+      toNonNegativeInt(
+        promptDetails.cache_creation_input_tokens ?? promptDetails.cache_write_tokens,
+      ),
+      toNonNegativeInt(usage.cache_creation_input_tokens),
+    ),
+  );
+  const reasoning = Math.min(
+    completion,
+    Math.max(
+      toNonNegativeInt(outputDetails.reasoning_tokens),
+      toNonNegativeInt(usage.reasoning_tokens),
+    ),
+  );
+  return {
+    input_tokens: Math.max(0, total - completion - cacheRead - cacheWrite),
+    cached_input_tokens: cacheRead,
+    cache_creation_input_tokens: cacheWrite,
+    output_tokens: Math.max(0, completion - reasoning),
+    reasoning_output_tokens: reasoning,
+    total_tokens: total,
+    billable_total_tokens: total,
+    total_cost_usd: 0,
+    conversation_count: 1,
+  };
+}
+
+function lmstudioResponseId(buffer) {
+  const id = lastLmstudioJsonStringField(buffer, "id");
+  return typeof id === "string" && ["chatcmpl-", "cmpl-", "resp_"].some((prefix) => id.startsWith(prefix))
+    ? id
+    : null;
+}
+
+function absorbLmstudioIdentity(identity, buffer) {
+  const responseId = lmstudioResponseId(buffer);
+  const model = lastLmstudioJsonStringField(buffer, "model");
+  const timestamp = lastLmstudioTimestamp(buffer);
+  if (responseId) identity.responseId = responseId;
+  if (model && model.trim()) identity.model = model.trim();
+  if (timestamp) identity.timestamp = timestamp;
+}
+
+function normalizeLmstudioResumeIdentity(value) {
+  const identity = {};
+  const responseId = typeof value?.responseId === "string" ? value.responseId : "";
+  if (["chatcmpl-", "cmpl-", "resp_"].some((prefix) => responseId.startsWith(prefix))) {
+    identity.responseId = responseId;
+  }
+  const model = typeof value?.model === "string" ? value.model.trim() : "";
+  if (model) identity.model = model;
+  const timestamp = Number(value?.timestamp);
+  if (Number.isFinite(timestamp) && timestamp > 0) identity.timestamp = timestamp;
+  return identity;
+}
+
+function lmstudioRecordFromSlices({ usageBuffer, metadataBuffer, carried, filePath, marker, fallbackTimestamp }) {
+  let usage;
+  try {
+    usage = JSON.parse(usageBuffer.toString("utf8"));
+  } catch (_e) {
+    return null;
+  }
+  const totals = normalizeLocalStudioTokens(usage);
+  if (!totals) return null;
+  const responseId = lmstudioResponseId(metadataBuffer) || carried.responseId || null;
+  const model = normalizeModelInput(
+    lastLmstudioJsonStringField(metadataBuffer, "model") || carried.model,
+  ) || DEFAULT_MODEL;
+  const timestampMs = lastLmstudioTimestamp(metadataBuffer) || carried.timestamp || fallbackTimestamp;
+  const bucketStart = timestampMs
+    ? toUtcHalfHourStart(new Date(timestampMs).toISOString())
+    : null;
+  if (!bucketStart) return null;
+  const fallback = crypto.createHash("sha256")
+    .update(filePath)
+    .update("\0")
+    .update(String(marker))
+    .update("\0")
+    .update(model)
+    .update("\0")
+    .update(totalsKey(totals))
+    .digest("base64url");
+  return {
+    key: responseId ? `lmstudio:${responseId}` : `lmstudio:${fallback}`,
+    model,
+    bucketStart,
+    totals,
+    marker,
+  };
+}
+
+async function readLmstudioFileRecords(
+  filePath,
+  {
+    windowBytes = LMSTUDIO_WINDOW_BYTES,
+    chunkBytes = LMSTUDIO_CHUNK_BYTES,
+    startOffset = 0,
+    resumeIdentity,
+  } = {},
+) {
+  const handle = await fs.open(filePath, "r");
+  try {
+    const initialStat = await handle.stat();
+    const fallbackTimestamp = initialStat.mtimeMs;
+    const records = [];
+    const initialOffset = Math.min(
+      initialStat.size,
+      Math.max(0, Number.isFinite(startOffset) ? Math.floor(startOffset) : 0),
+    );
+    let window = Buffer.alloc(0);
+    let windowStart = initialOffset;
+    let metadataStart = initialOffset;
+    let scannedTo = initialOffset;
+    let position = initialOffset;
+    let carried = normalizeLmstudioResumeIdentity(resumeIdentity);
+    const chunk = Buffer.alloc(Math.max(1, chunkBytes));
+
+    while (true) {
+      const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
+      if (bytesRead > 0) {
+        window = Buffer.concat([window, chunk.subarray(0, bytesRead)]);
+        position += bytesRead;
+      }
+      const atEof = bytesRead === 0;
+
+      while (true) {
+        const scanFrom = Math.min(window.length, Math.max(0, scannedTo - windowStart));
+        const scan = scanLmstudioUsageObjectStart(window, scanFrom);
+        if (!scan.found) {
+          scannedTo = windowStart + scan.certainTo;
+          break;
+        }
+        const objectEnd = lmstudioBalancedObjectEnd(window, scan.found.objectStart);
+        if (objectEnd < 0) {
+          scannedTo = atEof
+            ? windowStart + window.length
+            : windowStart + scan.found.marker;
+          break;
+        }
+        scannedTo = windowStart + objectEnd;
+        const absoluteMarker = windowStart + scan.found.marker;
+        const absoluteEnd = windowStart + objectEnd;
+        const metadataFrom = Math.min(
+          scan.found.marker,
+          Math.max(0, metadataStart - windowStart),
+        );
+        const record = lmstudioRecordFromSlices({
+          usageBuffer: window.subarray(scan.found.objectStart, objectEnd),
+          metadataBuffer: window.subarray(metadataFrom, scan.found.marker),
+          carried,
+          filePath,
+          marker: absoluteMarker,
+          fallbackTimestamp,
+        });
+        if (record) records.push(record);
+        metadataStart = absoluteEnd;
+        carried = {};
+      }
+
+      const keepFrom = Math.min(window.length, Math.max(0, metadataStart - windowStart));
+      if (keepFrom > 0) {
+        window = window.subarray(keepFrom);
+        windowStart += keepFrom;
+      }
+
+      if (window.length > windowBytes) {
+        const overflow = window.length - windowBytes;
+        const absorbTo = Math.min(
+          window.length,
+          overflow + LMSTUDIO_IDENTITY_OVERLAP_BYTES,
+        );
+        absorbLmstudioIdentity(carried, window.subarray(0, absorbTo));
+        window = window.subarray(overflow);
+        windowStart += overflow;
+        metadataStart = Math.max(metadataStart, windowStart);
+        scannedTo = Math.max(scannedTo, windowStart);
+      }
+
+      if (atEof) {
+        const finalStat = await handle.stat().catch(() => initialStat);
+        const nextIdentity = normalizeLmstudioResumeIdentity(carried);
+        return {
+          records,
+          stat: {
+            dev: finalStat.dev,
+            ino: finalStat.ino,
+            size: finalStat.size,
+            mtimeMs: finalStat.size === position ? finalStat.mtimeMs : -1,
+            resumeOffset: Math.max(metadataStart, position - windowBytes),
+            ...(Object.keys(nextIdentity).length > 0
+              ? { resumeIdentity: nextIdentity }
+              : {}),
+          },
+        };
+      }
+    }
+  } finally {
+    await handle.close().catch(() => {});
+  }
+}
+
+function retainNewestPassiveMessages(messages, maxEntries) {
+  const limit = Number.isInteger(maxEntries) && maxEntries > 0
+    ? maxEntries
+    : LMSTUDIO_MESSAGE_LIMIT;
+  const entries = Object.entries(messages);
+  if (entries.length <= limit) return messages;
+  entries.sort((left, right) => {
+    const leftTime = Date.parse(left[1]?.updatedAt || left[1]?.bucketStart || "") || 0;
+    const rightTime = Date.parse(right[1]?.updatedAt || right[1]?.bucketStart || "") || 0;
+    if (leftTime !== rightTime) return rightTime - leftTime;
+    return right[0].localeCompare(left[0]);
+  });
+  return Object.fromEntries(entries.slice(0, limit));
+}
+
+function reconcilePassiveUsageEvent({ event, source, messages, hourlyState, touchedBuckets }) {
+  const previous = messages[event.key];
+  const unchanged = previous
+    && previous.model === event.model
+    && previous.bucketStart === event.bucketStart
+    && totalsKey(previous.totals) === totalsKey(event.totals);
+  if (unchanged) return false;
+
+  if (previous?.model && previous?.bucketStart && previous?.totals) {
+    const oldBucket = getHourlyBucket(hourlyState, source, previous.model, previous.bucketStart);
+    subtractTotals(oldBucket.totals, previous.totals);
+    touchedBuckets.add(bucketKey(source, previous.model, previous.bucketStart));
+  }
+  const bucket = getHourlyBucket(hourlyState, source, event.model, event.bucketStart);
+  addTotals(bucket.totals, event.totals);
+  touchedBuckets.add(bucketKey(source, event.model, event.bucketStart));
+  messages[event.key] = {
+    model: event.model,
+    bucketStart: event.bucketStart,
+    totals: event.totals,
+    updatedAt: new Date().toISOString(),
+  };
+  return true;
+}
+
+async function parseLmstudioIncremental({
+  logFiles,
+  cursors,
+  queuePath,
+  onProgress,
+  messageLimit = LMSTUDIO_MESSAGE_LIMIT,
+  readerOptions,
+} = {}) {
+  await ensureDir(path.dirname(queuePath));
+  const files = Array.isArray(logFiles) ? [...logFiles].sort((a, b) => a.localeCompare(b)) : [];
+  const priorState = cursors.lmstudio && typeof cursors.lmstudio === "object"
+    ? cursors.lmstudio
+    : {};
+  let fileState = priorState.files && typeof priorState.files === "object"
+    ? { ...priorState.files }
+    : {};
+  let messages = priorState.messages && typeof priorState.messages === "object"
+    ? priorState.messages
+    : {};
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const touchedBuckets = new Set();
+  const presentFiles = new Set(files);
+  const cb = typeof onProgress === "function" ? onProgress : null;
+  const effectiveMessageLimit = Number.isInteger(messageLimit) && messageLimit > 0
+    ? messageLimit
+    : LMSTUDIO_MESSAGE_LIMIT;
+  let recordsProcessed = 0;
+  let eventsAggregated = 0;
+  let stateChanged = false;
+  const readOptions = readerOptions && typeof readerOptions === "object" ? readerOptions : {};
+  const filePlans = [];
+  let allFilesReadable = true;
+
+  for (const filePath of files) {
+    const previousFile = fileState[filePath];
+    let stat;
+    try {
+      stat = await fs.stat(filePath);
+    } catch (_e) {
+      allFilesReadable = false;
+      continue;
+    }
+    const unchanged = Boolean(previousFile
+      && previousFile.dev === stat.dev
+      && previousFile.ino === stat.ino
+      && previousFile.size === stat.size
+      && previousFile.mtimeMs === stat.mtimeMs
+      && Number.isFinite(previousFile.resumeOffset));
+    const canResume = Boolean(previousFile
+      && previousFile.dev === stat.dev
+      && previousFile.ino === stat.ino
+      && (stat.size > previousFile.size || previousFile.mtimeMs === -1)
+      && Number.isFinite(previousFile.resumeOffset)
+      && previousFile.resumeOffset >= 0
+      && previousFile.resumeOffset <= stat.size);
+    filePlans.push({ filePath, previousFile, unchanged, canResume });
+  }
+
+  const requiresRebuild = filePlans.some(
+    ({ previousFile, unchanged, canResume }) => previousFile && !unchanged && !canResume,
+  );
+  if (requiresRebuild) {
+    // Bounded message retention cannot safely deduplicate a replay from byte
+    // zero. Rebuild every current LM Studio log in isolation, then replace only
+    // this source's buckets while preserving their queue fingerprints.
+    if (!allFilesReadable || filePlans.length !== files.length) {
+      return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+    }
+    const rebuiltHourly = normalizeHourlyState(null);
+    const rebuiltMessages = {};
+    const rebuiltFiles = {};
+    const rebuiltTouched = new Set();
+    let rebuiltRecords = 0;
+    let rebuiltEvents = 0;
+    try {
+      for (let index = 0; index < filePlans.length; index++) {
+        const { filePath } = filePlans[index];
+        const parsed = await readLmstudioFileRecords(filePath, {
+          ...readOptions,
+          startOffset: 0,
+          resumeIdentity: undefined,
+        });
+        for (const event of parsed.records) {
+          if (reconcilePassiveUsageEvent({
+            event,
+            source: LMSTUDIO_SOURCE,
+            messages: rebuiltMessages,
+            hourlyState: rebuiltHourly,
+            touchedBuckets: rebuiltTouched,
+          })) rebuiltEvents += 1;
+        }
+        rebuiltFiles[filePath] = { ...parsed.stat, updatedAt: new Date().toISOString() };
+        rebuiltRecords += 1;
+        if (cb) {
+          cb({
+            index: index + 1,
+            total: files.length,
+            recordsProcessed: rebuiltRecords,
+            eventsAggregated: rebuiltEvents,
+            bucketsQueued: rebuiltTouched.size,
+          });
+        }
+      }
+    } catch (error) {
+      if (process.env.TOKENTRACKER_DEBUG) {
+        process.stderr.write(`[lmstudio] rebuild deferred: ${error?.message || error}\n`);
+      }
+      return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+    }
+
+    for (const [key, bucket] of Object.entries(hourlyState.buckets || {})) {
+      if (parseBucketKey(key).source !== LMSTUDIO_SOURCE || !bucket?.totals) continue;
+      bucket.totals = initTotals();
+      touchedBuckets.add(key);
+    }
+    for (const [key, bucket] of Object.entries(rebuiltHourly.buckets || {})) {
+      const current = hourlyState.buckets[key];
+      if (current && typeof current === "object") current.totals = cloneTotals(bucket.totals);
+      else hourlyState.buckets[key] = bucket;
+      touchedBuckets.add(key);
+    }
+    fileState = rebuiltFiles;
+    messages = rebuiltMessages;
+    recordsProcessed = rebuiltRecords;
+    eventsAggregated = rebuiltEvents;
+    stateChanged = true;
+  } else {
+    for (let index = 0; index < filePlans.length; index++) {
+      const { filePath, previousFile, unchanged, canResume } = filePlans[index];
+      if (!unchanged) {
+        try {
+          const parsed = await readLmstudioFileRecords(filePath, {
+            ...readOptions,
+            startOffset: canResume ? previousFile.resumeOffset : 0,
+            resumeIdentity: canResume ? previousFile.resumeIdentity : undefined,
+          });
+          for (const event of parsed.records) {
+            if (reconcilePassiveUsageEvent({
+              event,
+              source: LMSTUDIO_SOURCE,
+              messages,
+              hourlyState,
+              touchedBuckets,
+            })) eventsAggregated += 1;
+          }
+          fileState[filePath] = { ...parsed.stat, updatedAt: new Date().toISOString() };
+          recordsProcessed += 1;
+          stateChanged = true;
+        } catch (error) {
+          if (process.env.TOKENTRACKER_DEBUG) {
+            process.stderr.write(`[lmstudio] skipped ${filePath}: ${error?.message || error}\n`);
+          }
+        }
+      }
+      if (cb) {
+        cb({
+          index: index + 1,
+          total: files.length,
+          recordsProcessed,
+          eventsAggregated,
+          bucketsQueued: touchedBuckets.size,
+        });
+      }
+    }
+  }
+
+  for (const filePath of Object.keys(fileState)) {
+    if (!presentFiles.has(filePath)) {
+      delete fileState[filePath];
+      stateChanged = true;
+    }
+  }
+  if (
+    !stateChanged
+    && touchedBuckets.size === 0
+    && Object.keys(messages).length <= effectiveMessageLimit
+  ) {
+    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  }
+  const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
+  const updatedAt = new Date().toISOString();
+  hourlyState.updatedAt = updatedAt;
+  cursors.hourly = hourlyState;
+  cursors.lmstudio = {
+    files: fileState,
+    messages: retainNewestPassiveMessages(messages, effectiveMessageLimit),
+    updatedAt,
+  };
+  return { recordsProcessed, eventsAggregated, bucketsQueued };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Unsloth Studio
+//
+// Durable inference usage lives in `studio.db`. The SQL projections below
+// extract only stable IDs, timestamps, model identifiers, and scalar counters.
+// Message content, attachments, API subjects, credentials, and training data
+// never leave SQLite.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const UNSLOTH_SOURCE = "unsloth";
+const UNSLOTH_SQL_PAGE_SIZE = 500;
+const UNSLOTH_CURSOR_OVERLAP_ROWS = 256;
+const UNSLOTH_METERED_PROVIDER_TYPES = new Set([
+  "anthropic",
+  "deepseek",
+  "gemini",
+  "huggingface",
+  "kimi",
+  "mistral",
+  "openai",
+  "openrouter",
+  "qwen",
+]);
+
+function resolveUnslothDbPath(env = process.env) {
+  const override = typeof env.TOKENTRACKER_UNSLOTH_DB === "string"
+    ? env.TOKENTRACKER_UNSLOTH_DB.trim()
+    : "";
+  if (override) return path.resolve(override);
+  const studioHome = typeof env.UNSLOTH_STUDIO_HOME === "string"
+    ? env.UNSLOTH_STUDIO_HOME.trim()
+    : "";
+  if (studioHome) return path.join(path.resolve(studioHome), "studio.db");
+  return path.join(env.HOME || env.USERPROFILE || os.homedir(), ".unsloth", "studio", "studio.db");
+}
+
+function unslothSqliteFingerprint(dbPath) {
+  const fingerprint = sqliteSidecarFingerprint(dbPath);
+  delete fingerprint["-shm"];
+  return fingerprint;
+}
+
+function unslothRowPosition(row) {
+  const createdAt = row?.created_at == null ? "" : String(row.created_at);
+  const id = row?.id == null ? "" : String(row.id);
+  return createdAt && id ? { createdAt, id } : null;
+}
+
+function unslothPositionClause(rowAlias, position, inclusive) {
+  if (!position?.createdAt || !position?.id) return null;
+  const createdAt = sqliteStringLiteral(position.createdAt);
+  const id = sqliteStringLiteral(position.id);
+  const idOperator = inclusive ? ">=" : ">";
+  return `(${rowAlias}.created_at > ${createdAt} OR ` +
+    `(${rowAlias}.created_at = ${createdAt} AND ${rowAlias}.id ${idOperator} ${id}))`;
+}
+
+function readUnslothRowsPaged({
+  dbPath,
+  select,
+  from,
+  where,
+  rowAlias,
+  lowerBound,
+  options,
+}) {
+  const requestedPageSize = Number(options?.pageSize);
+  const pageSize = Number.isInteger(requestedPageSize) && requestedPageSize > 0
+    ? Math.min(requestedPageSize, UNSLOTH_SQL_PAGE_SIZE)
+    : UNSLOTH_SQL_PAGE_SIZE;
+  const rows = [];
+  let after = null;
+
+  while (true) {
+    const clauses = [where, `${rowAlias}.created_at IS NOT NULL`, `${rowAlias}.id IS NOT NULL`];
+    const positionClause = unslothPositionClause(rowAlias, after || lowerBound, !after);
+    if (positionClause) clauses.push(positionClause);
+    const page = readSqliteJsonRows(dbPath, `
+      SELECT
+        ${select}
+      FROM ${from}
+      WHERE ${clauses.filter(Boolean).join(" AND ")}
+      ORDER BY ${rowAlias}.created_at, ${rowAlias}.id
+      LIMIT ${pageSize}
+    `.trim(), options);
+    rows.push(...page);
+    if (page.length < pageSize) break;
+
+    const next = unslothRowPosition(page[page.length - 1]);
+    if (!next || (
+      after && next.createdAt === after.createdAt && next.id === after.id
+    )) {
+      throw new Error("Unsloth Studio pagination did not advance");
+    }
+    after = next;
+  }
+
+  return rows;
+}
+
+function readUnslothUsageRows(dbPath, sqliteOptions = {}, scanState = {}) {
+  if (!dbPath || !fssync.existsSync(dbPath)) return [];
+  const options = {
+    label: "Unsloth Studio",
+    maxBuffer: 32 * 1024 * 1024,
+    timeout: 30_000,
+    readOnly: true,
+    throwOnReadFailure: true,
+    ...sqliteOptions,
+  };
+  const tables = new Set(
+    readSqliteJsonRows(
+      dbPath,
+      "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('chat_messages','chat_threads','api_usage_events')",
+      options,
+    ).map((row) => row?.name).filter(Boolean),
+  );
+  const rows = [];
+
+  if (tables.has("chat_messages")) {
+    const joinThread = tables.has("chat_threads")
+      ? "LEFT JOIN chat_threads t ON t.id = m.thread_id"
+      : "";
+    const threadModel = tables.has("chat_threads") ? "t.model_id" : "NULL";
+    const field = (jsonPath, alias) =>
+      `CASE WHEN json_valid(m.metadata_json) THEN json_extract(m.metadata_json, '${jsonPath}') ELSE NULL END AS ${alias}`;
+    // Re-read a bounded tail because Studio finalizes recent rows in place and
+    // does not expose an updated_at column. Older immutable history stays behind
+    // the persisted timestamp/ID overlap cursor.
+    const chatRows = readUnslothRowsPaged({
+      dbPath,
+      select: `
+        'chat' AS usage_kind,
+        m.id,
+        m.created_at,
+        ${field("$.responseDetails.responseModelId", "response_model")},
+        ${field("$.contextUsage.modelId", "requested_model")},
+        ${field("$.responseDetails.providerType", "provider_type")},
+        ${threadModel} AS fallback_model,
+        ${field("$.contextUsage.promptTokens", "prompt_tokens")},
+        ${field("$.contextUsage.completionTokens", "completion_tokens")},
+        ${field("$.contextUsage.totalTokens", "total_tokens")},
+        ${field("$.contextUsage.cachedTokens", "cached_tokens")},
+        ${field("$.contextUsage.cacheWriteTokens", "cache_write_tokens")},
+        ${field("$.contextUsage.reasoningTokens", "reasoning_tokens")}
+      `.trim(),
+      from: `chat_messages m ${joinThread}`,
+      where: "m.role = 'assistant'",
+      rowAlias: "m",
+      lowerBound: scanState?.chat?.overlap,
+      options,
+    });
+    rows.push(...chatRows);
+  }
+
+  if (tables.has("api_usage_events")) {
+    const columns = new Set(
+      readSqliteJsonRows(dbPath, "PRAGMA table_info(api_usage_events)", options)
+        .map((row) => row?.name)
+        .filter(Boolean),
+    );
+    const required = ["id", "model", "prompt_tokens", "completion_tokens", "total_tokens", "created_at"];
+    if (required.every((column) => columns.has(column))) {
+      rows.push(...readUnslothRowsPaged({
+        dbPath,
+        select: `
+          'api' AS usage_kind,
+          e.id,
+          e.created_at,
+          e.model AS response_model,
+          e.model AS requested_model,
+          'local' AS provider_type,
+          NULL AS fallback_model,
+          e.prompt_tokens,
+          e.completion_tokens,
+          e.total_tokens,
+          0 AS cached_tokens,
+          0 AS cache_write_tokens,
+          0 AS reasoning_tokens
+        `.trim(),
+        from: "api_usage_events e",
+        where: "",
+        rowAlias: "e",
+        lowerBound: scanState?.api?.overlap,
+        options,
+      }));
+    }
+  }
+  return rows;
+}
+
+function qualifyUnslothModel(row) {
+  const rawModel = normalizeModelInput(row?.response_model)
+    || normalizeModelInput(row?.requested_model)
+    || normalizeModelInput(row?.fallback_model)
+    || DEFAULT_MODEL;
+  const providerType = normalizeMessageKeyPart(row?.provider_type).toLowerCase();
+  if (row?.usage_kind === "api" || providerType === "local") {
+    return `local/${rawModel}`;
+  }
+  if (!UNSLOTH_METERED_PROVIDER_TYPES.has(providerType)) {
+    return `unpriced/${providerType || "unknown"}/${rawModel}`;
+  }
+  const lowerModel = rawModel.toLowerCase();
+  return lowerModel.startsWith(`${providerType}/`)
+    ? rawModel
+    : `${providerType}/${rawModel}`;
+}
+
+function normalizeUnslothUsageRow(row) {
+  const totals = normalizeLocalStudioTokens({
+    prompt_tokens: row?.prompt_tokens,
+    completion_tokens: row?.completion_tokens,
+    total_tokens: row?.total_tokens,
+    cached_tokens: row?.cached_tokens,
+    cache_creation_input_tokens: row?.cache_write_tokens,
+    reasoning_tokens: row?.reasoning_tokens,
+  });
+  const id = normalizeMessageKeyPart(row?.id == null ? "" : String(row.id));
+  const timestampMs = coerceEpochMs(row?.created_at) || parseIsoTimestampMs(row?.created_at);
+  const bucketStart = timestampMs
+    ? toUtcHalfHourStart(new Date(timestampMs).toISOString())
+    : null;
+  if (!id || !totals || !bucketStart) return null;
+  const kind = row?.usage_kind === "api" ? "api" : "chat";
+  return {
+    key: `unsloth:${kind}:${id}`,
+    model: qualifyUnslothModel(row),
+    bucketStart,
+    totals,
+  };
+}
+
+async function parseUnslothIncremental({
+  dbPath,
+  cursors,
+  queuePath,
+  onProgress,
+  env,
+  sqliteOptions,
+  overlapRows = UNSLOTH_CURSOR_OVERLAP_ROWS,
+} = {}) {
+  await ensureDir(path.dirname(queuePath));
+  const resolvedDb = dbPath || resolveUnslothDbPath(env || process.env);
+  const priorState = cursors.unsloth && typeof cursors.unsloth === "object"
+    ? cursors.unsloth
+    : {};
+  const messages = priorState.messages && typeof priorState.messages === "object"
+    ? priorState.messages
+    : {};
+  if (!resolvedDb || !fssync.existsSync(resolvedDb)) {
+    cursors.unsloth = { ...priorState, messages, updatedAt: new Date().toISOString() };
+    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  }
+
+  const initialFingerprint = unslothSqliteFingerprint(resolvedDb);
+  const hasBoundedScan = priorState.scan
+    && typeof priorState.scan === "object"
+    && Object.keys(messages).length <= UNSLOTH_CURSOR_OVERLAP_ROWS * 2;
+  if (hasBoundedScan && sameSqliteFingerprint(initialFingerprint, priorState.fingerprint)) {
+    return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
+  }
+
+  const rows = readUnslothUsageRows(resolvedDb, sqliteOptions, priorState.scan);
+  const hourlyState = normalizeHourlyState(cursors?.hourly);
+  const touchedBuckets = new Set();
+  const cb = typeof onProgress === "function" ? onProgress : null;
+  let recordsProcessed = 0;
+  let eventsAggregated = 0;
+  for (let index = 0; index < rows.length; index++) {
+    const event = normalizeUnslothUsageRow(rows[index]);
+    recordsProcessed += 1;
+    if (event && reconcilePassiveUsageEvent({
+      event,
+      source: UNSLOTH_SOURCE,
+      messages,
+      hourlyState,
+      touchedBuckets,
+    })) eventsAggregated += 1;
+    if (cb) {
+      cb({
+        index: index + 1,
+        total: rows.length,
+        recordsProcessed,
+        eventsAggregated,
+        bucketsQueued: touchedBuckets.size,
+      });
+    }
+  }
+
+  const retention = Number.isInteger(overlapRows) && overlapRows > 0
+    ? Math.min(overlapRows, UNSLOTH_CURSOR_OVERLAP_ROWS)
+    : UNSLOTH_CURSOR_OVERLAP_ROWS;
+  const nextScan = {};
+  const retainedMessageKeys = new Set();
+  for (const kind of ["chat", "api"]) {
+    const kindRows = rows.filter((row) => (row?.usage_kind === "api" ? "api" : "chat") === kind);
+    if (kindRows.length === 0) {
+      if (priorState.scan?.[kind]) nextScan[kind] = priorState.scan[kind];
+      for (const key of Object.keys(messages)) {
+        if (key.startsWith(`unsloth:${kind}:`)) retainedMessageKeys.add(key);
+      }
+      continue;
+    }
+    const tail = kindRows.slice(-retention);
+    const overlap = unslothRowPosition(tail[0]);
+    const latest = unslothRowPosition(kindRows[kindRows.length - 1]);
+    if (overlap && latest) nextScan[kind] = { overlap, latest };
+    for (const row of tail) {
+      const event = normalizeUnslothUsageRow(row);
+      if (event) retainedMessageKeys.add(event.key);
+    }
+  }
+  for (const key of Object.keys(messages)) {
+    if (key.startsWith("unsloth:") && !retainedMessageKeys.has(key)) delete messages[key];
+  }
+
+  const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
+  const finalFingerprint = unslothSqliteFingerprint(resolvedDb);
+  const updatedAt = new Date().toISOString();
+  hourlyState.updatedAt = updatedAt;
+  cursors.hourly = hourlyState;
+  cursors.unsloth = {
+    messages,
+    scan: nextScan,
+    fingerprint: sameSqliteFingerprint(initialFingerprint, finalFingerprint)
+      ? finalFingerprint
+      : initialFingerprint,
+    updatedAt,
+  };
+  return { recordsProcessed, eventsAggregated, bucketsQueued };
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // AnythingLLM Desktop (Mintplex Labs)
 //
@@ -14251,6 +15243,13 @@ async function migrateCopilotChatLogRecordDedup({
   touchedBuckets,
   seenIds,
 } = {}) {
+  // A v2 cursor can retain an offset for a rotated or deleted OTEL file that
+  // is no longer returned by discovery. Its historical contribution cannot be
+  // reconciled, but the stale offset must not block migration of new files.
+  const availableFiles = new Set(Array.isArray(files) ? files : []);
+  for (const filePath of Object.keys(fileOffsets || {})) {
+    if (!availableFiles.has(filePath)) delete fileOffsets[filePath];
+  }
   const trackedFiles = Object.keys(fileOffsets || {}).filter(
     (filePath) => Number(fileOffsets[filePath]?.size) > 0,
   );
@@ -14258,11 +15257,6 @@ async function migrateCopilotChatLogRecordDedup({
     return { applied: true, changed: false };
   }
 
-  const availableFiles = new Set(Array.isArray(files) ? files : []);
-  if (trackedFiles.some((filePath) => !availableFiles.has(filePath))) {
-    return { applied: false, reason: "a previously parsed OTEL file is unavailable" };
-  }
-
   const oldSeen = new Set();
   const newSeen = new Set();
   const oldTotals = new Map();
@@ -14342,7 +15336,10 @@ async function migrateCopilotChatLogRecordDedup({
   }
 
   const keys = new Set([...oldTotals.keys(), ...newTotals.keys()]);
-  for (const key of keys) {
+  // Only keys that existed in the v2 contribution need coverage validation.
+  // A key that exists only in newTotals is a newly recovered Chat request; its
+  // old contribution is zero, so an absent old bucket is expected.
+  for (const key of oldTotals.keys()) {
     const [model, bucketStart] = JSON.parse(key);
     const oldUsage = oldTotals.get(key) || initTotals();
     if (!copilotTotalsCover(
@@ -17417,6 +18414,7 @@ async function parseAntigravityFile({
       // Match the mainstream convention (Codebuddy / Kilocode / OMP / Hermes):
       // total_tokens = sum of every token column. No cache columns here.
       delta.total_tokens = inputDelta + outputTokens + reasoningTokens;
+      delta.billable_total_tokens = delta.total_tokens;
       delta.conversation_count = 1;
       billedPlanner = delta.total_tokens > 0;
     }
@@ -18955,6 +19953,15 @@ module.exports = {
   sumZedRequestUsage,
   readZedUsage,
   parseZedIncremental,
+  resolveLmstudioHome,
+  resolveLmstudioLogFiles,
+  normalizeLocalStudioTokens,
+  readLmstudioFileRecords,
+  parseLmstudioIncremental,
+  resolveUnslothDbPath,
+  readUnslothUsageRows,
+  normalizeUnslothUsageRow,
+  parseUnslothIncremental,
   resolveAnythingllmDbPath,
   parseAnythingllmTimestamp,
   readAnythingllmUsageRows,