tokentracker-cli 0.94.1 → 0.95.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +9 -7
- package/README.ja.md +8 -6
- package/README.ko.md +8 -6
- package/README.md +23 -10
- package/README.zh-CN.md +8 -6
- package/dashboard/dist/assets/{AchievementBadge-DnmgEa4q.js → AchievementBadge-Cyju9OnL.js} +1 -1
- package/dashboard/dist/assets/{AchievementsPage-B7YTbAuK.js → AchievementsPage-Bh2dQR-I.js} +2 -2
- package/dashboard/dist/assets/{AchievementsSection-CwgT5sqH.js → AchievementsSection-DSFAM0Zp.js} +1 -1
- package/dashboard/dist/assets/{BadgeDetailModal-Dets6jzW.js → BadgeDetailModal-CgVF0VI0.js} +1 -1
- package/dashboard/dist/assets/{BotAnimated-5KwUvC9F.js → BotAnimated-DI-rTxI9.js} +1 -1
- package/dashboard/dist/assets/{Card-jf9w-4-N.js → Card-DAlV-PSI.js} +1 -1
- package/dashboard/dist/assets/{ClawdAnimated-CGxDVQ0c.js → ClawdAnimated-oOYKgcpq.js} +2 -2
- package/dashboard/dist/assets/{CommandPalette-B1bUUf32.js → CommandPalette-bYE5uD63.js} +1 -1
- package/dashboard/dist/assets/CommunityStatsModal-DCx0sQjk.js +1 -0
- package/dashboard/dist/assets/{ConfirmModal-BR-aIRH9.js → ConfirmModal-DrYf9m15.js} +1 -1
- package/dashboard/dist/assets/{Controls-CbHsJAqV.js → Controls-DAift9Z4.js} +1 -1
- package/dashboard/dist/assets/DashboardPage-CLswnOil.js +1 -0
- package/dashboard/dist/assets/{DevicePage-DDlicLLn.js → DevicePage-uWLgMC4Z.js} +1 -1
- package/dashboard/dist/assets/{DialogClose-B89uLXqp.js → DialogClose-NitVrdVC.js} +1 -1
- package/dashboard/dist/assets/DialogDescription-CQuJCP4L.js +1 -0
- package/dashboard/dist/assets/{DialogTitle-jOyycYKF.js → DialogTitle-CA_ZstJL.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-BEND42ze.js → FadeIn-CIkYDamA.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-Dacl1WGS.js → HeaderGithubStar-CYTJNLpL.js} +1 -1
- package/dashboard/dist/assets/{HoverTooltip-B2Mh2LTi.js → HoverTooltip-jh3sI8NC.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-CPEuRhiU.js → IpCheckPage-Brpe-N7B.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-BYWNR_dx.js → LandingPage-D4PjaNcN.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardAvatar-B9_5LDVj.js → LeaderboardAvatar-D9dc6frq.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-_Hl1NaIa.js → LeaderboardPage-CEuqopiC.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfileModal-Cydp7CyA.js → LeaderboardProfileModal-TeYdXeIA.js} +3 -3
- package/dashboard/dist/assets/{LeaderboardProfilePage-Dz0ecujs.js → LeaderboardProfilePage-DmtuwJkL.js} +1 -1
- package/dashboard/dist/assets/{LimitsPage-CmdT2qOZ.js → LimitsPage-BJW3teYA.js} +2 -2
- package/dashboard/dist/assets/{LocalOnlyNotice-DhuPGUkw.js → LocalOnlyNotice-V8_KFKOD.js} +1 -1
- package/dashboard/dist/assets/{LoginCard-DQtHaX_5.js → LoginCard-ud0lCY_u.js} +1 -1
- package/dashboard/dist/assets/{LoginModal-DvpPRnYV.js → LoginModal-Bnemx4UD.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-DOjfmdwb.js → LoginPage-v89BHssa.js} +1 -1
- package/dashboard/dist/assets/{PetPage-Dd6U9fh2.js → PetPage-Bg9kEi7o.js} +1 -1
- package/dashboard/dist/assets/{PopoverPopup-B1V01u10.js → PopoverPopup-CCHVwh0w.js} +1 -1
- package/dashboard/dist/assets/{ProviderIcon-BLRe55c7.js → ProviderIcon-BecpRylD.js} +1 -1
- package/dashboard/dist/assets/{ResetPasswordPage-BJ95vy1S.js → ResetPasswordPage-CsUtFeUr.js} +1 -1
- package/dashboard/dist/assets/{SegmentedControl-DacS_m9M.js → SegmentedControl-3rBqbzW_.js} +1 -1
- package/dashboard/dist/assets/{Select-bQDO5BEA.js → Select-DZ2ll7Qv.js} +1 -1
- package/dashboard/dist/assets/{SelectItemText-CUGrs9CY.js → SelectItemText-Dr1W47W6.js} +1 -1
- package/dashboard/dist/assets/{ServiceStatusPage-CsERvmVC.js → ServiceStatusPage-Cc1LXdAh.js} +1 -1
- package/dashboard/dist/assets/{SessionsPage-0VvqEfPx.js → SessionsPage-ytOFN1h9.js} +1 -1
- package/dashboard/dist/assets/{SettingsPage-9EHh2HfD.js → SettingsPage-2wYQ2ly6.js} +2 -2
- package/dashboard/dist/assets/SkillsPage-BEDAwBTN.js +1 -0
- package/dashboard/dist/assets/{TokenGalaxy-DSKTHQSs.js → TokenGalaxy-DxG0dBeG.js} +1 -1
- package/dashboard/dist/assets/{ToolbarRootContext-DP4mQLbq.js → ToolbarRootContext-DG1pr5UP.js} +1 -1
- package/dashboard/dist/assets/TrendMonitor-CeuEQ0XC.js +19 -0
- package/dashboard/dist/assets/{WidgetsPage-Cr6Wg9QX.js → WidgetsPage-ALDlZ5aR.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-DqQvRcnA.js → WrappedPage-BF59ouR-.js} +1 -1
- package/dashboard/dist/assets/{__vite-browser-external-BHz2N7Ih.css → __vite-browser-external-CCMdjCX2.css} +1 -1
- package/dashboard/dist/assets/__vite-browser-external-DXtgoDkV.js +45 -0
- package/dashboard/dist/assets/{agent-logos-BjIh_mqf.js → agent-logos-D6Ya1qw1.js} +1 -1
- package/dashboard/dist/assets/{arrow-up-right-BIhbN0NO.js → arrow-up-right-DUR69elJ.js} +1 -1
- package/dashboard/dist/assets/{check-U-wNzPvd.js → check-9_1xp5h5.js} +1 -1
- package/dashboard/dist/assets/{composite-BR91CxNl.js → composite-CEzovOpb.js} +1 -1
- package/dashboard/dist/assets/{copy-Bw3hXBCu.js → copy-DEoLQqJE.js} +1 -1
- package/dashboard/dist/assets/{download-WOqcQzvK.js → download-BNjfcJ1g.js} +1 -1
- package/dashboard/dist/assets/{external-link-BZtf6WfS.js → external-link-fV_WJ5f_.js} +1 -1
- package/dashboard/dist/assets/{flame-B5kp2VI9.js → flame-DqpuFJ14.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-BOCnuMuJ.js → format-tokens-CatddjmK.js} +1 -1
- package/dashboard/dist/assets/{icons-CA3FiIa_.js → icons-CBT1KI7V.js} +1 -1
- package/dashboard/dist/assets/{index-DjWC63Bs.js → index-BK3SzL7v.js} +1 -1
- package/dashboard/dist/assets/{index-DQaanK52.js → index-_2LEUZTl.js} +1 -1
- package/dashboard/dist/assets/{index-mEG0HZ-a.js → index-tK7s2SeO.js} +1 -1
- package/dashboard/dist/assets/{info-Djer9i3-.js → info-DBotcfT9.js} +1 -1
- package/dashboard/dist/assets/{limits-providers-F-RnNFAb.js → limits-providers-CjMU709b.js} +1 -1
- package/dashboard/dist/assets/{link-2-DV4ocD0l.js → link-2-Wc54x2fh.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-DJK72c-o.js → loader-circle-nRhGIddE.js} +1 -1
- package/dashboard/dist/assets/{maximize-2-BwBA_lMm.js → maximize-2-Dw3_GuKW.js} +1 -1
- package/dashboard/dist/assets/{modifiers.esm-B3sl4qyZ.js → modifiers.esm-rTZ5BUxp.js} +1 -1
- package/dashboard/dist/assets/module-xpSFyNBK.js +3 -0
- package/dashboard/dist/assets/{monitor-smartphone-BUTpM4ng.js → monitor-smartphone-CNVAqV_N.js} +1 -1
- package/dashboard/dist/assets/{plus-BA9RP5L6.js → plus-B5oPOXhp.js} +1 -1
- package/dashboard/dist/assets/{provider-display-DqsJzGSr.js → provider-display-DhAUWfC6.js} +2 -2
- package/dashboard/dist/assets/{react-BYw5AVSz.js → react-BKACWrsB.js} +1 -1
- package/dashboard/dist/assets/{search-BzcnFWVD.js → search-DgddmV09.js} +1 -1
- package/dashboard/dist/assets/{skills-api-vAME1P7G.js → skills-api-BT1uiqeq.js} +1 -1
- package/dashboard/dist/assets/{sortable.esm-BpEiRqqI.js → sortable.esm-UCoNfZ6z.js} +1 -1
- package/dashboard/dist/assets/{store-GeT5fhRX.js → store-ZswY8Voz.js} +1 -1
- package/dashboard/dist/assets/{terminal-CZCMYQ8K.js → terminal-BtQyotJS.js} +1 -1
- package/dashboard/dist/assets/{trash-2-DzlWEwKO.js → trash-2-Dc_NLbWs.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-DOvJfj-i.js → use-community-stats-yOKag_ou.js} +1 -1
- package/dashboard/dist/assets/use-latest-request-guard-BEiszvlI.js +1 -0
- package/dashboard/dist/assets/{use-limits-display-prefs-BmLs4HyN.js → use-limits-display-prefs-C-l_Mxvy.js} +1 -1
- package/dashboard/dist/assets/{use-ordered-list-BgozDSSV.js → use-ordered-list-CKjRaO6K.js} +1 -1
- package/dashboard/dist/assets/use-reduced-motion-BlOkflqU.js +1 -0
- package/dashboard/dist/assets/{use-session-efficiency-pref-HP1ckVkS.js → use-session-efficiency-pref-DPI_--W2.js} +1 -1
- package/dashboard/dist/assets/{use-transform-B4VSogOx.js → use-transform-BQUem5OA.js} +1 -1
- package/dashboard/dist/assets/use-usage-limits-lvkMBnNR.js +1 -0
- package/dashboard/dist/assets/{useCurrency-CSuzCBAL.js → useCurrency-C5xhS1QV.js} +1 -1
- package/dashboard/dist/assets/{useTokenFormat-ClnL9OFe.js → useTokenFormat-FBMikfEq.js} +1 -1
- package/dashboard/dist/assets/{zap-C_T9CCiS.js → zap-BsCNYelz.js} +1 -1
- package/dashboard/dist/brand-logos/lmstudio.svg +12 -0
- package/dashboard/dist/brand-logos/unsloth.svg +1 -0
- package/dashboard/dist/index.html +16 -12
- package/dashboard/dist/ip-check.html +2 -2
- package/dashboard/dist/leaderboard.html +2 -2
- package/dashboard/dist/llms.txt +4 -2
- package/dashboard/dist/share.html +2 -2
- package/package.json +2 -2
- package/src/commands/device-login.js +5 -10
- package/src/commands/init.js +28 -1
- package/src/commands/serve.js +6 -0
- package/src/commands/status.js +75 -8
- package/src/commands/sync.js +189 -18
- package/src/lib/cloud-account.js +102 -29
- package/src/lib/fs.js +30 -0
- package/src/lib/local-api.js +99 -13
- package/src/lib/pricing/index.js +5 -0
- package/src/lib/pricing/matcher.js +15 -0
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +1418 -28
- package/src/lib/usage-limits.js +507 -55
- package/src/lib/zcode-limits.js +65 -8
- package/dashboard/dist/assets/CommunityStatsModal-DRDpZycV.js +0 -1
- package/dashboard/dist/assets/DashboardPage-B5XMQoxy.js +0 -1
- package/dashboard/dist/assets/DialogDescription-DiVbwvTR.js +0 -1
- package/dashboard/dist/assets/SkillsPage-B1AcLX41.js +0 -1
- package/dashboard/dist/assets/TrendMonitor-rabtcE5w.js +0 -19
- package/dashboard/dist/assets/__vite-browser-external-C8KwSyTz.js +0 -45
- package/dashboard/dist/assets/chevron-right-Du-QVNj4.js +0 -1
- package/dashboard/dist/assets/module-D2D5r9gC.js +0 -3
- package/dashboard/dist/assets/use-reduced-motion-Bc3RFJdg.js +0 -1
- package/dashboard/dist/assets/use-usage-limits-DNNhD2H6.js +0 -1
package/src/lib/rollout.js
CHANGED
|
@@ -4900,6 +4900,42 @@ function isZcodeNativeMessage(data) {
|
|
|
4900
4900
|
);
|
|
4901
4901
|
}
|
|
4902
4902
|
|
|
4903
|
+
// ZCode persists inclusive parent counters in both its legacy OpenCode tables
|
|
4904
|
+
// and the newer model_usage table: cache read/write are already included in
|
|
4905
|
+
// input, and reasoning is already included in output. The shared OpenCode
|
|
4906
|
+
// parser expects disjoint columns, so split the subsets before it computes
|
|
4907
|
+
// queue totals and cost (issue #554).
|
|
4908
|
+
function normalizeZcodeInclusiveTokens(tokens) {
|
|
4909
|
+
if (!tokens || typeof tokens !== "object") return tokens;
|
|
4910
|
+
const rawInput = toNonNegativeInt(tokens.input);
|
|
4911
|
+
const rawOutput = toNonNegativeInt(tokens.output);
|
|
4912
|
+
const cacheRead = toNonNegativeInt(tokens.cache?.read);
|
|
4913
|
+
const cacheWrite = toNonNegativeInt(tokens.cache?.write);
|
|
4914
|
+
const reasoning = toNonNegativeInt(tokens.reasoning);
|
|
4915
|
+
return {
|
|
4916
|
+
...tokens,
|
|
4917
|
+
input: Math.max(0, rawInput - cacheRead - cacheWrite),
|
|
4918
|
+
output: Math.max(0, rawOutput - reasoning),
|
|
4919
|
+
reasoning,
|
|
4920
|
+
cache: {
|
|
4921
|
+
...(tokens.cache && typeof tokens.cache === "object" ? tokens.cache : {}),
|
|
4922
|
+
read: cacheRead,
|
|
4923
|
+
write: cacheWrite,
|
|
4924
|
+
},
|
|
4925
|
+
};
|
|
4926
|
+
}
|
|
4927
|
+
|
|
4928
|
+
function normalizeZcodeLegacyMessage(message) {
|
|
4929
|
+
if (!message?.data?.tokens) return message;
|
|
4930
|
+
return {
|
|
4931
|
+
...message,
|
|
4932
|
+
data: {
|
|
4933
|
+
...message.data,
|
|
4934
|
+
tokens: normalizeZcodeInclusiveTokens(message.data.tokens),
|
|
4935
|
+
},
|
|
4936
|
+
};
|
|
4937
|
+
}
|
|
4938
|
+
|
|
4903
4939
|
const ZCODE_NATIVE_USAGE_COLUMNS = new Set([
|
|
4904
4940
|
"id",
|
|
4905
4941
|
"logical_request_id",
|
|
@@ -5045,18 +5081,6 @@ function readZcodeNativeUsageMessages(dbPath, sqliteOptions = {}) {
|
|
|
5045
5081
|
const startedAt = coerceEpochMs(row?.started_at);
|
|
5046
5082
|
if (!providerID || !modelID || !sessionID || !id || !startedAt) continue;
|
|
5047
5083
|
|
|
5048
|
-
// ZCode's persisted input/output columns are inclusive parents:
|
|
5049
|
-
// cache read/write are subsets of input, and reasoning is a subset of
|
|
5050
|
-
// output. TokenTracker stores those five columns disjointly, so split the
|
|
5051
|
-
// subsets here before entering the shared OpenCode parser. This preserves
|
|
5052
|
-
// sum(input + cache read + cache write + output + reasoning) without
|
|
5053
|
-
// billing either subset twice.
|
|
5054
|
-
const rawInput = toNonNegativeInt(row?.input_tokens);
|
|
5055
|
-
const rawOutput = toNonNegativeInt(row?.output_tokens);
|
|
5056
|
-
const cacheRead = toNonNegativeInt(row?.cache_read_input_tokens);
|
|
5057
|
-
const cacheWrite = toNonNegativeInt(row?.cache_creation_input_tokens);
|
|
5058
|
-
const reasoning = toNonNegativeInt(row?.reasoning_tokens);
|
|
5059
|
-
|
|
5060
5084
|
const data = {
|
|
5061
5085
|
id,
|
|
5062
5086
|
sessionID,
|
|
@@ -5064,15 +5088,15 @@ function readZcodeNativeUsageMessages(dbPath, sqliteOptions = {}) {
|
|
|
5064
5088
|
providerID,
|
|
5065
5089
|
modelID,
|
|
5066
5090
|
time: { created: startedAt, completed: startedAt },
|
|
5067
|
-
tokens: {
|
|
5068
|
-
input:
|
|
5069
|
-
output:
|
|
5070
|
-
reasoning,
|
|
5091
|
+
tokens: normalizeZcodeInclusiveTokens({
|
|
5092
|
+
input: row?.input_tokens,
|
|
5093
|
+
output: row?.output_tokens,
|
|
5094
|
+
reasoning: row?.reasoning_tokens,
|
|
5071
5095
|
cache: {
|
|
5072
|
-
read:
|
|
5073
|
-
write:
|
|
5096
|
+
read: row?.cache_read_input_tokens,
|
|
5097
|
+
write: row?.cache_creation_input_tokens,
|
|
5074
5098
|
},
|
|
5075
|
-
},
|
|
5099
|
+
}),
|
|
5076
5100
|
};
|
|
5077
5101
|
if (typeof row?.directory === "string" && row.directory.trim()) {
|
|
5078
5102
|
data.path = { cwd: row.directory.trim() };
|
|
@@ -5085,12 +5109,31 @@ function readZcodeNativeUsageMessages(dbPath, sqliteOptions = {}) {
|
|
|
5085
5109
|
|
|
5086
5110
|
// Read only genuine ZCode assistant messages (its own GLM models via Z.ai /
|
|
5087
5111
|
// BigModel), dropping any bundled sub-agent turns. See isZcodeNativeMessage.
|
|
5112
|
+
//
|
|
5113
|
+
// ZCode started writing model_usage after many installations had already
|
|
5114
|
+
// accumulated months of history in the OpenCode message tables. The native
|
|
5115
|
+
// table is authoritative from its first completed row onward, but it is not a
|
|
5116
|
+
// historical backfill. Keep legacy rows before that boundary so merely adding
|
|
5117
|
+
// the new table cannot make older usage disappear from TokenTracker.
|
|
5088
5118
|
function readZcodeDbMessages(dbPath, sqliteOptions = {}) {
|
|
5089
5119
|
if (!dbPath || !fssync.existsSync(dbPath)) return [];
|
|
5090
5120
|
const nativeMessages = readZcodeNativeUsageMessages(dbPath, sqliteOptions);
|
|
5091
|
-
|
|
5092
|
-
|
|
5093
|
-
|
|
5121
|
+
const legacyMessages = readOpencodeDbMessages(dbPath, sqliteOptions)
|
|
5122
|
+
.filter((message) => isZcodeNativeMessage(message.data))
|
|
5123
|
+
.map(normalizeZcodeLegacyMessage);
|
|
5124
|
+
if (nativeMessages === null) return legacyMessages;
|
|
5125
|
+
|
|
5126
|
+
const nativeStartMs = nativeMessages.reduce((earliest, message) => {
|
|
5127
|
+
const timestampMs = coerceEpochMs(message?.timeUpdated);
|
|
5128
|
+
return timestampMs > 0 ? Math.min(earliest, timestampMs) : earliest;
|
|
5129
|
+
}, Number.POSITIVE_INFINITY);
|
|
5130
|
+
if (!Number.isFinite(nativeStartMs)) return legacyMessages;
|
|
5131
|
+
|
|
5132
|
+
const historicalMessages = legacyMessages.filter((message) => {
|
|
5133
|
+
const timestampMs = coerceEpochMs(message?.timeUpdated);
|
|
5134
|
+
return timestampMs > 0 && timestampMs < nativeStartMs;
|
|
5135
|
+
});
|
|
5136
|
+
return [...historicalMessages, ...nativeMessages];
|
|
5094
5137
|
}
|
|
5095
5138
|
|
|
5096
5139
|
async function parseOpencodeDbIncremental({
|
|
@@ -5718,6 +5761,384 @@ async function parseQoderDbIncremental({
|
|
|
5718
5761
|
return { messagesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
|
|
5719
5762
|
}
|
|
5720
5763
|
|
|
5764
|
+
// ── Qoder (new) — ~/.qoder/projects JSONL (com.qoder.app.stable, 2026-08+) ──
|
|
5765
|
+
//
|
|
5766
|
+
// Qoder 2026-08 (app 0.1.2+) migrated from SharedClientCache/cache/db/local.db
|
|
5767
|
+
// to Electron main.sqlite + ~/.qoder/projects/<slug>/<sessionId>.jsonl.
|
|
5768
|
+
// The new transcript's message.usage no longer carries prompt_tokens — it is a
|
|
5769
|
+
// credit-billed SDK: {input_tokens:0, output_tokens:0, credits:3.2, billable:true}.
|
|
5770
|
+
// Only rows with authoritative token fields are counted: credit-only usage
|
|
5771
|
+
// without tokens is intentionally not counted (usage stays unsupported, no
|
|
5772
|
+
// token delta) because there is no first-party evidence for a credit→token
|
|
5773
|
+
// rate. Cost was never estimated — no authoritative credit→USD rate is
|
|
5774
|
+
// published, so total_cost_usd stays 0.
|
|
5775
|
+
// Old local.db is kept as a legacy fallback; both sources now use distinct
|
|
5776
|
+
// cursor namespaces (qoder vs qoderNew) via disjoint messageKey prefixes
|
|
5777
|
+
// (row: vs jsonl:) but upload under the same source="qoder".
|
|
5778
|
+
|
|
5779
|
+
function resolveQoderProjectsDir({ home = os.homedir(), env = process.env, platform = process.platform, deps = {} } = {}) {
|
|
5780
|
+
const override = typeof env.QODER_PROJECTS_DIR === "string" && env.QODER_PROJECTS_DIR.trim()
|
|
5781
|
+
? path.resolve(env.QODER_PROJECTS_DIR.trim())
|
|
5782
|
+
: null;
|
|
5783
|
+
if (override) return override;
|
|
5784
|
+
// QODER_HOME points at the app support dir for the legacy DB; the new
|
|
5785
|
+
// projects dir is always ~/.qoder regardless of QODER_HOME.
|
|
5786
|
+
// On Windows also probe WSL distro home (same pattern as other providers).
|
|
5787
|
+
if (platform === "win32" && !env.QODER_PROJECTS_DIR) {
|
|
5788
|
+
const discoverWslHome = deps.discoverWslHome || wsl.discoverWslHome;
|
|
5789
|
+
const wslRoot = wsl.shouldProbeWsl(env) ? discoverWslHome(".qoder", { ...deps, env }) : null;
|
|
5790
|
+
if (wslRoot) {
|
|
5791
|
+
const wslProjects = path.join(wslRoot, "projects");
|
|
5792
|
+
if ((deps.existsSync || fssync.existsSync)(wslProjects)) return wslProjects;
|
|
5793
|
+
}
|
|
5794
|
+
}
|
|
5795
|
+
return path.join(home, ".qoder", "projects");
|
|
5796
|
+
}
|
|
5797
|
+
|
|
5798
|
+
function resolveQoderCnProjectsDir({ home = os.homedir(), env = process.env, platform = process.platform, deps = {} } = {}) {
|
|
5799
|
+
const override = typeof env.QODER_CN_PROJECTS_DIR === "string" && env.QODER_CN_PROJECTS_DIR.trim()
|
|
5800
|
+
? path.resolve(env.QODER_CN_PROJECTS_DIR.trim())
|
|
5801
|
+
: null;
|
|
5802
|
+
if (override) return override;
|
|
5803
|
+
// CN and international currently share ~/.qoder; keep a distinct override for
|
|
5804
|
+
// future split without double-counting by default.
|
|
5805
|
+
if (platform === "win32" && !env.QODER_CN_PROJECTS_DIR) {
|
|
5806
|
+
const discoverWslHome = deps.discoverWslHome || wsl.discoverWslHome;
|
|
5807
|
+
const wslRoot = wsl.shouldProbeWsl(env) ? discoverWslHome(".qoder", { ...deps, env }) : null;
|
|
5808
|
+
if (wslRoot) {
|
|
5809
|
+
const wslProjects = path.join(wslRoot, "projects");
|
|
5810
|
+
if ((deps.existsSync || fssync.existsSync)(wslProjects)) return wslProjects;
|
|
5811
|
+
}
|
|
5812
|
+
}
|
|
5813
|
+
return path.join(home, ".qoder", "projects");
|
|
5814
|
+
}
|
|
5815
|
+
|
|
5816
|
+
async function listQoderNewSessionFiles(projectsDir) {
|
|
5817
|
+
const out = [];
|
|
5818
|
+
if (!projectsDir || !fssync.existsSync(projectsDir)) return out;
|
|
5819
|
+
async function walk(dir) {
|
|
5820
|
+
const entries = await safeReadDir(dir);
|
|
5821
|
+
for (const e of entries) {
|
|
5822
|
+
const p = path.join(dir, e.name);
|
|
5823
|
+
if (e.isDirectory()) {
|
|
5824
|
+
await walk(p);
|
|
5825
|
+
} else if (e.isFile() && e.name.endsWith(".jsonl")) {
|
|
5826
|
+
out.push(p);
|
|
5827
|
+
}
|
|
5828
|
+
}
|
|
5829
|
+
}
|
|
5830
|
+
await walk(projectsDir);
|
|
5831
|
+
out.sort((a, b) => a.localeCompare(b));
|
|
5832
|
+
return out;
|
|
5833
|
+
}
|
|
5834
|
+
|
|
5835
|
+
function qoderNewModelFromRecord(record) {
|
|
5836
|
+
const msgModel = record?.message?.model;
|
|
5837
|
+
const direct = typeof msgModel === "string" ? msgModel.trim() : "";
|
|
5838
|
+
return normalizeModelInput(direct) || "qoder-agent";
|
|
5839
|
+
}
|
|
5840
|
+
|
|
5841
|
+
function qoderNewMessageKey(record, filePath, lineIndex = 0) {
|
|
5842
|
+
const msgId = normalizeMessageKeyPart(record?.message?.id)
|
|
5843
|
+
|| normalizeMessageKeyPart(record?.uuid)
|
|
5844
|
+
|| normalizeMessageKeyPart(record?.id)
|
|
5845
|
+
|| null;
|
|
5846
|
+
const sessionId = normalizeMessageKeyPart(record?.sessionId || record?.session_id);
|
|
5847
|
+
// Prefix to avoid collision with legacy row: keys; fallback includes line index
|
|
5848
|
+
// so multiple no-id records in the same file remain distinct.
|
|
5849
|
+
const fallbackSuffix = Number.isFinite(lineIndex) ? `${filePath}:${lineIndex}` : filePath;
|
|
5850
|
+
if (sessionId && msgId) return `jsonl:${sessionId}|${msgId}`;
|
|
5851
|
+
if (msgId) return `jsonl:${msgId}`;
|
|
5852
|
+
if (sessionId) return `jsonl:${sessionId}|${record?.uuid || fallbackSuffix}`;
|
|
5853
|
+
return `jsonl:${fallbackSuffix}|${record?.uuid || ""}`;
|
|
5854
|
+
}
|
|
5855
|
+
|
|
5856
|
+
function qoderNewTimestampMs(record) {
|
|
5857
|
+
return coerceEpochMs(record?.timestamp)
|
|
5858
|
+
|| coerceEpochMs(record?.message?.timestamp)
|
|
5859
|
+
|| parseIsoTimestampMs(record?.timestamp)
|
|
5860
|
+
|| parseIsoTimestampMs(record?.message?.timestamp)
|
|
5861
|
+
|| 0;
|
|
5862
|
+
}
|
|
5863
|
+
|
|
5864
|
+
function normalizeQoderNewTokens(usage) {
|
|
5865
|
+
if (!usage || typeof usage !== "object") return null;
|
|
5866
|
+
const credits = Number(usage.credits ?? usage.original_credits ?? 0);
|
|
5867
|
+
let input = Number(usage.input_tokens ?? 0);
|
|
5868
|
+
let cached = Number(usage.cache_read_input_tokens ?? usage.cached_tokens ?? 0);
|
|
5869
|
+
let cacheCreation = Number(usage.cache_creation_input_tokens ?? 0);
|
|
5870
|
+
let output = Number(usage.output_tokens ?? 0);
|
|
5871
|
+
// Guard against malformed numbers (NaN/Infinity/negative) — align with
|
|
5872
|
+
// legacy normalizeQoderTokens which returns null on such input.
|
|
5873
|
+
if (!Number.isFinite(input) || input < 0) input = 0;
|
|
5874
|
+
if (!Number.isFinite(cached) || cached < 0) cached = 0;
|
|
5875
|
+
if (!Number.isFinite(cacheCreation) || cacheCreation < 0) cacheCreation = 0;
|
|
5876
|
+
if (!Number.isFinite(output) || output < 0) output = 0;
|
|
5877
|
+
// Only rows with authoritative token fields are counted; anything else
|
|
5878
|
+
// falls through to null (unsupported, no token delta).
|
|
5879
|
+
if (input > 0 || cached > 0 || cacheCreation > 0 || output > 0) {
|
|
5880
|
+
const inp = Math.max(0, Math.trunc(input));
|
|
5881
|
+
const cach = Math.max(0, Math.trunc(cached));
|
|
5882
|
+
const out = Math.max(0, Math.trunc(output));
|
|
5883
|
+
const cc = Math.max(0, Math.trunc(cacheCreation));
|
|
5884
|
+
return {
|
|
5885
|
+
input_tokens: inp,
|
|
5886
|
+
cached_input_tokens: cach,
|
|
5887
|
+
cache_creation_input_tokens: cc,
|
|
5888
|
+
output_tokens: out,
|
|
5889
|
+
reasoning_output_tokens: 0,
|
|
5890
|
+
total_tokens: inp + cach + cc + out,
|
|
5891
|
+
billable_total_tokens: inp + cach + cc + out,
|
|
5892
|
+
credits: Number.isFinite(credits) && credits > 0 ? credits : 0,
|
|
5893
|
+
usage_precision: null,
|
|
5894
|
+
};
|
|
5895
|
+
}
|
|
5896
|
+
// Credit-only usage without authoritative token fields is intentionally
|
|
5897
|
+
// not counted (no token delta): there is no first-party evidence for a
|
|
5898
|
+
// credit→token rate. The caller still counts billable messages as
|
|
5899
|
+
// conversation activity.
|
|
5900
|
+
return null;
|
|
5901
|
+
}
|
|
5902
|
+
|
|
5903
|
+
async function parseQoderNewIncremental({
|
|
5904
|
+
sessionFiles,
|
|
5905
|
+
cursors,
|
|
5906
|
+
queuePath,
|
|
5907
|
+
projectQueuePath,
|
|
5908
|
+
onProgress,
|
|
5909
|
+
sourceKey = "qoder",
|
|
5910
|
+
cursorKey = "qoderNew",
|
|
5911
|
+
publicRepoResolver,
|
|
5912
|
+
} = {}) {
|
|
5913
|
+
await ensureDir(path.dirname(queuePath));
|
|
5914
|
+
const files = Array.isArray(sessionFiles) ? sessionFiles : [];
|
|
5915
|
+
// One-time migration: pre-#549 stored JSONL keys under the legacy "qoder"
|
|
5916
|
+
// cursor (same namespace as SQLite). Move them to the new isolated namespace
|
|
5917
|
+
// so history is not double-counted and legacy multi-install state is preserved.
|
|
5918
|
+
if (cursorKey === "qoderNew" && cursors?.qoder && !cursors?.qoderNew) {
|
|
5919
|
+
const legacy = normalizeQoderState(cursors.qoder);
|
|
5920
|
+
const jsonlEntries = Object.entries(legacy.messages).filter(([k]) => k.startsWith("jsonl:"));
|
|
5921
|
+
if (jsonlEntries.length > 0) {
|
|
5922
|
+
const migrated = {};
|
|
5923
|
+
for (const [k, v] of jsonlEntries) {
|
|
5924
|
+
migrated[k] = v;
|
|
5925
|
+
delete legacy.messages[k];
|
|
5926
|
+
}
|
|
5927
|
+
cursors.qoderNew = { messages: migrated, updatedAt: legacy.updatedAt || new Date().toISOString() };
|
|
5928
|
+
}
|
|
5929
|
+
}
|
|
5930
|
+
if (cursorKey === "qoderCnNew" && cursors?.["qoder-cn"] && !cursors?.["qoderCnNew"]) {
|
|
5931
|
+
const legacy = normalizeQoderState(cursors["qoder-cn"]);
|
|
5932
|
+
const jsonlEntries = Object.entries(legacy.messages).filter(([k]) => k.startsWith("jsonl:"));
|
|
5933
|
+
if (jsonlEntries.length > 0) {
|
|
5934
|
+
const migrated = {};
|
|
5935
|
+
for (const [k, v] of jsonlEntries) {
|
|
5936
|
+
migrated[k] = v;
|
|
5937
|
+
delete legacy.messages[k];
|
|
5938
|
+
}
|
|
5939
|
+
cursors["qoderCnNew"] = { messages: migrated, updatedAt: legacy.updatedAt || new Date().toISOString() };
|
|
5940
|
+
}
|
|
5941
|
+
}
|
|
5942
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
5943
|
+
const qoderState = normalizeQoderState(cursors?.[cursorKey]);
|
|
5944
|
+
const projectEnabled = typeof projectQueuePath === "string" && projectQueuePath.length > 0;
|
|
5945
|
+
const projectState = projectEnabled ? normalizeProjectState(cursors?.projectHourly) : null;
|
|
5946
|
+
const projectTouchedBuckets = projectEnabled ? new Set() : null;
|
|
5947
|
+
const projectMetaCache = projectEnabled ? new Map() : null;
|
|
5948
|
+
const publicRepoCache = projectEnabled ? new Map() : null;
|
|
5949
|
+
const touchedBuckets = new Set();
|
|
5950
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
5951
|
+
|
|
5952
|
+
// Build current snapshot from all JSONL files
|
|
5953
|
+
const currentByKey = new Map();
|
|
5954
|
+
const fileCount = files.length;
|
|
5955
|
+
for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
|
|
5956
|
+
const filePath = files[fileIdx];
|
|
5957
|
+
let raw;
|
|
5958
|
+
try {
|
|
5959
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
5960
|
+
} catch (_e) {
|
|
5961
|
+
continue;
|
|
5962
|
+
}
|
|
5963
|
+
const lines = raw.split("\n");
|
|
5964
|
+
for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
|
|
5965
|
+
const line = lines[lineIdx];
|
|
5966
|
+
if (!line.trim()) continue;
|
|
5967
|
+
let record;
|
|
5968
|
+
try {
|
|
5969
|
+
record = JSON.parse(line);
|
|
5970
|
+
} catch (_e) {
|
|
5971
|
+
continue;
|
|
5972
|
+
}
|
|
5973
|
+
if (record?.type !== "assistant") continue;
|
|
5974
|
+
const msg = record?.message;
|
|
5975
|
+
if (!msg || msg.role !== "assistant") continue;
|
|
5976
|
+
// Skip synthetic sub-agent streaming chunks that carry no usage
|
|
5977
|
+
const usage = msg.usage;
|
|
5978
|
+
if (!usage || typeof usage !== "object") continue;
|
|
5979
|
+
const base = normalizeQoderNewTokens(usage);
|
|
5980
|
+
// Allow billable zero-token messages to still count conversation (no token delta)
|
|
5981
|
+
const isBillable = usage.billable !== false;
|
|
5982
|
+
if (!base && !isBillable) continue;
|
|
5983
|
+
const timestampMs = qoderNewTimestampMs(record);
|
|
5984
|
+
if (!timestampMs) continue;
|
|
5985
|
+
const bucketStart = toUtcHalfHourStart(new Date(timestampMs).toISOString());
|
|
5986
|
+
if (!bucketStart) continue;
|
|
5987
|
+
const messageKey = qoderNewMessageKey(record, filePath, lineIdx);
|
|
5988
|
+
if (!messageKey) continue;
|
|
5989
|
+
const model = qoderNewModelFromRecord(record);
|
|
5990
|
+
const totals = base ? {
|
|
5991
|
+
input_tokens: base.input_tokens,
|
|
5992
|
+
cached_input_tokens: base.cached_input_tokens,
|
|
5993
|
+
cache_creation_input_tokens: base.cache_creation_input_tokens,
|
|
5994
|
+
output_tokens: base.output_tokens,
|
|
5995
|
+
reasoning_output_tokens: 0,
|
|
5996
|
+
total_tokens: base.total_tokens,
|
|
5997
|
+
billable_total_tokens: base.billable_total_tokens,
|
|
5998
|
+
total_cost_usd: 0,
|
|
5999
|
+
usage_precision: base.usage_precision || undefined,
|
|
6000
|
+
conversation_count: 1,
|
|
6001
|
+
} : {
|
|
6002
|
+
input_tokens: 0,
|
|
6003
|
+
cached_input_tokens: 0,
|
|
6004
|
+
cache_creation_input_tokens: 0,
|
|
6005
|
+
output_tokens: 0,
|
|
6006
|
+
reasoning_output_tokens: 0,
|
|
6007
|
+
total_tokens: 0,
|
|
6008
|
+
billable_total_tokens: 0,
|
|
6009
|
+
total_cost_usd: 0,
|
|
6010
|
+
conversation_count: 1,
|
|
6011
|
+
};
|
|
6012
|
+
let projectKey = null;
|
|
6013
|
+
let projectRef = null;
|
|
6014
|
+
if (projectEnabled) {
|
|
6015
|
+
const rawCwd = typeof record?.cwd === "string" ? record.cwd.trim() : "";
|
|
6016
|
+
if (rawCwd) {
|
|
6017
|
+
const startDir = wsl.mapWslCwdToUnc(rawCwd, filePath);
|
|
6018
|
+
const context = await resolveProjectContextForPath({
|
|
6019
|
+
startDir,
|
|
6020
|
+
projectMetaCache,
|
|
6021
|
+
publicRepoCache,
|
|
6022
|
+
publicRepoResolver,
|
|
6023
|
+
projectState,
|
|
6024
|
+
});
|
|
6025
|
+
projectKey = context?.projectKey || null;
|
|
6026
|
+
projectRef = context?.projectRef || null;
|
|
6027
|
+
}
|
|
6028
|
+
}
|
|
6029
|
+
currentByKey.set(messageKey, {
|
|
6030
|
+
totals,
|
|
6031
|
+
bucketStart,
|
|
6032
|
+
model,
|
|
6033
|
+
projectKey,
|
|
6034
|
+
projectRef,
|
|
6035
|
+
filePath,
|
|
6036
|
+
});
|
|
6037
|
+
}
|
|
6038
|
+
if (cb && (fileIdx % 50 === 0 || fileIdx === files.length - 1)) {
|
|
6039
|
+
cb({
|
|
6040
|
+
index: fileIdx + 1,
|
|
6041
|
+
total: fileCount,
|
|
6042
|
+
messagesProcessed: currentByKey.size,
|
|
6043
|
+
eventsAggregated: 0,
|
|
6044
|
+
bucketsQueued: 0,
|
|
6045
|
+
});
|
|
6046
|
+
}
|
|
6047
|
+
}
|
|
6048
|
+
|
|
6049
|
+
let messagesProcessed = currentByKey.size;
|
|
6050
|
+
let eventsAggregated = 0;
|
|
6051
|
+
|
|
6052
|
+
// Subtract contributions that disappeared or changed
|
|
6053
|
+
for (const [key, prev] of Object.entries(qoderState.messages)) {
|
|
6054
|
+
if (!key.startsWith("jsonl:")) continue;
|
|
6055
|
+
const cur = currentByKey.get(key);
|
|
6056
|
+
const unchanged = cur
|
|
6057
|
+
&& totalsKey(prev.totals) === totalsKey(cur.totals)
|
|
6058
|
+
&& prev.bucketStart === cur.bucketStart
|
|
6059
|
+
&& prev.model === cur.model
|
|
6060
|
+
&& (prev.projectKey || null) === (cur.projectKey || null)
|
|
6061
|
+
&& (prev.totals?.total_cost_usd || 0) === (cur.totals.total_cost_usd || 0);
|
|
6062
|
+
if (unchanged) continue;
|
|
6063
|
+
if (prev.totals && prev.bucketStart && prev.model) {
|
|
6064
|
+
const oldBucket = getHourlyBucket(hourlyState, sourceKey, prev.model, prev.bucketStart);
|
|
6065
|
+
subtractTotals(oldBucket.totals, prev.totals);
|
|
6066
|
+
touchedBuckets.add(bucketKey(sourceKey, prev.model, prev.bucketStart));
|
|
6067
|
+
if (projectEnabled && prev.projectKey) {
|
|
6068
|
+
const oldProjectBucket = getProjectBucket(projectState, prev.projectKey, sourceKey, prev.bucketStart, prev.projectRef || null);
|
|
6069
|
+
subtractTotals(oldProjectBucket.totals, prev.totals);
|
|
6070
|
+
projectTouchedBuckets.add(projectBucketKey(prev.projectKey, sourceKey, prev.bucketStart));
|
|
6071
|
+
}
|
|
6072
|
+
}
|
|
6073
|
+
if (cur) {
|
|
6074
|
+
const bucket = getHourlyBucket(hourlyState, sourceKey, cur.model, cur.bucketStart);
|
|
6075
|
+
addTotals(bucket.totals, cur.totals);
|
|
6076
|
+
if (cur.totals.usage_precision) bucket.usage_precision = cur.totals.usage_precision;
|
|
6077
|
+
// addTotals handles total_cost_usd via USD_TICKS, but credits cost is small; ensure it accumulates
|
|
6078
|
+
touchedBuckets.add(bucketKey(sourceKey, cur.model, cur.bucketStart));
|
|
6079
|
+
if (projectEnabled && cur.projectKey) {
|
|
6080
|
+
const projectBucket = getProjectBucket(projectState, cur.projectKey, sourceKey, cur.bucketStart, cur.projectRef);
|
|
6081
|
+
addTotals(projectBucket.totals, cur.totals);
|
|
6082
|
+
if (cur.totals.usage_precision) projectBucket.usage_precision = cur.totals.usage_precision;
|
|
6083
|
+
projectTouchedBuckets.add(projectBucketKey(cur.projectKey, sourceKey, cur.bucketStart));
|
|
6084
|
+
}
|
|
6085
|
+
qoderState.messages[key] = {
|
|
6086
|
+
totals: cur.totals,
|
|
6087
|
+
conversationCount: cur.totals.conversation_count,
|
|
6088
|
+
bucketStart: cur.bucketStart,
|
|
6089
|
+
model: cur.model,
|
|
6090
|
+
projectKey: cur.projectKey,
|
|
6091
|
+
projectRef: cur.projectRef,
|
|
6092
|
+
updatedAt: new Date().toISOString(),
|
|
6093
|
+
};
|
|
6094
|
+
eventsAggregated += 1;
|
|
6095
|
+
} else {
|
|
6096
|
+
delete qoderState.messages[key];
|
|
6097
|
+
eventsAggregated += 1;
|
|
6098
|
+
}
|
|
6099
|
+
}
|
|
6100
|
+
|
|
6101
|
+
// Add brand-new keys
|
|
6102
|
+
for (const [key, cur] of currentByKey.entries()) {
|
|
6103
|
+
if (qoderState.messages[key]) continue;
|
|
6104
|
+
const bucket = getHourlyBucket(hourlyState, sourceKey, cur.model, cur.bucketStart);
|
|
6105
|
+
addTotals(bucket.totals, cur.totals);
|
|
6106
|
+
if (cur.totals.usage_precision) bucket.usage_precision = cur.totals.usage_precision;
|
|
6107
|
+
touchedBuckets.add(bucketKey(sourceKey, cur.model, cur.bucketStart));
|
|
6108
|
+
if (projectEnabled && cur.projectKey) {
|
|
6109
|
+
const projectBucket = getProjectBucket(projectState, cur.projectKey, sourceKey, cur.bucketStart, cur.projectRef);
|
|
6110
|
+
addTotals(projectBucket.totals, cur.totals);
|
|
6111
|
+
if (cur.totals.usage_precision) projectBucket.usage_precision = cur.totals.usage_precision;
|
|
6112
|
+
projectTouchedBuckets.add(projectBucketKey(cur.projectKey, sourceKey, cur.bucketStart));
|
|
6113
|
+
}
|
|
6114
|
+
qoderState.messages[key] = {
|
|
6115
|
+
totals: cur.totals,
|
|
6116
|
+
conversationCount: cur.totals.conversation_count,
|
|
6117
|
+
bucketStart: cur.bucketStart,
|
|
6118
|
+
model: cur.model,
|
|
6119
|
+
projectKey: cur.projectKey,
|
|
6120
|
+
projectRef: cur.projectRef,
|
|
6121
|
+
updatedAt: new Date().toISOString(),
|
|
6122
|
+
};
|
|
6123
|
+
eventsAggregated += 1;
|
|
6124
|
+
}
|
|
6125
|
+
|
|
6126
|
+
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
6127
|
+
const projectBucketsQueued = projectEnabled
|
|
6128
|
+
? await enqueueTouchedProjectBuckets({ projectQueuePath, projectState, projectTouchedBuckets })
|
|
6129
|
+
: 0;
|
|
6130
|
+
const updatedAt = new Date().toISOString();
|
|
6131
|
+
hourlyState.updatedAt = updatedAt;
|
|
6132
|
+
qoderState.updatedAt = updatedAt;
|
|
6133
|
+
cursors.hourly = hourlyState;
|
|
6134
|
+
cursors[cursorKey] = qoderState;
|
|
6135
|
+
if (projectState) {
|
|
6136
|
+
projectState.updatedAt = updatedAt;
|
|
6137
|
+
cursors.projectHourly = projectState;
|
|
6138
|
+
}
|
|
6139
|
+
return { messagesProcessed, eventsAggregated, bucketsQueued, projectBucketsQueued };
|
|
6140
|
+
}
|
|
6141
|
+
|
|
5721
6142
|
// ── Claude Science (Anthropic's local research workbench, issue #246) ──
|
|
5722
6143
|
//
|
|
5723
6144
|
// Claude Science keeps all of its state in a SQLite database named
|
|
@@ -11160,6 +11581,955 @@ async function parseZedIncremental({
|
|
|
11160
11581
|
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
11161
11582
|
}
|
|
11162
11583
|
|
|
11584
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
11585
|
+
// LM Studio
|
|
11586
|
+
//
|
|
11587
|
+
// Data: pretty-printed OpenAI-compatible final responses beneath
|
|
11588
|
+
// `~/.lmstudio/server-logs/`. The reader recognizes both Chat Completions and
|
|
11589
|
+
// Responses API usage shapes. It scans only response identity, model, timestamp,
|
|
11590
|
+
// and the balanced `usage` object; prompt and response bodies are never parsed
|
|
11591
|
+
// or persisted.
|
|
11592
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
11593
|
+
|
|
11594
|
+
const LMSTUDIO_SOURCE = "lmstudio";
|
|
11595
|
+
const LMSTUDIO_WINDOW_BYTES = 8 * 1024 * 1024;
|
|
11596
|
+
const LMSTUDIO_CHUNK_BYTES = 64 * 1024;
|
|
11597
|
+
const LMSTUDIO_IDENTITY_OVERLAP_BYTES = 512;
|
|
11598
|
+
const LMSTUDIO_MESSAGE_LIMIT = 10_000;
|
|
11599
|
+
const LMSTUDIO_USAGE_MARKER = Buffer.from('"usage"');
|
|
11600
|
+
|
|
11601
|
+
function resolveLmstudioHome(env = process.env) {
|
|
11602
|
+
const override = typeof env.TOKENTRACKER_LMSTUDIO_HOME === "string"
|
|
11603
|
+
? env.TOKENTRACKER_LMSTUDIO_HOME.trim()
|
|
11604
|
+
: "";
|
|
11605
|
+
if (override) return path.resolve(override);
|
|
11606
|
+
const nativeHome = typeof env.LM_STUDIO_HOME === "string"
|
|
11607
|
+
? env.LM_STUDIO_HOME.trim()
|
|
11608
|
+
: "";
|
|
11609
|
+
if (nativeHome) return path.resolve(nativeHome);
|
|
11610
|
+
return path.join(env.HOME || env.USERPROFILE || os.homedir(), ".lmstudio");
|
|
11611
|
+
}
|
|
11612
|
+
|
|
11613
|
+
async function resolveLmstudioLogFiles(env = process.env) {
|
|
11614
|
+
const root = path.join(resolveLmstudioHome(env), "server-logs");
|
|
11615
|
+
const files = [];
|
|
11616
|
+
const pending = [root];
|
|
11617
|
+
while (pending.length > 0) {
|
|
11618
|
+
const dir = pending.pop();
|
|
11619
|
+
let entries;
|
|
11620
|
+
try {
|
|
11621
|
+
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
11622
|
+
} catch (_e) {
|
|
11623
|
+
continue;
|
|
11624
|
+
}
|
|
11625
|
+
for (const entry of entries) {
|
|
11626
|
+
const fullPath = path.join(dir, entry.name);
|
|
11627
|
+
if (entry.isDirectory()) pending.push(fullPath);
|
|
11628
|
+
else if (entry.isFile() && entry.name.toLowerCase().endsWith(".log")) files.push(fullPath);
|
|
11629
|
+
}
|
|
11630
|
+
}
|
|
11631
|
+
return files.sort((a, b) => a.localeCompare(b));
|
|
11632
|
+
}
|
|
11633
|
+
|
|
11634
|
+
function skipLmstudioWhitespace(buffer, index) {
|
|
11635
|
+
while (index < buffer.length && /\s/.test(String.fromCharCode(buffer[index]))) index += 1;
|
|
11636
|
+
return index;
|
|
11637
|
+
}
|
|
11638
|
+
|
|
11639
|
+
function scanLmstudioUsageObjectStart(buffer, from = 0) {
|
|
11640
|
+
let cursor = Math.max(0, from);
|
|
11641
|
+
while (cursor < buffer.length) {
|
|
11642
|
+
const marker = buffer.indexOf(LMSTUDIO_USAGE_MARKER, cursor);
|
|
11643
|
+
if (marker < 0) break;
|
|
11644
|
+
cursor = marker + LMSTUDIO_USAGE_MARKER.length;
|
|
11645
|
+
if (marker > 0 && buffer[marker - 1] === 0x5c) continue;
|
|
11646
|
+
const colon = skipLmstudioWhitespace(buffer, cursor);
|
|
11647
|
+
if (colon >= buffer.length) return { found: null, certainTo: marker };
|
|
11648
|
+
if (buffer[colon] !== 0x3a) continue;
|
|
11649
|
+
const brace = skipLmstudioWhitespace(buffer, colon + 1);
|
|
11650
|
+
if (brace >= buffer.length) return { found: null, certainTo: marker };
|
|
11651
|
+
if (buffer[brace] === 0x7b) return { found: { marker, objectStart: brace }, certainTo: marker };
|
|
11652
|
+
}
|
|
11653
|
+
return {
|
|
11654
|
+
found: null,
|
|
11655
|
+
certainTo: Math.max(0, buffer.length - Math.max(0, LMSTUDIO_USAGE_MARKER.length - 1)),
|
|
11656
|
+
};
|
|
11657
|
+
}
|
|
11658
|
+
|
|
11659
|
+
function lmstudioBalancedObjectEnd(buffer, start) {
|
|
11660
|
+
if (buffer[start] !== 0x7b) return -1;
|
|
11661
|
+
let depth = 0;
|
|
11662
|
+
let inString = false;
|
|
11663
|
+
let escaped = false;
|
|
11664
|
+
for (let index = start; index < buffer.length; index++) {
|
|
11665
|
+
const byte = buffer[index];
|
|
11666
|
+
if (inString) {
|
|
11667
|
+
if (escaped) escaped = false;
|
|
11668
|
+
else if (byte === 0x5c) escaped = true;
|
|
11669
|
+
else if (byte === 0x22) inString = false;
|
|
11670
|
+
continue;
|
|
11671
|
+
}
|
|
11672
|
+
if (byte === 0x22) inString = true;
|
|
11673
|
+
else if (byte === 0x7b) depth += 1;
|
|
11674
|
+
else if (byte === 0x7d) {
|
|
11675
|
+
depth -= 1;
|
|
11676
|
+
if (depth === 0) return index + 1;
|
|
11677
|
+
}
|
|
11678
|
+
}
|
|
11679
|
+
return -1;
|
|
11680
|
+
}
|
|
11681
|
+
|
|
11682
|
+
function lmstudioJsonStringEnd(buffer, start) {
|
|
11683
|
+
if (buffer[start] !== 0x22) return -1;
|
|
11684
|
+
let escaped = false;
|
|
11685
|
+
for (let index = start + 1; index < buffer.length; index++) {
|
|
11686
|
+
const byte = buffer[index];
|
|
11687
|
+
if (escaped) escaped = false;
|
|
11688
|
+
else if (byte === 0x5c) escaped = true;
|
|
11689
|
+
else if (byte === 0x22) return index + 1;
|
|
11690
|
+
}
|
|
11691
|
+
return -1;
|
|
11692
|
+
}
|
|
11693
|
+
|
|
11694
|
+
function lastLmstudioJsonStringField(buffer, field) {
|
|
11695
|
+
const markerBuffer = Buffer.from(`"${field}"`);
|
|
11696
|
+
let cursor = 0;
|
|
11697
|
+
let found = null;
|
|
11698
|
+
while (cursor < buffer.length) {
|
|
11699
|
+
const index = buffer.indexOf(markerBuffer, cursor);
|
|
11700
|
+
if (index < 0) break;
|
|
11701
|
+
cursor = index + markerBuffer.length;
|
|
11702
|
+
if (index > 0 && buffer[index - 1] === 0x5c) continue;
|
|
11703
|
+
let valueStart = skipLmstudioWhitespace(buffer, cursor);
|
|
11704
|
+
if (buffer[valueStart] !== 0x3a) continue;
|
|
11705
|
+
valueStart = skipLmstudioWhitespace(buffer, valueStart + 1);
|
|
11706
|
+
const valueEnd = lmstudioJsonStringEnd(buffer, valueStart);
|
|
11707
|
+
if (valueEnd < 0) continue;
|
|
11708
|
+
try {
|
|
11709
|
+
const value = JSON.parse(buffer.subarray(valueStart, valueEnd).toString("utf8"));
|
|
11710
|
+
if (typeof value === "string") found = value;
|
|
11711
|
+
} catch (_e) { }
|
|
11712
|
+
}
|
|
11713
|
+
return found;
|
|
11714
|
+
}
|
|
11715
|
+
|
|
11716
|
+
function lastLmstudioTimestamp(buffer) {
|
|
11717
|
+
const text = buffer.toString("utf8");
|
|
11718
|
+
let timestamp = null;
|
|
11719
|
+
for (const rawLine of text.split("\n")) {
|
|
11720
|
+
const line = rawLine.startsWith("\r") ? rawLine.slice(1) : rawLine;
|
|
11721
|
+
const match = /^\[(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\]\[/.exec(line);
|
|
11722
|
+
if (!match) continue;
|
|
11723
|
+
const date = new Date(
|
|
11724
|
+
Number(match[1]),
|
|
11725
|
+
Number(match[2]) - 1,
|
|
11726
|
+
Number(match[3]),
|
|
11727
|
+
Number(match[4]),
|
|
11728
|
+
Number(match[5]),
|
|
11729
|
+
Number(match[6]),
|
|
11730
|
+
);
|
|
11731
|
+
if (!Number.isNaN(date.getTime())) timestamp = date.getTime();
|
|
11732
|
+
}
|
|
11733
|
+
return timestamp;
|
|
11734
|
+
}
|
|
11735
|
+
|
|
11736
|
+
function normalizeLocalStudioTokens(usage) {
|
|
11737
|
+
if (!usage || typeof usage !== "object") return null;
|
|
11738
|
+
const prompt = toNonNegativeInt(
|
|
11739
|
+
usage.prompt_tokens ?? usage.promptTokens ?? usage.input_tokens ?? usage.inputTokens,
|
|
11740
|
+
);
|
|
11741
|
+
const completion = toNonNegativeInt(
|
|
11742
|
+
usage.completion_tokens ?? usage.completionTokens ?? usage.output_tokens ?? usage.outputTokens,
|
|
11743
|
+
);
|
|
11744
|
+
const total = Math.max(
|
|
11745
|
+
toNonNegativeInt(usage.total_tokens ?? usage.totalTokens),
|
|
11746
|
+
prompt + completion,
|
|
11747
|
+
);
|
|
11748
|
+
if (total <= 0) return null;
|
|
11749
|
+
const promptDetails = usage.prompt_tokens_details
|
|
11750
|
+
|| usage.input_tokens_details
|
|
11751
|
+
|| usage.inputTokensDetails
|
|
11752
|
+
|| {};
|
|
11753
|
+
const outputDetails = usage.completion_tokens_details
|
|
11754
|
+
|| usage.output_tokens_details
|
|
11755
|
+
|| usage.completionTokensDetails
|
|
11756
|
+
|| usage.outputTokensDetails
|
|
11757
|
+
|| {};
|
|
11758
|
+
const cacheRead = Math.min(
|
|
11759
|
+
prompt,
|
|
11760
|
+
Math.max(
|
|
11761
|
+
toNonNegativeInt(promptDetails.cached_tokens ?? promptDetails.cache_read_tokens),
|
|
11762
|
+
toNonNegativeInt(usage.cached_tokens),
|
|
11763
|
+
),
|
|
11764
|
+
);
|
|
11765
|
+
const cacheWrite = Math.min(
|
|
11766
|
+
Math.max(0, prompt - cacheRead),
|
|
11767
|
+
Math.max(
|
|
11768
|
+
toNonNegativeInt(
|
|
11769
|
+
promptDetails.cache_creation_input_tokens ?? promptDetails.cache_write_tokens,
|
|
11770
|
+
),
|
|
11771
|
+
toNonNegativeInt(usage.cache_creation_input_tokens),
|
|
11772
|
+
),
|
|
11773
|
+
);
|
|
11774
|
+
const reasoning = Math.min(
|
|
11775
|
+
completion,
|
|
11776
|
+
Math.max(
|
|
11777
|
+
toNonNegativeInt(outputDetails.reasoning_tokens),
|
|
11778
|
+
toNonNegativeInt(usage.reasoning_tokens),
|
|
11779
|
+
),
|
|
11780
|
+
);
|
|
11781
|
+
return {
|
|
11782
|
+
input_tokens: Math.max(0, total - completion - cacheRead - cacheWrite),
|
|
11783
|
+
cached_input_tokens: cacheRead,
|
|
11784
|
+
cache_creation_input_tokens: cacheWrite,
|
|
11785
|
+
output_tokens: Math.max(0, completion - reasoning),
|
|
11786
|
+
reasoning_output_tokens: reasoning,
|
|
11787
|
+
total_tokens: total,
|
|
11788
|
+
billable_total_tokens: total,
|
|
11789
|
+
total_cost_usd: 0,
|
|
11790
|
+
conversation_count: 1,
|
|
11791
|
+
};
|
|
11792
|
+
}
|
|
11793
|
+
|
|
11794
|
+
function lmstudioResponseId(buffer) {
|
|
11795
|
+
const id = lastLmstudioJsonStringField(buffer, "id");
|
|
11796
|
+
return typeof id === "string" && ["chatcmpl-", "cmpl-", "resp_"].some((prefix) => id.startsWith(prefix))
|
|
11797
|
+
? id
|
|
11798
|
+
: null;
|
|
11799
|
+
}
|
|
11800
|
+
|
|
11801
|
+
function absorbLmstudioIdentity(identity, buffer) {
|
|
11802
|
+
const responseId = lmstudioResponseId(buffer);
|
|
11803
|
+
const model = lastLmstudioJsonStringField(buffer, "model");
|
|
11804
|
+
const timestamp = lastLmstudioTimestamp(buffer);
|
|
11805
|
+
if (responseId) identity.responseId = responseId;
|
|
11806
|
+
if (model && model.trim()) identity.model = model.trim();
|
|
11807
|
+
if (timestamp) identity.timestamp = timestamp;
|
|
11808
|
+
}
|
|
11809
|
+
|
|
11810
|
+
function normalizeLmstudioResumeIdentity(value) {
|
|
11811
|
+
const identity = {};
|
|
11812
|
+
const responseId = typeof value?.responseId === "string" ? value.responseId : "";
|
|
11813
|
+
if (["chatcmpl-", "cmpl-", "resp_"].some((prefix) => responseId.startsWith(prefix))) {
|
|
11814
|
+
identity.responseId = responseId;
|
|
11815
|
+
}
|
|
11816
|
+
const model = typeof value?.model === "string" ? value.model.trim() : "";
|
|
11817
|
+
if (model) identity.model = model;
|
|
11818
|
+
const timestamp = Number(value?.timestamp);
|
|
11819
|
+
if (Number.isFinite(timestamp) && timestamp > 0) identity.timestamp = timestamp;
|
|
11820
|
+
return identity;
|
|
11821
|
+
}
|
|
11822
|
+
|
|
11823
|
+
function lmstudioRecordFromSlices({ usageBuffer, metadataBuffer, carried, filePath, marker, fallbackTimestamp }) {
|
|
11824
|
+
let usage;
|
|
11825
|
+
try {
|
|
11826
|
+
usage = JSON.parse(usageBuffer.toString("utf8"));
|
|
11827
|
+
} catch (_e) {
|
|
11828
|
+
return null;
|
|
11829
|
+
}
|
|
11830
|
+
const totals = normalizeLocalStudioTokens(usage);
|
|
11831
|
+
if (!totals) return null;
|
|
11832
|
+
const responseId = lmstudioResponseId(metadataBuffer) || carried.responseId || null;
|
|
11833
|
+
const model = normalizeModelInput(
|
|
11834
|
+
lastLmstudioJsonStringField(metadataBuffer, "model") || carried.model,
|
|
11835
|
+
) || DEFAULT_MODEL;
|
|
11836
|
+
const timestampMs = lastLmstudioTimestamp(metadataBuffer) || carried.timestamp || fallbackTimestamp;
|
|
11837
|
+
const bucketStart = timestampMs
|
|
11838
|
+
? toUtcHalfHourStart(new Date(timestampMs).toISOString())
|
|
11839
|
+
: null;
|
|
11840
|
+
if (!bucketStart) return null;
|
|
11841
|
+
const fallback = crypto.createHash("sha256")
|
|
11842
|
+
.update(filePath)
|
|
11843
|
+
.update("\0")
|
|
11844
|
+
.update(String(marker))
|
|
11845
|
+
.update("\0")
|
|
11846
|
+
.update(model)
|
|
11847
|
+
.update("\0")
|
|
11848
|
+
.update(totalsKey(totals))
|
|
11849
|
+
.digest("base64url");
|
|
11850
|
+
return {
|
|
11851
|
+
key: responseId ? `lmstudio:${responseId}` : `lmstudio:${fallback}`,
|
|
11852
|
+
model,
|
|
11853
|
+
bucketStart,
|
|
11854
|
+
totals,
|
|
11855
|
+
marker,
|
|
11856
|
+
};
|
|
11857
|
+
}
|
|
11858
|
+
|
|
11859
|
+
async function readLmstudioFileRecords(
|
|
11860
|
+
filePath,
|
|
11861
|
+
{
|
|
11862
|
+
windowBytes = LMSTUDIO_WINDOW_BYTES,
|
|
11863
|
+
chunkBytes = LMSTUDIO_CHUNK_BYTES,
|
|
11864
|
+
startOffset = 0,
|
|
11865
|
+
resumeIdentity,
|
|
11866
|
+
} = {},
|
|
11867
|
+
) {
|
|
11868
|
+
const handle = await fs.open(filePath, "r");
|
|
11869
|
+
try {
|
|
11870
|
+
const initialStat = await handle.stat();
|
|
11871
|
+
const fallbackTimestamp = initialStat.mtimeMs;
|
|
11872
|
+
const records = [];
|
|
11873
|
+
const initialOffset = Math.min(
|
|
11874
|
+
initialStat.size,
|
|
11875
|
+
Math.max(0, Number.isFinite(startOffset) ? Math.floor(startOffset) : 0),
|
|
11876
|
+
);
|
|
11877
|
+
let window = Buffer.alloc(0);
|
|
11878
|
+
let windowStart = initialOffset;
|
|
11879
|
+
let metadataStart = initialOffset;
|
|
11880
|
+
let scannedTo = initialOffset;
|
|
11881
|
+
let position = initialOffset;
|
|
11882
|
+
let carried = normalizeLmstudioResumeIdentity(resumeIdentity);
|
|
11883
|
+
const chunk = Buffer.alloc(Math.max(1, chunkBytes));
|
|
11884
|
+
|
|
11885
|
+
while (true) {
|
|
11886
|
+
const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
|
|
11887
|
+
if (bytesRead > 0) {
|
|
11888
|
+
window = Buffer.concat([window, chunk.subarray(0, bytesRead)]);
|
|
11889
|
+
position += bytesRead;
|
|
11890
|
+
}
|
|
11891
|
+
const atEof = bytesRead === 0;
|
|
11892
|
+
|
|
11893
|
+
while (true) {
|
|
11894
|
+
const scanFrom = Math.min(window.length, Math.max(0, scannedTo - windowStart));
|
|
11895
|
+
const scan = scanLmstudioUsageObjectStart(window, scanFrom);
|
|
11896
|
+
if (!scan.found) {
|
|
11897
|
+
scannedTo = windowStart + scan.certainTo;
|
|
11898
|
+
break;
|
|
11899
|
+
}
|
|
11900
|
+
const objectEnd = lmstudioBalancedObjectEnd(window, scan.found.objectStart);
|
|
11901
|
+
if (objectEnd < 0) {
|
|
11902
|
+
scannedTo = atEof
|
|
11903
|
+
? windowStart + window.length
|
|
11904
|
+
: windowStart + scan.found.marker;
|
|
11905
|
+
break;
|
|
11906
|
+
}
|
|
11907
|
+
scannedTo = windowStart + objectEnd;
|
|
11908
|
+
const absoluteMarker = windowStart + scan.found.marker;
|
|
11909
|
+
const absoluteEnd = windowStart + objectEnd;
|
|
11910
|
+
const metadataFrom = Math.min(
|
|
11911
|
+
scan.found.marker,
|
|
11912
|
+
Math.max(0, metadataStart - windowStart),
|
|
11913
|
+
);
|
|
11914
|
+
const record = lmstudioRecordFromSlices({
|
|
11915
|
+
usageBuffer: window.subarray(scan.found.objectStart, objectEnd),
|
|
11916
|
+
metadataBuffer: window.subarray(metadataFrom, scan.found.marker),
|
|
11917
|
+
carried,
|
|
11918
|
+
filePath,
|
|
11919
|
+
marker: absoluteMarker,
|
|
11920
|
+
fallbackTimestamp,
|
|
11921
|
+
});
|
|
11922
|
+
if (record) records.push(record);
|
|
11923
|
+
metadataStart = absoluteEnd;
|
|
11924
|
+
carried = {};
|
|
11925
|
+
}
|
|
11926
|
+
|
|
11927
|
+
const keepFrom = Math.min(window.length, Math.max(0, metadataStart - windowStart));
|
|
11928
|
+
if (keepFrom > 0) {
|
|
11929
|
+
window = window.subarray(keepFrom);
|
|
11930
|
+
windowStart += keepFrom;
|
|
11931
|
+
}
|
|
11932
|
+
|
|
11933
|
+
if (window.length > windowBytes) {
|
|
11934
|
+
const overflow = window.length - windowBytes;
|
|
11935
|
+
const absorbTo = Math.min(
|
|
11936
|
+
window.length,
|
|
11937
|
+
overflow + LMSTUDIO_IDENTITY_OVERLAP_BYTES,
|
|
11938
|
+
);
|
|
11939
|
+
absorbLmstudioIdentity(carried, window.subarray(0, absorbTo));
|
|
11940
|
+
window = window.subarray(overflow);
|
|
11941
|
+
windowStart += overflow;
|
|
11942
|
+
metadataStart = Math.max(metadataStart, windowStart);
|
|
11943
|
+
scannedTo = Math.max(scannedTo, windowStart);
|
|
11944
|
+
}
|
|
11945
|
+
|
|
11946
|
+
if (atEof) {
|
|
11947
|
+
const finalStat = await handle.stat().catch(() => initialStat);
|
|
11948
|
+
const nextIdentity = normalizeLmstudioResumeIdentity(carried);
|
|
11949
|
+
return {
|
|
11950
|
+
records,
|
|
11951
|
+
stat: {
|
|
11952
|
+
dev: finalStat.dev,
|
|
11953
|
+
ino: finalStat.ino,
|
|
11954
|
+
size: finalStat.size,
|
|
11955
|
+
mtimeMs: finalStat.size === position ? finalStat.mtimeMs : -1,
|
|
11956
|
+
resumeOffset: Math.max(metadataStart, position - windowBytes),
|
|
11957
|
+
...(Object.keys(nextIdentity).length > 0
|
|
11958
|
+
? { resumeIdentity: nextIdentity }
|
|
11959
|
+
: {}),
|
|
11960
|
+
},
|
|
11961
|
+
};
|
|
11962
|
+
}
|
|
11963
|
+
}
|
|
11964
|
+
} finally {
|
|
11965
|
+
await handle.close().catch(() => {});
|
|
11966
|
+
}
|
|
11967
|
+
}
|
|
11968
|
+
|
|
11969
|
+
function retainNewestPassiveMessages(messages, maxEntries) {
|
|
11970
|
+
const limit = Number.isInteger(maxEntries) && maxEntries > 0
|
|
11971
|
+
? maxEntries
|
|
11972
|
+
: LMSTUDIO_MESSAGE_LIMIT;
|
|
11973
|
+
const entries = Object.entries(messages);
|
|
11974
|
+
if (entries.length <= limit) return messages;
|
|
11975
|
+
entries.sort((left, right) => {
|
|
11976
|
+
const leftTime = Date.parse(left[1]?.updatedAt || left[1]?.bucketStart || "") || 0;
|
|
11977
|
+
const rightTime = Date.parse(right[1]?.updatedAt || right[1]?.bucketStart || "") || 0;
|
|
11978
|
+
if (leftTime !== rightTime) return rightTime - leftTime;
|
|
11979
|
+
return right[0].localeCompare(left[0]);
|
|
11980
|
+
});
|
|
11981
|
+
return Object.fromEntries(entries.slice(0, limit));
|
|
11982
|
+
}
|
|
11983
|
+
|
|
11984
|
+
function reconcilePassiveUsageEvent({ event, source, messages, hourlyState, touchedBuckets }) {
|
|
11985
|
+
const previous = messages[event.key];
|
|
11986
|
+
const unchanged = previous
|
|
11987
|
+
&& previous.model === event.model
|
|
11988
|
+
&& previous.bucketStart === event.bucketStart
|
|
11989
|
+
&& totalsKey(previous.totals) === totalsKey(event.totals);
|
|
11990
|
+
if (unchanged) return false;
|
|
11991
|
+
|
|
11992
|
+
if (previous?.model && previous?.bucketStart && previous?.totals) {
|
|
11993
|
+
const oldBucket = getHourlyBucket(hourlyState, source, previous.model, previous.bucketStart);
|
|
11994
|
+
subtractTotals(oldBucket.totals, previous.totals);
|
|
11995
|
+
touchedBuckets.add(bucketKey(source, previous.model, previous.bucketStart));
|
|
11996
|
+
}
|
|
11997
|
+
const bucket = getHourlyBucket(hourlyState, source, event.model, event.bucketStart);
|
|
11998
|
+
addTotals(bucket.totals, event.totals);
|
|
11999
|
+
touchedBuckets.add(bucketKey(source, event.model, event.bucketStart));
|
|
12000
|
+
messages[event.key] = {
|
|
12001
|
+
model: event.model,
|
|
12002
|
+
bucketStart: event.bucketStart,
|
|
12003
|
+
totals: event.totals,
|
|
12004
|
+
updatedAt: new Date().toISOString(),
|
|
12005
|
+
};
|
|
12006
|
+
return true;
|
|
12007
|
+
}
|
|
12008
|
+
|
|
12009
|
+
async function parseLmstudioIncremental({
|
|
12010
|
+
logFiles,
|
|
12011
|
+
cursors,
|
|
12012
|
+
queuePath,
|
|
12013
|
+
onProgress,
|
|
12014
|
+
messageLimit = LMSTUDIO_MESSAGE_LIMIT,
|
|
12015
|
+
readerOptions,
|
|
12016
|
+
} = {}) {
|
|
12017
|
+
await ensureDir(path.dirname(queuePath));
|
|
12018
|
+
const files = Array.isArray(logFiles) ? [...logFiles].sort((a, b) => a.localeCompare(b)) : [];
|
|
12019
|
+
const priorState = cursors.lmstudio && typeof cursors.lmstudio === "object"
|
|
12020
|
+
? cursors.lmstudio
|
|
12021
|
+
: {};
|
|
12022
|
+
let fileState = priorState.files && typeof priorState.files === "object"
|
|
12023
|
+
? { ...priorState.files }
|
|
12024
|
+
: {};
|
|
12025
|
+
let messages = priorState.messages && typeof priorState.messages === "object"
|
|
12026
|
+
? priorState.messages
|
|
12027
|
+
: {};
|
|
12028
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
12029
|
+
const touchedBuckets = new Set();
|
|
12030
|
+
const presentFiles = new Set(files);
|
|
12031
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
12032
|
+
const effectiveMessageLimit = Number.isInteger(messageLimit) && messageLimit > 0
|
|
12033
|
+
? messageLimit
|
|
12034
|
+
: LMSTUDIO_MESSAGE_LIMIT;
|
|
12035
|
+
let recordsProcessed = 0;
|
|
12036
|
+
let eventsAggregated = 0;
|
|
12037
|
+
let stateChanged = false;
|
|
12038
|
+
const readOptions = readerOptions && typeof readerOptions === "object" ? readerOptions : {};
|
|
12039
|
+
const filePlans = [];
|
|
12040
|
+
let allFilesReadable = true;
|
|
12041
|
+
|
|
12042
|
+
for (const filePath of files) {
|
|
12043
|
+
const previousFile = fileState[filePath];
|
|
12044
|
+
let stat;
|
|
12045
|
+
try {
|
|
12046
|
+
stat = await fs.stat(filePath);
|
|
12047
|
+
} catch (_e) {
|
|
12048
|
+
allFilesReadable = false;
|
|
12049
|
+
continue;
|
|
12050
|
+
}
|
|
12051
|
+
const unchanged = Boolean(previousFile
|
|
12052
|
+
&& previousFile.dev === stat.dev
|
|
12053
|
+
&& previousFile.ino === stat.ino
|
|
12054
|
+
&& previousFile.size === stat.size
|
|
12055
|
+
&& previousFile.mtimeMs === stat.mtimeMs
|
|
12056
|
+
&& Number.isFinite(previousFile.resumeOffset));
|
|
12057
|
+
const canResume = Boolean(previousFile
|
|
12058
|
+
&& previousFile.dev === stat.dev
|
|
12059
|
+
&& previousFile.ino === stat.ino
|
|
12060
|
+
&& (stat.size > previousFile.size || previousFile.mtimeMs === -1)
|
|
12061
|
+
&& Number.isFinite(previousFile.resumeOffset)
|
|
12062
|
+
&& previousFile.resumeOffset >= 0
|
|
12063
|
+
&& previousFile.resumeOffset <= stat.size);
|
|
12064
|
+
filePlans.push({ filePath, previousFile, unchanged, canResume });
|
|
12065
|
+
}
|
|
12066
|
+
|
|
12067
|
+
const requiresRebuild = filePlans.some(
|
|
12068
|
+
({ previousFile, unchanged, canResume }) => previousFile && !unchanged && !canResume,
|
|
12069
|
+
);
|
|
12070
|
+
if (requiresRebuild) {
|
|
12071
|
+
// Bounded message retention cannot safely deduplicate a replay from byte
|
|
12072
|
+
// zero. Rebuild every current LM Studio log in isolation, then replace only
|
|
12073
|
+
// this source's buckets while preserving their queue fingerprints.
|
|
12074
|
+
if (!allFilesReadable || filePlans.length !== files.length) {
|
|
12075
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
12076
|
+
}
|
|
12077
|
+
const rebuiltHourly = normalizeHourlyState(null);
|
|
12078
|
+
const rebuiltMessages = {};
|
|
12079
|
+
const rebuiltFiles = {};
|
|
12080
|
+
const rebuiltTouched = new Set();
|
|
12081
|
+
let rebuiltRecords = 0;
|
|
12082
|
+
let rebuiltEvents = 0;
|
|
12083
|
+
try {
|
|
12084
|
+
for (let index = 0; index < filePlans.length; index++) {
|
|
12085
|
+
const { filePath } = filePlans[index];
|
|
12086
|
+
const parsed = await readLmstudioFileRecords(filePath, {
|
|
12087
|
+
...readOptions,
|
|
12088
|
+
startOffset: 0,
|
|
12089
|
+
resumeIdentity: undefined,
|
|
12090
|
+
});
|
|
12091
|
+
for (const event of parsed.records) {
|
|
12092
|
+
if (reconcilePassiveUsageEvent({
|
|
12093
|
+
event,
|
|
12094
|
+
source: LMSTUDIO_SOURCE,
|
|
12095
|
+
messages: rebuiltMessages,
|
|
12096
|
+
hourlyState: rebuiltHourly,
|
|
12097
|
+
touchedBuckets: rebuiltTouched,
|
|
12098
|
+
})) rebuiltEvents += 1;
|
|
12099
|
+
}
|
|
12100
|
+
rebuiltFiles[filePath] = { ...parsed.stat, updatedAt: new Date().toISOString() };
|
|
12101
|
+
rebuiltRecords += 1;
|
|
12102
|
+
if (cb) {
|
|
12103
|
+
cb({
|
|
12104
|
+
index: index + 1,
|
|
12105
|
+
total: files.length,
|
|
12106
|
+
recordsProcessed: rebuiltRecords,
|
|
12107
|
+
eventsAggregated: rebuiltEvents,
|
|
12108
|
+
bucketsQueued: rebuiltTouched.size,
|
|
12109
|
+
});
|
|
12110
|
+
}
|
|
12111
|
+
}
|
|
12112
|
+
} catch (error) {
|
|
12113
|
+
if (process.env.TOKENTRACKER_DEBUG) {
|
|
12114
|
+
process.stderr.write(`[lmstudio] rebuild deferred: ${error?.message || error}\n`);
|
|
12115
|
+
}
|
|
12116
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
12117
|
+
}
|
|
12118
|
+
|
|
12119
|
+
for (const [key, bucket] of Object.entries(hourlyState.buckets || {})) {
|
|
12120
|
+
if (parseBucketKey(key).source !== LMSTUDIO_SOURCE || !bucket?.totals) continue;
|
|
12121
|
+
bucket.totals = initTotals();
|
|
12122
|
+
touchedBuckets.add(key);
|
|
12123
|
+
}
|
|
12124
|
+
for (const [key, bucket] of Object.entries(rebuiltHourly.buckets || {})) {
|
|
12125
|
+
const current = hourlyState.buckets[key];
|
|
12126
|
+
if (current && typeof current === "object") current.totals = cloneTotals(bucket.totals);
|
|
12127
|
+
else hourlyState.buckets[key] = bucket;
|
|
12128
|
+
touchedBuckets.add(key);
|
|
12129
|
+
}
|
|
12130
|
+
fileState = rebuiltFiles;
|
|
12131
|
+
messages = rebuiltMessages;
|
|
12132
|
+
recordsProcessed = rebuiltRecords;
|
|
12133
|
+
eventsAggregated = rebuiltEvents;
|
|
12134
|
+
stateChanged = true;
|
|
12135
|
+
} else {
|
|
12136
|
+
for (let index = 0; index < filePlans.length; index++) {
|
|
12137
|
+
const { filePath, previousFile, unchanged, canResume } = filePlans[index];
|
|
12138
|
+
if (!unchanged) {
|
|
12139
|
+
try {
|
|
12140
|
+
const parsed = await readLmstudioFileRecords(filePath, {
|
|
12141
|
+
...readOptions,
|
|
12142
|
+
startOffset: canResume ? previousFile.resumeOffset : 0,
|
|
12143
|
+
resumeIdentity: canResume ? previousFile.resumeIdentity : undefined,
|
|
12144
|
+
});
|
|
12145
|
+
for (const event of parsed.records) {
|
|
12146
|
+
if (reconcilePassiveUsageEvent({
|
|
12147
|
+
event,
|
|
12148
|
+
source: LMSTUDIO_SOURCE,
|
|
12149
|
+
messages,
|
|
12150
|
+
hourlyState,
|
|
12151
|
+
touchedBuckets,
|
|
12152
|
+
})) eventsAggregated += 1;
|
|
12153
|
+
}
|
|
12154
|
+
fileState[filePath] = { ...parsed.stat, updatedAt: new Date().toISOString() };
|
|
12155
|
+
recordsProcessed += 1;
|
|
12156
|
+
stateChanged = true;
|
|
12157
|
+
} catch (error) {
|
|
12158
|
+
if (process.env.TOKENTRACKER_DEBUG) {
|
|
12159
|
+
process.stderr.write(`[lmstudio] skipped ${filePath}: ${error?.message || error}\n`);
|
|
12160
|
+
}
|
|
12161
|
+
}
|
|
12162
|
+
}
|
|
12163
|
+
if (cb) {
|
|
12164
|
+
cb({
|
|
12165
|
+
index: index + 1,
|
|
12166
|
+
total: files.length,
|
|
12167
|
+
recordsProcessed,
|
|
12168
|
+
eventsAggregated,
|
|
12169
|
+
bucketsQueued: touchedBuckets.size,
|
|
12170
|
+
});
|
|
12171
|
+
}
|
|
12172
|
+
}
|
|
12173
|
+
}
|
|
12174
|
+
|
|
12175
|
+
for (const filePath of Object.keys(fileState)) {
|
|
12176
|
+
if (!presentFiles.has(filePath)) {
|
|
12177
|
+
delete fileState[filePath];
|
|
12178
|
+
stateChanged = true;
|
|
12179
|
+
}
|
|
12180
|
+
}
|
|
12181
|
+
if (
|
|
12182
|
+
!stateChanged
|
|
12183
|
+
&& touchedBuckets.size === 0
|
|
12184
|
+
&& Object.keys(messages).length <= effectiveMessageLimit
|
|
12185
|
+
) {
|
|
12186
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
12187
|
+
}
|
|
12188
|
+
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
12189
|
+
const updatedAt = new Date().toISOString();
|
|
12190
|
+
hourlyState.updatedAt = updatedAt;
|
|
12191
|
+
cursors.hourly = hourlyState;
|
|
12192
|
+
cursors.lmstudio = {
|
|
12193
|
+
files: fileState,
|
|
12194
|
+
messages: retainNewestPassiveMessages(messages, effectiveMessageLimit),
|
|
12195
|
+
updatedAt,
|
|
12196
|
+
};
|
|
12197
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
12198
|
+
}
|
|
12199
|
+
|
|
12200
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
12201
|
+
// Unsloth Studio
|
|
12202
|
+
//
|
|
12203
|
+
// Durable inference usage lives in `studio.db`. The SQL projections below
|
|
12204
|
+
// extract only stable IDs, timestamps, model identifiers, and scalar counters.
|
|
12205
|
+
// Message content, attachments, API subjects, credentials, and training data
|
|
12206
|
+
// never leave SQLite.
|
|
12207
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
12208
|
+
|
|
12209
|
+
const UNSLOTH_SOURCE = "unsloth";
|
|
12210
|
+
const UNSLOTH_SQL_PAGE_SIZE = 500;
|
|
12211
|
+
const UNSLOTH_CURSOR_OVERLAP_ROWS = 256;
|
|
12212
|
+
const UNSLOTH_METERED_PROVIDER_TYPES = new Set([
|
|
12213
|
+
"anthropic",
|
|
12214
|
+
"deepseek",
|
|
12215
|
+
"gemini",
|
|
12216
|
+
"huggingface",
|
|
12217
|
+
"kimi",
|
|
12218
|
+
"mistral",
|
|
12219
|
+
"openai",
|
|
12220
|
+
"openrouter",
|
|
12221
|
+
"qwen",
|
|
12222
|
+
]);
|
|
12223
|
+
|
|
12224
|
+
function resolveUnslothDbPath(env = process.env) {
|
|
12225
|
+
const override = typeof env.TOKENTRACKER_UNSLOTH_DB === "string"
|
|
12226
|
+
? env.TOKENTRACKER_UNSLOTH_DB.trim()
|
|
12227
|
+
: "";
|
|
12228
|
+
if (override) return path.resolve(override);
|
|
12229
|
+
const studioHome = typeof env.UNSLOTH_STUDIO_HOME === "string"
|
|
12230
|
+
? env.UNSLOTH_STUDIO_HOME.trim()
|
|
12231
|
+
: "";
|
|
12232
|
+
if (studioHome) return path.join(path.resolve(studioHome), "studio.db");
|
|
12233
|
+
return path.join(env.HOME || env.USERPROFILE || os.homedir(), ".unsloth", "studio", "studio.db");
|
|
12234
|
+
}
|
|
12235
|
+
|
|
12236
|
+
function unslothSqliteFingerprint(dbPath) {
|
|
12237
|
+
const fingerprint = sqliteSidecarFingerprint(dbPath);
|
|
12238
|
+
delete fingerprint["-shm"];
|
|
12239
|
+
return fingerprint;
|
|
12240
|
+
}
|
|
12241
|
+
|
|
12242
|
+
function unslothRowPosition(row) {
|
|
12243
|
+
const createdAt = row?.created_at == null ? "" : String(row.created_at);
|
|
12244
|
+
const id = row?.id == null ? "" : String(row.id);
|
|
12245
|
+
return createdAt && id ? { createdAt, id } : null;
|
|
12246
|
+
}
|
|
12247
|
+
|
|
12248
|
+
function unslothPositionClause(rowAlias, position, inclusive) {
|
|
12249
|
+
if (!position?.createdAt || !position?.id) return null;
|
|
12250
|
+
const createdAt = sqliteStringLiteral(position.createdAt);
|
|
12251
|
+
const id = sqliteStringLiteral(position.id);
|
|
12252
|
+
const idOperator = inclusive ? ">=" : ">";
|
|
12253
|
+
return `(${rowAlias}.created_at > ${createdAt} OR ` +
|
|
12254
|
+
`(${rowAlias}.created_at = ${createdAt} AND ${rowAlias}.id ${idOperator} ${id}))`;
|
|
12255
|
+
}
|
|
12256
|
+
|
|
12257
|
+
function readUnslothRowsPaged({
|
|
12258
|
+
dbPath,
|
|
12259
|
+
select,
|
|
12260
|
+
from,
|
|
12261
|
+
where,
|
|
12262
|
+
rowAlias,
|
|
12263
|
+
lowerBound,
|
|
12264
|
+
options,
|
|
12265
|
+
}) {
|
|
12266
|
+
const requestedPageSize = Number(options?.pageSize);
|
|
12267
|
+
const pageSize = Number.isInteger(requestedPageSize) && requestedPageSize > 0
|
|
12268
|
+
? Math.min(requestedPageSize, UNSLOTH_SQL_PAGE_SIZE)
|
|
12269
|
+
: UNSLOTH_SQL_PAGE_SIZE;
|
|
12270
|
+
const rows = [];
|
|
12271
|
+
let after = null;
|
|
12272
|
+
|
|
12273
|
+
while (true) {
|
|
12274
|
+
const clauses = [where, `${rowAlias}.created_at IS NOT NULL`, `${rowAlias}.id IS NOT NULL`];
|
|
12275
|
+
const positionClause = unslothPositionClause(rowAlias, after || lowerBound, !after);
|
|
12276
|
+
if (positionClause) clauses.push(positionClause);
|
|
12277
|
+
const page = readSqliteJsonRows(dbPath, `
|
|
12278
|
+
SELECT
|
|
12279
|
+
${select}
|
|
12280
|
+
FROM ${from}
|
|
12281
|
+
WHERE ${clauses.filter(Boolean).join(" AND ")}
|
|
12282
|
+
ORDER BY ${rowAlias}.created_at, ${rowAlias}.id
|
|
12283
|
+
LIMIT ${pageSize}
|
|
12284
|
+
`.trim(), options);
|
|
12285
|
+
rows.push(...page);
|
|
12286
|
+
if (page.length < pageSize) break;
|
|
12287
|
+
|
|
12288
|
+
const next = unslothRowPosition(page[page.length - 1]);
|
|
12289
|
+
if (!next || (
|
|
12290
|
+
after && next.createdAt === after.createdAt && next.id === after.id
|
|
12291
|
+
)) {
|
|
12292
|
+
throw new Error("Unsloth Studio pagination did not advance");
|
|
12293
|
+
}
|
|
12294
|
+
after = next;
|
|
12295
|
+
}
|
|
12296
|
+
|
|
12297
|
+
return rows;
|
|
12298
|
+
}
|
|
12299
|
+
|
|
12300
|
+
function readUnslothUsageRows(dbPath, sqliteOptions = {}, scanState = {}) {
|
|
12301
|
+
if (!dbPath || !fssync.existsSync(dbPath)) return [];
|
|
12302
|
+
const options = {
|
|
12303
|
+
label: "Unsloth Studio",
|
|
12304
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
12305
|
+
timeout: 30_000,
|
|
12306
|
+
readOnly: true,
|
|
12307
|
+
throwOnReadFailure: true,
|
|
12308
|
+
...sqliteOptions,
|
|
12309
|
+
};
|
|
12310
|
+
const tables = new Set(
|
|
12311
|
+
readSqliteJsonRows(
|
|
12312
|
+
dbPath,
|
|
12313
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name IN ('chat_messages','chat_threads','api_usage_events')",
|
|
12314
|
+
options,
|
|
12315
|
+
).map((row) => row?.name).filter(Boolean),
|
|
12316
|
+
);
|
|
12317
|
+
const rows = [];
|
|
12318
|
+
|
|
12319
|
+
if (tables.has("chat_messages")) {
|
|
12320
|
+
const joinThread = tables.has("chat_threads")
|
|
12321
|
+
? "LEFT JOIN chat_threads t ON t.id = m.thread_id"
|
|
12322
|
+
: "";
|
|
12323
|
+
const threadModel = tables.has("chat_threads") ? "t.model_id" : "NULL";
|
|
12324
|
+
const field = (jsonPath, alias) =>
|
|
12325
|
+
`CASE WHEN json_valid(m.metadata_json) THEN json_extract(m.metadata_json, '${jsonPath}') ELSE NULL END AS ${alias}`;
|
|
12326
|
+
// Re-read a bounded tail because Studio finalizes recent rows in place and
|
|
12327
|
+
// does not expose an updated_at column. Older immutable history stays behind
|
|
12328
|
+
// the persisted timestamp/ID overlap cursor.
|
|
12329
|
+
const chatRows = readUnslothRowsPaged({
|
|
12330
|
+
dbPath,
|
|
12331
|
+
select: `
|
|
12332
|
+
'chat' AS usage_kind,
|
|
12333
|
+
m.id,
|
|
12334
|
+
m.created_at,
|
|
12335
|
+
${field("$.responseDetails.responseModelId", "response_model")},
|
|
12336
|
+
${field("$.contextUsage.modelId", "requested_model")},
|
|
12337
|
+
${field("$.responseDetails.providerType", "provider_type")},
|
|
12338
|
+
${threadModel} AS fallback_model,
|
|
12339
|
+
${field("$.contextUsage.promptTokens", "prompt_tokens")},
|
|
12340
|
+
${field("$.contextUsage.completionTokens", "completion_tokens")},
|
|
12341
|
+
${field("$.contextUsage.totalTokens", "total_tokens")},
|
|
12342
|
+
${field("$.contextUsage.cachedTokens", "cached_tokens")},
|
|
12343
|
+
${field("$.contextUsage.cacheWriteTokens", "cache_write_tokens")},
|
|
12344
|
+
${field("$.contextUsage.reasoningTokens", "reasoning_tokens")}
|
|
12345
|
+
`.trim(),
|
|
12346
|
+
from: `chat_messages m ${joinThread}`,
|
|
12347
|
+
where: "m.role = 'assistant'",
|
|
12348
|
+
rowAlias: "m",
|
|
12349
|
+
lowerBound: scanState?.chat?.overlap,
|
|
12350
|
+
options,
|
|
12351
|
+
});
|
|
12352
|
+
rows.push(...chatRows);
|
|
12353
|
+
}
|
|
12354
|
+
|
|
12355
|
+
if (tables.has("api_usage_events")) {
|
|
12356
|
+
const columns = new Set(
|
|
12357
|
+
readSqliteJsonRows(dbPath, "PRAGMA table_info(api_usage_events)", options)
|
|
12358
|
+
.map((row) => row?.name)
|
|
12359
|
+
.filter(Boolean),
|
|
12360
|
+
);
|
|
12361
|
+
const required = ["id", "model", "prompt_tokens", "completion_tokens", "total_tokens", "created_at"];
|
|
12362
|
+
if (required.every((column) => columns.has(column))) {
|
|
12363
|
+
rows.push(...readUnslothRowsPaged({
|
|
12364
|
+
dbPath,
|
|
12365
|
+
select: `
|
|
12366
|
+
'api' AS usage_kind,
|
|
12367
|
+
e.id,
|
|
12368
|
+
e.created_at,
|
|
12369
|
+
e.model AS response_model,
|
|
12370
|
+
e.model AS requested_model,
|
|
12371
|
+
'local' AS provider_type,
|
|
12372
|
+
NULL AS fallback_model,
|
|
12373
|
+
e.prompt_tokens,
|
|
12374
|
+
e.completion_tokens,
|
|
12375
|
+
e.total_tokens,
|
|
12376
|
+
0 AS cached_tokens,
|
|
12377
|
+
0 AS cache_write_tokens,
|
|
12378
|
+
0 AS reasoning_tokens
|
|
12379
|
+
`.trim(),
|
|
12380
|
+
from: "api_usage_events e",
|
|
12381
|
+
where: "",
|
|
12382
|
+
rowAlias: "e",
|
|
12383
|
+
lowerBound: scanState?.api?.overlap,
|
|
12384
|
+
options,
|
|
12385
|
+
}));
|
|
12386
|
+
}
|
|
12387
|
+
}
|
|
12388
|
+
return rows;
|
|
12389
|
+
}
|
|
12390
|
+
|
|
12391
|
+
function qualifyUnslothModel(row) {
|
|
12392
|
+
const rawModel = normalizeModelInput(row?.response_model)
|
|
12393
|
+
|| normalizeModelInput(row?.requested_model)
|
|
12394
|
+
|| normalizeModelInput(row?.fallback_model)
|
|
12395
|
+
|| DEFAULT_MODEL;
|
|
12396
|
+
const providerType = normalizeMessageKeyPart(row?.provider_type).toLowerCase();
|
|
12397
|
+
if (row?.usage_kind === "api" || providerType === "local") {
|
|
12398
|
+
return `local/${rawModel}`;
|
|
12399
|
+
}
|
|
12400
|
+
if (!UNSLOTH_METERED_PROVIDER_TYPES.has(providerType)) {
|
|
12401
|
+
return `unpriced/${providerType || "unknown"}/${rawModel}`;
|
|
12402
|
+
}
|
|
12403
|
+
const lowerModel = rawModel.toLowerCase();
|
|
12404
|
+
return lowerModel.startsWith(`${providerType}/`)
|
|
12405
|
+
? rawModel
|
|
12406
|
+
: `${providerType}/${rawModel}`;
|
|
12407
|
+
}
|
|
12408
|
+
|
|
12409
|
+
function normalizeUnslothUsageRow(row) {
|
|
12410
|
+
const totals = normalizeLocalStudioTokens({
|
|
12411
|
+
prompt_tokens: row?.prompt_tokens,
|
|
12412
|
+
completion_tokens: row?.completion_tokens,
|
|
12413
|
+
total_tokens: row?.total_tokens,
|
|
12414
|
+
cached_tokens: row?.cached_tokens,
|
|
12415
|
+
cache_creation_input_tokens: row?.cache_write_tokens,
|
|
12416
|
+
reasoning_tokens: row?.reasoning_tokens,
|
|
12417
|
+
});
|
|
12418
|
+
const id = normalizeMessageKeyPart(row?.id == null ? "" : String(row.id));
|
|
12419
|
+
const timestampMs = coerceEpochMs(row?.created_at) || parseIsoTimestampMs(row?.created_at);
|
|
12420
|
+
const bucketStart = timestampMs
|
|
12421
|
+
? toUtcHalfHourStart(new Date(timestampMs).toISOString())
|
|
12422
|
+
: null;
|
|
12423
|
+
if (!id || !totals || !bucketStart) return null;
|
|
12424
|
+
const kind = row?.usage_kind === "api" ? "api" : "chat";
|
|
12425
|
+
return {
|
|
12426
|
+
key: `unsloth:${kind}:${id}`,
|
|
12427
|
+
model: qualifyUnslothModel(row),
|
|
12428
|
+
bucketStart,
|
|
12429
|
+
totals,
|
|
12430
|
+
};
|
|
12431
|
+
}
|
|
12432
|
+
|
|
12433
|
+
async function parseUnslothIncremental({
|
|
12434
|
+
dbPath,
|
|
12435
|
+
cursors,
|
|
12436
|
+
queuePath,
|
|
12437
|
+
onProgress,
|
|
12438
|
+
env,
|
|
12439
|
+
sqliteOptions,
|
|
12440
|
+
overlapRows = UNSLOTH_CURSOR_OVERLAP_ROWS,
|
|
12441
|
+
} = {}) {
|
|
12442
|
+
await ensureDir(path.dirname(queuePath));
|
|
12443
|
+
const resolvedDb = dbPath || resolveUnslothDbPath(env || process.env);
|
|
12444
|
+
const priorState = cursors.unsloth && typeof cursors.unsloth === "object"
|
|
12445
|
+
? cursors.unsloth
|
|
12446
|
+
: {};
|
|
12447
|
+
const messages = priorState.messages && typeof priorState.messages === "object"
|
|
12448
|
+
? priorState.messages
|
|
12449
|
+
: {};
|
|
12450
|
+
if (!resolvedDb || !fssync.existsSync(resolvedDb)) {
|
|
12451
|
+
cursors.unsloth = { ...priorState, messages, updatedAt: new Date().toISOString() };
|
|
12452
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
12453
|
+
}
|
|
12454
|
+
|
|
12455
|
+
const initialFingerprint = unslothSqliteFingerprint(resolvedDb);
|
|
12456
|
+
const hasBoundedScan = priorState.scan
|
|
12457
|
+
&& typeof priorState.scan === "object"
|
|
12458
|
+
&& Object.keys(messages).length <= UNSLOTH_CURSOR_OVERLAP_ROWS * 2;
|
|
12459
|
+
if (hasBoundedScan && sameSqliteFingerprint(initialFingerprint, priorState.fingerprint)) {
|
|
12460
|
+
return { recordsProcessed: 0, eventsAggregated: 0, bucketsQueued: 0 };
|
|
12461
|
+
}
|
|
12462
|
+
|
|
12463
|
+
const rows = readUnslothUsageRows(resolvedDb, sqliteOptions, priorState.scan);
|
|
12464
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
12465
|
+
const touchedBuckets = new Set();
|
|
12466
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
12467
|
+
let recordsProcessed = 0;
|
|
12468
|
+
let eventsAggregated = 0;
|
|
12469
|
+
for (let index = 0; index < rows.length; index++) {
|
|
12470
|
+
const event = normalizeUnslothUsageRow(rows[index]);
|
|
12471
|
+
recordsProcessed += 1;
|
|
12472
|
+
if (event && reconcilePassiveUsageEvent({
|
|
12473
|
+
event,
|
|
12474
|
+
source: UNSLOTH_SOURCE,
|
|
12475
|
+
messages,
|
|
12476
|
+
hourlyState,
|
|
12477
|
+
touchedBuckets,
|
|
12478
|
+
})) eventsAggregated += 1;
|
|
12479
|
+
if (cb) {
|
|
12480
|
+
cb({
|
|
12481
|
+
index: index + 1,
|
|
12482
|
+
total: rows.length,
|
|
12483
|
+
recordsProcessed,
|
|
12484
|
+
eventsAggregated,
|
|
12485
|
+
bucketsQueued: touchedBuckets.size,
|
|
12486
|
+
});
|
|
12487
|
+
}
|
|
12488
|
+
}
|
|
12489
|
+
|
|
12490
|
+
const retention = Number.isInteger(overlapRows) && overlapRows > 0
|
|
12491
|
+
? Math.min(overlapRows, UNSLOTH_CURSOR_OVERLAP_ROWS)
|
|
12492
|
+
: UNSLOTH_CURSOR_OVERLAP_ROWS;
|
|
12493
|
+
const nextScan = {};
|
|
12494
|
+
const retainedMessageKeys = new Set();
|
|
12495
|
+
for (const kind of ["chat", "api"]) {
|
|
12496
|
+
const kindRows = rows.filter((row) => (row?.usage_kind === "api" ? "api" : "chat") === kind);
|
|
12497
|
+
if (kindRows.length === 0) {
|
|
12498
|
+
if (priorState.scan?.[kind]) nextScan[kind] = priorState.scan[kind];
|
|
12499
|
+
for (const key of Object.keys(messages)) {
|
|
12500
|
+
if (key.startsWith(`unsloth:${kind}:`)) retainedMessageKeys.add(key);
|
|
12501
|
+
}
|
|
12502
|
+
continue;
|
|
12503
|
+
}
|
|
12504
|
+
const tail = kindRows.slice(-retention);
|
|
12505
|
+
const overlap = unslothRowPosition(tail[0]);
|
|
12506
|
+
const latest = unslothRowPosition(kindRows[kindRows.length - 1]);
|
|
12507
|
+
if (overlap && latest) nextScan[kind] = { overlap, latest };
|
|
12508
|
+
for (const row of tail) {
|
|
12509
|
+
const event = normalizeUnslothUsageRow(row);
|
|
12510
|
+
if (event) retainedMessageKeys.add(event.key);
|
|
12511
|
+
}
|
|
12512
|
+
}
|
|
12513
|
+
for (const key of Object.keys(messages)) {
|
|
12514
|
+
if (key.startsWith("unsloth:") && !retainedMessageKeys.has(key)) delete messages[key];
|
|
12515
|
+
}
|
|
12516
|
+
|
|
12517
|
+
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
12518
|
+
const finalFingerprint = unslothSqliteFingerprint(resolvedDb);
|
|
12519
|
+
const updatedAt = new Date().toISOString();
|
|
12520
|
+
hourlyState.updatedAt = updatedAt;
|
|
12521
|
+
cursors.hourly = hourlyState;
|
|
12522
|
+
cursors.unsloth = {
|
|
12523
|
+
messages,
|
|
12524
|
+
scan: nextScan,
|
|
12525
|
+
fingerprint: sameSqliteFingerprint(initialFingerprint, finalFingerprint)
|
|
12526
|
+
? finalFingerprint
|
|
12527
|
+
: initialFingerprint,
|
|
12528
|
+
updatedAt,
|
|
12529
|
+
};
|
|
12530
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
12531
|
+
}
|
|
12532
|
+
|
|
11163
12533
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
11164
12534
|
// AnythingLLM Desktop (Mintplex Labs)
|
|
11165
12535
|
//
|
|
@@ -14251,6 +15621,13 @@ async function migrateCopilotChatLogRecordDedup({
|
|
|
14251
15621
|
touchedBuckets,
|
|
14252
15622
|
seenIds,
|
|
14253
15623
|
} = {}) {
|
|
15624
|
+
// A v2 cursor can retain an offset for a rotated or deleted OTEL file that
|
|
15625
|
+
// is no longer returned by discovery. Its historical contribution cannot be
|
|
15626
|
+
// reconciled, but the stale offset must not block migration of new files.
|
|
15627
|
+
const availableFiles = new Set(Array.isArray(files) ? files : []);
|
|
15628
|
+
for (const filePath of Object.keys(fileOffsets || {})) {
|
|
15629
|
+
if (!availableFiles.has(filePath)) delete fileOffsets[filePath];
|
|
15630
|
+
}
|
|
14254
15631
|
const trackedFiles = Object.keys(fileOffsets || {}).filter(
|
|
14255
15632
|
(filePath) => Number(fileOffsets[filePath]?.size) > 0,
|
|
14256
15633
|
);
|
|
@@ -14258,11 +15635,6 @@ async function migrateCopilotChatLogRecordDedup({
|
|
|
14258
15635
|
return { applied: true, changed: false };
|
|
14259
15636
|
}
|
|
14260
15637
|
|
|
14261
|
-
const availableFiles = new Set(Array.isArray(files) ? files : []);
|
|
14262
|
-
if (trackedFiles.some((filePath) => !availableFiles.has(filePath))) {
|
|
14263
|
-
return { applied: false, reason: "a previously parsed OTEL file is unavailable" };
|
|
14264
|
-
}
|
|
14265
|
-
|
|
14266
15638
|
const oldSeen = new Set();
|
|
14267
15639
|
const newSeen = new Set();
|
|
14268
15640
|
const oldTotals = new Map();
|
|
@@ -14342,7 +15714,10 @@ async function migrateCopilotChatLogRecordDedup({
|
|
|
14342
15714
|
}
|
|
14343
15715
|
|
|
14344
15716
|
const keys = new Set([...oldTotals.keys(), ...newTotals.keys()]);
|
|
14345
|
-
|
|
15717
|
+
// Only keys that existed in the v2 contribution need coverage validation.
|
|
15718
|
+
// A key that exists only in newTotals is a newly recovered Chat request; its
|
|
15719
|
+
// old contribution is zero, so an absent old bucket is expected.
|
|
15720
|
+
for (const key of oldTotals.keys()) {
|
|
14346
15721
|
const [model, bucketStart] = JSON.parse(key);
|
|
14347
15722
|
const oldUsage = oldTotals.get(key) || initTotals();
|
|
14348
15723
|
if (!copilotTotalsCover(
|
|
@@ -17417,6 +18792,7 @@ async function parseAntigravityFile({
|
|
|
17417
18792
|
// Match the mainstream convention (Codebuddy / Kilocode / OMP / Hermes):
|
|
17418
18793
|
// total_tokens = sum of every token column. No cache columns here.
|
|
17419
18794
|
delta.total_tokens = inputDelta + outputTokens + reasoningTokens;
|
|
18795
|
+
delta.billable_total_tokens = delta.total_tokens;
|
|
17420
18796
|
delta.conversation_count = 1;
|
|
17421
18797
|
billedPlanner = delta.total_tokens > 0;
|
|
17422
18798
|
}
|
|
@@ -18866,6 +20242,10 @@ module.exports = {
|
|
|
18866
20242
|
resolveQoderDbPaths,
|
|
18867
20243
|
resolveQoderCnDbPaths,
|
|
18868
20244
|
readQoderDbMessages,
|
|
20245
|
+
resolveQoderProjectsDir,
|
|
20246
|
+
resolveQoderCnProjectsDir,
|
|
20247
|
+
listQoderNewSessionFiles,
|
|
20248
|
+
parseQoderNewIncremental,
|
|
18869
20249
|
resolveKiroBasePath,
|
|
18870
20250
|
resolveKiroDbPath,
|
|
18871
20251
|
resolveKiroJsonlPath,
|
|
@@ -18955,6 +20335,15 @@ module.exports = {
|
|
|
18955
20335
|
sumZedRequestUsage,
|
|
18956
20336
|
readZedUsage,
|
|
18957
20337
|
parseZedIncremental,
|
|
20338
|
+
resolveLmstudioHome,
|
|
20339
|
+
resolveLmstudioLogFiles,
|
|
20340
|
+
normalizeLocalStudioTokens,
|
|
20341
|
+
readLmstudioFileRecords,
|
|
20342
|
+
parseLmstudioIncremental,
|
|
20343
|
+
resolveUnslothDbPath,
|
|
20344
|
+
readUnslothUsageRows,
|
|
20345
|
+
normalizeUnslothUsageRow,
|
|
20346
|
+
parseUnslothIncremental,
|
|
18958
20347
|
resolveAnythingllmDbPath,
|
|
18959
20348
|
parseAnythingllmTimestamp,
|
|
18960
20349
|
readAnythingllmUsageRows,
|
|
@@ -19000,6 +20389,7 @@ module.exports = {
|
|
|
19000
20389
|
normalizeGeminiTokens,
|
|
19001
20390
|
normalizeOpencodeTokens,
|
|
19002
20391
|
normalizeQoderTokens,
|
|
20392
|
+
normalizeQoderNewTokens,
|
|
19003
20393
|
sameGeminiTotals,
|
|
19004
20394
|
diffGeminiTotals,
|
|
19005
20395
|
// Exposed so the queue-repair migration can mutate cursors state in the
|