tokentracker-cli 0.88.6 → 0.88.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +4 -4
- package/README.ja.md +3 -3
- package/README.ko.md +3 -3
- package/README.md +6 -5
- package/README.zh-CN.md +5 -4
- package/dashboard/dist/assets/{AchievementBadge-CgVttPJy.js → AchievementBadge-CzfK_8kl.js} +1 -1
- package/dashboard/dist/assets/{AchievementsPage-2MeyWMmO.js → AchievementsPage-PVft5lyI.js} +2 -2
- package/dashboard/dist/assets/{AchievementsSection-hZGNcWfs.js → AchievementsSection-DyzHGzys.js} +1 -1
- package/dashboard/dist/assets/{BadgeDetailModal-DBMexTtI.js → BadgeDetailModal-CFsRpXG1.js} +1 -1
- package/dashboard/dist/assets/{Card-Ca6NJlW2.js → Card-Bi9dmQTP.js} +1 -1
- package/dashboard/dist/assets/{ClawdAnimated-D5GOjZtF.js → ClawdAnimated-BxcZjV9b.js} +1 -1
- package/dashboard/dist/assets/{CommandPalette-CPLhHuIS.js → CommandPalette-DHVK5yTu.js} +1 -1
- package/dashboard/dist/assets/{CommunityStatsModal-BFyZtlgh.js → CommunityStatsModal-BmMh7Avp.js} +1 -1
- package/dashboard/dist/assets/{ConfirmModal-DzpUyVbg.js → ConfirmModal-BjTQXMg4.js} +1 -1
- package/dashboard/dist/assets/{Controls-CIPVemCR.js → Controls-CZUbEggT.js} +1 -1
- package/dashboard/dist/assets/DashboardPage-C3x4Hew6.js +1 -0
- package/dashboard/dist/assets/{DevicePage-CORcVYoS.js → DevicePage-1qf6aiEH.js} +1 -1
- package/dashboard/dist/assets/{DialogClose-BMizP3FN.js → DialogClose-CRrlkVf0.js} +1 -1
- package/dashboard/dist/assets/DialogDescription-tD0gEESM.js +1 -0
- package/dashboard/dist/assets/{DialogTitle-CAlyWmQy.js → DialogTitle-DhGyZ0Ue.js} +1 -1
- package/dashboard/dist/assets/{FadeIn-R8uHDC1n.js → FadeIn-M8ECNbPG.js} +1 -1
- package/dashboard/dist/assets/{HeaderGithubStar-D-H9BYBq.js → HeaderGithubStar-BKtQc1Si.js} +1 -1
- package/dashboard/dist/assets/{HoverTooltip-C6D0Qf3J.js → HoverTooltip-Bo1zVtke.js} +1 -1
- package/dashboard/dist/assets/{IpCheckPage-BDKN_XOv.js → IpCheckPage-B78HcgA1.js} +1 -1
- package/dashboard/dist/assets/{LandingPage-CVS7Gi16.js → LandingPage-DSlanH3Y.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardAvatar-By9ZRxCx.js → LeaderboardAvatar-Cs4IYwTC.js} +1 -1
- package/dashboard/dist/assets/{LeaderboardPage-B2FWLDbM.js → LeaderboardPage-DXlo-PNE.js} +2 -2
- package/dashboard/dist/assets/{LeaderboardProfileModal-bKfAVxsg.js → LeaderboardProfileModal-CVM6FjLt.js} +3 -3
- package/dashboard/dist/assets/{LeaderboardProfilePage-NSZpmCOS.js → LeaderboardProfilePage-BO-MTMcH.js} +1 -1
- package/dashboard/dist/assets/{LimitsPage-DEeKWZ3u.js → LimitsPage-BohIYn9J.js} +1 -1
- package/dashboard/dist/assets/{LocalOnlyNotice-CJRoGtCr.js → LocalOnlyNotice-BQ7eYwB8.js} +1 -1
- package/dashboard/dist/assets/{LoginCard-DsotnrVc.js → LoginCard-DhOJyCcc.js} +1 -1
- package/dashboard/dist/assets/{LoginModal-CBzSJfD-.js → LoginModal-Dfo2DSA9.js} +1 -1
- package/dashboard/dist/assets/{LoginPage-Dr7dP5R6.js → LoginPage-CN-cXdsD.js} +1 -1
- package/dashboard/dist/assets/{PetPage-D_FQtbbK.js → PetPage-CQY2YUNm.js} +1 -1
- package/dashboard/dist/assets/{PopoverPopup-BBIGg4i-.js → PopoverPopup-D0bYgqkq.js} +1 -1
- package/dashboard/dist/assets/{ProviderIcon-C6NT8xCP.js → ProviderIcon-BfiriW8d.js} +1 -1
- package/dashboard/dist/assets/{ResetPasswordPage-BbDZdmyR.js → ResetPasswordPage-CQyX1BF6.js} +1 -1
- package/dashboard/dist/assets/{SegmentedControl-C0YbTO7C.js → SegmentedControl-DWhf_XGR.js} +1 -1
- package/dashboard/dist/assets/{Select-CO90ysTb.js → Select-Dy6krqIZ.js} +1 -1
- package/dashboard/dist/assets/{SelectItemText-CI5GNKhv.js → SelectItemText-D1296h92.js} +1 -1
- package/dashboard/dist/assets/{ServiceStatusPage-CV71dVHd.js → ServiceStatusPage-BgC3i1y-.js} +1 -1
- package/dashboard/dist/assets/{SessionsPage-B5jvyh-X.js → SessionsPage-Dyq0Q8tm.js} +1 -1
- package/dashboard/dist/assets/{SettingsPage-DokT0qhX.js → SettingsPage-QCZksfny.js} +1 -1
- package/dashboard/dist/assets/{SkillsPage-BYAAiPpI.js → SkillsPage-BUWMr2_o.js} +1 -1
- package/dashboard/dist/assets/{TokenGalaxy-m_5kxlC_.js → TokenGalaxy-BHauFZfa.js} +1 -1
- package/dashboard/dist/assets/{ToolbarRootContext-TYR3c0G7.js → ToolbarRootContext-1sViYfay.js} +1 -1
- package/dashboard/dist/assets/{TrendMonitor-DjVbRbLd.js → TrendMonitor-DM26Zn7d.js} +1 -1
- package/dashboard/dist/assets/{WidgetsPage-DwImvB2Y.js → WidgetsPage-DhjPczRN.js} +1 -1
- package/dashboard/dist/assets/{WrappedPage-CEcqtP_W.js → WrappedPage-BsEFzfrx.js} +1 -1
- package/dashboard/dist/assets/{__vite-browser-external-BOpcJ7Bn.js → __vite-browser-external-BW4ueyBB.js} +6 -3
- package/dashboard/dist/assets/agent-logos-IRTgTlkV.js +1 -0
- package/dashboard/dist/assets/{arrow-up-right-C0nGIX35.js → arrow-up-right-BapNuXPQ.js} +1 -1
- package/dashboard/dist/assets/{check-Cpk8oXcn.js → check-lpIv67Hh.js} +1 -1
- package/dashboard/dist/assets/{chevron-right-DsKtZU0a.js → chevron-right-jYAPg2OX.js} +1 -1
- package/dashboard/dist/assets/{copy-COeB2IQx.js → copy-CEdsXYF2.js} +1 -1
- package/dashboard/dist/assets/{download-Dvw5E6xA.js → download-CGJA0mZj.js} +1 -1
- package/dashboard/dist/assets/{external-link-COe8flh4.js → external-link-DTmmieCB.js} +1 -1
- package/dashboard/dist/assets/{flame-Dza_-WNE.js → flame-C-orSEmd.js} +1 -1
- package/dashboard/dist/assets/{format-tokens-CKDbxlqC.js → format-tokens-BQuK3UBV.js} +1 -1
- package/dashboard/dist/assets/{icons-Dj3nlViF.js → icons-DuVOMXCd.js} +1 -1
- package/dashboard/dist/assets/{index-DvR1xY_C.js → index-5hluBV6Y.js} +1 -1
- package/dashboard/dist/assets/{index-CBgWOlov.js → index-CKG-dDk-.js} +1 -1
- package/dashboard/dist/assets/{index-DCpYik68.js → index-DvLGX2GN.js} +1 -1
- package/dashboard/dist/assets/{info-B9M6Luq-.js → info-EKHZ3tlo.js} +1 -1
- package/dashboard/dist/assets/{limits-providers-CAZ_eIVT.js → limits-providers-DgnwvabV.js} +1 -1
- package/dashboard/dist/assets/{link-2-PL-NJZyN.js → link-2-A2f7492I.js} +1 -1
- package/dashboard/dist/assets/{loader-circle-CQ-akiLL.js → loader-circle-DiBM_0cy.js} +1 -1
- package/dashboard/dist/assets/{maximize-2-9EZoXvlF.js → maximize-2-CmJrrSbJ.js} +1 -1
- package/dashboard/dist/assets/{modifiers.esm-Bc1a9Bqm.js → modifiers.esm-v5P5znue.js} +1 -1
- package/dashboard/dist/assets/{monitor-smartphone-DR9MHu8R.js → monitor-smartphone-jmB0EaLx.js} +1 -1
- package/dashboard/dist/assets/{provider-display-DAH8KAyP.js → provider-display-BJXN_8Ft.js} +2 -2
- package/dashboard/dist/assets/{react-gkeEFJtr.js → react-DRhZt4b6.js} +1 -1
- package/dashboard/dist/assets/{search-B67Xwb-N.js → search-D9j_51N0.js} +1 -1
- package/dashboard/dist/assets/{skills-api-aZca3hZV.js → skills-api-Dywy4E7M.js} +1 -1
- package/dashboard/dist/assets/{sortable.esm-Crm57xTH.js → sortable.esm-BYYlH2M3.js} +1 -1
- package/dashboard/dist/assets/{store-DisKKZeI.js → store-Dt6GNo2E.js} +1 -1
- package/dashboard/dist/assets/{terminal-DGueqM2Q.js → terminal-BfWjr1gW.js} +1 -1
- package/dashboard/dist/assets/{trash-2-b5d9XdRV.js → trash-2-Ceal5Y6z.js} +1 -1
- package/dashboard/dist/assets/{use-community-stats-DOPcq80h.js → use-community-stats-2G4i1ZwJ.js} +1 -1
- package/dashboard/dist/assets/{use-limits-display-prefs-DGfPkvee.js → use-limits-display-prefs--qpfQM72.js} +1 -1
- package/dashboard/dist/assets/{use-ordered-list-Dyul6pNM.js → use-ordered-list-BRg8W35v.js} +1 -1
- package/dashboard/dist/assets/use-reduced-motion-BDYfqZDy.js +1 -0
- package/dashboard/dist/assets/{use-session-efficiency-pref-BPzM7u7L.js → use-session-efficiency-pref-DHwJtj_8.js} +1 -1
- package/dashboard/dist/assets/{use-transform-C5yhWdHM.js → use-transform-DaIhV0Zy.js} +1 -1
- package/dashboard/dist/assets/{use-usage-limits-C6S69l9t.js → use-usage-limits-COggDJdz.js} +1 -1
- package/dashboard/dist/assets/{useCurrency-BfJWbenA.js → useCurrency-0g-kQuZf.js} +1 -1
- package/dashboard/dist/assets/{useScrollLock-Btq-Jtjh.js → useScrollLock-CMMjcyJf.js} +1 -1
- package/dashboard/dist/assets/{useTokenFormat-DaXiI8qf.js → useTokenFormat-BuVZXkVC.js} +1 -1
- package/dashboard/dist/assets/{zap-DMOIXlpH.js → zap-BhYCdFZP.js} +1 -1
- package/dashboard/dist/brand-logos/reasonix.png +0 -0
- package/dashboard/dist/index.html +9 -9
- package/dashboard/dist/ip-check.html +1 -1
- package/dashboard/dist/leaderboard.html +1 -1
- package/dashboard/dist/llms.txt +3 -2
- package/dashboard/dist/share.html +1 -1
- package/package.json +2 -2
- package/src/commands/init.js +10 -0
- package/src/commands/status.js +27 -5
- package/src/commands/sync.js +177 -5
- package/src/lib/install-resolver.js +35 -0
- package/src/lib/opencode-go-limits.js +122 -13
- package/src/lib/pricing/seed-snapshot.json +1 -1
- package/src/lib/rollout.js +768 -0
- package/src/lib/subscriptions.js +1 -0
- package/src/lib/usage-limits.js +17 -2
- package/dashboard/dist/assets/DashboardPage-h2LfrNwp.js +0 -1
- package/dashboard/dist/assets/DialogDescription-B_xemJjT.js +0 -1
- package/dashboard/dist/assets/agent-logos-Cwc0bFNx.js +0 -1
- package/dashboard/dist/assets/use-reduced-motion-UUonF8mY.js +0 -1
package/src/lib/rollout.js
CHANGED
|
@@ -12708,6 +12708,140 @@ function resolveCraftDefaultModel() {
|
|
|
12708
12708
|
return "craft-unknown";
|
|
12709
12709
|
}
|
|
12710
12710
|
|
|
12711
|
+
// Reasonix persists content-free cumulative usage beside each session JSONL.
|
|
12712
|
+
// Reading only these telemetry sidecars keeps prompts and tool output private.
|
|
12713
|
+
function resolveReasonixHome(env = process.env) {
|
|
12714
|
+
if (env.TOKENTRACKER_REASONIX_HOME) {
|
|
12715
|
+
return expandHomePath(env.TOKENTRACKER_REASONIX_HOME, env);
|
|
12716
|
+
}
|
|
12717
|
+
if (env.REASONIX_STATE_HOME) {
|
|
12718
|
+
return expandHomePath(env.REASONIX_STATE_HOME, env);
|
|
12719
|
+
}
|
|
12720
|
+
const home = env.HOME || require("node:os").homedir();
|
|
12721
|
+
return path.join(home, ".reasonix");
|
|
12722
|
+
}
|
|
12723
|
+
|
|
12724
|
+
function collectReasonixTelemetryFiles(dir, files) {
|
|
12725
|
+
if (!fssync.existsSync(dir)) return;
|
|
12726
|
+
let entries;
|
|
12727
|
+
try { entries = fssync.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
12728
|
+
for (const entry of entries) {
|
|
12729
|
+
const full = path.join(dir, entry.name);
|
|
12730
|
+
if (entry.isDirectory()) collectReasonixTelemetryFiles(full, files);
|
|
12731
|
+
else if (entry.isFile() && entry.name.endsWith(".jsonl.telemetry.json")) files.push(full);
|
|
12732
|
+
}
|
|
12733
|
+
}
|
|
12734
|
+
|
|
12735
|
+
function resolveReasonixTelemetryFiles(env = process.env) {
|
|
12736
|
+
const reasonixHome = resolveReasonixHome(env);
|
|
12737
|
+
const files = [];
|
|
12738
|
+
collectReasonixTelemetryFiles(path.join(reasonixHome, "projects"), files);
|
|
12739
|
+
collectReasonixTelemetryFiles(path.join(reasonixHome, "sessions"), files);
|
|
12740
|
+
return files.sort((a, b) => a.localeCompare(b));
|
|
12741
|
+
}
|
|
12742
|
+
|
|
12743
|
+
function readReasonixSnapshot(filePath) {
|
|
12744
|
+
const telemetry = JSON.parse(fssync.readFileSync(filePath, "utf8"));
|
|
12745
|
+
const usage = telemetry?.usage;
|
|
12746
|
+
if (!usage || typeof usage !== "object") return null;
|
|
12747
|
+
const metaPath = filePath.slice(0, -".telemetry.json".length) + ".meta";
|
|
12748
|
+
let meta = {};
|
|
12749
|
+
try { meta = JSON.parse(fssync.readFileSync(metaPath, "utf8")); } catch {}
|
|
12750
|
+
const stat = fssync.statSync(filePath);
|
|
12751
|
+
return { usage, meta, stat };
|
|
12752
|
+
}
|
|
12753
|
+
|
|
12754
|
+
function normalizeReasonixTotals(usage) {
|
|
12755
|
+
const prompt = toNonNegativeInt(usage.promptTokens);
|
|
12756
|
+
const reasoning = toNonNegativeInt(usage.reasoningTokens);
|
|
12757
|
+
const completion = toNonNegativeInt(usage.completionTokens);
|
|
12758
|
+
const cacheMiss = Math.min(prompt, toNonNegativeInt(usage.cacheMissTokens));
|
|
12759
|
+
const cacheHit = Math.min(prompt, toNonNegativeInt(usage.cacheHitTokens));
|
|
12760
|
+
const hasCacheMiss = usage.cacheMissTokens != null;
|
|
12761
|
+
const uncachedPrompt = hasCacheMiss ? cacheMiss : Math.max(0, prompt - cacheHit);
|
|
12762
|
+
const cacheWrite = Math.min(uncachedPrompt, toNonNegativeInt(usage.cacheWriteTokens));
|
|
12763
|
+
return {
|
|
12764
|
+
input: uncachedPrompt - cacheWrite,
|
|
12765
|
+
cacheRead: hasCacheMiss ? prompt - cacheMiss : cacheHit,
|
|
12766
|
+
cacheWrite,
|
|
12767
|
+
output: Math.max(0, completion - reasoning),
|
|
12768
|
+
reasoning,
|
|
12769
|
+
requests: toNonNegativeInt(usage.requestCount),
|
|
12770
|
+
};
|
|
12771
|
+
}
|
|
12772
|
+
|
|
12773
|
+
function diffReasonixTotals(current, previous = {}) {
|
|
12774
|
+
const delta = {};
|
|
12775
|
+
for (const key of ["input", "cacheRead", "cacheWrite", "output", "reasoning", "requests"]) {
|
|
12776
|
+
delta[key] = Math.max(0, current[key] - toNonNegativeInt(previous[key]));
|
|
12777
|
+
}
|
|
12778
|
+
return delta;
|
|
12779
|
+
}
|
|
12780
|
+
|
|
12781
|
+
function reasonixTimestamp(snapshot) {
|
|
12782
|
+
for (const value of [snapshot.meta.updated_at, snapshot.meta.created_at]) {
|
|
12783
|
+
const parsed = Date.parse(value);
|
|
12784
|
+
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
|
12785
|
+
}
|
|
12786
|
+
return snapshot.stat.mtimeMs;
|
|
12787
|
+
}
|
|
12788
|
+
|
|
12789
|
+
function normalizeReasonixModel(value) {
|
|
12790
|
+
const normalized = normalizeModelInput(value);
|
|
12791
|
+
if (!normalized) return "reasonix-unknown";
|
|
12792
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
12793
|
+
return segments.at(-1) || "reasonix-unknown";
|
|
12794
|
+
}
|
|
12795
|
+
|
|
12796
|
+
function addReasonixDelta(hourlyState, touchedBuckets, snapshot, delta) {
|
|
12797
|
+
const bucketStart = toUtcHalfHourStart(new Date(reasonixTimestamp(snapshot)).toISOString());
|
|
12798
|
+
if (!bucketStart) return false;
|
|
12799
|
+
const model = normalizeReasonixModel(snapshot.meta.model);
|
|
12800
|
+
const total = delta.input + delta.cacheRead + delta.cacheWrite + delta.output + delta.reasoning;
|
|
12801
|
+
const bucket = getHourlyBucket(hourlyState, "reasonix", model, bucketStart);
|
|
12802
|
+
addTotals(bucket.totals, {
|
|
12803
|
+
input_tokens: delta.input,
|
|
12804
|
+
cached_input_tokens: delta.cacheRead,
|
|
12805
|
+
cache_creation_input_tokens: delta.cacheWrite,
|
|
12806
|
+
output_tokens: delta.output,
|
|
12807
|
+
reasoning_output_tokens: delta.reasoning,
|
|
12808
|
+
total_tokens: total,
|
|
12809
|
+
conversation_count: delta.requests,
|
|
12810
|
+
});
|
|
12811
|
+
touchedBuckets.add(bucketKey("reasonix", model, bucketStart));
|
|
12812
|
+
return true;
|
|
12813
|
+
}
|
|
12814
|
+
|
|
12815
|
+
async function parseReasonixIncremental({ telemetryFiles, cursors, queuePath, onProgress, env } = {}) {
|
|
12816
|
+
await ensureDir(path.dirname(queuePath));
|
|
12817
|
+
const state = cursors.reasonix && typeof cursors.reasonix === "object" ? cursors.reasonix : {};
|
|
12818
|
+
const sessionTotals = { ...(state.sessionTotals || {}) };
|
|
12819
|
+
const files = Array.isArray(telemetryFiles) ? telemetryFiles : resolveReasonixTelemetryFiles(env);
|
|
12820
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
12821
|
+
const touchedBuckets = new Set();
|
|
12822
|
+
let recordsProcessed = 0;
|
|
12823
|
+
let eventsAggregated = 0;
|
|
12824
|
+
for (const [index, filePath] of files.entries()) {
|
|
12825
|
+
let snapshot;
|
|
12826
|
+
try { snapshot = readReasonixSnapshot(filePath); } catch { continue; }
|
|
12827
|
+
if (!snapshot) continue;
|
|
12828
|
+
recordsProcessed++;
|
|
12829
|
+
const current = normalizeReasonixTotals(snapshot.usage);
|
|
12830
|
+
const delta = diffReasonixTotals(current, sessionTotals[filePath]);
|
|
12831
|
+
const tokenDelta = delta.input + delta.cacheRead + delta.cacheWrite + delta.output + delta.reasoning;
|
|
12832
|
+
if ((tokenDelta > 0 || delta.requests > 0) &&
|
|
12833
|
+
addReasonixDelta(hourlyState, touchedBuckets, snapshot, delta)) eventsAggregated++;
|
|
12834
|
+
sessionTotals[filePath] = current;
|
|
12835
|
+
onProgress?.({ index: index + 1, total: files.length, recordsProcessed, eventsAggregated, bucketsQueued: touchedBuckets.size });
|
|
12836
|
+
}
|
|
12837
|
+
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
12838
|
+
const updatedAt = new Date().toISOString();
|
|
12839
|
+
hourlyState.updatedAt = updatedAt;
|
|
12840
|
+
cursors.hourly = hourlyState;
|
|
12841
|
+
cursors.reasonix = { ...state, sessionTotals, updatedAt };
|
|
12842
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
12843
|
+
}
|
|
12844
|
+
|
|
12711
12845
|
async function parseCraftIncremental({
|
|
12712
12846
|
sessionFiles,
|
|
12713
12847
|
cursors,
|
|
@@ -16244,6 +16378,626 @@ function readTraeEntitlementFromStorage(storagePath) {
|
|
|
16244
16378
|
};
|
|
16245
16379
|
}
|
|
16246
16380
|
|
|
16381
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
16382
|
+
// DeepSeek Harness (dsh) — passive reader of the harness's session logs.
|
|
16383
|
+
//
|
|
16384
|
+
// The DeepSeek Harness SDK persists each agent session as an append-only JSONL
|
|
16385
|
+
// log under `<harness-home>/sessions/<project-key>/<session-id>/session.jsonl[.zstd]`
|
|
16386
|
+
// (default `~/.dsh/sessions`; `$DSH_HOME` / TOKENTRACKER_DSH_HOME override). The first line is a `type: "session"` header;
|
|
16387
|
+
// every following line is a `SessionEvent` carrying `type`, `seq`, `time`
|
|
16388
|
+
// (epoch ms), and `data`. Token accounting rides on `assistant/message` events
|
|
16389
|
+
// (`data.usage`), and the model on the same event's `data.message.source.model`
|
|
16390
|
+
// (falling back to the most recent `request/header` config). Usage counts are
|
|
16391
|
+
// DISJOINT — `inputTokens` is uncached input only, cache reads/writes and
|
|
16392
|
+
// reasoning travel separately — so the mapping to our queue columns is direct
|
|
16393
|
+
// (no cache subtraction, unlike Codex).
|
|
16394
|
+
//
|
|
16395
|
+
// Zstd artifacts are a concatenated-frame container: Node's
|
|
16396
|
+
// `zlib.zstdDecompressSync` decodes only the first frame (silently dropping
|
|
16397
|
+
// every event line), so we identify the frame boundaries and feed each frame
|
|
16398
|
+
// through `@mongodb-js/zstd`, enforcing a cumulative plaintext bound as we go.
|
|
16399
|
+
// Dedup is a per-file `lastSeq` watermark — seq
|
|
16400
|
+
// is monotonic within a session log, so an append-only grow re-reads the file
|
|
16401
|
+
// and skips everything at or below the watermark; a torn-tail repair or full
|
|
16402
|
+
// rewrite re-reads the same seqs and is therefore idempotent.
|
|
16403
|
+
const DSH_SESSION_LOG_MAX_BYTES = 64 * 1024 * 1024;
|
|
16404
|
+
const DSH_SESSION_TEXT_MAX_BYTES = 128 * 1024 * 1024;
|
|
16405
|
+
const DSH_SOURCE = "dsh";
|
|
16406
|
+
|
|
16407
|
+
// Precedence mirrors the harness's own resolveDshHome: an explicit
|
|
16408
|
+
// TokenTracker override, then the harness's $DSH_HOME, then `~/.dsh`. Tests
|
|
16409
|
+
// isolate $DSH_HOME via the `withHome` helper, so honoring it here cannot leak
|
|
16410
|
+
// the real home past test isolation.
|
|
16411
|
+
function resolveDshHome(env = process.env) {
|
|
16412
|
+
const explicit =
|
|
16413
|
+
(typeof env?.TOKENTRACKER_DSH_HOME === "string" && env.TOKENTRACKER_DSH_HOME.trim()) ||
|
|
16414
|
+
(typeof env?.DSH_HOME === "string" && env.DSH_HOME.trim()) ||
|
|
16415
|
+
"";
|
|
16416
|
+
if (explicit) return path.resolve(explicit);
|
|
16417
|
+
return path.join(os.homedir(), ".dsh");
|
|
16418
|
+
}
|
|
16419
|
+
|
|
16420
|
+
function isDshSessionLogName(name) {
|
|
16421
|
+
return name === "session.jsonl" || name === "session.jsonl.zstd";
|
|
16422
|
+
}
|
|
16423
|
+
|
|
16424
|
+
// Walk the harness sessions root for per-session log artifacts. The tree is
|
|
16425
|
+
// `<sessions-root>/<project-key>/<session-id>/session.jsonl[.zstd]`; only the
|
|
16426
|
+
// exact leaf names are collected so unrelated harness files are ignored.
|
|
16427
|
+
async function resolveDshSessionFiles(env = process.env) {
|
|
16428
|
+
const sessionsRoot = path.join(resolveDshHome(env), "sessions");
|
|
16429
|
+
const out = [];
|
|
16430
|
+
const projects = await safeReadDir(sessionsRoot);
|
|
16431
|
+
for (const project of projects) {
|
|
16432
|
+
if (!project.isDirectory()) continue;
|
|
16433
|
+
const projectDir = path.join(sessionsRoot, project.name);
|
|
16434
|
+
const sessions = await safeReadDir(projectDir);
|
|
16435
|
+
for (const session of sessions) {
|
|
16436
|
+
if (!session.isDirectory()) continue;
|
|
16437
|
+
const sessionDir = path.join(projectDir, session.name);
|
|
16438
|
+
const artifacts = await safeReadDir(sessionDir);
|
|
16439
|
+
const transcripts = artifacts.filter(
|
|
16440
|
+
(artifact) => artifact.isFile() && isDshSessionLogName(artifact.name),
|
|
16441
|
+
);
|
|
16442
|
+
if (transcripts.length === 1) {
|
|
16443
|
+
out.push(path.join(sessionDir, transcripts[0].name));
|
|
16444
|
+
} else if (transcripts.length > 1) {
|
|
16445
|
+
// Harness itself rejects mixed encodings in one root. For a passive
|
|
16446
|
+
// reader, choose the actively-written artifact instead of counting the
|
|
16447
|
+
// same session twice; a tie prefers the default zstd encoding.
|
|
16448
|
+
const ranked = await Promise.all(transcripts.map(async (artifact) => {
|
|
16449
|
+
const full = path.join(sessionDir, artifact.name);
|
|
16450
|
+
const handle = await fs.open(full, "r").catch(() => null);
|
|
16451
|
+
if (!handle) return { full, name: artifact.name, mtimeMs: 0 };
|
|
16452
|
+
try {
|
|
16453
|
+
const stat = await handle.stat().catch(() => null);
|
|
16454
|
+
return {
|
|
16455
|
+
full,
|
|
16456
|
+
name: artifact.name,
|
|
16457
|
+
mtimeMs: stat?.isFile() ? Number(stat.mtimeMs || 0) : 0,
|
|
16458
|
+
};
|
|
16459
|
+
} finally {
|
|
16460
|
+
await handle.close().catch(() => {});
|
|
16461
|
+
}
|
|
16462
|
+
}));
|
|
16463
|
+
ranked.sort((left, right) =>
|
|
16464
|
+
right.mtimeMs - left.mtimeMs ||
|
|
16465
|
+
Number(right.name.endsWith(".zstd")) - Number(left.name.endsWith(".zstd")),
|
|
16466
|
+
);
|
|
16467
|
+
if (ranked[0]) out.push(ranked[0].full);
|
|
16468
|
+
}
|
|
16469
|
+
}
|
|
16470
|
+
}
|
|
16471
|
+
out.sort((a, b) => a.localeCompare(b));
|
|
16472
|
+
return out;
|
|
16473
|
+
}
|
|
16474
|
+
|
|
16475
|
+
// Inspect concatenated Zstandard frames without decompressing them. Harness
|
|
16476
|
+
// writes a content size into every independent append frame; summing those
|
|
16477
|
+
// declarations lets us reject a decompression bomb before the native decoder
|
|
16478
|
+
// allocates its plaintext buffer.
|
|
16479
|
+
function inspectDshZstdFrames(data, maxOutputBytes = DSH_SESSION_TEXT_MAX_BYTES) {
|
|
16480
|
+
const input = Buffer.isBuffer(data) ? data : Buffer.from(data || []);
|
|
16481
|
+
const limit = Number.isSafeInteger(maxOutputBytes) && maxOutputBytes >= 0
|
|
16482
|
+
? maxOutputBytes
|
|
16483
|
+
: DSH_SESSION_TEXT_MAX_BYTES;
|
|
16484
|
+
let offset = 0;
|
|
16485
|
+
let totalContentBytes = 0;
|
|
16486
|
+
let allContentSizesDeclared = true;
|
|
16487
|
+
let frames = 0;
|
|
16488
|
+
const frameRanges = [];
|
|
16489
|
+
|
|
16490
|
+
const need = (count, label) => {
|
|
16491
|
+
if (offset + count > input.length) throw new Error(`Invalid DeepSeek Harness zstd ${label}`);
|
|
16492
|
+
};
|
|
16493
|
+
const readUnsignedLE = (count) => {
|
|
16494
|
+
need(count, "frame header");
|
|
16495
|
+
let value = 0n;
|
|
16496
|
+
for (let i = 0; i < count; i++) value |= BigInt(input[offset + i]) << BigInt(i * 8);
|
|
16497
|
+
offset += count;
|
|
16498
|
+
return value;
|
|
16499
|
+
};
|
|
16500
|
+
|
|
16501
|
+
while (offset < input.length) {
|
|
16502
|
+
const frameStart = offset;
|
|
16503
|
+
need(4, "magic");
|
|
16504
|
+
const magic = input.readUInt32LE(offset);
|
|
16505
|
+
offset += 4;
|
|
16506
|
+
|
|
16507
|
+
if (magic >= 0x184d2a50 && magic <= 0x184d2a5f) {
|
|
16508
|
+
need(4, "skippable frame size");
|
|
16509
|
+
const skipBytes = input.readUInt32LE(offset);
|
|
16510
|
+
offset += 4;
|
|
16511
|
+
need(skipBytes, "skippable frame payload");
|
|
16512
|
+
offset += skipBytes;
|
|
16513
|
+
continue;
|
|
16514
|
+
}
|
|
16515
|
+
if (magic !== 0xfd2fb528) throw new Error("Invalid DeepSeek Harness zstd frame magic");
|
|
16516
|
+
|
|
16517
|
+
need(1, "descriptor");
|
|
16518
|
+
const descriptor = input[offset++];
|
|
16519
|
+
if ((descriptor & 0x08) !== 0) throw new Error("Invalid DeepSeek Harness zstd reserved frame bit");
|
|
16520
|
+
const contentSizeFlag = descriptor >>> 6;
|
|
16521
|
+
const singleSegment = (descriptor & 0x20) !== 0;
|
|
16522
|
+
const hasChecksum = (descriptor & 0x04) !== 0;
|
|
16523
|
+
const dictionaryIdBytes = [0, 1, 2, 4][descriptor & 0x03];
|
|
16524
|
+
let windowSize = null;
|
|
16525
|
+
if (!singleSegment) {
|
|
16526
|
+
need(1, "window descriptor");
|
|
16527
|
+
const windowDescriptor = input[offset++];
|
|
16528
|
+
const windowBase = 2 ** (10 + (windowDescriptor >>> 3));
|
|
16529
|
+
windowSize = windowBase + (windowBase / 8) * (windowDescriptor & 0x07);
|
|
16530
|
+
}
|
|
16531
|
+
need(dictionaryIdBytes, "dictionary id");
|
|
16532
|
+
offset += dictionaryIdBytes;
|
|
16533
|
+
|
|
16534
|
+
const contentSizeBytes = contentSizeFlag === 0
|
|
16535
|
+
? (singleSegment ? 1 : 0)
|
|
16536
|
+
: [0, 2, 4, 8][contentSizeFlag];
|
|
16537
|
+
let declaredContentBytes = null;
|
|
16538
|
+
if (contentSizeBytes > 0) {
|
|
16539
|
+
declaredContentBytes = readUnsignedLE(contentSizeBytes);
|
|
16540
|
+
if (contentSizeBytes === 2) declaredContentBytes += 256n;
|
|
16541
|
+
if (declaredContentBytes > BigInt(limit)) {
|
|
16542
|
+
throw new Error(`DeepSeek Harness decompressed session log exceeds ${limit} bytes`);
|
|
16543
|
+
}
|
|
16544
|
+
if (singleSegment) windowSize = Number(declaredContentBytes);
|
|
16545
|
+
} else {
|
|
16546
|
+
allContentSizesDeclared = false;
|
|
16547
|
+
}
|
|
16548
|
+
|
|
16549
|
+
let lastBlock = false;
|
|
16550
|
+
let frameUpperBound = 0;
|
|
16551
|
+
while (!lastBlock) {
|
|
16552
|
+
need(3, "block header");
|
|
16553
|
+
const blockHeader = input[offset] | (input[offset + 1] << 8) | (input[offset + 2] << 16);
|
|
16554
|
+
offset += 3;
|
|
16555
|
+
lastBlock = (blockHeader & 1) !== 0;
|
|
16556
|
+
const blockType = (blockHeader >>> 1) & 0x03;
|
|
16557
|
+
const blockSize = blockHeader >>> 3;
|
|
16558
|
+
if (blockType === 3) throw new Error("Invalid DeepSeek Harness zstd reserved block type");
|
|
16559
|
+
const payloadBytes = blockType === 1 ? 1 : blockSize;
|
|
16560
|
+
need(payloadBytes, "block payload");
|
|
16561
|
+
offset += payloadBytes;
|
|
16562
|
+
// A raw block emits blockSize bytes; an RLE block expands to blockSize
|
|
16563
|
+
// repeated bytes; a compressed block cannot emit more than the frame's
|
|
16564
|
+
// window or Zstandard's 128 KiB maximum block size. This gives frames
|
|
16565
|
+
// without FCS a safe allocation bound before native decompression.
|
|
16566
|
+
frameUpperBound += blockType === 2
|
|
16567
|
+
? Math.min(Number(windowSize) || 128 * 1024, 128 * 1024)
|
|
16568
|
+
: blockSize;
|
|
16569
|
+
if (declaredContentBytes == null && frameUpperBound > limit) {
|
|
16570
|
+
throw new Error(`DeepSeek Harness decompressed session log exceeds ${limit} bytes`);
|
|
16571
|
+
}
|
|
16572
|
+
}
|
|
16573
|
+
if (hasChecksum) {
|
|
16574
|
+
need(4, "checksum");
|
|
16575
|
+
offset += 4;
|
|
16576
|
+
}
|
|
16577
|
+
const boundedFrameBytes = declaredContentBytes == null
|
|
16578
|
+
? frameUpperBound
|
|
16579
|
+
: Number(declaredContentBytes);
|
|
16580
|
+
totalContentBytes += boundedFrameBytes;
|
|
16581
|
+
if (allContentSizesDeclared && totalContentBytes > limit) {
|
|
16582
|
+
throw new Error(`DeepSeek Harness decompressed session log exceeds ${limit} bytes`);
|
|
16583
|
+
}
|
|
16584
|
+
frameRanges.push({ start: frameStart, end: offset, maxContentBytes: boundedFrameBytes });
|
|
16585
|
+
frames += 1;
|
|
16586
|
+
}
|
|
16587
|
+
|
|
16588
|
+
if (frames === 0) throw new Error("DeepSeek Harness zstd log contains no data frames");
|
|
16589
|
+
return {
|
|
16590
|
+
frames,
|
|
16591
|
+
totalContentBytes: allContentSizesDeclared ? totalContentBytes : null,
|
|
16592
|
+
maxTotalContentBytes: totalContentBytes,
|
|
16593
|
+
frameRanges,
|
|
16594
|
+
};
|
|
16595
|
+
}
|
|
16596
|
+
|
|
16597
|
+
// Decode one independent Harness append frame at a time. The MongoDB binding
|
|
16598
|
+
// understands every Node version we ship; per-frame decoding also lets us
|
|
16599
|
+
// enforce the aggregate plaintext limit when the frame omits its content size.
|
|
16600
|
+
async function decodeDshZstd(
|
|
16601
|
+
data,
|
|
16602
|
+
{ maxOutputBytes = DSH_SESSION_TEXT_MAX_BYTES, inspected = null } = {},
|
|
16603
|
+
) {
|
|
16604
|
+
const input = Buffer.isBuffer(data) ? data : Buffer.from(data || []);
|
|
16605
|
+
const frameInfo = inspected || inspectDshZstdFrames(input, maxOutputBytes);
|
|
16606
|
+
const parts = [];
|
|
16607
|
+
let total = 0;
|
|
16608
|
+
for (const frame of frameInfo.frameRanges) {
|
|
16609
|
+
const decoded = Buffer.from(
|
|
16610
|
+
await require("@mongodb-js/zstd").decompress(input.subarray(frame.start, frame.end)),
|
|
16611
|
+
);
|
|
16612
|
+
total += decoded.length;
|
|
16613
|
+
if (total > maxOutputBytes) {
|
|
16614
|
+
throw new Error(`DeepSeek Harness decompressed session log exceeds ${maxOutputBytes} bytes`);
|
|
16615
|
+
}
|
|
16616
|
+
parts.push(decoded);
|
|
16617
|
+
}
|
|
16618
|
+
return Buffer.concat(parts, total);
|
|
16619
|
+
}
|
|
16620
|
+
|
|
16621
|
+
function dshSessionMetadata(stat) {
|
|
16622
|
+
return {
|
|
16623
|
+
inode: stat?.ino || 0,
|
|
16624
|
+
size: Number.isFinite(stat?.size) ? stat.size : 0,
|
|
16625
|
+
mtimeMs: Number.isFinite(stat?.mtimeMs) ? stat.mtimeMs : 0,
|
|
16626
|
+
};
|
|
16627
|
+
}
|
|
16628
|
+
|
|
16629
|
+
function sameDshSessionMetadata(previous, current) {
|
|
16630
|
+
return Boolean(
|
|
16631
|
+
previous &&
|
|
16632
|
+
previous.inode === current.inode &&
|
|
16633
|
+
previous.size === current.size &&
|
|
16634
|
+
previous.mtimeMs === current.mtimeMs
|
|
16635
|
+
);
|
|
16636
|
+
}
|
|
16637
|
+
|
|
16638
|
+
// Open, inspect and read through one handle so a path replacement cannot make
|
|
16639
|
+
// the metadata describe a different file from the bytes we parse.
|
|
16640
|
+
async function readDshSessionSnapshot(
|
|
16641
|
+
filePath,
|
|
16642
|
+
{ maxOutputBytes = DSH_SESSION_TEXT_MAX_BYTES, previous = null } = {},
|
|
16643
|
+
) {
|
|
16644
|
+
const handle = await fs.open(filePath, "r").catch(() => null);
|
|
16645
|
+
if (!handle) return null;
|
|
16646
|
+
try {
|
|
16647
|
+
const initialStat = await handle.stat().catch(() => null);
|
|
16648
|
+
if (!initialStat || !initialStat.isFile()) return null;
|
|
16649
|
+
const initialMetadata = dshSessionMetadata(initialStat);
|
|
16650
|
+
if (initialMetadata.size > DSH_SESSION_LOG_MAX_BYTES) {
|
|
16651
|
+
throw new Error(`DeepSeek Harness session log exceeds ${DSH_SESSION_LOG_MAX_BYTES} bytes`);
|
|
16652
|
+
}
|
|
16653
|
+
if (sameDshSessionMetadata(previous, initialMetadata)) {
|
|
16654
|
+
return { ...initialMetadata, unchanged: true, text: null };
|
|
16655
|
+
}
|
|
16656
|
+
const outputLimit = Number.isSafeInteger(maxOutputBytes) && maxOutputBytes >= 0
|
|
16657
|
+
? maxOutputBytes
|
|
16658
|
+
: DSH_SESSION_TEXT_MAX_BYTES;
|
|
16659
|
+
const data = await handle.readFile();
|
|
16660
|
+
if (data.length > DSH_SESSION_LOG_MAX_BYTES) {
|
|
16661
|
+
throw new Error(`DeepSeek Harness session log exceeds ${DSH_SESSION_LOG_MAX_BYTES} bytes`);
|
|
16662
|
+
}
|
|
16663
|
+
let text;
|
|
16664
|
+
if (filePath.endsWith(".zstd")) {
|
|
16665
|
+
if (data.length === 0) text = "";
|
|
16666
|
+
else {
|
|
16667
|
+
const inspected = inspectDshZstdFrames(data, outputLimit);
|
|
16668
|
+
const decoded = await decodeDshZstd(data, { maxOutputBytes: outputLimit, inspected });
|
|
16669
|
+
if (
|
|
16670
|
+
decoded.length > outputLimit ||
|
|
16671
|
+
(inspected.totalContentBytes != null && decoded.length !== inspected.totalContentBytes)
|
|
16672
|
+
) {
|
|
16673
|
+
throw new Error(`DeepSeek Harness decompressed session log exceeds ${outputLimit} bytes`);
|
|
16674
|
+
}
|
|
16675
|
+
text = decoded.toString("utf8");
|
|
16676
|
+
}
|
|
16677
|
+
} else {
|
|
16678
|
+
if (data.length > outputLimit) {
|
|
16679
|
+
throw new Error(`DeepSeek Harness decompressed session log exceeds ${outputLimit} bytes`);
|
|
16680
|
+
}
|
|
16681
|
+
text = data.toString("utf8");
|
|
16682
|
+
}
|
|
16683
|
+
|
|
16684
|
+
const finalStat = await handle.stat().catch(() => initialStat);
|
|
16685
|
+
const metadata = dshSessionMetadata(finalStat);
|
|
16686
|
+
// If Harness appended while this handle was being read, force a retry on
|
|
16687
|
+
// the next sync unless the bytes consumed reached the final compressed/raw
|
|
16688
|
+
// file size. This avoids acknowledging a tail that was never parsed.
|
|
16689
|
+
if (metadata.size !== data.length) metadata.mtimeMs = -1;
|
|
16690
|
+
return { ...metadata, unchanged: false, text };
|
|
16691
|
+
} finally {
|
|
16692
|
+
await handle.close().catch(() => {});
|
|
16693
|
+
}
|
|
16694
|
+
}
|
|
16695
|
+
|
|
16696
|
+
// Read one session log to plaintext, decompressing zstd artifacts. Returns
|
|
16697
|
+
// null for a missing/non-file path so callers treat it as a no-op.
|
|
16698
|
+
async function readDshSessionText(filePath, options = {}) {
|
|
16699
|
+
const snapshot = await readDshSessionSnapshot(filePath, options);
|
|
16700
|
+
return snapshot?.text ?? null;
|
|
16701
|
+
}
|
|
16702
|
+
|
|
16703
|
+
// Drop a provider-qualified model id prefix ("deepseek/deepseek-v4-pro" →
|
|
16704
|
+
// "deepseek-v4-pro"). The harness emits a bare id today, but the source
|
|
16705
|
+
// vocabulary is provider-qualified elsewhere; keep the mapping cheap.
|
|
16706
|
+
function normalizeDshModelName(value) {
|
|
16707
|
+
if (typeof value !== "string") return null;
|
|
16708
|
+
const trimmed = value.trim();
|
|
16709
|
+
if (!trimmed) return null;
|
|
16710
|
+
const slash = trimmed.lastIndexOf("/");
|
|
16711
|
+
const name = slash >= 0 ? trimmed.slice(slash + 1) : trimmed;
|
|
16712
|
+
return name ? name : null;
|
|
16713
|
+
}
|
|
16714
|
+
|
|
16715
|
+
// Convert a harness `TokenUsage` object into our queue `totals` delta. Counts
|
|
16716
|
+
// are disjoint, so every column maps 1:1; returns null for an all-zero usage.
|
|
16717
|
+
function dshUsageToTotals(usage) {
|
|
16718
|
+
if (!usage || typeof usage !== "object") return null;
|
|
16719
|
+
const input = toNonNegativeInt(usage.inputTokens);
|
|
16720
|
+
const output = toNonNegativeInt(usage.outputTokens);
|
|
16721
|
+
const cacheRead = toNonNegativeInt(usage.cacheReadTokens);
|
|
16722
|
+
const cacheWrite = toNonNegativeInt(usage.cacheWriteTokens);
|
|
16723
|
+
const reasoning = toNonNegativeInt(usage.reasoningTokens);
|
|
16724
|
+
if (input === 0 && output === 0 && cacheRead === 0 && cacheWrite === 0 && reasoning === 0) {
|
|
16725
|
+
return null;
|
|
16726
|
+
}
|
|
16727
|
+
return {
|
|
16728
|
+
input_tokens: input,
|
|
16729
|
+
cached_input_tokens: cacheRead,
|
|
16730
|
+
cache_creation_input_tokens: cacheWrite,
|
|
16731
|
+
output_tokens: output,
|
|
16732
|
+
reasoning_output_tokens: reasoning,
|
|
16733
|
+
total_tokens: input + output + cacheRead + cacheWrite + reasoning,
|
|
16734
|
+
conversation_count: 1,
|
|
16735
|
+
};
|
|
16736
|
+
}
|
|
16737
|
+
|
|
16738
|
+
// Return one top-level JSON object property's raw value slice without parsing
|
|
16739
|
+
// sibling values. In particular this lets the Harness reader skip message
|
|
16740
|
+
// content byte-for-byte and materialize only routing metadata plus token
|
|
16741
|
+
// counters. It is intentionally small and JSON-syntax aware (strings, escapes,
|
|
16742
|
+
// nested arrays/objects), not a regex over potentially nested content.
|
|
16743
|
+
function findDshJsonProperty(raw, wantedKey) {
|
|
16744
|
+
const text = String(raw || "");
|
|
16745
|
+
const skipWhitespace = (index) => {
|
|
16746
|
+
while (index < text.length && /\s/.test(text[index])) index += 1;
|
|
16747
|
+
return index;
|
|
16748
|
+
};
|
|
16749
|
+
const stringEnd = (start) => {
|
|
16750
|
+
if (text[start] !== '"') return -1;
|
|
16751
|
+
for (let index = start + 1; index < text.length; index++) {
|
|
16752
|
+
if (text[index] === "\\") {
|
|
16753
|
+
index += 1;
|
|
16754
|
+
} else if (text[index] === '"') {
|
|
16755
|
+
return index + 1;
|
|
16756
|
+
}
|
|
16757
|
+
}
|
|
16758
|
+
return -1;
|
|
16759
|
+
};
|
|
16760
|
+
const valueEnd = (start) => {
|
|
16761
|
+
const first = text[start];
|
|
16762
|
+
if (first === '"') return stringEnd(start);
|
|
16763
|
+
if (first === "{" || first === "[") {
|
|
16764
|
+
const stack = [first === "{" ? "}" : "]"];
|
|
16765
|
+
let inString = false;
|
|
16766
|
+
for (let index = start + 1; index < text.length; index++) {
|
|
16767
|
+
const char = text[index];
|
|
16768
|
+
if (inString) {
|
|
16769
|
+
if (char === "\\") index += 1;
|
|
16770
|
+
else if (char === '"') inString = false;
|
|
16771
|
+
continue;
|
|
16772
|
+
}
|
|
16773
|
+
if (char === '"') {
|
|
16774
|
+
inString = true;
|
|
16775
|
+
} else if (char === "{") {
|
|
16776
|
+
stack.push("}");
|
|
16777
|
+
} else if (char === "[") {
|
|
16778
|
+
stack.push("]");
|
|
16779
|
+
} else if (char === stack.at(-1)) {
|
|
16780
|
+
stack.pop();
|
|
16781
|
+
if (stack.length === 0) return index + 1;
|
|
16782
|
+
}
|
|
16783
|
+
}
|
|
16784
|
+
return -1;
|
|
16785
|
+
}
|
|
16786
|
+
let index = start;
|
|
16787
|
+
while (index < text.length && !/[\s,}\]]/.test(text[index])) index += 1;
|
|
16788
|
+
return index > start ? index : -1;
|
|
16789
|
+
};
|
|
16790
|
+
|
|
16791
|
+
let index = skipWhitespace(0);
|
|
16792
|
+
if (text[index] !== "{") return null;
|
|
16793
|
+
index += 1;
|
|
16794
|
+
while (index < text.length) {
|
|
16795
|
+
index = skipWhitespace(index);
|
|
16796
|
+
if (text[index] === "}") return null;
|
|
16797
|
+
const keyStart = index;
|
|
16798
|
+
const keyEnd = stringEnd(keyStart);
|
|
16799
|
+
if (keyEnd < 0) return null;
|
|
16800
|
+
let key;
|
|
16801
|
+
try {
|
|
16802
|
+
key = JSON.parse(text.slice(keyStart, keyEnd));
|
|
16803
|
+
} catch {
|
|
16804
|
+
return null;
|
|
16805
|
+
}
|
|
16806
|
+
index = skipWhitespace(keyEnd);
|
|
16807
|
+
if (text[index] !== ":") return null;
|
|
16808
|
+
index = skipWhitespace(index + 1);
|
|
16809
|
+
const start = index;
|
|
16810
|
+
const end = valueEnd(start);
|
|
16811
|
+
if (end < 0) return null;
|
|
16812
|
+
if (key === wantedKey) return text.slice(start, end);
|
|
16813
|
+
index = skipWhitespace(end);
|
|
16814
|
+
if (text[index] === ",") {
|
|
16815
|
+
index += 1;
|
|
16816
|
+
continue;
|
|
16817
|
+
}
|
|
16818
|
+
if (text[index] === "}") return null;
|
|
16819
|
+
return null;
|
|
16820
|
+
}
|
|
16821
|
+
return null;
|
|
16822
|
+
}
|
|
16823
|
+
|
|
16824
|
+
function parseDshJsonString(raw) {
|
|
16825
|
+
if (typeof raw !== "string" || raw[0] !== '"') return null;
|
|
16826
|
+
try {
|
|
16827
|
+
const value = JSON.parse(raw);
|
|
16828
|
+
return typeof value === "string" ? value : null;
|
|
16829
|
+
} catch {
|
|
16830
|
+
return null;
|
|
16831
|
+
}
|
|
16832
|
+
}
|
|
16833
|
+
|
|
16834
|
+
function parseDshJsonNumber(raw) {
|
|
16835
|
+
if (typeof raw !== "string") return NaN;
|
|
16836
|
+
const value = Number(raw);
|
|
16837
|
+
return Number.isFinite(value) ? value : NaN;
|
|
16838
|
+
}
|
|
16839
|
+
|
|
16840
|
+
function parseDshUsageSlice(raw) {
|
|
16841
|
+
if (typeof raw !== "string") return null;
|
|
16842
|
+
return {
|
|
16843
|
+
inputTokens: parseDshJsonNumber(findDshJsonProperty(raw, "inputTokens")),
|
|
16844
|
+
outputTokens: parseDshJsonNumber(findDshJsonProperty(raw, "outputTokens")),
|
|
16845
|
+
cacheReadTokens: parseDshJsonNumber(findDshJsonProperty(raw, "cacheReadTokens")),
|
|
16846
|
+
cacheWriteTokens: parseDshJsonNumber(findDshJsonProperty(raw, "cacheWriteTokens")),
|
|
16847
|
+
reasoningTokens: parseDshJsonNumber(findDshJsonProperty(raw, "reasoningTokens")),
|
|
16848
|
+
};
|
|
16849
|
+
}
|
|
16850
|
+
|
|
16851
|
+
// Parse one session log's plaintext into usage deltas, skipping events whose
|
|
16852
|
+
// seq is at or below the watermark. Returns the deltas (each carrying the
|
|
16853
|
+
// model and epoch-ms timestamp), the highest seq seen, and the session id.
|
|
16854
|
+
function extractDshSessionUsage(text, lastSeq = -1) {
|
|
16855
|
+
const deltas = [];
|
|
16856
|
+
const watermark = Number.isFinite(lastSeq) ? lastSeq : -1;
|
|
16857
|
+
let maxSeq = watermark;
|
|
16858
|
+
let sessionId = null;
|
|
16859
|
+
let headerModel = null;
|
|
16860
|
+
|
|
16861
|
+
const lines = String(text || "").split("\n");
|
|
16862
|
+
for (let i = 0; i < lines.length; i++) {
|
|
16863
|
+
const line = lines[i];
|
|
16864
|
+
if (!line || !line.trim()) continue;
|
|
16865
|
+
const eventType = parseDshJsonString(findDshJsonProperty(line, "type"));
|
|
16866
|
+
if (!eventType) continue;
|
|
16867
|
+
|
|
16868
|
+
if (eventType === "session") {
|
|
16869
|
+
const id = parseDshJsonString(findDshJsonProperty(line, "id"));
|
|
16870
|
+
if (id) sessionId = id;
|
|
16871
|
+
continue;
|
|
16872
|
+
}
|
|
16873
|
+
|
|
16874
|
+
const seq = parseDshJsonNumber(findDshJsonProperty(line, "seq"));
|
|
16875
|
+
const seqKnown = Number.isFinite(seq) && seq >= 0;
|
|
16876
|
+
if (seqKnown && seq > maxSeq) maxSeq = seq;
|
|
16877
|
+
|
|
16878
|
+
const data = findDshJsonProperty(line, "data");
|
|
16879
|
+
if (!data) continue;
|
|
16880
|
+
|
|
16881
|
+
if (eventType === "request/header") {
|
|
16882
|
+
const header = findDshJsonProperty(data, "header");
|
|
16883
|
+
const config = findDshJsonProperty(header, "config");
|
|
16884
|
+
const model = parseDshJsonString(findDshJsonProperty(config, "model"));
|
|
16885
|
+
const normalized = normalizeDshModelName(model);
|
|
16886
|
+
if (normalized) headerModel = normalized;
|
|
16887
|
+
continue;
|
|
16888
|
+
}
|
|
16889
|
+
|
|
16890
|
+
if (eventType !== "assistant/message") continue;
|
|
16891
|
+
if (seqKnown && seq <= watermark) continue;
|
|
16892
|
+
|
|
16893
|
+
const message = findDshJsonProperty(data, "message");
|
|
16894
|
+
const source = findDshJsonProperty(message, "source");
|
|
16895
|
+
const model = normalizeDshModelName(
|
|
16896
|
+
parseDshJsonString(findDshJsonProperty(source, "model")),
|
|
16897
|
+
) || headerModel;
|
|
16898
|
+
const totals = dshUsageToTotals(
|
|
16899
|
+
parseDshUsageSlice(findDshJsonProperty(data, "usage")),
|
|
16900
|
+
);
|
|
16901
|
+
if (!model || !totals) continue;
|
|
16902
|
+
|
|
16903
|
+
const timeMs = parseDshJsonNumber(findDshJsonProperty(line, "time"));
|
|
16904
|
+
if (!Number.isFinite(timeMs) || timeMs <= 0) continue;
|
|
16905
|
+
|
|
16906
|
+
deltas.push({ model, timeMs, totals });
|
|
16907
|
+
}
|
|
16908
|
+
|
|
16909
|
+
return { deltas, maxSeq, sessionId };
|
|
16910
|
+
}
|
|
16911
|
+
|
|
16912
|
+
// Incremental parser entrypoint. Mirrors the other passive JSONL readers:
|
|
16913
|
+
// identity-check each file (inode/size/mtime), re-read + re-parse only when it
|
|
16914
|
+
// changed, dedup via the per-file seq watermark, accumulate into hourly
|
|
16915
|
+
// buckets, then flush touched buckets to the queue.
|
|
16916
|
+
async function parseDshIncremental({ sessionFiles, cursors, queuePath, onProgress }) {
|
|
16917
|
+
await ensureDir(path.dirname(queuePath));
|
|
16918
|
+
if (!cursors || typeof cursors !== "object") cursors = {};
|
|
16919
|
+
if (!cursors.dsh || typeof cursors.dsh !== "object") cursors.dsh = {};
|
|
16920
|
+
const dshState = cursors.dsh;
|
|
16921
|
+
const fileState =
|
|
16922
|
+
dshState.files && typeof dshState.files === "object" ? dshState.files : {};
|
|
16923
|
+
|
|
16924
|
+
const hourlyState = normalizeHourlyState(cursors?.hourly);
|
|
16925
|
+
const touchedBuckets = new Set();
|
|
16926
|
+
const cb = typeof onProgress === "function" ? onProgress : null;
|
|
16927
|
+
|
|
16928
|
+
const files = Array.isArray(sessionFiles) ? sessionFiles : [];
|
|
16929
|
+
const presentFiles = new Set(files.filter((filePath) => typeof filePath === "string"));
|
|
16930
|
+
const total = files.length;
|
|
16931
|
+
let recordsProcessed = 0;
|
|
16932
|
+
let eventsAggregated = 0;
|
|
16933
|
+
|
|
16934
|
+
for (let idx = 0; idx < files.length; idx++) {
|
|
16935
|
+
const filePath = files[idx];
|
|
16936
|
+
const prev = fileState[filePath] || null;
|
|
16937
|
+
let snapshot;
|
|
16938
|
+
let parsed;
|
|
16939
|
+
try {
|
|
16940
|
+
snapshot = await readDshSessionSnapshot(filePath, { previous: prev });
|
|
16941
|
+
if (!snapshot || snapshot.unchanged) {
|
|
16942
|
+
if (cb) {
|
|
16943
|
+
cb({ index: idx + 1, total, recordsProcessed, eventsAggregated, bucketsQueued: touchedBuckets.size });
|
|
16944
|
+
}
|
|
16945
|
+
continue;
|
|
16946
|
+
}
|
|
16947
|
+
|
|
16948
|
+
const lastSeq = Number.isFinite(prev?.lastSeq) ? prev.lastSeq : -1;
|
|
16949
|
+
parsed = extractDshSessionUsage(snapshot.text, lastSeq);
|
|
16950
|
+
if (prev && parsed.sessionId && parsed.sessionId !== prev.sessionId) {
|
|
16951
|
+
parsed = extractDshSessionUsage(snapshot.text, -1);
|
|
16952
|
+
}
|
|
16953
|
+
} catch (error) {
|
|
16954
|
+
if (process.env.TOKENTRACKER_DEBUG) {
|
|
16955
|
+
process.stderr.write(`[dsh] skipped ${filePath}: ${error?.message || error}\n`);
|
|
16956
|
+
}
|
|
16957
|
+
if (cb) {
|
|
16958
|
+
cb({ index: idx + 1, total, recordsProcessed, eventsAggregated, bucketsQueued: touchedBuckets.size });
|
|
16959
|
+
}
|
|
16960
|
+
continue;
|
|
16961
|
+
}
|
|
16962
|
+
|
|
16963
|
+
for (const delta of parsed.deltas) {
|
|
16964
|
+
const bucketStart = toUtcHalfHourStart(delta.timeMs);
|
|
16965
|
+
if (!bucketStart) continue;
|
|
16966
|
+
const bucket = getHourlyBucket(hourlyState, DSH_SOURCE, delta.model, bucketStart);
|
|
16967
|
+
addTotals(bucket.totals, delta.totals);
|
|
16968
|
+
touchedBuckets.add(bucketKey(DSH_SOURCE, delta.model, bucketStart));
|
|
16969
|
+
eventsAggregated += 1;
|
|
16970
|
+
}
|
|
16971
|
+
|
|
16972
|
+
fileState[filePath] = {
|
|
16973
|
+
inode: snapshot.inode,
|
|
16974
|
+
size: snapshot.size,
|
|
16975
|
+
mtimeMs: snapshot.mtimeMs,
|
|
16976
|
+
sessionId: parsed.sessionId || null,
|
|
16977
|
+
lastSeq: parsed.maxSeq,
|
|
16978
|
+
updatedAt: new Date().toISOString(),
|
|
16979
|
+
};
|
|
16980
|
+
recordsProcessed += 1;
|
|
16981
|
+
|
|
16982
|
+
if (cb) {
|
|
16983
|
+
cb({ index: idx + 1, total, recordsProcessed, eventsAggregated, bucketsQueued: touchedBuckets.size });
|
|
16984
|
+
}
|
|
16985
|
+
}
|
|
16986
|
+
|
|
16987
|
+
for (const filePath of Object.keys(fileState)) {
|
|
16988
|
+
if (!presentFiles.has(filePath)) delete fileState[filePath];
|
|
16989
|
+
}
|
|
16990
|
+
|
|
16991
|
+
const bucketsQueued = await enqueueTouchedBuckets({ queuePath, hourlyState, touchedBuckets });
|
|
16992
|
+
hourlyState.updatedAt = new Date().toISOString();
|
|
16993
|
+
cursors.hourly = hourlyState;
|
|
16994
|
+
dshState.files = fileState;
|
|
16995
|
+
dshState.updatedAt = new Date().toISOString();
|
|
16996
|
+
|
|
16997
|
+
return { recordsProcessed, eventsAggregated, bucketsQueued };
|
|
16998
|
+
}
|
|
16999
|
+
|
|
17000
|
+
|
|
16247
17001
|
module.exports = {
|
|
16248
17002
|
listRolloutFiles,
|
|
16249
17003
|
listRolloutFilesDeep,
|
|
@@ -16380,6 +17134,10 @@ module.exports = {
|
|
|
16380
17134
|
resolveCraftSessionFiles,
|
|
16381
17135
|
resolveCraftDefaultModel,
|
|
16382
17136
|
parseCraftIncremental,
|
|
17137
|
+
resolveReasonixHome,
|
|
17138
|
+
resolveReasonixTelemetryFiles,
|
|
17139
|
+
normalizeReasonixModel,
|
|
17140
|
+
parseReasonixIncremental,
|
|
16383
17141
|
// Exposed for regression tests covering cache-token accounting.
|
|
16384
17142
|
normalizeGeminiTokens,
|
|
16385
17143
|
normalizeOpencodeTokens,
|
|
@@ -16415,4 +17173,14 @@ module.exports = {
|
|
|
16415
17173
|
resolveTraePath,
|
|
16416
17174
|
resolveTraeStoragePath,
|
|
16417
17175
|
readTraeEntitlementFromStorage,
|
|
17176
|
+
// DeepSeek Harness (dsh) — passive session-log reader
|
|
17177
|
+
resolveDshHome,
|
|
17178
|
+
resolveDshSessionFiles,
|
|
17179
|
+
readDshSessionText,
|
|
17180
|
+
decodeDshZstd,
|
|
17181
|
+
inspectDshZstdFrames,
|
|
17182
|
+
normalizeDshModelName,
|
|
17183
|
+
dshUsageToTotals,
|
|
17184
|
+
extractDshSessionUsage,
|
|
17185
|
+
parseDshIncremental,
|
|
16418
17186
|
};
|