sigrank-mcp 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/adapters.mjs +480 -0
- package/cascade.mjs +143 -0
- package/index.mjs +48 -0
- package/narrate.mjs +65 -0
- package/package.json +25 -0
- package/tokenpull.mjs +274 -0
- package/tools.mjs +323 -0
package/cascade.mjs
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cascade.mjs — pure SigRank yield cascade (no deps, no transport).
|
|
3
|
+
* Mirrors sigrank-app/lib/ingest/bridge.ts computeCascadeMetrics() so rank_paste
|
|
4
|
+
* reproduces the canonical MO§ES Υ 18436.98 from its 4 raw token pillars.
|
|
5
|
+
* Paper-and-pencil math, open by design; proprietary threshold cuts stay server-side.
|
|
6
|
+
*
|
|
7
|
+
* Degenerate-input policy (hardened):
|
|
8
|
+
* - Any pillar that collapses a denominator (i=0, o=0, cw=0, cr=0) returns null for
|
|
9
|
+
* the affected metrics rather than Infinity/NaN.
|
|
10
|
+
* - A `warnings[]` array is attached when any metric is null so callers can surface the
|
|
11
|
+
* reason without silently corrupting downstream calculations.
|
|
12
|
+
* - The cascade is NEVER thrown away — even partial results are useful for review/storage.
|
|
13
|
+
* Callers that require a fully-formed result should check `warnings.length === 0`.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export const round = (n, d) => (Number.isFinite(n) ? Number(n.toFixed(d)) : null)
|
|
17
|
+
|
|
18
|
+
/** The four raw token pillars → the cascade. */
|
|
19
|
+
export function cascade({ input, output, cacheCreate, cacheRead }) {
|
|
20
|
+
const i = Number(input), o = Number(output), cw = Number(cacheCreate), cr = Number(cacheRead)
|
|
21
|
+
const total = i + o + cw + cr
|
|
22
|
+
const warnings = []
|
|
23
|
+
|
|
24
|
+
// SNR: undefined when both i and o are 0 (empty session)
|
|
25
|
+
const snrDenom = i + o
|
|
26
|
+
const snr = snrDenom > 0 ? o / snrDenom : null
|
|
27
|
+
if (snr === null) warnings.push('snr_undefined: input+output=0')
|
|
28
|
+
|
|
29
|
+
// velocity: undefined when i=0 (no fresh input — pure cache-only session)
|
|
30
|
+
const velocity = i > 0 ? o / i : null
|
|
31
|
+
if (velocity === null) warnings.push('velocity_undefined: input=0')
|
|
32
|
+
|
|
33
|
+
// leverage: undefined when i=0
|
|
34
|
+
const leverage = i > 0 ? cr / i : null
|
|
35
|
+
if (leverage === null) warnings.push('leverage_undefined: input=0')
|
|
36
|
+
|
|
37
|
+
// Υ = leverage × velocity = (Cr·O)/I² — null when either component is null
|
|
38
|
+
const yield_ = leverage !== null && velocity !== null ? leverage * velocity : null
|
|
39
|
+
if (yield_ === null && !warnings.some((w) => w.startsWith('yield')))
|
|
40
|
+
warnings.push('yield_undefined: requires input>0')
|
|
41
|
+
|
|
42
|
+
// dev10x = log10(Cr/I) — collapses when cw=0 (no cache commits) or i=0 or cr=0.
|
|
43
|
+
// narrate.mjs already has a nonCompounding branch for this case; we return null
|
|
44
|
+
// explicitly so the branch triggers correctly (vs -Infinity on log10(0)).
|
|
45
|
+
let dev10x = null
|
|
46
|
+
if (i > 0 && o > 0 && cw > 0 && cr > 0) {
|
|
47
|
+
dev10x = Math.log10((o / i) * (cw / o) * (cr / cw)) // = log10(Cr/I)
|
|
48
|
+
} else {
|
|
49
|
+
warnings.push('dev10x_undefined: requires all four pillars > 0')
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const result = {
|
|
53
|
+
pillars: { input: i, output: o, cacheCreate: cw, cacheRead: cr, total },
|
|
54
|
+
yield: round(yield_, 2),
|
|
55
|
+
snr: round(snr, 4),
|
|
56
|
+
leverage: round(leverage, 1),
|
|
57
|
+
velocity: round(velocity, 3),
|
|
58
|
+
dev10x: round(dev10x, 2),
|
|
59
|
+
class: classify(yield_, dev10x),
|
|
60
|
+
}
|
|
61
|
+
if (warnings.length > 0) result.warnings = warnings
|
|
62
|
+
return result
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** MVP class tiering from Υ + 10xDEV (canon assigns from cascade SNR + 10xDEV; this
|
|
66
|
+
* is the open-MVP approximation — proprietary threshold cuts stay server-side). */
|
|
67
|
+
export function classify(yieldVal, dev10x) {
|
|
68
|
+
if (yieldVal >= 1000 || dev10x >= 3) return 'TRANSMITTER'
|
|
69
|
+
if (dev10x >= 1.45) return 'ARCH+'
|
|
70
|
+
if (dev10x >= 1.35) return 'ARCH'
|
|
71
|
+
if (dev10x >= 1.2) return 'POWER'
|
|
72
|
+
if (dev10x >= 1.0) return 'BASE'
|
|
73
|
+
if (dev10x >= 0) return 'SEEKER'
|
|
74
|
+
if (dev10x >= -0.3) return 'REFINER'
|
|
75
|
+
return 'IGNITER'
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Extract the 4 pillars from pasted text: JSON object OR 4 whitespace numbers.
|
|
80
|
+
*
|
|
81
|
+
* Hardened parse policy:
|
|
82
|
+
* - JSON path: requires named keys and numeric values. Rejects strings/null.
|
|
83
|
+
* - Positional path: requires the input to be ONLY numeric tokens (whitespace/commas
|
|
84
|
+
* allowed as separators). If the text contains non-numeric words the positional
|
|
85
|
+
* extractor attaches a `_parseWarnings` flag so downstream can route it to a review
|
|
86
|
+
* channel instead of treating it as authoritative data.
|
|
87
|
+
* - Negative values are accepted (not thrown away) but flagged — could be valid in
|
|
88
|
+
* some edge-case accounting or a data error; the server is the authority on validity.
|
|
89
|
+
* - We NEVER silently corrupt: if we can't parse 4 pillars we throw. If we parse but
|
|
90
|
+
* something looks suspicious we surface it in `_parseWarnings` on the returned object.
|
|
91
|
+
*/
|
|
92
|
+
export function parsePillars(text) {
|
|
93
|
+
const t = String(text || '').trim()
|
|
94
|
+
const pw = [] // parse warnings to attach
|
|
95
|
+
|
|
96
|
+
// ── JSON path ──────────────────────────────────────────────────────────────
|
|
97
|
+
try {
|
|
98
|
+
const j = JSON.parse(t)
|
|
99
|
+
if (j && typeof j === 'object' && !Array.isArray(j)) {
|
|
100
|
+
const g = (...keys) => { for (const k of keys) if (j[k] != null) return j[k]; return null }
|
|
101
|
+
const input = g('input', 'tokens_input_fresh', 'inputTokens', 'input_tokens')
|
|
102
|
+
const output = g('output', 'tokens_output', 'outputTokens', 'output_tokens')
|
|
103
|
+
const cacheCreate = g('cacheCreate', 'tokens_cache_creation', 'cache_creation_tokens')
|
|
104
|
+
const cacheRead = g('cacheRead', 'tokens_cache_read', 'cache_read_tokens')
|
|
105
|
+
if ([input, output, cacheCreate, cacheRead].every((v) => v != null)) {
|
|
106
|
+
const pillars = {
|
|
107
|
+
input: Number(input),
|
|
108
|
+
output: Number(output),
|
|
109
|
+
cacheCreate: Number(cacheCreate),
|
|
110
|
+
cacheRead: Number(cacheRead),
|
|
111
|
+
}
|
|
112
|
+
if ([pillars.input, pillars.output, pillars.cacheCreate, pillars.cacheRead].some((v) => !Number.isFinite(v)))
|
|
113
|
+
throw new Error('Non-numeric pillar value in JSON (got string or non-finite number).')
|
|
114
|
+
if ([pillars.input, pillars.output, pillars.cacheCreate, pillars.cacheRead].some((v) => v < 0))
|
|
115
|
+
pw.push('negative_pillar: one or more pillars is negative — may be a data error')
|
|
116
|
+
if (pw.length > 0) pillars._parseWarnings = pw
|
|
117
|
+
return pillars
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
} catch (e) {
|
|
121
|
+
// JSON.parse syntax error — fall through to positional. Re-throw parse errors we raised ourselves.
|
|
122
|
+
if (e.message.startsWith('Non-numeric')) throw e
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ── Positional path ────────────────────────────────────────────────────────
|
|
126
|
+
// Guard: if the text contains alphabetic words, the numeric extraction is unreliable.
|
|
127
|
+
// We still attempt it (don't throw away the data) but flag it for review.
|
|
128
|
+
if (/[a-zA-Z]/.test(t))
|
|
129
|
+
pw.push('positional_from_mixed_text: extracted numbers from text that contains alphabetic characters — verify these are the correct 4 pillars')
|
|
130
|
+
|
|
131
|
+
const nums = (t.match(/-?\d[\d,]*\.?\d*/g) || []).map((s) => Number(s.replace(/,/g, '')))
|
|
132
|
+
if (nums.length >= 4) {
|
|
133
|
+
const [input, output, cacheCreate, cacheRead] = nums
|
|
134
|
+
if (nums.length > 4)
|
|
135
|
+
pw.push(`positional_extra_numbers: found ${nums.length} numbers, using first 4 — inspect for positional order error`)
|
|
136
|
+
const pillars = { input, output, cacheCreate, cacheRead }
|
|
137
|
+
if ([input, output, cacheCreate, cacheRead].some((v) => v < 0))
|
|
138
|
+
pw.push('negative_pillar: one or more pillars is negative — may be a data error')
|
|
139
|
+
if (pw.length > 0) pillars._parseWarnings = pw
|
|
140
|
+
return pillars
|
|
141
|
+
}
|
|
142
|
+
throw new Error('Could not parse 4 token pillars (input, output, cacheCreate, cacheRead) from the input.')
|
|
143
|
+
}
|
package/index.mjs
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* SigRank MCP server — exposes the SigRank yield cascade + live board as MCP tools
|
|
4
|
+
* any client (Claude Code, Cursor, …) can call. Token-only, Claude/ccusage-first.
|
|
5
|
+
*
|
|
6
|
+
* Tools (see ./tools.mjs):
|
|
7
|
+
* - rank_paste(text) paste ccusage token counts → Υ/SNR/Leverage/…+class +card
|
|
8
|
+
* - get_leaderboard() the live public board (signalaf.com)
|
|
9
|
+
* - get_operator(codename) one operator's live profile
|
|
10
|
+
* - submit_paste(text, codename) rank AND publish to the board in one call
|
|
11
|
+
*
|
|
12
|
+
* Pure cascade math lives in ./cascade.mjs (mirrors lib/ingest/bridge.ts); the
|
|
13
|
+
* deterministic card in ./narrate.mjs; the tool table + dispatcher in ./tools.mjs.
|
|
14
|
+
* No transcript content, no auth. Reuses the live read/write endpoints over HTTP.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { Server } from '@modelcontextprotocol/sdk/server/index.js'
|
|
18
|
+
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
|
|
19
|
+
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'
|
|
20
|
+
import { TOOLS, callTool } from './tools.mjs'
|
|
21
|
+
|
|
22
|
+
// Prevent silent crashes — log to stderr (MCP clients read stdout; stderr is safe for
|
|
23
|
+
// diagnostics). The process exits so the client can respawn with a clean slate rather
|
|
24
|
+
// than hanging on a broken connection.
|
|
25
|
+
process.on('uncaughtException', (err) => {
|
|
26
|
+
process.stderr.write(`[sigrank-mcp] uncaughtException: ${err?.stack || err}\n`)
|
|
27
|
+
process.exit(1)
|
|
28
|
+
})
|
|
29
|
+
process.on('unhandledRejection', (reason) => {
|
|
30
|
+
process.stderr.write(`[sigrank-mcp] unhandledRejection: ${reason?.stack || reason}\n`)
|
|
31
|
+
process.exit(1)
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
async function main() {
|
|
35
|
+
const server = new Server({ name: 'sigrank', version: '0.6.2' }, { capabilities: { tools: {} } })
|
|
36
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }))
|
|
37
|
+
server.setRequestHandler(CallToolRequestSchema, async (req) => {
|
|
38
|
+
try {
|
|
39
|
+
const out = await callTool(req.params.name, req.params.arguments)
|
|
40
|
+
return { content: [{ type: 'text', text: JSON.stringify(out, null, 2) }] }
|
|
41
|
+
} catch (e) {
|
|
42
|
+
return { content: [{ type: 'text', text: `Error: ${e.message}` }], isError: true }
|
|
43
|
+
}
|
|
44
|
+
})
|
|
45
|
+
await server.connect(new StdioServerTransport())
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
main()
|
package/narrate.mjs
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* narrate.mjs — deterministic prose "card" for a cascade result.
|
|
3
|
+
*
|
|
4
|
+
* Port of _template() from ~/Desktop/moses-sigrank/narrate.py. The model path
|
|
5
|
+
* (MiniCPM4-0.5B) is intentionally SKIPPED: the template is the trustworthy,
|
|
6
|
+
* instant, auditable fallback — same numbers in → same card out, and it can never
|
|
7
|
+
* emit a metric the cascade didn't produce. A model hook can layer behind this same
|
|
8
|
+
* narrate() interface later without a rewrite.
|
|
9
|
+
*
|
|
10
|
+
* Token-only. No network, no randomness.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
// Safe formatters: never emit NaN/Infinity/undefined into the card.
|
|
14
|
+
const safeNum = (n) => (Number.isFinite(Number(n)) ? Number(n) : null)
|
|
15
|
+
const comma = (n, dec) => {
|
|
16
|
+
const v = safeNum(n)
|
|
17
|
+
return v !== null ? v.toLocaleString('en-US', { minimumFractionDigits: dec, maximumFractionDigits: dec }) : '—'
|
|
18
|
+
}
|
|
19
|
+
const plain = (n, dec) => {
|
|
20
|
+
const v = safeNum(n)
|
|
21
|
+
return v !== null ? v.toFixed(dec) : '—'
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Given a cascade result ({ velocity, leverage, dev10x, pillars, class }) and an
|
|
26
|
+
* optional subject name, return "**CLASS.** <one or two sentences>". Deterministic.
|
|
27
|
+
*/
|
|
28
|
+
export function narrate(cascade, name = 'This operator') {
|
|
29
|
+
const klass = cascade.class || cascade.klass || 'UNCLASSED'
|
|
30
|
+
const v = safeNum(cascade.velocity)
|
|
31
|
+
const l = safeNum(cascade.leverage)
|
|
32
|
+
|
|
33
|
+
// "non-compounding" = a stateless pipe: no cache commits, so the cascade can't
|
|
34
|
+
// form. cascade.mjs leaves dev10x null when cacheCreate is 0 (the cw/o term
|
|
35
|
+
// collapses), which is exactly metrics.py's non_compounding flag.
|
|
36
|
+
// Also catches zero-input sessions where velocity/leverage are null.
|
|
37
|
+
const cw = cascade.pillars ? Number(cascade.pillars.cacheCreate) : NaN
|
|
38
|
+
const nonCompounding = cascade.dev10x == null || !(cw > 0) || v === null || l === null
|
|
39
|
+
|
|
40
|
+
let body
|
|
41
|
+
if (nonCompounding) {
|
|
42
|
+
const leverageStr = l !== null ? `Leverage ${comma(l, 1)}x comes from reuse alone.` : 'Leverage is undefined (no fresh input recorded).'
|
|
43
|
+
const dev10xNote = cascade.dev10x == null ? ' 10xDEV is undefined — the compounding loop has not formed yet.' : ''
|
|
44
|
+
body =
|
|
45
|
+
`${name} runs a stateless pipe — no cache commits, so the cascade can't form. ` +
|
|
46
|
+
`High read volume, but nothing is being built forward. ${leverageStr}${dev10xNote}`
|
|
47
|
+
} else if (v >= 1 && l >= 100) {
|
|
48
|
+
body =
|
|
49
|
+
`${name} holds both axes at once: ${plain(v, 1)}x generation AND ${comma(l, 0)}x memory leverage. ` +
|
|
50
|
+
`A closed kinetic loop — the rare operator the leverage/generation tradeoff says shouldn't exist.`
|
|
51
|
+
} else if (l >= 10 && v < 1) {
|
|
52
|
+
body =
|
|
53
|
+
`${name} is an archival sponge — ${comma(l, 0)}x reuse but only ${plain(v, 2)}x generation. ` +
|
|
54
|
+
`Holds context beautifully, executes little with it. The reuse number is inflated by a weak commitment stage.`
|
|
55
|
+
} else if (v >= 0.8 && l < 2) {
|
|
56
|
+
body =
|
|
57
|
+
`${name} is a volatile ingestor — ${plain(v, 2)}x generation but ${plain(l, 1)}x leverage. ` +
|
|
58
|
+
`Fast on single shots, resets between turns. Memory doesn't persist into a compounding loop.`
|
|
59
|
+
} else {
|
|
60
|
+
body =
|
|
61
|
+
`${name} sits low on both axes: ${plain(v, 2)}x generation, ${plain(l, 1)}x leverage. ` +
|
|
62
|
+
`A transient profile — neither building state nor converting input to output efficiently.`
|
|
63
|
+
}
|
|
64
|
+
return `**${klass}.** ${body}`
|
|
65
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "sigrank-mcp",
|
|
3
|
+
"version": "0.6.3",
|
|
4
|
+
"description": "SigRank MCP server — the yield cascade + live leaderboard as MCP tools any agent can call",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": { "sigrank-mcp": "./index.mjs" },
|
|
7
|
+
"main": "index.mjs",
|
|
8
|
+
"engines": { "node": ">=18" },
|
|
9
|
+
"scripts": {
|
|
10
|
+
"start": "node index.mjs",
|
|
11
|
+
"test": "node test.mjs"
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"index.mjs",
|
|
15
|
+
"tools.mjs",
|
|
16
|
+
"cascade.mjs",
|
|
17
|
+
"narrate.mjs",
|
|
18
|
+
"tokenpull.mjs",
|
|
19
|
+
"adapters.mjs",
|
|
20
|
+
"README.md"
|
|
21
|
+
],
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"@modelcontextprotocol/sdk": "^1.0.0"
|
|
24
|
+
}
|
|
25
|
+
}
|
package/tokenpull.mjs
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tokenpull.mjs — SigRank's in-house local usage reader.
|
|
3
|
+
*
|
|
4
|
+
* Reads token telemetry straight from the platform's local session logs (Claude Code
|
|
5
|
+
* first: ~/.claude/projects/<project>/<session>.jsonl) and slices it into the four
|
|
6
|
+
* windows (7d / 30d / 90d / all) of raw pillars — the WINDOWED_PROFILES payload, pulled
|
|
7
|
+
* with zero paste. No ccusage, no tokscale: our own reader, our own numbers.
|
|
8
|
+
*
|
|
9
|
+
* Token-only: we read usage counts (input/output/cache), message id, and timestamp —
|
|
10
|
+
* never message text. Billing-accurate: messages are deduped by `message.id` so the
|
|
11
|
+
* tally matches what the API actually billed (the one trick token-dashboard gets right
|
|
12
|
+
* and sigrank-agent's line-index keying does not).
|
|
13
|
+
*
|
|
14
|
+
* Adapter-shaped for multi-system: add a Codex / Cursor / Gemini reader by implementing
|
|
15
|
+
* the same { platform, defaultRoot(), messages(root) } contract — Claude is just the first.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { readdir, readFile, lstat } from 'node:fs/promises'
|
|
19
|
+
import { join } from 'node:path'
|
|
20
|
+
import { homedir } from 'node:os'
|
|
21
|
+
import { ADAPTERS } from './adapters.mjs'
|
|
22
|
+
|
|
23
|
+
const DAY_MS = 86_400_000
|
|
24
|
+
|
|
25
|
+
// Background tooling that runs under your account but is NOT your work — memory plugins,
|
|
26
|
+
// observers, summarizers, sub-agent fleets. They pad output/Υ (claude-mem did ~27%).
|
|
27
|
+
// Excluded so pillars reflect the operator, not their tools. EXTENSIBLE — add tool
|
|
28
|
+
// dir-name patterns here. Strategic note: others' tools inflate the PUBLIC
|
|
29
|
+
// token-dashboard / tokscale boards; SigRank filters them, so SigRank stays honest.
|
|
30
|
+
// subagents/ are KEPT (real work). Future-robust signal: also drop entrypoint=sdk-cli.
|
|
31
|
+
export const EXCLUDE_TOOLING =
|
|
32
|
+
/(^|[/-])(claude-mem|mem0|claude-self-reflect|basic-memory|memento|cipher-mem|memory-keeper)\b|observer-(sessions|archive)/i
|
|
33
|
+
|
|
34
|
+
/** The four windows, in days. `all` = unbounded. */
|
|
35
|
+
export const WINDOWS = [
|
|
36
|
+
{ key: '7d', days: 7 },
|
|
37
|
+
{ key: '30d', days: 30 },
|
|
38
|
+
{ key: '90d', days: 90 },
|
|
39
|
+
{ key: 'all', days: Infinity },
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
/** Hard cap: stop walking after this many .jsonl files to prevent OOM on
|
|
43
|
+
* accidentally huge or circularly-symlinked directory trees.
|
|
44
|
+
* Override via SIGRANK_MAX_JSONL_FILES env var. */
|
|
45
|
+
const MAX_JSONL_FILES = Number(process.env.SIGRANK_MAX_JSONL_FILES) || 10_000
|
|
46
|
+
|
|
47
|
+
/** Recursively yield every *.jsonl path under dir (any depth), sorted. Must be
|
|
48
|
+
* recursive: Claude Code stores sub-agent transcripts in `<project>/subagents/`
|
|
49
|
+
* (and other nested logs) — a 2-level readdir silently drops them, which badly
|
|
50
|
+
* under-counts input (sub-agent runs are input-heavy). token-dashboard's rglob.
|
|
51
|
+
*
|
|
52
|
+
* Hardened: skips symlinked directories (prevents circular traversal) and stops
|
|
53
|
+
* after MAX_JSONL_FILES files (prevents OOM on pathological trees). */
|
|
54
|
+
async function* _walkJsonl(dir, _counter = { n: 0 }) {
|
|
55
|
+
if (_counter.n >= MAX_JSONL_FILES) return
|
|
56
|
+
let entries
|
|
57
|
+
try { entries = await readdir(dir, { withFileTypes: true }) } catch { return }
|
|
58
|
+
for (const e of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
|
|
59
|
+
if (_counter.n >= MAX_JSONL_FILES) return
|
|
60
|
+
const full = join(dir, e.name)
|
|
61
|
+
if (e.isDirectory()) {
|
|
62
|
+
// Skip symlinked directories — prevents circular traversal on machines where
|
|
63
|
+
// ~/.claude/projects/ contains a symlink to a large or self-referential tree.
|
|
64
|
+
let stat
|
|
65
|
+
try { stat = await lstat(full) } catch { continue }
|
|
66
|
+
if (stat.isSymbolicLink()) continue
|
|
67
|
+
yield* _walkJsonl(full, _counter)
|
|
68
|
+
} else if (e.isFile() && e.name.endsWith('.jsonl')) {
|
|
69
|
+
_counter.n++
|
|
70
|
+
yield full
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Claude Code adapter. Yields one record per billed assistant message:
|
|
77
|
+
* { id, sid, ts, input, output, cacheCreate, cacheRead, file }. Walks
|
|
78
|
+
* ~/.claude/projects RECURSIVELY (incl. subagents/) and carries sessionId so the
|
|
79
|
+
* caller can dedup by (session_id, message_id) like token-dashboard.
|
|
80
|
+
*/
|
|
81
|
+
export const claudeAdapter = {
|
|
82
|
+
platform: 'claude',
|
|
83
|
+
defaultRoot: () => join(homedir(), '.claude', 'projects'),
|
|
84
|
+
async *messages(root) {
|
|
85
|
+
for await (const path of _walkJsonl(root)) {
|
|
86
|
+
let text
|
|
87
|
+
try { text = await readFile(path, 'utf8') } catch { continue }
|
|
88
|
+
const rel = path.startsWith(root) ? path.slice(root.length + 1) : path
|
|
89
|
+
if (EXCLUDE_TOOLING.test(rel)) continue // skip claude-mem observer & other background tooling
|
|
90
|
+
for (const line of text.split('\n')) {
|
|
91
|
+
const s = line.trim()
|
|
92
|
+
if (!s) continue
|
|
93
|
+
let ev
|
|
94
|
+
try { ev = JSON.parse(s) } catch { continue }
|
|
95
|
+
const m = ev && ev.message
|
|
96
|
+
if (!m || typeof m !== 'object') continue
|
|
97
|
+
const u = m.usage
|
|
98
|
+
if (!u || typeof u !== 'object') continue
|
|
99
|
+
yield {
|
|
100
|
+
id: m.id || null,
|
|
101
|
+
sid: ev.sessionId || null,
|
|
102
|
+
ts: ev.timestamp || ev.ts || null,
|
|
103
|
+
input: Number(u.input_tokens) || 0,
|
|
104
|
+
output: Number(u.output_tokens) || 0,
|
|
105
|
+
cacheCreate: Number(u.cache_creation_input_tokens) || 0,
|
|
106
|
+
cacheRead: Number(u.cache_read_input_tokens) || 0,
|
|
107
|
+
file: rel,
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Pull local usage and slice it into the four windows of raw pillars.
|
|
116
|
+
* Deterministic given (adapter output, now). Inject `now`/`root`/`adapter` for tests.
|
|
117
|
+
* Returns { platform, root, generatedAt, files, totalMessages, windows:[{window,pillars,messages}] }.
|
|
118
|
+
*/
|
|
119
|
+
export async function tokenpull({ adapter = claudeAdapter, root, now } = {}) {
|
|
120
|
+
const r = root || adapter.defaultRoot()
|
|
121
|
+
const nowMs = now == null ? Date.now() : (typeof now === 'number' ? now : Date.parse(now))
|
|
122
|
+
|
|
123
|
+
// Dedup by (session_id, message_id), keeping the FINAL snapshot — matches
|
|
124
|
+
// token-dashboard: Claude Code writes 2-3 partial→final lines per response with
|
|
125
|
+
// the same message.id; only the final tally matches billing. No-id records each
|
|
126
|
+
// get a unique synthetic key so they always count.
|
|
127
|
+
const seen = new Map()
|
|
128
|
+
const files = new Set()
|
|
129
|
+
let noId = 0
|
|
130
|
+
for await (const msg of adapter.messages(r)) {
|
|
131
|
+
const key = msg.sid && msg.id ? `${msg.sid}|${msg.id}` : (msg.id || `__noid_${noId++}`)
|
|
132
|
+
seen.set(key, msg) // keep-last (final snapshot wins)
|
|
133
|
+
if (msg.file) files.add(msg.file)
|
|
134
|
+
}
|
|
135
|
+
const msgs = [...seen.values()]
|
|
136
|
+
|
|
137
|
+
const windows = WINDOWS.map((w) => {
|
|
138
|
+
const cutoff = w.days === Infinity ? -Infinity : nowMs - w.days * DAY_MS
|
|
139
|
+
const inWin = msgs.filter((m) => {
|
|
140
|
+
if (w.days === Infinity) return true
|
|
141
|
+
const t = m.ts ? Date.parse(m.ts) : NaN
|
|
142
|
+
return Number.isFinite(t) && t >= cutoff && t <= nowMs
|
|
143
|
+
})
|
|
144
|
+
const pillars = inWin.reduce(
|
|
145
|
+
(a, m) => ({ input: a.input + m.input, output: a.output + m.output, cacheCreate: a.cacheCreate + m.cacheCreate, cacheRead: a.cacheRead + m.cacheRead }),
|
|
146
|
+
{ input: 0, output: 0, cacheCreate: 0, cacheRead: 0 },
|
|
147
|
+
)
|
|
148
|
+
return { window: w.key, pillars, messages: inWin.length }
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
// Auto-detect background tooling we excluded → report it (the MCP "asks" by telling).
|
|
152
|
+
let excludedTooling = []
|
|
153
|
+
try {
|
|
154
|
+
const top = await readdir(r, { withFileTypes: true })
|
|
155
|
+
excludedTooling = top.filter((d) => d.isDirectory() && EXCLUDE_TOOLING.test(d.name)).map((d) => d.name)
|
|
156
|
+
} catch { /* ignore */ }
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
platform: adapter.platform,
|
|
160
|
+
root: r,
|
|
161
|
+
generatedAt: new Date(nowMs).toISOString(),
|
|
162
|
+
files: files.size,
|
|
163
|
+
totalMessages: msgs.length,
|
|
164
|
+
excludedTooling,
|
|
165
|
+
windows,
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// ── Codex ──────────────────────────────────────────────────────────────────
|
|
170
|
+
// Logs at ~/.codex/sessions/**/rollout-*.jsonl (+ archived_sessions). Token usage is
|
|
171
|
+
// on `payload.type=='token_count'` lines; `payload.info.last_token_usage` is the
|
|
172
|
+
// per-turn delta. Codex `input_tokens` is INCLUSIVE of cached, so the combined input to
|
|
173
|
+
// split = input_tokens − cached_input_tokens (= ccusage's inputTokens, verified ~1%).
|
|
174
|
+
// The input/cacheCreate split is WINDOW-level (estInput = output × io_ratio), so Codex
|
|
175
|
+
// has its own pull (tokenpullCodex) instead of the per-message claude pipeline.
|
|
176
|
+
export const codexAdapter = {
|
|
177
|
+
platform: 'codex',
|
|
178
|
+
defaultRoot: () => join(homedir(), '.codex'),
|
|
179
|
+
async *records(root) {
|
|
180
|
+
for (const sub of ['sessions', 'archived_sessions']) {
|
|
181
|
+
for await (const path of _walkJsonl(join(root, sub))) {
|
|
182
|
+
let text
|
|
183
|
+
try { text = await readFile(path, 'utf8') } catch { continue }
|
|
184
|
+
const rel = path.startsWith(root) ? path.slice(root.length + 1) : path
|
|
185
|
+
// Apply the same background-tooling exclusion as the Claude adapter — skips
|
|
186
|
+
// memory plugins / observers running under the Codex account.
|
|
187
|
+
if (EXCLUDE_TOOLING.test(rel)) continue
|
|
188
|
+
for (const line of text.split('\n')) {
|
|
189
|
+
if (!line.includes('"token_count"')) continue
|
|
190
|
+
let ev
|
|
191
|
+
try { ev = JSON.parse(line) } catch { continue }
|
|
192
|
+
const p = ev && ev.payload
|
|
193
|
+
if (!p || p.type !== 'token_count') continue
|
|
194
|
+
const u = (p.info || {}).last_token_usage || {}
|
|
195
|
+
const inputIncl = Number(u.input_tokens) || 0
|
|
196
|
+
const cached = Number(u.cached_input_tokens) || 0
|
|
197
|
+
yield {
|
|
198
|
+
ts: ev.timestamp || null,
|
|
199
|
+
output: (Number(u.output_tokens) || 0) + (Number(u.reasoning_output_tokens) || 0),
|
|
200
|
+
cacheRead: cached,
|
|
201
|
+
uncached: Math.max(0, inputIncl - cached), // (true input + cache-write), split window-level
|
|
202
|
+
file: rel,
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
},
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Pull local Codex usage → the 4 windows of CANONICAL pillars (always estimated).
|
|
212
|
+
* Window-level conversion: input = floor(output × ioRatio) (Beta = operator's Claude
|
|
213
|
+
* input/output ratio; Alpha = 2.0 default), cacheCreate = max(0, uncached − input).
|
|
214
|
+
* Inject ioRatio/root/now/adapter for tests.
|
|
215
|
+
*/
|
|
216
|
+
export async function tokenpullCodex({ adapter = codexAdapter, root, now, ioRatio = 2.0 } = {}) {
|
|
217
|
+
const r = root || adapter.defaultRoot()
|
|
218
|
+
const nowMs = now == null ? Date.now() : (typeof now === 'number' ? now : Date.parse(now))
|
|
219
|
+
const recs = []
|
|
220
|
+
const files = new Set()
|
|
221
|
+
for await (const m of adapter.records(r)) { recs.push(m); if (m.file) files.add(m.file) }
|
|
222
|
+
|
|
223
|
+
const windows = WINDOWS.map((w) => {
|
|
224
|
+
const cutoff = w.days === Infinity ? -Infinity : nowMs - w.days * DAY_MS
|
|
225
|
+
const inWin = recs.filter((m) => {
|
|
226
|
+
if (w.days === Infinity) return true
|
|
227
|
+
const t = m.ts ? Date.parse(m.ts) : NaN
|
|
228
|
+
return Number.isFinite(t) && t >= cutoff && t <= nowMs
|
|
229
|
+
})
|
|
230
|
+
const sum = inWin.reduce(
|
|
231
|
+
(a, m) => ({ output: a.output + m.output, cacheRead: a.cacheRead + m.cacheRead, uncached: a.uncached + m.uncached }),
|
|
232
|
+
{ output: 0, cacheRead: 0, uncached: 0 },
|
|
233
|
+
)
|
|
234
|
+
const input = Math.floor(sum.output * ioRatio) // estimated true input
|
|
235
|
+
const cacheCreate = Math.max(0, sum.uncached - input) // split cache-write out of the combined input
|
|
236
|
+
return { window: w.key, messages: inWin.length, pillars: { input, output: sum.output, cacheCreate, cacheRead: sum.cacheRead } }
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
return { platform: 'codex', root: r, generatedAt: new Date(nowMs).toISOString(), files: files.size, totalMessages: recs.length, estimated: true, ioRatio, windows }
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Unified pull for ANY platform by name. Routes to the right adapter:
|
|
244
|
+
* - 'claude' → tokenpull()
|
|
245
|
+
* - 'codex' → tokenpullCodex() (requires ioRatio via opts; auto-derives from Claude if available)
|
|
246
|
+
* - other → tokenpull() with the adapter from ADAPTERS registry
|
|
247
|
+
*
|
|
248
|
+
* Returns the same shape as tokenpull() with an added `estimated` flag when the
|
|
249
|
+
* adapter cannot provide full cacheCreate data, and a `dataGap` string when the
|
|
250
|
+
* source's log format doesn't expose token counts at all.
|
|
251
|
+
*/
|
|
252
|
+
export async function tokenpullAny(platform, opts = {}) {
|
|
253
|
+
if (!platform || platform === 'claude') return tokenpull({ adapter: claudeAdapter, ...opts })
|
|
254
|
+
if (platform === 'codex') {
|
|
255
|
+
// Auto-derive io_ratio from the operator's Claude data when not explicitly provided.
|
|
256
|
+
let ioRatio = opts.ioRatio || 2.0
|
|
257
|
+
if (!opts.ioRatio) {
|
|
258
|
+
try {
|
|
259
|
+
const c = await tokenpull({ adapter: claudeAdapter })
|
|
260
|
+
const all = c.windows.find((w) => w.window === 'all')
|
|
261
|
+
if (all && all.pillars.output > 0) ioRatio = all.pillars.input / all.pillars.output
|
|
262
|
+
} catch { /* no Claude data → Alpha 2.0 */ }
|
|
263
|
+
}
|
|
264
|
+
return tokenpullCodex({ ioRatio, ...opts })
|
|
265
|
+
}
|
|
266
|
+
const adapter = ADAPTERS[platform]
|
|
267
|
+
if (!adapter) throw new Error(`Unknown platform "${platform}". Valid platforms: claude, codex, ${Object.keys(ADAPTERS).join(', ')}`)
|
|
268
|
+
const result = await tokenpull({ adapter, ...opts })
|
|
269
|
+
// Surface adapter-level flags
|
|
270
|
+
if (adapter.estimated) result.estimated = true
|
|
271
|
+
if (adapter.dataGap) result.dataGap = adapter.dataGap
|
|
272
|
+
if (adapter.setupNote) result.setupNote = adapter.setupNote
|
|
273
|
+
return result
|
|
274
|
+
}
|