sigrank 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +301 -0
- package/adapters.mjs +480 -0
- package/cascade.mjs +143 -0
- package/cli.mjs +1394 -0
- package/index.mjs +82 -0
- package/keystore.mjs +142 -0
- package/narrate.mjs +65 -0
- package/package.json +45 -0
- package/sign.mjs +87 -0
- package/submit.mjs +164 -0
- package/tokenpull.mjs +282 -0
- package/tools.mjs +610 -0
- package/tui.mjs +1582 -0
package/adapters.mjs
ADDED
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* adapters.mjs — SigRank tokenpull adapters for all supported platforms.
|
|
3
|
+
*
|
|
4
|
+
* OKF (code-file form — fields live in this comment, not raw frontmatter, so the .mjs
|
|
5
|
+
* still parses as valid JS):
|
|
6
|
+
* type: Reference
|
|
7
|
+
* title: SigRank tokenpull adapters
|
|
8
|
+
* description: Per-platform adapters implementing the tokenpull contract — async-generate
|
|
9
|
+
* {id,sid,ts,input,output,cacheCreate,cacheRead,file} from local logs. Reasoning→output,
|
|
10
|
+
* cost fields dropped, missing cacheCreate → estimated:true. Token-only, read-only.
|
|
11
|
+
* tags: [sigrank, mcp, tokenpull, adapters, reference]
|
|
12
|
+
* timestamp: 2026-06-23
|
|
13
|
+
*
|
|
14
|
+
* Each adapter implements the tokenpull contract:
|
|
15
|
+
* messages(root): async generator → { id?, sid?, ts, input, output, cacheCreate, cacheRead, file }
|
|
16
|
+
*
|
|
17
|
+
* SigRank-specific mapping rules (applied consistently across all adapters):
|
|
18
|
+
* - Reasoning / thinking tokens → folded into `output` (they are output-side spend)
|
|
19
|
+
* - No cache-creation data available → cacheCreate: 0 + adapter sets `estimated: true`
|
|
20
|
+
* - Cost fields (USD) → NEVER used or forwarded (SigRank scores cost efficiency from
|
|
21
|
+
* token ratios, not from dollar amounts — cost efficiency is derived, not ingested)
|
|
22
|
+
* - Credits / provider-specific fields → dropped
|
|
23
|
+
*
|
|
24
|
+
* "estimated" flag: set on the ADAPTER OBJECT (not per-record) when the adapter
|
|
25
|
+
* cannot provide native cacheCreate values. tokenpull() and tokenpullCodex() already
|
|
26
|
+
* handle this pattern; new adapters with estimated=true get the same treatment.
|
|
27
|
+
*
|
|
28
|
+
* SQLite adapters shell out to `sqlite3 -json` (macOS/Linux system tool, no npm dep).
|
|
29
|
+
* If sqlite3 is unavailable the adapter returns an empty generator with a dataGap note.
|
|
30
|
+
*
|
|
31
|
+
* Data-gap notes (sources that can't provide full 4-pillar data) are attached on the
|
|
32
|
+
* adapter object as `dataGap: string` so tokenpull() can surface them to the user.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { readdir, readFile } from 'node:fs/promises'
|
|
36
|
+
import { join, extname } from 'node:path'
|
|
37
|
+
import { homedir } from 'node:os'
|
|
38
|
+
import { exec as execCb } from 'node:child_process'
|
|
39
|
+
import { promisify } from 'node:util'
|
|
40
|
+
|
|
41
|
+
const exec = promisify(execCb)
|
|
42
|
+
const DAY_MS = 86_400_000 // shared with tokenpull.mjs but kept local to avoid circular import
|
|
43
|
+
|
|
44
|
+
// ── File-system helpers ───────────────────────────────────────────────────────
|
|
45
|
+
|
|
46
|
+
/** Recursively yield every file whose name matches `pred` under dir (skips symlink dirs). */
|
|
47
|
+
async function* walkFiles(dir, pred, counter = { n: 0 }, max = 10_000) {
|
|
48
|
+
if (counter.n >= max) return
|
|
49
|
+
let entries
|
|
50
|
+
try { entries = await readdir(dir, { withFileTypes: true }) } catch { return }
|
|
51
|
+
for (const e of entries) {
|
|
52
|
+
if (counter.n >= max) return
|
|
53
|
+
const full = join(dir, e.name)
|
|
54
|
+
if (e.isSymbolicLink()) continue
|
|
55
|
+
if (e.isDirectory()) { yield* walkFiles(full, pred, counter, max) }
|
|
56
|
+
else if (e.isFile() && pred(e.name)) { counter.n++; yield full }
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const isJsonl = (n) => n.endsWith('.jsonl') || n.endsWith('.jsonl.deleted') || n.match(/\.jsonl\.reset\.\d+$/)
|
|
61
|
+
const isJson = (n) => n.endsWith('.json') && !n.endsWith('.jsonl')
|
|
62
|
+
|
|
63
|
+
/** Read a file as UTF-8, return null on error. */
|
|
64
|
+
async function readUtf8(path) {
|
|
65
|
+
try { return await readFile(path, 'utf8') } catch { return null }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Parse each newline-delimited JSON line, yield parsed objects silently skipping bad lines. */
|
|
69
|
+
function* parseJsonl(text, filePath) {
|
|
70
|
+
if (!text) return
|
|
71
|
+
for (const line of text.split('\n')) {
|
|
72
|
+
const s = line.trim()
|
|
73
|
+
if (!s) continue
|
|
74
|
+
try { yield [JSON.parse(s), filePath] } catch { /* skip malformed */ }
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Shell out to sqlite3 and return parsed JSON rows, or [] on error/unavailability. */
|
|
79
|
+
async function sqliteJson(dbPath, sql) {
|
|
80
|
+
try {
|
|
81
|
+
const { stdout } = await exec(`sqlite3 -json "${dbPath}" "${sql.replace(/"/g, '\\"')}"`, { timeout: 10_000 })
|
|
82
|
+
return JSON.parse(stdout || '[]')
|
|
83
|
+
} catch { return [] }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ── Env-var helper ────────────────────────────────────────────────────────────
|
|
87
|
+
/** Resolve roots from env var or default. Supports comma-separated list. */
|
|
88
|
+
function roots(envVar, defaultPath) {
|
|
89
|
+
const v = process.env[envVar]
|
|
90
|
+
if (v) return v.split(',').map((s) => s.trim()).filter(Boolean)
|
|
91
|
+
return [defaultPath]
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ── 1. Amp ────────────────────────────────────────────────────────────────────
|
|
95
|
+
// ~/.local/share/amp/threads/**/*.json
|
|
96
|
+
// Fields: assistant message usage: input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens
|
|
97
|
+
export const ampAdapter = {
|
|
98
|
+
platform: 'amp',
|
|
99
|
+
defaultRoot: () => join(homedir(), '.local', 'share', 'amp'),
|
|
100
|
+
async *messages(root) {
|
|
101
|
+
for (const r of roots('AMP_DATA_DIR', root)) {
|
|
102
|
+
const threadsDir = join(r, 'threads')
|
|
103
|
+
for await (const path of walkFiles(threadsDir, isJson)) {
|
|
104
|
+
const text = await readUtf8(path)
|
|
105
|
+
if (!text) continue
|
|
106
|
+
let thread
|
|
107
|
+
try { thread = JSON.parse(text) } catch { continue }
|
|
108
|
+
// Amp thread is an array or object with messages
|
|
109
|
+
const msgs = Array.isArray(thread) ? thread : (thread.messages || [])
|
|
110
|
+
for (const msg of msgs) {
|
|
111
|
+
if (!msg || msg.role !== 'assistant') continue
|
|
112
|
+
const u = msg.usage || (msg.metadata && msg.metadata.usage) || {}
|
|
113
|
+
const input = Number(u.input_tokens || u.inputTokens || 0)
|
|
114
|
+
const output = Number(u.output_tokens || u.outputTokens || 0)
|
|
115
|
+
const cacheCreate = Number(u.cache_creation_tokens || u.cacheCreationTokens || 0)
|
|
116
|
+
const cacheRead = Number(u.cache_read_tokens || u.cacheReadTokens || 0)
|
|
117
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
118
|
+
yield { id: msg.id || null, sid: thread.id || null, ts: msg.timestamp || msg.created_at || null, input, output, cacheCreate, cacheRead, file: path }
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ── 2. Kimi ───────────────────────────────────────────────────────────────────
|
|
126
|
+
// ~/.kimi/sessions/<group-id>/<session-id>/wire.jsonl
|
|
127
|
+
// StatusUpdate lines only; token_usage: { input_other, output, input_cache_read, input_cache_creation }
|
|
128
|
+
export const kimiAdapter = {
|
|
129
|
+
platform: 'kimi',
|
|
130
|
+
defaultRoot: () => join(homedir(), '.kimi'),
|
|
131
|
+
async *messages(root) {
|
|
132
|
+
const sessionsDir = join(roots('KIMI_DATA_DIR', root)[0], 'sessions')
|
|
133
|
+
for await (const path of walkFiles(sessionsDir, isJsonl)) {
|
|
134
|
+
const text = await readUtf8(path)
|
|
135
|
+
for (const [ev] of parseJsonl(text, path)) {
|
|
136
|
+
if (!ev || ev.type !== 'StatusUpdate') continue
|
|
137
|
+
const u = ev.token_usage || {}
|
|
138
|
+
const input = Number(u.input_other || 0)
|
|
139
|
+
const output = Number(u.output || 0)
|
|
140
|
+
const cacheCreate = Number(u.input_cache_creation || 0)
|
|
141
|
+
const cacheRead = Number(u.input_cache_read || 0)
|
|
142
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
143
|
+
yield { id: ev.id || null, sid: null, ts: ev.timestamp || ev.created_at || null, input, output, cacheCreate, cacheRead, file: path }
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ── 3. Qwen ───────────────────────────────────────────────────────────────────
|
|
150
|
+
// ~/.qwen/projects/{project}/chats/*.jsonl
|
|
151
|
+
// usageMetadata: { promptTokenCount, candidatesTokenCount, cachedContentTokenCount, thoughtsTokenCount }
|
|
152
|
+
// No cache creation field. Reasoning (thoughtsTokenCount) → output.
|
|
153
|
+
export const qwenAdapter = {
|
|
154
|
+
platform: 'qwen',
|
|
155
|
+
estimated: true, // no cacheCreate in logs
|
|
156
|
+
defaultRoot: () => join(homedir(), '.qwen'),
|
|
157
|
+
async *messages(root) {
|
|
158
|
+
const projectsDir = join(roots('QWEN_DATA_DIR', root)[0], 'projects')
|
|
159
|
+
for await (const path of walkFiles(projectsDir, isJsonl)) {
|
|
160
|
+
const text = await readUtf8(path)
|
|
161
|
+
for (const [ev] of parseJsonl(text, path)) {
|
|
162
|
+
if (!ev || !ev.usageMetadata) continue
|
|
163
|
+
const u = ev.usageMetadata
|
|
164
|
+
const rawInput = Number(u.promptTokenCount || 0)
|
|
165
|
+
const rawOutput = Number(u.candidatesTokenCount || 0)
|
|
166
|
+
const thoughts = Number(u.thoughtsTokenCount || 0) // reasoning → output
|
|
167
|
+
const cacheRead = Number(u.cachedContentTokenCount || 0)
|
|
168
|
+
// promptTokenCount is inclusive of cached; subtract to get fresh input
|
|
169
|
+
const input = Math.max(0, rawInput - cacheRead)
|
|
170
|
+
const output = rawOutput + thoughts
|
|
171
|
+
if (input + output + cacheRead === 0) continue
|
|
172
|
+
yield { id: ev.id || null, sid: null, ts: ev.timestamp || null, input, output, cacheCreate: 0, cacheRead, file: path }
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// ── 4. pi-agent ───────────────────────────────────────────────────────────────
|
|
179
|
+
// ~/.pi/agent/sessions/**/*.jsonl
|
|
180
|
+
// Fields: inputTokens, outputTokens, cacheCreationTokens, cacheReadTokens
|
|
181
|
+
export const piAdapter = {
|
|
182
|
+
platform: 'pi',
|
|
183
|
+
defaultRoot: () => join(homedir(), '.pi', 'agent', 'sessions'),
|
|
184
|
+
async *messages(root) {
|
|
185
|
+
for (const r of roots('PI_AGENT_DIR', root)) {
|
|
186
|
+
for await (const path of walkFiles(r, isJsonl)) {
|
|
187
|
+
const text = await readUtf8(path)
|
|
188
|
+
for (const [ev] of parseJsonl(text, path)) {
|
|
189
|
+
if (!ev) continue
|
|
190
|
+
// pi-agent stores usage in assistant messages or usage events
|
|
191
|
+
const u = ev.usage || ev
|
|
192
|
+
const input = Number(u.inputTokens || u.input_tokens || 0)
|
|
193
|
+
const output = Number(u.outputTokens || u.output_tokens || 0)
|
|
194
|
+
const cacheCreate = Number(u.cacheCreationTokens || u.cache_creation_tokens || 0)
|
|
195
|
+
const cacheRead = Number(u.cacheReadTokens || u.cache_read_tokens || 0)
|
|
196
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
197
|
+
yield { id: ev.id || null, sid: ev.sessionId || null, ts: ev.timestamp || null, input, output, cacheCreate, cacheRead, file: path }
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
},
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// ── 5. OpenClaw ───────────────────────────────────────────────────────────────
|
|
205
|
+
// ~/.openclaw/ (also ~/.clawdbot/, ~/.moltbot/, ~/.moldbot/)
|
|
206
|
+
// agents/<agentId>/sessions/<uuid>.jsonl (+ .deleted.<ts> and .reset.<ts> variants)
|
|
207
|
+
// Per-message: input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens
|
|
208
|
+
export const openclawAdapter = {
|
|
209
|
+
platform: 'openclaw',
|
|
210
|
+
defaultRoot: () => {
|
|
211
|
+
const env = process.env.OPENCLAW_DIR
|
|
212
|
+
if (env) return env.split(',')[0].trim()
|
|
213
|
+
for (const d of ['.openclaw', '.clawdbot', '.moltbot', '.moldbot']) {
|
|
214
|
+
const p = join(homedir(), d)
|
|
215
|
+
return p // return first; walkFiles silently skips missing
|
|
216
|
+
}
|
|
217
|
+
return join(homedir(), '.openclaw')
|
|
218
|
+
},
|
|
219
|
+
async *messages(root) {
|
|
220
|
+
const dirs = process.env.OPENCLAW_DIR
|
|
221
|
+
? process.env.OPENCLAW_DIR.split(',').map((s) => s.trim())
|
|
222
|
+
: [join(homedir(), '.openclaw'), join(homedir(), '.clawdbot'), join(homedir(), '.moltbot'), join(homedir(), '.moldbot')]
|
|
223
|
+
const pred = (n) => isJsonl(n) || n.endsWith('.json') // covers archived variants
|
|
224
|
+
for (const dir of dirs) {
|
|
225
|
+
for await (const path of walkFiles(dir, pred)) {
|
|
226
|
+
const text = await readUtf8(path)
|
|
227
|
+
for (const [ev] of parseJsonl(text, path)) {
|
|
228
|
+
if (!ev || ev.role !== 'assistant') continue
|
|
229
|
+
const u = ev.usage || ev.tokens || {}
|
|
230
|
+
const input = Number(u.input_tokens || u.inputTokens || 0)
|
|
231
|
+
const output = Number(u.output_tokens || u.outputTokens || 0)
|
|
232
|
+
const cacheCreate = Number(u.cache_creation_tokens || u.cacheCreationTokens || 0)
|
|
233
|
+
const cacheRead = Number(u.cache_read_tokens || u.cacheReadTokens || 0)
|
|
234
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
235
|
+
yield { id: ev.id || null, sid: null, ts: ev.timestamp || null, input, output, cacheCreate, cacheRead, file: path }
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
},
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// ── 6. Droid ──────────────────────────────────────────────────────────────────
|
|
243
|
+
// ~/.factory/sessions/**/*.settings.json
|
|
244
|
+
// Fields: input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens, thinking_tokens
|
|
245
|
+
// Thinking tokens → output. Per-settings-file (session-level granularity).
|
|
246
|
+
export const droidAdapter = {
|
|
247
|
+
platform: 'droid',
|
|
248
|
+
defaultRoot: () => join(homedir(), '.factory', 'sessions'),
|
|
249
|
+
async *messages(root) {
|
|
250
|
+
for (const r of roots('DROID_SESSIONS_DIR', root)) {
|
|
251
|
+
for await (const path of walkFiles(r, (n) => n.endsWith('.settings.json'))) {
|
|
252
|
+
const text = await readUtf8(path)
|
|
253
|
+
if (!text) continue
|
|
254
|
+
let s
|
|
255
|
+
try { s = JSON.parse(text) } catch { continue }
|
|
256
|
+
const input = Number(s.input_tokens || 0)
|
|
257
|
+
const output = Number(s.output_tokens || 0) + Number(s.thinking_tokens || 0)
|
|
258
|
+
const cacheCreate = Number(s.cache_creation_tokens || 0)
|
|
259
|
+
const cacheRead = Number(s.cache_read_tokens || 0)
|
|
260
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
261
|
+
yield { id: null, sid: s.session_id || s.id || null, ts: s.updated_at || s.created_at || null, input, output, cacheCreate, cacheRead, file: path }
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ── 7. Codebuff ───────────────────────────────────────────────────────────────
|
|
268
|
+
// ~/.config/manicode/projects/<project>/chats/<chat-id>/chat-messages.json
|
|
269
|
+
// assistant messages: metadata.usage or metadata.codebuff.usage
|
|
270
|
+
export const codebuffAdapter = {
|
|
271
|
+
platform: 'codebuff',
|
|
272
|
+
defaultRoot: () => join(homedir(), '.config', 'manicode'),
|
|
273
|
+
async *messages(root) {
|
|
274
|
+
const dirs = process.env.CODEBUFF_DATA_DIR
|
|
275
|
+
? process.env.CODEBUFF_DATA_DIR.split(',').map((s) => s.trim())
|
|
276
|
+
: [join(homedir(), '.config', 'manicode'), join(homedir(), '.config', 'manicode-dev'), join(homedir(), '.config', 'manicode-staging')]
|
|
277
|
+
for (const dir of dirs) {
|
|
278
|
+
for await (const path of walkFiles(dir, (n) => n === 'chat-messages.json')) {
|
|
279
|
+
const text = await readUtf8(path)
|
|
280
|
+
if (!text) continue
|
|
281
|
+
let msgs
|
|
282
|
+
try { msgs = JSON.parse(text) } catch { continue }
|
|
283
|
+
if (!Array.isArray(msgs)) continue
|
|
284
|
+
for (const msg of msgs) {
|
|
285
|
+
if (!msg || msg.role !== 'assistant') continue
|
|
286
|
+
const u = (msg.metadata && (msg.metadata.usage || msg.metadata.codebuff?.usage)) || {}
|
|
287
|
+
const input = Number(u.input_tokens || u.inputTokens || 0)
|
|
288
|
+
const output = Number(u.output_tokens || u.outputTokens || 0)
|
|
289
|
+
const cacheCreate = Number(u.cache_creation_tokens || u.cacheCreationTokens || 0)
|
|
290
|
+
const cacheRead = Number(u.cache_read_tokens || u.cacheReadTokens || 0)
|
|
291
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
292
|
+
yield { id: msg.id || null, sid: null, ts: msg.timestamp || msg.created_at || null, input, output, cacheCreate, cacheRead, file: path }
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
},
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ── 8. Gemini CLI ─────────────────────────────────────────────────────────────
|
|
300
|
+
// ~/.gemini/tmp/*/chats/*.json and *.jsonl
|
|
301
|
+
// Fields: input, output, cached, thought (reasoning), tool, total
|
|
302
|
+
// SigRank mapping: input = input−cached (fresh), cacheRead = cached, cacheCreate = 0 (not exposed),
|
|
303
|
+
// output = output + thought (reasoning→output)
|
|
304
|
+
export const geminiAdapter = {
|
|
305
|
+
platform: 'gemini',
|
|
306
|
+
estimated: true, // no cacheCreate field in Gemini logs
|
|
307
|
+
defaultRoot: () => join(homedir(), '.gemini', 'tmp'),
|
|
308
|
+
async *messages(root) {
|
|
309
|
+
for (const r of roots('GEMINI_DATA_DIR', root)) {
|
|
310
|
+
for await (const path of walkFiles(r, (n) => n.endsWith('.json') || n.endsWith('.jsonl'))) {
|
|
311
|
+
const text = await readUtf8(path)
|
|
312
|
+
if (!text) continue
|
|
313
|
+
// Try JSONL first, then single JSON
|
|
314
|
+
let parsed = []
|
|
315
|
+
try {
|
|
316
|
+
parsed = text.trim().split('\n').filter(Boolean).map((l) => JSON.parse(l))
|
|
317
|
+
} catch {
|
|
318
|
+
try { parsed = [JSON.parse(text)] } catch { continue }
|
|
319
|
+
}
|
|
320
|
+
for (const ev of parsed) {
|
|
321
|
+
if (!ev) continue
|
|
322
|
+
// Gemini usage may be at top level or nested in usageMetadata
|
|
323
|
+
const u = ev.usageMetadata || ev.usage || ev
|
|
324
|
+
const rawInput = Number(u.input || u.promptTokenCount || 0)
|
|
325
|
+
const rawOutput = Number(u.output || u.candidatesTokenCount || 0)
|
|
326
|
+
const cached = Number(u.cached || u.cachedContentTokenCount || 0)
|
|
327
|
+
const thought = Number(u.thought || u.thoughtsTokenCount || 0)
|
|
328
|
+
if (rawInput + rawOutput + cached + thought === 0) continue
|
|
329
|
+
const input = Math.max(0, rawInput - cached) // strip cached from input
|
|
330
|
+
const output = rawOutput + thought // reasoning → output
|
|
331
|
+
yield { id: ev.id || null, sid: null, ts: ev.timestamp || null, input, output, cacheCreate: 0, cacheRead: cached, file: path }
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
},
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// ── 9. GitHub Copilot CLI ─────────────────────────────────────────────────────
|
|
339
|
+
// ~/.copilot/otel/*.jsonl (requires COPILOT_OTEL_ENABLED=true before session start)
|
|
340
|
+
// OpenTelemetry spans; looks for llm.token_count.prompt / completion / cached attributes
|
|
341
|
+
export const copilotAdapter = {
|
|
342
|
+
platform: 'copilot',
|
|
343
|
+
defaultRoot: () => join(homedir(), '.copilot', 'otel'),
|
|
344
|
+
async *messages(root) {
|
|
345
|
+
const dir = process.env.COPILOT_OTEL_FILE_EXPORTER_PATH
|
|
346
|
+
? require('node:path').dirname(process.env.COPILOT_OTEL_FILE_EXPORTER_PATH)
|
|
347
|
+
: root
|
|
348
|
+
for await (const path of walkFiles(dir, isJsonl)) {
|
|
349
|
+
const text = await readUtf8(path)
|
|
350
|
+
for (const [ev] of parseJsonl(text, path)) {
|
|
351
|
+
if (!ev) continue
|
|
352
|
+
// OTel span: attributes may be an object or array of {key,value} pairs
|
|
353
|
+
const attrs = ev.attributes || ev.resource?.attributes || {}
|
|
354
|
+
const get = (k) => {
|
|
355
|
+
if (typeof attrs === 'object' && !Array.isArray(attrs)) return attrs[k]
|
|
356
|
+
if (Array.isArray(attrs)) { const a = attrs.find((x) => x.key === k); return a?.value?.intValue ?? a?.value?.stringValue ?? null }
|
|
357
|
+
return null
|
|
358
|
+
}
|
|
359
|
+
const input = Number(get('llm.token_count.prompt') || get('gen_ai.usage.input_tokens') || 0)
|
|
360
|
+
const output = Number(get('llm.token_count.completion') || get('gen_ai.usage.output_tokens') || 0)
|
|
361
|
+
const cacheCreate = Number(get('llm.token_count.cache_creation') || 0)
|
|
362
|
+
const cacheRead = Number(get('llm.token_count.cache_read') || get('gen_ai.usage.cache_read_input_tokens') || 0)
|
|
363
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
364
|
+
yield { id: ev.traceId || ev.spanId || null, sid: null, ts: ev.startTimeUnixNano ? new Date(Number(ev.startTimeUnixNano) / 1e6).toISOString() : null, input, output, cacheCreate, cacheRead, file: path }
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
},
|
|
368
|
+
setupNote: 'Requires COPILOT_OTEL_ENABLED=true and COPILOT_OTEL_EXPORTER_TYPE=file set BEFORE starting the Copilot session. Without this, no local token logs are written.',
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// ── 10. OpenCode ──────────────────────────────────────────────────────────────
|
|
372
|
+
// ~/.local/share/opencode — JSON message files, but costs stored as 0 and token fields
|
|
373
|
+
// are calculated via LiteLLM pricing (not stored in the log). No raw token fields.
|
|
374
|
+
// SigRank cannot derive pillars from OpenCode logs with current log format.
|
|
375
|
+
export const opencodeAdapter = {
|
|
376
|
+
platform: 'opencode',
|
|
377
|
+
defaultRoot: () => join(homedir(), '.local', 'share', 'opencode'),
|
|
378
|
+
estimated: true,
|
|
379
|
+
dataGap: 'OpenCode logs store cost:0 and derive tokens via LiteLLM at runtime — raw token counts are not persisted. SigRank cannot read pillars from OpenCode logs with the current format. Track https://github.com/ccusage/ccusage for format changes.',
|
|
380
|
+
// eslint-disable-next-line require-yield
|
|
381
|
+
async *messages() { /* no data available */ },
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// ── 11. Goose ─────────────────────────────────────────────────────────────────
|
|
385
|
+
// SQLite: sessions.db at standard Goose data roots or $GOOSE_PATH_ROOT/data/sessions/sessions.db
|
|
386
|
+
// Columns: accumulated_input_tokens (or input_tokens), accumulated_output_tokens (or output_tokens),
|
|
387
|
+
// accumulated_total_tokens (or total_tokens). NO cache fields. Reasoning = total-input-output.
|
|
388
|
+
export const gooseAdapter = {
|
|
389
|
+
platform: 'goose',
|
|
390
|
+
estimated: true, // no cacheCreate or cacheRead
|
|
391
|
+
defaultRoot: () => {
|
|
392
|
+
const env = process.env.GOOSE_PATH_ROOT
|
|
393
|
+
if (env) return env
|
|
394
|
+
// Standard locations (macOS first, then XDG)
|
|
395
|
+
return join(homedir(), 'Library', 'Application Support', 'goose')
|
|
396
|
+
},
|
|
397
|
+
async *messages(root) {
|
|
398
|
+
const dbCandidates = process.env.GOOSE_PATH_ROOT
|
|
399
|
+
? [join(process.env.GOOSE_PATH_ROOT, 'data', 'sessions', 'sessions.db')]
|
|
400
|
+
: [
|
|
401
|
+
join(homedir(), 'Library', 'Application Support', 'goose', 'sessions', 'sessions.db'),
|
|
402
|
+
join(homedir(), '.local', 'share', 'goose', 'sessions', 'sessions.db'),
|
|
403
|
+
join(homedir(), '.local', 'share', 'Block', 'goose', 'sessions', 'sessions.db'),
|
|
404
|
+
]
|
|
405
|
+
for (const db of dbCandidates) {
|
|
406
|
+
const rows = await sqliteJson(db, 'SELECT * FROM sessions')
|
|
407
|
+
for (const row of rows) {
|
|
408
|
+
const input = Number(row.accumulated_input_tokens || row.input_tokens || 0)
|
|
409
|
+
const output = Number(row.accumulated_output_tokens || row.output_tokens || 0)
|
|
410
|
+
const total = Number(row.accumulated_total_tokens || row.total_tokens || 0)
|
|
411
|
+
const reasoning = Math.max(0, total - input - output) // folded into output
|
|
412
|
+
if (input + output === 0) continue
|
|
413
|
+
yield { id: String(row.id || ''), sid: null, ts: row.created_at || row.updated_at || null, input, output: output + reasoning, cacheCreate: 0, cacheRead: 0, file: db }
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
},
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// ── 12. Kilo ──────────────────────────────────────────────────────────────────
|
|
420
|
+
// SQLite: ~/.local/share/kilo/kilo.db
|
|
421
|
+
// Per-message rows with model, input/output/cache token columns.
|
|
422
|
+
export const kiloAdapter = {
|
|
423
|
+
platform: 'kilo',
|
|
424
|
+
defaultRoot: () => join(homedir(), '.local', 'share', 'kilo'),
|
|
425
|
+
async *messages(root) {
|
|
426
|
+
const dbPath = join(roots('KILO_DATA_DIR', root)[0], 'kilo.db')
|
|
427
|
+
const rows = await sqliteJson(dbPath, 'SELECT * FROM messages WHERE role="assistant"')
|
|
428
|
+
for (const row of rows) {
|
|
429
|
+
const input = Number(row.input_tokens || row.inputTokens || 0)
|
|
430
|
+
const output = Number(row.output_tokens || row.outputTokens || 0)
|
|
431
|
+
const cacheCreate = Number(row.cache_creation_tokens || row.cacheCreationTokens || 0)
|
|
432
|
+
const cacheRead = Number(row.cache_read_tokens || row.cacheReadTokens || 0)
|
|
433
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
434
|
+
yield { id: String(row.id || ''), sid: String(row.session_id || row.sessionId || ''), ts: row.created_at || row.timestamp || null, input, output, cacheCreate, cacheRead, file: dbPath }
|
|
435
|
+
}
|
|
436
|
+
},
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// ── 13. Hermes Agent ─────────────────────────────────────────────────────────
|
|
440
|
+
// SQLite: ~/.hermes/state.db
|
|
441
|
+
// Per-session rows: input, output, cache_read, cache_write (=cacheCreate), reasoning_tokens → output
|
|
442
|
+
export const hermesAdapter = {
|
|
443
|
+
platform: 'hermes',
|
|
444
|
+
defaultRoot: () => join(homedir(), '.hermes'),
|
|
445
|
+
async *messages(root) {
|
|
446
|
+
for (const r of roots('HERMES_HOME', root)) {
|
|
447
|
+
const dbPath = join(r, 'state.db')
|
|
448
|
+
const rows = await sqliteJson(dbPath, 'SELECT * FROM sessions')
|
|
449
|
+
for (const row of rows) {
|
|
450
|
+
const input = Number(row.input || 0)
|
|
451
|
+
const reasoning = Number(row.reasoning_tokens || 0)
|
|
452
|
+
const output = Number(row.output || 0) + reasoning
|
|
453
|
+
const cacheCreate = Number(row.cache_write || row.cache_creation || 0)
|
|
454
|
+
const cacheRead = Number(row.cache_read || 0)
|
|
455
|
+
if (input + output + cacheCreate + cacheRead === 0) continue
|
|
456
|
+
yield { id: String(row.id || ''), sid: null, ts: row.created_at || row.updated_at || null, input, output, cacheCreate, cacheRead, file: dbPath }
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
},
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// ── Registry ──────────────────────────────────────────────────────────────────
|
|
463
|
+
/** All non-Claude, non-Codex adapters keyed by platform ID. */
|
|
464
|
+
export const ADAPTERS = {
|
|
465
|
+
amp: ampAdapter,
|
|
466
|
+
kimi: kimiAdapter,
|
|
467
|
+
qwen: qwenAdapter,
|
|
468
|
+
pi: piAdapter,
|
|
469
|
+
openclaw: openclawAdapter,
|
|
470
|
+
droid: droidAdapter,
|
|
471
|
+
codebuff: codebuffAdapter,
|
|
472
|
+
gemini: geminiAdapter,
|
|
473
|
+
copilot: copilotAdapter,
|
|
474
|
+
opencode: opencodeAdapter,
|
|
475
|
+
goose: gooseAdapter,
|
|
476
|
+
kilo: kiloAdapter,
|
|
477
|
+
hermes: hermesAdapter,
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
export const ALL_PLATFORMS = Object.keys(ADAPTERS).concat(['claude', 'codex'])
|
package/cascade.mjs
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cascade.mjs — pure SigRank yield cascade (no deps, no transport).
|
|
3
|
+
* Mirrors sigrank-app/lib/ingest/bridge.ts computeCascadeMetrics() so rank_paste
|
|
4
|
+
* reproduces the canonical MO§ES Υ 18436.98 from its 4 raw token pillars.
|
|
5
|
+
* Paper-and-pencil math, open by design; proprietary threshold cuts stay server-side.
|
|
6
|
+
*
|
|
7
|
+
* Degenerate-input policy (hardened):
|
|
8
|
+
* - Any pillar that collapses a denominator (i=0, o=0, cw=0, cr=0) returns null for
|
|
9
|
+
* the affected metrics rather than Infinity/NaN.
|
|
10
|
+
* - A `warnings[]` array is attached when any metric is null so callers can surface the
|
|
11
|
+
* reason without silently corrupting downstream calculations.
|
|
12
|
+
* - The cascade is NEVER thrown away — even partial results are useful for review/storage.
|
|
13
|
+
* Callers that require a fully-formed result should check `warnings.length === 0`.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export const round = (n, d) => (Number.isFinite(n) ? Number(n.toFixed(d)) : null)
|
|
17
|
+
|
|
18
|
+
/** The four raw token pillars → the cascade. */
|
|
19
|
+
export function cascade({ input, output, cacheCreate, cacheRead }) {
|
|
20
|
+
const i = Number(input), o = Number(output), cw = Number(cacheCreate), cr = Number(cacheRead)
|
|
21
|
+
const total = i + o + cw + cr
|
|
22
|
+
const warnings = []
|
|
23
|
+
|
|
24
|
+
// SNR: undefined when both i and o are 0 (empty session)
|
|
25
|
+
const snrDenom = i + o
|
|
26
|
+
const snr = snrDenom > 0 ? o / snrDenom : null
|
|
27
|
+
if (snr === null) warnings.push('snr_undefined: input+output=0')
|
|
28
|
+
|
|
29
|
+
// velocity: undefined when i=0 (no fresh input — pure cache-only session)
|
|
30
|
+
const velocity = i > 0 ? o / i : null
|
|
31
|
+
if (velocity === null) warnings.push('velocity_undefined: input=0')
|
|
32
|
+
|
|
33
|
+
// leverage: undefined when i=0
|
|
34
|
+
const leverage = i > 0 ? cr / i : null
|
|
35
|
+
if (leverage === null) warnings.push('leverage_undefined: input=0')
|
|
36
|
+
|
|
37
|
+
// Υ = leverage × velocity = (Cr·O)/I² — null when either component is null
|
|
38
|
+
const yield_ = leverage !== null && velocity !== null ? leverage * velocity : null
|
|
39
|
+
if (yield_ === null && !warnings.some((w) => w.startsWith('yield')))
|
|
40
|
+
warnings.push('yield_undefined: requires input>0')
|
|
41
|
+
|
|
42
|
+
// dev10x = log10(Cr/I) — collapses when cw=0 (no cache commits) or i=0 or cr=0.
|
|
43
|
+
// narrate.mjs already has a nonCompounding branch for this case; we return null
|
|
44
|
+
// explicitly so the branch triggers correctly (vs -Infinity on log10(0)).
|
|
45
|
+
let dev10x = null
|
|
46
|
+
if (i > 0 && o > 0 && cw > 0 && cr > 0) {
|
|
47
|
+
dev10x = Math.log10((o / i) * (cw / o) * (cr / cw)) // = log10(Cr/I)
|
|
48
|
+
} else {
|
|
49
|
+
warnings.push('dev10x_undefined: requires all four pillars > 0')
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const result = {
|
|
53
|
+
pillars: { input: i, output: o, cacheCreate: cw, cacheRead: cr, total },
|
|
54
|
+
yield: round(yield_, 2),
|
|
55
|
+
snr: round(snr, 4),
|
|
56
|
+
leverage: round(leverage, 1),
|
|
57
|
+
velocity: round(velocity, 3),
|
|
58
|
+
dev10x: round(dev10x, 2),
|
|
59
|
+
class: classify(yield_, dev10x),
|
|
60
|
+
}
|
|
61
|
+
if (warnings.length > 0) result.warnings = warnings
|
|
62
|
+
return result
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** MVP class tiering from Υ + 10xDEV (canon assigns from cascade SNR + 10xDEV; this
|
|
66
|
+
* is the open-MVP approximation — proprietary threshold cuts stay server-side). */
|
|
67
|
+
export function classify(yieldVal, dev10x) {
|
|
68
|
+
if (yieldVal >= 1000 || dev10x >= 3) return 'TRANSMITTER'
|
|
69
|
+
if (dev10x >= 1.45) return 'ARCH+'
|
|
70
|
+
if (dev10x >= 1.35) return 'ARCH'
|
|
71
|
+
if (dev10x >= 1.2) return 'POWER'
|
|
72
|
+
if (dev10x >= 1.0) return 'BASE'
|
|
73
|
+
if (dev10x >= 0) return 'SEEKER'
|
|
74
|
+
if (dev10x >= -0.3) return 'REFINER'
|
|
75
|
+
return 'IGNITER'
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Extract the 4 pillars from pasted text: JSON object OR 4 whitespace numbers.
|
|
80
|
+
*
|
|
81
|
+
* Hardened parse policy:
|
|
82
|
+
* - JSON path: requires named keys and numeric values. Rejects strings/null.
|
|
83
|
+
* - Positional path: requires the input to be ONLY numeric tokens (whitespace/commas
|
|
84
|
+
* allowed as separators). If the text contains non-numeric words the positional
|
|
85
|
+
* extractor attaches a `_parseWarnings` flag so downstream can route it to a review
|
|
86
|
+
* channel instead of treating it as authoritative data.
|
|
87
|
+
* - Negative values are accepted (not thrown away) but flagged — could be valid in
|
|
88
|
+
* some edge-case accounting or a data error; the server is the authority on validity.
|
|
89
|
+
* - We NEVER silently corrupt: if we can't parse 4 pillars we throw. If we parse but
|
|
90
|
+
* something looks suspicious we surface it in `_parseWarnings` on the returned object.
|
|
91
|
+
*/
|
|
92
|
+
export function parsePillars(text) {
|
|
93
|
+
const t = String(text || '').trim()
|
|
94
|
+
const pw = [] // parse warnings to attach
|
|
95
|
+
|
|
96
|
+
// ── JSON path ──────────────────────────────────────────────────────────────
|
|
97
|
+
try {
|
|
98
|
+
const j = JSON.parse(t)
|
|
99
|
+
if (j && typeof j === 'object' && !Array.isArray(j)) {
|
|
100
|
+
const g = (...keys) => { for (const k of keys) if (j[k] != null) return j[k]; return null }
|
|
101
|
+
const input = g('input', 'tokens_input_fresh', 'inputTokens', 'input_tokens')
|
|
102
|
+
const output = g('output', 'tokens_output', 'outputTokens', 'output_tokens')
|
|
103
|
+
const cacheCreate = g('cacheCreate', 'tokens_cache_creation', 'cache_creation_tokens')
|
|
104
|
+
const cacheRead = g('cacheRead', 'tokens_cache_read', 'cache_read_tokens')
|
|
105
|
+
if ([input, output, cacheCreate, cacheRead].every((v) => v != null)) {
|
|
106
|
+
const pillars = {
|
|
107
|
+
input: Number(input),
|
|
108
|
+
output: Number(output),
|
|
109
|
+
cacheCreate: Number(cacheCreate),
|
|
110
|
+
cacheRead: Number(cacheRead),
|
|
111
|
+
}
|
|
112
|
+
if ([pillars.input, pillars.output, pillars.cacheCreate, pillars.cacheRead].some((v) => !Number.isFinite(v)))
|
|
113
|
+
throw new Error('Non-numeric pillar value in JSON (got string or non-finite number).')
|
|
114
|
+
if ([pillars.input, pillars.output, pillars.cacheCreate, pillars.cacheRead].some((v) => v < 0))
|
|
115
|
+
pw.push('negative_pillar: one or more pillars is negative — may be a data error')
|
|
116
|
+
if (pw.length > 0) pillars._parseWarnings = pw
|
|
117
|
+
return pillars
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
} catch (e) {
|
|
121
|
+
// JSON.parse syntax error — fall through to positional. Re-throw parse errors we raised ourselves.
|
|
122
|
+
if (e.message.startsWith('Non-numeric')) throw e
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ── Positional path ────────────────────────────────────────────────────────
|
|
126
|
+
// Guard: if the text contains alphabetic words, the numeric extraction is unreliable.
|
|
127
|
+
// We still attempt it (don't throw away the data) but flag it for review.
|
|
128
|
+
if (/[a-zA-Z]/.test(t))
|
|
129
|
+
pw.push('positional_from_mixed_text: extracted numbers from text that contains alphabetic characters — verify these are the correct 4 pillars')
|
|
130
|
+
|
|
131
|
+
const nums = (t.match(/-?\d[\d,]*\.?\d*/g) || []).map((s) => Number(s.replace(/,/g, '')))
|
|
132
|
+
if (nums.length >= 4) {
|
|
133
|
+
const [input, output, cacheCreate, cacheRead] = nums
|
|
134
|
+
if (nums.length > 4)
|
|
135
|
+
pw.push(`positional_extra_numbers: found ${nums.length} numbers, using first 4 — inspect for positional order error`)
|
|
136
|
+
const pillars = { input, output, cacheCreate, cacheRead }
|
|
137
|
+
if ([input, output, cacheCreate, cacheRead].some((v) => v < 0))
|
|
138
|
+
pw.push('negative_pillar: one or more pillars is negative — may be a data error')
|
|
139
|
+
if (pw.length > 0) pillars._parseWarnings = pw
|
|
140
|
+
return pillars
|
|
141
|
+
}
|
|
142
|
+
throw new Error('Could not parse 4 token pillars (input, output, cacheCreate, cacheRead) from the input.')
|
|
143
|
+
}
|