token-goat 2.9.29 → 2.9.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -40
- package/SECURITY.md +28 -10
- package/THIRD_PARTY_NOTICES.md +40 -3
- package/dist/native/linux-arm64/tg-hook +0 -0
- package/dist/native/linux-x64/tg-hook +0 -0
- package/dist/ort-wasm-simd-threaded.mjs +59 -0
- package/dist/token-goat-chunk-3CHUNJ2O.mjs +53 -0
- package/dist/token-goat-chunk-3Z5S4KGM.mjs +918 -0
- package/dist/token-goat-chunk-4FVIAHSI.mjs +211 -0
- package/dist/{token-goat-chunk-PGTH2U4C.mjs → token-goat-chunk-4JJS54TR.mjs} +4 -3
- package/dist/token-goat-chunk-4LCCNRTG.mjs +313 -0
- package/dist/{token-goat-chunk-L7H3U27L.mjs → token-goat-chunk-54K3EUV3.mjs} +1 -1
- package/dist/token-goat-chunk-56EIMS3E.mjs +159 -0
- package/dist/token-goat-chunk-5BCIJXMO.mjs +609 -0
- package/dist/token-goat-chunk-5KDMCMF2.mjs +362 -0
- package/dist/token-goat-chunk-5RG5SBL6.mjs +221 -0
- package/dist/{token-goat-chunk-RRNZMM3A.mjs → token-goat-chunk-5Y6ZYL6U.mjs} +21 -2
- package/dist/{token-goat-chunk-GLXRTTLE.mjs → token-goat-chunk-64FBPDTK.mjs} +12 -8
- package/dist/{token-goat-chunk-G23DWHV4.mjs → token-goat-chunk-664RTTUV.mjs} +28 -15
- package/dist/token-goat-chunk-67TQJ76B.mjs +57 -0
- package/dist/token-goat-chunk-6QMAS7AB.mjs +42 -0
- package/dist/token-goat-chunk-7BXSNBY5.mjs +253 -0
- package/dist/{token-goat-chunk-IVWI2OCK.mjs → token-goat-chunk-7F2UBPD3.mjs} +40 -11
- package/dist/{token-goat-chunk-TGOXNAHW.mjs → token-goat-chunk-7IN2MKDS.mjs} +682 -501
- package/dist/token-goat-chunk-7NADEDKJ.mjs +124 -0
- package/dist/{token-goat-chunk-KOPIR3KZ.mjs → token-goat-chunk-A3CK6N5Z.mjs} +618 -72
- package/dist/{token-goat-chunk-O6I2XU4P.mjs → token-goat-chunk-AFLYLHDH.mjs} +44 -31
- package/dist/{token-goat-chunk-TCQANXYC.mjs → token-goat-chunk-AM5V6GSX.mjs} +18 -2
- package/dist/{token-goat-chunk-E4ZOJIZK.mjs → token-goat-chunk-APGZXSPS.mjs} +39 -146
- package/dist/{token-goat-chunk-D4ETHJD6.mjs → token-goat-chunk-ARDNEC2U.mjs} +9 -7
- package/dist/token-goat-chunk-ATCPY7EQ.mjs +134 -0
- package/dist/token-goat-chunk-B2RZZCSS.mjs +354 -0
- package/dist/{token-goat-chunk-BGYOZ6DO.mjs → token-goat-chunk-B45VMIIH.mjs} +6 -7
- package/dist/token-goat-chunk-BPGCUMR7.mjs +30 -0
- package/dist/{token-goat-chunk-TDUT2CL6.mjs → token-goat-chunk-C63JQV5K.mjs} +45 -6
- package/dist/{token-goat-chunk-ZER6EAXZ.mjs → token-goat-chunk-COBQGL5S.mjs} +2661 -2440
- package/dist/token-goat-chunk-COTXZHTS.mjs +39 -0
- package/dist/{token-goat-chunk-RQGGMPCR.mjs → token-goat-chunk-CYCEYAEO.mjs} +76 -16
- package/dist/token-goat-chunk-CZ3LUXYZ.mjs +4115 -0
- package/dist/{token-goat-chunk-ASLWOQ3I.mjs → token-goat-chunk-DZX2YDPT.mjs} +2 -1
- package/dist/token-goat-chunk-E3OQX2XT.mjs +824 -0
- package/dist/token-goat-chunk-EFHMMIGP.mjs +68 -0
- package/dist/token-goat-chunk-ETBHVDB6.mjs +328 -0
- package/dist/token-goat-chunk-EVMZJ27Z.mjs +147 -0
- package/dist/token-goat-chunk-EXKGNHEG.mjs +2096 -0
- package/dist/{token-goat-chunk-RTMVLHWX.mjs → token-goat-chunk-FBNQNSJK.mjs} +2555 -5127
- package/dist/{token-goat-chunk-NMTKNYGF.mjs → token-goat-chunk-FNFBKLRD.mjs} +74 -9
- package/dist/{token-goat-chunk-RLP6J7WR.mjs → token-goat-chunk-FZ3NEW5O.mjs} +33 -19
- package/dist/token-goat-chunk-G5PJ7AGM.mjs +36 -0
- package/dist/{token-goat-chunk-DNRPXHYH.mjs → token-goat-chunk-GFBSN7BD.mjs} +29 -401
- package/dist/{token-goat-chunk-XYWPOSCN.mjs → token-goat-chunk-GOLEOXZP.mjs} +763 -211
- package/dist/token-goat-chunk-H2K3QP6H.mjs +24 -0
- package/dist/token-goat-chunk-H2PHHGM5.mjs +188 -0
- package/dist/token-goat-chunk-HAVAOB3S.mjs +87 -0
- package/dist/token-goat-chunk-HG45AF2B.mjs +2228 -0
- package/dist/{token-goat-chunk-HUYUHODZ.mjs → token-goat-chunk-HOJLCGDP.mjs} +12965 -11748
- package/dist/{token-goat-chunk-435UQTKD.mjs → token-goat-chunk-HYBFA5X3.mjs} +118 -19
- package/dist/token-goat-chunk-IVALLT3S.mjs +48 -0
- package/dist/{token-goat-chunk-UXSXQQIY.mjs → token-goat-chunk-IVKIZWAB.mjs} +3 -3
- package/dist/{token-goat-chunk-52NSDWYY.mjs → token-goat-chunk-JB6QN6VF.mjs} +6 -4
- package/dist/{token-goat-chunk-HNFZZHFW.mjs → token-goat-chunk-JDHE26RP.mjs} +10 -3
- package/dist/{token-goat-chunk-FTP4U4QA.mjs → token-goat-chunk-JDU3EXSU.mjs} +12 -11
- package/dist/token-goat-chunk-JDWCRD6M.mjs +253 -0
- package/dist/{token-goat-chunk-6AK46ZT7.mjs → token-goat-chunk-JN4XMLSD.mjs} +5 -1
- package/dist/token-goat-chunk-JTQQGXZH.mjs +733 -0
- package/dist/token-goat-chunk-KKUFXAPD.mjs +25 -0
- package/dist/token-goat-chunk-KQNZDCBD.mjs +47 -0
- package/dist/{token-goat-chunk-2WOUYWLM.mjs → token-goat-chunk-KWPNJBDH.mjs} +27 -3
- package/dist/{token-goat-chunk-DPNLUJWB.mjs → token-goat-chunk-LOBGHXPB.mjs} +30 -8
- package/dist/token-goat-chunk-MKPF4CL5.mjs +37 -0
- package/dist/token-goat-chunk-MNWKPRAL.mjs +13 -0
- package/dist/{token-goat-chunk-DNSPXQXQ.mjs → token-goat-chunk-N2NMER56.mjs} +25 -15
- package/dist/token-goat-chunk-NYKZLXMK.mjs +218 -0
- package/dist/token-goat-chunk-OJBNYTJP.mjs +3087 -0
- package/dist/token-goat-chunk-P4QMM4WK.mjs +295 -0
- package/dist/token-goat-chunk-PD5VKH46.mjs +184 -0
- package/dist/token-goat-chunk-PEPTHRFF.mjs +129 -0
- package/dist/{token-goat-chunk-NMBWLRHW.mjs → token-goat-chunk-PU7V3FD6.mjs} +8721 -11104
- package/dist/token-goat-chunk-PY533T7F.mjs +46 -0
- package/dist/token-goat-chunk-Q5VTGRH7.mjs +761 -0
- package/dist/{token-goat-chunk-67BNIKKO.mjs → token-goat-chunk-QCLDAQTS.mjs} +31 -20
- package/dist/{token-goat-chunk-SE6E4BKJ.mjs → token-goat-chunk-QSBRTGM2.mjs} +9 -2
- package/dist/{token-goat-chunk-VZQEWRVH.mjs → token-goat-chunk-RC35RBUL.mjs} +4 -4
- package/dist/{token-goat-chunk-VMFWNHGR.mjs → token-goat-chunk-RHXKCZX3.mjs} +873 -504
- package/dist/{token-goat-chunk-PYSUADX4.mjs → token-goat-chunk-RZRTDST7.mjs} +17 -7
- package/dist/{token-goat-chunk-ONXH6NZ5.mjs → token-goat-chunk-SJHWVNDQ.mjs} +15 -21
- package/dist/token-goat-chunk-SMVRN2HU.mjs +188 -0
- package/dist/{token-goat-chunk-QO3LXVVR.mjs → token-goat-chunk-SQSSNQPE.mjs} +2295 -1718
- package/dist/{token-goat-chunk-2AWQ2R5X.mjs → token-goat-chunk-T4FK7VOV.mjs} +2 -2
- package/dist/token-goat-chunk-U4N3BN2W.mjs +92 -0
- package/dist/token-goat-chunk-UA5V6P3P.mjs +122 -0
- package/dist/{token-goat-chunk-UA5BKWWQ.mjs → token-goat-chunk-UQ5UK2T7.mjs} +15 -4
- package/dist/token-goat-chunk-USILZ3T3.mjs +53 -0
- package/dist/{token-goat-chunk-MELOKFDJ.mjs → token-goat-chunk-VES7EL7O.mjs} +7 -1
- package/dist/{token-goat-chunk-7CPMDFGB.mjs → token-goat-chunk-VL3YIFHZ.mjs} +389 -14
- package/dist/{token-goat-chunk-E5TK5DNO.mjs → token-goat-chunk-VPBBTSJR.mjs} +356 -129
- package/dist/{token-goat-chunk-YOOVFPTP.mjs → token-goat-chunk-W3QCJS6G.mjs} +10 -2
- package/dist/{token-goat-chunk-LZNBQTDK.mjs → token-goat-chunk-WY3L4DVT.mjs} +11 -7
- package/dist/token-goat-chunk-XKI4SEMX.mjs +66 -0
- package/dist/token-goat-chunk-XU4P6Z24.mjs +133 -0
- package/dist/{token-goat-chunk-H5J3C56Q.mjs → token-goat-chunk-Y776AENC.mjs} +112 -26
- package/dist/token-goat-chunk-YARZFCZS.mjs +359 -0
- package/dist/{token-goat-chunk-S4VSXURL.mjs → token-goat-chunk-Z44YO3QZ.mjs} +4 -0
- package/dist/token-goat-chunk-Z57YODA2.mjs +35 -0
- package/dist/{token-goat-chunk-PQ53BSCF.mjs → token-goat-chunk-ZCDTCVIS.mjs} +48 -29
- package/dist/token-goat-chunk-ZEJVMUQH.mjs +48 -0
- package/dist/{token-goat-chunk-BGHSSJUA.mjs → token-goat-chunk-ZFOBKUWH.mjs} +1 -1
- package/dist/token-goat-chunk-ZMBL6UVV.mjs +110 -0
- package/dist/{token-goat-chunk-PKRWMHVP.mjs → token-goat-chunk-ZQSJMO2R.mjs} +338 -170
- package/dist/token-goat-hook-client.cjs +87 -23
- package/dist/token-goat-hook-client.mjs +3 -3
- package/dist/token-goat-hook.mjs +46 -30
- package/dist/token-goat.core.mjs +6 -6
- package/docs/C4_RUNTIME_ARCHITECTURE.md +1 -1
- package/docs/cli.md +80 -46
- package/docs/install.md +47 -14
- package/docs/security.md +7 -7
- package/package.json +17 -7
- package/scripts/verify-native-dist.mjs +386 -0
- package/dist/token-goat-chunk-2NNUA2LR.mjs +0 -143
- package/dist/token-goat-chunk-44BM7DDQ.mjs +0 -294
- package/dist/token-goat-chunk-C5YPNZKQ.mjs +0 -318
- package/dist/token-goat-chunk-CIQ2FTY3.mjs +0 -1990
- package/dist/token-goat-chunk-EUSDXRXU.mjs +0 -1429
- package/dist/token-goat-chunk-G7IWXCFF.mjs +0 -456
- package/dist/token-goat-chunk-JTKCQMQA.mjs +0 -418
- package/dist/token-goat-chunk-ORNFMIER.mjs +0 -295
- package/dist/token-goat-chunk-VQ6F6T7X.mjs +0 -36
- package/dist/token-goat-chunk-YAGLESS3.mjs +0 -23
- package/dist/token-goat-chunk-YKSRAWUU.mjs +0 -33
- package/dist/token-goat-chunk-Z6KBMOBI.mjs +0 -53
|
@@ -0,0 +1,918 @@
|
|
|
1
|
+
import { createRequire as __cjsRequire } from 'node:module';
|
|
2
|
+
const require = __cjsRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
getDb,
|
|
5
|
+
isReadOnlyDb
|
|
6
|
+
} from "./token-goat-chunk-VL3YIFHZ.mjs";
|
|
7
|
+
import {
|
|
8
|
+
Database
|
|
9
|
+
} from "./token-goat-chunk-XB5KKOCG.mjs";
|
|
10
|
+
import {
|
|
11
|
+
getHarnessName
|
|
12
|
+
} from "./token-goat-chunk-U4N3BN2W.mjs";
|
|
13
|
+
import {
|
|
14
|
+
registerReset
|
|
15
|
+
} from "./token-goat-chunk-YZAAZU2S.mjs";
|
|
16
|
+
import {
|
|
17
|
+
countNoun,
|
|
18
|
+
ensureDirSync,
|
|
19
|
+
extractErrorMessage
|
|
20
|
+
} from "./token-goat-chunk-GOLEOXZP.mjs";
|
|
21
|
+
import {
|
|
22
|
+
displaySafeText
|
|
23
|
+
} from "./token-goat-chunk-FNFBKLRD.mjs";
|
|
24
|
+
import {
|
|
25
|
+
VERSION,
|
|
26
|
+
dataDir,
|
|
27
|
+
dataDirForHome
|
|
28
|
+
} from "./token-goat-chunk-VES7EL7O.mjs";
|
|
29
|
+
import {
|
|
30
|
+
init_define_import_meta_env
|
|
31
|
+
} from "./token-goat-chunk-A37V4PBF.mjs";
|
|
32
|
+
|
|
33
|
+
// src/stats.ts
|
|
34
|
+
init_define_import_meta_env();
|
|
35
|
+
import * as fs from "node:fs";
|
|
36
|
+
import * as path from "node:path";
|
|
37
|
+
|
|
38
|
+
// src/token_estimate.ts
|
|
39
|
+
init_define_import_meta_env();
|
|
40
|
+
var BYTES_PER_TOKEN = {
|
|
41
|
+
text: 4,
|
|
42
|
+
dense: 1.45
|
|
43
|
+
};
|
|
44
|
+
var GUARD_MARGIN = 0.75;
|
|
45
|
+
function creditDivisor(cls = "text") {
|
|
46
|
+
return BYTES_PER_TOKEN[cls];
|
|
47
|
+
}
|
|
48
|
+
function guardDivisor(cls = "text") {
|
|
49
|
+
return BYTES_PER_TOKEN[cls] * GUARD_MARGIN;
|
|
50
|
+
}
|
|
51
|
+
var DENSE_ALPHABET = /^[A-Za-z0-9+/=_-]$/;
|
|
52
|
+
var MIN_DENSE_SAMPLE = 200;
|
|
53
|
+
var MIN_DENSE_DISTINCT = 16;
|
|
54
|
+
var CLASSIFY_SAMPLE_CHARS = 4096;
|
|
55
|
+
function classifyContent(text) {
|
|
56
|
+
const sample = text.slice(0, CLASSIFY_SAMPLE_CHARS);
|
|
57
|
+
if (sample.length < MIN_DENSE_SAMPLE) return "text";
|
|
58
|
+
let whitespace = 0;
|
|
59
|
+
let dense = 0;
|
|
60
|
+
const distinct = /* @__PURE__ */ new Set();
|
|
61
|
+
for (const ch of sample) {
|
|
62
|
+
if (ch === " " || ch === "\n" || ch === " " || ch === "\r") whitespace++;
|
|
63
|
+
else if (DENSE_ALPHABET.test(ch)) {
|
|
64
|
+
dense++;
|
|
65
|
+
distinct.add(ch);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (whitespace / sample.length >= 0.05) return "text";
|
|
69
|
+
if (distinct.size < MIN_DENSE_DISTINCT) return "text";
|
|
70
|
+
return dense / sample.length >= 0.95 ? "dense" : "text";
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// src/stats.ts
|
|
74
|
+
var HARNESS_UNRECORDED = "unrecorded (pre-2.8.1)";
|
|
75
|
+
var PRICING_VERSION_UNRECORDED = "unrecorded (pre-tg_version column)";
|
|
76
|
+
function hasMixedPricingEras(summary) {
|
|
77
|
+
return Object.keys(summary.by_pricing_version).length > 1;
|
|
78
|
+
}
|
|
79
|
+
var SOURCE_IMAGE = "image";
|
|
80
|
+
var SOURCE_HINT = "hint";
|
|
81
|
+
var SOURCE_READ = "read";
|
|
82
|
+
var SOURCE_BASH = "bash";
|
|
83
|
+
var SOURCE_WEB = "web";
|
|
84
|
+
var SOURCE_MCP = "mcp";
|
|
85
|
+
var SOURCE_SKILL = "skill";
|
|
86
|
+
var SOURCE_CONTENT = "content";
|
|
87
|
+
var SOURCE_OTHER = "other";
|
|
88
|
+
var COUNT_ONLY_KINDS = /* @__PURE__ */ new Set(["secret_redacted"]);
|
|
89
|
+
function savedTokensFromBytes(bytes, cls = "text") {
|
|
90
|
+
return Math.round(Math.max(0, bytes) / creditDivisor(cls));
|
|
91
|
+
}
|
|
92
|
+
var KIND_TO_SOURCE = {
|
|
93
|
+
image_shrink: SOURCE_IMAGE,
|
|
94
|
+
image_shrink_cache_hit: SOURCE_IMAGE,
|
|
95
|
+
image_shrink_skipped: SOURCE_IMAGE,
|
|
96
|
+
image_shrink_over_pixel_limit: SOURCE_IMAGE,
|
|
97
|
+
image_ocr: SOURCE_IMAGE,
|
|
98
|
+
webfetch_image: SOURCE_IMAGE,
|
|
99
|
+
gdrive_image: SOURCE_IMAGE,
|
|
100
|
+
session_hint: SOURCE_HINT,
|
|
101
|
+
session_hint_suppressed: SOURCE_HINT,
|
|
102
|
+
diff_hint: SOURCE_HINT,
|
|
103
|
+
evidence_cache_hit: SOURCE_HINT,
|
|
104
|
+
structured_file_hint: SOURCE_HINT,
|
|
105
|
+
predictive_prefetch_hit: SOURCE_HINT,
|
|
106
|
+
grep_dedup_hint: SOURCE_HINT,
|
|
107
|
+
glob_dedup_hint: SOURCE_HINT,
|
|
108
|
+
write_rewrite_hint: SOURCE_HINT,
|
|
109
|
+
websearch_dedup_hint: SOURCE_HINT,
|
|
110
|
+
large_file_hint_followed: SOURCE_HINT,
|
|
111
|
+
large_file_hint_ignored: SOURCE_HINT,
|
|
112
|
+
read_count_deny: SOURCE_HINT,
|
|
113
|
+
read_served_deny: SOURCE_HINT,
|
|
114
|
+
// hooks_read.ts's subagent first-read markdown deny (hints.subagent_markdown_first_read_deny, off by default). Always recorded at 0 bytes / 0 tokens: no first-read deny of this shape exists in the transcript corpus, so its abandoned/substituted/shell-read/retried rates are unknown and can only be borrowed from the re-read heading-tree census. Booking withheld bytes against borrowed rates would claim a saving this path cannot back up. The kind exists to make the intervention countable in session-audit, not to claim a win.
|
|
115
|
+
subagent_markdown_first_read_deny: SOURCE_HINT,
|
|
116
|
+
// hooks_agent_spawn.ts's scoped-spawn deny (hints.agent_scoped_spawn_deny, off by default). Recorded at 0 bytes / 0 tokens: the lane prefix a redirected spawn avoids depends on which agent the retry picks, if any, and nothing in the hook can observe that, so booking a saving here would be a guess.
|
|
117
|
+
agent_scoped_spawn_deny: SOURCE_HINT,
|
|
118
|
+
// A measurement booked beside every whole-file re-read deny in hooks_read.ts: which branch fired and whether the refused file still held what the session last read (detail `branch=… identity=identical|changed|unknown basis=snapshot|stat|none edited=0|1`). SOURCE_OTHER, not SOURCE_HINT: stats_report.ts counts SOURCE_HINT events as hints fired, and this row is a second record of a deny its sibling session_hint or read_count_deny row already counts.
|
|
119
|
+
reread_deny_identity: SOURCE_OTHER,
|
|
120
|
+
// A measurement booked by postEditHandler when an edit lands on a file whose last Read in the session handed over only part of it (detail `last_read=outline|skeleton|fold|truncated|partial tool=<name>`), from the record read_shape.ts keeps. Zero bytes: it saves nothing, it counts edits composed from a partial view.
|
|
121
|
+
edit_after_fold: SOURCE_OTHER,
|
|
122
|
+
read_replacement: SOURCE_READ,
|
|
123
|
+
section_replacement: SOURCE_READ,
|
|
124
|
+
symbol_read: SOURCE_READ,
|
|
125
|
+
section_read: SOURCE_READ,
|
|
126
|
+
stub_view: SOURCE_READ,
|
|
127
|
+
symbol_lookup: SOURCE_READ,
|
|
128
|
+
semantic_search: SOURCE_READ,
|
|
129
|
+
map_lookup: SOURCE_READ,
|
|
130
|
+
changed_lookup: SOURCE_READ,
|
|
131
|
+
outline: SOURCE_READ,
|
|
132
|
+
exports: SOURCE_READ,
|
|
133
|
+
imports: SOURCE_READ,
|
|
134
|
+
dep_docs: SOURCE_READ,
|
|
135
|
+
csv_query: SOURCE_READ,
|
|
136
|
+
csv_profile: SOURCE_READ,
|
|
137
|
+
pdf_extract: SOURCE_READ,
|
|
138
|
+
pdf_locate: SOURCE_READ,
|
|
139
|
+
pdf_outline: SOURCE_READ,
|
|
140
|
+
pdf_meta: SOURCE_READ,
|
|
141
|
+
xlsx_sheets: SOURCE_READ,
|
|
142
|
+
xlsx_head: SOURCE_READ,
|
|
143
|
+
xlsx_range: SOURCE_READ,
|
|
144
|
+
xlsx_query: SOURCE_READ,
|
|
145
|
+
xlsx_columns: SOURCE_READ,
|
|
146
|
+
pptx_outline: SOURCE_READ,
|
|
147
|
+
pptx_slide: SOURCE_READ,
|
|
148
|
+
pptx_notes: SOURCE_READ,
|
|
149
|
+
pptx_text: SOURCE_READ,
|
|
150
|
+
docx_outline: SOURCE_READ,
|
|
151
|
+
docx_tables: SOURCE_READ,
|
|
152
|
+
docx_text: SOURCE_READ,
|
|
153
|
+
transcript_outline: SOURCE_READ,
|
|
154
|
+
transcript: SOURCE_READ,
|
|
155
|
+
video_chapters: SOURCE_READ,
|
|
156
|
+
image_meta: SOURCE_READ,
|
|
157
|
+
image_text: SOURCE_READ,
|
|
158
|
+
coverage_report_gaps: SOURCE_READ,
|
|
159
|
+
json_query: SOURCE_READ,
|
|
160
|
+
json_outline: SOURCE_READ,
|
|
161
|
+
yaml_query: SOURCE_READ,
|
|
162
|
+
yaml_outline: SOURCE_READ,
|
|
163
|
+
xml_query: SOURCE_READ,
|
|
164
|
+
xml_outline: SOURCE_READ,
|
|
165
|
+
html_query: SOURCE_READ,
|
|
166
|
+
html_outline: SOURCE_READ,
|
|
167
|
+
openapi_op: SOURCE_READ,
|
|
168
|
+
openapi_outline: SOURCE_READ,
|
|
169
|
+
zip_list: SOURCE_READ,
|
|
170
|
+
zip_read: SOURCE_READ,
|
|
171
|
+
sqlite_query: SOURCE_READ,
|
|
172
|
+
sqlite_schema: SOURCE_READ,
|
|
173
|
+
sqlite_tables: SOURCE_READ,
|
|
174
|
+
conflicts: SOURCE_READ,
|
|
175
|
+
brief_view: SOURCE_READ,
|
|
176
|
+
session_outline: SOURCE_READ,
|
|
177
|
+
session_slice: SOURCE_READ,
|
|
178
|
+
gdrive_sections: SOURCE_READ,
|
|
179
|
+
pr_slice: SOURCE_READ,
|
|
180
|
+
compact_doc: SOURCE_READ,
|
|
181
|
+
note_read: SOURCE_READ,
|
|
182
|
+
note_list: SOURCE_READ,
|
|
183
|
+
// note-add is a write (like insert-section/replace, which record no stat at all -- neither has a "full source it replaces" savings concept). It still gets an event-only entry here (no bytesSaved/tokensSaved argument, same as skill_load) purely so `token-goat note-add` usage is visible in `token-goat stats --full` at all -- SOURCE_OTHER, not SOURCE_READ, since it is not a token-savings substitute for a read.
|
|
184
|
+
note_write: SOURCE_OTHER,
|
|
185
|
+
web_fetch: SOURCE_WEB,
|
|
186
|
+
injection_detected: SOURCE_WEB,
|
|
187
|
+
skill_load: SOURCE_SKILL,
|
|
188
|
+
skill_oversized_first_load: SOURCE_SKILL,
|
|
189
|
+
// Cold first load of an oversized skill where preSkillHandler inlined the compact slice in its reply instead of pointing at `skill-body --compact`. Unlike its skill_oversized_first_load sibling (event-only, 0 bytes -- the pointer deny saves nothing by itself, the follow-up command does) this one records real savings: the full body never landed, the slice did, so bytesSaved is body minus slice.
|
|
190
|
+
skill_compact_inlined: SOURCE_SKILL,
|
|
191
|
+
// Cold first load of an oversized skill with no compact marker at all, where preSkillHandler inlined a heading tree in its reply instead of letting the whole body fall through. Same shape as skill_compact_inlined: real savings, bytesSaved is body minus the rendered tree.
|
|
192
|
+
skill_heading_tree_inlined: SOURCE_SKILL,
|
|
193
|
+
secret_redacted: SOURCE_OTHER,
|
|
194
|
+
// Fail-soft diagnostic counters from hooks_edit.ts: they record that a side task threw, never a byte saving, so "other" is the right home. Listed explicitly rather than left to kindToSource()'s fallback so the registration guard can tell a deliberate placement from an unregistered kind.
|
|
195
|
+
dirty_queue_append_failed: SOURCE_OTHER,
|
|
196
|
+
worker_healthcheck_failed: SOURCE_OTHER,
|
|
197
|
+
known_root_record_failed: SOURCE_OTHER,
|
|
198
|
+
// Same fail-soft shape, from hooks_session_start.ts's reconcileNote: a thrown reconcile sweep must not block session start, but it also must not vanish silently, so the catch that swallows it records why instead of returning null with nothing recorded.
|
|
199
|
+
reconcile_note_failed: SOURCE_OTHER,
|
|
200
|
+
// Measurement of what a compaction produced (hooks_compact.ts postCompactHandler): summary size and how many manifest paths survived into it. SOURCE_OTHER and always recorded at (0, 0) -- the summary was written whether or not token-goat was watching, so there is no counterfactual in which those bytes were saved. Filing it anywhere with a savings total would credit token-goat for the whole summary, which is the accounting mistake this registry exists to prevent.
|
|
201
|
+
compact_summary: SOURCE_OTHER,
|
|
202
|
+
// One row per failed tool call the post_tool_use_failure hook sees (hooks_tool_failure.ts), with its tool_error_class.ts verdict in `detail` so failure rates by reason accumulate going forward. A measurement at (0, 0): a failure saves nothing.
|
|
203
|
+
tool_failure: SOURCE_OTHER,
|
|
204
|
+
// Envelope compaction of an oversized subagent report (hooks_agent_spawn.ts). SOURCE_CONTENT, not SOURCE_HINT: the handler's sibling session_hint entry is advisory (it only appends a recall pointer and genuinely saves nothing), whereas this kind records a real rewrite with real bytes removed, so filing it under the advisory bucket would understate the compaction and repeat the zero-savings desync this registry keeps getting bitten by.
|
|
205
|
+
agent_report_compact: SOURCE_CONTENT,
|
|
206
|
+
// Decline counterpart to agent_report_compact: the fence-collapse net-benefit gate ran and found at least one over-long fence, but declined to rewrite because net savings did not clear the notice cost. Always recorded at (0, 0) -- see the recordStat call site -- so it never contributes to any savings total; it exists purely to make gate hit-rate and near-misses visible instead of the decline being invisible.
|
|
207
|
+
agent_report_compact_declined: SOURCE_CONTENT,
|
|
208
|
+
content_compress: SOURCE_CONTENT,
|
|
209
|
+
// Verbatim-repeat collapse of a browser tool's "Tab Context:" text block (hooks_browser_image.ts postBrowserImageHandler). SOURCE_CONTENT for the same reason as agent_report_compact above: it is a real rewrite with real bytes removed, not an advisory nudge. Deliberately not SOURCE_IMAGE -- it shares a handler with image_shrink but collapses text, and folding text bytes into the image ledger is the two-units-under-one-label mistake this file's image_shrink entry was just fixed for.
|
|
210
|
+
browser_tab_dedup: SOURCE_CONTENT,
|
|
211
|
+
// Collapse of the plan echo in an approved ExitPlanMode result (hooks_exitplanmode.ts). SOURCE_CONTENT for the same reason as agent_report_compact: real bytes removed from a tool result, not an advisory nudge. The handler shipped for releases emitting this rewrite and recording nothing at all, so the mechanism was invisible in `stats` and its net benefit could not be checked against the gate that admits it.
|
|
212
|
+
plan_echo_collapse: SOURCE_CONTENT,
|
|
213
|
+
// Lossless re-layout of Grep content-mode output (hooks_grep.ts foldGrepContentHandler). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as agent_report_compact above: its sibling grep_dedup_hint is advisory and saves nothing directly, whereas this is a real rewrite with real bytes removed. Filing it under the advisory bucket would silently add non-hint savings to hint_stats.ts's savedBytes (which reads by_source[SOURCE_HINT] wholesale) and overstate the hint ledger's net benefit.
|
|
214
|
+
"grep:fold": SOURCE_CONTENT,
|
|
215
|
+
// Withholding of already-served stretches from a completed Read (hooks_read_post.ts elideAlreadyServedLines). SOURCE_CONTENT, not SOURCE_HINT, for the same reason as grep:fold above: its siblings read_count_deny and read_served_deny are decisions about whether a read happens at all, whereas this is a rewrite of a result that did happen, with real bytes removed from it. Filing it under the advisory bucket would add non-hint savings to hint_stats.ts's savedBytes, which reads by_source[SOURCE_HINT] wholesale.
|
|
216
|
+
"read:served_elide": SOURCE_CONTENT,
|
|
217
|
+
// Same bucket and same reasoning as read:served_elide directly above: a rewrite of a Read that did happen, with real bytes removed, not an advisory about whether to read at all.
|
|
218
|
+
"read:body_fold": SOURCE_CONTENT,
|
|
219
|
+
// Same bucket and same reasoning as read:body_fold directly above: a coarser sibling rewrite of a large untargeted markdown Read (hooks_read_post.ts foldMarkdownOutline) that replaces the body with a heading tree plus preamble, with real bytes removed, not an advisory about whether to read at all.
|
|
220
|
+
"read:markdown_outline": SOURCE_CONTENT,
|
|
221
|
+
// Same bucket and same reasoning as read:markdown_outline directly above, on source instead of prose: the structural-skeleton replacement of a large untargeted source Read (hooks_read_post.ts foldSourceSkeleton), with real bytes removed, not an advisory about whether to read at all.
|
|
222
|
+
"read:source_skeleton": SOURCE_CONTENT,
|
|
223
|
+
content_retrieve: SOURCE_CONTENT,
|
|
224
|
+
handoff_create: SOURCE_CONTENT,
|
|
225
|
+
handoff_resolve: SOURCE_CONTENT
|
|
226
|
+
};
|
|
227
|
+
var KIND_PREFIX_TO_SOURCE = [
|
|
228
|
+
["bash_compress:", SOURCE_BASH],
|
|
229
|
+
["webfetch:", SOURCE_WEB],
|
|
230
|
+
["gdrive:", SOURCE_WEB],
|
|
231
|
+
["mcp:", SOURCE_MCP],
|
|
232
|
+
["skill_body:", SOURCE_SKILL],
|
|
233
|
+
["skill_compact:", SOURCE_SKILL],
|
|
234
|
+
["bashoutput:", SOURCE_BASH],
|
|
235
|
+
["taskoutput:", SOURCE_CONTENT],
|
|
236
|
+
// Hook wall-clock timing (relay.ts's relayInProcess), one row per invocation, always 0 bytes/0 tokens: it measures token-goat's own overhead, not a saving. See hook_latency.ts's hookLatencyBreakdown() for the dedicated read path and pruneHookStats() below for its own (shorter) retention.
|
|
237
|
+
["hook:", SOURCE_OTHER],
|
|
238
|
+
// See recordStaleServed (read_commands.ts).
|
|
239
|
+
["stale_served:", SOURCE_OTHER],
|
|
240
|
+
// See recordAnswer (answer_router.ts).
|
|
241
|
+
["answer:", SOURCE_OTHER]
|
|
242
|
+
];
|
|
243
|
+
var ANSWER_ROUTES = ["symbol", "callers", "impact", "test-for", "exports", "imports", "deps", "brief", "refused"];
|
|
244
|
+
var COMMAND_KINDS = {
|
|
245
|
+
symbol: /* @__PURE__ */ new Set(["symbol_lookup"]),
|
|
246
|
+
// runAnswer's routes and refusals; savings stay on the delegate's own kind.
|
|
247
|
+
answer: new Set(ANSWER_ROUTES.map((r) => `answer:${r}`)),
|
|
248
|
+
read: /* @__PURE__ */ new Set(["read_replacement"]),
|
|
249
|
+
section: /* @__PURE__ */ new Set(["section_replacement", "section_read"]),
|
|
250
|
+
semantic: /* @__PURE__ */ new Set(["semantic_search"]),
|
|
251
|
+
outline: /* @__PURE__ */ new Set(["outline"]),
|
|
252
|
+
exports: /* @__PURE__ */ new Set(["exports"]),
|
|
253
|
+
imports: /* @__PURE__ */ new Set(["imports"]),
|
|
254
|
+
skeleton: /* @__PURE__ */ new Set(["stub_view"]),
|
|
255
|
+
refs: /* @__PURE__ */ new Set(["symbol_read"]),
|
|
256
|
+
map: /* @__PURE__ */ new Set(["map_lookup"]),
|
|
257
|
+
changed: /* @__PURE__ */ new Set(["changed_lookup"]),
|
|
258
|
+
"dep-docs": /* @__PURE__ */ new Set(["dep_docs"]),
|
|
259
|
+
"csv-query": /* @__PURE__ */ new Set(["csv_query"]),
|
|
260
|
+
"csv-profile": /* @__PURE__ */ new Set(["csv_profile"]),
|
|
261
|
+
"pdf-extract": /* @__PURE__ */ new Set(["pdf_extract"]),
|
|
262
|
+
"pdf-locate": /* @__PURE__ */ new Set(["pdf_locate"]),
|
|
263
|
+
"pdf-outline": /* @__PURE__ */ new Set(["pdf_outline"]),
|
|
264
|
+
"pdf-meta": /* @__PURE__ */ new Set(["pdf_meta"]),
|
|
265
|
+
"xlsx-sheets": /* @__PURE__ */ new Set(["xlsx_sheets"]),
|
|
266
|
+
"xlsx-head": /* @__PURE__ */ new Set(["xlsx_head"]),
|
|
267
|
+
"xlsx-range": /* @__PURE__ */ new Set(["xlsx_range"]),
|
|
268
|
+
"xlsx-query": /* @__PURE__ */ new Set(["xlsx_query"]),
|
|
269
|
+
"xlsx-columns": /* @__PURE__ */ new Set(["xlsx_columns"]),
|
|
270
|
+
"pptx-outline": /* @__PURE__ */ new Set(["pptx_outline"]),
|
|
271
|
+
"pptx-slide": /* @__PURE__ */ new Set(["pptx_slide"]),
|
|
272
|
+
"pptx-notes": /* @__PURE__ */ new Set(["pptx_notes"]),
|
|
273
|
+
"pptx-text": /* @__PURE__ */ new Set(["pptx_text"]),
|
|
274
|
+
"docx-outline": /* @__PURE__ */ new Set(["docx_outline"]),
|
|
275
|
+
"docx-tables": /* @__PURE__ */ new Set(["docx_tables"]),
|
|
276
|
+
"docx-text": /* @__PURE__ */ new Set(["docx_text"]),
|
|
277
|
+
"transcript-outline": /* @__PURE__ */ new Set(["transcript_outline"]),
|
|
278
|
+
transcript: /* @__PURE__ */ new Set(["transcript"]),
|
|
279
|
+
"video-chapters": /* @__PURE__ */ new Set(["video_chapters"]),
|
|
280
|
+
"image-meta": /* @__PURE__ */ new Set(["image_meta"]),
|
|
281
|
+
"image-text": /* @__PURE__ */ new Set(["image_text"]),
|
|
282
|
+
"coverage-report-gaps": /* @__PURE__ */ new Set(["coverage_report_gaps"]),
|
|
283
|
+
"json-query": /* @__PURE__ */ new Set(["json_query"]),
|
|
284
|
+
"json-outline": /* @__PURE__ */ new Set(["json_outline"]),
|
|
285
|
+
"yaml-query": /* @__PURE__ */ new Set(["yaml_query"]),
|
|
286
|
+
"yaml-outline": /* @__PURE__ */ new Set(["yaml_outline"]),
|
|
287
|
+
"xml-query": /* @__PURE__ */ new Set(["xml_query"]),
|
|
288
|
+
"xml-outline": /* @__PURE__ */ new Set(["xml_outline"]),
|
|
289
|
+
"html-query": /* @__PURE__ */ new Set(["html_query"]),
|
|
290
|
+
"html-outline": /* @__PURE__ */ new Set(["html_outline"]),
|
|
291
|
+
"openapi-op": /* @__PURE__ */ new Set(["openapi_op"]),
|
|
292
|
+
"openapi-outline": /* @__PURE__ */ new Set(["openapi_outline"]),
|
|
293
|
+
"zip-list": /* @__PURE__ */ new Set(["zip_list"]),
|
|
294
|
+
"zip-read": /* @__PURE__ */ new Set(["zip_read"]),
|
|
295
|
+
"sqlite-query": /* @__PURE__ */ new Set(["sqlite_query"]),
|
|
296
|
+
"sqlite-schema": /* @__PURE__ */ new Set(["sqlite_schema"]),
|
|
297
|
+
"sqlite-tables": /* @__PURE__ */ new Set(["sqlite_tables"]),
|
|
298
|
+
conflicts: /* @__PURE__ */ new Set(["conflicts"]),
|
|
299
|
+
brief: /* @__PURE__ */ new Set(["brief_view"]),
|
|
300
|
+
"session-outline": /* @__PURE__ */ new Set(["session_outline"]),
|
|
301
|
+
"session-slice": /* @__PURE__ */ new Set(["session_slice"]),
|
|
302
|
+
"gdrive-sections": /* @__PURE__ */ new Set(["gdrive_sections"]),
|
|
303
|
+
"pr-slice": /* @__PURE__ */ new Set(["pr_slice"]),
|
|
304
|
+
"compact-doc": /* @__PURE__ */ new Set(["compact_doc"]),
|
|
305
|
+
"note-add": /* @__PURE__ */ new Set(["note_write"]),
|
|
306
|
+
"note-get": /* @__PURE__ */ new Set(["note_read"]),
|
|
307
|
+
"note-list": /* @__PURE__ */ new Set(["note_list"]),
|
|
308
|
+
"compress-text": /* @__PURE__ */ new Set(["content_compress"]),
|
|
309
|
+
retrieve: /* @__PURE__ */ new Set(["content_retrieve"]),
|
|
310
|
+
"handoff-create": /* @__PURE__ */ new Set(["handoff_create"]),
|
|
311
|
+
"handoff-resolve": /* @__PURE__ */ new Set(["handoff_resolve"]),
|
|
312
|
+
npm: /* @__PURE__ */ new Set([
|
|
313
|
+
"bash_compress:npm_install",
|
|
314
|
+
"bash_compress:npm_ci",
|
|
315
|
+
"bash_compress:npm_audit",
|
|
316
|
+
"bash_compress:npm_ls",
|
|
317
|
+
"bash_compress:npm_outdated",
|
|
318
|
+
"bash_compress:npx"
|
|
319
|
+
])
|
|
320
|
+
};
|
|
321
|
+
var OVERHEAD_SUFFIX = "_overhead";
|
|
322
|
+
function kindToSource(kind) {
|
|
323
|
+
const src = KIND_TO_SOURCE[kind];
|
|
324
|
+
if (src !== void 0) return src;
|
|
325
|
+
if (kind.endsWith(OVERHEAD_SUFFIX)) {
|
|
326
|
+
const base = kind.slice(0, -OVERHEAD_SUFFIX.length);
|
|
327
|
+
const baseSrc = KIND_TO_SOURCE[base];
|
|
328
|
+
if (baseSrc !== void 0) return baseSrc;
|
|
329
|
+
}
|
|
330
|
+
for (const [prefix, prefixSrc] of KIND_PREFIX_TO_SOURCE) {
|
|
331
|
+
if (kind.startsWith(prefix)) return prefixSrc;
|
|
332
|
+
}
|
|
333
|
+
return SOURCE_OTHER;
|
|
334
|
+
}
|
|
335
|
+
function toLocalDateKey(d) {
|
|
336
|
+
const year = d.getFullYear();
|
|
337
|
+
const month = String(d.getMonth() + 1).padStart(2, "0");
|
|
338
|
+
const day = String(d.getDate()).padStart(2, "0");
|
|
339
|
+
return `${year}-${month}-${day}`;
|
|
340
|
+
}
|
|
341
|
+
function formatLocalTimestamp(d) {
|
|
342
|
+
const hours = String(d.getHours()).padStart(2, "0");
|
|
343
|
+
const minutes = String(d.getMinutes()).padStart(2, "0");
|
|
344
|
+
const seconds = String(d.getSeconds()).padStart(2, "0");
|
|
345
|
+
return `${toLocalDateKey(d)}T${hours}:${minutes}:${seconds}`;
|
|
346
|
+
}
|
|
347
|
+
function zeroBucket() {
|
|
348
|
+
return { events: 0, bytes_saved: 0, tokens_saved: 0 };
|
|
349
|
+
}
|
|
350
|
+
function incBucket(bucket, bytesSaved, tokensSaved) {
|
|
351
|
+
bucket.events += 1;
|
|
352
|
+
bucket.bytes_saved += bytesSaved;
|
|
353
|
+
bucket.tokens_saved += tokensSaved;
|
|
354
|
+
}
|
|
355
|
+
var GLOBAL_SCHEMA_SQL = `
|
|
356
|
+
CREATE TABLE IF NOT EXISTS stats (
|
|
357
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
358
|
+
ts INTEGER NOT NULL,
|
|
359
|
+
kind TEXT NOT NULL,
|
|
360
|
+
tokens_saved INTEGER NOT NULL DEFAULT 0,
|
|
361
|
+
bytes_saved INTEGER NOT NULL DEFAULT 0,
|
|
362
|
+
detail TEXT,
|
|
363
|
+
harness TEXT,
|
|
364
|
+
traceparent TEXT,
|
|
365
|
+
tg_version TEXT,
|
|
366
|
+
duration_ms INTEGER
|
|
367
|
+
);
|
|
368
|
+
CREATE INDEX IF NOT EXISTS idx_stats_ts ON stats(ts);
|
|
369
|
+
CREATE INDEX IF NOT EXISTS idx_stats_kind ON stats(kind);
|
|
370
|
+
-- Day-granularity rollup of stats rows older than STATS_RETENTION_DAYS -- see
|
|
371
|
+
-- rollupAndPruneStats's doc comment for why raw rows are pruned instead of kept forever, and
|
|
372
|
+
-- summarize()'s rollup merge for how a --window-days report spanning pruned days still gets a
|
|
373
|
+
-- correct total. Deliberately day+kind+harness+tg_version, not day+kind alone: those are exactly
|
|
374
|
+
-- the dimensions summarize() breaks totals out by, and collapsing any of them here would make an
|
|
375
|
+
-- old --window-days report under-report a breakdown a recent one still shows in full.
|
|
376
|
+
CREATE TABLE IF NOT EXISTS stats_daily_rollup (
|
|
377
|
+
day TEXT NOT NULL,
|
|
378
|
+
kind TEXT NOT NULL,
|
|
379
|
+
harness TEXT NOT NULL DEFAULT '',
|
|
380
|
+
tg_version TEXT NOT NULL DEFAULT '',
|
|
381
|
+
events INTEGER NOT NULL DEFAULT 0,
|
|
382
|
+
bytes_saved INTEGER NOT NULL DEFAULT 0,
|
|
383
|
+
tokens_saved INTEGER NOT NULL DEFAULT 0,
|
|
384
|
+
PRIMARY KEY (day, kind, harness, tg_version)
|
|
385
|
+
);
|
|
386
|
+
CREATE INDEX IF NOT EXISTS idx_stats_daily_rollup_day ON stats_daily_rollup(day);
|
|
387
|
+
-- Single-row (id=1, same pattern as embedding_provenance in db.ts) throttle so the rollup+prune
|
|
388
|
+
-- pass -- an aggregate scan over the whole stats table -- runs at most once per
|
|
389
|
+
-- STATS_ROLLUP_INTERVAL_MS regardless of how often recordStat() itself is called (many times a
|
|
390
|
+
-- minute during active use).
|
|
391
|
+
CREATE TABLE IF NOT EXISTS stats_maintenance (
|
|
392
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
393
|
+
last_rollup_ts INTEGER NOT NULL DEFAULT 0
|
|
394
|
+
);
|
|
395
|
+
CREATE TABLE IF NOT EXISTS unmapped_tools (
|
|
396
|
+
harness TEXT NOT NULL,
|
|
397
|
+
tool_name TEXT NOT NULL,
|
|
398
|
+
event_name TEXT NOT NULL,
|
|
399
|
+
near_miss TEXT,
|
|
400
|
+
first_seen INTEGER NOT NULL,
|
|
401
|
+
last_seen INTEGER NOT NULL,
|
|
402
|
+
hits INTEGER NOT NULL DEFAULT 0,
|
|
403
|
+
PRIMARY KEY (harness, tool_name, event_name)
|
|
404
|
+
);
|
|
405
|
+
CREATE TABLE IF NOT EXISTS stats_flags (
|
|
406
|
+
name TEXT PRIMARY KEY,
|
|
407
|
+
set_ts INTEGER NOT NULL
|
|
408
|
+
);
|
|
409
|
+
-- One row per semantic query's dense outcome, for semantic --distances. Deliberately no query text: only distances and a project hash, so the ledger cannot leak what was asked. Pruned with the stats table's retention (pruneSemanticQueries).
|
|
410
|
+
CREATE TABLE IF NOT EXISTS semantic_queries (
|
|
411
|
+
ts INTEGER NOT NULL,
|
|
412
|
+
project_hash TEXT,
|
|
413
|
+
closest_distance REAL,
|
|
414
|
+
floor_rejected_min REAL,
|
|
415
|
+
weak INTEGER NOT NULL DEFAULT 0
|
|
416
|
+
);
|
|
417
|
+
CREATE INDEX IF NOT EXISTS idx_semantic_queries_ts ON semantic_queries(ts);
|
|
418
|
+
`;
|
|
419
|
+
var _globalSchemaApplied = /* @__PURE__ */ new Set();
|
|
420
|
+
registerReset(() => _globalSchemaApplied.clear(), { perRequest: true });
|
|
421
|
+
function migrateGlobalSchema(db) {
|
|
422
|
+
try {
|
|
423
|
+
db.exec("ALTER TABLE stats ADD COLUMN harness TEXT");
|
|
424
|
+
} catch (err) {
|
|
425
|
+
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
426
|
+
}
|
|
427
|
+
try {
|
|
428
|
+
db.exec("ALTER TABLE stats ADD COLUMN traceparent TEXT");
|
|
429
|
+
} catch (err) {
|
|
430
|
+
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
431
|
+
}
|
|
432
|
+
try {
|
|
433
|
+
db.exec("ALTER TABLE stats ADD COLUMN tg_version TEXT");
|
|
434
|
+
} catch (err) {
|
|
435
|
+
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
436
|
+
}
|
|
437
|
+
try {
|
|
438
|
+
db.exec("ALTER TABLE stats ADD COLUMN duration_ms INTEGER");
|
|
439
|
+
} catch (err) {
|
|
440
|
+
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
var TEST_ISOLATION_LEAK_WINDOW_START_TS = 1780406895;
|
|
444
|
+
var TEST_ISOLATION_LEAK_WINDOW_END_TS = 1782119331;
|
|
445
|
+
function pruneTestIsolationLeakRows(db) {
|
|
446
|
+
try {
|
|
447
|
+
db.prepare(
|
|
448
|
+
`DELETE FROM stats WHERE tg_version IS NULL AND ts BETWEEN ? AND ? AND (detail LIKE '%pytest-of-%' OR detail LIKE '%/fake/%')`
|
|
449
|
+
).run(TEST_ISOLATION_LEAK_WINDOW_START_TS, TEST_ISOLATION_LEAK_WINDOW_END_TS);
|
|
450
|
+
} catch {
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
function dropRetiredPythonTables(db) {
|
|
454
|
+
try {
|
|
455
|
+
db.prepare("DROP TABLE IF EXISTS grep_patterns").run();
|
|
456
|
+
db.prepare("DROP TABLE IF EXISTS miss_patterns").run();
|
|
457
|
+
db.prepare("DROP TABLE IF EXISTS wal_bloat").run();
|
|
458
|
+
} catch {
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
var _harnessColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
462
|
+
function statsHasHarnessColumn(db) {
|
|
463
|
+
const cached = _harnessColumnByDb.get(db);
|
|
464
|
+
if (cached !== void 0) return cached;
|
|
465
|
+
let present;
|
|
466
|
+
try {
|
|
467
|
+
present = db.prepare("PRAGMA table_info(stats)").all().some(
|
|
468
|
+
(c) => c.name === "harness"
|
|
469
|
+
);
|
|
470
|
+
} catch {
|
|
471
|
+
present = false;
|
|
472
|
+
}
|
|
473
|
+
_harnessColumnByDb.set(db, present);
|
|
474
|
+
return present;
|
|
475
|
+
}
|
|
476
|
+
var _traceparentColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
477
|
+
function statsHasTraceparentColumn(db) {
|
|
478
|
+
const cached = _traceparentColumnByDb.get(db);
|
|
479
|
+
if (cached !== void 0) return cached;
|
|
480
|
+
let present;
|
|
481
|
+
try {
|
|
482
|
+
present = db.prepare("PRAGMA table_info(stats)").all().some(
|
|
483
|
+
(c) => c.name === "traceparent"
|
|
484
|
+
);
|
|
485
|
+
} catch {
|
|
486
|
+
present = false;
|
|
487
|
+
}
|
|
488
|
+
_traceparentColumnByDb.set(db, present);
|
|
489
|
+
return present;
|
|
490
|
+
}
|
|
491
|
+
var _versionColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
492
|
+
function statsHasVersionColumn(db) {
|
|
493
|
+
const cached = _versionColumnByDb.get(db);
|
|
494
|
+
if (cached !== void 0) return cached;
|
|
495
|
+
let present;
|
|
496
|
+
try {
|
|
497
|
+
present = db.prepare("PRAGMA table_info(stats)").all().some(
|
|
498
|
+
(c) => c.name === "tg_version"
|
|
499
|
+
);
|
|
500
|
+
} catch {
|
|
501
|
+
present = false;
|
|
502
|
+
}
|
|
503
|
+
_versionColumnByDb.set(db, present);
|
|
504
|
+
return present;
|
|
505
|
+
}
|
|
506
|
+
var _durationColumnByDb = /* @__PURE__ */ new WeakMap();
|
|
507
|
+
function statsHasDurationColumn(db) {
|
|
508
|
+
const cached = _durationColumnByDb.get(db);
|
|
509
|
+
if (cached !== void 0) return cached;
|
|
510
|
+
let present;
|
|
511
|
+
try {
|
|
512
|
+
present = db.prepare("PRAGMA table_info(stats)").all().some(
|
|
513
|
+
(c) => c.name === "duration_ms"
|
|
514
|
+
);
|
|
515
|
+
} catch {
|
|
516
|
+
present = false;
|
|
517
|
+
}
|
|
518
|
+
_durationColumnByDb.set(db, present);
|
|
519
|
+
return present;
|
|
520
|
+
}
|
|
521
|
+
function getGlobalDb(homeDir) {
|
|
522
|
+
const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
|
|
523
|
+
const dbPath = path.join(basePath, "global.db");
|
|
524
|
+
const db = getDb(dbPath);
|
|
525
|
+
if (!_globalSchemaApplied.has(dbPath) && !isReadOnlyDb(dbPath)) {
|
|
526
|
+
db.exec(GLOBAL_SCHEMA_SQL);
|
|
527
|
+
migrateGlobalSchema(db);
|
|
528
|
+
_globalSchemaApplied.add(dbPath);
|
|
529
|
+
}
|
|
530
|
+
return db;
|
|
531
|
+
}
|
|
532
|
+
var STATS_RETENTION_DAYS = 180;
|
|
533
|
+
var UNMAPPED_TOOL_RETENTION_DAYS = 30;
|
|
534
|
+
var STATS_ROLLUP_INTERVAL_MS = 6 * 60 * 60 * 1e3;
|
|
535
|
+
function rollupAndPruneStats(db, retentionDays = STATS_RETENTION_DAYS) {
|
|
536
|
+
try {
|
|
537
|
+
const cutoffTs = Math.floor(Date.now() / 1e3) - retentionDays * 86400;
|
|
538
|
+
const run = db.transaction((cutoff) => {
|
|
539
|
+
db.prepare(
|
|
540
|
+
`INSERT INTO stats_daily_rollup (day, kind, harness, tg_version, events, bytes_saved, tokens_saved)
|
|
541
|
+
SELECT strftime('%Y-%m-%d', ts, 'unixepoch', 'localtime') AS day,
|
|
542
|
+
kind,
|
|
543
|
+
COALESCE(harness, '') AS harness,
|
|
544
|
+
COALESCE(tg_version, '') AS tg_version,
|
|
545
|
+
COUNT(*), SUM(bytes_saved), SUM(tokens_saved)
|
|
546
|
+
FROM stats
|
|
547
|
+
WHERE ts < ?
|
|
548
|
+
GROUP BY day, kind, harness, tg_version
|
|
549
|
+
ON CONFLICT(day, kind, harness, tg_version) DO UPDATE SET
|
|
550
|
+
events = events + excluded.events,
|
|
551
|
+
bytes_saved = bytes_saved + excluded.bytes_saved,
|
|
552
|
+
tokens_saved = tokens_saved + excluded.tokens_saved`
|
|
553
|
+
).run(cutoff);
|
|
554
|
+
db.prepare(`DELETE FROM stats WHERE ts < ?`).run(cutoff);
|
|
555
|
+
try {
|
|
556
|
+
const unmappedCutoff = Math.floor(Date.now() / 1e3) - UNMAPPED_TOOL_RETENTION_DAYS * 86400;
|
|
557
|
+
db.prepare(`DELETE FROM unmapped_tools WHERE last_seen < ?`).run(unmappedCutoff);
|
|
558
|
+
} catch {
|
|
559
|
+
}
|
|
560
|
+
});
|
|
561
|
+
run(cutoffTs);
|
|
562
|
+
} catch {
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
function pruneHintEmissions(db, retentionDays = STATS_RETENTION_DAYS) {
|
|
566
|
+
try {
|
|
567
|
+
const cutoffMs = Date.now() - retentionDays * 86400 * 1e3;
|
|
568
|
+
db.prepare(`DELETE FROM hint_emissions WHERE emitted_at < ?`).run(cutoffMs);
|
|
569
|
+
} catch {
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
function pruneSemanticQueries(db, retentionDays = STATS_RETENTION_DAYS) {
|
|
573
|
+
try {
|
|
574
|
+
db.prepare(`DELETE FROM semantic_queries WHERE ts < ?`).run(Math.floor(Date.now() / 1e3) - retentionDays * 86400);
|
|
575
|
+
} catch {
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
var HOOK_STATS_RETENTION_DAYS = 7;
|
|
579
|
+
var HOOK_PROBE_ENV = "TOKEN_GOAT_HOOK_PROBE";
|
|
580
|
+
function pruneHookStats(db, retentionDays = HOOK_STATS_RETENTION_DAYS) {
|
|
581
|
+
try {
|
|
582
|
+
const cutoffTs = Math.floor(Date.now() / 1e3) - retentionDays * 86400;
|
|
583
|
+
db.prepare(`DELETE FROM stats WHERE kind LIKE 'hook:%' AND ts < ?`).run(cutoffTs);
|
|
584
|
+
} catch {
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
function maybeRunStatsMaintenance(db) {
|
|
588
|
+
try {
|
|
589
|
+
const now = Date.now();
|
|
590
|
+
const row = db.prepare(`SELECT last_rollup_ts FROM stats_maintenance WHERE id = 1`).get();
|
|
591
|
+
const last = row?.last_rollup_ts ?? 0;
|
|
592
|
+
if (now - last < STATS_ROLLUP_INTERVAL_MS) return;
|
|
593
|
+
db.prepare(
|
|
594
|
+
`INSERT INTO stats_maintenance (id, last_rollup_ts) VALUES (1, ?)
|
|
595
|
+
ON CONFLICT(id) DO UPDATE SET last_rollup_ts = excluded.last_rollup_ts`
|
|
596
|
+
).run(now);
|
|
597
|
+
pruneHookStats(db);
|
|
598
|
+
rollupAndPruneStats(db);
|
|
599
|
+
pruneHintEmissions(db);
|
|
600
|
+
pruneSemanticQueries(db);
|
|
601
|
+
pruneTestIsolationLeakRows(db);
|
|
602
|
+
dropRetiredPythonTables(db);
|
|
603
|
+
} catch {
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
function noStatsMessage(windowDays, homeDir) {
|
|
607
|
+
if (windowDays <= 0) return "No stats recorded yet.";
|
|
608
|
+
const db = getGlobalDb(homeDir);
|
|
609
|
+
let total = db.prepare("SELECT COUNT(*) as c FROM stats").get().c;
|
|
610
|
+
try {
|
|
611
|
+
const rollupTable = db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'stats_daily_rollup'`).get();
|
|
612
|
+
if (rollupTable !== void 0) {
|
|
613
|
+
const rolledUp = db.prepare("SELECT COALESCE(SUM(events), 0) as c FROM stats_daily_rollup").get().c;
|
|
614
|
+
total += rolledUp;
|
|
615
|
+
}
|
|
616
|
+
} catch {
|
|
617
|
+
}
|
|
618
|
+
if (total === 0) return "No stats recorded yet.";
|
|
619
|
+
return `No stats in the last ${countNoun(windowDays, "day")} (${total} recorded outside this window; use --window-days 0 for all time).`;
|
|
620
|
+
}
|
|
621
|
+
var STATS_WRITE_FAILURE_LOG_MIN_INTERVAL_MS = 60 * 1e3;
|
|
622
|
+
function statsWriteFailureMarkerPath(dir) {
|
|
623
|
+
return path.join(dir, "stats-write-failed.marker");
|
|
624
|
+
}
|
|
625
|
+
function statsWriteFailureLogPath(dir) {
|
|
626
|
+
return path.join(dir, "stats-write-failed.log");
|
|
627
|
+
}
|
|
628
|
+
function recordStatWriteFailure(kind, err, dir = dataDir()) {
|
|
629
|
+
try {
|
|
630
|
+
const markerPath = statsWriteFailureMarkerPath(dir);
|
|
631
|
+
try {
|
|
632
|
+
if (Date.now() - fs.statSync(markerPath).mtimeMs < STATS_WRITE_FAILURE_LOG_MIN_INTERVAL_MS) return;
|
|
633
|
+
} catch {
|
|
634
|
+
}
|
|
635
|
+
ensureDirSync(dir);
|
|
636
|
+
fs.writeFileSync(markerPath, "");
|
|
637
|
+
const line = `${(/* @__PURE__ */ new Date()).toISOString()} recordStat write failed for kind=${kind}: ${extractErrorMessage(err)}`;
|
|
638
|
+
fs.appendFileSync(statsWriteFailureLogPath(dir), displaySafeText(line.replace(/[\n\r]+$/, "")) + "\n");
|
|
639
|
+
} catch {
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
var STATS_WRITE_BUSY_TIMEOUT_MS = 200;
|
|
643
|
+
function withTelemetryWriteDb(write) {
|
|
644
|
+
getGlobalDb();
|
|
645
|
+
const dbPath = path.join(dataDir(), "global.db");
|
|
646
|
+
if (isReadOnlyDb(dbPath)) return;
|
|
647
|
+
const db = new Database(dbPath, { timeout: STATS_WRITE_BUSY_TIMEOUT_MS });
|
|
648
|
+
try {
|
|
649
|
+
if (String(db.pragma("journal_mode", { simple: true })).toLowerCase() === "wal") db.pragma("synchronous = NORMAL");
|
|
650
|
+
write(db);
|
|
651
|
+
} finally {
|
|
652
|
+
try {
|
|
653
|
+
db.close();
|
|
654
|
+
} catch {
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
function recordStat(kind, bytesSaved = 0, tokensSaved = 0, _testDb, detail, traceparent, durationMs) {
|
|
659
|
+
const write = (db) => {
|
|
660
|
+
const ts = Math.floor(Date.now() / 1e3);
|
|
661
|
+
const tp = traceparent ?? process.env["TRACEPARENT"] ?? process.env["traceparent"] ?? null;
|
|
662
|
+
const cols = ["ts", "kind", "bytes_saved", "tokens_saved", "detail"];
|
|
663
|
+
const vals = [ts, kind, bytesSaved, tokensSaved, detail ?? null];
|
|
664
|
+
if (statsHasHarnessColumn(db)) {
|
|
665
|
+
cols.push("harness");
|
|
666
|
+
vals.push(getHarnessName());
|
|
667
|
+
}
|
|
668
|
+
if (statsHasTraceparentColumn(db)) {
|
|
669
|
+
cols.push("traceparent");
|
|
670
|
+
vals.push(tp);
|
|
671
|
+
}
|
|
672
|
+
if (statsHasVersionColumn(db)) {
|
|
673
|
+
cols.push("tg_version");
|
|
674
|
+
vals.push(VERSION);
|
|
675
|
+
}
|
|
676
|
+
if (durationMs !== void 0 && statsHasDurationColumn(db)) {
|
|
677
|
+
cols.push("duration_ms");
|
|
678
|
+
vals.push(Math.round(durationMs));
|
|
679
|
+
}
|
|
680
|
+
db.prepare(
|
|
681
|
+
`INSERT INTO stats (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`
|
|
682
|
+
).run(...vals);
|
|
683
|
+
maybeRunStatsMaintenance(db);
|
|
684
|
+
};
|
|
685
|
+
try {
|
|
686
|
+
if (_testDb) write(_testDb);
|
|
687
|
+
else withTelemetryWriteDb(write);
|
|
688
|
+
} catch (e) {
|
|
689
|
+
recordStatWriteFailure(kind, e);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
var MAX_TOOL_NAME_CHARS = 200;
|
|
693
|
+
function recordUnmappedTool(toolName, eventName, nearMiss, _testDb) {
|
|
694
|
+
try {
|
|
695
|
+
if (!toolName) return;
|
|
696
|
+
const db = _testDb ?? getGlobalDb();
|
|
697
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
698
|
+
db.prepare(
|
|
699
|
+
`INSERT INTO unmapped_tools (harness, tool_name, event_name, near_miss, first_seen, last_seen, hits)
|
|
700
|
+
VALUES (?, ?, ?, ?, ?, ?, 1)
|
|
701
|
+
ON CONFLICT(harness, tool_name, event_name) DO UPDATE SET
|
|
702
|
+
hits = hits + 1,
|
|
703
|
+
last_seen = excluded.last_seen,
|
|
704
|
+
near_miss = excluded.near_miss`
|
|
705
|
+
).run(getHarnessName(), toolName.slice(0, MAX_TOOL_NAME_CHARS), eventName, nearMiss, now, now);
|
|
706
|
+
} catch {
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
function readUnmappedTools(dbPath, homeDir) {
|
|
710
|
+
try {
|
|
711
|
+
const db = dbPath ? getDb(dbPath) : getGlobalDb(homeDir);
|
|
712
|
+
return db.prepare(
|
|
713
|
+
"SELECT harness, tool_name, event_name, near_miss, hits, last_seen FROM unmapped_tools ORDER BY hits DESC, tool_name ASC"
|
|
714
|
+
).all();
|
|
715
|
+
} catch {
|
|
716
|
+
return [];
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
function pruneStalePatternCoveredUnmappedTools(db, patterns) {
|
|
720
|
+
if (patterns.length === 0) return;
|
|
721
|
+
const regexes = patterns.map((p) => new RegExp(p));
|
|
722
|
+
try {
|
|
723
|
+
const rows = db.prepare("SELECT DISTINCT tool_name FROM unmapped_tools").all();
|
|
724
|
+
const stale = rows.filter((r) => regexes.some((re) => re.test(r.tool_name)));
|
|
725
|
+
if (stale.length === 0) return;
|
|
726
|
+
const del = db.prepare("DELETE FROM unmapped_tools WHERE tool_name = ?");
|
|
727
|
+
for (const r of stale) del.run(r.tool_name);
|
|
728
|
+
} catch {
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
var FIRST_RECEIPT_FLAG = "first_receipt_shown";
|
|
732
|
+
function allTimeTokensSaved(db) {
|
|
733
|
+
const kinds = [...COUNT_ONLY_KINDS];
|
|
734
|
+
const notCount = kinds.length > 0 ? `WHERE kind NOT IN (${kinds.map(() => "?").join(", ")})` : "";
|
|
735
|
+
const raw = db.prepare(`SELECT COALESCE(SUM(tokens_saved), 0) AS t FROM stats ${notCount}`).get(...kinds).t;
|
|
736
|
+
const rolled = db.prepare(`SELECT COALESCE(SUM(tokens_saved), 0) AS t FROM stats_daily_rollup ${notCount}`).get(...kinds).t;
|
|
737
|
+
return raw + rolled;
|
|
738
|
+
}
|
|
739
|
+
function claimFirstSavingsReceipt(testDb, homeDir, now = Date.now()) {
|
|
740
|
+
try {
|
|
741
|
+
const db = testDb ?? getGlobalDb(homeDir);
|
|
742
|
+
if (db.prepare("SELECT 1 FROM stats_flags WHERE name = ?").get(FIRST_RECEIPT_FLAG) !== void 0) return null;
|
|
743
|
+
const total = allTimeTokensSaved(db);
|
|
744
|
+
if (total <= 0) return null;
|
|
745
|
+
const claimed = db.prepare("INSERT OR IGNORE INTO stats_flags (name, set_ts) VALUES (?, ?)").run(FIRST_RECEIPT_FLAG, Math.floor(now / 1e3));
|
|
746
|
+
return claimed.changes === 1 ? total : null;
|
|
747
|
+
} catch {
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
function firstReceiptShownAt(testDb, homeDir) {
|
|
752
|
+
try {
|
|
753
|
+
const db = testDb ?? getGlobalDb(homeDir);
|
|
754
|
+
const row = db.prepare("SELECT set_ts FROM stats_flags WHERE name = ?").get(FIRST_RECEIPT_FLAG);
|
|
755
|
+
return row === void 0 ? null : row.set_ts * 1e3;
|
|
756
|
+
} catch {
|
|
757
|
+
return null;
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
function summarize(windowDays = 30, testDb, homeDir) {
|
|
761
|
+
const t0 = Date.now();
|
|
762
|
+
const sinceTs = windowDays > 0 ? Math.floor((Date.now() - windowDays * 24 * 60 * 60 * 1e3) / 1e3) : null;
|
|
763
|
+
const byKind = {};
|
|
764
|
+
const byDay = {};
|
|
765
|
+
const byHarness = {};
|
|
766
|
+
const byPricingVersion = {};
|
|
767
|
+
let totalEvents = 0;
|
|
768
|
+
let totalBytes = 0;
|
|
769
|
+
let totalTokens = 0;
|
|
770
|
+
const db = testDb ?? getGlobalDb(homeDir);
|
|
771
|
+
const hasHarness = statsHasHarnessColumn(db);
|
|
772
|
+
const hasVersion = statsHasVersionColumn(db);
|
|
773
|
+
const cols = [
|
|
774
|
+
"ts",
|
|
775
|
+
"kind",
|
|
776
|
+
"bytes_saved",
|
|
777
|
+
"tokens_saved",
|
|
778
|
+
...hasHarness ? ["harness"] : [],
|
|
779
|
+
...hasVersion ? ["tg_version"] : []
|
|
780
|
+
].join(", ");
|
|
781
|
+
const query = sinceTs !== null ? `SELECT ${cols} FROM stats WHERE ts >= ? ORDER BY ts DESC` : `SELECT ${cols} FROM stats ORDER BY ts DESC`;
|
|
782
|
+
const stmt = db.prepare(query);
|
|
783
|
+
const rows = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
|
|
784
|
+
const tsToDateCache = {};
|
|
785
|
+
const counts = {};
|
|
786
|
+
for (const row of rows) {
|
|
787
|
+
const bytesSaved = row.bytes_saved ?? 0;
|
|
788
|
+
const recorded = row.tokens_saved ?? 0;
|
|
789
|
+
const kind = row.kind;
|
|
790
|
+
const isCount = COUNT_ONLY_KINDS.has(kind);
|
|
791
|
+
if (isCount) counts[kind] = (counts[kind] ?? 0) + recorded;
|
|
792
|
+
const tokensSaved = isCount ? 0 : recorded;
|
|
793
|
+
const tsRaw = row.ts;
|
|
794
|
+
if (tsRaw === void 0) continue;
|
|
795
|
+
const ts = tsRaw;
|
|
796
|
+
totalEvents += 1;
|
|
797
|
+
totalBytes += bytesSaved;
|
|
798
|
+
totalTokens += tokensSaved;
|
|
799
|
+
if (!byKind[kind]) {
|
|
800
|
+
byKind[kind] = zeroBucket();
|
|
801
|
+
}
|
|
802
|
+
incBucket(byKind[kind], bytesSaved, tokensSaved);
|
|
803
|
+
const dateKey = tsToDateCache[ts] || toLocalDateKey(new Date(ts * 1e3));
|
|
804
|
+
tsToDateCache[ts] = dateKey;
|
|
805
|
+
if (!byDay[dateKey]) {
|
|
806
|
+
byDay[dateKey] = zeroBucket();
|
|
807
|
+
}
|
|
808
|
+
incBucket(byDay[dateKey], bytesSaved, tokensSaved);
|
|
809
|
+
const harness = row.harness || HARNESS_UNRECORDED;
|
|
810
|
+
if (!byHarness[harness]) {
|
|
811
|
+
byHarness[harness] = zeroBucket();
|
|
812
|
+
}
|
|
813
|
+
incBucket(byHarness[harness], bytesSaved, tokensSaved);
|
|
814
|
+
const pricingVersion = row.tg_version || PRICING_VERSION_UNRECORDED;
|
|
815
|
+
if (!byPricingVersion[pricingVersion]) {
|
|
816
|
+
byPricingVersion[pricingVersion] = zeroBucket();
|
|
817
|
+
}
|
|
818
|
+
incBucket(byPricingVersion[pricingVersion], bytesSaved, tokensSaved);
|
|
819
|
+
}
|
|
820
|
+
try {
|
|
821
|
+
const rollupTable = db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'stats_daily_rollup'`).get();
|
|
822
|
+
if (rollupTable !== void 0) {
|
|
823
|
+
const sinceDayKey = sinceTs !== null ? toLocalDateKey(new Date(sinceTs * 1e3)) : null;
|
|
824
|
+
const rollupRows = sinceDayKey !== null ? db.prepare(`SELECT day, kind, harness, tg_version, events, bytes_saved, tokens_saved FROM stats_daily_rollup WHERE day >= ?`).all(sinceDayKey) : db.prepare(`SELECT day, kind, harness, tg_version, events, bytes_saved, tokens_saved FROM stats_daily_rollup`).all();
|
|
825
|
+
const addBucket = (bucket, events, bytesSaved, tokensSaved) => {
|
|
826
|
+
bucket.events += events;
|
|
827
|
+
bucket.bytes_saved += bytesSaved;
|
|
828
|
+
bucket.tokens_saved += tokensSaved;
|
|
829
|
+
};
|
|
830
|
+
for (const r of rollupRows) {
|
|
831
|
+
const isCount = COUNT_ONLY_KINDS.has(r.kind);
|
|
832
|
+
if (isCount) counts[r.kind] = (counts[r.kind] ?? 0) + r.tokens_saved;
|
|
833
|
+
const tokensSaved = isCount ? 0 : r.tokens_saved;
|
|
834
|
+
const bytesSaved = r.bytes_saved;
|
|
835
|
+
totalEvents += r.events;
|
|
836
|
+
totalBytes += bytesSaved;
|
|
837
|
+
totalTokens += tokensSaved;
|
|
838
|
+
if (!byKind[r.kind]) byKind[r.kind] = zeroBucket();
|
|
839
|
+
addBucket(byKind[r.kind], r.events, bytesSaved, tokensSaved);
|
|
840
|
+
if (!byDay[r.day]) byDay[r.day] = zeroBucket();
|
|
841
|
+
addBucket(byDay[r.day], r.events, bytesSaved, tokensSaved);
|
|
842
|
+
const harness = r.harness || HARNESS_UNRECORDED;
|
|
843
|
+
if (!byHarness[harness]) byHarness[harness] = zeroBucket();
|
|
844
|
+
addBucket(byHarness[harness], r.events, bytesSaved, tokensSaved);
|
|
845
|
+
const pricingVersion = r.tg_version || PRICING_VERSION_UNRECORDED;
|
|
846
|
+
if (!byPricingVersion[pricingVersion]) byPricingVersion[pricingVersion] = zeroBucket();
|
|
847
|
+
addBucket(byPricingVersion[pricingVersion], r.events, bytesSaved, tokensSaved);
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
} catch {
|
|
851
|
+
}
|
|
852
|
+
const bySourceDict = {};
|
|
853
|
+
for (const [kind, bucket] of Object.entries(byKind)) {
|
|
854
|
+
const source = kindToSource(kind);
|
|
855
|
+
if (!bySourceDict[source]) {
|
|
856
|
+
bySourceDict[source] = zeroBucket();
|
|
857
|
+
}
|
|
858
|
+
bySourceDict[source].events += bucket.events;
|
|
859
|
+
bySourceDict[source].bytes_saved += bucket.bytes_saved;
|
|
860
|
+
bySourceDict[source].tokens_saved += bucket.tokens_saved;
|
|
861
|
+
}
|
|
862
|
+
const byCommandDict = {};
|
|
863
|
+
for (const [cmd, kinds] of Object.entries(COMMAND_KINDS)) {
|
|
864
|
+
byCommandDict[cmd] = zeroBucket();
|
|
865
|
+
for (const kind of kinds) {
|
|
866
|
+
if (byKind[kind]) {
|
|
867
|
+
byCommandDict[cmd].events += byKind[kind].events;
|
|
868
|
+
byCommandDict[cmd].bytes_saved += byKind[kind].bytes_saved;
|
|
869
|
+
byCommandDict[cmd].tokens_saved += byKind[kind].tokens_saved;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
const byDayList = Object.entries(byDay).map(([date, bucket]) => ({ ...bucket, date })).sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
|
|
874
|
+
const byProjectList = [];
|
|
875
|
+
const t1 = Date.now();
|
|
876
|
+
if (t1 - t0 > 1e3) {
|
|
877
|
+
console.warn(`summarize took ${t1 - t0}ms (window=${windowDays}d, total=${totalEvents} rows)`);
|
|
878
|
+
}
|
|
879
|
+
return {
|
|
880
|
+
total_events: totalEvents,
|
|
881
|
+
total_bytes_saved: totalBytes,
|
|
882
|
+
total_tokens_saved: totalTokens,
|
|
883
|
+
by_kind: byKind,
|
|
884
|
+
by_day: byDayList,
|
|
885
|
+
by_project: byProjectList,
|
|
886
|
+
by_source: bySourceDict,
|
|
887
|
+
by_harness: byHarness,
|
|
888
|
+
by_pricing_version: byPricingVersion,
|
|
889
|
+
counts,
|
|
890
|
+
by_command: Object.entries(byCommandDict).map(([command, bucket]) => ({ ...bucket, command })).filter((r) => r.events > 0),
|
|
891
|
+
window_days: windowDays
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
export {
|
|
896
|
+
BYTES_PER_TOKEN,
|
|
897
|
+
guardDivisor,
|
|
898
|
+
classifyContent,
|
|
899
|
+
PRICING_VERSION_UNRECORDED,
|
|
900
|
+
hasMixedPricingEras,
|
|
901
|
+
SOURCE_HINT,
|
|
902
|
+
savedTokensFromBytes,
|
|
903
|
+
formatLocalTimestamp,
|
|
904
|
+
statsHasHarnessColumn,
|
|
905
|
+
statsHasDurationColumn,
|
|
906
|
+
getGlobalDb,
|
|
907
|
+
HOOK_STATS_RETENTION_DAYS,
|
|
908
|
+
HOOK_PROBE_ENV,
|
|
909
|
+
noStatsMessage,
|
|
910
|
+
withTelemetryWriteDb,
|
|
911
|
+
recordStat,
|
|
912
|
+
recordUnmappedTool,
|
|
913
|
+
readUnmappedTools,
|
|
914
|
+
pruneStalePatternCoveredUnmappedTools,
|
|
915
|
+
claimFirstSavingsReceipt,
|
|
916
|
+
firstReceiptShownAt,
|
|
917
|
+
summarize
|
|
918
|
+
};
|