token-goat 2.9.13 → 2.9.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/dist/token-goat-chunk-2X2EBBC6.mjs +277 -0
- package/dist/token-goat-chunk-3BTK54F3.mjs +1733 -0
- package/dist/token-goat-chunk-3NSDDTGL.mjs +34 -0
- package/dist/token-goat-chunk-4EXFN2AW.mjs +29 -0
- package/dist/token-goat-chunk-5V7DAC7V.mjs +123 -0
- package/dist/token-goat-chunk-6DLVZDB6.mjs +34 -0
- package/dist/token-goat-chunk-7ZYK25AO.mjs +24 -0
- package/dist/token-goat-chunk-A4JYKD5H.mjs +144 -0
- package/dist/token-goat-chunk-AH6QILZM.mjs +26 -0
- package/dist/token-goat-chunk-ASYEPR3S.mjs +212 -0
- package/dist/{token-goat-chunk-RDITECDL.mjs → token-goat-chunk-ATIFTMRC.mjs} +31 -13
- package/dist/{token-goat-chunk-ZZI3IDQZ.mjs → token-goat-chunk-BL5LNGBG.mjs} +3935 -11949
- package/dist/{token-goat-chunk-4NXUKV7D.mjs → token-goat-chunk-C5JIO6HK.mjs} +8 -4
- package/dist/{token-goat-chunk-FZU7GMUS.mjs → token-goat-chunk-DQ4J5AFF.mjs} +50 -18
- package/dist/token-goat-chunk-ERTXEKB6.mjs +228 -0
- package/dist/{token-goat-chunk-6B44WLIF.mjs → token-goat-chunk-GIIHUSZX.mjs} +142 -21
- package/dist/token-goat-chunk-GMOUBOX4.mjs +386 -0
- package/dist/{token-goat-chunk-3ESRORNM.mjs → token-goat-chunk-GMQQA7E4.mjs} +12576 -12326
- package/dist/token-goat-chunk-IVUQLQWN.mjs +2046 -0
- package/dist/token-goat-chunk-K7F2BFIK.mjs +2430 -0
- package/dist/{token-goat-chunk-U7X6LQD2.mjs → token-goat-chunk-LCZBPOIN.mjs} +10197 -9717
- package/dist/token-goat-chunk-LMFO66YD.mjs +331 -0
- package/dist/token-goat-chunk-LT7JRU6K.mjs +22 -0
- package/dist/token-goat-chunk-MZDIJJ3R.mjs +420 -0
- package/dist/token-goat-chunk-NDPO7GAH.mjs +177 -0
- package/dist/token-goat-chunk-NDRP4KJQ.mjs +4371 -0
- package/dist/token-goat-chunk-NEI4NC54.mjs +424 -0
- package/dist/token-goat-chunk-NU7TLMQK.mjs +585 -0
- package/dist/token-goat-chunk-OSUFN2FV.mjs +326 -0
- package/dist/token-goat-chunk-OUGNPMDA.mjs +959 -0
- package/dist/token-goat-chunk-PM76YS22.mjs +1341 -0
- package/dist/token-goat-chunk-S4XRY446.mjs +2637 -0
- package/dist/{token-goat-chunk-YOA4N6WA.mjs → token-goat-chunk-SFAS46RE.mjs} +5 -3
- package/dist/{token-goat-chunk-QWSUZWFP.mjs → token-goat-chunk-SZWYESBS.mjs} +793 -650
- package/dist/{token-goat-chunk-B3CTCQTH.mjs → token-goat-chunk-XEPXYDPI.mjs} +3 -2
- package/dist/token-goat-chunk-XTQAOTSO.mjs +89 -0
- package/dist/token-goat-chunk-Y4AFKTHK.mjs +22 -0
- package/dist/token-goat-chunk-YKG35VHC.mjs +228 -0
- package/dist/{token-goat-chunk-JOXLE672.mjs → token-goat-chunk-YQ7WI2CO.mjs} +990 -106
- package/dist/{token-goat-chunk-2WC4ZUXN.mjs → token-goat-chunk-YZX7EFG4.mjs} +1 -1
- package/dist/token-goat-chunk-Z6UXPYJA.mjs +62 -0
- package/dist/token-goat-hook.mjs +16 -8
- package/dist/token-goat.core.mjs +27 -10
- package/docs/cli.md +9 -6
- package/package.json +4 -2
- package/dist/token-goat-chunk-FQCNJV4V.mjs +0 -693
- package/dist/token-goat-chunk-JVNPCQB7.mjs +0 -31
- package/dist/token-goat-chunk-P2PU4CR5.mjs +0 -26
- package/dist/token-goat-chunk-QKXBGBQR.mjs +0 -3653
- package/dist/token-goat-chunk-UM47DRD3.mjs +0 -242
- package/dist/token-goat-chunk-UMXJN7DI.mjs +0 -5521
package/README.md
CHANGED
|
@@ -138,6 +138,11 @@ The fastest way to reduce AI token costs is fixing these five, not writing short
|
|
|
138
138
|
| Large MCP tool result (≥2 KB) doesn't table-ify (an object-rooted config dump, a single-resource response, a nested settings tree) but most of its bytes are `null`/`""`/`[]`/`{}` | Deterministic empty-value pruning: recursively drops those four empty shapes to a fixed point (a container left empty by its own dropped children is dropped too) and renders the remainder as compact JSON with a trailing `dropped N empty value(s) (null, "", [], {})` summary line; `0` and `false` are never dropped; only applied when it saves ≥15%. Same full-recovery-by-id guarantee and `TOKEN_GOAT_MCP_COMPRESS=0` opt-out as the table pass |
|
|
139
139
|
| GitHub MCP tool result (`list_pull_requests`, `list_issues`, `search_code`, `get_file_contents`, `pull_request_read`, …) carries dozens of boilerplate fields per object | GitHub compression pack strips `_links`, `node_id`, `gravatar_id`, `site_admin`, and every `*_url` field (`avatar_url`, `html_url`, `events_url`, `gists_url`, `followers_url`, …) except `download_url`/`git_url`/`clone_url`/`ssh_url`, before handing the shrunk JSON to the same table-ifying pass — same `TOKEN_GOAT_MCP_COMPRESS=0` opt-out and full-recovery-by-id guarantee |
|
|
140
140
|
| Browser-automation MCP tool result (claude-in-chrome's `read_console_messages`/`read_network_requests`, chrome-devtools-mcp's `list_console_messages`/`list_network_requests`) carries verbose CDP plumbing per entry | Browser compression pack strips console `stackTrace` frames and network `requestHeaders`/`responseHeaders`/`timing`/`initiator`/`securityDetails`/cookie fields, keeping `url`/`method`/`status`/`resourceType`/`mimeType`/`reqid` and the actual log text, before the same table-ifying pass runs — same opt-out and full-recovery-by-id guarantee |
|
|
141
|
+
| Atlassian MCP tool result (Jira issues, Confluence pages, search results) carries repetitive UI chrome and schemas | Atlassian compression pack strips boilerplate fields (`avatarUrls`, `iconUrl`, `self`, `expand`, `schema`, `operations`, `editmeta`, `names`, `_links`, `timeZone`), reducing payloads by 60–80% before table compression |
|
|
142
|
+
| Oversized MCP tool result (≥25 KB) or dumped JSON tool spill file (e.g. `content.json`) re-read whole into context | Oversized MCP results return an elision preview and recovery ID; pre-read hooks intercept tool spill files, redirecting models to `token-goat mcp-output --json-query '<path>'` or `--file <path>` (~90–98% smaller) |
|
|
143
|
+
| Large XML package (.dtsx, .ampkg, .xaml) read in full or paged with sequential line ranges | Pre-read hook intercepts package reads (20KB threshold) with package-specific hierarchy advice; sequential 3+ range paging is actively denied and redirected to `xml-outline` and `xml-query` |
|
|
144
|
+
| Scratch terminal script or shell command used to inspect XML (`Select-Xml`, `[xml]`, `inspect_*.ps1`, Python `xml.etree`, `xmllint`) | Pre-bash hook intercepts terminal XML parsing commands, recommending `token-goat xml-query --xpath <expr>` or `xml-outline` instead of multi-step script generation (~85–95% smaller) |
|
|
145
|
+
| XML document with deep element hierarchies or entity-encoded XML/AML payloads | `token-goat xml-query <file> --xpath <expr>` extracts targeted elements with exact line ranges (`--with-lines`), and `--decode-embedded-xml` pretty-prints and bounds nested AML payloads (~80–95% smaller) |
|
|
141
146
|
| `curl -v` dumps TLS handshake + all request/response headers | Verbose lines stripped; request line, HTTP status, content-type, and body kept — typically 70–90% smaller |
|
|
142
147
|
| `jest --verbose` / `vitest --verbose` emits one `✓` line per passing test | Consecutive passing-test lines collapsed to a count per file; failures kept verbatim, ~95% smaller on passing suites |
|
|
143
148
|
| `go test -v` emits `--- PASS: TestName (Ns)` for every passing test | PASS lines collapsed to a count per package; FAIL lines and panic output kept, ~90% smaller on clean runs |
|
|
@@ -175,6 +180,7 @@ The fastest way to reduce AI token costs is fixing these five, not writing short
|
|
|
175
180
|
| Reading poetry.lock or package-lock.json to find a pinned version | `token-goat lockdeps` returns a name/version table of direct dependencies; optional packages and transitive entries excluded |
|
|
176
181
|
| Large SVG / diagram file (≥8 KB) read in full | Coordinate path flooding blocked; extracted layer/group IDs and title shown; redirects to `token-goat xml-outline` and `xml-query` |
|
|
177
182
|
| Broad recursive Glob sweep (`*`, `**/*`) on root directory | Pre-Glob hook warns against tree-dumping and points at `token-goat map --compact` for fast, lightweight structure inspection |
|
|
183
|
+
| Guessing database column names in `session_store_sql` / `sql` and falling back to `SELECT *` | `token-goat session-schema [table]` and `describe <target>` provide instant schema discovery; `post_tool_use_failure` hook intercepts unknown columns and guides the query — ~85–95% smaller than trial-and-error `SELECT *` dumps |
|
|
178
184
|
| Compound test/build pipeline (`npm run build && npm run typecheck && npm test`) | Post-Bash hook routes chained build/test/lint commands to `generic-ci` compression, dropping verbose passing steps and compiler noise |
|
|
179
185
|
|
|
180
186
|
On a per-token API plan, 100K wasted tokens per session runs about $0.30. Five sessions a week is ~$450/year. AI coding cost reduction at that scale comes from fixing the waste, not from using the product less. Token-goat is free. And on subscription plans, it can result in limits feeling 10x higher.
|
|
@@ -186,6 +192,7 @@ On a per-token API plan, 100K wasted tokens per session runs about $0.30. Five s
|
|
|
186
192
|
```bash
|
|
187
193
|
cd ~/notes # or any plain folder of .md files, no .git required
|
|
188
194
|
token-goat index . --walk # non-git folders need --walk (git repos: plain `token-goat index .`)
|
|
195
|
+
token-goat semantic --preflight # verify runtime, model weights & project coverage
|
|
189
196
|
token-goat semantic "how long to steep cold brew"
|
|
190
197
|
```
|
|
191
198
|
|
|
@@ -204,10 +211,15 @@ Numbers below come from synthetic-fixture benchmarks in the test suite, except t
|
|
|
204
211
|
| DB reindex | Batched single transaction + composite indexes on `(file_id, kind)` | 100 files / 10K rows: 84 s → 1 s (~80× faster) | `src/parser.ts`, `src/db.ts` (index migration) |
|
|
205
212
|
| Hook cold-start | Lazy import of heavy modules; unknown events short-circuit | 86 ms → 30 ms (~65% faster); unknown-event dispatch <1 ms | `src/hooks_cli.ts` |
|
|
206
213
|
| Symbol start_line | TypeScript decorators captured in symbol span | One `token-goat read` returns the decorator + signature + body; no re-read | `src/parser.ts` (TypeScript adapter) |
|
|
207
|
-
| Section extraction | Setext headings, h5/h6, anchor IDs, and `__frontmatter__` | `token-goat section` resolves more headings without falling back to a full file read | `src/parser.ts` (Markdown
|
|
214
|
+
| Section extraction | Setext headings, h5/h6, anchor IDs, shell comment banners (`##`, `# ---`, `# ===`, `# [...]`, `# REGION:`), and `__frontmatter__` | `token-goat section` resolves more headings and procedural shell checks without falling back to a full file read | `src/parser.ts` (Markdown & Shell adapters) |
|
|
208
215
|
| Image cache | Repeat Read of an unchanged image serves the stored re-encode, keyed on path + size + mtime, instead of re-encoding it again | Skips the re-encode entirely on a hit; the same bytes reach the model, so the reported saving is identical either way | `src/image_shrink.ts` (`findCachedShrink`) |
|
|
209
216
|
| Monorepo defaults | Reindex batch 500 → 2000; compact `min_events` 5 → 3 | Fewer worker wakeups; compact manifests fire on shorter sessions | `src/config.ts` defaults |
|
|
210
217
|
| Miss suggestions | `read` / `section` print "Did you mean…?" on a miss; `section` also auto-redirects on an unambiguous heading-prefix match | Keeps agents on the surgical-read path instead of falling back to full-file `Read` | `src/read_commands.ts` |
|
|
218
|
+
| Atlassian MCP compression | Strips UI chrome, avatars, operations, and schema links from Jira and Confluence tool responses | 60–80% smaller before generic table compression | `src/mcp_atlassian_compress.ts` |
|
|
219
|
+
| MCP tool spill slicing | Narrow-slice queries on cached MCP tool outputs or dumped JSON files with `mcp-output --json-query` | 90–98% smaller than whole-file JSON re-reads | `src/mcp_json_query.ts` |
|
|
220
|
+
| XML package & terminal script interception | Pre-read nudges on `.dtsx`/`.ampkg`/`.xaml` and pre-bash interception of `Select-Xml`/`[xml]`/scratch scripts | 85–95% smaller than scratch script creation and chunk paging | `src/bash_extractors.ts`, `src/hooks_read.ts` |
|
|
221
|
+
| Enhanced XML XPath & embedded AML | Granular XPath expressions, element source line ranges, and decoded embedded XML/AML payloads | 80–95% smaller than reading whole XML packages | `src/xml_query.ts` |
|
|
222
|
+
| Session Store Schema & SQL Error Intercept | Authoritative schema discovery via `session-schema` / `describe`, paired with runtime unknown-column diagnosis in `postToolUseFailureHandler` | 85–95% smaller than trial-and-error `SELECT *` dumps | `src/session_store_schema.ts`, `src/hooks_tool_failure.ts` |
|
|
211
223
|
|
|
212
224
|
## Token-savings examples
|
|
213
225
|
|
|
@@ -298,6 +310,15 @@ Built-in output compression covers 150+ dev tool CLIs: `pytest`, `jest` / `vites
|
|
|
298
310
|
|
|
299
311
|
A failing `pytest` / `jest` / `vitest` / `go test` / `cargo test` run (including bare `npm test`/`npm run test`/`yarn test`/`pnpm test`) also gets a one-line advisory naming the exact `token-goat bash-output <id> | token-goat failures` command to run instead of re-reading the raw dump. Silent on a passing run, a non-test command, or output too small to be worth reducing.
|
|
300
312
|
|
|
313
|
+
Give the command three ways: as positional arguments (`token-goat compress pytest -v tests/`), as one string with `-c` / `--cmd`, or base64-encoded with `--cmd-b64`. The base64 form preserves quotes, backslashes, and dollar signs that a harness or an intermediate shell would otherwise mangle; it takes precedence when more than one form is supplied. `--native` runs the command through the platform's own shell (`cmd.exe` on Windows) instead of bash, which keeps Windows path backslashes intact for tools that reject the forward-slash form.
|
|
314
|
+
|
|
315
|
+
`-q` / `--quiet-success` trades a clean run's output for one line. On exit code 0 the full output is stored for recall and only the summary prints; on any other exit code the compressed output is delivered as usual.
|
|
316
|
+
|
|
317
|
+
```
|
|
318
|
+
$ token-goat compress -q --cmd "npm run lint"
|
|
319
|
+
[tg: ok] npm run lint (22.2s, recall: token-goat bash-output 326d9800e732eb9d)
|
|
320
|
+
```
|
|
321
|
+
|
|
301
322
|
### 6. Context pressure
|
|
302
323
|
|
|
303
324
|
Token-goat tracks how close a session is to the autocompact trigger and tightens its hints as the window fills. Surgical-read suggestions kick in on progressively smaller files as pressure builds (500 lines at cool, down to 50 at critical), so large reads get flagged before they tip the session over. The PreCompact manifest also shrinks: capped at 500 tokens once the window runs hot, 300 once critical, so it stops contributing to the pressure it measures. The denominator is always the fixed 660,000-token autocompact trigger budget, not the model's raw context window, so the same thresholds apply across models. Run `token-goat doctor --context` to see the current footprint.
|
|
@@ -478,6 +499,8 @@ The file-by-file table for each harness, and the path that file sits at: **[What
|
|
|
478
499
|
|
|
479
500
|
Hooks fire automatically on every tool call once installed — nothing to start or restart there. The background worker is a separate, manual step: `token-goat worker start` launches it as a detached process, `token-goat worker status` checks it, `token-goat worker stop` kills it. It restarts itself automatically if it crashes or gets killed while the machine is running — an edit hook checks its liveness and respawns it, rate-limited to about once every 5 minutes. It does not survive a reboot or logout, though; re-run `worker start` after either. `token-goat uninstall` removes the hook entries, `CLAUDE.md` block, and skill directory, but does not touch a running worker — stop it separately with `token-goat worker stop` if you no longer want it running.
|
|
480
501
|
|
|
502
|
+
To move to a newer release, `token-goat upgrade` installs it from npm and then re-runs `token-goat install`, so your hooks and integration manifests point at the build that just landed. `token-goat upgrade --check` reports whether one is available without installing anything, and `--json` gives the same answer for a script. Both go quiet when `network.offline` is set, and `token-goat doctor` mentions an available update at the end of its report.
|
|
503
|
+
|
|
481
504
|
## Verify
|
|
482
505
|
|
|
483
506
|
```
|
|
@@ -701,12 +724,16 @@ Bug reports go to the same place. The most useful ones include:
|
|
|
701
724
|
- The matching log line from `%LOCALAPPDATA%\dfk-helper\token-goat\logs\` on Windows or `~/.local/share/token-goat/logs/` on Linux/WSL
|
|
702
725
|
- What you expected and what actually happened
|
|
703
726
|
|
|
727
|
+
To audit an existing AI session for missed savings, hook friction, or optimization opportunities before opening an issue, see the [Session Mining Prompt](docs/PROMPT_SESSION_MINING.md).
|
|
728
|
+
|
|
704
729
|
For private questions, commercial licensing, or anything you'd rather not post publicly, contact me at token-goat@dfkhelper.com.
|
|
705
730
|
|
|
706
731
|
## Disclaimer
|
|
707
732
|
|
|
708
733
|
Token-Goat runs on your machine and touches your files. The software is provided as-is, without warranty of any kind. DFK Helper LLC is not liable for any damages arising from use. Full terms, including the No Liability clause, are in the LICENSE file.
|
|
709
734
|
|
|
735
|
+
All product names, logos, brands, trademarks, and registered trademarks mentioned are property of their respective owners. All company, product, and service names used in this project are for identification and compatibility purposes only; their use does not imply any affiliation, sponsorship, or endorsement.
|
|
736
|
+
|
|
710
737
|
## License
|
|
711
738
|
|
|
712
739
|
Token-Goat is licensed under the PolyForm Noncommercial License 1.0.0. See the LICENSE file for the full terms.
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import { createRequire as __cjsRequire } from 'node:module';
|
|
2
|
+
const require = __cjsRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
getDb
|
|
5
|
+
} from "./token-goat-chunk-K7F2BFIK.mjs";
|
|
6
|
+
import {
|
|
7
|
+
foldPath,
|
|
8
|
+
globalDbPath,
|
|
9
|
+
isCaseInsensitiveFs
|
|
10
|
+
} from "./token-goat-chunk-PM76YS22.mjs";
|
|
11
|
+
import {
|
|
12
|
+
normalizePath
|
|
13
|
+
} from "./token-goat-chunk-ERTXEKB6.mjs";
|
|
14
|
+
import {
|
|
15
|
+
init_define_import_meta_env
|
|
16
|
+
} from "./token-goat-chunk-A37V4PBF.mjs";
|
|
17
|
+
|
|
18
|
+
// src/index_reader.ts
|
|
19
|
+
init_define_import_meta_env();
|
|
20
|
+
|
|
21
|
+
// src/sql_path.ts
|
|
22
|
+
init_define_import_meta_env();
|
|
23
|
+
function pathEqClause(column) {
|
|
24
|
+
return isCaseInsensitiveFs() ? `TG_LOWER(${column}) = ?` : `${column} = ?`;
|
|
25
|
+
}
|
|
26
|
+
function pathSuffixClause(column) {
|
|
27
|
+
const col = isCaseInsensitiveFs() ? `TG_LOWER(${column})` : column;
|
|
28
|
+
return {
|
|
29
|
+
// Both separators are accepted because not every writer into `symbols` stores a normalizePath'd (forward-slash) file_path, and the JS-side path-boundary test this narrowing feeds treats `/` and `\` alike.
|
|
30
|
+
clause: `(${col} = ? OR substr(${col}, -length(?)) = ? OR substr(${col}, -length(?)) = ?)`,
|
|
31
|
+
params: (baseName) => {
|
|
32
|
+
const folded = foldPath(baseName);
|
|
33
|
+
return [folded, `/${folded}`, `/${folded}`, `\\${folded}`, `\\${folded}`];
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function projectScopeClause(column) {
|
|
38
|
+
const caseInsensitive = isCaseInsensitiveFs();
|
|
39
|
+
const col = caseInsensitive ? `TG_LOWER(${column})` : column;
|
|
40
|
+
return {
|
|
41
|
+
// Parenthesised so an OR anywhere in the surrounding WHERE cannot bind tighter than the pair.
|
|
42
|
+
clause: `(${col} >= ? AND ${col} < ?)`,
|
|
43
|
+
params: (root) => {
|
|
44
|
+
const folded = caseInsensitive ? foldPath(normalizePath(root)) : normalizePath(root);
|
|
45
|
+
const lower = folded.endsWith("/") ? folded : `${folded}/`;
|
|
46
|
+
const upper = `${lower.slice(0, -1)}0`;
|
|
47
|
+
return [lower, upper];
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// src/index_reader.ts
|
|
53
|
+
function toSymbolEntry(row) {
|
|
54
|
+
return {
|
|
55
|
+
filePath: row.file_path,
|
|
56
|
+
name: row.name,
|
|
57
|
+
kind: row.kind,
|
|
58
|
+
lineStart: row.line_start,
|
|
59
|
+
lineEnd: row.line_end,
|
|
60
|
+
body: row.body ?? "",
|
|
61
|
+
docstring: row.docstring ?? "",
|
|
62
|
+
parent: row.parent ?? ""
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function toRefEntry(row) {
|
|
66
|
+
return {
|
|
67
|
+
filePath: row.file_path,
|
|
68
|
+
name: row.name,
|
|
69
|
+
line: row.line,
|
|
70
|
+
col: row.col,
|
|
71
|
+
context: row.context ?? ""
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function applyRootDirScope(rootDir, column, where, params) {
|
|
75
|
+
if (rootDir === void 0) return;
|
|
76
|
+
const { clause, params: bounds } = projectScopeClause(column);
|
|
77
|
+
where.push(clause);
|
|
78
|
+
params.push(...bounds(rootDir));
|
|
79
|
+
}
|
|
80
|
+
function buildSymbolWhere(opts) {
|
|
81
|
+
const where = [];
|
|
82
|
+
const params = [];
|
|
83
|
+
if (opts.name !== void 0) {
|
|
84
|
+
where.push("name = ?");
|
|
85
|
+
params.push(opts.name);
|
|
86
|
+
}
|
|
87
|
+
if (opts.filePath !== void 0) {
|
|
88
|
+
if (opts.filePath.includes("/") || opts.filePath.includes("\\")) {
|
|
89
|
+
const alt = opts.filePath.includes("/") ? opts.filePath.replace(/\//g, "\\") : opts.filePath.replace(/\\/g, "/");
|
|
90
|
+
where.push(`(${pathEqClause("file_path")} OR ${pathEqClause("file_path")})`);
|
|
91
|
+
params.push(foldPath(opts.filePath), foldPath(alt));
|
|
92
|
+
} else {
|
|
93
|
+
where.push(pathEqClause("file_path"));
|
|
94
|
+
params.push(foldPath(opts.filePath));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (opts.kind !== void 0) {
|
|
98
|
+
where.push("kind = ?");
|
|
99
|
+
params.push(opts.kind);
|
|
100
|
+
}
|
|
101
|
+
if (opts.fileBaseName !== void 0) {
|
|
102
|
+
const { clause: suffixClause, params: suffixParams } = pathSuffixClause("file_path");
|
|
103
|
+
where.push(suffixClause);
|
|
104
|
+
params.push(...suffixParams(opts.fileBaseName));
|
|
105
|
+
}
|
|
106
|
+
if (opts.enclosingLine !== void 0) {
|
|
107
|
+
where.push("line_start <= ? AND ? <= line_end");
|
|
108
|
+
params.push(opts.enclosingLine, opts.enclosingLine);
|
|
109
|
+
}
|
|
110
|
+
applyRootDirScope(opts.rootDir, "file_path", where, params);
|
|
111
|
+
return { clause: where.length > 0 ? `WHERE ${where.join(" AND ")}` : "", params };
|
|
112
|
+
}
|
|
113
|
+
var DEFAULT_QUERY_LIMIT = 100;
|
|
114
|
+
function querySymbols(opts = {}, dbPath = globalDbPath()) {
|
|
115
|
+
const { clause, params } = buildSymbolWhere(opts);
|
|
116
|
+
const limit = opts.limit ?? DEFAULT_QUERY_LIMIT;
|
|
117
|
+
const offset = opts.offset ?? 0;
|
|
118
|
+
const sql = `SELECT file_path, name, kind, line_start, line_end, body, docstring, parent FROM symbols ${clause} ORDER BY file_path, line_start, rowid LIMIT ? OFFSET ?`;
|
|
119
|
+
const db = getDb(dbPath);
|
|
120
|
+
const rows = db.prepare(sql).all(...params, limit, offset);
|
|
121
|
+
return rows.map(toSymbolEntry);
|
|
122
|
+
}
|
|
123
|
+
function distinctSymbolKinds(rootDir, dbPath = globalDbPath()) {
|
|
124
|
+
const where = [];
|
|
125
|
+
const params = [];
|
|
126
|
+
applyRootDirScope(rootDir, "file_path", where, params);
|
|
127
|
+
const clause = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
|
|
128
|
+
const db = getDb(dbPath);
|
|
129
|
+
const rows = db.prepare(`SELECT DISTINCT kind FROM symbols ${clause} ORDER BY kind`).all(...params);
|
|
130
|
+
return rows.map((r) => r.kind);
|
|
131
|
+
}
|
|
132
|
+
function countSymbols(opts = {}, dbPath = globalDbPath()) {
|
|
133
|
+
const { clause, params } = buildSymbolWhere(opts);
|
|
134
|
+
const sql = `SELECT COUNT(*) as cnt FROM symbols ${clause}`;
|
|
135
|
+
const db = getDb(dbPath);
|
|
136
|
+
const row = db.prepare(sql).get(...params);
|
|
137
|
+
return row.cnt;
|
|
138
|
+
}
|
|
139
|
+
function buildRefsWhere(opts) {
|
|
140
|
+
const where = ["name = ?"];
|
|
141
|
+
const params = [opts.name];
|
|
142
|
+
if (opts.filePath !== void 0) {
|
|
143
|
+
where.push(pathEqClause("file_path"));
|
|
144
|
+
params.push(foldPath(opts.filePath));
|
|
145
|
+
}
|
|
146
|
+
applyRootDirScope(opts.rootDir, "file_path", where, params);
|
|
147
|
+
return { clause: `WHERE ${where.join(" AND ")}`, params };
|
|
148
|
+
}
|
|
149
|
+
function queryRefs(opts, dbPath = globalDbPath()) {
|
|
150
|
+
const { clause, params } = buildRefsWhere(opts);
|
|
151
|
+
const limit = opts.limit ?? DEFAULT_QUERY_LIMIT;
|
|
152
|
+
const sql = `SELECT file_path, name, line, col, context FROM refs ${clause} ORDER BY file_path, line LIMIT ?`;
|
|
153
|
+
const db = getDb(dbPath);
|
|
154
|
+
const rows = db.prepare(sql).all(...params, limit);
|
|
155
|
+
return rows.map(toRefEntry);
|
|
156
|
+
}
|
|
157
|
+
function countRefs(opts, dbPath = globalDbPath()) {
|
|
158
|
+
const { clause, params } = buildRefsWhere(opts);
|
|
159
|
+
const sql = `SELECT COUNT(*) as cnt FROM refs ${clause}`;
|
|
160
|
+
const db = getDb(dbPath);
|
|
161
|
+
const row = db.prepare(sql).get(...params);
|
|
162
|
+
return row.cnt;
|
|
163
|
+
}
|
|
164
|
+
function queryRefsByContext(context, filePath, dbPath = globalDbPath()) {
|
|
165
|
+
const sql = `SELECT file_path, name, line, col, context FROM refs WHERE context = ? AND ${pathEqClause("file_path")} ORDER BY line LIMIT 20`;
|
|
166
|
+
const db = getDb(dbPath);
|
|
167
|
+
const rows = db.prepare(sql).all(context, foldPath(filePath));
|
|
168
|
+
return rows.map(toRefEntry);
|
|
169
|
+
}
|
|
170
|
+
var REF_COUNT_BATCH = 900;
|
|
171
|
+
function queryRefCounts(names, dbPath = globalDbPath(), rootDir) {
|
|
172
|
+
const counts = /* @__PURE__ */ new Map();
|
|
173
|
+
if (names.length === 0) return counts;
|
|
174
|
+
const db = getDb(dbPath);
|
|
175
|
+
const scopeWhere = [];
|
|
176
|
+
const scopeParams = [];
|
|
177
|
+
applyRootDirScope(rootDir, "file_path", scopeWhere, scopeParams);
|
|
178
|
+
const scopeSql = scopeWhere.length > 0 ? ` AND ${scopeWhere.join(" AND ")}` : "";
|
|
179
|
+
const sqlFor = (width) => `SELECT name, COUNT(*) as c FROM refs WHERE name IN (${Array.from({ length: width }, () => "?").join(", ")})${scopeSql} GROUP BY name`;
|
|
180
|
+
const full = names.length >= REF_COUNT_BATCH ? db.prepare(sqlFor(REF_COUNT_BATCH)) : void 0;
|
|
181
|
+
db.transaction(() => {
|
|
182
|
+
for (let start = 0; start < names.length; start += REF_COUNT_BATCH) {
|
|
183
|
+
const batch = names.slice(start, start + REF_COUNT_BATCH);
|
|
184
|
+
const stmt = batch.length === REF_COUNT_BATCH && full !== void 0 ? full : db.prepare(sqlFor(batch.length));
|
|
185
|
+
for (const row of stmt.all(...batch, ...scopeParams)) {
|
|
186
|
+
counts.set(row.name, row.c);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
})();
|
|
190
|
+
return counts;
|
|
191
|
+
}
|
|
192
|
+
function getProjectFileEntries(rootDir, dbPath = globalDbPath()) {
|
|
193
|
+
const db = getDb(dbPath);
|
|
194
|
+
const { clause, params } = projectScopeClause("path");
|
|
195
|
+
const rows = db.prepare(`SELECT path, sha, mtime, language, indexed_at, embed_sha, parser_sha FROM files WHERE ${clause}`).all(...params(rootDir));
|
|
196
|
+
const out = /* @__PURE__ */ new Map();
|
|
197
|
+
for (const row of rows) {
|
|
198
|
+
out.set(foldPath(row.path), {
|
|
199
|
+
filePath: row.path,
|
|
200
|
+
sha: row.sha ?? "",
|
|
201
|
+
mtime: row.mtime ?? 0,
|
|
202
|
+
language: row.language ?? "unknown",
|
|
203
|
+
indexedAt: row.indexed_at ?? 0,
|
|
204
|
+
embedSha: row.embed_sha ?? "",
|
|
205
|
+
parserSha: row.parser_sha ?? ""
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
return out;
|
|
209
|
+
}
|
|
210
|
+
function getFileEntry(filePath, dbPath = globalDbPath()) {
|
|
211
|
+
const db = getDb(dbPath);
|
|
212
|
+
let row = db.prepare(
|
|
213
|
+
`SELECT path, sha, mtime, language, indexed_at, embed_sha, parser_sha FROM files WHERE ${pathEqClause("path")}`
|
|
214
|
+
).get(foldPath(filePath));
|
|
215
|
+
if (row === void 0 && (filePath.includes("/") || filePath.includes("\\"))) {
|
|
216
|
+
const altPath = filePath.includes("/") ? filePath.replace(/\//g, "\\") : filePath.replace(/\\/g, "/");
|
|
217
|
+
row = db.prepare(
|
|
218
|
+
`SELECT path, sha, mtime, language, indexed_at, embed_sha, parser_sha FROM files WHERE ${pathEqClause("path")}`
|
|
219
|
+
).get(foldPath(altPath));
|
|
220
|
+
}
|
|
221
|
+
if (row === void 0) return null;
|
|
222
|
+
return {
|
|
223
|
+
filePath: row.path,
|
|
224
|
+
sha: row.sha ?? "",
|
|
225
|
+
mtime: row.mtime ?? 0,
|
|
226
|
+
language: row.language ?? "unknown",
|
|
227
|
+
indexedAt: row.indexed_at ?? 0,
|
|
228
|
+
embedSha: row.embed_sha ?? "",
|
|
229
|
+
parserSha: row.parser_sha ?? ""
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
function sanitizeFtsQuery(query, join = "AND") {
|
|
233
|
+
const terms = query.split(/\s+/).filter((t) => t.length > 0).map((t) => '"' + t.replace(/"/g, '""') + '"');
|
|
234
|
+
return terms.join(join === "OR" ? " OR " : " ");
|
|
235
|
+
}
|
|
236
|
+
function runFtsQuery(db, match, limit, scope, rootDir) {
|
|
237
|
+
const sql = `SELECT s.file_path, s.name, s.kind, s.line_start, s.line_end, s.body, s.docstring, s.parent FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE symbols_fts MATCH ?${scope !== void 0 ? ` AND ${scope.clause}` : ""} ORDER BY bm25(symbols_fts) LIMIT ?`;
|
|
238
|
+
const params = [match];
|
|
239
|
+
if (scope !== void 0 && rootDir !== void 0) {
|
|
240
|
+
params.push(...scope.params(rootDir));
|
|
241
|
+
}
|
|
242
|
+
params.push(limit);
|
|
243
|
+
const rows = db.prepare(sql).all(...params);
|
|
244
|
+
return rows.map(toSymbolEntry);
|
|
245
|
+
}
|
|
246
|
+
function searchSymbolsFts(query, limit = 50, dbPath = globalDbPath(), rootDir) {
|
|
247
|
+
const andMatch = sanitizeFtsQuery(query, "AND");
|
|
248
|
+
if (andMatch === "") return [];
|
|
249
|
+
const db = getDb(dbPath);
|
|
250
|
+
const scope = rootDir !== void 0 ? projectScopeClause("s.file_path") : void 0;
|
|
251
|
+
try {
|
|
252
|
+
const andResults = runFtsQuery(db, andMatch, limit, scope, rootDir);
|
|
253
|
+
if (andResults.length > 0) return andResults;
|
|
254
|
+
const orMatch = sanitizeFtsQuery(query, "OR");
|
|
255
|
+
if (orMatch === andMatch) return andResults;
|
|
256
|
+
return runFtsQuery(db, orMatch, limit, scope, rootDir);
|
|
257
|
+
} catch {
|
|
258
|
+
return [];
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export {
|
|
263
|
+
pathEqClause,
|
|
264
|
+
projectScopeClause,
|
|
265
|
+
DEFAULT_QUERY_LIMIT,
|
|
266
|
+
querySymbols,
|
|
267
|
+
distinctSymbolKinds,
|
|
268
|
+
countSymbols,
|
|
269
|
+
queryRefs,
|
|
270
|
+
countRefs,
|
|
271
|
+
queryRefsByContext,
|
|
272
|
+
queryRefCounts,
|
|
273
|
+
getProjectFileEntries,
|
|
274
|
+
getFileEntry,
|
|
275
|
+
sanitizeFtsQuery,
|
|
276
|
+
searchSymbolsFts
|
|
277
|
+
};
|