token-goat 2.9.14 → 2.9.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +15 -1
  2. package/dist/{token-goat-chunk-2ESBO4IN.mjs → token-goat-chunk-2IEIBVVD.mjs} +16 -9
  3. package/dist/{token-goat-chunk-RUDOKYPJ.mjs → token-goat-chunk-2PCFAQAS.mjs} +6582 -6194
  4. package/dist/{token-goat-chunk-AMYCQJX4.mjs → token-goat-chunk-3H6FQT62.mjs} +49 -256
  5. package/dist/{token-goat-chunk-PRJVGIC5.mjs → token-goat-chunk-4DC6DDIB.mjs} +7 -5
  6. package/dist/token-goat-chunk-4EXFN2AW.mjs +29 -0
  7. package/dist/{token-goat-chunk-VZYD4OZB.mjs → token-goat-chunk-5USHO7B6.mjs} +416 -4168
  8. package/dist/token-goat-chunk-63QWJ6UM.mjs +277 -0
  9. package/dist/token-goat-chunk-BFNTLGVA.mjs +4547 -0
  10. package/dist/token-goat-chunk-BGHSSJUA.mjs +102 -0
  11. package/dist/{token-goat-chunk-T2IWWTHB.mjs → token-goat-chunk-DMQ32ZSR.mjs} +35 -1142
  12. package/dist/{token-goat-chunk-DBNY4RLN.mjs → token-goat-chunk-EJCY6KHO.mjs} +5 -3
  13. package/dist/token-goat-chunk-ERTXEKB6.mjs +228 -0
  14. package/dist/{token-goat-chunk-Z6UXPYJA.mjs → token-goat-chunk-F2ARFHLJ.mjs} +1 -1
  15. package/dist/{token-goat-chunk-EG3663UT.mjs → token-goat-chunk-GIMGT34T.mjs} +4 -2
  16. package/dist/{token-goat-chunk-XVZ4MNQC.mjs → token-goat-chunk-GMXBQS6O.mjs} +360 -116
  17. package/dist/{token-goat-chunk-SAQ5PG4L.mjs → token-goat-chunk-H3DXI6VD.mjs} +8 -4
  18. package/dist/token-goat-chunk-LJL6FGDS.mjs +35 -0
  19. package/dist/{token-goat-chunk-NEI4NC54.mjs → token-goat-chunk-LKSXAMJB.mjs} +7 -2
  20. package/dist/token-goat-chunk-LMFO66YD.mjs +331 -0
  21. package/dist/{token-goat-chunk-7OGKZ7AP.mjs → token-goat-chunk-M2R4UQ3B.mjs} +5 -3
  22. package/dist/{token-goat-chunk-YHGTGG6K.mjs → token-goat-chunk-MA5BZ2KI.mjs} +68 -7
  23. package/dist/{token-goat-chunk-ASVVF4JV.mjs → token-goat-chunk-MIL62GMN.mjs} +171 -23
  24. package/dist/{token-goat-chunk-ZFM4PWXL.mjs → token-goat-chunk-NAMH252H.mjs} +10 -7
  25. package/dist/{token-goat-chunk-YTQHZJXW.mjs → token-goat-chunk-NOBZV7PM.mjs} +96 -22
  26. package/dist/{token-goat-chunk-4SDX3QP3.mjs → token-goat-chunk-NQZCSVN2.mjs} +13 -5
  27. package/dist/{token-goat-chunk-XEH6KBWW.mjs → token-goat-chunk-OCI25AXY.mjs} +4 -3
  28. package/dist/token-goat-chunk-OSUFN2FV.mjs +326 -0
  29. package/dist/{token-goat-chunk-O5WAMISC.mjs → token-goat-chunk-QMZMHQRR.mjs} +29 -18
  30. package/dist/{token-goat-chunk-POBYR64E.mjs → token-goat-chunk-R3X5TCPB.mjs} +801 -3
  31. package/dist/token-goat-chunk-RRNZMM3A.mjs +3364 -0
  32. package/dist/{token-goat-chunk-PGGDW7DZ.mjs → token-goat-chunk-ULA4WCED.mjs} +9 -6
  33. package/dist/token-goat-chunk-UMXD2M7O.mjs +975 -0
  34. package/dist/{token-goat-chunk-3XQPEJMV.mjs → token-goat-chunk-UXHDBVUR.mjs} +1 -1
  35. package/dist/{token-goat-chunk-EZNVAIR3.mjs → token-goat-chunk-VVIYEFBI.mjs} +4 -3
  36. package/dist/token-goat-chunk-XOHTFRDH.mjs +10760 -0
  37. package/dist/{token-goat-chunk-HF6H7RNK.mjs → token-goat-chunk-YPS64ACE.mjs} +13 -3
  38. package/dist/{token-goat-chunk-DAMXYVIW.mjs → token-goat-chunk-Z5VPAXTC.mjs} +2560 -820
  39. package/dist/{token-goat-chunk-XQF5J25J.mjs → token-goat-chunk-ZEHFYJ7Z.mjs} +5 -4
  40. package/dist/token-goat-hook.mjs +15 -10
  41. package/dist/token-goat.core.mjs +24 -18
  42. package/docs/cli.md +8 -6
  43. package/package.json +1 -1
  44. package/dist/token-goat-chunk-IT6O3PNN.mjs +0 -15463
  45. package/dist/token-goat-chunk-ZD4EM4LR.mjs +0 -30
  46. package/dist/token-goat-chunk-ZYNNQ36L.mjs +0 -479
@@ -1,40 +1,28 @@
1
1
  import { createRequire as __cjsRequire } from 'node:module';
2
2
  const require = __cjsRequire(import.meta.url);
3
+ import {
4
+ getDb
5
+ } from "./token-goat-chunk-R3X5TCPB.mjs";
3
6
  import {
4
7
  getHarnessName,
5
8
  loadConfig
6
- } from "./token-goat-chunk-SAQ5PG4L.mjs";
9
+ } from "./token-goat-chunk-H3DXI6VD.mjs";
7
10
  import {
8
11
  registerReset
9
12
  } from "./token-goat-chunk-EEIDFMEM.mjs";
10
13
  import {
11
- _detectOpenQuote,
12
- _lineClosesQuote,
13
- isAblSource,
14
- isLatexClassFile,
15
- isMatlabSource,
16
- isObjcHeader,
17
- isObjcSource,
18
- isPascalSource,
19
- isPerlSource,
20
- isPrologSource
21
- } from "./token-goat-chunk-POBYR64E.mjs";
22
- import {
23
- SYMBOL_BODY_CHAR_CAP,
24
14
  VERSION,
25
15
  atomicWriteText,
26
16
  countNoun,
27
17
  dataDir,
28
18
  dataDirForHome,
29
- displaySafeText,
30
19
  ensureDirSync,
31
- foldCase,
32
- foldPath,
33
- safeJoin,
34
20
  sanitizeIdForFilename,
35
- sleepSync,
36
21
  tokenGoatHome
37
- } from "./token-goat-chunk-AMYCQJX4.mjs";
22
+ } from "./token-goat-chunk-3H6FQT62.mjs";
23
+ import {
24
+ displaySafeText
25
+ } from "./token-goat-chunk-ERTXEKB6.mjs";
38
26
  import {
39
27
  growsExponentially,
40
28
  hasNestedQuantifier
@@ -43,1087 +31,6 @@ import {
43
31
  init_define_import_meta_env
44
32
  } from "./token-goat-chunk-A37V4PBF.mjs";
45
33
 
46
- // src/parser_types.ts
47
- init_define_import_meta_env();
48
- import * as fs from "node:fs";
49
- import * as path from "node:path";
50
-
51
- // src/language_specs.ts
52
- init_define_import_meta_env();
53
- var CODE = { symbolBearing: true, sourceHints: true, grepSource: true, diffable: true };
54
- var DATA = { symbolBearing: false, sourceHints: false, grepSource: false, diffable: false };
55
- var LANGUAGE_SPECS = [
56
- { id: "typescript", extraction: "tree-sitter", extensions: [".ts", ".tsx", ".mts", ".cts"], ...CODE, fence: "typescript", fenceByExtension: { ".tsx": "tsx" } },
57
- { id: "javascript", extraction: "tree-sitter", extensions: [".js", ".jsx", ".mjs", ".cjs"], ...CODE, fence: "javascript", fenceByExtension: { ".jsx": "jsx" } },
58
- // Bazel and Starlark (`.bzl`, `.star`, BUILD/WORKSPACE/MODULE.bazel) are a Python dialect: `def` and top-level calls such as `load(...)` parse with the Python grammar (https://github.com/bazelbuild/starlark/blob/master/spec.md).
59
- { id: "python", extraction: "tree-sitter", extensions: [".py", ".pyi", ".bzl", ".star"], basenames: ["build.bazel", "workspace.bazel", "module.bazel"], exactBasenames: ["BUILD", "WORKSPACE"], ...CODE, fence: "python" },
60
- { id: "go", extraction: "tree-sitter", extensions: [".go"], ...CODE, fence: "go" },
61
- { id: "rust", extraction: "tree-sitter", extensions: [".rs"], ...CODE, fence: "rust" },
62
- // Rake task files, Gemfile, Rakefile and the other extensionless Ruby DSL files are plain Ruby syntax.
63
- { id: "ruby", extraction: "tree-sitter", extensions: [".rb", ".ruby", ".rake"], basenames: ["gemfile", "rakefile", "vagrantfile", "guardfile", "podfile", "capfile", "fastfile", "brewfile"], ...CODE, fence: "ruby" },
64
- { id: "java", extraction: "tree-sitter", extensions: [".java"], ...CODE, fence: "java" },
65
- // `.h` is C unless it declares an Objective-C `@interface` or `@protocol` (refineLanguageByContent); a C++ header parses with the C grammar.
66
- { id: "c", extraction: "tree-sitter", extensions: [".c", ".h"], label: "C", ...CODE, fence: "c" },
67
- { id: "cpp", extraction: "tree-sitter", extensions: [".cpp", ".cc", ".cxx", ".hpp", ".hxx"], label: "C++", ...CODE, fence: "cpp" },
68
- // zsh, ksh and bats scripts share the POSIX `name() {` and `function name {` function forms the bash adapter reads.
69
- { id: "bash", extraction: "regex", extensions: [".sh", ".bash", ".zsh", ".ksh", ".bats"], ...DATA, symbolBearing: true, fence: "bash" },
70
- // MDX headings are plain ATX; `.rst` needs an underline heading parser this adapter lacks, so it stays unmapped.
71
- { id: "markdown", extraction: "regex", extensions: [".md", ".markdown", ".mdx"], ...DATA, fence: "markdown" },
72
- { id: "toml", extraction: "regex", extensions: [".toml"], basenames: ["cargo.toml", "pyproject.toml"], label: "TOML", ...DATA, diffable: true, fence: "toml" },
73
- // JSON with comments (`.jsonc`) and Avro schemas (`.avsc`) read with the JSON adapter, which skips `//` and `/* */` comments.
74
- { id: "json", extraction: "regex", extensions: [".json", ".jsonc", ".avsc"], basenames: ["package.json", "tsconfig.json"], label: "JSON", ...DATA, diffable: true, fence: "json" },
75
- { id: "yaml", extraction: "regex", extensions: [".yaml", ".yml"], label: "YAML", ...DATA, diffable: true, fence: "yaml" },
76
- { id: "css", extraction: "regex", extensions: [".css", ".scss", ".sass", ".less"], label: "CSS", ...DATA, grepSource: true, diffable: true, fence: "css", fenceByExtension: { ".scss": "scss", ".sass": "sass", ".less": "less" } },
77
- { id: "dockerfile", extraction: "regex", extensions: [], basenames: ["dockerfile"], ...DATA, fence: "dockerfile" },
78
- { id: "csharp", extraction: "regex", extensions: [".cs"], label: "C#", ...CODE, fence: "csharp" },
79
- { id: "php", extraction: "regex", extensions: [".php"], label: "PHP", ...CODE, fence: "php" },
80
- { id: "html", extraction: "regex", extensions: [".html", ".htm"], label: "HTML", ...DATA, fence: "html" },
81
- { id: "liquid", extraction: "regex", extensions: [".liquid"], ...DATA, fence: "liquid" },
82
- // Six template dialects, each masking its own delimiters out then handing off to the HTML
83
- // extractor (src/languages/templates_idx.ts) -- markup formats, so DATA defaults like html/liquid.
84
- { id: "jinja2", extraction: "regex", extensions: [".j2", ".jinja", ".jinja2"], label: "Jinja2", ...DATA, fence: "jinja" },
85
- { id: "handlebars", extraction: "regex", extensions: [".hbs", ".handlebars"], label: "Handlebars", ...DATA, fence: "handlebars" },
86
- { id: "erb", extraction: "regex", extensions: [".erb"], label: "ERB", ...DATA, fence: "erb" },
87
- { id: "ejs", extraction: "regex", extensions: [".ejs"], label: "EJS", ...DATA, fence: "ejs" },
88
- { id: "nunjucks", extraction: "regex", extensions: [".njk"], label: "Nunjucks", ...DATA, fence: "html" },
89
- { id: "twig", extraction: "regex", extensions: [".twig"], label: "Twig", ...DATA, fence: "twig" },
90
- { id: "kotlin", extraction: "regex", extensions: [".kt", ".kts"], ...CODE, fence: "kotlin" },
91
- { id: "swift", extraction: "regex", extensions: [".swift"], ...CODE, fence: "swift" },
92
- { id: "scala", extraction: "regex", extensions: [".scala", ".sc"], ...CODE, fence: "scala" },
93
- { id: "lua", extraction: "regex", extensions: [".lua"], ...CODE, fence: "lua" },
94
- { id: "elixir", extraction: "regex", extensions: [".ex", ".exs"], ...CODE, fence: "elixir" },
95
- { id: "dart", extraction: "regex", extensions: [".dart"], ...CODE, fence: "dart" },
96
- { id: "zig", extraction: "regex", extensions: [".zig"], ...CODE, fence: "zig" },
97
- { id: "r", extraction: "regex", extensions: [".r"], ...CODE, fence: "r" },
98
- { id: "graphql", extraction: "regex", extensions: [".graphql", ".gql"], label: "GraphQL", ...DATA, symbolBearing: true, fence: "graphql" },
99
- // Oracle PL/SQL sources: package spec and body, standalone procedure/function, trigger, and object type spec and body.
100
- { id: "sql", extraction: "regex", extensions: [".sql", ".pks", ".pkb", ".pls", ".plsql", ".pck", ".prc", ".fnc", ".trg", ".tps", ".tpb"], label: "SQL", ...DATA, symbolBearing: true, diffable: true, fence: "sql" },
101
- { id: "ini", extraction: "regex", extensions: [".ini", ".cfg", ".conf"], label: "INI", ...DATA, fence: "ini" },
102
- // `.mk` fragments (config.mk, rules.mk) share a bare Makefile's syntax.
103
- { id: "makefile", extraction: "regex", extensions: [".mk"], basenames: ["makefile", "gnumakefile", "bsdmakefile"], ...DATA, fence: "makefile", basenameImportsExtension: ".mk" },
104
- { id: "proto", extraction: "regex", extensions: [".proto"], label: "Protocol Buffers", ...DATA, symbolBearing: true, fence: "protobuf" },
105
- { id: "terraform", extraction: "regex", extensions: [".tf", ".tfvars", ".hcl"], ...DATA, symbolBearing: true, fence: "hcl" },
106
- // `.env` and `.env.<suffix>` are matched by parser_types.ts's DOTENV_VARIANT_RE before this table.
107
- { id: "env_file", extraction: "regex", extensions: [".env"], basenames: [".envrc"], label: "env file", ...DATA },
108
- { id: "powershell", extraction: "regex", extensions: [".ps1", ".psm1"], label: "PowerShell", ...CODE, fence: "powershell" },
109
- // VB.NET, VB6/VBA standard modules, VBScript and VB6 forms. A VB6 class module shares `.cls` with Apex (and LaTeX) and is told apart by content in refineLanguageByContent.
110
- { id: "vb", extraction: "regex", extensions: [".vb", ".bas", ".vbs", ".frm"], label: "Visual Basic", ...CODE, fence: "vb", fenceByExtension: { ".vb": "vbnet", ".vbs": "vbscript" } },
111
- {
112
- id: "cobol",
113
- extraction: "own-result",
114
- extensions: [".cbl", ".cob", ".cobol", ".cpy"],
115
- label: "COBOL",
116
- ...CODE,
117
- fence: "cobol",
118
- partialRefsReason: "PERFORM, GO TO and CALL 'literal' are recorded as references, but a paragraph also runs by falling through from the one above it and a program can be called through a data item holding its name, so `dead` skips COBOL"
119
- },
120
- // Natural object sources as NaturalONE and SYSOBJH export them. Maps (.nsm) and DDMs (.nsd) are layouts, not code, and stay unmapped.
121
- {
122
- id: "natural",
123
- extraction: "own-result",
124
- extensions: [".nsp", ".nsn", ".nss", ".nsa", ".nsl", ".nsg", ".nsc", ".nsh"],
125
- label: "Natural",
126
- ...CODE,
127
- fence: "natural",
128
- partialRefsReason: "PERFORM, CALLNAT 'literal' and FETCH 'literal' are recorded as references, but an object can also be called through a variable holding its name, so `dead` skips Natural"
129
- },
130
- { id: "abap", extraction: "regex", extensions: [".abap"], label: "ABAP", ...CODE, fence: "abap" },
131
- { id: "sas", extraction: "regex", extensions: [".sas"], label: "SAS", ...CODE, fence: "sas" },
132
- { id: "pli", extraction: "regex", extensions: [".pli", ".pl1"], label: "PL/I", ...CODE, fence: "pli" },
133
- // `.rpg` stays unmapped: RPG II and RPG III sources use it, and their fixed layout predates the ILE RPG forms this adapter reads.
134
- { id: "rpg", extraction: "regex", extensions: [".rpgle", ".sqlrpgle"], label: "RPG", ...CODE, fence: "rpgle" },
135
- { id: "jcl", extraction: "regex", extensions: [".jcl"], label: "JCL", ...CODE, fence: "jcl" },
136
- // `.mm` is always Objective-C++. A `.m` (MATLAB uses it too) is Objective-C only on an `#import`, `@interface`, `@implementation` or `@protocol` line, and a `.h` only on `@interface` or `@protocol`: refineLanguageByContent in parser_types.ts decides.
137
- { id: "objc", extraction: "regex", extensions: [".mm"], label: "Objective-C", ...CODE, fence: "objectivec" },
138
- // Gradle build scripts and Jenkinsfiles are Groovy.
139
- { id: "groovy", extraction: "regex", extensions: [".groovy", ".gvy", ".gradle"], basenames: ["jenkinsfile"], label: "Groovy", ...CODE, fence: "groovy", basenameImportsExtension: ".groovy" },
140
- // A Prolog `.pl` stays unknown (refineLanguageByContent), and a `.t` is Perl only on a Perl marker line.
141
- { id: "perl", extraction: "regex", extensions: [".pl", ".pm"], label: "Perl", ...CODE, fence: "perl" },
142
- { id: "solidity", extraction: "regex", extensions: [".sol"], label: "Solidity", ...CODE, fence: "solidity" },
143
- { id: "thrift", extraction: "regex", extensions: [".thrift"], label: "Thrift", ...DATA, symbolBearing: true, fence: "thrift" },
144
- { id: "glsl", extraction: "regex", extensions: [".glsl", ".vert", ".frag", ".comp", ".geom", ".tesc", ".tese"], label: "GLSL", ...CODE, fence: "glsl" },
145
- // `.fx` stays unmapped: other languages use it too.
146
- { id: "hlsl", extraction: "regex", extensions: [".hlsl", ".hlsli"], label: "HLSL", ...CODE, fence: "hlsl" },
147
- { id: "wgsl", extraction: "regex", extensions: [".wgsl"], label: "WGSL", ...CODE, fence: "wgsl" },
148
- { id: "metal", extraction: "regex", extensions: [".metal"], label: "Metal", ...CODE, fence: "metal" },
149
- // `.f`, `.for` and `.f77` are read as fixed form unless code starts in column 1. `.fpp` stays unmapped: it is used for both forms.
150
- { id: "fortran", extraction: "regex", extensions: [".f", ".for", ".f77", ".f90", ".f95", ".f03", ".f08"], label: "Fortran", ...CODE, fence: "fortran" },
151
- // `.inc` stays unmapped (many languages use it), and a `.pp` (Puppet uses it too) is Pascal only on a unit, program or library header: refineLanguageByContent in parser_types.ts decides. Text-form `.dfm` forms list their components.
152
- { id: "pascal", extraction: "regex", extensions: [".pas", ".dpr", ".dpk", ".lpr", ".dfm"], label: "Pascal", ...CODE, fence: "pascal" },
153
- // MATLAB has no extension of its own: a `.m` that is not Objective-C is MATLAB only on a `function` or `classdef` header line, which refineLanguageByContent in parser_types.ts checks, so a Mathematica or Mercury `.m` stays unknown.
154
- { id: "matlab", extraction: "regex", extensions: [], label: "MATLAB", ...CODE, fence: "matlab" },
155
- { id: "cmake", extraction: "regex", extensions: [".cmake"], basenames: ["cmakelists.txt"], label: "CMake", ...CODE, fence: "cmake", basenameImportsExtension: ".cmake" },
156
- // One adapter for the three assembly dialects that share these extensions: GNU as (`.s`, and `.S` through the lowercase lookup), NASM (`.asm`, `.nasm`) and IBM High Level Assembler (`.asm`), which the adapter tells apart by content. `.inc` stays unmapped: many languages use it.
157
- { id: "asm", extraction: "regex", extensions: [".s", ".asm", ".nasm"], label: "Assembly", ...CODE, fence: "asm" },
158
- { id: "batch", extraction: "regex", extensions: [".bat", ".cmd"], label: "Windows batch", ...CODE, fence: "batch" },
159
- { id: "erlang", extraction: "regex", extensions: [".erl", ".hrl"], label: "Erlang", ...CODE, fence: "erlang" },
160
- { id: "vhdl", extraction: "regex", extensions: [".vhd", ".vhdl"], label: "VHDL", ...CODE, fence: "vhdl" },
161
- // Five Lisp-family dialects, each its own row: none shares an extractor (see common_lisp.ts's
162
- // module doc for why their lexical rules stay separate rather than a single Lisp masker).
163
- { id: "common_lisp", extraction: "regex", extensions: [".lisp", ".lsp", ".cl"], label: "Common Lisp", ...CODE, fence: "lisp" },
164
- { id: "scheme", extraction: "regex", extensions: [".scm", ".ss"], label: "Scheme", ...CODE, fence: "scheme" },
165
- { id: "racket", extraction: "regex", extensions: [".rkt", ".rktl"], label: "Racket", ...CODE, fence: "racket" },
166
- { id: "clojure", extraction: "regex", extensions: [".clj", ".cljs", ".cljc"], label: "Clojure", ...CODE, fence: "clojure" },
167
- { id: "emacs_lisp", extraction: "regex", extensions: [".el"], label: "Emacs Lisp", ...CODE, fence: "lisp" },
168
- // `.lhs` (literate Haskell) is deliberately not claimed here -- see haskell.ts's module doc.
169
- { id: "haskell", extraction: "regex", extensions: [".hs"], label: "Haskell", ...CODE, fence: "haskell" },
170
- // `.mli` interface files share `.ml`'s lexical rules (comments, strings, quoted strings) and are read with the same extractor -- see ocaml.ts's module doc.
171
- { id: "ocaml", extraction: "regex", extensions: [".ml", ".mli"], label: "OCaml", ...CODE, fence: "ocaml" },
172
- // `.fsi` signature files and `.fsx` scripts share `.fs`'s lexical rules and are read with the same extractor -- see fsharp.ts's module doc.
173
- { id: "fsharp", extraction: "regex", extensions: [".fs", ".fsi", ".fsx"], label: "F#", ...CODE, fence: "fsharp" },
174
- { id: "nix", extraction: "regex", extensions: [".nix"], label: "Nix", ...CODE, fence: "nix" },
175
- // OpenEdge ABL has no extension of its own: a `.p` or `.w` (Pascal and CWEB use them too) or a `.cls` (Apex, VB6, LaTeX) is ABL only when its head carries an ABL marker, which refineLanguageByContent in parser_types.ts checks. The path-only hooks see a `.p` or `.w` as unknown and a `.cls` as Apex.
176
- { id: "abl", extraction: "regex", extensions: [], label: "OpenEdge ABL", ...CODE, fence: "abl" },
177
- { id: "apex", extraction: "regex", extensions: [".cls", ".trigger"], label: "Apex", ...CODE, fence: "apex" },
178
- // Matched by the `-meta.xml` suffix in detectLanguage, not by an extension.
179
- { id: "salesforce_metadata", extraction: "regex", extensions: [], label: "Salesforce metadata", ...DATA, symbolBearing: true, sourceHints: true, grepSource: true, fence: "xml" },
180
- { id: "salesforce_markup", extraction: "own-result", extensions: [".cmp", ".app", ".evt", ".intf", ".design", ".auradoc", ".tokens", ".page", ".component", ".email"], label: "Salesforce markup", ...DATA, symbolBearing: true, sourceHints: true, grepSource: true, fence: "xml" },
181
- { id: "vue", extraction: "own-result", extensions: [".vue"], ...DATA, fence: "vue" },
182
- { id: "svelte", extraction: "own-result", extensions: [".svelte"], ...DATA, fence: "svelte" },
183
- { id: "astro", extraction: "own-result", extensions: [".astro"], ...DATA, fence: "astro" },
184
- // Notebooks index through their code cells as Python, in parser.ts's ipynb branch.
185
- { id: "ipynb", extraction: "own-result", extensions: [".ipynb"], label: "Jupyter notebook", ...DATA, fence: "json" }
186
- ];
187
- var SPEC_BY_ID = new Map(LANGUAGE_SPECS.map((s) => [s.id, s]));
188
- function rows() {
189
- return LANGUAGE_SPECS;
190
- }
191
- var EXTENSION_LANGUAGE = new Map(
192
- LANGUAGE_SPECS.flatMap((s) => s.extensions.map((e) => [e, s.id]))
193
- );
194
- var FILENAME_LANGUAGE = new Map(
195
- rows().flatMap((s) => (s.basenames ?? []).map((b) => [b, s.id]))
196
- );
197
- var EXACT_FILENAME_LANGUAGE = new Map(
198
- rows().flatMap((s) => (s.exactBasenames ?? []).map((b) => [b, s.id]))
199
- );
200
- var TREE_SITTER_LANGUAGES = LANGUAGE_SPECS.filter((s) => s.extraction === "tree-sitter").map((s) => s.id);
201
- function languageHasFlag(language, flag) {
202
- return SPEC_BY_ID.get(language)?.[flag] === true;
203
- }
204
- function languageLabel(language) {
205
- if (language === "unknown") return "this file type";
206
- return SPEC_BY_ID.get(language)?.label ?? language;
207
- }
208
- function fenceFor(language, ext) {
209
- const spec = SPEC_BY_ID.get(language);
210
- if (spec === void 0) return "";
211
- return spec.fenceByExtension?.[ext] ?? spec.fence ?? "";
212
- }
213
- function basenameImportsExtension(language) {
214
- return SPEC_BY_ID.get(language)?.basenameImportsExtension;
215
- }
216
- function partialRefsReason(language) {
217
- return SPEC_BY_ID.get(language)?.partialRefsReason;
218
- }
219
-
220
- // src/parser_types.ts
221
- var DOTENV_VARIANT_RE = /^\.env(\..+)?$/;
222
- var VB6_HEADER_SCAN_LINES = 40;
223
- function isVb6ClassModule(content) {
224
- if (content.includes("\0")) return false;
225
- const lines = (content.charCodeAt(0) === 65279 ? content.slice(1) : content).split(/\r?\n/, VB6_HEADER_SCAN_LINES);
226
- const first = lines.find((l) => l.trim() !== "");
227
- if (first !== void 0 && /^VERSION\s+1\.0\s+CLASS\b/i.test(first.trim())) return true;
228
- return lines.some((l) => /^Attribute\s+VB_Name\s*=\s*"/i.test(l.trim()));
229
- }
230
- function refineLanguageByContent(filePath, language, content) {
231
- const sniff = CONTENT_SNIFFS.get(path.extname(filePath).toLowerCase());
232
- if (sniff === void 0 || sniff.from !== language) return language;
233
- return sniff.refine(content) ?? language;
234
- }
235
- var LANGUAGE_SNIFF_BYTES = 8192;
236
- function sniffHead(content) {
237
- const head = content.slice(0, LANGUAGE_SNIFF_BYTES);
238
- return Buffer.byteLength(head, "utf8") <= LANGUAGE_SNIFF_BYTES ? head : Buffer.from(head, "utf8").subarray(0, LANGUAGE_SNIFF_BYTES).toString("utf8");
239
- }
240
- var ablOrUnknown = (c) => isAblSource(c) ? "abl" : void 0;
241
- var clsRefine = (c) => {
242
- if (isVb6ClassModule(c)) return "vb";
243
- const abl = ablOrUnknown(c);
244
- if (abl !== void 0) return abl;
245
- if (isLatexClassFile(c)) return "unknown";
246
- return void 0;
247
- };
248
- var CONTENT_SNIFFS = /* @__PURE__ */ new Map([
249
- [".cls", { from: "apex", refine: clsRefine }],
250
- [".p", { from: "unknown", refine: ablOrUnknown }],
251
- [".w", { from: "unknown", refine: ablOrUnknown }],
252
- [".m", { from: "unknown", refine: (c) => isObjcSource(sniffHead(c)) ? "objc" : isMatlabSource(sniffHead(c)) ? "matlab" : void 0 }],
253
- [".pp", { from: "unknown", refine: (c) => isPascalSource(sniffHead(c)) ? "pascal" : void 0 }],
254
- [".h", { from: "c", refine: (c) => isObjcHeader(sniffHead(c)) ? "objc" : void 0 }],
255
- [".pl", { from: "perl", refine: (c) => isPrologSource(sniffHead(c)) ? "unknown" : void 0 }],
256
- [".t", { from: "unknown", refine: (c) => isPerlSource(sniffHead(c)) ? "perl" : void 0 }]
257
- ]);
258
- function needsContentSniff(filePath, language) {
259
- return CONTENT_SNIFFS.get(path.extname(filePath).toLowerCase())?.from === language;
260
- }
261
- function detectLanguageOfFile(filePath) {
262
- const language = detectLanguage(filePath);
263
- if (!needsContentSniff(filePath, language)) return language;
264
- try {
265
- const fd = fs.openSync(filePath, "r");
266
- try {
267
- const buf = Buffer.alloc(LANGUAGE_SNIFF_BYTES);
268
- const n = fs.readSync(fd, buf, 0, buf.length, 0);
269
- return refineLanguageByContent(filePath, language, buf.subarray(0, n).toString("utf8"));
270
- } finally {
271
- fs.closeSync(fd);
272
- }
273
- } catch {
274
- return language;
275
- }
276
- }
277
- function detectLanguage(filePath) {
278
- const exactBase = path.basename(filePath);
279
- const base = exactBase.toLowerCase();
280
- if (DOTENV_VARIANT_RE.test(base)) return "env_file";
281
- const byName = EXACT_FILENAME_LANGUAGE.get(exactBase) ?? FILENAME_LANGUAGE.get(base);
282
- if (byName !== void 0) return byName;
283
- if (base.endsWith("-meta.xml")) {
284
- return "salesforce_metadata";
285
- }
286
- const ext = path.extname(base).toLowerCase();
287
- return EXTENSION_LANGUAGE.get(ext) ?? "unknown";
288
- }
289
- var UNSUPPORTED_LANGUAGE_EXTENSIONS = /* @__PURE__ */ new Map([
290
- [".rpg", "RPG II or RPG III"],
291
- [".nsm", "Natural map"],
292
- [".nsd", "Natural DDM"]
293
- ]);
294
- function nonTreeSitterLanguageCount() {
295
- return LANGUAGE_SPECS.filter((s) => s.extraction !== "tree-sitter" && s.id !== "ipynb").length;
296
- }
297
- function unsupportedLanguageName(filePath) {
298
- const ext = path.extname(filePath).toLowerCase();
299
- return UNSUPPORTED_LANGUAGE_EXTENSIONS.get(ext);
300
- }
301
-
302
- // src/dotenv_redact.ts
303
- init_define_import_meta_env();
304
- var DOTENV_VALUE_PLACEHOLDER = "[REDACTED:dotenv_value]";
305
- var ASSIGNMENT_RE = /^(\s*(?:export\s+)?[A-Za-z_][\w.-]*\s*(?:\+?=|:(?!\/\/)))/;
306
- var SAFE_LINE_RE = /^\s*(?:[#;].*)?$/;
307
- function isDotenvPath(filePath) {
308
- return detectLanguage(filePath) === "env_file";
309
- }
310
- function redactDotenvValues(text) {
311
- const lines = text.split("\n");
312
- const out = [];
313
- let openQuote = null;
314
- for (const raw of lines) {
315
- const hasCr = raw.endsWith("\r");
316
- const line = hasCr ? raw.slice(0, -1) : raw;
317
- const eol = hasCr ? "\r" : "";
318
- const emit = (s) => {
319
- out.push(`${s}${eol}`);
320
- };
321
- if (openQuote !== null) {
322
- if (_lineClosesQuote(line, openQuote)) openQuote = null;
323
- emit(DOTENV_VALUE_PLACEHOLDER);
324
- continue;
325
- }
326
- if (SAFE_LINE_RE.test(line)) {
327
- emit(line);
328
- continue;
329
- }
330
- const m = ASSIGNMENT_RE.exec(line);
331
- if (m === null || m[1] === void 0) {
332
- emit(DOTENV_VALUE_PLACEHOLDER);
333
- continue;
334
- }
335
- const prefix = m[1];
336
- openQuote = _detectOpenQuote(line.slice(prefix.length));
337
- emit(`${prefix}${DOTENV_VALUE_PLACEHOLDER}`);
338
- }
339
- return out.join("\n");
340
- }
341
- function redactIfDotenv(filePath, text) {
342
- return isDotenvPath(filePath) ? redactDotenvValues(text) : text;
343
- }
344
-
345
- // src/db.ts
346
- init_define_import_meta_env();
347
- import * as fs3 from "node:fs";
348
- import { createRequire as createRequire2 } from "node:module";
349
- import * as path2 from "node:path";
350
-
351
- // src/sqlite_driver.ts
352
- init_define_import_meta_env();
353
- import * as fs2 from "node:fs";
354
- import { createRequire } from "node:module";
355
- var _require = createRequire(import.meta.url);
356
- function suppressSqliteExperimentalWarning() {
357
- const original = process.emit;
358
- let armed = true;
359
- const restore = () => {
360
- if (!armed) return;
361
- armed = false;
362
- process.emit = original;
363
- };
364
- process.emit = function patched(name, ...rest) {
365
- const data = rest[0];
366
- if (armed && name === "warning" && data instanceof Error && data.name === "ExperimentalWarning" && /sqlite/i.test(data.message)) {
367
- restore();
368
- return false;
369
- }
370
- return original.call(this, name, ...rest);
371
- };
372
- setImmediate(restore);
373
- return restore;
374
- }
375
- var restoreWarnings = suppressSqliteExperimentalWarning();
376
- var nodeSqlite;
377
- try {
378
- nodeSqlite = _require("node:sqlite");
379
- } catch (e) {
380
- restoreWarnings();
381
- throw e;
382
- }
383
- var { DatabaseSync } = nodeSqlite;
384
- var SQLITE_PRIMARY_CODES = [
385
- "SQLITE_OK",
386
- "SQLITE_ERROR",
387
- "SQLITE_INTERNAL",
388
- "SQLITE_PERM",
389
- "SQLITE_ABORT",
390
- "SQLITE_BUSY",
391
- "SQLITE_LOCKED",
392
- "SQLITE_NOMEM",
393
- "SQLITE_READONLY",
394
- "SQLITE_INTERRUPT",
395
- "SQLITE_IOERR",
396
- "SQLITE_CORRUPT",
397
- "SQLITE_NOTFOUND",
398
- "SQLITE_FULL",
399
- "SQLITE_CANTOPEN",
400
- "SQLITE_PROTOCOL",
401
- "SQLITE_EMPTY",
402
- "SQLITE_SCHEMA",
403
- "SQLITE_TOOBIG",
404
- "SQLITE_CONSTRAINT",
405
- "SQLITE_MISMATCH",
406
- "SQLITE_MISUSE",
407
- "SQLITE_NOLFS",
408
- "SQLITE_AUTH",
409
- "SQLITE_FORMAT",
410
- "SQLITE_RANGE",
411
- "SQLITE_NOTADB",
412
- "SQLITE_NOTICE",
413
- "SQLITE_WARNING"
414
- ];
415
- var SQLITE_EXTENDED_SUFFIXES = {
416
- SQLITE_OK: ["LOAD_PERMANENTLY", "SYMLINK"],
417
- SQLITE_ERROR: ["MISSING_COLLSEQ", "RETRY", "SNAPSHOT"],
418
- SQLITE_ABORT: [null, "ROLLBACK"],
419
- SQLITE_BUSY: ["RECOVERY", "SNAPSHOT", "TIMEOUT"],
420
- SQLITE_LOCKED: ["SHAREDCACHE", "VTAB"],
421
- SQLITE_READONLY: ["RECOVERY", "CANTLOCK", "ROLLBACK", "DBMOVED", "CANTINIT", "DIRECTORY"],
422
- SQLITE_IOERR: [
423
- "READ",
424
- "SHORT_READ",
425
- "WRITE",
426
- "FSYNC",
427
- "DIR_FSYNC",
428
- "TRUNCATE",
429
- "FSTAT",
430
- "UNLOCK",
431
- "RDLOCK",
432
- "DELETE",
433
- "BLOCKED",
434
- "NOMEM",
435
- "ACCESS",
436
- "CHECKRESERVEDLOCK",
437
- "LOCK",
438
- "CLOSE",
439
- "DIR_CLOSE",
440
- "SHMOPEN",
441
- "SHMSIZE",
442
- "SHMLOCK",
443
- "SHMMAP",
444
- "SEEK",
445
- "DELETE_NOENT",
446
- "MMAP",
447
- "GETTEMPPATH",
448
- "CONVPATH",
449
- "VNODE",
450
- "AUTH",
451
- "BEGIN_ATOMIC",
452
- "COMMIT_ATOMIC",
453
- "ROLLBACK_ATOMIC",
454
- "DATA",
455
- "CORRUPTFS",
456
- "IN_PAGE"
457
- ],
458
- SQLITE_CORRUPT: ["VTAB", "SEQUENCE", "INDEX"],
459
- SQLITE_CANTOPEN: ["NOTEMPDIR", "ISDIR", "FULLPATH", "CONVPATH", "DIRTYWAL", "SYMLINK"],
460
- SQLITE_CONSTRAINT: [
461
- "CHECK",
462
- "COMMITHOOK",
463
- "FOREIGNKEY",
464
- "FUNCTION",
465
- "NOTNULL",
466
- "PRIMARYKEY",
467
- "TRIGGER",
468
- "UNIQUE",
469
- "VTAB",
470
- "ROWID",
471
- "PINNED",
472
- "DATATYPE"
473
- ],
474
- SQLITE_AUTH: ["USER"],
475
- SQLITE_NOTICE: ["RECOVER_WAL", "RECOVER_ROLLBACK", "RBU"],
476
- SQLITE_WARNING: ["AUTOINDEX"]
477
- };
478
- function sqliteResultCodeName(errcode) {
479
- if (!Number.isInteger(errcode) || errcode < 0) return "ERR_SQLITE_ERROR";
480
- if (errcode === 100) return "SQLITE_ROW";
481
- if (errcode === 101) return "SQLITE_DONE";
482
- const primary = SQLITE_PRIMARY_CODES[errcode & 255];
483
- if (primary === void 0) return "ERR_SQLITE_ERROR";
484
- const subcode = errcode >> 8;
485
- if (subcode === 0) return primary;
486
- const suffix = SQLITE_EXTENDED_SUFFIXES[primary]?.[subcode - 1];
487
- return suffix === void 0 || suffix === null ? primary : `${primary}_${suffix}`;
488
- }
489
- function attempt(fn) {
490
- try {
491
- return fn();
492
- } catch (e) {
493
- const err = e;
494
- if (err.code === "ERR_SQLITE_ERROR" && typeof err.errcode === "number") {
495
- err.code = sqliteResultCodeName(err.errcode);
496
- }
497
- throw e;
498
- }
499
- }
500
- var Statement = class {
501
- #stmt;
502
- #pluck = false;
503
- constructor(stmt) {
504
- this.#stmt = stmt;
505
- }
506
- get source() {
507
- return this.#stmt.sourceSQL;
508
- }
509
- /**
510
- * better-sqlite3's `reader` flag: does this statement return rows?
511
- *
512
- * `node:sqlite` has no equivalent, so it is derived from the prepared statement's own column
513
- * count -- SQLite gives a row-producing statement its result columns at prepare time and gives a
514
- * non-producing one none. That is a derivation, and this is the third defence-in-depth layer in
515
- * `sqlite_query.ts`'s read-only guard, so it is not taken on faith: the driver tests run both
516
- * libraries side by side over SELECT, a CTE, VALUES, EXPLAIN, an empty-result SELECT, INSERT,
517
- * UPDATE, DELETE, CREATE, a reading PRAGMA and an assigning PRAGMA, and require every verdict to
518
- * agree. If a future SQLite statement form ever breaks the equivalence, that test fails rather
519
- * than the guard quietly weakening.
520
- */
521
- get reader() {
522
- return attempt(() => this.#stmt.columns()).length > 0;
523
- }
524
- // A plucked row is "the first column", which for an object row means the first *inserted* key. V8 preserves insertion order for string keys, and node:sqlite builds the row by walking the result columns left to right, so Object.values()[0] is the leftmost column -- not the column named in the SQL text, the same rule better-sqlite3 applies.
525
- #shape(row) {
526
- if (!this.#pluck || row === void 0 || row === null) return row;
527
- const values = Object.values(row);
528
- return values.length === 0 ? void 0 : values[0];
529
- }
530
- get(...params) {
531
- return this.#shape(attempt(() => this.#stmt.get(...params)));
532
- }
533
- all(...params) {
534
- const rows2 = attempt(() => this.#stmt.all(...params));
535
- return this.#pluck ? rows2.map((r) => this.#shape(r)) : rows2;
536
- }
537
- run(...params) {
538
- return attempt(() => this.#stmt.run(...params));
539
- }
540
- // Wrapped rather than returned directly so pluck applies lazily, one row at a time: the whole point of iterate() here is that sqlite_query.ts caps the row count without buffering the rest, and mapping the iterator through .all() first would defeat that.
541
- *iterate(...params) {
542
- const rows2 = attempt(() => this.#stmt.iterate(...params))[Symbol.iterator]();
543
- for (; ; ) {
544
- const next = attempt(() => rows2.next());
545
- if (next.done === true) return;
546
- yield this.#shape(next.value);
547
- }
548
- }
549
- pluck(toggle = true) {
550
- this.#pluck = toggle;
551
- return this;
552
- }
553
- safeIntegers(toggle = true) {
554
- this.#stmt.setReadBigInts(toggle);
555
- return this;
556
- }
557
- columns() {
558
- return attempt(() => this.#stmt.columns());
559
- }
560
- };
561
- var Database = class {
562
- #db;
563
- #path;
564
- #readonly;
565
- #savepoints = 0;
566
- constructor(dbPath, options = {}) {
567
- const wantsExisting = options.readonly === true || options.fileMustExist === true;
568
- if (wantsExisting && dbPath !== ":memory:" && !fs2.existsSync(dbPath)) {
569
- throw new Error("unable to open database file");
570
- }
571
- this.#db = attempt(() => new DatabaseSync(dbPath, {
572
- readOnly: options.readonly === true,
573
- // sqlite-vec is loaded through db.loadExtension by initConnection, which node:sqlite refuses unless the connection opted in at construction. Harmless when no extension is ever loaded.
574
- allowExtension: true,
575
- // better-sqlite3 opens every connection with busy_timeout at 5000ms; node:sqlite opens at 0, so a connection that named no timeout would silently go from five seconds of patience to none. db.ts overrides this to 15000 in initConnection, but sqlite_query.ts opens a user's arbitrary database readonly and takes whatever the default is -- which would have turned ordinary contention with another writer into an immediate "database is locked".
576
- timeout: options.timeout ?? 5e3
577
- }));
578
- this.#path = dbPath;
579
- this.#readonly = options.readonly === true;
580
- }
581
- get open() {
582
- return this.#db.isOpen;
583
- }
584
- get inTransaction() {
585
- return this.#db.isTransaction;
586
- }
587
- get readonly() {
588
- return this.#readonly;
589
- }
590
- get name() {
591
- return this.#path;
592
- }
593
- prepare(sql) {
594
- return new Statement(attempt(() => this.#db.prepare(sql)));
595
- }
596
- exec(sql) {
597
- attempt(() => this.#db.exec(sql));
598
- }
599
- pragma(source, options = {}) {
600
- const rows2 = attempt(() => this.#db.prepare(`PRAGMA ${source}`).all());
601
- if (options.simple !== true) return rows2;
602
- const first = rows2[0];
603
- if (first === void 0) return void 0;
604
- const values = Object.values(first);
605
- return values.length === 0 ? void 0 : values[0];
606
- }
607
- function(name, options, fn) {
608
- attempt(() => this.#db.function(name, options, fn));
609
- }
610
- loadExtension(extensionPath) {
611
- attempt(() => this.#db.loadExtension(extensionPath));
612
- }
613
- close() {
614
- attempt(() => this.#db.close());
615
- }
616
- /**
617
- * Wrap `fn` so it runs inside a transaction, committing on return and rolling back on throw.
618
- *
619
- * Nesting uses SAVEPOINT, which is what makes it safe for a transactional helper to call another
620
- * one: an inner `BEGIN` would throw ("cannot start a transaction within a transaction"), an inner
621
- * SAVEPOINT composes. Whether we are nested is read from SQLite via `isTransaction` rather than
622
- * tracked in a counter here, so a transaction some other code path opened still nests correctly.
623
- *
624
- * The rollback is best-effort and never replaces the caller's error: if the ROLLBACK itself fails
625
- * -- the connection died, the transaction was already unwound -- the original failure is still
626
- * what propagates, because that is the one that explains what went wrong.
627
- */
628
- transaction(fn) {
629
- const build = (beginSql) => (...args) => {
630
- if (this.#db.isTransaction) {
631
- const name = `tg_sp_${this.#savepoints++}`;
632
- attempt(() => this.#db.exec(`SAVEPOINT ${name}`));
633
- try {
634
- const result = fn(...args);
635
- attempt(() => this.#db.exec(`RELEASE ${name}`));
636
- return result;
637
- } catch (e) {
638
- try {
639
- attempt(() => this.#db.exec(`ROLLBACK TO ${name}`));
640
- attempt(() => this.#db.exec(`RELEASE ${name}`));
641
- } catch {
642
- }
643
- throw e;
644
- }
645
- }
646
- attempt(() => this.#db.exec(beginSql));
647
- try {
648
- const result = fn(...args);
649
- attempt(() => this.#db.exec("COMMIT"));
650
- return result;
651
- } catch (e) {
652
- try {
653
- attempt(() => this.#db.exec("ROLLBACK"));
654
- } catch {
655
- }
656
- throw e;
657
- }
658
- };
659
- const wrapped = build("BEGIN");
660
- wrapped.default = wrapped;
661
- wrapped.deferred = build("BEGIN");
662
- wrapped.immediate = build("BEGIN IMMEDIATE");
663
- wrapped.exclusive = build("BEGIN EXCLUSIVE");
664
- return wrapped;
665
- }
666
- };
667
-
668
- // src/db.ts
669
- var _require2 = createRequire2(import.meta.url);
670
- var _connections = /* @__PURE__ */ new Map();
671
- var SCHEMA_SQL = `
672
- CREATE TABLE IF NOT EXISTS files (
673
- path TEXT PRIMARY KEY,
674
- sha TEXT,
675
- mtime REAL,
676
- language TEXT,
677
- indexed_at REAL,
678
- embed_sha TEXT,
679
- retry_count INTEGER NOT NULL DEFAULT 0,
680
- parser_sha TEXT
681
- );
682
- -- Expression index on TG_LOWER(path) -- see pathEqClause (sql_path.ts) and TG_LOWER's
683
- -- registration above. TG_LOWER is registered { deterministic: true }, which is required for
684
- -- SQLite to index an expression at all; without it CREATE INDEX on a function call throws
685
- -- "non-deterministic functions prohibited in index expressions". Because pathEqClause emits
686
- -- this exact 'TG_LOWER(path) = ?' text for every case-insensitive-filesystem query, the planner
687
- -- matches it against this index and uses SEARCH instead of a full table SCAN, without requiring
688
- -- any writer to populate a separate folded column (verified via EXPLAIN QUERY PLAN in
689
- -- db.test.ts / sql_path.test.ts). CREATE INDEX IF NOT EXISTS is purely additive and safe to run
690
- -- against an already-populated table on every connection open, unlike an ALTER TABLE column add
691
- -- -- no MIGRATIONS entry or SCHEMA_VERSION bump is needed for this index.
692
- CREATE INDEX IF NOT EXISTS idx_files_path_folded ON files(TG_LOWER(path));
693
-
694
- CREATE TABLE IF NOT EXISTS symbols (
695
- id INTEGER PRIMARY KEY AUTOINCREMENT,
696
- file_path TEXT,
697
- name TEXT,
698
- kind TEXT,
699
- line_start INTEGER,
700
- line_end INTEGER,
701
- body TEXT,
702
- docstring TEXT,
703
- parent TEXT NOT NULL DEFAULT ''
704
- );
705
- CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);
706
- CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path);
707
- CREATE INDEX IF NOT EXISTS idx_symbols_name_kind ON symbols(name, kind);
708
- CREATE INDEX IF NOT EXISTS idx_symbols_file_folded ON symbols(TG_LOWER(file_path));
709
- CREATE INDEX IF NOT EXISTS idx_symbols_file_name_folded ON symbols(TG_LOWER(file_path), name);
710
- -- Partial index backing checkSymbolBodySize (cli_doctor.ts), which every SessionStart hook runs.
711
- -- Its predicate cannot be served by any index above, so the check had to read the whole symbols
712
- -- table -- 226 MB / 231324 rows here, 229 ms per session start, and the early-exit LIMIT 1 never
713
- -- fires on a healthy index because there is nothing to find. Indexing the *violating* rows only
714
- -- makes the check a lookup into a b-tree that is empty on a healthy index: measured 229 ms -> 0.0
715
- -- ms, 4 KB on disk, and no measurable insert cost (-0.2%, within noise, over 40000 real rows),
716
- -- because SQLite evaluates the predicate and skips the b-tree write for every row under the cap.
717
- -- SQLite uses a partial index only where the query's WHERE implies the index's, so the probe in
718
- -- cli_doctor.ts spells its comparison the same way and against the same constant. That makes the
719
- -- threshold part of the stored schema -- see SYMBOL_BODY_CHAR_CAP in constants.ts for what
720
- -- changing it requires. A query with a lower threshold correctly gets a full scan instead, so no
721
- -- other reader can be served stale rows by this index.
722
- CREATE INDEX IF NOT EXISTS idx_symbols_oversized_body ON symbols(id) WHERE LENGTH(body) > ${SYMBOL_BODY_CHAR_CAP};
723
-
724
- CREATE TABLE IF NOT EXISTS refs (
725
- id INTEGER PRIMARY KEY AUTOINCREMENT,
726
- file_path TEXT,
727
- name TEXT,
728
- line INTEGER,
729
- col INTEGER,
730
- context TEXT
731
- );
732
- CREATE INDEX IF NOT EXISTS idx_refs_name ON refs(name);
733
- CREATE INDEX IF NOT EXISTS idx_refs_file ON refs(file_path);
734
- CREATE INDEX IF NOT EXISTS idx_refs_file_folded ON refs(TG_LOWER(file_path));
735
- CREATE INDEX IF NOT EXISTS idx_refs_file_name_folded ON refs(TG_LOWER(file_path), name);
736
-
737
- CREATE TABLE IF NOT EXISTS chunks (
738
- id INTEGER PRIMARY KEY AUTOINCREMENT,
739
- file_path TEXT,
740
- start_line INTEGER,
741
- end_line INTEGER,
742
- text TEXT,
743
- kind TEXT
744
- );
745
- CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
746
- CREATE INDEX IF NOT EXISTS idx_chunks_file_folded ON chunks(TG_LOWER(file_path));
747
- CREATE INDEX IF NOT EXISTS idx_chunks_file_kind_folded ON chunks(TG_LOWER(file_path), kind);
748
-
749
- -- Tracks every project root a hook has ever seen an edit for, so the worker's periodic sweep
750
- -- (sweepKnownRoots in index_prune.ts) knows which roots to auto-prune without scanning the
751
- -- entire shared files table for distinct top-level directories on every cycle. Purely additive
752
- -- (no SCHEMA_VERSION bump needed): last_seen_ms is refreshed on every observed edit,
753
- -- first_missing_ms is set the first sweep that finds the root unreachable and cleared the
754
- -- moment it's seen reachable again -- see sweepKnownRoots' grace-period logic.
755
- CREATE TABLE IF NOT EXISTS known_roots (
756
- root TEXT PRIMARY KEY,
757
- last_seen_ms REAL NOT NULL,
758
- first_missing_ms REAL
759
- );
760
-
761
- -- Resume point for a budget-truncated reconcile sweep (reconcile.ts), one row per project root. A
762
- -- project too large to finish a sweep inside DEFAULT_RECONCILE_BUDGET_MS would otherwise scan the
763
- -- same deterministic (git ls-files) prefix every session forever, leaving every file after the
764
- -- budget cutoff permanently unchecked. last_scanned_path is the last tracked file the sweep
765
- -- finished examining before its budget ran out; the next sweep rotates its scan order to resume
766
- -- right after that file, wrapping back to the start, so repeated truncated sweeps eventually cover
767
- -- the whole project. Cleared (row deleted) the moment a sweep completes a full lap without running
768
- -- out of budget. Purely additive (no SCHEMA_VERSION bump needed): a missing row just means "start
769
- -- from the beginning", the same as a fresh database.
770
- CREATE TABLE IF NOT EXISTS reconcile_cursor (
771
- root TEXT PRIMARY KEY,
772
- last_scanned_path TEXT NOT NULL,
773
- updated_at REAL NOT NULL
774
- );
775
-
776
- -- Cross-cache full-text search index for 'token-goat recall' (recall_index.ts). One row
777
- -- per bash-output/web-output/mcp-output blob-store entry (see disk_cache.ts), refreshed
778
- -- in place (ON CONFLICT DO UPDATE) whenever storeBashOutput/storeWebOutput/storeMcpOutput
779
- -- write that entry, so recall never needs a separate rebuild step. row_id is a plain
780
- -- surrogate integer key -- entry_id is the real blob-store id (bash/mcp ids are hex,
781
- -- web ids are the cache's own scheme) and is not unique on its own since bash-output and
782
- -- mcp-output ids share one namespace (BASH_OUTPUT_SUBDIR) while web-output ids are a
783
- -- separate namespace; cache_type disambiguates.
784
- CREATE TABLE IF NOT EXISTS cache_recall (
785
- row_id INTEGER PRIMARY KEY AUTOINCREMENT,
786
- cache_type TEXT NOT NULL,
787
- entry_id TEXT NOT NULL,
788
- label TEXT,
789
- content TEXT,
790
- stored_at REAL,
791
- UNIQUE(cache_type, entry_id)
792
- );
793
- CREATE INDEX IF NOT EXISTS idx_cache_recall_type ON cache_recall(cache_type);
794
-
795
- -- Per-emission ledger for 'token-goat hint-stats' (hint_stats.ts). One row per hint
796
- -- emission event (a hook returning a 'context' HookOutput classified as a discretionary
797
- -- efficiency nudge, as opposed to a mandatory informational injection -- see hint_stats.ts's
798
- -- doc comment for the exact category list and what is deliberately excluded). correlator is a
799
- -- best-effort file-path/output-id substring extracted from the hint's own text, used to check
800
- -- whether a later Bash tool call in the same session actually followed the hint's specific
801
- -- pointer (see resolvePendingHintsForEvent) -- NULL when no such pointer could be extracted,
802
- -- in which case the row is inserted already resolved with acted_on=0 (counted as emitted, never
803
- -- eligible for auto-detected credit). calls_remaining is the countdown of subsequent tool-use
804
- -- events still eligible to resolve this row before it is considered timed out.
805
- CREATE TABLE IF NOT EXISTS hint_emissions (
806
- id INTEGER PRIMARY KEY AUTOINCREMENT,
807
- category TEXT NOT NULL,
808
- session_id TEXT NOT NULL,
809
- harness TEXT NOT NULL,
810
- correlator TEXT,
811
- emitted_at REAL NOT NULL,
812
- resolved INTEGER NOT NULL DEFAULT 0,
813
- acted_on INTEGER NOT NULL DEFAULT 0,
814
- calls_remaining INTEGER NOT NULL DEFAULT 0,
815
- bytes_emitted INTEGER
816
- );
817
- CREATE INDEX IF NOT EXISTS idx_hint_emissions_category ON hint_emissions(category);
818
- CREATE INDEX IF NOT EXISTS idx_hint_emissions_session_pending ON hint_emissions(session_id, resolved);
819
-
820
- -- Manual efficacy votes for a hint category (token-goat hint-stats --mark-effective/--mark-ineffective),
821
- -- kept separate from hint_emissions' automatic acted_on signal so the two are never silently
822
- -- blended -- see hint_stats.ts's doc comment on why some categories only support this manual signal.
823
- CREATE TABLE IF NOT EXISTS hint_manual_marks (
824
- category TEXT PRIMARY KEY,
825
- effective_count INTEGER NOT NULL DEFAULT 0,
826
- ineffective_count INTEGER NOT NULL DEFAULT 0
827
- );
828
-
829
- -- Durable counter backing hint_stats.ts's backoff-threshold probe-recovery schedule: how many
830
- -- CONSECUTIVE suppressed occasions have elapsed for (category, harness) since a hint in this
831
- -- category was last actually shown (either organically, because shouldSuppress no longer holds,
832
- -- or via a prior probe). shouldSuppress itself stays a pure function of hint_emissions -- this
833
- -- table exists only because a suppressed occasion is deliberately never written to
834
- -- hint_emissions (see that table's own comment), so without a separate durable counter here
835
- -- there would be no way to know "how many suppressed occasions have we seen" across the
836
- -- short-lived hook CLI processes that call applyHintTracking. Keyed by (category, harness), not
837
- -- category alone, to match shouldSuppress/categoryStats' own per-harness scoping -- unlike
838
- -- hint_manual_marks (a human-entered vote, deliberately not harness-split).
839
- CREATE TABLE IF NOT EXISTS hint_suppression_probes (
840
- category TEXT NOT NULL,
841
- harness TEXT NOT NULL,
842
- streak INTEGER NOT NULL DEFAULT 0,
843
- PRIMARY KEY (category, harness)
844
- );
845
-
846
- -- Free-text architecture/rationale notes (the "why" layer -- see notes.ts), attached either to
847
- -- a whole file (symbol = '') or to one specific indexed symbol within it (symbol = that
848
- -- symbol's name). '' rather than NULL for the whole-file case because SQLite's UNIQUE treats
849
- -- NULLs as pairwise-distinct (never conflicting with each other), which would let note-add
850
- -- accumulate unlimited duplicate whole-file notes for the same file instead of upserting one;
851
- -- '' is a real, comparable value so UNIQUE(file_path, symbol) enforces "at most one note per
852
- -- attachment point" for both cases identically. 'fingerprint' is a SHA-256 digest (see
853
- -- fingerprintContent in fingerprint.ts) captured at write time of exactly what the note
854
- -- describes -- the resolved symbol's current body text for a symbol-scoped note, or a stable
855
- -- digest of the file's current top-level symbol manifest (name:kind:line-range per symbol,
856
- -- sorted) for a file-scoped note -- so 'token-goat note-list --stale-only' can recompute the
857
- -- same fingerprint against the live index later and flag a mismatch (see notes.ts's
858
- -- isNoteStale). Staleness detection is purely advisory: nothing here ever auto-rewrites or
859
- -- deletes a note's content, only flags that the code it describes has moved since it was
860
- -- written -- a human/agent re-review decides what to do with a stale note.
861
- CREATE TABLE IF NOT EXISTS notes (
862
- id INTEGER PRIMARY KEY AUTOINCREMENT,
863
- file_path TEXT NOT NULL,
864
- symbol TEXT NOT NULL DEFAULT '',
865
- content TEXT NOT NULL,
866
- fingerprint TEXT NOT NULL,
867
- created_at REAL NOT NULL,
868
- updated_at REAL NOT NULL,
869
- UNIQUE(file_path, symbol)
870
- );
871
- CREATE INDEX IF NOT EXISTS idx_notes_file_folded ON notes(TG_LOWER(file_path));
872
-
873
- -- Baseline for skill_version_drift.ts's one-shot nudge: the token-goat CLI version (and its
874
- -- flat command-name set, JSON-encoded) active the moment the token-goat skill's body was
875
- -- last (re)loaded into this session -- see hooks_skill.ts's postSkillHandler. A session that
876
- -- keeps running after the CLI is upgraded has no other way to learn that new surgical-read
877
- -- commands now exist (the skill only re-announces itself on an explicit reload), so
878
- -- checkSkillVersionDrift compares this snapshot against the live command set on each user turn
879
- -- and fires the nudge exactly once (notified_at) per (re)load. session_id is the primary key
880
- -- because only one skill (token-goat) is ever tracked here.
881
- CREATE TABLE IF NOT EXISTS skill_version_snapshots (
882
- session_id TEXT PRIMARY KEY,
883
- skill_name TEXT NOT NULL,
884
- loaded_version TEXT NOT NULL,
885
- loaded_commands_json TEXT NOT NULL,
886
- notified_at REAL
887
- );
888
-
889
- -- Which embedding stack produced the vectors currently in chunk_vectors -- the model, its
890
- -- pinned revision, and the inference runtime (see embeddingProvenance in embeddings.ts). The
891
- -- vector table itself is vec0(rowid, embedding) and has nowhere to record this, so without
892
- -- this row a database that was embedded by one stack and then added to by another holds two
893
- -- incomparable sets of vectors under one index, with nothing able to tell them apart. That is
894
- -- not hypothetical: global.db is machine-wide across every project on the machine (see
895
- -- constants.ts), so upgrading the runtime, or changing the model or its pinned revision, mixes
896
- -- old and new vectors for as long as the old files go untouched. Measured drift between two
897
- -- runtime versions of the same quantized model is 0.9925-0.9978 cosine on the final vector --
898
- -- small, but enough to reorder near-ties, and invisible to every existing check.
899
- --
900
- -- Single-row by construction (the CHECK pins the key), because there is exactly one vector
901
- -- table per database. An EMPTY table on a database that already holds chunks means the vectors
902
- -- predate this stamp and their provenance is unknowable -- see ensureEmbeddingProvenance, which
903
- -- treats that exactly like a mismatch. That is what makes this work without a migration step.
904
- CREATE TABLE IF NOT EXISTS embedding_provenance (
905
- id INTEGER PRIMARY KEY CHECK (id = 1),
906
- provenance TEXT NOT NULL
907
- );
908
- `;
909
- var FTS_TOKENIZER = "unicode61 remove_diacritics 2";
910
- var FTS_SQL = `
911
- CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
912
- name,
913
- body,
914
- docstring,
915
- content='symbols',
916
- content_rowid='id',
917
- tokenize='${FTS_TOKENIZER}'
918
- );
919
- CREATE TRIGGER IF NOT EXISTS symbols_ai AFTER INSERT ON symbols BEGIN
920
- INSERT INTO symbols_fts(rowid, name, body, docstring)
921
- VALUES (new.id, new.name, new.body, new.docstring);
922
- END;
923
- CREATE TRIGGER IF NOT EXISTS symbols_ad AFTER DELETE ON symbols BEGIN
924
- INSERT INTO symbols_fts(symbols_fts, rowid, name, body, docstring)
925
- VALUES ('delete', old.id, old.name, old.body, old.docstring);
926
- END;
927
- CREATE TRIGGER IF NOT EXISTS symbols_au AFTER UPDATE ON symbols BEGIN
928
- INSERT INTO symbols_fts(symbols_fts, rowid, name, body, docstring)
929
- VALUES ('delete', old.id, old.name, old.body, old.docstring);
930
- INSERT INTO symbols_fts(rowid, name, body, docstring)
931
- VALUES (new.id, new.name, new.body, new.docstring);
932
- END;
933
-
934
- -- Content-linked FTS5 mirror of cache_recall (recall_index.ts), same shape as symbols_fts
935
- -- above. An INSERT ... ON CONFLICT DO UPDATE against cache_recall fires the AFTER UPDATE
936
- -- trigger (not AFTER INSERT) on the conflicting row, same as any other SQLite upsert, so the
937
- -- delete+reinsert pattern below keeps the fts index correct on a re-indexed (overwritten)
938
- -- entry, not just a brand-new one.
939
- CREATE VIRTUAL TABLE IF NOT EXISTS cache_recall_fts USING fts5(
940
- label,
941
- content,
942
- content='cache_recall',
943
- content_rowid='row_id',
944
- tokenize='${FTS_TOKENIZER}'
945
- );
946
- CREATE TRIGGER IF NOT EXISTS cache_recall_ai AFTER INSERT ON cache_recall BEGIN
947
- INSERT INTO cache_recall_fts(rowid, label, content)
948
- VALUES (new.row_id, new.label, new.content);
949
- END;
950
- CREATE TRIGGER IF NOT EXISTS cache_recall_ad AFTER DELETE ON cache_recall BEGIN
951
- INSERT INTO cache_recall_fts(cache_recall_fts, rowid, label, content)
952
- VALUES ('delete', old.row_id, old.label, old.content);
953
- END;
954
- CREATE TRIGGER IF NOT EXISTS cache_recall_au AFTER UPDATE ON cache_recall BEGIN
955
- INSERT INTO cache_recall_fts(cache_recall_fts, rowid, label, content)
956
- VALUES ('delete', old.row_id, old.label, old.content);
957
- INSERT INTO cache_recall_fts(rowid, label, content)
958
- VALUES (new.row_id, new.label, new.content);
959
- END;
960
- `;
961
- var SCHEMA_VERSION = 14;
962
- function alterTableIdempotent(conn, sql) {
963
- try {
964
- conn.exec(sql);
965
- } catch (err) {
966
- if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
967
- }
968
- }
969
- function purgeDotenvEmbeddings(conn) {
970
- let paths;
971
- try {
972
- paths = conn.prepare("SELECT DISTINCT file_path FROM chunks").all().map((r) => r.file_path).filter(isDotenvPath);
973
- } catch {
974
- return;
975
- }
976
- if (paths.length === 0) return;
977
- for (const p of paths) {
978
- try {
979
- conn.prepare("DELETE FROM chunk_vectors WHERE rowid IN (SELECT id FROM chunks WHERE file_path = ?)").run(p);
980
- } catch {
981
- }
982
- conn.prepare("DELETE FROM chunks WHERE file_path = ?").run(p);
983
- try {
984
- conn.prepare("UPDATE files SET embed_sha = NULL WHERE path = ?").run(p);
985
- } catch {
986
- }
987
- }
988
- }
989
- function rebuildFtsAtCurrentTokenizer(conn) {
990
- try {
991
- const declarations = conn.prepare("SELECT sql FROM sqlite_master WHERE name IN ('symbols_fts','cache_recall_fts')").all();
992
- const current = declarations.map((d) => /tokenize\s*=\s*'([^']*)'/.exec(d.sql ?? "")?.[1] ?? "");
993
- if (current.length === 2 && current.every((t) => t === FTS_TOKENIZER)) return;
994
- conn.transaction(() => {
995
- conn.exec("DROP TABLE IF EXISTS symbols_fts; DROP TABLE IF EXISTS cache_recall_fts;");
996
- conn.exec(FTS_SQL);
997
- conn.exec("INSERT INTO symbols_fts(symbols_fts) VALUES('rebuild');");
998
- conn.exec("INSERT INTO cache_recall_fts(cache_recall_fts) VALUES('rebuild');");
999
- }).immediate();
1000
- } catch {
1001
- }
1002
- }
1003
- var MIGRATIONS = {
1004
- // v1 -> v2: adds files.embed_sha, tracked separately from files.sha so embedding freshness can be gated independently of parse freshness (see makeIndexer in worker.ts). A pre-existing v1 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, so a genuine ALTER TABLE failure is never silently lost.
1005
- 1: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN embed_sha TEXT"),
1006
- // v2 -> v3: adds files.retry_count, a durable per-path counter for consecutive transient-read-failure requeues (see MAX_TRANSIENT_RETRIES / requeueDirtyPath / clearRetryCount in worker.ts). Previously this counter lived only in an in-memory Map inside worker.ts, which meant the retry-count reset -- run at the time in the short-lived hook CLI process -- could never actually reach the long-lived detached daemon process's own copy of that Map: they are different Node processes with no shared memory, so the reset was a silent no-op in the real deployed topology. Persisting the counter in `files` makes it visible to both processes via the one thing they do share: the index DB. Same swallow-duplicate-column pattern as v1 -> v2 above.
1007
- 2: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"),
1008
- // v8 -> v9: adds symbols.parent (see SCHEMA_VERSION comment above for why). A pre-existing v8 database's `symbols` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 above.
1009
- 8: (conn) => alterTableIdempotent(conn, "ALTER TABLE symbols ADD COLUMN parent TEXT NOT NULL DEFAULT ''"),
1010
- // v9 -> v10: adds hint_emissions.bytes_emitted (see SCHEMA_VERSION comment above for why). A pre-existing v9 database's `hint_emissions` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 above.
1011
- 9: (conn) => alterTableIdempotent(conn, "ALTER TABLE hint_emissions ADD COLUMN bytes_emitted INTEGER"),
1012
- // v10 -> v11: purge chunks (and their vectors) for dotenv files. Until this version, a tracked `.env` was chunked and embedded verbatim on the git path, so `semantic` returned its values -- see dotenv_redact.ts. Redacting from now on is not enough on its own: the embed-freshness gate (isEmbedFresh in parser.ts) skips a file whose bytes have not changed, so an already-indexed .env would have kept serving its pre-fix chunks indefinitely. Deleting the rows here both removes the stored secrets and, by clearing embed_sha, makes the next drain re-embed the file through the redacting path.
1013
- 10: purgeDotenvEmbeddings,
1014
- // v12 -> v13: adds files.parser_sha, the digest of the extraction logic that produced this file's rows, tracked separately from files.sha for the same reason embed_sha is -- content freshness and parse freshness are different questions, and the content sha alone could only ever answer the first. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 / v9 -> v10 above. Deliberately left NULL for every existing row rather than backfilled with the current fingerprint: NULL is the truthful answer (nobody recorded which parser wrote those rows), and it is also the answer that makes the freshness gates reparse them once, which is exactly what a database indexed by an older parser needs. v13 -> v14: changes both FTS5 tables' tokenizer to `unicode61 remove_diacritics 2`, so a search for `Noi` or `Viet` finds `Hà Nội` and `Việt Nam` -- combining marks that `remove_diacritics 1`, FTS5's default, leaves in place. This is the first schema change that `CREATE VIRTUAL TABLE IF NOT EXISTS` cannot express at all rather than merely cannot express on a populated table: against an existing virtual table that statement is a silent no-op, so without MIGRATIONS[13] the new tokenizer would reach only databases created after this release. The step drops both tables, re-runs FTS_SQL to re-create them at the current declaration, and rebuilds each from its content table.
1015
- 12: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN parser_sha TEXT"),
1016
- // v13 -> v14: re-creates both FTS5 tables at the tokenizer FTS_SQL currently declares (see the SCHEMA_VERSION comment above for why no `IF NOT EXISTS` form can do this).
1017
- 13: rebuildFtsAtCurrentTokenizer
1018
- };
1019
- function runMigrations(conn, fromVersion, toVersion) {
1020
- for (let v = fromVersion; v < toVersion; v++) {
1021
- MIGRATIONS[v]?.(conn);
1022
- }
1023
- }
1024
- var WAL_SWITCH_DEADLINE_MS = 15e3;
1025
- function enableWalWithRetry(conn, budgetMs = WAL_SWITCH_DEADLINE_MS) {
1026
- const deadline = Date.now() + budgetMs;
1027
- let lastError;
1028
- for (; ; ) {
1029
- try {
1030
- const mode = conn.pragma("journal_mode = WAL", { simple: true });
1031
- if (String(mode).toLowerCase() === "wal") return;
1032
- lastError = new Error(`got: ${String(mode)}`);
1033
- } catch (e) {
1034
- lastError = e;
1035
- }
1036
- try {
1037
- if (String(conn.pragma("journal_mode", { simple: true })).toLowerCase() === "wal") return;
1038
- } catch {
1039
- }
1040
- if (Date.now() >= deadline) {
1041
- throw new Error(`db: failed to enable WAL mode (${lastError instanceof Error ? lastError.message : String(lastError)})`);
1042
- }
1043
- sleepSync(25);
1044
- }
1045
- }
1046
- function initConnection(conn) {
1047
- conn.pragma("busy_timeout = 15000");
1048
- enableWalWithRetry(conn);
1049
- conn.pragma("synchronous = NORMAL");
1050
- conn.pragma("cache_size = -32000");
1051
- conn.pragma("temp_store = MEMORY");
1052
- conn.pragma("mmap_size = 134217728");
1053
- conn.function(
1054
- "TG_LOWER",
1055
- { deterministic: true },
1056
- (value) => value === null ? null : foldCase(String(value))
1057
- );
1058
- const storedVersion = Number(conn.pragma("user_version", { simple: true }));
1059
- if (storedVersion > SCHEMA_VERSION) {
1060
- throw new Error(
1061
- `db: index schema version ${storedVersion} is newer than this token-goat build supports (expected ${SCHEMA_VERSION}). Update token-goat, or delete the stale index database and let it rebuild.`
1062
- );
1063
- }
1064
- conn.exec(SCHEMA_SQL);
1065
- try {
1066
- conn.exec(FTS_SQL);
1067
- } catch {
1068
- }
1069
- try {
1070
- const sqliteVec = _require2("sqlite-vec");
1071
- sqliteVec.load(conn);
1072
- conn.exec(
1073
- `CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vectors USING vec0(
1074
- embedding float[384]
1075
- );`
1076
- );
1077
- } catch {
1078
- }
1079
- if (storedVersion < SCHEMA_VERSION) {
1080
- runMigrations(conn, storedVersion, SCHEMA_VERSION);
1081
- conn.pragma(`user_version = ${SCHEMA_VERSION}`);
1082
- }
1083
- }
1084
- function resolveDbPath(dbPath) {
1085
- if (path2.isAbsolute(dbPath)) return dbPath;
1086
- if (dbPath.includes("/") || dbPath.includes("\\")) return path2.resolve(dbPath);
1087
- return safeJoin(dataDir(), dbPath);
1088
- }
1089
- function connectionKey(dbPath) {
1090
- const resolved = resolveDbPath(dbPath);
1091
- return { resolved, key: foldPath(resolved) };
1092
- }
1093
- function getDb(dbPath) {
1094
- const { resolved, key } = connectionKey(dbPath);
1095
- const existing = _connections.get(key);
1096
- if (existing !== void 0) return existing;
1097
- const dir = path2.dirname(resolved);
1098
- try {
1099
- ensureDirSync(dir);
1100
- } catch (e) {
1101
- if (e.code !== "EEXIST" || !fs3.existsSync(dir)) throw e;
1102
- }
1103
- const conn = new Database(resolved);
1104
- try {
1105
- initConnection(conn);
1106
- } catch (e) {
1107
- try {
1108
- conn.close();
1109
- } catch {
1110
- }
1111
- throw e;
1112
- }
1113
- _connections.set(key, conn);
1114
- return conn;
1115
- }
1116
- function closeAllDbs() {
1117
- for (const conn of _connections.values()) {
1118
- try {
1119
- conn.close();
1120
- } catch {
1121
- }
1122
- }
1123
- _connections.clear();
1124
- }
1125
- registerReset(closeAllDbs);
1126
-
1127
34
  // src/render/ansi.ts
1128
35
  init_define_import_meta_env();
1129
36
  function _colorStream(isatty) {
@@ -1201,7 +108,7 @@ var C = {
1201
108
 
1202
109
  // src/stats.ts
1203
110
  init_define_import_meta_env();
1204
- import * as path3 from "node:path";
111
+ import * as path from "node:path";
1205
112
 
1206
113
  // src/render/stats_renderer.ts
1207
114
  init_define_import_meta_env();
@@ -1566,6 +473,7 @@ var _KIND_GROUPS = [
1566
473
  "gdrive_image",
1567
474
  "webfetch_image",
1568
475
  "image_shrink_skipped",
476
+ "image_shrink_over_pixel_limit",
1569
477
  "image_shrink_cache_hit",
1570
478
  "image_ocr"
1571
479
  ])
@@ -1965,6 +873,7 @@ var KIND_TO_SOURCE = {
1965
873
  image_shrink: SOURCE_IMAGE,
1966
874
  image_shrink_cache_hit: SOURCE_IMAGE,
1967
875
  image_shrink_skipped: SOURCE_IMAGE,
876
+ image_shrink_over_pixel_limit: SOURCE_IMAGE,
1968
877
  image_ocr: SOURCE_IMAGE,
1969
878
  webfetch_image: SOURCE_IMAGE,
1970
879
  gdrive_image: SOURCE_IMAGE,
@@ -2329,7 +1238,7 @@ function statsHasVersionColumn(db) {
2329
1238
  }
2330
1239
  function getGlobalDb(homeDir) {
2331
1240
  const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
2332
- const dbPath = path3.join(basePath, "global.db");
1241
+ const dbPath = path.join(basePath, "global.db");
2333
1242
  const db = getDb(dbPath);
2334
1243
  if (!_globalSchemaApplied.has(dbPath)) {
2335
1244
  db.exec(GLOBAL_SCHEMA_SQL);
@@ -2470,10 +1379,10 @@ function summarize(windowDays = 30, testDb, homeDir) {
2470
1379
  ].join(", ");
2471
1380
  const query = sinceTs !== null ? `SELECT ${cols} FROM stats WHERE ts >= ? ORDER BY ts DESC` : `SELECT ${cols} FROM stats ORDER BY ts DESC`;
2472
1381
  const stmt = db.prepare(query);
2473
- const rows2 = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
1382
+ const rows = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
2474
1383
  const tsToDateCache = {};
2475
1384
  const counts = {};
2476
- for (const row of rows2) {
1385
+ for (const row of rows) {
2477
1386
  const bytesSaved = row.bytes_saved ?? 0;
2478
1387
  const recorded = row.tokens_saved ?? 0;
2479
1388
  const kind = row.kind;
@@ -2895,23 +1804,23 @@ function redactSecrets(text, config = loadConfig()) {
2895
1804
 
2896
1805
  // src/disk_cache.ts
2897
1806
  init_define_import_meta_env();
2898
- import * as fs4 from "node:fs";
2899
- import * as path4 from "node:path";
1807
+ import * as fs from "node:fs";
1808
+ import * as path2 from "node:path";
2900
1809
  var DEFAULT_MAX_COUNT = 200;
2901
1810
  var DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
2902
1811
  function sanitizeId(id) {
2903
1812
  return sanitizeIdForFilename(id, 64);
2904
1813
  }
2905
1814
  function blobDir(subdir) {
2906
- return path4.join(tokenGoatHome(), subdir);
1815
+ return path2.join(tokenGoatHome(), subdir);
2907
1816
  }
2908
1817
  function blobPath(subdir, id) {
2909
1818
  const safe = sanitizeId(id);
2910
1819
  if (!safe) return null;
2911
1820
  const dir = blobDir(subdir);
2912
- const candidate = path4.join(dir, `${safe}.json`);
1821
+ const candidate = path2.join(dir, `${safe}.json`);
2913
1822
  try {
2914
- const rel = path4.relative(dir, candidate);
1823
+ const rel = path2.relative(dir, candidate);
2915
1824
  if (rel.startsWith("..")) return null;
2916
1825
  } catch {
2917
1826
  return null;
@@ -2922,7 +1831,7 @@ function isBlobStale(subdir, id) {
2922
1831
  const p = blobPath(subdir, id);
2923
1832
  if (p === null) return false;
2924
1833
  try {
2925
- const stat = fs4.statSync(p);
1834
+ const stat = fs.statSync(p);
2926
1835
  return Date.now() - stat.mtimeMs > DEFAULT_MAX_AGE_MS;
2927
1836
  } catch {
2928
1837
  return false;
@@ -2963,8 +1872,8 @@ function storeBlob(subdir, id, value, opts = {}) {
2963
1872
  }
2964
1873
  if (Number.isFinite(maxBytesPerItem) && Buffer.byteLength(json, "utf-8") > maxBytesPerItem) return false;
2965
1874
  try {
2966
- const dir = path4.dirname(p);
2967
- if (!fs4.existsSync(dir)) ensureDirSync(dir);
1875
+ const dir = path2.dirname(p);
1876
+ if (!fs.existsSync(dir)) ensureDirSync(dir);
2968
1877
  atomicWriteText(p, json);
2969
1878
  } catch {
2970
1879
  return false;
@@ -2982,8 +1891,8 @@ function loadBlob(subdir, id) {
2982
1891
  const p = blobPath(subdir, id);
2983
1892
  if (!p) return null;
2984
1893
  try {
2985
- if (!fs4.existsSync(p)) return null;
2986
- return JSON.parse(fs4.readFileSync(p, "utf8"));
1894
+ if (!fs.existsSync(p)) return null;
1895
+ return JSON.parse(fs.readFileSync(p, "utf8"));
2987
1896
  } catch {
2988
1897
  return null;
2989
1898
  }
@@ -2992,13 +1901,13 @@ function listBlobs(subdir) {
2992
1901
  const dir = blobDir(subdir);
2993
1902
  const out = [];
2994
1903
  try {
2995
- if (!fs4.existsSync(dir)) return out;
2996
- for (const file of fs4.readdirSync(dir)) {
1904
+ if (!fs.existsSync(dir)) return out;
1905
+ for (const file of fs.readdirSync(dir)) {
2997
1906
  if (!file.endsWith(".json")) continue;
2998
1907
  const id = file.slice(0, -5);
2999
1908
  let mtime = 0;
3000
1909
  try {
3001
- mtime = fs4.statSync(path4.join(dir, file)).mtimeMs;
1910
+ mtime = fs.statSync(path2.join(dir, file)).mtimeMs;
3002
1911
  } catch {
3003
1912
  }
3004
1913
  const value = loadBlob(subdir, id);
@@ -3015,15 +1924,15 @@ function pruneBlobs(subdir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX
3015
1924
  function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
3016
1925
  let removed = 0;
3017
1926
  try {
3018
- if (!fs4.existsSync(dir)) return 0;
1927
+ if (!fs.existsSync(dir)) return 0;
3019
1928
  const cutoff = Date.now() - maxAgeMs;
3020
1929
  let kept = [];
3021
1930
  let protectedEntry;
3022
- for (const file of fs4.readdirSync(dir)) {
3023
- const full = path4.join(dir, file);
1931
+ for (const file of fs.readdirSync(dir)) {
1932
+ const full = path2.join(dir, file);
3024
1933
  let stat;
3025
1934
  try {
3026
- stat = fs4.statSync(full);
1935
+ stat = fs.statSync(full);
3027
1936
  } catch {
3028
1937
  continue;
3029
1938
  }
@@ -3034,7 +1943,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
3034
1943
  }
3035
1944
  if (stat.mtimeMs < cutoff) {
3036
1945
  try {
3037
- fs4.unlinkSync(full);
1946
+ fs.unlinkSync(full);
3038
1947
  removed++;
3039
1948
  } catch {
3040
1949
  continue;
@@ -3050,7 +1959,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
3050
1959
  kept = kept.slice(kept.length - countBudget);
3051
1960
  for (const [full] of excess) {
3052
1961
  try {
3053
- fs4.unlinkSync(full);
1962
+ fs.unlinkSync(full);
3054
1963
  removed++;
3055
1964
  } catch {
3056
1965
  continue;
@@ -3065,7 +1974,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
3065
1974
  if (!oldest) break;
3066
1975
  const [full, , size] = oldest;
3067
1976
  try {
3068
- fs4.unlinkSync(full);
1977
+ fs.unlinkSync(full);
3069
1978
  removed++;
3070
1979
  total -= size;
3071
1980
  } catch {
@@ -3095,7 +2004,7 @@ function sweepCacheRoots(extraRoots = []) {
3095
2004
  try {
3096
2005
  const defaults = subdirCacheDefaults(subdir);
3097
2006
  removed += pruneBlobDir(
3098
- path4.join(root, subdir),
2007
+ path2.join(root, subdir),
3099
2008
  countCapped ? defaults.maxCount : Number.POSITIVE_INFINITY,
3100
2009
  DEFAULT_MAX_AGE_MS,
3101
2010
  countCapped ? defaults.maxBytes : Number.POSITIVE_INFINITY
@@ -3108,22 +2017,6 @@ function sweepCacheRoots(extraRoots = []) {
3108
2017
  }
3109
2018
 
3110
2019
  export {
3111
- Database,
3112
- FILENAME_LANGUAGE,
3113
- TREE_SITTER_LANGUAGES,
3114
- languageHasFlag,
3115
- languageLabel,
3116
- fenceFor,
3117
- basenameImportsExtension,
3118
- partialRefsReason,
3119
- refineLanguageByContent,
3120
- detectLanguageOfFile,
3121
- detectLanguage,
3122
- nonTreeSitterLanguageCount,
3123
- unsupportedLanguageName,
3124
- isDotenvPath,
3125
- redactIfDotenv,
3126
- getDb,
3127
2020
  colorStdout,
3128
2021
  RESET,
3129
2022
  stripAnsiEscapes,