token-goat 2.9.14 → 2.9.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -1
- package/dist/{token-goat-chunk-2ESBO4IN.mjs → token-goat-chunk-2IEIBVVD.mjs} +16 -9
- package/dist/{token-goat-chunk-RUDOKYPJ.mjs → token-goat-chunk-2PCFAQAS.mjs} +6582 -6194
- package/dist/{token-goat-chunk-AMYCQJX4.mjs → token-goat-chunk-3H6FQT62.mjs} +49 -256
- package/dist/{token-goat-chunk-PRJVGIC5.mjs → token-goat-chunk-4DC6DDIB.mjs} +7 -5
- package/dist/token-goat-chunk-4EXFN2AW.mjs +29 -0
- package/dist/{token-goat-chunk-VZYD4OZB.mjs → token-goat-chunk-5USHO7B6.mjs} +416 -4168
- package/dist/token-goat-chunk-63QWJ6UM.mjs +277 -0
- package/dist/token-goat-chunk-BFNTLGVA.mjs +4547 -0
- package/dist/token-goat-chunk-BGHSSJUA.mjs +102 -0
- package/dist/{token-goat-chunk-T2IWWTHB.mjs → token-goat-chunk-DMQ32ZSR.mjs} +35 -1142
- package/dist/{token-goat-chunk-DBNY4RLN.mjs → token-goat-chunk-EJCY6KHO.mjs} +5 -3
- package/dist/token-goat-chunk-ERTXEKB6.mjs +228 -0
- package/dist/{token-goat-chunk-Z6UXPYJA.mjs → token-goat-chunk-F2ARFHLJ.mjs} +1 -1
- package/dist/{token-goat-chunk-EG3663UT.mjs → token-goat-chunk-GIMGT34T.mjs} +4 -2
- package/dist/{token-goat-chunk-XVZ4MNQC.mjs → token-goat-chunk-GMXBQS6O.mjs} +360 -116
- package/dist/{token-goat-chunk-SAQ5PG4L.mjs → token-goat-chunk-H3DXI6VD.mjs} +8 -4
- package/dist/token-goat-chunk-LJL6FGDS.mjs +35 -0
- package/dist/{token-goat-chunk-NEI4NC54.mjs → token-goat-chunk-LKSXAMJB.mjs} +7 -2
- package/dist/token-goat-chunk-LMFO66YD.mjs +331 -0
- package/dist/{token-goat-chunk-7OGKZ7AP.mjs → token-goat-chunk-M2R4UQ3B.mjs} +5 -3
- package/dist/{token-goat-chunk-YHGTGG6K.mjs → token-goat-chunk-MA5BZ2KI.mjs} +68 -7
- package/dist/{token-goat-chunk-ASVVF4JV.mjs → token-goat-chunk-MIL62GMN.mjs} +171 -23
- package/dist/{token-goat-chunk-ZFM4PWXL.mjs → token-goat-chunk-NAMH252H.mjs} +10 -7
- package/dist/{token-goat-chunk-YTQHZJXW.mjs → token-goat-chunk-NOBZV7PM.mjs} +96 -22
- package/dist/{token-goat-chunk-4SDX3QP3.mjs → token-goat-chunk-NQZCSVN2.mjs} +13 -5
- package/dist/{token-goat-chunk-XEH6KBWW.mjs → token-goat-chunk-OCI25AXY.mjs} +4 -3
- package/dist/token-goat-chunk-OSUFN2FV.mjs +326 -0
- package/dist/{token-goat-chunk-O5WAMISC.mjs → token-goat-chunk-QMZMHQRR.mjs} +29 -18
- package/dist/{token-goat-chunk-POBYR64E.mjs → token-goat-chunk-R3X5TCPB.mjs} +801 -3
- package/dist/token-goat-chunk-RRNZMM3A.mjs +3364 -0
- package/dist/{token-goat-chunk-PGGDW7DZ.mjs → token-goat-chunk-ULA4WCED.mjs} +9 -6
- package/dist/token-goat-chunk-UMXD2M7O.mjs +975 -0
- package/dist/{token-goat-chunk-3XQPEJMV.mjs → token-goat-chunk-UXHDBVUR.mjs} +1 -1
- package/dist/{token-goat-chunk-EZNVAIR3.mjs → token-goat-chunk-VVIYEFBI.mjs} +4 -3
- package/dist/token-goat-chunk-XOHTFRDH.mjs +10760 -0
- package/dist/{token-goat-chunk-HF6H7RNK.mjs → token-goat-chunk-YPS64ACE.mjs} +13 -3
- package/dist/{token-goat-chunk-DAMXYVIW.mjs → token-goat-chunk-Z5VPAXTC.mjs} +2560 -820
- package/dist/{token-goat-chunk-XQF5J25J.mjs → token-goat-chunk-ZEHFYJ7Z.mjs} +5 -4
- package/dist/token-goat-hook.mjs +15 -10
- package/dist/token-goat.core.mjs +24 -18
- package/docs/cli.md +8 -6
- package/package.json +1 -1
- package/dist/token-goat-chunk-IT6O3PNN.mjs +0 -15463
- package/dist/token-goat-chunk-ZD4EM4LR.mjs +0 -30
- package/dist/token-goat-chunk-ZYNNQ36L.mjs +0 -479
|
@@ -1,40 +1,28 @@
|
|
|
1
1
|
import { createRequire as __cjsRequire } from 'node:module';
|
|
2
2
|
const require = __cjsRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
getDb
|
|
5
|
+
} from "./token-goat-chunk-R3X5TCPB.mjs";
|
|
3
6
|
import {
|
|
4
7
|
getHarnessName,
|
|
5
8
|
loadConfig
|
|
6
|
-
} from "./token-goat-chunk-
|
|
9
|
+
} from "./token-goat-chunk-H3DXI6VD.mjs";
|
|
7
10
|
import {
|
|
8
11
|
registerReset
|
|
9
12
|
} from "./token-goat-chunk-EEIDFMEM.mjs";
|
|
10
13
|
import {
|
|
11
|
-
_detectOpenQuote,
|
|
12
|
-
_lineClosesQuote,
|
|
13
|
-
isAblSource,
|
|
14
|
-
isLatexClassFile,
|
|
15
|
-
isMatlabSource,
|
|
16
|
-
isObjcHeader,
|
|
17
|
-
isObjcSource,
|
|
18
|
-
isPascalSource,
|
|
19
|
-
isPerlSource,
|
|
20
|
-
isPrologSource
|
|
21
|
-
} from "./token-goat-chunk-POBYR64E.mjs";
|
|
22
|
-
import {
|
|
23
|
-
SYMBOL_BODY_CHAR_CAP,
|
|
24
14
|
VERSION,
|
|
25
15
|
atomicWriteText,
|
|
26
16
|
countNoun,
|
|
27
17
|
dataDir,
|
|
28
18
|
dataDirForHome,
|
|
29
|
-
displaySafeText,
|
|
30
19
|
ensureDirSync,
|
|
31
|
-
foldCase,
|
|
32
|
-
foldPath,
|
|
33
|
-
safeJoin,
|
|
34
20
|
sanitizeIdForFilename,
|
|
35
|
-
sleepSync,
|
|
36
21
|
tokenGoatHome
|
|
37
|
-
} from "./token-goat-chunk-
|
|
22
|
+
} from "./token-goat-chunk-3H6FQT62.mjs";
|
|
23
|
+
import {
|
|
24
|
+
displaySafeText
|
|
25
|
+
} from "./token-goat-chunk-ERTXEKB6.mjs";
|
|
38
26
|
import {
|
|
39
27
|
growsExponentially,
|
|
40
28
|
hasNestedQuantifier
|
|
@@ -43,1087 +31,6 @@ import {
|
|
|
43
31
|
init_define_import_meta_env
|
|
44
32
|
} from "./token-goat-chunk-A37V4PBF.mjs";
|
|
45
33
|
|
|
46
|
-
// src/parser_types.ts
|
|
47
|
-
init_define_import_meta_env();
|
|
48
|
-
import * as fs from "node:fs";
|
|
49
|
-
import * as path from "node:path";
|
|
50
|
-
|
|
51
|
-
// src/language_specs.ts
|
|
52
|
-
init_define_import_meta_env();
|
|
53
|
-
var CODE = { symbolBearing: true, sourceHints: true, grepSource: true, diffable: true };
|
|
54
|
-
var DATA = { symbolBearing: false, sourceHints: false, grepSource: false, diffable: false };
|
|
55
|
-
var LANGUAGE_SPECS = [
|
|
56
|
-
{ id: "typescript", extraction: "tree-sitter", extensions: [".ts", ".tsx", ".mts", ".cts"], ...CODE, fence: "typescript", fenceByExtension: { ".tsx": "tsx" } },
|
|
57
|
-
{ id: "javascript", extraction: "tree-sitter", extensions: [".js", ".jsx", ".mjs", ".cjs"], ...CODE, fence: "javascript", fenceByExtension: { ".jsx": "jsx" } },
|
|
58
|
-
// Bazel and Starlark (`.bzl`, `.star`, BUILD/WORKSPACE/MODULE.bazel) are a Python dialect: `def` and top-level calls such as `load(...)` parse with the Python grammar (https://github.com/bazelbuild/starlark/blob/master/spec.md).
|
|
59
|
-
{ id: "python", extraction: "tree-sitter", extensions: [".py", ".pyi", ".bzl", ".star"], basenames: ["build.bazel", "workspace.bazel", "module.bazel"], exactBasenames: ["BUILD", "WORKSPACE"], ...CODE, fence: "python" },
|
|
60
|
-
{ id: "go", extraction: "tree-sitter", extensions: [".go"], ...CODE, fence: "go" },
|
|
61
|
-
{ id: "rust", extraction: "tree-sitter", extensions: [".rs"], ...CODE, fence: "rust" },
|
|
62
|
-
// Rake task files, Gemfile, Rakefile and the other extensionless Ruby DSL files are plain Ruby syntax.
|
|
63
|
-
{ id: "ruby", extraction: "tree-sitter", extensions: [".rb", ".ruby", ".rake"], basenames: ["gemfile", "rakefile", "vagrantfile", "guardfile", "podfile", "capfile", "fastfile", "brewfile"], ...CODE, fence: "ruby" },
|
|
64
|
-
{ id: "java", extraction: "tree-sitter", extensions: [".java"], ...CODE, fence: "java" },
|
|
65
|
-
// `.h` is C unless it declares an Objective-C `@interface` or `@protocol` (refineLanguageByContent); a C++ header parses with the C grammar.
|
|
66
|
-
{ id: "c", extraction: "tree-sitter", extensions: [".c", ".h"], label: "C", ...CODE, fence: "c" },
|
|
67
|
-
{ id: "cpp", extraction: "tree-sitter", extensions: [".cpp", ".cc", ".cxx", ".hpp", ".hxx"], label: "C++", ...CODE, fence: "cpp" },
|
|
68
|
-
// zsh, ksh and bats scripts share the POSIX `name() {` and `function name {` function forms the bash adapter reads.
|
|
69
|
-
{ id: "bash", extraction: "regex", extensions: [".sh", ".bash", ".zsh", ".ksh", ".bats"], ...DATA, symbolBearing: true, fence: "bash" },
|
|
70
|
-
// MDX headings are plain ATX; `.rst` needs an underline heading parser this adapter lacks, so it stays unmapped.
|
|
71
|
-
{ id: "markdown", extraction: "regex", extensions: [".md", ".markdown", ".mdx"], ...DATA, fence: "markdown" },
|
|
72
|
-
{ id: "toml", extraction: "regex", extensions: [".toml"], basenames: ["cargo.toml", "pyproject.toml"], label: "TOML", ...DATA, diffable: true, fence: "toml" },
|
|
73
|
-
// JSON with comments (`.jsonc`) and Avro schemas (`.avsc`) read with the JSON adapter, which skips `//` and `/* */` comments.
|
|
74
|
-
{ id: "json", extraction: "regex", extensions: [".json", ".jsonc", ".avsc"], basenames: ["package.json", "tsconfig.json"], label: "JSON", ...DATA, diffable: true, fence: "json" },
|
|
75
|
-
{ id: "yaml", extraction: "regex", extensions: [".yaml", ".yml"], label: "YAML", ...DATA, diffable: true, fence: "yaml" },
|
|
76
|
-
{ id: "css", extraction: "regex", extensions: [".css", ".scss", ".sass", ".less"], label: "CSS", ...DATA, grepSource: true, diffable: true, fence: "css", fenceByExtension: { ".scss": "scss", ".sass": "sass", ".less": "less" } },
|
|
77
|
-
{ id: "dockerfile", extraction: "regex", extensions: [], basenames: ["dockerfile"], ...DATA, fence: "dockerfile" },
|
|
78
|
-
{ id: "csharp", extraction: "regex", extensions: [".cs"], label: "C#", ...CODE, fence: "csharp" },
|
|
79
|
-
{ id: "php", extraction: "regex", extensions: [".php"], label: "PHP", ...CODE, fence: "php" },
|
|
80
|
-
{ id: "html", extraction: "regex", extensions: [".html", ".htm"], label: "HTML", ...DATA, fence: "html" },
|
|
81
|
-
{ id: "liquid", extraction: "regex", extensions: [".liquid"], ...DATA, fence: "liquid" },
|
|
82
|
-
// Six template dialects, each masking its own delimiters out then handing off to the HTML
|
|
83
|
-
// extractor (src/languages/templates_idx.ts) -- markup formats, so DATA defaults like html/liquid.
|
|
84
|
-
{ id: "jinja2", extraction: "regex", extensions: [".j2", ".jinja", ".jinja2"], label: "Jinja2", ...DATA, fence: "jinja" },
|
|
85
|
-
{ id: "handlebars", extraction: "regex", extensions: [".hbs", ".handlebars"], label: "Handlebars", ...DATA, fence: "handlebars" },
|
|
86
|
-
{ id: "erb", extraction: "regex", extensions: [".erb"], label: "ERB", ...DATA, fence: "erb" },
|
|
87
|
-
{ id: "ejs", extraction: "regex", extensions: [".ejs"], label: "EJS", ...DATA, fence: "ejs" },
|
|
88
|
-
{ id: "nunjucks", extraction: "regex", extensions: [".njk"], label: "Nunjucks", ...DATA, fence: "html" },
|
|
89
|
-
{ id: "twig", extraction: "regex", extensions: [".twig"], label: "Twig", ...DATA, fence: "twig" },
|
|
90
|
-
{ id: "kotlin", extraction: "regex", extensions: [".kt", ".kts"], ...CODE, fence: "kotlin" },
|
|
91
|
-
{ id: "swift", extraction: "regex", extensions: [".swift"], ...CODE, fence: "swift" },
|
|
92
|
-
{ id: "scala", extraction: "regex", extensions: [".scala", ".sc"], ...CODE, fence: "scala" },
|
|
93
|
-
{ id: "lua", extraction: "regex", extensions: [".lua"], ...CODE, fence: "lua" },
|
|
94
|
-
{ id: "elixir", extraction: "regex", extensions: [".ex", ".exs"], ...CODE, fence: "elixir" },
|
|
95
|
-
{ id: "dart", extraction: "regex", extensions: [".dart"], ...CODE, fence: "dart" },
|
|
96
|
-
{ id: "zig", extraction: "regex", extensions: [".zig"], ...CODE, fence: "zig" },
|
|
97
|
-
{ id: "r", extraction: "regex", extensions: [".r"], ...CODE, fence: "r" },
|
|
98
|
-
{ id: "graphql", extraction: "regex", extensions: [".graphql", ".gql"], label: "GraphQL", ...DATA, symbolBearing: true, fence: "graphql" },
|
|
99
|
-
// Oracle PL/SQL sources: package spec and body, standalone procedure/function, trigger, and object type spec and body.
|
|
100
|
-
{ id: "sql", extraction: "regex", extensions: [".sql", ".pks", ".pkb", ".pls", ".plsql", ".pck", ".prc", ".fnc", ".trg", ".tps", ".tpb"], label: "SQL", ...DATA, symbolBearing: true, diffable: true, fence: "sql" },
|
|
101
|
-
{ id: "ini", extraction: "regex", extensions: [".ini", ".cfg", ".conf"], label: "INI", ...DATA, fence: "ini" },
|
|
102
|
-
// `.mk` fragments (config.mk, rules.mk) share a bare Makefile's syntax.
|
|
103
|
-
{ id: "makefile", extraction: "regex", extensions: [".mk"], basenames: ["makefile", "gnumakefile", "bsdmakefile"], ...DATA, fence: "makefile", basenameImportsExtension: ".mk" },
|
|
104
|
-
{ id: "proto", extraction: "regex", extensions: [".proto"], label: "Protocol Buffers", ...DATA, symbolBearing: true, fence: "protobuf" },
|
|
105
|
-
{ id: "terraform", extraction: "regex", extensions: [".tf", ".tfvars", ".hcl"], ...DATA, symbolBearing: true, fence: "hcl" },
|
|
106
|
-
// `.env` and `.env.<suffix>` are matched by parser_types.ts's DOTENV_VARIANT_RE before this table.
|
|
107
|
-
{ id: "env_file", extraction: "regex", extensions: [".env"], basenames: [".envrc"], label: "env file", ...DATA },
|
|
108
|
-
{ id: "powershell", extraction: "regex", extensions: [".ps1", ".psm1"], label: "PowerShell", ...CODE, fence: "powershell" },
|
|
109
|
-
// VB.NET, VB6/VBA standard modules, VBScript and VB6 forms. A VB6 class module shares `.cls` with Apex (and LaTeX) and is told apart by content in refineLanguageByContent.
|
|
110
|
-
{ id: "vb", extraction: "regex", extensions: [".vb", ".bas", ".vbs", ".frm"], label: "Visual Basic", ...CODE, fence: "vb", fenceByExtension: { ".vb": "vbnet", ".vbs": "vbscript" } },
|
|
111
|
-
{
|
|
112
|
-
id: "cobol",
|
|
113
|
-
extraction: "own-result",
|
|
114
|
-
extensions: [".cbl", ".cob", ".cobol", ".cpy"],
|
|
115
|
-
label: "COBOL",
|
|
116
|
-
...CODE,
|
|
117
|
-
fence: "cobol",
|
|
118
|
-
partialRefsReason: "PERFORM, GO TO and CALL 'literal' are recorded as references, but a paragraph also runs by falling through from the one above it and a program can be called through a data item holding its name, so `dead` skips COBOL"
|
|
119
|
-
},
|
|
120
|
-
// Natural object sources as NaturalONE and SYSOBJH export them. Maps (.nsm) and DDMs (.nsd) are layouts, not code, and stay unmapped.
|
|
121
|
-
{
|
|
122
|
-
id: "natural",
|
|
123
|
-
extraction: "own-result",
|
|
124
|
-
extensions: [".nsp", ".nsn", ".nss", ".nsa", ".nsl", ".nsg", ".nsc", ".nsh"],
|
|
125
|
-
label: "Natural",
|
|
126
|
-
...CODE,
|
|
127
|
-
fence: "natural",
|
|
128
|
-
partialRefsReason: "PERFORM, CALLNAT 'literal' and FETCH 'literal' are recorded as references, but an object can also be called through a variable holding its name, so `dead` skips Natural"
|
|
129
|
-
},
|
|
130
|
-
{ id: "abap", extraction: "regex", extensions: [".abap"], label: "ABAP", ...CODE, fence: "abap" },
|
|
131
|
-
{ id: "sas", extraction: "regex", extensions: [".sas"], label: "SAS", ...CODE, fence: "sas" },
|
|
132
|
-
{ id: "pli", extraction: "regex", extensions: [".pli", ".pl1"], label: "PL/I", ...CODE, fence: "pli" },
|
|
133
|
-
// `.rpg` stays unmapped: RPG II and RPG III sources use it, and their fixed layout predates the ILE RPG forms this adapter reads.
|
|
134
|
-
{ id: "rpg", extraction: "regex", extensions: [".rpgle", ".sqlrpgle"], label: "RPG", ...CODE, fence: "rpgle" },
|
|
135
|
-
{ id: "jcl", extraction: "regex", extensions: [".jcl"], label: "JCL", ...CODE, fence: "jcl" },
|
|
136
|
-
// `.mm` is always Objective-C++. A `.m` (MATLAB uses it too) is Objective-C only on an `#import`, `@interface`, `@implementation` or `@protocol` line, and a `.h` only on `@interface` or `@protocol`: refineLanguageByContent in parser_types.ts decides.
|
|
137
|
-
{ id: "objc", extraction: "regex", extensions: [".mm"], label: "Objective-C", ...CODE, fence: "objectivec" },
|
|
138
|
-
// Gradle build scripts and Jenkinsfiles are Groovy.
|
|
139
|
-
{ id: "groovy", extraction: "regex", extensions: [".groovy", ".gvy", ".gradle"], basenames: ["jenkinsfile"], label: "Groovy", ...CODE, fence: "groovy", basenameImportsExtension: ".groovy" },
|
|
140
|
-
// A Prolog `.pl` stays unknown (refineLanguageByContent), and a `.t` is Perl only on a Perl marker line.
|
|
141
|
-
{ id: "perl", extraction: "regex", extensions: [".pl", ".pm"], label: "Perl", ...CODE, fence: "perl" },
|
|
142
|
-
{ id: "solidity", extraction: "regex", extensions: [".sol"], label: "Solidity", ...CODE, fence: "solidity" },
|
|
143
|
-
{ id: "thrift", extraction: "regex", extensions: [".thrift"], label: "Thrift", ...DATA, symbolBearing: true, fence: "thrift" },
|
|
144
|
-
{ id: "glsl", extraction: "regex", extensions: [".glsl", ".vert", ".frag", ".comp", ".geom", ".tesc", ".tese"], label: "GLSL", ...CODE, fence: "glsl" },
|
|
145
|
-
// `.fx` stays unmapped: other languages use it too.
|
|
146
|
-
{ id: "hlsl", extraction: "regex", extensions: [".hlsl", ".hlsli"], label: "HLSL", ...CODE, fence: "hlsl" },
|
|
147
|
-
{ id: "wgsl", extraction: "regex", extensions: [".wgsl"], label: "WGSL", ...CODE, fence: "wgsl" },
|
|
148
|
-
{ id: "metal", extraction: "regex", extensions: [".metal"], label: "Metal", ...CODE, fence: "metal" },
|
|
149
|
-
// `.f`, `.for` and `.f77` are read as fixed form unless code starts in column 1. `.fpp` stays unmapped: it is used for both forms.
|
|
150
|
-
{ id: "fortran", extraction: "regex", extensions: [".f", ".for", ".f77", ".f90", ".f95", ".f03", ".f08"], label: "Fortran", ...CODE, fence: "fortran" },
|
|
151
|
-
// `.inc` stays unmapped (many languages use it), and a `.pp` (Puppet uses it too) is Pascal only on a unit, program or library header: refineLanguageByContent in parser_types.ts decides. Text-form `.dfm` forms list their components.
|
|
152
|
-
{ id: "pascal", extraction: "regex", extensions: [".pas", ".dpr", ".dpk", ".lpr", ".dfm"], label: "Pascal", ...CODE, fence: "pascal" },
|
|
153
|
-
// MATLAB has no extension of its own: a `.m` that is not Objective-C is MATLAB only on a `function` or `classdef` header line, which refineLanguageByContent in parser_types.ts checks, so a Mathematica or Mercury `.m` stays unknown.
|
|
154
|
-
{ id: "matlab", extraction: "regex", extensions: [], label: "MATLAB", ...CODE, fence: "matlab" },
|
|
155
|
-
{ id: "cmake", extraction: "regex", extensions: [".cmake"], basenames: ["cmakelists.txt"], label: "CMake", ...CODE, fence: "cmake", basenameImportsExtension: ".cmake" },
|
|
156
|
-
// One adapter for the three assembly dialects that share these extensions: GNU as (`.s`, and `.S` through the lowercase lookup), NASM (`.asm`, `.nasm`) and IBM High Level Assembler (`.asm`), which the adapter tells apart by content. `.inc` stays unmapped: many languages use it.
|
|
157
|
-
{ id: "asm", extraction: "regex", extensions: [".s", ".asm", ".nasm"], label: "Assembly", ...CODE, fence: "asm" },
|
|
158
|
-
{ id: "batch", extraction: "regex", extensions: [".bat", ".cmd"], label: "Windows batch", ...CODE, fence: "batch" },
|
|
159
|
-
{ id: "erlang", extraction: "regex", extensions: [".erl", ".hrl"], label: "Erlang", ...CODE, fence: "erlang" },
|
|
160
|
-
{ id: "vhdl", extraction: "regex", extensions: [".vhd", ".vhdl"], label: "VHDL", ...CODE, fence: "vhdl" },
|
|
161
|
-
// Five Lisp-family dialects, each its own row: none shares an extractor (see common_lisp.ts's
|
|
162
|
-
// module doc for why their lexical rules stay separate rather than a single Lisp masker).
|
|
163
|
-
{ id: "common_lisp", extraction: "regex", extensions: [".lisp", ".lsp", ".cl"], label: "Common Lisp", ...CODE, fence: "lisp" },
|
|
164
|
-
{ id: "scheme", extraction: "regex", extensions: [".scm", ".ss"], label: "Scheme", ...CODE, fence: "scheme" },
|
|
165
|
-
{ id: "racket", extraction: "regex", extensions: [".rkt", ".rktl"], label: "Racket", ...CODE, fence: "racket" },
|
|
166
|
-
{ id: "clojure", extraction: "regex", extensions: [".clj", ".cljs", ".cljc"], label: "Clojure", ...CODE, fence: "clojure" },
|
|
167
|
-
{ id: "emacs_lisp", extraction: "regex", extensions: [".el"], label: "Emacs Lisp", ...CODE, fence: "lisp" },
|
|
168
|
-
// `.lhs` (literate Haskell) is deliberately not claimed here -- see haskell.ts's module doc.
|
|
169
|
-
{ id: "haskell", extraction: "regex", extensions: [".hs"], label: "Haskell", ...CODE, fence: "haskell" },
|
|
170
|
-
// `.mli` interface files share `.ml`'s lexical rules (comments, strings, quoted strings) and are read with the same extractor -- see ocaml.ts's module doc.
|
|
171
|
-
{ id: "ocaml", extraction: "regex", extensions: [".ml", ".mli"], label: "OCaml", ...CODE, fence: "ocaml" },
|
|
172
|
-
// `.fsi` signature files and `.fsx` scripts share `.fs`'s lexical rules and are read with the same extractor -- see fsharp.ts's module doc.
|
|
173
|
-
{ id: "fsharp", extraction: "regex", extensions: [".fs", ".fsi", ".fsx"], label: "F#", ...CODE, fence: "fsharp" },
|
|
174
|
-
{ id: "nix", extraction: "regex", extensions: [".nix"], label: "Nix", ...CODE, fence: "nix" },
|
|
175
|
-
// OpenEdge ABL has no extension of its own: a `.p` or `.w` (Pascal and CWEB use them too) or a `.cls` (Apex, VB6, LaTeX) is ABL only when its head carries an ABL marker, which refineLanguageByContent in parser_types.ts checks. The path-only hooks see a `.p` or `.w` as unknown and a `.cls` as Apex.
|
|
176
|
-
{ id: "abl", extraction: "regex", extensions: [], label: "OpenEdge ABL", ...CODE, fence: "abl" },
|
|
177
|
-
{ id: "apex", extraction: "regex", extensions: [".cls", ".trigger"], label: "Apex", ...CODE, fence: "apex" },
|
|
178
|
-
// Matched by the `-meta.xml` suffix in detectLanguage, not by an extension.
|
|
179
|
-
{ id: "salesforce_metadata", extraction: "regex", extensions: [], label: "Salesforce metadata", ...DATA, symbolBearing: true, sourceHints: true, grepSource: true, fence: "xml" },
|
|
180
|
-
{ id: "salesforce_markup", extraction: "own-result", extensions: [".cmp", ".app", ".evt", ".intf", ".design", ".auradoc", ".tokens", ".page", ".component", ".email"], label: "Salesforce markup", ...DATA, symbolBearing: true, sourceHints: true, grepSource: true, fence: "xml" },
|
|
181
|
-
{ id: "vue", extraction: "own-result", extensions: [".vue"], ...DATA, fence: "vue" },
|
|
182
|
-
{ id: "svelte", extraction: "own-result", extensions: [".svelte"], ...DATA, fence: "svelte" },
|
|
183
|
-
{ id: "astro", extraction: "own-result", extensions: [".astro"], ...DATA, fence: "astro" },
|
|
184
|
-
// Notebooks index through their code cells as Python, in parser.ts's ipynb branch.
|
|
185
|
-
{ id: "ipynb", extraction: "own-result", extensions: [".ipynb"], label: "Jupyter notebook", ...DATA, fence: "json" }
|
|
186
|
-
];
|
|
187
|
-
var SPEC_BY_ID = new Map(LANGUAGE_SPECS.map((s) => [s.id, s]));
|
|
188
|
-
function rows() {
|
|
189
|
-
return LANGUAGE_SPECS;
|
|
190
|
-
}
|
|
191
|
-
var EXTENSION_LANGUAGE = new Map(
|
|
192
|
-
LANGUAGE_SPECS.flatMap((s) => s.extensions.map((e) => [e, s.id]))
|
|
193
|
-
);
|
|
194
|
-
var FILENAME_LANGUAGE = new Map(
|
|
195
|
-
rows().flatMap((s) => (s.basenames ?? []).map((b) => [b, s.id]))
|
|
196
|
-
);
|
|
197
|
-
var EXACT_FILENAME_LANGUAGE = new Map(
|
|
198
|
-
rows().flatMap((s) => (s.exactBasenames ?? []).map((b) => [b, s.id]))
|
|
199
|
-
);
|
|
200
|
-
var TREE_SITTER_LANGUAGES = LANGUAGE_SPECS.filter((s) => s.extraction === "tree-sitter").map((s) => s.id);
|
|
201
|
-
function languageHasFlag(language, flag) {
|
|
202
|
-
return SPEC_BY_ID.get(language)?.[flag] === true;
|
|
203
|
-
}
|
|
204
|
-
function languageLabel(language) {
|
|
205
|
-
if (language === "unknown") return "this file type";
|
|
206
|
-
return SPEC_BY_ID.get(language)?.label ?? language;
|
|
207
|
-
}
|
|
208
|
-
function fenceFor(language, ext) {
|
|
209
|
-
const spec = SPEC_BY_ID.get(language);
|
|
210
|
-
if (spec === void 0) return "";
|
|
211
|
-
return spec.fenceByExtension?.[ext] ?? spec.fence ?? "";
|
|
212
|
-
}
|
|
213
|
-
function basenameImportsExtension(language) {
|
|
214
|
-
return SPEC_BY_ID.get(language)?.basenameImportsExtension;
|
|
215
|
-
}
|
|
216
|
-
function partialRefsReason(language) {
|
|
217
|
-
return SPEC_BY_ID.get(language)?.partialRefsReason;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// src/parser_types.ts
|
|
221
|
-
var DOTENV_VARIANT_RE = /^\.env(\..+)?$/;
|
|
222
|
-
var VB6_HEADER_SCAN_LINES = 40;
|
|
223
|
-
function isVb6ClassModule(content) {
|
|
224
|
-
if (content.includes("\0")) return false;
|
|
225
|
-
const lines = (content.charCodeAt(0) === 65279 ? content.slice(1) : content).split(/\r?\n/, VB6_HEADER_SCAN_LINES);
|
|
226
|
-
const first = lines.find((l) => l.trim() !== "");
|
|
227
|
-
if (first !== void 0 && /^VERSION\s+1\.0\s+CLASS\b/i.test(first.trim())) return true;
|
|
228
|
-
return lines.some((l) => /^Attribute\s+VB_Name\s*=\s*"/i.test(l.trim()));
|
|
229
|
-
}
|
|
230
|
-
function refineLanguageByContent(filePath, language, content) {
|
|
231
|
-
const sniff = CONTENT_SNIFFS.get(path.extname(filePath).toLowerCase());
|
|
232
|
-
if (sniff === void 0 || sniff.from !== language) return language;
|
|
233
|
-
return sniff.refine(content) ?? language;
|
|
234
|
-
}
|
|
235
|
-
var LANGUAGE_SNIFF_BYTES = 8192;
|
|
236
|
-
function sniffHead(content) {
|
|
237
|
-
const head = content.slice(0, LANGUAGE_SNIFF_BYTES);
|
|
238
|
-
return Buffer.byteLength(head, "utf8") <= LANGUAGE_SNIFF_BYTES ? head : Buffer.from(head, "utf8").subarray(0, LANGUAGE_SNIFF_BYTES).toString("utf8");
|
|
239
|
-
}
|
|
240
|
-
var ablOrUnknown = (c) => isAblSource(c) ? "abl" : void 0;
|
|
241
|
-
var clsRefine = (c) => {
|
|
242
|
-
if (isVb6ClassModule(c)) return "vb";
|
|
243
|
-
const abl = ablOrUnknown(c);
|
|
244
|
-
if (abl !== void 0) return abl;
|
|
245
|
-
if (isLatexClassFile(c)) return "unknown";
|
|
246
|
-
return void 0;
|
|
247
|
-
};
|
|
248
|
-
var CONTENT_SNIFFS = /* @__PURE__ */ new Map([
|
|
249
|
-
[".cls", { from: "apex", refine: clsRefine }],
|
|
250
|
-
[".p", { from: "unknown", refine: ablOrUnknown }],
|
|
251
|
-
[".w", { from: "unknown", refine: ablOrUnknown }],
|
|
252
|
-
[".m", { from: "unknown", refine: (c) => isObjcSource(sniffHead(c)) ? "objc" : isMatlabSource(sniffHead(c)) ? "matlab" : void 0 }],
|
|
253
|
-
[".pp", { from: "unknown", refine: (c) => isPascalSource(sniffHead(c)) ? "pascal" : void 0 }],
|
|
254
|
-
[".h", { from: "c", refine: (c) => isObjcHeader(sniffHead(c)) ? "objc" : void 0 }],
|
|
255
|
-
[".pl", { from: "perl", refine: (c) => isPrologSource(sniffHead(c)) ? "unknown" : void 0 }],
|
|
256
|
-
[".t", { from: "unknown", refine: (c) => isPerlSource(sniffHead(c)) ? "perl" : void 0 }]
|
|
257
|
-
]);
|
|
258
|
-
function needsContentSniff(filePath, language) {
|
|
259
|
-
return CONTENT_SNIFFS.get(path.extname(filePath).toLowerCase())?.from === language;
|
|
260
|
-
}
|
|
261
|
-
function detectLanguageOfFile(filePath) {
|
|
262
|
-
const language = detectLanguage(filePath);
|
|
263
|
-
if (!needsContentSniff(filePath, language)) return language;
|
|
264
|
-
try {
|
|
265
|
-
const fd = fs.openSync(filePath, "r");
|
|
266
|
-
try {
|
|
267
|
-
const buf = Buffer.alloc(LANGUAGE_SNIFF_BYTES);
|
|
268
|
-
const n = fs.readSync(fd, buf, 0, buf.length, 0);
|
|
269
|
-
return refineLanguageByContent(filePath, language, buf.subarray(0, n).toString("utf8"));
|
|
270
|
-
} finally {
|
|
271
|
-
fs.closeSync(fd);
|
|
272
|
-
}
|
|
273
|
-
} catch {
|
|
274
|
-
return language;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
function detectLanguage(filePath) {
|
|
278
|
-
const exactBase = path.basename(filePath);
|
|
279
|
-
const base = exactBase.toLowerCase();
|
|
280
|
-
if (DOTENV_VARIANT_RE.test(base)) return "env_file";
|
|
281
|
-
const byName = EXACT_FILENAME_LANGUAGE.get(exactBase) ?? FILENAME_LANGUAGE.get(base);
|
|
282
|
-
if (byName !== void 0) return byName;
|
|
283
|
-
if (base.endsWith("-meta.xml")) {
|
|
284
|
-
return "salesforce_metadata";
|
|
285
|
-
}
|
|
286
|
-
const ext = path.extname(base).toLowerCase();
|
|
287
|
-
return EXTENSION_LANGUAGE.get(ext) ?? "unknown";
|
|
288
|
-
}
|
|
289
|
-
var UNSUPPORTED_LANGUAGE_EXTENSIONS = /* @__PURE__ */ new Map([
|
|
290
|
-
[".rpg", "RPG II or RPG III"],
|
|
291
|
-
[".nsm", "Natural map"],
|
|
292
|
-
[".nsd", "Natural DDM"]
|
|
293
|
-
]);
|
|
294
|
-
function nonTreeSitterLanguageCount() {
|
|
295
|
-
return LANGUAGE_SPECS.filter((s) => s.extraction !== "tree-sitter" && s.id !== "ipynb").length;
|
|
296
|
-
}
|
|
297
|
-
function unsupportedLanguageName(filePath) {
|
|
298
|
-
const ext = path.extname(filePath).toLowerCase();
|
|
299
|
-
return UNSUPPORTED_LANGUAGE_EXTENSIONS.get(ext);
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
// src/dotenv_redact.ts
|
|
303
|
-
init_define_import_meta_env();
|
|
304
|
-
var DOTENV_VALUE_PLACEHOLDER = "[REDACTED:dotenv_value]";
|
|
305
|
-
var ASSIGNMENT_RE = /^(\s*(?:export\s+)?[A-Za-z_][\w.-]*\s*(?:\+?=|:(?!\/\/)))/;
|
|
306
|
-
var SAFE_LINE_RE = /^\s*(?:[#;].*)?$/;
|
|
307
|
-
function isDotenvPath(filePath) {
|
|
308
|
-
return detectLanguage(filePath) === "env_file";
|
|
309
|
-
}
|
|
310
|
-
function redactDotenvValues(text) {
|
|
311
|
-
const lines = text.split("\n");
|
|
312
|
-
const out = [];
|
|
313
|
-
let openQuote = null;
|
|
314
|
-
for (const raw of lines) {
|
|
315
|
-
const hasCr = raw.endsWith("\r");
|
|
316
|
-
const line = hasCr ? raw.slice(0, -1) : raw;
|
|
317
|
-
const eol = hasCr ? "\r" : "";
|
|
318
|
-
const emit = (s) => {
|
|
319
|
-
out.push(`${s}${eol}`);
|
|
320
|
-
};
|
|
321
|
-
if (openQuote !== null) {
|
|
322
|
-
if (_lineClosesQuote(line, openQuote)) openQuote = null;
|
|
323
|
-
emit(DOTENV_VALUE_PLACEHOLDER);
|
|
324
|
-
continue;
|
|
325
|
-
}
|
|
326
|
-
if (SAFE_LINE_RE.test(line)) {
|
|
327
|
-
emit(line);
|
|
328
|
-
continue;
|
|
329
|
-
}
|
|
330
|
-
const m = ASSIGNMENT_RE.exec(line);
|
|
331
|
-
if (m === null || m[1] === void 0) {
|
|
332
|
-
emit(DOTENV_VALUE_PLACEHOLDER);
|
|
333
|
-
continue;
|
|
334
|
-
}
|
|
335
|
-
const prefix = m[1];
|
|
336
|
-
openQuote = _detectOpenQuote(line.slice(prefix.length));
|
|
337
|
-
emit(`${prefix}${DOTENV_VALUE_PLACEHOLDER}`);
|
|
338
|
-
}
|
|
339
|
-
return out.join("\n");
|
|
340
|
-
}
|
|
341
|
-
function redactIfDotenv(filePath, text) {
|
|
342
|
-
return isDotenvPath(filePath) ? redactDotenvValues(text) : text;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
// src/db.ts
|
|
346
|
-
init_define_import_meta_env();
|
|
347
|
-
import * as fs3 from "node:fs";
|
|
348
|
-
import { createRequire as createRequire2 } from "node:module";
|
|
349
|
-
import * as path2 from "node:path";
|
|
350
|
-
|
|
351
|
-
// src/sqlite_driver.ts
|
|
352
|
-
init_define_import_meta_env();
|
|
353
|
-
import * as fs2 from "node:fs";
|
|
354
|
-
import { createRequire } from "node:module";
|
|
355
|
-
var _require = createRequire(import.meta.url);
|
|
356
|
-
function suppressSqliteExperimentalWarning() {
|
|
357
|
-
const original = process.emit;
|
|
358
|
-
let armed = true;
|
|
359
|
-
const restore = () => {
|
|
360
|
-
if (!armed) return;
|
|
361
|
-
armed = false;
|
|
362
|
-
process.emit = original;
|
|
363
|
-
};
|
|
364
|
-
process.emit = function patched(name, ...rest) {
|
|
365
|
-
const data = rest[0];
|
|
366
|
-
if (armed && name === "warning" && data instanceof Error && data.name === "ExperimentalWarning" && /sqlite/i.test(data.message)) {
|
|
367
|
-
restore();
|
|
368
|
-
return false;
|
|
369
|
-
}
|
|
370
|
-
return original.call(this, name, ...rest);
|
|
371
|
-
};
|
|
372
|
-
setImmediate(restore);
|
|
373
|
-
return restore;
|
|
374
|
-
}
|
|
375
|
-
var restoreWarnings = suppressSqliteExperimentalWarning();
|
|
376
|
-
var nodeSqlite;
|
|
377
|
-
try {
|
|
378
|
-
nodeSqlite = _require("node:sqlite");
|
|
379
|
-
} catch (e) {
|
|
380
|
-
restoreWarnings();
|
|
381
|
-
throw e;
|
|
382
|
-
}
|
|
383
|
-
var { DatabaseSync } = nodeSqlite;
|
|
384
|
-
var SQLITE_PRIMARY_CODES = [
|
|
385
|
-
"SQLITE_OK",
|
|
386
|
-
"SQLITE_ERROR",
|
|
387
|
-
"SQLITE_INTERNAL",
|
|
388
|
-
"SQLITE_PERM",
|
|
389
|
-
"SQLITE_ABORT",
|
|
390
|
-
"SQLITE_BUSY",
|
|
391
|
-
"SQLITE_LOCKED",
|
|
392
|
-
"SQLITE_NOMEM",
|
|
393
|
-
"SQLITE_READONLY",
|
|
394
|
-
"SQLITE_INTERRUPT",
|
|
395
|
-
"SQLITE_IOERR",
|
|
396
|
-
"SQLITE_CORRUPT",
|
|
397
|
-
"SQLITE_NOTFOUND",
|
|
398
|
-
"SQLITE_FULL",
|
|
399
|
-
"SQLITE_CANTOPEN",
|
|
400
|
-
"SQLITE_PROTOCOL",
|
|
401
|
-
"SQLITE_EMPTY",
|
|
402
|
-
"SQLITE_SCHEMA",
|
|
403
|
-
"SQLITE_TOOBIG",
|
|
404
|
-
"SQLITE_CONSTRAINT",
|
|
405
|
-
"SQLITE_MISMATCH",
|
|
406
|
-
"SQLITE_MISUSE",
|
|
407
|
-
"SQLITE_NOLFS",
|
|
408
|
-
"SQLITE_AUTH",
|
|
409
|
-
"SQLITE_FORMAT",
|
|
410
|
-
"SQLITE_RANGE",
|
|
411
|
-
"SQLITE_NOTADB",
|
|
412
|
-
"SQLITE_NOTICE",
|
|
413
|
-
"SQLITE_WARNING"
|
|
414
|
-
];
|
|
415
|
-
var SQLITE_EXTENDED_SUFFIXES = {
|
|
416
|
-
SQLITE_OK: ["LOAD_PERMANENTLY", "SYMLINK"],
|
|
417
|
-
SQLITE_ERROR: ["MISSING_COLLSEQ", "RETRY", "SNAPSHOT"],
|
|
418
|
-
SQLITE_ABORT: [null, "ROLLBACK"],
|
|
419
|
-
SQLITE_BUSY: ["RECOVERY", "SNAPSHOT", "TIMEOUT"],
|
|
420
|
-
SQLITE_LOCKED: ["SHAREDCACHE", "VTAB"],
|
|
421
|
-
SQLITE_READONLY: ["RECOVERY", "CANTLOCK", "ROLLBACK", "DBMOVED", "CANTINIT", "DIRECTORY"],
|
|
422
|
-
SQLITE_IOERR: [
|
|
423
|
-
"READ",
|
|
424
|
-
"SHORT_READ",
|
|
425
|
-
"WRITE",
|
|
426
|
-
"FSYNC",
|
|
427
|
-
"DIR_FSYNC",
|
|
428
|
-
"TRUNCATE",
|
|
429
|
-
"FSTAT",
|
|
430
|
-
"UNLOCK",
|
|
431
|
-
"RDLOCK",
|
|
432
|
-
"DELETE",
|
|
433
|
-
"BLOCKED",
|
|
434
|
-
"NOMEM",
|
|
435
|
-
"ACCESS",
|
|
436
|
-
"CHECKRESERVEDLOCK",
|
|
437
|
-
"LOCK",
|
|
438
|
-
"CLOSE",
|
|
439
|
-
"DIR_CLOSE",
|
|
440
|
-
"SHMOPEN",
|
|
441
|
-
"SHMSIZE",
|
|
442
|
-
"SHMLOCK",
|
|
443
|
-
"SHMMAP",
|
|
444
|
-
"SEEK",
|
|
445
|
-
"DELETE_NOENT",
|
|
446
|
-
"MMAP",
|
|
447
|
-
"GETTEMPPATH",
|
|
448
|
-
"CONVPATH",
|
|
449
|
-
"VNODE",
|
|
450
|
-
"AUTH",
|
|
451
|
-
"BEGIN_ATOMIC",
|
|
452
|
-
"COMMIT_ATOMIC",
|
|
453
|
-
"ROLLBACK_ATOMIC",
|
|
454
|
-
"DATA",
|
|
455
|
-
"CORRUPTFS",
|
|
456
|
-
"IN_PAGE"
|
|
457
|
-
],
|
|
458
|
-
SQLITE_CORRUPT: ["VTAB", "SEQUENCE", "INDEX"],
|
|
459
|
-
SQLITE_CANTOPEN: ["NOTEMPDIR", "ISDIR", "FULLPATH", "CONVPATH", "DIRTYWAL", "SYMLINK"],
|
|
460
|
-
SQLITE_CONSTRAINT: [
|
|
461
|
-
"CHECK",
|
|
462
|
-
"COMMITHOOK",
|
|
463
|
-
"FOREIGNKEY",
|
|
464
|
-
"FUNCTION",
|
|
465
|
-
"NOTNULL",
|
|
466
|
-
"PRIMARYKEY",
|
|
467
|
-
"TRIGGER",
|
|
468
|
-
"UNIQUE",
|
|
469
|
-
"VTAB",
|
|
470
|
-
"ROWID",
|
|
471
|
-
"PINNED",
|
|
472
|
-
"DATATYPE"
|
|
473
|
-
],
|
|
474
|
-
SQLITE_AUTH: ["USER"],
|
|
475
|
-
SQLITE_NOTICE: ["RECOVER_WAL", "RECOVER_ROLLBACK", "RBU"],
|
|
476
|
-
SQLITE_WARNING: ["AUTOINDEX"]
|
|
477
|
-
};
|
|
478
|
-
function sqliteResultCodeName(errcode) {
|
|
479
|
-
if (!Number.isInteger(errcode) || errcode < 0) return "ERR_SQLITE_ERROR";
|
|
480
|
-
if (errcode === 100) return "SQLITE_ROW";
|
|
481
|
-
if (errcode === 101) return "SQLITE_DONE";
|
|
482
|
-
const primary = SQLITE_PRIMARY_CODES[errcode & 255];
|
|
483
|
-
if (primary === void 0) return "ERR_SQLITE_ERROR";
|
|
484
|
-
const subcode = errcode >> 8;
|
|
485
|
-
if (subcode === 0) return primary;
|
|
486
|
-
const suffix = SQLITE_EXTENDED_SUFFIXES[primary]?.[subcode - 1];
|
|
487
|
-
return suffix === void 0 || suffix === null ? primary : `${primary}_${suffix}`;
|
|
488
|
-
}
|
|
489
|
-
function attempt(fn) {
|
|
490
|
-
try {
|
|
491
|
-
return fn();
|
|
492
|
-
} catch (e) {
|
|
493
|
-
const err = e;
|
|
494
|
-
if (err.code === "ERR_SQLITE_ERROR" && typeof err.errcode === "number") {
|
|
495
|
-
err.code = sqliteResultCodeName(err.errcode);
|
|
496
|
-
}
|
|
497
|
-
throw e;
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
var Statement = class {
|
|
501
|
-
#stmt;
|
|
502
|
-
#pluck = false;
|
|
503
|
-
constructor(stmt) {
|
|
504
|
-
this.#stmt = stmt;
|
|
505
|
-
}
|
|
506
|
-
get source() {
|
|
507
|
-
return this.#stmt.sourceSQL;
|
|
508
|
-
}
|
|
509
|
-
/**
|
|
510
|
-
* better-sqlite3's `reader` flag: does this statement return rows?
|
|
511
|
-
*
|
|
512
|
-
* `node:sqlite` has no equivalent, so it is derived from the prepared statement's own column
|
|
513
|
-
* count -- SQLite gives a row-producing statement its result columns at prepare time and gives a
|
|
514
|
-
* non-producing one none. That is a derivation, and this is the third defence-in-depth layer in
|
|
515
|
-
* `sqlite_query.ts`'s read-only guard, so it is not taken on faith: the driver tests run both
|
|
516
|
-
* libraries side by side over SELECT, a CTE, VALUES, EXPLAIN, an empty-result SELECT, INSERT,
|
|
517
|
-
* UPDATE, DELETE, CREATE, a reading PRAGMA and an assigning PRAGMA, and require every verdict to
|
|
518
|
-
* agree. If a future SQLite statement form ever breaks the equivalence, that test fails rather
|
|
519
|
-
* than the guard quietly weakening.
|
|
520
|
-
*/
|
|
521
|
-
get reader() {
|
|
522
|
-
return attempt(() => this.#stmt.columns()).length > 0;
|
|
523
|
-
}
|
|
524
|
-
// A plucked row is "the first column", which for an object row means the first *inserted* key. V8 preserves insertion order for string keys, and node:sqlite builds the row by walking the result columns left to right, so Object.values()[0] is the leftmost column -- not the column named in the SQL text, the same rule better-sqlite3 applies.
|
|
525
|
-
#shape(row) {
|
|
526
|
-
if (!this.#pluck || row === void 0 || row === null) return row;
|
|
527
|
-
const values = Object.values(row);
|
|
528
|
-
return values.length === 0 ? void 0 : values[0];
|
|
529
|
-
}
|
|
530
|
-
get(...params) {
|
|
531
|
-
return this.#shape(attempt(() => this.#stmt.get(...params)));
|
|
532
|
-
}
|
|
533
|
-
all(...params) {
|
|
534
|
-
const rows2 = attempt(() => this.#stmt.all(...params));
|
|
535
|
-
return this.#pluck ? rows2.map((r) => this.#shape(r)) : rows2;
|
|
536
|
-
}
|
|
537
|
-
run(...params) {
|
|
538
|
-
return attempt(() => this.#stmt.run(...params));
|
|
539
|
-
}
|
|
540
|
-
// Wrapped rather than returned directly so pluck applies lazily, one row at a time: the whole point of iterate() here is that sqlite_query.ts caps the row count without buffering the rest, and mapping the iterator through .all() first would defeat that.
|
|
541
|
-
*iterate(...params) {
|
|
542
|
-
const rows2 = attempt(() => this.#stmt.iterate(...params))[Symbol.iterator]();
|
|
543
|
-
for (; ; ) {
|
|
544
|
-
const next = attempt(() => rows2.next());
|
|
545
|
-
if (next.done === true) return;
|
|
546
|
-
yield this.#shape(next.value);
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
pluck(toggle = true) {
|
|
550
|
-
this.#pluck = toggle;
|
|
551
|
-
return this;
|
|
552
|
-
}
|
|
553
|
-
safeIntegers(toggle = true) {
|
|
554
|
-
this.#stmt.setReadBigInts(toggle);
|
|
555
|
-
return this;
|
|
556
|
-
}
|
|
557
|
-
columns() {
|
|
558
|
-
return attempt(() => this.#stmt.columns());
|
|
559
|
-
}
|
|
560
|
-
};
|
|
561
|
-
var Database = class {
|
|
562
|
-
#db;
|
|
563
|
-
#path;
|
|
564
|
-
#readonly;
|
|
565
|
-
#savepoints = 0;
|
|
566
|
-
constructor(dbPath, options = {}) {
|
|
567
|
-
const wantsExisting = options.readonly === true || options.fileMustExist === true;
|
|
568
|
-
if (wantsExisting && dbPath !== ":memory:" && !fs2.existsSync(dbPath)) {
|
|
569
|
-
throw new Error("unable to open database file");
|
|
570
|
-
}
|
|
571
|
-
this.#db = attempt(() => new DatabaseSync(dbPath, {
|
|
572
|
-
readOnly: options.readonly === true,
|
|
573
|
-
// sqlite-vec is loaded through db.loadExtension by initConnection, which node:sqlite refuses unless the connection opted in at construction. Harmless when no extension is ever loaded.
|
|
574
|
-
allowExtension: true,
|
|
575
|
-
// better-sqlite3 opens every connection with busy_timeout at 5000ms; node:sqlite opens at 0, so a connection that named no timeout would silently go from five seconds of patience to none. db.ts overrides this to 15000 in initConnection, but sqlite_query.ts opens a user's arbitrary database readonly and takes whatever the default is -- which would have turned ordinary contention with another writer into an immediate "database is locked".
|
|
576
|
-
timeout: options.timeout ?? 5e3
|
|
577
|
-
}));
|
|
578
|
-
this.#path = dbPath;
|
|
579
|
-
this.#readonly = options.readonly === true;
|
|
580
|
-
}
|
|
581
|
-
get open() {
|
|
582
|
-
return this.#db.isOpen;
|
|
583
|
-
}
|
|
584
|
-
get inTransaction() {
|
|
585
|
-
return this.#db.isTransaction;
|
|
586
|
-
}
|
|
587
|
-
get readonly() {
|
|
588
|
-
return this.#readonly;
|
|
589
|
-
}
|
|
590
|
-
get name() {
|
|
591
|
-
return this.#path;
|
|
592
|
-
}
|
|
593
|
-
prepare(sql) {
|
|
594
|
-
return new Statement(attempt(() => this.#db.prepare(sql)));
|
|
595
|
-
}
|
|
596
|
-
exec(sql) {
|
|
597
|
-
attempt(() => this.#db.exec(sql));
|
|
598
|
-
}
|
|
599
|
-
pragma(source, options = {}) {
|
|
600
|
-
const rows2 = attempt(() => this.#db.prepare(`PRAGMA ${source}`).all());
|
|
601
|
-
if (options.simple !== true) return rows2;
|
|
602
|
-
const first = rows2[0];
|
|
603
|
-
if (first === void 0) return void 0;
|
|
604
|
-
const values = Object.values(first);
|
|
605
|
-
return values.length === 0 ? void 0 : values[0];
|
|
606
|
-
}
|
|
607
|
-
function(name, options, fn) {
|
|
608
|
-
attempt(() => this.#db.function(name, options, fn));
|
|
609
|
-
}
|
|
610
|
-
loadExtension(extensionPath) {
|
|
611
|
-
attempt(() => this.#db.loadExtension(extensionPath));
|
|
612
|
-
}
|
|
613
|
-
close() {
|
|
614
|
-
attempt(() => this.#db.close());
|
|
615
|
-
}
|
|
616
|
-
/**
|
|
617
|
-
* Wrap `fn` so it runs inside a transaction, committing on return and rolling back on throw.
|
|
618
|
-
*
|
|
619
|
-
* Nesting uses SAVEPOINT, which is what makes it safe for a transactional helper to call another
|
|
620
|
-
* one: an inner `BEGIN` would throw ("cannot start a transaction within a transaction"), an inner
|
|
621
|
-
* SAVEPOINT composes. Whether we are nested is read from SQLite via `isTransaction` rather than
|
|
622
|
-
* tracked in a counter here, so a transaction some other code path opened still nests correctly.
|
|
623
|
-
*
|
|
624
|
-
* The rollback is best-effort and never replaces the caller's error: if the ROLLBACK itself fails
|
|
625
|
-
* -- the connection died, the transaction was already unwound -- the original failure is still
|
|
626
|
-
* what propagates, because that is the one that explains what went wrong.
|
|
627
|
-
*/
|
|
628
|
-
transaction(fn) {
|
|
629
|
-
const build = (beginSql) => (...args) => {
|
|
630
|
-
if (this.#db.isTransaction) {
|
|
631
|
-
const name = `tg_sp_${this.#savepoints++}`;
|
|
632
|
-
attempt(() => this.#db.exec(`SAVEPOINT ${name}`));
|
|
633
|
-
try {
|
|
634
|
-
const result = fn(...args);
|
|
635
|
-
attempt(() => this.#db.exec(`RELEASE ${name}`));
|
|
636
|
-
return result;
|
|
637
|
-
} catch (e) {
|
|
638
|
-
try {
|
|
639
|
-
attempt(() => this.#db.exec(`ROLLBACK TO ${name}`));
|
|
640
|
-
attempt(() => this.#db.exec(`RELEASE ${name}`));
|
|
641
|
-
} catch {
|
|
642
|
-
}
|
|
643
|
-
throw e;
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
attempt(() => this.#db.exec(beginSql));
|
|
647
|
-
try {
|
|
648
|
-
const result = fn(...args);
|
|
649
|
-
attempt(() => this.#db.exec("COMMIT"));
|
|
650
|
-
return result;
|
|
651
|
-
} catch (e) {
|
|
652
|
-
try {
|
|
653
|
-
attempt(() => this.#db.exec("ROLLBACK"));
|
|
654
|
-
} catch {
|
|
655
|
-
}
|
|
656
|
-
throw e;
|
|
657
|
-
}
|
|
658
|
-
};
|
|
659
|
-
const wrapped = build("BEGIN");
|
|
660
|
-
wrapped.default = wrapped;
|
|
661
|
-
wrapped.deferred = build("BEGIN");
|
|
662
|
-
wrapped.immediate = build("BEGIN IMMEDIATE");
|
|
663
|
-
wrapped.exclusive = build("BEGIN EXCLUSIVE");
|
|
664
|
-
return wrapped;
|
|
665
|
-
}
|
|
666
|
-
};
|
|
667
|
-
|
|
668
|
-
// src/db.ts
|
|
669
|
-
var _require2 = createRequire2(import.meta.url);
|
|
670
|
-
var _connections = /* @__PURE__ */ new Map();
|
|
671
|
-
var SCHEMA_SQL = `
|
|
672
|
-
CREATE TABLE IF NOT EXISTS files (
|
|
673
|
-
path TEXT PRIMARY KEY,
|
|
674
|
-
sha TEXT,
|
|
675
|
-
mtime REAL,
|
|
676
|
-
language TEXT,
|
|
677
|
-
indexed_at REAL,
|
|
678
|
-
embed_sha TEXT,
|
|
679
|
-
retry_count INTEGER NOT NULL DEFAULT 0,
|
|
680
|
-
parser_sha TEXT
|
|
681
|
-
);
|
|
682
|
-
-- Expression index on TG_LOWER(path) -- see pathEqClause (sql_path.ts) and TG_LOWER's
|
|
683
|
-
-- registration above. TG_LOWER is registered { deterministic: true }, which is required for
|
|
684
|
-
-- SQLite to index an expression at all; without it CREATE INDEX on a function call throws
|
|
685
|
-
-- "non-deterministic functions prohibited in index expressions". Because pathEqClause emits
|
|
686
|
-
-- this exact 'TG_LOWER(path) = ?' text for every case-insensitive-filesystem query, the planner
|
|
687
|
-
-- matches it against this index and uses SEARCH instead of a full table SCAN, without requiring
|
|
688
|
-
-- any writer to populate a separate folded column (verified via EXPLAIN QUERY PLAN in
|
|
689
|
-
-- db.test.ts / sql_path.test.ts). CREATE INDEX IF NOT EXISTS is purely additive and safe to run
|
|
690
|
-
-- against an already-populated table on every connection open, unlike an ALTER TABLE column add
|
|
691
|
-
-- -- no MIGRATIONS entry or SCHEMA_VERSION bump is needed for this index.
|
|
692
|
-
CREATE INDEX IF NOT EXISTS idx_files_path_folded ON files(TG_LOWER(path));
|
|
693
|
-
|
|
694
|
-
CREATE TABLE IF NOT EXISTS symbols (
|
|
695
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
696
|
-
file_path TEXT,
|
|
697
|
-
name TEXT,
|
|
698
|
-
kind TEXT,
|
|
699
|
-
line_start INTEGER,
|
|
700
|
-
line_end INTEGER,
|
|
701
|
-
body TEXT,
|
|
702
|
-
docstring TEXT,
|
|
703
|
-
parent TEXT NOT NULL DEFAULT ''
|
|
704
|
-
);
|
|
705
|
-
CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);
|
|
706
|
-
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path);
|
|
707
|
-
CREATE INDEX IF NOT EXISTS idx_symbols_name_kind ON symbols(name, kind);
|
|
708
|
-
CREATE INDEX IF NOT EXISTS idx_symbols_file_folded ON symbols(TG_LOWER(file_path));
|
|
709
|
-
CREATE INDEX IF NOT EXISTS idx_symbols_file_name_folded ON symbols(TG_LOWER(file_path), name);
|
|
710
|
-
-- Partial index backing checkSymbolBodySize (cli_doctor.ts), which every SessionStart hook runs.
|
|
711
|
-
-- Its predicate cannot be served by any index above, so the check had to read the whole symbols
|
|
712
|
-
-- table -- 226 MB / 231324 rows here, 229 ms per session start, and the early-exit LIMIT 1 never
|
|
713
|
-
-- fires on a healthy index because there is nothing to find. Indexing the *violating* rows only
|
|
714
|
-
-- makes the check a lookup into a b-tree that is empty on a healthy index: measured 229 ms -> 0.0
|
|
715
|
-
-- ms, 4 KB on disk, and no measurable insert cost (-0.2%, within noise, over 40000 real rows),
|
|
716
|
-
-- because SQLite evaluates the predicate and skips the b-tree write for every row under the cap.
|
|
717
|
-
-- SQLite uses a partial index only where the query's WHERE implies the index's, so the probe in
|
|
718
|
-
-- cli_doctor.ts spells its comparison the same way and against the same constant. That makes the
|
|
719
|
-
-- threshold part of the stored schema -- see SYMBOL_BODY_CHAR_CAP in constants.ts for what
|
|
720
|
-
-- changing it requires. A query with a lower threshold correctly gets a full scan instead, so no
|
|
721
|
-
-- other reader can be served stale rows by this index.
|
|
722
|
-
CREATE INDEX IF NOT EXISTS idx_symbols_oversized_body ON symbols(id) WHERE LENGTH(body) > ${SYMBOL_BODY_CHAR_CAP};
|
|
723
|
-
|
|
724
|
-
CREATE TABLE IF NOT EXISTS refs (
|
|
725
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
726
|
-
file_path TEXT,
|
|
727
|
-
name TEXT,
|
|
728
|
-
line INTEGER,
|
|
729
|
-
col INTEGER,
|
|
730
|
-
context TEXT
|
|
731
|
-
);
|
|
732
|
-
CREATE INDEX IF NOT EXISTS idx_refs_name ON refs(name);
|
|
733
|
-
CREATE INDEX IF NOT EXISTS idx_refs_file ON refs(file_path);
|
|
734
|
-
CREATE INDEX IF NOT EXISTS idx_refs_file_folded ON refs(TG_LOWER(file_path));
|
|
735
|
-
CREATE INDEX IF NOT EXISTS idx_refs_file_name_folded ON refs(TG_LOWER(file_path), name);
|
|
736
|
-
|
|
737
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
738
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
739
|
-
file_path TEXT,
|
|
740
|
-
start_line INTEGER,
|
|
741
|
-
end_line INTEGER,
|
|
742
|
-
text TEXT,
|
|
743
|
-
kind TEXT
|
|
744
|
-
);
|
|
745
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
|
|
746
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_file_folded ON chunks(TG_LOWER(file_path));
|
|
747
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_file_kind_folded ON chunks(TG_LOWER(file_path), kind);
|
|
748
|
-
|
|
749
|
-
-- Tracks every project root a hook has ever seen an edit for, so the worker's periodic sweep
|
|
750
|
-
-- (sweepKnownRoots in index_prune.ts) knows which roots to auto-prune without scanning the
|
|
751
|
-
-- entire shared files table for distinct top-level directories on every cycle. Purely additive
|
|
752
|
-
-- (no SCHEMA_VERSION bump needed): last_seen_ms is refreshed on every observed edit,
|
|
753
|
-
-- first_missing_ms is set the first sweep that finds the root unreachable and cleared the
|
|
754
|
-
-- moment it's seen reachable again -- see sweepKnownRoots' grace-period logic.
|
|
755
|
-
CREATE TABLE IF NOT EXISTS known_roots (
|
|
756
|
-
root TEXT PRIMARY KEY,
|
|
757
|
-
last_seen_ms REAL NOT NULL,
|
|
758
|
-
first_missing_ms REAL
|
|
759
|
-
);
|
|
760
|
-
|
|
761
|
-
-- Resume point for a budget-truncated reconcile sweep (reconcile.ts), one row per project root. A
|
|
762
|
-
-- project too large to finish a sweep inside DEFAULT_RECONCILE_BUDGET_MS would otherwise scan the
|
|
763
|
-
-- same deterministic (git ls-files) prefix every session forever, leaving every file after the
|
|
764
|
-
-- budget cutoff permanently unchecked. last_scanned_path is the last tracked file the sweep
|
|
765
|
-
-- finished examining before its budget ran out; the next sweep rotates its scan order to resume
|
|
766
|
-
-- right after that file, wrapping back to the start, so repeated truncated sweeps eventually cover
|
|
767
|
-
-- the whole project. Cleared (row deleted) the moment a sweep completes a full lap without running
|
|
768
|
-
-- out of budget. Purely additive (no SCHEMA_VERSION bump needed): a missing row just means "start
|
|
769
|
-
-- from the beginning", the same as a fresh database.
|
|
770
|
-
CREATE TABLE IF NOT EXISTS reconcile_cursor (
|
|
771
|
-
root TEXT PRIMARY KEY,
|
|
772
|
-
last_scanned_path TEXT NOT NULL,
|
|
773
|
-
updated_at REAL NOT NULL
|
|
774
|
-
);
|
|
775
|
-
|
|
776
|
-
-- Cross-cache full-text search index for 'token-goat recall' (recall_index.ts). One row
|
|
777
|
-
-- per bash-output/web-output/mcp-output blob-store entry (see disk_cache.ts), refreshed
|
|
778
|
-
-- in place (ON CONFLICT DO UPDATE) whenever storeBashOutput/storeWebOutput/storeMcpOutput
|
|
779
|
-
-- write that entry, so recall never needs a separate rebuild step. row_id is a plain
|
|
780
|
-
-- surrogate integer key -- entry_id is the real blob-store id (bash/mcp ids are hex,
|
|
781
|
-
-- web ids are the cache's own scheme) and is not unique on its own since bash-output and
|
|
782
|
-
-- mcp-output ids share one namespace (BASH_OUTPUT_SUBDIR) while web-output ids are a
|
|
783
|
-
-- separate namespace; cache_type disambiguates.
|
|
784
|
-
CREATE TABLE IF NOT EXISTS cache_recall (
|
|
785
|
-
row_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
786
|
-
cache_type TEXT NOT NULL,
|
|
787
|
-
entry_id TEXT NOT NULL,
|
|
788
|
-
label TEXT,
|
|
789
|
-
content TEXT,
|
|
790
|
-
stored_at REAL,
|
|
791
|
-
UNIQUE(cache_type, entry_id)
|
|
792
|
-
);
|
|
793
|
-
CREATE INDEX IF NOT EXISTS idx_cache_recall_type ON cache_recall(cache_type);
|
|
794
|
-
|
|
795
|
-
-- Per-emission ledger for 'token-goat hint-stats' (hint_stats.ts). One row per hint
|
|
796
|
-
-- emission event (a hook returning a 'context' HookOutput classified as a discretionary
|
|
797
|
-
-- efficiency nudge, as opposed to a mandatory informational injection -- see hint_stats.ts's
|
|
798
|
-
-- doc comment for the exact category list and what is deliberately excluded). correlator is a
|
|
799
|
-
-- best-effort file-path/output-id substring extracted from the hint's own text, used to check
|
|
800
|
-
-- whether a later Bash tool call in the same session actually followed the hint's specific
|
|
801
|
-
-- pointer (see resolvePendingHintsForEvent) -- NULL when no such pointer could be extracted,
|
|
802
|
-
-- in which case the row is inserted already resolved with acted_on=0 (counted as emitted, never
|
|
803
|
-
-- eligible for auto-detected credit). calls_remaining is the countdown of subsequent tool-use
|
|
804
|
-
-- events still eligible to resolve this row before it is considered timed out.
|
|
805
|
-
CREATE TABLE IF NOT EXISTS hint_emissions (
|
|
806
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
807
|
-
category TEXT NOT NULL,
|
|
808
|
-
session_id TEXT NOT NULL,
|
|
809
|
-
harness TEXT NOT NULL,
|
|
810
|
-
correlator TEXT,
|
|
811
|
-
emitted_at REAL NOT NULL,
|
|
812
|
-
resolved INTEGER NOT NULL DEFAULT 0,
|
|
813
|
-
acted_on INTEGER NOT NULL DEFAULT 0,
|
|
814
|
-
calls_remaining INTEGER NOT NULL DEFAULT 0,
|
|
815
|
-
bytes_emitted INTEGER
|
|
816
|
-
);
|
|
817
|
-
CREATE INDEX IF NOT EXISTS idx_hint_emissions_category ON hint_emissions(category);
|
|
818
|
-
CREATE INDEX IF NOT EXISTS idx_hint_emissions_session_pending ON hint_emissions(session_id, resolved);
|
|
819
|
-
|
|
820
|
-
-- Manual efficacy votes for a hint category (token-goat hint-stats --mark-effective/--mark-ineffective),
|
|
821
|
-
-- kept separate from hint_emissions' automatic acted_on signal so the two are never silently
|
|
822
|
-
-- blended -- see hint_stats.ts's doc comment on why some categories only support this manual signal.
|
|
823
|
-
CREATE TABLE IF NOT EXISTS hint_manual_marks (
|
|
824
|
-
category TEXT PRIMARY KEY,
|
|
825
|
-
effective_count INTEGER NOT NULL DEFAULT 0,
|
|
826
|
-
ineffective_count INTEGER NOT NULL DEFAULT 0
|
|
827
|
-
);
|
|
828
|
-
|
|
829
|
-
-- Durable counter backing hint_stats.ts's backoff-threshold probe-recovery schedule: how many
|
|
830
|
-
-- CONSECUTIVE suppressed occasions have elapsed for (category, harness) since a hint in this
|
|
831
|
-
-- category was last actually shown (either organically, because shouldSuppress no longer holds,
|
|
832
|
-
-- or via a prior probe). shouldSuppress itself stays a pure function of hint_emissions -- this
|
|
833
|
-
-- table exists only because a suppressed occasion is deliberately never written to
|
|
834
|
-
-- hint_emissions (see that table's own comment), so without a separate durable counter here
|
|
835
|
-
-- there would be no way to know "how many suppressed occasions have we seen" across the
|
|
836
|
-
-- short-lived hook CLI processes that call applyHintTracking. Keyed by (category, harness), not
|
|
837
|
-
-- category alone, to match shouldSuppress/categoryStats' own per-harness scoping -- unlike
|
|
838
|
-
-- hint_manual_marks (a human-entered vote, deliberately not harness-split).
|
|
839
|
-
CREATE TABLE IF NOT EXISTS hint_suppression_probes (
|
|
840
|
-
category TEXT NOT NULL,
|
|
841
|
-
harness TEXT NOT NULL,
|
|
842
|
-
streak INTEGER NOT NULL DEFAULT 0,
|
|
843
|
-
PRIMARY KEY (category, harness)
|
|
844
|
-
);
|
|
845
|
-
|
|
846
|
-
-- Free-text architecture/rationale notes (the "why" layer -- see notes.ts), attached either to
|
|
847
|
-
-- a whole file (symbol = '') or to one specific indexed symbol within it (symbol = that
|
|
848
|
-
-- symbol's name). '' rather than NULL for the whole-file case because SQLite's UNIQUE treats
|
|
849
|
-
-- NULLs as pairwise-distinct (never conflicting with each other), which would let note-add
|
|
850
|
-
-- accumulate unlimited duplicate whole-file notes for the same file instead of upserting one;
|
|
851
|
-
-- '' is a real, comparable value so UNIQUE(file_path, symbol) enforces "at most one note per
|
|
852
|
-
-- attachment point" for both cases identically. 'fingerprint' is a SHA-256 digest (see
|
|
853
|
-
-- fingerprintContent in fingerprint.ts) captured at write time of exactly what the note
|
|
854
|
-
-- describes -- the resolved symbol's current body text for a symbol-scoped note, or a stable
|
|
855
|
-
-- digest of the file's current top-level symbol manifest (name:kind:line-range per symbol,
|
|
856
|
-
-- sorted) for a file-scoped note -- so 'token-goat note-list --stale-only' can recompute the
|
|
857
|
-
-- same fingerprint against the live index later and flag a mismatch (see notes.ts's
|
|
858
|
-
-- isNoteStale). Staleness detection is purely advisory: nothing here ever auto-rewrites or
|
|
859
|
-
-- deletes a note's content, only flags that the code it describes has moved since it was
|
|
860
|
-
-- written -- a human/agent re-review decides what to do with a stale note.
|
|
861
|
-
CREATE TABLE IF NOT EXISTS notes (
|
|
862
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
863
|
-
file_path TEXT NOT NULL,
|
|
864
|
-
symbol TEXT NOT NULL DEFAULT '',
|
|
865
|
-
content TEXT NOT NULL,
|
|
866
|
-
fingerprint TEXT NOT NULL,
|
|
867
|
-
created_at REAL NOT NULL,
|
|
868
|
-
updated_at REAL NOT NULL,
|
|
869
|
-
UNIQUE(file_path, symbol)
|
|
870
|
-
);
|
|
871
|
-
CREATE INDEX IF NOT EXISTS idx_notes_file_folded ON notes(TG_LOWER(file_path));
|
|
872
|
-
|
|
873
|
-
-- Baseline for skill_version_drift.ts's one-shot nudge: the token-goat CLI version (and its
|
|
874
|
-
-- flat command-name set, JSON-encoded) active the moment the token-goat skill's body was
|
|
875
|
-
-- last (re)loaded into this session -- see hooks_skill.ts's postSkillHandler. A session that
|
|
876
|
-
-- keeps running after the CLI is upgraded has no other way to learn that new surgical-read
|
|
877
|
-
-- commands now exist (the skill only re-announces itself on an explicit reload), so
|
|
878
|
-
-- checkSkillVersionDrift compares this snapshot against the live command set on each user turn
|
|
879
|
-
-- and fires the nudge exactly once (notified_at) per (re)load. session_id is the primary key
|
|
880
|
-
-- because only one skill (token-goat) is ever tracked here.
|
|
881
|
-
CREATE TABLE IF NOT EXISTS skill_version_snapshots (
|
|
882
|
-
session_id TEXT PRIMARY KEY,
|
|
883
|
-
skill_name TEXT NOT NULL,
|
|
884
|
-
loaded_version TEXT NOT NULL,
|
|
885
|
-
loaded_commands_json TEXT NOT NULL,
|
|
886
|
-
notified_at REAL
|
|
887
|
-
);
|
|
888
|
-
|
|
889
|
-
-- Which embedding stack produced the vectors currently in chunk_vectors -- the model, its
|
|
890
|
-
-- pinned revision, and the inference runtime (see embeddingProvenance in embeddings.ts). The
|
|
891
|
-
-- vector table itself is vec0(rowid, embedding) and has nowhere to record this, so without
|
|
892
|
-
-- this row a database that was embedded by one stack and then added to by another holds two
|
|
893
|
-
-- incomparable sets of vectors under one index, with nothing able to tell them apart. That is
|
|
894
|
-
-- not hypothetical: global.db is machine-wide across every project on the machine (see
|
|
895
|
-
-- constants.ts), so upgrading the runtime, or changing the model or its pinned revision, mixes
|
|
896
|
-
-- old and new vectors for as long as the old files go untouched. Measured drift between two
|
|
897
|
-
-- runtime versions of the same quantized model is 0.9925-0.9978 cosine on the final vector --
|
|
898
|
-
-- small, but enough to reorder near-ties, and invisible to every existing check.
|
|
899
|
-
--
|
|
900
|
-
-- Single-row by construction (the CHECK pins the key), because there is exactly one vector
|
|
901
|
-
-- table per database. An EMPTY table on a database that already holds chunks means the vectors
|
|
902
|
-
-- predate this stamp and their provenance is unknowable -- see ensureEmbeddingProvenance, which
|
|
903
|
-
-- treats that exactly like a mismatch. That is what makes this work without a migration step.
|
|
904
|
-
CREATE TABLE IF NOT EXISTS embedding_provenance (
|
|
905
|
-
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
906
|
-
provenance TEXT NOT NULL
|
|
907
|
-
);
|
|
908
|
-
`;
|
|
909
|
-
var FTS_TOKENIZER = "unicode61 remove_diacritics 2";
|
|
910
|
-
var FTS_SQL = `
|
|
911
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
|
|
912
|
-
name,
|
|
913
|
-
body,
|
|
914
|
-
docstring,
|
|
915
|
-
content='symbols',
|
|
916
|
-
content_rowid='id',
|
|
917
|
-
tokenize='${FTS_TOKENIZER}'
|
|
918
|
-
);
|
|
919
|
-
CREATE TRIGGER IF NOT EXISTS symbols_ai AFTER INSERT ON symbols BEGIN
|
|
920
|
-
INSERT INTO symbols_fts(rowid, name, body, docstring)
|
|
921
|
-
VALUES (new.id, new.name, new.body, new.docstring);
|
|
922
|
-
END;
|
|
923
|
-
CREATE TRIGGER IF NOT EXISTS symbols_ad AFTER DELETE ON symbols BEGIN
|
|
924
|
-
INSERT INTO symbols_fts(symbols_fts, rowid, name, body, docstring)
|
|
925
|
-
VALUES ('delete', old.id, old.name, old.body, old.docstring);
|
|
926
|
-
END;
|
|
927
|
-
CREATE TRIGGER IF NOT EXISTS symbols_au AFTER UPDATE ON symbols BEGIN
|
|
928
|
-
INSERT INTO symbols_fts(symbols_fts, rowid, name, body, docstring)
|
|
929
|
-
VALUES ('delete', old.id, old.name, old.body, old.docstring);
|
|
930
|
-
INSERT INTO symbols_fts(rowid, name, body, docstring)
|
|
931
|
-
VALUES (new.id, new.name, new.body, new.docstring);
|
|
932
|
-
END;
|
|
933
|
-
|
|
934
|
-
-- Content-linked FTS5 mirror of cache_recall (recall_index.ts), same shape as symbols_fts
|
|
935
|
-
-- above. An INSERT ... ON CONFLICT DO UPDATE against cache_recall fires the AFTER UPDATE
|
|
936
|
-
-- trigger (not AFTER INSERT) on the conflicting row, same as any other SQLite upsert, so the
|
|
937
|
-
-- delete+reinsert pattern below keeps the fts index correct on a re-indexed (overwritten)
|
|
938
|
-
-- entry, not just a brand-new one.
|
|
939
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS cache_recall_fts USING fts5(
|
|
940
|
-
label,
|
|
941
|
-
content,
|
|
942
|
-
content='cache_recall',
|
|
943
|
-
content_rowid='row_id',
|
|
944
|
-
tokenize='${FTS_TOKENIZER}'
|
|
945
|
-
);
|
|
946
|
-
CREATE TRIGGER IF NOT EXISTS cache_recall_ai AFTER INSERT ON cache_recall BEGIN
|
|
947
|
-
INSERT INTO cache_recall_fts(rowid, label, content)
|
|
948
|
-
VALUES (new.row_id, new.label, new.content);
|
|
949
|
-
END;
|
|
950
|
-
CREATE TRIGGER IF NOT EXISTS cache_recall_ad AFTER DELETE ON cache_recall BEGIN
|
|
951
|
-
INSERT INTO cache_recall_fts(cache_recall_fts, rowid, label, content)
|
|
952
|
-
VALUES ('delete', old.row_id, old.label, old.content);
|
|
953
|
-
END;
|
|
954
|
-
CREATE TRIGGER IF NOT EXISTS cache_recall_au AFTER UPDATE ON cache_recall BEGIN
|
|
955
|
-
INSERT INTO cache_recall_fts(cache_recall_fts, rowid, label, content)
|
|
956
|
-
VALUES ('delete', old.row_id, old.label, old.content);
|
|
957
|
-
INSERT INTO cache_recall_fts(rowid, label, content)
|
|
958
|
-
VALUES (new.row_id, new.label, new.content);
|
|
959
|
-
END;
|
|
960
|
-
`;
|
|
961
|
-
var SCHEMA_VERSION = 14;
|
|
962
|
-
function alterTableIdempotent(conn, sql) {
|
|
963
|
-
try {
|
|
964
|
-
conn.exec(sql);
|
|
965
|
-
} catch (err) {
|
|
966
|
-
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err;
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
function purgeDotenvEmbeddings(conn) {
|
|
970
|
-
let paths;
|
|
971
|
-
try {
|
|
972
|
-
paths = conn.prepare("SELECT DISTINCT file_path FROM chunks").all().map((r) => r.file_path).filter(isDotenvPath);
|
|
973
|
-
} catch {
|
|
974
|
-
return;
|
|
975
|
-
}
|
|
976
|
-
if (paths.length === 0) return;
|
|
977
|
-
for (const p of paths) {
|
|
978
|
-
try {
|
|
979
|
-
conn.prepare("DELETE FROM chunk_vectors WHERE rowid IN (SELECT id FROM chunks WHERE file_path = ?)").run(p);
|
|
980
|
-
} catch {
|
|
981
|
-
}
|
|
982
|
-
conn.prepare("DELETE FROM chunks WHERE file_path = ?").run(p);
|
|
983
|
-
try {
|
|
984
|
-
conn.prepare("UPDATE files SET embed_sha = NULL WHERE path = ?").run(p);
|
|
985
|
-
} catch {
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
function rebuildFtsAtCurrentTokenizer(conn) {
|
|
990
|
-
try {
|
|
991
|
-
const declarations = conn.prepare("SELECT sql FROM sqlite_master WHERE name IN ('symbols_fts','cache_recall_fts')").all();
|
|
992
|
-
const current = declarations.map((d) => /tokenize\s*=\s*'([^']*)'/.exec(d.sql ?? "")?.[1] ?? "");
|
|
993
|
-
if (current.length === 2 && current.every((t) => t === FTS_TOKENIZER)) return;
|
|
994
|
-
conn.transaction(() => {
|
|
995
|
-
conn.exec("DROP TABLE IF EXISTS symbols_fts; DROP TABLE IF EXISTS cache_recall_fts;");
|
|
996
|
-
conn.exec(FTS_SQL);
|
|
997
|
-
conn.exec("INSERT INTO symbols_fts(symbols_fts) VALUES('rebuild');");
|
|
998
|
-
conn.exec("INSERT INTO cache_recall_fts(cache_recall_fts) VALUES('rebuild');");
|
|
999
|
-
}).immediate();
|
|
1000
|
-
} catch {
|
|
1001
|
-
}
|
|
1002
|
-
}
|
|
1003
|
-
var MIGRATIONS = {
|
|
1004
|
-
// v1 -> v2: adds files.embed_sha, tracked separately from files.sha so embedding freshness can be gated independently of parse freshness (see makeIndexer in worker.ts). A pre-existing v1 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, so a genuine ALTER TABLE failure is never silently lost.
|
|
1005
|
-
1: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN embed_sha TEXT"),
|
|
1006
|
-
// v2 -> v3: adds files.retry_count, a durable per-path counter for consecutive transient-read-failure requeues (see MAX_TRANSIENT_RETRIES / requeueDirtyPath / clearRetryCount in worker.ts). Previously this counter lived only in an in-memory Map inside worker.ts, which meant the retry-count reset -- run at the time in the short-lived hook CLI process -- could never actually reach the long-lived detached daemon process's own copy of that Map: they are different Node processes with no shared memory, so the reset was a silent no-op in the real deployed topology. Persisting the counter in `files` makes it visible to both processes via the one thing they do share: the index DB. Same swallow-duplicate-column pattern as v1 -> v2 above.
|
|
1007
|
-
2: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"),
|
|
1008
|
-
// v8 -> v9: adds symbols.parent (see SCHEMA_VERSION comment above for why). A pre-existing v8 database's `symbols` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 above.
|
|
1009
|
-
8: (conn) => alterTableIdempotent(conn, "ALTER TABLE symbols ADD COLUMN parent TEXT NOT NULL DEFAULT ''"),
|
|
1010
|
-
// v9 -> v10: adds hint_emissions.bytes_emitted (see SCHEMA_VERSION comment above for why). A pre-existing v9 database's `hint_emissions` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 above.
|
|
1011
|
-
9: (conn) => alterTableIdempotent(conn, "ALTER TABLE hint_emissions ADD COLUMN bytes_emitted INTEGER"),
|
|
1012
|
-
// v10 -> v11: purge chunks (and their vectors) for dotenv files. Until this version, a tracked `.env` was chunked and embedded verbatim on the git path, so `semantic` returned its values -- see dotenv_redact.ts. Redacting from now on is not enough on its own: the embed-freshness gate (isEmbedFresh in parser.ts) skips a file whose bytes have not changed, so an already-indexed .env would have kept serving its pre-fix chunks indefinitely. Deleting the rows here both removes the stored secrets and, by clearing embed_sha, makes the next drain re-embed the file through the redacting path.
|
|
1013
|
-
10: purgeDotenvEmbeddings,
|
|
1014
|
-
// v12 -> v13: adds files.parser_sha, the digest of the extraction logic that produced this file's rows, tracked separately from files.sha for the same reason embed_sha is -- content freshness and parse freshness are different questions, and the content sha alone could only ever answer the first. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 / v9 -> v10 above. Deliberately left NULL for every existing row rather than backfilled with the current fingerprint: NULL is the truthful answer (nobody recorded which parser wrote those rows), and it is also the answer that makes the freshness gates reparse them once, which is exactly what a database indexed by an older parser needs. v13 -> v14: changes both FTS5 tables' tokenizer to `unicode61 remove_diacritics 2`, so a search for `Noi` or `Viet` finds `Hà Nội` and `Việt Nam` -- combining marks that `remove_diacritics 1`, FTS5's default, leaves in place. This is the first schema change that `CREATE VIRTUAL TABLE IF NOT EXISTS` cannot express at all rather than merely cannot express on a populated table: against an existing virtual table that statement is a silent no-op, so without MIGRATIONS[13] the new tokenizer would reach only databases created after this release. The step drops both tables, re-runs FTS_SQL to re-create them at the current declaration, and rebuilds each from its content table.
|
|
1015
|
-
12: (conn) => alterTableIdempotent(conn, "ALTER TABLE files ADD COLUMN parser_sha TEXT"),
|
|
1016
|
-
// v13 -> v14: re-creates both FTS5 tables at the tokenizer FTS_SQL currently declares (see the SCHEMA_VERSION comment above for why no `IF NOT EXISTS` form can do this).
|
|
1017
|
-
13: rebuildFtsAtCurrentTokenizer
|
|
1018
|
-
};
|
|
1019
|
-
function runMigrations(conn, fromVersion, toVersion) {
|
|
1020
|
-
for (let v = fromVersion; v < toVersion; v++) {
|
|
1021
|
-
MIGRATIONS[v]?.(conn);
|
|
1022
|
-
}
|
|
1023
|
-
}
|
|
1024
|
-
var WAL_SWITCH_DEADLINE_MS = 15e3;
|
|
1025
|
-
function enableWalWithRetry(conn, budgetMs = WAL_SWITCH_DEADLINE_MS) {
|
|
1026
|
-
const deadline = Date.now() + budgetMs;
|
|
1027
|
-
let lastError;
|
|
1028
|
-
for (; ; ) {
|
|
1029
|
-
try {
|
|
1030
|
-
const mode = conn.pragma("journal_mode = WAL", { simple: true });
|
|
1031
|
-
if (String(mode).toLowerCase() === "wal") return;
|
|
1032
|
-
lastError = new Error(`got: ${String(mode)}`);
|
|
1033
|
-
} catch (e) {
|
|
1034
|
-
lastError = e;
|
|
1035
|
-
}
|
|
1036
|
-
try {
|
|
1037
|
-
if (String(conn.pragma("journal_mode", { simple: true })).toLowerCase() === "wal") return;
|
|
1038
|
-
} catch {
|
|
1039
|
-
}
|
|
1040
|
-
if (Date.now() >= deadline) {
|
|
1041
|
-
throw new Error(`db: failed to enable WAL mode (${lastError instanceof Error ? lastError.message : String(lastError)})`);
|
|
1042
|
-
}
|
|
1043
|
-
sleepSync(25);
|
|
1044
|
-
}
|
|
1045
|
-
}
|
|
1046
|
-
function initConnection(conn) {
|
|
1047
|
-
conn.pragma("busy_timeout = 15000");
|
|
1048
|
-
enableWalWithRetry(conn);
|
|
1049
|
-
conn.pragma("synchronous = NORMAL");
|
|
1050
|
-
conn.pragma("cache_size = -32000");
|
|
1051
|
-
conn.pragma("temp_store = MEMORY");
|
|
1052
|
-
conn.pragma("mmap_size = 134217728");
|
|
1053
|
-
conn.function(
|
|
1054
|
-
"TG_LOWER",
|
|
1055
|
-
{ deterministic: true },
|
|
1056
|
-
(value) => value === null ? null : foldCase(String(value))
|
|
1057
|
-
);
|
|
1058
|
-
const storedVersion = Number(conn.pragma("user_version", { simple: true }));
|
|
1059
|
-
if (storedVersion > SCHEMA_VERSION) {
|
|
1060
|
-
throw new Error(
|
|
1061
|
-
`db: index schema version ${storedVersion} is newer than this token-goat build supports (expected ${SCHEMA_VERSION}). Update token-goat, or delete the stale index database and let it rebuild.`
|
|
1062
|
-
);
|
|
1063
|
-
}
|
|
1064
|
-
conn.exec(SCHEMA_SQL);
|
|
1065
|
-
try {
|
|
1066
|
-
conn.exec(FTS_SQL);
|
|
1067
|
-
} catch {
|
|
1068
|
-
}
|
|
1069
|
-
try {
|
|
1070
|
-
const sqliteVec = _require2("sqlite-vec");
|
|
1071
|
-
sqliteVec.load(conn);
|
|
1072
|
-
conn.exec(
|
|
1073
|
-
`CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vectors USING vec0(
|
|
1074
|
-
embedding float[384]
|
|
1075
|
-
);`
|
|
1076
|
-
);
|
|
1077
|
-
} catch {
|
|
1078
|
-
}
|
|
1079
|
-
if (storedVersion < SCHEMA_VERSION) {
|
|
1080
|
-
runMigrations(conn, storedVersion, SCHEMA_VERSION);
|
|
1081
|
-
conn.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
1082
|
-
}
|
|
1083
|
-
}
|
|
1084
|
-
function resolveDbPath(dbPath) {
|
|
1085
|
-
if (path2.isAbsolute(dbPath)) return dbPath;
|
|
1086
|
-
if (dbPath.includes("/") || dbPath.includes("\\")) return path2.resolve(dbPath);
|
|
1087
|
-
return safeJoin(dataDir(), dbPath);
|
|
1088
|
-
}
|
|
1089
|
-
function connectionKey(dbPath) {
|
|
1090
|
-
const resolved = resolveDbPath(dbPath);
|
|
1091
|
-
return { resolved, key: foldPath(resolved) };
|
|
1092
|
-
}
|
|
1093
|
-
function getDb(dbPath) {
|
|
1094
|
-
const { resolved, key } = connectionKey(dbPath);
|
|
1095
|
-
const existing = _connections.get(key);
|
|
1096
|
-
if (existing !== void 0) return existing;
|
|
1097
|
-
const dir = path2.dirname(resolved);
|
|
1098
|
-
try {
|
|
1099
|
-
ensureDirSync(dir);
|
|
1100
|
-
} catch (e) {
|
|
1101
|
-
if (e.code !== "EEXIST" || !fs3.existsSync(dir)) throw e;
|
|
1102
|
-
}
|
|
1103
|
-
const conn = new Database(resolved);
|
|
1104
|
-
try {
|
|
1105
|
-
initConnection(conn);
|
|
1106
|
-
} catch (e) {
|
|
1107
|
-
try {
|
|
1108
|
-
conn.close();
|
|
1109
|
-
} catch {
|
|
1110
|
-
}
|
|
1111
|
-
throw e;
|
|
1112
|
-
}
|
|
1113
|
-
_connections.set(key, conn);
|
|
1114
|
-
return conn;
|
|
1115
|
-
}
|
|
1116
|
-
function closeAllDbs() {
|
|
1117
|
-
for (const conn of _connections.values()) {
|
|
1118
|
-
try {
|
|
1119
|
-
conn.close();
|
|
1120
|
-
} catch {
|
|
1121
|
-
}
|
|
1122
|
-
}
|
|
1123
|
-
_connections.clear();
|
|
1124
|
-
}
|
|
1125
|
-
registerReset(closeAllDbs);
|
|
1126
|
-
|
|
1127
34
|
// src/render/ansi.ts
|
|
1128
35
|
init_define_import_meta_env();
|
|
1129
36
|
function _colorStream(isatty) {
|
|
@@ -1201,7 +108,7 @@ var C = {
|
|
|
1201
108
|
|
|
1202
109
|
// src/stats.ts
|
|
1203
110
|
init_define_import_meta_env();
|
|
1204
|
-
import * as
|
|
111
|
+
import * as path from "node:path";
|
|
1205
112
|
|
|
1206
113
|
// src/render/stats_renderer.ts
|
|
1207
114
|
init_define_import_meta_env();
|
|
@@ -1566,6 +473,7 @@ var _KIND_GROUPS = [
|
|
|
1566
473
|
"gdrive_image",
|
|
1567
474
|
"webfetch_image",
|
|
1568
475
|
"image_shrink_skipped",
|
|
476
|
+
"image_shrink_over_pixel_limit",
|
|
1569
477
|
"image_shrink_cache_hit",
|
|
1570
478
|
"image_ocr"
|
|
1571
479
|
])
|
|
@@ -1965,6 +873,7 @@ var KIND_TO_SOURCE = {
|
|
|
1965
873
|
image_shrink: SOURCE_IMAGE,
|
|
1966
874
|
image_shrink_cache_hit: SOURCE_IMAGE,
|
|
1967
875
|
image_shrink_skipped: SOURCE_IMAGE,
|
|
876
|
+
image_shrink_over_pixel_limit: SOURCE_IMAGE,
|
|
1968
877
|
image_ocr: SOURCE_IMAGE,
|
|
1969
878
|
webfetch_image: SOURCE_IMAGE,
|
|
1970
879
|
gdrive_image: SOURCE_IMAGE,
|
|
@@ -2329,7 +1238,7 @@ function statsHasVersionColumn(db) {
|
|
|
2329
1238
|
}
|
|
2330
1239
|
function getGlobalDb(homeDir) {
|
|
2331
1240
|
const basePath = homeDir ? dataDirForHome(homeDir) : dataDir();
|
|
2332
|
-
const dbPath =
|
|
1241
|
+
const dbPath = path.join(basePath, "global.db");
|
|
2333
1242
|
const db = getDb(dbPath);
|
|
2334
1243
|
if (!_globalSchemaApplied.has(dbPath)) {
|
|
2335
1244
|
db.exec(GLOBAL_SCHEMA_SQL);
|
|
@@ -2470,10 +1379,10 @@ function summarize(windowDays = 30, testDb, homeDir) {
|
|
|
2470
1379
|
].join(", ");
|
|
2471
1380
|
const query = sinceTs !== null ? `SELECT ${cols} FROM stats WHERE ts >= ? ORDER BY ts DESC` : `SELECT ${cols} FROM stats ORDER BY ts DESC`;
|
|
2472
1381
|
const stmt = db.prepare(query);
|
|
2473
|
-
const
|
|
1382
|
+
const rows = sinceTs !== null ? stmt.all(sinceTs) : stmt.all();
|
|
2474
1383
|
const tsToDateCache = {};
|
|
2475
1384
|
const counts = {};
|
|
2476
|
-
for (const row of
|
|
1385
|
+
for (const row of rows) {
|
|
2477
1386
|
const bytesSaved = row.bytes_saved ?? 0;
|
|
2478
1387
|
const recorded = row.tokens_saved ?? 0;
|
|
2479
1388
|
const kind = row.kind;
|
|
@@ -2895,23 +1804,23 @@ function redactSecrets(text, config = loadConfig()) {
|
|
|
2895
1804
|
|
|
2896
1805
|
// src/disk_cache.ts
|
|
2897
1806
|
init_define_import_meta_env();
|
|
2898
|
-
import * as
|
|
2899
|
-
import * as
|
|
1807
|
+
import * as fs from "node:fs";
|
|
1808
|
+
import * as path2 from "node:path";
|
|
2900
1809
|
var DEFAULT_MAX_COUNT = 200;
|
|
2901
1810
|
var DEFAULT_MAX_AGE_MS = 24 * 3600 * 1e3;
|
|
2902
1811
|
function sanitizeId(id) {
|
|
2903
1812
|
return sanitizeIdForFilename(id, 64);
|
|
2904
1813
|
}
|
|
2905
1814
|
function blobDir(subdir) {
|
|
2906
|
-
return
|
|
1815
|
+
return path2.join(tokenGoatHome(), subdir);
|
|
2907
1816
|
}
|
|
2908
1817
|
function blobPath(subdir, id) {
|
|
2909
1818
|
const safe = sanitizeId(id);
|
|
2910
1819
|
if (!safe) return null;
|
|
2911
1820
|
const dir = blobDir(subdir);
|
|
2912
|
-
const candidate =
|
|
1821
|
+
const candidate = path2.join(dir, `${safe}.json`);
|
|
2913
1822
|
try {
|
|
2914
|
-
const rel =
|
|
1823
|
+
const rel = path2.relative(dir, candidate);
|
|
2915
1824
|
if (rel.startsWith("..")) return null;
|
|
2916
1825
|
} catch {
|
|
2917
1826
|
return null;
|
|
@@ -2922,7 +1831,7 @@ function isBlobStale(subdir, id) {
|
|
|
2922
1831
|
const p = blobPath(subdir, id);
|
|
2923
1832
|
if (p === null) return false;
|
|
2924
1833
|
try {
|
|
2925
|
-
const stat =
|
|
1834
|
+
const stat = fs.statSync(p);
|
|
2926
1835
|
return Date.now() - stat.mtimeMs > DEFAULT_MAX_AGE_MS;
|
|
2927
1836
|
} catch {
|
|
2928
1837
|
return false;
|
|
@@ -2963,8 +1872,8 @@ function storeBlob(subdir, id, value, opts = {}) {
|
|
|
2963
1872
|
}
|
|
2964
1873
|
if (Number.isFinite(maxBytesPerItem) && Buffer.byteLength(json, "utf-8") > maxBytesPerItem) return false;
|
|
2965
1874
|
try {
|
|
2966
|
-
const dir =
|
|
2967
|
-
if (!
|
|
1875
|
+
const dir = path2.dirname(p);
|
|
1876
|
+
if (!fs.existsSync(dir)) ensureDirSync(dir);
|
|
2968
1877
|
atomicWriteText(p, json);
|
|
2969
1878
|
} catch {
|
|
2970
1879
|
return false;
|
|
@@ -2982,8 +1891,8 @@ function loadBlob(subdir, id) {
|
|
|
2982
1891
|
const p = blobPath(subdir, id);
|
|
2983
1892
|
if (!p) return null;
|
|
2984
1893
|
try {
|
|
2985
|
-
if (!
|
|
2986
|
-
return JSON.parse(
|
|
1894
|
+
if (!fs.existsSync(p)) return null;
|
|
1895
|
+
return JSON.parse(fs.readFileSync(p, "utf8"));
|
|
2987
1896
|
} catch {
|
|
2988
1897
|
return null;
|
|
2989
1898
|
}
|
|
@@ -2992,13 +1901,13 @@ function listBlobs(subdir) {
|
|
|
2992
1901
|
const dir = blobDir(subdir);
|
|
2993
1902
|
const out = [];
|
|
2994
1903
|
try {
|
|
2995
|
-
if (!
|
|
2996
|
-
for (const file of
|
|
1904
|
+
if (!fs.existsSync(dir)) return out;
|
|
1905
|
+
for (const file of fs.readdirSync(dir)) {
|
|
2997
1906
|
if (!file.endsWith(".json")) continue;
|
|
2998
1907
|
const id = file.slice(0, -5);
|
|
2999
1908
|
let mtime = 0;
|
|
3000
1909
|
try {
|
|
3001
|
-
mtime =
|
|
1910
|
+
mtime = fs.statSync(path2.join(dir, file)).mtimeMs;
|
|
3002
1911
|
} catch {
|
|
3003
1912
|
}
|
|
3004
1913
|
const value = loadBlob(subdir, id);
|
|
@@ -3015,15 +1924,15 @@ function pruneBlobs(subdir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX
|
|
|
3015
1924
|
function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_AGE_MS, maxBytes = Number.POSITIVE_INFINITY, protectedPath) {
|
|
3016
1925
|
let removed = 0;
|
|
3017
1926
|
try {
|
|
3018
|
-
if (!
|
|
1927
|
+
if (!fs.existsSync(dir)) return 0;
|
|
3019
1928
|
const cutoff = Date.now() - maxAgeMs;
|
|
3020
1929
|
let kept = [];
|
|
3021
1930
|
let protectedEntry;
|
|
3022
|
-
for (const file of
|
|
3023
|
-
const full =
|
|
1931
|
+
for (const file of fs.readdirSync(dir)) {
|
|
1932
|
+
const full = path2.join(dir, file);
|
|
3024
1933
|
let stat;
|
|
3025
1934
|
try {
|
|
3026
|
-
stat =
|
|
1935
|
+
stat = fs.statSync(full);
|
|
3027
1936
|
} catch {
|
|
3028
1937
|
continue;
|
|
3029
1938
|
}
|
|
@@ -3034,7 +1943,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
3034
1943
|
}
|
|
3035
1944
|
if (stat.mtimeMs < cutoff) {
|
|
3036
1945
|
try {
|
|
3037
|
-
|
|
1946
|
+
fs.unlinkSync(full);
|
|
3038
1947
|
removed++;
|
|
3039
1948
|
} catch {
|
|
3040
1949
|
continue;
|
|
@@ -3050,7 +1959,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
3050
1959
|
kept = kept.slice(kept.length - countBudget);
|
|
3051
1960
|
for (const [full] of excess) {
|
|
3052
1961
|
try {
|
|
3053
|
-
|
|
1962
|
+
fs.unlinkSync(full);
|
|
3054
1963
|
removed++;
|
|
3055
1964
|
} catch {
|
|
3056
1965
|
continue;
|
|
@@ -3065,7 +1974,7 @@ function pruneBlobDir(dir, maxCount = DEFAULT_MAX_COUNT, maxAgeMs = DEFAULT_MAX_
|
|
|
3065
1974
|
if (!oldest) break;
|
|
3066
1975
|
const [full, , size] = oldest;
|
|
3067
1976
|
try {
|
|
3068
|
-
|
|
1977
|
+
fs.unlinkSync(full);
|
|
3069
1978
|
removed++;
|
|
3070
1979
|
total -= size;
|
|
3071
1980
|
} catch {
|
|
@@ -3095,7 +2004,7 @@ function sweepCacheRoots(extraRoots = []) {
|
|
|
3095
2004
|
try {
|
|
3096
2005
|
const defaults = subdirCacheDefaults(subdir);
|
|
3097
2006
|
removed += pruneBlobDir(
|
|
3098
|
-
|
|
2007
|
+
path2.join(root, subdir),
|
|
3099
2008
|
countCapped ? defaults.maxCount : Number.POSITIVE_INFINITY,
|
|
3100
2009
|
DEFAULT_MAX_AGE_MS,
|
|
3101
2010
|
countCapped ? defaults.maxBytes : Number.POSITIVE_INFINITY
|
|
@@ -3108,22 +2017,6 @@ function sweepCacheRoots(extraRoots = []) {
|
|
|
3108
2017
|
}
|
|
3109
2018
|
|
|
3110
2019
|
export {
|
|
3111
|
-
Database,
|
|
3112
|
-
FILENAME_LANGUAGE,
|
|
3113
|
-
TREE_SITTER_LANGUAGES,
|
|
3114
|
-
languageHasFlag,
|
|
3115
|
-
languageLabel,
|
|
3116
|
-
fenceFor,
|
|
3117
|
-
basenameImportsExtension,
|
|
3118
|
-
partialRefsReason,
|
|
3119
|
-
refineLanguageByContent,
|
|
3120
|
-
detectLanguageOfFile,
|
|
3121
|
-
detectLanguage,
|
|
3122
|
-
nonTreeSitterLanguageCount,
|
|
3123
|
-
unsupportedLanguageName,
|
|
3124
|
-
isDotenvPath,
|
|
3125
|
-
redactIfDotenv,
|
|
3126
|
-
getDb,
|
|
3127
2020
|
colorStdout,
|
|
3128
2021
|
RESET,
|
|
3129
2022
|
stripAnsiEscapes,
|