sift-light 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/CHANGELOG.md +0 -0
  2. package/CONTRIBUTING.md +19 -0
  3. package/LICENSE +661 -0
  4. package/README.md +177 -0
  5. package/README.zh-CN.md +177 -0
  6. package/SECURITY.md +25 -0
  7. package/package.json +142 -0
  8. package/plugins/sift-light/.claude-plugin/plugin.json +10 -0
  9. package/plugins/sift-light/.codex-plugin/plugin.json +20 -0
  10. package/plugins/sift-light/.mcp.json +24 -0
  11. package/plugins/sift-light/LICENSE +661 -0
  12. package/plugins/sift-light/concept-worker.mjs +567 -0
  13. package/plugins/sift-light/hooks/THIRD_PARTY_NOTICES.txt +70 -0
  14. package/plugins/sift-light/hooks/hooks.json +16 -0
  15. package/plugins/sift-light/hooks/search-policy.mjs +4113 -0
  16. package/plugins/sift-light/hooks/tree-sitter-bash.wasm +0 -0
  17. package/plugins/sift-light/hooks/tree-sitter-powershell.wasm +0 -0
  18. package/plugins/sift-light/hooks/tree-sitter.wasm +0 -0
  19. package/plugins/sift-light/kimi.plugin.json +36 -0
  20. package/plugins/sift-light/mcp-server.mjs +1 -0
  21. package/plugins/sift-light/omp-extension.mjs +20914 -0
  22. package/plugins/sift-light/package.json +11 -0
  23. package/plugins/sift-light/syntax-worker.mjs +280 -0
  24. package/plugins/sift-light/syntax-worker.toml +2 -0
  25. package/src/analysis-evidence.ts +43 -0
  26. package/src/analysis-limits.ts +27 -0
  27. package/src/analysis-store.ts +559 -0
  28. package/src/analysis-term-pages.ts +73 -0
  29. package/src/analysis-types.ts +162 -0
  30. package/src/audit-search.ts +391 -0
  31. package/src/capped-lines.ts +112 -0
  32. package/src/concept-embedding-cache.ts +186 -0
  33. package/src/concept-inference.ts +218 -0
  34. package/src/concept-model.ts +96 -0
  35. package/src/concept-search.ts +577 -0
  36. package/src/concept-setup.ts +75 -0
  37. package/src/concept-source-generation.ts +268 -0
  38. package/src/concept-worker.mjs +567 -0
  39. package/src/concept-worker.ts +210 -0
  40. package/src/config-reader.ts +283 -0
  41. package/src/config.ts +37 -0
  42. package/src/context-budget.ts +26 -0
  43. package/src/discovery-errors.ts +2 -0
  44. package/src/errors.ts +95 -0
  45. package/src/evidence-candidates.ts +344 -0
  46. package/src/evidence-ranking.ts +48 -0
  47. package/src/evidence-service.ts +1047 -0
  48. package/src/evidence-validation.ts +466 -0
  49. package/src/evidence-validity.ts +33 -0
  50. package/src/excerpt.ts +53 -0
  51. package/src/file-discovery.ts +150 -0
  52. package/src/file-metadata-filter.ts +46 -0
  53. package/src/format.ts +407 -0
  54. package/src/git-diff.ts +195 -0
  55. package/src/git-process.ts +189 -0
  56. package/src/git-repository.ts +367 -0
  57. package/src/git-source.ts +385 -0
  58. package/src/historical-paths.ts +205 -0
  59. package/src/hybrid-search.ts +340 -0
  60. package/src/import-model.ts +611 -0
  61. package/src/import-navigation.ts +100 -0
  62. package/src/import-resolution.ts +240 -0
  63. package/src/index.ts +97 -0
  64. package/src/inspect-batch.ts +265 -0
  65. package/src/inspect.ts +271 -0
  66. package/src/language-capabilities.ts +157 -0
  67. package/src/language-capability-definitions.ts +204 -0
  68. package/src/literal-search.ts +25 -0
  69. package/src/mcp-cli.ts +21 -0
  70. package/src/mcp-model-output.ts +120 -0
  71. package/src/mcp-output.ts +9 -0
  72. package/src/mcp-semantic-judge.ts +39 -0
  73. package/src/mcp-server.mjs +14064 -0
  74. package/src/mcp-server.ts +157 -0
  75. package/src/mcp-stdio.ts +114 -0
  76. package/src/mcp.ts +631 -0
  77. package/src/model-error.ts +120 -0
  78. package/src/multi-term-search.ts +122 -0
  79. package/src/omp-index.ts +277 -0
  80. package/src/omp-marketplace-entry.ts +7 -0
  81. package/src/operation-lifecycle.ts +413 -0
  82. package/src/operation-output.ts +100 -0
  83. package/src/owned-parallel.ts +22 -0
  84. package/src/owned-process.ts +102 -0
  85. package/src/owned-task-queue.ts +38 -0
  86. package/src/package-version.ts +3 -0
  87. package/src/path-policy.ts +203 -0
  88. package/src/pi-search-policy.ts +33 -0
  89. package/src/prompt-guidelines.ts +54 -0
  90. package/src/python-outline.ts +231 -0
  91. package/src/record-value.ts +3 -0
  92. package/src/redaction.ts +95 -0
  93. package/src/request-contract-catalog.ts +215 -0
  94. package/src/request-contract-error.ts +96 -0
  95. package/src/request-contract.ts +481 -0
  96. package/src/request.ts +132 -0
  97. package/src/result-statistics.ts +178 -0
  98. package/src/rg.ts +574 -0
  99. package/src/ripgrep-diagnostics.ts +122 -0
  100. package/src/ripgrep-executable.ts +43 -0
  101. package/src/runtime.ts +53 -0
  102. package/src/scan-revisions.ts +148 -0
  103. package/src/script-runtime.ts +11 -0
  104. package/src/search-policy-commands.ts +170 -0
  105. package/src/search-policy-hook.ts +61 -0
  106. package/src/search-policy-recovery.ts +388 -0
  107. package/src/search-policy-shell.ts +280 -0
  108. package/src/search-policy.ts +102 -0
  109. package/src/search-retention.ts +77 -0
  110. package/src/semantic-judge-batches.ts +135 -0
  111. package/src/semantic-judge.ts +519 -0
  112. package/src/service.ts +945 -0
  113. package/src/session-summary.ts +100 -0
  114. package/src/snapshot-store.ts +203 -0
  115. package/src/source-access.ts +288 -0
  116. package/src/source-continuations.ts +168 -0
  117. package/src/source-document.ts +277 -0
  118. package/src/source-inspection.ts +731 -0
  119. package/src/source-pages.ts +112 -0
  120. package/src/source.ts +286 -0
  121. package/src/structural-search.ts +125 -0
  122. package/src/structure.ts +259 -0
  123. package/src/summary-previews.ts +96 -0
  124. package/src/summary.ts +79 -0
  125. package/src/syntax-facts.ts +464 -0
  126. package/src/syntax-fields.ts +161 -0
  127. package/src/syntax-search.ts +361 -0
  128. package/src/syntax-tree.ts +30 -0
  129. package/src/syntax-types.ts +72 -0
  130. package/src/syntax-worker.mjs +280 -0
  131. package/src/syntax-worker.toml +2 -0
  132. package/src/syntax-worker.ts +143 -0
  133. package/src/syntax.ts +259 -0
  134. package/src/test-navigation-facts.ts +261 -0
  135. package/src/test-navigation.ts +543 -0
  136. package/src/tool-schema.ts +321 -0
  137. package/src/tui/dashboard.ts +107 -0
  138. package/src/tui/layout.ts +291 -0
  139. package/src/tui/palette.ts +73 -0
  140. package/src/tui/presentation.ts +240 -0
  141. package/src/tui/renderers.ts +106 -0
  142. package/src/types.ts +368 -0
  143. package/src/validation-output.ts +148 -0
  144. package/src/validation-types.ts +48 -0
  145. package/src/workspace-files.ts +142 -0
@@ -0,0 +1,280 @@
1
+ // src/syntax-worker.ts
2
+ import { parse, registerDynamicLanguage } from "@ast-grep/napi";
3
+ import go from "@ast-grep/lang-go";
4
+
5
+ // src/analysis-limits.ts
6
+ var MAX_STRUCTURE_BYTES = 32 * 1024 * 1024;
7
+ var MAX_SYNTAX_NODES = 1e5;
8
+ var MAX_ANALYSIS_STORAGE_BYTES = 32 * 1024 * 1024;
9
+ var ANALYSIS_METADATA_RESERVE_BYTES = 64 * 1024;
10
+ var MAX_ANALYSIS_REASON_BYTES = 4 * 1024;
11
+ var ANALYSIS_TTL_MS = 10 * 60 * 1000;
12
+ var MAX_SOURCE_CONTINUATION_BYTES = 1024 * 1024;
13
+
14
+ // src/syntax-fields.ts
15
+ var JAVASCRIPT_FIELDS = {
16
+ arrow_function: ["body", "parameter", "parameters", "return_type", "type_parameters"],
17
+ assignment_expression: ["left", "right"],
18
+ assignment_pattern: ["left", "right"],
19
+ augmented_assignment_expression: ["left", "operator", "right"],
20
+ binary_expression: ["left", "operator", "right"],
21
+ break_statement: ["label"],
22
+ call_expression: ["arguments", "function", "optional_chain", "type_arguments"],
23
+ catch_clause: ["body", "parameter", "type"],
24
+ class: ["body", "decorator", "name", "type_parameters"],
25
+ class_body: ["decorator", "member"],
26
+ class_declaration: ["body", "decorator", "name", "type_parameters"],
27
+ class_static_block: ["body"],
28
+ continue_statement: ["label"],
29
+ do_statement: ["body", "condition"],
30
+ export_specifier: ["alias", "name"],
31
+ export_statement: ["declaration", "decorator", "source", "value"],
32
+ field_definition: ["decorator", "property", "value"],
33
+ finally_clause: ["body"],
34
+ for_in_statement: ["body", "kind", "left", "operator", "right", "value"],
35
+ for_statement: ["body", "condition", "increment", "initializer"],
36
+ function_declaration: ["body", "name", "parameters", "return_type", "type_parameters"],
37
+ function_expression: ["body", "name", "parameters", "return_type", "type_parameters"],
38
+ generator_function: ["body", "name", "parameters", "return_type", "type_parameters"],
39
+ generator_function_declaration: ["body", "name", "parameters", "return_type", "type_parameters"],
40
+ if_statement: ["alternative", "condition", "consequence"],
41
+ import_specifier: ["alias", "name"],
42
+ import_statement: ["source"],
43
+ jsx_closing_element: ["name"],
44
+ jsx_element: ["close_tag", "open_tag"],
45
+ jsx_opening_element: ["attribute", "name", "type_arguments"],
46
+ jsx_self_closing_element: ["attribute", "name", "type_arguments"],
47
+ labeled_statement: ["body", "label"],
48
+ lexical_declaration: ["kind"],
49
+ member_expression: ["object", "optional_chain", "property"],
50
+ method_definition: ["body", "decorator", "name", "parameters", "return_type", "type_parameters"],
51
+ new_expression: ["arguments", "constructor", "type_arguments"],
52
+ object_assignment_pattern: ["left", "right"],
53
+ pair: ["key", "value"],
54
+ pair_pattern: ["key", "value"],
55
+ regex: ["flags", "pattern"],
56
+ subscript_expression: ["index", "object", "optional_chain"],
57
+ switch_case: ["body", "value"],
58
+ switch_default: ["body"],
59
+ switch_statement: ["body", "value"],
60
+ ternary_expression: ["alternative", "condition", "consequence"],
61
+ try_statement: ["body", "finalizer", "handler"],
62
+ unary_expression: ["argument", "operator"],
63
+ update_expression: ["argument", "operator"],
64
+ using_declaration: ["kind"],
65
+ variable_declarator: ["name", "type", "value"],
66
+ while_statement: ["body", "condition"],
67
+ with_statement: ["body", "object"],
68
+ abstract_class_declaration: ["body", "decorator", "name", "type_parameters"],
69
+ abstract_method_signature: ["name", "parameters", "return_type", "type_parameters"],
70
+ call_signature: ["parameters", "return_type", "type_parameters"],
71
+ conditional_type: ["alternative", "consequence", "left", "right"],
72
+ construct_signature: ["parameters", "type", "type_parameters"],
73
+ constructor_type: ["parameters", "type", "type_parameters"],
74
+ enum_assignment: ["name", "value"],
75
+ enum_body: ["name"],
76
+ enum_declaration: ["body", "name"],
77
+ extends_clause: ["type_arguments", "value"],
78
+ extends_type_clause: ["type"],
79
+ function_signature: ["name", "parameters", "return_type", "type_parameters"],
80
+ function_type: ["parameters", "return_type", "type_parameters"],
81
+ generic_type: ["name", "type_arguments"],
82
+ import_require_clause: ["source"],
83
+ index_signature: ["index_type", "name", "sign", "type"],
84
+ instantiation_expression: ["function", "type_arguments"],
85
+ interface_declaration: ["body", "name", "type_parameters"],
86
+ internal_module: ["body", "name"],
87
+ mapped_type_clause: ["alias", "name", "type"],
88
+ method_signature: ["name", "parameters", "return_type", "type_parameters"],
89
+ module: ["body", "name"],
90
+ nested_identifier: ["object", "property"],
91
+ nested_type_identifier: ["module", "name"],
92
+ optional_parameter: ["decorator", "name", "pattern", "type", "value"],
93
+ parenthesized_expression: ["type"],
94
+ property_signature: ["name", "type"],
95
+ public_field_definition: ["decorator", "name", "type", "value"],
96
+ required_parameter: ["decorator", "name", "pattern", "type", "value"],
97
+ type_alias_declaration: ["name", "type_parameters", "value"],
98
+ type_parameter: ["constraint", "name", "value"],
99
+ type_predicate: ["name", "type"]
100
+ };
101
+ var GO_FIELDS = {
102
+ array_type: ["element", "length"],
103
+ assignment_statement: ["left", "operator", "right"],
104
+ binary_expression: ["left", "operator", "right"],
105
+ call_expression: ["arguments", "function", "type_arguments"],
106
+ channel_type: ["value"],
107
+ communication_case: ["communication"],
108
+ composite_literal: ["body", "type"],
109
+ const_spec: ["name", "type", "value"],
110
+ expression_case: ["value"],
111
+ expression_switch_statement: ["initializer", "value"],
112
+ field_declaration: ["name", "tag", "type"],
113
+ for_clause: ["condition", "initializer", "update"],
114
+ for_statement: ["body"],
115
+ func_literal: ["body", "parameters", "result"],
116
+ function_declaration: ["body", "name", "parameters", "result", "type_parameters"],
117
+ function_type: ["parameters", "result"],
118
+ generic_type: ["type", "type_arguments"],
119
+ if_statement: ["alternative", "condition", "consequence", "initializer"],
120
+ implicit_length_array_type: ["element"],
121
+ import_spec: ["name", "path"],
122
+ index_expression: ["index", "operand"],
123
+ keyed_element: ["key", "value"],
124
+ labeled_statement: ["label"],
125
+ map_type: ["key", "value"],
126
+ method_declaration: ["body", "name", "parameters", "receiver", "result"],
127
+ method_elem: ["name", "parameters", "result"],
128
+ parameter_declaration: ["name", "type"],
129
+ qualified_type: ["name", "package"],
130
+ range_clause: ["left", "right"],
131
+ receive_statement: ["left", "right"],
132
+ selector_expression: ["field", "operand"],
133
+ send_statement: ["channel", "value"],
134
+ short_var_declaration: ["left", "right"],
135
+ slice_expression: ["capacity", "end", "operand", "start"],
136
+ slice_type: ["element"],
137
+ type_alias: ["name", "type", "type_parameters"],
138
+ type_assertion_expression: ["operand", "type"],
139
+ type_case: ["type"],
140
+ type_conversion_expression: ["operand", "type"],
141
+ type_instantiation_expression: ["type"],
142
+ type_parameter_declaration: ["name", "type"],
143
+ type_spec: ["name", "type", "type_parameters"],
144
+ type_switch_statement: ["alias", "initializer", "value"],
145
+ unary_expression: ["operand", "operator"],
146
+ var_spec: ["name", "type", "value"],
147
+ variadic_parameter_declaration: ["name", "type"]
148
+ };
149
+
150
+ // src/types.ts
151
+ var MAX_SEARCH_STORAGE_BYTES = 32 * 1024 * 1024;
152
+ var MAX_RESULT_BYTES = 16 * 1024;
153
+ var MAX_PROTOCOL_LINE_BYTES = 16 * 1024 * 1024;
154
+ var MAX_SOURCE_FILE_BYTES = 5 * 1024 * 1024;
155
+ var MAX_PATTERN_CHARACTERS = 64 * 1024;
156
+
157
+ // src/syntax-worker.ts
158
+ function fieldsFor(node, language) {
159
+ const fields = new Map;
160
+ const names = (language === "go" ? GO_FIELDS : JAVASCRIPT_FIELDS)[node.kind()] ?? [];
161
+ for (const name of names) {
162
+ for (const child of node.fieldChildren(name))
163
+ fields.set(child.id(), name);
164
+ }
165
+ return fields;
166
+ }
167
+ function languageName(value) {
168
+ if (value === "javascript" || value === "typescript" || value === "tsx" || value === "go") {
169
+ return value;
170
+ }
171
+ throw new Error("Invalid syntax worker language");
172
+ }
173
+ function parseInput(input) {
174
+ if (!input || typeof input !== "object" || !("language" in input) || !("text" in input)) {
175
+ throw new Error("Invalid syntax worker input");
176
+ }
177
+ const language = languageName(input.language);
178
+ if (typeof input.text !== "string" || !input.text.isWellFormed()) {
179
+ throw new Error("Syntax worker input must be well-formed Unicode");
180
+ }
181
+ if (Buffer.byteLength(input.text) > MAX_SOURCE_FILE_BYTES) {
182
+ throw new Error("Syntax worker source exceeds the file limit");
183
+ }
184
+ if ("pattern" in input && (typeof input.pattern !== "string" || Buffer.byteLength(input.pattern) > 4096))
185
+ throw new Error("Invalid AST pattern");
186
+ return {
187
+ language,
188
+ text: input.text,
189
+ ..."pattern" in input && typeof input.pattern === "string" ? { pattern: input.pattern } : {}
190
+ };
191
+ }
192
+ function extract(language, text, pattern) {
193
+ if (language === "go")
194
+ registerDynamicLanguage({ go });
195
+ const names = { javascript: "JavaScript", typescript: "TypeScript", tsx: "Tsx", go: "go" };
196
+ if (pattern !== undefined) {
197
+ const prepared = language === "go" ? pattern.replaceAll("$", "µ") : pattern;
198
+ const templates = language === "go" ? [
199
+ prepared,
200
+ `package pattern
201
+ ${prepared}`,
202
+ `package pattern
203
+ func pattern() {
204
+ ${prepared}
205
+ }`
206
+ ] : [prepared];
207
+ const valid = templates.some((candidate) => {
208
+ const template = parse(names[language], candidate).root();
209
+ return template.kind() !== "ERROR" && !template.find({ rule: { kind: "ERROR" } });
210
+ });
211
+ if (!valid)
212
+ throw new Error("Malformed structural pattern for selected language");
213
+ }
214
+ const root = parse(names[language], text).root();
215
+ const rootRange = root.range();
216
+ const nodes = [
217
+ {
218
+ kind: String(root.kind()),
219
+ start: rootRange.start.index,
220
+ end: rootRange.end.index,
221
+ parent: null,
222
+ named: root.isNamed()
223
+ }
224
+ ];
225
+ const stack = [{ node: root, index: 0, next: 0, fields: fieldsFor(root, language) }];
226
+ let malformed = root.kind() === "ERROR";
227
+ while (stack.length > 0) {
228
+ const frame = stack.at(-1);
229
+ if (!frame)
230
+ break;
231
+ const child = frame.node.child(frame.next++);
232
+ if (!child) {
233
+ stack.pop();
234
+ continue;
235
+ }
236
+ if (nodes.length === MAX_SYNTAX_NODES)
237
+ return { status: "limit", nodes };
238
+ const range = child.range();
239
+ const kind = String(child.kind());
240
+ const field = frame.fields.get(child.id());
241
+ const node = {
242
+ kind,
243
+ start: range.start.index,
244
+ end: range.end.index,
245
+ parent: frame.index,
246
+ named: child.isNamed(),
247
+ ...field ? { field } : {}
248
+ };
249
+ if (kind === "ERROR" || node.start === node.end)
250
+ malformed = true;
251
+ const index = nodes.length;
252
+ nodes.push(node);
253
+ stack.push({ node: child, index, next: 0, fields: fieldsFor(child, language) });
254
+ }
255
+ const patternMatches = pattern === undefined || malformed ? undefined : root.findAll(pattern).map((node) => ({ start: node.range().start.index, end: node.range().end.index }));
256
+ return {
257
+ status: malformed ? "parse-error" : "ok",
258
+ nodes,
259
+ ...patternMatches ? { patternMatches } : {}
260
+ };
261
+ }
262
+ async function main() {
263
+ const chunks = [];
264
+ let bytes = 0;
265
+ for await (const chunk of process.stdin) {
266
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
267
+ bytes += buffer.length;
268
+ if (bytes > MAX_STRUCTURE_BYTES)
269
+ throw new Error("Syntax worker input exceeds protocol limit");
270
+ chunks.push(buffer);
271
+ }
272
+ const input = JSON.parse(Buffer.concat(chunks).toString("utf8"));
273
+ const { language, text, pattern } = parseInput(input);
274
+ const output = JSON.stringify(extract(language, text, pattern));
275
+ if (Buffer.byteLength(output) > MAX_STRUCTURE_BYTES) {
276
+ throw new Error("Syntax worker output exceeds protocol limit");
277
+ }
278
+ process.stdout.write(output);
279
+ }
280
+ await main();
@@ -0,0 +1,2 @@
1
+ # Parser runtime configuration, not user/project configuration.
2
+ preload = []
@@ -0,0 +1,143 @@
1
+ import { parse, registerDynamicLanguage, type SgNode } from "@ast-grep/napi";
2
+ import go from "@ast-grep/lang-go";
3
+ import { MAX_STRUCTURE_BYTES, MAX_SYNTAX_NODES } from "./analysis-limits.ts";
4
+ import { GO_FIELDS, JAVASCRIPT_FIELDS } from "./syntax-fields.ts";
5
+ import type { SyntaxLanguage, SyntaxNode, SyntaxWorkerResult } from "./syntax-types.ts";
6
+ import { MAX_SOURCE_FILE_BYTES } from "./types.ts";
7
+
8
+ interface Frame {
9
+ node: SgNode;
10
+ index: number;
11
+ next: number;
12
+ fields: Map<number, string>;
13
+ }
14
+
15
+ function fieldsFor(node: SgNode, language: SyntaxLanguage): Map<number, string> {
16
+ const fields = new Map<number, string>();
17
+ const names = (language === "go" ? GO_FIELDS : JAVASCRIPT_FIELDS)[node.kind()] ?? [];
18
+ for (const name of names) {
19
+ for (const child of node.fieldChildren(name)) fields.set(child.id(), name);
20
+ }
21
+ return fields;
22
+ }
23
+
24
+ function languageName(value: unknown): SyntaxLanguage {
25
+ if (value === "javascript" || value === "typescript" || value === "tsx" || value === "go") {
26
+ return value;
27
+ }
28
+ throw new Error("Invalid syntax worker language");
29
+ }
30
+
31
+ function parseInput(input: unknown): { language: SyntaxLanguage; text: string; pattern?: string } {
32
+ if (!input || typeof input !== "object" || !("language" in input) || !("text" in input)) {
33
+ throw new Error("Invalid syntax worker input");
34
+ }
35
+ const language = languageName(input.language);
36
+ if (typeof input.text !== "string" || !input.text.isWellFormed()) {
37
+ throw new Error("Syntax worker input must be well-formed Unicode");
38
+ }
39
+ if (Buffer.byteLength(input.text) > MAX_SOURCE_FILE_BYTES) {
40
+ throw new Error("Syntax worker source exceeds the file limit");
41
+ }
42
+ if (
43
+ "pattern" in input &&
44
+ (typeof input.pattern !== "string" || Buffer.byteLength(input.pattern) > 4_096)
45
+ )
46
+ throw new Error("Invalid AST pattern");
47
+ return {
48
+ language,
49
+ text: input.text,
50
+ ...("pattern" in input && typeof input.pattern === "string" ? { pattern: input.pattern } : {}),
51
+ };
52
+ }
53
+
54
+ function extract(language: SyntaxLanguage, text: string, pattern?: string): SyntaxWorkerResult {
55
+ if (language === "go") registerDynamicLanguage({ go });
56
+ const names = { javascript: "JavaScript", typescript: "TypeScript", tsx: "Tsx", go: "go" };
57
+ if (pattern !== undefined) {
58
+ // Go's registered expando character makes metavariables legal identifiers during validation.
59
+ const prepared = language === "go" ? pattern.replaceAll("$", "µ") : pattern;
60
+ const templates =
61
+ language === "go"
62
+ ? [
63
+ prepared,
64
+ `package pattern\n${prepared}`,
65
+ `package pattern\nfunc pattern() {\n${prepared}\n}`,
66
+ ]
67
+ : [prepared];
68
+ const valid = templates.some((candidate) => {
69
+ const template = parse(names[language], candidate).root();
70
+ return template.kind() !== "ERROR" && !template.find({ rule: { kind: "ERROR" } });
71
+ });
72
+ if (!valid) throw new Error("Malformed structural pattern for selected language");
73
+ }
74
+ const root = parse(names[language], text).root();
75
+ const rootRange = root.range();
76
+ const nodes: SyntaxNode[] = [
77
+ {
78
+ kind: String(root.kind()),
79
+ start: rootRange.start.index,
80
+ end: rootRange.end.index,
81
+ parent: null,
82
+ named: root.isNamed(),
83
+ },
84
+ ];
85
+ const stack: Frame[] = [{ node: root, index: 0, next: 0, fields: fieldsFor(root, language) }];
86
+ let malformed = root.kind() === "ERROR";
87
+ while (stack.length > 0) {
88
+ const frame = stack.at(-1);
89
+ if (!frame) break;
90
+ const child = frame.node.child(frame.next++);
91
+ if (!child) {
92
+ stack.pop();
93
+ continue;
94
+ }
95
+ if (nodes.length === MAX_SYNTAX_NODES) return { status: "limit", nodes };
96
+ const range = child.range();
97
+ const kind = String(child.kind());
98
+ const field = frame.fields.get(child.id());
99
+ const node: SyntaxNode = {
100
+ kind,
101
+ start: range.start.index,
102
+ end: range.end.index,
103
+ parent: frame.index,
104
+ named: child.isNamed(),
105
+ ...(field ? { field } : {}),
106
+ };
107
+ if (kind === "ERROR" || node.start === node.end) malformed = true;
108
+ const index = nodes.length;
109
+ nodes.push(node);
110
+ stack.push({ node: child, index, next: 0, fields: fieldsFor(child, language) });
111
+ }
112
+ const patternMatches =
113
+ pattern === undefined || malformed
114
+ ? undefined
115
+ : root
116
+ .findAll(pattern)
117
+ .map((node) => ({ start: node.range().start.index, end: node.range().end.index }));
118
+ return {
119
+ status: malformed ? "parse-error" : "ok",
120
+ nodes,
121
+ ...(patternMatches ? { patternMatches } : {}),
122
+ };
123
+ }
124
+
125
+ async function main(): Promise<void> {
126
+ const chunks: Buffer[] = [];
127
+ let bytes = 0;
128
+ for await (const chunk of process.stdin) {
129
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
130
+ bytes += buffer.length;
131
+ if (bytes > MAX_STRUCTURE_BYTES) throw new Error("Syntax worker input exceeds protocol limit");
132
+ chunks.push(buffer);
133
+ }
134
+ const input: unknown = JSON.parse(Buffer.concat(chunks).toString("utf8"));
135
+ const { language, text, pattern } = parseInput(input);
136
+ const output = JSON.stringify(extract(language, text, pattern));
137
+ if (Buffer.byteLength(output) > MAX_STRUCTURE_BYTES) {
138
+ throw new Error("Syntax worker output exceeds protocol limit");
139
+ }
140
+ process.stdout.write(output);
141
+ }
142
+
143
+ await main();
package/src/syntax.ts ADDED
@@ -0,0 +1,259 @@
1
+ import { dirname } from "node:path";
2
+ import { fileURLToPath } from "node:url";
3
+ import { MAX_PARSE_TIME_MS, MAX_STRUCTURE_BYTES, MAX_SYNTAX_NODES } from "./analysis-limits.js";
4
+ import { abortError, SiftLightError } from "./errors.js";
5
+ import { runOwnedProcess } from "./owned-process.js";
6
+ import { scriptRuntimeEnvironment } from "./script-runtime.js";
7
+ import { deriveSyntaxFacts } from "./syntax-facts.js";
8
+ import { syntaxChildren } from "./syntax-tree.js";
9
+ import type {
10
+ SyntaxAnalysis,
11
+ SyntaxLanguage,
12
+ SyntaxNode,
13
+ SyntaxStatus,
14
+ SyntaxWorkerResult,
15
+ } from "./syntax-types.js";
16
+ import { MAX_SOURCE_FILE_BYTES } from "./types.js";
17
+ import {
18
+ DEFAULT_LANGUAGE_CAPABILITIES,
19
+ languageForPath,
20
+ } from "./language-capability-definitions.js";
21
+
22
+ export type {
23
+ SyntaxAnalysis,
24
+ SyntaxDiagnostic,
25
+ SyntaxLanguage,
26
+ SyntaxNode,
27
+ SyntaxRole,
28
+ SyntaxRoleName,
29
+ SyntaxStatus,
30
+ SyntaxSymbol,
31
+ } from "./syntax-types.js";
32
+ export { classifySyntaxRange } from "./syntax-facts.js";
33
+ export { syntaxField, syntaxFields, syntaxText } from "./syntax-tree.js";
34
+
35
+ export function syntaxLanguage(path: string): SyntaxLanguage | undefined {
36
+ switch (languageForPath(DEFAULT_LANGUAGE_CAPABILITIES, path)) {
37
+ case "javascript":
38
+ return "javascript";
39
+ case "typescript":
40
+ return "typescript";
41
+ case "tsx":
42
+ return "tsx";
43
+ case "go":
44
+ return "go";
45
+ default:
46
+ return undefined;
47
+ }
48
+ }
49
+
50
+ function emptyAnalysis(status: SyntaxStatus, language?: SyntaxLanguage): SyntaxAnalysis {
51
+ return {
52
+ status,
53
+ ...(language ? { language } : {}),
54
+ nodes: [],
55
+ children: [],
56
+ symbols: [],
57
+ roles: [],
58
+ diagnostics: [],
59
+ limited: status === "limit",
60
+ };
61
+ }
62
+
63
+ function invalidProtocol(): never {
64
+ throw new SiftLightError("Invalid syntax parser protocol");
65
+ }
66
+
67
+ function readNode(
68
+ value: unknown,
69
+ index: number,
70
+ nodes: readonly SyntaxNode[],
71
+ length: number,
72
+ ): SyntaxNode {
73
+ if (!value || typeof value !== "object") return invalidProtocol();
74
+ if (
75
+ !("kind" in value) ||
76
+ typeof value.kind !== "string" ||
77
+ value.kind.length === 0 ||
78
+ !("start" in value) ||
79
+ typeof value.start !== "number" ||
80
+ !Number.isSafeInteger(value.start) ||
81
+ !("end" in value) ||
82
+ typeof value.end !== "number" ||
83
+ !Number.isSafeInteger(value.end) ||
84
+ value.start < 0 ||
85
+ value.end < value.start ||
86
+ value.end > length ||
87
+ !("named" in value) ||
88
+ typeof value.named !== "boolean" ||
89
+ !("parent" in value)
90
+ )
91
+ return invalidProtocol();
92
+ const parent = value.parent;
93
+ if (
94
+ index === 0
95
+ ? parent !== null
96
+ : typeof parent !== "number" || !Number.isSafeInteger(parent) || parent < 0 || parent >= index
97
+ ) {
98
+ return invalidProtocol();
99
+ }
100
+ if (typeof parent === "number") {
101
+ const owner = nodes[parent];
102
+ if (!owner || owner.start > value.start || owner.end < value.end) return invalidProtocol();
103
+ }
104
+ if ("field" in value && typeof value.field !== "string") return invalidProtocol();
105
+ return {
106
+ kind: value.kind,
107
+ start: value.start,
108
+ end: value.end,
109
+ parent: typeof parent === "number" ? parent : null,
110
+ named: value.named,
111
+ ...("field" in value && typeof value.field === "string" ? { field: value.field } : {}),
112
+ };
113
+ }
114
+
115
+ function readResult(output: string, length: number): SyntaxWorkerResult {
116
+ let result: unknown;
117
+ try {
118
+ result = JSON.parse(output);
119
+ } catch {
120
+ return invalidProtocol();
121
+ }
122
+ if (
123
+ !result ||
124
+ typeof result !== "object" ||
125
+ !("status" in result) ||
126
+ !("nodes" in result) ||
127
+ !["ok", "parse-error", "limit"].includes(String(result.status)) ||
128
+ !Array.isArray(result.nodes) ||
129
+ result.nodes.length === 0 ||
130
+ result.nodes.length > MAX_SYNTAX_NODES
131
+ ) {
132
+ return invalidProtocol();
133
+ }
134
+ const nodes: SyntaxNode[] = [];
135
+ for (const value of result.nodes) nodes.push(readNode(value, nodes.length, nodes, length));
136
+ if (result.status !== "ok" && result.status !== "parse-error" && result.status !== "limit")
137
+ return invalidProtocol();
138
+ const patternMatches: { start: number; end: number }[] = [];
139
+ if ("patternMatches" in result) {
140
+ if (!Array.isArray(result.patternMatches) || result.patternMatches.length > MAX_SYNTAX_NODES)
141
+ return invalidProtocol();
142
+ for (const match of result.patternMatches) {
143
+ if (
144
+ !match ||
145
+ typeof match !== "object" ||
146
+ !("start" in match) ||
147
+ !("end" in match) ||
148
+ typeof match.start !== "number" ||
149
+ typeof match.end !== "number" ||
150
+ !Number.isSafeInteger(match.start) ||
151
+ !Number.isSafeInteger(match.end) ||
152
+ match.start < 0 ||
153
+ match.end < match.start ||
154
+ match.end > length
155
+ )
156
+ return invalidProtocol();
157
+ patternMatches.push({ start: match.start, end: match.end });
158
+ }
159
+ }
160
+ return { status: result.status, nodes, patternMatches };
161
+ }
162
+
163
+ /** The service serializes requests; this owns exactly one short-lived native parser. */
164
+ export async function parseSyntax(
165
+ path: string,
166
+ text: string,
167
+ signal?: AbortSignal,
168
+ pattern?: string,
169
+ ): Promise<SyntaxAnalysis> {
170
+ if (signal?.aborted) throw abortError();
171
+ const language = syntaxLanguage(path);
172
+ if (!language) return emptyAnalysis("unsupported");
173
+ if (Buffer.byteLength(text) > MAX_SOURCE_FILE_BYTES) return emptyAnalysis("limit", language);
174
+ if (!text.isWellFormed()) {
175
+ return {
176
+ ...emptyAnalysis("parse-error", language),
177
+ diagnostics: [{ kind: "invalid-unicode", start: 0, end: text.length }],
178
+ };
179
+ }
180
+ const worker = fileURLToPath(new URL("./syntax-worker.mjs", import.meta.url));
181
+ const config = fileURLToPath(new URL("./syntax-worker.toml", import.meta.url));
182
+ const args = process.versions.bun
183
+ ? [`--config=${config}`, "--no-env-file", "--no-macros", "--no-install", worker]
184
+ : [worker];
185
+ const env = scriptRuntimeEnvironment();
186
+ const controller = new AbortController();
187
+ let timedOut = false;
188
+ const abort = () => controller.abort();
189
+ signal?.addEventListener("abort", abort, { once: true });
190
+ if (signal?.aborted) controller.abort();
191
+ const timer = setTimeout(() => {
192
+ timedOut = true;
193
+ controller.abort();
194
+ }, MAX_PARSE_TIME_MS);
195
+ const chunks: Buffer[] = [];
196
+ let bytes = 0;
197
+ try {
198
+ const result = await runOwnedProcess(
199
+ {
200
+ executable: process.execPath,
201
+ args,
202
+ cwd: dirname(worker),
203
+ env,
204
+ signal: controller.signal,
205
+ input: Buffer.from(JSON.stringify({ language, text, pattern })),
206
+ },
207
+ async (stdout) => {
208
+ for await (const chunk of stdout) {
209
+ bytes += chunk.byteLength;
210
+ if (bytes > MAX_STRUCTURE_BYTES)
211
+ throw new SiftLightError("Syntax parser output exceeds protocol limit");
212
+ chunks.push(Buffer.from(chunk));
213
+ }
214
+ },
215
+ );
216
+ if (signal?.aborted) throw abortError();
217
+ if (result.code !== 0) {
218
+ throw new SiftLightError(
219
+ `Syntax parser process failed (${String(result.code)}): ${result.stderr.trim()}`,
220
+ );
221
+ }
222
+ const parsed = readResult(Buffer.concat(chunks).toString("utf8"), text.length);
223
+ const children = syntaxChildren(parsed.nodes);
224
+ const diagnostics = parsed.nodes.flatMap((node, index) =>
225
+ node.kind === "ERROR" || (index > 0 && node.start === node.end)
226
+ ? [
227
+ {
228
+ kind: node.kind === "ERROR" ? "syntax-error" : "missing-token",
229
+ start: node.start,
230
+ end: node.end,
231
+ },
232
+ ]
233
+ : [],
234
+ );
235
+ const facts =
236
+ parsed.status === "ok"
237
+ ? deriveSyntaxFacts({ nodes: parsed.nodes, children }, language, text)
238
+ : { symbols: [], roles: [] };
239
+ if (signal?.aborted) throw abortError();
240
+ return {
241
+ language,
242
+ status: parsed.status,
243
+ nodes: parsed.nodes,
244
+ children,
245
+ ...(parsed.patternMatches ? { patternMatches: parsed.patternMatches } : {}),
246
+ ...facts,
247
+ diagnostics,
248
+ limited: parsed.status === "limit",
249
+ };
250
+ } catch (error) {
251
+ const aborted = error instanceof Error && error.name === "AbortError";
252
+ if (aborted && signal?.aborted) throw abortError();
253
+ if (aborted && timedOut) return emptyAnalysis("timeout", language);
254
+ throw error;
255
+ } finally {
256
+ clearTimeout(timer);
257
+ signal?.removeEventListener("abort", abort);
258
+ }
259
+ }