pi-knowledge 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,179 +1,767 @@
1
- import { buildChunkEmbeddingText, contentHash, preTokenizeForFTS } from "../chunker.js";
1
+ import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "../chunker.js";
2
+ function moduleField(moduleValue, field) {
3
+ if (typeof moduleValue !== "object" || moduleValue === null)
4
+ return undefined;
5
+ return moduleValue[field];
6
+ }
7
+ function moduleDefault(moduleValue) {
8
+ return moduleField(moduleValue, "default");
9
+ }
2
10
  const LANGS = {
3
11
  typescript: {
4
12
  grammar: async () => {
5
13
  const m = await import("tree-sitter-typescript");
6
- return m.typescript ?? m.default?.typescript;
14
+ return moduleField(m, "typescript") ?? moduleField(moduleDefault(m), "typescript");
7
15
  },
8
- fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
9
- classTypes: new Set(["class_declaration", "interface_declaration"]),
10
- classBody: "class_body",
11
16
  fileType: "typescript",
12
17
  },
13
18
  javascript: {
14
19
  grammar: async () => {
15
- const m = await import("tree-sitter-typescript");
16
- return m.typescript ?? m.default?.typescript;
20
+ const m = await import("tree-sitter-javascript");
21
+ return moduleDefault(m) ?? m;
17
22
  },
18
- fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
19
- classTypes: new Set(["class_declaration"]),
20
- classBody: "class_body",
21
23
  fileType: "javascript",
22
24
  },
23
25
  python: {
24
26
  grammar: async () => {
25
27
  const m = await import("tree-sitter-python");
26
- return m.default ?? m;
28
+ return moduleDefault(m) ?? m;
27
29
  },
28
- fnTypes: new Set(["function_definition"]),
29
- classTypes: new Set(["class_definition"]),
30
- classBody: "block",
31
30
  fileType: "python",
32
31
  },
33
32
  go: {
34
33
  grammar: async () => {
35
34
  const m = await import("tree-sitter-go");
36
- return m.default ?? m;
35
+ return moduleDefault(m) ?? m;
37
36
  },
38
- fnTypes: new Set(["function_declaration", "method_declaration"]),
39
- classTypes: new Set(["type_declaration"]),
40
37
  fileType: "go",
41
38
  },
42
39
  rust: {
43
40
  grammar: async () => {
44
41
  const m = await import("tree-sitter-rust");
45
- return m.default ?? m;
42
+ return moduleDefault(m) ?? m;
46
43
  },
47
- fnTypes: new Set(["function_item"]),
48
- classTypes: new Set(["struct_item", "enum_item"]),
49
- methodContainer: "impl_item",
50
44
  fileType: "rust",
51
45
  },
52
46
  java: {
53
47
  grammar: async () => {
54
48
  const m = await import("tree-sitter-java");
55
- return m.default ?? m;
49
+ return moduleDefault(m) ?? m;
56
50
  },
57
- fnTypes: new Set(["method_declaration", "constructor_declaration"]),
58
- classTypes: new Set(["class_declaration", "interface_declaration", "enum_declaration"]),
59
- classBody: "class_body",
60
51
  fileType: "java",
61
52
  },
53
+ bash: {
54
+ grammar: async () => {
55
+ const m = await import("tree-sitter-bash");
56
+ return moduleDefault(m) ?? m;
57
+ },
58
+ fileType: "bash",
59
+ fallbackOnParseError: true,
60
+ },
61
+ c: {
62
+ grammar: async () => {
63
+ const m = await import("tree-sitter-c");
64
+ return moduleDefault(m) ?? m;
65
+ },
66
+ fileType: "c",
67
+ fallbackOnParseError: true,
68
+ },
69
+ cpp: {
70
+ grammar: async () => {
71
+ const m = await import("tree-sitter-cpp");
72
+ return moduleDefault(m) ?? m;
73
+ },
74
+ fileType: "cpp",
75
+ fallbackOnParseError: true,
76
+ },
77
+ qml: {
78
+ grammar: async () => {
79
+ const m = await import("tree-sitter-qmljs");
80
+ return moduleDefault(m) ?? m;
81
+ },
82
+ fileType: "qml",
83
+ fallbackOnParseError: true,
84
+ },
62
85
  };
86
+ const NAMESPACE_TYPES = new Set(["namespace_definition"]);
87
+ const QML_TYPES = new Set(["ui_binding", "ui_import", "ui_object_definition", "ui_property", "ui_signal"]);
88
+ const CLASS_TYPES = new Set([
89
+ "abstract_class_declaration",
90
+ "class_declaration",
91
+ "class_definition",
92
+ "class_specifier",
93
+ "enum_declaration",
94
+ ]);
95
+ const INTERFACE_TYPES = new Set(["interface_declaration"]);
96
+ const TYPE_TYPES = new Set([
97
+ "annotation_type_declaration",
98
+ "enum_item",
99
+ "enum_specifier",
100
+ "struct_item",
101
+ "struct_specifier",
102
+ "trait_item",
103
+ "type_alias_declaration",
104
+ "type_declaration",
105
+ "type_definition",
106
+ "union_specifier",
107
+ ]);
108
+ const FUNCTION_TYPES = new Set([
109
+ "declaration",
110
+ "field_declaration",
111
+ "function_declaration",
112
+ "function_definition",
113
+ "function_item",
114
+ "generator_function_declaration",
115
+ "method_declaration",
116
+ ]);
117
+ const METHOD_TYPES = new Set(["method_definition"]);
118
+ const CONSTRUCTOR_TYPES = new Set(["constructor_declaration"]);
119
+ const DECLARATION_WRAPPERS = new Set(["export_statement", "decorated_definition", "template_declaration"]);
120
+ const VARIABLE_DECLARATOR_TYPES = new Set(["variable_declarator"]);
121
+ const FIELD_DEFINITION_TYPES = new Set(["public_field_definition", "field_definition", "property_declaration"]);
122
+ const FUNCTION_VALUE_TYPES = new Set([
123
+ "arrow_function",
124
+ "function",
125
+ "function_expression",
126
+ "function_declaration",
127
+ "generator_function",
128
+ "generator_function_declaration",
129
+ ]);
130
+ const AST_TARGET_TOKENS = 900;
131
+ const AST_MAX_TOKENS = 1_800;
132
+ const MAX_AST_FALLBACK_CHARS = 6_000;
133
+ function estimateTokens(text) {
134
+ return Math.ceil(text.length / 3);
135
+ }
136
+ function buildByteIndexMap(content) {
137
+ const byByte = new Map();
138
+ let byteOffset = 0;
139
+ for (let index = 0; index < content.length;) {
140
+ byByte.set(byteOffset, index);
141
+ const codePoint = content.codePointAt(index);
142
+ if (codePoint === undefined)
143
+ break;
144
+ const char = String.fromCodePoint(codePoint);
145
+ byteOffset += Buffer.byteLength(char);
146
+ index += char.length;
147
+ }
148
+ byByte.set(byteOffset, content.length);
149
+ return { content, byByte };
150
+ }
151
+ function stringIndexForByte(map, byteOffset) {
152
+ const exact = map.byByte.get(byteOffset);
153
+ if (exact !== undefined)
154
+ return exact;
155
+ let closestByte = 0;
156
+ let closestIndex = 0;
157
+ for (const [candidateByte, candidateIndex] of map.byByte) {
158
+ if (candidateByte > byteOffset)
159
+ break;
160
+ closestByte = candidateByte;
161
+ closestIndex = candidateIndex;
162
+ }
163
+ return closestByte === byteOffset ? closestIndex : closestIndex;
164
+ }
165
+ function sliceByBytes(map, startByte, endByte) {
166
+ return map.content.slice(stringIndexForByte(map, startByte), stringIndexForByte(map, endByte));
167
+ }
168
+ function nodeChildren(node) {
169
+ return node.namedChildren && node.namedChildren.length > 0 ? node.namedChildren : node.children;
170
+ }
171
+ function nameFromDeclarator(node) {
172
+ if (!node)
173
+ return undefined;
174
+ if (node.type === "qualified_identifier" || node.type === "destructor_name") {
175
+ const text = node.text.trim();
176
+ return text || undefined;
177
+ }
178
+ const direct = node.childForFieldName("name")?.text;
179
+ if (direct?.trim())
180
+ return direct.trim();
181
+ const nested = node.childForFieldName("declarator");
182
+ if (nested && nested !== node)
183
+ return nameFromDeclarator(nested);
184
+ if (["field_identifier", "identifier", "type_identifier", "variable_name", "word"].includes(node.type)) {
185
+ const text = node.text.trim();
186
+ return text || undefined;
187
+ }
188
+ return nodeChildren(node)
189
+ .map(nameFromDeclarator)
190
+ .find((name) => Boolean(name));
191
+ }
192
+ function firstNamedChildText(node, type) {
193
+ const match = nodeChildren(node).find((child) => child.type === type);
194
+ const text = match?.text.trim();
195
+ return text || undefined;
196
+ }
197
+ function qmlBindingName(node) {
198
+ const key = node.childForFieldName("name")?.text.trim() ?? firstNamedChildText(node, "identifier");
199
+ if (key !== "id")
200
+ return key || undefined;
201
+ const value = node.childForFieldName("value");
202
+ return value ? (firstNamedChildText(value, "identifier") ?? value.text.trim()) : key;
203
+ }
63
204
  function getName(node) {
64
- return node.childForFieldName("name")?.text ?? node.childForFieldName("type")?.text ?? "anonymous";
65
- }
66
- function collectChunks(root, config) {
67
- const chunks = [];
68
- function walk(node) {
69
- if (node.type === "export_statement") {
70
- for (const child of node.children) {
71
- if (config.fnTypes.has(child.type)) {
72
- chunks.push({
73
- name: getName(child),
74
- text: node.text,
75
- start: node.startPosition.row + 1,
76
- end: node.endPosition.row + 1,
77
- });
78
- return;
79
- }
80
- if (config.classTypes.has(child.type)) {
81
- walk(child);
82
- return;
83
- }
84
- }
205
+ if (node.type === "constructor_declaration")
206
+ return "constructor";
207
+ if (node.type === "ui_object_definition")
208
+ return firstNamedChildText(node, "identifier");
209
+ if (node.type === "ui_import") {
210
+ return firstNamedChildText(node, "nested_identifier") ?? firstNamedChildText(node, "identifier");
211
+ }
212
+ if (node.type === "ui_binding")
213
+ return qmlBindingName(node);
214
+ if (node.type === "type_definition" ||
215
+ node.type === "declaration" ||
216
+ node.type === "field_declaration" ||
217
+ node.type === "function_definition") {
218
+ const declaratorName = nameFromDeclarator(node.childForFieldName("declarator"));
219
+ if (declaratorName)
220
+ return declaratorName;
221
+ }
222
+ const name = node.childForFieldName("name")?.text ?? node.childForFieldName("type")?.text;
223
+ if (name?.trim())
224
+ return name.trim();
225
+ return nameFromDeclarator(node.childForFieldName("declarator"));
226
+ }
227
+ function nodeHasFunctionValue(node) {
228
+ return nodeChildren(node).some((child) => FUNCTION_VALUE_TYPES.has(child.type) || nodeHasFunctionValue(child));
229
+ }
230
+ function firstStructuralChild(node) {
231
+ return nodeChildren(node).find((child) => isStructuralCandidate(child) || child.type === "lexical_declaration");
232
+ }
233
+ function decoratorsFor(node, inherited = []) {
234
+ const decorators = node.children.filter((child) => child.type === "decorator").map((child) => child.text.trim());
235
+ return [...inherited, ...decorators].filter(Boolean);
236
+ }
237
+ function modifiersFor(node) {
238
+ return node.children
239
+ .filter((child) => child.type === "storage_class_specifier")
240
+ .map((child) => child.text.trim())
241
+ .filter(Boolean);
242
+ }
243
+ function leafName(name) {
244
+ if (!name)
245
+ return undefined;
246
+ return name.split("::").at(-1);
247
+ }
248
+ function parentFromQualifiedName(name) {
249
+ if (!name?.includes("::"))
250
+ return undefined;
251
+ const parts = name.split("::");
252
+ return parts.length > 1 ? parts.at(-2) : undefined;
253
+ }
254
+ function isStructuralCandidate(node) {
255
+ return (QML_TYPES.has(node.type) ||
256
+ NAMESPACE_TYPES.has(node.type) ||
257
+ CLASS_TYPES.has(node.type) ||
258
+ INTERFACE_TYPES.has(node.type) ||
259
+ TYPE_TYPES.has(node.type) ||
260
+ FUNCTION_TYPES.has(node.type) ||
261
+ METHOD_TYPES.has(node.type) ||
262
+ CONSTRUCTOR_TYPES.has(node.type));
263
+ }
264
+ function kindForNode(node, context) {
265
+ if (node.type === "ui_import")
266
+ return "import";
267
+ if (node.type === "ui_object_definition")
268
+ return "class";
269
+ if (node.type === "ui_property")
270
+ return "property";
271
+ if (node.type === "ui_signal")
272
+ return "signal";
273
+ if (node.type === "ui_binding") {
274
+ const key = node.childForFieldName("name")?.text.trim() ?? firstNamedChildText(node, "identifier");
275
+ if (key === "id")
276
+ return "id";
277
+ return key?.startsWith("on") ? "handler" : "binding";
278
+ }
279
+ if (node.type === "preproc_def" || node.type === "preproc_function_def")
280
+ return "variable";
281
+ if ((node.type === "declaration" || node.type === "field_declaration") &&
282
+ !nodeChildren(node).some((child) => child.type === "function_declarator")) {
283
+ return undefined;
284
+ }
285
+ if (NAMESPACE_TYPES.has(node.type))
286
+ return "namespace";
287
+ if (CLASS_TYPES.has(node.type))
288
+ return "class";
289
+ if (INTERFACE_TYPES.has(node.type))
290
+ return "interface";
291
+ if (TYPE_TYPES.has(node.type))
292
+ return "type";
293
+ if (METHOD_TYPES.has(node.type) || node.type === "method_declaration")
294
+ return "method";
295
+ if (CONSTRUCTOR_TYPES.has(node.type))
296
+ return "constructor";
297
+ if (FUNCTION_TYPES.has(node.type)) {
298
+ const name = getName(node);
299
+ const leaf = leafName(name);
300
+ if ((context?.parentSymbol && leaf === context.parentSymbol) || parentFromQualifiedName(name) === leaf) {
301
+ return "constructor";
85
302
  }
86
- if (node.type === "decorated_definition") {
87
- for (const child of node.children) {
88
- if (config.fnTypes.has(child.type) || config.classTypes.has(child.type)) {
89
- chunks.push({
90
- name: getName(child),
91
- text: node.text,
92
- start: node.startPosition.row + 1,
93
- end: node.endPosition.row + 1,
94
- });
95
- return;
96
- }
97
- }
303
+ if (leaf?.startsWith("~"))
304
+ return "destructor";
305
+ if (node.type === "field_declaration" || (context?.parentSymbol && node.type === "declaration"))
306
+ return "method";
307
+ if (name?.includes("::"))
308
+ return leaf?.startsWith("~") ? "destructor" : "method";
309
+ return "function";
310
+ }
311
+ return undefined;
312
+ }
313
+ function signatureFromText(text, kind) {
314
+ const normalized = text.trim();
315
+ if (!normalized)
316
+ return undefined;
317
+ const bodyMarkers = kind === "function" || kind === "method" || kind === "constructor" || kind === "destructor"
318
+ ? normalized.startsWith("def ") || normalized.startsWith("async def ")
319
+ ? [":"]
320
+ : ["{", "=>"]
321
+ : ["{"];
322
+ let end = normalized.length;
323
+ for (const marker of bodyMarkers) {
324
+ const index = normalized.indexOf(marker);
325
+ if (index > 0)
326
+ end = Math.min(end, marker === "=>" ? index + marker.length : index);
327
+ }
328
+ const firstLine = normalized.slice(0, end).replace(/\s+/g, " ").trim();
329
+ return firstLine.length > 280 ? `${firstLine.slice(0, 277)}...` : firstLine;
330
+ }
331
+ function createStructureNode(node, kind, name, context, map) {
332
+ const span = context.spanNode ?? node;
333
+ const qualifiedParent = parentFromQualifiedName(name);
334
+ const parentSymbol = qualifiedParent ?? context.parentSymbol;
335
+ const baseScope = qualifiedParent && context.scope.at(-1) !== qualifiedParent ? [...context.scope, qualifiedParent] : context.scope;
336
+ const scope = name ? [...baseScope, name] : [...baseScope];
337
+ const text = sliceByBytes(map, span.startIndex, span.endIndex);
338
+ const astPath = [...context.astPath, node.type];
339
+ return {
340
+ kind,
341
+ astType: node.type,
342
+ name,
343
+ signature: signatureFromText(text, kind),
344
+ language: context.language,
345
+ startLine: span.startPosition.row + 1,
346
+ endLine: span.endPosition.row + 1,
347
+ startByte: span.startIndex,
348
+ endByte: span.endIndex,
349
+ exported: context.exported,
350
+ decorators: decoratorsFor(span, context.decorators),
351
+ modifiers: modifiersFor(span),
352
+ visibility: context.visibility,
353
+ parentSymbol,
354
+ scope,
355
+ astPath,
356
+ children: [],
357
+ };
358
+ }
359
+ function childContext(parent, context) {
360
+ return {
361
+ language: context.language,
362
+ exported: false,
363
+ decorators: [],
364
+ scope: parent.scope,
365
+ parentSymbol: parent.name ?? context.parentSymbol,
366
+ visibility: undefined,
367
+ astPath: parent.astPath,
368
+ };
369
+ }
370
+ function buildVariableFunctionNodes(node, context, map) {
371
+ const declarators = nodeChildren(node).filter((child) => VARIABLE_DECLARATOR_TYPES.has(child.type));
372
+ const singleDeclarator = declarators.length === 1;
373
+ const nodes = [];
374
+ for (const declarator of declarators) {
375
+ if (!nodeHasFunctionValue(declarator))
376
+ continue;
377
+ const name = getName(declarator);
378
+ const spanNode = context.spanNode ?? (singleDeclarator ? node : declarator);
379
+ const structural = createStructureNode(declarator, "function", name, { ...context, spanNode, astPath: [...context.astPath, node.type] }, map);
380
+ nodes.push(structural);
381
+ }
382
+ return nodes;
383
+ }
384
+ function buildFieldFunctionNode(node, context, map) {
385
+ if (!nodeHasFunctionValue(node))
386
+ return undefined;
387
+ const structural = createStructureNode(node, context.parentSymbol ? "method" : "function", getName(node), context, map);
388
+ return structural;
389
+ }
390
+ function collectStructureNodes(node, context, map) {
391
+ if (node.type === "export_statement") {
392
+ const structuralChild = firstStructuralChild(node);
393
+ if (structuralChild) {
394
+ return collectStructureNodes(structuralChild, { ...context, exported: true, spanNode: node }, map);
98
395
  }
99
- if (config.fnTypes.has(node.type)) {
100
- chunks.push({
101
- name: getName(node),
102
- text: node.text,
103
- start: node.startPosition.row + 1,
104
- end: node.endPosition.row + 1,
105
- });
106
- return;
396
+ }
397
+ if (node.type === "decorated_definition") {
398
+ const structuralChild = firstStructuralChild(node);
399
+ if (structuralChild) {
400
+ return collectStructureNodes(structuralChild, {
401
+ ...context,
402
+ decorators: decoratorsFor(node, context.decorators),
403
+ spanNode: node,
404
+ }, map);
107
405
  }
108
- if (config.classTypes.has(node.type)) {
109
- const name = getName(node);
110
- let hasChildren = false;
111
- for (const child of node.children) {
112
- if (config.classBody && child.type === config.classBody) {
113
- for (const m of child.children) {
114
- if (config.fnTypes.has(m.type) || m.type === "method_definition") {
115
- hasChildren = true;
116
- chunks.push({
117
- name: `${name}.${getName(m)}`,
118
- text: m.text,
119
- start: m.startPosition.row + 1,
120
- end: m.endPosition.row + 1,
121
- });
122
- }
123
- }
124
- }
125
- }
126
- if (!hasChildren)
127
- chunks.push({ name, text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 });
128
- return;
406
+ }
407
+ if (node.type === "lexical_declaration" || node.type === "variable_declaration") {
408
+ const variableNodes = buildVariableFunctionNodes(node, context, map);
409
+ if (variableNodes.length > 0)
410
+ return variableNodes;
411
+ }
412
+ if (VARIABLE_DECLARATOR_TYPES.has(node.type) && nodeHasFunctionValue(node)) {
413
+ return [createStructureNode(node, "function", getName(node), context, map)];
414
+ }
415
+ if (FIELD_DEFINITION_TYPES.has(node.type)) {
416
+ const field = buildFieldFunctionNode(node, context, map);
417
+ if (field)
418
+ return [field];
419
+ }
420
+ const kind = kindForNode(node, context);
421
+ if (kind) {
422
+ const structural = createStructureNode(node, kind, getName(node), context, map);
423
+ const nested = scanChildren(node, childContext(structural, context), map);
424
+ structural.children.push(...nested.filter((child) => child.startByte >= structural.startByte && child.endByte <= structural.endByte));
425
+ return [structural];
426
+ }
427
+ return scanChildren(node, context, map);
428
+ }
429
+ function scanChildren(node, context, map) {
430
+ const nodes = [];
431
+ let visibility = context.visibility;
432
+ for (const child of nodeChildren(node)) {
433
+ if (child.type === "access_specifier") {
434
+ visibility = child.text.trim();
435
+ continue;
129
436
  }
130
- if (config.methodContainer && node.type === config.methodContainer) {
131
- const typeName = getName(node);
132
- for (const child of node.children) {
133
- if (child.type === "declaration_list") {
134
- for (const m of child.children) {
135
- if (config.fnTypes.has(m.type))
136
- chunks.push({
137
- name: `${typeName}.${getName(m)}`,
138
- text: m.text,
139
- start: m.startPosition.row + 1,
140
- end: m.endPosition.row + 1,
141
- });
142
- }
143
- }
437
+ const nextContext = { ...context, visibility };
438
+ if (DECLARATION_WRAPPERS.has(child.type) ||
439
+ FIELD_DEFINITION_TYPES.has(child.type) ||
440
+ isStructuralCandidate(child) ||
441
+ child.type === "preproc_def" ||
442
+ child.type === "preproc_function_def" ||
443
+ child.type.includes("declaration")) {
444
+ nodes.push(...collectStructureNodes(child, { ...nextContext, spanNode: undefined }, map));
445
+ continue;
446
+ }
447
+ if (child.type === "class_body" ||
448
+ child.type === "block" ||
449
+ child.type === "declaration_list" ||
450
+ child.type === "field_declaration_list" ||
451
+ child.type === "ui_object_initializer") {
452
+ nodes.push(...scanChildren(child, nextContext, map));
453
+ }
454
+ }
455
+ return nodes.sort((a, b) => a.startByte - b.startByte || a.endByte - b.endByte);
456
+ }
457
+ function metadataForNode(node) {
458
+ const symbol = node.name ?? node.kind;
459
+ return {
460
+ language: node.language,
461
+ symbol,
462
+ function_name: symbol,
463
+ symbol_kind: node.kind,
464
+ scope: node.scope,
465
+ parent_symbol: node.parentSymbol ?? null,
466
+ signature: node.signature ?? null,
467
+ exported: node.exported,
468
+ ast_depth: Math.max(0, node.scope.length - 1),
469
+ ast_path: node.astPath.join("/"),
470
+ decorators: node.decorators.length > 0 ? node.decorators : undefined,
471
+ modifiers: node.modifiers.length > 0 ? node.modifiers : undefined,
472
+ visibility: node.visibility ?? undefined,
473
+ static: node.modifiers.includes("static") ? true : undefined,
474
+ start_line: node.startLine,
475
+ end_line: node.endLine,
476
+ };
477
+ }
478
+ function makeChunk(content, filePath, fileType, startLine, endLine, metadata) {
479
+ const metadata_json = JSON.stringify(metadata);
480
+ const chunk = {
481
+ content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
482
+ content,
483
+ content_tokenized: "",
484
+ file_path: filePath,
485
+ file_type: fileType,
486
+ start_line: startLine,
487
+ end_line: endLine,
488
+ metadata_json,
489
+ };
490
+ return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
491
+ }
492
+ function draftForNode(node, map) {
493
+ const text = sliceByBytes(map, node.startByte, node.endByte).trim();
494
+ return {
495
+ content: text,
496
+ startLine: node.startLine,
497
+ endLine: node.endLine,
498
+ startByte: node.startByte,
499
+ endByte: node.endByte,
500
+ metadata: metadataForNode(node),
501
+ packable: ["function", "method", "constructor", "destructor", "variable"].includes(node.kind) &&
502
+ estimateTokens(text) < AST_TARGET_TOKENS,
503
+ packKey: node.parentSymbol ?? "module",
504
+ symbols: node.name ? [node.name] : [],
505
+ };
506
+ }
507
+ function fallbackDrafts(node, map) {
508
+ const text = sliceByBytes(map, node.startByte, node.endByte);
509
+ const lines = text.split("\n");
510
+ const drafts = [];
511
+ let buffer = [];
512
+ let bufferChars = 0;
513
+ let startLine = node.startLine;
514
+ let part = 1;
515
+ function pushDraft(content, draftStartLine, draftEndLine) {
516
+ const trimmed = content.trim();
517
+ if (!trimmed)
518
+ return;
519
+ drafts.push({
520
+ content: trimmed,
521
+ startLine: draftStartLine,
522
+ endLine: draftEndLine,
523
+ startByte: node.startByte,
524
+ endByte: node.endByte,
525
+ metadata: { ...metadataForNode(node), chunk_part: part, start_line: draftStartLine, end_line: draftEndLine },
526
+ packable: false,
527
+ packKey: `${node.parentSymbol ?? "module"}:${node.name ?? node.kind}`,
528
+ symbols: node.name ? [node.name] : [],
529
+ });
530
+ part++;
531
+ }
532
+ function flush(endLine) {
533
+ pushDraft(buffer.join("\n"), startLine, endLine);
534
+ buffer = [];
535
+ bufferChars = 0;
536
+ }
537
+ for (let index = 0; index < lines.length; index++) {
538
+ const line = lines[index];
539
+ const absoluteLine = node.startLine + index;
540
+ if (line.length > MAX_AST_FALLBACK_CHARS) {
541
+ if (buffer.length > 0)
542
+ flush(absoluteLine - 1);
543
+ for (let offset = 0; offset < line.length; offset += MAX_AST_FALLBACK_CHARS) {
544
+ pushDraft(line.slice(offset, offset + MAX_AST_FALLBACK_CHARS), absoluteLine, absoluteLine);
144
545
  }
546
+ startLine = absoluteLine + 1;
547
+ continue;
548
+ }
549
+ if (buffer.length > 0 && bufferChars + line.length + 1 > MAX_AST_FALLBACK_CHARS)
550
+ flush(absoluteLine - 1);
551
+ if (buffer.length === 0)
552
+ startLine = absoluteLine;
553
+ buffer.push(line);
554
+ bufferChars += line.length + 1;
555
+ }
556
+ if (buffer.length > 0)
557
+ flush(node.endLine);
558
+ return drafts;
559
+ }
560
+ function packDrafts(drafts, map) {
561
+ const packed = [];
562
+ let buffer = [];
563
+ let bufferTokens = 0;
564
+ function flush() {
565
+ if (buffer.length === 0)
145
566
  return;
567
+ if (buffer.length === 1) {
568
+ packed.push(buffer[0]);
569
+ }
570
+ else {
571
+ const first = buffer[0];
572
+ const last = buffer[buffer.length - 1];
573
+ const symbols = buffer.flatMap((draft) => draft.symbols);
574
+ const content = sliceByBytes(map, first.startByte, last.endByte).trim();
575
+ packed.push({
576
+ content,
577
+ startLine: first.startLine,
578
+ endLine: last.endLine,
579
+ startByte: first.startByte,
580
+ endByte: last.endByte,
581
+ metadata: {
582
+ ...first.metadata,
583
+ symbol: symbols.join(", "),
584
+ function_name: symbols[0] ?? first.metadata.function_name,
585
+ symbol_kind: "group",
586
+ symbols,
587
+ signature: null,
588
+ start_line: first.startLine,
589
+ end_line: last.endLine,
590
+ },
591
+ packable: false,
592
+ packKey: first.packKey,
593
+ symbols,
594
+ });
146
595
  }
147
- for (const child of node.children)
148
- walk(child);
596
+ buffer = [];
597
+ bufferTokens = 0;
149
598
  }
150
- walk(root);
151
- return chunks;
599
+ for (const draft of drafts) {
600
+ const tokens = estimateTokens(draft.content);
601
+ const compatible = draft.packable &&
602
+ buffer.length > 0 &&
603
+ buffer.every((item) => item.packable && item.packKey === draft.packKey) &&
604
+ bufferTokens + tokens <= AST_TARGET_TOKENS;
605
+ if (!draft.packable) {
606
+ flush();
607
+ packed.push(draft);
608
+ continue;
609
+ }
610
+ if (!compatible && buffer.length > 0)
611
+ flush();
612
+ buffer.push(draft);
613
+ bufferTokens += tokens;
614
+ }
615
+ flush();
616
+ return packed;
152
617
  }
153
- export async function chunkWithAST(content, filePath, language) {
618
+ function buildDrafts(nodes, map) {
619
+ const drafts = [];
620
+ for (const node of nodes.sort((a, b) => a.startByte - b.startByte || a.endByte - b.endByte)) {
621
+ if (node.kind === "namespace" && node.children.length > 0) {
622
+ drafts.push(...buildDrafts(node.children, map));
623
+ continue;
624
+ }
625
+ const text = sliceByBytes(map, node.startByte, node.endByte).trim();
626
+ if (!text)
627
+ continue;
628
+ if (estimateTokens(text) <= AST_MAX_TOKENS) {
629
+ drafts.push(draftForNode(node, map));
630
+ continue;
631
+ }
632
+ if (node.children.length > 0) {
633
+ drafts.push(...buildDrafts(node.children, map));
634
+ continue;
635
+ }
636
+ drafts.push(...fallbackDrafts(node, map));
637
+ }
638
+ return packDrafts(drafts, map);
639
+ }
640
+ function symbolKindForNode(node) {
641
+ if (!node.name)
642
+ return undefined;
643
+ if (node.kind === "class")
644
+ return "class";
645
+ if (node.kind === "namespace" || node.kind === "import" || node.kind === "type")
646
+ return "type";
647
+ if (node.kind === "variable" || node.kind === "property" || node.kind === "id" || node.kind === "binding")
648
+ return "variable";
649
+ if (node.kind === "function" ||
650
+ node.kind === "method" ||
651
+ node.kind === "constructor" ||
652
+ node.kind === "destructor" ||
653
+ node.kind === "signal" ||
654
+ node.kind === "handler")
655
+ return "function";
656
+ return undefined;
657
+ }
658
+ function symbolMetadataForNode(node) {
659
+ return {
660
+ language: node.language,
661
+ symbol_kind: node.kind,
662
+ scope: node.scope,
663
+ parent_symbol: node.parentSymbol ?? null,
664
+ exported: node.exported,
665
+ ast_depth: Math.max(0, node.scope.length - 1),
666
+ ast_path: node.astPath.join("/"),
667
+ modifiers: node.modifiers.length > 0 ? node.modifiers : undefined,
668
+ static: node.modifiers.includes("static") ? true : undefined,
669
+ visibility: node.visibility ?? undefined,
670
+ decorators: node.decorators.length > 0 ? node.decorators : undefined,
671
+ };
672
+ }
673
+ function collectSymbols(nodes, map) {
674
+ const symbols = [];
675
+ for (const node of nodes) {
676
+ const kind = symbolKindForNode(node);
677
+ if (kind && node.name) {
678
+ const source = sliceByBytes(map, node.startByte, node.endByte).trim();
679
+ symbols.push({
680
+ name: node.name,
681
+ kind,
682
+ file_path: "",
683
+ file_type: node.language,
684
+ start_line: node.startLine,
685
+ end_line: node.endLine,
686
+ container_name: node.parentSymbol ?? null,
687
+ signature: node.signature ?? null,
688
+ text: node.signature ?? source.split("\n")[0]?.trim() ?? node.name,
689
+ metadata_json: JSON.stringify(symbolMetadataForNode(node)),
690
+ });
691
+ }
692
+ symbols.push(...collectSymbols(node.children, map));
693
+ }
694
+ return symbols;
695
+ }
696
+ function withSymbolLocation(symbols, filePath, fileType) {
697
+ return symbols.map((symbol) => ({ ...symbol, file_path: filePath, file_type: fileType }));
698
+ }
699
+ function emptyStructure(language) {
700
+ return {
701
+ kind: "module",
702
+ astType: "program",
703
+ language,
704
+ startLine: 1,
705
+ endLine: 1,
706
+ startByte: 0,
707
+ endByte: 0,
708
+ exported: false,
709
+ decorators: [],
710
+ modifiers: [],
711
+ scope: [],
712
+ astPath: ["program"],
713
+ children: [],
714
+ };
715
+ }
716
+ async function parseStructure(content, language) {
154
717
  const config = LANGS[language];
155
718
  if (!config)
156
- return [];
719
+ throw new Error(`Unsupported AST language: ${language}`);
157
720
  const Parser = (await import("tree-sitter")).default;
158
721
  const grammar = await config.grammar();
159
722
  const parser = new Parser();
160
723
  parser.setLanguage(grammar);
161
724
  const tree = parser.parse(content);
162
- const fns = collectChunks(tree.rootNode, config);
163
- if (fns.length === 0)
725
+ const rootNode = tree.rootNode;
726
+ if (config.fallbackOnParseError && rootNode.hasError)
727
+ throw new Error(`AST parse failed for ${language}`);
728
+ const map = buildByteIndexMap(content);
729
+ const root = {
730
+ kind: "module",
731
+ astType: rootNode.type,
732
+ language: config.fileType,
733
+ startLine: 1,
734
+ endLine: content.split("\n").length,
735
+ startByte: rootNode.startIndex,
736
+ endByte: rootNode.endIndex,
737
+ exported: false,
738
+ decorators: [],
739
+ modifiers: [],
740
+ visibility: undefined,
741
+ scope: [],
742
+ astPath: [rootNode.type],
743
+ children: [],
744
+ };
745
+ root.children.push(...scanChildren(rootNode, { language: config.fileType, exported: false, decorators: [], scope: [], astPath: [rootNode.type] }, map));
746
+ return { root, map };
747
+ }
748
+ export async function parseCodeStructure(content, language) {
749
+ return (await parseStructure(content, language)).root;
750
+ }
751
+ export async function analyzeCodeWithAST(content, filePath, language) {
752
+ const config = LANGS[language];
753
+ if (!config)
754
+ return { chunks: [], symbols: [], structure: emptyStructure(language) };
755
+ const { root, map } = await parseStructure(content, language);
756
+ const drafts = buildDrafts(root.children, map);
757
+ const chunks = drafts.map((draft) => makeChunk(draft.content, filePath, config.fileType, draft.startLine, draft.endLine, draft.metadata));
758
+ const symbols = withSymbolLocation(collectSymbols(root.children, map), filePath, config.fileType);
759
+ return { chunks, symbols, structure: root };
760
+ }
761
+ export async function chunkWithAST(content, filePath, language) {
762
+ const config = LANGS[language];
763
+ if (!config)
164
764
  return [];
165
- return fns.map((fn) => {
166
- const chunk = {
167
- content_hash: contentHash(fn.text),
168
- content: fn.text,
169
- content_tokenized: "",
170
- file_path: filePath,
171
- file_type: config.fileType,
172
- start_line: fn.start,
173
- end_line: fn.end,
174
- metadata_json: JSON.stringify({ function_name: fn.name }),
175
- };
176
- return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
177
- });
765
+ return (await analyzeCodeWithAST(content, filePath, language)).chunks;
178
766
  }
179
767
  export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));