pi-knowledge 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,28 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.10.0] - 2026-08-27
4
+
5
+ ### Added
6
+ - Added Bash AST indexing for `.sh` and `.bash` scripts, including both `name() {}` and `function name {}` forms with bounded fallback chunking (#12).
7
+ - Added GNU C AST indexing for `.c` source files, covering functions, prototypes, structs, unions, enums, typedefs, reliable preprocessor symbols, and `static` metadata (#12).
8
+ - Added C++ AST indexing for `.cpp`, `.cc`, `.cxx`, `.hpp`, `.hh`, and `.hxx` files, covering namespaces, classes, methods, constructors, destructors, enums, templates, and access visibility where the parser reports it reliably (#12).
9
+ - Added QML AST indexing for `.qml` files, covering imports, component/object hierarchy, ids, properties, signals, handlers, bindings, and embedded JavaScript functions (#12).
10
+
11
+ ### Changed
12
+ - Kept ambiguous `.h` headers classified as text by default; use typed C++ header extensions for C++ AST indexing (#12).
13
+ - Parser errors in Bash, C, C++, and QML now intentionally fall back to existing text chunking instead of indexing partial structural claims (#12).
14
+
15
+ ## [0.9.0] - 2026-08-26
16
+
17
+ ### Added
18
+ - Added recursive AST chunking for supported code files, including same-parent sibling packing, bounded oversized leaf fallback, and structural chunk metadata for language, symbol kind, scope, parent symbol, signature, export state, AST path, and source line range (#11).
19
+ - Added AST-backed symbol extraction for supported code files so method symbols remain searchable independently from retrieval chunk boundaries (#11).
20
+ - Added metadata-driven AST-aware adaptive context expansion so `adaptive` search can prefer same-file parent/sibling chunks while preserving bounded context and clean returned source.
21
+
22
+ ### Changed
23
+ - Code indexing now analyzes supported source files once for both retrieval chunks and symbols, while preserving regex/text fallbacks for unsupported or unparsable content.
24
+ - Searchable embedding/FTS text now includes richer deterministic code structure; existing KBs should be updated or rebuilt to benefit from the new metadata.
25
+
3
26
 
4
27
  ## [0.8.1] - 2026-08-08
5
28
 
package/README.md CHANGED
@@ -23,8 +23,8 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
23
23
 
24
24
  - **Local-first project memory**: stores indexes under `~/.pi/knowledge/`; no project files are modified.
25
25
  - **Hybrid retrieval**: lexical-anchored BM25 + semantic vectors + normalized weighted score fusion, with semantic mode for vector-only conceptual recall.
26
- - **Code-aware indexing**: AST-aware chunking for TypeScript/JavaScript, Python, Go, Rust, and Java.
27
- - **Lightweight symbol lookup**: indexes functions, classes, interfaces, types, variables, route-like handlers, Markdown headings, config keys, and env vars for exact agent lookup.
26
+ - **Code-aware indexing**: recursive AST chunking for TypeScript/JavaScript, Python, Go, Rust, Java, Bash, GNU C, C++, and QML, with bounded fallback chunks for oversized declarations.
27
+ - **Lightweight symbol lookup**: indexes functions, methods, classes, interfaces, types, variables, route-like handlers, Markdown headings, config keys, and env vars for exact agent lookup.
28
28
  - **Better agent answers**: adaptive context windows, diversity reranking, optional cross-encoder reranking, and diagnostics.
29
29
  - **Large-project stability**: persisted indexing progress, capped batches, streaming vector scans, and stuck-job detection.
30
30
  - **Private by default**: local embeddings run without an API key.
@@ -43,7 +43,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
43
43
  | **Diversity reranking** | ✅ | ❌ | ❌ |
44
44
  | **Incremental re-indexing** | ✅ | ❌ | ❌ |
45
45
  | **File watcher (auto-update)** | ✅ | ❌ | ❌ |
46
- | **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java) | ❌ | ❌ |
46
+ | **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java/Bash/C/C++/QML) | ❌ | ❌ |
47
47
  | **Symbol/config/heading lookup** | ✅ | ❌ | ❌ |
48
48
  | **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
49
49
  | **Index quality diagnostics** | ✅ | ❌ | ❌ |
@@ -64,9 +64,9 @@ TUI rendering currently uses Pi's stable default tool rendering plus targeted re
64
64
 
65
65
  - **RAG-native project memory**: follows the Retrieval-Augmented Generation pattern from Lewis et al. 2020: keep source truth outside the model, retrieve it at answer time, and inject only relevant context.
66
66
  - **Dense semantic recall**: uses multilingual dense embeddings in the spirit of Dense Passage Retrieval (Karpukhin et al. 2020), so conceptual queries can find code/docs even when wording differs.
67
- - **Contextual Retrieval without remote chunk rewriting**: applies Anthropic's Contextual Retrieval insight locally by embedding file path, file type, Markdown breadcrumbs, and code symbols with each chunk. This improves standalone chunk meaning without sending private source chunks to an LLM for context generation.
67
+ - **Contextual Retrieval without remote chunk rewriting**: applies Anthropic's Contextual Retrieval insight locally by embedding file path, file type, Markdown breadcrumbs, and AST-derived code structure such as symbols, scope, parent symbol, and signatures with each chunk. This improves standalone chunk meaning without sending private source chunks to an LLM for context generation.
68
68
  - **Hybrid retrieval with diagnosable scores**: combines lexical BM25 anchors and vectors with normalized weighted score fusion. RRF (Cormack et al. 2009) remains the baseline reference, but weighted fusion is used by default because project dogfood showed RRF compressed scores too much for ranking diagnostics.
69
- - **MMR-style diversity**: uses Maximal Marginal Relevance ideas (Goldstein and Carbonell 1998), file interleaving, vector redundancy checks, and adaptive-window overlap collapse so repeated README or same-file chunks do not dominate top results.
69
+ - **MMR-style diversity**: uses Maximal Marginal Relevance ideas (Goldstein and Carbonell 1998), file interleaving, vector redundancy checks, and bounded adaptive-window overlap collapse so repeated README or same-file chunks do not dominate top results. Code KBs with AST metadata can also use same-parent/sibling structure during adaptive expansion.
70
70
  - **Intent-aware and self-correcting agent UX**: mode selection (`auto`, `fast`, `semantic`, `hybrid`, `adaptive`, `deep`), ranking diagnostics, and `knowledge_doctor` turn retrieval failures into concrete next actions instead of silent bad answers.
71
71
  - **Confidence gating**: low-evidence hybrid matches can return zero results instead of unrelated chunks, reducing false confidence when the KB does not contain the answer.
72
72
 
@@ -132,14 +132,14 @@ Mode selection contract:
132
132
  - Start with `hybrid` for most project questions that contain useful lexical anchors.
133
133
  - Use `fast` for exact symbols, filenames, commands, error codes, API names, config keys, or quoted strings.
134
134
  - Use `semantic` when the query is conceptual, exact terms may differ from indexed wording, or hybrid returns no lexical matches.
135
- - Use `adaptive` when the answer needs nearby code, neighboring documentation sections, or enough context to make a safe edit.
135
+ - Use `adaptive` when the answer needs nearby code, neighboring documentation sections, AST-related sibling methods where indexed, or enough context to make a safe edit.
136
136
  - Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
137
137
  - If results are empty or weak but the KB should contain the answer, retry once with a different mode before concluding no answer exists.
138
138
 
139
- Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
139
+ Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. In `adaptive` mode, code chunks with AST metadata can prefer same-parent/sibling chunks inside the same bounded file window; this is metadata-driven context expansion, not caller/callee analysis. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, source freshness, and adaptive `source_chunk_ids` where available.
140
140
 
141
- For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
142
- Symbol/config/heading metadata is also rebuilt during `knowledge_update`; older KBs created before this feature may show a doctor action recommending update.
141
+ For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and AST-derived code structure such as symbol names, scopes, parent symbols, and signatures while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, classes, methods, or functions, and reduces duplicate-looking chunk hits.
142
+ Symbol/config/heading metadata is also rebuilt during `knowledge_update`; code symbol metadata includes AST-backed methods for supported languages. Older KBs created before this feature may show a doctor action recommending update.
143
143
 
144
144
  ## Embedding Configuration
145
145
 
@@ -224,7 +224,7 @@ All data is stored globally at `~/.pi/knowledge/` under Pi or `~/.omp/knowledge/
224
224
  - **Override**: set `PI_KNOWLEDGE_DIR` or `OMP_KNOWLEDGE_DIR`
225
225
  - **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
226
226
  - **Updates**: extension updates do not affect existing indexed data. Schema migrations run automatically if needed.
227
- - **Symbol index**: `knowledge.db` also stores lightweight symbol/config/heading metadata used by `knowledge_symbol_search`; it is derived from indexed source. File, directory, and URL KBs with retained source paths can rebuild it with `knowledge_update`; imported portable KBs and inline text KBs should be re-imported or re-added because they intentionally do not store an active local source manifest.
227
+ - **Symbol index**: `knowledge.db` also stores lightweight symbol/config/heading metadata used by `knowledge_symbol_search`; it is derived from indexed source. Supported code files use the same AST analysis as chunking, so method symbols can be looked up even when their parent class remains one retrieval chunk. File, directory, and URL KBs with retained source paths can rebuild it with `knowledge_update`; imported portable KBs and inline text KBs should be re-imported or re-added because they intentionally do not store an active local source manifest.
228
228
 
229
229
  ## Development
230
230
 
@@ -3,7 +3,7 @@ import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
4
  import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
- import { addSkippedScanEntry, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
6
+ import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
7
  import { extractSymbols } from "./indexer/symbols.js";
8
8
  import { shutdownModelWorker } from "./model-worker-client.js";
9
9
  import { searchBM25 } from "./search/bm25.js";
@@ -254,7 +254,67 @@ function buildQuerySnippet(content, query, maxLength = 240) {
254
254
  const suffix = end < content.length ? "..." : "";
255
255
  return `${prefix}${content.slice(start, end)}${suffix}`;
256
256
  }
257
+ function metadataString(record, key) {
258
+ const value = record[key];
259
+ return typeof value === "string" && value.trim() ? value.trim() : undefined;
260
+ }
261
+ function metadataNumber(record, key) {
262
+ const value = record[key];
263
+ return typeof value === "number" && Number.isFinite(value) ? value : undefined;
264
+ }
265
+ function metadataStringArray(record, key) {
266
+ const value = record[key];
267
+ return Array.isArray(value)
268
+ ? value.filter((item) => typeof item === "string" && Boolean(item.trim()))
269
+ : [];
270
+ }
271
+ function parseAdaptiveMetadata(chunk) {
272
+ try {
273
+ const parsed = JSON.parse(chunk.metadata_json);
274
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
275
+ return { scope: [] };
276
+ const record = parsed;
277
+ return {
278
+ symbol: metadataString(record, "symbol") ?? metadataString(record, "function_name"),
279
+ symbolKind: metadataString(record, "symbol_kind"),
280
+ parentSymbol: metadataString(record, "parent_symbol"),
281
+ scope: metadataStringArray(record, "scope"),
282
+ startLine: metadataNumber(record, "start_line"),
283
+ endLine: metadataNumber(record, "end_line"),
284
+ };
285
+ }
286
+ catch {
287
+ return { scope: [] };
288
+ }
289
+ }
290
+ function hasAdaptiveStructure(metadata) {
291
+ return Boolean(metadata.symbol || metadata.symbolKind || metadata.parentSymbol || metadata.scope.length > 0);
292
+ }
293
+ function parentScope(scope) {
294
+ return scope.length > 1 ? scope.slice(0, -1) : [];
295
+ }
296
+ function sameScope(left, right) {
297
+ return left.length > 0 && left.length === right.length && left.every((item, index) => item === right[index]);
298
+ }
299
+ function isScopePrefix(parent, child) {
300
+ return parent.length > 0 && parent.length < child.length && parent.every((item, index) => item === child[index]);
301
+ }
302
+ function adaptiveRelationBoost(seed, candidate) {
303
+ if (!hasAdaptiveStructure(seed) || !hasAdaptiveStructure(candidate))
304
+ return 0;
305
+ const seedParentScope = parentScope(seed.scope);
306
+ const candidateParentScope = parentScope(candidate.scope);
307
+ if (sameScope(seedParentScope, candidateParentScope))
308
+ return seed.symbol === candidate.symbol ? 0.35 : 1.35;
309
+ if (seed.parentSymbol && seed.parentSymbol === candidate.parentSymbol) {
310
+ return seed.symbol === candidate.symbol ? 0.35 : 1.2;
311
+ }
312
+ if (isScopePrefix(seed.scope, candidate.scope) || isScopePrefix(candidate.scope, seed.scope))
313
+ return 0.9;
314
+ return 0;
315
+ }
257
316
  function buildAdaptiveContext(seed, chunks, queryTokens, options) {
317
+ const seedMetadata = parseAdaptiveMetadata(seed);
258
318
  const scored = chunks
259
319
  .map((chunk) => {
260
320
  const distance = chunk.id === seed.id
@@ -263,9 +323,10 @@ function buildAdaptiveContext(seed, chunks, queryTokens, options) {
263
323
  const proximity = 1 / (1 + distance / 20);
264
324
  const coverage = queryCoverage(chunk.content, queryTokens);
265
325
  const seedBoost = chunk.id === seed.id ? 2 : 0;
266
- return { chunk, score: seedBoost + proximity + coverage };
326
+ const relationBoost = chunk.id === seed.id ? 0 : adaptiveRelationBoost(seedMetadata, parseAdaptiveMetadata(chunk));
327
+ return { chunk, score: seedBoost + relationBoost + proximity + coverage };
267
328
  })
268
- .sort((a, b) => b.score - a.score)
329
+ .sort((a, b) => b.score - a.score || a.chunk.start_line - b.chunk.start_line || a.chunk.end_line - b.chunk.end_line)
269
330
  .slice(0, options.neighborTarget);
270
331
  if (!scored.some((item) => item.chunk.id === seed.id))
271
332
  scored.push({ chunk: seed, score: Number.MAX_SAFE_INTEGER });
@@ -275,6 +336,7 @@ function buildAdaptiveContext(seed, chunks, queryTokens, options) {
275
336
  .sort((a, b) => a.start_line - b.start_line || a.end_line - b.end_line);
276
337
  const parts = [];
277
338
  const sourceChunkIds = [];
339
+ const includedChunks = [];
278
340
  let totalLength = 0;
279
341
  for (const chunk of ordered) {
280
342
  if (totalLength >= options.maxContextChars)
@@ -283,12 +345,13 @@ function buildAdaptiveContext(seed, chunks, queryTokens, options) {
283
345
  const text = chunk.content.slice(0, remaining);
284
346
  parts.push(text);
285
347
  sourceChunkIds.push(chunk.id);
348
+ includedChunks.push(chunk);
286
349
  totalLength += text.length + 2;
287
350
  }
288
351
  return {
289
352
  content: parts.join("\n\n"),
290
- startLine: Math.min(...ordered.map((chunk) => chunk.start_line)),
291
- endLine: Math.max(...ordered.map((chunk) => chunk.end_line)),
353
+ startLine: Math.min(...includedChunks.map((chunk) => chunk.start_line)),
354
+ endLine: Math.max(...includedChunks.map((chunk) => chunk.end_line)),
292
355
  sourceChunkIds,
293
356
  };
294
357
  }
@@ -712,6 +775,11 @@ export class KnowledgeEngine {
712
775
  const addSymbols = (content, filePath, fileType) => {
713
776
  insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
714
777
  };
778
+ const analyzeAndAddSymbols = async (content, filePath, fileType) => {
779
+ const analysis = await analyzeIndexableContent(content, filePath, fileType);
780
+ insertSymbols(db, kb.id, analysis.symbols);
781
+ return analysis.chunks;
782
+ };
715
783
  if (isUrl) {
716
784
  const message = `Fetching ${source}...`;
717
785
  updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
@@ -724,8 +792,8 @@ export class KnowledgeEngine {
724
792
  else if (isFile) {
725
793
  const extracted = await extractSourceFileContent(resolvedSource, signal);
726
794
  fileCount = 1;
727
- addSymbols(extracted.content, resolvedSource, extracted.fileType);
728
- await addChunks(await chunkFile(extracted.content, resolvedSource));
795
+ const chunks = await analyzeAndAddSymbols(extracted.content, resolvedSource, extracted.fileType);
796
+ await addChunks(chunks);
729
797
  }
730
798
  else if (isDir) {
731
799
  const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
@@ -755,12 +823,11 @@ export class KnowledgeEngine {
755
823
  latestSkippedTotal = skipped.total;
756
824
  continue;
757
825
  }
758
- const chunks = await chunkFile(extracted.content, file.relPath);
826
+ const chunks = await analyzeAndAddSymbols(extracted.content, file.relPath, extracted.fileType);
759
827
  processedFiles++;
760
828
  latestSkippedTotal = skipped.total;
761
829
  if (chunks.length > 0)
762
830
  fileCount++;
763
- addSymbols(extracted.content, file.relPath, extracted.fileType);
764
831
  await addChunks(chunks, processedFiles, plan.files);
765
832
  if (processedFiles % 25 === 0)
766
833
  reportProgress("Chunking", processedFiles, plan.files, skipped.total);
@@ -781,8 +848,8 @@ export class KnowledgeEngine {
781
848
  }
782
849
  else {
783
850
  fileCount = 1;
784
- addSymbols(source, "inline-text", "text");
785
- await addChunks(await chunkFile(source, "inline-text"));
851
+ const chunks = await analyzeAndAddSymbols(source, "inline-text", "text");
852
+ await addChunks(chunks);
786
853
  }
787
854
  await flushPending();
788
855
  vectorWriter.close();
@@ -986,9 +1053,10 @@ export class KnowledgeEngine {
986
1053
  const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
987
1054
  if (!extracted)
988
1055
  continue;
1056
+ const analysis = await analyzeIndexableContent(extracted.content, file.relPath, extracted.fileType);
989
1057
  scannedFiles++;
990
- stagedSymbols.push(...extractSymbols(extracted.content, file.relPath, extracted.fileType));
991
- await processChunks(await chunkFile(extracted.content, file.relPath));
1058
+ stagedSymbols.push(...analysis.symbols);
1059
+ await processChunks(analysis.chunks);
992
1060
  if (scannedFiles % 25 === 0) {
993
1061
  const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
994
1062
  updateIndexingJob(this.db, kb.id, {
@@ -1022,8 +1090,9 @@ export class KnowledgeEngine {
1022
1090
  else {
1023
1091
  const extracted = await extractSourceFileContent(kb.source_path, signal);
1024
1092
  scannedFiles = 1;
1025
- stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
1026
- await processChunks(await chunkFile(extracted.content, kb.source_path));
1093
+ const analysis = await analyzeIndexableContent(extracted.content, kb.source_path, extracted.fileType);
1094
+ stagedSymbols.push(...analysis.symbols);
1095
+ await processChunks(analysis.chunks);
1027
1096
  }
1028
1097
  await flushPending();
1029
1098
  addedVectorWriter.close();
@@ -1,4 +1,4 @@
1
- import type { ChunkInsert } from "../storage/sqlite.ts";
1
+ import type { ChunkInsert, KnowledgeSymbolInsert } from "../storage/sqlite.ts";
2
2
  export interface ScanOptions {
3
3
  includeSuggestedText?: boolean;
4
4
  includePaths?: string[];
@@ -52,4 +52,9 @@ export declare function chunkIdentityHash(opts: {
52
52
  export declare function buildChunkEmbeddingText(chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">): string;
53
53
  export declare function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[];
54
54
  export declare function chunkText(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[];
55
+ export declare function isCodeFileType(fileType: string): boolean;
56
+ export declare function analyzeIndexableContent(content: string, filePath: string, fileType?: string): Promise<{
57
+ chunks: Omit<ChunkInsert, "kb_id">[];
58
+ symbols: KnowledgeSymbolInsert[];
59
+ }>;
55
60
  export declare function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]>;
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
2
2
  import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
3
3
  import { extname, join, relative, sep } from "node:path";
4
4
  import ignore from "ignore";
5
+ import { extractSymbols } from "./symbols.js";
5
6
  const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
6
7
  const DOCUMENT_EXTENSIONS = { ".doc": true, ".docx": true, ".pdf": true };
7
8
  const BINARY_EXTENSIONS = {
@@ -142,8 +143,13 @@ function detectFileType(filePath) {
142
143
  ".java": "java",
143
144
  ".c": "c",
144
145
  ".cpp": "cpp",
145
- ".h": "c",
146
+ ".cc": "cpp",
147
+ ".cxx": "cpp",
148
+ ".h": "text",
146
149
  ".hpp": "cpp",
150
+ ".hh": "cpp",
151
+ ".hxx": "cpp",
152
+ ".qml": "qml",
147
153
  ".md": "markdown",
148
154
  ".mdx": "markdown",
149
155
  ".json": "json",
@@ -153,8 +159,8 @@ function detectFileType(filePath) {
153
159
  ".html": "html",
154
160
  ".css": "css",
155
161
  ".scss": "css",
156
- ".sh": "shell",
157
- ".bash": "shell",
162
+ ".sh": "bash",
163
+ ".bash": "bash",
158
164
  ".zsh": "shell",
159
165
  ".sql": "sql",
160
166
  ".graphql": "graphql",
@@ -339,13 +345,32 @@ function buildContextPrefix(filePath, fileType, metadata) {
339
345
  const parts = [`File: ${filePath}`, `Type: ${fileType}`];
340
346
  const heading = typeof metadata.heading === "string" ? metadata.heading : "";
341
347
  const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
342
- const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
348
+ const language = typeof metadata.language === "string" ? metadata.language : "";
349
+ const symbol = typeof metadata.symbol === "string"
350
+ ? metadata.symbol
351
+ : typeof metadata.function_name === "string"
352
+ ? metadata.function_name
353
+ : "";
354
+ const symbolKind = typeof metadata.symbol_kind === "string" ? metadata.symbol_kind : "";
355
+ const parentSymbol = typeof metadata.parent_symbol === "string" ? metadata.parent_symbol : "";
356
+ const signature = typeof metadata.signature === "string" ? metadata.signature : "";
357
+ const scope = Array.isArray(metadata.scope) ? metadata.scope.filter((item) => typeof item === "string") : [];
343
358
  if (breadcrumb)
344
359
  parts.push(`Section: ${breadcrumb}`);
345
360
  else if (heading)
346
361
  parts.push(`Section: ${normalizeHeading(heading)}`);
362
+ if (language)
363
+ parts.push(`Language: ${language}`);
364
+ if (scope.length > 0)
365
+ parts.push(`Scope: ${scope.join(" > ")}`);
366
+ else if (parentSymbol)
367
+ parts.push(`Parent: ${parentSymbol}`);
368
+ if (symbolKind)
369
+ parts.push(`Kind: ${symbolKind}`);
347
370
  if (symbol)
348
371
  parts.push(`Symbol: ${symbol}`);
372
+ if (signature)
373
+ parts.push(`Signature: ${signature}`);
349
374
  return parts.join("\n");
350
375
  }
351
376
  export function buildChunkEmbeddingText(chunk) {
@@ -504,13 +529,51 @@ export function chunkText(content, filePath) {
504
529
  flush();
505
530
  return chunks;
506
531
  }
532
+ export function isCodeFileType(fileType) {
533
+ return ["typescript", "javascript", "python", "go", "rust", "java", "bash", "c", "cpp", "qml"].includes(fileType);
534
+ }
535
+ export async function analyzeIndexableContent(content, filePath, fileType = detectFileType(filePath)) {
536
+ const detectedFileType = detectFileType(filePath);
537
+ const analysisFileType = isCodeFileType(detectedFileType) ? detectedFileType : fileType;
538
+ if (isCodeFileType(analysisFileType)) {
539
+ try {
540
+ const { analyzeCodeWithAST } = await import("./chunkers/code-ast.js");
541
+ const analysis = await analyzeCodeWithAST(content, filePath, analysisFileType);
542
+ const chunks = analysis.chunks.length > 0 ? analysis.chunks : await chunkFile(content, filePath);
543
+ const symbols = dedupeSymbols([...analysis.symbols, ...extractSymbols(content, filePath, analysisFileType)]);
544
+ return { chunks, symbols };
545
+ }
546
+ catch {
547
+ /* fallback below */
548
+ }
549
+ }
550
+ return { chunks: await chunkFile(content, filePath), symbols: extractSymbols(content, filePath, analysisFileType) };
551
+ }
552
+ function dedupeSymbols(symbols) {
553
+ const seen = new Set();
554
+ const deduped = [];
555
+ for (const symbol of symbols) {
556
+ const key = [
557
+ symbol.kind,
558
+ symbol.name,
559
+ symbol.file_path,
560
+ String(symbol.start_line),
561
+ symbol.container_name ?? "",
562
+ ].join("\0");
563
+ if (seen.has(key))
564
+ continue;
565
+ seen.add(key);
566
+ deduped.push(symbol);
567
+ }
568
+ return deduped;
569
+ }
507
570
  export async function chunkFile(content, filePath) {
508
571
  const fileType = detectFileType(filePath);
509
572
  let chunks = [];
510
573
  if (fileType === "markdown") {
511
574
  chunks = chunkMarkdown(content, filePath);
512
575
  }
513
- else if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
576
+ else if (isCodeFileType(fileType)) {
514
577
  try {
515
578
  const { chunkWithAST } = await import("./chunkers/code-ast.js");
516
579
  chunks = await chunkWithAST(content, filePath, fileType);
@@ -1,3 +1,31 @@
1
- import type { ChunkInsert } from "../../storage/sqlite.ts";
1
+ import type { ChunkInsert, KnowledgeSymbolInsert } from "../../storage/sqlite.ts";
2
+ type StructureKind = "module" | "namespace" | "import" | "class" | "interface" | "type" | "function" | "method" | "constructor" | "destructor" | "property" | "signal" | "handler" | "id" | "binding" | "variable";
3
+ export interface CodeStructureNode {
4
+ kind: StructureKind;
5
+ astType: string;
6
+ name?: string;
7
+ signature?: string;
8
+ language: string;
9
+ startLine: number;
10
+ endLine: number;
11
+ startByte: number;
12
+ endByte: number;
13
+ exported: boolean;
14
+ decorators: string[];
15
+ modifiers: string[];
16
+ visibility?: string;
17
+ parentSymbol?: string;
18
+ scope: string[];
19
+ astPath: string[];
20
+ children: CodeStructureNode[];
21
+ }
22
+ export interface CodeAnalysisResult {
23
+ chunks: Omit<ChunkInsert, "kb_id">[];
24
+ symbols: KnowledgeSymbolInsert[];
25
+ structure: CodeStructureNode;
26
+ }
27
+ export declare function parseCodeStructure(content: string, language: string): Promise<CodeStructureNode>;
28
+ export declare function analyzeCodeWithAST(content: string, filePath: string, language: string): Promise<CodeAnalysisResult>;
2
29
  export declare function chunkWithAST(content: string, filePath: string, language: string): Promise<Omit<ChunkInsert, "kb_id">[]>;
3
30
  export declare const SUPPORTED_LANGUAGES: Set<string>;
31
+ export {};