sweet-search 2.6.0 → 2.6.2

package/core/indexing/ast-chunker.js

@@ -349,7 +349,11 @@ export class ASTChunker {
     // chunker uses tree-sitter-cpp instead of tree-sitter-c.
     const langInfo = resolveLanguage(filePath, content);
     if (!langInfo || !langInfo.chunker) {
-      return this.parseGenericFile(filePath, content);
+      // Mapped-but-chunkerless languages (e.g. Clojure: no tree-sitter grammar,
+      // and paren-delimited forms don't fit the brace/indent/endKeyword parsers)
+      // fall back to lossless generic windowing but keep their resolved language
+      // id so chunks are tagged 'clojure' rather than generic 'text'.
+      return this.parseGenericFile(filePath, content, langInfo?.id || 'text');
     }
 
     // Try tree-sitter WASM first for supported languages
@@ -909,7 +913,7 @@ export class ASTChunker {
     return subChunks;
   }
 
-  parseGenericFile(filePath, content) {
+  parseGenericFile(filePath, content, language = 'text') {
     const lines = content.split('\n');
     const chunks = [];
     const CHUNK_SIZE = 50;
@@ -921,7 +925,7 @@ export class ASTChunker {
       const chunkContent = lines.slice(start, end).join('\n');
 
       if (chunkContent.trim().length > 20) {
-        chunks.push(this.buildChunk(chunkContent, filePath, 'text', 'code', 'unknown', start, end - 1));
+        chunks.push(this.buildChunk(chunkContent, filePath, language, 'code', 'unknown', start, end - 1));
       }
 
       start = end - OVERLAP;

package/core/infrastructure/config/search.js

@@ -51,8 +51,29 @@ export const ROUTING_CONFIG = {
 export const FILE_PATTERNS = {
   include: [
     // Source code (all major languages)
-    '**/*.{js,jsx,ts,tsx,mjs,cjs}',      // JavaScript/TypeScript
+    '**/*.{js,jsx,ts,tsx,mjs,cjs,cts,mts}', // JavaScript/TypeScript (incl. CommonJS/ESM TS)
     '**/*.{java,kt,kts,scala,groovy}',    // JVM
+    '**/*.{clj,cljc,cljs,edn}',           // Clojure / ClojureScript / EDN
+    '**/*.jl',                             // Julia
+    '**/*.{R,r,Rd,rd,Rmd,rmd}',            // R (case-sensitive matcher: list both cases)
+    '**/*.{ml,mli,mll,mly}',               // OCaml
+    '**/*.{res,resi}',                     // ReScript
+    '**/*.{hs,lhs}',                       // Haskell
+    '**/*.{erl,hrl}',                      // Erlang
+    '**/*.{pl,pm,pod}',                    // Perl
+    '**/*.{f,for,f90,f95,f03,f08,F,F90,F95}', // Fortran (case-sensitive)
+    '**/*.{cob,cbl}',                      // COBOL
+    '**/*.{asm,s,S}',                      // Assembly (case-sensitive)
+    '**/*.{cr,vala,hx,pas,nix,vim}',       // Crystal / Vala / Haxe / Pascal / Nix / Vim
+    '**/*.{elm,sol,tla,rdl,el,ejs}',       // Elm / Solidity / TLA+ / SystemRDL / Emacs Lisp / EJS
+    '**/*.{ql,qll}',                       // CodeQL
+    '**/*.{zeek,bro}',                     // Zeek
+    '**/*.{tcl,tk}',                       // Tcl
+    '**/*.astro',                          // Astro (SFC)
+    // GPU shaders
+    '**/*.{glsl,vert,frag,comp,geom,tesc,tese}', // GLSL
+    '**/*.{hlsl,hlsli}',                   // HLSL
+    '**/*.{metal,wgsl,shader,cg,cginc}',   // Metal / WGSL / ShaderLab / Cg
     '**/*.{py,pyi}',                       // Python
     '**/*.go',                              // Go
     '**/*.rs',                              // Rust
@@ -71,7 +92,10 @@ export const FILE_PATTERNS = {
     '**/*.{yaml,yml}',                      // YAML
     '**/*.toml',                             // TOML
     '**/*.{xml,xsl,xsd,wsdl,pom,csproj}',  // XML
-    '**/*.{md,mdx,mdc,rst,txt}',           // Documentation + Cursor rules
+    '**/*.{tf,tfvars,hcl}',                // Terraform / HCL
+    '**/*.{ini,cfg}',                      // INI / config
+    '**/*.properties',                     // Java properties
+    '**/*.{md,mdx,mdc,rst,txt,markdown}',  // Documentation + Cursor rules
     '**/*.{html,htm,xhtml,vue,svelte}',    // Web markup/SFC
     '**/*.{css,scss,sass,less}',           // Stylesheets
     '**/*.svg',                             // SVG
@@ -81,6 +105,15 @@ export const FILE_PATTERNS = {
     '**/*.dockerfile',                     // Dockerfile alt extension
     '**/Makefile',                         // Makefile
     '**/*.mk',                            // Makefile includes
+    '**/*.cmake',                          // CMake modules
+    '**/*.gradle',                         // Gradle (Groovy DSL)
+    '**/*.ninja',                          // Ninja
+    '**/*.{bzl,star}',                     // Bazel / Starlark
+    '**/BUILD', '**/BUILD.bazel',          // Bazel BUILD
+    '**/WORKSPACE', '**/WORKSPACE.bazel',  // Bazel WORKSPACE
+    '**/meson.build',                      // Meson
+    '**/Earthfile',                        // Earthly
+    '**/justfile', '**/Justfile',          // Just
     // Project markers
     '**/CLAUDE.md',
     '**/AGENTS.md',

package/core/infrastructure/language-patterns/maps.js

@@ -42,6 +42,15 @@ export const EXTENSION_MAP = {
   // Scala
   '.scala': 'scala',
 
+  // Clojure / ClojureScript / EDN
+  // No tree-sitter-clojure grammar ships in the bundle and the brace-based
+  // regex chunker would fragment paren-delimited Lisp forms (and drop sub-30-char
+  // defns via MIN_CONTENT_LENGTH), so there is intentionally no LANGUAGES.clojure
+  // entry — getLanguageByExtension returns the chunker-less fallback and
+  // ast-chunker routes these through parseGenericFile (lossless 50-line windows),
+  // tagged with this 'clojure' language id rather than generic 'text'.
+  '.clj': 'clojure', '.cljc': 'clojure', '.cljs': 'clojure', '.edn': 'clojure',
+
   // Dart
   '.dart': 'dart',
 
@@ -110,8 +119,72 @@ export const EXTENSION_MAP = {
   // F# / VB (.NET additional)
   // '.vb' omitted — no LANGUAGES.vb entry exists
 
+  // Julia
+  '.jl': 'julia',
+
+  // R — the lookup lowercases the key, so '.r'/'.rd'/'.rmd' also resolve the
+  // canonical uppercase on-disk forms (.R/.Rd/.Rmd). Keep keys lowercase only.
+  '.r': 'r', '.rd': 'r', '.rmd': 'r',
+
+  // OCaml (tree-sitter-ocaml ships in the bundle but is unwired → generic chunking)
+  '.ml': 'ocaml', '.mli': 'ocaml', '.mll': 'ocaml', '.mly': 'ocaml',
+
+  // TypeScript modules (CommonJS / ESM) — reuse the wired typescript grammar.
+  // Map to 'typescript' not 'tsx': .cts/.mts cannot contain JSX.
+  '.cts': 'typescript', '.mts': 'typescript',
+
+  // Other bundled-grammar languages (unwired → lossless generic chunking)
+  '.elm': 'elm',
+  '.sol': 'solidity',
+  '.res': 'rescript', '.resi': 'rescript',
+  '.ql': 'ql', '.qll': 'ql',
+  '.tla': 'tlaplus',
+  '.rdl': 'systemrdl',
+  '.el': 'elisp',
+  '.ejs': 'embedded_template',
+
+  // Functional / systems / scripting
+  '.hs': 'haskell', '.lhs': 'haskell',
+  '.erl': 'erlang', '.hrl': 'erlang',
+  '.cr': 'crystal',
+  '.vala': 'vala',
+  '.hx': 'haxe',
+  '.pas': 'pascal',
+  '.nix': 'nix',
+  '.pl': 'perl', '.pm': 'perl', '.pod': 'perl',
+  '.vim': 'vim',
+  '.tcl': 'tcl', '.tk': 'tcl',
+  '.zeek': 'zeek', '.bro': 'zeek',
+
+  // Scientific / legacy (lowercase keys cover the uppercase .F/.F90/.S forms)
+  '.f': 'fortran', '.for': 'fortran', '.f90': 'fortran', '.f95': 'fortran',
+  '.f03': 'fortran', '.f08': 'fortran',
+  '.cob': 'cobol', '.cbl': 'cobol',
+  '.asm': 'assembly', '.s': 'assembly',
+
+  // GPU shaders
+  '.glsl': 'glsl', '.vert': 'glsl', '.frag': 'glsl', '.comp': 'glsl',
+  '.geom': 'glsl', '.tesc': 'glsl', '.tese': 'glsl',
+  '.hlsl': 'hlsl', '.hlsli': 'hlsl',
+  '.metal': 'metal',
+  '.wgsl': 'wgsl',
+  '.shader': 'shaderlab',
+  '.cg': 'cg', '.cginc': 'cg',
+
+  // Web framework SFC — reuse the html SFC path (like .vue/.svelte)
+  '.astro': 'html',
+
+  // Infra / build / config DSLs
+  '.tf': 'hcl', '.tfvars': 'hcl', '.hcl': 'hcl',
+  '.ini': 'ini', '.cfg': 'ini',
+  '.properties': 'properties',
+  '.cmake': 'cmake',
+  '.gradle': 'gradle',
+  '.ninja': 'ninja',
+  '.bzl': 'starlark', '.star': 'starlark',
+
   // Document formats (dispatched to DocumentChunker in ast-chunker.js)
-  '.md': 'markdown', '.mdx': 'markdown',
+  '.md': 'markdown', '.mdx': 'markdown', '.markdown': 'markdown',
   '.rst': 'rst',
   '.txt': 'plaintext',
 };
@@ -121,4 +194,11 @@ export const FILENAME_MAP = {
   Dockerfile: 'dockerfile',
   Makefile: 'makefile',
   GNUmakefile: 'makefile',
+  // Extensionless build / project files (mirrors the Dockerfile/Makefile pattern)
+  BUILD: 'starlark',
+  WORKSPACE: 'starlark',
+  'meson.build': 'meson',
+  Earthfile: 'earthfile',
+  justfile: 'just',
+  Justfile: 'just',
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sweet-search",
-  "version": "2.6.0",
+  "version": "2.6.2",
   "description": "Sweet Search - SOTA Hybrid Code Search Engine with WASM CatBoost Query Router, Semantic/Lexical/Structural Search, and Multilingual Support",
   "type": "module",
   "main": "core/search/sweet-search.js",
@@ -167,13 +167,13 @@
     "vitest": "^4.0.16"
   },
   "optionalDependencies": {
-    "@sweet-search/native-darwin-arm64": "2.6.0",
-    "@sweet-search/native-darwin-x64": "2.6.0",
-    "@sweet-search/native-linux-arm64-gnu": "2.6.0",
-    "@sweet-search/native-linux-arm64-gnu-cuda": "2.6.0",
-    "@sweet-search/native-linux-x64-gnu": "2.6.0",
-    "@sweet-search/native-linux-x64-gnu-cuda": "2.6.0",
-    "@sweet-search/bg-priority": "2.6.0"
+    "@sweet-search/native-darwin-arm64": "2.6.2",
+    "@sweet-search/native-darwin-x64": "2.6.2",
+    "@sweet-search/native-linux-arm64-gnu": "2.6.2",
+    "@sweet-search/native-linux-arm64-gnu-cuda": "2.6.2",
+    "@sweet-search/native-linux-x64-gnu": "2.6.2",
+    "@sweet-search/native-linux-x64-gnu-cuda": "2.6.2",
+    "@sweet-search/bg-priority": "2.6.2"
   },
   "engines": {
     "node": ">=18.0.0"