coderay 1.0.2__tar.gz → 1.0.4__tar.gz

{coderay-1.0.2/src/coderay.egg-info → coderay-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coderay
-Version: 1.0.2
+Version: 1.0.4
 Summary: X-ray your codebase — semantic search, code graphs, file skeletons, and MCP server
 Author-email: Bogdan Copocean <bogdancopocean@gmail.com>
 License-Expression: MIT
@@ -20,35 +20,31 @@ Classifier: Topic :: Text Processing :: Indexing
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: python-dotenv>=1.0.0
-Requires-Dist: networkx>=3.0
-Requires-Dist: tree-sitter>=0.24.0
+Requires-Dist: python-dotenv>=1.2.0
+Requires-Dist: networkx>=3.4
+Requires-Dist: tree-sitter>=0.25.0
 Requires-Dist: tree-sitter-python>=0.25.0
-Requires-Dist: lancedb>=0.5.0
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: click>=8.0
-Requires-Dist: filelock>=3.0
-Requires-Dist: fastembed>=0.4.0
-Requires-Dist: mcp>=1.0.0
-Requires-Dist: watchdog>=4.0.0
-Requires-Dist: pathspec>=0.12.0
-Provides-Extra: openai
-Requires-Dist: openai>=1.0.0; extra == "openai"
+Requires-Dist: lancedb>=0.29.0
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: click>=8.3.0
+Requires-Dist: filelock>=3.25.0
+Requires-Dist: fastembed>=0.7.0
+Requires-Dist: fastmcp==3.1.0
+Requires-Dist: watchdog>=6.0.0
+Requires-Dist: pathspec>=1.0.0
 Provides-Extra: languages
-Requires-Dist: tree-sitter-javascript>=0.23.0; extra == "languages"
-Requires-Dist: tree-sitter-typescript>=0.23.0; extra == "languages"
-Requires-Dist: tree-sitter-go>=0.23.0; extra == "languages"
+Requires-Dist: tree-sitter-javascript>=0.25.0; extra == "languages"
+Requires-Dist: tree-sitter-typescript>=0.23.2; extra == "languages"
+Requires-Dist: tree-sitter-go>=0.25.0; extra == "languages"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
 Requires-Dist: ruff>=0.8.0; extra == "dev"
 Requires-Dist: mypy>=1.0.0; extra == "dev"
-Requires-Dist: openai>=1.0.0; extra == "dev"
-Requires-Dist: httpx>=0.27.0; extra == "dev"
 Provides-Extra: maintain
 Requires-Dist: pylance>=0.15.0; extra == "maintain"
 Provides-Extra: all
-Requires-Dist: coderay[dev,languages,maintain,openai]; extra == "all"
+Requires-Dist: coderay[dev,languages,maintain]; extra == "all"
 Dynamic: license-file
 
 # CodeRay
@@ -72,7 +68,7 @@ AI coding assistants and a standalone CLI.
 pip install coderay
 ```
 
-With all extras (OpenAI embeddings, JS/TS/Go support, MCP server):
+With all extras (JS/TS/Go support, MCP server tools):
 
 ```bash
 pip install "coderay[all]"
@@ -132,7 +128,7 @@ Optional `config.yaml` in the index directory:
 
 ```yaml
 embedder:
-  provider: local           # local | openai
+  provider: local
   model: all-MiniLM-L6-v2
   dimensions: 384
 

{coderay-1.0.2 → coderay-1.0.4}/README.md

@@ -19,7 +19,7 @@ AI coding assistants and a standalone CLI.
 pip install coderay
 ```
 
-With all extras (OpenAI embeddings, JS/TS/Go support, MCP server):
+With all extras (JS/TS/Go support, MCP server tools):
 
 ```bash
 pip install "coderay[all]"
@@ -79,7 +79,7 @@ Optional `config.yaml` in the index directory:
 
 ```yaml
 embedder:
-  provider: local           # local | openai
+  provider: local
   model: all-MiniLM-L6-v2
   dimensions: 384
 

{coderay-1.0.2 → coderay-1.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "coderay"
-version = "1.0.2"
+version = "1.0.4"
 description = "X-ray your codebase — semantic search, code graphs, file skeletons, and MCP server"
 readme = "README.md"
 license = "MIT"
@@ -25,18 +25,18 @@ classifiers = [
     "Topic :: Text Processing :: Indexing",
 ]
 dependencies = [
-    "python-dotenv>=1.0.0",
-    "networkx>=3.0",
-    "tree-sitter>=0.24.0",
+    "python-dotenv>=1.2.0",
+    "networkx>=3.4",
+    "tree-sitter>=0.25.0",
     "tree-sitter-python>=0.25.0",
-    "lancedb>=0.5.0",
-    "pyyaml>=6.0",
-    "click>=8.0",
-    "filelock>=3.0",
-    "fastembed>=0.4.0",
-    "mcp>=1.0.0",
-    "watchdog>=4.0.0",
-    "pathspec>=0.12.0",
+    "lancedb>=0.29.0",
+    "pyyaml>=6.0.3",
+    "click>=8.3.0",
+    "filelock>=3.25.0",
+    "fastembed>=0.7.0",
+    "fastmcp==3.1.0",
+    "watchdog>=6.0.0",
+    "pathspec>=1.0.0",
 ]
 
 [project.urls]
@@ -45,25 +45,22 @@ Repository = "https://github.com/bogdan-copocean/coderay"
 Issues = "https://github.com/bogdan-copocean/coderay/issues"
 
 [project.optional-dependencies]
-openai = ["openai>=1.0.0"]
 languages = [
-    "tree-sitter-javascript>=0.23.0",
-    "tree-sitter-typescript>=0.23.0",
-    "tree-sitter-go>=0.23.0",
+    "tree-sitter-javascript>=0.25.0",
+    "tree-sitter-typescript>=0.23.2",
+    "tree-sitter-go>=0.25.0",
 ]
 dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
     "ruff>=0.8.0",
     "mypy>=1.0.0",
-    "openai>=1.0.0",
-    "httpx>=0.27.0",
 ]
 maintain = [
     "pylance>=0.15.0",
 ]
 all = [
-    "coderay[openai,languages,dev,maintain]",
+    "coderay[languages,dev,maintain]",
 ]
 
 [project.scripts]

coderay-1.0.4/src/coderay/chunking/chunker.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from coderay.core.models import Chunk
+from coderay.parsing.base import BaseTreeSitterParser, parse_file
+
+logger = logging.getLogger(__name__)
+
+MODULE_SYMBOL = "<module>"
+
+
+class ChunkingTreeSitterParser(BaseTreeSitterParser):
+    """Tree-sitter based chunker for source files."""
+
+    def collect_chunks(self) -> list[Chunk]:
+        """Collect all chunks for the configured file and language."""
+        tree = self.get_tree()
+        root = tree.root_node
+        chunks: list[Chunk] = []
+
+        if preamble_lines := self._collect_preamble_lines(root):
+            chunks.append(
+                Chunk(
+                    path=self.file_path,
+                    start_line=1,
+                    end_line=root.end_point[0] + 1,
+                    symbol=MODULE_SYMBOL,
+                    content="\n".join(preamble_lines),
+                )
+            )
+
+        def _dfs(node) -> None:
+            if node.type in self._ctx.lang_cfg.chunker.chunk_types:
+                if (
+                    node.parent
+                    and node.parent.type in self._ctx.lang_cfg.chunker.chunk_types
+                ):
+                    for child in node.children:
+                        _dfs(child)
+                    return
+                start_line = node.start_point[0] + 1
+                end_line = node.end_point[0] + 1
+                text = self.node_text(node)
+                symbol = self.identifier_from_node(node) or f"<{node.type}>"
+                chunks.append(
+                    Chunk(
+                        path=self.file_path,
+                        start_line=start_line,
+                        end_line=end_line,
+                        symbol=symbol,
+                        content=text,
+                    )
+                )
+            for child in node.children:
+                _dfs(child)
+
+        _dfs(root)
+        logger.debug("Chunked %s: %d chunks", self.file_path, len(chunks))
+        return chunks
+
+    def _collect_preamble_lines(self, root) -> list[str]:
+        """Collect top-level lines that are NOT part of any chunk_type definition."""
+        lines: list[str] = []
+        for child in root.children:
+            if child.type in self._ctx.lang_cfg.chunker.chunk_types:
+                continue
+            text = self.node_text(child).strip()
+            if text:
+                lines.append(text)
+        return lines
+
+
+def chunk_file(path: str | Path, content: str) -> list[Chunk]:
+    """Chunk a source file into semantic units (functions, classes, preamble)."""
+    ctx = parse_file(path, content)
+    if ctx is None:
+        logger.warning("No language config for %s", path)
+        return []
+    parser = ChunkingTreeSitterParser(ctx)
+    try:
+        return parser.collect_chunks()
+    except Exception as e:  # pragma: no cover - defensive logging
+        logger.warning("Chunking failed for %s: %s", path, e)
+        return []

{coderay-1.0.2 → coderay-1.0.4}/src/coderay/cli/commands.py

@@ -14,7 +14,7 @@ from coderay.retrieval.search import Retrieval
 from coderay.state.machine import StateMachine
 from coderay.storage.lancedb import index_exists
 
-# Load .env so OPENAI_API_KEY etc. are available (e.g. for embedder).
+# Load .env so configuration and environment variables are available.
 load_dotenv()
 
 # ANSI colors (safe when not TTY we can strip or leave)
@@ -39,21 +39,22 @@ def _setup_logging(verbose: bool = False) -> None:
         level=level,
         datefmt="%H:%M:%S",
     )
-    # Suppress noisy OpenAI/HTTP logging; keep only warnings and errors
-    for name in ("openai", "httpx", "httpcore"):
+    # Suppress noisy HTTP logging; keep only warnings and errors
+    for name in ("httpx", "httpcore"):
         logging.getLogger(name).setLevel(logging.WARNING)
 
 
 @click.group()
-@click.option("--index-dir", default=".index", help="Index directory (default .index)")
 @click.option("-v", "--verbose", is_flag=True, default=False, help="Verbose logging")
 @click.pass_context
-def cli(ctx: click.Context, index_dir: str, verbose: bool) -> None:
+def cli(ctx: click.Context, verbose: bool) -> None:
     """CodeRay — build, update, search, and inspect the index."""
     _setup_logging(verbose)
     ctx.ensure_object(dict)
-    ctx.obj["index_dir"] = Path(index_dir)
     ctx.obj["verbose"] = verbose
+    from coderay.core.config import get_config
+
+    get_config()
 
 
 @cli.command()
@@ -67,9 +68,12 @@ def cli(ctx: click.Context, index_dir: str, verbose: bool) -> None:
 @click.pass_context
 def build(ctx: click.Context, full: bool, repo: Path) -> None:
     """Build or rebuild the index."""
-    index_dir = ctx.obj["index_dir"]
+    from coderay.core.config import get_config
+
+    config = get_config()
+    index_dir = Path(config.index.path)
     index_dir.mkdir(parents=True, exist_ok=True)
-    indexer = Indexer(repo, index_dir)
+    indexer = Indexer(repo)
     t0 = time.time()
     try:
         with acquire_indexer_lock(index_dir):
@@ -108,8 +112,11 @@ def build(ctx: click.Context, full: bool, repo: Path) -> None:
 @click.pass_context
 def update(ctx: click.Context, repo: Path) -> None:
     """Incremental update (only changed files). Uses file lock."""
-    index_dir = ctx.obj["index_dir"]
-    indexer = Indexer(repo, index_dir)
+    from coderay.core.config import get_config
+
+    config = get_config()
+    index_dir = Path(config.index.path)
+    indexer = Indexer(repo)
     t0 = time.time()
 
     if not indexer.index_exists():
@@ -132,28 +139,29 @@ def update(ctx: click.Context, repo: Path) -> None:
 @click.argument("query_text", required=True)
 @click.option("--top-k", "top_k", default=10, help="Number of results")
 @click.option("--path-prefix", help="Filter by path prefix")
-@click.option("--language", help="Filter by language (e.g. python)")
 @click.pass_context
 def search_cmd(
     ctx: click.Context,
     query_text: str,
     top_k: int,
     path_prefix: str | None,
-    language: str | None,
 ) -> None:
     """Semantic search the index."""
-    index_dir = ctx.obj["index_dir"]
+    from coderay.core.config import get_config
+
+    config = get_config()
+    index_dir = Path(config.index.path)
     if not index_exists(index_dir):
         click.echo(_color("No index found. Run 'coderay build' first.", YELLOW))
         ctx.exit(1)
 
-    sm = StateMachine(index_dir)
+    sm = StateMachine()
     current_state = sm.current_state
     if current_state is None:
         click.echo(_color("No index state. Run 'coderay build' first.", YELLOW))
         ctx.exit(1)
 
-    retrieval = Retrieval(index_dir)
+    retrieval = Retrieval()
     click.echo(_color(f"Searching: {query_text!r}", CYAN))
     t0 = time.perf_counter()
 
@@ -162,7 +170,6 @@ def search_cmd(
         current_state=current_state,
         top_k=top_k,
         path_prefix=path_prefix,
-        language=language,
     )
     elapsed = time.perf_counter() - t0
     click.echo(_color(f"Query took {elapsed:.2f}s", BOLD))
@@ -171,31 +178,16 @@ def search_cmd(
         click.echo(_color("No results.", YELLOW))
         return
 
-    score_type = results[0].get("score_type", "cosine")
-    if score_type == "rrf":
-        click.echo(
-            _color("Scoring: hybrid (RRF) — relative ranking, not a percentage", CYAN)
-        )
-    else:
-        click.echo(_color("Scoring: cosine similarity (0-1)", CYAN))
-
     for i, r in enumerate(results, 1):
         path = r.get("path", "?")
         start = r.get("start_line", 0)
         end = r.get("end_line", 0)
         symbol = r.get("symbol", "?")
-        score = r.get("score", 0)
-        if score_type == "cosine":
-            score_str = f"score={score:.4f} ({score:.0%})"
-        else:
-            score_str = f"score={score:.4f} (rrf)"
         preview = (r.get("content") or "")[:200].replace("\n", " ")
         if len(r.get("content") or "") > 200:
             preview += "..."
         click.echo("")
-        click.echo(
-            _color(f"  {i}. {path}:{start}-{end} ({symbol})  {score_str}", GREEN)
-        )
+        click.echo(_color(f"  {i}. {path}:{start}-{end} ({symbol})", GREEN))
         click.echo(f"     {preview}")
 
 
@@ -226,8 +218,11 @@ def list_cmd(
     show_content: bool,
 ) -> None:
     """Show what is in the index: chunk counts and/or chunk list."""
-    index_dir = ctx.obj["index_dir"]
-    retrieval = Retrieval(index_dir)
+    from coderay.core.config import get_config
+
+    config = get_config()
+    index_dir = Path(config.index.path)
+    retrieval = Retrieval()
     if not index_exists(index_dir):
         click.echo(_color("No index found. Run 'coderay build' first.", YELLOW))
         ctx.exit(1)
@@ -261,23 +256,23 @@ def list_cmd(
 @click.pass_context
 def status(ctx: click.Context) -> None:
     """Show index status: state, branch, commit, chunk count."""
-    index_dir = ctx.obj["index_dir"]
+    from coderay.core.config import get_config
+    from coderay.state.version import read_index_version
+    from coderay.storage.lancedb import Store
+
+    config = get_config()
+    index_dir = Path(config.index.path)
     if not index_exists(index_dir):
         click.echo(_color("No index found. Run 'coderay build' first.", YELLOW))
         ctx.exit(1)
 
-    sm = StateMachine(index_dir)
+    sm = StateMachine()
     state = sm.current_state
     if state is None:
         click.echo(_color("No index state found.", YELLOW))
         ctx.exit(1)
 
-    from coderay.core.config import get_embedding_dimensions, load_config
-    from coderay.state.version import read_index_version
-    from coderay.storage.lancedb import Store
-
-    config = load_config(index_dir)
-    store = Store(index_dir, dimensions=get_embedding_dimensions(config))
+    store = Store()
     chunks = store.chunk_count()
     version = read_index_version(index_dir)
 
@@ -302,12 +297,15 @@ def status(ctx: click.Context) -> None:
 @click.pass_context
 def maintain(ctx: click.Context, repo: Path) -> None:
     """Reclaim space and compact the index."""
-    index_dir = ctx.obj["index_dir"]
+    from coderay.core.config import get_config
+
+    config = get_config()
+    index_dir = Path(config.index.path)
     if not index_exists(index_dir):
         click.echo(_color("No index found. Run 'coderay build' first.", YELLOW))
         ctx.exit(1)
     click.echo(_color("Maintaining index...", CYAN))
-    indexer = Indexer(repo, index_dir)
+    indexer = Indexer(repo)
     with acquire_indexer_lock(index_dir):
         result = indexer.maintain()
     if result.get("cleanup_done"):
@@ -357,8 +355,11 @@ def graph_cmd(
     limit: int,
 ) -> None:
     """List call and import graph edges (who calls who, who imports what)."""
-    index_dir = ctx.obj["index_dir"]
-    retrieval = Retrieval(index_dir)
+    from coderay.core.config import get_config
+
+    config = get_config()
+    index_dir = Path(config.index.path)
+    retrieval = Retrieval()
     if not index_exists(index_dir):
         click.echo(_color("No index found. Run 'coderay build' first.", YELLOW))
         ctx.exit(1)
@@ -394,12 +395,6 @@ def graph_cmd(
     type=click.Path(exists=True, path_type=Path),
     help="Repo root",
 )
-@click.option(
-    "--debounce",
-    type=float,
-    default=None,
-    help="Debounce seconds (default from config, typically 2s)",
-)
 @click.option(
     "--quiet",
     is_flag=True,
@@ -409,14 +404,14 @@ def graph_cmd(
 def watch(
     ctx: click.Context,
     repo: Path,
-    debounce: float | None,
     quiet: bool,
 ) -> None:
     """Watch for file changes and re-index automatically."""
-    from coderay.core.config import load_config
+    from coderay.core.config import get_config
     from coderay.pipeline.watcher import FileWatcher
 
-    index_dir = ctx.obj["index_dir"]
+    config = get_config()
+    index_dir = Path(config.index.path)
     if not index_exists(index_dir):
         click.echo(
             _color(
@@ -426,27 +421,20 @@ def watch(
         )
         ctx.exit(1)
 
-    config = load_config(index_dir)
-    if debounce is not None:
-        config.setdefault("watch", {})["debounce_seconds"] = debounce
-
     if quiet:
         logging.getLogger("coderay.pipeline.watcher").setLevel(logging.WARNING)
 
-    watcher = FileWatcher(repo, index_dir, config=config)
+    watcher = FileWatcher(repo, index_dir)
 
     click.echo(
         _color(
             f"Watching {repo.resolve()} "
-            f"(debounce={config.get('watch', {}).get('debounce_seconds', 2)}s, "
-            f"Ctrl+C to stop)",
+            f"(debounce={config.watcher.debounce}s, Ctrl+C to stop)",
             CYAN,
         )
     )
-    # Do an incremental update at start-up
-    index_dir = ctx.obj["index_dir"]
     index_dir.mkdir(parents=True, exist_ok=True)
-    indexer = Indexer(repo, index_dir)
+    indexer = Indexer(repo)
     indexer.update_incremental()
 
     watcher.start()