nexus-dev 3.2.0__py3-none-any.whl

nexus_dev/chunkers/docs_chunker.py

@@ -0,0 +1,291 @@
+"""Documentation chunker for Markdown and RST files."""
+
+from __future__ import annotations
+
+import re
+
+from .base import BaseChunker, ChunkType, CodeChunk
+
+
+class DocumentationChunker(BaseChunker):
+    """Chunker for documentation files (Markdown, RST, plain text).
+
+    Splits documentation by headers/sections while keeping code blocks intact.
+    """
+
+    @property
+    def supported_extensions(self) -> list[str]:
+        return [".md", ".markdown", ".rst", ".txt"]
+
+    def chunk_file(self, file_path: str, content: str) -> list[CodeChunk]:
+        """Parse documentation file and extract sections as chunks.
+
+        Args:
+            file_path: Path to the documentation file.
+            content: File content.
+
+        Returns:
+            List of documentation chunks split by headers.
+        """
+        if not content.strip():
+            return []
+
+        ext = file_path.split(".")[-1].lower()
+
+        if ext in ("md", "markdown"):
+            return self._chunk_markdown(file_path, content)
+        elif ext == "rst":
+            return self._chunk_rst(file_path, content)
+        else:
+            return self._chunk_plain_text(file_path, content)
+
+    def _chunk_markdown(self, file_path: str, content: str) -> list[CodeChunk]:
+        """Chunk Markdown content by headers.
+
+        Args:
+            file_path: File path.
+            content: Markdown content.
+
+        Returns:
+            List of section chunks.
+        """
+        lines = content.split("\n")
+        chunks: list[CodeChunk] = []
+
+        # Pattern for Markdown headers (## or ###)
+        header_pattern = re.compile(r"^(#{1,6})\s+(.+)$")
+
+        current_section: list[str] = []
+        current_header = ""
+        current_start_line = 1
+        parent_header: str | None = None
+
+        for i, line in enumerate(lines):
+            match = header_pattern.match(line)
+
+            if match:
+                # Save previous section if it has content
+                if current_section:
+                    section_content = "\n".join(current_section).strip()
+                    if section_content:
+                        chunks.append(
+                            CodeChunk(
+                                content=section_content,
+                                chunk_type=ChunkType.SECTION,
+                                name=current_header or file_path.split("/")[-1],
+                                start_line=current_start_line,
+                                end_line=i,
+                                language="markdown",
+                                file_path=file_path,
+                                parent=parent_header,
+                            )
+                        )
+
+                # Start new section
+                level = len(match.group(1))
+                header_text = match.group(2).strip()
+
+                # Track parent for level 2+ headers
+                if level == 1:
+                    parent_header = None
+                elif level == 2:
+                    parent_header = current_header if current_header else None
+
+                current_header = header_text
+                current_section = [line]
+                current_start_line = i + 1
+            else:
+                current_section.append(line)
+
+        # Save final section
+        if current_section:
+            section_content = "\n".join(current_section).strip()
+            if section_content:
+                chunks.append(
+                    CodeChunk(
+                        content=section_content,
+                        chunk_type=ChunkType.SECTION,
+                        name=current_header or file_path.split("/")[-1],
+                        start_line=current_start_line,
+                        end_line=len(lines),
+                        language="markdown",
+                        file_path=file_path,
+                        parent=parent_header,
+                    )
+                )
+
+        # If no headers found, return whole file as one chunk
+        if not chunks:
+            chunks.append(
+                CodeChunk(
+                    content=content,
+                    chunk_type=ChunkType.DOCUMENTATION,
+                    name=file_path.split("/")[-1] if "/" in file_path else file_path,
+                    start_line=1,
+                    end_line=len(lines),
+                    language="markdown",
+                    file_path=file_path,
+                )
+            )
+
+        return chunks
+
+    def _chunk_rst(self, file_path: str, content: str) -> list[CodeChunk]:
+        """Chunk RST content by title underlines.
+
+        Args:
+            file_path: File path.
+            content: RST content.
+
+        Returns:
+            List of section chunks.
+        """
+        lines = content.split("\n")
+        chunks: list[CodeChunk] = []
+
+        # RST title underlines: =, -, ~, ^, ", etc.
+        underline_pattern = re.compile(r"^[=\-~^\"\'`\*\+#]+$")
+
+        current_section: list[str] = []
+        current_header = ""
+        current_start_line = 1
+
+        i = 0
+        while i < len(lines):
+            # Check if next line is an underline (making current line a header)
+            if (
+                i + 1 < len(lines)
+                and lines[i].strip()
+                and underline_pattern.match(lines[i + 1].strip())
+                and len(lines[i + 1].strip()) >= len(lines[i].strip())
+            ):
+                # Save previous section
+                if current_section:
+                    section_content = "\n".join(current_section).strip()
+                    if section_content:
+                        chunks.append(
+                            CodeChunk(
+                                content=section_content,
+                                chunk_type=ChunkType.SECTION,
+                                name=current_header or file_path.split("/")[-1],
+                                start_line=current_start_line,
+                                end_line=i,
+                                language="rst",
+                                file_path=file_path,
+                            )
+                        )
+
+                # Start new section with header
+                current_header = lines[i].strip()
+                current_section = [lines[i], lines[i + 1]]
+                current_start_line = i + 1
+                i += 2
+            else:
+                current_section.append(lines[i])
+                i += 1
+
+        # Save final section
+        if current_section:
+            section_content = "\n".join(current_section).strip()
+            if section_content:
+                chunks.append(
+                    CodeChunk(
+                        content=section_content,
+                        chunk_type=ChunkType.SECTION,
+                        name=current_header or file_path.split("/")[-1],
+                        start_line=current_start_line,
+                        end_line=len(lines),
+                        language="rst",
+                        file_path=file_path,
+                    )
+                )
+
+        if not chunks:
+            chunks.append(
+                CodeChunk(
+                    content=content,
+                    chunk_type=ChunkType.DOCUMENTATION,
+                    name=file_path.split("/")[-1] if "/" in file_path else file_path,
+                    start_line=1,
+                    end_line=len(lines),
+                    language="rst",
+                    file_path=file_path,
+                )
+            )
+
+        return chunks
+
+    def _chunk_plain_text(self, file_path: str, content: str) -> list[CodeChunk]:
+        """Chunk plain text by paragraph breaks or fixed size.
+
+        Args:
+            file_path: File path.
+            content: Text content.
+
+        Returns:
+            List of text chunks.
+        """
+        # Split by double newlines (paragraphs)
+        paragraphs = re.split(r"\n\s*\n", content)
+
+        chunks: list[CodeChunk] = []
+        current_chunk: list[str] = []
+        current_length = 0
+        chunk_index = 0
+        max_chunk_size = 1500
+
+        for para in paragraphs:
+            para = para.strip()
+            if not para:
+                continue
+
+            if current_length + len(para) > max_chunk_size and current_chunk:
+                # Save current chunk
+                chunk_content = "\n\n".join(current_chunk)
+                chunks.append(
+                    CodeChunk(
+                        content=chunk_content,
+                        chunk_type=ChunkType.DOCUMENTATION,
+                        name=f"{file_path.split('/')[-1]}:section_{chunk_index}",
+                        start_line=1,  # Not tracking exact lines for plain text
+                        end_line=1,
+                        language="text",
+                        file_path=file_path,
+                    )
+                )
+                current_chunk = []
+                current_length = 0
+                chunk_index += 1
+
+            current_chunk.append(para)
+            current_length += len(para)
+
+        # Save final chunk
+        if current_chunk:
+            chunk_content = "\n\n".join(current_chunk)
+            chunks.append(
+                CodeChunk(
+                    content=chunk_content,
+                    chunk_type=ChunkType.DOCUMENTATION,
+                    name=f"{file_path.split('/')[-1]}:section_{chunk_index}",
+                    start_line=1,
+                    end_line=1,
+                    language="text",
+                    file_path=file_path,
+                )
+            )
+
+        if not chunks:
+            chunks.append(
+                CodeChunk(
+                    content=content,
+                    chunk_type=ChunkType.DOCUMENTATION,
+                    name=file_path.split("/")[-1] if "/" in file_path else file_path,
+                    start_line=1,
+                    end_line=content.count("\n") + 1,
+                    language="text",
+                    file_path=file_path,
+                )
+            )
+
+        return chunks

nexus_dev/chunkers/java_chunker.py

@@ -0,0 +1,343 @@
+"""Java code chunker using tree-sitter."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from tree_sitter_language_pack import get_parser
+
+from .base import BaseChunker, ChunkType, CodeChunk
+
+
+class JavaChunker(BaseChunker):
+    """Tree-sitter based chunker for Java files.
+
+    Extracts classes, interfaces, methods, and constructors as semantic chunks.
+    """
+
+    @property
+    def supported_extensions(self) -> list[str]:
+        return [".java"]
+
+    def __init__(self) -> None:
+        """Initialize the Java parser."""
+        self._parser = get_parser("java")
+
+    def chunk_file(self, file_path: str, content: str) -> list[CodeChunk]:
+        """Parse Java file and extract semantic chunks.
+
+        Args:
+            file_path: Path to the Java file.
+            content: File content.
+
+        Returns:
+            List of code chunks.
+        """
+        if not content.strip():
+            return []
+
+        try:
+            tree = self._parser.parse(content.encode("utf-8"))
+        except Exception:
+            return [self._create_module_chunk(file_path, content)]
+
+        chunks: list[CodeChunk] = []
+        lines = content.split("\n")
+
+        # Extract package and imports for context
+        imports = self._extract_imports(tree.root_node, lines)
+
+        self._walk_tree(tree.root_node, lines, file_path, chunks, imports, parent=None)
+
+        if not chunks:
+            return [self._create_module_chunk(file_path, content)]
+
+        return chunks
+
+    def _walk_tree(
+        self,
+        node: Any,
+        lines: list[str],
+        file_path: str,
+        chunks: list[CodeChunk],
+        imports: list[str],
+        parent: str | None,
+    ) -> None:
+        """Recursively walk the AST to find classes and methods.
+
+        Args:
+            node: Current AST node.
+            lines: Source lines.
+            file_path: File path.
+            chunks: List to append chunks to.
+            imports: Package and import statements.
+            parent: Parent class name if inside a class.
+        """
+        if node.type == "class_declaration":
+            class_name = self._get_identifier(node)
+
+            # Extract the whole class
+            chunk = self._extract_class(node, lines, file_path, imports)
+            if chunk:
+                chunks.append(chunk)
+
+            # Extract methods and constructors
+            for child in node.children:
+                if child.type == "class_body":
+                    for member in child.children:
+                        if member.type == "method_declaration":
+                            method_chunk = self._extract_method(
+                                member, lines, file_path, imports, class_name
+                            )
+                            if method_chunk:
+                                chunks.append(method_chunk)
+                        elif member.type == "constructor_declaration":
+                            constructor_chunk = self._extract_constructor(
+                                member, lines, file_path, imports, class_name
+                            )
+                            if constructor_chunk:
+                                chunks.append(constructor_chunk)
+                        # Recurse for inner classes
+                        elif member.type == "class_declaration":
+                            self._walk_tree(member, lines, file_path, chunks, imports, class_name)
+
+        elif node.type == "interface_declaration":
+            chunk = self._extract_interface(node, lines, file_path, imports)
+            if chunk:
+                chunks.append(chunk)
+
+        elif node.type == "enum_declaration":
+            chunk = self._extract_enum(node, lines, file_path, imports)
+            if chunk:
+                chunks.append(chunk)
+
+        else:
+            for child in node.children:
+                self._walk_tree(child, lines, file_path, chunks, imports, parent)
+
+    def _extract_imports(self, root: Any, lines: list[str]) -> list[str]:
+        """Extract package and import statements.
+
+        Args:
+            root: Root AST node.
+            lines: Source lines.
+
+        Returns:
+            List of package and import statements.
+        """
+        imports = []
+        for child in root.children:
+            if child.type == "package_declaration" or child.type == "import_declaration":
+                start = child.start_point[0]
+                end = child.end_point[0]
+                imports.append("\n".join(lines[start : end + 1]))
+        return imports
+
+    def _get_identifier(self, node: Any) -> str:
+        """Get identifier (name) from a declaration."""
+        for child in node.children:
+            if child.type == "identifier":
+                return child.text.decode("utf-8")
+        return "Unknown"
+
+    def _get_javadoc(self, node: Any, lines: list[str]) -> str | None:
+        """Extract Javadoc comment if present before the node.
+
+        Args:
+            node: AST node.
+            lines: Source lines.
+
+        Returns:
+            Javadoc text or None.
+        """
+        start_line = node.start_point[0]
+        if start_line == 0:
+            return None
+
+        # Look for block comment ending just before this node
+        for i in range(start_line - 1, max(-1, start_line - 20), -1):
+            line = lines[i].strip()
+            if line.endswith("*/"):
+                # Found end of comment, find start
+                doc_lines: list[str] = []
+                for j in range(i, max(-1, i - 50), -1):
+                    doc_lines.insert(0, lines[j])
+                    if lines[j].strip().startswith("/**"):
+                        return "\n".join(doc_lines)
+            elif line and not line.startswith("*") and not line.startswith("@"):
+                break
+
+        return None
+
+    def _extract_class(
+        self,
+        node: Any,
+        lines: list[str],
+        file_path: str,
+        imports: list[str],
+    ) -> CodeChunk | None:
+        """Extract a class as a chunk."""
+        try:
+            name = self._get_identifier(node)
+            start_line = node.start_point[0]
+            end_line = node.end_point[0]
+            content = "\n".join(lines[start_line : end_line + 1])
+            docstring = self._get_javadoc(node, lines)
+
+            # Build signature from first line
+            signature = lines[start_line].strip()
+
+            return CodeChunk(
+                content=content,
+                chunk_type=ChunkType.CLASS,
+                name=name,
+                start_line=start_line + 1,
+                end_line=end_line + 1,
+                language="java",
+                file_path=file_path,
+                docstring=docstring,
+                imports=imports[:5],
+                signature=signature,
+            )
+        except Exception:
+            return None
+
+    def _extract_interface(
+        self,
+        node: Any,
+        lines: list[str],
+        file_path: str,
+        imports: list[str],
+    ) -> CodeChunk | None:
+        """Extract an interface as a chunk."""
+        try:
+            name = self._get_identifier(node)
+            start_line = node.start_point[0]
+            end_line = node.end_point[0]
+            content = "\n".join(lines[start_line : end_line + 1])
+            docstring = self._get_javadoc(node, lines)
+            signature = lines[start_line].strip()
+
+            return CodeChunk(
+                content=content,
+                chunk_type=ChunkType.CLASS,  # Treat interface as class type
+                name=name,
+                start_line=start_line + 1,
+                end_line=end_line + 1,
+                language="java",
+                file_path=file_path,
+                docstring=docstring,
+                imports=imports[:5],
+                signature=signature,
+            )
+        except Exception:
+            return None
+
+    def _extract_enum(
+        self,
+        node: Any,
+        lines: list[str],
+        file_path: str,
+        imports: list[str],
+    ) -> CodeChunk | None:
+        """Extract an enum as a chunk."""
+        try:
+            name = self._get_identifier(node)
+            start_line = node.start_point[0]
+            end_line = node.end_point[0]
+            content = "\n".join(lines[start_line : end_line + 1])
+            docstring = self._get_javadoc(node, lines)
+            signature = lines[start_line].strip()
+
+            return CodeChunk(
+                content=content,
+                chunk_type=ChunkType.CLASS,
+                name=name,
+                start_line=start_line + 1,
+                end_line=end_line + 1,
+                language="java",
+                file_path=file_path,
+                docstring=docstring,
+                imports=imports[:5],
+                signature=signature,
+            )
+        except Exception:
+            return None
+
+    def _extract_method(
+        self,
+        node: Any,
+        lines: list[str],
+        file_path: str,
+        imports: list[str],
+        parent: str,
+    ) -> CodeChunk | None:
+        """Extract a method as a chunk."""
+        try:
+            name = self._get_identifier(node)
+            start_line = node.start_point[0]
+            end_line = node.end_point[0]
+            content = "\n".join(lines[start_line : end_line + 1])
+            docstring = self._get_javadoc(node, lines)
+            signature = lines[start_line].strip()
+
+            return CodeChunk(
+                content=content,
+                chunk_type=ChunkType.METHOD,
+                name=name,
+                start_line=start_line + 1,
+                end_line=end_line + 1,
+                language="java",
+                file_path=file_path,
+                parent=parent,
+                docstring=docstring,
+                imports=imports[:3],
+                signature=signature,
+            )
+        except Exception:
+            return None
+
+    def _extract_constructor(
+        self,
+        node: Any,
+        lines: list[str],
+        file_path: str,
+        imports: list[str],
+        parent: str,
+    ) -> CodeChunk | None:
+        """Extract a constructor as a chunk."""
+        try:
+            start_line = node.start_point[0]
+            end_line = node.end_point[0]
+            content = "\n".join(lines[start_line : end_line + 1])
+            docstring = self._get_javadoc(node, lines)
+            signature = lines[start_line].strip()
+
+            return CodeChunk(
+                content=content,
+                chunk_type=ChunkType.METHOD,
+                name=f"{parent}",  # Constructor has same name as class
+                start_line=start_line + 1,
+                end_line=end_line + 1,
+                language="java",
+                file_path=file_path,
+                parent=parent,
+                docstring=docstring,
+                imports=imports[:3],
+                signature=signature,
+            )
+        except Exception:
+            return None
+
+    def _create_module_chunk(self, file_path: str, content: str) -> CodeChunk:
+        """Create a module-level chunk for the entire file."""
+        name = file_path.split("/")[-1] if "/" in file_path else file_path
+        return CodeChunk(
+            content=content,
+            chunk_type=ChunkType.MODULE,
+            name=name,
+            start_line=1,
+            end_line=content.count("\n") + 1,
+            language="java",
+            file_path=file_path,
+        )