vox-code 2.0.0__py3-none-any.whl

voxcli/rag/analyzer.py

@@ -0,0 +1,89 @@
+"""代码分析器 - 分析代码结构和关系"""
+
+import ast
+import logging
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+from .relation import CodeRelation
+
+logger = logging.getLogger(__name__)
+
+
+class CodeAnalyzer:
+    def __init__(self, project_path: str):
+        self._project_path = Path(project_path)
+
+    def analyze_file(self, file_path: str) -> List[CodeRelation]:
+        path = Path(file_path)
+        if not path.exists():
+            return []
+        try:
+            content = path.read_text(encoding="utf-8")
+        except Exception as e:
+            logger.warning("Failed to read %s: %s", file_path, e)
+            return []
+
+        ext = path.suffix.lower()
+        if ext == ".py":
+            return self._analyze_python(file_path, content)
+        elif ext == ".java":
+            return self._analyze_java(file_path, content)
+        return []
+
+    def _analyze_python(self, file_path: str, content: str) -> List[CodeRelation]:
+        relations = []
+        try:
+            tree = ast.parse(content)
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        target = self._resolve_module(alias.name)
+                        if target:
+                            relations.append(CodeRelation(
+                                source_file=file_path,
+                                target_file=target,
+                                relation_type="import",
+                            ))
+                elif isinstance(node, ast.ImportFrom):
+                    if node.module:
+                        for alias in node.names:
+                            target = self._resolve_module(f"{node.module}.{alias.name}")
+                            if target:
+                                relations.append(CodeRelation(
+                                    source_file=file_path,
+                                    target_file=target,
+                                    relation_type="import",
+                                ))
+        except SyntaxError:
+            pass
+        return relations
+
+    def _analyze_java(self, file_path: str, content: str) -> List[CodeRelation]:
+        relations = []
+        import_pattern = re.compile(r"^import\s+(?:static\s+)?([\w.]+);", re.MULTILINE)
+        for match in import_pattern.finditer(content):
+            import_path = match.group(1)
+            target = self._resolve_java_class(import_path)
+            if target:
+                relations.append(CodeRelation(
+                    source_file=file_path,
+                    target_file=str(target),
+                    relation_type="import",
+                ))
+        return relations
+
+    def _resolve_module(self, module_name: str) -> Optional[str]:
+        path = self._project_path / (module_name.replace(".", "/") + ".py")
+        if path.exists():
+            return str(path)
+        init_path = path.parent / "__init__.py"
+        if init_path.exists():
+            return str(init_path)
+        return None
+
+    def _resolve_java_class(self, class_name: str) -> Optional[Path]:
+        rel_path = class_name.replace(".", "/") + ".java"
+        candidates = list(self._project_path.rglob(rel_path))
+        return candidates[0] if candidates else None

voxcli/rag/chunk.py

@@ -0,0 +1,17 @@
+"""代码块数据模型"""
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+
+@dataclass
+class CodeChunk:
+    id: str
+    file_path: str
+    content: str
+    language: str
+    start_line: int
+    end_line: int
+    chunk_type: str = "code"  # code, comment, import, class, function
+    metadata: dict = field(default_factory=dict)
+    embedding: Optional[List[float]] = None

voxcli/rag/chunker.py

@@ -0,0 +1,137 @@
+"""代码分块器 - 将源代码文件分割为可检索的块"""
+
+import ast
+import logging
+import re
+from pathlib import Path
+from typing import List, Optional
+
+from .chunk import CodeChunk
+
+logger = logging.getLogger(__name__)
+
+_LINE_CHUNK_SIZE = 50
+_LINE_OVERLAP = 10
+
+_LANGUAGE_MAP = {
+    ".py": "python", ".java": "java", ".js": "javascript",
+    ".ts": "typescript", ".tsx": "typescriptreact", ".jsx": "javascriptreact",
+    ".go": "go", ".rs": "rust", ".rb": "ruby", ".php": "php",
+    ".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp",
+    ".swift": "swift", ".kt": "kotlin", ".scala": "scala",
+    ".md": "markdown", ".json": "json", ".yaml": "yaml", ".yml": "yaml",
+    ".xml": "xml", ".toml": "toml", ".sql": "sql", ".sh": "bash",
+    ".zsh": "bash", ".bash": "bash", ".txt": "text",
+}
+
+
+class CodeChunker:
+    def __init__(self, line_chunk_size: int = _LINE_CHUNK_SIZE,
+                 line_overlap: int = _LINE_OVERLAP):
+        self._chunk_size = line_chunk_size
+        self._overlap = line_overlap
+
+    def chunk_file(self, file_path: str) -> List[CodeChunk]:
+        path = Path(file_path)
+        if not path.exists():
+            return []
+        try:
+            content = path.read_text(encoding="utf-8")
+        except Exception as e:
+            logger.warning("Failed to read %s: %s", file_path, e)
+            return []
+
+        ext = path.suffix.lower()
+        language = _LANGUAGE_MAP.get(ext, "text")
+        chunks = []
+
+        if language == "python":
+            chunks = self._chunk_python(file_path, content)
+        elif language == "java":
+            chunks = self._chunk_java(file_path, content)
+
+        if not chunks:
+            chunks = self._chunk_by_lines(file_path, content, language)
+
+        return chunks
+
+    def _chunk_python(self, file_path: str, content: str) -> List[CodeChunk]:
+        chunks = []
+        try:
+            tree = ast.parse(content)
+            for node in ast.walk(tree):
+                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                    start = node.lineno or 1
+                    end = node.end_lineno or start
+                    lines = content.split("\n")[start - 1:end]
+                    chunk_type = "class" if isinstance(node, ast.ClassDef) else "function"
+                    chunks.append(CodeChunk(
+                        id=f"{file_path}:{start}-{end}",
+                        file_path=file_path,
+                        content="\n".join(lines),
+                        language="python",
+                        start_line=start,
+                        end_line=end,
+                        chunk_type=chunk_type,
+                    ))
+        except SyntaxError:
+            pass
+        return chunks
+
+    def _chunk_java(self, file_path: str, content: str) -> List[CodeChunk]:
+        chunks = []
+        pattern = re.compile(
+            r'(?:public|private|protected)\s+(?:static\s+)?(?:class|interface|enum)\s+(\w+)|'
+            r'(?:public|private|protected)\s+(?:static\s+)?\w+\s+(\w+)\s*\([^)]*\)\s*(?:throws\s+\w+)?\s*\{'
+        )
+        lines = content.split("\n")
+        for match in pattern.finditer(content):
+            start_line = content[:match.start()].count("\n") + 1
+            brace_pos = content.find("{", match.end())
+            if brace_pos == -1:
+                continue
+            end_line = self._find_matching_brace(content, brace_pos, lines)
+            chunk_type = "class" if match.group(1) else "function"
+            chunk_lines = lines[start_line - 1:end_line]
+            chunks.append(CodeChunk(
+                id=f"{file_path}:{start_line}-{end_line}",
+                file_path=file_path,
+                content="\n".join(chunk_lines),
+                language="java",
+                start_line=start_line,
+                end_line=end_line,
+                chunk_type=chunk_type,
+            ))
+        return chunks
+
+    def _chunk_by_lines(self, file_path: str, content: str, language: str) -> List[CodeChunk]:
+        lines = content.split("\n")
+        chunks = []
+        for i in range(0, len(lines), self._chunk_size - self._overlap):
+            chunk_lines = lines[i:i + self._chunk_size]
+            if not chunk_lines:
+                break
+            start = i + 1
+            end = i + len(chunk_lines)
+            chunks.append(CodeChunk(
+                id=f"{file_path}:{start}-{end}",
+                file_path=file_path,
+                content="\n".join(chunk_lines),
+                language=language,
+                start_line=start,
+                end_line=end,
+                chunk_type="code",
+            ))
+        return chunks
+
+    @staticmethod
+    def _find_matching_brace(content: str, open_pos: int, lines: List[str]) -> int:
+        depth = 1
+        pos = open_pos + 1
+        while pos < len(content) and depth > 0:
+            if content[pos] == "{":
+                depth += 1
+            elif content[pos] == "}":
+                depth -= 1
+            pos += 1
+        return content[:pos].count("\n") + 1

voxcli/rag/embedding.py

@@ -0,0 +1,75 @@
+"""嵌入客户端 - 使用 Ollama 或 OpenAI 兼容 API 生成文本嵌入"""
+
+import json
+import logging
+from typing import List, Optional
+
+import httpx
+
+from ..config import pai_config
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_MODEL = "nomic-embed-text"
+_DEFAULT_BASE_URL = "http://localhost:11434"
+
+
+class EmbeddingClient:
+    def __init__(self, model: str = _DEFAULT_MODEL, base_url: str = _DEFAULT_BASE_URL,
+                 api_key: str = ""):
+        self._model = model
+        self._base_url = base_url
+        self._api_key = api_key
+
+    @classmethod
+    def from_env(cls) -> "EmbeddingClient":
+        provider = pai_config.get_provider("ollama")
+        if provider:
+            return cls(
+                base_url=provider.get("base_url", _DEFAULT_BASE_URL),
+                model=provider.get("model", _DEFAULT_MODEL),
+            )
+        provider = pai_config.get_provider("glm")
+        if provider:
+            return cls(
+                base_url=provider.get("base_url", "https://open.bigmodel.cn/api/paas/v4"),
+                model=provider.get("model", "embedding-2"),
+                api_key=provider.get("api_key", ""),
+            )
+        return cls()
+
+    def embed(self, text: str) -> List[float]:
+        if self._base_url.startswith("http://localhost") or "ollama" in self._base_url:
+            return self._embed_ollama(text)
+        return self._embed_openai(text)
+
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        return [self.embed(t) for t in texts]
+
+    @property
+    def dimension(self) -> int:
+        return 768
+
+    def _embed_ollama(self, text: str) -> List[float]:
+        response = httpx.post(
+            f"{self._base_url}/api/embeddings",
+            json={"model": self._model, "prompt": text},
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return data.get("embedding", [])
+
+    def _embed_openai(self, text: str) -> List[float]:
+        response = httpx.post(
+            f"{self._base_url}/embeddings",
+            headers={
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json",
+            },
+            json={"model": self._model, "input": text},
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return data.get("data", [{}])[0].get("embedding", [])

voxcli/rag/formatter.py

@@ -0,0 +1,40 @@
+"""搜索结果格式化器"""
+
+from typing import List
+
+from .store import SearchResult
+
+
+class SearchResultFormatter:
+    @staticmethod
+    def format_for_tool(query: str, results: List[SearchResult]) -> str:
+        lines = [f"🔍 代码检索: {query}\n"]
+        if not results:
+            lines.append("未找到相关结果。")
+        else:
+            for i, r in enumerate(results, 1):
+                chunk = r.chunk
+                lines.append(f"{i}. [{chunk.chunk_type}] {chunk.file_path}:{chunk.start_line}-{chunk.end_line}")
+                lines.append(f"   语言: {chunk.language} | 相关度: {r.score:.3f}")
+                code = chunk.content.strip()
+                if len(code) > 500:
+                    code = code[:500] + "..."
+                lines.append(f"   ```{chunk.language}")
+                lines.append(code)
+                lines.append("   ```")
+                lines.append("")
+        return "\n".join(lines).strip()
+
+    @staticmethod
+    def format_for_llm(query: str, results: List[SearchResult]) -> str:
+        parts = []
+        for r in results:
+            chunk = r.chunk
+            parts.append(
+                f"File: {chunk.file_path}:{chunk.start_line}-{chunk.end_line}\n"
+                f"Type: {chunk.chunk_type}\n"
+                f"```{chunk.language}\n{chunk.content}\n```"
+            )
+        if parts:
+            return f"Related code for query '{query}':\n\n" + "\n---\n".join(parts)
+        return ""

voxcli/rag/index.py

@@ -0,0 +1,96 @@
+"""代码索引器 - 将项目文件索引到向量存储"""
+
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import List, Optional, Set
+
+from .chunk import CodeChunk
+from .chunker import CodeChunker
+from .embedding import EmbeddingClient
+from .store import VectorStore
+
+logger = logging.getLogger(__name__)
+
+_IGNORE_DIRS = {
+    ".git", ".svn", "__pycache__", "node_modules", ".mvn", ".gradle",
+    "target", "build", "dist", ".idea", ".vscode", ".venv", "venv",
+    "env", ".egg-info", "site-packages", ".tox", ".nox",
+}
+
+_IGNORE_EXTENSIONS = {
+    ".pyc", ".pyo", ".so", ".o", ".class", ".jar", ".war", ".zip",
+    ".tar", ".gz", ".7z", ".rar", ".exe", ".dll", ".dylib", ".bin",
+    ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".webp",
+    ".mp4", ".mp3", ".wav", ".avi", ".mov", ".pdf", ".doc", ".docx",
+    ".xls", ".xlsx", ".ppt", ".pptx", ".ttf", ".woff", ".woff2",
+    ".DS_Store", ".gitkeep", ".gitignore",
+}
+
+_INCLUDE_EXTENSIONS = {
+    ".py", ".java", ".js", ".ts", ".tsx", ".jsx", ".go", ".rs", ".rb",
+    ".php", ".c", ".cpp", ".h", ".hpp", ".swift", ".kt", ".scala",
+    ".md", ".json", ".yaml", ".yml", ".xml", ".toml", ".sql", ".sh",
+    ".bash", ".zsh", ".txt", ".cfg", ".conf", ".ini", ".properties",
+}
+
+
+class CodeIndex:
+    def __init__(self, project_path: str,
+                 vector_store: Optional[VectorStore] = None,
+                 chunker: Optional[CodeChunker] = None,
+                 embedding_client: Optional[EmbeddingClient] = None):
+        self._project_path = Path(project_path)
+        self._store = vector_store or VectorStore()
+        self._chunker = chunker or CodeChunker()
+        self._embedding = embedding_client or EmbeddingClient()
+
+    def index_project(self, max_workers: int = 4) -> int:
+        files = self._discover_files()
+        if not files:
+            logger.info("No indexable files found in %s", self._project_path)
+            return 0
+
+        total_chunks = 0
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = [executor.submit(self._index_file, f) for f in files]
+            for future in futures:
+                try:
+                    total_chunks += future.result()
+                except Exception as e:
+                    logger.warning("Indexing file failed: %s", e)
+
+        logger.info("Indexed %d chunks from %d files", total_chunks, len(files))
+        return total_chunks
+
+    def _discover_files(self) -> List[Path]:
+        files = []
+        for ext in _INCLUDE_EXTENSIONS:
+            files.extend(self._project_path.rglob(f"*{ext}"))
+
+        return [
+            f for f in files
+            if f.is_file()
+            and not any(part.startswith(".") or part in _IGNORE_DIRS
+                        for part in f.relative_to(self._project_path).parts)
+            and f.suffix.lower() not in _IGNORE_EXTENSIONS
+        ]
+
+    def _index_file(self, file_path: Path) -> int:
+        try:
+            # Remove stale entries
+            self._store.remove_file(str(file_path))
+            chunks = self._chunker.chunk_file(str(file_path))
+            if not chunks:
+                return 0
+
+            for chunk in chunks:
+                embedding = self._embedding.embed(chunk.content)
+                chunk.embedding = embedding
+
+            self._store.store_chunks(chunks)
+            return len(chunks)
+        except Exception as e:
+            logger.warning("Failed to index %s: %s", file_path, e)
+            return 0

voxcli/rag/relation.py

@@ -0,0 +1,14 @@
+"""代码关系 - 代码块之间的调用/引用关系"""
+
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+
+@dataclass
+class CodeRelation:
+    source_file: str
+    target_file: str
+    relation_type: str  # import, call, extend, implement, reference
+    source_chunk_id: Optional[str] = None
+    target_chunk_id: Optional[str] = None
+    metadata: dict = field(default_factory=dict)

voxcli/rag/retriever.py

@@ -0,0 +1,58 @@
+"""代码检索器 - 混合检索（语义 + 关键词）"""
+
+import logging
+from dataclasses import dataclass
+from typing import List, Optional
+
+from .embedding import EmbeddingClient
+from .store import VectorStore, SearchResult
+from .chunker import CodeChunker
+from .index import CodeIndex
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class IndexStats:
+    chunk_count: int
+    file_count: int
+
+
+class CodeRetriever:
+    def __init__(self, project_path: str,
+                 vector_store: Optional[VectorStore] = None,
+                 embedding_client: Optional[EmbeddingClient] = None):
+        self._project_path = project_path
+        self._store = vector_store or VectorStore()
+        self._embedding = embedding_client or EmbeddingClient.from_env()
+
+    def hybrid_search(self, query: str, top_k: int = 5) -> List[SearchResult]:
+        query_embedding = self._embedding.embed(query)
+        semantic_results = self._store.search_by_embedding(query_embedding, top_k)
+        keyword_results = self._store.search_by_keyword(query, top_k)
+
+        seen = set()
+        merged = []
+
+        for r in semantic_results + keyword_results:
+            if r.chunk.id not in seen:
+                seen.add(r.chunk.id)
+                merged.append(r)
+
+        merged.sort(key=lambda x: x.score, reverse=True)
+        return merged[:top_k]
+
+    def get_stats(self) -> IndexStats:
+        return IndexStats(
+            chunk_count=self._store.chunk_count,
+            file_count=self._store.file_count,
+        )
+
+    def close(self):
+        self._store.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

voxcli/rag/store.py

@@ -0,0 +1,155 @@
+"""向量存储 - SQLite + 余弦相似度检索"""
+
+import json
+import logging
+import math
+import sqlite3
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from .chunk import CodeChunk
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SearchResult:
+    chunk: CodeChunk
+    score: float
+
+
+class VectorStore:
+    def __init__(self, db_path: Optional[str] = None):
+        if db_path:
+            self._db_path = Path(db_path)
+        else:
+            mem_dir = Path.home() / ".vox-code" / "memory"
+            mem_dir.mkdir(parents=True, exist_ok=True)
+            self._db_path = mem_dir / "vector_store.db"
+        self._conn = sqlite3.connect(str(self._db_path))
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._init_db()
+
+    def _init_db(self):
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS chunks (
+                id TEXT PRIMARY KEY,
+                file_path TEXT NOT NULL,
+                content TEXT NOT NULL,
+                language TEXT NOT NULL,
+                start_line INTEGER NOT NULL,
+                end_line INTEGER NOT NULL,
+                chunk_type TEXT DEFAULT 'code',
+                metadata TEXT DEFAULT '{}',
+                embedding TEXT,
+                indexed_at REAL NOT NULL
+            )
+        """)
+        self._conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_chunks_file_path ON chunks(file_path)
+        """)
+        self._conn.commit()
+
+    def store_chunk(self, chunk: CodeChunk):
+        self._conn.execute("""
+            INSERT OR REPLACE INTO chunks
+            (id, file_path, content, language, start_line, end_line,
+             chunk_type, metadata, embedding, indexed_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (
+            chunk.id, chunk.file_path, chunk.content, chunk.language,
+            chunk.start_line, chunk.end_line, chunk.chunk_type,
+            json.dumps(chunk.metadata),
+            json.dumps(chunk.embedding) if chunk.embedding else None,
+            time.time(),
+        ))
+        self._conn.commit()
+
+    def store_chunks(self, chunks: List[CodeChunk]):
+        for chunk in chunks:
+            self.store_chunk(chunk)
+
+    def search_by_embedding(self, query_embedding: List[float], top_k: int = 5) -> List[SearchResult]:
+        cursor = self._conn.execute(
+            "SELECT id, file_path, content, language, start_line, end_line, "
+            "chunk_type, metadata, embedding FROM chunks WHERE embedding IS NOT NULL"
+        )
+        scored: List[Tuple[float, dict]] = []
+        for row in cursor:
+            try:
+                stored_emb = json.loads(row[8])
+                score = self._cosine_similarity(query_embedding, stored_emb)
+                scored.append((score, {
+                    "id": row[0], "file_path": row[1], "content": row[2],
+                    "language": row[3], "start_line": row[4], "end_line": row[5],
+                    "chunk_type": row[6], "metadata": json.loads(row[7]),
+                }))
+            except (json.JSONDecodeError, IndexError):
+                continue
+
+        scored.sort(key=lambda x: x[0], reverse=True)
+        results = []
+        for score, data in scored[:top_k]:
+            chunk = CodeChunk(**data)
+            results.append(SearchResult(chunk=chunk, score=score))
+        return results
+
+    def search_by_keyword(self, query: str, top_k: int = 5) -> List[SearchResult]:
+        terms = query.lower().split()
+        cursor = self._conn.execute(
+            "SELECT id, file_path, content, language, start_line, end_line, "
+            "chunk_type, metadata FROM chunks"
+        )
+        scored: List[Tuple[float, dict]] = []
+        for row in cursor:
+            content_lower = (row[2] or "").lower()
+            path_lower = (row[1] or "").lower()
+            match_count = sum(1 for t in terms if t in content_lower or t in path_lower)
+            if match_count > 0:
+                score = match_count / len(terms)
+                scored.append((score, {
+                    "id": row[0], "file_path": row[1], "content": row[2],
+                    "language": row[3], "start_line": row[4], "end_line": row[5],
+                    "chunk_type": row[6], "metadata": json.loads(row[7]),
+                }))
+
+        scored.sort(key=lambda x: x[0], reverse=True)
+        results = []
+        for score, data in scored[:top_k]:
+            chunk = CodeChunk(**data)
+            results.append(SearchResult(chunk=chunk, score=score))
+        return results
+
+    def remove_file(self, file_path: str):
+        self._conn.execute("DELETE FROM chunks WHERE file_path = ?", (file_path,))
+        self._conn.commit()
+
+    def clear(self):
+        self._conn.execute("DELETE FROM chunks")
+        self._conn.commit()
+
+    @property
+    def chunk_count(self) -> int:
+        cursor = self._conn.execute("SELECT COUNT(*) FROM chunks")
+        return cursor.fetchone()[0]
+
+    @property
+    def file_count(self) -> int:
+        cursor = self._conn.execute("SELECT COUNT(DISTINCT file_path) FROM chunks")
+        return cursor.fetchone()[0]
+
+    def close(self):
+        self._conn.close()
+
+    @staticmethod
+    def _cosine_similarity(a: List[float], b: List[float]) -> float:
+        if not a or not b:
+            return 0.0
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(y * y for y in b))
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        return dot / (norm_a * norm_b)