code-explore 0.1.0__py3-none-any.whl

code_explore/database.py

@@ -0,0 +1,207 @@
+"""SQLite database with FTS5 for Code Explore."""
+
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+
+from code_explore.models import Project
+
+DEFAULT_DB_PATH = Path.home() / ".code-explore" / "code-explore.db"
+
+
+def get_db_path() -> Path:
+    path = DEFAULT_DB_PATH
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def get_connection(db_path: Path | None = None) -> sqlite3.Connection:
+    path = db_path or get_db_path()
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    return conn
+
+
+def _migrate_db(conn: sqlite3.Connection) -> None:
+    """Add columns/tables that may not exist in older databases."""
+    cursor = conn.execute("PRAGMA table_info(projects)")
+    columns = {row["name"] for row in cursor.fetchall()}
+
+    if not columns:
+        # Table doesn't exist yet, nothing to migrate
+        return
+
+    if "readme_snippet" not in columns:
+        conn.execute("ALTER TABLE projects ADD COLUMN readme_snippet TEXT")
+    if "git_head" not in columns:
+        conn.execute("ALTER TABLE projects ADD COLUMN git_head TEXT")
+
+    # Rebuild FTS table to include readme_snippet if needed
+    try:
+        conn.execute("SELECT readme_snippet FROM projects_fts LIMIT 1")
+    except sqlite3.OperationalError:
+        conn.execute("DROP TABLE IF EXISTS projects_fts")
+        conn.execute("DROP TRIGGER IF EXISTS projects_ai")
+        conn.execute("DROP TRIGGER IF EXISTS projects_ad")
+        conn.execute("DROP TRIGGER IF EXISTS projects_au")
+    conn.commit()
+
+
+def init_db(db_path: Path | None = None) -> None:
+    conn = get_connection(db_path)
+    _migrate_db(conn)
+    conn.executescript("""
+        CREATE TABLE IF NOT EXISTS projects (
+            id TEXT PRIMARY KEY,
+            name TEXT NOT NULL,
+            path TEXT,
+            remote_url TEXT,
+            source TEXT DEFAULT 'local',
+            status TEXT DEFAULT 'pending',
+            data JSON NOT NULL,
+            summary TEXT,
+            tags TEXT,
+            readme_snippet TEXT,
+            git_head TEXT,
+            scanned_at TEXT,
+            analyzed_at TEXT,
+            indexed_at TEXT,
+            created_at TEXT DEFAULT (datetime('now')),
+            updated_at TEXT DEFAULT (datetime('now'))
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_projects_name ON projects(name);
+        CREATE INDEX IF NOT EXISTS idx_projects_source ON projects(source);
+        CREATE INDEX IF NOT EXISTS idx_projects_status ON projects(status);
+
+        CREATE VIRTUAL TABLE IF NOT EXISTS projects_fts USING fts5(
+            name,
+            summary,
+            tags,
+            readme_snippet,
+            content='projects',
+            content_rowid='rowid',
+            tokenize='porter unicode61'
+        );
+
+        CREATE TRIGGER IF NOT EXISTS projects_ai AFTER INSERT ON projects BEGIN
+            INSERT INTO projects_fts(rowid, name, summary, tags, readme_snippet)
+            VALUES (new.rowid, new.name, new.summary, new.tags, new.readme_snippet);
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS projects_ad AFTER DELETE ON projects BEGIN
+            INSERT INTO projects_fts(projects_fts, rowid, name, summary, tags, readme_snippet)
+            VALUES ('delete', old.rowid, old.name, old.summary, old.tags, old.readme_snippet);
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS projects_au AFTER UPDATE ON projects BEGIN
+            INSERT INTO projects_fts(projects_fts, rowid, name, summary, tags, readme_snippet)
+            VALUES ('delete', old.rowid, old.name, old.summary, old.tags, old.readme_snippet);
+            INSERT INTO projects_fts(rowid, name, summary, tags, readme_snippet)
+            VALUES (new.rowid, new.name, new.summary, new.tags, new.readme_snippet);
+        END;
+    """)
+    conn.commit()
+    conn.close()
+
+
+def save_project(project: Project, db_path: Path | None = None) -> None:
+    conn = get_connection(db_path)
+    now = datetime.now().isoformat()
+    tags_str = ", ".join(project.tags) if project.tags else ""
+
+    readme_str = (project.readme_snippet or "")[:2000]
+
+    conn.execute(
+        """
+        INSERT OR REPLACE INTO projects
+        (id, name, path, remote_url, source, status, data, summary, tags,
+         readme_snippet, git_head, scanned_at, analyzed_at, indexed_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """,
+        (
+            project.id, project.name, project.path, project.remote_url,
+            project.source.value, project.status.value,
+            project.model_dump_json(),
+            project.summary, tags_str,
+            readme_str, project.git_head,
+            project.scanned_at.isoformat() if project.scanned_at else None,
+            project.analyzed_at.isoformat() if project.analyzed_at else None,
+            project.indexed_at.isoformat() if project.indexed_at else None,
+            now,
+        ),
+    )
+    conn.commit()
+    conn.close()
+
+
+def get_project(project_id: str, db_path: Path | None = None) -> Project | None:
+    conn = get_connection(db_path)
+    row = conn.execute("SELECT data FROM projects WHERE id = ?", (project_id,)).fetchone()
+    conn.close()
+    if row:
+        return Project.model_validate_json(row["data"])
+    return None
+
+
+def get_all_projects(db_path: Path | None = None) -> list[Project]:
+    conn = get_connection(db_path)
+    rows = conn.execute("SELECT data FROM projects ORDER BY name").fetchall()
+    conn.close()
+    return [Project.model_validate_json(row["data"]) for row in rows]
+
+
+def search_fulltext(
+    query: str, limit: int = 20, db_path: Path | None = None
+) -> list[tuple[str, float]]:
+    conn = get_connection(db_path)
+    try:
+        rows = conn.execute(
+            """
+            SELECT p.id, fts.rank
+            FROM projects_fts fts
+            JOIN projects p ON p.rowid = fts.rowid
+            WHERE projects_fts MATCH ?
+            ORDER BY fts.rank
+            LIMIT ?
+            """,
+            (query, limit),
+        ).fetchall()
+    except sqlite3.OperationalError:
+        # Query may contain special characters that break FTS5 syntax.
+        # Escape by wrapping each token in double quotes.
+        tokens = query.split()
+        escaped = " ".join(f'"{t}"' for t in tokens if t)
+        try:
+            rows = conn.execute(
+                """
+                SELECT p.id, fts.rank
+                FROM projects_fts fts
+                JOIN projects p ON p.rowid = fts.rowid
+                WHERE projects_fts MATCH ?
+                ORDER BY fts.rank
+                LIMIT ?
+                """,
+                (escaped, limit),
+            ).fetchall()
+        except sqlite3.OperationalError:
+            conn.close()
+            return []
+    conn.close()
+    return [(row["id"], row["rank"]) for row in rows]
+
+
+def delete_project(project_id: str, db_path: Path | None = None) -> None:
+    conn = get_connection(db_path)
+    conn.execute("DELETE FROM projects WHERE id = ?", (project_id,))
+    conn.commit()
+    conn.close()
+
+
+def get_project_count(db_path: Path | None = None) -> int:
+    conn = get_connection(db_path)
+    count = conn.execute("SELECT COUNT(*) as cnt FROM projects").fetchone()["cnt"]
+    conn.close()
+    return count

code_explore/indexer/__init__.py

@@ -0,0 +1 @@
+"""Embedding generation and vector indexing."""

code_explore/indexer/embeddings.py

@@ -0,0 +1,181 @@
+"""Generate and store embeddings using Ollama + LanceDB."""
+
+import logging
+from pathlib import Path
+
+import httpx
+import lancedb
+import pyarrow as pa
+
+from code_explore.models import Project
+
+logger = logging.getLogger(__name__)
+
+OLLAMA_BASE_URL = "http://localhost:11434"
+EMBEDDING_MODEL = "qwen3-embedding:8b"
+EMBEDDING_DIM = 4096
+VECTOR_DB_PATH = Path.home() / ".code-explore" / "vectors"
+TABLE_NAME = "project_embeddings"
+
+SCHEMA = pa.schema([
+    pa.field("id", pa.string()),
+    pa.field("text", pa.string()),
+    pa.field("vector", pa.list_(pa.float32(), EMBEDDING_DIM)),
+])
+
+
+def _ollama_available() -> bool:
+    try:
+        resp = httpx.get(f"{OLLAMA_BASE_URL}/api/tags", timeout=5.0)
+        return resp.status_code == 200
+    except (httpx.ConnectError, httpx.TimeoutException):
+        return False
+
+
+def generate_embedding(text: str) -> list[float] | None:
+    try:
+        resp = httpx.post(
+            f"{OLLAMA_BASE_URL}/api/embeddings",
+            json={"model": EMBEDDING_MODEL, "prompt": text},
+            timeout=30.0,
+        )
+        resp.raise_for_status()
+        return resp.json()["embedding"]
+    except (httpx.ConnectError, httpx.TimeoutException):
+        logger.warning("Ollama is not running at %s. Skipping embedding generation.", OLLAMA_BASE_URL)
+        return None
+    except (httpx.HTTPStatusError, KeyError) as e:
+        logger.error("Failed to generate embedding: %s", e)
+        return None
+
+
+def _generate_embeddings_batch(texts: list[str]) -> list[list[float] | None]:
+    results = []
+    for text in texts:
+        results.append(generate_embedding(text))
+    return results
+
+
+def _project_to_text(project: Project) -> str:
+    parts: list[str] = []
+
+    # Project name - repeated for weighting
+    parts.append(project.name)
+
+    # Name with summary for context
+    if project.summary:
+        parts.append(f"{project.name} - {project.summary}")
+
+    # README snippet - most valuable signal
+    if project.readme_snippet:
+        parts.append(f"README: {project.readme_snippet}")
+
+    # Summary standalone
+    if project.summary:
+        parts.append(f"Summary: {project.summary}")
+
+    # Dependency names (without versions) - critical for discovery
+    dep_names = [dep.name for dep in project.dependencies]
+    if dep_names:
+        parts.append(f"Dependencies: {', '.join(dep_names)}")
+
+    # Tags and concepts
+    if project.tags:
+        parts.append(f"Tags: {', '.join(project.tags)}")
+
+    if project.concepts:
+        parts.append(f"Concepts: {', '.join(project.concepts)}")
+
+    # Language names
+    languages = [lang.name for lang in project.languages]
+    if languages:
+        parts.append(f"Languages: {', '.join(languages)}")
+
+    # Framework names
+    if project.frameworks:
+        parts.append(f"Frameworks: {', '.join(project.frameworks)}")
+
+    # Pattern names
+    patterns = [p.name for p in project.patterns]
+    if patterns:
+        parts.append(f"Patterns: {', '.join(patterns)}")
+
+    # Key file names
+    if project.key_files:
+        parts.append(f"Files: {', '.join(project.key_files)}")
+
+    # Git remote URL contains useful project name info
+    remote = project.remote_url or (project.git.remote_url if project.git else None)
+    if remote:
+        parts.append(f"Remote: {remote}")
+
+    return "\n".join(parts)
+
+
+def _get_table(db: lancedb.DBConnection) -> lancedb.table.Table:
+    if TABLE_NAME in db.table_names():
+        return db.open_table(TABLE_NAME)
+    return db.create_table(TABLE_NAME, schema=SCHEMA)
+
+
+def index_project(project: Project) -> None:
+    if not _ollama_available():
+        logger.warning("Ollama is not available. Skipping indexing for project '%s'.", project.name)
+        return
+
+    text = _project_to_text(project)
+    vector = generate_embedding(text)
+    if vector is None:
+        return
+
+    VECTOR_DB_PATH.mkdir(parents=True, exist_ok=True)
+    db = lancedb.connect(str(VECTOR_DB_PATH))
+    table = _get_table(db)
+
+    data = [{"id": project.id, "text": text, "vector": vector}]
+
+    existing = table.search().where(f"id = '{project.id}'", prefilter=True).limit(1).to_list()
+    if existing:
+        table.delete(f"id = '{project.id}'")
+
+    table.add(data)
+    logger.info("Indexed project '%s' in vector store.", project.name)
+
+
+def index_all_projects(projects: list[Project]) -> None:
+    if not projects:
+        return
+
+    if not _ollama_available():
+        logger.warning(
+            "Ollama is not available. Skipping vector indexing for %d projects.", len(projects)
+        )
+        return
+
+    texts = [_project_to_text(p) for p in projects]
+    vectors = _generate_embeddings_batch(texts)
+
+    data = []
+    for project, text, vector in zip(projects, texts, vectors):
+        if vector is not None:
+            data.append({"id": project.id, "text": text, "vector": vector})
+
+    if not data:
+        logger.warning("No embeddings generated. Skipping vector store update.")
+        return
+
+    VECTOR_DB_PATH.mkdir(parents=True, exist_ok=True)
+    db = lancedb.connect(str(VECTOR_DB_PATH))
+    table = _get_table(db)
+
+    existing_ids = {item["id"] for item in data}
+    for pid in existing_ids:
+        try:
+            found = table.search().where(f"id = '{pid}'", prefilter=True).limit(1).to_list()
+            if found:
+                table.delete(f"id = '{pid}'")
+        except Exception:
+            pass
+
+    table.add(data)
+    logger.info("Indexed %d/%d projects in vector store.", len(data), len(projects))

code_explore/models.py

@@ -0,0 +1,106 @@
+"""Data models for Code Explore."""
+
+from datetime import datetime
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+
+class ProjectSource(str, Enum):
+    LOCAL = "local"
+    GITHUB = "github"
+    GITLAB = "gitlab"
+
+
+class ProjectStatus(str, Enum):
+    PENDING = "pending"
+    SCANNING = "scanning"
+    ANALYZED = "analyzed"
+    INDEXED = "indexed"
+    ERROR = "error"
+
+
+class LanguageInfo(BaseModel):
+    name: str
+    files: int = 0
+    lines: int = 0
+    percentage: float = 0.0
+
+
+class DependencyInfo(BaseModel):
+    name: str
+    version: str | None = None
+    dev: bool = False
+    source: str = ""
+
+
+class PatternInfo(BaseModel):
+    name: str
+    category: str
+    confidence: float = 0.0
+    evidence: list[str] = Field(default_factory=list)
+
+
+class QualityMetrics(BaseModel):
+    total_files: int = 0
+    total_lines: int = 0
+    avg_file_size: float = 0.0
+    max_file_size: int = 0
+    has_tests: bool = False
+    has_ci: bool = False
+    has_docs: bool = False
+    has_readme: bool = False
+    has_license: bool = False
+    has_gitignore: bool = False
+
+
+class GitInfo(BaseModel):
+    remote_url: str | None = None
+    default_branch: str | None = None
+    total_commits: int = 0
+    last_commit_date: datetime | None = None
+    last_commit_message: str | None = None
+    contributors: list[str] = Field(default_factory=list)
+
+
+class Project(BaseModel):
+    id: str
+    name: str
+    path: str | None = None
+    remote_url: str | None = None
+    source: ProjectSource = ProjectSource.LOCAL
+    status: ProjectStatus = ProjectStatus.PENDING
+
+    languages: list[LanguageInfo] = Field(default_factory=list)
+    primary_language: str | None = None
+    frameworks: list[str] = Field(default_factory=list)
+    dependencies: list[DependencyInfo] = Field(default_factory=list)
+    patterns: list[PatternInfo] = Field(default_factory=list)
+    quality: QualityMetrics = Field(default_factory=QualityMetrics)
+    git: GitInfo = Field(default_factory=GitInfo)
+
+    summary: str | None = None
+    tags: list[str] = Field(default_factory=list)
+    concepts: list[str] = Field(default_factory=list)
+
+    readme_snippet: str | None = None
+    key_files: list[str] = Field(default_factory=list)
+    git_head: str | None = None
+
+    scanned_at: datetime | None = None
+    analyzed_at: datetime | None = None
+    indexed_at: datetime | None = None
+
+
+class SearchResult(BaseModel):
+    project: Project
+    score: float
+    match_type: str
+    highlights: list[str] = Field(default_factory=list)
+
+
+class SearchQuery(BaseModel):
+    query: str
+    mode: str = "hybrid"
+    limit: int = 20
+    filters: dict = Field(default_factory=dict)

code_explore/scanner/__init__.py

@@ -0,0 +1 @@
+"""Project scanning - local filesystem and git repos."""

code_explore/scanner/git_info.py

@@ -0,0 +1,94 @@
+"""Extract git metadata from a local repository."""
+
+from pathlib import Path
+
+from git import InvalidGitRepositoryError, Repo
+
+from code_explore.models import GitInfo
+
+
+def get_git_head(repo_path: str | Path) -> str | None:
+    """Return the HEAD commit SHA for the repository, or None on error."""
+    repo_path = Path(repo_path).expanduser().resolve()
+    try:
+        repo = Repo(repo_path)
+        return repo.head.commit.hexsha
+    except (InvalidGitRepositoryError, ValueError, TypeError):
+        return None
+
+
+def extract_git_info(repo_path: str | Path) -> GitInfo:
+    """Read git metadata from the repository at repo_path and return a GitInfo model."""
+    repo_path = Path(repo_path).expanduser().resolve()
+
+    try:
+        repo = Repo(repo_path)
+    except InvalidGitRepositoryError:
+        return GitInfo()
+
+    remote_url = _get_remote_url(repo)
+    default_branch = _get_default_branch(repo)
+    total_commits, last_commit_date, last_commit_message, contributors = _get_commit_info(repo)
+
+    return GitInfo(
+        remote_url=remote_url,
+        default_branch=default_branch,
+        total_commits=total_commits,
+        last_commit_date=last_commit_date,
+        last_commit_message=last_commit_message,
+        contributors=contributors,
+    )
+
+
+def _get_remote_url(repo: Repo) -> str | None:
+    if not repo.remotes:
+        return None
+    try:
+        return repo.remotes.origin.url
+    except AttributeError:
+        return repo.remotes[0].url
+
+
+def _get_default_branch(repo: Repo) -> str | None:
+    if repo.bare:
+        try:
+            return repo.head.reference.name
+        except TypeError:
+            return None
+
+    for name in ("main", "master", "develop"):
+        if name in repo.heads:
+            return name
+
+    try:
+        return repo.active_branch.name
+    except TypeError:
+        return None
+
+
+def _get_commit_info(
+    repo: Repo,
+) -> tuple[int, "datetime | None", str | None, list[str]]:
+    from datetime import datetime, timezone
+
+    try:
+        head_commit = repo.head.commit
+    except ValueError:
+        return 0, None, None, []
+
+    total_commits = head_commit.count()
+
+    last_commit_date = datetime.fromtimestamp(
+        head_commit.committed_date, tz=timezone.utc
+    )
+    last_commit_message = head_commit.message.strip()
+
+    seen: set[str] = set()
+    contributors: list[str] = []
+    for commit in repo.iter_commits(max_count=500):
+        name = commit.author.name
+        if name and name not in seen:
+            seen.add(name)
+            contributors.append(name)
+
+    return total_commits, last_commit_date, last_commit_message, contributors

code_explore/scanner/local.py

@@ -0,0 +1,70 @@
+"""Scan local directories to discover git repositories."""
+
+import asyncio
+import os
+from pathlib import Path
+
+from rich.console import Console
+
+console = Console()
+
+SKIP_DIRS = frozenset({
+    ".git",
+    "node_modules",
+    "venv",
+    ".venv",
+    "__pycache__",
+    ".cache",
+    "vendor",
+    "dist",
+    "build",
+    "target",
+})
+
+
+async def scan_local_repos(
+    root: str | Path,
+    max_depth: int = 4,
+) -> list[Path]:
+    """Walk root directory recursively and return paths of discovered git repositories."""
+    root = Path(root).expanduser().resolve()
+    if not root.is_dir():
+        console.print(f"[red]Root path does not exist: {root}[/red]")
+        return []
+
+    console.print(f"[blue]Scanning[/blue] {root} (max depth {max_depth})")
+    discovered: list[Path] = await asyncio.to_thread(_walk_for_repos, root, max_depth)
+    console.print(f"[green]Found {len(discovered)} repositories[/green]")
+    return discovered
+
+
+def _walk_for_repos(root: Path, max_depth: int) -> list[Path]:
+    repos: list[Path] = []
+    stack: list[tuple[Path, int]] = [(root, 0)]
+
+    while stack:
+        current, depth = stack.pop()
+
+        if (current / ".git").is_dir():
+            repos.append(current)
+            continue
+
+        if depth >= max_depth:
+            continue
+
+        try:
+            entries = sorted(current.iterdir())
+        except PermissionError:
+            continue
+
+        for entry in entries:
+            if entry.is_symlink() or not entry.is_dir():
+                continue
+            if entry.name in SKIP_DIRS:
+                continue
+            stack.append((entry, depth + 1))
+
+        if len(repos) % 50 == 0 and repos:
+            console.print(f"  [dim]...discovered {len(repos)} repos so far[/dim]")
+
+    return repos