code-explore 0.1.0__py3-none-any.whl

code_explore/scanner/readme.py

@@ -0,0 +1,70 @@
+"""Read README files and list key files from repositories."""
+
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+README_CANDIDATES = ("README.md", "readme.md", "README.rst", "README", "README.txt", "Readme.md")
+
+SKIP_ENTRIES = frozenset({
+    ".git",
+    "node_modules",
+    "__pycache__",
+    ".DS_Store",
+    ".gitignore",
+    ".editorconfig",
+    ".vscode",
+    ".idea",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".tox",
+    ".eggs",
+    ".env",
+})
+
+MAX_KEY_FILES = 50
+MAX_README_CHARS = 1500
+
+
+def read_readme(repo_path: Path) -> str | None:
+    """Read the README file from a repository and return its first 1500 characters.
+
+    Tries several common README filenames. Returns None if no README is found
+    or if the file cannot be read.
+    """
+    for candidate in README_CANDIDATES:
+        readme_path = repo_path / candidate
+        try:
+            if readme_path.is_file():
+                text = readme_path.read_text(encoding="utf-8", errors="replace")
+                return text[:MAX_README_CHARS] if text else None
+        except (OSError, PermissionError) as e:
+            logger.debug("Could not read %s: %s", readme_path, e)
+            continue
+    return None
+
+
+def list_key_files(repo_path: Path) -> list[str]:
+    """List important top-level files and directories in a repository.
+
+    Skips common non-informative entries like .git, node_modules, etc.
+    Returns up to 50 file/directory names (not full paths).
+    """
+    try:
+        entries = sorted(repo_path.iterdir())
+    except (OSError, PermissionError) as e:
+        logger.debug("Could not list directory %s: %s", repo_path, e)
+        return []
+
+    names: list[str] = []
+    for entry in entries:
+        if entry.name in SKIP_ENTRIES:
+            continue
+        name = entry.name + "/" if entry.is_dir() and not entry.is_symlink() else entry.name
+        names.append(name)
+        if len(names) >= MAX_KEY_FILES:
+            break
+
+    return names

code_explore/search/__init__.py

@@ -0,0 +1 @@
+"""Search engine - fulltext and semantic search."""

code_explore/search/fulltext.py

@@ -0,0 +1,137 @@
+"""SQLite FTS5 fulltext search."""
+
+import logging
+import sqlite3
+from pathlib import Path
+
+from code_explore.database import get_connection, get_db_path, get_project
+from code_explore.database import search_fulltext as db_search_fulltext
+from code_explore.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+# Minimum word length to try as an individual search term
+_MIN_WORD_LEN = 3
+
+
+def _extract_snippets(
+    query: str, project_id: str, db_path: Path | None = None
+) -> list[str]:
+    conn = get_connection(db_path)
+    try:
+        rows = conn.execute(
+            """
+            SELECT
+                snippet(projects_fts, 0, '**', '**', '...', 32) as name_snip,
+                snippet(projects_fts, 1, '**', '**', '...', 64) as summary_snip,
+                snippet(projects_fts, 2, '**', '**', '...', 32) as tags_snip,
+                snippet(projects_fts, 3, '**', '**', '...', 64) as readme_snip
+            FROM projects_fts fts
+            JOIN projects p ON p.rowid = fts.rowid
+            WHERE projects_fts MATCH ? AND p.id = ?
+            LIMIT 1
+            """,
+            (query, project_id),
+        ).fetchall()
+    except sqlite3.OperationalError:
+        return []
+    finally:
+        conn.close()
+
+    if not rows:
+        return []
+
+    snippets = []
+    row = rows[0]
+    for key in ("name_snip", "summary_snip", "tags_snip", "readme_snip"):
+        val = row[key]
+        if val and "**" in val:
+            snippets.append(val)
+    return snippets
+
+
+def _or_query(query: str) -> str:
+    """Convert a multi-word query to OR-separated FTS5 query."""
+    tokens = query.split()
+    if len(tokens) <= 1:
+        return query
+    return " OR ".join(tokens)
+
+
+def _significant_words(query: str) -> list[str]:
+    """Return words from the query that are long enough to search individually."""
+    return [w for w in query.split() if len(w) >= _MIN_WORD_LEN]
+
+
+def search(
+    query: str, limit: int = 20, db_path: Path | None = None
+) -> list[SearchResult]:
+    # Strategy 1: Try the original query (AND logic)
+    raw_results = _safe_fts_search(query, limit=limit, db_path=db_path)
+
+    # Strategy 2: If no results, try OR between words
+    if not raw_results:
+        or_query = _or_query(query)
+        if or_query != query:
+            raw_results = _safe_fts_search(or_query, limit=limit, db_path=db_path)
+
+    # Strategy 3: If still no results, try each significant word individually
+    if not raw_results:
+        seen_ids: set[str] = set()
+        merged: list[tuple[str, float]] = []
+        score_penalty = 0.0
+
+        for word in _significant_words(query):
+            word_results = _safe_fts_search(word, limit=limit, db_path=db_path)
+            for project_id, rank in word_results:
+                if project_id not in seen_ids:
+                    seen_ids.add(project_id)
+                    # Apply increasing penalty so earlier word matches rank higher
+                    adjusted_rank = rank - score_penalty if rank < 0 else rank + score_penalty
+                    merged.append((project_id, adjusted_rank))
+            score_penalty += 1.0
+
+        raw_results = merged[:limit]
+
+    # Build SearchResult objects
+    # For snippets, try the original query first, then fall back to OR query
+    results = []
+    for project_id, rank in raw_results:
+        project = get_project(project_id, db_path=db_path)
+        if project is None:
+            continue
+
+        score = -rank if rank < 0 else rank
+        highlights = _extract_snippets(query, project_id, db_path=db_path)
+        if not highlights:
+            or_q = _or_query(query)
+            if or_q != query:
+                highlights = _extract_snippets(or_q, project_id, db_path=db_path)
+        if not highlights:
+            # Try individual significant words for snippet extraction
+            for word in _significant_words(query):
+                highlights = _extract_snippets(word, project_id, db_path=db_path)
+                if highlights:
+                    break
+
+        results.append(
+            SearchResult(
+                project=project,
+                score=score,
+                match_type="fulltext",
+                highlights=highlights,
+            )
+        )
+
+    return results
+
+
+def _safe_fts_search(
+    query: str, limit: int = 20, db_path: Path | None = None
+) -> list[tuple[str, float]]:
+    """Run FTS5 search, returning empty list on any error."""
+    try:
+        return db_search_fulltext(query, limit=limit, db_path=db_path)
+    except sqlite3.OperationalError as e:
+        logger.debug("FTS5 search failed for query %r: %s", query, e)
+        return []

code_explore/search/hybrid.py

@@ -0,0 +1,92 @@
+"""Hybrid search combining fulltext and semantic search with reciprocal rank fusion."""
+
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from code_explore.models import SearchResult
+from code_explore.search.fulltext import search as fulltext_search
+from code_explore.search.semantic import search as semantic_search
+
+logger = logging.getLogger(__name__)
+
+RRF_K = 60
+
+
+def _reciprocal_rank_fusion(
+    fulltext_results: list[SearchResult],
+    semantic_results: list[SearchResult],
+) -> list[SearchResult]:
+    scores: dict[str, float] = {}
+    results_map: dict[str, SearchResult] = {}
+    highlights_map: dict[str, list[str]] = {}
+
+    for rank, result in enumerate(fulltext_results):
+        pid = result.project.id
+        scores[pid] = scores.get(pid, 0.0) + 1.0 / (RRF_K + rank + 1)
+        if pid not in results_map:
+            results_map[pid] = result
+            highlights_map[pid] = list(result.highlights)
+        else:
+            for h in result.highlights:
+                if h not in highlights_map[pid]:
+                    highlights_map[pid].append(h)
+
+    for rank, result in enumerate(semantic_results):
+        pid = result.project.id
+        scores[pid] = scores.get(pid, 0.0) + 1.0 / (RRF_K + rank + 1)
+        if pid not in results_map:
+            results_map[pid] = result
+            highlights_map[pid] = list(result.highlights)
+        else:
+            for h in result.highlights:
+                if h not in highlights_map[pid]:
+                    highlights_map[pid].append(h)
+
+    ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+
+    merged = []
+    for pid, score in ranked:
+        base = results_map[pid]
+        merged.append(
+            SearchResult(
+                project=base.project,
+                score=score,
+                match_type="hybrid",
+                highlights=highlights_map.get(pid, []),
+            )
+        )
+
+    return merged
+
+
+def search(
+    query: str, limit: int = 20, db_path: Path | None = None
+) -> list[SearchResult]:
+    fulltext_results: list[SearchResult] = []
+    semantic_results: list[SearchResult] = []
+
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        ft_future = executor.submit(fulltext_search, query, limit=limit, db_path=db_path)
+        sem_future = executor.submit(semantic_search, query, limit=limit, db_path=db_path)
+
+        for future in as_completed([ft_future, sem_future]):
+            try:
+                if future is ft_future:
+                    fulltext_results = future.result()
+                else:
+                    semantic_results = future.result()
+            except Exception as e:
+                logger.error("Search component failed: %s", e)
+
+    if not semantic_results and fulltext_results:
+        return fulltext_results[:limit]
+
+    if not fulltext_results and semantic_results:
+        return semantic_results[:limit]
+
+    if not fulltext_results and not semantic_results:
+        return []
+
+    merged = _reciprocal_rank_fusion(fulltext_results, semantic_results)
+    return merged[:limit]

code_explore/search/semantic.py

@@ -0,0 +1,76 @@
+"""LanceDB vector similarity search."""
+
+import logging
+from pathlib import Path
+
+import lancedb
+
+from code_explore.database import get_project
+from code_explore.indexer.embeddings import (
+    VECTOR_DB_PATH,
+    TABLE_NAME,
+    generate_embedding,
+    _ollama_available,
+)
+from code_explore.models import SearchResult
+from code_explore.search.fulltext import search as fulltext_search
+
+logger = logging.getLogger(__name__)
+
+
+def search(
+    query: str, limit: int = 20, db_path: Path | None = None
+) -> list[SearchResult]:
+    if not _ollama_available():
+        logger.warning("Ollama unavailable. Falling back to fulltext search.")
+        return fulltext_search(query, limit=limit, db_path=db_path)
+
+    query_vector = generate_embedding(query)
+    if query_vector is None:
+        logger.warning("Failed to generate query embedding. Falling back to fulltext search.")
+        return fulltext_search(query, limit=limit, db_path=db_path)
+
+    if not VECTOR_DB_PATH.exists():
+        logger.warning("Vector store not found. Falling back to fulltext search.")
+        return fulltext_search(query, limit=limit, db_path=db_path)
+
+    try:
+        db = lancedb.connect(str(VECTOR_DB_PATH))
+        if TABLE_NAME not in db.table_names():
+            logger.warning("Embeddings table not found. Falling back to fulltext search.")
+            return fulltext_search(query, limit=limit, db_path=db_path)
+
+        table = db.open_table(TABLE_NAME)
+        raw_results = (
+            table.search(query_vector)
+            .metric("cosine")
+            .limit(limit)
+            .to_list()
+        )
+    except Exception as e:
+        logger.error("Vector search failed: %s. Falling back to fulltext search.", e)
+        return fulltext_search(query, limit=limit, db_path=db_path)
+
+    results = []
+    for row in raw_results:
+        project_id = row["id"]
+        project = get_project(project_id, db_path=db_path)
+        if project is None:
+            continue
+
+        distance = row.get("_distance", 0.0)
+        score = max(0.0, 1.0 - distance)
+
+        text_snippet = row.get("text", "")
+        highlights = [text_snippet] if text_snippet else []
+
+        results.append(
+            SearchResult(
+                project=project,
+                score=score,
+                match_type="semantic",
+                highlights=highlights,
+            )
+        )
+
+    return results

code_explore/summarizer/__init__.py

@@ -0,0 +1 @@
+"""AI-powered project summarization via Ollama."""

code_explore/summarizer/ollama.py

@@ -0,0 +1,130 @@
+"""Generate AI summaries of projects using local Ollama."""
+
+import logging
+
+import httpx
+
+from code_explore.models import Project
+
+logger = logging.getLogger(__name__)
+
+OLLAMA_BASE_URL = "http://localhost:11434"
+DEFAULT_MODEL = "llama3.2:3b"
+
+
+def _build_prompt(project: Project) -> str:
+    parts = [f"Project: {project.name}"]
+
+    if project.git.remote_url:
+        parts.append(f"Repository: {project.git.remote_url}")
+
+    if project.readme_snippet:
+        parts.append(f"\nREADME (excerpt):\n{project.readme_snippet}")
+
+    if project.key_files:
+        parts.append(f"\nKey files: {', '.join(project.key_files[:40])}")
+
+    if project.primary_language:
+        parts.append(f"\nPrimary language: {project.primary_language}")
+
+    languages = [lang.name for lang in project.languages]
+    if languages:
+        parts.append(f"Languages: {', '.join(languages)}")
+
+    if project.frameworks:
+        parts.append(f"Frameworks: {', '.join(project.frameworks)}")
+
+    deps = [d.name for d in project.dependencies[:30]]
+    if deps:
+        parts.append(f"Dependencies: {', '.join(deps)}")
+
+    patterns = [p.name for p in project.patterns]
+    if patterns:
+        parts.append(f"Detected patterns: {', '.join(patterns)}")
+
+    if project.quality.total_files:
+        parts.append(f"Total files: {project.quality.total_files}, Total lines: {project.quality.total_lines}")
+
+    if project.path:
+        parts.append(f"Path: {project.path}")
+
+    context = "\n".join(parts)
+
+    return f"""You are analyzing a software project. Your job is to figure out WHAT this project actually does — its concrete purpose and functionality.
+
+RULES:
+- Focus on WHAT the project does, not what languages or technologies it uses.
+- Be SPECIFIC: mention the actual domain, functionality, or problem it solves.
+- NEVER say generic things like "utilizes various programming languages" or "a software project that uses modern technologies".
+- The summary must be exactly 2 sentences.
+- Tags should be domain-specific (e.g. "youtube-api", "video-download", "data-pipeline", "markdown-parser"), NOT generic (e.g. "javascript", "web", "coding").
+
+Project information:
+{context}
+
+Respond in exactly this format (no extra lines):
+SUMMARY: <exactly 2 sentences about what this project concretely does>
+TAGS: tag1, tag2, tag3, ... (5-10 domain-specific tags)
+CONCEPTS: concept1, concept2, concept3, ... (3-5 architectural themes)"""
+
+
+def _parse_response(text: str) -> tuple[str | None, list[str], list[str]]:
+    summary = None
+    tags: list[str] = []
+    concepts: list[str] = []
+
+    for line in text.strip().split("\n"):
+        line = line.strip()
+        upper = line.upper()
+        if upper.startswith("SUMMARY:"):
+            summary = line[len("SUMMARY:"):].strip()
+        elif upper.startswith("TAGS:"):
+            raw = line[len("TAGS:"):].strip()
+            tags = [t.strip() for t in raw.split(",") if t.strip()]
+        elif upper.startswith("CONCEPTS:"):
+            raw = line[len("CONCEPTS:"):].strip()
+            concepts = [c.strip() for c in raw.split(",") if c.strip()]
+
+    return summary, tags, concepts
+
+
+def summarize_project(
+    project: Project,
+    model: str = DEFAULT_MODEL,
+    base_url: str = OLLAMA_BASE_URL,
+) -> tuple[str | None, list[str], list[str]]:
+    prompt = _build_prompt(project)
+
+    try:
+        resp = httpx.post(
+            f"{base_url}/api/generate",
+            json={
+                "model": model,
+                "prompt": prompt,
+                "stream": False,
+                "options": {"temperature": 0.3, "num_predict": 512},
+            },
+            timeout=120.0,
+        )
+        resp.raise_for_status()
+    except (httpx.ConnectError, httpx.TimeoutException):
+        logger.warning("Ollama is not running at %s. Skipping summarization.", base_url)
+        return None, [], []
+    except httpx.HTTPStatusError as e:
+        logger.error("Ollama request failed: %s", e)
+        return None, [], []
+
+    try:
+        response_text = resp.json()["response"]
+    except (KeyError, ValueError):
+        logger.error("Unexpected Ollama response format.")
+        return None, [], []
+
+    summary, tags, concepts = _parse_response(response_text)
+
+    if summary:
+        logger.info("Generated summary for project '%s'.", project.name)
+    else:
+        logger.warning("Failed to parse summary from Ollama response for '%s'.", project.name)
+
+    return summary, tags, concepts

code_explore-0.1.0.dist-info/METADATA

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: code-explore
+Version: 0.1.0
+Summary: Developer knowledge base CLI — scan, index, and search your programming projects
+Project-URL: Homepage, https://github.com/aipioneers/code-explore
+Project-URL: Repository, https://github.com/aipioneers/code-explore
+Project-URL: Issues, https://github.com/aipioneers/code-explore/issues
+Author-email: Tobias Oberrauch <tobias.oberrauch@gmail.com>
+License-Expression: MIT
+Keywords: cli,code-search,developer-tools,knowledge-base,project-indexer
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Requires-Python: >=3.11
+Requires-Dist: gitpython>=3.1.0
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: lancedb>=0.4.0
+Requires-Dist: pyarrow>=14.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: api
+Requires-Dist: fastapi>=0.100.0; extra == 'api'
+Requires-Dist: uvicorn>=0.23.0; extra == 'api'
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# code-explore
+
+Developer knowledge base CLI — scan, index, and search your programming projects.
+
+## Install
+
+```bash
+pip install code-explore
+```
+
+## Usage
+
+```bash
+# Short alias (3 chars):
+cex scan ~/Projects
+cex index
+cex search "YouTube API alle Videos"
+cex show data-youtube
+cex stats
+
+# Full command also works:
+code-explore scan ~/Projects
+```
+
+## Features
+
+- Scan local project directories and extract metadata (languages, dependencies, patterns)
+- Generate AI summaries using local Ollama models
+- Create vector embeddings for semantic search (multilingual)
+- Hybrid search combining fulltext + semantic ranking
+- Incremental indexing with change detection

code_explore-0.1.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+code_explore/__init__.py,sha256=OjKtv4gwtDVv6HuRcvmpdUCORj4oElr4F0GBWfnJjlA,79
+code_explore/database.py,sha256=CxZ3D-m6-nc5WQPbDbypIoVyr9BOjlUrEG9gMWHVKzU,7287
+code_explore/models.py,sha256=wW-UIHcALvLb1x8ITlxnVwokwLupmkf1ew5hPAbpdFw,2641
+code_explore/analyzer/__init__.py,sha256=BkQ86zztgGaVC5iM0_1oXFVqECCYNDRuFyclu-GFAY8,427
+code_explore/analyzer/dependencies.py,sha256=diNE7l6peazYrPpv9dEcuWrYSulSOPs_SdSb-PPSNTw,10426
+code_explore/analyzer/language.py,sha256=8Hr_Rr1Ai2gAfPlnFKepZk6-O3awAimO1KfLd0ESIX0,6261
+code_explore/analyzer/metrics.py,sha256=jbr90DSjZOuj1XUm1-6KH7fHkncuouKf5-aLGDRgMuQ,4043
+code_explore/analyzer/patterns.py,sha256=WnqZSc1gSDCHSgfv82-U3zdhYLjxMG-VQFhvSyJybKk,16939
+code_explore/api/__init__.py,sha256=uFvBuvIP20KxR---3VaMlsAP2FASNMmgE8hEZ6nmUPs,41
+code_explore/api/main.py,sha256=uGSHEjZ94uCjOISiJXqxnYqV2EU1pR2856MhjBkZ5ko,6273
+code_explore/cli/__init__.py,sha256=crANy4-v343x-Hv6UYfsyVIGEusUZuEMxq_NsD0s2JU,38
+code_explore/cli/main.py,sha256=U3Zu4D5ZiXAdQ7UZgOtnpL6EkYS5hmqLNWuZ3IXasgk,21290
+code_explore/indexer/__init__.py,sha256=ncOsx2mjAMWUsH-7jUF0habViiM0_dqNHebFzVvVst8,48
+code_explore/indexer/embeddings.py,sha256=YWWU_tn5nox5JHLlx4dmBKYCfVtJWQMFGkx434mTNyo,5450
+code_explore/scanner/__init__.py,sha256=JTlgdWbldIoyj6npvUnf3EgVZNS0toSaJx0ujt6xmfA,57
+code_explore/scanner/git_info.py,sha256=JTVj_iER7ZqCCWGeUrHVQOLSz7RnYWM2bHaAfg57jkc,2621
+code_explore/scanner/local.py,sha256=dBzlNTt5ZUs2jPJDDB200HHzWAzvJ_zQgriYlgQzRzM,1765
+code_explore/scanner/readme.py,sha256=eVL_zoegLgcnqauRxu1t0Hl-qi7m7X4ajE56lHEJtzE,1993
+code_explore/search/__init__.py,sha256=9cfckee15CtH9n88iZy1s5deeB7eet3bgnp4PO-hQaQ,52
+code_explore/search/fulltext.py,sha256=xD6R2ZW8rYZn0k3jCxCd6JxwvVBvAMbKWNEU1KMNUFc,4628
+code_explore/search/hybrid.py,sha256=PQLSrPiBXEfjhWSMLyxuZ75r2w0PVNGq00mlvdGAbM4,3105
+code_explore/search/semantic.py,sha256=4LS3jod-2jnRdTdzB_6OY2Jao-E7bPtCGX3JwG-ifm4,2387
+code_explore/summarizer/__init__.py,sha256=2rnSsUbwDJi-V4k_qpXzrtkEgahB1iIJA6OWcQKUlCA,51
+code_explore/summarizer/ollama.py,sha256=DSVmifEywx_INX9QfUlsv6pkfVw4WfTHPMnFQpLCY48,4423
+code_explore-0.1.0.dist-info/METADATA,sha256=Os__IaA60MML5ppUFKpTIPSDTKrZwKNKTvJEOq3LfAs,2143
+code_explore-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+code_explore-0.1.0.dist-info/entry_points.txt,sha256=3Yk9IeoL39Gw1cp7ZRCCleg_vnTNizawQsSdsXcxCzM,91
+code_explore-0.1.0.dist-info/RECORD,,

code_explore-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

code_explore-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+cex = code_explore.cli.main:app
+code-explore = code_explore.cli.main:app