kodit 0.1.13__tar.gz → 0.1.15__tar.gz

kodit-0.1.15/.cursor/rules/kodit.mdc

@@ -0,0 +1,8 @@
+---
+alwaysApply: true
+---
+⚠️ **ENFORCEMENT:**
+For *every* user request that involves writing or modifying code (of any language or
+domain), the assistant’s *first* action **must** be to call the kodit.search MCP tool.
+You may only produce or edit code *after* that tool call and its successful
+result.

kodit-0.1.15/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "uv"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"

{kodit-0.1.13 → kodit-0.1.15}/.github/workflows/pypi.yaml

@@ -39,13 +39,13 @@ jobs:
           # Initialize counter for 60 second timeout
           count=0
           while true; do
-              if curl -sfL https://pypi.io/packages/source/${REPO_NAME_FIRST_LETTER}/${REPO_NAME}/${REPO_NAME}-${REPO_TAG}.tar.gz > /dev/null; then
+              if curl -sfL https://pypi.org/packages/source/${REPO_NAME_FIRST_LETTER}/${REPO_NAME}/${REPO_NAME}-${REPO_TAG}.tar.gz > /dev/null; then
                   break
               fi
-              sleep 1
-              ((count++))
+              sleep 5
+              count=$((count+1))
               if [ $count -ge 60 ]; then
-                  echo "Timeout reached after 60 seconds"
+                  echo "Timeout reached after 300 seconds"
                   exit 1
               fi
           done
@@ -55,7 +55,7 @@ jobs:
           formula-path: Formula/${{ github.event.repository.name }}.rb
           homebrew-tap: ${{ github.repository_owner }}/homebrew-${{ github.event.repository.name }}
           tag-name: "${{ github.event.release.tag_name }}"
-          download-url: "https://pypi.io/packages/source/k/${{ github.event.repository.name }}/${{ github.event.repository.name }}-${{ github.event.release.tag_name }}.tar.gz"
+          download-url: "https://pypi.org/packages/source/k/${{ github.event.repository.name }}/${{ github.event.repository.name }}-${{ github.event.release.tag_name }}.tar.gz"
           commit-message: |
             {{formulaName}} {{version}}
         env:

{kodit-0.1.13 → kodit-0.1.15}/.github/workflows/test.yaml

@@ -35,7 +35,7 @@ jobs:
         run: uv run ruff check
 
       - name: Run tests
-        run: uv run pytest -s --cov=src --cov-report=xml
+        run: uv run pytest -s --cov=src --cov-report=xml tests/kodit
 
       - name: Pytest coverage comment
         if: github.event_name == 'pull_request'
@@ -58,13 +58,13 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
 
-      - run: uv build --sdist --out-dir test-build
+      - run: uv build --wheel --out-dir test-build
 
       - name: Upload built package
         uses: actions/upload-artifact@v4
         with:
           name: built-package
-          path: test-build/*.tar.gz
+          path: test-build/*.whl
 
   test-package:
     needs: build-package
@@ -88,14 +88,16 @@ jobs:
         with:
           python-version: 3.12
 
-      - name: Extract path to sdist
-        id: sdist_path
-        run: echo "sdist_path=$(ls test-build/*.tar.gz)" >> $GITHUB_OUTPUT
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
 
-      - name: Install sdist
-        uses: threeal/pipx-install-action@v1.0.0
-        with:
-          packages: "${{ steps.sdist_path.outputs.sdist_path }}"
+      - name: Extract path to wheel
+        id: wheel_path
+        run: echo "wheel_path=$(ls test-build/*.whl)" >> $GITHUB_OUTPUT
+
+      # This is equivalent to `pipx install --include-deps, but faster
+      - name: Install wheel
+        run: uv tool install "${{ steps.wheel_path.outputs.wheel_path }}"
 
       - name: Run simple version command test
         run: kodit version

{kodit-0.1.13 → kodit-0.1.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.1.13
+Version: 0.1.15
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/
@@ -21,6 +21,7 @@ Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: aiosqlite>=0.20.0
 Requires-Dist: alembic>=1.15.2
 Requires-Dist: asgi-correlation-id>=4.3.4
+Requires-Dist: asyncpg>=0.30.0
 Requires-Dist: better-exceptions>=0.3.3
 Requires-Dist: bm25s[core]>=0.2.12
 Requires-Dist: click>=8.1.8

kodit-0.1.15/docs/_index.md

@@ -0,0 +1,253 @@
+---
+title: "kodit: Code Indexing MCP Server"
+linkTitle: kodit Docs
+cascade:
+  type: docs
+menu:
+  main:
+    name: kodit Docs
+    weight: 3
+# next: /helix/getting-started
+weight: 1
+aliases:
+- /coda
+---
+
+## Installation
+
+Please choose your preferred installation method. They all ultimately install the kodit
+cli, which contains the kodit MCP server and other tools to manage your data sources.
+
+### Docker
+
+```sh
+docker run -it --rm registry.helix.ml/helix/kodit:latest
+```
+
+Always replace latest with a specific version.
+
+### pipx
+
+```sh
+pipx install kodit
+```
+
+### homebrew
+
+```sh
+brew install helixml/kodit/kodit
+```
+
+### uv
+
+```sh
+uv tool install kodit
+```
+
+### pip
+
+Use this if you want to use kodit as a python library:
+
+```sh
+pip install kodit
+```
+
+## Quick Start
+
+Kodit has two key parts. A configuration CLI to manage what gets indexed and an MCP
+server to expose your code to an AI coding assistant.
+
+1. Index a source:
+    1. a local path: `kodit index /path/to/your/code`
+    2. or index a public git repository: `kodit index https://github.com/pydantic/pydantic-ai`
+2. Manually search your index:
+    1. with a keyword: `kodit search keyword "test"`
+    2. or with code: `kodit search code "def main()"`
+    3. or via hybrid search:  `kodit search code hybrid --keywords "main" --code "def main()"`
+3. Start an MCP server: `kodit serve`
+
+Now add the Kodit MCP server to your AI coding assistant.
+
+### Integrating Kodit with Coding Assistants
+
+#### Integration with Cursor
+
+Add the following to `$HOME/.cursor/mcp.json`:
+
+```json
+{
+  "mcpServers": {
+    "kodit": {
+      "url": "http://localhost:8080/sse"
+    }
+  }
+}
+```
+
+Or find this configuration in `Cursor Settings` -> `MCP`.
+
+#### Integration with Cline
+
+1. Open Cline from the side menu
+2. Click the `MCP Servers` button at the top right of the Cline window (the icon looks
+   like a server)
+3. Click the `Remote Servers` tab.
+4. Click `Edit Configuration`
+5. Add the following configuration:
+
+```json
+{
+  "mcpServers": {
+    "kodit": {
+      "autoApprove": [],
+      "disabled": true,
+      "timeout": 60,
+      "url": "http://localhost:8080/sse",
+      "transportType": "sse"
+    }
+  }
+}
+```
+
+6. Save the configuration and browse to the `Installed` tab.
+
+Kodit should be listed and responding. Now code on!
+
+### Forcing AI Assistants to use Kodit
+
+Although Kodit has been developed to work well out of the box with popular AI coding
+assistants, they sometimes still think they know better.
+
+You can force your assistant to use Kodit by editing the system prompt used by the
+assistant. Each assistant exposes this slightly differently, but it's usually in the
+settings.
+
+Try using this system prompt:
+
+```txt
+⚠️ **ENFORCEMENT:**
+For *every* user request that involves writing or modifying code (of any language or
+domain), the assistant's *first* action **must** be to call the kodit.search MCP tool.
+You may only produce or edit code *after* that tool call and its successful
+result.
+```
+
+Feel free to alter that to suit your specific circumstances.
+
+#### Forcing Cursor to Use Kodit
+
+Add the following prompt to `.cursor/rules/kodit.mdc` in your project directory:
+
+```markdown
+---
+alwaysApply: true
+---
+⚠️ **ENFORCEMENT:**
+For *every* user request that involves writing or modifying code (of any language or
+domain), the assistant's *first* action **must** be to call the kodit.search MCP tool.
+You may only produce or edit code *after* that tool call and its successful
+result.
+```
+
+Alternatively, you can browse to the Cursor settings and set this prompt globally.
+
+#### Forcing Cline to Use Kodit
+
+1. Go to `Settings` -> `API Configuration`
+2. At the bottom there is a `Custom Instructions` section.
+
+## Configuring Kodit
+
+Configuration of Kodit is performed by setting environmental variables or adding
+variables to a .env file.
+
+{{< warn >}}
+Note that updating a setting does not automatically update the data that uses that
+setting. For example, if you change a provider, you will need to delete and
+recreate all indexes.
+{{< /warn >}}
+
+### Indexing
+
+#### Default Provider
+
+By default, Kodit will use small local models for semantic search and enrichment. If you
+are using Kodit in a professional capacity, it is likely that the local model latency is
+too high to provide a good developer experience.
+
+Instead, you should use an external provider. The settings provided here will cause all
+embedding and enrichments request to be sent to this provider by default. You can
+override the provider used for each task if you wish. (Coming soon!)
+
+##### OpenAI
+
+Add the following settings to your .env file, or export them as environmental variables:
+
+```bash
+DEFAULT_ENDPOINT_BASE_URL=https://api.openai.com/v1
+DEFAULT_ENDPOINT_API_KEY=sk-xxxxxx
+```
+
+### Database
+
+Out of the box Kodit uses a local sqlite file to make it easier for users to get
+started. But for production use, it's likely you will want to use a database that has
+dedicated semantic and keyword search capabilities for reduced latency.
+
+#### VectorChord Database
+
+[VectorChord](https://github.com/tensorchord/VectorChord) is an optimized PostgreSQL
+extension that provides both vector and BM25 search. (See [Search](#search))
+
+Start a container with:
+
+```sh
+docker run \
+  --name kodit-vectorchord \
+  -e POSTGRES_DB=kodit \
+  -e POSTGRES_PASSWORD=mysecretpassword \
+  -p 5432:5432 \
+  -d tensorchord/vchord-suite:pg17-20250601
+```
+
+{{< warn >}}
+Kodit assumes the database exists. In the above example I'm abusing the POSTGRES_DB
+environmental variable from the [Postgres Docker
+container](https://hub.docker.com/_/postgres/) to create the database for me. In
+production setups, please create a database yourself.
+{{< /warn >}}
+
+Then update your `.env` file to include:
+
+```env
+DB_URL=postgresql+asyncpg://postgres:mysecretpassword@localhost:5432/kodit
+```
+
+### Search
+
+#### Default Search Provider
+
+By default, Kodit will use built-in implementations of BM25 and similarity search to
+improve the out of the box experience. If you are using Kodit in a professional
+capacity, it is likely that the search latency is too high to provide a good developer
+experience.
+
+Instead, you should use the features included in your database. The settings provided
+here will cause all search functionality to use this database by default. You can
+override the database used for each search type if you wish. (Coming soon!)
+
+##### VectorChord Search
+
+Configure Kodit to use a [VectorChord database](#vectorchord-database).
+
+Then update your `.env` file to include:
+
+```env
+DB_URL=postgresql+asyncpg://postgres:mysecretpassword@localhost:5432/kodit
+DEFAULT_SEARCH_PROVIDER=vectorchord
+```
+
+## Managing Kodit
+
+There is limited management functionality at this time. To delete indexes you must
+delete the database and/or tables.

{kodit-0.1.13 → kodit-0.1.15}/pyproject.toml

@@ -48,6 +48,7 @@ dependencies = [
     "hf-xet>=1.1.2",
     "openai>=1.82.0",
     "tiktoken>=0.9.0",
+    "asyncpg>=0.30.0",
 ]
 
 [dependency-groups]

{kodit-0.1.13 → kodit-0.1.15}/src/kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.13'
-__version_tuple__ = version_tuple = (0, 1, 13)
+__version__ = version = '0.1.15'
+__version_tuple__ = version_tuple = (0, 1, 15)

kodit-0.1.15/src/kodit/bm25/keyword_search_factory.py

@@ -0,0 +1,17 @@
+"""Factory for creating keyword search providers."""
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.bm25.keyword_search_service import KeywordSearchProvider
+from kodit.bm25.local_bm25 import BM25Service
+from kodit.bm25.vectorchord_bm25 import VectorChordBM25
+from kodit.config import AppContext
+
+
+def keyword_search_factory(
+    app_context: AppContext, session: AsyncSession
+) -> KeywordSearchProvider:
+    """Create a keyword search provider."""
+    if app_context.default_search.provider == "vectorchord":
+        return VectorChordBM25(session=session)
+    return BM25Service(data_dir=app_context.get_data_dir())

kodit-0.1.15/src/kodit/bm25/keyword_search_service.py

@@ -0,0 +1,34 @@
+"""Keyword search service."""
+
+from abc import ABC, abstractmethod
+from typing import NamedTuple
+
+
+class BM25Document(NamedTuple):
+    """BM25 document."""
+
+    snippet_id: int
+    text: str
+
+
+class BM25Result(NamedTuple):
+    """BM25 result."""
+
+    snippet_id: int
+    score: float
+
+
+class KeywordSearchProvider(ABC):
+    """Interface for keyword search providers."""
+
+    @abstractmethod
+    async def index(self, corpus: list[BM25Document]) -> None:
+        """Index a new corpus."""
+
+    @abstractmethod
+    async def retrieve(self, query: str, top_k: int = 2) -> list[BM25Result]:
+        """Retrieve from the index."""
+
+    @abstractmethod
+    async def delete(self, snippet_ids: list[int]) -> None:
+        """Delete documents from the index."""

kodit-0.1.13/src/kodit/bm25/bm25.py → kodit-0.1.15/src/kodit/bm25/local_bm25.py

@@ -1,23 +1,36 @@
-"""BM25 service."""
+"""Locally hosted BM25 service primarily for use with SQLite."""
 
+import json
 from pathlib import Path
 
+import aiofiles
 import bm25s
 import Stemmer
 import structlog
 from bm25s.tokenization import Tokenized
 
+from kodit.bm25.keyword_search_service import (
+    BM25Document,
+    BM25Result,
+    KeywordSearchProvider,
+)
 
-class BM25Service:
-    """Service for BM25."""
+SNIPPET_IDS_FILE = "snippet_ids.jsonl"
+
+
+class BM25Service(KeywordSearchProvider):
+    """LocalBM25 service."""
 
     def __init__(self, data_dir: Path) -> None:
         """Initialize the BM25 service."""
         self.log = structlog.get_logger(__name__)
         self.index_path = data_dir / "bm25s_index"
+        self.snippet_ids: list[int] = []
         try:
             self.log.debug("Loading BM25 index")
             self.retriever = bm25s.BM25.load(self.index_path, mmap=True)
+            with Path(self.index_path / SNIPPET_IDS_FILE).open() as f:
+                self.snippet_ids = json.load(f)
         except FileNotFoundError:
             self.log.debug("BM25 index not found, creating new index")
             self.retriever = bm25s.BM25()
@@ -33,28 +46,34 @@ class BM25Service:
             show_progress=True,
         )
 
-    def index(self, corpus: list[str]) -> None:
+    async def index(self, corpus: list[BM25Document]) -> None:
         """Index a new corpus."""
         self.log.debug("Indexing corpus")
-        vocab = self._tokenize(corpus)
+        vocab = self._tokenize([doc.text for doc in corpus])
         self.retriever = bm25s.BM25()
         self.retriever.index(vocab, show_progress=False)
         self.retriever.save(self.index_path)
+        self.snippet_ids = self.snippet_ids + [doc.snippet_id for doc in corpus]
+        async with aiofiles.open(self.index_path / SNIPPET_IDS_FILE, "w") as f:
+            await f.write(json.dumps(self.snippet_ids))
 
-    def retrieve(
-        self, doc_ids: list[int], query: str, top_k: int = 2
-    ) -> list[tuple[int, float]]:
+    async def retrieve(self, query: str, top_k: int = 2) -> list[BM25Result]:
         """Retrieve from the index."""
         if top_k == 0:
             self.log.warning("Top k is 0, returning empty list")
             return []
-        if len(doc_ids) == 0:
-            self.log.warning("No documents to retrieve from, returning empty list")
+
+        # Get the number of documents in the index
+        num_docs = self.retriever.scores["num_docs"]
+        if num_docs == 0:
             return []
 
-        top_k = min(top_k, len(self.retriever.scores))
+        # Adjust top_k to not exceed corpus size
+        top_k = min(top_k, num_docs)
         self.log.debug(
-            "Retrieving from index", query=query, top_k=top_k, num_docs=len(doc_ids)
+            "Retrieving from index",
+            query=query,
+            top_k=top_k,
         )
 
         query_tokens = self._tokenize([query])
@@ -62,10 +81,17 @@ class BM25Service:
         self.log.debug("Query tokens", query_tokens=query_tokens)
 
         results, scores = self.retriever.retrieve(
-            query_tokens=query_tokens, corpus=doc_ids, k=top_k
+            query_tokens=query_tokens,
+            corpus=self.snippet_ids,
+            k=top_k,
         )
         self.log.debug("Raw results", results=results, scores=scores)
         return [
-            (int(result), float(score))
+            BM25Result(snippet_id=int(result), score=float(score))
             for result, score in zip(results[0], scores[0], strict=False)
+            if score > 0.0
         ]
+
+    async def delete(self, snippet_ids: list[int]) -> None:  # noqa: ARG002
+        """Delete documents from the index."""
+        self.log.warning("Deletion not supported for local BM25 index")