kodit 0.1.7__tar.gz → 0.1.9__tar.gz

{kodit-0.1.7 → kodit-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.1.7
+Version: 0.1.9
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/
@@ -28,6 +28,7 @@ Requires-Dist: colorama>=0.4.6
 Requires-Dist: dotenv>=0.9.9
 Requires-Dist: fastapi[standard]>=0.115.12
 Requires-Dist: fastmcp>=2.3.3
+Requires-Dist: gitpython>=3.1.44
 Requires-Dist: httpx-retries>=0.3.2
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: posthog>=4.0.1

{kodit-0.1.7 → kodit-0.1.9}/docs/_index.md

@@ -52,14 +52,15 @@ Use this if you want to use kodit as a python library:
 pip install kodit
 ```
 
-## Usage
+## Quick Start
 
 Kodit has two key parts. A configuration CLI to manage what gets indexed and an MCP
 server to expose your code to an AI coding assistant.
 
-1. Add an index: `kodit index /path/to/your/code`
-2. Test retrieval on your index: `kodit retrieve "test"`
-3. Start an MCP server: `kodit serve`
+1. Index a local path: `kodit index /path/to/your/code`
+2. Or index a public git repository: `kodit index https://github.com/pydantic/pydantic-ai`
+3. Test retrieval on your index: `kodit retrieve "test"`
+4. Start an MCP server: `kodit serve`
 
 Now browse to your AI coding assistant and add the MCP server. You will also need to
 tell your assistant to use this server in coding tasks, otherwise it won't get called!

{kodit-0.1.7 → kodit-0.1.9}/pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
     "fastmcp>=2.3.3",
     "pydantic-settings>=2.9.1",
     "bm25s[core]>=0.2.12",
+    "gitpython>=3.1.44",
 ]
 
 [dependency-groups]

{kodit-0.1.7 → kodit-0.1.9}/src/kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.7'
-__version_tuple__ = version_tuple = (0, 1, 7)
+__version__ = version = '0.1.9'
+__version_tuple__ = version_tuple = (0, 1, 9)

{kodit-0.1.7 → kodit-0.1.9}/src/kodit/cli.py

@@ -123,12 +123,6 @@ async def index(
         return
     # Handle source indexing
     for source in sources:
-        if source.startswith("https://"):
-            msg = "Web or git indexing is not implemented yet"
-            raise click.UsageError(msg)
-        if source.startswith("git"):
-            msg = "Git indexing is not implemented yet"
-            raise click.UsageError(msg)
         if Path(source).is_file():
             msg = "File indexing is not implemented yet"
             raise click.UsageError(msg)

{kodit-0.1.7 → kodit-0.1.9}/src/kodit/indexing/repository.py

@@ -8,7 +8,7 @@ and retrieving index information with their associated metadata.
 from datetime import UTC, datetime
 from typing import TypeVar
 
-from sqlalchemy import func, select
+from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.indexing.models import Index, Snippet
@@ -63,6 +63,17 @@ class IndexRepository:
         result = await self.session.execute(query)
         return result.scalar_one_or_none()
 
+    async def get_by_source_id(self, source_id: int) -> Index | None:
+        """Get an index by its source ID.
+
+        Args:
+            source_id: The ID of the source to retrieve an index for.
+
+        """
+        query = select(Index).where(Index.source_id == source_id)
+        result = await self.session.execute(query)
+        return result.scalar_one_or_none()
+
     async def files_for_index(self, index_id: int) -> list[File]:
         """Get all files for an index.
 
@@ -122,6 +133,17 @@ class IndexRepository:
         self.session.add(snippet)
         await self.session.commit()
 
+    async def delete_all_snippets(self, index_id: int) -> None:
+        """Delete all snippets for an index.
+
+        Args:
+            index_id: The ID of the index to delete snippets for.
+
+        """
+        query = delete(Snippet).where(Snippet.index_id == index_id)
+        await self.session.execute(query)
+        await self.session.commit()
+
     async def get_snippets_for_index(self, index_id: int) -> list[Snippet]:
         """Get all snippets for an index.
 

{kodit-0.1.7 → kodit-0.1.9}/src/kodit/indexing/service.py

@@ -83,7 +83,10 @@ class IndexService:
         # Check if the source exists
         source = await self.source_service.get(source_id)
 
-        index = await self.repository.create(source.id)
+        # Check if the index already exists
+        index = await self.repository.get_by_source_id(source.id)
+        if not index:
+            index = await self.repository.create(source.id)
         return IndexView(
             id=index.id,
             created_at=index.created_at,
@@ -119,6 +122,9 @@ class IndexService:
             msg = f"Index not found: {index_id}"
             raise ValueError(msg)
 
+        # First delete all old snippets, if they exist
+        await self.repository.delete_all_snippets(index_id)
+
         # Create snippets for supported file types
         await self._create_snippets(index_id)
 

{kodit-0.1.7 → kodit-0.1.9}/src/kodit/sources/service.py

@@ -13,6 +13,7 @@ from hashlib import sha256
 from pathlib import Path
 
 import aiofiles
+import git
 import pydantic
 import structlog
 from tqdm import tqdm
@@ -98,8 +99,19 @@ class SourceService:
             parsed = urisplit(uri_or_path_like)
             if parsed.scheme == "file":
                 return await self._create_folder_source(Path(parsed.path))
-            msg = f"Unsupported source type: {uri_or_path_like}"
-            raise ValueError(msg)
+            if parsed.scheme in ("git", "http", "https") and parsed.path.endswith(
+                ".git"
+            ):
+                return await self._create_git_source(uri_or_path_like)
+
+            # Try adding a .git suffix, sometimes people just pass the url
+            if not uri_or_path_like.endswith(".git"):
+                uri_or_path_like = uri_or_path_like + ".git"
+                try:
+                    return await self._create_git_source(uri_or_path_like)
+                except ValueError:
+                    pass
+
         msg = f"Unsupported source type: {uri_or_path_like}"
         raise ValueError(msg)
 
@@ -110,46 +122,98 @@ class SourceService:
             directory: The path to the local directory.
 
         Raises:
-            ValueError: If the folder doesn't exist or is already added.
+            ValueError: If the folder doesn't exist.
+            SourceAlreadyExistsError: If the folder is already added.
 
         """
         # Resolve the directory to an absolute path
         directory = directory.expanduser().resolve()
 
-        # Check if the folder exists
-        if not directory.exists():
-            msg = f"Folder does not exist: {directory}"
-            raise ValueError(msg)
+        source = await self.repository.get_source_by_uri(directory.as_uri())
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+        else:
+            # Check if the folder exists
+            if not directory.exists():
+                msg = f"Folder does not exist: {directory}"
+                raise ValueError(msg)
+
+            # Check if the folder is already added
+            if await self.repository.get_source_by_uri(directory.as_uri()):
+                msg = f"Directory already added: {directory}"
+                raise ValueError(msg)
+
+            # Clone into a local directory
+            clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
+            clone_path.mkdir(parents=True, exist_ok=True)
+
+            # Copy all files recursively, preserving directory structure, ignoring
+            # hidden files
+            shutil.copytree(
+                directory,
+                clone_path,
+                ignore=shutil.ignore_patterns(".*"),
+                dirs_exist_ok=True,
+            )
 
-        # Check if the folder is already added
-        if await self.repository.get_source_by_uri(directory.as_uri()):
-            msg = f"Directory already added: {directory}"
-            raise ValueError(msg)
+            source = await self.repository.create_source(
+                Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+            )
 
-        # Clone into a local directory
-        clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
-        clone_path.mkdir(parents=True, exist_ok=True)
-
-        # Copy all files recursively, preserving directory structure, ignoring hidden
-        # files
-        shutil.copytree(
-            directory,
-            clone_path,
-            ignore=shutil.ignore_patterns(".*"),
-            dirs_exist_ok=True,
-        )
+            # Add all files to the source
+            # Count total files for progress bar
+            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
+
+            # Process each file in the source directory
+            for path in tqdm(clone_path.rglob("*"), total=file_count):
+                await self._process_file(source.id, path.absolute())
 
-        source = await self.repository.create_source(
-            Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+        return SourceView(
+            id=source.id,
+            uri=source.uri,
+            cloned_path=Path(source.cloned_path),
+            created_at=source.created_at,
+            num_files=await self.repository.num_files_for_source(source.id),
         )
 
-        # Add all files to the source
-        # Count total files for progress bar
-        file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
+    async def _create_git_source(self, uri: str) -> SourceView:
+        """Create a git source.
+
+        Args:
+            uri: The URI of the git repository.
+
+        Raises:
+            ValueError: If the repository cloning fails.
+
+        """
+        # Check if the repository is already added
+        source = await self.repository.get_source_by_uri(uri)
+
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+        else:
+            # Create a unique directory name for the clone
+            clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
+            clone_path.mkdir(parents=True, exist_ok=True)
+
+            try:
+                # Clone the repository
+                git.Repo.clone_from(uri, clone_path)
+            except git.GitCommandError as e:
+                msg = f"Failed to clone repository: {e}"
+                raise ValueError(msg) from e
+
+            source = await self.repository.create_source(
+                Source(uri=uri, cloned_path=str(clone_path)),
+            )
+
+            # Add all files to the source
+            # Count total files for progress bar
+            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
 
-        # Process each file in the source directory
-        for path in tqdm(clone_path.rglob("*"), total=file_count):
-            await self._process_file(source.id, path.absolute())
+            # Process each file in the source directory
+            for path in tqdm(clone_path.rglob("*"), total=file_count):
+                await self._process_file(source.id, path.absolute())
 
         return SourceView(
             id=source.id,

{kodit-0.1.7 → kodit-0.1.9}/tests/kodit/indexing/test_service.py

@@ -88,9 +88,8 @@ async def test_create_index_already_exists(
     # Create first index
     await service.create(source.id)
 
-    # Try to create second index
-    with pytest.raises(IntegrityError, match="UNIQUE constraint failed"):
-        await service.create(source.id)
+    # Try to create second index, should be fine
+    await service.create(source.id)
 
 
 @pytest.mark.asyncio

{kodit-0.1.7 → kodit-0.1.9}/tests/kodit/sources/test_service.py

@@ -2,7 +2,9 @@
 
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
+import shutil
 
+import git
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -55,9 +57,8 @@ async def test_create_source_already_added(
     # Create a folder source
     await service.create(str(test_dir))
 
-    # Try to create the same source again
-    with pytest.raises(ValueError, match=f"Directory already added: {test_dir}"):
-        await service.create(str(test_dir))
+    # Try to create the same source again, should be fine
+    await service.create(str(test_dir))
 
 
 @pytest.mark.asyncio
@@ -108,3 +109,49 @@ async def test_create_source_list_source(
     assert not (cloned_path / ".hidden-file").exists()
     assert (cloned_path / "file1.txt").exists()
     assert (cloned_path / "subdir" / "file2.txt").exists()
+
+
+@pytest.mark.asyncio
+async def test_create_git_source(service: SourceService, tmp_path: Path) -> None:
+    """Test creating a git source."""
+    # Create a temporary git repository
+    repo_path = tmp_path / "test_repo"
+    repo_path.mkdir()
+    repo = git.Repo.init(repo_path)
+
+    # Add some files to the repository
+    (repo_path / "file1.txt").write_text("Hello, world!")
+    (repo_path / "subdir").mkdir()
+    (repo_path / "subdir" / "file2.txt").write_text("Hello, world!")
+
+    # Commit the files
+    repo.index.add(["file1.txt", "subdir/file2.txt"])
+    repo.index.commit("Initial commit")
+
+    # Create a git source
+    source = await service.create(repo_path.as_uri())
+    assert source.id is not None
+    assert source.uri == repo_path.as_uri()
+    assert source.cloned_path.is_dir()
+    assert source.created_at is not None
+    assert source.num_files == 2
+
+    # Check that the files are present in the cloned directory
+    cloned_path = Path(source.cloned_path)
+    assert cloned_path.exists()
+    assert cloned_path.is_dir()
+    assert (cloned_path / "file1.txt").exists()
+    assert (cloned_path / "subdir" / "file2.txt").exists()
+
+    # Clean up
+    shutil.rmtree(repo_path)
+
+
+@pytest.mark.asyncio
+async def test_create_source_relative_path(
+    service: SourceService, tmp_path: Path
+) -> None:
+    """Test creating a source with a relative path, i.e. the current directory."""
+
+    # Should not raise an error
+    await service.create(".")

{kodit-0.1.7 → kodit-0.1.9}/tests/smoke.sh

@@ -26,6 +26,7 @@ $prefix kodit version
 
 # Test index command
 $prefix kodit index $tmp_dir
+$prefix kodit index https://github.com/winderai/analytics-ai-agent-demo
 $prefix kodit index
 
 # Test retrieve command

{kodit-0.1.7 → kodit-0.1.9}/uv.lock

@@ -433,6 +433,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a0/e6/310d1fe6708b7338e1f48915a13d8bf00fd0599acdc7bf98da4fd20fcb66/fastmcp-2.3.4-py3-none-any.whl", hash = "sha256:12a45f72dd95aeaa1a6a56281fff96ca46929def3ccd9f9eb125cb97b722fbab", size = 96393, upload-time = "2025-05-15T00:54:49.714Z" },
 ]
 
+[[package]]
+name = "gitdb"
+version = "4.0.12"
+source = { registry = "https://pypi.org/simple/" }
+dependencies = [
+    { name = "smmap" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" },
+]
+
+[[package]]
+name = "gitpython"
+version = "3.1.44"
+source = { registry = "https://pypi.org/simple/" }
+dependencies = [
+    { name = "gitdb" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/89/37df0b71473153574a5cdef8f242de422a0f5d26d7a9e231e6f169b4ad14/gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269", size = 214196, upload-time = "2025-01-02T07:32:43.59Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110", size = 207599, upload-time = "2025-01-02T07:32:40.731Z" },
+]
+
 [[package]]
 name = "greenlet"
 version = "3.2.2"
@@ -691,6 +715,7 @@ dependencies = [
     { name = "dotenv" },
     { name = "fastapi", extra = ["standard"] },
     { name = "fastmcp" },
+    { name = "gitpython" },
     { name = "httpx" },
     { name = "httpx-retries" },
     { name = "posthog" },
@@ -727,6 +752,7 @@ requires-dist = [
     { name = "dotenv", specifier = ">=0.9.9" },
     { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" },
     { name = "fastmcp", specifier = ">=2.3.3" },
+    { name = "gitpython", specifier = ">=3.1.44" },
     { name = "httpx", specifier = ">=0.28.1" },
     { name = "httpx-retries", specifier = ">=0.3.2" },
     { name = "posthog", specifier = ">=4.0.1" },
@@ -1528,6 +1554,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
 ]
 
+[[package]]
+name = "smmap"
+version = "5.0.2"
+source = { registry = "https://pypi.org/simple/" }
+sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" },
+]
+
 [[package]]
 name = "sniffio"
 version = "1.3.1"