kodit 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.7'
-__version_tuple__ = version_tuple = (0, 1, 7)
+__version__ = version = '0.1.9'
+__version_tuple__ = version_tuple = (0, 1, 9)

kodit/cli.py

@@ -123,12 +123,6 @@ async def index(
         return
     # Handle source indexing
     for source in sources:
-        if source.startswith("https://"):
-            msg = "Web or git indexing is not implemented yet"
-            raise click.UsageError(msg)
-        if source.startswith("git"):
-            msg = "Git indexing is not implemented yet"
-            raise click.UsageError(msg)
         if Path(source).is_file():
             msg = "File indexing is not implemented yet"
             raise click.UsageError(msg)

kodit/indexing/repository.py

@@ -8,7 +8,7 @@ and retrieving index information with their associated metadata.
 from datetime import UTC, datetime
 from typing import TypeVar
 
-from sqlalchemy import func, select
+from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.indexing.models import Index, Snippet
@@ -63,6 +63,17 @@ class IndexRepository:
         result = await self.session.execute(query)
         return result.scalar_one_or_none()
 
+    async def get_by_source_id(self, source_id: int) -> Index | None:
+        """Get an index by its source ID.
+
+        Args:
+            source_id: The ID of the source to retrieve an index for.
+
+        """
+        query = select(Index).where(Index.source_id == source_id)
+        result = await self.session.execute(query)
+        return result.scalar_one_or_none()
+
     async def files_for_index(self, index_id: int) -> list[File]:
         """Get all files for an index.
 
@@ -122,6 +133,17 @@ class IndexRepository:
         self.session.add(snippet)
         await self.session.commit()
 
+    async def delete_all_snippets(self, index_id: int) -> None:
+        """Delete all snippets for an index.
+
+        Args:
+            index_id: The ID of the index to delete snippets for.
+
+        """
+        query = delete(Snippet).where(Snippet.index_id == index_id)
+        await self.session.execute(query)
+        await self.session.commit()
+
     async def get_snippets_for_index(self, index_id: int) -> list[Snippet]:
         """Get all snippets for an index.
 

kodit/indexing/service.py

@@ -83,7 +83,10 @@ class IndexService:
         # Check if the source exists
         source = await self.source_service.get(source_id)
 
-        index = await self.repository.create(source.id)
+        # Check if the index already exists
+        index = await self.repository.get_by_source_id(source.id)
+        if not index:
+            index = await self.repository.create(source.id)
         return IndexView(
             id=index.id,
             created_at=index.created_at,
@@ -119,6 +122,9 @@ class IndexService:
             msg = f"Index not found: {index_id}"
             raise ValueError(msg)
 
+        # First delete all old snippets, if they exist
+        await self.repository.delete_all_snippets(index_id)
+
         # Create snippets for supported file types
         await self._create_snippets(index_id)
 

kodit/sources/service.py

@@ -13,6 +13,7 @@ from hashlib import sha256
 from pathlib import Path
 
 import aiofiles
+import git
 import pydantic
 import structlog
 from tqdm import tqdm
@@ -98,8 +99,19 @@ class SourceService:
             parsed = urisplit(uri_or_path_like)
             if parsed.scheme == "file":
                 return await self._create_folder_source(Path(parsed.path))
-            msg = f"Unsupported source type: {uri_or_path_like}"
-            raise ValueError(msg)
+            if parsed.scheme in ("git", "http", "https") and parsed.path.endswith(
+                ".git"
+            ):
+                return await self._create_git_source(uri_or_path_like)
+
+            # Try adding a .git suffix, sometimes people just pass the url
+            if not uri_or_path_like.endswith(".git"):
+                uri_or_path_like = uri_or_path_like + ".git"
+                try:
+                    return await self._create_git_source(uri_or_path_like)
+                except ValueError:
+                    pass
+
         msg = f"Unsupported source type: {uri_or_path_like}"
         raise ValueError(msg)
 
@@ -110,46 +122,98 @@ class SourceService:
             directory: The path to the local directory.
 
         Raises:
-            ValueError: If the folder doesn't exist or is already added.
+            ValueError: If the folder doesn't exist.
+            SourceAlreadyExistsError: If the folder is already added.
 
         """
         # Resolve the directory to an absolute path
         directory = directory.expanduser().resolve()
 
-        # Check if the folder exists
-        if not directory.exists():
-            msg = f"Folder does not exist: {directory}"
-            raise ValueError(msg)
+        source = await self.repository.get_source_by_uri(directory.as_uri())
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+        else:
+            # Check if the folder exists
+            if not directory.exists():
+                msg = f"Folder does not exist: {directory}"
+                raise ValueError(msg)
+
+            # Check if the folder is already added
+            if await self.repository.get_source_by_uri(directory.as_uri()):
+                msg = f"Directory already added: {directory}"
+                raise ValueError(msg)
+
+            # Clone into a local directory
+            clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
+            clone_path.mkdir(parents=True, exist_ok=True)
+
+            # Copy all files recursively, preserving directory structure, ignoring
+            # hidden files
+            shutil.copytree(
+                directory,
+                clone_path,
+                ignore=shutil.ignore_patterns(".*"),
+                dirs_exist_ok=True,
+            )
 
-        # Check if the folder is already added
-        if await self.repository.get_source_by_uri(directory.as_uri()):
-            msg = f"Directory already added: {directory}"
-            raise ValueError(msg)
+            source = await self.repository.create_source(
+                Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+            )
 
-        # Clone into a local directory
-        clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
-        clone_path.mkdir(parents=True, exist_ok=True)
-
-        # Copy all files recursively, preserving directory structure, ignoring hidden
-        # files
-        shutil.copytree(
-            directory,
-            clone_path,
-            ignore=shutil.ignore_patterns(".*"),
-            dirs_exist_ok=True,
-        )
+            # Add all files to the source
+            # Count total files for progress bar
+            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
+
+            # Process each file in the source directory
+            for path in tqdm(clone_path.rglob("*"), total=file_count):
+                await self._process_file(source.id, path.absolute())
 
-        source = await self.repository.create_source(
-            Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+        return SourceView(
+            id=source.id,
+            uri=source.uri,
+            cloned_path=Path(source.cloned_path),
+            created_at=source.created_at,
+            num_files=await self.repository.num_files_for_source(source.id),
         )
 
-        # Add all files to the source
-        # Count total files for progress bar
-        file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
+    async def _create_git_source(self, uri: str) -> SourceView:
+        """Create a git source.
+
+        Args:
+            uri: The URI of the git repository.
+
+        Raises:
+            ValueError: If the repository cloning fails.
+
+        """
+        # Check if the repository is already added
+        source = await self.repository.get_source_by_uri(uri)
+
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+        else:
+            # Create a unique directory name for the clone
+            clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
+            clone_path.mkdir(parents=True, exist_ok=True)
+
+            try:
+                # Clone the repository
+                git.Repo.clone_from(uri, clone_path)
+            except git.GitCommandError as e:
+                msg = f"Failed to clone repository: {e}"
+                raise ValueError(msg) from e
+
+            source = await self.repository.create_source(
+                Source(uri=uri, cloned_path=str(clone_path)),
+            )
+
+            # Add all files to the source
+            # Count total files for progress bar
+            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
 
-        # Process each file in the source directory
-        for path in tqdm(clone_path.rglob("*"), total=file_count):
-            await self._process_file(source.id, path.absolute())
+            # Process each file in the source directory
+            for path in tqdm(clone_path.rglob("*"), total=file_count):
+                await self._process_file(source.id, path.absolute())
 
         return SourceView(
             id=source.id,

{kodit-0.1.7.dist-info → kodit-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.1.7
+Version: 0.1.9
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/
@@ -28,6 +28,7 @@ Requires-Dist: colorama>=0.4.6
 Requires-Dist: dotenv>=0.9.9
 Requires-Dist: fastapi[standard]>=0.115.12
 Requires-Dist: fastmcp>=2.3.3
+Requires-Dist: gitpython>=3.1.44
 Requires-Dist: httpx-retries>=0.3.2
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: posthog>=4.0.1

{kodit-0.1.7.dist-info → kodit-0.1.9.dist-info}/RECORD

@@ -1,8 +1,8 @@
 kodit/.gitignore,sha256=ztkjgRwL9Uud1OEi36hGQeDGk3OLK1NfDEO8YqGYy8o,11
 kodit/__init__.py,sha256=aEKHYninUq1yh6jaNfvJBYg-6fenpN132nJt1UU6Jxs,59
-kodit/_version.py,sha256=W_EoL8cAL4KhujvbYWEpb9NqRLbbrH0T024lJvRRWHI,511
+kodit/_version.py,sha256=bhntibG3PKk5Ai3XlSNEV8gj-ffItuKloY6vzWn6swo,511
 kodit/app.py,sha256=Mr5BFHOHx5zppwjC4XPWVvHjwgl1yrKbUjTWXKubJQM,891
-kodit/cli.py,sha256=x1zw2zOlGhhU6D3E-GU3cMw3l9CqKC76geQREAQKweY,6915
+kodit/cli.py,sha256=bsfURvGKZzpHkChnTlatI0nXHV3KV_6vJnUJ2fQEAfM,6637
 kodit/config.py,sha256=nlm9U-nVx5riH2SrU1XY4XcCMhQK4DrwO_1H8bPOBjA,2927
 kodit/database.py,sha256=vtTlmrXHyHJH3Ek-twZTCqEjB0jun-NncALFze2fqhA,2350
 kodit/logging.py,sha256=cFEQXWI27LzWScSxly9ApwkbBDamUG17pA-jEfVakXQ,5316
@@ -18,8 +18,8 @@ kodit/bm25/__init__.py,sha256=j8zyriNWhbwE5Lbybzg1hQAhANlU9mKHWw4beeUR6og,19
 kodit/bm25/bm25.py,sha256=3wyNRSrTaYqV7s4R1D6X0NpCf22PuFK2_uc8YapzYLE,2263
 kodit/indexing/__init__.py,sha256=cPyi2Iej3G1JFWlWr7X80_UrsMaTu5W5rBwgif1B3xo,75
 kodit/indexing/models.py,sha256=sZIhGwvL4Dw0QTWFxrjfWctSLkAoDT6fv5DlGz8-Fr8,1258
-kodit/indexing/repository.py,sha256=C020FGpIfTZmVZg7NH04kVuffWv7r7m-82Pdex8CItg,4388
-kodit/indexing/service.py,sha256=7vHqevve-PnKHP2pDfyrW5n3AXVOghABWNsNTw588KY,5499
+kodit/indexing/repository.py,sha256=ZicLPXPKQxW6NnY_anmZ4nI1-FGkrJsqjg0NK-vvnTY,5117
+kodit/indexing/service.py,sha256=rLWYI70VytlJAyZtQC5Xpqtj9f3EzbivzgeM_1L9BUU,5751
 kodit/retreival/__init__.py,sha256=33PhJU-3gtsqYq6A1UkaLNKbev_Zee9Lq6dYC59-CsA,69
 kodit/retreival/repository.py,sha256=1lqGgJHsBmvMGMzEYa-hrdXg2q7rqtYPl1cvBb7jMRE,3119
 kodit/retreival/service.py,sha256=9wvURtPPJVvPUWNIC2waIrJMxcm1Ka1J_xDEOEedAFU,2007
@@ -32,9 +32,9 @@ kodit/snippets/languages/python.scm,sha256=ee85R9PBzwye3IMTE7-iVoKWd_ViU3EJISTyr
 kodit/sources/__init__.py,sha256=1NTZyPdjThVQpZO1Mp1ColVsS7sqYanOVLqnoqV9Ipo,83
 kodit/sources/models.py,sha256=xb42CaNDO1CUB8SIW-xXMrB6Ji8cFw-yeJ550xBEg9Q,2398
 kodit/sources/repository.py,sha256=mGJrHWH6Uo8YABdoojHFbzaf_jW-2ywJpAHIa1gnc3U,3401
-kodit/sources/service.py,sha256=cBCxnOQKwGNi2e13_3Vue8MylAaUxb9XG4IgM636la0,6712
-kodit-0.1.7.dist-info/METADATA,sha256=8doJ6TfmVkn-OTLSbgRlDut6YxFbmBKs1L0rgZUlxUQ,2181
-kodit-0.1.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kodit-0.1.7.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
-kodit-0.1.7.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kodit-0.1.7.dist-info/RECORD,,
+kodit/sources/service.py,sha256=hqAjGFVhvtePhMrK1Aprj__Mq2PLjVq8CsWMBoA3_Qw,9217
+kodit-0.1.9.dist-info/METADATA,sha256=MAqVxrLPrTV3Ihcix_3YHQNq9qyuD1OEavYHV76qli8,2214
+kodit-0.1.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kodit-0.1.9.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
+kodit-0.1.9.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kodit-0.1.9.dist-info/RECORD,,