kodit 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.6'
-__version_tuple__ = version_tuple = (0, 1, 6)
+__version__ = version = '0.1.8'
+__version_tuple__ = version_tuple = (0, 1, 8)

kodit/cli.py

@@ -8,7 +8,7 @@ from typing import Any
 import click
 import structlog
 import uvicorn
-from pytable_formatter import Table
+from pytable_formatter import Cell, Table
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.config import (
@@ -84,110 +84,53 @@ def cli(  # noqa: PLR0913
     ctx.obj = config
 
 
-@cli.group()
-def sources() -> None:
-    """Manage code sources."""
-
-
-@sources.command(name="list")
-@with_app_context
-@with_session
-async def list_sources(session: AsyncSession, app_context: AppContext) -> None:
-    """List all code sources."""
-    repository = SourceRepository(session)
-    service = SourceService(app_context.get_clone_dir(), repository)
-    sources = await service.list_sources()
-
-    # Define headers and data
-    headers = ["ID", "Created At", "URI"]
-    data = [[source.id, source.created_at, source.uri] for source in sources]
-
-    # Create and display the table
-    table = Table(headers=headers, data=data)
-    click.echo(table)
-
-
-@sources.command(name="create")
-@click.argument("uri")
-@with_app_context
-@with_session
-async def create_source(
-    session: AsyncSession, app_context: AppContext, uri: str
-) -> None:
-    """Add a new code source."""
-    repository = SourceRepository(session)
-    service = SourceService(app_context.get_clone_dir(), repository)
-    source = await service.create(uri)
-    click.echo(f"Source created: {source.id}")
-
-
-@cli.group()
-def indexes() -> None:
-    """Manage indexes."""
-
-
-@indexes.command(name="create")
-@click.argument("source_id")
+@cli.command()
+@click.argument("sources", nargs=-1)
 @with_app_context
 @with_session
-async def create_index(
-    session: AsyncSession, app_context: AppContext, source_id: int
+async def index(
+    session: AsyncSession,
+    app_context: AppContext,
+    sources: list[str],
 ) -> None:
-    """Create an index for a source."""
+    """List indexes, or index data sources."""
     source_repository = SourceRepository(session)
     source_service = SourceService(app_context.get_clone_dir(), source_repository)
     repository = IndexRepository(session)
     service = IndexService(repository, source_service, app_context.get_data_dir())
-    index = await service.create(source_id)
-    click.echo(f"Index created: {index.id}")
 
-
-@indexes.command(name="list")
-@with_app_context
-@with_session
-async def list_indexes(session: AsyncSession, app_context: AppContext) -> None:
-    """List all indexes."""
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(repository, source_service, app_context.get_data_dir())
-    indexes = await service.list_indexes()
-
-    # Define headers and data
-    headers = [
-        "ID",
-        "Created At",
-        "Updated At",
-        "Num Snippets",
-    ]
-    data = [
-        [
-            index.id,
-            index.created_at,
-            index.updated_at,
-            index.num_snippets,
+    if not sources:
+        # No source specified, list all indexes
+        indexes = await service.list_indexes()
+        headers: list[str | Cell] = [
+            "ID",
+            "Created At",
+            "Updated At",
+            "Source",
+            "Num Snippets",
         ]
-        for index in indexes
-    ]
-
-    # Create and display the table
-    table = Table(headers=headers, data=data)
-    click.echo(table)
-
-
-@indexes.command(name="run")
-@click.argument("index_id")
-@with_app_context
-@with_session
-async def run_index(
-    session: AsyncSession, app_context: AppContext, index_id: int
-) -> None:
-    """Run an index."""
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(repository, source_service, app_context.get_data_dir())
-    await service.run(index_id)
+        data = [
+            [
+                index.id,
+                index.created_at,
+                index.updated_at,
+                index.source,
+                index.num_snippets,
+            ]
+            for index in indexes
+        ]
+        click.echo(Table(headers=headers, data=data))
+        return
+    # Handle source indexing
+    for source in sources:
+        if Path(source).is_file():
+            msg = "File indexing is not implemented yet"
+            raise click.UsageError(msg)
+
+        # Index directory
+        s = await source_service.create(source)
+        index = await service.create(s.id)
+        await service.run(index.id)
 
 
 @cli.command()

kodit/indexing/repository.py

@@ -8,7 +8,7 @@ and retrieving index information with their associated metadata.
 from datetime import UTC, datetime
 from typing import TypeVar
 
-from sqlalchemy import func, select
+from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.indexing.models import Index, Snippet
@@ -63,6 +63,17 @@ class IndexRepository:
         result = await self.session.execute(query)
         return result.scalar_one_or_none()
 
+    async def get_by_source_id(self, source_id: int) -> Index | None:
+        """Get an index by its source ID.
+
+        Args:
+            source_id: The ID of the source to retrieve an index for.
+
+        """
+        query = select(Index).where(Index.source_id == source_id)
+        result = await self.session.execute(query)
+        return result.scalar_one_or_none()
+
     async def files_for_index(self, index_id: int) -> list[File]:
         """Get all files for an index.
 
@@ -82,7 +93,7 @@ class IndexRepository:
         result = await self.session.execute(query)
         return list(result.scalars())
 
-    async def list_indexes(self) -> list[Index]:
+    async def list_indexes(self) -> list[tuple[Index, Source]]:
         """List all indexes.
 
         Returns:
@@ -90,9 +101,11 @@ class IndexRepository:
             and counts of files and snippets.
 
         """
-        query = select(Index).limit(10)
+        query = select(Index, Source).join(
+            Source, Index.source_id == Source.id, full=True
+        )
         result = await self.session.execute(query)
-        return list(result.scalars())
+        return list(result.tuples())
 
     async def num_snippets_for_index(self, index_id: int) -> int:
         """Get the number of snippets for an index."""
@@ -120,6 +133,17 @@ class IndexRepository:
         self.session.add(snippet)
         await self.session.commit()
 
+    async def delete_all_snippets(self, index_id: int) -> None:
+        """Delete all snippets for an index.
+
+        Args:
+            index_id: The ID of the index to delete snippets for.
+
+        """
+        query = delete(Snippet).where(Snippet.index_id == index_id)
+        await self.session.execute(query)
+        await self.session.commit()
+
     async def get_snippets_for_index(self, index_id: int) -> list[Snippet]:
         """Get all snippets for an index.
 

kodit/indexing/service.py

@@ -33,6 +33,7 @@ class IndexView(pydantic.BaseModel):
     id: int
     created_at: datetime
     updated_at: datetime | None = None
+    source: str | None = None
     num_snippets: int | None = None
 
 
@@ -82,7 +83,10 @@ class IndexService:
         # Check if the source exists
         source = await self.source_service.get(source_id)
 
-        index = await self.repository.create(source.id)
+        # Check if the index already exists
+        index = await self.repository.get_by_source_id(source.id)
+        if not index:
+            index = await self.repository.create(source.id)
         return IndexView(
             id=index.id,
             created_at=index.created_at,
@@ -105,8 +109,9 @@ class IndexService:
                 created_at=index.created_at,
                 updated_at=index.updated_at,
                 num_snippets=await self.repository.num_snippets_for_index(index.id),
+                source=source.uri,
             )
-            for index in indexes
+            for index, source in indexes
         ]
 
     async def run(self, index_id: int) -> None:
@@ -117,6 +122,9 @@ class IndexService:
             msg = f"Index not found: {index_id}"
             raise ValueError(msg)
 
+        # First delete all old snippets, if they exist
+        await self.repository.delete_all_snippets(index_id)
+
         # Create snippets for supported file types
         await self._create_snippets(index_id)
 

kodit/sources/service.py

@@ -13,6 +13,7 @@ from hashlib import sha256
 from pathlib import Path
 
 import aiofiles
+import git
 import pydantic
 import structlog
 from tqdm import tqdm
@@ -98,8 +99,19 @@ class SourceService:
             parsed = urisplit(uri_or_path_like)
             if parsed.scheme == "file":
                 return await self._create_folder_source(Path(parsed.path))
-            msg = f"Unsupported source type: {uri_or_path_like}"
-            raise ValueError(msg)
+            if parsed.scheme in ("git", "http", "https") and parsed.path.endswith(
+                ".git"
+            ):
+                return await self._create_git_source(uri_or_path_like)
+
+            # Try adding a .git suffix, sometimes people just pass the url
+            if not uri_or_path_like.endswith(".git"):
+                uri_or_path_like = uri_or_path_like + ".git"
+                try:
+                    return await self._create_git_source(uri_or_path_like)
+                except ValueError:
+                    pass
+
         msg = f"Unsupported source type: {uri_or_path_like}"
         raise ValueError(msg)
 
@@ -110,46 +122,98 @@ class SourceService:
             directory: The path to the local directory.
 
         Raises:
-            ValueError: If the folder doesn't exist or is already added.
+            ValueError: If the folder doesn't exist.
+            SourceAlreadyExistsError: If the folder is already added.
 
         """
-        # Resolve the directory to an absolute path
-        directory = directory.expanduser().resolve()
+        source = await self.repository.get_source_by_uri(directory.as_uri())
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+        else:
+            # Resolve the directory to an absolute path
+            directory = directory.expanduser().resolve()
+
+            # Check if the folder exists
+            if not directory.exists():
+                msg = f"Folder does not exist: {directory}"
+                raise ValueError(msg)
+
+            # Check if the folder is already added
+            if await self.repository.get_source_by_uri(directory.as_uri()):
+                msg = f"Directory already added: {directory}"
+                raise ValueError(msg)
+
+            # Clone into a local directory
+            clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
+            clone_path.mkdir(parents=True, exist_ok=True)
+
+            # Copy all files recursively, preserving directory structure, ignoring
+            # hidden files
+            shutil.copytree(
+                directory,
+                clone_path,
+                ignore=shutil.ignore_patterns(".*"),
+                dirs_exist_ok=True,
+            )
 
-        # Check if the folder exists
-        if not directory.exists():
-            msg = f"Folder does not exist: {directory}"
-            raise ValueError(msg)
+            source = await self.repository.create_source(
+                Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+            )
 
-        # Check if the folder is already added
-        if await self.repository.get_source_by_uri(directory.as_uri()):
-            msg = f"Directory already added: {directory}"
-            raise ValueError(msg)
+            # Add all files to the source
+            # Count total files for progress bar
+            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
 
-        # Clone into a local directory
-        clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
-        clone_path.mkdir(parents=True, exist_ok=True)
-
-        # Copy all files recursively, preserving directory structure, ignoring hidden
-        # files
-        shutil.copytree(
-            directory,
-            clone_path,
-            ignore=shutil.ignore_patterns(".*"),
-            dirs_exist_ok=True,
-        )
+            # Process each file in the source directory
+            for path in tqdm(clone_path.rglob("*"), total=file_count):
+                await self._process_file(source.id, path.absolute())
 
-        source = await self.repository.create_source(
-            Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+        return SourceView(
+            id=source.id,
+            uri=source.uri,
+            cloned_path=Path(source.cloned_path),
+            created_at=source.created_at,
+            num_files=await self.repository.num_files_for_source(source.id),
         )
 
-        # Add all files to the source
-        # Count total files for progress bar
-        file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
+    async def _create_git_source(self, uri: str) -> SourceView:
+        """Create a git source.
+
+        Args:
+            uri: The URI of the git repository.
+
+        Raises:
+            ValueError: If the repository cloning fails.
+
+        """
+        # Check if the repository is already added
+        source = await self.repository.get_source_by_uri(uri)
+
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+        else:
+            # Create a unique directory name for the clone
+            clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
+            clone_path.mkdir(parents=True, exist_ok=True)
+
+            try:
+                # Clone the repository
+                git.Repo.clone_from(uri, clone_path)
+            except git.GitCommandError as e:
+                msg = f"Failed to clone repository: {e}"
+                raise ValueError(msg) from e
+
+            source = await self.repository.create_source(
+                Source(uri=uri, cloned_path=str(clone_path)),
+            )
+
+            # Add all files to the source
+            # Count total files for progress bar
+            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
 
-        # Process each file in the source directory
-        for path in tqdm(clone_path.rglob("*"), total=file_count):
-            await self._process_file(source.id, path.absolute())
+            # Process each file in the source directory
+            for path in tqdm(clone_path.rglob("*"), total=file_count):
+                await self._process_file(source.id, path.absolute())
 
         return SourceView(
             id=source.id,

{kodit-0.1.6.dist-info → kodit-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.1.6
+Version: 0.1.8
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/
@@ -28,6 +28,7 @@ Requires-Dist: colorama>=0.4.6
 Requires-Dist: dotenv>=0.9.9
 Requires-Dist: fastapi[standard]>=0.115.12
 Requires-Dist: fastmcp>=2.3.3
+Requires-Dist: gitpython>=3.1.44
 Requires-Dist: httpx-retries>=0.3.2
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: posthog>=4.0.1

{kodit-0.1.6.dist-info → kodit-0.1.8.dist-info}/RECORD

@@ -1,8 +1,8 @@
 kodit/.gitignore,sha256=ztkjgRwL9Uud1OEi36hGQeDGk3OLK1NfDEO8YqGYy8o,11
 kodit/__init__.py,sha256=aEKHYninUq1yh6jaNfvJBYg-6fenpN132nJt1UU6Jxs,59
-kodit/_version.py,sha256=ESbJO0YD7TYfOUv_WDIJJgWELGepEWsoyhqVifEcXPA,511
+kodit/_version.py,sha256=AjUi5zEL_BoWoXMXR1FnWc3mD6FHX7snDXjDHVLoens,511
 kodit/app.py,sha256=Mr5BFHOHx5zppwjC4XPWVvHjwgl1yrKbUjTWXKubJQM,891
-kodit/cli.py,sha256=CNewAn4aCDBbBRKfTaO5hosklCySvJM7afDN04h3lvY,8307
+kodit/cli.py,sha256=bsfURvGKZzpHkChnTlatI0nXHV3KV_6vJnUJ2fQEAfM,6637
 kodit/config.py,sha256=nlm9U-nVx5riH2SrU1XY4XcCMhQK4DrwO_1H8bPOBjA,2927
 kodit/database.py,sha256=vtTlmrXHyHJH3Ek-twZTCqEjB0jun-NncALFze2fqhA,2350
 kodit/logging.py,sha256=cFEQXWI27LzWScSxly9ApwkbBDamUG17pA-jEfVakXQ,5316
@@ -18,8 +18,8 @@ kodit/bm25/__init__.py,sha256=j8zyriNWhbwE5Lbybzg1hQAhANlU9mKHWw4beeUR6og,19
 kodit/bm25/bm25.py,sha256=3wyNRSrTaYqV7s4R1D6X0NpCf22PuFK2_uc8YapzYLE,2263
 kodit/indexing/__init__.py,sha256=cPyi2Iej3G1JFWlWr7X80_UrsMaTu5W5rBwgif1B3xo,75
 kodit/indexing/models.py,sha256=sZIhGwvL4Dw0QTWFxrjfWctSLkAoDT6fv5DlGz8-Fr8,1258
-kodit/indexing/repository.py,sha256=kvAlNfMSQYboF0TB1huw2qoBdLJ4UsEPiM7ZG-e6rrg,4300
-kodit/indexing/service.py,sha256=eopx_IZeaUjCI-5LeSqZq7W7m76JZsDcVvOFhLquHpI,5426
+kodit/indexing/repository.py,sha256=ZicLPXPKQxW6NnY_anmZ4nI1-FGkrJsqjg0NK-vvnTY,5117
+kodit/indexing/service.py,sha256=rLWYI70VytlJAyZtQC5Xpqtj9f3EzbivzgeM_1L9BUU,5751
 kodit/retreival/__init__.py,sha256=33PhJU-3gtsqYq6A1UkaLNKbev_Zee9Lq6dYC59-CsA,69
 kodit/retreival/repository.py,sha256=1lqGgJHsBmvMGMzEYa-hrdXg2q7rqtYPl1cvBb7jMRE,3119
 kodit/retreival/service.py,sha256=9wvURtPPJVvPUWNIC2waIrJMxcm1Ka1J_xDEOEedAFU,2007
@@ -32,9 +32,9 @@ kodit/snippets/languages/python.scm,sha256=ee85R9PBzwye3IMTE7-iVoKWd_ViU3EJISTyr
 kodit/sources/__init__.py,sha256=1NTZyPdjThVQpZO1Mp1ColVsS7sqYanOVLqnoqV9Ipo,83
 kodit/sources/models.py,sha256=xb42CaNDO1CUB8SIW-xXMrB6Ji8cFw-yeJ550xBEg9Q,2398
 kodit/sources/repository.py,sha256=mGJrHWH6Uo8YABdoojHFbzaf_jW-2ywJpAHIa1gnc3U,3401
-kodit/sources/service.py,sha256=cBCxnOQKwGNi2e13_3Vue8MylAaUxb9XG4IgM636la0,6712
-kodit-0.1.6.dist-info/METADATA,sha256=1HHbQJLVCV6-1Kim8-_pwXq_HmSYUgBLObA_Q3Ck2l8,2181
-kodit-0.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kodit-0.1.6.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
-kodit-0.1.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kodit-0.1.6.dist-info/RECORD,,
+kodit/sources/service.py,sha256=aEW43wpYr3AJvXRuYzCHCNqNWUaQSJy8lqr5TYlGIRs,9225
+kodit-0.1.8.dist-info/METADATA,sha256=2TlteBGPvV-oVpOWibx6PzgpzTOfMyCWyLpyBuMpgzs,2214
+kodit-0.1.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kodit-0.1.8.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
+kodit-0.1.8.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kodit-0.1.8.dist-info/RECORD,,