kodit 0.0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.1'
-__version_tuple__ = version_tuple = (0, 0, 1)
+__version__ = version = '0.1.1'
+__version_tuple__ = version_tuple = (0, 1, 1)

kodit/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration with an async dbapi.

kodit/alembic/__init__.py

@@ -0,0 +1 @@
+"""Database configuration for kodit."""

kodit/alembic/env.py

@@ -0,0 +1,83 @@
+# ruff: noqa: F401
+"""Alembic environment file for kodit."""
+
+import asyncio
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+import kodit.indexing.models
+import kodit.sources.models
+from kodit.database import Base
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+# We skip this to preserve the existing logging configuration
+# if config.config_file_name is not None:
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    """Run migrations in 'online' mode."""
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    """Run migrations in 'async' mode."""
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

kodit/alembic/script.py.mako

@@ -0,0 +1,30 @@
+# ruff: noqa
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

kodit/alembic/versions/85155663351e_initial.py

@@ -0,0 +1,82 @@
+# ruff: noqa
+"""initial
+
+Revision ID: 85155663351e
+Revises: 
+Create Date: 2025-05-08 13:45:16.687162
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '85155663351e'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('sources',
+    sa.Column('uri', sa.String(length=1024), nullable=False),
+    sa.Column('cloned_path', sa.String(length=1024), nullable=False),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_sources_uri'), 'sources', ['uri'], unique=True)
+    op.create_table('files',
+    sa.Column('source_id', sa.Integer(), nullable=False),
+    sa.Column('mime_type', sa.String(length=255), nullable=False),
+    sa.Column('uri', sa.String(length=1024), nullable=False),
+    sa.Column('cloned_path', sa.String(length=1024), nullable=False),
+    sa.Column('sha256', sa.String(length=64), nullable=False),
+    sa.Column('size_bytes', sa.Integer(), nullable=False),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_files_sha256'), 'files', ['sha256'], unique=False)
+    op.create_table('indexes',
+    sa.Column('source_id', sa.Integer(), nullable=False),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_indexes_source_id'), 'indexes', ['source_id'], unique=True)
+    op.create_table('snippets',
+    sa.Column('file_id', sa.Integer(), nullable=False),
+    sa.Column('index_id', sa.Integer(), nullable=False),
+    sa.Column('content', sa.UnicodeText(), nullable=False),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['file_id'], ['files.id'], ),
+    sa.ForeignKeyConstraint(['index_id'], ['indexes.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('snippets')
+    op.drop_index(op.f('ix_indexes_source_id'), table_name='indexes')
+    op.drop_table('indexes')
+    op.drop_index(op.f('ix_files_sha256'), table_name='files')
+    op.drop_table('files')
+    op.drop_index(op.f('ix_sources_uri'), table_name='sources')
+    op.drop_table('sources')
+    # ### end Alembic commands ###

kodit/alembic/versions/__init__.py

@@ -0,0 +1 @@
+"""Alembic migrations for kodit."""

kodit/cli.py

@@ -6,8 +6,17 @@ import click
 import structlog
 import uvicorn
 from dotenv import dotenv_values
+from pytable_formatter import Table
+from sqlalchemy.ext.asyncio import AsyncSession
 
+from kodit.database import configure_database, with_session
+from kodit.indexing.repository import IndexRepository
+from kodit.indexing.service import IndexService
 from kodit.logging import LogFormat, configure_logging, disable_posthog, log_event
+from kodit.retreival.repository import RetrievalRepository
+from kodit.retreival.service import RetrievalRequest, RetrievalService
+from kodit.sources.repository import SourceRepository
+from kodit.sources.service import SourceService
 
 env_vars = dict(dotenv_values())
 os.environ.update(env_vars)
@@ -26,6 +35,119 @@ def cli(
     configure_logging(log_level, log_format)
     if disable_telemetry:
         disable_posthog()
+    configure_database()
+
+
+@cli.group()
+def sources() -> None:
+    """Manage code sources."""
+
+
+@sources.command(name="list")
+@with_session
+async def list_sources(session: AsyncSession) -> None:
+    """List all code sources."""
+    repository = SourceRepository(session)
+    service = SourceService(repository)
+    sources = await service.list_sources()
+
+    # Define headers and data
+    headers = ["ID", "Created At", "URI"]
+    data = [[source.id, source.created_at, source.uri] for source in sources]
+
+    # Create and display the table
+    table = Table(headers=headers, data=data)
+    click.echo(table)
+
+
+@sources.command(name="create")
+@click.argument("uri")
+@with_session
+async def create_source(session: AsyncSession, uri: str) -> None:
+    """Add a new code source."""
+    repository = SourceRepository(session)
+    service = SourceService(repository)
+    source = await service.create(uri)
+    click.echo(f"Source created: {source.id}")
+
+
+@cli.group()
+def indexes() -> None:
+    """Manage indexes."""
+
+
+@indexes.command(name="create")
+@click.argument("source_id")
+@with_session
+async def create_index(session: AsyncSession, source_id: int) -> None:
+    """Create an index for a source."""
+    source_repository = SourceRepository(session)
+    source_service = SourceService(source_repository)
+    repository = IndexRepository(session)
+    service = IndexService(repository, source_service)
+    index = await service.create(source_id)
+    click.echo(f"Index created: {index.id}")
+
+
+@indexes.command(name="list")
+@with_session
+async def list_indexes(session: AsyncSession) -> None:
+    """List all indexes."""
+    source_repository = SourceRepository(session)
+    source_service = SourceService(source_repository)
+    repository = IndexRepository(session)
+    service = IndexService(repository, source_service)
+    indexes = await service.list_indexes()
+
+    # Define headers and data
+    headers = [
+        "ID",
+        "Created At",
+        "Updated At",
+        "Source URI",
+        "Num Snippets",
+    ]
+    data = [
+        [
+            index.id,
+            index.created_at,
+            index.updated_at,
+            index.source_uri,
+            index.num_snippets,
+        ]
+        for index in indexes
+    ]
+
+    # Create and display the table
+    table = Table(headers=headers, data=data)
+    click.echo(table)
+
+
+@indexes.command(name="run")
+@click.argument("index_id")
+@with_session
+async def run_index(session: AsyncSession, index_id: int) -> None:
+    """Run an index."""
+    source_repository = SourceRepository(session)
+    source_service = SourceService(source_repository)
+    repository = IndexRepository(session)
+    service = IndexService(repository, source_service)
+    await service.run(index_id)
+
+
+@cli.command()
+@click.argument("query")
+@with_session
+async def retrieve(session: AsyncSession, query: str) -> None:
+    """Retrieve snippets from the database."""
+    repository = RetrievalRepository(session)
+    service = RetrievalService(repository)
+    snippets = await service.retrieve(RetrievalRequest(query=query))
+
+    for snippet in snippets:
+        click.echo(f"{snippet.uri}")
+        click.echo(snippet.content)
+        click.echo()
 
 
 @cli.command()

kodit/database.py

@@ -0,0 +1,85 @@
+"""Database configuration for kodit."""
+
+import asyncio
+from collections.abc import AsyncGenerator, Callable
+from datetime import UTC, datetime
+from functools import wraps
+from pathlib import Path
+from typing import Any, TypeVar
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import DateTime
+from sqlalchemy.ext.asyncio import (
+    AsyncAttrs,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+# Constants
+DATA_DIR = Path.cwd() / ".kodit"
+DB_URL = f"sqlite+aiosqlite:///{DATA_DIR}/kodit.db"
+
+# Create data directory if it doesn't exist
+DATA_DIR.mkdir(exist_ok=True)
+
+# Create async engine with file-based SQLite
+engine = create_async_engine(DB_URL, echo=False)
+
+# Create async session factory
+async_session_factory = async_sessionmaker(
+    engine,
+    class_=AsyncSession,
+    expire_on_commit=False,
+)
+
+
+class Base(AsyncAttrs, DeclarativeBase):
+    """Base class for all models."""
+
+
+class CommonMixin:
+    """Common mixin for all models."""
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime, default=lambda: datetime.now(UTC)
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
+    )
+
+
+async def get_session() -> AsyncGenerator[AsyncSession, None]:
+    """Get a database session."""
+    async with async_session_factory() as session:
+        try:
+            yield session
+        finally:
+            await session.close()
+
+
+T = TypeVar("T")
+
+
+def with_session(func: Callable[..., T]) -> Callable[..., T]:
+    """Provide an async session to CLI commands."""
+
+    @wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> T:
+        async def _run() -> T:
+            async with async_session_factory() as session:
+                return await func(session, *args, **kwargs)
+
+        return asyncio.run(_run())
+
+    return wrapper
+
+
+def configure_database() -> None:
+    """Configure the database by initializing it and running any pending migrations."""
+    # Create Alembic configuration and run migrations
+    alembic_cfg = Config("alembic.ini")
+    command.upgrade(alembic_cfg, "head")

kodit/indexing/__init__.py

@@ -0,0 +1 @@
+"""Indexing package for managing code indexes and search functionality."""

kodit/indexing/models.py

@@ -0,0 +1,43 @@
+"""Index models for managing code indexes.
+
+This module defines the SQLAlchemy models used for storing and managing code indexes,
+including files and snippets. It provides the data structures for tracking indexed
+files and their content.
+"""
+
+from sqlalchemy import ForeignKey, UnicodeText
+from sqlalchemy.orm import Mapped, mapped_column
+
+from kodit.database import Base, CommonMixin
+
+
+class Index(Base, CommonMixin):
+    """Index model."""
+
+    __tablename__ = "indexes"
+
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("sources.id"), unique=True, index=True
+    )
+
+    def __init__(self, source_id: int) -> None:
+        """Initialize the index."""
+        super().__init__()
+        self.source_id = source_id
+
+
+class Snippet(Base, CommonMixin):
+    """Snippet model."""
+
+    __tablename__ = "snippets"
+
+    file_id: Mapped[int] = mapped_column(ForeignKey("files.id"))
+    index_id: Mapped[int] = mapped_column(ForeignKey("indexes.id"))
+    content: Mapped[str] = mapped_column(UnicodeText, default="")
+
+    def __init__(self, file_id: int, index_id: int, content: str) -> None:
+        """Initialize the snippet."""
+        super().__init__()
+        self.file_id = file_id
+        self.index_id = index_id
+        self.content = content

kodit/indexing/repository.py

@@ -0,0 +1,132 @@
+"""Repository for managing code indexes and their associated files and snippets.
+
+This module provides the IndexRepository class which handles all database operations
+related to code indexes, including creating indexes, managing files and snippets,
+and retrieving index information with their associated metadata.
+"""
+
+from datetime import UTC, datetime
+from typing import TypeVar
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.indexing.models import Index, Snippet
+from kodit.sources.models import File, Source
+
+T = TypeVar("T")
+
+
+class IndexRepository:
+    """Repository for managing code indexes and their associated data.
+
+    This class provides methods for creating and managing code indexes, including
+    their associated files and snippets. It handles all database operations related
+    to indexing code sources.
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        """Initialize the index repository.
+
+        Args:
+            session: The SQLAlchemy async session to use for database operations.
+
+        """
+        self.session = session
+
+    async def create(self, source_id: int) -> Index:
+        """Create a new index for a source.
+
+        Args:
+            source_id: The ID of the source to create an index for.
+
+        Returns:
+            The newly created Index instance.
+
+        """
+        index = Index(source_id=source_id)
+        self.session.add(index)
+        await self.session.commit()
+        return index
+
+    async def get_by_id(self, index_id: int) -> Index | None:
+        """Get an index by its ID.
+
+        Args:
+            index_id: The ID of the index to retrieve.
+
+        Returns:
+            The Index instance if found, None otherwise.
+
+        """
+        query = select(Index).where(Index.id == index_id)
+        result = await self.session.execute(query)
+        return result.scalar_one_or_none()
+
+    async def files_for_index(self, index_id: int) -> list[File]:
+        """Get all files for an index.
+
+        Args:
+            index_id: The ID of the index to get files for.
+
+        Returns:
+            A list of File instances.
+
+        """
+        query = (
+            select(File)
+            .join(Source, File.source_id == Source.id)
+            .join(Index, Index.source_id == Source.id)
+            .where(Index.id == index_id)
+        )
+        result = await self.session.execute(query)
+        return list(result.scalars())
+
+    async def list_indexes(self) -> list[Index]:
+        """List all indexes.
+
+        Returns:
+            A list of tuples containing index information, source details,
+            and counts of files and snippets.
+
+        """
+        query = select(Index).limit(10)
+        result = await self.session.execute(query)
+        return list(result.scalars())
+
+    async def num_snippets_for_index(self, index_id: int) -> int:
+        """Get the number of snippets for an index."""
+        query = select(func.count()).where(Snippet.index_id == index_id)
+        result = await self.session.execute(query)
+        return result.scalar_one()
+
+    async def update_index_timestamp(self, index: Index) -> None:
+        """Update the updated_at timestamp of an index.
+
+        Args:
+            index: The Index instance to update.
+
+        """
+        index.updated_at = datetime.now(UTC)
+        await self.session.commit()
+
+    async def add_snippet(self, snippet: Snippet) -> None:
+        """Add a new snippet to the database.
+
+        Args:
+            snippet: The Snippet instance to add.
+
+        """
+        self.session.add(snippet)
+        await self.session.commit()
+
+    async def get_snippets_for_index(self, index_id: int) -> list[Snippet]:
+        """Get all snippets for an index.
+
+        Args:
+            index_id: The ID of the index to get snippets for.
+
+        """
+        query = select(Snippet).where(Snippet.index_id == index_id)
+        result = await self.session.execute(query)
+        return list(result.scalars())