kodit 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

kodit/source/source_service.py

@@ -6,20 +6,23 @@ system, database operations (via SourceRepository), and provides a clean API for
 source management.
 """
 
-import mimetypes
-import shutil
 from datetime import datetime
-from hashlib import sha256
 from pathlib import Path
 
-import aiofiles
-import git
 import pydantic
 import structlog
-from tqdm import tqdm
-from uritools import isuri, urisplit
 
-from kodit.source.source_models import File, Source
+from kodit.source.git import is_valid_clone_target
+from kodit.source.source_factories import (
+    FolderFileMetadataExtractor,
+    FolderSourceFactory,
+    FolderWorkingCopyProvider,
+    GitAuthorExtractor,
+    GitFileMetadataExtractor,
+    GitSourceFactory,
+    GitWorkingCopyProvider,
+    NoOpAuthorExtractor,
+)
 from kodit.source.source_repository import SourceRepository
 
 
@@ -62,6 +65,32 @@ class SourceService:
         self.repository = repository
         self.log = structlog.get_logger(__name__)
 
+        # Initialize factories
+        self._setup_factories()
+
+    def _setup_factories(self) -> None:
+        # Git-specific dependencies
+        git_working_copy = GitWorkingCopyProvider(self.clone_dir)
+        git_metadata_extractor = GitFileMetadataExtractor()
+        git_author_extractor = GitAuthorExtractor(self.repository)
+        self.git_factory = GitSourceFactory(
+            working_copy=git_working_copy,
+            metadata_extractor=git_metadata_extractor,
+            author_extractor=git_author_extractor,
+            repository=self.repository,
+        )
+
+        # Folder-specific dependencies
+        folder_working_copy = FolderWorkingCopyProvider(self.clone_dir)
+        folder_metadata_extractor = FolderFileMetadataExtractor()
+        no_op_author_extractor = NoOpAuthorExtractor()
+        self.folder_factory = FolderSourceFactory(
+            working_copy=folder_working_copy,
+            metadata_extractor=folder_metadata_extractor,
+            author_extractor=no_op_author_extractor,
+            repository=self.repository,
+        )
+
     async def get(self, source_id: int) -> SourceView:
         """Get a source by ID.
 
@@ -83,126 +112,15 @@ class SourceService:
 
     async def create(self, uri_or_path_like: str) -> SourceView:
         """Create a new source from a URI or path."""
-        if Path(uri_or_path_like).is_dir():
-            return await self._create_folder_source(Path(uri_or_path_like))
-        if isuri(uri_or_path_like):
-            parsed = urisplit(uri_or_path_like)
-            if parsed.scheme == "file":
-                return await self._create_folder_source(Path(parsed.path))
-            if parsed.scheme in ("git", "http", "https") and parsed.path.endswith(
-                ".git"
-            ):
-                return await self._create_git_source(uri_or_path_like)
-            if not uri_or_path_like.endswith(".git"):
-                uri_or_path_like = uri_or_path_like.strip("/") + ".git"
-                return await self._create_git_source(uri_or_path_like)
-
-        msg = f"Unsupported source: {uri_or_path_like}"
-        raise ValueError(msg)
-
-    async def _create_folder_source(self, directory: Path) -> SourceView:
-        """Create a folder source.
-
-        Args:
-            directory: The path to the local directory.
-
-        Raises:
-            ValueError: If the folder doesn't exist.
-            SourceAlreadyExistsError: If the folder is already added.
-
-        """
-        # Resolve the directory to an absolute path
-        directory = directory.expanduser().resolve()
-
-        source = await self.repository.get_source_by_uri(directory.as_uri())
-        if source:
-            self.log.info("Source already exists, reusing...", source_id=source.id)
+        # If it's possible to clone it, then do so
+        if is_valid_clone_target(uri_or_path_like):
+            source = await self.git_factory.create(uri_or_path_like)
+        # Otherwise just treat it as a directory
+        elif Path(uri_or_path_like).is_dir():
+            source = await self.folder_factory.create(uri_or_path_like)
         else:
-            # Check if the folder exists
-            if not directory.exists():
-                msg = f"Folder does not exist: {directory}"
-                raise ValueError(msg)
-
-            # Check if the folder is already added
-            if await self.repository.get_source_by_uri(directory.as_uri()):
-                msg = f"Directory already added: {directory}"
-                raise ValueError(msg)
-
-            # Clone into a local directory
-            clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
-            clone_path.mkdir(parents=True, exist_ok=True)
-
-            # Copy all files recursively, preserving directory structure, ignoring
-            # hidden files
-            shutil.copytree(
-                directory,
-                clone_path,
-                ignore=shutil.ignore_patterns(".*"),
-                dirs_exist_ok=True,
-            )
-
-            source = await self.repository.create_source(
-                Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
-            )
-
-            # Add all files to the source
-            # Count total files for progress bar
-            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
-
-            # Process each file in the source directory
-            for path in tqdm(clone_path.rglob("*"), total=file_count, leave=False):
-                await self._process_file(source.id, path.absolute())
-
-        return SourceView(
-            id=source.id,
-            uri=source.uri,
-            cloned_path=Path(source.cloned_path),
-            created_at=source.created_at,
-            num_files=await self.repository.num_files_for_source(source.id),
-        )
-
-    async def _create_git_source(self, uri: str) -> SourceView:
-        """Create a git source.
-
-        Args:
-            uri: The URI of the git repository.
-
-        Raises:
-            ValueError: If the repository cloning fails.
-
-        """
-        # Check if the repository is already added
-        source = await self.repository.get_source_by_uri(uri)
-
-        if source:
-            self.log.info("Source already exists, reusing...", source_id=source.id)
-        else:
-            # Create a unique directory name for the clone
-            clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
-            clone_path.mkdir(parents=True, exist_ok=True)
-
-            try:
-                self.log.info("Cloning repository", uri=uri, clone_path=str(clone_path))
-                git.Repo.clone_from(uri, clone_path)
-            except git.GitCommandError as e:
-                if "already exists and is not an empty directory" in str(e):
-                    self.log.info("Repository already exists, reusing...", uri=uri)
-                else:
-                    msg = f"Failed to clone repository: {e}"
-                    raise ValueError(msg) from e
-
-            source = await self.repository.create_source(
-                Source(uri=uri, cloned_path=str(clone_path)),
-            )
-
-            # Add all files to the source
-            # Count total files for progress bar
-            file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
-
-            # Process each file in the source directory
-            self.log.info("Inspecting files", source_id=source.id)
-            for path in tqdm(clone_path.rglob("*"), total=file_count, leave=False):
-                await self._process_file(source.id, path.absolute())
+            msg = f"Unsupported source: {uri_or_path_like}"
+            raise ValueError(msg)
 
         return SourceView(
             id=source.id,
@@ -212,34 +130,6 @@ class SourceService:
             num_files=await self.repository.num_files_for_source(source.id),
         )
 
-    async def _process_file(
-        self,
-        source_id: int,
-        cloned_path: Path,
-    ) -> None:
-        """Process a single file for indexing."""
-        if not cloned_path.is_file():
-            return
-
-        async with aiofiles.open(cloned_path, "rb") as f:
-            content = await f.read()
-            mime_type = mimetypes.guess_type(cloned_path)
-            sha = sha256(content).hexdigest()
-
-            # Create file record
-            file = File(
-                source_id=source_id,
-                cloned_path=cloned_path.as_posix(),
-                mime_type=mime_type[0]
-                if mime_type and mime_type[0]
-                else "application/octet-stream",
-                uri=cloned_path.as_uri(),
-                sha256=sha,
-                size_bytes=len(content),
-            )
-
-            await self.repository.create_file(file)
-
     async def list_sources(self) -> list[SourceView]:
         """List all available sources.
 

{kodit-0.2.1.dist-info → kodit-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.2.1
+Version: 0.2.3
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/
@@ -36,9 +36,10 @@ Requires-Dist: hf-xet>=1.1.2
 Requires-Dist: httpx-retries>=0.3.2
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: openai>=1.82.0
-Requires-Dist: posthog>=4.0.1
+Requires-Dist: pathspec>=0.12.1
 Requires-Dist: pydantic-settings>=2.9.1
 Requires-Dist: pytable-formatter>=0.1.1
+Requires-Dist: rudder-sdk-python>=2.1.4
 Requires-Dist: sentence-transformers>=4.1.0
 Requires-Dist: sqlalchemy[asyncio]>=2.0.40
 Requires-Dist: structlog>=25.3.0
@@ -93,6 +94,7 @@ code. This index is used to build a snippet library, ready for ingestion into an
 - Build comprehensive snippet libraries for LLM ingestion
 - Support for multiple codebase types and languages
 - Efficient indexing and search capabilities
+- Privacy first: respects .gitignore and .noindex files.
 
 ### MCP Server
 

{kodit-0.2.1.dist-info → kodit-0.2.3.dist-info}/RECORD

@@ -1,48 +1,49 @@
 kodit/.gitignore,sha256=ztkjgRwL9Uud1OEi36hGQeDGk3OLK1NfDEO8YqGYy8o,11
 kodit/__init__.py,sha256=aEKHYninUq1yh6jaNfvJBYg-6fenpN132nJt1UU6Jxs,59
-kodit/_version.py,sha256=UoNvMtd4wCG76RwoSpNCUtaFyTwakGcZolfjXzNVSMY,511
-kodit/app.py,sha256=Mr5BFHOHx5zppwjC4XPWVvHjwgl1yrKbUjTWXKubJQM,891
-kodit/cli.py,sha256=i7eEt0FdIQGEfXKFte-8fBcZZGE8BPXBp40aGwJDQGI,11323
-kodit/config.py,sha256=KAxs6qdIvgRZEGTdDIl30QC0g56U0mdk-W8VlaVzghs,4472
+kodit/_version.py,sha256=wD8hnA5gV5UmPkQnpT3xR6V2csgj9K5NEADogbLK79M,511
+kodit/app.py,sha256=qKBWJ0VNSY_M6G3VFfAQ0133q5bnS99cUFD0p396taw,1032
+kodit/cli.py,sha256=wKFXGUMX-fDLooaK-3po2TBpNNRBwgSD7BRbUddg-_M,11562
+kodit/config.py,sha256=3yh7hfLSILjZK_qJMhcExwRcrWJ0b5Eb1JjjOvMPJZo,4146
 kodit/database.py,sha256=WB1KpVxUYPgiJGU0gJa2hqytYB8wJEJ5z3WayhWzNMU,2403
-kodit/log.py,sha256=HU1OmuxO4FcVw61k4WW7Y4WM7BrDaeplw1PcBHhuIZY,5434
+kodit/log.py,sha256=sHPHYetlMcKTor2VaFLMyao1_fZ_xhuzqXCAt5F5UMU,8575
 kodit/mcp.py,sha256=QruyPskWB0_x59pkfj5BBeXuR13GMny5TAZEa2j4U9s,5752
 kodit/middleware.py,sha256=I6FOkqG9-8RH5kR1-0ZoQWfE4qLCB8lZYv8H_OCH29o,2714
 kodit/bm25/__init__.py,sha256=j8zyriNWhbwE5Lbybzg1hQAhANlU9mKHWw4beeUR6og,19
 kodit/bm25/keyword_search_factory.py,sha256=rp-wx3DJsc2KlELK1V337EyeYvmwnMQwUqOo1WVPSmg,631
 kodit/bm25/keyword_search_service.py,sha256=aBbWQKgQmi2re3EIHdXFS00n7Wj3b2D0pZsLZ4qmHfE,754
-kodit/bm25/local_bm25.py,sha256=cDx_hT9hXyEWz8LlFGyQs5-0ufK2Y4gArMfZv7-D9zQ,3621
-kodit/bm25/vectorchord_bm25.py,sha256=_nGrkUReYLLV-L8RIuIVLwjuhSYZl9T532n5OVf0kWs,6393
+kodit/bm25/local_bm25.py,sha256=nokrd_xAeqXi3m68X5P1R5KBhRRB1E2L_J6Zgm26PCg,3869
+kodit/bm25/vectorchord_bm25.py,sha256=0p_FgliaoevB8GLSmzWnV3zUjdcWgCgOKIpLURr7Qfo,6549
 kodit/embedding/__init__.py,sha256=h9NXzDA1r-K23nvBajBV-RJzHJN0p3UJ7UQsmdnOoRw,24
-kodit/embedding/embedding_factory.py,sha256=UGnFRyyQXazSUOwyW4Hg7Vq2-kfAoDj9lD4CTLu8x04,1630
+kodit/embedding/embedding_factory.py,sha256=-WuXNleQ_mqdw1E4TczjtOawNeaXKAiDPFqN_XX7Mmg,2419
 kodit/embedding/embedding_models.py,sha256=rN90vSs86dYiqoawcp8E9jtwY31JoJXYfaDlsJK7uqc,656
 kodit/embedding/embedding_repository.py,sha256=-ux3scpBzel8c0pMH9fNOEsSXFIzl-IfgaWrkTb1szo,6907
-kodit/embedding/local_vector_search_service.py,sha256=hkF0qlfzjyGt400qIX9Mr6B7b7i8WvYIYWN2Z2C_pcs,1907
+kodit/embedding/local_vector_search_service.py,sha256=dgMi8hQNUbYEgHnEYmLIpon4yLduoNUpu7k7VP6sOHI,2042
 kodit/embedding/vector_search_service.py,sha256=pQJ129QjGrAWOXzqkywmgtDRpy8_gtzYgkivyqF9Vrs,1009
-kodit/embedding/vectorchord_vector_search_service.py,sha256=63Xf7_nAz3xWOwrmZibw8Q-xoRdCrPDDpdSA_WE7mrc,5131
+kodit/embedding/vectorchord_vector_search_service.py,sha256=TKNR3HgWHwwWtJ1SsvSaj_BXLJ_uw6Bdr_tpaePMeAA,5383
 kodit/embedding/embedding_provider/__init__.py,sha256=h9NXzDA1r-K23nvBajBV-RJzHJN0p3UJ7UQsmdnOoRw,24
-kodit/embedding/embedding_provider/embedding_provider.py,sha256=IC7fZaZ_ze-DxpxKfK44pRDwHWUQhVIqVKKQ3alO5Qc,1882
+kodit/embedding/embedding_provider/embedding_provider.py,sha256=T6AKMWwzEJ1vDe-cEIg-qxdjhUEZ0PKs9YqQMWaLaQ0,1928
 kodit/embedding/embedding_provider/hash_embedding_provider.py,sha256=nAhlhh8j8PqqCCbhVl26Y8ntFBm2vJBCtB4X04g5Wwg,2638
 kodit/embedding/embedding_provider/local_embedding_provider.py,sha256=WP8lw6XG7v1_5Mw4_rhIOETooYRsxhkwmFaXCqCouQU,1977
-kodit/embedding/embedding_provider/openai_embedding_provider.py,sha256=V_jdUXiaGdslplwxMlfgFc4_hAVS2eaJXMTs2C7RiLI,2666
+kodit/embedding/embedding_provider/openai_embedding_provider.py,sha256=-phz5FKYM_tI3Q4_3SPzjzIOK3k92Uk52TAOTmoVoWI,2722
 kodit/enrichment/__init__.py,sha256=vBEolHpKaHUhfINX0dSGyAPlvgpLNAer9YzFtdvCB24,18
-kodit/enrichment/enrichment_factory.py,sha256=vKjkUTdhj74IW2S4GENDWdWMJx6BwUSZjJGDC0i7DSk,787
+kodit/enrichment/enrichment_factory.py,sha256=AAzvxgjo-FQU5aAm9Zla4DAwUMKGrcw8mQwJsMhIsHY,1566
 kodit/enrichment/enrichment_service.py,sha256=87Sd3gGbEMJYb_wVrHG8L1yGIZmQNR7foUS4_y94azI,977
 kodit/enrichment/enrichment_provider/__init__.py,sha256=klf8iuLVWX4iRz-DZQauFFNAoJC5CByczh48TBZPW-o,27
 kodit/enrichment/enrichment_provider/enrichment_provider.py,sha256=E0H5rq3OENM0yYbA8K_3nSnj5lUHCpoIOqpWLo-2MVU,413
-kodit/enrichment/enrichment_provider/local_enrichment_provider.py,sha256=kh_9X9m-WEziRi5TV6QflKXyXQTos9kzpmpGil7pywM,3196
-kodit/enrichment/enrichment_provider/openai_enrichment_provider.py,sha256=gYuFTAeIVdQNlCUvNSPgRoiRwCvRD0C8419h8ubyABA,2725
+kodit/enrichment/enrichment_provider/local_enrichment_provider.py,sha256=RqwUD0BnwRQ8zlkFNkaKq8d58r33k2jIdnSdf6zla1w,3325
+kodit/enrichment/enrichment_provider/openai_enrichment_provider.py,sha256=0Yw7h9RXptoI4bKuqJSKIRQXPUUhNV7eACavgoy_T8s,2874
 kodit/indexing/__init__.py,sha256=cPyi2Iej3G1JFWlWr7X80_UrsMaTu5W5rBwgif1B3xo,75
 kodit/indexing/fusion.py,sha256=TZb4fPAedXdEUXzwzOofW98QIOymdbclBOP1KOijuEk,1674
 kodit/indexing/indexing_models.py,sha256=6NX9HVcj6Pu9ePwHC7n-PWSyAgukpJq0nCNmUIigtbo,1282
 kodit/indexing/indexing_repository.py,sha256=dqOS0pxKM6bUjMXWqYukAK8XdiD36OnskFASgZRXRQM,6955
-kodit/indexing/indexing_service.py,sha256=_uhoqBic3_zXNJOsKt_w-TgX5ebf7OBwbqMdO9zectM,10779
+kodit/indexing/indexing_service.py,sha256=3hW7vbFyabLEkLU-PRoGR49yVLewANdOKlye4GhR-tw,11467
 kodit/migrations/README,sha256=ISVtAOvqvKk_5ThM5ioJE-lMkvf9IbknFUFVU_vPma4,58
 kodit/migrations/__init__.py,sha256=lP5MuwlyWRMO6UcDWnQcQ3G-GYHcFb6rl9gYPHJ1sjo,40
 kodit/migrations/env.py,sha256=w1M7OZh-ZeR2dPHS0ByXAUxQjfZQ8xIzMseWuzLDTWw,2469
 kodit/migrations/script.py.mako,sha256=zWziKtiwYKEWuwPV_HBNHwa9LCT45_bi01-uSNFaOOE,703
 kodit/migrations/versions/7c3bbc2ab32b_add_embeddings_table.py,sha256=-61qol9PfQKILCDQRA5jEaats9aGZs9Wdtp-j-38SF4,1644
 kodit/migrations/versions/85155663351e_initial.py,sha256=Cg7zlF871o9ShV5rQMQ1v7hRV7fI59veDY9cjtTrs-8,3306
+kodit/migrations/versions/9e53ea8bb3b0_add_authors.py,sha256=a32Zm8KUQyiiLkjKNPYdaJDgjW6VsV-GhaLnPnK_fpI,3884
 kodit/migrations/versions/__init__.py,sha256=9-lHzptItTzq_fomdIRBegQNm4Znx6pVjwD4MiqRIdo,36
 kodit/migrations/versions/c3f5137d30f5_index_all_the_things.py,sha256=rI8LmjF-I2OMxZ2nOIF_NRmqOLXe45hL_iz_nx97DTQ,1680
 kodit/snippets/__init__.py,sha256=-2coNoCRjTixU9KcP6alpmt7zqf37tCRWH3D7FPJ8dg,48
@@ -55,13 +56,16 @@ kodit/snippets/languages/javascript.scm,sha256=Ini5TsVNmcBKQ8aL46a5Id9ut0g9Udmvm
 kodit/snippets/languages/python.scm,sha256=ee85R9PBzwye3IMTE7-iVoKWd_ViU3EJISTyrFGrVeo,429
 kodit/snippets/languages/typescript.scm,sha256=U-ujbbv4tylbUBj9wuhL-e5cW6hmgPCNs4xrIX3r_hE,448
 kodit/source/__init__.py,sha256=1NTZyPdjThVQpZO1Mp1ColVsS7sqYanOVLqnoqV9Ipo,83
-kodit/source/source_models.py,sha256=kcC59XPSDDMth2mOYK3FakqTN0jxKFaTDch0ejyD9Sw,2446
-kodit/source/source_repository.py,sha256=0EksMpoLzdkfe8S4eeCm4Sf7TuxsOzOzaF4BBsMYo-4,3163
-kodit/source/source_service.py,sha256=dyXWf_t2qGvD9YAY3C5Zcxc63BbyrheL-jgTzXV7gqo,9156
+kodit/source/git.py,sha256=CpNczc06SbxpzfQKq76lZFzuol10ZJvTRSzeXW9DFUs,363
+kodit/source/ignore.py,sha256=W7cuIrYlgfu3S1qyoIepXe8PqYmtFv61Tt5RO8cbZbg,1701
+kodit/source/source_factories.py,sha256=qTjKBKu62vynHUxf3-lgxQCoj1nGn8aI9ExFUWNk5B8,11678
+kodit/source/source_models.py,sha256=XlgrY2dlpJkOFZQjrwMGwebAcwQ4rSfjshvCrHKA18Q,3916
+kodit/source/source_repository.py,sha256=eme0C3pRqwFZ1ZSbqq4Z6SV9CC6AvRmiOjy3eHQoEN0,5443
+kodit/source/source_service.py,sha256=E1KPG7TrorqdreJVHxZPx8CVLncOxGEvZ5uDQ6yZugo,5050
 kodit/util/__init__.py,sha256=bPu6CtqDWCRGU7VgW2_aiQrCBi8G89FS6k1PjvDajJ0,37
 kodit/util/spinner.py,sha256=R9bzrHtBiIH6IfLbmsIVHL53s8vg-tqW4lwGGALu4dw,1932
-kodit-0.2.1.dist-info/METADATA,sha256=Frs10m-Bc3DzgvxXSA27u0dFXDdV0fR_I4zzyhhPYfY,5768
-kodit-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kodit-0.2.1.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
-kodit-0.2.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kodit-0.2.1.dist-info/RECORD,,
+kodit-0.2.3.dist-info/METADATA,sha256=ccZ0bl5PCGFNC30XSr-4ljL-JrkTgTfYlqZUlUPYba8,5867
+kodit-0.2.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kodit-0.2.3.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
+kodit-0.2.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kodit-0.2.3.dist-info/RECORD,,