graph-dependency-analyzer 0.2.0__py3-none-any.whl

src/application/use_cases/indexing_orchestrator.py

@@ -0,0 +1,181 @@
+"""
+IndexingOrchestrator Implementation (Task 8.2)
+
+Manages parallel processing of multiple repositories with concurrency control.
+Uses semaphore to limit concurrent executions.
+"""
+from pathlib import Path
+from typing import Callable
+import asyncio
+import structlog
+
+from src.application.dtos.batch_processing_result import BatchProcessingResult
+from src.application.dtos.index_result import IndexResult
+from src.application.use_cases.index_repository_use_case import IndexRepositoryUseCase
+from src.application.services.progress_tracker import ProgressUpdate
+
+logger = structlog.get_logger(__name__)
+
+
+class IndexingOrchestrator:
+    """
+    Orchestrates parallel processing of multiple repositories.
+
+    Features:
+    - Concurrent processing with configurable limit (semaphore)
+    - Individual failure isolation (one failure doesn't stop others)
+    - Progress callback for each repository
+    - Aggregated results (total, success, failed counts)
+
+    Requirements: 2.6, 2.7, 2.8
+    """
+
+    def __init__(self, index_repository_use_case: IndexRepositoryUseCase):
+        """
+        Initialize orchestrator with use case.
+
+        Args:
+            index_repository_use_case: Use case for indexing single repository
+        """
+        self.index_repository_use_case = index_repository_use_case
+
+    async def process_batch(
+        self,
+        repositories: list[Path],
+        parallel: int = 3,
+        progress_callback: Callable[[str, ProgressUpdate], None] | None = None
+    ) -> BatchProcessingResult:
+        """
+        Process multiple repositories in parallel with concurrency control.
+
+        Args:
+            repositories: List of repository paths to process
+            parallel: Maximum concurrent processes (default: 3)
+            progress_callback: Optional callback(repo_name, progress_update)
+
+        Returns:
+            BatchProcessingResult with aggregated counts and results
+        """
+        # Handle empty list
+        if not repositories:
+            logger.info("batch_processing_empty", repository_count=0)
+            return BatchProcessingResult(
+                total_processed=0,
+                success_count=0,
+                failed_count=0,
+                results=[]
+            )
+
+        logger.info(
+            "batch_processing_started",
+            repository_count=len(repositories),
+            parallel_limit=parallel
+        )
+
+        # Create semaphore for concurrency control
+        semaphore = asyncio.Semaphore(parallel)
+
+        # Process each repository with semaphore
+        async def process_with_semaphore(repo_path: Path, index: int) -> IndexResult:
+            """Process single repository with semaphore control."""
+            async with semaphore:
+                return await self._process_single_repository(
+                    repo_path,
+                    index,
+                    progress_callback
+                )
+
+        # Create tasks for all repositories
+        tasks = [
+            process_with_semaphore(repo, i)
+            for i, repo in enumerate(repositories)
+        ]
+
+        # Execute all tasks concurrently (but limited by semaphore)
+        results = await asyncio.gather(*tasks, return_exceptions=False)
+
+        # Aggregate results
+        success_count = sum(1 for r in results if r.status == "success")
+        failed_count = sum(1 for r in results if r.status == "error")
+
+        logger.info(
+            "batch_processing_completed",
+            total_processed=len(results),
+            success_count=success_count,
+            failed_count=failed_count
+        )
+
+        return BatchProcessingResult(
+            total_processed=len(results),
+            success_count=success_count,
+            failed_count=failed_count,
+            results=results
+        )
+
+    async def _process_single_repository(
+        self,
+        repository_path: Path,
+        index: int,
+        progress_callback: Callable[[str, ProgressUpdate], None] | None = None
+    ) -> IndexResult:
+        """
+        Process a single repository with error handling.
+
+        Args:
+            repository_path: Path to repository
+            index: Index in batch (for logging)
+            progress_callback: Optional progress callback
+
+        Returns:
+            IndexResult (either success or error)
+        """
+        repository_name = repository_path.name
+
+        logger.info(
+            "repository_processing_started",
+            repository=repository_name,
+            index=index,
+            path=str(repository_path)
+        )
+
+        try:
+            # Create wrapper callback that adds repository name
+            repo_progress_callback = None
+            if progress_callback:
+                def wrapped_callback(update: ProgressUpdate):
+                    progress_callback(repository_name, update)
+                repo_progress_callback = wrapped_callback
+
+            # Execute indexing
+            result = await self.index_repository_use_case.execute(
+                repository_path,
+                progress_callback=repo_progress_callback
+            )
+
+            logger.info(
+                "repository_processing_completed",
+                repository=repository_name,
+                status=result.status
+            )
+
+            return result
+
+        except Exception as e:
+            # Capture exception and return error result
+            logger.error(
+                "repository_processing_failed",
+                repository=repository_name,
+                error=str(e),
+                error_type=type(e).__name__
+            )
+
+            # Return error result instead of raising
+            return IndexResult(
+                status="error",
+                repository_name=repository_name,
+                files_processed=0,
+                chunks_created=0,
+                nodes_created=0,
+                relationships_created=0,
+                error_message=str(e)
+            )

src/application/use_cases/list_repositories_use_case.py

@@ -0,0 +1,84 @@
+"""Use case for listing indexed repositories."""
+from typing import List
+import structlog
+
+from src.infrastructure.persistence.neo4j_dependency_repository import Neo4jDependencyGraphRepository
+
+logger = structlog.get_logger(__name__)
+
+
+class ListRepositoriesUseCase:
+    """
+    Use case for listing all indexed repositories.
+
+    Queries Neo4j to find all unique repository names from indexed nodes.
+    """
+
+    def __init__(self, graph_repository: Neo4jDependencyGraphRepository):
+        """
+        Initialize use case.
+
+        Args:
+            graph_repository: Neo4j dependency graph repository
+        """
+        self.graph_repository = graph_repository
+
+    async def execute(self) -> List[str]:
+        """
+        Execute use case to list repositories.
+
+        Returns:
+            List of unique repository names
+        """
+        logger.info("list_repositories_starting")
+
+        try:
+            # Query Neo4j for distinct repository names
+            repositories = await self._query_repositories()
+
+            logger.info(
+                "list_repositories_completed",
+                count=len(repositories)
+            )
+
+            return repositories
+
+        except Exception as e:
+            logger.error(
+                "list_repositories_failed",
+                error=str(e),
+                error_type=type(e).__name__
+            )
+            raise
+
+    async def _query_repositories(self) -> List[str]:
+        """
+        Query Neo4j for unique repository names.
+
+        Filters out 'external' and 'unknown' repositories (libraries).
+
+        Returns:
+            List of repository names sorted alphabetically
+        """
+        query = """
+        MATCH (n)
+        WHERE n.repository IS NOT NULL
+          AND n.repository <> 'external'
+          AND n.repository <> 'unknown'
+        RETURN DISTINCT n.repository as repository
+        ORDER BY repository ASC
+        """
+
+        async with self.graph_repository.driver.session() as session:
+            result = await session.run(query)
+            records = await result.data()
+
+            repositories = [record["repository"] for record in records]
+
+            logger.debug(
+                "repositories_queried",
+                repositories=repositories,
+                count=len(repositories)
+            )
+
+            return repositories