graph-dependency-analyzer 0.2.0__py3-none-any.whl

src/application/use_cases/index_repository_use_case.py

@@ -0,0 +1,683 @@
+"""
+IndexRepositoryUseCase Implementation (Task 8.1)
+
+Orchestrates the complete indexing pipeline for a single repository.
+Executes 8 sequential stages from validation to Neo4j storage.
+"""
+from pathlib import Path
+from typing import Callable
+import structlog
+
+from src.application.dtos.index_result import IndexResult
+from src.application.services.file_scanner_service import FileScannerService
+from src.application.services.progress_tracker import ProgressTracker, ProgressUpdate
+from src.infrastructure.parsing.tree_sitter_java_parser import TreeSitterJavaParser, ParseError
+from src.infrastructure.parsing.tree_sitter_typescript_parser import TreeSitterTypeScriptParser  # NEW: Task 7.1
+from src.domain.services.dependency_extractor import DependencyExtractor
+from src.domain.services.semantic_chunk_builder import SemanticChunkBuilder
+from src.infrastructure.external_apis.openai_embedding_provider import OpenAIEmbeddingProvider
+from src.infrastructure.persistence.qdrant_vector_repository import QdrantVectorRepository
+from src.infrastructure.persistence.neo4j_dependency_repository import Neo4jDependencyGraphRepository
+
+logger = structlog.get_logger(__name__)
+
+
+class IndexRepositoryUseCase:
+    """
+    Use case for indexing a single repository through complete pipeline.
+
+    Executes 8 sequential stages:
+    1. Validation (0%)
+    2. File scanning (5%)
+    3. AST parsing (25%)
+    4. Dependency extraction (50%)
+    5. Chunking (65%)
+    6. Embedding generation (77%)
+    7. Vector storage in Qdrant (90%)
+    8. Graph storage in Neo4j (97%)
+
+    Requirements: 3.1, 3.2, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.15
+    """
+
+    def __init__(
+        self,
+        file_scanner: FileScannerService,
+        java_parser: TreeSitterJavaParser,
+        typescript_parser=None,
+        dependency_extractor=None,
+        chunk_builder=None,
+        embedding_provider=None,
+        qdrant_repository=None,
+        neo4j_repository=None,
+        sql_parser=None,
+        python_parser=None,
+        groovy_parser=None,
+    ):
+        """
+        Initialize use case with all dependencies.
+
+        Args:
+            file_scanner: Service for scanning source files
+            java_parser: Parser for Java AST
+            typescript_parser: Optional parser for TypeScript/JavaScript AST
+            dependency_extractor: Service for extracting dependencies
+            chunk_builder: Service for creating semantic chunks
+            embedding_provider: Provider for generating embeddings
+            qdrant_repository: Repository for vector storage
+            neo4j_repository: Repository for graph storage
+            sql_parser: Optional parser for SQL/DB schema files
+        """
+        self.file_scanner = file_scanner
+        self.java_parser = java_parser
+        self.typescript_parser = typescript_parser
+        self.sql_parser = sql_parser
+        self.python_parser = python_parser
+        self.groovy_parser = groovy_parser
+        self.dependency_extractor = dependency_extractor
+        self.chunk_builder = chunk_builder
+        self.embedding_provider = embedding_provider
+        self.qdrant_repository = qdrant_repository
+        self.neo4j_repository = neo4j_repository
+
+    async def execute(
+        self,
+        repository_path: Path,
+        progress_callback: Callable[[ProgressUpdate], None] | None = None
+    ) -> IndexResult:
+        """
+        Execute complete indexing pipeline for one repository.
+
+        Args:
+            repository_path: Absolute path to repository root
+            progress_callback: Optional callback for progress updates
+
+        Returns:
+            IndexResult with status and counts
+
+        Raises:
+            ValueError: If repository_path does not exist
+        """
+        # Convert to Path if string
+        if isinstance(repository_path, str):
+            repository_path = Path(repository_path)
+
+        # Get repository name
+        repository_name = repository_path.name
+
+        # Create progress tracker
+        tracker = ProgressTracker(repository_name, progress_callback)
+
+        try:
+            # STAGE 1: Validation (0%)
+            tracker.update("validation", "Validating repository path")
+            if not repository_path.exists():
+                raise ValueError(f"Repository path does not exist: {repository_path}")
+
+            logger.info(
+                "indexing_started",
+                repository=repository_name,
+                path=str(repository_path)
+            )
+
+            # STAGE 2: File Scanning (5%) - Task 6.1 extension
+            tracker.update("scan", f"Scanning files in {repository_name}")
+            scanned_files = self.file_scanner.scan(repository_path)
+
+            # Extract separate counts for progress tracking
+            java_files = scanned_files.get("java", [])
+            typescript_files = scanned_files.get("typescript", [])
+            javascript_files = scanned_files.get("javascript", [])
+            sql_files = scanned_files.get("sql", [])
+            python_files = scanned_files.get("python", [])
+            groovy_files = scanned_files.get("groovy", [])
+
+            total_files = len(java_files) + len(typescript_files) + len(javascript_files) + len(sql_files) + len(python_files) + len(groovy_files)
+
+            logger.info(
+                "files_scanned",
+                repository=repository_name,
+                java_files_count=len(java_files),
+                typescript_files_count=len(typescript_files),
+                javascript_files_count=len(javascript_files),
+                sql_files_count=len(sql_files),
+                total_files=total_files
+            )
+
+            # STAGE 3: AST Parsing (25%)
+            tracker.update("ast", f"Parsing {len(java_files)} Java, {len(typescript_files)} TypeScript, {len(javascript_files)} JS, {len(sql_files)} SQL files")
+
+            parsed_results = []
+            parse_errors = {"java": 0, "typescript": 0}
+
+            # Task 7.2: Parallel parsing using asyncio.gather()
+            import asyncio
+
+            async def parse_java_files():
+                """Parse all Java files"""
+                java_results = []
+                java_errors = 0
+                for java_file in java_files:
+                    try:
+                        parsed = self.java_parser.parse_file(java_file)  # FIXED: Renamed
+                        java_results.append(parsed)
+                    except ParseError as e:
+                        java_errors += 1
+                        logger.warning(
+                            "parse_error_skipped",
+                            language="java",
+                            file=str(java_file),
+                            error=str(e)
+                        )
+                return java_results, java_errors
+
+            async def parse_typescript_files():
+                """Parse all TypeScript/JavaScript files"""
+                ts_results = []
+                ts_errors = 0
+                if self.typescript_parser:
+                    for ts_file in (typescript_files + javascript_files):
+                        try:
+                            parsed = self.typescript_parser.parse_file(ts_file)
+                            ts_results.append(parsed)
+                        except Exception as e:
+                            ts_errors += 1
+                            logger.warning(
+                                "parse_error_skipped",
+                                language="typescript",
+                                file=str(ts_file),
+                                error=str(e)
+                            )
+                return ts_results, ts_errors
+
+            async def parse_sql_files_fn():
+                """Parse all SQL/DB schema files"""
+                sql_results = []
+                sql_errors = 0
+                if self.sql_parser and sql_files:
+                    for sql_file in sql_files:
+                        try:
+                            parsed = self.sql_parser.parse_file(sql_file)
+                            sql_results.append(parsed)
+                        except Exception as e:
+                            sql_errors += 1
+                            logger.warning(
+                                "parse_error_skipped",
+                                language="sql",
+                                file=str(sql_file),
+                                error=str(e)
+                            )
+                return sql_results, sql_errors
+
+            async def parse_python_files_fn():
+                """Parse all Python source files"""
+                py_results = []
+                py_errors = 0
+                if self.python_parser and python_files:
+                    for py_file in python_files:
+                        try:
+                            parsed = self.python_parser.parse_file(py_file)
+                            py_results.append(parsed)
+                        except Exception as e:
+                            py_errors += 1
+                            logger.warning(
+                                "parse_error_skipped",
+                                language="python",
+                                file=str(py_file),
+                                error=str(e)
+                            )
+                return py_results, py_errors
+
+            async def parse_groovy_files_fn():
+                """Parse all Groovy/Jenkinsfile source files"""
+                groovy_results = []
+                groovy_errors = 0
+                if self.groovy_parser and groovy_files:
+                    for groovy_file in groovy_files:
+                        try:
+                            parsed = self.groovy_parser.parse_file(groovy_file)
+                            groovy_results.append(parsed)
+                        except Exception as e:
+                            groovy_errors += 1
+                            logger.warning(
+                                "parse_error_skipped",
+                                language="groovy",
+                                file=str(groovy_file),
+                                error=str(e)
+                            )
+                return groovy_results, groovy_errors
+
+            # Execute parsing in parallel
+            (java_parsed, java_errors), (ts_parsed, ts_errors), \
+            (sql_parsed, sql_errors), (py_parsed, py_errors), \
+            (groovy_parsed, groovy_errors) = \
+                await asyncio.gather(
+                    parse_java_files(),
+                    parse_typescript_files(),
+                    parse_sql_files_fn(),
+                    parse_python_files_fn(),
+                    parse_groovy_files_fn(),
+                )
+
+            # Combine results
+            parsed_results = java_parsed + ts_parsed + sql_parsed + py_parsed + groovy_parsed
+            parse_errors["java"] = java_errors
+            parse_errors["typescript"] = ts_errors
+            parse_errors["sql"] = sql_errors
+
+            logger.info(
+                "ast_parsing_complete",
+                repository=repository_name,
+                files_parsed=len(parsed_results),
+                java_parse_errors=parse_errors["java"],
+                typescript_parse_errors=parse_errors["typescript"]
+            )
+
+            # STAGE 4: Dependency Extraction (50%)
+            tracker.update("deps", "Extracting dependencies from AST")
+
+            # Convert parsed results to AST nodes format expected by DependencyExtractor
+            from src.domain.entities.ast_node import ASTNode
+
+            ast_nodes = []
+            for parse_result in parsed_results:
+                # Convert classes to ASTNode objects
+                for cls in parse_result.classes:
+                    if isinstance(cls, dict):
+                        # Add repository metadata
+                        cls_metadata = cls.copy()
+                        cls_metadata["repository"] = repository_name
+                        cls_metadata["file_path"] = str(parse_result.file_path)
+
+                        # Ensure qualified_name exists (use name as fallback)
+                        qualified_name = cls.get("qualified_name", cls.get("name", "Unknown"))
+                        if not qualified_name or qualified_name == "Unknown":
+                            file_path_str = str(parse_result.file_path)
+                            component_name = cls.get("name", Path(file_path_str).stem)
+                            qualified_name = f"{repository_name}.{component_name}"
+
+                        # Preserve DB object types from SqlSchemaParser; default to CLASS
+                        _db_types = {"DB_TABLE","DB_VIEW","DB_PROCEDURE","DB_PACKAGE",
+                                     "DB_TRIGGER","DB_SEQUENCE","DB_FUNCTION","DB_INDEX",
+                                     "DB_TYPE","DB_COLLECTION"}
+                        raw_node_type = cls.get("node_type", "CLASS")
+                        node_type = raw_node_type if raw_node_type in _db_types else "CLASS"
+
+                        ast_node = ASTNode(
+                            node_type=node_type,
+                            name=cls.get("name", "Unknown"),
+                            qualified_name=qualified_name,
+                            line_start=cls.get("line_start", cls.get("line", 0)),
+                            line_end=cls.get("line_end", cls.get("line", 0)),
+                            metadata=cls_metadata
+                        )
+                        ast_nodes.append(ast_node)
+                    else:
+                        ast_nodes.append(cls)  # Already an ASTNode
+
+                # Convert methods to ASTNode objects
+                for method in parse_result.methods:
+                    if isinstance(method, dict):
+                        # Add repository metadata
+                        method_metadata = method.copy()
+                        method_metadata["repository"] = repository_name
+                        method_metadata["file_path"] = str(parse_result.file_path)
+
+                        # Add string_literals from ParseResult for API detection (Task 4.2)
+                        # TypeScript parser extracts all string literals from the file
+                        if hasattr(parse_result, 'string_literals') and parse_result.string_literals:
+                            method_metadata["string_literals"] = parse_result.string_literals
+
+                        # Generate qualified_name if not present (TypeScript components/functions)
+                        method_name = method.get("name", "Unknown")
+                        qualified_name = method.get("qualified_name", "")
+                        if not qualified_name:
+                            # Generate from repository + file path + method name
+                            file_path_str = str(parse_result.file_path)
+                            file_name = Path(file_path_str).stem
+                            qualified_name = f"{repository_name}.{file_name}.{method_name}"
+
+                        ast_node = ASTNode(
+                            node_type="METHOD",
+                            name=method_name,
+                            qualified_name=qualified_name,
+                            line_start=method.get("line_start", method.get("line", 0)),
+                            line_end=method.get("line_end", method.get("line", 0)),
+                            metadata=method_metadata
+                        )
+                        ast_nodes.append(ast_node)
+                    else:
+                        ast_nodes.append(method)
+
+                # Convert Spring endpoints and HTTP wrapper calls to ASTNode objects
+                for call in (parse_result.method_calls if hasattr(parse_result, 'method_calls') else []):
+                    if not isinstance(call, dict):
+                        continue
+                    file_stem = Path(str(parse_result.file_path)).stem
+
+                    # Spring REST endpoint extracted by Java parser
+                    if call.get("_is_endpoint"):
+                        ep_metadata = call.copy()
+                        ep_metadata["repository"] = repository_name
+                        ep_metadata["file_path"] = str(parse_result.file_path)
+                        qualified_name = f"{repository_name}.{file_stem}"
+                        ast_node = ASTNode(
+                            node_type="BACKEND_ENDPOINT",
+                            name=call.get("path", ""),
+                            qualified_name=qualified_name,
+                            line_start=call.get("line", 0),
+                            line_end=call.get("line", 0),
+                            metadata=ep_metadata
+                        )
+                        ast_nodes.append(ast_node)
+
+                    # Frontend HTTP wrapper call
+                    elif call.get("is_api_call") and call.get("api_path"):
+                        call_metadata = call.copy()
+                        call_metadata["repository"] = repository_name
+                        call_metadata["file_path"] = str(parse_result.file_path)
+                        qualified_name = f"{repository_name}.{file_stem}"
+                        ast_node = ASTNode(
+                            node_type="API_CALL",
+                            name=call.get("callee", ""),
+                            qualified_name=qualified_name,
+                            line_start=call.get("line", 0),
+                            line_end=call.get("line", 0),
+                            metadata=call_metadata
+                        )
+                        ast_nodes.append(ast_node)
+
+                # Determine the source class for imports in this file.
+                # Use the first class defined in the file as the owning class.
+                source_class_qname = None
+                for cls in parse_result.classes:
+                    if isinstance(cls, dict):
+                        qn = cls.get("qualified_name") or cls.get("name")
+                        if qn and qn not in ("Unknown", ""):
+                            source_class_qname = qn
+                            break
+
+                # Convert imports to ASTNode objects
+                for imp in parse_result.imports:
+                    if not isinstance(imp, dict):
+                        ast_nodes.append(imp)
+                        continue
+
+                    # Skip SQL cross-refs (handled separately below)
+                    if imp.get("type") in {"READS_TABLE", "WRITES_TABLE", "CALLS_PROCEDURE"}:
+                        continue
+
+                    # Resolve import path — Java parser uses "import_path",
+                    # TypeScript parser uses "module_path".
+                    module_path = (
+                        imp.get("import_path")       # Java: com.medline.wms.Foo
+                        or imp.get("module_path")    # TypeScript: ../services/Foo
+                        or imp.get("qualified_name")
+                        or ""
+                    )
+                    if not module_path:
+                        continue
+
+                    imp_metadata = imp.copy()
+                    imp_metadata["repository"] = repository_name
+                    imp_metadata["file_path"] = str(parse_result.file_path)
+                    imp_metadata["import_path"] = module_path
+
+                    # Use the owning class as source (for CLASS→CLASS IMPORTS edge).
+                    # Fall back to file-based package only if no class found.
+                    if source_class_qname:
+                        imp_metadata["current_package"] = source_class_qname
+                    else:
+                        file_path_str = str(parse_result.file_path)
+                        if "src" in file_path_str:
+                            parts = file_path_str.split("src")[-1].strip("/\\").replace("\\", "/")
+                            package_parts = parts.split("/")[:-1]
+                            imp_metadata["current_package"] = ".".join(package_parts) if package_parts else "src"
+                        else:
+                            imp_metadata["current_package"] = repository_name
+
+                    ast_nodes.append(ASTNode(
+                        node_type="IMPORT",
+                        name=imp.get("name", module_path),
+                        qualified_name=module_path,
+                        line_start=imp.get("line", imp.get("line_start", 0)),
+                        line_end=imp.get("line", imp.get("line_end", 0)),
+                        metadata=imp_metadata,
+                    ))
+
+                # Convert SQL cross-references (reads/writes/calls between DB objects)
+                # These come from SqlSchemaParser as parse_result.imports = list[dict]
+                # with keys: type (READS_TABLE/WRITES_TABLE/CALLS_PROCEDURE), target, line
+                for ref in (parse_result.imports if hasattr(parse_result, 'imports') else []):
+                    if not isinstance(ref, dict):
+                        continue
+                    ref_type = ref.get("type", "")
+                    if ref_type not in {"READS_TABLE", "WRITES_TABLE", "CALLS_PROCEDURE"}:
+                        continue
+                    target = ref.get("target", "")
+                    if not target:
+                        continue
+                    ref_metadata = {
+                        "repository": repository_name,
+                        "file_path": str(parse_result.file_path),
+                        "ref_type": ref_type,
+                        "target": target,
+                    }
+                    ast_nodes.append(ASTNode(
+                        node_type="DB_REF",
+                        name=target,
+                        qualified_name=target,
+                        line_start=ref.get("line", 0),
+                        line_end=ref.get("line", 0),
+                        metadata=ref_metadata,
+                    ))
+
+            # Scan build files (pom.xml, build.gradle) and config files
+            build_files: dict[str, str] = {}
+            config_files: dict[str, str] = {}
+            _BUILD_NAMES = {"pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle"}
+            _SKIP_DIRS = {"node_modules", ".git", "target", "dist", ".next", "__pycache__"}
+            for f in repository_path.rglob("*"):
+                if not f.is_file():
+                    continue
+                # skip unwanted directories
+                if _SKIP_DIRS & set(f.parts):
+                    continue
+                fname = f.name
+                fname_lower = fname.lower()
+                try:
+                    # Build files — exact name match
+                    if fname in _BUILD_NAMES:
+                        build_files[fname] = f.read_text(encoding="utf-8", errors="ignore")
+                    # Config files — pattern match (all profiles and formats)
+                    elif (
+                        fname_lower.startswith("application") and fname_lower.endswith((".properties", ".yml", ".yaml"))
+                        or fname_lower.startswith("bootstrap") and fname_lower.endswith((".properties", ".yml", ".yaml"))
+                        or fname_lower in (".env", ".env.production", ".env.staging", ".env.development", ".env.local")
+                        or fname_lower.startswith(".env.") and not fname_lower.endswith((".js", ".ts", ".py"))
+                        # SQL query properties files (e.g. webServiceSql.properties, commonSql.properties)
+                        or fname_lower.endswith("sql.properties")
+                        or fname_lower.endswith("sql.props")
+                    ):
+                        # Use relative path as key to avoid collisions between profiles
+                        rel_key = str(f.relative_to(repository_path))
+                        config_files[rel_key] = f.read_text(encoding="utf-8", errors="ignore")
+                except Exception:
+                    pass
+
+            dependency_graph = self.dependency_extractor.extract_dependencies(
+                ast_nodes=ast_nodes,
+                build_files=build_files,
+                config_files=config_files,
+                repository=repository_name,
+            )
+
+            # Cross-language linking: match frontend API calls to backend endpoints
+            cross_links = self.dependency_extractor.link_cross_language(dependency_graph)
+
+            logger.info(
+                "dependencies_extracted",
+                repository=repository_name,
+                nodes_count=len(dependency_graph.nodes) if hasattr(dependency_graph, 'nodes') else 0,
+                cross_language_links=cross_links
+            )
+
+            # STAGE 5: Chunking (65%)
+            tracker.update("chunking", "Creating semantic code chunks")
+
+            # Build source_code dictionary from all files (Java + TypeScript + JavaScript)
+            source_code = {}
+
+            # Add Java files
+            for java_file in java_files:
+                try:
+                    source_code[str(java_file)] = java_file.read_text(encoding='utf-8')
+                except Exception as e:
+                    logger.warning(
+                        "source_code_read_failed",
+                        file=str(java_file),
+                        error=str(e)
+                    )
+
+            # Add TypeScript files
+            for ts_file in typescript_files:
+                try:
+                    source_code[str(ts_file)] = ts_file.read_text(encoding='utf-8')
+                except Exception as e:
+                    logger.warning(
+                        "source_code_read_failed",
+                        file=str(ts_file),
+                        error=str(e)
+                    )
+
+            # Add JavaScript files
+            for js_file in javascript_files:
+                try:
+                    source_code[str(js_file)] = js_file.read_text(encoding='utf-8')
+                except Exception as e:
+                    logger.warning(
+                        "source_code_read_failed",
+                        file=str(js_file),
+                        error=str(e)
+                    )
+
+            # Build chunks from Java files (existing logic)
+            chunks = self.chunk_builder.build_chunks(
+                ast_nodes=ast_nodes,
+                source_code=source_code
+            )
+
+            # Build chunks from TypeScript/JavaScript files (Task 5.1-5.2)
+            if self.typescript_parser and (typescript_files or javascript_files):
+                ts_chunks = self.chunk_builder.build_typescript_chunks(
+                    parse_results=ts_parsed,
+                    source_code=source_code
+                )
+                chunks.extend(ts_chunks)
+
+            logger.info(
+                "chunks_created",
+                repository=repository_name,
+                chunks_count=len(chunks)
+            )
+
+            # STAGE 6: Embedding Generation (77%)
+            tracker.update("embeddings", f"Generating embeddings for {len(chunks)} chunks")
+
+            # Extract chunk contents for embedding
+            chunk_contents = [chunk.content for chunk in chunks]
+
+            # Generate embeddings with batch_size=10
+            embeddings = await self.embedding_provider.embed_batch(
+                chunk_contents,
+                batch_size=10
+            )
+
+            logger.info(
+                "embeddings_generated",
+                repository=repository_name,
+                embeddings_count=len(embeddings)
+            )
+
+            # STAGE 7: Qdrant Storage (90%)
+            tracker.update("qdrant", "Storing vectors in Qdrant")
+
+            # Stamp repository name on every chunk before storing
+            for chunk in chunks:
+                if not chunk.metadata.get("repository"):
+                    chunk.metadata["repository"] = repository_name
+
+            qdrant_result = await self.qdrant_repository.upsert_batch(
+                chunks,
+                embeddings,
+                batch_size=1000
+            )
+
+            logger.info(
+                "vectors_stored",
+                repository=repository_name,
+                points_stored=qdrant_result.points_count if hasattr(qdrant_result, 'points_count') else len(chunks)
+            )
+
+            # STAGE 8: Neo4j Storage (97%)
+            tracker.update("neo4j", "Creating dependency graph in Neo4j")
+
+            neo4j_result = await self.neo4j_repository.store_dependency_graph(
+                dependency_graph,
+                repository_name
+            )
+
+            logger.info(
+                "graph_stored",
+                repository=repository_name,
+                nodes_created=neo4j_result.nodes_created,
+                relationships_created=neo4j_result.relationships_created
+            )
+
+            # COMPLETE (100%)
+            tracker.complete(f"Repository {repository_name} indexed successfully")
+
+            logger.info(
+                "indexing_completed",
+                repository=repository_name,
+                status="success"
+            )
+
+            # Return success result
+            return IndexResult(
+                status="success",
+                repository_name=repository_name,
+                files_processed=len(java_files),
+                chunks_created=len(chunks),
+                nodes_created=neo4j_result.nodes_created,
+                relationships_created=neo4j_result.relationships_created
+            )
+
+        except ValueError as e:
+            # Validation errors
+            logger.error(
+                "indexing_failed_validation",
+                repository=repository_name,
+                error=str(e)
+            )
+            tracker.fail(f"Validation error: {str(e)}")
+            raise
+
+        except Exception as e:
+            # Critical errors
+            logger.error(
+                "indexing_failed",
+                repository=repository_name,
+                error=str(e),
+                error_type=type(e).__name__
+            )
+            tracker.fail(f"Indexing failed: {str(e)}")
+
+            return IndexResult(
+                status="error",
+                repository_name=repository_name,
+                files_processed=0,
+                chunks_created=0,
+                nodes_created=0,
+                relationships_created=0,
+                error_message=str(e)
+            )