graph-dependency-analyzer 0.2.0__py3-none-any.whl

src/application/exceptions.py

@@ -0,0 +1,41 @@
+"""
+Application Layer Exceptions (Task 9.3)
+
+Exception hierarchy for the application layer.
+"""
+
+
+class ApplicationException(Exception):
+    """Base exception for application layer errors."""
+
+    def __init__(self, message: str, details: dict | None = None):
+        """
+        Initialize application exception.
+
+        Args:
+            message: Human-readable error message
+            details: Optional dict with additional context
+        """
+        super().__init__(message)
+        self.message = message
+        self.details = details or {}
+
+
+class IndexingError(ApplicationException):
+    """Error during indexing operations."""
+    pass
+
+
+class RepositoryNotFoundError(ApplicationException):
+    """Repository not found in filesystem."""
+    pass
+
+
+class NoValidRepositoriesError(ApplicationException):
+    """No valid repositories found in the provided list."""
+    pass
+
+
+class EmptyRepositoryListError(ApplicationException):
+    """Repository list is empty."""
+    pass

src/application/services/file_scanner_service.py

@@ -0,0 +1,258 @@
+"""
+FileScannerService Implementation (Task 7.1)
+
+Scans repository directory for Java files, filtering out build artifacts
+and large/binary files.
+"""
+from pathlib import Path
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+class FileScannerService:
+    """
+    Application service for scanning repository for valid Java and TypeScript/JavaScript files.
+
+    Filters out:
+    - Build directories (target/, build/, .git/, .idea/, node_modules/, dist/)
+    - Large files (>1MB)
+    - Binary files
+    - Non-source files
+
+    Requirements: 3.3, 3.4, 3.5, 8.1, 12.1 (Task 6.1)
+    """
+
+    # Directories to skip during scanning
+    SKIP_DIRECTORIES = {
+        "target", "build", ".git", ".idea", "node_modules", ".gradle", "dist",
+        # test / load-test dirs
+        "k6-tests", "__tests__", "e2e", "cypress", "playwright",
+        # frontend build outputs
+        ".next", ".nuxt", ".svelte-kit", "coverage", "storybook-static",
+    }
+
+    # File name suffixes to skip (test files)
+    SKIP_FILE_SUFFIXES = {
+        ".test.ts", ".test.tsx", ".test.js", ".test.jsx",
+        ".spec.ts", ".spec.tsx", ".spec.js", ".spec.jsx",
+        ".stories.ts", ".stories.tsx", ".stories.js",
+    }
+
+    # TypeScript/JavaScript file extensions
+    TYPESCRIPT_EXTENSIONS = {".ts", ".tsx"}
+    JAVASCRIPT_EXTENSIONS = {".js", ".jsx"}
+
+    # SQL / DB schema file extensions
+    SQL_EXTENSIONS = {".sql", ".SQL", ".pks", ".pkb", ".prc", ".fnc",
+                      ".trg", ".vw", ".tbl", ".seq", ".idx", ".typ"}
+
+    # Python file extensions
+    PYTHON_EXTENSIONS = {".py", ".pyw"}
+
+    # Groovy / Jenkinsfile extensions
+    GROOVY_EXTENSIONS = {".groovy"}
+
+    # Maximum file size (1MB)
+    MAX_FILE_SIZE = 1024 * 1024  # 1MB in bytes
+
+    def scan(self, repository_path: Path) -> dict[str, list[Path]]:
+        """
+        Scan repository for valid Java, TypeScript, and JavaScript files.
+
+        Args:
+            repository_path: Absolute path to repository root
+
+        Returns:
+            Dict with keys "java", "typescript", "javascript" containing lists of file paths
+
+        Raises:
+            ValueError: If path does not exist or is not a directory
+        """
+        # Convert to Path if string
+        if isinstance(repository_path, str):
+            repository_path = Path(repository_path)
+
+        # Validate path exists
+        if not repository_path.exists():
+            raise ValueError(f"Path does not exist: {repository_path}")
+
+        # Validate path is a directory
+        if not repository_path.is_dir():
+            raise ValueError(f"Path is not a directory: {repository_path}")
+
+        # Resolve to absolute path
+        repository_path = repository_path.resolve()
+
+        java_files = []
+        typescript_files = []
+        javascript_files = []
+        sql_files = []
+        python_files = []
+        groovy_files = []
+        other_extensions = {}
+        large_files_count = 0
+        binary_files_count = 0
+
+        # Walk directory tree
+        for file_path in repository_path.rglob("*"):
+            # Skip if not a file
+            if not file_path.is_file():
+                continue
+
+            # Skip if in excluded directory
+            if self._is_in_skip_directory(file_path, repository_path):
+                continue
+
+            # Skip test/spec files by name suffix
+            if self._is_test_file(file_path):
+                continue
+
+            # Count all file extensions
+            ext = file_path.suffix
+            fname_lower = file_path.name.lower()
+            if ext:
+                other_extensions[ext] = other_extensions.get(ext, 0) + 1
+
+            # SQL query properties files (e.g. webServiceSql.properties)
+            is_sql_props = (
+                ext == ".properties"
+                and (fname_lower.endswith("sql.properties") or fname_lower.endswith("sql.props"))
+            )
+
+            # Skip files without supported extensions
+            all_supported = {".java"} | self.TYPESCRIPT_EXTENSIONS | self.JAVASCRIPT_EXTENSIONS | self.SQL_EXTENSIONS | self.PYTHON_EXTENSIONS | self.GROOVY_EXTENSIONS
+            if ext not in all_supported and not is_sql_props:
+                continue
+
+            # Check file size
+            try:
+                file_size = file_path.stat().st_size
+                if file_size > self.MAX_FILE_SIZE:
+                    large_files_count += 1
+                    logger.warning(
+                        "file_too_large",
+                        file_path=str(file_path),
+                        size_mb=file_size / (1024 * 1024)
+                    )
+                    continue
+            except OSError as e:
+                logger.warning(
+                    "file_stat_error",
+                    file_path=str(file_path),
+                    error=str(e)
+                )
+                continue
+
+            # Check if file is text (not binary)
+            if not self._is_text_file(file_path):
+                binary_files_count += 1
+                logger.warning(
+                    "binary_file_skipped",
+                    file_path=str(file_path)
+                )
+                continue
+
+            # Categorize by extension
+            if ext == ".java":
+                java_files.append(file_path)
+            elif ext in self.TYPESCRIPT_EXTENSIONS:
+                typescript_files.append(file_path)
+            elif ext in self.JAVASCRIPT_EXTENSIONS:
+                javascript_files.append(file_path)
+            elif ext in self.SQL_EXTENSIONS or is_sql_props:
+                sql_files.append(file_path)
+            elif ext in self.PYTHON_EXTENSIONS:
+                python_files.append(file_path)
+            elif ext in self.GROOVY_EXTENSIONS:
+                groovy_files.append(file_path)
+            elif fname_lower in ('jenkinsfile',) or fname_lower.startswith('jenkinsfile'):
+                groovy_files.append(file_path)
+
+        # Log summary
+        logger.info(
+            "repository_scan_complete",
+            repository=str(repository_path),
+            java_files_found=len(java_files),
+            typescript_files_found=len(typescript_files),
+            javascript_files_found=len(javascript_files),
+            sql_files_found=len(sql_files),
+            python_files_found=len(python_files),
+            groovy_files_found=len(groovy_files),
+            large_files_filtered=large_files_count,
+            binary_files_filtered=binary_files_count,
+            extensions_found=dict(other_extensions)
+        )
+
+        return {
+            "java": java_files,
+            "typescript": typescript_files,
+            "javascript": javascript_files,
+            "sql": sql_files,
+            "python": python_files,
+            "groovy": groovy_files,
+        }
+
+    def _is_in_skip_directory(self, file_path: Path, repository_root: Path) -> bool:
+        """
+        Check if file is inside a directory that should be skipped.
+
+        Args:
+            file_path: Absolute path to file
+            repository_root: Repository root path
+
+        Returns:
+            True if file is in a skip directory
+        """
+        # Get relative path from repository root
+        try:
+            relative_path = file_path.relative_to(repository_root)
+        except ValueError:
+            # File is not relative to repository root
+            return False
+
+        # Check all parts of the path
+        for part in relative_path.parts:
+            if part in self.SKIP_DIRECTORIES:
+                return True
+
+        return False
+
+    def _is_test_file(self, file_path: Path) -> bool:
+        """Check if file is a test/spec file by name pattern."""
+        name = file_path.name
+        return any(name.endswith(suffix) for suffix in self.SKIP_FILE_SUFFIXES)
+
+    def _is_text_file(self, file_path: Path) -> bool:
+        """
+        Check if file is a text file (not binary).
+
+        Args:
+            file_path: Path to file
+
+        Returns:
+            True if file appears to be text
+        """
+        try:
+            # Read first 8192 bytes
+            with open(file_path, 'rb') as f:
+                chunk = f.read(8192)
+
+            # Check for null bytes (indicator of binary file)
+            if b'\x00' in chunk:
+                return False
+
+            # Try to decode as UTF-8
+            try:
+                chunk.decode('utf-8')
+                return True
+            except UnicodeDecodeError:
+                # Try other common encodings
+                try:
+                    chunk.decode('latin-1')
+                    return True
+                except UnicodeDecodeError:
+                    return False
+
+        except Exception:
+            return False

src/application/services/progress_tracker.py

@@ -0,0 +1,132 @@
+"""
+ProgressTracker Implementation (Task 7.2)
+
+Tracks progress of repository indexing through 8 stages with percentage mapping.
+"""
+from typing import Callable, TypedDict
+
+
+class ProgressUpdate(TypedDict):
+    """Progress update data structure"""
+    repository_name: str
+    stage: str
+    message: str
+    percentage: int
+    status: str
+
+
+class ProgressTracker:
+    """
+    Application service for tracking indexing progress.
+
+    Maps 8 stages to percentage ranges:
+    - validation: 0%
+    - scan: 0-10% (midpoint: 5%)
+    - ast: 10-40% (midpoint: 25%)
+    - deps: 40-60% (midpoint: 50%)
+    - chunking: 60-70% (midpoint: 65%)
+    - embeddings: 70-85% (midpoint: 77%)
+    - qdrant: 85-95% (midpoint: 90%)
+    - neo4j: 95-100% (midpoint: 97%)
+
+    Requirements: 2.7, 3.15
+    """
+
+    # Stage to percentage mapping (midpoint of ranges)
+    STAGE_PERCENTAGES = {
+        "validation": 0,
+        "scan": 5,
+        "ast": 25,
+        "deps": 50,
+        "chunking": 65,
+        "embeddings": 77,
+        "qdrant": 90,
+        "neo4j": 97,
+    }
+
+    def __init__(
+        self,
+        repository_name: str,
+        callback: Callable[[ProgressUpdate], None] | None = None
+    ):
+        """
+        Initialize progress tracker.
+
+        Args:
+            repository_name: Name of repository being indexed
+            callback: Optional callback to invoke on progress updates
+        """
+        self.repository_name = repository_name
+        self.callback = callback
+        self.current_percentage = 0
+        self.current_stage = ""
+        self.current_status = "in_progress"
+
+    def update(self, stage: str, message: str) -> None:
+        """
+        Update progress to a specific stage.
+
+        Args:
+            stage: Stage name (validation, scan, ast, deps, chunking, embeddings, qdrant, neo4j)
+            message: Progress message
+
+        Raises:
+            ValueError: If stage is not recognized
+        """
+        if stage not in self.STAGE_PERCENTAGES:
+            raise ValueError(f"Unknown stage: {stage}")
+
+        self.current_stage = stage
+        self.current_percentage = self.STAGE_PERCENTAGES[stage]
+        self.current_status = "in_progress"
+
+        if self.callback:
+            progress_data: ProgressUpdate = {
+                "repository_name": self.repository_name,
+                "stage": stage,
+                "message": message,
+                "percentage": self.current_percentage,
+                "status": self.current_status,
+            }
+            self.callback(progress_data)
+
+    def complete(self, message: str = "Completed") -> None:
+        """
+        Mark indexing as complete (100%).
+
+        Args:
+            message: Completion message
+        """
+        self.current_percentage = 100
+        self.current_status = "completed"
+
+        if self.callback:
+            progress_data: ProgressUpdate = {
+                "repository_name": self.repository_name,
+                "stage": "complete",
+                "message": message,
+                "percentage": 100,
+                "status": "completed",
+            }
+            self.callback(progress_data)
+
+    def fail(self, message: str) -> None:
+        """
+        Mark indexing as failed.
+
+        Keeps current percentage but sets status to error.
+
+        Args:
+            message: Error message
+        """
+        self.current_status = "error"
+
+        if self.callback:
+            progress_data: ProgressUpdate = {
+                "repository_name": self.repository_name,
+                "stage": self.current_stage,
+                "message": message,
+                "percentage": self.current_percentage,
+                "status": "error",
+            }
+            self.callback(progress_data)

src/application/use_cases/clear_all_data_use_case.py

@@ -0,0 +1,180 @@
+"""Use case for clearing all data from Qdrant and Neo4j."""
+import structlog
+
+from src.infrastructure.persistence.neo4j_dependency_repository import Neo4jDependencyGraphRepository
+from qdrant_client import QdrantClient
+
+logger = structlog.get_logger(__name__)
+
+
+class ClearAllDataUseCase:
+    """
+    Use case for clearing all indexed data.
+
+    Deletes:
+    - All collections in Qdrant
+    - All nodes and relationships in Neo4j
+
+    ⚠️ DESTRUCTIVE OPERATION - Use with caution!
+    """
+
+    def __init__(
+        self,
+        graph_repository: Neo4jDependencyGraphRepository,
+        qdrant_client: QdrantClient,
+        collection_name: str
+    ):
+        """
+        Initialize use case.
+
+        Args:
+            graph_repository: Neo4j dependency graph repository
+            qdrant_client: Qdrant client
+            collection_name: Name of Qdrant collection to delete
+        """
+        self.graph_repository = graph_repository
+        self.qdrant_client = qdrant_client
+        self.collection_name = collection_name
+
+    async def execute(self) -> dict:
+        """
+        Execute clear all operation.
+
+        Returns:
+            Dict with operation results
+        """
+        logger.warning("clear_all_data_starting", operation="DESTRUCTIVE")
+
+        results = {
+            "neo4j": {"success": False, "nodes_deleted": 0, "relationships_deleted": 0},
+            "qdrant": {"success": False, "collection_deleted": False}
+        }
+
+        # 1. Clear Neo4j
+        try:
+            neo4j_result = await self._clear_neo4j()
+            results["neo4j"] = neo4j_result
+            logger.info(
+                "neo4j_cleared",
+                nodes_deleted=neo4j_result["nodes_deleted"],
+                relationships_deleted=neo4j_result["relationships_deleted"]
+            )
+        except Exception as e:
+            logger.error(
+                "neo4j_clear_failed",
+                error=str(e),
+                error_type=type(e).__name__
+            )
+            results["neo4j"]["error"] = str(e)
+
+        # 2. Clear Qdrant
+        try:
+            qdrant_result = await self._clear_qdrant()
+            results["qdrant"] = qdrant_result
+            logger.info(
+                "qdrant_cleared",
+                collection_deleted=qdrant_result["collection_deleted"]
+            )
+        except Exception as e:
+            logger.error(
+                "qdrant_clear_failed",
+                error=str(e),
+                error_type=type(e).__name__
+            )
+            results["qdrant"]["error"] = str(e)
+
+        logger.warning("clear_all_data_completed", results=results)
+
+        return results
+
+    async def _clear_neo4j(self) -> dict:
+        """
+        Clear all nodes and relationships from Neo4j.
+
+        Returns:
+            Dict with deletion metrics
+        """
+        async with self.graph_repository.driver.session() as session:
+            # Get counts before deletion
+            count_result = await session.run("""
+                MATCH (n)
+                OPTIONAL MATCH (n)-[r]->()
+                RETURN count(DISTINCT n) as nodes, count(r) as rels
+            """)
+            count_record = await count_result.single()
+            nodes_before = count_record["nodes"]
+            rels_before = count_record["rels"]
+
+            # Delete in batches of 10k to avoid Neo4j heap overflow
+            # (single DETACH DELETE on large graphs silently fails with OOM)
+            total_deleted = 0
+            while True:
+                result = await session.run("""
+                    MATCH (n)
+                    WITH n LIMIT 10000
+                    DETACH DELETE n
+                    RETURN count(*) AS deleted
+                """)
+                row = await result.single()
+                batch = row["deleted"] if row else 0
+                total_deleted += batch
+                if batch == 0:
+                    break
+
+            logger.debug(
+                "neo4j_deletion_executed",
+                nodes_deleted=total_deleted,
+                relationships_deleted=rels_before
+            )
+
+            return {
+                "success": True,
+                "nodes_deleted": total_deleted,
+                "relationships_deleted": rels_before
+            }
+
+    async def _clear_qdrant(self) -> dict:
+        """
+        Clear Qdrant collection.
+
+        Returns:
+            Dict with deletion result
+        """
+        try:
+            # Check if collection exists
+            collections = self.qdrant_client.get_collections().collections
+            collection_exists = any(c.name == self.collection_name for c in collections)
+
+            if collection_exists:
+                # Delete collection
+                self.qdrant_client.delete_collection(collection_name=self.collection_name)
+
+                logger.debug(
+                    "qdrant_collection_deleted",
+                    collection=self.collection_name
+                )
+
+                return {
+                    "success": True,
+                    "collection_deleted": True,
+                    "collection_name": self.collection_name
+                }
+            else:
+                logger.debug(
+                    "qdrant_collection_not_found",
+                    collection=self.collection_name
+                )
+
+                return {
+                    "success": True,
+                    "collection_deleted": False,
+                    "message": f"Collection '{self.collection_name}' does not exist"
+                }
+
+        except Exception as e:
+            logger.error(
+                "qdrant_deletion_failed",
+                error=str(e),
+                collection=self.collection_name
+            )
+            raise