emdash-core 0.1.7__py3-none-any.whl

emdash_core/ingestion/parsers/typescript_parser.py

@@ -0,0 +1,278 @@
+"""TypeScript/JavaScript parser using external TypeScript compiler."""
+
+import json
+import subprocess
+import hashlib
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+from .base_parser import BaseLanguageParser
+from ...core.models import (
+    FileEntity, FileEntities, ClassEntity, FunctionEntity,
+    ModuleEntity, ImportStatement
+)
+from ...core.exceptions import ParsingError
+from ...utils.logger import log
+
+
+class TypeScriptParser(BaseLanguageParser):
+    """Parses TypeScript/JavaScript files using TypeScript compiler API."""
+
+    def __init__(self, file_path: Path, repo_root: Optional[Path] = None):
+        """Initialize TypeScript parser.
+
+        Args:
+            file_path: Path to TypeScript/JavaScript file
+            repo_root: Root directory of repository (for resolving qualified names)
+        """
+        super().__init__(file_path, repo_root)
+        self.content: Optional[str] = None
+
+    @classmethod
+    def get_supported_extensions(cls) -> List[str]:
+        """Return list of supported file extensions.
+
+        Returns:
+            List containing '.ts', '.tsx', '.js', '.jsx'
+        """
+        return ['.ts', '.tsx', '.js', '.jsx']
+
+    def parse(self) -> FileEntities:
+        """Parse TypeScript/JavaScript file and extract entities.
+
+        Returns:
+            FileEntities containing all extracted entities
+
+        Raises:
+            ParsingError: If file cannot be parsed
+        """
+        try:
+            # Read file content
+            with open(self.file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                self.content = f.read()
+
+            # Parse using TypeScript compiler (via Node.js)
+            ast = self._parse_typescript_ast()
+
+            if not ast:
+                log.warning(f"Failed to parse TypeScript AST for {self.file_path}")
+                return FileEntities()
+
+            # Extract file metadata
+            file_entity = self._extract_file_entity()
+
+            # Extract entities from AST
+            classes, class_methods = self._extract_classes(ast)
+            functions = self._extract_functions(ast)
+            # Add class methods to functions list
+            functions.extend(class_methods)
+            imports, modules = self._extract_imports(ast)
+
+            return FileEntities(
+                file=file_entity,
+                classes=classes,
+                functions=functions,
+                modules=modules,
+                imports=imports,
+            )
+
+        except Exception as e:
+            log.warning(f"Failed to parse TypeScript file {self.file_path}: {e}")
+            return FileEntities()
+
+    def _parse_typescript_ast(self) -> Optional[Dict]:
+        """Parse TypeScript using ts-node parser script.
+
+        Returns:
+            Dictionary containing AST data or None if parsing fails
+        """
+        # Call Node.js script that uses TypeScript compiler API
+        parser_script = Path(__file__).parent / "ts_ast_parser.js"
+
+        if not parser_script.exists():
+            log.warning(f"TypeScript parser script not found: {parser_script}")
+            return None
+
+        try:
+            result = subprocess.run(
+                ['node', str(parser_script), str(self.file_path)],
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+
+            if result.returncode != 0:
+                log.warning(f"TypeScript parser failed for {self.file_path}: {result.stderr}")
+                return None
+
+            return json.loads(result.stdout)
+
+        except subprocess.TimeoutExpired:
+            log.warning(f"TypeScript parsing timeout for {self.file_path}")
+            return None
+        except json.JSONDecodeError as e:
+            log.warning(f"Failed to parse TypeScript AST JSON: {e}")
+            return None
+        except FileNotFoundError:
+            log.warning("Node.js not found. Please install Node.js to parse TypeScript/JavaScript files.")
+            return None
+
+    def _extract_file_entity(self) -> FileEntity:
+        """Extract file metadata.
+
+        Returns:
+            FileEntity with file metadata
+        """
+        content_hash = hashlib.sha256(self.content.encode()).hexdigest()
+        return FileEntity.from_path(self.file_path, content_hash)
+
+    def _extract_classes(self, ast: Dict) -> tuple[List[ClassEntity], List[FunctionEntity]]:
+        """Extract class/interface definitions from TypeScript AST.
+
+        Args:
+            ast: Parsed AST dictionary
+
+        Returns:
+            Tuple of (ClassEntity list, FunctionEntity list for methods)
+        """
+        classes = []
+        all_methods = []
+
+        for cls_data in ast.get('classes', []):
+            try:
+                # Build qualified name
+                class_qualified_name = f"{self.module_name}.{cls_data['name']}" if self.module_name else cls_data['name']
+
+                class_entity = ClassEntity(
+                    name=cls_data['name'],
+                    qualified_name=class_qualified_name,
+                    file_path=str(self.file_path),
+                    line_start=cls_data.get('line_start', 0),
+                    line_end=cls_data.get('line_end', 0),
+                    docstring=cls_data.get('docstring'),
+                    is_abstract=cls_data.get('is_abstract', False),
+                    decorators=cls_data.get('decorators', []),
+                    base_classes=cls_data.get('base_classes', []),
+                    attributes=cls_data.get('properties', []),
+                    methods=cls_data.get('methods', []),
+                )
+                classes.append(class_entity)
+
+                # Extract method entities with calls
+                for method_data in cls_data.get('method_entities', []):
+                    try:
+                        method_qualified_name = f"{class_qualified_name}.{method_data['name']}"
+                        method_entity = FunctionEntity(
+                            name=method_data['name'],
+                            qualified_name=method_qualified_name,
+                            file_path=str(self.file_path),
+                            line_start=method_data.get('line_start', 0),
+                            line_end=method_data.get('line_end', 0),
+                            docstring=method_data.get('docstring'),
+                            parameters=method_data.get('parameters', []),
+                            return_annotation=method_data.get('return_type'),
+                            is_async=method_data.get('is_async', False),
+                            is_method=True,
+                            is_static=method_data.get('is_static', False),
+                            is_classmethod=False,
+                            decorators=method_data.get('decorators', []),
+                            cyclomatic_complexity=0,
+                            calls=method_data.get('calls', []),
+                        )
+                        all_methods.append(method_entity)
+                    except Exception as e:
+                        log.warning(f"Failed to extract method {method_data.get('name', 'unknown')}: {e}")
+
+            except Exception as e:
+                log.warning(f"Failed to extract class {cls_data.get('name', 'unknown')}: {e}")
+
+        return classes, all_methods
+
+    def _extract_functions(self, ast: Dict) -> List[FunctionEntity]:
+        """Extract function/method definitions from TypeScript AST.
+
+        Args:
+            ast: Parsed AST dictionary
+
+        Returns:
+            List of FunctionEntity objects
+        """
+        functions = []
+
+        for func_data in ast.get('functions', []):
+            try:
+                # Build qualified name
+                qualified_name = f"{self.module_name}.{func_data['name']}" if self.module_name else func_data['name']
+
+                function_entity = FunctionEntity(
+                    name=func_data['name'],
+                    qualified_name=qualified_name,
+                    file_path=str(self.file_path),
+                    line_start=func_data.get('line_start', 0),
+                    line_end=func_data.get('line_end', 0),
+                    docstring=func_data.get('docstring'),
+                    parameters=func_data.get('parameters', []),
+                    return_annotation=func_data.get('return_type'),
+                    is_async=func_data.get('is_async', False),
+                    is_method=func_data.get('is_method', False),
+                    is_static=func_data.get('is_static', False),
+                    is_classmethod=False,  # TypeScript doesn't have classmethods like Python
+                    decorators=func_data.get('decorators', []),
+                    cyclomatic_complexity=0,  # TODO: Calculate complexity
+                    calls=func_data.get('calls', []),
+                )
+                functions.append(function_entity)
+            except Exception as e:
+                log.warning(f"Failed to extract function {func_data.get('name', 'unknown')}: {e}")
+
+        return functions
+
+    def _extract_imports(self, ast: Dict) -> tuple[List[ImportStatement], List[ModuleEntity]]:
+        """Extract import statements from TypeScript AST.
+
+        Args:
+            ast: Parsed AST dictionary
+
+        Returns:
+            Tuple of (import_statements, module_entities)
+        """
+        imports = []
+        modules = {}
+
+        for import_data in ast.get('imports', []):
+            try:
+                module_name = import_data['module']
+                line_number = import_data.get('line_number', 0)
+
+                # Determine if module is external (from node_modules)
+                is_external = not module_name.startswith('.') and not module_name.startswith('/')
+
+                # Create import statement
+                import_stmt = ImportStatement(
+                    module=module_name,
+                    file_path=str(self.file_path),
+                    line_number=line_number,
+                    import_type='import',  # TypeScript uses 'import'
+                    alias=import_data.get('alias'),
+                    imported_names=import_data.get('imported_names', []),
+                )
+                imports.append(import_stmt)
+
+                # Create module entity if not already exists
+                if module_name not in modules:
+                    modules[module_name] = ModuleEntity(
+                        name=module_name,
+                        import_path=module_name,
+                        is_external=is_external,
+                        package=module_name.split('/')[0] if '/' in module_name else module_name,
+                    )
+
+            except Exception as e:
+                log.warning(f"Failed to extract import: {e}")
+
+        return imports, list(modules.values())
+
+
+# Auto-register TypeScript parser with registry
+from .registry import ParserRegistry
+ParserRegistry.register(TypeScriptParser)

emdash_core/ingestion/repository.py

@@ -0,0 +1,346 @@
+"""Repository management - clone, fetch, and track Git repositories."""
+
+import hashlib
+import os
+import shutil
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+import git
+from git import Repo, GitCommandError
+
+from ..core.exceptions import RepositoryError
+from ..core.models import RepositoryEntity
+from ..utils.logger import log
+
+
+class RepositoryManager:
+    """Manages Git repository operations (clone, fetch, status)."""
+
+    def __init__(self, cache_dir: Optional[Path] = None):
+        """Initialize repository manager.
+
+        Args:
+            cache_dir: Directory to cache cloned repositories.
+                      Defaults to ~/.emdash/repos
+        """
+        if cache_dir is None:
+            cache_dir = Path.home() / ".emdash" / "repos"
+
+        self.cache_dir = cache_dir
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def get_or_clone(
+        self,
+        repo_path: str,
+        skip_commit_count: bool = False
+    ) -> tuple[Repo, RepositoryEntity]:
+        """Get a repository from cache or clone it.
+
+        Args:
+            repo_path: URL or local path to repository
+            skip_commit_count: Whether to skip counting commits
+
+        Returns:
+            Tuple of (git.Repo, RepositoryEntity)
+
+        Raises:
+            RepositoryError: If repository cannot be accessed
+        """
+        # Check if it's a local path
+        if Path(repo_path).exists():
+            return self._open_local_repo(repo_path)
+
+        # It's a URL - clone or fetch
+        return self._clone_or_fetch(repo_path, skip_commit_count)
+
+    def _open_local_repo(self, path: str) -> tuple[Repo, RepositoryEntity]:
+        """Open a local repository.
+
+        Args:
+            path: Local path to repository
+
+        Returns:
+            Tuple of (git.Repo, RepositoryEntity)
+        """
+        log.info(f"Opening local repository: {path}")
+
+        try:
+            repo = Repo(path)
+
+            # Get repository info
+            origin_url = self._get_origin_url(repo)
+            repo_name = Path(path).name
+
+            entity = RepositoryEntity(
+                url=origin_url or f"file://{path}",
+                name=repo_name,
+                owner=None,
+                default_branch=repo.active_branch.name,
+                last_ingested=None,
+                ingestion_status="pending",
+            )
+
+            return repo, entity
+
+        except Exception as e:
+            raise RepositoryError(f"Failed to open local repository {path}: {e}")
+
+    def _clone_or_fetch(
+        self,
+        url: str,
+        skip_commit_count: bool
+    ) -> tuple[Repo, RepositoryEntity]:
+        """Clone a repository or fetch updates if already cloned.
+
+        Args:
+            url: Repository URL
+            skip_commit_count: Whether to skip counting commits
+
+        Returns:
+            Tuple of (git.Repo, RepositoryEntity)
+        """
+        # Generate cache path from URL
+        cache_path = self._get_cache_path(url)
+
+        if cache_path.exists():
+            log.info(f"Repository already cached at {cache_path}")
+            return self._fetch_updates(cache_path, url, skip_commit_count)
+        else:
+            log.info(f"Cloning repository: {url}")
+            return self._clone_repo(url, cache_path, skip_commit_count)
+
+    def _clone_repo(
+        self,
+        url: str,
+        cache_path: Path,
+        skip_commit_count: bool
+    ) -> tuple[Repo, RepositoryEntity]:
+        """Clone a repository.
+
+        Args:
+            url: Repository URL
+            cache_path: Path to clone into
+            skip_commit_count: Whether to skip counting commits
+
+        Returns:
+            Tuple of (git.Repo, RepositoryEntity)
+        """
+        try:
+            repo = Repo.clone_from(url, cache_path, depth=None)
+            log.info(f"Successfully cloned {url}")
+
+            entity = self._create_repository_entity(
+                repo,
+                url,
+                skip_commit_count=skip_commit_count
+            )
+            return repo, entity
+
+        except GitCommandError as e:
+            raise RepositoryError(f"Failed to clone repository {url}: {e}")
+        except Exception as e:
+            raise RepositoryError(f"Unexpected error cloning {url}: {e}")
+
+    def _fetch_updates(
+        self,
+        cache_path: Path,
+        url: str,
+        skip_commit_count: bool
+    ) -> tuple[Repo, RepositoryEntity]:
+        """Fetch updates for an existing repository.
+
+        Args:
+            cache_path: Path to cached repository
+            url: Repository URL
+            skip_commit_count: Whether to skip counting commits
+
+        Returns:
+            Tuple of (git.Repo, RepositoryEntity)
+        """
+        try:
+            repo = Repo(cache_path)
+
+            log.info("Fetching updates from remote...")
+            repo.remotes.origin.fetch()
+
+            # Pull latest changes
+            repo.remotes.origin.pull()
+
+            log.info("Repository updated successfully")
+
+            entity = self._create_repository_entity(
+                repo,
+                url,
+                skip_commit_count=skip_commit_count
+            )
+            return repo, entity
+
+        except GitCommandError as e:
+            raise RepositoryError(f"Failed to fetch updates for {url}: {e}")
+        except Exception as e:
+            raise RepositoryError(f"Unexpected error fetching updates: {e}")
+
+    def _create_repository_entity(
+        self,
+        repo: Repo,
+        url: str,
+        skip_commit_count: bool = False
+    ) -> RepositoryEntity:
+        """Create a RepositoryEntity from a git.Repo.
+
+        Args:
+            repo: Git repository
+            url: Repository URL
+            skip_commit_count: Whether to skip counting commits
+
+        Returns:
+            RepositoryEntity
+        """
+        # Parse URL to extract owner and name
+        parsed = urlparse(url)
+        path_parts = parsed.path.strip("/").split("/")
+
+        if len(path_parts) >= 2:
+            owner = path_parts[-2]
+            repo_name = path_parts[-1].replace(".git", "")
+        else:
+            owner = None
+            repo_name = path_parts[-1].replace(".git", "") if path_parts else "unknown"
+
+        commit_count = 0
+        if not skip_commit_count:
+            try:
+                commit_count = sum(1 for _ in repo.iter_commits())
+            except Exception:
+                commit_count = 0
+
+        return RepositoryEntity(
+            url=url,
+            name=repo_name,
+            owner=owner,
+            default_branch=repo.active_branch.name if repo.active_branch else "main",
+            last_ingested=None,
+            ingestion_status="pending",
+            commit_count=commit_count,
+        )
+
+    def _get_cache_path(self, url: str) -> Path:
+        """Get the cache path for a repository URL.
+
+        Args:
+            url: Repository URL
+
+        Returns:
+            Path to cache directory
+        """
+        # Create a unique directory name from URL
+        url_hash = hashlib.md5(url.encode()).hexdigest()[:12]
+
+        # Extract repo name from URL
+        parsed = urlparse(url)
+        path_parts = parsed.path.strip("/").split("/")
+        repo_name = path_parts[-1].replace(".git", "")
+
+        return self.cache_dir / f"{repo_name}_{url_hash}"
+
+    def _get_origin_url(self, repo: Repo) -> Optional[str]:
+        """Get the origin URL of a repository.
+
+        Args:
+            repo: Git repository
+
+        Returns:
+            Origin URL or None
+        """
+        try:
+            if hasattr(repo.remotes, "origin"):
+                return repo.remotes.origin.url
+        except Exception:
+            pass
+        return None
+
+    def get_source_files(
+        self,
+        repo: Repo,
+        extensions: list[str],
+        ignore_patterns: list[str] = None
+    ) -> list[Path]:
+        """Get all source files matching given extensions.
+
+        Args:
+            repo: Git repository
+            extensions: List of file extensions (e.g., ['.py', '.ts', '.js'])
+            ignore_patterns: Patterns to ignore (e.g., "__pycache__", "venv")
+
+        Returns:
+            List of source file paths
+        """
+        if ignore_patterns is None:
+            ignore_patterns = [
+                "__pycache__",
+                "*.pyc",
+                "*.pyo",
+                ".git",
+                ".venv",
+                "venv",
+                "env",
+                "node_modules",
+                ".tox",
+                ".pytest_cache",
+                "*.egg-info",
+                "dist",
+                "build",
+            ]
+
+        repo_path = Path(repo.working_dir)
+        source_files = []
+
+        # Normalize extensions to lowercase
+        extensions = [ext.lower() for ext in extensions]
+
+        for source_file in repo_path.rglob("*"):
+            # Check if file (not directory)
+            if not source_file.is_file():
+                continue
+
+            # Check extension
+            if source_file.suffix.lower() not in extensions:
+                continue
+
+            # Check ignore patterns
+            relative_path = source_file.relative_to(repo_path)
+            if any(pattern in str(relative_path) for pattern in ignore_patterns):
+                continue
+
+            source_files.append(source_file)
+
+        log.info(f"Found {len(source_files)} source files with extensions {extensions}")
+        return source_files
+
+    def get_python_files(self, repo: Repo, ignore_patterns: list[str] = None) -> list[Path]:
+        """Get all Python files in a repository.
+
+        Args:
+            repo: Git repository
+            ignore_patterns: Patterns to ignore (e.g., "__pycache__", "venv")
+
+        Returns:
+            List of Python file paths
+
+        Note:
+            This is a convenience wrapper around get_source_files() for backward compatibility.
+        """
+        return self.get_source_files(repo, ['.py'], ignore_patterns)
+
+    def clear_cache(self):
+        """Clear all cached repositories."""
+        log.warning("Clearing repository cache...")
+
+        if self.cache_dir.exists():
+            shutil.rmtree(self.cache_dir)
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        log.info("Cache cleared successfully")

emdash_core/models/__init__.py

@@ -0,0 +1,38 @@
+"""Pydantic models for API requests and responses."""
+
+from .agent import (
+    AgentChatRequest,
+    AgentMode,
+    ImageData,
+)
+from .query import (
+    EntityType,
+    SearchRequest,
+    SearchResult,
+    SearchResponse,
+    ExpandRequest,
+)
+from .index import (
+    IndexRequest,
+    IndexOptions,
+    IndexStatus,
+    IndexStats,
+)
+
+__all__ = [
+    # Agent
+    "AgentChatRequest",
+    "AgentMode",
+    "ImageData",
+    # Query
+    "EntityType",
+    "SearchRequest",
+    "SearchResult",
+    "SearchResponse",
+    "ExpandRequest",
+    # Index
+    "IndexRequest",
+    "IndexOptions",
+    "IndexStatus",
+    "IndexStats",
+]