awslabs.git-repo-research-mcp-server 0.0.1__py3-none-any.whl

awslabs/git_repo_research_mcp_server/models.py

@@ -0,0 +1,291 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+# with the License. A copy of the License is located at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+"""Data models for Git Repository Research MCP Server."""
+
+from datetime import datetime
+from enum import Enum
+from pydantic import BaseModel, Field
+from typing import Dict, List, Optional
+
+
+class GitHubConfig(BaseModel):
+    """GitHub API configuration.
+
+    This model defines the configuration for the GitHub API, including
+    the optional token for authentication and the API URL.
+    """
+
+    token: Optional[str] = Field(None, description='GitHub API token for increased rate limits')
+    api_url: str = Field(
+        default='https://api.github.com/graphql', description='GitHub GraphQL API URL'
+    )
+
+
+class IndexMetadata(BaseModel):
+    """Metadata for a repository index.
+
+    This model stores information about an indexed repository, including
+    its location, creation time, and statistics about the indexed content.
+    """
+
+    repository_name: str = Field(..., description='Name of the repository')
+    repository_path: str = Field(..., description='Path or URL of the repository')
+    index_path: str = Field(..., description='Path to the index file')
+    created_at: datetime = Field(
+        default_factory=datetime.now, description='When the index was created'
+    )
+    last_accessed: Optional[datetime] = Field(None, description='When the index was last accessed')
+    file_count: int = Field(0, description='Number of files indexed')
+    chunk_count: int = Field(0, description='Number of text chunks indexed')
+    embedding_model: str = Field(..., description='Model used for embeddings')
+    file_types: Dict[str, int] = Field(
+        default_factory=dict, description='Count of file types indexed'
+    )
+    total_tokens: Optional[int] = Field(None, description='Total number of tokens processed')
+    index_size_bytes: Optional[int] = Field(None, description='Size of the index in bytes')
+    last_commit_id: Optional[str] = Field(
+        None, description='ID of the last commit in the repository'
+    )
+    repository_directory: Optional[str] = Field(
+        None, description='Path to the cloned repository directory'
+    )
+
+
+class SearchResult(BaseModel):
+    """Result from a repository search.
+
+    This model represents a single search result, including the file path,
+    relevant content, and similarity score.
+    """
+
+    file_path: str = Field(..., description='Path to the file within the repository')
+    content: str = Field(..., description='Relevant content snippet')
+    score: float = Field(..., description='Similarity score (0-1)')
+    line_numbers: Optional[List[int]] = Field(None, description='Line numbers for the content')
+    metadata: Optional[Dict[str, str]] = Field(
+        None, description='Additional metadata about the result'
+    )
+
+
+class SearchResponse(BaseModel):
+    """Response from a repository search.
+
+    This model represents the complete response from a search operation,
+    including all matching results and query metadata.
+    """
+
+    results: List[SearchResult] = Field(default_factory=list, description='Search results')
+    query: str = Field(..., description='Original search query')
+    index_path: str = Field(..., description='Path to the index that was searched')
+    repository_name: str = Field(..., description='Name of the repository')
+    repository_directory: Optional[str] = Field(
+        None, description='Path to the cloned repository directory'
+    )
+    timestamp: datetime = Field(
+        default_factory=datetime.now, description='When the search was performed'
+    )
+    total_results: int = Field(0, description='Total number of results found')
+    execution_time_ms: Optional[float] = Field(
+        None, description='Search execution time in milliseconds'
+    )
+
+
+class IndexedRepositoryInfo(BaseModel):
+    """Information about an indexed repository.
+
+    This model provides a summary of an indexed repository for listing purposes.
+    """
+
+    repository_name: str = Field(..., description='Name of the repository')
+    repository_path: str = Field(..., description='Path or URL of the repository')
+    index_path: str = Field(..., description='Path to the index file')
+    repository_directory: Optional[str] = Field(
+        None, description='Path to the cloned repository directory'
+    )
+    created_at: datetime = Field(..., description='When the index was created')
+    last_accessed: Optional[datetime] = Field(None, description='When the index was last accessed')
+    file_count: int = Field(0, description='Number of files indexed')
+    embedding_model: str = Field(..., description='Model used for embeddings')
+
+
+class IndexedRepositoriesResponse(BaseModel):
+    """Response containing a list of indexed repositories.
+
+    This model represents the complete response from a list operation,
+    including all indexed repositories and summary statistics.
+    """
+
+    repositories: List[IndexedRepositoryInfo] = Field(
+        default_factory=list, description='List of indexed repositories'
+    )
+    total_count: int = Field(0, description='Total number of indexed repositories')
+    index_directory: str = Field(..., description='Directory containing the indices')
+
+
+class DetailedIndexedRepositoryInfo(IndexedRepositoryInfo):
+    """Detailed information about an indexed repository.
+
+    This model extends the basic repository info with additional details
+    about the indexed content.
+    """
+
+    chunk_count: int = Field(0, description='Number of text chunks indexed')
+    file_types: Dict[str, int] = Field(
+        default_factory=dict, description='Count of file types indexed'
+    )
+    total_tokens: Optional[int] = Field(None, description='Total number of tokens processed')
+    index_size_bytes: Optional[int] = Field(None, description='Size of the index in bytes')
+    last_commit_id: Optional[str] = Field(
+        None, description='ID of the last commit in the repository'
+    )
+
+
+class DetailedIndexedRepositoriesResponse(BaseModel):
+    """Response containing detailed information about indexed repositories.
+
+    This model represents the complete response from a detailed list operation,
+    including all indexed repositories with detailed information.
+    """
+
+    repositories: List[DetailedIndexedRepositoryInfo] = Field(
+        default_factory=list,
+        description='List of indexed repositories with detailed information',
+    )
+    total_count: int = Field(0, description='Total number of indexed repositories')
+    index_directory: str = Field(..., description='Directory containing the indices')
+    total_index_size_bytes: Optional[int] = Field(
+        None, description='Total size of all indices in bytes'
+    )
+
+
+class EmbeddingModel(str, Enum):
+    """Available embedding models.
+
+    This enum defines the available embedding models that can be used
+    for generating embeddings from repository content.
+    """
+
+    AMAZON_TITAN_EMBED_TEXT_V1 = 'amazon.titan-embed-text-v1'
+    AMAZON_TITAN_EMBED_TEXT_V2 = 'amazon.titan-embed-text-v2:0'
+    COHERE_EMBED_ENGLISH_V3 = 'cohere.embed-english-v3'
+    COHERE_EMBED_MULTILINGUAL_V3 = 'cohere.embed-multilingual-v3'
+
+
+class IndexRepositoryResponse(BaseModel):
+    """Response from indexing a repository.
+
+    This model represents the complete response from an indexing operation,
+    including metadata about the created index.
+    """
+
+    status: str = Field(..., description='Status of the indexing operation')
+    repository_name: str = Field(..., description='Name of the repository')
+    repository_path: str = Field(..., description='Path or URL of the repository')
+    index_path: str = Field(..., description='Path to the created index')
+    repository_directory: Optional[str] = Field(
+        None, description='Path to the cloned repository directory'
+    )
+    file_count: int = Field(0, description='Number of files indexed')
+    chunk_count: int = Field(0, description='Number of text chunks indexed')
+    embedding_model: str = Field(..., description='Model used for embeddings')
+    execution_time_ms: Optional[float] = Field(
+        None, description='Indexing execution time in milliseconds'
+    )
+    message: Optional[str] = Field(
+        None, description='Additional information about the indexing operation'
+    )
+
+
+class GitHubRepoSearchInput(BaseModel):
+    """Input for GitHub repository search.
+
+    This model defines the input parameters for searching GitHub repositories
+    based on keywords and organizations.
+    """
+
+    keywords: List[str] = Field(description='List of keywords to search for GitHub repositories')
+    organizations: Optional[List[str]] = Field(
+        default=['aws-samples', 'aws-solutions-library-samples', 'awslabs'],
+        description='List of GitHub organizations to scope the search to',
+    )
+    num_results: Optional[int] = Field(default=5, description='Number of results to return')
+    license_filter: Optional[List[str]] = Field(
+        default=None,
+        description="List of licenses to filter by (e.g., 'Apache License 2.0', 'MIT No Attribution')",
+    )
+
+
+class GitHubRepoSearchResult(BaseModel):
+    """Result from a GitHub repository search.
+
+    This model represents a single GitHub repository search result.
+    """
+
+    url: str = Field(..., description='URL of the GitHub repository')
+    title: str = Field(..., description='Title of the repository')
+    description: Optional[str] = Field(None, description='Description of the repository')
+    organization: str = Field(..., description='GitHub organization that owns the repository')
+    stars: Optional[int] = Field(None, description='Number of stars the repository has')
+    updated_at: Optional[str] = Field(None, description='When the repository was last updated')
+    language: Optional[str] = Field(
+        None, description='Primary programming language of the repository'
+    )
+    topics: Optional[List[str]] = Field(
+        None, description='Topics/tags associated with the repository'
+    )
+    license: Optional[str] = Field(None, description='License of the repository')
+    forks: Optional[int] = Field(None, description='Number of forks the repository has')
+    open_issues: Optional[int] = Field(None, description='Number of open issues in the repository')
+    homepage: Optional[str] = Field(None, description='Homepage URL of the repository')
+
+
+class GitHubRepoSearchResponse(BaseModel):
+    """Response from a GitHub repository search.
+
+    This model represents the complete response from a GitHub repository search.
+    """
+
+    status: str = Field(..., description='Status of the search operation')
+    query: str = Field(..., description='Original search query')
+    organizations: List[str] = Field(..., description='Organizations that were searched')
+    results: List[GitHubRepoSearchResult] = Field(
+        default_factory=list, description='Search results'
+    )
+    total_results: int = Field(0, description='Total number of results found')
+    execution_time_ms: Optional[float] = Field(
+        None, description='Search execution time in milliseconds'
+    )
+
+
+class DeleteRepositoryResponse(BaseModel):
+    """Response from deleting a repository.
+
+    This model represents the complete response from a delete operation,
+    including status and information about the deleted repository.
+    """
+
+    status: str = Field(
+        ..., description='Status of the delete operation (success, partial, or error)'
+    )
+    message: str = Field(..., description='Information about the delete operation')
+    repository_name: Optional[str] = Field(None, description='Name of the deleted repository')
+    execution_time_ms: Optional[float] = Field(
+        None, description='Delete operation execution time in milliseconds'
+    )
+    deleted_files: Optional[List[str]] = Field(
+        None, description='List of files that were successfully deleted'
+    )
+    errors: Optional[List[str]] = Field(
+        None, description='List of errors encountered during deletion'
+    )
+    permission_issues: Optional[List[str]] = Field(
+        None, description='List of files with permission issues'
+    )

awslabs/git_repo_research_mcp_server/repository.py

@@ -0,0 +1,321 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+# with the License. A copy of the License is located at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions
+# and limitations under the License.
+"""Repository handling for Git Repository Research MCP Server.
+
+This module provides functionality for cloning, accessing, and processing
+Git repositories for indexing and searching.
+"""
+
+import fnmatch
+import os
+import shutil
+import tempfile
+from awslabs.git_repo_research_mcp_server.defaults import Constants
+from git import Repo
+from loguru import logger
+from typing import Dict, List, Optional, Tuple
+from urllib.parse import urlparse
+
+
+def is_git_url(repo_path: str) -> bool:
+    """Check if a string is a Git URL.
+
+    Args:
+        repo_path: Path or URL to check
+
+    Returns:
+        True if the string is a Git URL, False otherwise
+    """
+    parsed = urlparse(repo_path)
+    return parsed.scheme in ('http', 'https', 'git', 'ssh')
+
+
+def is_git_repo(path: str) -> bool:
+    """Check if a path is a Git repository.
+
+    Args:
+        path: Path to check
+
+    Returns:
+        True if the path is a Git repository, False otherwise
+    """
+    try:
+        Repo(path)
+        return True
+    except Exception:
+        return False
+
+
+def clone_repository(url: str, target_dir: Optional[str] = None) -> str:
+    """Clone a Git repository from a URL.
+
+    Args:
+        url: URL of the repository to clone
+        target_dir: Directory to clone into (optional, uses temp dir if not provided)
+
+    Returns:
+        Path to the cloned repository
+
+    Raises:
+        Exception: If cloning fails
+    """
+    if target_dir is None:
+        target_dir = tempfile.mkdtemp(prefix='git_repo_research_')
+
+    logger.info(f'Cloning repository from {url} to {target_dir}')
+    try:
+        # Clone the repository with GitPython
+        Repo.clone_from(url, target_dir)
+
+        # Check if .git directory exists after cloning
+        git_dir = os.path.join(target_dir, '.git')
+        if os.path.exists(git_dir):
+            logger.info(f'.git directory exists at {git_dir}')
+        else:
+            logger.warning(f'.git directory not found after cloning at {git_dir}')
+            # List the contents of the directory to debug
+            logger.info(f'Contents of {target_dir}: {os.listdir(target_dir)}')
+
+        return target_dir
+    except Exception as e:
+        # Clean up the target directory if it was created
+        if os.path.exists(target_dir):
+            shutil.rmtree(target_dir, ignore_errors=True)
+        logger.error(f'Failed to clone repository: {e}')
+        raise
+
+
+def get_repository_name(repo_path: str) -> str:
+    """Get the name of a repository.
+
+    Args:
+        repo_path: Path or URL of the repository
+
+    Returns:
+        Name of the repository, including GitHub organization/username if available
+        Note: For GitHub repositories, the format is "org_repo" (with underscore)
+        instead of "org/repo" for file path compatibility
+    """
+    if is_git_url(repo_path):
+        # Extract the repository name from the URL
+        parsed = urlparse(repo_path)
+        path_parts = parsed.path.strip('/').split('/')
+
+        # Check if this is a GitHub URL with org/username
+        if parsed.netloc in ['github.com', 'www.github.com'] and len(path_parts) >= 2:
+            # Include the organization/username in the repository name
+            org_name = path_parts[-2]
+            repo_name = path_parts[-1]
+            if repo_name.endswith('.git'):
+                repo_name = repo_name[:-4]
+            # Use underscore instead of slash for file path compatibility
+            return f'{org_name}_{repo_name}'
+        else:
+            # For non-GitHub URLs or URLs without clear org structure,
+            # just use the last part of the path
+            repo_name = path_parts[-1]
+            if repo_name.endswith('.git'):
+                repo_name = repo_name[:-4]
+            return repo_name
+    else:
+        # Use the directory name as the repository name
+        return os.path.basename(os.path.abspath(repo_path))
+
+
+def get_text_files(
+    repo_path: str,
+    include_patterns: Optional[List[str]] = None,
+    exclude_patterns: Optional[List[str]] = None,
+) -> List[str]:
+    """Get all text files in a repository.
+
+    Args:
+        repo_path: Path to the repository
+        include_patterns: Glob patterns for files to include (optional)
+        exclude_patterns: Glob patterns for files to exclude (optional)
+
+    Returns:
+        List of paths to text files
+    """
+    if include_patterns is None:
+        include_patterns = Constants.TEXT_FILE_INCLUDE_PATTERNS
+    if exclude_patterns is None:
+        exclude_patterns = Constants.TEXT_FILE_EXCLUDE_PATTERNS
+
+    text_files = []
+    for root, _, files in os.walk(repo_path):
+        for file in files:
+            file_path = os.path.join(root, file)
+            rel_path = os.path.relpath(file_path, repo_path)
+
+            # Check if the file matches any include pattern
+            included = any(fnmatch.fnmatch(rel_path, pattern) for pattern in include_patterns)
+            if not included:
+                continue
+
+            # Check if the file matches any exclude pattern
+            excluded = any(fnmatch.fnmatch(rel_path, pattern) for pattern in exclude_patterns)
+            if excluded:
+                continue
+
+            # Try to read the file as text
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    # Read a small sample to check if it's text
+                    sample = f.read(1024)
+                    # If we can decode it as UTF-8, it's probably text
+                    if sample:
+                        text_files.append(file_path)
+            except UnicodeDecodeError:
+                # Not a text file
+                pass
+            except Exception as e:
+                logger.warning(f'Error reading file {file_path}: {e}')
+
+    return text_files
+
+
+def get_file_extension_stats(file_paths: List[str]) -> Dict[str, int]:
+    """Get statistics about file extensions.
+
+    Args:
+        file_paths: List of file paths
+
+    Returns:
+        Dictionary mapping file extensions to counts
+    """
+    extension_counts = {}
+    for file_path in file_paths:
+        _, ext = os.path.splitext(file_path)
+        if ext:
+            # Remove the dot from the extension
+            ext = ext[1:].lower()
+            extension_counts[ext] = extension_counts.get(ext, 0) + 1
+        else:
+            extension_counts['no_extension'] = extension_counts.get('no_extension', 0) + 1
+    return extension_counts
+
+
+def read_file_content(file_path: str) -> str:
+    """Read the content of a file.
+
+    Args:
+        file_path: Path to the file
+
+    Returns:
+        Content of the file as a string
+
+    Raises:
+        Exception: If reading fails
+    """
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return f.read()
+    except Exception as e:
+        logger.error(f'Failed to read file {file_path}: {e}')
+        raise
+
+
+def chunk_text(text: str, chunk_size: int = 1000, chunk_overlap: int = 200) -> List[str]:
+    """Split text into chunks.
+
+    Args:
+        text: Text to split
+        chunk_size: Maximum size of each chunk in characters
+        chunk_overlap: Overlap between chunks in characters
+
+    Returns:
+        List of text chunks
+    """
+    if not text or len(text) <= chunk_size:
+        return [text] if text else []
+
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        if end >= len(text):
+            chunks.append(text[start:])
+            break
+
+        # Try to find a good breaking point (newline or space)
+        break_point = text.rfind('\n', start + chunk_size - chunk_overlap, end)
+        if break_point == -1:
+            break_point = text.rfind(' ', start + chunk_size - chunk_overlap, end)
+        if break_point == -1:
+            break_point = end
+
+        chunks.append(text[start:break_point])
+        start = break_point + 1 if text[break_point] in ['\n', ' '] else break_point
+
+    return chunks
+
+
+def process_repository(
+    repo_path: str,
+    include_patterns: Optional[List[str]] = None,
+    exclude_patterns: Optional[List[str]] = None,
+    chunk_size: int = 1000,
+    chunk_overlap: int = 200,
+) -> Tuple[List[str], Dict[str, str], Dict[str, int]]:
+    """Process a repository for indexing.
+
+    Args:
+        repo_path: Path to the repository
+        include_patterns: Glob patterns for files to include (optional)
+        exclude_patterns: Glob patterns for files to exclude (optional)
+        chunk_size: Maximum size of each chunk in characters
+        chunk_overlap: Overlap between chunks in characters
+
+    Returns:
+        Tuple containing:
+        - List of text chunks
+        - Dictionary mapping chunks to file paths
+        - Dictionary of file extension statistics
+    """
+    logger.info(f'Processing repository at {repo_path}')
+    text_files = get_text_files(repo_path, include_patterns, exclude_patterns)
+    logger.info(f'Found {len(text_files)} text files')
+
+    extension_stats = get_file_extension_stats(text_files)
+    logger.info(f'File extension statistics: {extension_stats}')
+
+    chunks = []
+    chunk_to_file = {}
+
+    for file_path in text_files:
+        try:
+            content = read_file_content(file_path)
+            file_chunks = chunk_text(content, chunk_size, chunk_overlap)
+
+            rel_path = os.path.relpath(file_path, repo_path)
+            for chunk in file_chunks:
+                chunks.append(chunk)
+                chunk_to_file[chunk] = rel_path
+        except Exception as e:
+            logger.warning(f'Error processing file {file_path}: {e}')
+
+    logger.info(f'Created {len(chunks)} text chunks')
+    return chunks, chunk_to_file, extension_stats
+
+
+def cleanup_repository(repo_path: str) -> None:
+    """Clean up a cloned repository.
+
+    Args:
+        repo_path: Path to the repository
+    """
+    if os.path.exists(repo_path) and os.path.isdir(repo_path):
+        logger.info(f'Cleaning up repository at {repo_path}')
+        try:
+            shutil.rmtree(repo_path, ignore_errors=True)
+        except Exception as e:
+            logger.warning(f'Error cleaning up repository: {e}')