github-haystack 1.0.0__py3-none-any.whl

haystack_integrations/components/connectors/github/repo_forker.py

@@ -0,0 +1,306 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+import re
+from typing import Any, Dict, Optional
+
+import requests
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class GitHubRepoForker:
+    """
+    Forks a GitHub repository from an issue URL.
+
+    The component takes a GitHub issue URL, extracts the repository information,
+    creates or syncs a fork of that repository, and optionally creates an issue-specific branch.
+
+    ### Usage example
+    ```python
+    from haystack_integrations.components.connectors.github import GitHubRepoForker
+    from haystack.utils import Secret
+
+    # Using direct token with auto-sync and branch creation
+    forker = GitHubRepoForker(
+        github_token=Secret.from_env_var("GITHUB_TOKEN"),
+        auto_sync=True,
+        create_branch=True
+    )
+
+    result = forker.run(url="https://github.com/owner/repo/issues/123")
+    print(result)
+    # Will create or sync fork and create branch "fix-123"
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        github_token: Secret = Secret.from_env_var("GITHUB_TOKEN"),
+        raise_on_failure: bool = True,
+        wait_for_completion: bool = False,
+        max_wait_seconds: int = 300,
+        poll_interval: int = 2,
+        auto_sync: bool = True,
+        create_branch: bool = True,
+    ):
+        """
+        Initialize the component.
+
+        :param github_token: GitHub personal access token for API authentication
+        :param raise_on_failure: If True, raises exceptions on API errors
+        :param wait_for_completion: If True, waits until fork is fully created
+        :param max_wait_seconds: Maximum time to wait for fork completion in seconds
+        :param poll_interval: Time between status checks in seconds
+        :param auto_sync: If True, syncs fork with original repository if it already exists
+        :param create_branch: If True, creates a fix branch based on the issue number
+        """
+        error_message = "github_token must be a Secret"
+        if not isinstance(github_token, Secret):
+            raise TypeError(error_message)
+
+        self.github_token = github_token
+        self.raise_on_failure = raise_on_failure
+        self.wait_for_completion = wait_for_completion
+        self.max_wait_seconds = max_wait_seconds
+        self.poll_interval = poll_interval
+        self.auto_sync = auto_sync
+        self.create_branch = create_branch
+
+        self.base_headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "Haystack/GitHubRepoForker",
+        }
+
+    def _get_request_headers(self) -> dict:
+        """
+        Get headers with resolved token for the request.
+
+        :return: Dictionary of headers including authorization if token is present
+        """
+        headers = self.base_headers.copy()
+        if self.github_token is not None:
+            headers["Authorization"] = f"Bearer {self.github_token.resolve_value()}"
+        return headers
+
+    def _parse_github_url(self, url: str) -> tuple[str, str, str]:
+        """
+        Parse GitHub URL into owner, repo, and issue number.
+
+        :param url: GitHub issue URL
+        :return: Tuple of (owner, repo, issue_number)
+        :raises ValueError: If URL format is invalid
+        """
+        pattern = r"https?://github\.com/([^/]+)/([^/]+)/issues/(\d+)"
+        match = re.match(pattern, url)
+        if not match:
+            error_message = f"Invalid GitHub issue URL format: {url}"
+            raise ValueError(error_message)
+
+        owner, repo, issue_number = match.groups()
+        return owner, repo, issue_number
+
+    def _check_fork_status(self, fork_path: str) -> bool:
+        """
+        Check if a forked repository exists and is ready.
+
+        :param fork_path: Repository path in owner/repo format
+        :return: True if fork exists and is ready, False otherwise
+        """
+        url = f"https://api.github.com/repos/{fork_path}"
+        try:
+            response = requests.get(
+                url,
+                headers=self._get_request_headers(),
+                timeout=10,
+            )
+            return response.status_code == 200  # noqa: PLR2004
+        except requests.RequestException:
+            return False
+
+    def _get_authenticated_user(self) -> str:
+        """
+        Get the authenticated user's username.
+
+        :return: Username of the authenticated user
+        :raises requests.RequestException: If API call fails
+        """
+        url = "https://api.github.com/user"
+        response = requests.get(url, headers=self._get_request_headers(), timeout=10)
+        response.raise_for_status()
+        return response.json()["login"]
+
+    def _get_existing_repository(self, repo_name: str) -> Optional[str]:
+        """
+        Check if a repository with the given name already exists in the authenticated user's account.
+
+        :param repo_name: Repository name to check
+        :return: Full repository name if it exists, None otherwise
+        """
+        url = f"https://api.github.com/repos/{self._get_authenticated_user()}/{repo_name}"
+        try:
+            response = requests.get(
+                url,
+                headers=self._get_request_headers(),
+                timeout=10,
+            )
+            if response.status_code == 200:  # noqa: PLR2004
+                return repo_name
+            return None
+        except requests.RequestException as e:
+            logger.warning(f"Failed to check repository existence: {e!s}")
+            return None
+
+    def _sync_fork(self, fork_path: str) -> None:
+        """
+        Sync a fork with its upstream repository.
+
+        :param fork_path: Fork path in owner/repo format
+        :raises requests.RequestException: If sync fails
+        """
+        url = f"https://api.github.com/repos/{fork_path}/merge-upstream"
+        response = requests.post(
+            url,
+            headers=self._get_request_headers(),
+            json={"branch": "main"},
+            timeout=10,
+        )
+        response.raise_for_status()
+
+    def _create_issue_branch(self, fork_path: str, issue_number: str) -> None:
+        """
+        Create a new branch for the issue.
+
+        :param fork_path: Fork path in owner/repo format
+        :param issue_number: Issue number to use in branch name
+        :raises requests.RequestException: If branch creation fails
+        """
+        # First, get the default branch SHA
+        url = f"https://api.github.com/repos/{fork_path}"
+        response = requests.get(url, headers=self._get_request_headers(), timeout=10)
+        response.raise_for_status()
+        default_branch = response.json()["default_branch"]
+
+        # Get the SHA of the default branch
+        url = f"https://api.github.com/repos/{fork_path}/git/ref/heads/{default_branch}"
+        response = requests.get(url, headers=self._get_request_headers(), timeout=10)
+        response.raise_for_status()
+        sha = response.json()["object"]["sha"]
+
+        # Create the new branch
+        branch_name = f"fix-{issue_number}"
+        url = f"https://api.github.com/repos/{fork_path}/git/refs"
+        response = requests.post(
+            url,
+            headers=self._get_request_headers(),
+            json={"ref": f"refs/heads/{branch_name}", "sha": sha},
+            timeout=10,
+        )
+        response.raise_for_status()
+
+    def _create_fork(self, owner: str, repo: str) -> str:
+        """
+        Create a fork of the repository.
+
+        :param owner: Original repository owner
+        :param repo: Repository name
+        :return: Fork path in owner/repo format
+        :raises requests.RequestException: If fork creation fails
+        """
+        url = f"https://api.github.com/repos/{owner}/{repo}/forks"
+        response = requests.post(url, headers=self._get_request_headers(), timeout=10)
+        response.raise_for_status()
+
+        fork_data = response.json()
+        return f"{fork_data['owner']['login']}/{fork_data['name']}"
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serialize the component to a dictionary.
+
+        :returns: Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            github_token=self.github_token.to_dict() if self.github_token else None,
+            raise_on_failure=self.raise_on_failure,
+            wait_for_completion=self.wait_for_completion,
+            max_wait_seconds=self.max_wait_seconds,
+            poll_interval=self.poll_interval,
+            auto_sync=self.auto_sync,
+            create_branch=self.create_branch,
+        )
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "GitHubRepoForker":
+        """
+        Deserialize the component from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :returns: Deserialized component.
+        """
+        init_params = data["init_parameters"]
+        deserialize_secrets_inplace(init_params, keys=["github_token"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(repo=str, issue_branch=str)
+    def run(self, url: str) -> dict:
+        """
+        Process a GitHub issue URL and create or sync a fork of the repository.
+
+        :param url: GitHub issue URL
+        :return: Dictionary containing repository path in owner/repo format
+        """
+        try:
+            # Extract repository information
+            owner, repo, issue_number = self._parse_github_url(url)
+
+            # Check if fork already exists
+            user = self._get_authenticated_user()
+            existing_fork = self._get_existing_repository(repo)
+
+            if existing_fork and self.auto_sync:
+                # If fork exists and auto_sync is enabled, sync with upstream
+                fork_path = f"{user}/{repo}"
+                logger.info("Fork already exists, syncing with upstream repository")
+                self._sync_fork(fork_path)
+            else:
+                # Create new fork
+                fork_path = self._create_fork(owner, repo)
+
+            # Wait for fork completion if requested
+            if self.wait_for_completion:
+                import time
+
+                start_time = time.time()
+
+                while time.time() - start_time < self.max_wait_seconds:
+                    if self._check_fork_status(fork_path):
+                        logger.info("Fork creation completed successfully")
+                        break
+                    logger.debug("Waiting for fork creation to complete...")
+                    time.sleep(self.poll_interval)
+                else:
+                    msg = f"Fork creation timed out after {self.max_wait_seconds} seconds"
+                    if self.raise_on_failure:
+                        raise TimeoutError(msg)
+                    logger.warning(msg)
+
+            # Create issue branch if enabled
+            issue_branch = None
+            if self.create_branch:
+                issue_branch = f"fix-{issue_number}"
+                logger.info(f"Creating branch for issue #{issue_number}")
+                self._create_issue_branch(fork_path, issue_number)
+
+            return {"repo": fork_path, "issue_branch": issue_branch}
+
+        except Exception as e:
+            if self.raise_on_failure:
+                raise
+            logger.warning("Error forking repository from {url}: {error}", url=url, error=str(e))
+            return {"repo": "", "issue_branch": None}

haystack_integrations/components/connectors/github/repo_viewer.py

@@ -0,0 +1,272 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+import base64
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+import requests
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GitHubItem:
+    """Represents an item (file or directory) in a GitHub repository"""
+
+    name: str
+    type: str  # "file" or "dir"
+    path: str
+    size: int
+    url: str
+    content: Optional[str] = None
+
+
+@component
+class GitHubRepoViewer:
+    """
+    Navigates and fetches content from GitHub repositories.
+
+    For directories:
+    - Returns a list of Documents, one for each item
+    - Each Document's content is the item name
+    - Full path and metadata in Document.meta
+
+    For files:
+    - Returns a single Document
+    - Document's content is the file content
+    - Full path and metadata in Document.meta
+
+    For errors:
+    - Returns a single Document
+    - Document's content is the error message
+    - Document's meta contains type="error"
+
+    ### Usage example
+    ```python
+    from haystack_integrations.components.connectors.github import GitHubRepoViewer
+
+    viewer = GitHubRepoViewer()
+
+    # List directory contents - returns multiple documents
+    result = viewer.run(
+        repo="owner/repository",
+        path="docs/",
+        branch="main"
+    )
+    print(result)
+
+    # Get specific file - returns single document
+    result = viewer.run(
+        repo="owner/repository",
+        path="README.md",
+        branch="main"
+    )
+    print(result)
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        github_token: Optional[Secret] = None,
+        raise_on_failure: bool = True,
+        max_file_size: int = 1_000_000,  # 1MB default limit
+        repo: Optional[str] = None,
+        branch: str = "main",
+    ):
+        """
+        Initialize the component.
+
+        :param github_token: GitHub personal access token for API authentication
+        :param raise_on_failure: If True, raises exceptions on API errors
+        :param max_file_size: Maximum file size in bytes to fetch (default: 1MB)
+        :param repo: Repository in format "owner/repo"
+        :param branch: Git reference (branch, tag, commit) to use
+        """
+        if github_token is not None and not isinstance(github_token, Secret):
+            msg = "github_token must be a Secret"
+            raise TypeError(msg)
+
+        self.github_token = github_token
+        self.raise_on_failure = raise_on_failure
+        self.max_file_size = max_file_size
+        self.repo = repo
+        self.branch = branch
+
+        self.base_headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "Haystack/GitHubRepoViewer",
+        }
+
+    def _get_request_headers(self) -> dict:
+        """
+        Get headers with resolved token for the request.
+
+        :return: Dictionary of headers including authorization if token is present
+        """
+        headers = self.base_headers.copy()
+        if self.github_token is not None:
+            headers["Authorization"] = f"Bearer {self.github_token.resolve_value()}"
+        return headers
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serialize the component to a dictionary.
+
+        :returns: Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            github_token=self.github_token.to_dict() if self.github_token else None,
+            raise_on_failure=self.raise_on_failure,
+            max_file_size=self.max_file_size,
+            repo=self.repo,
+            branch=self.branch,
+        )
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "GitHubRepoViewer":
+        """
+        Deserialize the component from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :returns: Deserialized component.
+        """
+        init_params = data["init_parameters"]
+        deserialize_secrets_inplace(init_params, keys=["github_token"])
+        return default_from_dict(cls, data)
+
+    def _parse_repo(self, repo: str) -> tuple[str, str]:
+        """Parse owner/repo string"""
+        parts = repo.split("/")
+        if len(parts) != 2:  # noqa: PLR2004
+            msg = f"Invalid repository format. Expected 'owner/repo', got '{repo}'"
+            raise ValueError(msg)
+        return parts[0], parts[1]
+
+    def _normalize_path(self, path: str) -> str:
+        """Normalize repository path"""
+        return path.strip("/")
+
+    def _fetch_contents(self, owner: str, repo: str, path: str, ref: str) -> Any:
+        """Fetch repository contents from GitHub API"""
+        url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}"
+        if ref:
+            url += f"?ref={ref}"
+
+        response = requests.get(url, headers=self._get_request_headers(), timeout=10)
+        response.raise_for_status()
+        return response.json()
+
+    def _process_file_content(self, content: str, encoding: str) -> str:
+        """Process file content based on encoding"""
+        if encoding == "base64":
+            return base64.b64decode(content).decode("utf-8")
+        return content
+
+    def _create_file_document(self, item: GitHubItem) -> Document:
+        """Create a Document from a file"""
+        return Document(
+            content=item.content if item.content else item.name,
+            meta={
+                "path": item.path,
+                "type": "file_content",
+                "size": item.size,
+                "url": item.url,
+            },
+        )
+
+    def _create_directory_documents(self, items: List[GitHubItem]) -> List[Document]:
+        """Create a list of Documents from directory contents"""
+        return [
+            Document(
+                content=item.name,
+                meta={
+                    "path": item.path,
+                    "type": item.type,
+                    "size": item.size,
+                    "url": item.url,
+                },
+            )
+            for item in sorted(items, key=lambda x: (x.type != "dir", x.name.lower()))
+        ]
+
+    def _create_error_document(self, error: Exception, path: str) -> Document:
+        """Create a Document from an error"""
+        return Document(
+            content=str(error),
+            meta={
+                "type": "error",
+                "path": path,
+            },
+        )
+
+    @component.output_types(documents=List[Document])
+    def run(self, path: str, repo: Optional[str] = None, branch: Optional[str] = None) -> Dict[str, List[Document]]:
+        """
+        Process a GitHub repository path and return documents.
+
+        :param repo: Repository in format "owner/repo"
+        :param path: Path within repository (default: root)
+        :param branch: Git reference (branch, tag, commit) to use
+        :return: Dictionary containing list of documents
+        """
+        if repo is None:
+            repo = self.repo
+            if repo is None:
+                msg = "Repository not provided in initialization or run() method"
+                raise ValueError(msg)
+        if branch is None:
+            branch = self.branch
+
+        try:
+            owner, repo_name = self._parse_repo(repo)
+            normalized_path = self._normalize_path(path)
+
+            contents = self._fetch_contents(owner, repo_name, normalized_path, branch)
+
+            # Handle single file response
+            if not isinstance(contents, list):
+                if contents.get("size", 0) > self.max_file_size:
+                    error_message = f"File size {contents['size']} exceeds limit of {self.max_file_size}"
+                    raise ValueError(error_message)
+
+                item = GitHubItem(
+                    name=contents["name"],
+                    type="file",
+                    path=contents["path"],
+                    size=contents["size"],
+                    url=contents["html_url"],
+                    content=self._process_file_content(contents["content"], contents["encoding"]),
+                )
+                return {"documents": [self._create_file_document(item)]}
+
+            # Handle directory listing
+            items = [
+                GitHubItem(
+                    name=item["name"],
+                    type="dir" if item["type"] == "dir" else "file",
+                    path=item["path"],
+                    size=item.get("size", 0),
+                    url=item["html_url"],
+                )
+                for item in contents
+            ]
+
+            return {"documents": self._create_directory_documents(items)}
+
+        except Exception as e:
+            error_doc = self._create_error_document(
+                Exception(f"Error processing repository path {path}: {e!s}. Seems like the file does not exist."), path
+            )
+            if self.raise_on_failure:
+                raise
+            logger.warning(
+                "Error processing repository path {path}: {error}",
+                path=path,
+                error=str(e),
+            )
+            return {"documents": [error_doc]}

haystack_integrations/prompts/github/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+from .context_prompt import CONTEXT_PROMPT
+from .file_editor_prompt import FILE_EDITOR_PROMPT, FILE_EDITOR_SCHEMA
+from .issue_commenter_prompt import ISSUE_COMMENTER_PROMPT, ISSUE_COMMENTER_SCHEMA
+from .pr_creator_prompt import PR_CREATOR_PROMPT, PR_CREATOR_SCHEMA
+from .repo_viewer_prompt import REPO_VIEWER_PROMPT, REPO_VIEWER_SCHEMA
+from .system_prompt import SYSTEM_PROMPT
+
+__all__ = [
+    "CONTEXT_PROMPT",
+    "FILE_EDITOR_PROMPT",
+    "FILE_EDITOR_SCHEMA",
+    "ISSUE_COMMENTER_PROMPT",
+    "ISSUE_COMMENTER_SCHEMA",
+    "PR_CREATOR_PROMPT",
+    "PR_CREATOR_SCHEMA",
+    "REPO_VIEWER_PROMPT",
+    "REPO_VIEWER_SCHEMA",
+    "SYSTEM_PROMPT",
+]