cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/github/__init__.py

@@ -0,0 +1,309 @@
+"""
+GitHub connector for CortexDB.
+
+Polls GitHub repositories for events (pushes, PRs, issues, comments,
+reviews) and ingests them as CortexDB Episodes via the Connector SDK.
+Uses the ``PyGithub`` library for API access.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="github")
+
+_DEFAULT_EVENT_TYPES = frozenset(
+    [
+        "push",
+        "pull_request",
+        "issues",
+        "issue_comment",
+        "pull_request_review",
+    ]
+)
+
+# Map GitHub event type strings to CortexDB episode types.
+_EVENT_TYPE_MAP: dict[str, EpisodeType] = {
+    "PushEvent": EpisodeType.CODE_CHANGE,
+    "PullRequestEvent": EpisodeType.CODE_CHANGE,
+    "IssuesEvent": EpisodeType.ISSUE,
+    "IssueCommentEvent": EpisodeType.COMMENT,
+    "PullRequestReviewEvent": EpisodeType.REVIEW,
+}
+
+
+class GitHubConnector(CortexConnector):
+    """Connector that syncs GitHub repository events into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    github_token:
+        GitHub personal access token or GitHub App installation token.
+    repos:
+        List of repository full names (``owner/repo``) to sync.
+    events:
+        GitHub event types to capture.  Defaults to push, PR, issues,
+        issue_comment, and pull_request_review.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "github"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        github_token: str,
+        repos: list[str] | None = None,
+        events: list[str] | None = None,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.github_token = github_token
+        self.repos = repos or []
+        self.event_filter = set(events) if events else set(_DEFAULT_EVENT_TYPES)
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch repository events from GitHub since *since*.
+
+        Uses the repository events API endpoint which returns up to 300
+        events (paginated, 30 per page, max 10 pages).
+        """
+        from github import Github
+
+        gh = Github(self.github_token)
+        raw_events: list[RawEvent] = []
+
+        for repo_name in self.repos:
+            try:
+                repo = gh.get_repo(repo_name)
+            except Exception:
+                logger.error("Could not access repository %s", repo_name)
+                continue
+
+            for event in repo.get_events():
+                created_at = event.created_at.replace(tzinfo=timezone.utc)
+
+                if since and created_at <= since:
+                    # Events are returned newest-first; once we pass the
+                    # cursor we can stop for this repo.
+                    break
+
+                if not self._matches_filter(event.type):
+                    continue
+
+                raw = RawEvent(
+                    source="github",
+                    external_id=str(event.id),
+                    timestamp=created_at,
+                    data={
+                        "type": event.type,
+                        "actor": {
+                            "login": event.actor.login if event.actor else "unknown",
+                            "id": event.actor.id if event.actor else 0,
+                            "avatar_url": (
+                                event.actor.avatar_url if event.actor else None
+                            ),
+                        },
+                        "repo": repo_name,
+                        "payload": event.payload,
+                        "public": not repo.private,
+                    },
+                    permissions={"private": repo.private},
+                )
+                raw_events.append(raw)
+
+        return raw_events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a GitHub event ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        payload = data.get("payload", {})
+        event_type_str = data.get("type", "")
+        episode_type = _EVENT_TYPE_MAP.get(event_type_str, EpisodeType.CUSTOM)
+
+        actor_data = data.get("actor", {})
+        actor = Actor(
+            id=str(actor_data.get("id", "")),
+            name=actor_data.get("login", "unknown"),
+            avatar_url=actor_data.get("avatar_url"),
+        )
+
+        content = self._build_content(event_type_str, payload)
+        entities = self._extract_entities(data, payload)
+        tags = ["github", event_type_str.lower()]
+
+        metadata: dict[str, Any] = {"github_event_type": event_type_str}
+        if "action" in payload:
+            metadata["action"] = payload["action"]
+
+        # Thread grouping: PR events share the PR number as thread_id.
+        thread_id = self._derive_thread_id(event_type_str, payload, data)
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=episode_type,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace="github",
+            entities=entities,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            idempotency_key=raw.external_id,
+        )
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map GitHub repository visibility to CortexDB visibility.
+
+        Public repositories yield ``Organization`` visibility; private
+        repositories yield ``Restricted``.
+        """
+        if raw.permissions.get("private"):
+            return Visibility(level=VisibilityLevel.RESTRICTED)
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- helpers -------------------------------------------------------------
+
+    def _matches_filter(self, event_type: str) -> bool:
+        """Check whether *event_type* matches the configured filter set."""
+        mapping = {
+            "PushEvent": "push",
+            "PullRequestEvent": "pull_request",
+            "IssuesEvent": "issues",
+            "IssueCommentEvent": "issue_comment",
+            "PullRequestReviewEvent": "pull_request_review",
+        }
+        short = mapping.get(event_type, event_type)
+        return short in self.event_filter
+
+    @staticmethod
+    def _build_content(event_type: str, payload: dict[str, Any]) -> str:
+        """Produce a human-readable summary of the event."""
+        if event_type == "PushEvent":
+            commits = payload.get("commits", [])
+            count = len(commits)
+            ref = payload.get("ref", "")
+            messages = "; ".join(
+                c.get("message", "").split("\n")[0] for c in commits[:5]
+            )
+            return f"Pushed {count} commit(s) to {ref}: {messages}"
+
+        if event_type == "PullRequestEvent":
+            pr = payload.get("pull_request", {})
+            action = payload.get("action", "")
+            title = pr.get("title", "")
+            return f"PR {action}: {title}"
+
+        if event_type == "IssuesEvent":
+            issue = payload.get("issue", {})
+            action = payload.get("action", "")
+            title = issue.get("title", "")
+            return f"Issue {action}: {title}"
+
+        if event_type == "IssueCommentEvent":
+            comment = payload.get("comment", {})
+            body = (comment.get("body") or "")[:200]
+            return f"Comment: {body}"
+
+        if event_type == "PullRequestReviewEvent":
+            review = payload.get("review", {})
+            state = review.get("state", "")
+            pr = payload.get("pull_request", {})
+            title = pr.get("title", "")
+            return f"Review ({state}) on PR: {title}"
+
+        return f"GitHub event: {event_type}"
+
+    @staticmethod
+    def _extract_entities(
+        data: dict[str, Any], payload: dict[str, Any]
+    ) -> list[EntityRef]:
+        """Extract entity references from the event."""
+        entities: list[EntityRef] = []
+
+        repo_name = data.get("repo", "")
+        if repo_name:
+            entities.append(
+                EntityRef(
+                    entity_type="repository",
+                    entity_id=repo_name,
+                    display_name=repo_name,
+                )
+            )
+
+        # PR author and reviewers.
+        pr = payload.get("pull_request", {})
+        if pr:
+            pr_user = pr.get("user", {})
+            if pr_user:
+                entities.append(
+                    EntityRef(
+                        entity_type="user",
+                        entity_id=str(pr_user.get("id", "")),
+                        display_name=pr_user.get("login", ""),
+                    )
+                )
+            for reviewer in pr.get("requested_reviewers", []):
+                entities.append(
+                    EntityRef(
+                        entity_type="user",
+                        entity_id=str(reviewer.get("id", "")),
+                        display_name=reviewer.get("login", ""),
+                    )
+                )
+
+        # Issue references.
+        issue = payload.get("issue", {})
+        if issue:
+            entities.append(
+                EntityRef(
+                    entity_type="issue",
+                    entity_id=str(issue.get("number", "")),
+                    display_name=issue.get("title", ""),
+                )
+            )
+
+        return entities
+
+    @staticmethod
+    def _derive_thread_id(
+        event_type: str, payload: dict[str, Any], data: dict[str, Any]
+    ) -> str | None:
+        """Derive a thread_id for grouping related events."""
+        repo = data.get("repo", "")
+        pr = payload.get("pull_request", {})
+        if pr and pr.get("number"):
+            return f"{repo}:pr:{pr['number']}"
+        issue = payload.get("issue", {})
+        if issue and issue.get("number"):
+            return f"{repo}:issue:{issue['number']}"
+        return None