cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/gitlab/__init__.py

@@ -0,0 +1,749 @@
+"""
+GitLab connector for CortexDB.
+
+Polls GitLab REST API v4 for Merge Requests, Issues, Comments/Notes,
+Commits, Pipelines, and Releases, converting them into CortexDB
+Episodes.  Uses the ``python-gitlab`` library for API access.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="gitlab")
+
+_DEFAULT_EVENTS = frozenset(
+    ["merge_request", "issue", "note", "commit", "pipeline", "release"]
+)
+
+_EVENT_TYPE_MAP: dict[str, EpisodeType] = {
+    "merge_request": EpisodeType.CODE_CHANGE,
+    "issue": EpisodeType.ISSUE,
+    "note": EpisodeType.COMMENT,
+    "commit": EpisodeType.CODE_CHANGE,
+    "pipeline": EpisodeType.DEPLOYMENT,
+    "release": EpisodeType.DEPLOYMENT,
+}
+
+
+class GitLabConnector(CortexConnector):
+    """Connector that syncs GitLab project events into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    gitlab_token:
+        GitLab personal access token or OAuth2 token.
+    gitlab_url:
+        GitLab instance URL.  Defaults to ``https://gitlab.com``.
+    project_ids:
+        List of GitLab project IDs to sync.
+    group_ids:
+        List of GitLab group IDs (all projects in each group are synced).
+    events:
+        Event types to capture.  Defaults to merge_request, issue, note,
+        commit, pipeline, release.
+    namespace:
+        CortexDB namespace for ingested episodes.
+    backfill_days:
+        Number of days of historical data to backfill on first sync.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "gitlab"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        gitlab_token: str,
+        gitlab_url: str = "https://gitlab.com",
+        project_ids: list[int] | None = None,
+        group_ids: list[int] | None = None,
+        events: list[str] | None = None,
+        namespace: str = "gitlab",
+        backfill_days: int = 90,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.gitlab_token = gitlab_token
+        self.gitlab_url = gitlab_url.rstrip("/")
+        self.project_ids = project_ids or []
+        self.group_ids = group_ids or []
+        self.event_filter = set(events) if events else set(_DEFAULT_EVENTS)
+        self.namespace = namespace
+        self.backfill_days = backfill_days
+        self._gl: Any = None
+
+    # -- GitLab client management --------------------------------------------
+
+    def _get_client(self) -> Any:
+        """Lazy-initialise and return the python-gitlab client."""
+        if self._gl is not None:
+            return self._gl
+
+        import gitlab
+
+        self._gl = gitlab.Gitlab(
+            self.gitlab_url,
+            private_token=self.gitlab_token,
+            per_page=100,
+        )
+        self._gl.auth()
+        return self._gl
+
+    def _resolve_projects(self) -> list[Any]:
+        """Resolve project IDs and group IDs into project objects."""
+        gl = self._get_client()
+        projects: list[Any] = []
+
+        for pid in self.project_ids:
+            try:
+                projects.append(gl.projects.get(pid))
+            except Exception:
+                logger.error("Could not access GitLab project %s", pid)
+
+        for gid in self.group_ids:
+            try:
+                group = gl.groups.get(gid)
+                for proj in group.projects.list(
+                    iterator=True, include_subgroups=True
+                ):
+                    try:
+                        projects.append(gl.projects.get(proj.id))
+                    except Exception:
+                        logger.debug(
+                            "Could not access project %s in group %s",
+                            proj.id,
+                            gid,
+                        )
+            except Exception:
+                logger.error("Could not access GitLab group %s", gid)
+
+        return projects
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch events from configured GitLab projects.
+
+        Iterates over each project and fetches merge requests, issues,
+        notes, commits, pipelines, and releases as configured.
+        """
+        cutoff = since or datetime.now(timezone.utc) - timedelta(
+            days=self.backfill_days
+        )
+        raw_events: list[RawEvent] = []
+
+        projects = self._resolve_projects()
+
+        for project in projects:
+            proj_name = project.path_with_namespace
+            try:
+                if "merge_request" in self.event_filter:
+                    raw_events.extend(
+                        self._fetch_merge_requests(project, proj_name, cutoff)
+                    )
+                if "issue" in self.event_filter:
+                    raw_events.extend(
+                        self._fetch_issues(project, proj_name, cutoff)
+                    )
+                if "commit" in self.event_filter:
+                    raw_events.extend(
+                        self._fetch_commits(project, proj_name, cutoff)
+                    )
+                if "pipeline" in self.event_filter:
+                    raw_events.extend(
+                        self._fetch_pipelines(project, proj_name, cutoff)
+                    )
+                if "release" in self.event_filter:
+                    raw_events.extend(
+                        self._fetch_releases(project, proj_name, cutoff)
+                    )
+            except Exception:
+                logger.exception(
+                    "Error fetching events from project %s", proj_name
+                )
+
+        return raw_events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a GitLab ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        event_kind = data.get("event_kind", "unknown")
+        episode_type = _EVENT_TYPE_MAP.get(event_kind, EpisodeType.CUSTOM)
+
+        actor = self._extract_actor(data)
+        content = self._build_content(event_kind, data)
+        entities = self._extract_entities(event_kind, data)
+        tags = ["gitlab", event_kind]
+        thread_id = self._derive_thread_id(event_kind, data)
+
+        metadata: dict[str, Any] = {
+            "gitlab_event": event_kind,
+            "project": data.get("project", ""),
+        }
+        if data.get("state"):
+            metadata["state"] = data["state"]
+        if data.get("action"):
+            metadata["action"] = data["action"]
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=episode_type,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            idempotency_key=raw.external_id,
+        )
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map GitLab project visibility to CortexDB visibility.
+
+        Public projects yield ``Organization``; internal or private
+        projects yield ``Restricted``.
+        """
+        if raw.data.get("visibility") in ("private", "internal"):
+            return Visibility(level=VisibilityLevel.RESTRICTED)
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- fetchers ------------------------------------------------------------
+
+    def _fetch_merge_requests(
+        self, project: Any, proj_name: str, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch merge requests and their notes."""
+        events: list[RawEvent] = []
+        cutoff_str = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        for mr in project.mergerequests.list(
+            iterator=True,
+            updated_after=cutoff_str,
+            order_by="updated_at",
+            sort="desc",
+        ):
+            ts = self._parse_ts(mr.updated_at)
+            events.append(
+                RawEvent(
+                    source="gitlab",
+                    external_id=f"mr:{project.id}:{mr.iid}",
+                    timestamp=ts,
+                    data={
+                        "event_kind": "merge_request",
+                        "project": proj_name,
+                        "project_id": project.id,
+                        "visibility": project.visibility,
+                        "iid": mr.iid,
+                        "title": mr.title,
+                        "description": (mr.description or "")[:1000],
+                        "state": mr.state,
+                        "source_branch": mr.source_branch,
+                        "target_branch": mr.target_branch,
+                        "author": self._user_dict(mr.author),
+                        "assignees": [
+                            self._user_dict(a)
+                            for a in (mr.assignees or [])
+                        ],
+                        "reviewers": [
+                            self._user_dict(r)
+                            for r in (getattr(mr, "reviewers", None) or [])
+                        ],
+                        "labels": mr.labels or [],
+                        "merged_by": self._user_dict(
+                            getattr(mr, "merged_by", None)
+                        ),
+                        "merge_commit_sha": getattr(
+                            mr, "merge_commit_sha", None
+                        ),
+                    },
+                    permissions={"visibility": project.visibility},
+                )
+            )
+
+            # Fetch notes on the MR.
+            if "note" in self.event_filter:
+                events.extend(
+                    self._fetch_notes_for(
+                        project,
+                        proj_name,
+                        "merge_request",
+                        mr.iid,
+                        mr.title,
+                        cutoff,
+                    )
+                )
+
+        return events
+
+    def _fetch_issues(
+        self, project: Any, proj_name: str, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch issues and their notes."""
+        events: list[RawEvent] = []
+        cutoff_str = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        for issue in project.issues.list(
+            iterator=True,
+            updated_after=cutoff_str,
+            order_by="updated_at",
+            sort="desc",
+        ):
+            ts = self._parse_ts(issue.updated_at)
+            events.append(
+                RawEvent(
+                    source="gitlab",
+                    external_id=f"issue:{project.id}:{issue.iid}",
+                    timestamp=ts,
+                    data={
+                        "event_kind": "issue",
+                        "project": proj_name,
+                        "project_id": project.id,
+                        "visibility": project.visibility,
+                        "iid": issue.iid,
+                        "title": issue.title,
+                        "description": (issue.description or "")[:1000],
+                        "state": issue.state,
+                        "author": self._user_dict(issue.author),
+                        "assignees": [
+                            self._user_dict(a)
+                            for a in (issue.assignees or [])
+                        ],
+                        "labels": issue.labels or [],
+                        "milestone": (
+                            issue.milestone.get("title")
+                            if issue.milestone
+                            else None
+                        ),
+                    },
+                    permissions={"visibility": project.visibility},
+                )
+            )
+
+            if "note" in self.event_filter:
+                events.extend(
+                    self._fetch_notes_for(
+                        project,
+                        proj_name,
+                        "issue",
+                        issue.iid,
+                        issue.title,
+                        cutoff,
+                    )
+                )
+
+        return events
+
+    def _fetch_notes_for(
+        self,
+        project: Any,
+        proj_name: str,
+        parent_type: str,
+        parent_iid: int,
+        parent_title: str,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch notes (comments) for a merge request or issue."""
+        events: list[RawEvent] = []
+
+        try:
+            if parent_type == "merge_request":
+                parent = project.mergerequests.get(parent_iid)
+            else:
+                parent = project.issues.get(parent_iid)
+
+            for note in parent.notes.list(
+                iterator=True, order_by="updated_at", sort="desc"
+            ):
+                ts = self._parse_ts(note.updated_at)
+                if ts < cutoff:
+                    break
+
+                # Skip system notes (e.g. "assigned to", "changed label").
+                if note.system:
+                    continue
+
+                events.append(
+                    RawEvent(
+                        source="gitlab",
+                        external_id=f"note:{project.id}:{parent_type}:{parent_iid}:{note.id}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "note",
+                            "project": proj_name,
+                            "project_id": project.id,
+                            "visibility": project.visibility,
+                            "parent_type": parent_type,
+                            "parent_iid": parent_iid,
+                            "parent_title": parent_title,
+                            "note_id": note.id,
+                            "body": (note.body or "")[:1000],
+                            "author": self._user_dict(note.author),
+                            "confidential": getattr(
+                                note, "confidential", False
+                            ),
+                        },
+                        permissions={
+                            "visibility": project.visibility,
+                            "confidential": getattr(
+                                note, "confidential", False
+                            ),
+                        },
+                    )
+                )
+        except Exception:
+            logger.debug(
+                "Could not fetch notes for %s !%d in %s",
+                parent_type,
+                parent_iid,
+                proj_name,
+            )
+
+        return events
+
+    def _fetch_commits(
+        self, project: Any, proj_name: str, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch recent commits from the default branch."""
+        events: list[RawEvent] = []
+        cutoff_str = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        try:
+            for commit in project.commits.list(
+                iterator=True,
+                since=cutoff_str,
+                ref_name=project.default_branch,
+            ):
+                ts = self._parse_ts(
+                    commit.committed_date or commit.created_at
+                )
+                events.append(
+                    RawEvent(
+                        source="gitlab",
+                        external_id=f"commit:{project.id}:{commit.id}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "commit",
+                            "project": proj_name,
+                            "project_id": project.id,
+                            "visibility": project.visibility,
+                            "sha": commit.id,
+                            "short_id": commit.short_id,
+                            "title": commit.title,
+                            "message": (commit.message or "")[:500],
+                            "author": {
+                                "name": commit.author_name,
+                                "email": commit.author_email,
+                            },
+                            "committer": {
+                                "name": commit.committer_name,
+                                "email": commit.committer_email,
+                            },
+                            "stats": getattr(commit, "stats", {}),
+                        },
+                        permissions={"visibility": project.visibility},
+                    )
+                )
+        except Exception:
+            logger.debug("Could not fetch commits for %s", proj_name)
+
+        return events
+
+    def _fetch_pipelines(
+        self, project: Any, proj_name: str, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch pipeline runs."""
+        events: list[RawEvent] = []
+        cutoff_str = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        try:
+            for pipeline in project.pipelines.list(
+                iterator=True,
+                updated_after=cutoff_str,
+                order_by="updated_at",
+                sort="desc",
+            ):
+                ts = self._parse_ts(pipeline.updated_at)
+                events.append(
+                    RawEvent(
+                        source="gitlab",
+                        external_id=f"pipeline:{project.id}:{pipeline.id}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "pipeline",
+                            "project": proj_name,
+                            "project_id": project.id,
+                            "visibility": project.visibility,
+                            "pipeline_id": pipeline.id,
+                            "ref": pipeline.ref,
+                            "status": pipeline.status,
+                            "source": getattr(pipeline, "source", ""),
+                            "sha": pipeline.sha,
+                        },
+                        permissions={"visibility": project.visibility},
+                    )
+                )
+        except Exception:
+            logger.debug("Could not fetch pipelines for %s", proj_name)
+
+        return events
+
+    def _fetch_releases(
+        self, project: Any, proj_name: str, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch releases."""
+        events: list[RawEvent] = []
+
+        try:
+            for release in project.releases.list(iterator=True):
+                ts = self._parse_ts(release.released_at or release.created_at)
+                if ts < cutoff:
+                    break
+                events.append(
+                    RawEvent(
+                        source="gitlab",
+                        external_id=f"release:{project.id}:{release.tag_name}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "release",
+                            "project": proj_name,
+                            "project_id": project.id,
+                            "visibility": project.visibility,
+                            "tag_name": release.tag_name,
+                            "name": release.name or release.tag_name,
+                            "description": (release.description or "")[:1000],
+                            "author": self._user_dict(
+                                getattr(release, "author", None)
+                            ),
+                        },
+                        permissions={"visibility": project.visibility},
+                    )
+                )
+        except Exception:
+            logger.debug("Could not fetch releases for %s", proj_name)
+
+        return events
+
+    # -- helpers -------------------------------------------------------------
+
+    @staticmethod
+    def _user_dict(user: Any) -> dict[str, Any]:
+        """Safely extract user data from a GitLab user object or dict."""
+        if user is None:
+            return {"id": "", "username": "unknown", "name": "Unknown"}
+        if isinstance(user, dict):
+            return {
+                "id": str(user.get("id", "")),
+                "username": user.get("username", "unknown"),
+                "name": user.get("name", user.get("username", "Unknown")),
+                "avatar_url": user.get("avatar_url"),
+            }
+        return {
+            "id": str(getattr(user, "id", "")),
+            "username": getattr(user, "username", "unknown"),
+            "name": getattr(user, "name", "Unknown"),
+            "avatar_url": getattr(user, "avatar_url", None),
+        }
+
+    @staticmethod
+    def _extract_actor(data: dict[str, Any]) -> Actor:
+        """Extract actor from event data."""
+        author = data.get("author", {})
+        if not author:
+            return Actor(id="unknown", name="Unknown")
+        return Actor(
+            id=str(author.get("id") or author.get("email", "")),
+            name=author.get("name") or author.get("username", "Unknown"),
+            email=author.get("email"),
+            avatar_url=author.get("avatar_url"),
+        )
+
+    @staticmethod
+    def _build_content(event_kind: str, data: dict[str, Any]) -> str:
+        """Build human-readable content from event data."""
+        if event_kind == "merge_request":
+            state = data.get("state", "")
+            title = data.get("title", "")
+            source = data.get("source_branch", "")
+            target = data.get("target_branch", "")
+            desc = data.get("description", "")[:200]
+            return f"MR [{state}]: {title} ({source} -> {target})\n{desc}".strip()
+
+        if event_kind == "issue":
+            state = data.get("state", "")
+            title = data.get("title", "")
+            desc = data.get("description", "")[:200]
+            return f"Issue [{state}]: {title}\n{desc}".strip()
+
+        if event_kind == "note":
+            parent_type = data.get("parent_type", "")
+            parent_title = data.get("parent_title", "")
+            body = data.get("body", "")[:500]
+            return f"Comment on {parent_type} [{parent_title}]: {body}"
+
+        if event_kind == "commit":
+            title = data.get("title", "")
+            sha = data.get("short_id", "")
+            return f"Commit {sha}: {title}"
+
+        if event_kind == "pipeline":
+            status = data.get("status", "")
+            ref = data.get("ref", "")
+            pid = data.get("pipeline_id", "")
+            return f"Pipeline #{pid} [{status}] on {ref}"
+
+        if event_kind == "release":
+            name = data.get("name", "")
+            tag = data.get("tag_name", "")
+            desc = data.get("description", "")[:200]
+            return f"Release {tag}: {name}\n{desc}".strip()
+
+        return f"GitLab event: {event_kind}"
+
+    @staticmethod
+    def _extract_entities(
+        event_kind: str, data: dict[str, Any]
+    ) -> list[EntityRef]:
+        """Extract entity references from event data."""
+        entities: list[EntityRef] = []
+
+        project = data.get("project", "")
+        if project:
+            entities.append(
+                EntityRef(
+                    entity_type="project",
+                    entity_id=str(data.get("project_id", project)),
+                    display_name=project,
+                )
+            )
+
+        author = data.get("author", {})
+        if author and author.get("id"):
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=str(author["id"]),
+                    display_name=author.get("name") or author.get("username", ""),
+                )
+            )
+
+        if event_kind == "merge_request":
+            for assignee in data.get("assignees", []):
+                if assignee.get("id"):
+                    entities.append(
+                        EntityRef(
+                            entity_type="user",
+                            entity_id=str(assignee["id"]),
+                            display_name=assignee.get("username", ""),
+                        )
+                    )
+            for reviewer in data.get("reviewers", []):
+                if reviewer.get("id"):
+                    entities.append(
+                        EntityRef(
+                            entity_type="user",
+                            entity_id=str(reviewer["id"]),
+                            display_name=reviewer.get("username", ""),
+                        )
+                    )
+            for label in data.get("labels", []):
+                entities.append(
+                    EntityRef(
+                        entity_type="label",
+                        entity_id=label,
+                        display_name=label,
+                    )
+                )
+
+        if event_kind == "issue":
+            milestone = data.get("milestone")
+            if milestone:
+                entities.append(
+                    EntityRef(
+                        entity_type="milestone",
+                        entity_id=str(milestone),
+                        display_name=str(milestone),
+                    )
+                )
+            for label in data.get("labels", []):
+                entities.append(
+                    EntityRef(
+                        entity_type="label",
+                        entity_id=label,
+                        display_name=label,
+                    )
+                )
+
+        return entities
+
+    @staticmethod
+    def _derive_thread_id(
+        event_kind: str, data: dict[str, Any]
+    ) -> str | None:
+        """Derive thread_id for grouping related events."""
+        project = data.get("project", "")
+
+        if event_kind == "merge_request":
+            iid = data.get("iid")
+            if iid:
+                return f"gl:{project}:mr:{iid}"
+
+        if event_kind == "issue":
+            iid = data.get("iid")
+            if iid:
+                return f"gl:{project}:issue:{iid}"
+
+        if event_kind == "note":
+            parent_type = data.get("parent_type", "")
+            parent_iid = data.get("parent_iid")
+            if parent_iid:
+                return f"gl:{project}:{parent_type}:{parent_iid}"
+
+        if event_kind == "pipeline":
+            pid = data.get("pipeline_id")
+            if pid:
+                return f"gl:{project}:pipeline:{pid}"
+
+        if event_kind == "release":
+            tag = data.get("tag_name")
+            if tag:
+                return f"gl:{project}:release:{tag}"
+
+        return None
+
+    @staticmethod
+    def _parse_ts(value: str | None) -> datetime:
+        if not value:
+            return datetime.now(timezone.utc)
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            return datetime.now(timezone.utc)