cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/linear/__init__.py

@@ -0,0 +1,705 @@
+"""
+Linear connector for CortexDB.
+
+Ingests issues, comments, and projects from Linear workspaces into CortexDB
+as Episodes.  Uses raw GraphQL queries via ``httpx`` against the Linear API
+and respects the 1500 requests/hour rate limit.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="linear")
+
+_GRAPHQL_URL = "https://api.linear.app/graphql"
+
+# Linear rate limit: 1500 req/hour -> 0.5 s between requests.
+_RATE_LIMIT_DELAY = 0.5
+
+_PAGE_SIZE = 50
+
+
+# ---------------------------------------------------------------------------
+# GraphQL query fragments
+# ---------------------------------------------------------------------------
+
+_ISSUES_QUERY = """
+query FetchIssues($teamId: String!, $after: String, $updatedAfter: DateTime) {
+  team(id: $teamId) {
+    issues(
+      first: %d,
+      after: $after,
+      filter: { updatedAt: { gte: $updatedAfter } },
+      orderBy: updatedAt
+    ) {
+      pageInfo { hasNextPage endCursor }
+      nodes {
+        id identifier title description priority priorityLabel
+        state { id name type }
+        assignee { id name email avatarUrl }
+        creator { id name email avatarUrl }
+        team { id name key }
+        project { id name }
+        cycle { id name number }
+        labels { nodes { id name } }
+        createdAt updatedAt completedAt canceledAt
+        estimate url
+      }
+    }
+  }
+}
+""" % _PAGE_SIZE
+
+_COMMENTS_QUERY = """
+query FetchComments($issueId: String!, $after: String) {
+  issue(id: $issueId) {
+    comments(first: %d, after: $after, orderBy: createdAt) {
+      pageInfo { hasNextPage endCursor }
+      nodes {
+        id body createdAt updatedAt
+        user { id name email avatarUrl }
+      }
+    }
+  }
+}
+""" % _PAGE_SIZE
+
+_PROJECTS_QUERY = """
+query FetchProjects($after: String, $updatedAfter: DateTime) {
+  projects(
+    first: %d,
+    after: $after,
+    filter: { updatedAt: { gte: $updatedAfter } },
+    orderBy: updatedAt
+  ) {
+    pageInfo { hasNextPage endCursor }
+    nodes {
+      id name description state
+      lead { id name email avatarUrl }
+      teams { nodes { id name key } }
+      members { nodes { id name email } }
+      startDate targetDate
+      createdAt updatedAt url
+    }
+  }
+}
+""" % _PAGE_SIZE
+
+_TEAMS_QUERY = """
+query FetchTeams {
+  teams {
+    nodes { id name key }
+  }
+}
+"""
+
+
+class LinearConnector(CortexConnector):
+    """Connector that syncs Linear issues, comments, and projects into
+    CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    linear_api_key:
+        Linear API key or OAuth token.
+    team_ids:
+        List of Linear team IDs to sync.  If empty, all teams in the
+        workspace are discovered automatically.
+    tenant_id:
+        CortexDB tenant identifier.
+    namespace:
+        CortexDB namespace for ingested episodes.
+    backfill_days:
+        Number of days of history to backfill on first sync.
+    """
+
+    connector_name: str = "linear"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        linear_api_key: str,
+        team_ids: list[str] | None = None,
+        tenant_id: str = "default",
+        namespace: str = "linear",
+        backfill_days: int = 30,
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.linear_api_key = linear_api_key
+        self.team_ids = team_ids or []
+        self.namespace = namespace
+        self.backfill_days = backfill_days
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch issues, comments, and projects from Linear.
+
+        If *since* is provided, only items updated after that timestamp
+        are returned.  Pagination is handled for all queries.
+        """
+        import httpx
+
+        cutoff = since or datetime.now(timezone.utc) - timedelta(
+            days=self.backfill_days
+        )
+        events: list[RawEvent] = []
+
+        async with httpx.AsyncClient(
+            headers={
+                "Authorization": self.linear_api_key,
+                "Content-Type": "application/json",
+            },
+            timeout=30.0,
+        ) as client:
+            # Discover teams if none configured.
+            team_ids = self.team_ids or await self._discover_teams(client)
+
+            # 1. Fetch issues per team.
+            for team_id in team_ids:
+                issue_events = await self._fetch_issues(
+                    client, team_id, cutoff
+                )
+                events.extend(issue_events)
+
+            # 2. Fetch comments on each issue.
+            issue_ids_seen: set[str] = set()
+            for event in list(events):
+                iid = event.data.get("issue_id", "")
+                if iid and iid not in issue_ids_seen:
+                    issue_ids_seen.add(iid)
+                    comment_events = await self._fetch_comments(client, iid)
+                    events.extend(comment_events)
+
+            # 3. Fetch projects.
+            project_events = await self._fetch_projects(client, cutoff)
+            events.extend(project_events)
+
+        return events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a Linear ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        event_kind = data.get("_event_kind", "issue")
+
+        if event_kind == "comment":
+            return self._normalize_comment(raw)
+        if event_kind == "project":
+            return self._normalize_project(raw)
+        return self._normalize_issue(raw)
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map Linear access to CortexDB visibility.
+
+        Linear workspaces are private by default, so all episodes are
+        ``Organization``.  If the team is marked private, we use
+        ``Restricted`` with team members as principals.
+        """
+        perms = raw.permissions
+        if perms.get("private_team"):
+            members = tuple(perms.get("team_member_ids", []))
+            return Visibility(
+                level=VisibilityLevel.RESTRICTED,
+                allowed_principals=members,
+            )
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- issue normalization -------------------------------------------------
+
+    def _normalize_issue(self, raw: RawEvent) -> Episode:
+        """Normalize a Linear issue into an Episode."""
+        data = raw.data
+        identifier = data.get("identifier", "")
+        title = data.get("title", "Untitled")
+        description = data.get("description", "") or ""
+        state_name = data.get("state_name", "Unknown")
+        priority_label = data.get("priority_label", "No priority")
+
+        content = f"[{identifier}] {title}"
+        if description:
+            content += f"\n\n{description[:1000]}"
+
+        actor = self._extract_actor(data, role="creator")
+        entities = self._extract_issue_entities(data)
+
+        tags = [
+            "linear",
+            state_name.lower().replace(" ", "_"),
+            priority_label.lower().replace(" ", "_"),
+        ]
+        for label in data.get("labels", []):
+            tags.append(label.get("name", "").lower().replace(" ", "_"))
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.ISSUE,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=tags,
+            metadata={
+                "identifier": identifier,
+                "state": state_name,
+                "state_type": data.get("state_type", ""),
+                "priority": data.get("priority", 0),
+                "priority_label": priority_label,
+                "estimate": data.get("estimate"),
+                "cycle_name": data.get("cycle_name"),
+                "cycle_number": data.get("cycle_number"),
+                "project_name": data.get("project_name"),
+                "completed_at": data.get("completed_at"),
+                "canceled_at": data.get("canceled_at"),
+                "url": data.get("url", ""),
+            },
+            thread_id=f"linear:{identifier}",
+            idempotency_key=(
+                f"linear:{raw.external_id}:{data.get('updated_at', '')}"
+            ),
+        )
+
+    def _normalize_comment(self, raw: RawEvent) -> Episode:
+        """Normalize a Linear comment into an Episode."""
+        data = raw.data
+        actor = self._extract_actor(data, role="commenter")
+        identifier = data.get("issue_identifier", "")
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.COMMENT,
+            occurred_at=raw.timestamp,
+            content=data.get("body", ""),
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=[
+                EntityRef(
+                    entity_type="issue",
+                    entity_id=data.get("issue_id", ""),
+                    display_name=identifier,
+                ),
+            ],
+            tags=["linear", "comment"],
+            metadata={
+                "comment_id": data.get("comment_id", ""),
+                "issue_id": data.get("issue_id", ""),
+                "issue_identifier": identifier,
+            },
+            thread_id=f"linear:{identifier}",
+            parent_id=f"linear:{data.get('issue_id', '')}",
+            idempotency_key=f"linear:comment:{raw.external_id}",
+        )
+
+    def _normalize_project(self, raw: RawEvent) -> Episode:
+        """Normalize a Linear project into an Episode."""
+        data = raw.data
+        name = data.get("name", "Untitled")
+        description = data.get("description", "") or ""
+        state = data.get("state", "planned")
+
+        content = f"[Project] {name}"
+        if description:
+            content += f"\n\n{description[:1000]}"
+
+        actor = self._extract_actor(data, role="lead")
+        entities: list[EntityRef] = [
+            EntityRef(
+                entity_type="project",
+                entity_id=data.get("project_id", ""),
+                display_name=name,
+            ),
+        ]
+        for team in data.get("teams", []):
+            entities.append(
+                EntityRef(
+                    entity_type="team",
+                    entity_id=team.get("id", ""),
+                    display_name=team.get("name", ""),
+                )
+            )
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.DOCUMENT,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=["linear", "project", state.lower().replace(" ", "_")],
+            metadata={
+                "project_id": data.get("project_id", ""),
+                "state": state,
+                "start_date": data.get("start_date"),
+                "target_date": data.get("target_date"),
+                "member_count": len(data.get("members", [])),
+                "url": data.get("url", ""),
+            },
+            thread_id=f"linear:project:{data.get('project_id', '')}",
+            idempotency_key=(
+                f"linear:project:{raw.external_id}:{data.get('updated_at', '')}"
+            ),
+        )
+
+    # -- fetchers ------------------------------------------------------------
+
+    async def _graphql(
+        self, client: Any, query: str, variables: dict[str, Any]
+    ) -> dict[str, Any]:
+        """Execute a GraphQL query against the Linear API."""
+        resp = await client.post(
+            _GRAPHQL_URL,
+            json={"query": query, "variables": variables},
+        )
+        resp.raise_for_status()
+        body = resp.json()
+
+        if body.get("errors"):
+            error_msgs = [e.get("message", "") for e in body["errors"]]
+            logger.error("Linear GraphQL errors: %s", error_msgs)
+
+        await asyncio.sleep(_RATE_LIMIT_DELAY)
+        return body.get("data", {})
+
+    async def _discover_teams(self, client: Any) -> list[str]:
+        """Discover all teams in the Linear workspace."""
+        data = await self._graphql(client, _TEAMS_QUERY, {})
+        teams = data.get("teams", {}).get("nodes", [])
+        team_ids = [t["id"] for t in teams if t.get("id")]
+        logger.info("Discovered %d Linear teams", len(team_ids))
+        return team_ids
+
+    async def _fetch_issues(
+        self, client: Any, team_id: str, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch issues for a team updated after *cutoff*."""
+        events: list[RawEvent] = []
+        has_next = True
+        after: str | None = None
+
+        while has_next:
+            variables: dict[str, Any] = {
+                "teamId": team_id,
+                "after": after,
+                "updatedAfter": cutoff.isoformat(),
+            }
+            data = await self._graphql(client, _ISSUES_QUERY, variables)
+
+            team_data = data.get("team", {})
+            issues_data = team_data.get("issues", {})
+            page_info = issues_data.get("pageInfo", {})
+
+            for node in issues_data.get("nodes", []):
+                raw = self._issue_node_to_raw(node)
+                events.append(raw)
+
+            has_next = page_info.get("hasNextPage", False)
+            after = page_info.get("endCursor")
+
+        return events
+
+    async def _fetch_comments(
+        self, client: Any, issue_id: str
+    ) -> list[RawEvent]:
+        """Fetch comments on a Linear issue."""
+        events: list[RawEvent] = []
+        has_next = True
+        after: str | None = None
+
+        while has_next:
+            variables: dict[str, Any] = {
+                "issueId": issue_id,
+                "after": after,
+            }
+            data = await self._graphql(client, _COMMENTS_QUERY, variables)
+
+            issue_data = data.get("issue", {})
+            comments_data = issue_data.get("comments", {})
+            page_info = comments_data.get("pageInfo", {})
+
+            for node in comments_data.get("nodes", []):
+                created_str = node.get("createdAt", "")
+                try:
+                    created_at = datetime.fromisoformat(
+                        created_str.replace("Z", "+00:00")
+                    )
+                except (ValueError, TypeError):
+                    created_at = datetime.now(timezone.utc)
+
+                user = node.get("user", {}) or {}
+
+                events.append(
+                    RawEvent(
+                        source="linear",
+                        external_id=node.get("id", ""),
+                        timestamp=created_at,
+                        data={
+                            "_event_kind": "comment",
+                            "comment_id": node.get("id", ""),
+                            "issue_id": issue_id,
+                            "issue_identifier": "",
+                            "body": node.get("body", ""),
+                            "commenter_id": user.get("id", ""),
+                            "commenter_name": user.get("name", ""),
+                            "commenter_email": user.get("email"),
+                            "commenter_avatar": user.get("avatarUrl"),
+                            "created_at": created_str,
+                            "updated_at": node.get("updatedAt", ""),
+                        },
+                        permissions={},
+                    )
+                )
+
+            has_next = page_info.get("hasNextPage", False)
+            after = page_info.get("endCursor")
+
+        return events
+
+    async def _fetch_projects(
+        self, client: Any, cutoff: datetime
+    ) -> list[RawEvent]:
+        """Fetch projects updated after *cutoff*."""
+        events: list[RawEvent] = []
+        has_next = True
+        after: str | None = None
+
+        while has_next:
+            variables: dict[str, Any] = {
+                "after": after,
+                "updatedAfter": cutoff.isoformat(),
+            }
+            data = await self._graphql(client, _PROJECTS_QUERY, variables)
+
+            projects_data = data.get("projects", {})
+            page_info = projects_data.get("pageInfo", {})
+
+            for node in projects_data.get("nodes", []):
+                updated_str = node.get("updatedAt", "")
+                try:
+                    updated_at = datetime.fromisoformat(
+                        updated_str.replace("Z", "+00:00")
+                    )
+                except (ValueError, TypeError):
+                    updated_at = datetime.now(timezone.utc)
+
+                lead = node.get("lead") or {}
+                teams = [
+                    {"id": t.get("id", ""), "name": t.get("name", ""),
+                     "key": t.get("key", "")}
+                    for t in node.get("teams", {}).get("nodes", [])
+                ]
+                members = [
+                    {"id": m.get("id", ""), "name": m.get("name", ""),
+                     "email": m.get("email")}
+                    for m in node.get("members", {}).get("nodes", [])
+                ]
+
+                events.append(
+                    RawEvent(
+                        source="linear",
+                        external_id=node.get("id", ""),
+                        timestamp=updated_at,
+                        data={
+                            "_event_kind": "project",
+                            "project_id": node.get("id", ""),
+                            "name": node.get("name", ""),
+                            "description": node.get("description", ""),
+                            "state": node.get("state", "planned"),
+                            "lead_id": lead.get("id", ""),
+                            "lead_name": lead.get("name", ""),
+                            "lead_email": lead.get("email"),
+                            "lead_avatar": lead.get("avatarUrl"),
+                            "teams": teams,
+                            "members": members,
+                            "start_date": node.get("startDate"),
+                            "target_date": node.get("targetDate"),
+                            "created_at": node.get("createdAt", ""),
+                            "updated_at": updated_str,
+                            "url": node.get("url", ""),
+                        },
+                        permissions={},
+                    )
+                )
+
+            has_next = page_info.get("hasNextPage", False)
+            after = page_info.get("endCursor")
+
+        return events
+
+    # -- helpers -------------------------------------------------------------
+
+    def _issue_node_to_raw(self, node: dict[str, Any]) -> RawEvent:
+        """Convert a Linear issue GraphQL node into a RawEvent."""
+        updated_str = node.get("updatedAt", "")
+        try:
+            updated_at = datetime.fromisoformat(
+                updated_str.replace("Z", "+00:00")
+            )
+        except (ValueError, TypeError):
+            updated_at = datetime.now(timezone.utc)
+
+        state = node.get("state", {}) or {}
+        assignee = node.get("assignee", {}) or {}
+        creator = node.get("creator", {}) or {}
+        team = node.get("team", {}) or {}
+        project = node.get("project", {}) or {}
+        cycle = node.get("cycle", {}) or {}
+        labels = [
+            {"id": l.get("id", ""), "name": l.get("name", "")}
+            for l in node.get("labels", {}).get("nodes", [])
+        ]
+
+        return RawEvent(
+            source="linear",
+            external_id=node.get("id", ""),
+            timestamp=updated_at,
+            data={
+                "_event_kind": "issue",
+                "issue_id": node.get("id", ""),
+                "identifier": node.get("identifier", ""),
+                "title": node.get("title", ""),
+                "description": node.get("description", ""),
+                "priority": node.get("priority", 0),
+                "priority_label": node.get("priorityLabel", "No priority"),
+                "state_name": state.get("name", "Unknown"),
+                "state_type": state.get("type", ""),
+                "assignee_id": assignee.get("id", ""),
+                "assignee_name": assignee.get("name", ""),
+                "assignee_email": assignee.get("email"),
+                "assignee_avatar": assignee.get("avatarUrl"),
+                "creator_id": creator.get("id", ""),
+                "creator_name": creator.get("name", ""),
+                "creator_email": creator.get("email"),
+                "creator_avatar": creator.get("avatarUrl"),
+                "team_id": team.get("id", ""),
+                "team_name": team.get("name", ""),
+                "team_key": team.get("key", ""),
+                "project_id": project.get("id", ""),
+                "project_name": project.get("name", ""),
+                "cycle_name": cycle.get("name"),
+                "cycle_number": cycle.get("number"),
+                "labels": labels,
+                "estimate": node.get("estimate"),
+                "created_at": node.get("createdAt", ""),
+                "updated_at": updated_str,
+                "completed_at": node.get("completedAt"),
+                "canceled_at": node.get("canceledAt"),
+                "url": node.get("url", ""),
+            },
+            permissions={
+                "team_id": team.get("id", ""),
+            },
+        )
+
+    @staticmethod
+    def _extract_actor(data: dict[str, Any], role: str = "creator") -> Actor:
+        """Extract the actor for the given role from event data."""
+        if role == "commenter":
+            return Actor(
+                id=data.get("commenter_id", "unknown"),
+                name=data.get("commenter_name", "Unknown"),
+                email=data.get("commenter_email"),
+                avatar_url=data.get("commenter_avatar"),
+            )
+        if role == "lead":
+            return Actor(
+                id=data.get("lead_id", "unknown"),
+                name=data.get("lead_name", "Unknown"),
+                email=data.get("lead_email"),
+                avatar_url=data.get("lead_avatar"),
+            )
+        # Default: creator for issues.
+        return Actor(
+            id=data.get("creator_id", "unknown"),
+            name=data.get("creator_name", "Unknown"),
+            email=data.get("creator_email"),
+            avatar_url=data.get("creator_avatar"),
+        )
+
+    @staticmethod
+    def _extract_issue_entities(data: dict[str, Any]) -> list[EntityRef]:
+        """Extract entity references from issue data."""
+        entities: list[EntityRef] = []
+
+        team_id = data.get("team_id", "")
+        if team_id:
+            entities.append(
+                EntityRef(
+                    entity_type="team",
+                    entity_id=team_id,
+                    display_name=data.get("team_name", ""),
+                )
+            )
+
+        project_id = data.get("project_id", "")
+        if project_id:
+            entities.append(
+                EntityRef(
+                    entity_type="project",
+                    entity_id=project_id,
+                    display_name=data.get("project_name", ""),
+                )
+            )
+
+        assignee_id = data.get("assignee_id", "")
+        if assignee_id:
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=assignee_id,
+                    display_name=data.get("assignee_name", ""),
+                )
+            )
+
+        creator_id = data.get("creator_id", "")
+        if creator_id and creator_id != assignee_id:
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=creator_id,
+                    display_name=data.get("creator_name", ""),
+                )
+            )
+
+        for label in data.get("labels", []):
+            if label.get("id"):
+                entities.append(
+                    EntityRef(
+                        entity_type="label",
+                        entity_id=label["id"],
+                        display_name=label.get("name", ""),
+                    )
+                )
+
+        return entities