PyPI - cortexdb-connectors - Versions diffs - 0.2.0__py3-none-any.whl - Mend

cortexdb-connectors 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (31) hide show

cortexdb_connectors/__init__.py +40 -0
cortexdb_connectors/base.py +364 -0
cortexdb_connectors/cli.py +476 -0
cortexdb_connectors/confluence/__init__.py +359 -0
cortexdb_connectors/discord/__init__.py +646 -0
cortexdb_connectors/github/__init__.py +309 -0
cortexdb_connectors/gitlab/__init__.py +749 -0
cortexdb_connectors/google_workspace/__init__.py +1033 -0
cortexdb_connectors/hubspot/__init__.py +489 -0
cortexdb_connectors/insights/__init__.py +40 -0
cortexdb_connectors/insights/api.py +252 -0
cortexdb_connectors/insights/detectors.py +771 -0
cortexdb_connectors/insights/engine.py +308 -0
cortexdb_connectors/intercom/__init__.py +631 -0
cortexdb_connectors/jira/__init__.py +298 -0
cortexdb_connectors/linear/__init__.py +705 -0
cortexdb_connectors/notion/__init__.py +702 -0
cortexdb_connectors/pagerduty/__init__.py +298 -0
cortexdb_connectors/salesforce/__init__.py +399 -0
cortexdb_connectors/servicenow/__init__.py +460 -0
cortexdb_connectors/slack/__init__.py +331 -0
cortexdb_connectors/state.py +145 -0
cortexdb_connectors/teams/__init__.py +656 -0
cortexdb_connectors/webhooks.py +672 -0
cortexdb_connectors/worker.py +192 -0
cortexdb_connectors/zendesk/__init__.py +496 -0
cortexdb_connectors-0.2.0.dist-info/METADATA +207 -0
cortexdb_connectors-0.2.0.dist-info/RECORD +31 -0
cortexdb_connectors-0.2.0.dist-info/WHEEL +5 -0
cortexdb_connectors-0.2.0.dist-info/entry_points.txt +2 -0
cortexdb_connectors-0.2.0.dist-info/top_level.txt +1 -0

cortexdb_connectors/webhooks.py ADDED Viewed

@@ -0,0 +1,672 @@
+"""Webhook receivers for real-time event ingestion from Slack and GitHub.
+Provides a Starlette-based HTTP server that accepts webhook payloads from
+Slack (Events API) and GitHub, normalizes them into CortexDB Episodes using
+the existing connector classes, and ingests them via the CortexDB HTTP API.
+Environment variables:
+    CORTEXDB_URL            Base URL of the CortexDB API (default: http://localhost:8080)
+    CORTEXDB_API_KEY        Bearer token for CortexDB authentication
+    SLACK_SIGNING_SECRET    Used to verify Slack request signatures
+    GITHUB_WEBHOOK_SECRET   Used to verify GitHub X-Hub-Signature-256 headers
+    WEBHOOK_PORT            Port to listen on (default: 8081)
+"""
+from __future__ import annotations
+import hashlib
+import hmac
+import json
+import logging
+import os
+import time
+from datetime import datetime, timezone
+from typing import Any
+import httpx
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import JSONResponse, PlainTextResponse
+from starlette.routing import Route
+from cortexdb_connectors.base import (
+    Actor,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+logger = logging.getLogger(__name__)
+_SLACK_SOURCE = Source(system="slack")
+_GITHUB_SOURCE = Source(system="github")
+# Map GitHub webhook event types to CortexDB episode types.
+_GITHUB_WEBHOOK_TYPE_MAP: dict[str, EpisodeType] = {
+    "push": EpisodeType.CODE_CHANGE,
+    "pull_request": EpisodeType.CODE_CHANGE,
+    "issues": EpisodeType.ISSUE,
+    "issue_comment": EpisodeType.COMMENT,
+    "pull_request_review": EpisodeType.REVIEW,
+    "deployment_status": EpisodeType.DEPLOYMENT,
+}
+# ---------------------------------------------------------------------------
+# Slack webhook handler
+# ---------------------------------------------------------------------------
+class SlackWebhookHandler:
+    """Handles incoming Slack Events API webhook requests.
+    Responsibilities:
+    - Respond to Slack URL verification challenges.
+    - Verify request authenticity using the Slack signing secret.
+    - Normalize incoming event_callback events into CortexDB Episodes.
+    - Forward episodes to the CortexDB ingestion API.
+    """
+    SUPPORTED_EVENT_TYPES = {"message", "message_changed", "reaction_added"}
+    def __init__(
+        self,
+        signing_secret: str,
+        cortexdb_url: str,
+        cortexdb_api_key: str,
+        tenant_id: str = "default",
+    ) -> None:
+        self.signing_secret = signing_secret
+        self.cortexdb_url = cortexdb_url.rstrip("/")
+        self.cortexdb_api_key = cortexdb_api_key
+        self.tenant_id = tenant_id
+    def verify_signature(
+        self,
+        body: bytes,
+        timestamp: str,
+        signature: str,
+    ) -> bool:
+        """Verify the Slack request signature.
+        Uses HMAC-SHA256 over ``v0:<timestamp>:<body>`` and compares
+        against the ``X-Slack-Signature`` header value.
+        Returns True if the signature is valid.
+        """
+        if not timestamp or not signature:
+            return False
+        # Reject requests older than 5 minutes to prevent replay attacks.
+        try:
+            ts = int(timestamp)
+        except (ValueError, TypeError):
+            return False
+        if abs(time.time() - ts) > 300:
+            return False
+        basestring = f"v0:{timestamp}:{body.decode('utf-8')}"
+        computed = (
+            "v0="
+            + hmac.new(
+                self.signing_secret.encode("utf-8"),
+                basestring.encode("utf-8"),
+                hashlib.sha256,
+            ).hexdigest()
+        )
+        return hmac.compare_digest(computed, signature)
+    async def handle(self, request: Request) -> JSONResponse:
+        """Process an incoming Slack Events API request."""
+        body = await request.body()
+        # -- URL verification challenge ---------------------------------------
+        try:
+            payload = json.loads(body)
+        except json.JSONDecodeError:
+            return JSONResponse({"error": "invalid json"}, status_code=400)
+        if payload.get("type") == "url_verification":
+            return JSONResponse({"challenge": payload.get("challenge", "")})
+        # -- Signature verification -------------------------------------------
+        timestamp = request.headers.get("X-Slack-Request-Timestamp", "")
+        signature = request.headers.get("X-Slack-Signature", "")
+        if not self.verify_signature(body, timestamp, signature):
+            logger.warning("Slack signature verification failed")
+            return JSONResponse({"error": "invalid signature"}, status_code=401)
+        # -- Event dispatch ---------------------------------------------------
+        if payload.get("type") != "event_callback":
+            return JSONResponse({"ok": True})
+        event = payload.get("event", {})
+        event_type = event.get("type", "")
+        if event_type not in self.SUPPORTED_EVENT_TYPES:
+            logger.debug("Ignoring unsupported Slack event type: %s", event_type)
+            return JSONResponse({"ok": True})
+        try:
+            episode = self.normalize(payload, event)
+            await self._ingest(episode)
+        except Exception:
+            logger.exception("Failed to process Slack event %s", event_type)
+            return JSONResponse(
+                {"error": "processing failed"}, status_code=500
+            )
+        return JSONResponse({"ok": True})
+    def normalize(
+        self,
+        payload: dict[str, Any],
+        event: dict[str, Any],
+    ) -> Episode:
+        """Normalize a Slack Events API event_callback into an Episode.
+        This mirrors SlackConnector.normalize() but operates on the
+        webhook payload structure rather than a RawEvent fetched via polling.
+        """
+        event_type = event.get("type", "")
+        user_id = event.get("user", event.get("user_id", "unknown"))
+        channel_id = event.get("channel", "")
+        ts = event.get("ts", event.get("event_ts", "0"))
+        thread_ts = event.get("thread_ts")
+        # Determine content based on event type.
+        if event_type == "message" or event_type == "message_changed":
+            text = event.get("text", "")
+            if event_type == "message_changed":
+                message = event.get("message", {})
+                text = message.get("text", text)
+                user_id = message.get("user", user_id)
+                ts = message.get("ts", ts)
+            episode_type = EpisodeType.MESSAGE
+        elif event_type == "reaction_added":
+            reaction = event.get("reaction", "")
+            text = f"Reaction :{reaction}: added"
+            item = event.get("item", {})
+            channel_id = item.get("channel", channel_id)
+            ts = item.get("ts", ts)
+            episode_type = EpisodeType.MESSAGE
+        else:
+            text = ""
+            episode_type = EpisodeType.CUSTOM
+        actor = Actor(id=user_id, name=user_id)
+        parent_id = None
+        if thread_ts and thread_ts != ts:
+            parent_id = f"{channel_id}:{thread_ts}"
+        return Episode(
+            actor=actor,
+            source=_SLACK_SOURCE,
+            episode_type=episode_type,
+            occurred_at=datetime.fromtimestamp(float(ts), tz=timezone.utc),
+            content=text,
+            raw_payload=payload,
+            tenant_id=self.tenant_id,
+            namespace="slack",
+            entities=[
+                EntityRef(
+                    entity_type="channel",
+                    entity_id=channel_id,
+                    display_name=channel_id,
+                ),
+            ],
+            tags=["slack", f"slack:{event_type}"],
+            metadata={
+                "event_type": event_type,
+                "team_id": payload.get("team_id", ""),
+            },
+            thread_id=(
+                f"{channel_id}:{thread_ts}" if thread_ts else None
+            ),
+            parent_id=parent_id,
+            idempotency_key=f"{channel_id}:{ts}",
+        )
+    async def _ingest(self, episode: Episode) -> None:
+        """Send a normalized episode to CortexDB."""
+        payload = _episode_to_dict(episode)
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(
+                f"{self.cortexdb_url}/api/v1/episodes",
+                json=payload,
+                headers={
+                    "Authorization": f"Bearer {self.cortexdb_api_key}",
+                    "Content-Type": "application/json",
+                },
+                timeout=30.0,
+            )
+            resp.raise_for_status()
+# ---------------------------------------------------------------------------
+# GitHub webhook handler
+# ---------------------------------------------------------------------------
+class GitHubWebhookHandler:
+    """Handles incoming GitHub webhook requests.
+    Responsibilities:
+    - Verify request authenticity using the GitHub webhook secret.
+    - Normalize incoming events into CortexDB Episodes.
+    - Forward episodes to the CortexDB ingestion API.
+    """
+    SUPPORTED_EVENTS = {
+        "push",
+        "pull_request",
+        "issues",
+        "issue_comment",
+        "pull_request_review",
+        "deployment_status",
+    }
+    def __init__(
+        self,
+        webhook_secret: str,
+        cortexdb_url: str,
+        cortexdb_api_key: str,
+        tenant_id: str = "default",
+    ) -> None:
+        self.webhook_secret = webhook_secret
+        self.cortexdb_url = cortexdb_url.rstrip("/")
+        self.cortexdb_api_key = cortexdb_api_key
+        self.tenant_id = tenant_id
+    def verify_signature(self, body: bytes, signature: str) -> bool:
+        """Verify the GitHub X-Hub-Signature-256 header.
+        Computes HMAC-SHA256 of the request body using the webhook secret
+        and compares it against the provided signature.
+        Returns True if the signature is valid.
+        """
+        if not signature or not signature.startswith("sha256="):
+            return False
+        expected = (
+            "sha256="
+            + hmac.new(
+                self.webhook_secret.encode("utf-8"),
+                body,
+                hashlib.sha256,
+            ).hexdigest()
+        )
+        return hmac.compare_digest(expected, signature)
+    async def handle(self, request: Request) -> JSONResponse:
+        """Process an incoming GitHub webhook request."""
+        body = await request.body()
+        # -- Signature verification -------------------------------------------
+        signature = request.headers.get("X-Hub-Signature-256", "")
+        if not self.verify_signature(body, signature):
+            logger.warning("GitHub signature verification failed")
+            return JSONResponse({"error": "invalid signature"}, status_code=401)
+        # -- Parse payload ----------------------------------------------------
+        try:
+            payload = json.loads(body)
+        except json.JSONDecodeError:
+            return JSONResponse({"error": "invalid json"}, status_code=400)
+        event_type = request.headers.get("X-GitHub-Event", "")
+        delivery_id = request.headers.get("X-GitHub-Delivery", "")
+        if event_type == "ping":
+            return JSONResponse({"ok": True, "msg": "pong"})
+        if event_type not in self.SUPPORTED_EVENTS:
+            logger.debug("Ignoring unsupported GitHub event: %s", event_type)
+            return JSONResponse({"ok": True})
+        try:
+            episode = self.normalize(event_type, delivery_id, payload)
+            await self._ingest(episode)
+        except Exception:
+            logger.exception(
+                "Failed to process GitHub event %s (%s)", event_type, delivery_id
+            )
+            return JSONResponse(
+                {"error": "processing failed"}, status_code=500
+            )
+        return JSONResponse({"ok": True})
+    def normalize(
+        self,
+        event_type: str,
+        delivery_id: str,
+        payload: dict[str, Any],
+    ) -> Episode:
+        """Normalize a GitHub webhook event into a CortexDB Episode.
+        This mirrors GitHubConnector.normalize() but operates on the
+        raw webhook payload rather than a RawEvent from the polling API.
+        """
+        episode_type = _GITHUB_WEBHOOK_TYPE_MAP.get(event_type, EpisodeType.CUSTOM)
+        # Extract actor.
+        sender = payload.get("sender", {})
+        actor = Actor(
+            id=str(sender.get("id", "")),
+            name=sender.get("login", "unknown"),
+            avatar_url=sender.get("avatar_url"),
+        )
+        # Extract repository.
+        repo_data = payload.get("repository", {})
+        repo_name = repo_data.get("full_name", "")
+        is_private = repo_data.get("private", False)
+        content = self._build_content(event_type, payload)
+        entities = self._extract_entities(repo_name, payload)
+        thread_id = self._derive_thread_id(event_type, payload, repo_name)
+        action = payload.get("action", "")
+        tags = ["github", event_type]
+        if action:
+            tags.append(f"github:{action}")
+        metadata: dict[str, Any] = {"github_event_type": event_type}
+        if action:
+            metadata["action"] = action
+        if delivery_id:
+            metadata["delivery_id"] = delivery_id
+        visibility = Visibility(
+            level=VisibilityLevel.RESTRICTED if is_private
+            else VisibilityLevel.ORGANIZATION
+        )
+        return Episode(
+            actor=actor,
+            source=_GITHUB_SOURCE,
+            episode_type=episode_type,
+            occurred_at=datetime.now(timezone.utc),
+            content=content,
+            raw_payload=payload,
+            tenant_id=self.tenant_id,
+            namespace="github",
+            entities=entities,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            visibility=visibility,
+            idempotency_key=delivery_id or None,
+        )
+    @staticmethod
+    def _build_content(event_type: str, payload: dict[str, Any]) -> str:
+        """Produce a human-readable summary of the GitHub webhook event."""
+        action = payload.get("action", "")
+        if event_type == "push":
+            commits = payload.get("commits", [])
+            ref = payload.get("ref", "")
+            messages = "; ".join(
+                c.get("message", "").split("\n")[0] for c in commits[:5]
+            )
+            return f"Pushed {len(commits)} commit(s) to {ref}: {messages}"
+        if event_type == "pull_request":
+            pr = payload.get("pull_request", {})
+            title = pr.get("title", "")
+            return f"PR {action}: {title}"
+        if event_type == "issues":
+            issue = payload.get("issue", {})
+            title = issue.get("title", "")
+            return f"Issue {action}: {title}"
+        if event_type == "issue_comment":
+            comment = payload.get("comment", {})
+            body = (comment.get("body") or "")[:200]
+            return f"Comment: {body}"
+        if event_type == "pull_request_review":
+            review = payload.get("review", {})
+            state = review.get("state", "")
+            pr = payload.get("pull_request", {})
+            title = pr.get("title", "")
+            return f"Review ({state}) on PR: {title}"
+        if event_type == "deployment_status":
+            deployment = payload.get("deployment_status", {})
+            state = deployment.get("state", "")
+            env = deployment.get("environment", "")
+            return f"Deployment to {env}: {state}"
+        return f"GitHub event: {event_type}"
+    @staticmethod
+    def _extract_entities(
+        repo_name: str, payload: dict[str, Any]
+    ) -> list[EntityRef]:
+        """Extract entity references from the webhook payload."""
+        entities: list[EntityRef] = []
+        if repo_name:
+            entities.append(
+                EntityRef(
+                    entity_type="repository",
+                    entity_id=repo_name,
+                    display_name=repo_name,
+                )
+            )
+        pr = payload.get("pull_request", {})
+        if pr:
+            pr_user = pr.get("user", {})
+            if pr_user:
+                entities.append(
+                    EntityRef(
+                        entity_type="user",
+                        entity_id=str(pr_user.get("id", "")),
+                        display_name=pr_user.get("login", ""),
+                    )
+                )
+        issue = payload.get("issue", {})
+        if issue:
+            entities.append(
+                EntityRef(
+                    entity_type="issue",
+                    entity_id=str(issue.get("number", "")),
+                    display_name=issue.get("title", ""),
+                )
+            )
+        return entities
+    @staticmethod
+    def _derive_thread_id(
+        event_type: str, payload: dict[str, Any], repo_name: str
+    ) -> str | None:
+        """Derive a thread_id for grouping related events."""
+        pr = payload.get("pull_request", {})
+        if pr and pr.get("number"):
+            return f"{repo_name}:pr:{pr['number']}"
+        issue = payload.get("issue", {})
+        if issue and issue.get("number"):
+            return f"{repo_name}:issue:{issue['number']}"
+        return None
+    async def _ingest(self, episode: Episode) -> None:
+        """Send a normalized episode to CortexDB."""
+        payload = _episode_to_dict(episode)
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(
+                f"{self.cortexdb_url}/api/v1/episodes",
+                json=payload,
+                headers={
+                    "Authorization": f"Bearer {self.cortexdb_api_key}",
+                    "Content-Type": "application/json",
+                },
+                timeout=30.0,
+            )
+            resp.raise_for_status()
+# ---------------------------------------------------------------------------
+# Shared utilities
+# ---------------------------------------------------------------------------
+def _episode_to_dict(episode: Episode) -> dict[str, Any]:
+    """Serialize an Episode to a dictionary suitable for JSON ingestion."""
+    payload: dict[str, Any] = {
+        "id": episode.id,
+        "episode_type": episode.episode_type.value,
+        "content": episode.content,
+        "occurred_at": episode.occurred_at.isoformat(),
+        "tenant_id": episode.tenant_id,
+        "namespace": episode.namespace,
+        "tags": episode.tags,
+        "metadata": episode.metadata,
+        "raw_payload": episode.raw_payload,
+        "visibility": {
+            "level": episode.visibility.level.value,
+            "allowed_principals": list(episode.visibility.allowed_principals),
+        },
+        "idempotency_key": episode.idempotency_key,
+    }
+    if episode.actor:
+        payload["actor"] = {
+            "id": episode.actor.id,
+            "name": episode.actor.name,
+            "email": episode.actor.email,
+        }
+    if episode.source:
+        payload["source"] = {
+            "system": episode.source.system,
+            "connector_version": episode.source.connector_version,
+        }
+    if episode.entities:
+        payload["entities"] = [
+            {
+                "entity_type": e.entity_type,
+                "entity_id": e.entity_id,
+                "display_name": e.display_name,
+            }
+            for e in episode.entities
+        ]
+    if episode.thread_id:
+        payload["thread_id"] = episode.thread_id
+    if episode.parent_id:
+        payload["parent_id"] = episode.parent_id
+    if episode.ttl_seconds is not None:
+        payload["ttl_seconds"] = episode.ttl_seconds
+    return payload
+# ---------------------------------------------------------------------------
+# Webhook server
+# ---------------------------------------------------------------------------
+class WebhookServer:
+    """Configurable Starlette application serving Slack and GitHub webhooks.
+    Reads configuration from environment variables and wires up the
+    handler classes with appropriate routes.
+    Parameters
+    ----------
+    cortexdb_url:
+        Override for CORTEXDB_URL env var.
+    cortexdb_api_key:
+        Override for CORTEXDB_API_KEY env var.
+    slack_signing_secret:
+        Override for SLACK_SIGNING_SECRET env var.
+    github_webhook_secret:
+        Override for GITHUB_WEBHOOK_SECRET env var.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+    def __init__(
+        self,
+        cortexdb_url: str | None = None,
+        cortexdb_api_key: str | None = None,
+        slack_signing_secret: str | None = None,
+        github_webhook_secret: str | None = None,
+        tenant_id: str = "default",
+    ) -> None:
+        self.cortexdb_url = cortexdb_url or os.environ.get(
+            "CORTEXDB_URL", "http://localhost:8080"
+        )
+        self.cortexdb_api_key = cortexdb_api_key or os.environ.get(
+            "CORTEXDB_API_KEY", ""
+        )
+        self.slack_signing_secret = slack_signing_secret or os.environ.get(
+            "SLACK_SIGNING_SECRET", ""
+        )
+        self.github_webhook_secret = github_webhook_secret or os.environ.get(
+            "GITHUB_WEBHOOK_SECRET", ""
+        )
+        self.tenant_id = tenant_id
+        self.slack_handler = SlackWebhookHandler(
+            signing_secret=self.slack_signing_secret,
+            cortexdb_url=self.cortexdb_url,
+            cortexdb_api_key=self.cortexdb_api_key,
+            tenant_id=self.tenant_id,
+        )
+        self.github_handler = GitHubWebhookHandler(
+            webhook_secret=self.github_webhook_secret,
+            cortexdb_url=self.cortexdb_url,
+            cortexdb_api_key=self.cortexdb_api_key,
+            tenant_id=self.tenant_id,
+        )
+    def build_app(self) -> Starlette:
+        """Construct and return the Starlette ASGI application."""
+        routes = [
+            Route("/health", self._health, methods=["GET"]),
+            Route(
+                "/webhooks/slack/events",
+                self.slack_handler.handle,
+                methods=["POST"],
+            ),
+            Route(
+                "/webhooks/github/events",
+                self.github_handler.handle,
+                methods=["POST"],
+            ),
+        ]
+        return Starlette(routes=routes)
+    @staticmethod
+    async def _health(request: Request) -> JSONResponse:
+        """Health check endpoint."""
+        return JSONResponse({"status": "ok"})
+def main() -> None:
+    """Entry point for running the webhook server via ``python -m`` or CLI."""
+    import uvicorn
+    port = int(os.environ.get("WEBHOOK_PORT", "8081"))
+    server = WebhookServer()
+    app = server.build_app()
+    logger.info("Starting CortexDB webhook server on port %d", port)
+    uvicorn.run(app, host="0.0.0.0", port=port, log_level="info")
+if __name__ == "__main__":
+    main()